# # Copyright (c) 2014 ThoughtWorks, Inc. # # Pixelated is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Pixelated is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PCULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see . import json import os import re import logging import dateutil.parser as dateparser from uuid import uuid4 from email import message_from_file from email.mime.text import MIMEText from email.header import decode_header from email.MIMEMultipart import MIMEMultipart from pycryptopp.hash import sha256 from leap.mail.adaptors import soledad_indexes as fields import leap.mail.walk as walk from pixelated.adapter.model.status import Status from pixelated.support import date from pixelated.support.functional import compact logger = logging.getLogger(__name__) TYPE_KEY = 'type' CONTENT_HASH_KEY = 'chash' HEADERS_KEY = 'headers' DATE_KEY = 'date' SUBJECT_KEY = 'subject' PARTS_MAP_KEY = 'part_map' BODY_KEY = 'body' MSGID_KEY = 'msgid' MULTIPART_KEY = 'multi' SIZE_KEY = 'size' class Mail(object): @property def to(self): return self.headers['To'] @property def cc(self): return self.headers['Cc'] @property def bcc(self): return self.headers['Bcc'] @property def date(self): return self.headers['Date'] @property def status(self): return Status.from_flags(self.flags) @property def flags(self): return self.fdoc.content.get('flags') @property def mailbox_name(self): # FIXME mbox is no longer available, instead we now have mbox_uuid return self.fdoc.content.get('mbox_uuid') @property def _mime_multipart(self): if self._mime: return self._mime mime = MIMEMultipart() for key, value in self.headers.items(): mime[str(key)] = str(value) try: body_to_use = self.body except AttributeError: body_to_use = self.text_plain_body mime.attach(MIMEText(body_to_use, 'plain', self._charset())) self._mime = mime return mime def _charset(self): if 'content_type' in self.headers and 'charset' in self.headers['content_type']: return self._parse_charset_header(self.headers['content_type']) else: return 'utf-8' def _parse_charset_header(self, charset_header, default_charset='utf-8'): try: return re.compile('.*charset=([a-zA-Z0-9-]+)', re.MULTILINE | re.DOTALL).match(charset_header).group(1) except: return default_charset @property def raw(self): return self._mime_multipart.as_string() def _get_chash(self): return sha256.SHA256(self.raw).hexdigest() class InputMail(Mail): FROM_EMAIL_ADDRESS = None def __init__(self): self._raw_message = None self._fd = None self._hd = None self._bd = None self._chash = None self._mime = None self.headers = {} self.body = '' self._status = [] @property def ident(self): return self._get_chash() def get_for_save(self, next_uid, mailbox): docs = [self._fdoc(next_uid, mailbox), self._hdoc()] docs.extend([m for m in self._cdocs()]) return docs def _fdoc(self, next_uid, mailbox): if self._fd: return self._fd fd = {} fd[fields.MBOX] = mailbox fd[fields.MBOX_UUID] = next_uid fd[fields.CONTENT_HASH] = self._get_chash() fd[SIZE_KEY] = len(self.raw) fd[MULTIPART_KEY] = True fd[fields.RECENT] = True fd[fields.TYPE] = fields.FLAGS fd[fields.FLAGS] = Status.to_flags(self._status) self._fd = fd return fd def _get_body_phash(self): return walk.get_body_phash(self._mime_multipart) def _hdoc(self): if self._hd: return self._hd # InputMail does not have a from header but we need it when persisted into soledad. headers = self.headers.copy() headers['From'] = InputMail.FROM_EMAIL_ADDRESS hd = {} hd[HEADERS_KEY] = headers hd[DATE_KEY] = headers['Date'] hd[CONTENT_HASH_KEY] = self._get_chash() hd[MSGID_KEY] = '' hd[MULTIPART_KEY] = True hd[SUBJECT_KEY] = headers.get('Subject') hd[TYPE_KEY] = fields.HEADERS hd[BODY_KEY] = self._get_body_phash() hd[PARTS_MAP_KEY] = \ walk.walk_msg_tree(walk.get_parts(self._mime_multipart), body_phash=self._get_body_phash())['part_map'] self._hd = hd return hd def _cdocs(self): return walk.get_raw_docs(self._mime_multipart, self._mime_multipart.walk()) def to_mime_multipart(self): mime_multipart = MIMEMultipart() for header in ['To', 'Cc', 'Bcc']: if self.headers[header]: mime_multipart[header] = ", ".join(self.headers[header]) if self.headers['Subject']: mime_multipart['Subject'] = self.headers['Subject'] mime_multipart['Date'] = self.headers['Date'] if type(self.body) is list: for part in self.body: mime_multipart.attach(MIMEText(part['raw'], part['content-type'])) else: mime_multipart.attach(MIMEText(self.body, 'plain', 'utf-8')) return mime_multipart def to_smtp_format(self): mime_multipart = self.to_mime_multipart() mime_multipart['From'] = InputMail.FROM_EMAIL_ADDRESS return mime_multipart.as_string() @staticmethod def delivery_error_template(delivery_address): return InputMail.from_dict({ 'body': "Mail undelivered for %s" % delivery_address, 'header': { 'bcc': [], 'cc': [], 'subject': "Mail undelivered for %s" % delivery_address } }) @staticmethod def from_dict(mail_dict): input_mail = InputMail() input_mail.headers = {key.capitalize(): value for key, value in mail_dict.get('header', {}).items()} # XXX this is overriding the property in PixelatedMail input_mail.headers['Date'] = date.iso_now() # XXX this is overriding the property in PixelatedMail input_mail.body = mail_dict.get('body', '') # XXX this is overriding the property in the PixelatedMail input_mail.tags = set(mail_dict.get('tags', [])) input_mail._status = set(mail_dict.get('status', [])) return input_mail @staticmethod def from_python_mail(mail): input_mail = InputMail() input_mail.headers = {key.capitalize(): value for key, value in mail.items()} input_mail.headers['Date'] = date.iso_now() input_mail.headers['Subject'] = mail['Subject'] input_mail.headers['To'] = InputMail.FROM_EMAIL_ADDRESS input_mail._mime = MIMEMultipart() for payload in mail.get_payload(): input_mail._mime.attach(payload) if payload.get_content_type() == 'text/plain': input_mail.body = payload.as_string() return input_mail class PixelatedMail(Mail): @staticmethod def from_soledad(fdoc, hdoc, bdoc, parts=None, soledad_querier=None): mail = PixelatedMail() mail.parts = parts mail.boundary = str(uuid4()).replace('-', '') mail.bdoc = bdoc mail.fdoc = fdoc mail.hdoc = hdoc mail.querier = soledad_querier mail._mime = None return mail def _decode_part(self, part): encoding = part['headers'].get('Content-Transfer-Encoding', '') content_type = self._parse_charset_header(part['headers'].get('Content-Type')) try: decoding_func = self._decoding_function_for_encoding(encoding) return self._decode_content_with_fallback(part['content'], decoding_func, content_type) except Exception: logger.error('Failed to decode mail part with:') logger.error('Content-Transfer-Encoding: %s' % encoding) logger.error('Content-Type: %s' % part['headers'].get('Content-Type')) raise def _decoding_function_for_encoding(self, encoding): decoding_map = { 'quoted-printable': lambda content, content_type: content.decode('quopri').decode(content_type), 'base64': lambda content, content_type: content.decode('base64').decode('utf-8'), '7bit': lambda content, content_type: content.encode(content_type), '8bit': lambda content, content_type: content.encode(content_type) } if encoding in decoding_map: return decoding_map[encoding] else: return decoding_map['8bit'] def _decode_content_with_fallback(self, content, decode_func, content_type): try: return decode_func(content, content_type) # return content.encode(content_type) except ValueError: return content.encode('ascii', 'ignore') @property def alternatives(self): return self.parts.get('alternatives') @property def text_plain_body(self): if self.parts and len(self.alternatives) >= 1: return self._decode_part(self.alternatives[0]) else: return self.bdoc.content['raw'] # plain @property def html_body(self): if self.parts and len(self.alternatives) > 1: html_parts = [e for e in self.alternatives if re.match('text/html', e['headers'].get('Content-Type', ''))] if len(html_parts): return self._decode_part(html_parts[0]) @property def headers(self): _headers = { 'To': [], 'Cc': [], 'Bcc': [] } hdoc_headers = self.hdoc.content['headers'] for header in ['To', 'Cc', 'Bcc']: header_value = self._decode_header(hdoc_headers.get(header)) if not header_value: continue _headers[header] = header_value if type(header_value) is list else header_value.split(',') _headers[header] = [head.strip() for head in compact(_headers[header])] for header in ['From', 'Subject']: _headers[header] = self._decode_header(hdoc_headers.get(header)) try: _headers['Date'] = self._get_date() except Exception: _headers['Date'] = date.iso_now() if self.parts and len(self.parts['alternatives']) > 1: _headers['content_type'] = 'multipart/alternative; boundary="%s"' % self.boundary elif self.hdoc.content['headers'].get('Content-Type'): _headers['content_type'] = hdoc_headers.get('Content-Type') if hdoc_headers.get('Reply-To'): _headers['Reply-To'] = hdoc_headers.get('Reply-To') return _headers def _decode_header_with_fallback(self, entry): try: return decode_header(entry)[0][0] except Exception: return entry.encode('ascii', 'ignore') def _decode_header(self, header): if not header: return None if isinstance(header, list): return [self._decode_header_with_fallback(entry) for entry in header] else: return self._decode_header_with_fallback(header) def _get_date(self): date = self.hdoc.content.get('date', None) try: if not date: received = self.hdoc.content.get('received', None) if received: date = received.split(";")[-1].strip() else: # we can't get a date for this mail, so lets just use now logger.warning('Encountered a mail with missing date and received header fields. ID %s' % self.fdoc.content.get('uid', None)) date = date.iso_now() return dateparser.parse(date).isoformat() except (ValueError, TypeError): date = date.iso_now() return dateparser.parse(date).isoformat() @property def security_casing(self): casing = {"imprints": [], "locks": []} casing["imprints"] = self.signature_information if self.encrypted == "true": casing["locks"] = [{"state": "valid"}] elif self.encrypted == "fail": casing["locks"] = [{"state": "failure"}] return casing @property def tags(self): _tags = self.fdoc.content.get('tags', '[]') return set(_tags) if type(_tags) is list or type(_tags) is set else set(json.loads(_tags)) @property def ident(self): return self.fdoc.content.get('chash') @property def mailbox_name(self): # FIXME mbox is no longer available, instead we now have mbox_uuid return self.fdoc.content.get('mbox_uuid') @property def is_recent(self): return Status('recent') in self.status @property def uid(self): return self.fdoc.content['uid'] @property def flags(self): return self.fdoc.content['flags'] def save(self): return self.querier.save_mail(self) def set_mailbox(self, mailbox_name): self.fdoc.content['mbox'] = mailbox_name def remove_all_tags(self): self.update_tags(set([])) def update_tags(self, tags): self._persist_mail_tags(tags) return self.tags def mark_as_read(self): if Status.SEEN in self.flags: return self self.flags.append(Status.SEEN) self.save() return self def mark_as_unread(self): if Status.SEEN in self.flags: self.flags.remove(Status.SEEN) self.save() return self def mark_as_not_recent(self): if Status.RECENT in self.flags: self.flags.remove(Status.RECENT) self.save() return self def _persist_mail_tags(self, current_tags): self.fdoc.content['tags'] = json.dumps(list(current_tags)) self.save() def has_tag(self, tag): return tag in self.tags @property def signature_information(self): signature = self.hdoc.content["headers"].get("X-Leap-Signature", None) if signature is None or signature.startswith("could not verify"): return [{"state": "no_signature_information"}] else: if signature.startswith("valid"): return [{"state": "valid", "seal": {"validity": "valid"}}] else: return [] @property def encrypted(self): return self.hdoc.content["headers"].get("X-Pixelated-encryption-status", "false") @property def bounced(self): content_type = self.hdoc.content["headers"].get("Content-Type", '') if re.compile('delivery-status').search(content_type): bounce_recipient = self._extract_bounced_address(self.hdoc.content) bounce_daemon = self.headers["From"] return [bounce_recipient, bounce_daemon] if bounce_recipient else False return False def _extract_bounced_address(self, part): part_header = dict(part.get('headers', {})) if 'Final-Recipient' in part_header: if self._bounce_permanent(part_header): return part_header['Final-Recipient'].split(';')[1].strip() else: return False elif 'part_map' in part: for subpart in part['part_map'].values(): result = self._extract_bounced_address(subpart) if result: return result else: continue return False def _bounce_permanent(self, part_headers): status = part_headers.get('Status', '') return status.startswith('5') def as_dict(self): dict_mail = {'header': {k.lower(): v for k, v in self.headers.items()}, 'ident': self.ident, 'tags': list(self.tags), 'status': list(self.status), 'security_casing': self.security_casing, 'textPlainBody': self.text_plain_body, 'htmlBody': self.html_body, 'mailbox': self.mailbox_name.lower(), 'attachments': self.parts['attachments'] if self.parts else []} dict_mail['replying'] = {'single': None, 'all': {'to-field': [], 'cc-field': []}} sender_mail = self.headers.get('Reply-To', self.headers.get('From')) # Issue #215: Fix for existing mails without any from address. if sender_mail is None: sender_mail = InputMail.FROM_EMAIL_ADDRESS recipients = [recipient for recipient in self.headers['To'] if recipient != InputMail.FROM_EMAIL_ADDRESS] recipients.append(sender_mail) ccs = [cc for cc in self.headers['Cc'] if cc != InputMail.FROM_EMAIL_ADDRESS] dict_mail['replying']['single'] = sender_mail dict_mail['replying']['all']['to-field'] = recipients dict_mail['replying']['all']['cc-field'] = ccs return dict_mail def welcome_mail(): current_path = os.path.dirname(os.path.abspath(__file__)) with open(os.path.join(current_path, '..', '..', 'assets', 'welcome.mail')) as mail_template_file: mail_template = message_from_file(mail_template_file) return InputMail.from_python_mail(mail_template)