diff options
Diffstat (limited to 'service/pixelated/adapter/model/mail.py')
| -rw-r--r-- | service/pixelated/adapter/model/mail.py | 378 |
1 files changed, 69 insertions, 309 deletions
diff --git a/service/pixelated/adapter/model/mail.py b/service/pixelated/adapter/model/mail.py index 464e0343..b89e511a 100644 --- a/service/pixelated/adapter/model/mail.py +++ b/service/pixelated/adapter/model/mail.py @@ -21,21 +21,38 @@ import dateutil.parser as dateparser from uuid import uuid4 from email import message_from_file from email.mime.text import MIMEText -from email.header import decode_header +from email.header import decode_header, Header from email.MIMEMultipart import MIMEMultipart from pycryptopp.hash import sha256 -from leap.mail.imap.fields import fields +from leap.mail.adaptors import soledad_indexes as fields import leap.mail.walk as walk from pixelated.adapter.model.status import Status from pixelated.support import date from pixelated.support.functional import compact +from twisted.internet import defer + logger = logging.getLogger(__name__) +TYPE_KEY = 'type' +CONTENT_HASH_KEY = 'chash' +HEADERS_KEY = 'headers' +DATE_KEY = 'date' +SUBJECT_KEY = 'subject' +PARTS_MAP_KEY = 'part_map' +BODY_KEY = 'body' +MSGID_KEY = 'msgid' +MULTIPART_KEY = 'multi' +SIZE_KEY = 'size' + class Mail(object): @property + def from_sender(self): + return self.headers['From'] + + @property def to(self): return self.headers['To'] @@ -48,6 +65,10 @@ class Mail(object): return self.headers['Bcc'] @property + def subject(self): + return self.headers['Subject'] + + @property def date(self): return self.headers['Date'] @@ -61,7 +82,17 @@ class Mail(object): @property def mailbox_name(self): - return self.fdoc.content.get('mbox') + # FIXME mbox is no longer available, instead we now have mbox_uuid + return self.fdoc.content.get('mbox', 'INBOX') + + def _encode_header_value_list(self, header_value_list): + return [self._encode_header_value(v) for v in header_value_list] + + def _encode_header_value(self, header_value): + if isinstance(header_value, unicode): + return str(Header(header_value, 'utf-8')) + else: + return str(header_value) @property def _mime_multipart(self): @@ -69,7 +100,10 @@ class Mail(object): return self._mime mime = MIMEMultipart() for key, value in self.headers.items(): - mime[str(key)] = str(value) + if isinstance(value, list): + mime[str(key)] = ', '.join(self._encode_header_value_list(value)) + else: + mime[str(key)] = self._encode_header_value(value) try: body_to_use = self.body @@ -128,19 +162,19 @@ class InputMail(Mail): return self._fd fd = {} - fd[fields.MBOX_KEY] = mailbox - fd[fields.UID_KEY] = next_uid - fd[fields.CONTENT_HASH_KEY] = self._get_chash() - fd[fields.SIZE_KEY] = len(self.raw) - fd[fields.MULTIPART_KEY] = True - fd[fields.RECENT_KEY] = True - fd[fields.TYPE_KEY] = fields.TYPE_FLAGS_VAL - fd[fields.FLAGS_KEY] = Status.to_flags(self._status) + fd[fields.MBOX] = mailbox + fd[fields.MBOX_UUID] = next_uid + fd[fields.CONTENT_HASH] = self._get_chash() + fd[SIZE_KEY] = len(self.raw) + fd[MULTIPART_KEY] = True + fd[fields.RECENT] = True + fd[fields.TYPE] = fields.FLAGS + fd[fields.FLAGS] = Status.to_flags(self._status) self._fd = fd return fd def _get_body_phash(self): - return walk.get_body_phash_multi(walk.get_payloads(self._mime_multipart)) + return walk.get_body_phash(self._mime_multipart) def _hdoc(self): if self._hd: @@ -151,15 +185,15 @@ class InputMail(Mail): headers['From'] = InputMail.FROM_EMAIL_ADDRESS hd = {} - hd[fields.HEADERS_KEY] = headers - hd[fields.DATE_KEY] = headers['Date'] - hd[fields.CONTENT_HASH_KEY] = self._get_chash() - hd[fields.MSGID_KEY] = '' - hd[fields.MULTIPART_KEY] = True - hd[fields.SUBJECT_KEY] = headers.get('Subject') - hd[fields.TYPE_KEY] = fields.TYPE_HEADERS_VAL - hd[fields.BODY_KEY] = self._get_body_phash() - hd[fields.PARTS_MAP_KEY] = \ + hd[HEADERS_KEY] = headers + hd[DATE_KEY] = headers['Date'] + hd[CONTENT_HASH_KEY] = self._get_chash() + hd[MSGID_KEY] = '' + hd[MULTIPART_KEY] = True + hd[SUBJECT_KEY] = headers.get('Subject') + hd[TYPE_KEY] = fields.HEADERS + hd[BODY_KEY] = self._get_body_phash() + hd[PARTS_MAP_KEY] = \ walk.walk_msg_tree(walk.get_parts(self._mime_multipart), body_phash=self._get_body_phash())['part_map'] self._hd = hd @@ -172,12 +206,15 @@ class InputMail(Mail): mime_multipart = MIMEMultipart() for header in ['To', 'Cc', 'Bcc']: - if self.headers[header]: + if self.headers.get(header): mime_multipart[header] = ", ".join(self.headers[header]) - if self.headers['Subject']: + if self.headers.get('Subject'): mime_multipart['Subject'] = self.headers['Subject'] + if self.headers.get('From'): + mime_multipart['From'] = self.headers['From'] + mime_multipart['Date'] = self.headers['Date'] if type(self.body) is list: for part in self.body: @@ -207,13 +244,10 @@ class InputMail(Mail): input_mail = InputMail() input_mail.headers = {key.capitalize(): value for key, value in mail_dict.get('header', {}).items()} - # XXX this is overriding the property in PixelatedMail - input_mail.headers['Date'] = date.iso_now() + input_mail.headers['Date'] = date.mail_date_now() - # XXX this is overriding the property in PixelatedMail input_mail.body = mail_dict.get('body', '') - # XXX this is overriding the property in the PixelatedMail input_mail.tags = set(mail_dict.get('tags', [])) input_mail._status = set(mail_dict.get('status', [])) @@ -222,292 +256,18 @@ class InputMail(Mail): @staticmethod def from_python_mail(mail): input_mail = InputMail() - input_mail.headers = {key.capitalize(): value for key, value in mail.items()} - input_mail.headers['Date'] = date.iso_now() - input_mail.headers['Subject'] = mail['Subject'] - input_mail.headers['To'] = InputMail.FROM_EMAIL_ADDRESS - input_mail._mime = MIMEMultipart() + input_mail.headers = {unicode(key.capitalize()): unicode(value) for key, value in mail.items()} + input_mail.headers[u'Date'] = unicode(date.mail_date_now()) + input_mail.headers[u'To'] = [u''] + for payload in mail.get_payload(): - input_mail._mime.attach(payload) + input_mail._mime_multipart.attach(payload) if payload.get_content_type() == 'text/plain': - input_mail.body = payload.as_string() + input_mail.body = unicode(payload.as_string()) + input_mail._mime = input_mail.to_mime_multipart() return input_mail -class PixelatedMail(Mail): - - @staticmethod - def from_soledad(fdoc, hdoc, bdoc, parts=None, soledad_querier=None): - mail = PixelatedMail() - mail.parts = parts - mail.boundary = str(uuid4()).replace('-', '') - mail.bdoc = bdoc - mail.fdoc = fdoc - mail.hdoc = hdoc - mail.querier = soledad_querier - mail._mime = None - return mail - - def _decode_part(self, part): - encoding = part['headers'].get('Content-Transfer-Encoding', '') - content_type = self._parse_charset_header(part['headers'].get('Content-Type')) - - try: - decoding_func = self._decoding_function_for_encoding(encoding) - return self._decode_content_with_fallback(part['content'], decoding_func, content_type) - except Exception: - logger.error('Failed to decode mail part with:') - logger.error('Content-Transfer-Encoding: %s' % encoding) - logger.error('Content-Type: %s' % part['headers'].get('Content-Type')) - raise - - def _decoding_function_for_encoding(self, encoding): - decoding_map = { - 'quoted-printable': lambda content, content_type: content.decode('quopri').decode(content_type), - 'base64': lambda content, content_type: content.decode('base64').decode('utf-8'), - '7bit': lambda content, content_type: content.encode(content_type), - '8bit': lambda content, content_type: content.encode(content_type) - } - if encoding in decoding_map: - return decoding_map[encoding] - else: - return decoding_map['8bit'] - - def _decode_content_with_fallback(self, content, decode_func, content_type): - try: - return decode_func(content, content_type) - # return content.encode(content_type) - except ValueError: - return content.encode('ascii', 'ignore') - - @property - def alternatives(self): - return self.parts.get('alternatives') - - @property - def text_plain_body(self): - if self.parts and len(self.alternatives) >= 1: - return self._decode_part(self.alternatives[0]) - else: - return self.bdoc.content['raw'] # plain - - @property - def html_body(self): - if self.parts and len(self.alternatives) > 1: - html_parts = [e for e in self.alternatives if re.match('text/html', e['headers'].get('Content-Type', ''))] - if len(html_parts): - return self._decode_part(html_parts[0]) - - @property - def headers(self): - _headers = { - 'To': [], - 'Cc': [], - 'Bcc': [] - } - hdoc_headers = self.hdoc.content['headers'] - - for header in ['To', 'Cc', 'Bcc']: - header_value = self._decode_header(hdoc_headers.get(header)) - if not header_value: - continue - _headers[header] = header_value if type(header_value) is list else header_value.split(',') - _headers[header] = [head.strip() for head in compact(_headers[header])] - - for header in ['From', 'Subject']: - _headers[header] = self._decode_header(hdoc_headers.get(header)) - - try: - _headers['Date'] = self._get_date() - except Exception: - _headers['Date'] = date.iso_now() - - if self.parts and len(self.parts['alternatives']) > 1: - _headers['content_type'] = 'multipart/alternative; boundary="%s"' % self.boundary - elif self.hdoc.content['headers'].get('Content-Type'): - _headers['content_type'] = hdoc_headers.get('Content-Type') - - if hdoc_headers.get('Reply-To'): - _headers['Reply-To'] = hdoc_headers.get('Reply-To') - - return _headers - - def _decode_header_with_fallback(self, entry): - try: - return decode_header(entry)[0][0] - except Exception: - return entry.encode('ascii', 'ignore') - - def _decode_header(self, header): - if not header: - return None - if isinstance(header, list): - return [self._decode_header_with_fallback(entry) for entry in header] - else: - return self._decode_header_with_fallback(header) - - def _get_date(self): - date = self.hdoc.content.get('date', None) - try: - if not date: - received = self.hdoc.content.get('received', None) - if received: - date = received.split(";")[-1].strip() - else: - # we can't get a date for this mail, so lets just use now - logger.warning('Encountered a mail with missing date and received header fields. ID %s' % self.fdoc.content.get('uid', None)) - date = date.iso_now() - return dateparser.parse(date).isoformat() - except (ValueError, TypeError): - date = date.iso_now() - return dateparser.parse(date).isoformat() - - @property - def security_casing(self): - casing = {"imprints": [], "locks": []} - casing["imprints"] = self.signature_information - if self.encrypted == "true": - casing["locks"] = [{"state": "valid"}] - elif self.encrypted == "fail": - casing["locks"] = [{"state": "failure"}] - return casing - - @property - def tags(self): - _tags = self.fdoc.content.get('tags', '[]') - return set(_tags) if type(_tags) is list or type(_tags) is set else set(json.loads(_tags)) - - @property - def ident(self): - return self.fdoc.content.get('chash') - - @property - def mailbox_name(self): - return self.fdoc.content.get('mbox') - - @property - def is_recent(self): - return Status('recent') in self.status - - @property - def uid(self): - return self.fdoc.content['uid'] - - @property - def flags(self): - return self.fdoc.content['flags'] - - def save(self): - return self.querier.save_mail(self) - - def set_mailbox(self, mailbox_name): - self.fdoc.content['mbox'] = mailbox_name - - def remove_all_tags(self): - self.update_tags(set([])) - - def update_tags(self, tags): - self._persist_mail_tags(tags) - return self.tags - - def mark_as_read(self): - if Status.SEEN in self.flags: - return self - self.flags.append(Status.SEEN) - self.save() - return self - - def mark_as_unread(self): - if Status.SEEN in self.flags: - self.flags.remove(Status.SEEN) - self.save() - return self - - def mark_as_not_recent(self): - if Status.RECENT in self.flags: - self.flags.remove(Status.RECENT) - self.save() - return self - - def _persist_mail_tags(self, current_tags): - self.fdoc.content['tags'] = json.dumps(list(current_tags)) - self.save() - - def has_tag(self, tag): - return tag in self.tags - - @property - def signature_information(self): - signature = self.hdoc.content["headers"].get("X-Leap-Signature", None) - if signature is None or signature.startswith("could not verify"): - return [{"state": "no_signature_information"}] - else: - if signature.startswith("valid"): - return [{"state": "valid", "seal": {"validity": "valid"}}] - else: - return [] - - @property - def encrypted(self): - return self.hdoc.content["headers"].get("X-Pixelated-encryption-status", "false") - - @property - def bounced(self): - content_type = self.hdoc.content["headers"].get("Content-Type", '') - if re.compile('delivery-status').search(content_type): - bounce_recipient = self._extract_bounced_address(self.hdoc.content) - bounce_daemon = self.headers["From"] - return [bounce_recipient, bounce_daemon] if bounce_recipient else False - - return False - - def _extract_bounced_address(self, part): - part_header = dict(part.get('headers', {})) - if 'Final-Recipient' in part_header: - if self._bounce_permanent(part_header): - return part_header['Final-Recipient'].split(';')[1].strip() - else: - return False - elif 'part_map' in part: - for subpart in part['part_map'].values(): - result = self._extract_bounced_address(subpart) - if result: - return result - else: - continue - return False - - def _bounce_permanent(self, part_headers): - status = part_headers.get('Status', '') - return status.startswith('5') - - def as_dict(self): - dict_mail = {'header': {k.lower(): v for k, v in self.headers.items()}, - 'ident': self.ident, - 'tags': list(self.tags), - 'status': list(self.status), - 'security_casing': self.security_casing, - 'textPlainBody': self.text_plain_body, - 'htmlBody': self.html_body, - 'mailbox': self.mailbox_name.lower(), - 'attachments': self.parts['attachments'] if self.parts else []} - dict_mail['replying'] = {'single': None, 'all': {'to-field': [], 'cc-field': []}} - - sender_mail = self.headers.get('Reply-To', self.headers.get('From')) - # Issue #215: Fix for existing mails without any from address. - if sender_mail is None: - sender_mail = InputMail.FROM_EMAIL_ADDRESS - - recipients = [recipient for recipient in self.headers['To'] if recipient != InputMail.FROM_EMAIL_ADDRESS] - recipients.append(sender_mail) - ccs = [cc for cc in self.headers['Cc'] if cc != InputMail.FROM_EMAIL_ADDRESS] - - dict_mail['replying']['single'] = sender_mail - dict_mail['replying']['all']['to-field'] = recipients - dict_mail['replying']['all']['cc-field'] = ccs - return dict_mail - - def welcome_mail(): current_path = os.path.dirname(os.path.abspath(__file__)) with open(os.path.join(current_path, '..', '..', 'assets', 'welcome.mail')) as mail_template_file: |