summaryrefslogtreecommitdiff
path: root/service/src/pixelated/adapter/mailstore/leap_mailstore.py
diff options
context:
space:
mode:
authorKali Kaneko <kali@leap.se>2017-07-25 11:40:11 -0400
committerKali Kaneko <kali@leap.se>2017-07-25 11:40:29 -0400
commit91e4481c450eb7eb928debc1cb7fa59bdb63dd7b (patch)
tree8fd7e6e77b6df669c33d96b7edad6db3cbe14dfe /service/src/pixelated/adapter/mailstore/leap_mailstore.py
parente4f755309d4cf5cfb6b0bcc62ed73d6070956ab5 (diff)
[pkg] packaging and path changes
- move all the pixelated python package under src/ - move the pixelated_www package under the leap namespace - allow to set globally the static folder - add hours and minutes to the timestamp in package version, to allow for several releases a day.
Diffstat (limited to 'service/src/pixelated/adapter/mailstore/leap_mailstore.py')
-rw-r--r--service/src/pixelated/adapter/mailstore/leap_mailstore.py406
1 files changed, 406 insertions, 0 deletions
diff --git a/service/src/pixelated/adapter/mailstore/leap_mailstore.py b/service/src/pixelated/adapter/mailstore/leap_mailstore.py
new file mode 100644
index 00000000..288223dd
--- /dev/null
+++ b/service/src/pixelated/adapter/mailstore/leap_mailstore.py
@@ -0,0 +1,406 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+import re
+from email.header import decode_header
+from uuid import uuid4
+
+from leap.bitmask.mail.adaptors.soledad import SoledadMailAdaptor
+from leap.bitmask.mail.mail import Message
+from twisted.internet import defer
+from twisted.internet.defer import FirstError, DeferredList
+
+from pixelated.adapter.mailstore.body_parser import BodyParser
+from pixelated.adapter.mailstore.mailstore import MailStore, underscore_uuid
+from pixelated.adapter.model.mail import Mail, InputMail
+from pixelated.support.functional import to_unicode
+from pixelated.support import date
+
+
+class AttachmentInfo(object):
+ def __init__(self, ident, name, encoding=None, ctype='application/octet-stream', size=0):
+ self.ident = ident
+ self.name = name
+ self.encoding = encoding
+ self.ctype = ctype
+ self.size = size
+
+ def __repr__(self):
+ return 'AttachmentInfo[%s, %s, %s]' % (self.ident, self.name, self.encoding)
+
+ def __str__(self):
+ return 'AttachmentInfo[%s, %s, %s]' % (self.ident, self.name, self.encoding)
+
+ def as_dict(self):
+ return {'ident': self.ident, 'name': self.name, 'encoding': self.encoding, 'size': self.size, 'content-type': self.ctype}
+
+
+class LeapMail(Mail):
+
+ def __init__(self, mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None, attachments=[]):
+ self._mail_id = mail_id
+ self._mailbox_name = mailbox_name
+ self._headers = headers if headers is not None else {}
+ self._body = to_unicode(body)
+ self.tags = set(tags) # TODO test that asserts copy
+ self._flags = set(flags) # TODO test that asserts copy
+ self._attachments = attachments
+
+ @property
+ def headers(self):
+ cpy = dict(self._headers)
+ for name in set(self._headers.keys()).intersection(['To', 'Cc', 'Bcc']):
+ cpy[name] = [address.strip() for address in (self._headers[name].split(',') if self._headers[name] else [])]
+
+ return cpy
+
+ @property
+ def ident(self):
+ return self._mail_id
+
+ @property
+ def mail_id(self):
+ return self._mail_id
+
+ @property
+ def body(self):
+ return self._body
+
+ @property
+ def flags(self):
+ return self._flags
+
+ @property
+ def mailbox_name(self):
+ return self._mailbox_name
+
+ @property
+ def security_casing(self):
+ casing = dict(imprints=self._signature_information(), locks=[])
+ if self._encrypted() == "decrypted":
+ casing["locks"] = [{"state": "valid"}]
+ return casing
+
+ def _encrypted(self):
+ return self.headers.get("X-Leap-Encryption", "false")
+
+ def _signature_information(self):
+ signature = self.headers.get("X-Leap-Signature", None)
+ if signature is None or signature.startswith("could not verify"):
+ return [{"state": "no_signature_information"}]
+ else:
+ if signature.startswith("valid"):
+ return [{"state": "valid", "seal": {"validity": "valid"}}]
+ else:
+ return []
+
+ @property
+ def raw(self):
+ result = u''
+ for k, v in self._headers.items():
+ content, encoding = decode_header(v)[0]
+ if encoding:
+ result += '%s: %s\n' % (k, unicode(content, encoding=encoding))
+ else:
+ result += '%s: %s\n' % (k, v)
+ result += '\n'
+
+ if self._body:
+ result = result + self._body
+
+ return result
+
+ def _remove_duplicates(self, values):
+ return list(set(values))
+
+ def _decoded_header_utf_8(self, header_value):
+ if isinstance(header_value, list):
+ return self._remove_duplicates([self._decoded_header_utf_8(v) for v in header_value])
+ elif header_value is not None:
+ def encode_chunk(content, encoding):
+ return unicode(content.strip(), encoding=encoding or 'ascii', errors='ignore')
+
+ try:
+ encoded_chunks = [encode_chunk(content, encoding) for content, encoding in decode_header(header_value)]
+ return ' '.join(encoded_chunks) # decode_header strips whitespaces on all chunks, joining over ' ' is only a workaround, not a proper fix
+ except UnicodeEncodeError:
+ return unicode(header_value.encode('ascii', errors='ignore'))
+
+ def as_dict(self):
+ return {
+ 'header': {k.lower(): self._decoded_header_utf_8(v) for k, v in self.headers.items()},
+ 'ident': self._mail_id,
+ 'tags': self.tags,
+ 'status': list(self.status),
+ 'body': self._body,
+ 'security_casing': self.security_casing,
+ 'textPlainBody': self._body,
+ 'mailbox': self._mailbox_name.lower(),
+ 'attachments': [attachment.as_dict() for attachment in self._attachments]
+ }
+
+ @staticmethod
+ def from_dict(mail_dict):
+ # TODO: implement this method and also write tests for it
+ headers = {key.capitalize(): value for key, value in mail_dict.get('header', {}).items()}
+ headers['Date'] = date.mail_date_now()
+ body = mail_dict.get('body', '')
+ tags = set(mail_dict.get('tags', []))
+ status = set(mail_dict.get('status', []))
+ attachments = []
+
+ # mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None, attachments=[]
+ return LeapMail(None, None, headers, tags, set(), body, attachments)
+
+
+def _extract_filename(headers, default_filename='UNNAMED'):
+ content_disposition = headers.get('Content-Disposition') or headers.get('content-disposition', '')
+ filename = _extract_filename_from_name_header_part(content_disposition)
+ if not filename:
+ filename = headers.get('Content-Description', '')
+ if not filename:
+ content_type = headers.get('Content-Type', '')
+ filename = _extract_filename_from_name_header_part(content_type)
+
+ if not filename:
+ filename = default_filename
+
+ return filename
+
+
+def _extract_filename_from_name_header_part(header_value):
+ match = re.compile('.*name=\"?(.*[^\"\'])').search(header_value)
+ filename = ''
+ if match:
+ filename = match.group(1)
+ return filename
+
+
+class LeapMailStore(MailStore):
+ __slots__ = ('soledad')
+
+ def __init__(self, soledad):
+ self.soledad = soledad
+
+ @defer.inlineCallbacks
+ def get_mail(self, mail_id, include_body=False):
+ message = yield self._fetch_msg_from_soledad(mail_id)
+ if not _is_empty_message(message):
+ leap_mail = yield self._leap_message_to_leap_mail(mail_id, message, include_body)
+ else:
+ leap_mail = None
+
+ defer.returnValue(leap_mail)
+
+ @defer.inlineCallbacks
+ def get_mails(self, mail_ids, gracefully_ignore_errors=False, include_body=False):
+ deferreds = []
+ for mail_id in mail_ids:
+ deferreds.append(self.get_mail(mail_id, include_body=include_body))
+
+ if gracefully_ignore_errors:
+ results = yield DeferredList(deferreds, consumeErrors=True)
+ defer.returnValue([mail for ok, mail in results if ok and mail is not None])
+ else:
+ result = yield defer.gatherResults(deferreds, consumeErrors=True)
+ defer.returnValue(result)
+
+ @defer.inlineCallbacks
+ def update_mail(self, mail):
+ message = yield self._fetch_msg_from_soledad(mail.mail_id)
+ message.get_wrapper().set_tags(tuple(mail.tags))
+ message.get_wrapper().set_flags(tuple(mail.flags))
+ yield self._update_mail(message) # TODO assert this is yielded (otherwise asynchronous)
+
+ @defer.inlineCallbacks
+ def all_mails(self, gracefully_ignore_errors=False):
+ mdocs = yield self.soledad.get_from_index('by-type', 'meta')
+
+ mail_ids = map(lambda doc: doc.doc_id, mdocs)
+
+ mails = yield self.get_mails(mail_ids, gracefully_ignore_errors=gracefully_ignore_errors, include_body=True)
+ defer.returnValue(mails)
+
+ @defer.inlineCallbacks
+ def add_mailbox(self, mailbox_name):
+ mailbox = yield self._get_or_create_mailbox(mailbox_name)
+ defer.returnValue(mailbox)
+
+ @defer.inlineCallbacks
+ def get_mailbox_names(self):
+ mbox_map = set((yield self._mailbox_uuid_to_name_map()).values())
+
+ defer.returnValue(mbox_map.union({'INBOX'}))
+
+ @defer.inlineCallbacks
+ def _mailbox_uuid_to_name_map(self):
+ map = {}
+ mbox_docs = yield self.soledad.get_from_index('by-type', 'mbox')
+ for doc in mbox_docs:
+ map[underscore_uuid(doc.content.get('uuid'))] = doc.content.get('mbox')
+
+ defer.returnValue(map)
+
+ @defer.inlineCallbacks
+ def add_mail(self, mailbox_name, raw_msg):
+ mailbox = yield self._get_or_create_mailbox(mailbox_name)
+ message = SoledadMailAdaptor().get_msg_from_string(Message, raw_msg)
+ message.get_wrapper().set_mbox_uuid(mailbox.uuid)
+
+ yield SoledadMailAdaptor().create_msg(self.soledad, message)
+
+ # add behavious from insert_mdoc_id from mail.py
+ mail = yield self._leap_message_to_leap_mail(message.get_wrapper().mdoc.doc_id, message, include_body=True) # TODO test that asserts include_body
+ defer.returnValue(mail)
+
+ @defer.inlineCallbacks
+ def delete_mail(self, mail_id):
+ message = yield self._fetch_msg_from_soledad(mail_id)
+ if message and message.get_wrapper().mdoc.doc_id:
+ yield message.get_wrapper().delete(self.soledad)
+ defer.returnValue(True)
+ defer.returnValue(False)
+
+ @defer.inlineCallbacks
+ def get_mailbox_mail_ids(self, mailbox_name):
+ mailbox = yield self._get_or_create_mailbox(mailbox_name)
+ fdocs = yield self.soledad.get_from_index('by-type-and-mbox-uuid', 'flags', underscore_uuid(mailbox.uuid))
+
+ mail_ids = map(lambda doc: _fdoc_id_to_mdoc_id(doc.doc_id), fdocs)
+
+ defer.returnValue(mail_ids)
+
+ @defer.inlineCallbacks
+ def delete_mailbox(self, mailbox_name):
+ mbx_wrapper = yield self._get_or_create_mailbox(mailbox_name)
+ yield SoledadMailAdaptor().delete_mbox(self.soledad, mbx_wrapper)
+
+ @defer.inlineCallbacks
+ def copy_mail_to_mailbox(self, mail_id, mailbox_name):
+ message = yield self._fetch_msg_from_soledad(mail_id, load_body=True)
+ mailbox = yield self._get_or_create_mailbox(mailbox_name)
+ copy_wrapper = yield message.get_wrapper().copy(self.soledad, mailbox.uuid)
+
+ leap_message = Message(copy_wrapper)
+
+ mail = yield self._leap_message_to_leap_mail(copy_wrapper.mdoc.doc_id, leap_message, include_body=False)
+
+ defer.returnValue(mail)
+
+ @defer.inlineCallbacks
+ def move_mail_to_mailbox(self, mail_id, mailbox_name):
+ mail_copy = yield self.copy_mail_to_mailbox(mail_id, mailbox_name)
+ yield self.delete_mail(mail_id)
+ defer.returnValue(mail_copy)
+
+ def _update_mail(self, message):
+ return message.get_wrapper().update(self.soledad)
+
+ @defer.inlineCallbacks
+ def _leap_message_to_leap_mail(self, mail_id, message, include_body):
+ if include_body:
+ # TODO use body from message if available
+ body = yield self._raw_message_body(message)
+ else:
+ body = None
+
+ # fetch mailbox name by mbox_uuid
+ mbox_uuid = message.get_wrapper().fdoc.mbox_uuid
+ mbox_name = yield self._mailbox_name_from_uuid(mbox_uuid)
+ attachments = self._extract_attachment_info_from(message)
+ mail = LeapMail(mail_id, mbox_name, message.get_wrapper().hdoc.headers, set(message.get_tags()), set(message.get_flags()), body=body, attachments=attachments) # TODO assert flags are passed on
+
+ defer.returnValue(mail)
+
+ @defer.inlineCallbacks
+ def _raw_message_body(self, message):
+ content_doc = (yield message.get_wrapper().get_body(self.soledad))
+ parser = BodyParser('', content_type='text/plain', content_transfer_encoding='UTF-8')
+ # It fix the problem when leap doesn'r found body_phash and returns empty string
+ if not isinstance(content_doc, str):
+ parser = BodyParser(content_doc.raw, content_type=content_doc.content_type,
+ content_transfer_encoding=content_doc.content_transfer_encoding, charset=content_doc.charset)
+
+ defer.returnValue(parser.parsed_content())
+
+ @defer.inlineCallbacks
+ def _mailbox_name_from_uuid(self, uuid):
+ map = (yield self._mailbox_uuid_to_name_map())
+ defer.returnValue(map.get(uuid, ''))
+
+ @defer.inlineCallbacks
+ def _get_or_create_mailbox(self, mailbox_name):
+ mailbox_name_upper = mailbox_name.upper()
+ mbx = yield SoledadMailAdaptor().get_or_create_mbox(self.soledad, mailbox_name_upper)
+ if mbx.uuid is None:
+ mbx.uuid = str(uuid4())
+ yield mbx.update(self.soledad)
+ defer.returnValue(mbx)
+
+ def _fetch_msg_from_soledad(self, mail_id, load_body=False):
+ return SoledadMailAdaptor().get_msg_from_mdoc_id(Message, self.soledad, mail_id, get_cdocs=load_body)
+
+ @defer.inlineCallbacks
+ def _dump_soledad(self):
+ gen, docs = yield self.soledad.get_all_docs()
+ for doc in docs:
+ print '\n%s\n' % doc
+
+ def _extract_attachment_info_from(self, message):
+ wrapper = message.get_wrapper()
+ part_maps = wrapper.hdoc.part_map
+ return self._extract_part_map(part_maps)
+
+ def _is_attachment(self, part_map, headers):
+ disposition = headers.get('Content-Disposition') or headers.get('content-disposition')
+ content_type = part_map['ctype']
+
+ if 'multipart' in content_type:
+ return False
+
+ if 'text/plain' == content_type and ((disposition == 'inline') or (disposition is None)):
+ return False
+
+ return True
+
+ def _create_attachment_info_from(self, part_map, headers):
+ ident = part_map['phash']
+ name = _extract_filename(headers)
+ encoding = headers.get('Content-Transfer-Encoding', None)
+ ctype = part_map.get('ctype') or headers.get('Content-Type')
+ size = part_map.get('size', 0)
+
+ return AttachmentInfo(ident, name, encoding, ctype, size)
+
+ def _extract_part_map(self, part_maps):
+ result = []
+
+ for nr, part_map in part_maps.items():
+ if 'headers' in part_map and 'phash' in part_map:
+ headers = {header[0]: header[1] for header in part_map['headers']}
+ if self._is_attachment(part_map, headers):
+ result.append(self._create_attachment_info_from(part_map, headers))
+ if 'part_map' in part_map:
+ result += self._extract_part_map(part_map['part_map'])
+
+ return result
+
+
+def _is_empty_message(message):
+ return (message is None) or (message.get_wrapper().mdoc.doc_id is None)
+
+
+def _fdoc_id_to_mdoc_id(fdoc_id):
+ return 'M' + fdoc_id[1:]