From 91e4481c450eb7eb928debc1cb7fa59bdb63dd7b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 25 Jul 2017 11:40:11 -0400 Subject: [pkg] packaging and path changes - move all the pixelated python package under src/ - move the pixelated_www package under the leap namespace - allow to set globally the static folder - add hours and minutes to the timestamp in package version, to allow for several releases a day. --- .../pixelated/adapter/mailstore/leap_mailstore.py | 406 --------------------- 1 file changed, 406 deletions(-) delete mode 100644 service/pixelated/adapter/mailstore/leap_mailstore.py (limited to 'service/pixelated/adapter/mailstore/leap_mailstore.py') diff --git a/service/pixelated/adapter/mailstore/leap_mailstore.py b/service/pixelated/adapter/mailstore/leap_mailstore.py deleted file mode 100644 index 288223dd..00000000 --- a/service/pixelated/adapter/mailstore/leap_mailstore.py +++ /dev/null @@ -1,406 +0,0 @@ -# -# Copyright (c) 2015 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . -import re -from email.header import decode_header -from uuid import uuid4 - -from leap.bitmask.mail.adaptors.soledad import SoledadMailAdaptor -from leap.bitmask.mail.mail import Message -from twisted.internet import defer -from twisted.internet.defer import FirstError, DeferredList - -from pixelated.adapter.mailstore.body_parser import BodyParser -from pixelated.adapter.mailstore.mailstore import MailStore, underscore_uuid -from pixelated.adapter.model.mail import Mail, InputMail -from pixelated.support.functional import to_unicode -from pixelated.support import date - - -class AttachmentInfo(object): - def __init__(self, ident, name, encoding=None, ctype='application/octet-stream', size=0): - self.ident = ident - self.name = name - self.encoding = encoding - self.ctype = ctype - self.size = size - - def __repr__(self): - return 'AttachmentInfo[%s, %s, %s]' % (self.ident, self.name, self.encoding) - - def __str__(self): - return 'AttachmentInfo[%s, %s, %s]' % (self.ident, self.name, self.encoding) - - def as_dict(self): - return {'ident': self.ident, 'name': self.name, 'encoding': self.encoding, 'size': self.size, 'content-type': self.ctype} - - -class LeapMail(Mail): - - def __init__(self, mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None, attachments=[]): - self._mail_id = mail_id - self._mailbox_name = mailbox_name - self._headers = headers if headers is not None else {} - self._body = to_unicode(body) - self.tags = set(tags) # TODO test that asserts copy - self._flags = set(flags) # TODO test that asserts copy - self._attachments = attachments - - @property - def headers(self): - cpy = dict(self._headers) - for name in set(self._headers.keys()).intersection(['To', 'Cc', 'Bcc']): - cpy[name] = [address.strip() for address in (self._headers[name].split(',') if self._headers[name] else [])] - - return cpy - - @property - def ident(self): - return self._mail_id - - @property - def mail_id(self): - return self._mail_id - - @property - def body(self): - return self._body - - @property - def flags(self): - return self._flags - - @property - def mailbox_name(self): - return self._mailbox_name - - @property - def security_casing(self): - casing = dict(imprints=self._signature_information(), locks=[]) - if self._encrypted() == "decrypted": - casing["locks"] = [{"state": "valid"}] - return casing - - def _encrypted(self): - return self.headers.get("X-Leap-Encryption", "false") - - def _signature_information(self): - signature = self.headers.get("X-Leap-Signature", None) - if signature is None or signature.startswith("could not verify"): - return [{"state": "no_signature_information"}] - else: - if signature.startswith("valid"): - return [{"state": "valid", "seal": {"validity": "valid"}}] - else: - return [] - - @property - def raw(self): - result = u'' - for k, v in self._headers.items(): - content, encoding = decode_header(v)[0] - if encoding: - result += '%s: %s\n' % (k, unicode(content, encoding=encoding)) - else: - result += '%s: %s\n' % (k, v) - result += '\n' - - if self._body: - result = result + self._body - - return result - - def _remove_duplicates(self, values): - return list(set(values)) - - def _decoded_header_utf_8(self, header_value): - if isinstance(header_value, list): - return self._remove_duplicates([self._decoded_header_utf_8(v) for v in header_value]) - elif header_value is not None: - def encode_chunk(content, encoding): - return unicode(content.strip(), encoding=encoding or 'ascii', errors='ignore') - - try: - encoded_chunks = [encode_chunk(content, encoding) for content, encoding in decode_header(header_value)] - return ' '.join(encoded_chunks) # decode_header strips whitespaces on all chunks, joining over ' ' is only a workaround, not a proper fix - except UnicodeEncodeError: - return unicode(header_value.encode('ascii', errors='ignore')) - - def as_dict(self): - return { - 'header': {k.lower(): self._decoded_header_utf_8(v) for k, v in self.headers.items()}, - 'ident': self._mail_id, - 'tags': self.tags, - 'status': list(self.status), - 'body': self._body, - 'security_casing': self.security_casing, - 'textPlainBody': self._body, - 'mailbox': self._mailbox_name.lower(), - 'attachments': [attachment.as_dict() for attachment in self._attachments] - } - - @staticmethod - def from_dict(mail_dict): - # TODO: implement this method and also write tests for it - headers = {key.capitalize(): value for key, value in mail_dict.get('header', {}).items()} - headers['Date'] = date.mail_date_now() - body = mail_dict.get('body', '') - tags = set(mail_dict.get('tags', [])) - status = set(mail_dict.get('status', [])) - attachments = [] - - # mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None, attachments=[] - return LeapMail(None, None, headers, tags, set(), body, attachments) - - -def _extract_filename(headers, default_filename='UNNAMED'): - content_disposition = headers.get('Content-Disposition') or headers.get('content-disposition', '') - filename = _extract_filename_from_name_header_part(content_disposition) - if not filename: - filename = headers.get('Content-Description', '') - if not filename: - content_type = headers.get('Content-Type', '') - filename = _extract_filename_from_name_header_part(content_type) - - if not filename: - filename = default_filename - - return filename - - -def _extract_filename_from_name_header_part(header_value): - match = re.compile('.*name=\"?(.*[^\"\'])').search(header_value) - filename = '' - if match: - filename = match.group(1) - return filename - - -class LeapMailStore(MailStore): - __slots__ = ('soledad') - - def __init__(self, soledad): - self.soledad = soledad - - @defer.inlineCallbacks - def get_mail(self, mail_id, include_body=False): - message = yield self._fetch_msg_from_soledad(mail_id) - if not _is_empty_message(message): - leap_mail = yield self._leap_message_to_leap_mail(mail_id, message, include_body) - else: - leap_mail = None - - defer.returnValue(leap_mail) - - @defer.inlineCallbacks - def get_mails(self, mail_ids, gracefully_ignore_errors=False, include_body=False): - deferreds = [] - for mail_id in mail_ids: - deferreds.append(self.get_mail(mail_id, include_body=include_body)) - - if gracefully_ignore_errors: - results = yield DeferredList(deferreds, consumeErrors=True) - defer.returnValue([mail for ok, mail in results if ok and mail is not None]) - else: - result = yield defer.gatherResults(deferreds, consumeErrors=True) - defer.returnValue(result) - - @defer.inlineCallbacks - def update_mail(self, mail): - message = yield self._fetch_msg_from_soledad(mail.mail_id) - message.get_wrapper().set_tags(tuple(mail.tags)) - message.get_wrapper().set_flags(tuple(mail.flags)) - yield self._update_mail(message) # TODO assert this is yielded (otherwise asynchronous) - - @defer.inlineCallbacks - def all_mails(self, gracefully_ignore_errors=False): - mdocs = yield self.soledad.get_from_index('by-type', 'meta') - - mail_ids = map(lambda doc: doc.doc_id, mdocs) - - mails = yield self.get_mails(mail_ids, gracefully_ignore_errors=gracefully_ignore_errors, include_body=True) - defer.returnValue(mails) - - @defer.inlineCallbacks - def add_mailbox(self, mailbox_name): - mailbox = yield self._get_or_create_mailbox(mailbox_name) - defer.returnValue(mailbox) - - @defer.inlineCallbacks - def get_mailbox_names(self): - mbox_map = set((yield self._mailbox_uuid_to_name_map()).values()) - - defer.returnValue(mbox_map.union({'INBOX'})) - - @defer.inlineCallbacks - def _mailbox_uuid_to_name_map(self): - map = {} - mbox_docs = yield self.soledad.get_from_index('by-type', 'mbox') - for doc in mbox_docs: - map[underscore_uuid(doc.content.get('uuid'))] = doc.content.get('mbox') - - defer.returnValue(map) - - @defer.inlineCallbacks - def add_mail(self, mailbox_name, raw_msg): - mailbox = yield self._get_or_create_mailbox(mailbox_name) - message = SoledadMailAdaptor().get_msg_from_string(Message, raw_msg) - message.get_wrapper().set_mbox_uuid(mailbox.uuid) - - yield SoledadMailAdaptor().create_msg(self.soledad, message) - - # add behavious from insert_mdoc_id from mail.py - mail = yield self._leap_message_to_leap_mail(message.get_wrapper().mdoc.doc_id, message, include_body=True) # TODO test that asserts include_body - defer.returnValue(mail) - - @defer.inlineCallbacks - def delete_mail(self, mail_id): - message = yield self._fetch_msg_from_soledad(mail_id) - if message and message.get_wrapper().mdoc.doc_id: - yield message.get_wrapper().delete(self.soledad) - defer.returnValue(True) - defer.returnValue(False) - - @defer.inlineCallbacks - def get_mailbox_mail_ids(self, mailbox_name): - mailbox = yield self._get_or_create_mailbox(mailbox_name) - fdocs = yield self.soledad.get_from_index('by-type-and-mbox-uuid', 'flags', underscore_uuid(mailbox.uuid)) - - mail_ids = map(lambda doc: _fdoc_id_to_mdoc_id(doc.doc_id), fdocs) - - defer.returnValue(mail_ids) - - @defer.inlineCallbacks - def delete_mailbox(self, mailbox_name): - mbx_wrapper = yield self._get_or_create_mailbox(mailbox_name) - yield SoledadMailAdaptor().delete_mbox(self.soledad, mbx_wrapper) - - @defer.inlineCallbacks - def copy_mail_to_mailbox(self, mail_id, mailbox_name): - message = yield self._fetch_msg_from_soledad(mail_id, load_body=True) - mailbox = yield self._get_or_create_mailbox(mailbox_name) - copy_wrapper = yield message.get_wrapper().copy(self.soledad, mailbox.uuid) - - leap_message = Message(copy_wrapper) - - mail = yield self._leap_message_to_leap_mail(copy_wrapper.mdoc.doc_id, leap_message, include_body=False) - - defer.returnValue(mail) - - @defer.inlineCallbacks - def move_mail_to_mailbox(self, mail_id, mailbox_name): - mail_copy = yield self.copy_mail_to_mailbox(mail_id, mailbox_name) - yield self.delete_mail(mail_id) - defer.returnValue(mail_copy) - - def _update_mail(self, message): - return message.get_wrapper().update(self.soledad) - - @defer.inlineCallbacks - def _leap_message_to_leap_mail(self, mail_id, message, include_body): - if include_body: - # TODO use body from message if available - body = yield self._raw_message_body(message) - else: - body = None - - # fetch mailbox name by mbox_uuid - mbox_uuid = message.get_wrapper().fdoc.mbox_uuid - mbox_name = yield self._mailbox_name_from_uuid(mbox_uuid) - attachments = self._extract_attachment_info_from(message) - mail = LeapMail(mail_id, mbox_name, message.get_wrapper().hdoc.headers, set(message.get_tags()), set(message.get_flags()), body=body, attachments=attachments) # TODO assert flags are passed on - - defer.returnValue(mail) - - @defer.inlineCallbacks - def _raw_message_body(self, message): - content_doc = (yield message.get_wrapper().get_body(self.soledad)) - parser = BodyParser('', content_type='text/plain', content_transfer_encoding='UTF-8') - # It fix the problem when leap doesn'r found body_phash and returns empty string - if not isinstance(content_doc, str): - parser = BodyParser(content_doc.raw, content_type=content_doc.content_type, - content_transfer_encoding=content_doc.content_transfer_encoding, charset=content_doc.charset) - - defer.returnValue(parser.parsed_content()) - - @defer.inlineCallbacks - def _mailbox_name_from_uuid(self, uuid): - map = (yield self._mailbox_uuid_to_name_map()) - defer.returnValue(map.get(uuid, '')) - - @defer.inlineCallbacks - def _get_or_create_mailbox(self, mailbox_name): - mailbox_name_upper = mailbox_name.upper() - mbx = yield SoledadMailAdaptor().get_or_create_mbox(self.soledad, mailbox_name_upper) - if mbx.uuid is None: - mbx.uuid = str(uuid4()) - yield mbx.update(self.soledad) - defer.returnValue(mbx) - - def _fetch_msg_from_soledad(self, mail_id, load_body=False): - return SoledadMailAdaptor().get_msg_from_mdoc_id(Message, self.soledad, mail_id, get_cdocs=load_body) - - @defer.inlineCallbacks - def _dump_soledad(self): - gen, docs = yield self.soledad.get_all_docs() - for doc in docs: - print '\n%s\n' % doc - - def _extract_attachment_info_from(self, message): - wrapper = message.get_wrapper() - part_maps = wrapper.hdoc.part_map - return self._extract_part_map(part_maps) - - def _is_attachment(self, part_map, headers): - disposition = headers.get('Content-Disposition') or headers.get('content-disposition') - content_type = part_map['ctype'] - - if 'multipart' in content_type: - return False - - if 'text/plain' == content_type and ((disposition == 'inline') or (disposition is None)): - return False - - return True - - def _create_attachment_info_from(self, part_map, headers): - ident = part_map['phash'] - name = _extract_filename(headers) - encoding = headers.get('Content-Transfer-Encoding', None) - ctype = part_map.get('ctype') or headers.get('Content-Type') - size = part_map.get('size', 0) - - return AttachmentInfo(ident, name, encoding, ctype, size) - - def _extract_part_map(self, part_maps): - result = [] - - for nr, part_map in part_maps.items(): - if 'headers' in part_map and 'phash' in part_map: - headers = {header[0]: header[1] for header in part_map['headers']} - if self._is_attachment(part_map, headers): - result.append(self._create_attachment_info_from(part_map, headers)) - if 'part_map' in part_map: - result += self._extract_part_map(part_map['part_map']) - - return result - - -def _is_empty_message(message): - return (message is None) or (message.get_wrapper().mdoc.doc_id is None) - - -def _fdoc_id_to_mdoc_id(fdoc_id): - return 'M' + fdoc_id[1:] -- cgit v1.2.3