diff options
Diffstat (limited to 'service/src/pixelated/adapter/mailstore')
7 files changed, 817 insertions, 0 deletions
diff --git a/service/src/pixelated/adapter/mailstore/__init__.py b/service/src/pixelated/adapter/mailstore/__init__.py new file mode 100644 index 00000000..978df45d --- /dev/null +++ b/service/src/pixelated/adapter/mailstore/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from pixelated.adapter.mailstore.mailstore import MailStore, underscore_uuid +from pixelated.adapter.mailstore.leap_mailstore import LeapMailStore + +__all__ = ['MailStore', 'LeapMailStore', 'underscore_uuid'] diff --git a/service/src/pixelated/adapter/mailstore/body_parser.py b/service/src/pixelated/adapter/mailstore/body_parser.py new file mode 100644 index 00000000..8cac75cf --- /dev/null +++ b/service/src/pixelated/adapter/mailstore/body_parser.py @@ -0,0 +1,69 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import re +from email.parser import Parser + +from twisted.logger import Logger + +logger = Logger() + + +def _parse_charset_header(content_type_and_charset_header, default_charset='us-ascii'): + try: + return re.compile('.*charset="?([a-zA-Z0-9-]+)"?', re.MULTILINE | re.DOTALL).match(content_type_and_charset_header).group(1) + except: + return default_charset + + +class BodyParser(object): + + def __init__(self, content, content_type='text/plain; charset="us-ascii"', content_transfer_encoding=None, charset=None): + self._content = content + self._content_type = content_type + self._content_transfer_encoding = content_transfer_encoding + self._charset = charset + + def parsed_content(self): + charset = self._charset or _parse_charset_header(self._content_type) + text = self._serialize_for_parser(charset) + + decoded_body = self._parse_and_decode(text) + return unicode(decoded_body, charset, errors='replace') + + def _parse_and_decode(self, text): + parsed_body = Parser().parsestr(text) + decoded_body = self._unwrap_content_transfer_encoding(parsed_body) + return decoded_body + + def _unwrap_content_transfer_encoding(self, parsed_body): + return parsed_body.get_payload(decode=True) + + def _serialize_for_parser(self, charset): + text = u'Content-Type: %s\n' % self._content_type + if self._content_transfer_encoding is not None: + text += u'Content-Transfer-Encoding: %s\n' % self._content_transfer_encoding + + text += u'\n' + encoded_text = text.encode(charset) + if isinstance(self._content, unicode): + try: + return encoded_text + self._content.encode(charset) + except UnicodeError, e: + logger.warn('Failed to encode content for charset %s. Ignoring invalid chars: %s' % (charset, e)) + return encoded_text + self._content.encode(charset, 'ignore') + else: + return encoded_text + self._content diff --git a/service/src/pixelated/adapter/mailstore/leap_attachment_store.py b/service/src/pixelated/adapter/mailstore/leap_attachment_store.py new file mode 100644 index 00000000..b297f9e6 --- /dev/null +++ b/service/src/pixelated/adapter/mailstore/leap_attachment_store.py @@ -0,0 +1,81 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + + +import quopri +import base64 +from email import encoders +from leap.bitmask.mail.adaptors.soledad import SoledadMailAdaptor, ContentDocWrapper +from twisted.internet import defer +from email.mime.nonmultipart import MIMENonMultipart +from email.mime.multipart import MIMEMultipart +from leap.bitmask.mail.mail import Message + + +class LeapAttachmentStore(object): + + def __init__(self, soledad): + self.soledad = soledad + + @defer.inlineCallbacks + def get_mail_attachment(self, attachment_id): + results = yield self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', attachment_id) if attachment_id else [] + if results: + content = ContentDocWrapper(**results[0].content) + defer.returnValue({'content-type': content.content_type, 'content': self._try_decode( + content.raw, content.content_transfer_encoding)}) + else: + raise ValueError('No attachment with id %s found!' % attachment_id) + + @defer.inlineCallbacks + def add_attachment(self, content, content_type): + cdoc = self._attachment_to_cdoc(content, content_type) + attachment_id = cdoc.phash + try: + yield self.get_mail_attachment(attachment_id) + except ValueError: + yield self.soledad.create_doc(cdoc.serialize(), doc_id=attachment_id) + defer.returnValue(attachment_id) + + def _try_decode(self, raw, encoding): + encoding = encoding.lower() + if encoding == 'base64': + data = base64.decodestring(raw) + elif encoding == 'quoted-printable': + data = quopri.decodestring(raw) + else: + data = str(raw) + + return bytearray(data) + + def _attachment_to_cdoc(self, content, content_type, encoder=encoders.encode_base64): + major, sub = content_type.split('/') + attachment = MIMENonMultipart(major, sub) + attachment.set_payload(content) + encoder(attachment) + attachment.add_header('Content-Disposition', 'attachment', filename='does_not_matter.txt') + + pseudo_mail = MIMEMultipart() + pseudo_mail.attach(attachment) + + tmp_mail = SoledadMailAdaptor().get_msg_from_string(MessageClass=Message, raw_msg=pseudo_mail.as_string()) + + cdoc = tmp_mail.get_wrapper().cdocs[1] + return cdoc + + def _calc_attachment_id_(self, content, content_type, encoder=encoders.encode_base64): + cdoc = self._attachment_to_cdoc(content, content_type, encoder) + return cdoc.phash diff --git a/service/src/pixelated/adapter/mailstore/leap_mailstore.py b/service/src/pixelated/adapter/mailstore/leap_mailstore.py new file mode 100644 index 00000000..288223dd --- /dev/null +++ b/service/src/pixelated/adapter/mailstore/leap_mailstore.py @@ -0,0 +1,406 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import re +from email.header import decode_header +from uuid import uuid4 + +from leap.bitmask.mail.adaptors.soledad import SoledadMailAdaptor +from leap.bitmask.mail.mail import Message +from twisted.internet import defer +from twisted.internet.defer import FirstError, DeferredList + +from pixelated.adapter.mailstore.body_parser import BodyParser +from pixelated.adapter.mailstore.mailstore import MailStore, underscore_uuid +from pixelated.adapter.model.mail import Mail, InputMail +from pixelated.support.functional import to_unicode +from pixelated.support import date + + +class AttachmentInfo(object): + def __init__(self, ident, name, encoding=None, ctype='application/octet-stream', size=0): + self.ident = ident + self.name = name + self.encoding = encoding + self.ctype = ctype + self.size = size + + def __repr__(self): + return 'AttachmentInfo[%s, %s, %s]' % (self.ident, self.name, self.encoding) + + def __str__(self): + return 'AttachmentInfo[%s, %s, %s]' % (self.ident, self.name, self.encoding) + + def as_dict(self): + return {'ident': self.ident, 'name': self.name, 'encoding': self.encoding, 'size': self.size, 'content-type': self.ctype} + + +class LeapMail(Mail): + + def __init__(self, mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None, attachments=[]): + self._mail_id = mail_id + self._mailbox_name = mailbox_name + self._headers = headers if headers is not None else {} + self._body = to_unicode(body) + self.tags = set(tags) # TODO test that asserts copy + self._flags = set(flags) # TODO test that asserts copy + self._attachments = attachments + + @property + def headers(self): + cpy = dict(self._headers) + for name in set(self._headers.keys()).intersection(['To', 'Cc', 'Bcc']): + cpy[name] = [address.strip() for address in (self._headers[name].split(',') if self._headers[name] else [])] + + return cpy + + @property + def ident(self): + return self._mail_id + + @property + def mail_id(self): + return self._mail_id + + @property + def body(self): + return self._body + + @property + def flags(self): + return self._flags + + @property + def mailbox_name(self): + return self._mailbox_name + + @property + def security_casing(self): + casing = dict(imprints=self._signature_information(), locks=[]) + if self._encrypted() == "decrypted": + casing["locks"] = [{"state": "valid"}] + return casing + + def _encrypted(self): + return self.headers.get("X-Leap-Encryption", "false") + + def _signature_information(self): + signature = self.headers.get("X-Leap-Signature", None) + if signature is None or signature.startswith("could not verify"): + return [{"state": "no_signature_information"}] + else: + if signature.startswith("valid"): + return [{"state": "valid", "seal": {"validity": "valid"}}] + else: + return [] + + @property + def raw(self): + result = u'' + for k, v in self._headers.items(): + content, encoding = decode_header(v)[0] + if encoding: + result += '%s: %s\n' % (k, unicode(content, encoding=encoding)) + else: + result += '%s: %s\n' % (k, v) + result += '\n' + + if self._body: + result = result + self._body + + return result + + def _remove_duplicates(self, values): + return list(set(values)) + + def _decoded_header_utf_8(self, header_value): + if isinstance(header_value, list): + return self._remove_duplicates([self._decoded_header_utf_8(v) for v in header_value]) + elif header_value is not None: + def encode_chunk(content, encoding): + return unicode(content.strip(), encoding=encoding or 'ascii', errors='ignore') + + try: + encoded_chunks = [encode_chunk(content, encoding) for content, encoding in decode_header(header_value)] + return ' '.join(encoded_chunks) # decode_header strips whitespaces on all chunks, joining over ' ' is only a workaround, not a proper fix + except UnicodeEncodeError: + return unicode(header_value.encode('ascii', errors='ignore')) + + def as_dict(self): + return { + 'header': {k.lower(): self._decoded_header_utf_8(v) for k, v in self.headers.items()}, + 'ident': self._mail_id, + 'tags': self.tags, + 'status': list(self.status), + 'body': self._body, + 'security_casing': self.security_casing, + 'textPlainBody': self._body, + 'mailbox': self._mailbox_name.lower(), + 'attachments': [attachment.as_dict() for attachment in self._attachments] + } + + @staticmethod + def from_dict(mail_dict): + # TODO: implement this method and also write tests for it + headers = {key.capitalize(): value for key, value in mail_dict.get('header', {}).items()} + headers['Date'] = date.mail_date_now() + body = mail_dict.get('body', '') + tags = set(mail_dict.get('tags', [])) + status = set(mail_dict.get('status', [])) + attachments = [] + + # mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None, attachments=[] + return LeapMail(None, None, headers, tags, set(), body, attachments) + + +def _extract_filename(headers, default_filename='UNNAMED'): + content_disposition = headers.get('Content-Disposition') or headers.get('content-disposition', '') + filename = _extract_filename_from_name_header_part(content_disposition) + if not filename: + filename = headers.get('Content-Description', '') + if not filename: + content_type = headers.get('Content-Type', '') + filename = _extract_filename_from_name_header_part(content_type) + + if not filename: + filename = default_filename + + return filename + + +def _extract_filename_from_name_header_part(header_value): + match = re.compile('.*name=\"?(.*[^\"\'])').search(header_value) + filename = '' + if match: + filename = match.group(1) + return filename + + +class LeapMailStore(MailStore): + __slots__ = ('soledad') + + def __init__(self, soledad): + self.soledad = soledad + + @defer.inlineCallbacks + def get_mail(self, mail_id, include_body=False): + message = yield self._fetch_msg_from_soledad(mail_id) + if not _is_empty_message(message): + leap_mail = yield self._leap_message_to_leap_mail(mail_id, message, include_body) + else: + leap_mail = None + + defer.returnValue(leap_mail) + + @defer.inlineCallbacks + def get_mails(self, mail_ids, gracefully_ignore_errors=False, include_body=False): + deferreds = [] + for mail_id in mail_ids: + deferreds.append(self.get_mail(mail_id, include_body=include_body)) + + if gracefully_ignore_errors: + results = yield DeferredList(deferreds, consumeErrors=True) + defer.returnValue([mail for ok, mail in results if ok and mail is not None]) + else: + result = yield defer.gatherResults(deferreds, consumeErrors=True) + defer.returnValue(result) + + @defer.inlineCallbacks + def update_mail(self, mail): + message = yield self._fetch_msg_from_soledad(mail.mail_id) + message.get_wrapper().set_tags(tuple(mail.tags)) + message.get_wrapper().set_flags(tuple(mail.flags)) + yield self._update_mail(message) # TODO assert this is yielded (otherwise asynchronous) + + @defer.inlineCallbacks + def all_mails(self, gracefully_ignore_errors=False): + mdocs = yield self.soledad.get_from_index('by-type', 'meta') + + mail_ids = map(lambda doc: doc.doc_id, mdocs) + + mails = yield self.get_mails(mail_ids, gracefully_ignore_errors=gracefully_ignore_errors, include_body=True) + defer.returnValue(mails) + + @defer.inlineCallbacks + def add_mailbox(self, mailbox_name): + mailbox = yield self._get_or_create_mailbox(mailbox_name) + defer.returnValue(mailbox) + + @defer.inlineCallbacks + def get_mailbox_names(self): + mbox_map = set((yield self._mailbox_uuid_to_name_map()).values()) + + defer.returnValue(mbox_map.union({'INBOX'})) + + @defer.inlineCallbacks + def _mailbox_uuid_to_name_map(self): + map = {} + mbox_docs = yield self.soledad.get_from_index('by-type', 'mbox') + for doc in mbox_docs: + map[underscore_uuid(doc.content.get('uuid'))] = doc.content.get('mbox') + + defer.returnValue(map) + + @defer.inlineCallbacks + def add_mail(self, mailbox_name, raw_msg): + mailbox = yield self._get_or_create_mailbox(mailbox_name) + message = SoledadMailAdaptor().get_msg_from_string(Message, raw_msg) + message.get_wrapper().set_mbox_uuid(mailbox.uuid) + + yield SoledadMailAdaptor().create_msg(self.soledad, message) + + # add behavious from insert_mdoc_id from mail.py + mail = yield self._leap_message_to_leap_mail(message.get_wrapper().mdoc.doc_id, message, include_body=True) # TODO test that asserts include_body + defer.returnValue(mail) + + @defer.inlineCallbacks + def delete_mail(self, mail_id): + message = yield self._fetch_msg_from_soledad(mail_id) + if message and message.get_wrapper().mdoc.doc_id: + yield message.get_wrapper().delete(self.soledad) + defer.returnValue(True) + defer.returnValue(False) + + @defer.inlineCallbacks + def get_mailbox_mail_ids(self, mailbox_name): + mailbox = yield self._get_or_create_mailbox(mailbox_name) + fdocs = yield self.soledad.get_from_index('by-type-and-mbox-uuid', 'flags', underscore_uuid(mailbox.uuid)) + + mail_ids = map(lambda doc: _fdoc_id_to_mdoc_id(doc.doc_id), fdocs) + + defer.returnValue(mail_ids) + + @defer.inlineCallbacks + def delete_mailbox(self, mailbox_name): + mbx_wrapper = yield self._get_or_create_mailbox(mailbox_name) + yield SoledadMailAdaptor().delete_mbox(self.soledad, mbx_wrapper) + + @defer.inlineCallbacks + def copy_mail_to_mailbox(self, mail_id, mailbox_name): + message = yield self._fetch_msg_from_soledad(mail_id, load_body=True) + mailbox = yield self._get_or_create_mailbox(mailbox_name) + copy_wrapper = yield message.get_wrapper().copy(self.soledad, mailbox.uuid) + + leap_message = Message(copy_wrapper) + + mail = yield self._leap_message_to_leap_mail(copy_wrapper.mdoc.doc_id, leap_message, include_body=False) + + defer.returnValue(mail) + + @defer.inlineCallbacks + def move_mail_to_mailbox(self, mail_id, mailbox_name): + mail_copy = yield self.copy_mail_to_mailbox(mail_id, mailbox_name) + yield self.delete_mail(mail_id) + defer.returnValue(mail_copy) + + def _update_mail(self, message): + return message.get_wrapper().update(self.soledad) + + @defer.inlineCallbacks + def _leap_message_to_leap_mail(self, mail_id, message, include_body): + if include_body: + # TODO use body from message if available + body = yield self._raw_message_body(message) + else: + body = None + + # fetch mailbox name by mbox_uuid + mbox_uuid = message.get_wrapper().fdoc.mbox_uuid + mbox_name = yield self._mailbox_name_from_uuid(mbox_uuid) + attachments = self._extract_attachment_info_from(message) + mail = LeapMail(mail_id, mbox_name, message.get_wrapper().hdoc.headers, set(message.get_tags()), set(message.get_flags()), body=body, attachments=attachments) # TODO assert flags are passed on + + defer.returnValue(mail) + + @defer.inlineCallbacks + def _raw_message_body(self, message): + content_doc = (yield message.get_wrapper().get_body(self.soledad)) + parser = BodyParser('', content_type='text/plain', content_transfer_encoding='UTF-8') + # It fix the problem when leap doesn'r found body_phash and returns empty string + if not isinstance(content_doc, str): + parser = BodyParser(content_doc.raw, content_type=content_doc.content_type, + content_transfer_encoding=content_doc.content_transfer_encoding, charset=content_doc.charset) + + defer.returnValue(parser.parsed_content()) + + @defer.inlineCallbacks + def _mailbox_name_from_uuid(self, uuid): + map = (yield self._mailbox_uuid_to_name_map()) + defer.returnValue(map.get(uuid, '')) + + @defer.inlineCallbacks + def _get_or_create_mailbox(self, mailbox_name): + mailbox_name_upper = mailbox_name.upper() + mbx = yield SoledadMailAdaptor().get_or_create_mbox(self.soledad, mailbox_name_upper) + if mbx.uuid is None: + mbx.uuid = str(uuid4()) + yield mbx.update(self.soledad) + defer.returnValue(mbx) + + def _fetch_msg_from_soledad(self, mail_id, load_body=False): + return SoledadMailAdaptor().get_msg_from_mdoc_id(Message, self.soledad, mail_id, get_cdocs=load_body) + + @defer.inlineCallbacks + def _dump_soledad(self): + gen, docs = yield self.soledad.get_all_docs() + for doc in docs: + print '\n%s\n' % doc + + def _extract_attachment_info_from(self, message): + wrapper = message.get_wrapper() + part_maps = wrapper.hdoc.part_map + return self._extract_part_map(part_maps) + + def _is_attachment(self, part_map, headers): + disposition = headers.get('Content-Disposition') or headers.get('content-disposition') + content_type = part_map['ctype'] + + if 'multipart' in content_type: + return False + + if 'text/plain' == content_type and ((disposition == 'inline') or (disposition is None)): + return False + + return True + + def _create_attachment_info_from(self, part_map, headers): + ident = part_map['phash'] + name = _extract_filename(headers) + encoding = headers.get('Content-Transfer-Encoding', None) + ctype = part_map.get('ctype') or headers.get('Content-Type') + size = part_map.get('size', 0) + + return AttachmentInfo(ident, name, encoding, ctype, size) + + def _extract_part_map(self, part_maps): + result = [] + + for nr, part_map in part_maps.items(): + if 'headers' in part_map and 'phash' in part_map: + headers = {header[0]: header[1] for header in part_map['headers']} + if self._is_attachment(part_map, headers): + result.append(self._create_attachment_info_from(part_map, headers)) + if 'part_map' in part_map: + result += self._extract_part_map(part_map['part_map']) + + return result + + +def _is_empty_message(message): + return (message is None) or (message.get_wrapper().mdoc.doc_id is None) + + +def _fdoc_id_to_mdoc_id(fdoc_id): + return 'M' + fdoc_id[1:] diff --git a/service/src/pixelated/adapter/mailstore/mailstore.py b/service/src/pixelated/adapter/mailstore/mailstore.py new file mode 100644 index 00000000..fbd7fc9e --- /dev/null +++ b/service/src/pixelated/adapter/mailstore/mailstore.py @@ -0,0 +1,60 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + + +class MailStore(object): + def get_mail(self, mail_id): + pass + + def get_mail_attachment(self, attachment_id): + pass + + def get_mails(self, mail_ids, gracefully_ignore_errors=False, include_body=False): + pass + + def all_mails(self): + pass + + def delete_mail(self, mail_id): + pass + + def update_mail(self, mail): + pass + + def add_mail(self, mailbox_name, mail): + pass + + def get_mailbox_names(self): + pass + + def add_mailbox(self, mailbox_name): + pass + + def delete_mailbox(self, mailbox_name): + pass + + def get_mailbox_mail_ids(self, mailbox_name): + pass + + def copy_mail_to_mailbox(self, mail_id, mailbox_name): + pass + + def move_mail_to_mailbox(self, mail_id, mailbox_name): + pass + + +def underscore_uuid(uuid): + return uuid.replace('-', '_') diff --git a/service/src/pixelated/adapter/mailstore/maintenance/__init__.py b/service/src/pixelated/adapter/mailstore/maintenance/__init__.py new file mode 100644 index 00000000..9a1007cc --- /dev/null +++ b/service/src/pixelated/adapter/mailstore/maintenance/__init__.py @@ -0,0 +1,100 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from leap.bitmask.keymanager import documents as leap_doc +from leap.bitmask.keymanager.keys import OpenPGPKey + +from twisted.internet import defer +from twisted.logger import Logger + + +TYPE_OPENPGP_KEY = 'OpenPGPKey' +TYPE_OPENPGP_ACTIVE = 'OpenPGPKey-active' + +KEY_DOC_TYPES = {TYPE_OPENPGP_ACTIVE, TYPE_OPENPGP_KEY} + +logger = Logger() + + +def _is_key_doc(doc): + return doc.content.get(leap_doc.KEY_TYPE_KEY, None) in KEY_DOC_TYPES + + +def _is_private_key_doc(doc): + return _is_key_doc(doc) and doc.content.get(leap_doc.KEY_PRIVATE_KEY, False) + + +def _is_active_key_doc(doc): + return _is_key_doc(doc) and doc.content.get(leap_doc.KEY_TYPE_KEY, None) == TYPE_OPENPGP_ACTIVE + + +def _is_public_key(doc): + return _is_key_doc(doc) and not doc.content.get(leap_doc.KEY_PRIVATE_KEY, False) + + +def _key_fingerprint(doc): + return doc.content.get(leap_doc.KEY_FINGERPRINT_KEY, None) + + +def _address(doc): + return doc.content.get(leap_doc.KEY_ADDRESS_KEY, None) + + +class SoledadMaintenance(object): + def __init__(self, soledad): + self._soledad = soledad + + @defer.inlineCallbacks + def repair(self): + _, docs = yield self._soledad.get_all_docs() + + private_key_fingerprints = self._key_fingerprints_with_private_key(docs) + + for doc in docs: + if _is_key_doc(doc) and _key_fingerprint(doc) not in private_key_fingerprints: + logger.warn('Deleting doc %s for key %s of <%s>' % (doc.doc_id, _key_fingerprint(doc), _address(doc))) + yield self._soledad.delete_doc(doc) + + yield self._repair_missing_active_docs(docs, private_key_fingerprints) + + @defer.inlineCallbacks + def _repair_missing_active_docs(self, docs, private_key_fingerprints): + missing = self._missing_active_docs(docs, private_key_fingerprints) + for fingerprint in missing: + emails = self._emails_for_key_fingerprint(docs, fingerprint) + for email in emails: + logger.warn('Re-creating active doc for key %s, email %s' % (fingerprint, email)) + yield self._soledad.create_doc_from_json(OpenPGPKey(email, fingerprint=fingerprint, private=False).get_active_json()) + + def _key_fingerprints_with_private_key(self, docs): + return [doc.content[leap_doc.KEY_FINGERPRINT_KEY] for doc in docs if _is_private_key_doc(doc)] + + def _missing_active_docs(self, docs, private_key_fingerprints): + active_doc_ids = self._active_docs_for_key_fingerprint(docs) + + return set([private_key_fingerprint for private_key_fingerprint in private_key_fingerprints if private_key_fingerprint not in active_doc_ids]) + + def _emails_for_key_fingerprint(self, docs, fingerprint): + for doc in docs: + if _is_private_key_doc(doc) and _key_fingerprint(doc) == fingerprint: + email = _address(doc) + if email is None: + return [] + if isinstance(email, list): + return email + return [email] + + def _active_docs_for_key_fingerprint(self, docs): + return [doc.content[leap_doc.KEY_FINGERPRINT_KEY] for doc in docs if _is_active_key_doc(doc) and _is_public_key(doc)] diff --git a/service/src/pixelated/adapter/mailstore/searchable_mailstore.py b/service/src/pixelated/adapter/mailstore/searchable_mailstore.py new file mode 100644 index 00000000..07e99ba7 --- /dev/null +++ b/service/src/pixelated/adapter/mailstore/searchable_mailstore.py @@ -0,0 +1,81 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from twisted.internet import defer +from types import FunctionType +from pixelated.adapter.mailstore import MailStore + + +class SearchableMailStore(object): # implementes MailStore + + def __init__(self, delegate, search_engine): + self._delegate = delegate + self._search_engine = search_engine + + @classmethod + def _create_delegator(cls, method_name): + def delegator(self, *args, **kw): + return getattr(self._delegate, method_name)(*args, **kw) + + setattr(cls, method_name, delegator) + + @defer.inlineCallbacks + def add_mail(self, mailbox_name, mail): + stored_mail = yield self._delegate.add_mail(mailbox_name, mail) + self._search_engine.index_mail(stored_mail) + defer.returnValue(stored_mail) + + @defer.inlineCallbacks + def delete_mail(self, mail_id): + removed = yield self._delegate.delete_mail(mail_id) + self._search_engine.remove_from_index(mail_id) + defer.returnValue(removed) + + @defer.inlineCallbacks + def update_mail(self, mail): + yield self._delegate.update_mail(mail) + self._search_engine.index_mail(mail) + + @defer.inlineCallbacks + def move_mail_to_mailbox(self, mail_id, mailbox_name): + moved_mail = yield self._delegate.move_mail_to_mailbox(mail_id, mailbox_name) + self._search_engine.remove_from_index(mail_id) + self._search_engine.index_mail(moved_mail) + defer.returnValue(moved_mail) + + @defer.inlineCallbacks + def copy_mail_to_mailbox(self, mail_id, mailbox_name): + copied_mail = yield self._delegate.copy_mail_to_mailbox(mail_id, mailbox_name) + self._search_engine.index_mail(copied_mail) + defer.returnValue(copied_mail) + + def delete_mailbox(self, mailbox_name): + raise NotImplementedError() + + def __getattr__(self, name): + """ + Acts like method missing. If a method of MailStore is not implemented in this class, + a delegate method is created. + + :param name: attribute name + :return: method or attribute + """ + methods = ([key for key, value in MailStore.__dict__.items() if type(value) == FunctionType]) + + if name in methods: + SearchableMailStore._create_delegator(name) + return super(SearchableMailStore, self).__getattribute__(name) + else: + raise NotImplementedError('No attribute %s' % name) |