diff options
Diffstat (limited to 'service/pixelated')
54 files changed, 1501 insertions, 1140 deletions
diff --git a/service/pixelated/adapter/errors/__init__.py b/service/pixelated/adapter/errors/__init__.py new file mode 100644 index 00000000..31ad4947 --- /dev/null +++ b/service/pixelated/adapter/errors/__init__.py @@ -0,0 +1,3 @@ +class DuplicatedDraftException(Exception): + def __init__(self, message): + super(Exception, self).__init__(message) diff --git a/service/pixelated/adapter/listeners/mailbox_indexer_listener.py b/service/pixelated/adapter/listeners/mailbox_indexer_listener.py index d8e0f81e..8896d742 100644 --- a/service/pixelated/adapter/listeners/mailbox_indexer_listener.py +++ b/service/pixelated/adapter/listeners/mailbox_indexer_listener.py @@ -13,6 +13,11 @@ # # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import logging +from twisted.internet import defer + + +logger = logging.getLogger(__name__) class MailboxIndexerListener(object): @@ -21,22 +26,31 @@ class MailboxIndexerListener(object): SEARCH_ENGINE = None @classmethod - def listen(cls, account, mailbox_name, soledad_querier): - listener = MailboxIndexerListener(mailbox_name, soledad_querier) - if listener not in account.getMailbox(mailbox_name).listeners: - account.getMailbox(mailbox_name).addListener(listener) + @defer.inlineCallbacks + def listen(cls, account, mailbox_name, mail_store): + listener = MailboxIndexerListener(mailbox_name, mail_store) + if listener not in (yield account.getMailbox(mailbox_name)).listeners: + mbx = yield account.getMailbox(mailbox_name) + mbx.addListener(listener) + + defer.returnValue(listener) - def __init__(self, mailbox_name, soledad_querier): + def __init__(self, mailbox_name, mail_store): self.mailbox_name = mailbox_name - self.querier = soledad_querier + self.mail_store = mail_store + @defer.inlineCallbacks def newMessages(self, exists, recent): - indexed_idents = set(self.SEARCH_ENGINE.search('tag:' + self.mailbox_name.lower(), all_mails=True)) - soledad_idents = self.querier.idents_by_mailbox(self.mailbox_name) + try: + indexed_idents = set(self.SEARCH_ENGINE.search('tag:' + self.mailbox_name.lower(), all_mails=True)) + soledad_idents = yield self.mail_store.get_mailbox_mail_ids(self.mailbox_name) + soledad_idents = set(soledad_idents) - missing_idents = soledad_idents.difference(indexed_idents) + missing_idents = soledad_idents.difference(indexed_idents) - self.SEARCH_ENGINE.index_mails(self.querier.mails(missing_idents)) + self.SEARCH_ENGINE.index_mails((yield self.mail_store.get_mails(missing_idents))) + except Exception, e: # this is a event handler, don't let exceptions escape + logger.error(e) def __eq__(self, other): return other and other.mailbox_name == self.mailbox_name @@ -46,3 +60,11 @@ class MailboxIndexerListener(object): def __repr__(self): return 'MailboxListener: ' + self.mailbox_name + + +@defer.inlineCallbacks +def listen_all_mailboxes(account, search_engine, mail_store): + MailboxIndexerListener.SEARCH_ENGINE = search_engine + mailboxes = yield account.account.list_all_mailbox_names() + for mailbox_name in mailboxes: + yield MailboxIndexerListener.listen(account, mailbox_name, mail_store) diff --git a/service/pixelated/adapter/soledad/__init__.py b/service/pixelated/adapter/mailstore/__init__.py index 2756a319..978df45d 100644 --- a/service/pixelated/adapter/soledad/__init__.py +++ b/service/pixelated/adapter/mailstore/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2014 ThoughtWorks, Inc. +# Copyright (c) 2015 ThoughtWorks, Inc. # # Pixelated is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -13,3 +13,8 @@ # # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from pixelated.adapter.mailstore.mailstore import MailStore, underscore_uuid +from pixelated.adapter.mailstore.leap_mailstore import LeapMailStore + +__all__ = ['MailStore', 'LeapMailStore', 'underscore_uuid'] diff --git a/service/pixelated/adapter/mailstore/body_parser.py b/service/pixelated/adapter/mailstore/body_parser.py new file mode 100644 index 00000000..a6017833 --- /dev/null +++ b/service/pixelated/adapter/mailstore/body_parser.py @@ -0,0 +1,68 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from email.parser import Parser +import re +import logging + +logger = logging.getLogger(__name__) + + +def _parse_charset_header(content_type_and_charset_header, default_charset='us-ascii'): + try: + return re.compile('.*charset="?([a-zA-Z0-9-]+)"?', re.MULTILINE | re.DOTALL).match(content_type_and_charset_header).group(1) + except: + return default_charset + + +class BodyParser(object): + + def __init__(self, content, content_type='text/plain; charset="us-ascii"', content_transfer_encoding=None): + self._content = content + self._content_type = content_type + self._content_transfer_encoding = content_transfer_encoding + + def parsed_content(self): + charset = _parse_charset_header(self._content_type) + text = self._serialize_for_parser(charset) + + decoded_body = self._parse_and_decode(text) + + return unicode(decoded_body, encoding=charset) + + def _parse_and_decode(self, text): + parsed_body = Parser().parsestr(text) + decoded_body = self._unwrap_content_transfer_encoding(parsed_body) + return decoded_body + + def _unwrap_content_transfer_encoding(self, parsed_body): + return parsed_body.get_payload(decode=True) + + def _serialize_for_parser(self, charset): + text = u'Content-Type: %s\n' % self._content_type + if self._content_transfer_encoding is not None: + text += u'Content-Transfer-Encoding: %s\n' % self._content_transfer_encoding + + text += u'\n' + encoded_text = text.encode(charset) + if isinstance(self._content, unicode): + try: + return encoded_text + self._content.encode(charset) + except UnicodeError, e: + logger.warn('Failed to encode content for charset %s. Ignoring invalid chars: %s' % (charset, e)) + return encoded_text + self._content.encode(charset, 'ignore') + else: + return encoded_text + self._content diff --git a/service/pixelated/adapter/mailstore/leap_mailstore.py b/service/pixelated/adapter/mailstore/leap_mailstore.py new file mode 100644 index 00000000..2754c624 --- /dev/null +++ b/service/pixelated/adapter/mailstore/leap_mailstore.py @@ -0,0 +1,413 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import base64 +from email.header import decode_header +from email.utils import parseaddr +import quopri +from uuid import uuid4 + +import re +from leap.mail.adaptors.soledad import SoledadMailAdaptor, ContentDocWrapper +from twisted.internet import defer +from pixelated.adapter.mailstore.body_parser import BodyParser +from pixelated.adapter.mailstore.mailstore import MailStore, underscore_uuid +from leap.mail.mail import Message +from pixelated.adapter.model.mail import Mail, InputMail + + +class AttachmentInfo(object): + def __init__(self, ident, name, encoding): + self.ident = ident + self.name = name + self.encoding = encoding + + +class LeapMail(Mail): + + def __init__(self, mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None, attachments=[]): + self._mail_id = mail_id + self._mailbox_name = mailbox_name + self._headers = headers if headers is not None else {} + self._body = body + self.tags = set(tags) # TODO test that asserts copy + self._flags = set(flags) # TODO test that asserts copy + self._attachments = attachments + + @property + def headers(self): + cpy = dict(self._headers) + + for name in set(self._headers.keys()).intersection(['To', 'Cc', 'Bcc']): + cpy[name] = self._headers[name].split(',') if self._headers[name] else [] + + return cpy + + @property + def ident(self): + return self._mail_id + + @property + def mail_id(self): + return self._mail_id + + @property + def body(self): + return self._body + + @property + def flags(self): + return self._flags + + @property + def mailbox_name(self): + return self._mailbox_name + + @property + def security_casing(self): + casing = dict(imprints=self._signature_information(), locks=[]) + if self._encrypted() == "decrypted": + casing["locks"] = [{"state": "valid"}] + return casing + + def _encrypted(self): + return self.headers.get("X-Leap-Encryption", "false") + + def _signature_information(self): + signature = self.headers.get("X-Leap-Signature", None) + if signature is None or signature.startswith("could not verify"): + return [{"state": "no_signature_information"}] + else: + if signature.startswith("valid"): + return [{"state": "valid", "seal": {"validity": "valid"}}] + else: + return [] + + @property + def raw(self): + result = u'' + for k, v in self._headers.items(): + content, encoding = decode_header(v)[0] + if encoding: + result += '%s: %s\n' % (k, unicode(content, encoding=encoding)) + else: + result += '%s: %s\n' % (k, v) + result += '\n' + + if self._body: + result = result + self._body + + return result + + def _decoded_header_utf_8(self, header_value): + if isinstance(header_value, list): + return self.remove_duplicates([self._decoded_header_utf_8(v) for v in header_value]) + elif header_value is not None: + def encode_chunk(content, encoding): + return unicode(content, encoding=encoding or 'ascii', errors='ignore') + + try: + encoded_chunks = [encode_chunk(content, encoding) for content, encoding in decode_header(header_value)] + return ' '.join(encoded_chunks) # decode_header strips whitespaces on all chunks, joining over ' ' is only a workaround, not a proper fix + except UnicodeEncodeError: + return unicode(header_value.encode('ascii', errors='ignore')) + + def as_dict(self): + return { + 'header': {k.lower(): self._decoded_header_utf_8(v) for k, v in self.headers.items()}, + 'ident': self._mail_id, + 'tags': self.tags, + 'status': list(self.status), + 'body': self._body, + 'security_casing': self.security_casing, + 'textPlainBody': self._body, + 'replying': self._replying_dict(), + 'mailbox': self._mailbox_name.lower(), + 'attachments': [{'ident': attachment.ident, 'name': attachment.name, 'encoding': attachment.encoding} for attachment in self._attachments] + } + + def _replying_dict(self): + result = {'single': None, 'all': {'to-field': [], 'cc-field': []}} + + sender_mail = self._decoded_header_utf_8(self.headers.get('Reply-To', self.headers.get('From'))) + # Issue #215: Fix for existing mails without any from address. + if sender_mail is None: + sender_mail = InputMail.FROM_EMAIL_ADDRESS + + recipients = self._decoded_header_utf_8(self._reply_recipient('To')) + if not self._parsed_mail_matches(sender_mail, InputMail.FROM_EMAIL_ADDRESS) or len(recipients) == 0: + recipients.append(sender_mail) + recipients = self.remove_duplicates(recipients) + ccs = self._decoded_header_utf_8(self._reply_recipient('Cc')) + + result['single'] = self._single_reply_recipient(recipients, sender_mail) + result['all']['to-field'] = recipients + result['all']['cc-field'] = ccs + return result + + def _single_reply_recipient(self, recipients, sender_mail): + """ + Currently the domain model expects only one single recipient for reply action. But it should support an array, + or even better: there should not be any conceptual difference between reply and reply all for this logic + """ + if self._parsed_mail_matches(sender_mail, InputMail.FROM_EMAIL_ADDRESS): + return recipients[0] + else: + return sender_mail + + def remove_duplicates(self, recipients): + return list(set(recipients)) + + def _reply_recipient(self, kind): + recipients = self.headers.get(kind, []) + if not recipients: + recipients = [] + + return [recipient for recipient in recipients if not self._parsed_mail_matches(recipient, InputMail.FROM_EMAIL_ADDRESS)] + + def _parsed_mail_matches(self, to_parse, expected): + if InputMail.FROM_EMAIL_ADDRESS is None: + return False + return parseaddr(self._decoded_header_utf_8(to_parse))[1] == expected + + @staticmethod + def from_dict(mail_dict): + # TODO: implement this method and also write tests for it + headers = {key.capitalize(): value for key, value in mail_dict.get('header', {}).items()} + headers['Date'] = date.mail_date_now() + body = mail_dict.get('body', '') + tags = set(mail_dict.get('tags', [])) + status = set(mail_dict.get('status', [])) + attachments = [] + + # mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None, attachments=[] + return LeapMail(None, None, headers, tags, set(), body, attachments) + + +def _extract_filename(content_disposition): + match = re.compile('.*name=\"(.*)\".*').search(content_disposition) + filename = '' + if match: + filename = match.group(1) + return filename + + +class LeapMailStore(MailStore): + __slots__ = ('soledad') + + def __init__(self, soledad): + self.soledad = soledad + + @defer.inlineCallbacks + def get_mail(self, mail_id, include_body=False): + message = yield self._fetch_msg_from_soledad(mail_id) + if not _is_empty_message(message): + leap_mail = yield self._leap_message_to_leap_mail(mail_id, message, include_body) + else: + leap_mail = None + + defer.returnValue(leap_mail) + + def get_mails(self, mail_ids): + deferreds = [] + for mail_id in mail_ids: + deferreds.append(self.get_mail(mail_id)) + + return defer.gatherResults(deferreds, consumeErrors=True) + + @defer.inlineCallbacks + def get_mail_attachment(self, attachment_id): + results = yield self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', attachment_id) if attachment_id else [] + if len(results): + content = ContentDocWrapper(**results[0].content) + defer.returnValue({'content-type': content.content_type, 'content': self._try_decode( + content.raw, content.content_transfer_encoding)}) + else: + raise ValueError('No attachment with id %s found!' % attachment_id) + + def _try_decode(self, raw, encoding): + encoding = encoding.lower() + if encoding == 'base64': + data = base64.decodestring(raw) + elif encoding == 'quoted-printable': + data = quopri.decodestring(raw) + else: + data = str(raw) + + return bytearray(data) + + @defer.inlineCallbacks + def update_mail(self, mail): + message = yield self._fetch_msg_from_soledad(mail.mail_id) + message.get_wrapper().set_tags(tuple(mail.tags)) + message.get_wrapper().set_flags(tuple(mail.flags)) + yield self._update_mail(message) # TODO assert this is yielded (otherwise asynchronous) + + @defer.inlineCallbacks + def all_mails(self): + mdocs = yield self.soledad.get_from_index('by-type', 'meta') + + mail_ids = map(lambda doc: doc.doc_id, mdocs) + + mails = yield self.get_mails(mail_ids) + defer.returnValue(mails) + + @defer.inlineCallbacks + def add_mailbox(self, mailbox_name): + mailbox = yield self._get_or_create_mailbox(mailbox_name) + defer.returnValue(mailbox) + + @defer.inlineCallbacks + def get_mailbox_names(self): + mbox_map = set((yield self._mailbox_uuid_to_name_map()).values()) + + defer.returnValue(mbox_map.union({'INBOX'})) + + @defer.inlineCallbacks + def _mailbox_uuid_to_name_map(self): + map = {} + mbox_docs = yield self.soledad.get_from_index('by-type', 'mbox') + for doc in mbox_docs: + map[underscore_uuid(doc.content.get('uuid'))] = doc.content.get('mbox') + + defer.returnValue(map) + + @defer.inlineCallbacks + def add_mail(self, mailbox_name, raw_msg): + mailbox = yield self._get_or_create_mailbox(mailbox_name) + message = SoledadMailAdaptor().get_msg_from_string(Message, raw_msg) + message.get_wrapper().set_mbox_uuid(mailbox.uuid) + + yield SoledadMailAdaptor().create_msg(self.soledad, message) + + # add behavious from insert_mdoc_id from mail.py + mail = yield self._leap_message_to_leap_mail(message.get_wrapper().mdoc.doc_id, message, include_body=True) # TODO test that asserts include_body + defer.returnValue(mail) + + @defer.inlineCallbacks + def delete_mail(self, mail_id): + message = yield self._fetch_msg_from_soledad(mail_id) + yield message.get_wrapper().delete(self.soledad) + + @defer.inlineCallbacks + def get_mailbox_mail_ids(self, mailbox_name): + mailbox = yield self._get_or_create_mailbox(mailbox_name) + fdocs = yield self.soledad.get_from_index('by-type-and-mbox-uuid', 'flags', underscore_uuid(mailbox.uuid)) + + mail_ids = map(lambda doc: _fdoc_id_to_mdoc_id(doc.doc_id), fdocs) + + defer.returnValue(mail_ids) + + @defer.inlineCallbacks + def delete_mailbox(self, mailbox_name): + mbx_wrapper = yield self._get_or_create_mailbox(mailbox_name) + yield SoledadMailAdaptor().delete_mbox(self.soledad, mbx_wrapper) + + @defer.inlineCallbacks + def copy_mail_to_mailbox(self, mail_id, mailbox_name): + message = yield self._fetch_msg_from_soledad(mail_id, load_body=True) + mailbox = yield self._get_or_create_mailbox(mailbox_name) + copy_wrapper = yield message.get_wrapper().copy(self.soledad, mailbox.uuid) + + leap_message = Message(copy_wrapper) + + mail = yield self._leap_message_to_leap_mail(copy_wrapper.mdoc.doc_id, leap_message, include_body=False) + + defer.returnValue(mail) + + @defer.inlineCallbacks + def move_mail_to_mailbox(self, mail_id, mailbox_name): + mail_copy = yield self.copy_mail_to_mailbox(mail_id, mailbox_name) + yield self.delete_mail(mail_id) + defer.returnValue(mail_copy) + + def _update_mail(self, message): + return message.get_wrapper().update(self.soledad) + + @defer.inlineCallbacks + def _leap_message_to_leap_mail(self, mail_id, message, include_body): + if include_body: + # TODO use body from message if available + body = yield self._raw_message_body(message) + else: + body = None + + # fetch mailbox name by mbox_uuid + mbox_uuid = message.get_wrapper().fdoc.mbox_uuid + mbox_name = yield self._mailbox_name_from_uuid(mbox_uuid) + + mail = LeapMail(mail_id, mbox_name, message.get_wrapper().hdoc.headers, set(message.get_tags()), set(message.get_flags()), body=body, attachments=self._extract_attachment_info_from(message)) # TODO assert flags are passed on + + defer.returnValue(mail) + + @defer.inlineCallbacks + def _raw_message_body(self, message): + content_doc = (yield message.get_wrapper().get_body(self.soledad)) + parser = BodyParser(content_doc.raw, content_type=content_doc.content_type, content_transfer_encoding=content_doc.content_transfer_encoding) + defer.returnValue(parser.parsed_content()) + + @defer.inlineCallbacks + def _mailbox_name_from_uuid(self, uuid): + map = (yield self._mailbox_uuid_to_name_map()) + defer.returnValue(map.get(uuid, '')) + + @defer.inlineCallbacks + def _get_or_create_mailbox(self, mailbox_name): + mailbox_name_upper = mailbox_name.upper() + mbx = yield SoledadMailAdaptor().get_or_create_mbox(self.soledad, mailbox_name_upper) + if mbx.uuid is None: + mbx.uuid = str(uuid4()) + yield mbx.update(self.soledad) + defer.returnValue(mbx) + + def _fetch_msg_from_soledad(self, mail_id, load_body=False): + return SoledadMailAdaptor().get_msg_from_mdoc_id(Message, self.soledad, mail_id, get_cdocs=load_body) + + @defer.inlineCallbacks + def _dump_soledad(self): + gen, docs = yield self.soledad.get_all_docs() + for doc in docs: + print '\n%s\n' % doc + + def _extract_attachment_info_from(self, message): + wrapper = message.get_wrapper() + part_maps = wrapper.hdoc.part_map + return self._extract_part_map(part_maps) + + def _extract_part_map(self, part_maps): + result = [] + + for nr, part_map in part_maps.items(): + if 'headers' in part_map and 'phash' in part_map: + headers = {header[0]: header[1] for header in part_map['headers']} + phash = part_map['phash'] + if 'Content-Disposition' in headers: + disposition = headers['Content-Disposition'] + if 'attachment' in disposition: + filename = _extract_filename(disposition) + encoding = headers.get('Content-Transfer-Encoding', None) + result.append(AttachmentInfo(phash, filename, encoding)) + if 'part_map' in part_map: + result += self._extract_part_map(part_map['part_map']) + + return result + + +def _is_empty_message(message): + return (message is None) or (message.get_wrapper().mdoc.doc_id is None) + + +def _fdoc_id_to_mdoc_id(fdoc_id): + return 'M' + fdoc_id[1:] diff --git a/service/pixelated/adapter/mailstore/mailstore.py b/service/pixelated/adapter/mailstore/mailstore.py new file mode 100644 index 00000000..60716dfe --- /dev/null +++ b/service/pixelated/adapter/mailstore/mailstore.py @@ -0,0 +1,60 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + + +class MailStore(object): + def get_mail(self, mail_id): + pass + + def get_mail_attachment(self, attachment_id): + pass + + def get_mails(self, mail_ids): + pass + + def all_mails(self): + pass + + def delete_mail(self, mail_id): + pass + + def update_mail(self, mail): + pass + + def add_mail(self, mailbox_name, mail): + pass + + def get_mailbox_names(self): + pass + + def add_mailbox(self, mailbox_name): + pass + + def delete_mailbox(self, mailbox_name): + pass + + def get_mailbox_mail_ids(self, mailbox_name): + pass + + def copy_mail_to_mailbox(self, mail_id, mailbox_name): + pass + + def move_mail_to_mailbox(self, mail_id, mailbox_name): + pass + + +def underscore_uuid(uuid): + return uuid.replace('-', '_') diff --git a/service/pixelated/adapter/mailstore/maintenance/__init__.py b/service/pixelated/adapter/mailstore/maintenance/__init__.py new file mode 100644 index 00000000..edc442c2 --- /dev/null +++ b/service/pixelated/adapter/mailstore/maintenance/__init__.py @@ -0,0 +1,99 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from leap.keymanager.keys import KEY_TYPE_KEY, KEY_PRIVATE_KEY, KEY_ID_KEY, KEY_ADDRESS_KEY +from leap.keymanager.openpgp import OpenPGPKey + +from twisted.internet import defer +import logging + + +TYPE_OPENPGP_KEY = 'OpenPGPKey' +TYPE_OPENPGP_ACTIVE = 'OpenPGPKey-active' + +KEY_DOC_TYPES = {TYPE_OPENPGP_ACTIVE, TYPE_OPENPGP_KEY} + +logger = logging.getLogger(__name__) + + +def _is_key_doc(doc): + return doc.content.get(KEY_TYPE_KEY, None) in KEY_DOC_TYPES + + +def _is_private_key_doc(doc): + return _is_key_doc(doc) and doc.content.get(KEY_PRIVATE_KEY, False) + + +def _is_active_key_doc(doc): + return _is_key_doc(doc) and doc.content.get(KEY_TYPE_KEY, None) == TYPE_OPENPGP_ACTIVE + + +def _is_public_key(doc): + return _is_key_doc(doc) and not doc.content.get(KEY_PRIVATE_KEY, False) + + +def _key_id(doc): + return doc.content.get(KEY_ID_KEY, None) + + +def _address(doc): + return doc.content.get(KEY_ADDRESS_KEY, None) + + +class SoledadMaintenance(object): + def __init__(self, soledad): + self._soledad = soledad + + @defer.inlineCallbacks + def repair(self): + _, docs = yield self._soledad.get_all_docs() + + private_key_ids = self._key_ids_with_private_key(docs) + + for doc in docs: + if _is_key_doc(doc) and _key_id(doc) not in private_key_ids: + logger.warn('Deleting doc %s for key %s of <%s>' % (doc.doc_id, _key_id(doc), _address(doc))) + yield self._soledad.delete_doc(doc) + + yield self._repair_missing_active_docs(docs, private_key_ids) + + @defer.inlineCallbacks + def _repair_missing_active_docs(self, docs, private_key_ids): + missing = self._missing_active_docs(docs, private_key_ids) + for key_id in missing: + emails = self._emails_for_key_id(docs, key_id) + for email in emails: + logger.warn('Re-creating active doc for key %s, email %s' % (key_id, email)) + yield self._soledad.create_doc_from_json(OpenPGPKey(email, key_id=key_id, private=False).get_active_json(email)) + + def _key_ids_with_private_key(self, docs): + return [doc.content[KEY_ID_KEY] for doc in docs if _is_private_key_doc(doc)] + + def _missing_active_docs(self, docs, private_key_ids): + active_doc_ids = self._active_docs_for_key_id(docs) + + return set([private_key_id for private_key_id in private_key_ids if private_key_id not in active_doc_ids]) + + def _emails_for_key_id(self, docs, key_id): + for doc in docs: + if _is_private_key_doc(doc) and _key_id(doc) == key_id: + email = _address(doc) + if isinstance(email, list): + return email + else: + return [email] + + def _active_docs_for_key_id(self, docs): + return [doc.content[KEY_ID_KEY] for doc in docs if _is_active_key_doc(doc) and _is_public_key(doc)] diff --git a/service/pixelated/adapter/mailstore/searchable_mailstore.py b/service/pixelated/adapter/mailstore/searchable_mailstore.py new file mode 100644 index 00000000..0c5310eb --- /dev/null +++ b/service/pixelated/adapter/mailstore/searchable_mailstore.py @@ -0,0 +1,80 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from twisted.internet import defer +from types import FunctionType +from pixelated.adapter.mailstore import MailStore + + +class SearchableMailStore(object): # implementes MailStore + + def __init__(self, delegate, search_engine): + self._delegate = delegate + self._search_engine = search_engine + + @classmethod + def _create_delegator(cls, method_name): + def delegator(self, *args, **kw): + return getattr(self._delegate, method_name)(*args, **kw) + + setattr(cls, method_name, delegator) + + @defer.inlineCallbacks + def add_mail(self, mailbox_name, mail): + stored_mail = yield self._delegate.add_mail(mailbox_name, mail) + self._search_engine.index_mail(stored_mail) + defer.returnValue(stored_mail) + + @defer.inlineCallbacks + def delete_mail(self, mail_id): + yield self._delegate.delete_mail(mail_id) + self._search_engine.remove_from_index(mail_id) + + @defer.inlineCallbacks + def update_mail(self, mail): + yield self._delegate.update_mail(mail) + self._search_engine.index_mail(mail) + + @defer.inlineCallbacks + def move_mail_to_mailbox(self, mail_id, mailbox_name): + moved_mail = yield self._delegate.move_mail_to_mailbox(mail_id, mailbox_name) + self._search_engine.remove_from_index(mail_id) + self._search_engine.index_mail(moved_mail) + defer.returnValue(moved_mail) + + @defer.inlineCallbacks + def copy_mail_to_mailbox(self, mail_id, mailbox_name): + copied_mail = yield self._delegate.copy_mail_to_mailbox(mail_id, mailbox_name) + self._search_engine.index_mail(copied_mail) + defer.returnValue(copied_mail) + + def delete_mailbox(self, mailbox_name): + raise NotImplementedError() + + def __getattr__(self, name): + """ + Acts like method missing. If a method of MailStore is not implemented in this class, + a delegate method is created. + + :param name: attribute name + :return: method or attribute + """ + methods = ([key for key, value in MailStore.__dict__.items() if type(value) == FunctionType]) + + if name in methods: + SearchableMailStore._create_delegator(name) + return super(SearchableMailStore, self).__getattribute__(name) + else: + raise NotImplementedError('No attribute %s' % name) diff --git a/service/pixelated/adapter/model/mail.py b/service/pixelated/adapter/model/mail.py index 464e0343..b89e511a 100644 --- a/service/pixelated/adapter/model/mail.py +++ b/service/pixelated/adapter/model/mail.py @@ -21,21 +21,38 @@ import dateutil.parser as dateparser from uuid import uuid4 from email import message_from_file from email.mime.text import MIMEText -from email.header import decode_header +from email.header import decode_header, Header from email.MIMEMultipart import MIMEMultipart from pycryptopp.hash import sha256 -from leap.mail.imap.fields import fields +from leap.mail.adaptors import soledad_indexes as fields import leap.mail.walk as walk from pixelated.adapter.model.status import Status from pixelated.support import date from pixelated.support.functional import compact +from twisted.internet import defer + logger = logging.getLogger(__name__) +TYPE_KEY = 'type' +CONTENT_HASH_KEY = 'chash' +HEADERS_KEY = 'headers' +DATE_KEY = 'date' +SUBJECT_KEY = 'subject' +PARTS_MAP_KEY = 'part_map' +BODY_KEY = 'body' +MSGID_KEY = 'msgid' +MULTIPART_KEY = 'multi' +SIZE_KEY = 'size' + class Mail(object): @property + def from_sender(self): + return self.headers['From'] + + @property def to(self): return self.headers['To'] @@ -48,6 +65,10 @@ class Mail(object): return self.headers['Bcc'] @property + def subject(self): + return self.headers['Subject'] + + @property def date(self): return self.headers['Date'] @@ -61,7 +82,17 @@ class Mail(object): @property def mailbox_name(self): - return self.fdoc.content.get('mbox') + # FIXME mbox is no longer available, instead we now have mbox_uuid + return self.fdoc.content.get('mbox', 'INBOX') + + def _encode_header_value_list(self, header_value_list): + return [self._encode_header_value(v) for v in header_value_list] + + def _encode_header_value(self, header_value): + if isinstance(header_value, unicode): + return str(Header(header_value, 'utf-8')) + else: + return str(header_value) @property def _mime_multipart(self): @@ -69,7 +100,10 @@ class Mail(object): return self._mime mime = MIMEMultipart() for key, value in self.headers.items(): - mime[str(key)] = str(value) + if isinstance(value, list): + mime[str(key)] = ', '.join(self._encode_header_value_list(value)) + else: + mime[str(key)] = self._encode_header_value(value) try: body_to_use = self.body @@ -128,19 +162,19 @@ class InputMail(Mail): return self._fd fd = {} - fd[fields.MBOX_KEY] = mailbox - fd[fields.UID_KEY] = next_uid - fd[fields.CONTENT_HASH_KEY] = self._get_chash() - fd[fields.SIZE_KEY] = len(self.raw) - fd[fields.MULTIPART_KEY] = True - fd[fields.RECENT_KEY] = True - fd[fields.TYPE_KEY] = fields.TYPE_FLAGS_VAL - fd[fields.FLAGS_KEY] = Status.to_flags(self._status) + fd[fields.MBOX] = mailbox + fd[fields.MBOX_UUID] = next_uid + fd[fields.CONTENT_HASH] = self._get_chash() + fd[SIZE_KEY] = len(self.raw) + fd[MULTIPART_KEY] = True + fd[fields.RECENT] = True + fd[fields.TYPE] = fields.FLAGS + fd[fields.FLAGS] = Status.to_flags(self._status) self._fd = fd return fd def _get_body_phash(self): - return walk.get_body_phash_multi(walk.get_payloads(self._mime_multipart)) + return walk.get_body_phash(self._mime_multipart) def _hdoc(self): if self._hd: @@ -151,15 +185,15 @@ class InputMail(Mail): headers['From'] = InputMail.FROM_EMAIL_ADDRESS hd = {} - hd[fields.HEADERS_KEY] = headers - hd[fields.DATE_KEY] = headers['Date'] - hd[fields.CONTENT_HASH_KEY] = self._get_chash() - hd[fields.MSGID_KEY] = '' - hd[fields.MULTIPART_KEY] = True - hd[fields.SUBJECT_KEY] = headers.get('Subject') - hd[fields.TYPE_KEY] = fields.TYPE_HEADERS_VAL - hd[fields.BODY_KEY] = self._get_body_phash() - hd[fields.PARTS_MAP_KEY] = \ + hd[HEADERS_KEY] = headers + hd[DATE_KEY] = headers['Date'] + hd[CONTENT_HASH_KEY] = self._get_chash() + hd[MSGID_KEY] = '' + hd[MULTIPART_KEY] = True + hd[SUBJECT_KEY] = headers.get('Subject') + hd[TYPE_KEY] = fields.HEADERS + hd[BODY_KEY] = self._get_body_phash() + hd[PARTS_MAP_KEY] = \ walk.walk_msg_tree(walk.get_parts(self._mime_multipart), body_phash=self._get_body_phash())['part_map'] self._hd = hd @@ -172,12 +206,15 @@ class InputMail(Mail): mime_multipart = MIMEMultipart() for header in ['To', 'Cc', 'Bcc']: - if self.headers[header]: + if self.headers.get(header): mime_multipart[header] = ", ".join(self.headers[header]) - if self.headers['Subject']: + if self.headers.get('Subject'): mime_multipart['Subject'] = self.headers['Subject'] + if self.headers.get('From'): + mime_multipart['From'] = self.headers['From'] + mime_multipart['Date'] = self.headers['Date'] if type(self.body) is list: for part in self.body: @@ -207,13 +244,10 @@ class InputMail(Mail): input_mail = InputMail() input_mail.headers = {key.capitalize(): value for key, value in mail_dict.get('header', {}).items()} - # XXX this is overriding the property in PixelatedMail - input_mail.headers['Date'] = date.iso_now() + input_mail.headers['Date'] = date.mail_date_now() - # XXX this is overriding the property in PixelatedMail input_mail.body = mail_dict.get('body', '') - # XXX this is overriding the property in the PixelatedMail input_mail.tags = set(mail_dict.get('tags', [])) input_mail._status = set(mail_dict.get('status', [])) @@ -222,292 +256,18 @@ class InputMail(Mail): @staticmethod def from_python_mail(mail): input_mail = InputMail() - input_mail.headers = {key.capitalize(): value for key, value in mail.items()} - input_mail.headers['Date'] = date.iso_now() - input_mail.headers['Subject'] = mail['Subject'] - input_mail.headers['To'] = InputMail.FROM_EMAIL_ADDRESS - input_mail._mime = MIMEMultipart() + input_mail.headers = {unicode(key.capitalize()): unicode(value) for key, value in mail.items()} + input_mail.headers[u'Date'] = unicode(date.mail_date_now()) + input_mail.headers[u'To'] = [u''] + for payload in mail.get_payload(): - input_mail._mime.attach(payload) + input_mail._mime_multipart.attach(payload) if payload.get_content_type() == 'text/plain': - input_mail.body = payload.as_string() + input_mail.body = unicode(payload.as_string()) + input_mail._mime = input_mail.to_mime_multipart() return input_mail -class PixelatedMail(Mail): - - @staticmethod - def from_soledad(fdoc, hdoc, bdoc, parts=None, soledad_querier=None): - mail = PixelatedMail() - mail.parts = parts - mail.boundary = str(uuid4()).replace('-', '') - mail.bdoc = bdoc - mail.fdoc = fdoc - mail.hdoc = hdoc - mail.querier = soledad_querier - mail._mime = None - return mail - - def _decode_part(self, part): - encoding = part['headers'].get('Content-Transfer-Encoding', '') - content_type = self._parse_charset_header(part['headers'].get('Content-Type')) - - try: - decoding_func = self._decoding_function_for_encoding(encoding) - return self._decode_content_with_fallback(part['content'], decoding_func, content_type) - except Exception: - logger.error('Failed to decode mail part with:') - logger.error('Content-Transfer-Encoding: %s' % encoding) - logger.error('Content-Type: %s' % part['headers'].get('Content-Type')) - raise - - def _decoding_function_for_encoding(self, encoding): - decoding_map = { - 'quoted-printable': lambda content, content_type: content.decode('quopri').decode(content_type), - 'base64': lambda content, content_type: content.decode('base64').decode('utf-8'), - '7bit': lambda content, content_type: content.encode(content_type), - '8bit': lambda content, content_type: content.encode(content_type) - } - if encoding in decoding_map: - return decoding_map[encoding] - else: - return decoding_map['8bit'] - - def _decode_content_with_fallback(self, content, decode_func, content_type): - try: - return decode_func(content, content_type) - # return content.encode(content_type) - except ValueError: - return content.encode('ascii', 'ignore') - - @property - def alternatives(self): - return self.parts.get('alternatives') - - @property - def text_plain_body(self): - if self.parts and len(self.alternatives) >= 1: - return self._decode_part(self.alternatives[0]) - else: - return self.bdoc.content['raw'] # plain - - @property - def html_body(self): - if self.parts and len(self.alternatives) > 1: - html_parts = [e for e in self.alternatives if re.match('text/html', e['headers'].get('Content-Type', ''))] - if len(html_parts): - return self._decode_part(html_parts[0]) - - @property - def headers(self): - _headers = { - 'To': [], - 'Cc': [], - 'Bcc': [] - } - hdoc_headers = self.hdoc.content['headers'] - - for header in ['To', 'Cc', 'Bcc']: - header_value = self._decode_header(hdoc_headers.get(header)) - if not header_value: - continue - _headers[header] = header_value if type(header_value) is list else header_value.split(',') - _headers[header] = [head.strip() for head in compact(_headers[header])] - - for header in ['From', 'Subject']: - _headers[header] = self._decode_header(hdoc_headers.get(header)) - - try: - _headers['Date'] = self._get_date() - except Exception: - _headers['Date'] = date.iso_now() - - if self.parts and len(self.parts['alternatives']) > 1: - _headers['content_type'] = 'multipart/alternative; boundary="%s"' % self.boundary - elif self.hdoc.content['headers'].get('Content-Type'): - _headers['content_type'] = hdoc_headers.get('Content-Type') - - if hdoc_headers.get('Reply-To'): - _headers['Reply-To'] = hdoc_headers.get('Reply-To') - - return _headers - - def _decode_header_with_fallback(self, entry): - try: - return decode_header(entry)[0][0] - except Exception: - return entry.encode('ascii', 'ignore') - - def _decode_header(self, header): - if not header: - return None - if isinstance(header, list): - return [self._decode_header_with_fallback(entry) for entry in header] - else: - return self._decode_header_with_fallback(header) - - def _get_date(self): - date = self.hdoc.content.get('date', None) - try: - if not date: - received = self.hdoc.content.get('received', None) - if received: - date = received.split(";")[-1].strip() - else: - # we can't get a date for this mail, so lets just use now - logger.warning('Encountered a mail with missing date and received header fields. ID %s' % self.fdoc.content.get('uid', None)) - date = date.iso_now() - return dateparser.parse(date).isoformat() - except (ValueError, TypeError): - date = date.iso_now() - return dateparser.parse(date).isoformat() - - @property - def security_casing(self): - casing = {"imprints": [], "locks": []} - casing["imprints"] = self.signature_information - if self.encrypted == "true": - casing["locks"] = [{"state": "valid"}] - elif self.encrypted == "fail": - casing["locks"] = [{"state": "failure"}] - return casing - - @property - def tags(self): - _tags = self.fdoc.content.get('tags', '[]') - return set(_tags) if type(_tags) is list or type(_tags) is set else set(json.loads(_tags)) - - @property - def ident(self): - return self.fdoc.content.get('chash') - - @property - def mailbox_name(self): - return self.fdoc.content.get('mbox') - - @property - def is_recent(self): - return Status('recent') in self.status - - @property - def uid(self): - return self.fdoc.content['uid'] - - @property - def flags(self): - return self.fdoc.content['flags'] - - def save(self): - return self.querier.save_mail(self) - - def set_mailbox(self, mailbox_name): - self.fdoc.content['mbox'] = mailbox_name - - def remove_all_tags(self): - self.update_tags(set([])) - - def update_tags(self, tags): - self._persist_mail_tags(tags) - return self.tags - - def mark_as_read(self): - if Status.SEEN in self.flags: - return self - self.flags.append(Status.SEEN) - self.save() - return self - - def mark_as_unread(self): - if Status.SEEN in self.flags: - self.flags.remove(Status.SEEN) - self.save() - return self - - def mark_as_not_recent(self): - if Status.RECENT in self.flags: - self.flags.remove(Status.RECENT) - self.save() - return self - - def _persist_mail_tags(self, current_tags): - self.fdoc.content['tags'] = json.dumps(list(current_tags)) - self.save() - - def has_tag(self, tag): - return tag in self.tags - - @property - def signature_information(self): - signature = self.hdoc.content["headers"].get("X-Leap-Signature", None) - if signature is None or signature.startswith("could not verify"): - return [{"state": "no_signature_information"}] - else: - if signature.startswith("valid"): - return [{"state": "valid", "seal": {"validity": "valid"}}] - else: - return [] - - @property - def encrypted(self): - return self.hdoc.content["headers"].get("X-Pixelated-encryption-status", "false") - - @property - def bounced(self): - content_type = self.hdoc.content["headers"].get("Content-Type", '') - if re.compile('delivery-status').search(content_type): - bounce_recipient = self._extract_bounced_address(self.hdoc.content) - bounce_daemon = self.headers["From"] - return [bounce_recipient, bounce_daemon] if bounce_recipient else False - - return False - - def _extract_bounced_address(self, part): - part_header = dict(part.get('headers', {})) - if 'Final-Recipient' in part_header: - if self._bounce_permanent(part_header): - return part_header['Final-Recipient'].split(';')[1].strip() - else: - return False - elif 'part_map' in part: - for subpart in part['part_map'].values(): - result = self._extract_bounced_address(subpart) - if result: - return result - else: - continue - return False - - def _bounce_permanent(self, part_headers): - status = part_headers.get('Status', '') - return status.startswith('5') - - def as_dict(self): - dict_mail = {'header': {k.lower(): v for k, v in self.headers.items()}, - 'ident': self.ident, - 'tags': list(self.tags), - 'status': list(self.status), - 'security_casing': self.security_casing, - 'textPlainBody': self.text_plain_body, - 'htmlBody': self.html_body, - 'mailbox': self.mailbox_name.lower(), - 'attachments': self.parts['attachments'] if self.parts else []} - dict_mail['replying'] = {'single': None, 'all': {'to-field': [], 'cc-field': []}} - - sender_mail = self.headers.get('Reply-To', self.headers.get('From')) - # Issue #215: Fix for existing mails without any from address. - if sender_mail is None: - sender_mail = InputMail.FROM_EMAIL_ADDRESS - - recipients = [recipient for recipient in self.headers['To'] if recipient != InputMail.FROM_EMAIL_ADDRESS] - recipients.append(sender_mail) - ccs = [cc for cc in self.headers['Cc'] if cc != InputMail.FROM_EMAIL_ADDRESS] - - dict_mail['replying']['single'] = sender_mail - dict_mail['replying']['all']['to-field'] = recipients - dict_mail['replying']['all']['cc-field'] = ccs - return dict_mail - - def welcome_mail(): current_path = os.path.dirname(os.path.abspath(__file__)) with open(os.path.join(current_path, '..', '..', 'assets', 'welcome.mail')) as mail_template_file: diff --git a/service/pixelated/adapter/search/__init__.py b/service/pixelated/adapter/search/__init__.py index b8d3e7ca..56ab2255 100644 --- a/service/pixelated/adapter/search/__init__.py +++ b/service/pixelated/adapter/search/__init__.py @@ -18,6 +18,8 @@ from pixelated.support.encrypted_file_storage import EncryptedFileStorage import os import re +import dateutil.parser +import time from pixelated.adapter.model.status import Status from pixelated.adapter.search.contacts import contacts_suggestions from whoosh.index import FileIndex @@ -27,7 +29,6 @@ from whoosh.qparser import MultifieldParser from whoosh.writing import AsyncWriter from whoosh import sorting from pixelated.support.functional import unique -from pixelated.support.date import milliseconds import traceback @@ -102,7 +103,6 @@ class SearchEngine(object): to=KEYWORD(stored=False, commas=True), cc=KEYWORD(stored=False, commas=True), bcc=KEYWORD(stored=False, commas=True), - bounced=KEYWORD(stored=False, commas=True), subject=TEXT(stored=False), date=NUMERIC(stored=False, sortable=True, bits=64, signed=False), body=TEXT(stored=False), @@ -121,32 +121,38 @@ class SearchEngine(object): def _index_mail(self, writer, mail): mdict = mail.as_dict() header = mdict['header'] - tags = mdict.get('tags', []) - tags.append(mail.mailbox_name.lower()) - bounced = mail.bounced if mail.bounced else [''] + tags = set(mdict.get('tags', {})) + tags.add(mail.mailbox_name.lower()) index_data = { - 'sender': self._unicode_header_field(header.get('from', '')), - 'subject': self._unicode_header_field(header.get('subject', '')), - 'date': milliseconds(header.get('date', '')), - 'to': u','.join([h.decode('utf-8') for h in header.get('to', [''])]), - 'cc': u','.join([h.decode('utf-8') for h in header.get('cc', [''])]), - 'bcc': u','.join([h.decode('utf-8') for h in header.get('bcc', [''])]), + 'sender': self._empty_string_to_none(header.get('from', '')), + 'subject': self._empty_string_to_none(header.get('subject', '')), + 'date': self._format_utc_integer(header.get('date', '')), + 'to': self._format_recipient(header, 'to'), + 'cc': self._format_recipient(header, 'cc'), + 'bcc': self._format_recipient(header, 'bcc'), 'tag': u','.join(unique(tags)), - 'bounced': u','.join(bounced), - 'body': unicode(mdict['textPlainBody']), + 'body': unicode(mdict['textPlainBody'] if 'textPlainBody' in mdict else mdict['body']), 'ident': unicode(mdict['ident']), 'flags': unicode(','.join(unique(mail.flags))), - 'raw': unicode(mail.raw.decode('utf-8')) + 'raw': unicode(mail.raw) } writer.update_document(**index_data) - def _unicode_header_field(self, field_value): + def _format_utc_integer(self, date): + timetuple = dateutil.parser.parse(date).utctimetuple() + return time.strftime('%s', timetuple) + + def _format_recipient(self, headers, name): + list = headers.get(name, ['']) + return u','.join(list) if list else u'' + + def _empty_string_to_none(self, field_value): if not field_value: return None - - return unicode(field_value.decode('utf-8')) + else: + return field_value def index_mails(self, mails, callback=None): try: diff --git a/service/pixelated/adapter/search/contacts.py b/service/pixelated/adapter/search/contacts.py index 0dfeb15b..0729e146 100644 --- a/service/pixelated/adapter/search/contacts.py +++ b/service/pixelated/adapter/search/contacts.py @@ -31,21 +31,12 @@ def address_duplication_filter(contacts): return contacts_by_mail.values() -def bounced_addresses_filter(searcher, contacts): - query = QueryParser('bounced', searcher.schema).parse('*') - bounced_addresses = searcher.search(query, - limit=None, - groupedby=sorting.FieldFacet('bounced', - allow_overlap=True)).groups() - return set(contacts) - set(flatten([bounced_addresses])) - - def extract_mail_address(text): return parseaddr(text)[1] def contacts_suggestions(query, searcher): - return address_duplication_filter(bounced_addresses_filter(searcher, search_addresses(searcher, query))) if query else [] + return address_duplication_filter(search_addresses(searcher, query)) if query else [] def search_addresses(searcher, query): diff --git a/service/pixelated/adapter/search/index_storage_key.py b/service/pixelated/adapter/search/index_storage_key.py new file mode 100644 index 00000000..b2761849 --- /dev/null +++ b/service/pixelated/adapter/search/index_storage_key.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import base64 +from twisted.internet import defer +import os + + +class SearchIndexStorageKey(object): + __slots__ = '_soledad' + + def __init__(self, soledad): + self._soledad = soledad + + @defer.inlineCallbacks + def get_or_create_key(self): + docs = yield self._soledad.get_from_index('by-type', 'index_key') + + if len(docs): + key = docs[0].content['value'] + else: + key = self._new_index_key() + yield self._store_key_in_soledad(key) + defer.returnValue(key) + + def _new_index_key(self): + return os.urandom(64) # 32 for encryption, 32 for hmac + + def _store_key_in_soledad(self, index_key): + return self._soledad.create_doc(dict(type='index_key', value=base64.encodestring(index_key))) diff --git a/service/pixelated/adapter/services/draft_service.py b/service/pixelated/adapter/services/draft_service.py index c8df0a05..65794f04 100644 --- a/service/pixelated/adapter/services/draft_service.py +++ b/service/pixelated/adapter/services/draft_service.py @@ -13,19 +13,36 @@ # # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from twisted.internet import defer +from pixelated.adapter.errors import DuplicatedDraftException class DraftService(object): - __slots__ = '_mailboxes' + __slots__ = '_mail_store' - def __init__(self, mailboxes): - self._mailboxes = mailboxes + def __init__(self, mail_store): + self._mail_store = mail_store + @defer.inlineCallbacks def create_draft(self, input_mail): - pixelated_mail = self._mailboxes.drafts.add(input_mail) - return pixelated_mail + mail = yield self._mail_store.add_mail('DRAFTS', input_mail.raw) + defer.returnValue(mail) + # pixelated_mail = yield (yield self._mailboxes.drafts).add(input_mail) + # defer.returnValue(pixelated_mail) + @defer.inlineCallbacks def update_draft(self, ident, input_mail): - pixelated_mail = self.create_draft(input_mail) - self._mailboxes.drafts.remove(ident) - return pixelated_mail + new_draft = yield self.create_draft(input_mail) + try: + yield self._mail_store.delete_mail(ident) + defer.returnValue(new_draft) + except Exception as error: + errorMessage = error.args[0].getErrorMessage() + + if errorMessage == 'Need to create doc before deleting': + yield self._mail_store.delete_mail(new_draft.ident) + raise DuplicatedDraftException(errorMessage) + + # pixelated_mail = yield self.create_draft(input_mail) + # yield (yield self._mailboxes.drafts).remove(ident) + # defer.returnValue(pixelated_mail) diff --git a/service/pixelated/adapter/services/feedback_service.py b/service/pixelated/adapter/services/feedback_service.py new file mode 100644 index 00000000..5200a9ff --- /dev/null +++ b/service/pixelated/adapter/services/feedback_service.py @@ -0,0 +1,20 @@ +import os +import requests + + +class FeedbackService(object): + FEEDBACK_URL = os.environ.get('FEEDBACK_URL') + + def __init__(self, leap_session): + self.leap_session = leap_session + + def open_ticket(self, feedback): + account_mail = self.leap_session.account_email() + data = { + "ticket[comments_attributes][0][body]": feedback, + "ticket[subject]": "Feedback user-agent from {0}".format(account_mail), + "ticket[email]": account_mail, + "ticket[regarding_user]": account_mail + } + + return requests.post(self.FEEDBACK_URL, data=data, verify=False) diff --git a/service/pixelated/adapter/services/mail_service.py b/service/pixelated/adapter/services/mail_service.py index 233d4d4a..44c4c145 100644 --- a/service/pixelated/adapter/services/mail_service.py +++ b/service/pixelated/adapter/services/mail_service.py @@ -13,37 +13,50 @@ # # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from twisted.internet import defer from pixelated.adapter.model.mail import InputMail +from pixelated.adapter.model.status import Status from pixelated.adapter.services.tag_service import extract_reserved_tags +from email import message_from_file +import os class MailService(object): - def __init__(self, mailboxes, mail_sender, soledad_querier, search_engine): - self.mailboxes = mailboxes - self.querier = soledad_querier + def __init__(self, mail_sender, mail_store, search_engine): + self.mail_store = mail_store self.search_engine = search_engine self.mail_sender = mail_sender + @defer.inlineCallbacks def all_mails(self): - return self.querier.all_mails() + mails = yield self.mail_store.all_mails() + defer.returnValue(mails) + @defer.inlineCallbacks def mails(self, query, window_size, page): mail_ids, total = self.search_engine.search(query, window_size, page) - return self.querier.mails(mail_ids), total + try: + mails = yield self.mail_store.get_mails(mail_ids) + defer.returnValue((mails, total)) + except Exception, e: + import traceback + traceback.print_exc() + raise + @defer.inlineCallbacks def update_tags(self, mail_id, new_tags): new_tags = self._filter_white_space_tags(new_tags) reserved_words = extract_reserved_tags(new_tags) if len(reserved_words): raise ValueError('None of the following words can be used as tags: ' + ' '.join(reserved_words)) new_tags = self._favor_existing_tags_casing(new_tags) - mail = self.mail(mail_id) - mail.update_tags(set(new_tags)) - self.search_engine.index_mail(mail) + mail = yield self.mail(mail_id) + mail.tags = set(new_tags) + yield self.mail_store.update_mail(mail) - return mail + defer.returnValue(mail) def _filter_white_space_tags(self, tags): return [tag.strip() for tag in tags if not tag.isspace()] @@ -58,53 +71,65 @@ class MailService(object): return [_use_current_casing(new_tag.lower()) if new_tag.lower() in current_tags_lower else new_tag for new_tag in new_tags] def mail(self, mail_id): - return self.querier.mail(mail_id) + return self.mail_store.get_mail(mail_id, include_body=True) - def attachment(self, attachment_id, encoding): - return self.querier.attachment(attachment_id, encoding) + def attachment(self, attachment_id): + return self.mail_store.get_mail_attachment(attachment_id) + @defer.inlineCallbacks def mail_exists(self, mail_id): - return not(not(self.querier.get_header_by_chash(mail_id))) + try: + mail = yield self.mail_store.get_mail(mail_id, include_body=False) + defer.returnValue(mail is not None) + except Exception, e: + defer.returnValue(False) + @defer.inlineCallbacks def send_mail(self, content_dict): mail = InputMail.from_dict(content_dict) draft_id = content_dict.get('ident') - def move_to_sent(_): - return self.move_to_sent(draft_id, mail) - - deferred = self.mail_sender.sendmail(mail) - deferred.addCallback(move_to_sent) - return deferred + yield self.mail_sender.sendmail(mail) + sent_mail = yield self.move_to_sent(draft_id, mail) + defer.returnValue(sent_mail) + @defer.inlineCallbacks def move_to_sent(self, last_draft_ident, mail): if last_draft_ident: - self.mailboxes.drafts.remove(last_draft_ident) - return self.mailboxes.sent.add(mail) + try: + yield self.mail_store.delete_mail(last_draft_ident) + except Exception as error: + pass + + sent_mail = yield self.mail_store.add_mail('SENT', mail.raw) + sent_mail.flags.add(Status.SEEN) + yield self.mail_store.update_mail(sent_mail) + defer.returnValue(sent_mail) + @defer.inlineCallbacks def mark_as_read(self, mail_id): - mail = self.mail(mail_id) - mail.mark_as_read() - self.search_engine.index_mail(mail) + mail = yield self.mail(mail_id) + mail.flags.add(Status.SEEN) + yield self.mail_store.update_mail(mail) + @defer.inlineCallbacks def mark_as_unread(self, mail_id): - mail = self.mail(mail_id) - mail.mark_as_unread() - self.search_engine.index_mail(mail) + mail = yield self.mail(mail_id) + mail.flags.remove(Status.SEEN) + yield self.mail_store.update_mail(mail) + @defer.inlineCallbacks def delete_mail(self, mail_id): - mail = self.mail(mail_id) - if mail.mailbox_name == 'TRASH': - self.delete_permanent(mail_id) + mail = yield self.mail(mail_id) + if mail.mailbox_name.upper() == u'TRASH': + yield self.mail_store.delete_mail(mail_id) else: - trashed_mail = self.mailboxes.move_to_trash(mail_id) - self.search_engine.index_mail(trashed_mail) + yield self.mail_store.move_mail_to_mailbox(mail_id, 'TRASH') + @defer.inlineCallbacks def recover_mail(self, mail_id): - recovered_mail = self.mailboxes.move_to_inbox(mail_id) - self.search_engine.index_mail(recovered_mail) + yield self.mail_store.move_mail_to_mailbox(mail_id, 'INBOX') + @defer.inlineCallbacks def delete_permanent(self, mail_id): - mail = self.mail(mail_id) - self.search_engine.remove_from_index(mail_id) - self.querier.remove_mail(mail) + yield self.mail_store.delete_mail(mail_id) diff --git a/service/pixelated/adapter/services/mailbox.py b/service/pixelated/adapter/services/mailbox.py deleted file mode 100644 index a4029d78..00000000 --- a/service/pixelated/adapter/services/mailbox.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. - - -class Mailbox(object): - - def __init__(self, mailbox_name, querier, search_engine): - self.mailbox_name = mailbox_name - self.mailbox_tag = mailbox_name.lower() - self.search_engine = search_engine - self.querier = querier - - @property - def fresh(self): - return self.querier.get_lastuid(self.mailbox_name) == 0 - - def mail(self, mail_id): - return self.querier.mail(mail_id) - - def add(self, mail): - added_mail = self.querier.create_mail(mail, self.mailbox_name) - self.search_engine.index_mail(added_mail) - return added_mail - - def remove(self, ident): - mail = self.querier.mail(ident) - self.search_engine.remove_from_index(mail.ident) - mail.remove_all_tags() - self.querier.remove_mail(mail) - - @classmethod - def create(cls, mailbox_name, soledad_querier, search_engine): - return Mailbox(mailbox_name, soledad_querier, search_engine) diff --git a/service/pixelated/adapter/services/mailboxes.py b/service/pixelated/adapter/services/mailboxes.py deleted file mode 100644 index c2b61ca8..00000000 --- a/service/pixelated/adapter/services/mailboxes.py +++ /dev/null @@ -1,74 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. -from pixelated.adapter.services.mailbox import Mailbox -from pixelated.adapter.listeners.mailbox_indexer_listener import MailboxIndexerListener -from pixelated.adapter.model.mail import welcome_mail - - -class Mailboxes(object): - - def __init__(self, account, soledad_querier, search_engine): - self.account = account - self.querier = soledad_querier - self.search_engine = search_engine - for mailbox_name in account.mailboxes: - MailboxIndexerListener.listen(self.account, mailbox_name, soledad_querier) - - def _create_or_get(self, mailbox_name): - mailbox_name = mailbox_name.upper() - if mailbox_name not in self.account.mailboxes: - self.account.addMailbox(mailbox_name) - MailboxIndexerListener.listen(self.account, mailbox_name, self.querier) - return Mailbox.create(mailbox_name, self.querier, self.search_engine) - - @property - def inbox(self): - return self._create_or_get('INBOX') - - @property - def drafts(self): - return self._create_or_get('DRAFTS') - - @property - def trash(self): - return self._create_or_get('TRASH') - - @property - def sent(self): - return self._create_or_get('SENT') - - def mailboxes(self): - return [self._create_or_get(leap_mailbox_name) for leap_mailbox_name in self.account.mailboxes] - - def move_to_trash(self, mail_id): - return self._move_to(mail_id, self.trash) - - def move_to_inbox(self, mail_id): - return self._move_to(mail_id, self.inbox) - - def _move_to(self, mail_id, mailbox): - mail = self.querier.mail(mail_id) - mail.set_mailbox(mailbox.mailbox_name) - mail.save() - return mail - - def mail(self, mail_id): - return self.querier.mail(mail_id) - - def add_welcome_mail_for_fresh_user(self): - if self.inbox.fresh: - mail = welcome_mail() - self.inbox.add(mail) diff --git a/service/pixelated/adapter/soledad/soledad_duplicate_removal_mixin.py b/service/pixelated/adapter/soledad/soledad_duplicate_removal_mixin.py deleted file mode 100644 index 0dd3d497..00000000 --- a/service/pixelated/adapter/soledad/soledad_duplicate_removal_mixin.py +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. -from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin - - -class SoledadDuplicateRemovalMixin(SoledadDbFacadeMixin, object): - - def remove_duplicates(self): - for mailbox in ['INBOX', 'DRAFTS', 'SENT', 'TRASH']: - self._remove_dup_inboxes(mailbox) - self._remove_dup_recent(mailbox) - - def _remove_many(self, docs): - [self.delete_doc(doc) for doc in docs] - - def _remove_dup_inboxes(self, mailbox_name): - mailboxes = self.get_mbox(mailbox_name) - if len(mailboxes) == 0: - return - mailboxes_to_remove = sorted(mailboxes, key=lambda x: x.content['created'])[1:len(mailboxes)] - self._remove_many(mailboxes_to_remove) - - def _remove_dup_recent(self, mailbox_name): - rct = self.get_recent_by_mbox(mailbox_name) - if len(rct) == 0: - return - rct_to_remove = sorted(rct, key=lambda x: len(x.content['rct']), reverse=True)[1:len(rct)] - self._remove_many(rct_to_remove) diff --git a/service/pixelated/adapter/soledad/soledad_facade_mixin.py b/service/pixelated/adapter/soledad/soledad_facade_mixin.py deleted file mode 100644 index 2a50b17d..00000000 --- a/service/pixelated/adapter/soledad/soledad_facade_mixin.py +++ /dev/null @@ -1,72 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. - - -class SoledadDbFacadeMixin(object): - - def get_all_flags(self): - return self.soledad.get_from_index('by-type', 'flags') - - def get_all_flags_by_mbox(self, mbox): - return self.soledad.get_from_index('by-type-and-mbox', 'flags', mbox) if mbox else [] - - def get_content_by_phash(self, phash): - content = self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', phash) if phash else [] - if len(content): - return content[0] - - def get_flags_by_chash(self, chash): - flags = self.soledad.get_from_index('by-type-and-contenthash', 'flags', chash) if chash else [] - if len(flags): - return flags[0] - - def get_header_by_chash(self, chash): - header = self.soledad.get_from_index('by-type-and-contenthash', 'head', chash) if chash else [] - if len(header): - return header[0] - - def get_recent_by_mbox(self, mbox): - return self.soledad.get_from_index('by-type-and-mbox', 'rct', mbox) if mbox else [] - - def put_doc(self, doc): - return self.soledad.put_doc(doc) - - def create_doc(self, doc): - return self.soledad.create_doc(doc) - - def create_docs(self, docs): - for doc in docs: - self.create_doc(doc) - - def delete_doc(self, doc): - return self.soledad.delete_doc(doc) - - def idents_by_mailbox(self, mbox): - return set(doc.content['chash'] for doc in self.soledad.get_from_index('by-type-and-mbox-and-deleted', 'flags', mbox, '0')) if mbox else set() - - def get_all_mbox(self): - return self.soledad.get_from_index('by-type', 'mbox') - - def get_mbox(self, mbox): - return self.soledad.get_from_index('by-type-and-mbox', 'mbox', mbox) if mbox else [] - - def get_lastuid(self, mbox): - if isinstance(mbox, str): - mbox = self.get_mbox(mbox)[0] - return mbox.content['lastuid'] - - def get_search_index_masterkey(self): - return self.soledad.get_from_index('by-type', 'index_key') diff --git a/service/pixelated/adapter/soledad/soledad_querier.py b/service/pixelated/adapter/soledad/soledad_querier.py deleted file mode 100644 index e0b215d3..00000000 --- a/service/pixelated/adapter/soledad/soledad_querier.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. -from pixelated.adapter.soledad.soledad_duplicate_removal_mixin import SoledadDuplicateRemovalMixin -from pixelated.adapter.soledad.soledad_reader_mixin import SoledadReaderMixin -from pixelated.adapter.soledad.soledad_search_key_masterkey_retrieval_mixin import SoledadSearchIndexMasterkeyRetrievalMixin -from pixelated.adapter.soledad.soledad_writer_mixin import SoledadWriterMixin - - -class SoledadQuerier(SoledadWriterMixin, - SoledadReaderMixin, - SoledadDuplicateRemovalMixin, - SoledadSearchIndexMasterkeyRetrievalMixin, - object): - - def __init__(self, soledad): - self.soledad = soledad diff --git a/service/pixelated/adapter/soledad/soledad_reader_mixin.py b/service/pixelated/adapter/soledad/soledad_reader_mixin.py deleted file mode 100644 index 347938ed..00000000 --- a/service/pixelated/adapter/soledad/soledad_reader_mixin.py +++ /dev/null @@ -1,121 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. -import base64 -import logging -import quopri -import re - -from pixelated.adapter.model.mail import PixelatedMail -from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin - - -logger = logging.getLogger(__name__) - - -class SoledadReaderMixin(SoledadDbFacadeMixin, object): - - def all_mails(self): - fdocs_chash = [(fdoc, fdoc.content['chash']) for fdoc in self.get_all_flags()] - if len(fdocs_chash) == 0: - return [] - return self._build_mails_from_fdocs(fdocs_chash) - - def _build_mails_from_fdocs(self, fdocs_chash): - if len(fdocs_chash) == 0: - return [] - - fdocs_hdocs = [] - for fdoc, chash in fdocs_chash: - hdoc = self.get_header_by_chash(chash) - if not hdoc: - continue - fdocs_hdocs.append((fdoc, hdoc)) - - fdocs_hdocs_bodyphash = [(f[0], f[1], f[1].content.get('body')) for f in fdocs_hdocs] - fdocs_hdocs_bdocs_parts = [] - for fdoc, hdoc, body_phash in fdocs_hdocs_bodyphash: - bdoc = self.get_content_by_phash(body_phash) - if not bdoc: - continue - parts = self._extract_parts(hdoc.content) - fdocs_hdocs_bdocs_parts.append((fdoc, hdoc, bdoc, parts)) - - return [PixelatedMail.from_soledad(*raw_mail, soledad_querier=self) for raw_mail in fdocs_hdocs_bdocs_parts] - - def mail_exists(self, ident): - return self.get_flags_by_chash(ident) - - def mail(self, ident): - fdoc = self.get_flags_by_chash(ident) - hdoc = self.get_header_by_chash(ident) - bdoc = self.get_content_by_phash(hdoc.content['body']) - parts = self._extract_parts(hdoc.content) - - return PixelatedMail.from_soledad(fdoc, hdoc, bdoc, parts=parts, soledad_querier=self) - - def mails(self, idents): - fdocs_chash = [(self.get_flags_by_chash(ident), ident) for ident in - idents] - fdocs_chash = [(result, ident) for result, ident in fdocs_chash if result] - return self._build_mails_from_fdocs(fdocs_chash) - - def attachment(self, attachment_ident, encoding): - bdoc = self.get_content_by_phash(attachment_ident) - return {'content': self._try_decode(bdoc.content['raw'], encoding), - 'content-type': bdoc.content['content-type']} - - def _try_decode(self, raw, encoding): - encoding = encoding.lower() - if encoding == 'base64': - return base64.decodestring(raw) - elif encoding == 'quoted-printable': - return quopri.decodestring(raw) - else: - return str(raw) - - def _extract_parts(self, hdoc, parts=None): - if not parts: - parts = {'alternatives': [], 'attachments': []} - - if hdoc['multi']: - for part_key in hdoc.get('part_map', {}).keys(): - self._extract_parts(hdoc['part_map'][part_key], parts) - else: - headers_dict = {elem[0]: elem[1] for elem in hdoc.get('headers', [])} - if 'attachment' in headers_dict.get('Content-Disposition', ''): - parts['attachments'].append(self._extract_attachment(hdoc, headers_dict)) - else: - parts['alternatives'].append(self._extract_alternative(hdoc, headers_dict)) - return parts - - def _extract_alternative(self, hdoc, headers_dict): - bdoc = self.get_content_by_phash(hdoc['phash']) - - if bdoc is None: - logger.warning("No BDOC content found for message!!!") - raw_content = "" - else: - raw_content = bdoc.content['raw'] - - return {'headers': headers_dict, 'content': raw_content} - - def _extract_attachment(self, hdoc, headers_dict): - content_disposition = headers_dict['Content-Disposition'] - match = re.compile('.*name=\"(.*)\".*').search(content_disposition) - filename = '' - if match: - filename = match.group(1) - return {'headers': headers_dict, 'ident': hdoc['phash'], 'name': filename} diff --git a/service/pixelated/adapter/soledad/soledad_search_key_masterkey_retrieval_mixin.py b/service/pixelated/adapter/soledad/soledad_search_key_masterkey_retrieval_mixin.py deleted file mode 100644 index 05d32779..00000000 --- a/service/pixelated/adapter/soledad/soledad_search_key_masterkey_retrieval_mixin.py +++ /dev/null @@ -1,31 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. -from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin -import os -import base64 - - -class SoledadSearchIndexMasterkeyRetrievalMixin(SoledadDbFacadeMixin, object): - - def get_index_masterkey(self): - result = self.get_search_index_masterkey() - index_key_doc = result[0] if result else None - - if not index_key_doc: - new_index_key = os.urandom(64) # 32 for encryption, 32 for hmac - self.create_doc(dict(type='index_key', value=base64.encodestring(new_index_key))) - return new_index_key - return base64.decodestring(index_key_doc.content['value']) diff --git a/service/pixelated/adapter/soledad/soledad_writer_mixin.py b/service/pixelated/adapter/soledad/soledad_writer_mixin.py deleted file mode 100644 index b0d21b93..00000000 --- a/service/pixelated/adapter/soledad/soledad_writer_mixin.py +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. -from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin - - -class SoledadWriterMixin(SoledadDbFacadeMixin, object): - - def mark_all_as_not_recent(self): - for mailbox in ['INBOX', 'DRAFTS', 'SENT', 'TRASH']: - rct = self.get_recent_by_mbox(mailbox) - if not rct or not rct[0].content['rct']: - return - rct = rct[0] - rct.content['rct'] = [] - self.put_doc(rct) - - def save_mail(self, mail): - self.put_doc(mail.fdoc) - - def create_mail(self, mail, mailbox_name): - mbox_doc = self.get_mbox(mailbox_name)[0] - uid = self.get_lastuid(mbox_doc) - self.create_docs(mail.get_for_save(next_uid=uid, mailbox=mailbox_name)) - - mbox_doc.content['lastuid'] = uid + 1 - self.put_doc(mbox_doc) - - return self.mail(mail.ident) - - def remove_mail(self, mail): - # FIX-ME: Must go through all the part_map phash to delete all the cdocs - self.delete_doc(mail.fdoc) - self.delete_doc(mail.hdoc) - self.delete_doc(mail.bdoc) diff --git a/service/pixelated/application.py b/service/pixelated/application.py index 6d83c6f7..dfeb8d82 100644 --- a/service/pixelated/application.py +++ b/service/pixelated/application.py @@ -15,9 +15,7 @@ # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. from twisted.internet import reactor -from twisted.internet.threads import deferToThread from twisted.internet import defer -from twisted.web.server import Site from twisted.internet import ssl from OpenSSL import SSL from OpenSSL import crypto @@ -26,15 +24,22 @@ from pixelated.config import arguments from pixelated.config.services import Services from pixelated.config.leap import initialize_leap from pixelated.config import logger +from pixelated.config.site import PixelatedSite from pixelated.resources.loading_page import LoadingResource from pixelated.resources.root_resource import RootResource +from leap.common.events import ( + register, + catalog as events +) + @defer.inlineCallbacks def start_user_agent(loading_app, host, port, sslkey, sslcert, leap_home, leap_session): yield loading_app.stopListening() services = Services(leap_home, leap_session) + yield services.setup(leap_home, leap_session) resource = RootResource() @@ -42,12 +47,13 @@ def start_user_agent(loading_app, host, port, sslkey, sslcert, leap_home, leap_s services.keymanager, services.search_engine, services.mail_service, - services.draft_service) + services.draft_service, + services.feedback_service) if sslkey and sslcert: - reactor.listenSSL(port, Site(resource), _ssl_options(sslkey, sslcert), interface=host) + reactor.listenSSL(port, PixelatedSite(resource), _ssl_options(sslkey, sslcert), interface=host) else: - reactor.listenTCP(port, Site(resource), interface=host) + reactor.listenTCP(port, PixelatedSite(resource), interface=host) # soledad needs lots of threads reactor.threadpool.adjustPoolsize(5, 15) @@ -71,16 +77,13 @@ def _ssl_options(sslkey, sslcert): def initialize(): args = arguments.parse_user_agent_args() logger.init(debug=args.debug) + loading_app = reactor.listenTCP(args.port, PixelatedSite(LoadingResource()), interface=args.host) - loading_app = reactor.listenTCP(args.port, Site(LoadingResource()), interface=args.host) - - deferred = deferToThread( - lambda: initialize_leap( - args.leap_provider_cert, - args.leap_provider_cert_fingerprint, - args.credentials_file, - args.organization_mode, - args.leap_home)) + deferred = initialize_leap(args.leap_provider_cert, + args.leap_provider_cert_fingerprint, + args.credentials_file, + args.organization_mode, + args.leap_home) deferred.addCallback( lambda leap_session: start_user_agent( @@ -96,6 +99,11 @@ def initialize(): failure.printTraceback() reactor.stop() + def _register_shutdown_on_token_expire(leap_session): + register(events.SOLEDAD_INVALID_AUTH_TOKEN, lambda _: reactor.stop()) + return leap_session + + deferred.addCallback(_register_shutdown_on_token_expire) deferred.addErrback(_quit_on_error) reactor.run() diff --git a/service/pixelated/assets/Interstitial.js b/service/pixelated/assets/Interstitial.js index a4c689b9..cf9ef8e4 100644 --- a/service/pixelated/assets/Interstitial.js +++ b/service/pixelated/assets/Interstitial.js @@ -4,13 +4,14 @@ if ($('#hive').length) { var left_pos = img_width * .5; var pixelated = hive.path("M12.4,20.3v31.8l28,15.8l28-15.8V20.3l-28-15.8L12.4,20.3z M39.2,56.4l-16.3-9V27.9l16.3,9.3L39.2,56.4z M57.7,47.4l-16.1,9l0-19.2l16.1-9.4V47.4z M57.7,25.2L40.4,35.5L22.9,25.2l17.5-9.4L57.7,25.2z").transform("translate(319, 50)").attr("fill", "#908e8e"); - var all = hive.group().transform("matrix(2, 0, 0, 2, "+(left_pos - 950)+", -40)"); + var all = hive.group().transform("matrix(2, 0, 0, 2, -100, -100)"); var height = 50; var width = 58; - var rows = $(window).height() / height; + var rows = (($(window).height() / height) / 2) + 1; var cols = (($(window).width() / width) / 2) + 1; + for (var j = 0; j < rows; j++) { for (var i = 0; i < cols; i++) { x = i * width + (j%2*width/2); diff --git a/service/pixelated/assets/welcome.mail b/service/pixelated/assets/welcome.mail index e85694f1..3f233143 100644 --- a/service/pixelated/assets/welcome.mail +++ b/service/pixelated/assets/welcome.mail @@ -5,9 +5,7 @@ To: Replace <will.be@the.user> Content-Type: multipart/alternative; boundary=000boundary000 --000boundary000 -Content-Type: text/plain; charset=UTF-8 - -Welcome to Pixelated Mail, a modern email with encryption. +Welcome to Pixelated Mail, a modern email with encryption. Pixelated Mail is an open source project that aims to provide secure email on the browser with all the functionality we've come to expect of a modern email client. diff --git a/service/pixelated/bitmask_libraries/config.py b/service/pixelated/bitmask_libraries/config.py index efb43411..c521a093 100644 --- a/service/pixelated/bitmask_libraries/config.py +++ b/service/pixelated/bitmask_libraries/config.py @@ -36,7 +36,6 @@ class LeapConfig(object): def __init__(self, leap_home=None, - fetch_interval_in_s=30, timeout_in_s=15, start_background_jobs=False, gpg_binary=discover_gpg_binary()): @@ -45,4 +44,3 @@ class LeapConfig(object): self.timeout_in_s = timeout_in_s self.start_background_jobs = start_background_jobs self.gpg_binary = gpg_binary - self.fetch_interval_in_s = fetch_interval_in_s diff --git a/service/pixelated/bitmask_libraries/nicknym.py b/service/pixelated/bitmask_libraries/nicknym.py index 220d75e5..826ecb58 100644 --- a/service/pixelated/bitmask_libraries/nicknym.py +++ b/service/pixelated/bitmask_libraries/nicknym.py @@ -15,6 +15,7 @@ # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. from leap.keymanager import KeyManager, openpgp, KeyNotFound from .certs import LeapCertificate +from twisted.internet import defer class NickNym(object): @@ -23,28 +24,35 @@ class NickNym(object): self._email = email_address self.keymanager = KeyManager(self._email, nicknym_url, soledad_session.soledad, - token, LeapCertificate(provider).provider_api_cert, provider.api_uri, - provider.api_version, - uuid, config.gpg_binary) + token=token, ca_cert_path=LeapCertificate(provider).provider_api_cert, api_uri=provider.api_uri, + api_version=provider.api_version, + uid=uuid, gpgbinary=config.gpg_binary) + @defer.inlineCallbacks def generate_openpgp_key(self): - if not self._key_exists(self._email): + key_present = yield self._key_exists(self._email) + if not key_present: print "Generating keys - this could take a while..." - self._gen_key() - self._send_key_to_leap() + yield self._gen_key() + # Sending it anyway for now. TODO: This can be better with real checking (downloading pubkey from nicknym) + yield self._send_key_to_leap() + @defer.inlineCallbacks def _key_exists(self, email): try: - self.keymanager.get_key(email, openpgp.OpenPGPKey, private=True, fetch_remote=False) - return True + yield self.fetch_key(email, private=True, fetch_remote=False) + defer.returnValue(True) except KeyNotFound: - return False + defer.returnValue(False) + + def fetch_key(self, email, private=False, fetch_remote=True): + return self.keymanager.get_key(email, openpgp.OpenPGPKey, private=private, fetch_remote=fetch_remote) def _gen_key(self): - self.keymanager.gen_key(openpgp.OpenPGPKey) + return self.keymanager.gen_key(openpgp.OpenPGPKey) def _send_key_to_leap(self): - self.keymanager.send_key(openpgp.OpenPGPKey) + return self.keymanager.send_key(openpgp.OpenPGPKey) def _discover_nicknym_server(provider): diff --git a/service/pixelated/bitmask_libraries/provider.py b/service/pixelated/bitmask_libraries/provider.py index 315ea7f1..b7f82f8a 100644 --- a/service/pixelated/bitmask_libraries/provider.py +++ b/service/pixelated/bitmask_libraries/provider.py @@ -93,7 +93,7 @@ class LeapProvider(object): digest = get_digest(cert_data, method) if fingerprint.strip() != digest: - raise Exception('Certificate fingerprints don\'t match') + raise Exception('Certificate fingerprints don\'t match! Expected [%s] but got [%s]' % (fingerprint.strip(), digest)) def _validated_get(self, url): session = requests.session() diff --git a/service/pixelated/bitmask_libraries/session.py b/service/pixelated/bitmask_libraries/session.py index a9cb15f2..7abe2a63 100644 --- a/service/pixelated/bitmask_libraries/session.py +++ b/service/pixelated/bitmask_libraries/session.py @@ -18,16 +18,15 @@ import traceback import sys import os -from leap.mail.imap.fetch import LeapIncomingMail -from leap.mail.imap.account import SoledadBackedAccount -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.soledadstore import SoledadStore +from leap.mail.incoming.service import IncomingMail from twisted.internet import reactor from .nicknym import NickNym from leap.auth import SRPAuth +from pixelated.adapter.mailstore import LeapMailStore from .soledad import SoledadSessionFactory from .smtp import LeapSmtp - +from leap.mail.imap.account import IMAPAccount +from twisted.internet import defer SESSIONS = {} @@ -47,48 +46,66 @@ class LeapSession(object): - ``user_auth`` the secure remote password session data after authenticating with LEAP. See http://en.wikipedia.org/wiki/Secure_Remote_Password_protocol (SRPSession) + - ``mail_store`` the MailStore to access the users mails + - ``soledad_session`` the soledad session. See https://leap.se/soledad (LeapSecureRemotePassword) - ``nicknym`` the nicknym instance. See https://leap.se/nicknym (NickNym) - - ``account`` the actual leap mail account. Implements Twisted imap4.IAccount and imap4.INamespacePresenter (SoledadBackedAccount) - - ``incoming_mail_fetcher`` Background job for fetching incoming mails from LEAP server (LeapIncomingMail) """ - def __init__(self, provider, user_auth, soledad_session, nicknym, soledad_account, incoming_mail_fetcher, smtp): + def __init__(self, provider, user_auth, mail_store, soledad_session, nicknym, smtp): self.smtp = smtp self.config = provider.config self.provider = provider self.user_auth = user_auth + self.mail_store = mail_store self.soledad_session = soledad_session self.nicknym = nicknym - self.account = soledad_account - self.incoming_mail_fetcher = incoming_mail_fetcher - self.soledad_session.soledad.sync(defer_decryption=False) - self.nicknym.generate_openpgp_key() - if self.config.start_background_jobs: - self.start_background_jobs() + @defer.inlineCallbacks + def initial_sync(self): + yield self.sync() + yield self.after_first_sync() + defer.returnValue(self) + + @defer.inlineCallbacks + def after_first_sync(self): + yield self.nicknym.generate_openpgp_key() + self.account = self._create_account(self.account_email, self.soledad_session) + self.incoming_mail_fetcher = yield self._create_incoming_mail_fetcher( + self.nicknym, + self.soledad_session, + self.account, + self.account_email()) + reactor.callFromThread(self.incoming_mail_fetcher.startService) + + def _create_account(self, user_mail, soledad_session): + account = IMAPAccount(user_mail, soledad_session.soledad) + return account def account_email(self): name = self.user_auth.username return self.provider.address_for(name) def close(self): - self.stop_background_jobs() + self.stop_background_jobs - def start_background_jobs(self): - self.smtp.ensure_running() - reactor.callFromThread(self.incoming_mail_fetcher.start_loop) + @defer.inlineCallbacks + def _create_incoming_mail_fetcher(self, nicknym, soledad_session, account, user_mail): + inbox = yield account.callWhenReady(lambda _: account.getMailbox('INBOX')) + defer.returnValue(IncomingMail(nicknym.keymanager, + soledad_session.soledad, + inbox.collection, + user_mail)) def stop_background_jobs(self): - self.smtp.stop() - reactor.callFromThread(self.incoming_mail_fetcher.stop) + reactor.callFromThread(self.incoming_mail_fetcher.stopService) def sync(self): try: - self.soledad_session.sync() + return self.soledad_session.sync() except: traceback.print_exc(file=sys.stderr) raise @@ -117,14 +134,13 @@ class LeapSessionFactory(object): account_email = self._provider.address_for(username) soledad = SoledadSessionFactory.create(self._provider, auth.token, auth.uuid, password) + mail_store = LeapMailStore(soledad.soledad) nicknym = self._create_nicknym(account_email, auth.token, auth.uuid, soledad) - account = self._create_account(auth.uuid, soledad) - incoming_mail_fetcher = self._create_incoming_mail_fetcher(nicknym, soledad, account, account_email) smtp = LeapSmtp(self._provider, auth, nicknym.keymanager) - return LeapSession(self._provider, auth, soledad, nicknym, account, incoming_mail_fetcher, smtp) + return LeapSession(self._provider, auth, mail_store, soledad, nicknym, smtp) def _lookup_session(self, key): global SESSIONS @@ -152,10 +168,5 @@ class LeapSessionFactory(object): def _create_nicknym(self, email_address, token, uuid, soledad_session): return NickNym(self._provider, self._config, soledad_session, email_address, token, uuid) - def _create_account(self, uuid, soledad_session): - memstore = MemoryStore(permanent_store=SoledadStore(soledad_session.soledad)) - return SoledadBackedAccount(uuid, soledad_session.soledad, memstore) - - def _create_incoming_mail_fetcher(self, nicknym, soledad_session, account, email_address): - return LeapIncomingMail(nicknym.keymanager, soledad_session.soledad, account, - self._config.fetch_interval_in_s, email_address) + # memstore = MemoryStore(permanent_store=SoledadStore(soledad_session.soledad)) + # return SoledadBackedAccount(uuid, soledad_session.soledad, memstore) diff --git a/service/pixelated/bitmask_libraries/smtp.py b/service/pixelated/bitmask_libraries/smtp.py index 31e56995..ff2792fb 100644 --- a/service/pixelated/bitmask_libraries/smtp.py +++ b/service/pixelated/bitmask_libraries/smtp.py @@ -88,7 +88,7 @@ class LeapSmtp(object): self._local_smtp_service, self._local_smtp_service_socket = setup_smtp_gateway( port=self.local_smtp_port_number, - userid=email, + userid=str(email), keymanager=self._keymanager, smtp_host=self._remote_hostname.encode('UTF-8'), smtp_port=self._remote_port, diff --git a/service/pixelated/bitmask_libraries/soledad.py b/service/pixelated/bitmask_libraries/soledad.py index f0cd9f2f..0546a158 100644 --- a/service/pixelated/bitmask_libraries/soledad.py +++ b/service/pixelated/bitmask_libraries/soledad.py @@ -17,7 +17,7 @@ import errno import os from leap.soledad.client import Soledad -from leap.soledad.common.crypto import WrongMac, UnknownMacMethod +from leap.soledad.common.crypto import WrongMacError, UnknownMacMethodError from pixelated.bitmask_libraries.certs import LeapCertificate SOLEDAD_TIMEOUT = 120 @@ -57,10 +57,16 @@ class SoledadSession(object): secrets = self._secrets_path() local_db = self._local_db_path() - return Soledad(self.user_uuid, unicode(encryption_passphrase), secrets, - local_db, server_url, LeapCertificate(self.provider).provider_api_cert, self.user_token, defer_encryption=False) + return Soledad(self.user_uuid, + passphrase=unicode(encryption_passphrase), + secrets_path=secrets, + local_db_path=local_db, server_url=server_url, + cert_file=LeapCertificate(self.provider).provider_api_cert, + shared_db=None, + auth_token=self.user_token, + defer_encryption=False) - except (WrongMac, UnknownMacMethod), e: + except (WrongMacError, UnknownMacMethodError), e: raise SoledadWrongPassphraseException(e) def _leap_path(self): @@ -82,8 +88,7 @@ class SoledadSession(object): raise def sync(self): - if self.soledad.need_sync(self.soledad.server_url): - self.soledad.sync() + return self.soledad.sync() def _discover_soledad_server(self): try: diff --git a/service/pixelated/config/arguments.py b/service/pixelated/config/arguments.py index fa7fdae4..7a7abe49 100644 --- a/service/pixelated/config/arguments.py +++ b/service/pixelated/config/arguments.py @@ -43,6 +43,7 @@ def parse_maintenance_args(): subparsers.add_parser('dump-soledad', help='dump the soledad database') subparsers.add_parser('sync', help='sync the soledad database') + subparsers.add_parser('repair', help='repair database if possible') return parser.parse_args() diff --git a/service/pixelated/config/leap.py b/service/pixelated/config/leap.py index 52cd4c8f..0409e54f 100644 --- a/service/pixelated/config/leap.py +++ b/service/pixelated/config/leap.py @@ -1,37 +1,69 @@ from __future__ import absolute_import -import random +from leap.common.events import (server as events_server, + register, catalog as events) +from email import message_from_file from pixelated.config import credentials -from leap.common.events import server as events_server from pixelated.bitmask_libraries.config import LeapConfig from pixelated.bitmask_libraries.certs import LeapCertificate from pixelated.bitmask_libraries.provider import LeapProvider from pixelated.bitmask_libraries.session import LeapSessionFactory +from pixelated.adapter.model.mail import InputMail +from twisted.internet import defer +import os +import logging +fresh_account = False + + +@defer.inlineCallbacks def initialize_leap(leap_provider_cert, leap_provider_cert_fingerprint, credentials_file, organization_mode, - leap_home): + leap_home, + initial_sync=True): init_monkeypatches() - events_server.ensure_server(random.randrange(8000, 11999)) - provider, username, password = credentials.read(organization_mode, credentials_file) - LeapCertificate.set_cert_and_fingerprint(leap_provider_cert, leap_provider_cert_fingerprint) + events_server.ensure_server() + register(events.KEYMANAGER_FINISHED_KEY_GENERATION, + set_fresh_account) + provider, username, password = credentials.read(organization_mode, + credentials_file) + LeapCertificate.set_cert_and_fingerprint(leap_provider_cert, + leap_provider_cert_fingerprint) config = LeapConfig(leap_home=leap_home, start_background_jobs=True) provider = LeapProvider(provider, config) LeapCertificate(provider).setup_ca_bundle() leap_session = LeapSessionFactory(provider).create(username, password) - return leap_session + if initial_sync: + leap_session = yield leap_session.initial_sync() + + global fresh_account + if fresh_account: + add_welcome_mail(leap_session.mail_store) + + defer.returnValue(leap_session) + + +def add_welcome_mail(mail_store): + current_path = os.path.dirname(os.path.abspath(__file__)) + with open(os.path.join(current_path, + '..', + 'assets', + 'welcome.mail')) as mail_template_file: + mail_template = message_from_file(mail_template_file) + + input_mail = InputMail.from_python_mail(mail_template) + logging.getLogger('pixelated.config.leap').info('Adding the welcome mail') + mail_store.add_mail('INBOX', input_mail.raw) def init_monkeypatches(): - import pixelated.extensions.protobuf_socket - import pixelated.extensions.sqlcipher_wal - import pixelated.extensions.esmtp_sender_factory - import pixelated.extensions.incoming_decrypt_header - import pixelated.extensions.soledad_sync_exception - import pixelated.extensions.keymanager_fetch_key import pixelated.extensions.requests_urllib3 - import pixelated.extensions.shared_db + + +def set_fresh_account(_, x): + global fresh_account + fresh_account = True diff --git a/service/pixelated/config/logger.py b/service/pixelated/config/logger.py index 52f3f3a5..5c711981 100644 --- a/service/pixelated/config/logger.py +++ b/service/pixelated/config/logger.py @@ -21,7 +21,7 @@ from twisted.python import log def init(debug=False): debug_enabled = debug or os.environ.get('DEBUG', False) - logging_level = logging.DEBUG if debug_enabled else logging.INFO + logging_level = logging.DEBUG if debug_enabled else logging.WARN log_format = "%(asctime)s [%(name)s] %(levelname)s %(message)s" date_format = '%Y-%m-%d %H:%M:%S' @@ -31,4 +31,5 @@ def init(debug=False): filemode='a') observer = log.PythonLoggingObserver() + logging.getLogger('gnupg').setLevel(logging.WARN) observer.start() diff --git a/service/pixelated/config/services.py b/service/pixelated/config/services.py index f1c7a540..41a357dc 100644 --- a/service/pixelated/config/services.py +++ b/service/pixelated/config/services.py @@ -1,65 +1,75 @@ +from pixelated.adapter.mailstore.searchable_mailstore import SearchableMailStore from pixelated.adapter.services.mail_service import MailService from pixelated.adapter.model.mail import InputMail from pixelated.adapter.services.mail_sender import MailSender -from pixelated.adapter.services.mailboxes import Mailboxes -from pixelated.adapter.soledad.soledad_querier import SoledadQuerier from pixelated.adapter.search import SearchEngine from pixelated.adapter.services.draft_service import DraftService -from pixelated.adapter.listeners.mailbox_indexer_listener import MailboxIndexerListener +from pixelated.adapter.listeners.mailbox_indexer_listener import listen_all_mailboxes +from twisted.internet import defer +from pixelated.adapter.search.index_storage_key import SearchIndexStorageKey +from pixelated.adapter.services.feedback_service import FeedbackService class Services(object): def __init__(self, leap_home, leap_session): + pass - soledad_querier = SoledadQuerier(soledad=leap_session.soledad_session.soledad) + @defer.inlineCallbacks + def setup(self, leap_home, leap_session): + InputMail.FROM_EMAIL_ADDRESS = leap_session.account_email() - self.search_engine = self.setup_search_engine( + search_index_storage_key = self.setup_search_index_storage_key(leap_session.soledad_session.soledad) + yield self.setup_search_engine( leap_home, - soledad_querier) + search_index_storage_key) - pixelated_mailboxes = Mailboxes( - leap_session.account, - soledad_querier, - self.search_engine) + self.wrap_mail_store_with_indexing_mail_store(leap_session) + + yield listen_all_mailboxes(leap_session.account, self.search_engine, leap_session.mail_store) self.mail_service = self.setup_mail_service( leap_session, - soledad_querier, - self.search_engine, - pixelated_mailboxes) + self.search_engine) - self.keymanager = self.setup_keymanager(leap_session) - self.draft_service = self.setup_draft_service(pixelated_mailboxes) + self.keymanager = leap_session.nicknym + self.draft_service = self.setup_draft_service(leap_session.mail_store) + self.feedback_service = self.setup_feedback_service(leap_session) - self.post_setup(soledad_querier, leap_session) + yield self.index_all_mails() - def post_setup(self, soledad_querier, leap_session): - self.search_engine.index_mails( - mails=self.mail_service.all_mails(), - callback=soledad_querier.mark_all_as_not_recent) - soledad_querier.remove_duplicates() - InputMail.FROM_EMAIL_ADDRESS = leap_session.account_email() + def wrap_mail_store_with_indexing_mail_store(self, leap_session): + leap_session.mail_store = SearchableMailStore(leap_session.mail_store, self.search_engine) - def setup_keymanager(self, leap_session): - return leap_session.nicknym.keymanager + @defer.inlineCallbacks + def index_all_mails(self): + all_mails = yield self.mail_service.all_mails() + self.search_engine.index_mails(all_mails) - def setup_search_engine(self, leap_home, soledad_querier): - key = soledad_querier.get_index_masterkey() + @defer.inlineCallbacks + def setup_search_engine(self, leap_home, search_index_storage_key): + key_unicode = yield search_index_storage_key.get_or_create_key() + key = str(key_unicode) + print 'The key len is: %s' % len(key) search_engine = SearchEngine(key, agent_home=leap_home) - MailboxIndexerListener.SEARCH_ENGINE = search_engine - return search_engine + self.search_engine = search_engine - def setup_mail_service(self, leap_session, soledad_querier, search_engine, pixelated_mailboxes): - pixelated_mailboxes.add_welcome_mail_for_fresh_user() + def setup_mail_service(self, leap_session, search_engine): + # if False: FIXME + # yield pixelated_mailboxes.add_welcome_mail_for_fresh_user() pixelated_mail_sender = MailSender( leap_session.account_email(), leap_session.smtp) return MailService( - pixelated_mailboxes, pixelated_mail_sender, - soledad_querier, + leap_session.mail_store, search_engine) - def setup_draft_service(self, pixelated_mailboxes): - return DraftService(pixelated_mailboxes) + def setup_draft_service(self, mail_store): + return DraftService(mail_store) + + def setup_search_index_storage_key(self, soledad): + return SearchIndexStorageKey(soledad) + + def setup_feedback_service(self, leap_session): + return FeedbackService(leap_session) diff --git a/service/pixelated/config/site.py b/service/pixelated/config/site.py new file mode 100644 index 00000000..bd149914 --- /dev/null +++ b/service/pixelated/config/site.py @@ -0,0 +1,15 @@ +from twisted.web.server import Site, Request + + +class AddCSPHeaderRequest(Request): + HEADER_VALUES = "default-src 'self'; style-src 'self' 'unsafe-inline'" + + def process(self): + self.setHeader("Content-Security-Policy", self.HEADER_VALUES) + self.setHeader("X-Content-Security-Policy", self.HEADER_VALUES) + self.setHeader("X-Webkit-CSP", self.HEADER_VALUES) + Request.process(self) + + +class PixelatedSite(Site): + requestFactory = AddCSPHeaderRequest diff --git a/service/pixelated/extensions/incoming_decrypt_header.py b/service/pixelated/extensions/incoming_decrypt_header.py deleted file mode 100644 index 2db5dd1d..00000000 --- a/service/pixelated/extensions/incoming_decrypt_header.py +++ /dev/null @@ -1,35 +0,0 @@ -import leap.mail.imap.fetch as fetch - - -def mark_as_encrypted_inline(f): - - def w(*args, **kwargs): - msg, valid_sign = f(*args) - is_encrypted = fetch.PGP_BEGIN in args[1].as_string() and fetch.PGP_END in args[1].as_string() - decrypted_successfully = fetch.PGP_BEGIN not in msg.as_string() and fetch.PGP_END not in msg.as_string() - - if not is_encrypted: - encrypted = 'false' - else: - if decrypted_successfully: - encrypted = 'true' - else: - encrypted = 'fail' - - msg.add_header('X-Pixelated-encryption-status', encrypted) - return msg, valid_sign - - return w - - -def mark_as_encrypted_multipart(f): - - def w(*args, **kwargs): - msg, valid_sign = f(*args) - msg.add_header('X-Pixelated-encryption-status', 'true') - return msg, valid_sign - return w - - -fetch.LeapIncomingMail._maybe_decrypt_inline_encrypted_msg = mark_as_encrypted_inline(fetch.LeapIncomingMail._maybe_decrypt_inline_encrypted_msg) -fetch.LeapIncomingMail._decrypt_multipart_encrypted_msg = mark_as_encrypted_multipart(fetch.LeapIncomingMail._decrypt_multipart_encrypted_msg) diff --git a/service/pixelated/extensions/keymanager_fetch_key.py b/service/pixelated/extensions/keymanager_fetch_key.py index d39d1f96..114e852e 100644 --- a/service/pixelated/extensions/keymanager_fetch_key.py +++ b/service/pixelated/extensions/keymanager_fetch_key.py @@ -57,4 +57,4 @@ def patched_fetch_keys_from_server(self, address): raise KeyNotFound(address) -leap.keymanager.KeyManager._fetch_keys_from_server = patched_fetch_keys_from_server +# leap.keymanager.KeyManager._fetch_keys_from_server = patched_fetch_keys_from_server diff --git a/service/pixelated/extensions/shared_db.py b/service/pixelated/extensions/shared_db.py index 3e8a978e..b433dd50 100644 --- a/service/pixelated/extensions/shared_db.py +++ b/service/pixelated/extensions/shared_db.py @@ -13,4 +13,4 @@ def patched_sign_request(self, method, url_query, params): 'Wrong credentials: %s' % self._creds) -TokenBasedAuth._sign_request = patched_sign_request +# TokenBasedAuth._sign_request = patched_sign_request diff --git a/service/pixelated/extensions/soledad_sync_exception.py b/service/pixelated/extensions/soledad_sync_exception.py index cb3204ad..c3ef5176 100644 --- a/service/pixelated/extensions/soledad_sync_exception.py +++ b/service/pixelated/extensions/soledad_sync_exception.py @@ -19,4 +19,4 @@ def patched_sync(self, defer_decryption=True): client.logger.error("Soledad exception when syncing: %s - %s" % (e.__class__.__name__, e.message)) -client.Soledad.sync = patched_sync +# client.Soledad.sync = patched_sync diff --git a/service/pixelated/maintenance.py b/service/pixelated/maintenance.py index 7170055c..f011658d 100644 --- a/service/pixelated/maintenance.py +++ b/service/pixelated/maintenance.py @@ -14,64 +14,81 @@ # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import logging from mailbox import Maildir from twisted.internet import reactor, defer from twisted.internet.threads import deferToThread +from pixelated.adapter.mailstore.maintenance import SoledadMaintenance from pixelated.config.leap import initialize_leap from pixelated.config import logger, arguments -from leap.mail.imap.fields import WithMsgFields -import time +from leap.mail.constants import MessageFlags + + +REPAIR_COMMAND = 'repair' def initialize(): - import time args = arguments.parse_maintenance_args() logger.init(debug=args.debug) - leap_session = initialize_leap( - args.leap_provider_cert, - args.leap_provider_cert_fingerprint, - args.credentials_file, - organization_mode=False, - leap_home=args.leap_home) + @defer.inlineCallbacks + def _run(): + leap_session = yield initialize_leap( + args.leap_provider_cert, + args.leap_provider_cert_fingerprint, + args.credentials_file, + organization_mode=False, + leap_home=args.leap_home, + initial_sync=_do_initial_sync(args)) - execute_command = create_execute_command(args, leap_session) + execute_command(args, leap_session) - reactor.callWhenRunning(execute_command) + reactor.callWhenRunning(_run) reactor.run() -def create_execute_command(args, leap_session): - def execute_command(): +def _do_initial_sync(args): + return not _is_repair_command(args) + + +def _is_repair_command(args): + return args.command == REPAIR_COMMAND + + +def execute_command(args, leap_session): - def init_soledad(): - return leap_session + def init_soledad(): + return leap_session - def get_soledad_handle(leap_session): - soledad = leap_session.soledad_session.soledad + def get_soledad_handle(leap_session): + soledad = leap_session.soledad_session.soledad - return leap_session, soledad + return leap_session, soledad - def soledad_sync(args): - leap_session, soledad = args + @defer.inlineCallbacks + def soledad_sync(args): + leap_session, soledad = args + log = logging.getLogger('some logger') - soledad.sync() + log.warn('Before sync') - return args + yield soledad.sync() - tearDown = defer.Deferred() + log.warn('after sync') - prepare = deferToThread(init_soledad) - prepare.addCallback(get_soledad_handle) - prepare.addCallback(soledad_sync) - add_command_callback(args, prepare, tearDown) - tearDown.addCallback(soledad_sync) - tearDown.addCallback(shutdown) - tearDown.addErrback(shutdown_on_error) + defer.returnValue(args) - return execute_command + tearDown = defer.Deferred() + + prepare = deferToThread(init_soledad) + prepare.addCallback(get_soledad_handle) + prepare.addCallback(soledad_sync) + add_command_callback(args, prepare, tearDown) + tearDown.addCallback(soledad_sync) + tearDown.addCallback(shutdown) + tearDown.addErrback(shutdown_on_error) def add_command_callback(args, prepareDeferred, finalizeDeferred): @@ -87,6 +104,9 @@ def add_command_callback(args, prepareDeferred, finalizeDeferred): elif args.command == 'sync': # nothing to do here, sync is already part of the chain prepareDeferred.chainDeferred(finalizeDeferred) + elif args.command == REPAIR_COMMAND: + prepareDeferred.addCallback(repair) + prepareDeferred.chainDeferred(finalizeDeferred) else: print 'Unsupported command: %s' % args.command prepareDeferred.chainDeferred(finalizeDeferred) @@ -94,90 +114,95 @@ def add_command_callback(args, prepareDeferred, finalizeDeferred): return finalizeDeferred +@defer.inlineCallbacks def delete_all_mails(args): leap_session, soledad = args - generation, docs = soledad.get_all_docs() + generation, docs = yield soledad.get_all_docs() for doc in docs: if doc.content.get('type', None) in ['head', 'cnt', 'flags']: soledad.delete_doc(doc) - return args + defer.returnValue(args) def is_keep_file(mail): return mail['subject'] is None -def add_mail_folder(account, maildir, folder_name, deferreds): - if folder_name not in account.mailboxes: - account.addMailbox(folder_name) +@defer.inlineCallbacks +def add_mail_folder(store, maildir, folder_name, deferreds): + yield store.add_mailbox(folder_name) - mbx = account.getMailbox(folder_name) for mail in maildir: if is_keep_file(mail): continue - flags = (WithMsgFields.RECENT_FLAG,) if mail.get_subdir() == 'new' else () + flags = (MessageFlags.RECENT_FLAG,) if mail.get_subdir() == 'new' else () if 'S' in mail.get_flags(): - flags = (WithMsgFields.SEEN_FLAG,) + flags + flags = (MessageFlags.SEEN_FLAG,) + flags if 'R' in mail.get_flags(): - flags = (WithMsgFields.ANSWERED_FLAG,) + flags + flags = (MessageFlags.ANSWERED_FLAG,) + flags - deferreds.append(mbx.addMessage(mail.as_string(), flags=flags, notify_on_disk=False)) + deferreds.append(store.add_mail(folder_name, mail.as_string())) + # FIXME support flags @defer.inlineCallbacks def load_mails(args, mail_paths): leap_session, soledad = args - account = leap_session.account + store = leap_session.mail_store deferreds = [] for path in mail_paths: maildir = Maildir(path, factory=None) - add_mail_folder(account, maildir, 'INBOX', deferreds) + yield add_mail_folder(store, maildir, 'INBOX', deferreds) for mail_folder_name in maildir.list_folders(): mail_folder = maildir.get_folder(mail_folder_name) - add_mail_folder(account, mail_folder, mail_folder_name, deferreds) + yield add_mail_folder(store, mail_folder, mail_folder_name, deferreds) + + yield defer.gatherResults(deferreds, consumeErrors=True) - yield defer.DeferredList(deferreds) defer.returnValue(args) def flush_to_soledad(args, finalize): leap_session, soledad = args - account = leap_session.account - memstore = account._memstore - permanent_store = memstore._permanent_store - - d = memstore.write_messages(permanent_store) - def check_flushed(args): - if memstore.is_writing: - reactor.callLater(1, check_flushed, args) - else: - finalize.callback((leap_session, soledad)) + def after_sync(_): + finalize.callback((leap_session, soledad)) - d.addCallback(check_flushed) + d = soledad.sync() + d.addCallback(after_sync) return args +@defer.inlineCallbacks def dump_soledad(args): leap_session, soledad = args - generation, docs = soledad.get_all_docs() + generation, docs = yield soledad.get_all_docs() for doc in docs: print doc print '\n' - return args + defer.returnValue(args) + + +@defer.inlineCallbacks +def repair(args): + leap_session, soledad = args + + yield SoledadMaintenance(soledad).repair() + + defer.returnValue(args) def shutdown(args): - time.sleep(30) + # time.sleep(30) reactor.stop() diff --git a/service/pixelated/register.py b/service/pixelated/register.py index 97f19d2e..2bdbb27b 100644 --- a/service/pixelated/register.py +++ b/service/pixelated/register.py @@ -16,6 +16,7 @@ import re import getpass import logging +import sys from pixelated.config import arguments from pixelated.config import logger as logger_config @@ -24,6 +25,7 @@ from pixelated.bitmask_libraries.config import LeapConfig from pixelated.bitmask_libraries.provider import LeapProvider from pixelated.bitmask_libraries.session import LeapSessionFactory from leap.auth import SRPAuth +from leap.common.events import server as events_server import pixelated.extensions.shared_db @@ -38,14 +40,17 @@ def register( provider_cert, provider_cert_fingerprint): - try: - validate_username(username) - except ValueError: - print('Only lowercase letters, digits, . - and _ allowed.') - if not password: password = getpass.getpass('Please enter password for %s: ' % username) + try: + validate_username(username) + validate_password(password) + except ValueError, e: + print(e.message) + sys.exit(1) + + events_server.ensure_server() LeapCertificate.set_cert_and_fingerprint(provider_cert, provider_cert_fingerprint) config = LeapConfig(leap_home=leap_home) provider = LeapProvider(server_name, config) @@ -60,8 +65,13 @@ def register( def validate_username(username): accepted_characters = '^[a-z0-9\-\_\.]*$' - if not re.match(accepted_characters, username): - raise ValueError + if (not re.match(accepted_characters, username)): + raise ValueError('Only lowercase letters, digits, . - and _ allowed.') + + +def validate_password(password): + if len(password) < 8: + raise ValueError('The password must have at least 8 characters') def initialize(): diff --git a/service/pixelated/resources/__init__.py b/service/pixelated/resources/__init__.py index b244900a..c65e19f3 100644 --- a/service/pixelated/resources/__init__.py +++ b/service/pixelated/resources/__init__.py @@ -17,15 +17,22 @@ import json +class SetEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, set): + return list(obj) + return super(SetEncoder, self).default(obj) + + def respond_json(entity, request, status_code=200): - json_response = json.dumps(entity) + json_response = json.dumps(entity, cls=SetEncoder) request.responseHeaders.addRawHeader(b"content-type", b"application/json") request.code = status_code return json_response def respond_json_deferred(entity, request, status_code=200): - json_response = json.dumps(entity) + json_response = json.dumps(entity, cls=SetEncoder) request.responseHeaders.addRawHeader(b"content-type", b"application/json") request.code = status_code request.write(json_response) diff --git a/service/pixelated/resources/attachments_resource.py b/service/pixelated/resources/attachments_resource.py index 83c7156d..a78022ec 100644 --- a/service/pixelated/resources/attachments_resource.py +++ b/service/pixelated/resources/attachments_resource.py @@ -18,9 +18,10 @@ import io import re from twisted.protocols.basic import FileSender -from twisted.python.log import err +from twisted.python.log import msg from twisted.web import server from twisted.web.resource import Resource +from twisted.internet import defer class AttachmentResource(Resource): @@ -33,23 +34,33 @@ class AttachmentResource(Resource): self.mail_service = mail_service def render_GET(self, request): + def error_handler(failure): + msg(failure, 'attachment not found') + request.code = 404 + request.finish() encoding = request.args.get('encoding', [None])[0] filename = request.args.get('filename', [self.attachment_id])[0] - attachment = self.mail_service.attachment(self.attachment_id, encoding) - request.setHeader(b'Content-Type', b'application/force-download') request.setHeader(b'Content-Disposition', bytes('attachment; filename=' + filename)) + + d = self._send_attachment(encoding, filename, request) + d.addErrback(error_handler) + + return server.NOT_DONE_YET + + @defer.inlineCallbacks + def _send_attachment(self, encoding, filename, request): + attachment = yield self.mail_service.attachment(self.attachment_id) + bytes_io = io.BytesIO(attachment['content']) - d = FileSender().beginFileTransfer(bytes_io, request) - def cb_finished(_): + try: + request.code = 200 + yield FileSender().beginFileTransfer(bytes_io, request) + finally: bytes_io.close() request.finish() - d.addErrback(err).addCallback(cb_finished) - - return server.NOT_DONE_YET - def _extract_mimetype(self, content_type): match = re.compile('([A-Za-z-]+\/[A-Za-z-]+)').search(content_type) return match.group(1) diff --git a/service/pixelated/resources/contacts_resource.py b/service/pixelated/resources/contacts_resource.py index 5ec39761..c9b81f54 100644 --- a/service/pixelated/resources/contacts_resource.py +++ b/service/pixelated/resources/contacts_resource.py @@ -29,8 +29,16 @@ class ContactsResource(Resource): self._search_engine = search_engine def render_GET(self, request): - query = request.args.get('q', [''])[0] + query = request.args.get('q', ['']) d = deferToThread(lambda: self._search_engine.contacts(query)) d.addCallback(lambda tags: respond_json_deferred(tags, request)) + def handle_error(error): + print 'Something went wrong' + import traceback + traceback.print_exc() + print error + + d.addErrback(handle_error) + return server.NOT_DONE_YET diff --git a/service/pixelated/resources/features_resource.py b/service/pixelated/resources/features_resource.py index 6a1a49ca..927cd9e9 100644 --- a/service/pixelated/resources/features_resource.py +++ b/service/pixelated/resources/features_resource.py @@ -21,12 +21,16 @@ from twisted.web.resource import Resource class FeaturesResource(Resource): DISABLED_FEATURES = ['draftReply'] - isLeaf = True def render_GET(self, request): - try: - disabled_features = {'logout': os.environ['DISPATCHER_LOGOUT_URL']} - except KeyError: - disabled_features = {} - return respond_json({'disabled_features': self.DISABLED_FEATURES, 'dispatcher_features': disabled_features}, request) + dispatcher_features = {} + + if os.environ.get('DISPATCHER_LOGOUT_URL'): + dispatcher_features['logout'] = os.environ.get('DISPATCHER_LOGOUT_URL') + + if os.environ.get('FEEDBACK_URL') is None: + self.DISABLED_FEATURES.append('feedback') + + return respond_json( + {'disabled_features': self.DISABLED_FEATURES, 'dispatcher_features': dispatcher_features}, request) diff --git a/service/pixelated/resources/feedback_resource.py b/service/pixelated/resources/feedback_resource.py new file mode 100644 index 00000000..b989b273 --- /dev/null +++ b/service/pixelated/resources/feedback_resource.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import json + +from twisted.web.resource import Resource +from pixelated.resources import respond_json + + +class FeedbackResource(Resource): + isLeaf = True + + def __init__(self, feedback_service): + Resource.__init__(self) + self.feedback_service = feedback_service + + def render_POST(self, request): + feedback = json.loads(request.content.read()).get('feedback') + self.feedback_service.open_ticket(feedback) + return respond_json({}, request) diff --git a/service/pixelated/resources/keys_resource.py b/service/pixelated/resources/keys_resource.py index 8afb2bf6..6df95b28 100644 --- a/service/pixelated/resources/keys_resource.py +++ b/service/pixelated/resources/keys_resource.py @@ -1,7 +1,5 @@ from email.utils import parseaddr -from leap.keymanager import OpenPGPKey from pixelated.resources import respond_json_deferred -from twisted.internet.threads import deferToThread from twisted.web import server from twisted.web.resource import Resource @@ -25,7 +23,7 @@ class KeysResource(Resource): respond_json_deferred(None, request, status_code=404) _, key_to_find = parseaddr(request.args.get('search')[0]) - d = deferToThread(lambda: self._keymanager.get_key_from_cache(key_to_find, OpenPGPKey)) + d = self._keymanager.fetch_key(key_to_find) d.addCallback(finish_request) d.addErrback(key_not_found) diff --git a/service/pixelated/resources/mail_resource.py b/service/pixelated/resources/mail_resource.py index dff594b0..436842fb 100644 --- a/service/pixelated/resources/mail_resource.py +++ b/service/pixelated/resources/mail_resource.py @@ -1,6 +1,8 @@ import json -from pixelated.resources import respond_json +from pixelated.resources import respond_json, respond_json_deferred from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET +from twisted.python.log import err class MailTags(Resource): @@ -15,11 +17,14 @@ class MailTags(Resource): def render_POST(self, request): new_tags = json.loads(request.content.read()).get('newtags') - try: - mail = self._mail_service.update_tags(self._mail_id, new_tags) - except ValueError as ve: - return respond_json(ve.message, request, 403) - return respond_json(mail.as_dict(), request) + d = self._mail_service.update_tags(self._mail_id, new_tags) + d.addCallback(lambda mail: respond_json_deferred(mail.as_dict(), request)) + + def handle403(failure): + failure.trap(ValueError) + return respond_json_deferred(failure.getErrorMessage(), request, 403) + d.addErrback(handle403) + return NOT_DONE_YET class Mail(Resource): @@ -31,12 +36,21 @@ class Mail(Resource): self._mail_service = mail_service def render_GET(self, request): - mail = self._mail_service.mail(self._mail_id) - return respond_json(mail.as_dict(), request) + d = self._mail_service.mail(self._mail_id) + + d.addCallback(lambda mail: respond_json_deferred(mail.as_dict(), request)) + + return NOT_DONE_YET def render_DELETE(self, request): - self._mail_service.delete_mail(self._mail_id) - return respond_json(None, request) + def response_failed(failure): + err(failure, 'something failed') + request.finish() + + d = self._mail_service.delete_mail(self._mail_id) + d.addCallback(lambda _: respond_json_deferred(None, request)) + d.addErrback(response_failed) + return NOT_DONE_YET class MailResource(Resource): diff --git a/service/pixelated/resources/mails_resource.py b/service/pixelated/resources/mails_resource.py index c4b578ba..93a19a9b 100644 --- a/service/pixelated/resources/mails_resource.py +++ b/service/pixelated/resources/mails_resource.py @@ -1,12 +1,15 @@ import json from pixelated.adapter.services.mail_sender import SMTPDownException from pixelated.adapter.model.mail import InputMail -from pixelated.resources import respond_json, respond_json_deferred +from twisted.web.server import NOT_DONE_YET +from pixelated.resources import respond_json_deferred from twisted.web.resource import Resource from twisted.web import server +from twisted.internet import defer +from twisted.python.log import err from leap.common.events import ( register, - events_pb2 as proto + catalog as events ) @@ -19,9 +22,15 @@ class MailsUnreadResource(Resource): def render_POST(self, request): idents = json.load(request.content).get('idents') + deferreds = [] for ident in idents: - self._mail_service.mark_as_unread(ident) - return respond_json(None, request) + deferreds.append(self._mail_service.mark_as_unread(ident)) + + d = defer.gatherResults(deferreds, consumeErrors=True) + d.addCallback(lambda _: respond_json_deferred(None, request)) + d.addErrback(lambda _: respond_json_deferred(None, request, status_code=500)) + + return NOT_DONE_YET class MailsReadResource(Resource): @@ -33,10 +42,15 @@ class MailsReadResource(Resource): def render_POST(self, request): idents = json.load(request.content).get('idents') + deferreds = [] for ident in idents: - self._mail_service.mark_as_read(ident) + deferreds.append(self._mail_service.mark_as_read(ident)) + + d = defer.gatherResults(deferreds, consumeErrors=True) + d.addCallback(lambda _: respond_json_deferred(None, request)) + d.addErrback(lambda _: respond_json_deferred(None, request, status_code=500)) - return respond_json(None, request) + return NOT_DONE_YET class MailsDeleteResource(Resource): @@ -47,10 +61,19 @@ class MailsDeleteResource(Resource): self._mail_service = mail_service def render_POST(self, request): + def response_failed(failure): + err(failure, 'something failed') + request.finish() + idents = json.loads(request.content.read())['idents'] + deferreds = [] for ident in idents: - self._mail_service.delete_mail(ident) - return respond_json(None, request) + deferreds.append(self._mail_service.delete_mail(ident)) + + d = defer.gatherResults(deferreds, consumeErrors=True) + d.addCallback(lambda _: respond_json_deferred(None, request)) + d.addErrback(response_failed) + return NOT_DONE_YET class MailsRecoverResource(Resource): @@ -62,9 +85,13 @@ class MailsRecoverResource(Resource): def render_POST(self, request): idents = json.loads(request.content.read())['idents'] + deferreds = [] for ident in idents: - self._mail_service.recover_mail(ident) - return respond_json(None, request) + deferreds.append(self._mail_service.recover_mail(ident)) + d = defer.gatherResults(deferreds, consumeErrors=True) + d.addCallback(lambda _: respond_json_deferred(None, request)) + d.addErrback(lambda _: respond_json_deferred(None, request, status_code=500)) + return NOT_DONE_YET class MailsResource(Resource): @@ -75,7 +102,7 @@ class MailsResource(Resource): delivery_error_mail = InputMail.delivery_error_template(delivery_address=event.content) self._mail_service.mailboxes.inbox.add(delivery_error_mail) - register(signal=proto.SMTP_SEND_MESSAGE_ERROR, callback=on_error) + register(events.SMTP_SEND_MESSAGE_ERROR, callback=on_error) def __init__(self, mail_service, draft_service): Resource.__init__(self) @@ -90,16 +117,22 @@ class MailsResource(Resource): def render_GET(self, request): query, window_size, page = request.args.get('q')[0], request.args.get('w')[0], request.args.get('p')[0] - mails, total = self._mail_service.mails(query, window_size, page) + d = self._mail_service.mails(query, window_size, page) - response = { + d.addCallback(lambda (mails, total): { "stats": { "total": total, }, "mails": [mail.as_dict() for mail in mails] - } + }) + d.addCallback(lambda res: respond_json_deferred(res, request)) + + def error_handler(error): + print error - return respond_json(response, request) + d.addErrback(error_handler) + + return NOT_DONE_YET def render_POST(self, request): content_dict = json.loads(request.content.read()) @@ -114,7 +147,8 @@ class MailsResource(Resource): if isinstance(error.value, SMTPDownException): respond_json_deferred({'message': str(error.value)}, request, status_code=503) else: - respond_json_deferred({'message': str(error)}, request, status_code=422) + err(error, 'something failed') + respond_json_deferred({'message': 'an error occurred while sending'}, request, status_code=422) deferred.addCallback(onSuccess) deferred.addErrback(onError) @@ -126,11 +160,25 @@ class MailsResource(Resource): _mail = InputMail.from_dict(content_dict) draft_id = content_dict.get('ident') + def defer_response(deferred): + deferred.addCallback(lambda pixelated_mail: respond_json_deferred({'ident': pixelated_mail.ident}, request)) + if draft_id: - if not self._mail_service.mail_exists(draft_id): - return respond_json("", request, status_code=422) - pixelated_mail = self._draft_service.update_draft(draft_id, _mail) + deferred_check = self._mail_service.mail_exists(draft_id) + + def handleDuplicatedDraftException(error): + respond_json_deferred("", request, status_code=422) + + def return422otherwise(mail_exists): + if not mail_exists: + respond_json_deferred("", request, status_code=422) + else: + new_draft = self._draft_service.update_draft(draft_id, _mail) + new_draft.addErrback(handleDuplicatedDraftException) + defer_response(new_draft) + + deferred_check.addCallback(return422otherwise) else: - pixelated_mail = self._draft_service.create_draft(_mail) + defer_response(self._draft_service.create_draft(_mail)) - return respond_json({'ident': pixelated_mail.ident}, request) + return server.NOT_DONE_YET diff --git a/service/pixelated/resources/root_resource.py b/service/pixelated/resources/root_resource.py index c1111269..8b536450 100644 --- a/service/pixelated/resources/root_resource.py +++ b/service/pixelated/resources/root_resource.py @@ -1,7 +1,9 @@ import os +import requests from pixelated.resources.attachments_resource import AttachmentsResource from pixelated.resources.contacts_resource import ContactsResource from pixelated.resources.features_resource import FeaturesResource +from pixelated.resources.feedback_resource import FeedbackResource from pixelated.resources.mail_resource import MailResource from pixelated.resources.mails_resource import MailsResource from pixelated.resources.tags_resource import TagsResource @@ -21,7 +23,7 @@ class RootResource(Resource): return self return Resource.getChild(self, path, request) - def initialize(self, keymanager, search_engine, mail_service, draft_service): + def initialize(self, keymanager, search_engine, mail_service, draft_service, feedback_service): self.putChild('assets', File(self._static_folder)) self.putChild('keys', KeysResource(keymanager)) self.putChild('attachment', AttachmentsResource(mail_service)) @@ -30,6 +32,7 @@ class RootResource(Resource): self.putChild('tags', TagsResource(search_engine)) self.putChild('mails', MailsResource(mail_service, draft_service)) self.putChild('mail', MailResource(mail_service)) + self.putChild('feedback', FeedbackResource(feedback_service)) def _get_static_folder(self): static_folder = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..", "web-ui", "app")) diff --git a/service/pixelated/support/date.py b/service/pixelated/support/date.py index e7cdbb30..0012aeea 100644 --- a/service/pixelated/support/date.py +++ b/service/pixelated/support/date.py @@ -16,6 +16,7 @@ import datetime import dateutil.parser +from email.utils import formatdate from dateutil.tz import tzlocal @@ -23,9 +24,6 @@ def iso_now(): return datetime.datetime.now(tzlocal()).isoformat() -def milliseconds(date): - date = dateutil.parser.parse(date) - date = date.replace(tzinfo=None) - epoch = datetime.datetime.utcfromtimestamp(0) - delta = date - epoch - return int(delta.total_seconds() * 1000) +def mail_date_now(): + date = dateutil.parser.parse(iso_now()) + return formatdate(float(date.strftime('%s'))) diff --git a/service/pixelated/support/encrypted_file_storage.py b/service/pixelated/support/encrypted_file_storage.py index 67036054..567a348a 100644 --- a/service/pixelated/support/encrypted_file_storage.py +++ b/service/pixelated/support/encrypted_file_storage.py @@ -23,7 +23,7 @@ from whoosh.filedb.filestore import FileStorage from whoosh.filedb.structfile import StructFile, BufferFile from leap.soledad.client.crypto import encrypt_sym from leap.soledad.client.crypto import decrypt_sym -from leap.soledad.client.crypto import EncryptionMethods +from leap.soledad.common.crypto import EncryptionMethods from whoosh.util import random_name @@ -56,16 +56,16 @@ class EncryptedFileStorage(FileStorage): return hmac.new(self.signkey, verifiable_payload, sha256).digest() def encrypt(self, content): - iv, ciphertext = encrypt_sym(content, self.masterkey, EncryptionMethods.XSALSA20) + iv, ciphertext = encrypt_sym(content, self.masterkey) mac = self.gen_mac(iv, ciphertext) return ''.join((mac, iv, ciphertext)) def decrypt(self, payload): - payload_mac, iv, ciphertext = payload[:32], payload[32:65], payload[65:] + payload_mac, iv, ciphertext = payload[:32], payload[32:57], payload[57:] generated_mac = self.gen_mac(iv, ciphertext) if sha256(payload_mac).digest() != sha256(generated_mac).digest(): raise Exception("EncryptedFileStorage - Error opening file. Wrong MAC") - return decrypt_sym(ciphertext, self.masterkey, EncryptionMethods.XSALSA20, iv=iv) + return decrypt_sym(ciphertext, self.masterkey, iv) def _encrypt_index_on_close(self, name): def wrapper(struct_file): |