diff options
Diffstat (limited to 'service/src/pixelated')
88 files changed, 6893 insertions, 0 deletions
diff --git a/service/src/pixelated/__init__.py b/service/src/pixelated/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/service/src/pixelated/__init__.py diff --git a/service/src/pixelated/account_recovery.py b/service/src/pixelated/account_recovery.py new file mode 100644 index 00000000..58242a7d --- /dev/null +++ b/service/src/pixelated/account_recovery.py @@ -0,0 +1,85 @@ +# +# Copyright (c) 2017 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import pkg_resources +import binascii +from email import message_from_string + +from twisted.internet.defer import inlineCallbacks, returnValue +from twisted.logger import Logger +from twisted.mail import smtp + +from pixelated.support import date + +log = Logger() + + +class AccountRecovery(object): + def __init__(self, session, soledad, smtp_config, backup_email, domain, language='en-US'): + self._bonafide_session = session + self._soledad = soledad + self._smtp_config = smtp_config + self._backup_email = backup_email + self._domain = domain + self._language = language + + @inlineCallbacks + def update_recovery_code(self): + log.info('Updating user\'s recovery code') + + try: + code = self._soledad.create_recovery_code() + response = yield self._bonafide_session.update_recovery_code(code) + yield self._send_mail(code, self._backup_email) + + returnValue(response) + + except Exception as e: + log.error('Something went wrong when trying to save the recovery code') + log.error(e) + raise e + + @inlineCallbacks + def _send_mail(self, code, backup_email): + log.info('Sending mail containing the user\'s recovery code') + + sender = 'team@{}'.format(self._domain) + msg = self._get_recovery_mail(code, sender, backup_email) + + try: + send_mail_result = yield smtp.sendmail( + str(self._smtp_config.remote_smtp_host), + sender, + [backup_email], + msg.as_string()) + returnValue(send_mail_result) + except Exception as e: + log.error('Failed trying to send the email with the recovery code') + raise e + + def _get_recovery_mail(self, code, sender, backup_email): + email_date = date.mail_date_now() + recovery_mail = pkg_resources.resource_filename( + 'pixelated.assets', + 'recovery.mail.%s' % (self._language)) + + with open(recovery_mail) as mail_template_file: + return message_from_string(mail_template_file.read().format( + domain=self._domain, + recovery_code=binascii.hexlify(code), + backup_email=backup_email, + sender=sender, + date=email_date)) diff --git a/service/src/pixelated/adapter/__init__.py b/service/src/pixelated/adapter/__init__.py new file mode 100644 index 00000000..2756a319 --- /dev/null +++ b/service/src/pixelated/adapter/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. diff --git a/service/src/pixelated/adapter/listeners/__init__.py b/service/src/pixelated/adapter/listeners/__init__.py new file mode 100644 index 00000000..2756a319 --- /dev/null +++ b/service/src/pixelated/adapter/listeners/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. diff --git a/service/src/pixelated/adapter/listeners/mailbox_indexer_listener.py b/service/src/pixelated/adapter/listeners/mailbox_indexer_listener.py new file mode 100644 index 00000000..9cc522cf --- /dev/null +++ b/service/src/pixelated/adapter/listeners/mailbox_indexer_listener.py @@ -0,0 +1,66 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PCULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from twisted.internet import defer +from twisted.logger import Logger + +logger = Logger() + + +class MailboxIndexerListener(object): + """ Listens for new mails, keeping the index updated """ + + @classmethod + @defer.inlineCallbacks + def listen(cls, account, mailbox_name, mail_store, search_engine): + listener = MailboxIndexerListener(mailbox_name, mail_store, search_engine) + mail_collection = yield account.get_collection_by_mailbox(mailbox_name) + mail_collection.addListener(listener) + + defer.returnValue(listener) + + def __init__(self, mailbox_name, mail_store, search_engine): + self.mailbox_name = mailbox_name + self.mail_store = mail_store + self.search_engine = search_engine + + @defer.inlineCallbacks + def notify_new(self): + try: + indexed_idents = set(self.search_engine.search('tag:' + self.mailbox_name.lower(), all_mails=True)) + soledad_idents = yield self.mail_store.get_mailbox_mail_ids(self.mailbox_name) + soledad_idents = set(soledad_idents) + + missing_idents = soledad_idents.difference(indexed_idents) + + self.search_engine.index_mails((yield self.mail_store.get_mails(missing_idents, include_body=True))) + except Exception, e: # this is a event handler, don't let exceptions escape + logger.error(e) + + def __eq__(self, other): + return other and other.mailbox_name == self.mailbox_name + + def __hash__(self): + return self.mailbox_name.__hash__() + + def __repr__(self): + return 'MailboxListener: ' + self.mailbox_name + + +@defer.inlineCallbacks +def listen_all_mailboxes(account, search_engine, mail_store): + mailboxes = yield account.list_all_mailbox_names() + for mailbox_name in mailboxes: + yield MailboxIndexerListener.listen(account, mailbox_name, mail_store, search_engine) diff --git a/service/src/pixelated/adapter/mailstore/__init__.py b/service/src/pixelated/adapter/mailstore/__init__.py new file mode 100644 index 00000000..978df45d --- /dev/null +++ b/service/src/pixelated/adapter/mailstore/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from pixelated.adapter.mailstore.mailstore import MailStore, underscore_uuid +from pixelated.adapter.mailstore.leap_mailstore import LeapMailStore + +__all__ = ['MailStore', 'LeapMailStore', 'underscore_uuid'] diff --git a/service/src/pixelated/adapter/mailstore/body_parser.py b/service/src/pixelated/adapter/mailstore/body_parser.py new file mode 100644 index 00000000..8cac75cf --- /dev/null +++ b/service/src/pixelated/adapter/mailstore/body_parser.py @@ -0,0 +1,69 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import re +from email.parser import Parser + +from twisted.logger import Logger + +logger = Logger() + + +def _parse_charset_header(content_type_and_charset_header, default_charset='us-ascii'): + try: + return re.compile('.*charset="?([a-zA-Z0-9-]+)"?', re.MULTILINE | re.DOTALL).match(content_type_and_charset_header).group(1) + except: + return default_charset + + +class BodyParser(object): + + def __init__(self, content, content_type='text/plain; charset="us-ascii"', content_transfer_encoding=None, charset=None): + self._content = content + self._content_type = content_type + self._content_transfer_encoding = content_transfer_encoding + self._charset = charset + + def parsed_content(self): + charset = self._charset or _parse_charset_header(self._content_type) + text = self._serialize_for_parser(charset) + + decoded_body = self._parse_and_decode(text) + return unicode(decoded_body, charset, errors='replace') + + def _parse_and_decode(self, text): + parsed_body = Parser().parsestr(text) + decoded_body = self._unwrap_content_transfer_encoding(parsed_body) + return decoded_body + + def _unwrap_content_transfer_encoding(self, parsed_body): + return parsed_body.get_payload(decode=True) + + def _serialize_for_parser(self, charset): + text = u'Content-Type: %s\n' % self._content_type + if self._content_transfer_encoding is not None: + text += u'Content-Transfer-Encoding: %s\n' % self._content_transfer_encoding + + text += u'\n' + encoded_text = text.encode(charset) + if isinstance(self._content, unicode): + try: + return encoded_text + self._content.encode(charset) + except UnicodeError, e: + logger.warn('Failed to encode content for charset %s. Ignoring invalid chars: %s' % (charset, e)) + return encoded_text + self._content.encode(charset, 'ignore') + else: + return encoded_text + self._content diff --git a/service/src/pixelated/adapter/mailstore/leap_attachment_store.py b/service/src/pixelated/adapter/mailstore/leap_attachment_store.py new file mode 100644 index 00000000..b297f9e6 --- /dev/null +++ b/service/src/pixelated/adapter/mailstore/leap_attachment_store.py @@ -0,0 +1,81 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + + +import quopri +import base64 +from email import encoders +from leap.bitmask.mail.adaptors.soledad import SoledadMailAdaptor, ContentDocWrapper +from twisted.internet import defer +from email.mime.nonmultipart import MIMENonMultipart +from email.mime.multipart import MIMEMultipart +from leap.bitmask.mail.mail import Message + + +class LeapAttachmentStore(object): + + def __init__(self, soledad): + self.soledad = soledad + + @defer.inlineCallbacks + def get_mail_attachment(self, attachment_id): + results = yield self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', attachment_id) if attachment_id else [] + if results: + content = ContentDocWrapper(**results[0].content) + defer.returnValue({'content-type': content.content_type, 'content': self._try_decode( + content.raw, content.content_transfer_encoding)}) + else: + raise ValueError('No attachment with id %s found!' % attachment_id) + + @defer.inlineCallbacks + def add_attachment(self, content, content_type): + cdoc = self._attachment_to_cdoc(content, content_type) + attachment_id = cdoc.phash + try: + yield self.get_mail_attachment(attachment_id) + except ValueError: + yield self.soledad.create_doc(cdoc.serialize(), doc_id=attachment_id) + defer.returnValue(attachment_id) + + def _try_decode(self, raw, encoding): + encoding = encoding.lower() + if encoding == 'base64': + data = base64.decodestring(raw) + elif encoding == 'quoted-printable': + data = quopri.decodestring(raw) + else: + data = str(raw) + + return bytearray(data) + + def _attachment_to_cdoc(self, content, content_type, encoder=encoders.encode_base64): + major, sub = content_type.split('/') + attachment = MIMENonMultipart(major, sub) + attachment.set_payload(content) + encoder(attachment) + attachment.add_header('Content-Disposition', 'attachment', filename='does_not_matter.txt') + + pseudo_mail = MIMEMultipart() + pseudo_mail.attach(attachment) + + tmp_mail = SoledadMailAdaptor().get_msg_from_string(MessageClass=Message, raw_msg=pseudo_mail.as_string()) + + cdoc = tmp_mail.get_wrapper().cdocs[1] + return cdoc + + def _calc_attachment_id_(self, content, content_type, encoder=encoders.encode_base64): + cdoc = self._attachment_to_cdoc(content, content_type, encoder) + return cdoc.phash diff --git a/service/src/pixelated/adapter/mailstore/leap_mailstore.py b/service/src/pixelated/adapter/mailstore/leap_mailstore.py new file mode 100644 index 00000000..288223dd --- /dev/null +++ b/service/src/pixelated/adapter/mailstore/leap_mailstore.py @@ -0,0 +1,406 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import re +from email.header import decode_header +from uuid import uuid4 + +from leap.bitmask.mail.adaptors.soledad import SoledadMailAdaptor +from leap.bitmask.mail.mail import Message +from twisted.internet import defer +from twisted.internet.defer import FirstError, DeferredList + +from pixelated.adapter.mailstore.body_parser import BodyParser +from pixelated.adapter.mailstore.mailstore import MailStore, underscore_uuid +from pixelated.adapter.model.mail import Mail, InputMail +from pixelated.support.functional import to_unicode +from pixelated.support import date + + +class AttachmentInfo(object): + def __init__(self, ident, name, encoding=None, ctype='application/octet-stream', size=0): + self.ident = ident + self.name = name + self.encoding = encoding + self.ctype = ctype + self.size = size + + def __repr__(self): + return 'AttachmentInfo[%s, %s, %s]' % (self.ident, self.name, self.encoding) + + def __str__(self): + return 'AttachmentInfo[%s, %s, %s]' % (self.ident, self.name, self.encoding) + + def as_dict(self): + return {'ident': self.ident, 'name': self.name, 'encoding': self.encoding, 'size': self.size, 'content-type': self.ctype} + + +class LeapMail(Mail): + + def __init__(self, mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None, attachments=[]): + self._mail_id = mail_id + self._mailbox_name = mailbox_name + self._headers = headers if headers is not None else {} + self._body = to_unicode(body) + self.tags = set(tags) # TODO test that asserts copy + self._flags = set(flags) # TODO test that asserts copy + self._attachments = attachments + + @property + def headers(self): + cpy = dict(self._headers) + for name in set(self._headers.keys()).intersection(['To', 'Cc', 'Bcc']): + cpy[name] = [address.strip() for address in (self._headers[name].split(',') if self._headers[name] else [])] + + return cpy + + @property + def ident(self): + return self._mail_id + + @property + def mail_id(self): + return self._mail_id + + @property + def body(self): + return self._body + + @property + def flags(self): + return self._flags + + @property + def mailbox_name(self): + return self._mailbox_name + + @property + def security_casing(self): + casing = dict(imprints=self._signature_information(), locks=[]) + if self._encrypted() == "decrypted": + casing["locks"] = [{"state": "valid"}] + return casing + + def _encrypted(self): + return self.headers.get("X-Leap-Encryption", "false") + + def _signature_information(self): + signature = self.headers.get("X-Leap-Signature", None) + if signature is None or signature.startswith("could not verify"): + return [{"state": "no_signature_information"}] + else: + if signature.startswith("valid"): + return [{"state": "valid", "seal": {"validity": "valid"}}] + else: + return [] + + @property + def raw(self): + result = u'' + for k, v in self._headers.items(): + content, encoding = decode_header(v)[0] + if encoding: + result += '%s: %s\n' % (k, unicode(content, encoding=encoding)) + else: + result += '%s: %s\n' % (k, v) + result += '\n' + + if self._body: + result = result + self._body + + return result + + def _remove_duplicates(self, values): + return list(set(values)) + + def _decoded_header_utf_8(self, header_value): + if isinstance(header_value, list): + return self._remove_duplicates([self._decoded_header_utf_8(v) for v in header_value]) + elif header_value is not None: + def encode_chunk(content, encoding): + return unicode(content.strip(), encoding=encoding or 'ascii', errors='ignore') + + try: + encoded_chunks = [encode_chunk(content, encoding) for content, encoding in decode_header(header_value)] + return ' '.join(encoded_chunks) # decode_header strips whitespaces on all chunks, joining over ' ' is only a workaround, not a proper fix + except UnicodeEncodeError: + return unicode(header_value.encode('ascii', errors='ignore')) + + def as_dict(self): + return { + 'header': {k.lower(): self._decoded_header_utf_8(v) for k, v in self.headers.items()}, + 'ident': self._mail_id, + 'tags': self.tags, + 'status': list(self.status), + 'body': self._body, + 'security_casing': self.security_casing, + 'textPlainBody': self._body, + 'mailbox': self._mailbox_name.lower(), + 'attachments': [attachment.as_dict() for attachment in self._attachments] + } + + @staticmethod + def from_dict(mail_dict): + # TODO: implement this method and also write tests for it + headers = {key.capitalize(): value for key, value in mail_dict.get('header', {}).items()} + headers['Date'] = date.mail_date_now() + body = mail_dict.get('body', '') + tags = set(mail_dict.get('tags', [])) + status = set(mail_dict.get('status', [])) + attachments = [] + + # mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None, attachments=[] + return LeapMail(None, None, headers, tags, set(), body, attachments) + + +def _extract_filename(headers, default_filename='UNNAMED'): + content_disposition = headers.get('Content-Disposition') or headers.get('content-disposition', '') + filename = _extract_filename_from_name_header_part(content_disposition) + if not filename: + filename = headers.get('Content-Description', '') + if not filename: + content_type = headers.get('Content-Type', '') + filename = _extract_filename_from_name_header_part(content_type) + + if not filename: + filename = default_filename + + return filename + + +def _extract_filename_from_name_header_part(header_value): + match = re.compile('.*name=\"?(.*[^\"\'])').search(header_value) + filename = '' + if match: + filename = match.group(1) + return filename + + +class LeapMailStore(MailStore): + __slots__ = ('soledad') + + def __init__(self, soledad): + self.soledad = soledad + + @defer.inlineCallbacks + def get_mail(self, mail_id, include_body=False): + message = yield self._fetch_msg_from_soledad(mail_id) + if not _is_empty_message(message): + leap_mail = yield self._leap_message_to_leap_mail(mail_id, message, include_body) + else: + leap_mail = None + + defer.returnValue(leap_mail) + + @defer.inlineCallbacks + def get_mails(self, mail_ids, gracefully_ignore_errors=False, include_body=False): + deferreds = [] + for mail_id in mail_ids: + deferreds.append(self.get_mail(mail_id, include_body=include_body)) + + if gracefully_ignore_errors: + results = yield DeferredList(deferreds, consumeErrors=True) + defer.returnValue([mail for ok, mail in results if ok and mail is not None]) + else: + result = yield defer.gatherResults(deferreds, consumeErrors=True) + defer.returnValue(result) + + @defer.inlineCallbacks + def update_mail(self, mail): + message = yield self._fetch_msg_from_soledad(mail.mail_id) + message.get_wrapper().set_tags(tuple(mail.tags)) + message.get_wrapper().set_flags(tuple(mail.flags)) + yield self._update_mail(message) # TODO assert this is yielded (otherwise asynchronous) + + @defer.inlineCallbacks + def all_mails(self, gracefully_ignore_errors=False): + mdocs = yield self.soledad.get_from_index('by-type', 'meta') + + mail_ids = map(lambda doc: doc.doc_id, mdocs) + + mails = yield self.get_mails(mail_ids, gracefully_ignore_errors=gracefully_ignore_errors, include_body=True) + defer.returnValue(mails) + + @defer.inlineCallbacks + def add_mailbox(self, mailbox_name): + mailbox = yield self._get_or_create_mailbox(mailbox_name) + defer.returnValue(mailbox) + + @defer.inlineCallbacks + def get_mailbox_names(self): + mbox_map = set((yield self._mailbox_uuid_to_name_map()).values()) + + defer.returnValue(mbox_map.union({'INBOX'})) + + @defer.inlineCallbacks + def _mailbox_uuid_to_name_map(self): + map = {} + mbox_docs = yield self.soledad.get_from_index('by-type', 'mbox') + for doc in mbox_docs: + map[underscore_uuid(doc.content.get('uuid'))] = doc.content.get('mbox') + + defer.returnValue(map) + + @defer.inlineCallbacks + def add_mail(self, mailbox_name, raw_msg): + mailbox = yield self._get_or_create_mailbox(mailbox_name) + message = SoledadMailAdaptor().get_msg_from_string(Message, raw_msg) + message.get_wrapper().set_mbox_uuid(mailbox.uuid) + + yield SoledadMailAdaptor().create_msg(self.soledad, message) + + # add behavious from insert_mdoc_id from mail.py + mail = yield self._leap_message_to_leap_mail(message.get_wrapper().mdoc.doc_id, message, include_body=True) # TODO test that asserts include_body + defer.returnValue(mail) + + @defer.inlineCallbacks + def delete_mail(self, mail_id): + message = yield self._fetch_msg_from_soledad(mail_id) + if message and message.get_wrapper().mdoc.doc_id: + yield message.get_wrapper().delete(self.soledad) + defer.returnValue(True) + defer.returnValue(False) + + @defer.inlineCallbacks + def get_mailbox_mail_ids(self, mailbox_name): + mailbox = yield self._get_or_create_mailbox(mailbox_name) + fdocs = yield self.soledad.get_from_index('by-type-and-mbox-uuid', 'flags', underscore_uuid(mailbox.uuid)) + + mail_ids = map(lambda doc: _fdoc_id_to_mdoc_id(doc.doc_id), fdocs) + + defer.returnValue(mail_ids) + + @defer.inlineCallbacks + def delete_mailbox(self, mailbox_name): + mbx_wrapper = yield self._get_or_create_mailbox(mailbox_name) + yield SoledadMailAdaptor().delete_mbox(self.soledad, mbx_wrapper) + + @defer.inlineCallbacks + def copy_mail_to_mailbox(self, mail_id, mailbox_name): + message = yield self._fetch_msg_from_soledad(mail_id, load_body=True) + mailbox = yield self._get_or_create_mailbox(mailbox_name) + copy_wrapper = yield message.get_wrapper().copy(self.soledad, mailbox.uuid) + + leap_message = Message(copy_wrapper) + + mail = yield self._leap_message_to_leap_mail(copy_wrapper.mdoc.doc_id, leap_message, include_body=False) + + defer.returnValue(mail) + + @defer.inlineCallbacks + def move_mail_to_mailbox(self, mail_id, mailbox_name): + mail_copy = yield self.copy_mail_to_mailbox(mail_id, mailbox_name) + yield self.delete_mail(mail_id) + defer.returnValue(mail_copy) + + def _update_mail(self, message): + return message.get_wrapper().update(self.soledad) + + @defer.inlineCallbacks + def _leap_message_to_leap_mail(self, mail_id, message, include_body): + if include_body: + # TODO use body from message if available + body = yield self._raw_message_body(message) + else: + body = None + + # fetch mailbox name by mbox_uuid + mbox_uuid = message.get_wrapper().fdoc.mbox_uuid + mbox_name = yield self._mailbox_name_from_uuid(mbox_uuid) + attachments = self._extract_attachment_info_from(message) + mail = LeapMail(mail_id, mbox_name, message.get_wrapper().hdoc.headers, set(message.get_tags()), set(message.get_flags()), body=body, attachments=attachments) # TODO assert flags are passed on + + defer.returnValue(mail) + + @defer.inlineCallbacks + def _raw_message_body(self, message): + content_doc = (yield message.get_wrapper().get_body(self.soledad)) + parser = BodyParser('', content_type='text/plain', content_transfer_encoding='UTF-8') + # It fix the problem when leap doesn'r found body_phash and returns empty string + if not isinstance(content_doc, str): + parser = BodyParser(content_doc.raw, content_type=content_doc.content_type, + content_transfer_encoding=content_doc.content_transfer_encoding, charset=content_doc.charset) + + defer.returnValue(parser.parsed_content()) + + @defer.inlineCallbacks + def _mailbox_name_from_uuid(self, uuid): + map = (yield self._mailbox_uuid_to_name_map()) + defer.returnValue(map.get(uuid, '')) + + @defer.inlineCallbacks + def _get_or_create_mailbox(self, mailbox_name): + mailbox_name_upper = mailbox_name.upper() + mbx = yield SoledadMailAdaptor().get_or_create_mbox(self.soledad, mailbox_name_upper) + if mbx.uuid is None: + mbx.uuid = str(uuid4()) + yield mbx.update(self.soledad) + defer.returnValue(mbx) + + def _fetch_msg_from_soledad(self, mail_id, load_body=False): + return SoledadMailAdaptor().get_msg_from_mdoc_id(Message, self.soledad, mail_id, get_cdocs=load_body) + + @defer.inlineCallbacks + def _dump_soledad(self): + gen, docs = yield self.soledad.get_all_docs() + for doc in docs: + print '\n%s\n' % doc + + def _extract_attachment_info_from(self, message): + wrapper = message.get_wrapper() + part_maps = wrapper.hdoc.part_map + return self._extract_part_map(part_maps) + + def _is_attachment(self, part_map, headers): + disposition = headers.get('Content-Disposition') or headers.get('content-disposition') + content_type = part_map['ctype'] + + if 'multipart' in content_type: + return False + + if 'text/plain' == content_type and ((disposition == 'inline') or (disposition is None)): + return False + + return True + + def _create_attachment_info_from(self, part_map, headers): + ident = part_map['phash'] + name = _extract_filename(headers) + encoding = headers.get('Content-Transfer-Encoding', None) + ctype = part_map.get('ctype') or headers.get('Content-Type') + size = part_map.get('size', 0) + + return AttachmentInfo(ident, name, encoding, ctype, size) + + def _extract_part_map(self, part_maps): + result = [] + + for nr, part_map in part_maps.items(): + if 'headers' in part_map and 'phash' in part_map: + headers = {header[0]: header[1] for header in part_map['headers']} + if self._is_attachment(part_map, headers): + result.append(self._create_attachment_info_from(part_map, headers)) + if 'part_map' in part_map: + result += self._extract_part_map(part_map['part_map']) + + return result + + +def _is_empty_message(message): + return (message is None) or (message.get_wrapper().mdoc.doc_id is None) + + +def _fdoc_id_to_mdoc_id(fdoc_id): + return 'M' + fdoc_id[1:] diff --git a/service/src/pixelated/adapter/mailstore/mailstore.py b/service/src/pixelated/adapter/mailstore/mailstore.py new file mode 100644 index 00000000..fbd7fc9e --- /dev/null +++ b/service/src/pixelated/adapter/mailstore/mailstore.py @@ -0,0 +1,60 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + + +class MailStore(object): + def get_mail(self, mail_id): + pass + + def get_mail_attachment(self, attachment_id): + pass + + def get_mails(self, mail_ids, gracefully_ignore_errors=False, include_body=False): + pass + + def all_mails(self): + pass + + def delete_mail(self, mail_id): + pass + + def update_mail(self, mail): + pass + + def add_mail(self, mailbox_name, mail): + pass + + def get_mailbox_names(self): + pass + + def add_mailbox(self, mailbox_name): + pass + + def delete_mailbox(self, mailbox_name): + pass + + def get_mailbox_mail_ids(self, mailbox_name): + pass + + def copy_mail_to_mailbox(self, mail_id, mailbox_name): + pass + + def move_mail_to_mailbox(self, mail_id, mailbox_name): + pass + + +def underscore_uuid(uuid): + return uuid.replace('-', '_') diff --git a/service/src/pixelated/adapter/mailstore/maintenance/__init__.py b/service/src/pixelated/adapter/mailstore/maintenance/__init__.py new file mode 100644 index 00000000..9a1007cc --- /dev/null +++ b/service/src/pixelated/adapter/mailstore/maintenance/__init__.py @@ -0,0 +1,100 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from leap.bitmask.keymanager import documents as leap_doc +from leap.bitmask.keymanager.keys import OpenPGPKey + +from twisted.internet import defer +from twisted.logger import Logger + + +TYPE_OPENPGP_KEY = 'OpenPGPKey' +TYPE_OPENPGP_ACTIVE = 'OpenPGPKey-active' + +KEY_DOC_TYPES = {TYPE_OPENPGP_ACTIVE, TYPE_OPENPGP_KEY} + +logger = Logger() + + +def _is_key_doc(doc): + return doc.content.get(leap_doc.KEY_TYPE_KEY, None) in KEY_DOC_TYPES + + +def _is_private_key_doc(doc): + return _is_key_doc(doc) and doc.content.get(leap_doc.KEY_PRIVATE_KEY, False) + + +def _is_active_key_doc(doc): + return _is_key_doc(doc) and doc.content.get(leap_doc.KEY_TYPE_KEY, None) == TYPE_OPENPGP_ACTIVE + + +def _is_public_key(doc): + return _is_key_doc(doc) and not doc.content.get(leap_doc.KEY_PRIVATE_KEY, False) + + +def _key_fingerprint(doc): + return doc.content.get(leap_doc.KEY_FINGERPRINT_KEY, None) + + +def _address(doc): + return doc.content.get(leap_doc.KEY_ADDRESS_KEY, None) + + +class SoledadMaintenance(object): + def __init__(self, soledad): + self._soledad = soledad + + @defer.inlineCallbacks + def repair(self): + _, docs = yield self._soledad.get_all_docs() + + private_key_fingerprints = self._key_fingerprints_with_private_key(docs) + + for doc in docs: + if _is_key_doc(doc) and _key_fingerprint(doc) not in private_key_fingerprints: + logger.warn('Deleting doc %s for key %s of <%s>' % (doc.doc_id, _key_fingerprint(doc), _address(doc))) + yield self._soledad.delete_doc(doc) + + yield self._repair_missing_active_docs(docs, private_key_fingerprints) + + @defer.inlineCallbacks + def _repair_missing_active_docs(self, docs, private_key_fingerprints): + missing = self._missing_active_docs(docs, private_key_fingerprints) + for fingerprint in missing: + emails = self._emails_for_key_fingerprint(docs, fingerprint) + for email in emails: + logger.warn('Re-creating active doc for key %s, email %s' % (fingerprint, email)) + yield self._soledad.create_doc_from_json(OpenPGPKey(email, fingerprint=fingerprint, private=False).get_active_json()) + + def _key_fingerprints_with_private_key(self, docs): + return [doc.content[leap_doc.KEY_FINGERPRINT_KEY] for doc in docs if _is_private_key_doc(doc)] + + def _missing_active_docs(self, docs, private_key_fingerprints): + active_doc_ids = self._active_docs_for_key_fingerprint(docs) + + return set([private_key_fingerprint for private_key_fingerprint in private_key_fingerprints if private_key_fingerprint not in active_doc_ids]) + + def _emails_for_key_fingerprint(self, docs, fingerprint): + for doc in docs: + if _is_private_key_doc(doc) and _key_fingerprint(doc) == fingerprint: + email = _address(doc) + if email is None: + return [] + if isinstance(email, list): + return email + return [email] + + def _active_docs_for_key_fingerprint(self, docs): + return [doc.content[leap_doc.KEY_FINGERPRINT_KEY] for doc in docs if _is_active_key_doc(doc) and _is_public_key(doc)] diff --git a/service/src/pixelated/adapter/mailstore/searchable_mailstore.py b/service/src/pixelated/adapter/mailstore/searchable_mailstore.py new file mode 100644 index 00000000..07e99ba7 --- /dev/null +++ b/service/src/pixelated/adapter/mailstore/searchable_mailstore.py @@ -0,0 +1,81 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from twisted.internet import defer +from types import FunctionType +from pixelated.adapter.mailstore import MailStore + + +class SearchableMailStore(object): # implementes MailStore + + def __init__(self, delegate, search_engine): + self._delegate = delegate + self._search_engine = search_engine + + @classmethod + def _create_delegator(cls, method_name): + def delegator(self, *args, **kw): + return getattr(self._delegate, method_name)(*args, **kw) + + setattr(cls, method_name, delegator) + + @defer.inlineCallbacks + def add_mail(self, mailbox_name, mail): + stored_mail = yield self._delegate.add_mail(mailbox_name, mail) + self._search_engine.index_mail(stored_mail) + defer.returnValue(stored_mail) + + @defer.inlineCallbacks + def delete_mail(self, mail_id): + removed = yield self._delegate.delete_mail(mail_id) + self._search_engine.remove_from_index(mail_id) + defer.returnValue(removed) + + @defer.inlineCallbacks + def update_mail(self, mail): + yield self._delegate.update_mail(mail) + self._search_engine.index_mail(mail) + + @defer.inlineCallbacks + def move_mail_to_mailbox(self, mail_id, mailbox_name): + moved_mail = yield self._delegate.move_mail_to_mailbox(mail_id, mailbox_name) + self._search_engine.remove_from_index(mail_id) + self._search_engine.index_mail(moved_mail) + defer.returnValue(moved_mail) + + @defer.inlineCallbacks + def copy_mail_to_mailbox(self, mail_id, mailbox_name): + copied_mail = yield self._delegate.copy_mail_to_mailbox(mail_id, mailbox_name) + self._search_engine.index_mail(copied_mail) + defer.returnValue(copied_mail) + + def delete_mailbox(self, mailbox_name): + raise NotImplementedError() + + def __getattr__(self, name): + """ + Acts like method missing. If a method of MailStore is not implemented in this class, + a delegate method is created. + + :param name: attribute name + :return: method or attribute + """ + methods = ([key for key, value in MailStore.__dict__.items() if type(value) == FunctionType]) + + if name in methods: + SearchableMailStore._create_delegator(name) + return super(SearchableMailStore, self).__getattribute__(name) + else: + raise NotImplementedError('No attribute %s' % name) diff --git a/service/src/pixelated/adapter/model/__init__.py b/service/src/pixelated/adapter/model/__init__.py new file mode 100644 index 00000000..2756a319 --- /dev/null +++ b/service/src/pixelated/adapter/model/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. diff --git a/service/src/pixelated/adapter/model/mail.py b/service/src/pixelated/adapter/model/mail.py new file mode 100644 index 00000000..b6a8beb0 --- /dev/null +++ b/service/src/pixelated/adapter/model/mail.py @@ -0,0 +1,224 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PCULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import re +import binascii +from email.mime.text import MIMEText +from email.header import Header +from hashlib import sha256 +from email.MIMEMultipart import MIMEMultipart +from email.mime.nonmultipart import MIMENonMultipart + +from twisted.logger import Logger + +from leap.bitmask.mail import walk + +from pixelated.adapter.model.status import Status +from pixelated.support import date + + +logger = Logger() + + +class Mail(object): + @property + def from_sender(self): + return self.headers['From'] + + @property + def to(self): + return self.headers['To'] + + @property + def cc(self): + return self.headers['Cc'] + + @property + def bcc(self): + return self.headers['Bcc'] + + @property + def subject(self): + return self.headers['Subject'] + + @property + def date(self): + return self.headers['Date'] + + @property + def status(self): + return Status.from_flags(self.flags) + + @property + def flags(self): + return self.fdoc.content.get('flags') + + @property + def mailbox_name(self): + # FIXME mbox is no longer available, instead we now have mbox_uuid + return self.fdoc.content.get('mbox', 'INBOX') + + def _encode_header_value_list(self, header_value_list): + encoded_header_list = [self._encode_header_value(v) for v in header_value_list] + return ', '.join(encoded_header_list) + + def _encode_header_value(self, header_value): + if isinstance(header_value, unicode): + return str(Header(header_value, 'utf-8')) + return str(header_value) + + def _add_message_content(self, mime_multipart, body_to_use=None): + body_to_use = body_to_use or self.body + if isinstance(body_to_use, list): + for part in body_to_use: + mime_multipart.attach(MIMEText(part['raw'], part['content-type'])) + else: + mime_multipart.attach(MIMEText(body_to_use, 'plain', self._charset())) + + def _add_body(self, mime): + body_to_use = getattr(self, 'body', None) or getattr(self, 'text_plain_body', None) + self._add_message_content(mime, body_to_use) + self._add_attachments(mime) + + def _generate_mime_multipart(self): + mime = MIMEMultipart() + self._add_headers(mime) + self._add_body(mime) + return mime + + @property + def _mime_multipart(self): + self._mime = self._mime or self._generate_mime_multipart() + return self._mime + + def _add_headers(self, mime): + for key, value in self.headers.items(): + if isinstance(value, list): + mime[str(key)] = self._encode_header_value_list(value) + else: + mime[str(key)] = self._encode_header_value(value) + + def _add_attachments(self, mime): + for attachment in getattr(self, '_attachments', []): + major, sub = attachment['content-type'].split('/') + attachment_mime = MIMENonMultipart(major, sub) + base64_attachment_file = binascii.b2a_base64(attachment['raw']) + attachment_mime.set_payload(base64_attachment_file) + attachment_mime['Content-Disposition'] = 'attachment; filename="%s"' % attachment['name'] + attachment_mime['Content-Transfer-Encoding'] = 'base64' + mime.attach(attachment_mime) + + def _charset(self): + content_type = self.headers.get('content_type', {}) + if 'charset' in content_type: + return self._parse_charset_header(content_type) + return 'utf-8' + + def _parse_charset_header(self, charset_header, default_charset='utf-8'): + try: + return re.compile('.*charset=([a-zA-Z0-9-]+)', re.MULTILINE | re.DOTALL).match(charset_header).group(1) + except: + return default_charset + + @property + def raw(self): + return self._mime_multipart.as_string() + + def _get_chash(self): + return sha256(self.raw).hexdigest() + + +class InputMail(Mail): + def __init__(self): + self._raw_message = None + self._fd = None + self._hd = None + self._bd = None + self._chash = None + self._mime = None + self.headers = {} + self.body = '' + self._status = [] + self._attachments = [] + + @property + def ident(self): + return self._get_chash() + + def _get_body_phash(self): + return walk.get_body_phash(self._mime_multipart) + + def _add_predefined_headers(self, mime_multipart): + for header in ['To', 'Cc', 'Bcc']: + if self.headers.get(header): + mime_multipart[header] = ", ".join(self.headers[header]) + for header in ['Subject', 'From']: + if self.headers.get(header): + mime_multipart[header] = self.headers[header] + mime_multipart['Date'] = self.headers['Date'] + + def to_mime_multipart(self): + mime = MIMEMultipart() + self._add_predefined_headers(mime) + self._add_body(mime) + return mime + + def to_smtp_format(self): + mime_multipart = self.to_mime_multipart() + return mime_multipart.as_string() + + @staticmethod + def delivery_error_template(delivery_address): + return InputMail.from_dict({ + 'body': "Mail undelivered for %s" % delivery_address, + 'header': { + 'bcc': [], + 'cc': [], + 'subject': "Mail undelivered for %s" % delivery_address + } + }) + + @staticmethod + def from_dict(mail_dict, from_address): + input_mail = InputMail() + input_mail.headers = {key.capitalize(): value for key, value in mail_dict.get('header', {}).items()} + + input_mail.headers['Date'] = date.mail_date_now() + input_mail.headers['From'] = from_address + + input_mail.body = mail_dict.get('body', '') + input_mail.tags = set(mail_dict.get('tags', [])) + input_mail._status = set(mail_dict.get('status', [])) + input_mail._attachments = mail_dict.get('attachments', []) + return input_mail + + @staticmethod + def from_python_mail(mail): + input_mail = InputMail() + input_mail.headers = {unicode(key.capitalize()): unicode(value) for key, value in mail.items()} + input_mail.headers[u'Date'] = unicode(date.mail_date_now()) + input_mail.headers[u'To'] = [u''] + + for payload in mail.get_payload(): + input_mail._mime_multipart.attach(payload) + if payload.get_content_type() == 'text/plain': + content_charset = payload.get_content_charset() + try: + input_mail.body = unicode( + payload.get_payload(decode=True), content_charset) + except TypeError: + input_mail.body = unicode(payload.get_payload(decode=True)) + input_mail._mime = input_mail.to_mime_multipart() + return input_mail diff --git a/service/src/pixelated/adapter/model/status.py b/service/src/pixelated/adapter/model/status.py new file mode 100644 index 00000000..5a11ee7b --- /dev/null +++ b/service/src/pixelated/adapter/model/status.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + + +class Status: + + SEEN = u'\\Seen' + ANSWERED = u'\\Answered' + DELETED = u'\\Deleted' + RECENT = u'\\Recent' + + FLAGS_TO_STATUSES = { + SEEN: 'read', + ANSWERED: 'replied', + RECENT: 'recent' + } + + @staticmethod + def from_flag(flag): + return Status.FLAGS_TO_STATUSES[flag] + + @staticmethod + def from_flags(flags): + return set(Status.from_flag(flag) for flag in flags if flag in Status.FLAGS_TO_STATUSES.keys()) + + @staticmethod + def to_flags(statuses): + statuses_to_flags = dict(zip(Status.FLAGS_TO_STATUSES.values(), Status.FLAGS_TO_STATUSES.keys())) + return [statuses_to_flags[status] for status in statuses] diff --git a/service/src/pixelated/adapter/model/tag.py b/service/src/pixelated/adapter/model/tag.py new file mode 100644 index 00000000..ca62a1fe --- /dev/null +++ b/service/src/pixelated/adapter/model/tag.py @@ -0,0 +1,73 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import json + + +class Tag(object): + + @classmethod + def from_dict(cls, tag_dict): + tag = Tag(tag_dict['name'], tag_dict['default']) + tag.mails = set(tag_dict['mails']) + return tag + + @classmethod + def from_json_string(cls, json_string): + tag_dict = json.loads(json_string) + tag_dict['mails'] = set(tag_dict['mails']) + return Tag.from_dict(tag_dict) + + @property + def total(self): + return len(self.mails) + + def __init__(self, name, default=False): + self.name = name.lower() + self.ident = self.name.__hash__() + self.default = default + self.mails = set() + + def __eq__(self, other): + return self.name == other.name + + def __hash__(self): + return self.name.__hash__() + + def increment(self, mail_ident): + self.mails.add(mail_ident) + + def decrement(self, mail_ident): + self.mails.discard(mail_ident) + + def as_dict(self): + return { + 'name': self.name, + 'default': self.default, + 'ident': self.ident, + 'counts': {'total': self.total, + 'read': 0, + 'starred': 0, + 'replied': 0}, + 'mails': list(self.mails) + } + + def as_json_string(self): + tag_dict = self.as_dict() + return json.dumps(tag_dict) + + def __repr__(self): + return self.name diff --git a/service/src/pixelated/adapter/search/__init__.py b/service/src/pixelated/adapter/search/__init__.py new file mode 100644 index 00000000..e7a4e8c6 --- /dev/null +++ b/service/src/pixelated/adapter/search/__init__.py @@ -0,0 +1,209 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from pixelated.support.encrypted_file_storage import EncryptedFileStorage + +import os +import re +import dateutil.parser +import time +from pixelated.adapter.model.status import Status +from pixelated.adapter.search.contacts import contacts_suggestions +from whoosh.index import FileIndex +from whoosh.fields import Schema, ID, KEYWORD, TEXT, NUMERIC, NGRAMWORDS +from whoosh.qparser import QueryParser +from whoosh.qparser import MultifieldParser +from whoosh.writing import AsyncWriter +from whoosh import sorting +from pixelated.support.functional import unique, to_unicode +import traceback +from pixelated.support import date + + +class SearchEngine(object): + DEFAULT_INDEX_HOME = os.path.join(os.environ['HOME'], '.leap') + DEFAULT_TAGS = ['inbox', 'sent', 'drafts', 'trash'] + + def __init__(self, key, user_home=DEFAULT_INDEX_HOME): + self.key = key + self.index_folder = os.path.join(user_home, 'search_index') + if not os.path.exists(self.index_folder): + os.makedirs(self.index_folder) + self._index = self._create_index() + + def _add_to_tags(self, tags, group, skip_default_tags, count_type, query=None): + query_matcher = re.compile(".*%s.*" % query.lower()) if query else re.compile(".*") + + for tag, count in group.iteritems(): + + if skip_default_tags and tag in self.DEFAULT_TAGS or not query_matcher.match(tag): + continue + + if not tags.get(tag): + tags[tag] = {'ident': tag, 'name': tag, 'default': False, 'counts': {'total': 0, 'read': 0}, + 'mails': []} + tags[tag]['counts'][count_type] += count + + def _search_tag_groups(self, is_filtering_tags): + seen = None + query_parser = QueryParser('tag', self._index.schema) + options = {'limit': None, 'groupedby': sorting.FieldFacet('tag', allow_overlap=True), 'maptype': sorting.Count} + + with self._index.searcher() as searcher: + total = searcher.search(query_parser.parse('*'), **options).groups() + if not is_filtering_tags: + seen = searcher.search(query_parser.parse("* AND flags:%s" % Status.SEEN), **options).groups() + return seen, total + + def _init_tags_defaults(self): + tags = {} + for default_tag in self.DEFAULT_TAGS: + tags[default_tag] = { + 'ident': default_tag, + 'name': default_tag, + 'default': True, + 'counts': { + 'total': 0, + 'read': 0 + }, + 'mails': [] + } + return tags + + def _build_tags(self, seen, total, skip_default_tags, query): + tags = {} + if not skip_default_tags: + tags = self._init_tags_defaults() + self._add_to_tags(tags, total, skip_default_tags, count_type='total', query=query) + if seen: + self._add_to_tags(tags, seen, skip_default_tags, count_type='read') + return tags.values() + + def tags(self, query, skip_default_tags): + is_filtering_tags = True if query else False + seen, total = self._search_tag_groups(is_filtering_tags=is_filtering_tags) + return self._build_tags(seen, total, skip_default_tags, query) + + def _mail_schema(self): + return Schema( + ident=ID(stored=True, unique=True), + sender=ID(stored=False), + to=KEYWORD(stored=False, commas=True), + cc=KEYWORD(stored=False, commas=True), + bcc=KEYWORD(stored=False, commas=True), + subject=NGRAMWORDS(stored=False), + date=NUMERIC(stored=False, sortable=True, bits=64, signed=False), + body=NGRAMWORDS(stored=False), + tag=KEYWORD(stored=True, commas=True), + flags=KEYWORD(stored=True, commas=True), + raw=TEXT(stored=False)) + + def _create_index(self): + storage = EncryptedFileStorage(self.index_folder, self.key) + return FileIndex.create(storage, self._mail_schema(), indexname='mails') + + def index_mail(self, mail): + if mail is not None: + with AsyncWriter(self._index) as writer: + self._index_mail(writer, mail) + + def _index_mail(self, writer, mail): + mdict = mail.as_dict() + header = mdict['header'] + tags = set(mdict.get('tags', {})) + tags.add(mail.mailbox_name.lower()) + + index_data = { + 'sender': self._empty_string_to_none(header.get('from', '')), + 'subject': self._empty_string_to_none(header.get('subject', '')), + 'date': self._format_utc_integer(header.get('date', date.mail_date_now())), + 'to': self._format_recipient(header, 'to'), + 'cc': self._format_recipient(header, 'cc'), + 'bcc': self._format_recipient(header, 'bcc'), + 'tag': u','.join(unique(tags)), + 'body': to_unicode(mdict.get('textPlainBody', mdict.get('body', ''))), + 'ident': unicode(mdict['ident']), + 'flags': unicode(','.join(unique(mail.flags))), + 'raw': unicode(mail.raw) + } + + writer.update_document(**index_data) + + def _format_utc_integer(self, date): + timetuple = dateutil.parser.parse(date).utctimetuple() + return time.strftime('%s', timetuple) + + def _format_recipient(self, headers, name): + list = headers.get(name, ['']) + return u','.join(list) if list else u'' + + def _empty_string_to_none(self, field_value): + if not field_value: + return None + else: + return field_value + + def index_mails(self, mails, callback=None): + try: + with AsyncWriter(self._index) as writer: + for mail in mails: + self._index_mail(writer, mail) + if callback: + callback() + except Exception, e: + traceback.print_exc(e) + raise + + def _search_with_options(self, options, query): + with self._index.searcher() as searcher: + query = QueryParser('raw', self._index.schema).parse(query) + results = searcher.search(query, **options) + return results + + def search(self, query, window=25, page=1, all_mails=False): + query = self.prepare_query(query) + return self._search_all_mails(query) if all_mails else self._paginated_search_mails(query, window, page) + + def _search_all_mails(self, query): + with self._index.searcher() as searcher: + sorting_facet = sorting.FieldFacet('date', reverse=True) + results = searcher.search(query, sortedby=sorting_facet, reverse=True, limit=None) + return unique([mail['ident'] for mail in results]) + + def _paginated_search_mails(self, query, window, page): + page = int(page) if page is not None and int(page) > 1 else 1 + window = int(window) if window is not None else 25 + + with self._index.searcher() as searcher: + tags_facet = sorting.FieldFacet('tag', allow_overlap=True, maptype=sorting.Count) + sorting_facet = sorting.FieldFacet('date', reverse=True) + results = searcher.search_page(query, page, pagelen=window, groupedby=tags_facet, sortedby=sorting_facet) + return unique([mail['ident'] for mail in results]), sum(results.results.groups().values()) + + def prepare_query(self, query): + query = ( + query + .replace('-in:', 'AND NOT tag:') + .replace('in:all', '*') + ) + return MultifieldParser(['body', 'subject', 'raw'], self._index.schema).parse(query) + + def remove_from_index(self, mail_id): + with AsyncWriter(self._index) as writer: + writer.delete_by_term('ident', mail_id) + + def contacts(self, query): + with self._index.searcher() as searcher: + return contacts_suggestions(query, searcher) diff --git a/service/src/pixelated/adapter/search/contacts.py b/service/src/pixelated/adapter/search/contacts.py new file mode 100644 index 00000000..733489b0 --- /dev/null +++ b/service/src/pixelated/adapter/search/contacts.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from email.utils import parseaddr +from pixelated.support.functional import flatten +from whoosh.qparser import QueryParser +from whoosh import sorting +from whoosh.query import Term + + +def address_duplication_filter(contacts): + contacts_by_mail = dict() + + for contact in contacts: + mail_address = extract_mail_address(contact) + current = contacts_by_mail.get(mail_address, '') + current = contact if len(contact) > len(current) else current + contacts_by_mail[mail_address] = current + return contacts_by_mail.values() + + +def extract_mail_address(text): + return parseaddr(text)[1] + + +def contacts_suggestions(query, searcher): + return address_duplication_filter(search_addresses(searcher, query)) if query else [] + + +def search_addresses(searcher, query): + restrict_q = Term("tag", "drafts") | Term("tag", "trash") + results = [] + for field in ['to', 'cc', 'bcc', 'sender']: + query_parser = QueryParser(field, searcher.schema) + results.append( + searcher.search( + query_parser.parse("*%s* OR *%s*" % (query.title(), query)), + limit=None, + mask=restrict_q, + groupedby=sorting.FieldFacet( + field, + allow_overlap=True), + terms=True).matched_terms()) + return [address[1] for address in flatten(results)] diff --git a/service/src/pixelated/adapter/search/index_storage_key.py b/service/src/pixelated/adapter/search/index_storage_key.py new file mode 100644 index 00000000..b2761849 --- /dev/null +++ b/service/src/pixelated/adapter/search/index_storage_key.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import base64 +from twisted.internet import defer +import os + + +class SearchIndexStorageKey(object): + __slots__ = '_soledad' + + def __init__(self, soledad): + self._soledad = soledad + + @defer.inlineCallbacks + def get_or_create_key(self): + docs = yield self._soledad.get_from_index('by-type', 'index_key') + + if len(docs): + key = docs[0].content['value'] + else: + key = self._new_index_key() + yield self._store_key_in_soledad(key) + defer.returnValue(key) + + def _new_index_key(self): + return os.urandom(64) # 32 for encryption, 32 for hmac + + def _store_key_in_soledad(self, index_key): + return self._soledad.create_doc(dict(type='index_key', value=base64.encodestring(index_key))) diff --git a/service/src/pixelated/adapter/services/__init__.py b/service/src/pixelated/adapter/services/__init__.py new file mode 100644 index 00000000..2756a319 --- /dev/null +++ b/service/src/pixelated/adapter/services/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. diff --git a/service/src/pixelated/adapter/services/draft_service.py b/service/src/pixelated/adapter/services/draft_service.py new file mode 100644 index 00000000..504d92db --- /dev/null +++ b/service/src/pixelated/adapter/services/draft_service.py @@ -0,0 +1,40 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from twisted.internet import defer + + +class DraftService(object): + __slots__ = '_mail_store' + + def __init__(self, mail_store): + self._mail_store = mail_store + + @defer.inlineCallbacks + def create_draft(self, input_mail): + mail = yield self._mail_store.add_mail('DRAFTS', input_mail.raw) + defer.returnValue(mail) + + @defer.inlineCallbacks + def update_draft(self, ident, input_mail): + removed = yield self._mail_store.delete_mail(ident) + if removed: + new_draft = yield self.create_draft(input_mail) + defer.returnValue(new_draft) + + def process_draft(self, ident, input_mail): + if ident: + return self.update_draft(ident, input_mail) + return self.create_draft(input_mail) diff --git a/service/src/pixelated/adapter/services/feedback_service.py b/service/src/pixelated/adapter/services/feedback_service.py new file mode 100644 index 00000000..0cc595eb --- /dev/null +++ b/service/src/pixelated/adapter/services/feedback_service.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import os +import requests + + +class FeedbackService(object): + FEEDBACK_URL = os.environ.get('FEEDBACK_URL') + + def __init__(self, leap_session): + self.leap_session = leap_session + + def open_ticket(self, feedback): + account_mail = self.leap_session.account_email() + data = { + "ticket[comments_attributes][0][body]": feedback, + "ticket[subject]": "Feedback user-agent from {0}".format(account_mail), + "ticket[email]": account_mail, + "ticket[regarding_user]": account_mail + } + + return requests.post(self.FEEDBACK_URL, data=data, verify=False) diff --git a/service/src/pixelated/adapter/services/mail_sender.py b/service/src/pixelated/adapter/services/mail_sender.py new file mode 100644 index 00000000..063ea156 --- /dev/null +++ b/service/src/pixelated/adapter/services/mail_sender.py @@ -0,0 +1,106 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from StringIO import StringIO +from email.utils import parseaddr +from copy import deepcopy +from leap.bitmask.mail.outgoing.service import OutgoingMail + +from twisted.internet.defer import Deferred, fail +from twisted.mail.smtp import SMTPSenderFactory +from twisted.internet import reactor, defer +from pixelated.support.functional import flatten +from twisted.mail.smtp import User + +from twisted.logger import Logger + + +logger = Logger() + + +class SMTPDownException(Exception): + def __init__(self): + Exception.__init__(self, "Couldn't send mail now, try again later.") + + +NOT_NEEDED = None + + +class MailSenderException(Exception): + + def __init__(self, message, email_error_map): + super(MailSenderException, self).__init__(message, email_error_map) + self.email_error_map = email_error_map + + +class MailSender(object): + + def __init__(self, smtp_config, keymanager): + self._smtp_config = smtp_config + self._keymanager = keymanager + + @defer.inlineCallbacks + def sendmail(self, mail): + # message is changed in sending, but should be saved unaltered + mail = deepcopy(mail) + + recipients = flatten([mail.to, mail.cc, mail.bcc]) + + results = yield self._send_mail_to_all_recipients(mail, recipients) + all_succeeded = reduce(lambda a, b: a and b, [r[0] for r in results]) + + if not all_succeeded: + error_map = self._build_error_map(recipients, results) + raise MailSenderException('Failed to send mail to all recipients', error_map) + + defer.returnValue(all_succeeded) + + def _send_mail_to_all_recipients(self, mail, recipients): + outgoing_mail = self._create_outgoing_mail() + bccs = mail.bcc + deferreds = [] + + for recipient in recipients: + logger.info('_send_mail_to_all_recipients: Sending mail to recipient %s' % recipient) + self._define_bcc_field(mail, recipient, bccs) + smtp_recipient = self._create_twisted_smtp_recipient(recipient) + logger.info('_send_mail_to_all_recipients: Sending mail to smtp_recipient %s' % smtp_recipient) + deferreds.append(outgoing_mail.send_message(mail.to_smtp_format(), smtp_recipient)) + + return defer.DeferredList(deferreds, fireOnOneErrback=False, consumeErrors=True) + + def _define_bcc_field(self, mail, recipient, bccs): + if recipient in bccs: + mail.headers['Bcc'] = [recipient] + else: + mail.headers['Bcc'] = [] + + def _build_error_map(self, recipients, results): + error_map = {} + for email, error in [(recipients[idx], r[1]) for idx, r in enumerate(results)]: + error_map[email] = error + return error_map + + def _create_outgoing_mail(self): + return OutgoingMail(str(self._smtp_config.account_email), + self._keymanager, + self._smtp_config.cert_path, + self._smtp_config.cert_path, + str(self._smtp_config.remote_smtp_host), + int(self._smtp_config.remote_smtp_port)) + + def _create_twisted_smtp_recipient(self, recipient): + # TODO: Better is fix Twisted instead + return User(str(recipient), NOT_NEEDED, NOT_NEEDED, NOT_NEEDED) diff --git a/service/src/pixelated/adapter/services/mail_service.py b/service/src/pixelated/adapter/services/mail_service.py new file mode 100644 index 00000000..e5343997 --- /dev/null +++ b/service/src/pixelated/adapter/services/mail_service.py @@ -0,0 +1,172 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from email import encoders +from email.mime.nonmultipart import MIMENonMultipart +from email.mime.multipart import MIMEMultipart +from leap.bitmask.mail.mail import Message + +from twisted.internet import defer + +from pixelated.adapter.model.mail import InputMail +from pixelated.adapter.model.status import Status +from pixelated.adapter.services.tag_service import extract_reserved_tags +from leap.bitmask.mail.adaptors.soledad import SoledadMailAdaptor + + +class MailService(object): + + def __init__(self, mail_sender, mail_store, search_engine, account_email, attachment_store): + self.mail_store = mail_store + self.search_engine = search_engine + self.mail_sender = mail_sender + self.account_email = account_email + self.attachment_store = attachment_store + + @defer.inlineCallbacks + def all_mails(self): + mails = yield self.mail_store.all_mails(gracefully_ignore_errors=True) + defer.returnValue(mails) + + def save_attachment(self, content, content_type): + return self.attachment_store.add_attachment(content, content_type) + + @defer.inlineCallbacks + def mails(self, query, window_size, page): + mail_ids, total = self.search_engine.search(query, window_size, page) + + try: + mails = yield self.mail_store.get_mails(mail_ids) + defer.returnValue((mails, total)) + except Exception, e: + import traceback + traceback.print_exc() + raise + + @defer.inlineCallbacks + def update_tags(self, mail_id, new_tags): + new_tags = self._filter_white_space_tags(new_tags) + reserved_words = extract_reserved_tags(new_tags) + if len(reserved_words): + raise ValueError('None of the following words can be used as tags: ' + ' '.join(reserved_words)) + new_tags = self._favor_existing_tags_casing(new_tags) + mail = yield self.mail(mail_id) + mail.tags = set(new_tags) + yield self.mail_store.update_mail(mail) + + defer.returnValue(mail) + + def _filter_white_space_tags(self, tags): + return [tag.strip() for tag in tags if not tag.isspace()] + + def _favor_existing_tags_casing(self, new_tags): + current_tags = [tag['name'] for tag in self.search_engine.tags(query='', skip_default_tags=True)] + current_tags_lower = [tag.lower() for tag in current_tags] + + def _use_current_casing(new_tag_lower): + return current_tags[current_tags_lower.index(new_tag_lower)] + + return [_use_current_casing(new_tag.lower()) if new_tag.lower() in current_tags_lower else new_tag for new_tag in new_tags] + + def mail(self, mail_id): + return self.mail_store.get_mail(mail_id, include_body=True) + + def attachment(self, attachment_id): + return self.attachment_store.get_mail_attachment(attachment_id) + + @defer.inlineCallbacks + def mail_exists(self, mail_id): + try: + mail = yield self.mail_store.get_mail(mail_id, include_body=False) + defer.returnValue(mail is not None) + except Exception, e: + defer.returnValue(False) + + @defer.inlineCallbacks + def send_mail(self, content_dict): + mail = InputMail.from_dict(content_dict, self.account_email) + draft_id = content_dict.get('ident') + self._deduplicate_recipients(mail) + yield self.mail_sender.sendmail(mail) + + sent_mail = yield self.move_to_sent(draft_id, mail) + defer.returnValue(sent_mail) + + def _deduplicate_recipients(self, mail): + self._remove_canonical(mail) + self._remove_duplicates_form_cc_and_to(mail) + + def _remove_canonical(self, mail): + mail.headers['To'] = map(self._remove_canonical_recipient, mail.to) + mail.headers['Cc'] = map(self._remove_canonical_recipient, mail.cc) + mail.headers['Bcc'] = map(self._remove_canonical_recipient, mail.bcc) + + def _remove_duplicates_form_cc_and_to(self, mail): + mail.headers['To'] = list(set(self._remove_duplicates(mail.to)).difference(set(mail.bcc))) + mail.headers['Cc'] = list((set(self._remove_duplicates(mail.cc)).difference(set(mail.bcc)).difference(set(mail.to)))) + mail.headers['Bcc'] = self._remove_duplicates(mail.bcc) + + def _remove_duplicates(self, recipient): + return list(set(recipient)) + + # TODO removing canocical should, be added back later + def _remove_canonical_recipient(self, recipient): + return recipient.split('<')[1][0:-1] if '<' in recipient else recipient + + @defer.inlineCallbacks + def move_to_sent(self, last_draft_ident, mail): + if last_draft_ident: + try: + yield self.mail_store.delete_mail(last_draft_ident) + except Exception as error: + pass + sent_mail = yield self.mail_store.add_mail('SENT', mail.raw) + sent_mail.flags.add(Status.SEEN) + yield self.mail_store.update_mail(sent_mail) + defer.returnValue(sent_mail) + + @defer.inlineCallbacks + def mark_as_read(self, mail_id): + mail = yield self.mail(mail_id) + mail.flags.add(Status.SEEN) + yield self.mail_store.update_mail(mail) + + @defer.inlineCallbacks + def mark_as_unread(self, mail_id): + mail = yield self.mail(mail_id) + mail.flags.remove(Status.SEEN) + yield self.mail_store.update_mail(mail) + + @defer.inlineCallbacks + def delete_mail(self, mail_id): + mail = yield self.mail(mail_id) + if mail is not None: + if mail.mailbox_name.upper() in (u'TRASH', u'DRAFTS'): + yield self.mail_store.delete_mail(mail_id) + else: + yield self.mail_store.move_mail_to_mailbox(mail_id, 'TRASH') + + @defer.inlineCallbacks + def recover_mail(self, mail_id): + yield self.mail_store.move_mail_to_mailbox(mail_id, 'INBOX') + + @defer.inlineCallbacks + def archive_mail(self, mail_id): + yield self.mail_store.add_mailbox('ARCHIVE') + yield self.mail_store.move_mail_to_mailbox(mail_id, 'ARCHIVE') + + @defer.inlineCallbacks + def delete_permanent(self, mail_id): + yield self.mail_store.delete_mail(mail_id) diff --git a/service/src/pixelated/adapter/services/tag_service.py b/service/src/pixelated/adapter/services/tag_service.py new file mode 100644 index 00000000..c51da625 --- /dev/null +++ b/service/src/pixelated/adapter/services/tag_service.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from pixelated.adapter.model.tag import Tag + +SPECIAL_TAGS = {Tag('inbox', True), Tag('sent', True), Tag('drafts', True), Tag('trash', True), Tag('ALL', True)} + + +def extract_reserved_tags(tags): + tags = [tag.lower() for tag in tags] + return {tag.name for tag in SPECIAL_TAGS if tag.name in tags} diff --git a/service/src/pixelated/adapter/welcome_mail.py b/service/src/pixelated/adapter/welcome_mail.py new file mode 100644 index 00000000..8d3cdd7a --- /dev/null +++ b/service/src/pixelated/adapter/welcome_mail.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import pkg_resources +from email import message_from_file +from pixelated.adapter.model.mail import InputMail + + +def add_welcome_mail(mail_store, language='en-US'): + welcome_mail = pkg_resources.resource_filename( + 'pixelated.assets', + 'welcome.mail.%s' % (language)) + + with open(welcome_mail) as mail_template_file: + mail_template = message_from_file(mail_template_file) + + input_mail = InputMail.from_python_mail(mail_template) + mail_store.add_mail('INBOX', input_mail.raw) diff --git a/service/src/pixelated/application.py b/service/src/pixelated/application.py new file mode 100644 index 00000000..ef99d47c --- /dev/null +++ b/service/src/pixelated/application.py @@ -0,0 +1,219 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import os + +from OpenSSL import SSL +from OpenSSL import crypto +from leap.common.events import (server as events_server, + register, catalog as events) +from leap.soledad.common.errors import InvalidAuthTokenError +from twisted.logger import Logger +from twisted.conch import manhole_tap +from twisted.cred import portal +from twisted.cred.checkers import AllowAnonymousAccess +from twisted.internet import defer +from twisted.internet import reactor +from twisted.internet import ssl + +from pixelated.adapter.welcome_mail import add_welcome_mail +from pixelated.authentication import Authenticator +from pixelated.config import arguments +from pixelated.config import logger +from pixelated.config import services +from pixelated.config.leap import initialize_leap_single_user, init_monkeypatches, initialize_leap_provider +from pixelated.config.services import ServicesFactory, SingleUserServicesFactory +from pixelated.config.site import PixelatedSite +from pixelated.resources.auth import PixelatedRealm, PixelatedAuthSessionWrapper, SessionChecker +from pixelated.resources.login_resource import LoginResource +from pixelated.resources.root_resource import RootResource + +log = Logger() + + +class UserAgentMode(object): + def __init__(self, is_single_user): + self.is_single_user = is_single_user + + +@defer.inlineCallbacks +def start_user_agent_in_single_user_mode(root_resource, services_factory, leap_home, leap_session): + log.info('Bootstrap done, loading services for user %s' % leap_session.user_auth.username) + + _services = services.Services(leap_session) + yield _services.setup() + + if leap_session.fresh_account: + yield add_welcome_mail(leap_session.mail_store) + + services_factory.add_session(leap_session.user_auth.uuid, _services) + + authenticator = Authenticator(leap_session.provider) + root_resource.initialize(provider=leap_session.provider, authenticator=authenticator) + + # soledad needs lots of threads + reactor.getThreadPool().adjustPoolsize(5, 15) + log.info('Done, the user agent is ready to be used') + + +def _ssl_options(sslkey, sslcert): + with open(sslkey) as keyfile: + pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, keyfile.read()) + with open(sslcert) as certfile: + cert = crypto.load_certificate(crypto.FILETYPE_PEM, certfile.read()) + + acceptable = ssl.AcceptableCiphers.fromOpenSSLCipherString( + u'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:!RC4:HIGH:!MD5:!aNULL:!EDH') + options = ssl.CertificateOptions(privateKey=pkey, + certificate=cert, + method=SSL.TLSv1_2_METHOD, + acceptableCiphers=acceptable) + return options + + +def _create_service_factory(args): + if args.single_user: + return SingleUserServicesFactory(UserAgentMode(is_single_user=True)) + else: + return ServicesFactory(UserAgentMode(is_single_user=False)) + + +def initialize(): + log.info('Starting the Pixelated user agent') + args = arguments.parse_user_agent_args() + logger.init(debug=args.debug) + services_factory = _create_service_factory(args) + resource = RootResource(services_factory) + + def start(): + start_async = _start_mode(args, resource, services_factory) + add_top_level_system_callbacks(start_async, services_factory) + + log.info('Running the reactor') + reactor.callWhenRunning(start) + reactor.run() + + +def add_top_level_system_callbacks(deferred, services_factory): + + def _quit_on_error(failure): + failure.printTraceback() + reactor.stop() + + def _log_user_out(event, user_data): + log.info('Invalid soledad token, logging out %s' % user_data) + user_data = {'user_id': user_data['uuid']} if 'uuid' in user_data else {'user_id': user_data, 'using_email': True} + services_factory.destroy_session(**user_data) + + def _log_user_out_on_token_expire(leap_session): + register(events.SOLEDAD_INVALID_AUTH_TOKEN, _log_user_out) + return leap_session + + deferred.addCallback(_log_user_out_on_token_expire) + deferred.addErrback(_quit_on_error) + + +def _start_mode(args, resource, services_factory): + if services_factory.mode.is_single_user: + deferred = _start_in_single_user_mode(args, resource, services_factory) + else: + deferred = _start_in_multi_user_mode(args, resource, services_factory) + return deferred + + +def _start_in_multi_user_mode(args, root_resource, services_factory): + try: + protected_resources = _setup_multi_user(args, root_resource, services_factory) + start_site(args, protected_resources) + reactor.getThreadPool().adjustPoolsize(5, 15) + return defer.succeed(None) + except Exception as e: + return defer.fail(e) + + +def _setup_multi_user(args, root_resource, services_factory): + if args.provider is None: + raise ValueError('Multi-user mode: provider name is required') + init_monkeypatches() + events_server.ensure_server() + provider = initialize_leap_provider(args.provider, args.leap_provider_cert, args.leap_provider_cert_fingerprint, args.leap_home) + protected_resource = set_up_protected_resources(root_resource, provider, services_factory, banner=args.banner) + return protected_resource + + +def set_up_protected_resources(root_resource, provider, services_factory, banner=None, authenticator=None): + auth = authenticator or Authenticator(provider) + session_checker = SessionChecker(services_factory) + + realm = PixelatedRealm() + _portal = portal.Portal(realm, [session_checker, AllowAnonymousAccess()]) + + anonymous_resource = LoginResource(services_factory, provider, disclaimer_banner=banner, authenticator=auth) + protected_resource = PixelatedAuthSessionWrapper(_portal, root_resource, anonymous_resource, []) + root_resource.initialize(provider, disclaimer_banner=banner, authenticator=auth) + return protected_resource + + +def _start_in_single_user_mode(args, resource, services_factory): + start_site(args, resource) + deferred = initialize_leap_single_user(args.leap_provider_cert, + args.leap_provider_cert_fingerprint, + args.credentials_file, + args.leap_home) + + def _handle_error(exception): + if(exception.type is InvalidAuthTokenError): + log.critical('Got an invalid soledad token, the user agent can\'t synchronize data, exiting') + os._exit(1) + else: + exception.raiseException() + + deferred.addCallbacks( + lambda leap_session: start_user_agent_in_single_user_mode( + resource, + services_factory, + args.leap_home, + leap_session), _handle_error) + return deferred + + +def start_site(config, resource): + log.info('Starting the API on port %s' % config.port) + + if config.manhole: + log.info('Starting the manhole on port 8008') + + multiService = manhole_tap.makeService(dict(namespace=globals(), + telnetPort='8008', + sshPort='8009', + sshKeyDir='sshKeyDir', + sshKeyName='id_rsa', + sshKeySize=4096, + passwd='passwd')) + telnetService, sshService = multiService.services + telnetFactory = telnetService.factory + sshFactory = sshService.factory + + reactor.listenTCP(8008, telnetFactory, interface='localhost') + reactor.listenTCP(8009, sshFactory, interface='localhost') + + site = PixelatedSite(resource) + site.displayTracebacks = False + if config.sslkey and config.sslcert: + reactor.listenSSL(config.port, site, _ssl_options(config.sslkey, config.sslcert), + interface=config.host) + else: + reactor.listenTCP(config.port, site, interface=config.host) diff --git a/service/src/pixelated/assets/__init__.py b/service/src/pixelated/assets/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/service/src/pixelated/assets/__init__.py diff --git a/service/src/pixelated/assets/recovery.mail.en-US b/service/src/pixelated/assets/recovery.mail.en-US new file mode 100644 index 00000000..e7a09f1b --- /dev/null +++ b/service/src/pixelated/assets/recovery.mail.en-US @@ -0,0 +1,28 @@ +From: {sender} +Date: {date} +Subject: Recovery Code for {domain} +To: {backup_email} +Content-Type: text/plain; charset=UTF-8 + +Hello, + +You are receiving this message because you registered an email account at https://{domain}. +If you ever forget your password, you'll need the code below to recover it. Save it! It is the only way to access to your account again. + +{recovery_code} + +Save this message or write this code in a safe place. + +-- +Why is this so important? + +Pixelated is an email client that respects your privacy and uses PGP Encryption to do so. Your password also gives you access to your keys, so if you forget it you will lose access to your account and the ability to read your messages. +Forgetting passwords is a common thing, so we developed a more secure way to recover access to your account. + +1) This code is half of a big code to recover your account. +2) The other half is with the account administrator. +3) In case you forget your password, use this code and your administrator code to recover access to your account. It's like those locks with two keys :) + + + +PS: If you didn't create an account at https://{domain}, please ignore this email. diff --git a/service/src/pixelated/assets/recovery.mail.pt-BR b/service/src/pixelated/assets/recovery.mail.pt-BR new file mode 100644 index 00000000..558c6905 --- /dev/null +++ b/service/src/pixelated/assets/recovery.mail.pt-BR @@ -0,0 +1,26 @@ +From: {sender} +Date: {date} +Subject: Codigo de Recuperacao de {domain} +To: {backup_email} +Content-Type: text/plain; charset=UTF-8 + +Olá, + +Você está recebendo isso porque você registrou um email no https://{domain}. +Guarde o código abaixo para se um dia esquecer sua senha. Ele é a única forma de recuperar uma senha para acessar sua conta outra vez: + +{recovery_code} + +Salve essa mensagem ou anote o código em um lugar seguro. + +-- +Por que isso é importante? + +O Pixelated é um cliente de email que respeita sua privacidade e usa criptografia PGP para isso. Sua senha também dá acesso às suas chaves, então se você esquecê-la você perderá acesso a sua conta e a habilidade de ler suas mensagens. Esquecer a senha é algo comum, por isso desenvolvemos uma forma mais segura de recuperar sua conta. + +1) Esse código é uma metade de um código necessário para recuperar a conta. +2) A outra metade está com o administrador da conta. +3) Se você esquecer a senha, use esse código e o do administrador para recuperar acesso a conta. É como se fosse um cadeado com duas chaves :) + + +PS: Se você não criou uma conta no site https://{domain}, por favor ignore esse email. diff --git a/service/src/pixelated/assets/welcome.mail.en-US b/service/src/pixelated/assets/welcome.mail.en-US new file mode 100644 index 00000000..34d30812 --- /dev/null +++ b/service/src/pixelated/assets/welcome.mail.en-US @@ -0,0 +1,94 @@ +From: Pixelated Team <pixelated-team@pixelated-project.org> +Date: Sat, 21 Mar 2015 19:30:09 -0300 +Subject: Welcome to Pixelated Mail +To: Replace <will.be@the.user> +Content-Type: multipart/alternative; boundary=000boundary000 + +--000boundary000 +Content-Type: text/plain; charset=UTF-8 + +Welcome to Pixelated Mail, modern email with encryption. + +Pixelated Mail is an open source project that aims to provide secure email in the browser. + + + +How to use it + +Pixelated Mail works as any other email client. On the left, you will find a navigation bar where you can find your mailboxes and tags. Clicking them will load the corresponding messages in the middle area - the mail list. Clicking on a message will show it on the right side, but you already know that, right? + +To compose a message look for the big blue button on the top left. You can add tags by clicking on the "+" sign below the message subject. You can find the encryption status of a message just above the sender information. + + + +About this message and encryption status + +This message wasn't encrypted. Like any email, it could have been read by others before reaching your provider. + +There are two ways to send encrypted messages from Pixelated: + +• when you send an email to an account on your provider, the message is encrypted and signed by default; +• when you send an email to another account, Pixelated will still try to encrypt the message, and if it succeeds you will see a closed padlock next to the email address; if the padlock is open, the message will be sent normally as any other e-mail client. Pixelated does all of this automatically. :D + + + +A bit more about Pixelated + +Pixelated is an open source project licensed under AGPL 3.0. It is composed of two main parts, the web client (that you are using now) and the provider (which will send and receive your emails, the server behind the "@" sign on your new mail address). You can learn more by visiting https://github.com/pixelated. + +Enjoy your secure email! + +--000boundary000 +Content-Type: text/html; charset=UTF-8 +Content-Transfer-Encoding: quoted-printable + +<h1>Welcome to Pixelated Mail, modern email with encryption.</h1> + +<p>Pixelated Mail is an open source project that aims to provide secure ema= +il in the browser.</p> + +<h2>How to use it<h2> + +<p> +Pixelated Mail works as any other email client. On the left, you will find = +a navigation bar where you can find your mailboxes and tags. Clicking them = +will load the corresponding messages in the middle area - the mail list. Cl= +icking on a message will show it on the right side, but you already know th= +at, right? +</p> +<p> +To compose a message look for the big blue button on the top left. You can = +add tags by clicking on the "+" sign below the message subject. You can fin= +d the encryption status of a message just above the sender information. +</p> + +<h2>About this message and encryption status</h2> + +<p> +This message wasn't encrypted. Like any email, it could have been read by o= +thers before reaching your provider. +</p> +<p> +There are two ways to send encrypted messages from Pixelated: +</p> + +<ul> +<li>when you send an email to an account on your provider, the message is e= +ncrypted and signed by default;</li> +<li>when you send an email to another account, Pixelated will still try to = +encrypt the message, and if it succeeds you will see a closed padlock next = +to the email address; if the padlock is open, the message will be sent norm= +ally as any other e-mail client. Pixelated does all of this automatically. = +:D</li> +</ul> + +<h2>A bit more about Pixelated</h2> + +<p>Pixelated is an open source project licensed under AGPL 3.0. It is compo= +sed of two main parts, the web client (that you are using now) and the prov= +ider (which will send and receive your emails, the server behind the @ sign= + on your new mail address). You can learn more by visiting https://github.c= +om/pixelated.</p> + +<p>Enjoy your secure email!</p> +--000boundary000-- diff --git a/service/src/pixelated/assets/welcome.mail.pt-BR b/service/src/pixelated/assets/welcome.mail.pt-BR new file mode 100644 index 00000000..e57405d2 --- /dev/null +++ b/service/src/pixelated/assets/welcome.mail.pt-BR @@ -0,0 +1,102 @@ +From: Pixelated Team <pixelated-team@pixelated-project.org> +Date: Sat, 21 Mar 2015 19:30:09 -0300 +Subject: Bem vindo ao Pixelated +To: Replace <will.be@the.user> +Content-Type: multipart/alternative; boundary=000boundary000 + +--000boundary000 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: base64 + +QmVtIHZpbmRvIGFvIFBpeGVsYXRlZCBNYWlsLCB1bSBlbWFpbCBtb2Rlcm5vIGNvbSBjcmlwdG9ncm +FmaWEuCgpQaXhlbGF0ZWQgTWFpbCDDqSB1bSBwcm9qZXRvIGRlIGPDs2RpZ28tYWJlcnRvIHF1ZSBk +ZXNlbnZvbHZlIHVtIGVtYWlsIHNlZ3VybyBwYXJhIG8gc2V1IG5hdmVnYWRvci4KCgoKQ29tbyB1c2 +FyCgpPIFBpeGVsYXRlZCBNYWlsIGZ1bmNpb25hIGNvbW8gcXVhbHF1ZXIgb3V0cm8gY2xpZW50ZSBk +ZSBlbWFpbC4gw4AgZXNxdWVyZGEsIHZvY8OqIHZhaSBlbmNvbnRyYXIgYSBiYXJyYSBkZSBuYXZlZ2 +HDp8OjbyBjb20gYWNlc3NvIGEgdG9kYXMgYXMgc3VhcyBwYXN0YXMgZSBtYXJjYWRvcmVzLiBDbGlj +YW5kbyBuZWxhcyB2b2PDqiBjYXJyZWdhcsOhIGFzIG1lbnNhZ2VucyBjb3JyZXNwb25kZW50ZXMgbm +8gcGFpbmVsIGRvIG1laW8gLSBhIGxpc3RhIGRlIGVtYWlscy4gQ2xpY2FuZG8gZW0gdW0gZW1haWwg +ZWxlIHNlcsOhIGFiZXJ0byBuZXN0ZSBwYWluZWwgYXF1aSwgbWFzIHZvY8OqIGrDoSBzYWJlIGRpc3 +NvLCBuw6k/CgpQYXJhIGNyaWFyIHVtYSBub3ZhIG1lbnNhZ2VtIHByb2N1cmUgcG9yIHVtIGdyYW5k +ZSBib3TDo28gYXp1bCBubyBjYW50byBzdXBlcmlvciBlc3F1ZXJkby4gVm9jw6ogcG9kZSBhZGljaW +9uYXIgbWFyY2Fkb3JlcyBjbGljYW5kbyBubyBzaW5hbCBkZSAiKyIgYWJhaXhvIGRvIGFzc3VudG8g +ZGEgbWVuc2FnZW0uIFZvY8OqIHZpc3VhbGl6YSBvIGVzdGFkbyBkZSBjcmlwdG9ncmFmaWEgZGFzIG +1lbnNhZ2VucyBsb2dvIGFiYWl4byBkYXMgaW5mb3JtYcOnw7VlcyBkbyByZW1ldGVudGUuCgoKClNv +YnJlIGVzdGEgbWVuc2FnZW0gZSBzZXUgZXN0YWRvIGRlIGNyaXB0b2dyYWZpYQoKRXN0YSBtZW5zYW +dlbSBuw6NvIGZvaSBjcmlwdG9ncmFmYWRhLiBFbSBvdXRyYXMgcGFsYXZyYXMsIGVsYSBwb2Rlcmlh +IHRlciBzaWRvIGxpZGEgcG9yIG91dHJhcyBwZXNzb2FzIGVtIGFsZ3VtIHBvbnRvIGR1cmFudGUgYS +B0cmFuc21pc3PDo28sIGNvbW8gZW0gcXVhbHF1ZXIgb3V0cm8gY2xpZW50ZSBkZSBlbWFpbC4KClBh +cmEgZW52aWFyIG1lbnNhZ2VucyBjcmlwdG9ncmFmYWRhcyBubyBQaXhlbGF0ZWQsIGV4aXN0ZW0gMi +Bwb3NzaWJpbGlkYWRlczoKCuKAoiBlbnZpYXIgdW0gZW1haWwgcGFyYSBvdXRyYSBjb250YSBkbyBQ +aXhlbGF0ZWQ6IGEgbWVuc2FnZW0gw6kgZW52aWFkYSBjcmlwdG9ncmFmYWRhIGUgYXNzaW5hZGEgcG +9yIHBhZHLDo287CuKAoiBlbnZpYXIgdW0gZW1haWwgcGFyYSBvdXRyYXMgY29udGFzOiBhbyBjcmlh +ciB1bWEgbWVuc2FnZW0sIGRpZ2l0ZSBvIGVuZGVyZcOnbyBkZSBlbWFpbCBkbyBkZXN0aW5hdMOhcm +lvLiBTZSBqdW50byBhbyBlbmRlcmXDp28gYXBhcmVjZXIgdW0gY2FkZWFkbyBmZWNoYWRvLCBvIGVt +YWlsIGVzdGFyw6EgY3JpcHRvZ3JhZmFkbzsgc2UgZXN0aXZlciBhYmVydG8sIGEgbWVuc2FnZW0gdm +FpIG5vcm1hbG1lbnRlIGNvbW8gcXVhbHF1ZXIgb3V0cm8gZW1haWwuIE8gUGl4ZWxhdGVkIGZheiB0 +dWRvIGF1dG9tYXRpY2FtZW50ZS4gOkQKCgoKVW0gcG91Y28gbWFpcyBzb2JyZSBvIFBpeGVsYXRlZA +oKUGl4ZWxhdGVkIMOpIHVtIHByb2pldG8gZGUgY8OzZGlnby1hYmVydG8gbGljZW5jaWFkbyBzb2Ig +QUdQTCAzLjAuIMOJIGNvbXBvc3RvIGRlIGR1YXMgcGFydGVzIHByaW5jaXBhaXMsIG8gY2xpZW50ZS +B3ZWIgKHF1ZSB2b2PDqiBlc3TDoSB1c2FuZG8gYWdvcmEpIGUgYSBwbGF0YWZvcm1hIChvIHNlcnZp +ZG9yIHBvciB0csOhcyBkbyBzaW5hbCBkZSAiQCIsIHF1ZSBwcm92w6ogbyBzZXJ2acOnbyBkZSBlbW +FpbCBxdWUgdm9jw6ogdXNhcsOhIHBhcmEgbWFuZGFyIGUgcmVjZWJlciBtZW5zYWdlbnMpLiBWb2PD +qiBwb2RlIGFwcmVuZGVyIG1haXMgdmlzaXRhbmRvIGh0dHBzOi8vZ2l0aHViLmNvbS9waXhlbGF0ZW +QuCgpBcHJvdmVpdGUgc2V1IGVtYWlsIHNlZ3VybyEK= +--000boundary000 +Content-Type: text/html; charset=UTF-8 +Content-Transfer-Encoding: quoted-printable + +<h1>Bem vindo ao Pixelated Mail, um email moderno com criptografia.</h1> + +<p>Pixelated Mail =C3=A9 um projeto de c=C3=B3digo-aberto que desenvolve um em= +ail seguro para o seu navegador.</p> + +<h2>Como usar</h2> + +<p> +O Pixelated Mail funciona como qualquer outro cliente de email. =C3=80 esqu= +erda, voc=C3=AA vai encontrar a barra de navega=C3=A7=C3=A3o com acesso a t= +odas as suas pastas e marcadores. Clicando nelas voc=C3=AA carregar=C3=A1 a= +s mensagens correspondentes no painel do meio - a lista de emails. Clicando= + em um email ele ser=C3=A1 aberto neste painel aqui, mas voc=C3=AA j=C3=A1 = +sabe disso, n=C3=A9? +</p> +<p> +Para criar uma nova mensagem procure por um grande bot=C3=A3o azul no canto= + superior esquerdo. Voc=C3=AA pode adicionar marcadores clicando no sinal d= +e "+" abaixo do assunto da mensagem. Voc=C3=AA visualiza o estado de cripto= +grafia das mensagens logo abaixo das informa=C3=A7=C3=B5es do remetente. +</p> + +<h2>Sobre esta mensagem e seu estado de criptografia</h2> + +<p> +Esta mensagem n=C3=A3o foi criptografada. Em outras palavras, ela poderia t= +er sido lida por outras pessoas em algum ponto durante a transmiss=C3=A3o, = +como em qualquer outro cliente de email. +</p> +<p> +Para enviar mensagens criptografadas no Pixelated, existem 2 possibilidades: +</p> + +<ul> +<li>enviar um email para outra conta do Pixelated: a mensagem =C3=A9 = +enviada criptografada e assinada por padr=C3=A3o;</li> +<li>enviar um email para outras contas: ao criar uma mensagem, digite= + o endere=C3=A7o de email do destinat=C3=A1rio. Se junto ao endere=C3=A7o a= +parecer um cadeado fechado, o email estar=C3=A1 criptografado; se estiver a= +berto, a mensagem vai normalmente como qualquer outro email. O Pixelated fa= +z tudo automaticamente. :D</li> +</ul> + +<h2>Um pouco mais sobre o Pixelated</h2> + +<p>Pixelated =C3=A9 um projeto de c=C3=B3digo-aberto licenciado sob AGPL 3.0. = +=C3=89 composto de duas partes principais, o cliente web (que voc=C3=AA est= +=C3=A1 usando agora) e a plataforma (o servidor por tr=C3=A1s do sinal de "= +@", que prov=C3=AA o servi=C3=A7o de email que voc=C3=AA usar=C3=A1 para ma= +ndar e receber mensagens). Voc=C3=AA pode aprender mais visitando https://g= +ithub.com/pixelated.</p> + +<p>Aproveite seu email seguro!</p> +--000boundary000-- diff --git a/service/src/pixelated/assets/welcome.mail.pt-BR.txt b/service/src/pixelated/assets/welcome.mail.pt-BR.txt new file mode 100644 index 00000000..810d31a6 --- /dev/null +++ b/service/src/pixelated/assets/welcome.mail.pt-BR.txt @@ -0,0 +1,30 @@ +Bem vindo ao Pixelated Mail, um email moderno com criptografia. + +Pixelated Mail é um projeto de código-aberto que desenvolve um email seguro para o seu navegador. + + + +Como usar + +O Pixelated Mail funciona como qualquer outro cliente de email. À esquerda, você vai encontrar a barra de navegação com acesso a todas as suas pastas e marcadores. Clicando nelas você carregará as mensagens correspondentes no painel do meio - a lista de emails. Clicando em um email ele será aberto neste painel aqui, mas você já sabe disso, né? + +Para criar uma nova mensagem procure por um grande botão azul no canto superior esquerdo. Você pode adicionar marcadores clicando no sinal de "+" abaixo do assunto da mensagem. Você visualiza o estado de criptografia das mensagens logo abaixo das informações do remetente. + + + +Sobre esta mensagem e seu estado de criptografia + +Esta mensagem não foi criptografada. Em outras palavras, ela poderia ter sido lida por outras pessoas em algum ponto durante a transmissão, como em qualquer outro cliente de email. + +Para enviar mensagens criptografadas no Pixelated, existem 2 possibilidades: + +• enviar um email para outra conta do Pixelated: a mensagem é enviada criptografada e assinada por padrão; +• enviar um email para outras contas: ao criar uma mensagem, digite o endereço de email do destinatário. Se junto ao endereço aparecer um cadeado fechado, o email estará criptografado; se estiver aberto, a mensagem vai normalmente como qualquer outro email. O Pixelated faz tudo automaticamente. :D + + + +Um pouco mais sobre o Pixelated + +Pixelated é um projeto de código-aberto licenciado sob AGPL 3.0. É composto de duas partes principais, o cliente web (que você está usando agora) e a plataforma (o servidor por trás do sinal de "@", que provê o serviço de email que você usará para mandar e receber mensagens). Você pode aprender mais visitando https://github.com/pixelated. + +Aproveite seu email seguro! diff --git a/service/src/pixelated/authentication.py b/service/src/pixelated/authentication.py new file mode 100644 index 00000000..b5edbec0 --- /dev/null +++ b/service/src/pixelated/authentication.py @@ -0,0 +1,85 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import re +from collections import namedtuple + +from leap.bitmask.bonafide.provider import Api +from leap.bitmask.bonafide.session import Session +from leap.bitmask.bonafide._srp import SRPAuthError + +from twisted.cred.error import UnauthorizedLogin +from twisted.internet.defer import inlineCallbacks, returnValue + +Credentials = namedtuple('Credentials', 'username, password') + + +class Authenticator(object): + def __init__(self, leap_provider): + self._leap_provider = leap_provider + self.domain = leap_provider.server_name + self.bonafide_session = None + + @inlineCallbacks + def authenticate(self, username, password): + username = self.clean_username(username) + auth = yield self._srp_auth(username, password) + returnValue(auth) + + @inlineCallbacks + def _srp_auth(self, username, password): + try: + auth = yield self._bonafide_auth(username, password) + except SRPAuthError: + raise UnauthorizedLogin("User typed wrong password/username combination.") + returnValue(auth) + + @inlineCallbacks + def _bonafide_auth(self, user, password): + srp_provider = Api(self._leap_provider.api_uri) + credentials = Credentials(user, password) + self.bonafide_session = Session(credentials, srp_provider, self._leap_provider.local_ca_crt) + yield self.bonafide_session.authenticate() + returnValue(Authentication(user, + self.bonafide_session.token, + self.bonafide_session.uuid, + 'session_id', + {'is_admin': False})) + + def clean_username(self, username): + if '@' not in username: + return username + extracted_username = self.extract_username(username) + if self.username_with_domain(extracted_username) == username: + return extracted_username + raise UnauthorizedLogin('User typed a wrong domain.') + + def extract_username(self, username): + return re.search('^([^@]+)@?.*$', username).group(1) + + def username_with_domain(self, username): + return '%s@%s' % (username, self.domain) + + +class Authentication(object): + def __init__(self, username, token, uuid, session_id, user_attributes): + self.username = username + self.token = token + self.uuid = uuid + self.session_id = session_id + self._user_attributes = user_attributes + + def is_admin(self): + return self._user_attributes.get('is_admin', False) diff --git a/service/src/pixelated/bitmask_libraries/__init__.py b/service/src/pixelated/bitmask_libraries/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/service/src/pixelated/bitmask_libraries/__init__.py diff --git a/service/src/pixelated/bitmask_libraries/certs.py b/service/src/pixelated/bitmask_libraries/certs.py new file mode 100644 index 00000000..9a76a01d --- /dev/null +++ b/service/src/pixelated/bitmask_libraries/certs.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import os + +from pixelated.config import leap_config + + +class LeapCertificate(object): + + LEAP_CERT = None + LEAP_FINGERPRINT = None + + def __init__(self, provider): + self._server_name = provider.server_name + self._provider = provider + + @staticmethod + def set_cert_and_fingerprint(cert_file=None, cert_fingerprint=None): + if cert_fingerprint is None: + LeapCertificate.LEAP_CERT = str(cert_file) if cert_file else True + LeapCertificate.LEAP_FINGERPRINT = None + else: + LeapCertificate.LEAP_FINGERPRINT = cert_fingerprint + LeapCertificate.LEAP_CERT = False + + @property + def provider_web_cert(self): + return self.LEAP_CERT diff --git a/service/src/pixelated/bitmask_libraries/keymanager.py b/service/src/pixelated/bitmask_libraries/keymanager.py new file mode 100644 index 00000000..9a1b730e --- /dev/null +++ b/service/src/pixelated/bitmask_libraries/keymanager.py @@ -0,0 +1,111 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from twisted.internet import defer +from twisted.logger import Logger + +from leap.bitmask.keymanager import KeyManager, KeyNotFound + +from pixelated.config import leap_config + +logger = Logger() + + +class UploadKeyError(Exception): + pass + + +TWO_MONTHS = 60 +DEFAULT_EXTENSION_THRESHOLD = TWO_MONTHS + + +class Keymanager(object): + + def __init__(self, provider, soledad, email_address, token, uuid): + nicknym_url = provider._discover_nicknym_server() + self._email = email_address + self.keymanager = KeyManager(self._email, nicknym_url, + soledad, + token=token, ca_cert_path=provider.provider_api_cert, api_uri=provider.api_uri, + api_version=provider.api_version, + uid=uuid, gpgbinary=leap_config.gpg_binary, + combined_ca_bundle=provider.combined_cerfificates_path) + + @defer.inlineCallbacks + def generate_openpgp_key(self): + current_key = yield self._key_exists(self._email) + if not current_key: + current_key = yield self._generate_key_and_send_to_leap() + elif current_key.needs_renewal(DEFAULT_EXTENSION_THRESHOLD): + current_key = yield self._regenerate_key_and_send_to_leap() + + self._synchronize_remote_key(current_key) + logger.debug("Current key for {}: {}".format(self._email, current_key.fingerprint)) + + @defer.inlineCallbacks + def _synchronize_remote_key(self, current_key): + if not self._is_key_synchronized_with_server(current_key): + try: + yield self.keymanager.send_key() + except Exception as e: + raise UploadKeyError(e.message) + + @defer.inlineCallbacks + def _is_key_synchronized_with_server(self, current_key): + remote_key = yield self.get_key(self._email, private=False, fetch_remote=True) + defer.returnValue(remote_key.fingerprint == current_key.fingerprint) + + @defer.inlineCallbacks + def _regenerate_key_and_send_to_leap(self): + logger.info("Regenerating keys - this could take a while...") + key = yield self.keymanager.regenerate_key() + try: + yield self.keymanager.send_key() + defer.returnValue(key) + except Exception as e: + raise UploadKeyError(e.message) + + @defer.inlineCallbacks + def _generate_key_and_send_to_leap(self): + logger.info("Generating keys - this could take a while...") + key = yield self.keymanager.gen_key() + try: + yield self.keymanager.send_key() + defer.returnValue(key) + except Exception as e: + yield self.delete_key_pair() + raise UploadKeyError(e.message) + + @defer.inlineCallbacks + def _key_exists(self, email): + try: + current_key = yield self.get_key(email, private=True, fetch_remote=False) + defer.returnValue(current_key) + except KeyNotFound: + defer.returnValue(None) + + @defer.inlineCallbacks + def get_key(self, email, private=False, fetch_remote=True): + key = yield self.keymanager.get_key(email, private=private, fetch_remote=fetch_remote) + defer.returnValue(key) + + @defer.inlineCallbacks + def delete_key_pair(self): + private_key = yield self.get_key(self._email, private=True, fetch_remote=False) + public_key = yield self.get_key(self._email, private=False, fetch_remote=False) + + self.keymanager.delete_key(private_key) + self.keymanager.delete_key(public_key) diff --git a/service/src/pixelated/bitmask_libraries/provider.py b/service/src/pixelated/bitmask_libraries/provider.py new file mode 100644 index 00000000..96935fbc --- /dev/null +++ b/service/src/pixelated/bitmask_libraries/provider.py @@ -0,0 +1,213 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import json +import os +import fileinput +import tempfile +import requests + +from leap.common.certs import get_digest +from leap.common import ca_bundle +from .certs import LeapCertificate +from pixelated.config import leap_config +from pixelated.support.tls_adapter import EnforceTLSv1Adapter + +REQUESTS_TIMEOUT = 15 + + +class LeapProvider(object): + def __init__(self, server_name): + self.server_name = server_name + self.local_ca_crt = '%s/ca.crt' % leap_config.leap_home + self.provider_json = self.fetch_provider_json() + + @property + def provider_api_cert(self): + return str(os.path.join(leap_config.leap_home, 'providers', self.server_name, 'keys', 'client', 'api.pem')) + + @property + def combined_cerfificates_path(self): + return str(os.path.join(leap_config.leap_home, 'providers', self.server_name, 'keys', 'client', 'ca_bundle')) + + @property + def api_uri(self): + return self.provider_json.get('api_uri') + + @property + def ca_cert_fingerprint(self): + return self.provider_json.get('ca_cert_fingerprint') + + @property + def ca_cert_uri(self): + return self.provider_json.get('ca_cert_uri') + + @property + def api_version(self): + return self.provider_json.get('api_version') + + @property + def domain(self): + return self.provider_json.get('domain') + + @property + def services(self): + return self.provider_json.get('services') + + def __hash__(self): + return hash(self.server_name) + + def __eq__(self, other): + return self.server_name == other.server_name + + def ensure_supports_mx(self): + if 'mx' not in self.services: + raise Exception + + def download_soledad_json(self): + self.soledad_json = self.fetch_soledad_json() + + def download_smtp_json(self): + self.smtp_json = self.fetch_smtp_json() + + def download_certificate(self, filename=None): + """ + Downloads the server certificate, validates it against the provided fingerprint and stores it to file + """ + path = filename or self.local_ca_crt + + directory = self._extract_directory(path) + if not os.path.exists(directory): + os.makedirs(directory) + + cert = self.fetch_valid_certificate() + with open(path, 'w') as out: + out.write(cert) + + def _extract_directory(self, path): + splited = path.split('/') + splited.pop(-1) + directory = '/'.join(splited) + return directory + + def fetch_valid_certificate(self): + cert = self._fetch_certificate() + self.validate_certificate(cert) + return cert + + def _fetch_certificate(self): + cert_url = '%s/ca.crt' % self._provider_base_url() + response = self._validated_get(cert_url) + cert_data = response.content + return cert_data + + def validate_certificate(self, cert_data=None): + if cert_data is None: + cert_data = self._fetch_certificate() + + parts = str(self.ca_cert_fingerprint).split(':') + method = parts[0].strip() + fingerprint = parts[1].strip() + + digest = get_digest(cert_data, method) + + if fingerprint.strip() != digest: + raise Exception('Certificate fingerprints don\'t match! Expected [%s] but got [%s]' % (fingerprint.strip(), digest)) + + def smtp_info(self): + hosts = self.smtp_json['hosts'] + hostname = hosts.keys()[0] + host = hosts[hostname] + return host['hostname'], host['port'] + + def _validated_get(self, url): + session = requests.session() + try: + session.mount('https://', EnforceTLSv1Adapter(assert_fingerprint=LeapCertificate.LEAP_FINGERPRINT)) + response = session.get(url, verify=LeapCertificate(self).provider_web_cert, timeout=REQUESTS_TIMEOUT) + response.raise_for_status() + return response + finally: + session.close() + + def fetch_provider_json(self): + url = '%s/provider.json' % self._provider_base_url() + response = self._validated_get(url) + json_data = json.loads(response.content) + return json_data + + def fetch_soledad_json(self): + service_url = "%s/%s/config/soledad-service.json" % ( + self.api_uri, self.api_version) + response = requests.get(service_url, verify=self.provider_api_cert, timeout=REQUESTS_TIMEOUT) + response.raise_for_status() + return json.loads(response.content) + + def fetch_smtp_json(self): + service_url = '%s/%s/config/smtp-service.json' % ( + self.api_uri, self.api_version) + response = requests.get(service_url, verify=self.provider_api_cert, timeout=REQUESTS_TIMEOUT) + response.raise_for_status() + return json.loads(response.content) + + def _provider_base_url(self): + return 'https://%s' % self.server_name + + def address_for(self, username): + return '%s@%s' % (username, self.domain) + + def discover_soledad_server(self, user_uuid): + hosts = self.soledad_json['hosts'] + host = hosts.keys()[0] + server_url = 'https://%s:%d/user-%s' % \ + (hosts[host]['hostname'], hosts[host]['port'], user_uuid) + return server_url + + def _discover_nicknym_server(self): + return 'https://nicknym.%s:6425/' % self.domain + + def create_combined_bundle_file(self): + leap_ca_bundle = ca_bundle.where() + + if self.provider_api_cert == leap_ca_bundle: + return self.provider_api_cert + elif not self.provider_api_cert: + return leap_ca_bundle + + with open(self.combined_cerfificates_path, 'w') as fout: + fin = fileinput.input(files=(leap_ca_bundle, self.provider_api_cert)) + for line in fin: + fout.write(line) + fin.close() + + def setup_ca_bundle(self): + path = os.path.join(leap_config.leap_home, 'providers', self.server_name, 'keys', 'client') + if not os.path.isdir(path): + os.makedirs(path, 0700) + self._download_cert(self.provider_api_cert) + + def _download_cert(self, cert_file_name): + cert = self.fetch_valid_certificate() + with open(cert_file_name, 'w') as file: + file.write(cert) + + def setup_ca(self): + self.download_certificate() + self.setup_ca_bundle() + self.create_combined_bundle_file() + + def download_settings(self): + self.download_soledad_json() + self.download_smtp_json() diff --git a/service/src/pixelated/bitmask_libraries/smtp.py b/service/src/pixelated/bitmask_libraries/smtp.py new file mode 100644 index 00000000..643d4d4a --- /dev/null +++ b/service/src/pixelated/bitmask_libraries/smtp.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + + +class LeapSMTPConfig(object): + + def __init__(self, account_email, cert_path, remote_smtp_host, remote_smtp_port): + self.account_email = account_email + self.cert_path = cert_path + self.remote_smtp_host = remote_smtp_host + self.remote_smtp_port = remote_smtp_port diff --git a/service/src/pixelated/certificates/__init__.py b/service/src/pixelated/certificates/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/service/src/pixelated/certificates/__init__.py diff --git a/service/src/pixelated/certificates/unstable.pixelated-project.org.ca.crt b/service/src/pixelated/certificates/unstable.pixelated-project.org.ca.crt new file mode 100644 index 00000000..22ccb662 --- /dev/null +++ b/service/src/pixelated/certificates/unstable.pixelated-project.org.ca.crt @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFqzCCA5OgAwIBAgIBATANBgkqhkiG9w0BAQ0FADBoMRowGAYDVQQKDBFQaXhl +bGF0ZWQgUHJvamVjdDEmMCQGA1UECwwdaHR0cHM6Ly9waXhlbGF0ZWQtcHJvamVj +dC5vcmcxIjAgBgNVBAMMGVBpeGVsYXRlZCBQcm9qZWN0IFJvb3QgQ0EwHhcNMTUw +NDAxMDAwMDAwWhcNMjUwNDAxMDAwMDAwWjBoMRowGAYDVQQKDBFQaXhlbGF0ZWQg +UHJvamVjdDEmMCQGA1UECwwdaHR0cHM6Ly9waXhlbGF0ZWQtcHJvamVjdC5vcmcx +IjAgBgNVBAMMGVBpeGVsYXRlZCBQcm9qZWN0IFJvb3QgQ0EwggIiMA0GCSqGSIb3 +DQEBAQUAA4ICDwAwggIKAoICAQDIKlYvQciyw1gq7NgyXxo4OPnlB6cyjLb0vr6V +cnY00F3pwa3cuOASYtZ5p4IMGod1fZoBUlGFbxay0NO/FyOlPVc3FaF1cL5oYurl +a/koAjAihGMGEjsc2gcdODJOHoidgZG4a4WUCVfXP54QFkG6tXve1uWnZAt7W6Z8 +M4DobBqh4HE8aAn1vSIdOltKvuH6Jz9+O2SZdXczEKuHM/Zv8o5qI4+3CKQ/enLS +WUfqZx217zJw4xax4kTsZxaU56XkdXGjAoqKmP3am8sAfSPyt5J6yPvKzUM2hbyQ +v/ZR3AzKX6E5w/kPWfrjr+ABPtMEbcUEQYTR5Ic/46umzseEEUTHaeBhn5XZfan/ +K3+iOXHgukuNX5BF28D10RKrIWutOr6pMWXtFNLsmSvbii6SYg4sr80RhPsox3Lf +QuO36aH/jrkmyeEgTVVyniPUYLeg/IU+lQzOQpoHR7FiS2/39QP9HNbO7kQzi2Fh +aFQW35sRRZuz+Gt38y0DObKOwEZaR2NM2kIHiR6cYWP9fYCekOfJi7rp0hfxcO+J +o+kVoZM53SS0QJGzHnKYo+hHAy01Hj9Kh7N269rruZSjDXsWF/eZp5oYQeItBolq +J3Va8VyQk12Pd+4vcIERWNx8JCBDnYY1+sRBsrsH1mSMRQxOkbOhlx0y64dRNQRa +GV4xxQIDAQABo2AwXjAdBgNVHQ4EFgQUdXb2FIbWofTZ00R/OoQQu2K1mscwDgYD +VR0PAQH/BAQDAgIEMAwGA1UdEwQFMAMBAf8wHwYDVR0jBBgwFoAUdXb2FIbWofTZ +00R/OoQQu2K1mscwDQYJKoZIhvcNAQENBQADggIBAKR6XuNIZGWccQtVxuOKRuNY +5KE5N2Gqi4erN1XkJvOD60an2F6aYjFUN1ZZfq31Oi57GHZ8lwPKZZuAn/pn2vQG +1YpGFXndnoFkKDGb3XVRMHMtFIcV/2u2rX8zfRmZGpHvgSZo+9g4PN2jCipKkuki +pVnLjO7SZi4/AAuzlFPBUHE441xJI6Hl3lLKFezLD4akxrBpFbW+rBOSFupxyBcD +hedH0MyHY6HUQLL4Mm2tobouFStUcBZT0wGEeh/tlaJKAhk/kzryle4IZcqpv01a +6BfsyO9kRABc4b9w26tvd93utgXkpmwni+I31QSFVH/s8oU1pYvrpHO2MQf5rE7X +gyVvUaXSNETE2Nj1/Bp0Bbk/A2IyzOyoIALU+5LBOzgnW8K+Ojg9FQSRuIFZF+fw +VRruRFMziKtFBYaTclWX9mf+tn3MFSe8FcIcpMa0yXu9pDtQxZeeFG/TbseE0VjS +20HHEPnA6YIKbi7vg7LZP+dZoPBc30tJeG/fDCo6EDmFhddtDJpmnFPnr3OVf1AC +B90gGoXQ2RJsP8ZLkYlMn5JYKQbXLQSf806eMYJr3Mo6IBOR+o9dxAyBtEBgAUG6 +jgfKRzGizhHA3LDbxUlJt1k16SU58QBY3p8FHwjbjon7GDEOhSNOtuHAHUc83sHH +SC1pIYpuR4H4lTW5fuyF +-----END CERTIFICATE----- diff --git a/service/src/pixelated/config/__init__.py b/service/src/pixelated/config/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/service/src/pixelated/config/__init__.py diff --git a/service/src/pixelated/config/arguments.py b/service/src/pixelated/config/arguments.py new file mode 100644 index 00000000..01152a34 --- /dev/null +++ b/service/src/pixelated/config/arguments.py @@ -0,0 +1,78 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import os +import argparse + + +def parse_user_agent_args(): + parser = argparse.ArgumentParser(description='Pixelated user agent.') + + parser_add_default_arguments(parser) + + parser.add_argument('--host', default='127.0.0.1', help='the host to run the user agent on') + parser.add_argument('--port', type=int, default=3333, help='the port to run the user agent on') + parser.add_argument('-sk', '--sslkey', metavar='<server.key>', default=None, help='use specified file as web server\'s SSL key (when using the user-agent in server-mode)') + parser.add_argument('-sc', '--sslcert', metavar='<server.crt>', default=None, help='use specified file as web server\'s SSL certificate (when using the user-agent in server-mode)') + parser.add_argument('--multi-user', help='Run user agent in multi user mode', action='store_false', default=True, dest='single_user') + parser.add_argument('-p', '--provider', help='specify a provider for mutli-user mode', metavar='<provider host>', default=None, dest='provider') + parser.add_argument('--banner', help='banner file to show on login screen') + parser.add_argument('--manhole', help='Run an interactive Python shell on port 8008', action='store_true', default=False, dest='manhole') + + args = parser.parse_args() + + return args + + +def parse_maintenance_args(): + parser = argparse.ArgumentParser(description='Pixelated maintenance') + parser_add_default_arguments(parser) + subparsers = parser.add_subparsers(help='commands', dest='command') + subparsers.add_parser('reset', help='reset account command') + mails_parser = subparsers.add_parser('load-mails', help='load mails into account') + mails_parser.add_argument('file', nargs='+', help='file(s) with mail data') + + markov_mails_parser = subparsers.add_parser('markov-generate', help='generate mails using markov chains') + markov_mails_parser.add_argument('--seed', default=None, help='Specify a seed to always generate the same output') + markov_mails_parser.add_argument('-l', '--limit', metavar='count', default='5', help='limit number of generated mails', dest='limit') + markov_mails_parser.add_argument('file', nargs='+', help='file(s) with mail data') + + subparsers.add_parser('dump-soledad', help='dump the soledad database') + subparsers.add_parser('sync', help='sync the soledad database') + subparsers.add_parser('repair', help='repair database if possible') + subparsers.add_parser('integrity-check', help='run integrity check on database') + + return parser.parse_args() + + +def parse_register_args(): + parser = argparse.ArgumentParser(description='Pixelated register') + parser.add_argument('provider', metavar='provider', action='store') + parser.add_argument('username', metavar='username', action='store') + parser.add_argument('-p', '--password', metavar='password', action='store', default=None, help='used just to register account automatically by scripts') + parser.add_argument('-lc', '--leap-provider-cert', metavar='<leap-provider.crt>', default=None, help='use specified file for LEAP provider cert authority certificate (url https://<LEAP-provider-domain>/ca.crt)') + parser.add_argument('-lf', '--leap-provider-cert-fingerprint', metavar='<leap provider certificate fingerprint>', default=None, help='use specified fingerprint to validate connection with LEAP provider', dest='leap_provider_cert_fingerprint') + parser.add_argument('--leap-home', help='The folder where the user agent stores its data. Defaults to ~/.leap', dest='leap_home', default=os.path.join(os.path.expanduser("~"), '.leap')) + parser.add_argument('--invite-code', help='invite code to register a user, if required', dest='invite_code', default=None) + return parser.parse_args() + + +def parser_add_default_arguments(parser): + parser.add_argument('--debug', action='store_true', help='DEBUG mode.') + parser.add_argument('-c', '--config', dest='credentials_file', metavar='<credentials_file>', default=None, help='use specified file for credentials (for test purposes only)') + parser.add_argument('--leap-home', help='The folder where the user agent stores its data. Defaults to ~/.leap', dest='leap_home', default=os.path.join(os.path.expanduser("~"), '.leap')) + parser.add_argument('-lc', '--leap-provider-cert', metavar='<leap-provider.crt>', default=None, help='use specified file for LEAP provider cert authority certificate (url https://<LEAP-provider-domain>/ca.crt)') + parser.add_argument('-lf', '--leap-provider-cert-fingerprint', metavar='<leap provider certificate fingerprint>', default=None, help='use specified fingerprint to validate connection with LEAP provider', dest='leap_provider_cert_fingerprint') diff --git a/service/src/pixelated/config/credentials.py b/service/src/pixelated/config/credentials.py new file mode 100644 index 00000000..89901b3f --- /dev/null +++ b/service/src/pixelated/config/credentials.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import os +import getpass +import json +import sys +import ConfigParser + + +def read(credentials_file): + if credentials_file: + return read_from_file(credentials_file) + return prompt_for_credentials() + + +def prompt_for_credentials(): + provider = raw_input('Which provider do you want to connect to:\n') + username = raw_input('What\'s your username registered on the provider:\n') + password = getpass.getpass('Type your password:\n') + return provider, username, password + + +def read_from_file(credentials_file): + config_parser = ConfigParser.ConfigParser() + credentials_file_path = os.path.abspath(os.path.expanduser(credentials_file)) + config_parser.read(credentials_file_path) + provider, user, password = \ + config_parser.get('pixelated', 'leap_server_name'), \ + config_parser.get('pixelated', 'leap_username'), \ + config_parser.get('pixelated', 'leap_password') + return provider, user, password diff --git a/service/src/pixelated/config/leap.py b/service/src/pixelated/config/leap.py new file mode 100644 index 00000000..2b3a242a --- /dev/null +++ b/service/src/pixelated/config/leap.py @@ -0,0 +1,114 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import absolute_import + +from leap.common.events import (server as events_server) +from pixelated.adapter.welcome_mail import add_welcome_mail +from pixelated.authentication import Authenticator +from pixelated.bitmask_libraries.certs import LeapCertificate +from pixelated.bitmask_libraries.provider import LeapProvider +from pixelated.config import credentials +from pixelated.config import leap_config +from pixelated.config.sessions import LeapSessionFactory +from twisted.internet import defer +from twisted.logger import Logger + +log = Logger() + + +def initialize_leap_provider(provider_hostname, provider_cert, provider_fingerprint, leap_home): + LeapCertificate.set_cert_and_fingerprint(provider_cert, + provider_fingerprint) + leap_config.set_leap_home(leap_home) + provider = LeapProvider(provider_hostname) + provider.setup_ca() + provider.download_settings() + return provider + + +@defer.inlineCallbacks +def initialize_leap_multi_user(provider_hostname, + leap_provider_cert, + leap_provider_cert_fingerprint, + credentials_file, + leap_home): + + config, provider = initialize_leap_provider(provider_hostname, leap_provider_cert, leap_provider_cert_fingerprint, leap_home) + + defer.returnValue((config, provider)) + + +@defer.inlineCallbacks +def create_leap_session(provider, username, password, auth=None): + leap_session = yield LeapSessionFactory(provider).create(username, password, auth) + defer.returnValue(leap_session) + + +@defer.inlineCallbacks +def initialize_leap_single_user(leap_provider_cert, + leap_provider_cert_fingerprint, + credentials_file, + leap_home): + + init_monkeypatches() + events_server.ensure_server() + + provider, username, password = credentials.read(credentials_file) + + provider = initialize_leap_provider(provider, leap_provider_cert, leap_provider_cert_fingerprint, leap_home) + + auth = yield Authenticator(provider).authenticate(username, password) + + leap_session = yield create_leap_session(provider, username, password, auth) + + defer.returnValue(leap_session) + + +def init_monkeypatches(): + import pixelated.extensions.requests_urllib3 + + +class BootstrapUserServices(object): + + def __init__(self, services_factory, provider): + self._services_factory = services_factory + self._provider = provider + + @defer.inlineCallbacks + def setup(self, user_auth, password, language='pt-BR'): + leap_session = None + try: + leap_session = yield create_leap_session(self._provider, user_auth.username, password, user_auth) + yield self._setup_user_services(leap_session) + yield self._add_welcome_email(leap_session, language) + except Exception as e: + log.warn('{0}: {1}. Closing session for user: {2}'.format(e.__class__.__name__, e, user_auth.username)) + if leap_session: + leap_session.close() + raise + + @defer.inlineCallbacks + def _setup_user_services(self, leap_session): + user_id = leap_session.user_auth.uuid + if not self._services_factory.has_session(user_id): + yield self._services_factory.create_services_from(leap_session) + self._services_factory.map_email(leap_session.user_auth.username, user_id) + + @defer.inlineCallbacks + def _add_welcome_email(self, leap_session, language): + if leap_session.fresh_account: + yield add_welcome_mail(leap_session.mail_store, language) diff --git a/service/src/pixelated/config/leap_config.py b/service/src/pixelated/config/leap_config.py new file mode 100644 index 00000000..7319d82b --- /dev/null +++ b/service/src/pixelated/config/leap_config.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import os +from distutils.spawn import find_executable + + +def discover_gpg_binary(): + path = find_executable('gpg') + if path is None: + raise Exception('Did not find a gpg executable!') + + if os.path.islink(path): + path = os.path.realpath(path) + + return path + + +SYSTEM_CA_BUNDLE = True +leap_home = os.path.expanduser('~/.leap/') +gpg_binary = discover_gpg_binary() + + +def set_leap_home(new_home): + leap_home = new_home + + +def set_gpg_binary(new_binary): + gpg_binary = binary diff --git a/service/src/pixelated/config/logger.py b/service/src/pixelated/config/logger.py new file mode 100644 index 00000000..bc4ab8d4 --- /dev/null +++ b/service/src/pixelated/config/logger.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import logging +import os +import sys +import time +from twisted.logger import globalLogBeginner, FileLogObserver + + +class PrivateKeyFilter(logging.Filter): + + def filter(self, record): + if '-----BEGIN PGP PRIVATE KEY BLOCK-----' in record.msg: + record.msg = '*** private key removed by %s.%s ***' % (type(self).__module__, type(self).__name__) + return True + + +def init(debug=False): + debug_enabled = debug or os.environ.get('DEBUG', False) + logging_level = logging.DEBUG if debug_enabled else logging.INFO + + logging.basicConfig(level=logging_level, + format='%(asctime)s [%(name)s] %(levelname)s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + filemode='a') + + logging.getLogger('gnupg').setLevel(logging.WARN) + logging.getLogger('gnupg').addFilter(PrivateKeyFilter()) + + def formatter(event): + try: + event['log_time'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(event['log_time'])) + event['log_level'] = event['log_level'].name.upper() + event['log_format'] = str(event['log_format']) + '\n' if event.get('log_format') else '' + logstring = u'{log_time} [{log_namespace}] {log_level} ' + event['log_format'] + return logstring.format(**event) + except Exception as e: + return "Error while formatting log event: {!r}\nOriginal event: {!r}\n".format(e, event) + + observers = [FileLogObserver(sys.stdout, formatter)] + globalLogBeginner.beginLoggingTo(observers) diff --git a/service/src/pixelated/config/services.py b/service/src/pixelated/config/services.py new file mode 100644 index 00000000..48c1a528 --- /dev/null +++ b/service/src/pixelated/config/services.py @@ -0,0 +1,158 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import os + +from twisted.internet import defer, reactor +from twisted.logger import Logger + +from pixelated.adapter.mailstore.leap_attachment_store import LeapAttachmentStore +from pixelated.adapter.mailstore.searchable_mailstore import SearchableMailStore +from pixelated.adapter.services.mail_service import MailService +from pixelated.adapter.model.mail import InputMail +from pixelated.adapter.services.mail_sender import MailSender +from pixelated.adapter.search import SearchEngine +from pixelated.adapter.services.draft_service import DraftService +from pixelated.adapter.listeners.mailbox_indexer_listener import listen_all_mailboxes +from pixelated.adapter.search.index_storage_key import SearchIndexStorageKey +from pixelated.adapter.services.feedback_service import FeedbackService +from pixelated.config import leap_config + +logger = Logger() + + +class Services(object): + + def __init__(self, leap_session): + self._leap_home = leap_config.leap_home + self._pixelated_home = os.path.join(self._leap_home, 'pixelated') + self._leap_session = leap_session + + @defer.inlineCallbacks + def setup(self): + search_index_storage_key = self._setup_search_index_storage_key(self._leap_session.soledad) + yield self._setup_search_engine(self._leap_session.user_auth.uuid, search_index_storage_key) + + self._wrap_mail_store_with_indexing_mail_store(self._leap_session) + + yield listen_all_mailboxes(self._leap_session.account, self.search_engine, self._leap_session.mail_store) + + self.mail_service = self._setup_mail_service(self.search_engine) + + self.keymanager = self._leap_session.keymanager + self.draft_service = self._setup_draft_service(self._leap_session.mail_store) + self.feedback_service = self._setup_feedback_service() + yield self._index_all_mails() + + def close(self): + self._leap_session.close() + + def _wrap_mail_store_with_indexing_mail_store(self, leap_session): + leap_session.mail_store = SearchableMailStore(leap_session.mail_store, self.search_engine) + + @defer.inlineCallbacks + def _index_all_mails(self): + all_mails = yield self.mail_service.all_mails() + self.search_engine.index_mails(all_mails) + + @defer.inlineCallbacks + def _setup_search_engine(self, namespace, search_index_storage_key): + key_unicode = yield search_index_storage_key.get_or_create_key() + key = str(key_unicode) + logger.debug('The key len is: %s' % len(key)) + user_id = self._leap_session.user_auth.uuid + user_folder = os.path.join(self._pixelated_home, user_id) + search_engine = SearchEngine(key, user_home=user_folder) + self.search_engine = search_engine + + def _setup_mail_service(self, search_engine): + pixelated_mail_sender = MailSender(self._leap_session.smtp_config, self._leap_session.keymanager.keymanager) + + return MailService( + pixelated_mail_sender, + self._leap_session.mail_store, + search_engine, + self._leap_session.account_email(), + LeapAttachmentStore(self._leap_session.soledad)) + + def _setup_draft_service(self, mail_store): + return DraftService(mail_store) + + def _setup_search_index_storage_key(self, soledad): + return SearchIndexStorageKey(soledad) + + def _setup_feedback_service(self): + return FeedbackService(self._leap_session) + + +class ServicesFactory(object): + + def __init__(self, mode): + self._services_by_user = {} + self.mode = mode + self._map_email = {} + + def map_email(self, username, user_id): + self._map_email[username] = user_id + + def has_session(self, user_id): + return user_id in self._services_by_user + + def services(self, user_id): + return self._services_by_user[user_id] + + def destroy_session(self, user_id, using_email=False): + if using_email: + username = user_id.split('@')[0] + user_id = self._map_email.get(username, None) + + if user_id is not None and self.has_session(user_id): + _services = self._services_by_user[user_id] + _services.close() + del self._services_by_user[user_id] + + def add_session(self, user_id, services): + self._services_by_user[user_id] = services + + def online_sessions(self): + return len(self._services_by_user.keys()) + + @defer.inlineCallbacks + def create_services_from(self, leap_session): + _services = Services(leap_session) + yield _services.setup() + self._services_by_user[leap_session.user_auth.uuid] = _services + + +class SingleUserServicesFactory(object): + def __init__(self, mode): + self._services = None + self.mode = mode + + def add_session(self, user_id, services): + self._services = services + + def services(self, user_id): + return self._services + + def has_session(self, user_id): + return True + + def destroy_session(self, user_id, using_email=False): + reactor.stop() + + def online_sessions(self): + return 1 diff --git a/service/src/pixelated/config/sessions.py b/service/src/pixelated/config/sessions.py new file mode 100644 index 00000000..594b8e35 --- /dev/null +++ b/service/src/pixelated/config/sessions.py @@ -0,0 +1,311 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import absolute_import + +import os +import errno +import requests + +from twisted.internet import defer, threads, reactor +from twisted.logger import Logger + +from leap.soledad.common.crypto import WrongMacError, UnknownMacMethodError +from leap.soledad.client import Soledad +from leap.bitmask.mail.incoming.service import IncomingMail +from leap.bitmask.mail.mail import Account +import leap.common.certs as leap_certs +from leap.common.events import ( + register, unregister, + catalog as events +) + +from pixelated.bitmask_libraries.keymanager import Keymanager +from pixelated.adapter.mailstore import LeapMailStore +from pixelated.config import leap_config +from pixelated.bitmask_libraries.smtp import LeapSMTPConfig + +logger = Logger() + + +class LeapSessionFactory(object): + def __init__(self, provider): + self._provider = provider + + @defer.inlineCallbacks + def create(self, username, password, auth): + key = SessionCache.session_key(self._provider, username) + session = SessionCache.lookup_session(key) + if not session: + session = yield self._create_new_session(username, password, auth) + yield session.first_required_sync() + SessionCache.remember_session(key, session) + defer.returnValue(session) + + @defer.inlineCallbacks + def _create_new_session(self, username, password, auth): + account_email = self._provider.address_for(username) + + self._create_database_dir(auth.uuid) + + api_cert = self._provider.provider_api_cert + + soledad = yield self.setup_soledad(auth.token, auth.uuid, password, api_cert) + + mail_store = LeapMailStore(soledad) + + keymanager = yield self.setup_keymanager(self._provider, soledad, account_email, auth.token, auth.uuid) + + smtp_client_cert = self._download_smtp_cert(auth) + smtp_host, smtp_port = self._provider.smtp_info() + smtp_config = LeapSMTPConfig(account_email, smtp_client_cert, smtp_host, smtp_port) + + leap_session = LeapSession(self._provider, auth, mail_store, soledad, keymanager, smtp_config) + + defer.returnValue(leap_session) + + @defer.inlineCallbacks + def setup_soledad(self, + user_token, + user_uuid, + password, + api_cert): + secrets = self._secrets_path(user_uuid) + local_db = self._local_db_path(user_uuid) + server_url = self._provider.discover_soledad_server(user_uuid) + try: + soledad = yield threads.deferToThread(Soledad, + user_uuid.encode('utf-8'), + passphrase=unicode(password, 'utf-8'), + secrets_path=secrets, + local_db_path=local_db, + server_url=server_url, + cert_file=api_cert, + shared_db=None, + auth_token=user_token) + defer.returnValue(soledad) + except (WrongMacError, UnknownMacMethodError), e: + raise SoledadWrongPassphraseException(e) + + @defer.inlineCallbacks + def setup_keymanager(self, provider, soledad, account_email, token, uuid): + keymanager = yield threads.deferToThread(Keymanager, + provider, + soledad, + account_email, + token, + uuid) + defer.returnValue(keymanager) + + def _download_smtp_cert(self, auth): + cert = SmtpClientCertificate(self._provider, auth, self._user_path(auth.uuid)) + return cert.cert_path() + + def _user_path(self, user_uuid): + return os.path.join(leap_config.leap_home, user_uuid) + + def _soledad_path(self, user_uuid): + return os.path.join(leap_config.leap_home, user_uuid, 'soledad') + + def _secrets_path(self, user_uuid): + return os.path.join(self._soledad_path(user_uuid), 'secrets') + + def _local_db_path(self, user_uuid): + return os.path.join(self._soledad_path(user_uuid), 'soledad.db') + + def _create_database_dir(self, user_uuid): + try: + os.makedirs(self._soledad_path(user_uuid)) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(self._soledad_path(user_uuid)): + pass + else: + raise + + +class LeapSession(object): + + def __init__(self, provider, user_auth, mail_store, soledad, keymanager, smtp_config): + self.smtp_config = smtp_config + self.provider = provider + self.user_auth = user_auth + self.mail_store = mail_store + self.soledad = soledad + self.keymanager = keymanager + self.fresh_account = False + self.incoming_mail_fetcher = None + self.account = None + self._has_been_initially_synced = False + self._is_closed = False + register(events.KEYMANAGER_FINISHED_KEY_GENERATION, self._set_fresh_account, uid=self.account_email(), replace=True) + + @defer.inlineCallbacks + def first_required_sync(self): + yield self.sync() + yield self.finish_bootstrap() + + @defer.inlineCallbacks + def finish_bootstrap(self): + yield self.keymanager.generate_openpgp_key() + yield self._create_account(self.soledad, self.user_auth.uuid) + self.incoming_mail_fetcher = yield self._create_incoming_mail_fetcher( + self.keymanager, + self.soledad, + self.account, + self.account_email()) + reactor.callFromThread(self.incoming_mail_fetcher.startService) + + def _create_account(self, soledad, user_id): + self.account = Account(soledad, user_id) + return self.account.deferred_initialization + + def _set_fresh_account(self, event, email_address): + logger.debug('Key for email %s has been generated' % email_address) + if email_address == self.account_email(): + self.fresh_account = True + + def account_email(self): + name = self.user_auth.username + return self.provider.address_for(name) + + def close(self): + self.stop_background_jobs() + unregister(events.KEYMANAGER_FINISHED_KEY_GENERATION, uid=self.account_email()) + self.soledad.close() + self._close_account() + self.remove_from_cache() + self._is_closed = True + + @property + def is_closed(self): + return self._is_closed + + def _close_account(self): + if self.account: + self.account.end_session() + + def remove_from_cache(self): + key = SessionCache.session_key(self.provider, self.user_auth.username) + SessionCache.remove_session(key) + + @defer.inlineCallbacks + def _create_incoming_mail_fetcher(self, keymanager, soledad, account, user_mail): + inbox = yield account.callWhenReady(lambda _: account.get_collection_by_mailbox('INBOX')) + defer.returnValue(IncomingMail(keymanager.keymanager, + soledad, + inbox, + user_mail)) + + def stop_background_jobs(self): + if self.incoming_mail_fetcher: + reactor.callFromThread(self.incoming_mail_fetcher.stopService) + self.incoming_mail_fetcher = None + + def sync(self): + try: + return self.soledad.sync() + except Exception as e: + logger.error(e) + raise + + +class SessionCache(object): + + sessions = {} + + @staticmethod + def lookup_session(key): + session = SessionCache.sessions.get(key, None) + if session is not None and session.is_closed: + SessionCache.remove_session(key) + return None + return session + + @staticmethod + def remember_session(key, session): + SessionCache.sessions[key] = session + + @staticmethod + def remove_session(key): + if key in SessionCache.sessions: + del SessionCache.sessions[key] + + @staticmethod + def session_key(provider, username): + return hash((provider, username)) + + +class SmtpClientCertificate(object): + def __init__(self, provider, auth, user_path): + self._provider = provider + self._auth = auth + self._user_path = user_path + + def cert_path(self): + if not self._is_cert_already_downloaded() or self._should_redownload(): + self._download_smtp_cert() + + return self._smtp_client_cert_path() + + def _is_cert_already_downloaded(self): + return os.path.exists(self._smtp_client_cert_path()) + + def _should_redownload(self): + return leap_certs.should_redownload(self._smtp_client_cert_path()) + + def _download_smtp_cert(self): + cert_path = self._smtp_client_cert_path() + + if not os.path.exists(os.path.dirname(cert_path)): + os.makedirs(os.path.dirname(cert_path)) + + self.download_to(cert_path) + + def _smtp_client_cert_path(self): + return os.path.join( + self._user_path, + "providers", + self._provider.domain, + "keys", "client", "smtp.pem") + + def download(self): + cert_url = '%s/%s/smtp_cert' % (self._provider.api_uri, self._provider.api_version) + headers = {} + headers["Authorization"] = 'Token token="{0}"'.format(self._auth.token) + params = {'address': self._auth.username} + response = requests.post( + cert_url, + params=params, + data=params, + verify=self._provider.provider_api_cert, + timeout=15, + headers=headers) + response.raise_for_status() + + client_cert = response.content + + return client_cert + + def download_to(self, target_file): + client_cert = self.download() + + with open(target_file, 'w') as f: + f.write(client_cert) + + +class SoledadWrongPassphraseException(Exception): + def __init__(self, *args, **kwargs): + super(SoledadWrongPassphraseException, self).__init__(*args, **kwargs) diff --git a/service/src/pixelated/config/site.py b/service/src/pixelated/config/site.py new file mode 100644 index 00000000..96554584 --- /dev/null +++ b/service/src/pixelated/config/site.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from twisted.web.server import Site, Request + + +class AddSecurityHeadersRequest(Request): + CSP_HEADER_VALUES = "default-src 'self'; style-src 'self' 'unsafe-inline'" + + def process(self): + self.setHeader('Content-Security-Policy', self.CSP_HEADER_VALUES) + self.setHeader('X-Content-Security-Policy', self.CSP_HEADER_VALUES) + self.setHeader('X-Webkit-CSP', self.CSP_HEADER_VALUES) + self.setHeader('X-Frame-Options', 'SAMEORIGIN') + self.setHeader('X-XSS-Protection', '1; mode=block') + self.setHeader('X-Content-Type-Options', 'nosniff') + + if self.isSecure(): + self.setHeader('Strict-Transport-Security', 'max-age=31536000; includeSubDomains') + + Request.process(self) + + +class PixelatedSite(Site): + + requestFactory = AddSecurityHeadersRequest + + @classmethod + def enable_csp_requests(cls): + cls.requestFactory = AddSecurityHeadersRequest + + @classmethod + def disable_csp_requests(cls): + cls.requestFactory = Site.requestFactory diff --git a/service/src/pixelated/extensions/__init__.py b/service/src/pixelated/extensions/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/service/src/pixelated/extensions/__init__.py diff --git a/service/src/pixelated/extensions/esmtp_sender_factory.py b/service/src/pixelated/extensions/esmtp_sender_factory.py new file mode 100644 index 00000000..59aa90c8 --- /dev/null +++ b/service/src/pixelated/extensions/esmtp_sender_factory.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from twisted.mail import smtp + + +def no_require_transport_security(f): + def wrapper(*args, **kwargs): + kwargs['requireTransportSecurity'] = False + return f(*args, **kwargs) + return wrapper + + +smtp.ESMTPSenderFactory = no_require_transport_security(smtp.ESMTPSenderFactory) diff --git a/service/src/pixelated/extensions/protobuf_socket.py b/service/src/pixelated/extensions/protobuf_socket.py new file mode 100644 index 00000000..548f5fd6 --- /dev/null +++ b/service/src/pixelated/extensions/protobuf_socket.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import print_function +from sys import platform as _platform + +import protobuf.socketrpc.server + +# protobuf throws a lot of 'Socket is not connected' exceptions on OSX but they are not an issue. +# refer too https://code.google.com/p/protobuf-socket-rpc/issues/detail?id=10 and +# or https://leap.se/code/issues/2187 +if _platform == 'darwin': + def try_except_decorator(func): + def wrapper(*args, **kwargs): + try: + func(*args, **kwargs) + pass + except: + pass + + return wrapper + + protobuf.socketrpc.server.SocketHandler.handle = try_except_decorator( + protobuf.socketrpc.server.SocketHandler.handle) diff --git a/service/src/pixelated/extensions/requests_urllib3.py b/service/src/pixelated/extensions/requests_urllib3.py new file mode 100644 index 00000000..c4ec2438 --- /dev/null +++ b/service/src/pixelated/extensions/requests_urllib3.py @@ -0,0 +1,82 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import requests + + +if requests.__version__ == '2.0.0': + try: + import requests.packages.urllib3.connectionpool + from socket import error as SocketError, timeout as SocketTimeout + from requests.packages.urllib3.packages.ssl_match_hostname import CertificateError, match_hostname + import socket + import ssl + + from requests.packages.urllib3.exceptions import ( + ClosedPoolError, + ConnectTimeoutError, + EmptyPoolError, + HostChangedError, + MaxRetryError, + SSLError, + ReadTimeoutError, + ProxyError, + ) + + from requests.packages.urllib3.util import ( + assert_fingerprint, + get_host, + is_connection_dropped, + resolve_cert_reqs, + resolve_ssl_version, + ssl_wrap_socket, + Timeout, + ) + + def patched_connect(self): + # Add certificate verification + try: + sock = socket.create_connection(address=(self.host, self.port), timeout=self.timeout) + except SocketTimeout: + raise ConnectTimeoutError(self, "Connection to %s timed out. (connect timeout=%s)" % (self.host, self.timeout)) + + resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs) + resolved_ssl_version = resolve_ssl_version(self.ssl_version) + + if self._tunnel_host: + self.sock = sock + # Calls self._set_hostport(), so self.host is + # self._tunnel_host below. + self._tunnel() + + # Wrap socket using verification with the root certs in + # trusted_root_certs + self.sock = ssl_wrap_socket(sock, self.key_file, self.cert_file, + cert_reqs=resolved_cert_reqs, + ca_certs=self.ca_certs, + server_hostname=self.host, + ssl_version=resolved_ssl_version) + + if self.assert_fingerprint: + assert_fingerprint(self.sock.getpeercert(binary_form=True), + self.assert_fingerprint) + elif resolved_cert_reqs != ssl.CERT_NONE and self.assert_hostname is not False: + match_hostname(self.sock.getpeercert(), + self.assert_hostname or self.host) + + requests.packages.urllib3.connectionpool.VerifiedHTTPSConnection.connect = patched_connect + except ImportError: + pass # The patch is specific for the debian package. Ignore it if it can't be found diff --git a/service/src/pixelated/extensions/sqlcipher_wal.py b/service/src/pixelated/extensions/sqlcipher_wal.py new file mode 100644 index 00000000..776087bf --- /dev/null +++ b/service/src/pixelated/extensions/sqlcipher_wal.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from sys import platform as _platform + +import leap.soledad.client.sqlcipher + +# WAL is breaking for the debian sqlcipher package so we need to disable it +# refer to https://leap.se/code/issues/5562 +if _platform == 'linux2': + leap.soledad.client.sqlcipher.SQLCipherDatabase._pragma_write_ahead_logging = lambda x, y: None diff --git a/service/src/pixelated/maintenance.py b/service/src/pixelated/maintenance.py new file mode 100644 index 00000000..a2286034 --- /dev/null +++ b/service/src/pixelated/maintenance.py @@ -0,0 +1,318 @@ + +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import random +from os.path import isfile +from mailbox import Maildir, mbox, MaildirMessage + +from twisted.internet import reactor, defer +from twisted.internet.threads import deferToThread +from twisted.logger import Logger + +from leap.bitmask.mail.adaptors.soledad import MetaMsgDocWrapper +from leap.bitmask.mail.constants import MessageFlags + +from pixelated.adapter.mailstore.maintenance import SoledadMaintenance +from pixelated.config.leap import initialize_leap_single_user +from pixelated.config import logger, arguments +from pixelated.support.mail_generator import MailGenerator + +REPAIR_COMMAND = 'repair' +INTEGRITY_CHECK_COMMAND = 'integrity-check' + + +log = Logger() + + +def initialize(): + args = arguments.parse_maintenance_args() + + logger.init(debug=args.debug) + + @defer.inlineCallbacks + def _run(): + leap_session = yield initialize_leap_single_user( + args.leap_provider_cert, + args.leap_provider_cert_fingerprint, + args.credentials_file, + leap_home=args.leap_home) + + execute_command(args, leap_session) + + reactor.callWhenRunning(_run) + reactor.run() + + +def _is_repair_command(args): + return args.command == REPAIR_COMMAND + + +def _is_integrity_check_command(args): + return args.command == INTEGRITY_CHECK_COMMAND + + +def execute_command(args, leap_session): + + def init_soledad(): + return leap_session + + def get_soledad_handle(leap_session): + soledad = leap_session.soledad + + return leap_session, soledad + + @defer.inlineCallbacks + def soledad_sync(args): + leap_session, soledad = args + + log.warn('Before sync') + + yield soledad.sync() + + log.warn('after sync') + + defer.returnValue(args) + + tearDown = defer.Deferred() + + prepare = deferToThread(init_soledad) + prepare.addCallback(get_soledad_handle) + prepare.addCallback(soledad_sync) + add_command_callback(args, prepare, tearDown) + tearDown.addCallback(soledad_sync) + tearDown.addCallback(shutdown) + tearDown.addErrback(shutdown_on_error) + + +def add_command_callback(args, prepareDeferred, finalizeDeferred): + if args.command == 'reset': + prepareDeferred.addCallback(delete_all_mails) + prepareDeferred.addCallback(flush_to_soledad, finalizeDeferred) + elif args.command == 'load-mails': + prepareDeferred.addCallback(load_mails, args.file) + prepareDeferred.addCallback(flush_to_soledad, finalizeDeferred) + elif args.command == 'markov-generate': + prepareDeferred.addCallback( + markov_generate, args.file, int(args.limit), args.seed) + prepareDeferred.addCallback(flush_to_soledad, finalizeDeferred) + elif args.command == 'dump-soledad': + prepareDeferred.addCallback(dump_soledad) + prepareDeferred.chainDeferred(finalizeDeferred) + elif args.command == 'sync': + # nothing to do here, sync is already part of the chain + prepareDeferred.chainDeferred(finalizeDeferred) + elif args.command == INTEGRITY_CHECK_COMMAND: + prepareDeferred.addCallback(integrity_check) + prepareDeferred.chainDeferred(finalizeDeferred) + elif args.command == REPAIR_COMMAND: + prepareDeferred.addCallback(repair) + prepareDeferred.chainDeferred(finalizeDeferred) + else: + print 'Unsupported command: %s' % args.command + prepareDeferred.chainDeferred(finalizeDeferred) + + return finalizeDeferred + + +@defer.inlineCallbacks +def delete_all_mails(args): + leap_session, soledad = args + generation, docs = yield soledad.get_all_docs() + + for doc in docs: + if doc.content.get('type', None) in ['head', 'cnt', 'flags']: + soledad.delete_doc(doc) + + defer.returnValue(args) + + +def is_keep_file(mail): + return mail['subject'] is None + + +def _is_new_mail(mail): + return _is_maildir_msg(mail) and mail.get_subdir() == 'new' + + +def _is_maildir_msg(mail): + return isinstance(mail, MaildirMessage) + + +@defer.inlineCallbacks +def _add_mail(store, folder_name, mail, flags, tags): + created_mail = yield store.add_mail(folder_name, mail.as_string()) + leap_mail = yield store.get_mail(created_mail.mail_id) + leap_mail.tags |= set(tags) + for flag in flags: + leap_mail.flags.add(flag) + + yield store.update_mail(leap_mail) + + +@defer.inlineCallbacks +def add_mail_folder(store, mailbox, folder_name, deferreds): + yield store.add_mailbox(folder_name) + + for mail in mailbox: + if is_keep_file(mail): + continue + + if _is_maildir_msg(mail): + flags = {MessageFlags.RECENT_FLAG} if _is_new_mail(mail) else set() + + if 'S' in mail.get_flags(): + flags = flags.add(MessageFlags.SEEN_FLAG) + if 'R' in mail.get_flags(): + flags = flags.add(MessageFlags.ANSWERED_FLAG) + else: + flags = {MessageFlags.RECENT_FLAG} + + tags = mail['X-Tags'].split() if mail['X-Tags'] else [] + + deferreds.append(_add_mail(store, folder_name, mail, flags, tags)) + + +@defer.inlineCallbacks +def load_mails(args, mail_paths): + leap_session, soledad = args + store = leap_session.mail_store + + yield _load_mails_as_is(mail_paths, store) + + defer.returnValue(args) + + +@defer.inlineCallbacks +def _load_mails_as_is(mail_paths, store): + deferreds = [] + + for path in mail_paths: + if isfile(path): + mbox_mails = mbox(path, factory=None) + yield add_mail_folder(store, mbox_mails, 'INBOX', deferreds) + else: + maildir = Maildir(path, factory=None) + yield add_mail_folder(store, maildir, 'INBOX', deferreds) + for mail_folder_name in maildir.list_folders(): + mail_folder = maildir.get_folder(mail_folder_name) + yield add_mail_folder(store, mail_folder, mail_folder_name, deferreds) + + yield defer.gatherResults(deferreds, consumeErrors=True) + + +@defer.inlineCallbacks +def markov_generate(args, mail_paths, limit, seed): + leap_session, soledad = args + store = leap_session.mail_store + + username = leap_session.user_auth.username + server_name = leap_session.provider.server_name + + markov_mails = _generate_mails(limit, mail_paths, seed, server_name, username) + deferreds = [] + yield add_mail_folder(store, markov_mails, 'INBOX', deferreds) + yield defer.gatherResults(deferreds, consumeErrors=True) + + defer.returnValue(args) + + +def _generate_mails(limit, mail_paths, seed, server_name, username): + mails = [] + for path in mail_paths: + mbox_mails = mbox(path, factory=None) + mails.extend(mbox_mails) + gen = MailGenerator(username, server_name, mails, random=random.Random(seed)) + markov_mails = [gen.generate_mail() for _ in range(limit)] + return markov_mails + + +def flush_to_soledad(args, finalize): + leap_session, soledad = args + + def after_sync(_): + finalize.callback((leap_session, soledad)) + + d = soledad.sync() + d.addCallback(after_sync) + + return args + + +@defer.inlineCallbacks +def dump_soledad(args): + leap_session, soledad = args + + generation, docs = yield soledad.get_all_docs() + + for doc in docs: + print doc + print '\n' + + defer.returnValue(args) + + +@defer.inlineCallbacks +def integrity_check(args): + leap_session, soledad = args + + generation, docs = yield soledad.get_all_docs() + + known_docs = {} + + print 'Analysing %d docs\n' % len(docs) + + # learn about all docs + for doc in docs: + known_docs[doc.doc_id] = doc + + for doc in docs: + if doc.doc_id.startswith('M-'): + meta = MetaMsgDocWrapper(doc_id=doc.doc_id, **doc.content) + + # validate header doc + if meta.hdoc not in known_docs: + print 'Error: Could not find header doc %s for meta %s' % (meta.hdoc, doc.doc_id) + + if meta.fdoc not in known_docs: + print 'Error: Could not find flags doc %s for meta %s' % (meta.fdoc, doc.doc_id) + + for cdoc in meta.cdocs: + if cdoc not in known_docs: + print 'Error: Could not find content doc %s for meta %s' % (cdoc, meta.doc_id) + + defer.returnValue(args) + + +@defer.inlineCallbacks +def repair(args): + leap_session, soledad = args + + yield SoledadMaintenance(soledad).repair() + + defer.returnValue(args) + + +def shutdown(args): + # time.sleep(30) + reactor.stop() + + +def shutdown_on_error(error): + print error + shutdown(None) + +if __name__ == '__main__': + initialize() diff --git a/service/src/pixelated/register.py b/service/src/pixelated/register.py new file mode 100644 index 00000000..b6faf454 --- /dev/null +++ b/service/src/pixelated/register.py @@ -0,0 +1,115 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import getpass +import re +from collections import namedtuple + +from leap.bitmask.bonafide.provider import Api +from leap.bitmask.bonafide.session import Session +from pixelated.bitmask_libraries.certs import LeapCertificate +from pixelated.bitmask_libraries.provider import LeapProvider +from pixelated.config import arguments +from pixelated.config import leap_config +from pixelated.config import logger as logger_config +from twisted.internet import reactor +from twisted.internet.defer import inlineCallbacks +from twisted.logger import Logger + +Credentials = namedtuple('Credentials', 'username, password') + +logger = Logger() + + +def _validate(username, password): + validate_username(username) + validate_password(password) + + +def _set_provider(provider_cert, provider_cert_fingerprint, server_name, leap_home=None): + if leap_home: + leap_config.set_leap_home(leap_home) + + LeapCertificate.set_cert_and_fingerprint(provider_cert, provider_cert_fingerprint) + provider = LeapProvider(server_name) + provider.setup_ca() + provider.download_settings() + return provider + + +def _set_leap_provider(args): + return _set_provider(args.leap_provider_cert, args.leap_provider_cert_fingerprint, args.provider, args.leap_home) + + +def _bonafide_session(username, password, provider): + srp_provider = Api(provider.api_uri) + credentials = Credentials(username, password) + return Session(credentials, srp_provider, provider.local_ca_crt) + + +def log_results(created, username): + if created: + logger.info('User %s successfully registered' % username) + else: + logger.error("Register failed") + + +@inlineCallbacks +def register(username, password, leap_provider, invite=None): + if not password: + password = getpass.getpass('Please enter password for %s: ' % username) + + _validate(username, password) + logger.info('password validated...') + srp_auth = _bonafide_session(username, password, leap_provider) + + created, user = yield srp_auth.signup(username, password, invite) + log_results(created, username) + + +def validate_username(username): + accepted_characters = '^[a-z0-9\-\_\.]*$' + if not re.match(accepted_characters, username): + raise ValueError('Only lowercase letters, digits, . - and _ allowed.') + + +def validate_password(password): + if len(password) < 8: + logger.info('password not validated...') + raise ValueError('The password must have at least 8 characters') + + +def initialize(): + logger_config.init(debug=False) + args = arguments.parse_register_args() + leap_provider = _set_leap_provider(args) + + def show_error(err): + logger.info('error: %s' % err) + + def shut_down(_): + reactor.stop() + + def _register(): + d = register( + args.username, + args.password, + leap_provider, + args.invite_code) + d.addErrback(show_error) + d.addBoth(shut_down) + + reactor.callWhenRunning(_register) + reactor.run() diff --git a/service/src/pixelated/resources/__init__.py b/service/src/pixelated/resources/__init__.py new file mode 100644 index 00000000..58b56786 --- /dev/null +++ b/service/src/pixelated/resources/__init__.py @@ -0,0 +1,159 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import json +import os + +from twisted.web.http import UNAUTHORIZED +from twisted.web.resource import Resource +from twisted.logger import Logger + +from pixelated.resources.session import IPixelatedSession + +from twisted.web.http import INTERNAL_SERVER_ERROR, SERVICE_UNAVAILABLE + +log = Logger() + + +STATIC = None + + +class SetEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, set): + return list(obj) + return super(SetEncoder, self).default(obj) + + +def respond_json(entity, request, status_code=200): + json_response = json.dumps(entity, cls=SetEncoder) + request.responseHeaders.setRawHeaders(b"content-type", [b"application/json"]) + request.code = status_code + return json_response + + +def respond_json_deferred(entity, request, status_code=200): + json_response = json.dumps(entity, cls=SetEncoder) + request.responseHeaders.setRawHeaders(b"content-type", [b"application/json"]) + request.code = status_code + request.write(json_response) + request.finish() + + +def handle_error_deferred(e, request): + log.error(e) + request.setResponseCode(INTERNAL_SERVER_ERROR) + request.write('Something went wrong!') + request.finish() + + +def set_static_folder(static_folder): + global STATIC + STATIC = static_folder + + +def get_protected_static_folder(static_folder=None): + static = static_folder or _get_static_folder() + return os.path.join(static, 'protected') + + +def get_public_static_folder(static_folder=None): + static = static_folder or _get_static_folder() + return os.path.join(static, 'public') + + +def _get_static_folder(): + if not STATIC: + static_folder = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..", "..", "web-ui", "dist")) + if not os.path.exists(static_folder): + static_folder = os.path.join('/', 'usr', 'share', 'pixelated-user-agent') + else: + static_folder = STATIC + return static_folder + + +class BaseResource(Resource): + + def __init__(self, services_factory): + Resource.__init__(self) + self._services_factory = services_factory + + def _get_user_id_from_request(self, request): + if self._services_factory.mode.is_single_user: + return None # it doesn't matter + session = self.get_session(request) + if session.is_logged_in(): + return session.user_uuid + raise ValueError('Not logged in') + + def is_logged_in(self, request): + session = self.get_session(request) + return session.is_logged_in() and self._services_factory.has_session(session.user_uuid) + + def get_session(self, request): + return IPixelatedSession(request.getSession()) + + def is_admin(self, request): + services = self._services(request) + return services._leap_session.user_auth.is_admin() + + def _services(self, request): + user_id = self._get_user_id_from_request(request) + return self._services_factory.services(user_id) + + def _service(self, request, attribute): + return getattr(self._services(request), attribute) + + def keymanager(self, request): + return self._service(request, 'keymanager') + + def mail_service(self, request): + return self._service(request, 'mail_service') + + def search_engine(self, request): + return self._service(request, 'search_engine') + + def draft_service(self, request): + return self._service(request, 'draft_service') + + def feedback_service(self, request): + return self._service(request, 'feedback_service') + + def soledad(self, request): + return self._service(request, '_leap_session').soledad + + +class UnAuthorizedResource(Resource): + + def __init__(self): + Resource.__init__(self) + + def render_GET(self, request): + request.setResponseCode(UNAUTHORIZED) + return "Unauthorized!" + + def render_POST(self, request): + request.setResponseCode(UNAUTHORIZED) + return "Unauthorized!" + + +class UnavailableResource(Resource): + def __init__(self): + Resource.__init__(self) + + def render(self, request): + request.setResponseCode(SERVICE_UNAVAILABLE) + return "Service Unavailable" diff --git a/service/src/pixelated/resources/account_recovery_resource.py b/service/src/pixelated/resources/account_recovery_resource.py new file mode 100644 index 00000000..209a7693 --- /dev/null +++ b/service/src/pixelated/resources/account_recovery_resource.py @@ -0,0 +1,87 @@ +# +# Copyright (c) 2017 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import os +import json + +from twisted.python.filepath import FilePath +from twisted.web.http import OK, INTERNAL_SERVER_ERROR +from twisted.web.template import Element, XMLFile, renderElement +from twisted.web.server import NOT_DONE_YET +from twisted.internet import defer +from twisted.logger import Logger + +from pixelated.resources import BaseResource +from pixelated.resources import get_public_static_folder + +log = Logger() + + +class InvalidPasswordError(Exception): + pass + + +class AccountRecoveryPage(Element): + loader = XMLFile(FilePath(os.path.join(get_public_static_folder(), 'account_recovery.html'))) + + def __init__(self): + super(AccountRecoveryPage, self).__init__() + + +class AccountRecoveryResource(BaseResource): + BASE_URL = 'account-recovery' + isLeaf = True + + def __init__(self, services_factory): + BaseResource.__init__(self, services_factory) + + def render_GET(self, request): + request.setResponseCode(OK) + return self._render_template(request) + + def _render_template(self, request): + site = AccountRecoveryPage() + return renderElement(request, site) + + def render_POST(self, request): + def success_response(response): + request.setResponseCode(OK) + request.finish() + + def error_response(failure): + log.warn(failure) + request.setResponseCode(INTERNAL_SERVER_ERROR) + request.finish() + + d = self._handle_post(request) + d.addCallbacks(success_response, error_response) + return NOT_DONE_YET + + def _get_post_form(self, request): + return json.loads(request.content.getvalue()) + + def _validate_password(self, password, confirm_password): + return password == confirm_password and len(password) >= 8 and len(password) <= 9999 + + def _handle_post(self, request): + form = self._get_post_form(request) + password = form.get('password') + confirm_password = form.get('confirmPassword') + + if not self._validate_password(password, confirm_password): + return defer.fail(InvalidPasswordError('The user entered an invalid password or confirmation')) + + return defer.succeed('Done!') diff --git a/service/src/pixelated/resources/attachments_resource.py b/service/src/pixelated/resources/attachments_resource.py new file mode 100644 index 00000000..1081b4b8 --- /dev/null +++ b/service/src/pixelated/resources/attachments_resource.py @@ -0,0 +1,110 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import cgi +import io +import re + +from twisted.internet import defer +from twisted.protocols.basic import FileSender +from twisted.python.log import msg +from twisted.web import server +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET +from twisted.logger import Logger + +from pixelated.resources import respond_json_deferred, BaseResource + + +logger = Logger() + + +class AttachmentResource(Resource): + isLeaf = True + + def __init__(self, mail_service, attachment_id): + Resource.__init__(self) + self.attachment_id = attachment_id + self.mail_service = mail_service + + def render_GET(self, request): + def error_handler(failure): + msg(failure, 'attachment not found') + request.code = 404 + request.finish() + + encoding = request.args.get('encoding', [None])[0] + filename = request.args.get('filename', [self.attachment_id])[0] + content_type = request.args.get('content_type', ['application/octet-stream'])[0] + request.setHeader(b'Content-Type', content_type) + request.setHeader(b'Content-Disposition', bytes('attachment; filename="' + filename + '"')) + + d = self._send_attachment(encoding, filename, request) + d.addErrback(error_handler) + + return server.NOT_DONE_YET + + @defer.inlineCallbacks + def _send_attachment(self, encoding, filename, request): + attachment = yield self.mail_service.attachment(self.attachment_id) + + bytes_io = io.BytesIO(attachment['content']) + + try: + request.code = 200 + yield FileSender().beginFileTransfer(bytes_io, request) + finally: + bytes_io.close() + request.finish() + + def _extract_mimetype(self, content_type): + match = re.compile('([A-Za-z-]+\/[A-Za-z-]+)').search(content_type) + return match.group(1) + + +class AttachmentsResource(BaseResource): + BASE_URL = 'attachment' + + def __init__(self, services_factory): + BaseResource.__init__(self, services_factory) + + def getChild(self, attachment_id, request): + _mail_service = self.mail_service(request) + return AttachmentResource(_mail_service, attachment_id) + + def render_POST(self, request): + _mail_service = self.mail_service(request) + fields = cgi.FieldStorage(fp=request.content, headers=(request.getAllHeaders()), + environ={'REQUEST_METHOD': 'POST'}) + _file = fields['attachment'] + deferred = _mail_service.save_attachment(_file.value, _file.type) + + def send_location(attachment_id): + request.setHeader('Location', '/%s/%s' % (self.BASE_URL, attachment_id)) + response_json = {"ident": attachment_id, + "content-type": _file.type, + "encoding": "base64", # hard coded for now -- not really used + "name": _file.filename, + "size": len(_file.value)} + respond_json_deferred(response_json, request, status_code=201) + + def error_handler(error): + logger.error(error) + respond_json_deferred({"message": "Something went wrong. Attachment not saved."}, request, status_code=500) + + deferred.addCallback(send_location) + deferred.addErrback(error_handler) + + return NOT_DONE_YET diff --git a/service/src/pixelated/resources/auth.py b/service/src/pixelated/resources/auth.py new file mode 100644 index 00000000..adac985f --- /dev/null +++ b/service/src/pixelated/resources/auth.py @@ -0,0 +1,117 @@ +# +# Copyright (c) 2016 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import re + +from pixelated.resources import IPixelatedSession +from twisted.cred import error +from twisted.cred import portal, checkers +from twisted.cred.checkers import ANONYMOUS +from twisted.cred.credentials import ICredentials +from twisted.internet import defer +from twisted.logger import Logger +from twisted.web import util +from twisted.web._auth.wrapper import UnauthorizedResource +from twisted.web.error import UnsupportedMethod +from twisted.web.resource import IResource, ErrorPage +from zope.interface import implements, implementer, Attribute + + +log = Logger() + + +class ISessionCredential(ICredentials): + + request = Attribute('the current request') + + +@implementer(ISessionCredential) +class SessionCredential(object): + def __init__(self, request): + self.request = request + + +@implementer(checkers.ICredentialsChecker) +class SessionChecker(object): + credentialInterfaces = (ISessionCredential,) + + def __init__(self, services_factory): + self._services_factory = services_factory + + def requestAvatarId(self, credentials): + session = self.get_session(credentials.request) + if session.is_logged_in() and self._services_factory.has_session(session.user_uuid): + return defer.succeed(session.user_uuid) + return defer.succeed(ANONYMOUS) + + def get_session(self, request): + return IPixelatedSession(request.getSession()) + + +class PixelatedRealm(object): + implements(portal.IRealm) + + def requestAvatar(self, avatarId, mind, *interfaces): + if IResource in interfaces: + return IResource, avatarId, lambda: None + raise NotImplementedError() + + +@implementer(IResource) +class PixelatedAuthSessionWrapper(object): + + isLeaf = False + + def __init__(self, portal, root_resource, anonymous_resource, credentialFactories): + self._portal = portal + self._credentialFactories = credentialFactories + self._root_resource = root_resource + self._anonymous_resource = anonymous_resource + + def render(self, request): + raise UnsupportedMethod(()) + + def getChildWithDefault(self, path, request): + request.postpath.insert(0, request.prepath.pop()) + return self._authorizedResource(request) + + def _authorizedResource(self, request): + creds = SessionCredential(request) + return util.DeferredResource(self._login(creds, request)) + + def _login(self, credentials, request): + pattern = re.compile("^/sandbox/") + + def loginSucceeded(args): + interface, avatar, logout = args + if avatar == checkers.ANONYMOUS and not pattern.match(request.path): + return self._anonymous_resource + else: + return self._root_resource + + def loginFailed(result): + if result.check(error.Unauthorized, error.LoginFailed): + return UnauthorizedResource(self._credentialFactories) + else: + log.err( + result, + "HTTPAuthSessionWrapper.getChildWithDefault encountered " + "unexpected error") + return ErrorPage(500, None, None) + + d = self._portal.login(credentials, None, IResource) + d.addCallbacks(loginSucceeded, loginFailed) + return d diff --git a/service/src/pixelated/resources/backup_account_resource.py b/service/src/pixelated/resources/backup_account_resource.py new file mode 100644 index 00000000..94129122 --- /dev/null +++ b/service/src/pixelated/resources/backup_account_resource.py @@ -0,0 +1,79 @@ +# +# Copyright (c) 2017 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import os +import json + +from twisted.python.filepath import FilePath +from twisted.web.http import OK, NO_CONTENT, INTERNAL_SERVER_ERROR +from twisted.web.server import NOT_DONE_YET +from twisted.web.template import Element, XMLFile, renderElement + +from pixelated.resources import BaseResource +from pixelated.resources import get_protected_static_folder +from pixelated.account_recovery import AccountRecovery +from pixelated.support.language import parse_accept_language + + +class BackupAccountPage(Element): + loader = XMLFile(FilePath(os.path.join(get_protected_static_folder(), 'backup_account.html'))) + + def __init__(self): + super(BackupAccountPage, self).__init__() + + +class BackupAccountResource(BaseResource): + isLeaf = True + + def __init__(self, services_factory, authenticator, leap_provider): + BaseResource.__init__(self, services_factory) + self._authenticator = authenticator + self._leap_provider = leap_provider + + def render_GET(self, request): + request.setResponseCode(OK) + return self._render_template(request) + + def _render_template(self, request): + site = BackupAccountPage() + return renderElement(request, site) + + def render_POST(self, request): + account_recovery = AccountRecovery( + self._authenticator.bonafide_session, + self.soledad(request), + self._service(request, '_leap_session').smtp_config, + self._get_backup_email(request), + self._leap_provider.server_name, + language=self._get_language(request)) + + def update_response(response): + request.setResponseCode(NO_CONTENT) + request.finish() + + def error_response(response): + request.setResponseCode(INTERNAL_SERVER_ERROR) + request.finish() + + d = account_recovery.update_recovery_code() + d.addCallbacks(update_response, error_response) + return NOT_DONE_YET + + def _get_backup_email(self, request): + return json.loads(request.content.getvalue()).get('backupEmail') + + def _get_language(self, request): + return parse_accept_language(request.getAllHeaders()) diff --git a/service/src/pixelated/resources/contacts_resource.py b/service/src/pixelated/resources/contacts_resource.py new file mode 100644 index 00000000..dc17d1ac --- /dev/null +++ b/service/src/pixelated/resources/contacts_resource.py @@ -0,0 +1,44 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from pixelated.resources import respond_json_deferred, BaseResource +from twisted.internet.threads import deferToThread +from twisted.web import server +from twisted.web.resource import Resource + + +class ContactsResource(BaseResource): + + isLeaf = True + + def __init__(self, services_factory): + BaseResource.__init__(self, services_factory) + + def render_GET(self, request): + _search_engine = self.search_engine(request) + query = request.args.get('q', [''])[-1] + d = deferToThread(lambda: _search_engine.contacts(query)) + d.addCallback(lambda tags: respond_json_deferred(tags, request)) + + def handle_error(error): + print 'Something went wrong' + import traceback + traceback.print_exc() + print error + + d.addErrback(handle_error) + + return server.NOT_DONE_YET diff --git a/service/src/pixelated/resources/features_resource.py b/service/src/pixelated/resources/features_resource.py new file mode 100644 index 00000000..c1b61f12 --- /dev/null +++ b/service/src/pixelated/resources/features_resource.py @@ -0,0 +1,46 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from pixelated.resources import respond_json +import os +from twisted.web.resource import Resource + +from pixelated.resources.logout_resource import LogoutResource + + +class FeaturesResource(Resource): + DISABLED_FEATURES = ['draftReply'] + isLeaf = True + + def __init__(self, multi_user=False): + Resource.__init__(self) + self._multi_user = multi_user + + def render_GET(self, request): + disabled_features = self._disabled_features() + features = {'disabled_features': disabled_features} + self._add_multi_user_to(features) + return respond_json(features, request) + + def _disabled_features(self): + disabled_features = [default_disabled_feature for default_disabled_feature in self.DISABLED_FEATURES] + if not os.environ.get('FEEDBACK_URL'): + disabled_features.append('feedback') + return disabled_features + + def _add_multi_user_to(self, features): + if self._multi_user: + features.update({'multi_user': {'logout': LogoutResource.BASE_URL}}) diff --git a/service/src/pixelated/resources/feedback_resource.py b/service/src/pixelated/resources/feedback_resource.py new file mode 100644 index 00000000..aeead401 --- /dev/null +++ b/service/src/pixelated/resources/feedback_resource.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import json + +from pixelated.resources import respond_json, BaseResource + + +class FeedbackResource(BaseResource): + isLeaf = True + + def __init__(self, services_factory): + BaseResource.__init__(self, services_factory) + + def render_POST(self, request): + _feedback_service = self.feedback_service(request) + feedback = json.loads(request.content.read()).get('feedback') + _feedback_service.open_ticket(feedback) + return respond_json({}, request) diff --git a/service/src/pixelated/resources/keys_resource.py b/service/src/pixelated/resources/keys_resource.py new file mode 100644 index 00000000..091c27d0 --- /dev/null +++ b/service/src/pixelated/resources/keys_resource.py @@ -0,0 +1,46 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from email.utils import parseaddr +from pixelated.resources import respond_json_deferred, BaseResource +from twisted.web import server + + +class KeysResource(BaseResource): + + isLeaf = True + + def __init__(self, services_factory): + BaseResource.__init__(self, services_factory) + + def render_GET(self, request): + _keymanager = self.keymanager(request) + + def finish_request(key): + if key.private: + respond_json_deferred(None, request, status_code=401) + else: + respond_json_deferred(key.get_active_json(), request) + + def key_not_found(_): + respond_json_deferred(None, request, status_code=404) + + _, key_to_find = parseaddr(request.args.get('search')[0]) + d = _keymanager.get_key(key_to_find) + d.addCallback(finish_request) + d.addErrback(key_not_found) + + return server.NOT_DONE_YET diff --git a/service/src/pixelated/resources/login_resource.py b/service/src/pixelated/resources/login_resource.py new file mode 100644 index 00000000..5b0b70d0 --- /dev/null +++ b/service/src/pixelated/resources/login_resource.py @@ -0,0 +1,173 @@ +# +# Copyright (c) 2016 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import os +from xml.sax import SAXParseException + +from pixelated.authentication import Authenticator +from pixelated.config.leap import BootstrapUserServices +from pixelated.resources import BaseResource, UnAuthorizedResource, IPixelatedSession +from pixelated.resources.account_recovery_resource import AccountRecoveryResource +from pixelated.resources import get_public_static_folder, respond_json +from pixelated.support.language import parse_accept_language + +from twisted.cred.error import UnauthorizedLogin +from twisted.internet import defer +from twisted.logger import Logger +from twisted.python.filepath import FilePath +from twisted.web import util +from twisted.web.http import UNAUTHORIZED, OK +from twisted.web.resource import NoResource +from twisted.web.server import NOT_DONE_YET +from twisted.web.static import File +from twisted.web.template import Element, XMLFile, renderElement, renderer + +log = Logger() + + +class DisclaimerElement(Element): + loader = XMLFile(FilePath(os.path.join(get_public_static_folder(), '_login_disclaimer_banner.html'))) + + def __init__(self, banner): + super(DisclaimerElement, self).__init__() + self._set_loader(banner) + self._banner_filename = banner or "_login_disclaimer_banner.html" + + def _set_loader(self, banner): + if banner: + current_path = os.path.dirname(os.path.abspath(__file__)) + banner_file_path = os.path.join(current_path, "..", "..", "..", banner) + self.loader = XMLFile(FilePath(banner_file_path)) + + def render(self, request): + try: + return super(DisclaimerElement, self).render(request) + except SAXParseException: + return ["Invalid XML template format for %s." % self._banner_filename] + except IOError: + return ["Disclaimer banner file %s could not be read or does not exit." % self._banner_filename] + + +class LoginWebSite(Element): + loader = XMLFile(FilePath(os.path.join(get_public_static_folder(), 'login.html'))) + + def __init__(self, disclaimer_banner_file=None): + super(LoginWebSite, self).__init__() + self.disclaimer_banner_file = disclaimer_banner_file + + @renderer + def disclaimer(self, request, tag): + return DisclaimerElement(self.disclaimer_banner_file).render(request) + + +class LoginResource(BaseResource): + BASE_URL = 'login' + + def __init__(self, services_factory, provider=None, disclaimer_banner=None, authenticator=None): + BaseResource.__init__(self, services_factory) + self._disclaimer_banner = disclaimer_banner + self._provider = provider + self._authenticator = authenticator + self._bootstrap_user_services = BootstrapUserServices(services_factory, provider) + + static_folder = get_public_static_folder() + self.putChild('public', File(static_folder)) + with open(os.path.join(static_folder, 'interstitial.html')) as f: + self.interstitial = f.read() + + def getChild(self, path, request): + if path == '': + return self + if path == 'login': + return self + if path == 'status': + return LoginStatusResource(self._services_factory) + if path == AccountRecoveryResource.BASE_URL: + return AccountRecoveryResource(self._services_factory) + if not self.is_logged_in(request): + return UnAuthorizedResource() + return NoResource() + + def render_GET(self, request): + request.setResponseCode(OK) + return self._render_template(request) + + def _render_template(self, request): + site = LoginWebSite(disclaimer_banner_file=self._disclaimer_banner) + return renderElement(request, site) + + def render_POST(self, request): + if self.is_logged_in(request): + return util.redirectTo("/", request) + + def render_response(user_auth): + request.setResponseCode(OK) + request.write(self.interstitial) + request.finish() + self._complete_bootstrap(user_auth, request) + + def render_error(error): + if error.type is UnauthorizedLogin: + log.info('Unauthorized login for %s. User typed wrong username/password combination.' % request.args['username'][0]) + else: + log.error('Authentication error for %s' % request.args['username'][0]) + log.error('%s' % error) + request.setResponseCode(UNAUTHORIZED) + content = util.redirectTo("/login?auth-error", request) + request.write(content) + request.finish() + + d = self._handle_login(request) + d.addCallbacks(render_response, render_error) + return NOT_DONE_YET + + @defer.inlineCallbacks + def _handle_login(self, request): + username = request.args['username'][0] + password = request.args['password'][0] + user_auth = yield self._authenticator.authenticate(username, password) + defer.returnValue(user_auth) + + def _complete_bootstrap(self, user_auth, request): + def login_error(error, session): + log.error('Login error during %s services setup: %s \n %s' % (user_auth.username, error.getErrorMessage(), error.getTraceback())) + session.login_error() + + def login_successful(_, session): + session.login_successful(user_auth.uuid) + + language = parse_accept_language(request.getAllHeaders()) + password = request.args['password'][0] + session = IPixelatedSession(request.getSession()) + session.login_started() + + d = self._bootstrap_user_services.setup(user_auth, password, language) + d.addCallback(login_successful, session) + d.addErrback(login_error, session) + + +class LoginStatusResource(BaseResource): + isLeaf = True + + def __init__(self, services_factory): + BaseResource.__init__(self, services_factory) + + def render_GET(self, request): + session = IPixelatedSession(request.getSession()) + status = 'completed' if self._services_factory.mode.is_single_user else str(session.check_login_status()) + + response = {'status': status} + return respond_json(response, request) diff --git a/service/src/pixelated/resources/logout_resource.py b/service/src/pixelated/resources/logout_resource.py new file mode 100644 index 00000000..a4fe584f --- /dev/null +++ b/service/src/pixelated/resources/logout_resource.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from twisted.internet import defer +from twisted.web import util +from twisted.web.server import NOT_DONE_YET + +from pixelated.resources import BaseResource, handle_error_deferred +from pixelated.resources.login_resource import LoginResource + + +class LogoutResource(BaseResource): + BASE_URL = "logout" + isLeaf = True + + @defer.inlineCallbacks + def _execute_logout(self, request): + http_session = self.get_session(request) + yield self._services_factory.destroy_session(http_session.user_uuid) + http_session.expire() + + def render_POST(self, request): + def _redirect_to_login(_): + content = util.redirectTo("/%s" % LoginResource.BASE_URL, request) + request.write(content) + request.finish() + + d = self._execute_logout(request) + d.addCallback(_redirect_to_login) + d.addErrback(handle_error_deferred, request) + + return NOT_DONE_YET diff --git a/service/src/pixelated/resources/mail_resource.py b/service/src/pixelated/resources/mail_resource.py new file mode 100644 index 00000000..e1ba6087 --- /dev/null +++ b/service/src/pixelated/resources/mail_resource.py @@ -0,0 +1,92 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import json + +from twisted.python.log import err +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET + +from pixelated.resources import respond_json_deferred, BaseResource, handle_error_deferred +from pixelated.support import replier + + +class MailTags(Resource): + + isLeaf = True + + def __init__(self, mail_id, mail_service): + Resource.__init__(self) + self._mail_service = mail_service + self._mail_id = mail_id + + def render_POST(self, request): + new_tags = json.loads(request.content.read()).get('newtags') + + d = self._mail_service.update_tags(self._mail_id, new_tags) + d.addCallback(lambda mail: respond_json_deferred(mail.as_dict(), request)) + + def handle403(failure): + failure.trap(ValueError) + return respond_json_deferred(failure.getErrorMessage(), request, 403) + d.addErrback(handle403) + return NOT_DONE_YET + + +class Mail(Resource): + + def __init__(self, mail_id, mail_service): + Resource.__init__(self) + self.putChild('tags', MailTags(mail_id, mail_service)) + self._mail_id = mail_id + self._mail_service = mail_service + + def render_GET(self, request): + def populate_reply(mail): + mail_dict = mail.as_dict() + current_user = self._mail_service.account_email + sender = mail.headers.get('Reply-to', mail.headers.get('From')) + to = mail.headers.get('To', []) + ccs = mail.headers.get('Cc', []) + mail_dict['replying'] = replier.generate_recipients(sender, to, ccs, current_user) + return mail_dict + + d = self._mail_service.mail(self._mail_id) + d.addCallback(lambda mail: populate_reply(mail)) + d.addCallback(lambda mail_dict: respond_json_deferred(mail_dict, request)) + d.addErrback(handle_error_deferred, request) + + return NOT_DONE_YET + + def render_DELETE(self, request): + def response_failed(failure): + err(failure, 'something failed') + request.finish() + + d = self._mail_service.delete_mail(self._mail_id) + d.addCallback(lambda _: respond_json_deferred(None, request)) + d.addErrback(response_failed) + return NOT_DONE_YET + + +class MailResource(BaseResource): + + def __init__(self, services_factory): + BaseResource.__init__(self, services_factory) + + def getChild(self, mail_id, request): + _mail_service = self.mail_service(request) + return Mail(mail_id, _mail_service) diff --git a/service/src/pixelated/resources/mails_resource.py b/service/src/pixelated/resources/mails_resource.py new file mode 100644 index 00000000..d911e0d2 --- /dev/null +++ b/service/src/pixelated/resources/mails_resource.py @@ -0,0 +1,244 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import time +import json + +from twisted.internet import defer +from twisted.logger import Logger +from twisted.web.server import NOT_DONE_YET +from twisted.web.resource import Resource +from twisted.web import server + +from leap.common import events + +from pixelated.adapter.model.mail import InputMail +from pixelated.resources import respond_json_deferred, BaseResource +from pixelated.adapter.services.mail_sender import SMTPDownException +from pixelated.support.functional import to_unicode + + +log = Logger() + + +class MailsUnreadResource(Resource): + isLeaf = True + + def __init__(self, mail_service): + Resource.__init__(self) + self._mail_service = mail_service + + def render_POST(self, request): + idents = json.load(request.content).get('idents') + deferreds = [] + for ident in idents: + deferreds.append(self._mail_service.mark_as_unread(ident)) + + d = defer.gatherResults(deferreds, consumeErrors=True) + d.addCallback(lambda _: respond_json_deferred(None, request)) + d.addErrback(lambda _: respond_json_deferred(None, request, status_code=500)) + + return NOT_DONE_YET + + +class MailsReadResource(Resource): + isLeaf = True + + def __init__(self, mail_service): + Resource.__init__(self) + self._mail_service = mail_service + + def render_POST(self, request): + idents = json.load(request.content).get('idents') + deferreds = [] + for ident in idents: + deferreds.append(self._mail_service.mark_as_read(ident)) + + d = defer.gatherResults(deferreds, consumeErrors=True) + d.addCallback(lambda _: respond_json_deferred(None, request)) + d.addErrback(lambda _: respond_json_deferred(None, request, status_code=500)) + + return NOT_DONE_YET + + +class MailsDeleteResource(Resource): + isLeaf = True + + def __init__(self, mail_service): + Resource.__init__(self) + self._mail_service = mail_service + + def render_POST(self, request): + def response_failed(failure): + log.error('something failed: %s' % failure.getErrorMessage()) + request.finish() + + idents = json.loads(request.content.read())['idents'] + deferreds = [] + for ident in idents: + deferreds.append(self._mail_service.delete_mail(ident)) + + d = defer.gatherResults(deferreds, consumeErrors=True) + d.addCallback(lambda _: respond_json_deferred(None, request)) + d.addErrback(response_failed) + return NOT_DONE_YET + + +class MailsRecoverResource(Resource): + isLeaf = True + + def __init__(self, mail_service): + Resource.__init__(self) + self._mail_service = mail_service + + def render_POST(self, request): + idents = json.loads(request.content.read())['idents'] + deferreds = [] + for ident in idents: + deferreds.append(self._mail_service.recover_mail(ident)) + d = defer.gatherResults(deferreds, consumeErrors=True) + d.addCallback(lambda _: respond_json_deferred(None, request)) + d.addErrback(lambda _: respond_json_deferred(None, request, status_code=500)) + return NOT_DONE_YET + + +class MailsArchiveResource(Resource): + isLeaf = True + + def __init__(self, mail_service): + Resource.__init__(self) + self._mail_service = mail_service + + def render_POST(self, request): + idents = json.loads(request.content.read())['idents'] + deferreds = [] + for ident in idents: + deferreds.append(self._mail_service.archive_mail(ident)) + d = defer.gatherResults(deferreds, consumeErrors=True) + d.addCallback(lambda _: respond_json_deferred({'successMessage': 'your-message-was-archived'}, request)) + d.addErrback(lambda _: respond_json_deferred(None, request, status_code=500)) + return NOT_DONE_YET + + +class MailsResource(BaseResource): + + def _register_smtp_error_handler(self): + + def on_error(event, content): + delivery_error_mail = InputMail.delivery_error_template(delivery_address=event.content) + self._mail_service.mailboxes.inbox.add(delivery_error_mail) + + events.register(events.catalog.SMTP_SEND_MESSAGE_ERROR, callback=on_error) + + def __init__(self, services_factory): + BaseResource.__init__(self, services_factory) + self._register_smtp_error_handler() + + def getChild(self, action, request): + _mail_service = self.mail_service(request) + + if action == 'delete': + return MailsDeleteResource(_mail_service) + if action == 'recover': + return MailsRecoverResource(_mail_service) + if action == 'archive': + return MailsArchiveResource(_mail_service) + if action == 'read': + return MailsReadResource(_mail_service) + if action == 'unread': + return MailsUnreadResource(_mail_service) + + def _build_mails_response(self, (mails, total)): + return { + "stats": { + "total": total, + }, + "mails": [mail.as_dict() for mail in mails] + } + + def render_GET(self, request): + + _mail_service = self.mail_service(request) + query, window_size, page = request.args.get('q')[0], request.args.get('w')[0], request.args.get('p')[0] + unicode_query = to_unicode(query) + d = _mail_service.mails(unicode_query, window_size, page) + + d.addCallback(self._build_mails_response) + d.addCallback(lambda res: respond_json_deferred(res, request)) + + def error_handler(error): + print error + + d.addErrback(error_handler) + + return NOT_DONE_YET + + def render_POST(self, request): + def onError(error): + if isinstance(error.value, SMTPDownException): + respond_json_deferred({'message': str(error.value)}, request, status_code=503) + else: + log.error('error occurred while sending: %s' % error.getErrorMessage()) + respond_json_deferred({'message': 'an error occurred while sending'}, request, status_code=422) + + deferred = self._handle_post(request) + deferred.addErrback(onError) + + return server.NOT_DONE_YET + + def render_PUT(self, request): + def onError(error): + log.error('error saving draft: %s' % error.getErrorMessage()) + respond_json_deferred("", request, status_code=422) + + deferred = self._handle_put(request) + deferred.addErrback(onError) + + return server.NOT_DONE_YET + + @defer.inlineCallbacks + def _fetch_attachment_contents(self, content_dict, _mail_service): + attachments = content_dict.get('attachments', []) if content_dict else [] + for attachment in attachments: + retrieved_attachment = yield _mail_service.attachment(attachment['ident']) + attachment['raw'] = retrieved_attachment['content'] + content_dict['attachments'] = attachments + defer.returnValue(content_dict) + + @defer.inlineCallbacks + def _handle_post(self, request): + _mail_service = self.mail_service(request) + content_dict = json.loads(request.content.read()) + with_attachment_content = yield self._fetch_attachment_contents(content_dict, _mail_service) + + sent_mail = yield _mail_service.send_mail(with_attachment_content) + respond_json_deferred(sent_mail.as_dict(), request, status_code=201) + + @defer.inlineCallbacks + def _handle_put(self, request): + _draft_service = self.draft_service(request) + _mail_service = self.mail_service(request) + content_dict = json.loads(request.content.read()) + with_attachment_content = yield self._fetch_attachment_contents(content_dict, _mail_service) + + _mail = InputMail.from_dict(with_attachment_content, from_address=_mail_service.account_email) + draft_id = content_dict.get('ident') + pixelated_mail = yield _draft_service.process_draft(draft_id, _mail) + + if not pixelated_mail: + respond_json_deferred("", request, status_code=422) + else: + respond_json_deferred({'ident': pixelated_mail.ident}, request) diff --git a/service/src/pixelated/resources/root_resource.py b/service/src/pixelated/resources/root_resource.py new file mode 100644 index 00000000..b014a590 --- /dev/null +++ b/service/src/pixelated/resources/root_resource.py @@ -0,0 +1,139 @@ +# +# Copyright (c) 2016 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import hashlib +import json +import os +from string import Template +from pixelated.resources.users import UsersResource + +from pixelated.resources import BaseResource, UnAuthorizedResource, UnavailableResource +from pixelated.resources import get_public_static_folder, get_protected_static_folder +from pixelated.resources.attachments_resource import AttachmentsResource +from pixelated.resources.sandbox_resource import SandboxResource +from pixelated.resources.account_recovery_resource import AccountRecoveryResource +from pixelated.resources.backup_account_resource import BackupAccountResource +from pixelated.resources.contacts_resource import ContactsResource +from pixelated.resources.features_resource import FeaturesResource +from pixelated.resources.feedback_resource import FeedbackResource +from pixelated.resources.login_resource import LoginResource, LoginStatusResource +from pixelated.resources.logout_resource import LogoutResource +from pixelated.resources.user_settings_resource import UserSettingsResource +from pixelated.resources.mail_resource import MailResource +from pixelated.resources.mails_resource import MailsResource +from pixelated.resources.tags_resource import TagsResource +from pixelated.resources.keys_resource import KeysResource +from twisted.web.resource import NoResource +from twisted.web.static import File + +from twisted.logger import Logger + +log = Logger() + + +CSRF_TOKEN_LENGTH = 32 + +MODE_STARTUP = 1 +MODE_RUNNING = 2 + + +class RootResource(BaseResource): + def __init__(self, services_factory, static_folder=None): + BaseResource.__init__(self, services_factory) + self._public_static_folder = get_public_static_folder(static_folder) + self._protected_static_folder = get_protected_static_folder(static_folder) + self._html_template = open(os.path.join(self._protected_static_folder, 'index.html')).read() + self._services_factory = services_factory + self._child_resources = ChildResourcesMap() + with open(os.path.join(self._public_static_folder, 'interstitial.html')) as f: + self.interstitial = f.read() + self._startup_mode() + + def _startup_mode(self): + self.putChild('public', File(self._public_static_folder)) + self.putChild('status', LoginStatusResource(self._services_factory)) + self._mode = MODE_STARTUP + + def getChild(self, path, request): + if path == '': + return self + if self._mode == MODE_STARTUP: + return UnavailableResource() + if self._is_xsrf_valid(request): + return self._child_resources.get(path) + return UnAuthorizedResource() + + def _is_xsrf_valid(self, request): + get_request = (request.method == 'GET') + if get_request: + return True + + xsrf_token = request.getCookie('XSRF-TOKEN') + + ajax_request = (request.getHeader('x-requested-with') == 'XMLHttpRequest') + if ajax_request: + xsrf_header = request.getHeader('x-xsrf-token') + return xsrf_header and xsrf_header == xsrf_token + + csrf_input = request.args.get('csrftoken', [None])[0] or json.loads(request.content.read()).get('csrftoken', [None])[0] + return csrf_input and csrf_input == xsrf_token + + def initialize(self, provider=None, disclaimer_banner=None, authenticator=None): + self._child_resources.add('assets', File(self._protected_static_folder)) + self._child_resources.add(AccountRecoveryResource.BASE_URL, AccountRecoveryResource(self._services_factory)) + self._child_resources.add('backup-account', BackupAccountResource(self._services_factory, authenticator, provider)) + self._child_resources.add('sandbox', SandboxResource(self._protected_static_folder)) + self._child_resources.add('keys', KeysResource(self._services_factory)) + self._child_resources.add(AttachmentsResource.BASE_URL, AttachmentsResource(self._services_factory)) + self._child_resources.add('contacts', ContactsResource(self._services_factory)) + self._child_resources.add('features', FeaturesResource(provider)) + self._child_resources.add('tags', TagsResource(self._services_factory)) + self._child_resources.add('mails', MailsResource(self._services_factory)) + self._child_resources.add('mail', MailResource(self._services_factory)) + self._child_resources.add('feedback', FeedbackResource(self._services_factory)) + self._child_resources.add('user-settings', UserSettingsResource(self._services_factory)) + self._child_resources.add('users', UsersResource(self._services_factory)) + self._child_resources.add(LoginResource.BASE_URL, + LoginResource(self._services_factory, provider, disclaimer_banner=disclaimer_banner, authenticator=authenticator)) + self._child_resources.add(LogoutResource.BASE_URL, LogoutResource(self._services_factory)) + + self._mode = MODE_RUNNING + + def _is_starting(self): + return self._mode == MODE_STARTUP + + def _add_csrf_cookie(self, request): + csrf_token = hashlib.sha256(os.urandom(CSRF_TOKEN_LENGTH)).hexdigest() + request.addCookie('XSRF-TOKEN', csrf_token) + + def render_GET(self, request): + self._add_csrf_cookie(request) + if self._is_starting(): + return self.interstitial + else: + account_email = self.mail_service(request).account_email + response = Template(self._html_template).safe_substitute(account_email=account_email) + return str(response) + + +class ChildResourcesMap(object): + def __init__(self): + self._registry = {} + + def add(self, path, resource): + self._registry[path] = resource + + def get(self, path): + return self._registry.get(path) or NoResource() diff --git a/service/src/pixelated/resources/sandbox_resource.py b/service/src/pixelated/resources/sandbox_resource.py new file mode 100644 index 00000000..35f99774 --- /dev/null +++ b/service/src/pixelated/resources/sandbox_resource.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2016 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from twisted.web.static import File + + +class SandboxResource(File): + CSP_HEADER_VALUES = "sandbox allow-popups allow-scripts;" \ + "default-src 'self';" \ + "style-src *;" \ + "script-src *;" \ + "font-src *;" \ + "img-src *;" \ + "object-src 'none';" \ + "connect-src 'none';" + + def render_GET(self, request): + request.setHeader('Content-Security-Policy', self.CSP_HEADER_VALUES) + request.setHeader('X-Content-Security-Policy', self.CSP_HEADER_VALUES) + request.setHeader('X-Webkit-CSP', self.CSP_HEADER_VALUES) + request.setHeader('Access-Control-Allow-Origin', '*') + request.setHeader('Access-Control-Allow-Methods', 'GET') + + return super(SandboxResource, self).render_GET(request) diff --git a/service/src/pixelated/resources/session.py b/service/src/pixelated/resources/session.py new file mode 100644 index 00000000..5dfa52e6 --- /dev/null +++ b/service/src/pixelated/resources/session.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2016 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from zope.interface import Interface, Attribute, implements +from twisted.python.components import registerAdapter +from twisted.web.server import Session + + +class IPixelatedSession(Interface): + user_uuid = Attribute('The uuid of the currently logged in user') + login_status = Attribute('The status during user login') + + +class PixelatedSession(object): + implements(IPixelatedSession) + + def __init__(self, session): + self.user_uuid = None + self.login_status = None + + def is_logged_in(self): + return self.user_uuid is not None + + def expire(self): + self.user_uuid = None + self.login_status = None + + def login_started(self): + self.login_status = 'started' + + def login_successful(self, user_uuid): + self.user_uuid = user_uuid + self.login_status = 'completed' + + def login_error(self): + self.login_status = 'error' + + def check_login_status(self): + return self.login_status + + +registerAdapter(PixelatedSession, Session, IPixelatedSession) diff --git a/service/src/pixelated/resources/tags_resource.py b/service/src/pixelated/resources/tags_resource.py new file mode 100644 index 00000000..4cea4ca7 --- /dev/null +++ b/service/src/pixelated/resources/tags_resource.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from pixelated.resources import respond_json_deferred, BaseResource, handle_error_deferred +from twisted.internet.threads import deferToThread +from twisted.web.server import NOT_DONE_YET + + +class TagsResource(BaseResource): + + isLeaf = True + + def __init__(self, services_factory): + BaseResource.__init__(self, services_factory) + + def render_GET(self, request): + _search_engine = self.search_engine(request) + query = request.args.get('q', [''])[0] + skip_default_tags = request.args.get('skipDefaultTags', [False])[0] + + d = deferToThread(lambda: _search_engine.tags(query=query, skip_default_tags=skip_default_tags)) + d.addCallback(lambda tags: respond_json_deferred(tags, request)) + d.addErrback(handle_error_deferred, request) + + return NOT_DONE_YET diff --git a/service/src/pixelated/resources/user_settings_resource.py b/service/src/pixelated/resources/user_settings_resource.py new file mode 100644 index 00000000..04b434bd --- /dev/null +++ b/service/src/pixelated/resources/user_settings_resource.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from pixelated.resources import respond_json_deferred, BaseResource +from twisted.web import server + +FINGERPRINT_NOT_FOUND = 'Fingerprint not found' + + +class UserSettingsResource(BaseResource): + isLeaf = True + + def __init__(self, services_factory): + BaseResource.__init__(self, services_factory) + + def render_GET(self, request): + _account_email = self.mail_service(request).account_email + + def finish_request(key): + _fingerprint = key.fingerprint + respond_json_deferred({'account_email': _account_email, 'fingerprint': _fingerprint}, request) + + def key_not_found(_): + respond_json_deferred({'account_email': _account_email, 'fingerprint': FINGERPRINT_NOT_FOUND}, request) + + d = self.keymanager(request).get_key(_account_email) + d.addCallback(finish_request) + d.addErrback(key_not_found) + + return server.NOT_DONE_YET diff --git a/service/src/pixelated/resources/users.py b/service/src/pixelated/resources/users.py new file mode 100644 index 00000000..a3e6118e --- /dev/null +++ b/service/src/pixelated/resources/users.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2016 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from pixelated.resources import respond_json_deferred, BaseResource, respond_json, UnAuthorizedResource +from twisted.web import server + + +class UsersResource(BaseResource): + isLeaf = True + + def __init__(self, services_factory): + BaseResource.__init__(self, services_factory) + + def render_GET(self, request): + if self.is_admin(request): + return respond_json({"count": self._services_factory.online_sessions()}, request) + return UnAuthorizedResource().render_GET(request) diff --git a/service/src/pixelated/support/__init__.py b/service/src/pixelated/support/__init__.py new file mode 100644 index 00000000..0685f48d --- /dev/null +++ b/service/src/pixelated/support/__init__.py @@ -0,0 +1,76 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import time +from functools import wraps +from twisted.internet import defer +from twisted.logger import Logger + + +log = Logger() + + +def _start_stopwatch(): + return (time.time(), time.clock()) + + +def _stop_stopwatch(start): + start_time, start_clock = start + end_clock = time.clock() + end_time = time.time() + clock_duration = end_clock - start_clock + time_duration = end_time - start_time + if time_duration < 0.00000001: # avoid division by zero + time_duration = 0.00000001 + + estimate_percent_io = ((time_duration - clock_duration) / time_duration) * 100.0 + + return time_duration, clock_duration, estimate_percent_io + + +def log_time(f): + + @wraps(f) + def wrapper(*args, **kwds): + start = _start_stopwatch() + + result = f(*args, **kwds) + + time_duration, clock_duration, estimate_percent_io = _stop_stopwatch(start) + log.info('Needed %fs (%fs cpu time, %.2f%% spent outside process) to execute %s' % (time_duration, clock_duration, estimate_percent_io, f)) + + return result + + return wrapper + + +def log_time_deferred(f): + + def log_time(result, start): + time_duration, clock_duration, estimate_percent_io = _stop_stopwatch(start) + log.info('after callback: Needed %fs (%fs cpu time, %.2f%% spent outside process) to execute %s' % (time_duration, clock_duration, estimate_percent_io, f)) + return result + + @wraps(f) + def wrapper(*args, **kwds): + start = _start_stopwatch() + result = f(*args, **kwds) + if isinstance(result, defer.Deferred): + result.addCallback(log_time, start=start) + else: + log.warn('No Deferred returned, perhaps need to re-order annotations?') + return result + + return wrapper diff --git a/service/src/pixelated/support/clock.py b/service/src/pixelated/support/clock.py new file mode 100644 index 00000000..9cab8857 --- /dev/null +++ b/service/src/pixelated/support/clock.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from datetime import datetime +from os.path import expanduser + + +class Clock(): + + def __init__(self, label, user=None): + self.start = datetime.now() + self.label = label + self.user = user + + def stop(self, fresh=False, user=None): + end = datetime.now() + with open(expanduser('~/MetricsTime'), 'a') as f: + flag = ' fresh-account' if fresh else '' + f.write('{} {:.5f} {} {}\n'.format((self.user or user or 'Unknown'), (end - self.start).total_seconds(), self.label, flag)) diff --git a/service/src/pixelated/support/date.py b/service/src/pixelated/support/date.py new file mode 100644 index 00000000..0012aeea --- /dev/null +++ b/service/src/pixelated/support/date.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import datetime + +import dateutil.parser +from email.utils import formatdate +from dateutil.tz import tzlocal + + +def iso_now(): + return datetime.datetime.now(tzlocal()).isoformat() + + +def mail_date_now(): + date = dateutil.parser.parse(iso_now()) + return formatdate(float(date.strftime('%s'))) diff --git a/service/src/pixelated/support/encrypted_file_storage.py b/service/src/pixelated/support/encrypted_file_storage.py new file mode 100644 index 00000000..a1dbffa8 --- /dev/null +++ b/service/src/pixelated/support/encrypted_file_storage.py @@ -0,0 +1,147 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import with_statement + +import hmac +import io +import os +from hashlib import sha256 + +from leap.soledad.client.crypto import decrypt_sym +from leap.soledad.client.crypto import encrypt_sym +from whoosh.filedb.filestore import FileStorage +from whoosh.filedb.structfile import BufferFile, StructFile +from whoosh.util import random_name + + +class DelayedCloseBytesIO(io.BytesIO): + def __init__(self, name): + super(DelayedCloseBytesIO, self).__init__() + self._name = name + self.shouldClose = False + + def close(self): + + self.shouldClose = True + + def explicit_close(self): + super(DelayedCloseBytesIO, self).close() + + +class DelayedCloseStructFile(StructFile): + def __init__(self, fileobj, name=None, onclose=None): + super(DelayedCloseStructFile, self).__init__(fileobj, name, onclose) + + def close(self): + """Closes the wrapped file. + """ + + if self.is_closed: + raise Exception("This file is already closed") + if self.onclose: + self.onclose(self) + if hasattr(self.file, "explicit_close"): + self.file.explicit_close() + self.is_closed = True + + +class EncryptedFileStorage(FileStorage): + def __init__(self, path, masterkey=None): + FileStorage.__init__(self, path, supports_mmap=False) + self.masterkey = masterkey[:32] + self.signkey = masterkey[32:] + self._tmp_storage = self.temp_storage + self.length_cache = {} + self._open_files = {} + + def open_file(self, name, **kwargs): + return self._open_encrypted_file(name) + + def create_file(self, name, excl=False, mode="w+b", **kwargs): + f = DelayedCloseStructFile(DelayedCloseBytesIO(name), name=name, onclose=self._encrypt_index_on_close(name)) + f.is_real = False + self._open_files[name] = f + return f + + def delete_file(self, name): + super(EncryptedFileStorage, self).delete_file(name) + if name in self._open_files: + del self._open_files[name] + + def temp_storage(self, name=None): + name = name or "%s.tmp" % random_name() + path = os.path.join(self.folder, name) + return EncryptedFileStorage(path, self.masterkey).create() + + def file_length(self, name): + return self.length_cache[name][0] + + def gen_mac(self, iv, ciphertext): + verifiable_payload = ''.join((iv, ciphertext)) + return hmac.new(self.signkey, verifiable_payload, sha256).digest() + + def encrypt(self, content): + iv, ciphertext = encrypt_sym(content, self.masterkey) + mac = self.gen_mac(iv, ciphertext) + return ''.join((mac, iv, ciphertext)) + + def decrypt(self, payload): + payload_mac, iv, ciphertext = payload[:32], payload[32:57], payload[57:] + generated_mac = self.gen_mac(iv, ciphertext) + if sha256(payload_mac).digest() != sha256(generated_mac).digest(): + raise Exception("EncryptedFileStorage - Error opening file. Wrong MAC") + return decrypt_sym(ciphertext, self.masterkey, iv) + + def _encrypt_index_on_close(self, name): + def wrapper(struct_file): + struct_file.seek(0) + content = struct_file.file.read() + file_hash = sha256(content).digest() + if name in self.length_cache and file_hash == self.length_cache[name][1]: + return + self.length_cache[name] = (len(content), file_hash) + encrypted_content = self.encrypt(content) + with open(self._fpath(name), 'w+b') as f: + f.write(encrypted_content) + + return wrapper + + def _open_encrypted_file(self, name, onclose=lambda x: None): + if not self.file_exists(name): + if name in self._open_files: + f = self._open_files[name] + if not f.is_closed: + state = 'closed' if f.file.shouldClose else 'open' + if state == 'closed': + self._store_file(name, f.file.getvalue()) + f.close() + del self._open_files[name] + else: + raise NameError(name) + file_content = open(self._fpath(name), "rb").read() + decrypted = self.decrypt(file_content) + self.length_cache[name] = (len(decrypted), sha256(decrypted).digest()) + return BufferFile(buffer(decrypted), name=name, onclose=onclose) + + def _store_file(self, name, content): + try: + encrypted_content = self.encrypt(content) + with open(self._fpath(name), 'w+b') as f: + f.write(encrypted_content) + except Exception, e: + print e + raise diff --git a/service/src/pixelated/support/error_handler.py b/service/src/pixelated/support/error_handler.py new file mode 100644 index 00000000..1a0e1a11 --- /dev/null +++ b/service/src/pixelated/support/error_handler.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from requests.exceptions import SSLError + + +def error_handler(excp): + if excp.type is SSLError: + print """ + SSL Error: Please check your certificates or read our wiki for further info: + https://github.com/pixelated-project/pixelated-user-agent/wiki/Configuring-and-using-SSL-Certificates-for-LEAP-provider + Error reference: %s + """ % excp.getErrorMessage() + else: + raise excp diff --git a/service/src/pixelated/support/functional.py b/service/src/pixelated/support/functional.py new file mode 100644 index 00000000..2e293625 --- /dev/null +++ b/service/src/pixelated/support/functional.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from itertools import chain + + +def flatten(_list): + return list(chain.from_iterable(_list)) + + +def unique(_list): + seen = set() + seen_add = seen.add + return [x for x in _list if not (x in seen or seen_add(x))] + + +def compact(_list): + return [a for a in _list if a] + + +def to_unicode(text): + if text and not isinstance(text, unicode): + encoding = 'utf-8' + return unicode(text, encoding=encoding) + return text diff --git a/service/src/pixelated/support/language.py b/service/src/pixelated/support/language.py new file mode 100644 index 00000000..cd455f89 --- /dev/null +++ b/service/src/pixelated/support/language.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2017 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + + +def parse_accept_language(all_headers): + accepted_languages = ['pt-BR', 'en-US'] + languages = all_headers.get('accept-language', '').split(';')[0] + for language in accepted_languages: + if language in languages: + return language + return 'en-US' diff --git a/service/src/pixelated/support/loglinegenerator.py b/service/src/pixelated/support/loglinegenerator.py new file mode 100644 index 00000000..d8a8fd5b --- /dev/null +++ b/service/src/pixelated/support/loglinegenerator.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from zope.interface import Interface + + +class ILogLineGenerator(Interface): + def getLogLine(self): + """ Return a string that will be logged, or None. This method will be called every second. + """ diff --git a/service/src/pixelated/support/mail_generator.py b/service/src/pixelated/support/mail_generator.py new file mode 100644 index 00000000..e5232370 --- /dev/null +++ b/service/src/pixelated/support/mail_generator.py @@ -0,0 +1,154 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + + +from email.mime.text import MIMEText +from email.utils import formatdate +from random import Random +from pixelated.support.markov import MarkovGenerator +import re +from collections import Counter +import time + + +def filter_two_line_on_wrote(lines): + skip_next = False + if len(lines) > 0: + for i in xrange(len(lines) - 1): + if skip_next: + skip_next = False + continue + + if lines[i].startswith('On') and lines[i + 1].endswith('wrote:'): + skip_next = True + else: + yield lines[i].strip() + + yield lines[-1] + + +def filter_lines(text): + pattern = re.compile('\s*[>-].*') + wrote_pattern = re.compile('\s*On.*wrote.*') + + lines = text.splitlines() + + lines = filter(lambda line: not pattern.match(line), lines) + lines = filter(lambda line: not len(line.strip()) == 0, lines) + lines = filter(lambda line: not wrote_pattern.match(line), lines) + lines = filter(lambda line: not line.endswith('writes:'), lines) + lines = filter(lambda line: ' ' in line.strip(), lines) + + lines = filter_two_line_on_wrote(lines) + + return ' '.join(lines) + + +def decode_multipart_mail_text(mail): + for payload in mail.get_payload(): + if payload.get_content_type() == 'text/plain': + return payload.get_payload(decode=True) + return '' + + +def search_for_tags(content): + words = content.split() + + only_alnum = filter(lambda word: word.isalnum(), words) + only_longer = filter(lambda word: len(word) > 5, only_alnum) + lower_case = map(lambda word: word.lower(), only_longer) + + counter = Counter(lower_case) + potential_tags = counter.most_common(10) + + return map(lambda tag: tag[0], potential_tags) + + +def filter_too_short_texts(texts): + return [text for text in texts if text is not None and len(text.split()) >= 3] + + +def load_all_mails(mail_list): + subjects = set() + mail_bodies = [] + + for mail in mail_list: + subjects.add(mail['Subject']) + if mail.is_multipart(): + mail_bodies.append(filter_lines(decode_multipart_mail_text(mail))) + else: + if mail.get_content_type() == 'text/plain': + mail_bodies.append(filter_lines(mail.get_payload(decode=True))) + else: + raise Exception(mail.get_content_type()) + + return filter_too_short_texts(subjects), filter_too_short_texts(mail_bodies) + + +class MailGenerator(object): + + NAMES = ['alice', 'bob', 'eve'] + + def __init__(self, receiver, domain_name, sample_mail_list, random=None): + self._random = random if random else Random() + self._receiver = receiver + self._domain_name = domain_name + self._subjects, self._bodies = load_all_mails(sample_mail_list) + + self._potential_tags = search_for_tags(' '.join(self._bodies)) + self._subject_markov = MarkovGenerator(self._subjects, random=self._random) + self._body_markov = MarkovGenerator(self._bodies, random=self._random, add_paragraph_on_empty_chain=True) + + def generate_mail(self): + body = self._body_markov.generate(150) + mail = MIMEText(body) + + mail['Subject'] = self._subject_markov.generate(8) + mail['To'] = '%s@%s' % (self._receiver, self._domain_name) + mail['From'] = self._random_from() + mail['Date'] = self._random_date() + mail['X-Tags'] = self._random_tags() + mail['X-Leap-Encryption'] = self._random_encryption_state() + mail['X-Leap-Signature'] = self._random_signature_state() + + return mail + + def _random_date(self): + now = int(time.time()) + ten_days = 60 * 60 * 24 * 10 + mail_time = self._random.randint(now - ten_days, now) + + return formatdate(mail_time) + + def _random_encryption_state(self): + return self._random.choice(['true', 'decrypted']) + + def _random_signature_state(self): + return self._random.choice(['could not verify', 'valid']) + + def _random_from(self): + name = self._random.choice(filter(lambda name: name != self._receiver, MailGenerator.NAMES)) + + return '%s@%s' % (name, self._domain_name) + + def _random_tags(self): + barrier = 0.5 + tags = set() + while self._random.random() > barrier: + tags.add(self._random.choice(self._potential_tags)) + barrier += 0.15 + + return ' '.join(tags) diff --git a/service/src/pixelated/support/markov.py b/service/src/pixelated/support/markov.py new file mode 100644 index 00000000..8f7c0ef3 --- /dev/null +++ b/service/src/pixelated/support/markov.py @@ -0,0 +1,94 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from random import Random + +NEW_PARAGRAPH = '\n\n' + + +class MarkovGenerator(object): + + def __init__(self, texts, random=None, add_paragraph_on_empty_chain=False): + self._markov_chain = {} + self._random = random if random else Random() + self._add_paragraph_on_empty_chain = add_paragraph_on_empty_chain + + for text in filter(lambda _: _ is not None, texts): + self._extend_chain_with(text) + + def add(self, text): + self._extend_chain_with(text) + + @staticmethod + def _triplet_generator(words): + if len(words) < 3: + raise ValueError('Expected input with at least three words') + + for i in xrange(len(words) - 2): + yield ((words[i], words[i + 1]), words[i + 2]) + + def _extend_chain_with(self, input_text): + words = input_text.split() + gen = self._triplet_generator(words) + + for key, value in gen: + if key in self._markov_chain: + self._markov_chain[key].add(value) + else: + self._markov_chain[key] = {value} + + def _generate_chain(self, length): + seed_pair = self._find_good_seed() + word, next_word = seed_pair + new_seed = False + + for i in xrange(length): + yield word + + if new_seed: + word, next_word = self._find_good_seed() + if self._add_paragraph_on_empty_chain: + yield NEW_PARAGRAPH + new_seed = False + else: + prev_word, word = word, next_word + + try: + next_word = self._random_next_word(prev_word, word) + except KeyError: + new_seed = True + + def _random_next_word(self, prev_word, word): + return self._random.choice(list(self._markov_chain[(prev_word, word)])) + + def _find_good_seed(self): + max_tries = len(self._markov_chain.keys()) + try_count = 0 + + seed_pair = self._random.choice(self._markov_chain.keys()) + while not seed_pair[0][0].isupper() and try_count <= max_tries: + seed_pair = self._random.choice(self._markov_chain.keys()) + try_count += 1 + + if try_count > max_tries: + raise ValueError('Not able find start word with captial letter') + + return seed_pair + + def generate(self, length): + if len(self._markov_chain.keys()) == 0: + raise ValueError('Expected at least three words input') + return ' '.join(self._generate_chain(length)) diff --git a/service/src/pixelated/support/replier.py b/service/src/pixelated/support/replier.py new file mode 100644 index 00000000..bab23179 --- /dev/null +++ b/service/src/pixelated/support/replier.py @@ -0,0 +1,48 @@ +# +# Copyright (c) 2015 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +from email.utils import parseaddr + + +def generate_recipients(sender, to, ccs, current_user): + result = {'single': None, 'all': {'to-field': [], 'cc-field': []}} + + to.append(sender) + to = remove_duplicates(to) + ccs = remove_duplicates(ccs) + + result['single'] = swap_recipient_if_needed(sender, remove_address(to, current_user), current_user) + result['all']['to-field'] = remove_address(to, current_user) if len(to) > 1 else to + result['all']['cc-field'] = remove_address(ccs, current_user) if len(ccs) > 1 else ccs + return result + + +def remove_duplicates(recipients): + return list(set(recipients)) + + +def remove_address(recipients, current_user): + return [recipient for recipient in recipients if not parsed_mail_matches(recipient, current_user)] + + +def parsed_mail_matches(to_parse, expected): + return parseaddr(to_parse)[1] == expected + + +def swap_recipient_if_needed(sender, recipients, current_user): + if len(recipients) == 1 and parsed_mail_matches(sender, current_user): + return recipients[0] + return sender diff --git a/service/src/pixelated/support/tls_adapter.py b/service/src/pixelated/support/tls_adapter.py new file mode 100644 index 00000000..301a2123 --- /dev/null +++ b/service/src/pixelated/support/tls_adapter.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import ssl +from requests.adapters import HTTPAdapter +try: + from urllib3.poolmanager import PoolManager +except: + from requests.packages.urllib3.poolmanager import PoolManager + +VERIFY_HOSTNAME = None + + +def latest_available_ssl_version(): + try: + return ssl.PROTOCOL_TLSv1_2 + except AttributeError: + return ssl.PROTOCOL_TLSv1 + + +class EnforceTLSv1Adapter(HTTPAdapter): + __slots__ = ('_assert_hostname', '_assert_fingerprint') + + def __init__(self, assert_hostname=VERIFY_HOSTNAME, assert_fingerprint=None): + self._assert_hostname = assert_hostname + self._assert_fingerprint = assert_fingerprint + super(EnforceTLSv1Adapter, self).__init__() + + def init_poolmanager(self, connections, maxsize, block=False): + self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize, + block=block, + assert_hostname=self._assert_hostname, + assert_fingerprint=self._assert_fingerprint, + cert_reqs=ssl.CERT_REQUIRED) |