From 66839f191708a0725c8d9841d5266ad13af8ee81 Mon Sep 17 00:00:00 2001 From: Duda Dornelles Date: Mon, 22 Dec 2014 09:35:01 -0200 Subject: refactoring package structure --- service/pixelated/adapter/__init__.py | 15 + service/pixelated/adapter/draft_service.py | 19 -- service/pixelated/adapter/listeners/__init__.py | 15 + .../adapter/listeners/mailbox_indexer_listener.py | 48 +++ service/pixelated/adapter/mail.py | 361 --------------------- service/pixelated/adapter/mail_sender.py | 38 --- service/pixelated/adapter/mail_service.py | 89 ----- service/pixelated/adapter/mailbox.py | 51 --- .../pixelated/adapter/mailbox_indexer_listener.py | 48 --- service/pixelated/adapter/mailboxes.py | 68 ---- service/pixelated/adapter/model/__init__.py | 15 + service/pixelated/adapter/model/mail.py | 361 +++++++++++++++++++++ service/pixelated/adapter/model/status.py | 42 +++ service/pixelated/adapter/model/tag.py | 73 +++++ service/pixelated/adapter/search.py | 204 ------------ service/pixelated/adapter/search/__init__.py | 204 ++++++++++++ service/pixelated/adapter/services/__init__.py | 15 + .../pixelated/adapter/services/draft_service.py | 34 ++ service/pixelated/adapter/services/mail_sender.py | 38 +++ service/pixelated/adapter/services/mail_service.py | 89 +++++ service/pixelated/adapter/services/mailbox.py | 51 +++ service/pixelated/adapter/services/mailboxes.py | 68 ++++ service/pixelated/adapter/services/tag_service.py | 26 ++ service/pixelated/adapter/soledad/__init__.py | 15 + .../pixelated/adapter/soledad/soledad_querier.py | 194 +++++++++++ service/pixelated/adapter/soledad_querier.py | 191 ----------- service/pixelated/adapter/status.py | 42 --- service/pixelated/adapter/tag.py | 73 ----- service/pixelated/adapter/tag_service.py | 26 -- service/pixelated/config/app_factory.py | 16 +- service/pixelated/controllers/mails_controller.py | 2 +- service/setup.py | 19 +- service/test/integration/contacts_test.py | 15 + .../test/integration/mark_as_read_unread_test.py | 2 +- .../test/support/integration/app_test_client.py | 12 +- service/test/support/integration/model.py | 4 +- service/test/support/test_helper.py | 2 +- service/test/unit/adapter/draft_service_test.py | 4 +- service/test/unit/adapter/mail_service_test.py | 2 +- service/test/unit/adapter/mail_test.py | 2 +- .../unit/adapter/mailbox_indexer_listener_test.py | 2 +- service/test/unit/adapter/mailbox_test.py | 4 +- service/test/unit/adapter/mailboxes_test.py | 4 +- service/test/unit/adapter/soledad_querier_test.py | 2 +- service/test/unit/adapter/test_status.py | 2 +- service/test/unit/adapter/test_tag.py | 2 +- 46 files changed, 1366 insertions(+), 1243 deletions(-) delete mode 100644 service/pixelated/adapter/draft_service.py create mode 100644 service/pixelated/adapter/listeners/__init__.py create mode 100644 service/pixelated/adapter/listeners/mailbox_indexer_listener.py delete mode 100644 service/pixelated/adapter/mail.py delete mode 100644 service/pixelated/adapter/mail_sender.py delete mode 100644 service/pixelated/adapter/mail_service.py delete mode 100644 service/pixelated/adapter/mailbox.py delete mode 100644 service/pixelated/adapter/mailbox_indexer_listener.py delete mode 100644 service/pixelated/adapter/mailboxes.py create mode 100644 service/pixelated/adapter/model/__init__.py create mode 100644 service/pixelated/adapter/model/mail.py create mode 100644 service/pixelated/adapter/model/status.py create mode 100644 service/pixelated/adapter/model/tag.py delete mode 100644 service/pixelated/adapter/search.py create mode 100644 service/pixelated/adapter/search/__init__.py create mode 100644 service/pixelated/adapter/services/__init__.py create mode 100644 service/pixelated/adapter/services/draft_service.py create mode 100644 service/pixelated/adapter/services/mail_sender.py create mode 100644 service/pixelated/adapter/services/mail_service.py create mode 100644 service/pixelated/adapter/services/mailbox.py create mode 100644 service/pixelated/adapter/services/mailboxes.py create mode 100644 service/pixelated/adapter/services/tag_service.py create mode 100644 service/pixelated/adapter/soledad/__init__.py create mode 100644 service/pixelated/adapter/soledad/soledad_querier.py delete mode 100644 service/pixelated/adapter/soledad_querier.py delete mode 100644 service/pixelated/adapter/status.py delete mode 100644 service/pixelated/adapter/tag.py delete mode 100644 service/pixelated/adapter/tag_service.py diff --git a/service/pixelated/adapter/__init__.py b/service/pixelated/adapter/__init__.py index e69de29b..2756a319 100644 --- a/service/pixelated/adapter/__init__.py +++ b/service/pixelated/adapter/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . diff --git a/service/pixelated/adapter/draft_service.py b/service/pixelated/adapter/draft_service.py deleted file mode 100644 index d9d6b12f..00000000 --- a/service/pixelated/adapter/draft_service.py +++ /dev/null @@ -1,19 +0,0 @@ - - -class DraftService(object): - __slots__ = '_mailboxes' - - def __init__(self, mailboxes): - self._mailboxes = mailboxes - - def create_draft(self, input_mail): - self._drafts().add(input_mail) - return input_mail - - def update_draft(self, ident, input_mail): - new_mail = self.create_draft(input_mail) - self._drafts().remove(ident) - return new_mail - - def _drafts(self): - return self._mailboxes.drafts() diff --git a/service/pixelated/adapter/listeners/__init__.py b/service/pixelated/adapter/listeners/__init__.py new file mode 100644 index 00000000..2756a319 --- /dev/null +++ b/service/pixelated/adapter/listeners/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . diff --git a/service/pixelated/adapter/listeners/mailbox_indexer_listener.py b/service/pixelated/adapter/listeners/mailbox_indexer_listener.py new file mode 100644 index 00000000..d8e0f81e --- /dev/null +++ b/service/pixelated/adapter/listeners/mailbox_indexer_listener.py @@ -0,0 +1,48 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PCULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . + + +class MailboxIndexerListener(object): + """ Listens for new mails, keeping the index updated """ + + SEARCH_ENGINE = None + + @classmethod + def listen(cls, account, mailbox_name, soledad_querier): + listener = MailboxIndexerListener(mailbox_name, soledad_querier) + if listener not in account.getMailbox(mailbox_name).listeners: + account.getMailbox(mailbox_name).addListener(listener) + + def __init__(self, mailbox_name, soledad_querier): + self.mailbox_name = mailbox_name + self.querier = soledad_querier + + def newMessages(self, exists, recent): + indexed_idents = set(self.SEARCH_ENGINE.search('tag:' + self.mailbox_name.lower(), all_mails=True)) + soledad_idents = self.querier.idents_by_mailbox(self.mailbox_name) + + missing_idents = soledad_idents.difference(indexed_idents) + + self.SEARCH_ENGINE.index_mails(self.querier.mails(missing_idents)) + + def __eq__(self, other): + return other and other.mailbox_name == self.mailbox_name + + def __hash__(self): + return self.mailbox_name.__hash__() + + def __repr__(self): + return 'MailboxListener: ' + self.mailbox_name diff --git a/service/pixelated/adapter/mail.py b/service/pixelated/adapter/mail.py deleted file mode 100644 index b6b566bb..00000000 --- a/service/pixelated/adapter/mail.py +++ /dev/null @@ -1,361 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PCULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . -import json -from uuid import uuid4 -from email.mime.text import MIMEText - -from leap.mail.imap.fields import fields -import leap.mail.walk as walk -import dateutil.parser as dateparser -from pixelated.adapter.status import Status -import pixelated.support.date -from email.MIMEMultipart import MIMEMultipart -from pycryptopp.hash import sha256 -import re -import base64 - - -class Mail(object): - @property - def to(self): - return self.headers['To'] - - @property - def cc(self): - return self.headers['Cc'] - - @property - def bcc(self): - return self.headers['Bcc'] - - @property - def date(self): - return self.headers['Date'] - - @property - def status(self): - return Status.from_flags(self.flags) - - @property - def flags(self): - return self.fdoc.content.get('flags') - - @property - def mailbox_name(self): - return self.fdoc.content.get('mbox') - - @property - def _mime_multipart(self): - if self._mime: - return self._mime - mime = MIMEMultipart() - for key, value in self.headers.items(): - mime[str(key)] = str(value) - mime.attach(MIMEText(self.body, 'plain', self._charset())) - self._mime = mime - return mime - - def _charset(self): - if 'content_type' in self.headers and 'charset' in self.headers['content_type']: - return re.compile('.*charset=(.*)').match(self.headers['content_type']).group(1) - else: - return 'utf-8' - - @property - def raw(self): - return self._mime_multipart.as_string() - - def _get_chash(self): - return sha256.SHA256(self.raw).hexdigest() - - -class InputMail(Mail): - FROM_EMAIL_ADDRESS = None - - def __init__(self): - self._raw_message = None - self._fd = None - self._hd = None - self._bd = None - self._chash = None - self._mime = None - - @property - def ident(self): - return self._get_chash() - - def get_for_save(self, next_uid, mailbox): - docs = [self._fdoc(next_uid, mailbox), self._hdoc()] - docs.extend([m for m in self._cdocs()]) - return docs - - def _fdoc(self, next_uid, mailbox): - if self._fd: - return self._fd - - fd = {} - fd[fields.MBOX_KEY] = mailbox - fd[fields.UID_KEY] = next_uid - fd[fields.CONTENT_HASH_KEY] = self._get_chash() - fd[fields.SIZE_KEY] = len(self.raw) - fd[fields.MULTIPART_KEY] = True - fd[fields.RECENT_KEY] = True - fd[fields.TYPE_KEY] = fields.TYPE_FLAGS_VAL - fd[fields.FLAGS_KEY] = Status.to_flags(self._status) - self._fd = fd - return fd - - def _get_body_phash(self): - return walk.get_body_phash_multi(walk.get_payloads(self._mime_multipart)) - - def _hdoc(self): - if self._hd: - return self._hd - - hd = {} - hd[fields.HEADERS_KEY] = self.headers - hd[fields.DATE_KEY] = self.headers['Date'] - hd[fields.CONTENT_HASH_KEY] = self._get_chash() - hd[fields.MSGID_KEY] = '' - hd[fields.MULTIPART_KEY] = True - hd[fields.SUBJECT_KEY] = self.headers.get('Subject') - hd[fields.TYPE_KEY] = fields.TYPE_HEADERS_VAL - hd[fields.BODY_KEY] = self._get_body_phash() - hd[fields.PARTS_MAP_KEY] = \ - walk.walk_msg_tree(walk.get_parts(self._mime_multipart), body_phash=self._get_body_phash())['part_map'] - - self._hd = hd - return hd - - def _cdocs(self): - return walk.get_raw_docs(self._mime_multipart, self._mime_multipart.walk()) - - def to_mime_multipart(self): - mime_multipart = MIMEMultipart() - - for header in ['To', 'Cc', 'Bcc']: - if self.headers[header]: - mime_multipart[header] = ", ".join(self.headers[header]) - - if self.headers['Subject']: - mime_multipart['Subject'] = self.headers['Subject'] - - mime_multipart['Date'] = self.headers['Date'] - if type(self.body) is list: - for part in self.body: - mime_multipart.attach(MIMEText(part['raw'], part['content-type'])) - else: - mime_multipart.attach(MIMEText(self.body, 'plain', 'utf-8')) - return mime_multipart - - def to_smtp_format(self): - mime_multipart = self.to_mime_multipart() - mime_multipart['From'] = InputMail.FROM_EMAIL_ADDRESS - return mime_multipart.as_string() - - @staticmethod - def from_dict(mail_dict): - input_mail = InputMail() - input_mail.headers = {key.capitalize(): value for key, value in mail_dict.get('header', {}).items()} - - # XXX this is overriding the property in PixelatedMail - input_mail.headers['Date'] = pixelated.support.date.iso_now() - - # XXX this is overriding the property in PixelatedMail - input_mail.body = mail_dict.get('body', '') - - # XXX this is overriding the property in the PixelatedMail - input_mail.tags = set(mail_dict.get('tags', [])) - - input_mail._status = set(mail_dict.get('status', [])) - return input_mail - - -class PixelatedMail(Mail): - @staticmethod - def from_soledad(fdoc, hdoc, bdoc, parts=None, soledad_querier=None): - mail = PixelatedMail() - mail.parts = parts - mail.boundary = str(uuid4()).replace('-', '') - mail.bdoc = bdoc - mail.fdoc = fdoc - mail.hdoc = hdoc - mail.querier = soledad_querier - mail._mime = None - return mail - - @property - def body(self): - if self.parts and len(self.parts['alternatives']) > 1: - body = '' - for alternative in self.parts['alternatives']: - body += '--' + self.boundary + '\n' - for header, value in alternative['headers'].items(): - body += '%s: %s\n' % (header, value) - body += '\n' - body += alternative['content'] - body += '\n' - body += '--' + self.boundary + '--' - return body - else: - if self.parts and self.parts['alternatives'][0]['headers'].get('Content-Transfer-Encoding', '') == 'base64': - return unicode(base64.b64decode(self.parts['alternatives'][0]['content']), 'utf-8') - else: - return self.bdoc.content['raw'] - - @property - def headers(self): - _headers = { - 'To': [], - 'Cc': [], - 'Bcc': [] - } - hdoc_headers = self.hdoc.content['headers'] - - for header in ['To', 'Cc', 'Bcc']: - header_value = hdoc_headers.get(header) - if not header_value: - continue - _headers[header] = header_value if type(header_value) is list else header_value.split(',') - _headers[header] = map(lambda x: x.strip(), _headers[header]) - - for header in ['From', 'Subject']: - _headers[header] = hdoc_headers.get(header) - - _headers['Date'] = self._get_date() - - if self.parts and len(self.parts['alternatives']) > 1: - _headers['content_type'] = 'multipart/alternative; boundary="%s"' % self.boundary - elif self.hdoc.content['headers'].get('Content-Type'): - _headers['content_type'] = hdoc_headers.get('Content-Type') - - if hdoc_headers.get('Reply-To'): - _headers['Reply-To'] = hdoc_headers.get('Reply-To') - - return _headers - - def _get_date(self): - date = self.hdoc.content.get('date', None) - if not date: - date = self.hdoc.content['received'].split(";")[-1].strip() - return dateparser.parse(date).isoformat() - - @property - def security_casing(self): - casing = {"imprints": [], "locks": []} - if self.signed: - casing["imprints"].append({"state": "valid", "seal": {"validity": "valid"}}) - elif self.signed is None: - casing["imprints"].append({"state": "no_signature_information"}) - - if self.encrypted: - casing["locks"].append({"state": "valid"}) - - return casing - - @property - def tags(self): - _tags = self.fdoc.content.get('tags', '[]') - return set(_tags) if type(_tags) is list or type(_tags) is set else set(json.loads(_tags)) - - @property - def ident(self): - return self.fdoc.content.get('chash') - - @property - def mailbox_name(self): - return self.fdoc.content.get('mbox') - - @property - def is_recent(self): - return Status('recent') in self.status - - @property - def uid(self): - return self.fdoc.content['uid'] - - def save(self): - return self.querier.save_mail(self) - - def set_mailbox(self, mailbox_name): - self.fdoc.content['mbox'] = mailbox_name - - def remove_all_tags(self): - self.update_tags(set([])) - - def update_tags(self, tags): - self._persist_mail_tags(tags) - return self.tags - - def mark_as_read(self): - if Status.SEEN in self.fdoc.content['flags']: - return self - self.fdoc.content['flags'].append(Status.SEEN) - self.save() - return self - - def mark_as_unread(self): - if Status.SEEN in self.fdoc.content['flags']: - self.fdoc.content['flags'].remove(Status.SEEN) - self.save() - return self - - def mark_as_not_recent(self): - if Status.RECENT in self.fdoc.content['flags']: - self.fdoc.content['flags'].remove(Status.RECENT) - self.save() - return self - - def _persist_mail_tags(self, current_tags): - self.fdoc.content['tags'] = json.dumps(list(current_tags)) - self.save() - - def has_tag(self, tag): - return tag in self.tags - - @property - def signed(self): - signature = self.hdoc.content["headers"].get("X-Leap-Signature", None) - if signature is None: - return None - else: - return signature.startswith("valid") - - @property - def encrypted(self): - return self.hdoc.content["headers"].get("OpenPGP", None) is not None - - def as_dict(self): - dict_mail = {'header': {k.lower(): v for k, v in self.headers.items()}, - 'ident': self.ident, - 'tags': list(self.tags), - 'status': list(self.status), - 'security_casing': self.security_casing, - 'body': self.body, - 'mailbox': self.mailbox_name.lower(), - 'attachments': self.parts['attachments'] if self.parts else []} - dict_mail['replying'] = {'single': None, 'all': {'to-field': [], 'cc-field': []}} - - sender_mail = self.headers.get('Reply-To', self.headers['From']) - - recipients = [recipient for recipient in self.headers['To'] if recipient != InputMail.FROM_EMAIL_ADDRESS] - recipients.append(sender_mail) - ccs = [cc for cc in self.headers['Cc'] if cc != InputMail.FROM_EMAIL_ADDRESS] - - dict_mail['replying']['single'] = sender_mail - dict_mail['replying']['all']['to-field'] = recipients - dict_mail['replying']['all']['cc-field'] = ccs - return dict_mail diff --git a/service/pixelated/adapter/mail_sender.py b/service/pixelated/adapter/mail_sender.py deleted file mode 100644 index 50c17ba5..00000000 --- a/service/pixelated/adapter/mail_sender.py +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . -from StringIO import StringIO - -from twisted.internet.defer import Deferred -from twisted.mail.smtp import SMTPSenderFactory -from twisted.internet import reactor -from pixelated.support.functional import flatten - - -class MailSender(): - def __init__(self, account_email_address, smtp_client=None): - self.account_email_address = account_email_address - - def sendmail(self, mail): - recipients = flatten([mail.to, mail.cc, mail.bcc]) - - resultDeferred = Deferred() - senderFactory = SMTPSenderFactory( - fromEmail=self.account_email_address, - toEmail=recipients, - file=StringIO(mail.to_smtp_format()), - deferred=resultDeferred) - - return reactor.connectTCP('localhost', 4650, senderFactory) diff --git a/service/pixelated/adapter/mail_service.py b/service/pixelated/adapter/mail_service.py deleted file mode 100644 index 722b9a29..00000000 --- a/service/pixelated/adapter/mail_service.py +++ /dev/null @@ -1,89 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . - - -class MailService: - __slots__ = ['leap_session', 'account', 'mailbox_name'] - - def __init__(self, mailboxes, mail_sender, tag_service, soledad_querier): - self.tag_service = tag_service - self.mailboxes = mailboxes - self.querier = soledad_querier - self.mail_sender = mail_sender - - def all_mails(self): - return self.querier.all_mails() - - def mails(self, ids): - return self.querier.mails(ids) - - def update_tags(self, mail_id, new_tags): - reserved_words = self.tag_service.extract_reserved(new_tags) - if len(reserved_words): - raise ValueError('None of the following words can be used as tags: ' + ' '.join(reserved_words)) - mail = self.mail(mail_id) - mail.update_tags(set(new_tags)) - return mail - - def mail(self, mail_id): - return self.mailboxes.mail(mail_id) - - def send(self, last_draft_ident, mail): - self.mail_sender.sendmail(mail) - if last_draft_ident: - self.mailboxes.drafts().remove(last_draft_ident) - return self.mailboxes.sent().add(mail) - - def thread(self, thread_id): - raise NotImplementedError() - - def mark_as_read(self, mail_id): - return self.mail(mail_id).mark_as_read() - - def mark_as_unread(self, mail_id): - return self.mail(mail_id).mark_as_unread() - - def tags_for_thread(self, thread): - raise NotImplementedError() - - def add_tag_to_thread(self, thread_id, tag): - raise NotImplementedError() - - def remove_tag_from_thread(self, thread_id, tag): - raise NotImplementedError() - - def delete_mail(self, mail_id): - return self.mailboxes.move_to_trash(mail_id) - - def delete_permanent(self, mail_id): - mail = self.mail(mail_id) - self.querier.remove_mail(mail) - - def save_draft(self, draft): - raise NotImplementedError() - - def draft_reply_for(self, mail_id): - raise NotImplementedError() - - def all_contacts(self, query): - raise NotImplementedError() - - def drafts(self): - raise NotImplementedError() - - def reply_all_template(self, mail_id): - mail = self.mail(mail_id) - return mail.to_reply_template() diff --git a/service/pixelated/adapter/mailbox.py b/service/pixelated/adapter/mailbox.py deleted file mode 100644 index fbdbfc30..00000000 --- a/service/pixelated/adapter/mailbox.py +++ /dev/null @@ -1,51 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . - - -class Mailbox: - - def __init__(self, mailbox_name, querier): - self.mailbox_name = mailbox_name - self.mailbox_tag = mailbox_name.lower() - self.querier = querier - - def mails(self): - _mails = self.querier.all_mails_by_mailbox(self.mailbox_name) - - result = [] - for mail in _mails: - result.append(mail) - return result - - def mails_by_tags(self, tags): - if 'all' in tags or self.mailbox_tag in tags: - return self.mails() - return [mail for mail in self.mails() if len(mail.tags.intersection(tags)) > 0] - - def mail(self, mail_id): - return self.querier.mail(mail_id) - - def add(self, mail): - return self.querier.create_mail(mail, self.mailbox_name) - - def remove(self, ident): - mail = self.querier.mail(ident) - mail.remove_all_tags() - self.querier.remove_mail(mail) - - @classmethod - def create(cls, mailbox_name, soledad_querier): - return Mailbox(mailbox_name, soledad_querier) diff --git a/service/pixelated/adapter/mailbox_indexer_listener.py b/service/pixelated/adapter/mailbox_indexer_listener.py deleted file mode 100644 index d8e0f81e..00000000 --- a/service/pixelated/adapter/mailbox_indexer_listener.py +++ /dev/null @@ -1,48 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PCULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . - - -class MailboxIndexerListener(object): - """ Listens for new mails, keeping the index updated """ - - SEARCH_ENGINE = None - - @classmethod - def listen(cls, account, mailbox_name, soledad_querier): - listener = MailboxIndexerListener(mailbox_name, soledad_querier) - if listener not in account.getMailbox(mailbox_name).listeners: - account.getMailbox(mailbox_name).addListener(listener) - - def __init__(self, mailbox_name, soledad_querier): - self.mailbox_name = mailbox_name - self.querier = soledad_querier - - def newMessages(self, exists, recent): - indexed_idents = set(self.SEARCH_ENGINE.search('tag:' + self.mailbox_name.lower(), all_mails=True)) - soledad_idents = self.querier.idents_by_mailbox(self.mailbox_name) - - missing_idents = soledad_idents.difference(indexed_idents) - - self.SEARCH_ENGINE.index_mails(self.querier.mails(missing_idents)) - - def __eq__(self, other): - return other and other.mailbox_name == self.mailbox_name - - def __hash__(self): - return self.mailbox_name.__hash__() - - def __repr__(self): - return 'MailboxListener: ' + self.mailbox_name diff --git a/service/pixelated/adapter/mailboxes.py b/service/pixelated/adapter/mailboxes.py deleted file mode 100644 index 241a8050..00000000 --- a/service/pixelated/adapter/mailboxes.py +++ /dev/null @@ -1,68 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . -from pixelated.adapter.mailbox import Mailbox -from pixelated.adapter.mailbox_indexer_listener import MailboxIndexerListener - - -class Mailboxes(): - - def __init__(self, account, soledad_querier): - self.account = account - self.querier = soledad_querier - for mailbox_name in account.mailboxes: - MailboxIndexerListener.listen(self.account, mailbox_name, soledad_querier) - - def _create_or_get(self, mailbox_name): - mailbox_name = mailbox_name.upper() - if mailbox_name not in self.account.mailboxes: - self.account.addMailbox(mailbox_name) - MailboxIndexerListener.listen(self.account, mailbox_name, self.querier) - return Mailbox.create(mailbox_name, self.querier) - - def inbox(self): - return self._create_or_get('INBOX') - - def drafts(self): - return self._create_or_get('DRAFTS') - - def trash(self): - return self._create_or_get('TRASH') - - def sent(self): - return self._create_or_get('SENT') - - def mailboxes(self): - return [self._create_or_get(leap_mailbox_name) for leap_mailbox_name in self.account.mailboxes] - - def mails_by_tag(self, query_tags): - mails = [] - for mailbox in self.mailboxes(): - mails.extend(mailbox.mails_by_tags(query_tags)) - - return mails - - def move_to_trash(self, mail_id): - mail = self.querier.mail(mail_id) - mail.remove_all_tags() - mail.set_mailbox(self.trash().mailbox_name) - mail.save() - return mail - - def mail(self, mail_id): - for mailbox in self.mailboxes(): - mail = mailbox.mail(mail_id) - if mail: - return mail diff --git a/service/pixelated/adapter/model/__init__.py b/service/pixelated/adapter/model/__init__.py new file mode 100644 index 00000000..2756a319 --- /dev/null +++ b/service/pixelated/adapter/model/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . diff --git a/service/pixelated/adapter/model/mail.py b/service/pixelated/adapter/model/mail.py new file mode 100644 index 00000000..0beb2111 --- /dev/null +++ b/service/pixelated/adapter/model/mail.py @@ -0,0 +1,361 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PCULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . +import json +from uuid import uuid4 +from email.mime.text import MIMEText + +from leap.mail.imap.fields import fields +import leap.mail.walk as walk +import dateutil.parser as dateparser +from pixelated.adapter.model.status import Status +import pixelated.support.date +from email.MIMEMultipart import MIMEMultipart +from pycryptopp.hash import sha256 +import re +import base64 + + +class Mail(object): + @property + def to(self): + return self.headers['To'] + + @property + def cc(self): + return self.headers['Cc'] + + @property + def bcc(self): + return self.headers['Bcc'] + + @property + def date(self): + return self.headers['Date'] + + @property + def status(self): + return Status.from_flags(self.flags) + + @property + def flags(self): + return self.fdoc.content.get('flags') + + @property + def mailbox_name(self): + return self.fdoc.content.get('mbox') + + @property + def _mime_multipart(self): + if self._mime: + return self._mime + mime = MIMEMultipart() + for key, value in self.headers.items(): + mime[str(key)] = str(value) + mime.attach(MIMEText(self.body, 'plain', self._charset())) + self._mime = mime + return mime + + def _charset(self): + if 'content_type' in self.headers and 'charset' in self.headers['content_type']: + return re.compile('.*charset=(.*)').match(self.headers['content_type']).group(1) + else: + return 'utf-8' + + @property + def raw(self): + return self._mime_multipart.as_string() + + def _get_chash(self): + return sha256.SHA256(self.raw).hexdigest() + + +class InputMail(Mail): + FROM_EMAIL_ADDRESS = None + + def __init__(self): + self._raw_message = None + self._fd = None + self._hd = None + self._bd = None + self._chash = None + self._mime = None + + @property + def ident(self): + return self._get_chash() + + def get_for_save(self, next_uid, mailbox): + docs = [self._fdoc(next_uid, mailbox), self._hdoc()] + docs.extend([m for m in self._cdocs()]) + return docs + + def _fdoc(self, next_uid, mailbox): + if self._fd: + return self._fd + + fd = {} + fd[fields.MBOX_KEY] = mailbox + fd[fields.UID_KEY] = next_uid + fd[fields.CONTENT_HASH_KEY] = self._get_chash() + fd[fields.SIZE_KEY] = len(self.raw) + fd[fields.MULTIPART_KEY] = True + fd[fields.RECENT_KEY] = True + fd[fields.TYPE_KEY] = fields.TYPE_FLAGS_VAL + fd[fields.FLAGS_KEY] = Status.to_flags(self._status) + self._fd = fd + return fd + + def _get_body_phash(self): + return walk.get_body_phash_multi(walk.get_payloads(self._mime_multipart)) + + def _hdoc(self): + if self._hd: + return self._hd + + hd = {} + hd[fields.HEADERS_KEY] = self.headers + hd[fields.DATE_KEY] = self.headers['Date'] + hd[fields.CONTENT_HASH_KEY] = self._get_chash() + hd[fields.MSGID_KEY] = '' + hd[fields.MULTIPART_KEY] = True + hd[fields.SUBJECT_KEY] = self.headers.get('Subject') + hd[fields.TYPE_KEY] = fields.TYPE_HEADERS_VAL + hd[fields.BODY_KEY] = self._get_body_phash() + hd[fields.PARTS_MAP_KEY] = \ + walk.walk_msg_tree(walk.get_parts(self._mime_multipart), body_phash=self._get_body_phash())['part_map'] + + self._hd = hd + return hd + + def _cdocs(self): + return walk.get_raw_docs(self._mime_multipart, self._mime_multipart.walk()) + + def to_mime_multipart(self): + mime_multipart = MIMEMultipart() + + for header in ['To', 'Cc', 'Bcc']: + if self.headers[header]: + mime_multipart[header] = ", ".join(self.headers[header]) + + if self.headers['Subject']: + mime_multipart['Subject'] = self.headers['Subject'] + + mime_multipart['Date'] = self.headers['Date'] + if type(self.body) is list: + for part in self.body: + mime_multipart.attach(MIMEText(part['raw'], part['content-type'])) + else: + mime_multipart.attach(MIMEText(self.body, 'plain', 'utf-8')) + return mime_multipart + + def to_smtp_format(self): + mime_multipart = self.to_mime_multipart() + mime_multipart['From'] = InputMail.FROM_EMAIL_ADDRESS + return mime_multipart.as_string() + + @staticmethod + def from_dict(mail_dict): + input_mail = InputMail() + input_mail.headers = {key.capitalize(): value for key, value in mail_dict.get('header', {}).items()} + + # XXX this is overriding the property in PixelatedMail + input_mail.headers['Date'] = pixelated.support.date.iso_now() + + # XXX this is overriding the property in PixelatedMail + input_mail.body = mail_dict.get('body', '') + + # XXX this is overriding the property in the PixelatedMail + input_mail.tags = set(mail_dict.get('tags', [])) + + input_mail._status = set(mail_dict.get('status', [])) + return input_mail + + +class PixelatedMail(Mail): + @staticmethod + def from_soledad(fdoc, hdoc, bdoc, parts=None, soledad_querier=None): + mail = PixelatedMail() + mail.parts = parts + mail.boundary = str(uuid4()).replace('-', '') + mail.bdoc = bdoc + mail.fdoc = fdoc + mail.hdoc = hdoc + mail.querier = soledad_querier + mail._mime = None + return mail + + @property + def body(self): + if self.parts and len(self.parts['alternatives']) > 1: + body = '' + for alternative in self.parts['alternatives']: + body += '--' + self.boundary + '\n' + for header, value in alternative['headers'].items(): + body += '%s: %s\n' % (header, value) + body += '\n' + body += alternative['content'] + body += '\n' + body += '--' + self.boundary + '--' + return body + else: + if self.parts and self.parts['alternatives'][0]['headers'].get('Content-Transfer-Encoding', '') == 'base64': + return unicode(base64.b64decode(self.parts['alternatives'][0]['content']), 'utf-8') + else: + return self.bdoc.content['raw'] + + @property + def headers(self): + _headers = { + 'To': [], + 'Cc': [], + 'Bcc': [] + } + hdoc_headers = self.hdoc.content['headers'] + + for header in ['To', 'Cc', 'Bcc']: + header_value = hdoc_headers.get(header) + if not header_value: + continue + _headers[header] = header_value if type(header_value) is list else header_value.split(',') + _headers[header] = map(lambda x: x.strip(), _headers[header]) + + for header in ['From', 'Subject']: + _headers[header] = hdoc_headers.get(header) + + _headers['Date'] = self._get_date() + + if self.parts and len(self.parts['alternatives']) > 1: + _headers['content_type'] = 'multipart/alternative; boundary="%s"' % self.boundary + elif self.hdoc.content['headers'].get('Content-Type'): + _headers['content_type'] = hdoc_headers.get('Content-Type') + + if hdoc_headers.get('Reply-To'): + _headers['Reply-To'] = hdoc_headers.get('Reply-To') + + return _headers + + def _get_date(self): + date = self.hdoc.content.get('date', None) + if not date: + date = self.hdoc.content['received'].split(";")[-1].strip() + return dateparser.parse(date).isoformat() + + @property + def security_casing(self): + casing = {"imprints": [], "locks": []} + if self.signed: + casing["imprints"].append({"state": "valid", "seal": {"validity": "valid"}}) + elif self.signed is None: + casing["imprints"].append({"state": "no_signature_information"}) + + if self.encrypted: + casing["locks"].append({"state": "valid"}) + + return casing + + @property + def tags(self): + _tags = self.fdoc.content.get('tags', '[]') + return set(_tags) if type(_tags) is list or type(_tags) is set else set(json.loads(_tags)) + + @property + def ident(self): + return self.fdoc.content.get('chash') + + @property + def mailbox_name(self): + return self.fdoc.content.get('mbox') + + @property + def is_recent(self): + return Status('recent') in self.status + + @property + def uid(self): + return self.fdoc.content['uid'] + + def save(self): + return self.querier.save_mail(self) + + def set_mailbox(self, mailbox_name): + self.fdoc.content['mbox'] = mailbox_name + + def remove_all_tags(self): + self.update_tags(set([])) + + def update_tags(self, tags): + self._persist_mail_tags(tags) + return self.tags + + def mark_as_read(self): + if Status.SEEN in self.fdoc.content['flags']: + return self + self.fdoc.content['flags'].append(Status.SEEN) + self.save() + return self + + def mark_as_unread(self): + if Status.SEEN in self.fdoc.content['flags']: + self.fdoc.content['flags'].remove(Status.SEEN) + self.save() + return self + + def mark_as_not_recent(self): + if Status.RECENT in self.fdoc.content['flags']: + self.fdoc.content['flags'].remove(Status.RECENT) + self.save() + return self + + def _persist_mail_tags(self, current_tags): + self.fdoc.content['tags'] = json.dumps(list(current_tags)) + self.save() + + def has_tag(self, tag): + return tag in self.tags + + @property + def signed(self): + signature = self.hdoc.content["headers"].get("X-Leap-Signature", None) + if signature is None: + return None + else: + return signature.startswith("valid") + + @property + def encrypted(self): + return self.hdoc.content["headers"].get("OpenPGP", None) is not None + + def as_dict(self): + dict_mail = {'header': {k.lower(): v for k, v in self.headers.items()}, + 'ident': self.ident, + 'tags': list(self.tags), + 'status': list(self.status), + 'security_casing': self.security_casing, + 'body': self.body, + 'mailbox': self.mailbox_name.lower(), + 'attachments': self.parts['attachments'] if self.parts else []} + dict_mail['replying'] = {'single': None, 'all': {'to-field': [], 'cc-field': []}} + + sender_mail = self.headers.get('Reply-To', self.headers['From']) + + recipients = [recipient for recipient in self.headers['To'] if recipient != InputMail.FROM_EMAIL_ADDRESS] + recipients.append(sender_mail) + ccs = [cc for cc in self.headers['Cc'] if cc != InputMail.FROM_EMAIL_ADDRESS] + + dict_mail['replying']['single'] = sender_mail + dict_mail['replying']['all']['to-field'] = recipients + dict_mail['replying']['all']['cc-field'] = ccs + return dict_mail diff --git a/service/pixelated/adapter/model/status.py b/service/pixelated/adapter/model/status.py new file mode 100644 index 00000000..5a11ee7b --- /dev/null +++ b/service/pixelated/adapter/model/status.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . + + +class Status: + + SEEN = u'\\Seen' + ANSWERED = u'\\Answered' + DELETED = u'\\Deleted' + RECENT = u'\\Recent' + + FLAGS_TO_STATUSES = { + SEEN: 'read', + ANSWERED: 'replied', + RECENT: 'recent' + } + + @staticmethod + def from_flag(flag): + return Status.FLAGS_TO_STATUSES[flag] + + @staticmethod + def from_flags(flags): + return set(Status.from_flag(flag) for flag in flags if flag in Status.FLAGS_TO_STATUSES.keys()) + + @staticmethod + def to_flags(statuses): + statuses_to_flags = dict(zip(Status.FLAGS_TO_STATUSES.values(), Status.FLAGS_TO_STATUSES.keys())) + return [statuses_to_flags[status] for status in statuses] diff --git a/service/pixelated/adapter/model/tag.py b/service/pixelated/adapter/model/tag.py new file mode 100644 index 00000000..d75022f9 --- /dev/null +++ b/service/pixelated/adapter/model/tag.py @@ -0,0 +1,73 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . + +import json + + +class Tag: + + @classmethod + def from_dict(cls, tag_dict): + tag = Tag(tag_dict['name'], tag_dict['default']) + tag.mails = set(tag_dict['mails']) + return tag + + @classmethod + def from_json_string(cls, json_string): + tag_dict = json.loads(json_string) + tag_dict['mails'] = set(tag_dict['mails']) + return Tag.from_dict(tag_dict) + + @property + def total(self): + return len(self.mails) + + def __init__(self, name, default=False): + self.name = name.lower() + self.ident = self.name.__hash__() + self.default = default + self.mails = set() + + def __eq__(self, other): + return self.name == other.name + + def __hash__(self): + return self.name.__hash__() + + def increment(self, mail_ident): + self.mails.add(mail_ident) + + def decrement(self, mail_ident): + self.mails.discard(mail_ident) + + def as_dict(self): + return { + 'name': self.name, + 'default': self.default, + 'ident': self.ident, + 'counts': {'total': self.total, + 'read': 0, + 'starred': 0, + 'replied': 0}, + 'mails': list(self.mails) + } + + def as_json_string(self): + tag_dict = self.as_dict() + return json.dumps(tag_dict) + + def __repr__(self): + return self.name diff --git a/service/pixelated/adapter/search.py b/service/pixelated/adapter/search.py deleted file mode 100644 index 3d71bab7..00000000 --- a/service/pixelated/adapter/search.py +++ /dev/null @@ -1,204 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . - -from pixelated.support.encrypted_file_storage import EncryptedFileStorage - -import os -from pixelated.adapter.status import Status -from pixelated.support.functional import flatten -from whoosh.index import FileIndex -from whoosh.fields import * -from whoosh.qparser import QueryParser -from whoosh.qparser import MultifieldParser -from whoosh.query import Term -from whoosh import sorting -from pixelated.support.functional import unique -from pixelated.support.date import milliseconds - - -class SearchEngine(object): - INDEX_FOLDER = os.path.join(os.environ['HOME'], '.leap', 'search_index') - DEFAULT_TAGS = ['inbox', 'sent', 'drafts', 'trash'] - - def __init__(self, soledad_querier): - self.soledad_querier = soledad_querier - if not os.path.exists(self.INDEX_FOLDER): - os.makedirs(self.INDEX_FOLDER) - self._index = self._create_index() - - def _add_to_tags(self, tags, group, skip_default_tags, count_type, query=None): - query_matcher = re.compile(".*%s.*" % query.lower()) if query else re.compile(".*") - - for tag, count in group.iteritems(): - - if skip_default_tags and tag in self.DEFAULT_TAGS or not query_matcher.match(tag): - continue - - if not tags.get(tag): - tags[tag] = {'ident': tag, 'name': tag, 'default': False, 'counts': {'total': 0, 'read': 0}, - 'mails': []} - tags[tag]['counts'][count_type] += count - - def _search_tag_groups(self, is_filtering_tags): - seen = None - query_parser = QueryParser('tag', self._index.schema) - options = {'limit': None, 'groupedby': sorting.FieldFacet('tag', allow_overlap=True), 'maptype': sorting.Count} - - with self._index.searcher() as searcher: - total = searcher.search(query_parser.parse('*'), **options).groups() - if not is_filtering_tags: - seen = searcher.search(query_parser.parse("* AND flags:%s" % Status.SEEN), **options).groups() - return seen, total - - def _init_tags_defaults(self): - tags = {} - for default_tag in self.DEFAULT_TAGS: - tags[default_tag] = { - 'ident': default_tag, - 'name': default_tag, - 'default': True, - 'counts': { - 'total': 0, - 'read': 0 - }, - 'mails': [] - } - return tags - - def _build_tags(self, seen, total, skip_default_tags, query): - tags = {} - if not skip_default_tags: - tags = self._init_tags_defaults() - self._add_to_tags(tags, total, skip_default_tags, count_type='total', query=query) - if seen: - self._add_to_tags(tags, seen, skip_default_tags, count_type='read') - return tags.values() - - def tags(self, query, skip_default_tags): - is_filtering_tags = True if query else False - seen, total = self._search_tag_groups(is_filtering_tags=is_filtering_tags) - return self._build_tags(seen, total, skip_default_tags, query) - - def _mail_schema(self): - return Schema( - ident=ID(stored=True, unique=True), - sender=ID(stored=False), - to=KEYWORD(stored=False, commas=True), - cc=KEYWORD(stored=False, commas=True), - bcc=KEYWORD(stored=False, commas=True), - subject=TEXT(stored=False), - date=NUMERIC(stored=False, sortable=True, bits=64, signed=False), - body=TEXT(stored=False), - tag=KEYWORD(stored=True, commas=True), - flags=KEYWORD(stored=True, commas=True), - raw=TEXT(stored=False)) - - def _create_index(self): - masterkey = self.soledad_querier.get_index_masterkey() - storage = EncryptedFileStorage(self.INDEX_FOLDER, masterkey) - return FileIndex.create(storage, self._mail_schema(), indexname='mails') - - def index_mail(self, mail): - with self._index.writer() as writer: - self._index_mail(writer, mail) - - def _index_mail(self, writer, mail): - mdict = mail.as_dict() - header = mdict['header'] - tags = mdict.get('tags', []) - tags.append(mail.mailbox_name.lower()) - index_data = { - 'sender': unicode(header.get('from', '')), - 'subject': unicode(header.get('subject', '')), - 'date': milliseconds(header.get('date', '')), - 'to': u','.join(header.get('to', [''])), - 'cc': u','.join(header.get('cc', [''])), - 'bcc': u','.join(header.get('bcc', [''])), - 'tag': u','.join(unique(tags)), - 'body': unicode(mdict['body']), - 'ident': unicode(mdict['ident']), - 'flags': unicode(','.join(unique(mail.flags))), - 'raw': unicode(mail.raw) - } - - writer.update_document(**index_data) - - def index_mails(self, mails, callback=None): - with self._index.writer() as writer: - for mail in mails: - self._index_mail(writer, mail) - if callback: - callback() - - def _search_with_options(self, options, query): - with self._index.searcher() as searcher: - query = QueryParser('raw', self._index.schema).parse(query) - results = searcher.search(query, **options) - return results - - def search(self, query, window=25, page=1, all_mails=False): - query = self.prepare_query(query) - return self._search_all_mails(query) if all_mails else self._paginated_search_mails(query, window, page) - - def _search_all_mails(self, query): - with self._index.searcher() as searcher: - sorting_facet = sorting.FieldFacet('date', reverse=True) - results = searcher.search(query, sortedby=sorting_facet, reverse=True, limit=None) - return unique([mail['ident'] for mail in results]) - - def _paginated_search_mails(self, query, window, page): - page = int(page) if page is not None and int(page) > 1 else 1 - window = int(window) if window is not None else 25 - - with self._index.searcher() as searcher: - tags_facet = sorting.FieldFacet('tag', allow_overlap=True, maptype=sorting.Count) - sorting_facet = sorting.FieldFacet('date', reverse=True) - results = searcher.search_page(query, page, pagelen=window, groupedby=tags_facet, sortedby=sorting_facet) - return unique([mail['ident'] for mail in results]), sum(results.results.groups().values()) - - def prepare_query(self, query): - query = ( - query - .replace('\"', '') - .replace('-in:', 'AND NOT tag:') - .replace('in:all', '*') - ) - return MultifieldParser(['raw', 'body'], self._index.schema).parse(query) - - def remove_from_index(self, mail_id): - writer = self._index.writer() - try: - writer.delete_by_term('ident', mail_id) - finally: - writer.commit() - - def contacts(self, query): - restrict_q = Term("tag", "drafts") | Term("tag", "trash") - - if query: - to = QueryParser('to', self._index.schema) - cc = QueryParser('cc', self._index.schema) - bcc = QueryParser('bcc', self._index.schema) - with self._index.searcher() as searcher: - to = searcher.search(to.parse("*%s*" % query), limit=None, mask=restrict_q, - groupedby=sorting.FieldFacet('to', allow_overlap=True)).groups() - cc = searcher.search(cc.parse("*%s*" % query), limit=None, mask=restrict_q, - groupedby=sorting.FieldFacet('cc', allow_overlap=True)).groups() - bcc = searcher.search(bcc.parse("*%s*" % query), limit=None, mask=restrict_q, - groupedby=sorting.FieldFacet('bcc', allow_overlap=True)).groups() - return flatten([to, cc, bcc]) - - return [] diff --git a/service/pixelated/adapter/search/__init__.py b/service/pixelated/adapter/search/__init__.py new file mode 100644 index 00000000..bbefc487 --- /dev/null +++ b/service/pixelated/adapter/search/__init__.py @@ -0,0 +1,204 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . + +from pixelated.support.encrypted_file_storage import EncryptedFileStorage + +import os +from pixelated.adapter.model.status import Status +from pixelated.support.functional import flatten +from whoosh.index import FileIndex +from whoosh.fields import * +from whoosh.qparser import QueryParser +from whoosh.qparser import MultifieldParser +from whoosh.query import Term +from whoosh import sorting +from pixelated.support.functional import unique +from pixelated.support.date import milliseconds + + +class SearchEngine(object): + INDEX_FOLDER = os.path.join(os.environ['HOME'], '.leap', 'search_index') + DEFAULT_TAGS = ['inbox', 'sent', 'drafts', 'trash'] + + def __init__(self, soledad_querier): + self.soledad_querier = soledad_querier + if not os.path.exists(self.INDEX_FOLDER): + os.makedirs(self.INDEX_FOLDER) + self._index = self._create_index() + + def _add_to_tags(self, tags, group, skip_default_tags, count_type, query=None): + query_matcher = re.compile(".*%s.*" % query.lower()) if query else re.compile(".*") + + for tag, count in group.iteritems(): + + if skip_default_tags and tag in self.DEFAULT_TAGS or not query_matcher.match(tag): + continue + + if not tags.get(tag): + tags[tag] = {'ident': tag, 'name': tag, 'default': False, 'counts': {'total': 0, 'read': 0}, + 'mails': []} + tags[tag]['counts'][count_type] += count + + def _search_tag_groups(self, is_filtering_tags): + seen = None + query_parser = QueryParser('tag', self._index.schema) + options = {'limit': None, 'groupedby': sorting.FieldFacet('tag', allow_overlap=True), 'maptype': sorting.Count} + + with self._index.searcher() as searcher: + total = searcher.search(query_parser.parse('*'), **options).groups() + if not is_filtering_tags: + seen = searcher.search(query_parser.parse("* AND flags:%s" % Status.SEEN), **options).groups() + return seen, total + + def _init_tags_defaults(self): + tags = {} + for default_tag in self.DEFAULT_TAGS: + tags[default_tag] = { + 'ident': default_tag, + 'name': default_tag, + 'default': True, + 'counts': { + 'total': 0, + 'read': 0 + }, + 'mails': [] + } + return tags + + def _build_tags(self, seen, total, skip_default_tags, query): + tags = {} + if not skip_default_tags: + tags = self._init_tags_defaults() + self._add_to_tags(tags, total, skip_default_tags, count_type='total', query=query) + if seen: + self._add_to_tags(tags, seen, skip_default_tags, count_type='read') + return tags.values() + + def tags(self, query, skip_default_tags): + is_filtering_tags = True if query else False + seen, total = self._search_tag_groups(is_filtering_tags=is_filtering_tags) + return self._build_tags(seen, total, skip_default_tags, query) + + def _mail_schema(self): + return Schema( + ident=ID(stored=True, unique=True), + sender=ID(stored=False), + to=KEYWORD(stored=False, commas=True), + cc=KEYWORD(stored=False, commas=True), + bcc=KEYWORD(stored=False, commas=True), + subject=TEXT(stored=False), + date=NUMERIC(stored=False, sortable=True, bits=64, signed=False), + body=TEXT(stored=False), + tag=KEYWORD(stored=True, commas=True), + flags=KEYWORD(stored=True, commas=True), + raw=TEXT(stored=False)) + + def _create_index(self): + masterkey = self.soledad_querier.get_index_masterkey() + storage = EncryptedFileStorage(self.INDEX_FOLDER, masterkey) + return FileIndex.create(storage, self._mail_schema(), indexname='mails') + + def index_mail(self, mail): + with self._index.writer() as writer: + self._index_mail(writer, mail) + + def _index_mail(self, writer, mail): + mdict = mail.as_dict() + header = mdict['header'] + tags = mdict.get('tags', []) + tags.append(mail.mailbox_name.lower()) + index_data = { + 'sender': unicode(header.get('from', '')), + 'subject': unicode(header.get('subject', '')), + 'date': milliseconds(header.get('date', '')), + 'to': u','.join(header.get('to', [''])), + 'cc': u','.join(header.get('cc', [''])), + 'bcc': u','.join(header.get('bcc', [''])), + 'tag': u','.join(unique(tags)), + 'body': unicode(mdict['body']), + 'ident': unicode(mdict['ident']), + 'flags': unicode(','.join(unique(mail.flags))), + 'raw': unicode(mail.raw) + } + + writer.update_document(**index_data) + + def index_mails(self, mails, callback=None): + with self._index.writer() as writer: + for mail in mails: + self._index_mail(writer, mail) + if callback: + callback() + + def _search_with_options(self, options, query): + with self._index.searcher() as searcher: + query = QueryParser('raw', self._index.schema).parse(query) + results = searcher.search(query, **options) + return results + + def search(self, query, window=25, page=1, all_mails=False): + query = self.prepare_query(query) + return self._search_all_mails(query) if all_mails else self._paginated_search_mails(query, window, page) + + def _search_all_mails(self, query): + with self._index.searcher() as searcher: + sorting_facet = sorting.FieldFacet('date', reverse=True) + results = searcher.search(query, sortedby=sorting_facet, reverse=True, limit=None) + return unique([mail['ident'] for mail in results]) + + def _paginated_search_mails(self, query, window, page): + page = int(page) if page is not None and int(page) > 1 else 1 + window = int(window) if window is not None else 25 + + with self._index.searcher() as searcher: + tags_facet = sorting.FieldFacet('tag', allow_overlap=True, maptype=sorting.Count) + sorting_facet = sorting.FieldFacet('date', reverse=True) + results = searcher.search_page(query, page, pagelen=window, groupedby=tags_facet, sortedby=sorting_facet) + return unique([mail['ident'] for mail in results]), sum(results.results.groups().values()) + + def prepare_query(self, query): + query = ( + query + .replace('\"', '') + .replace('-in:', 'AND NOT tag:') + .replace('in:all', '*') + ) + return MultifieldParser(['raw', 'body'], self._index.schema).parse(query) + + def remove_from_index(self, mail_id): + writer = self._index.writer() + try: + writer.delete_by_term('ident', mail_id) + finally: + writer.commit() + + def contacts(self, query): + restrict_q = Term("tag", "drafts") | Term("tag", "trash") + + if query: + to = QueryParser('to', self._index.schema) + cc = QueryParser('cc', self._index.schema) + bcc = QueryParser('bcc', self._index.schema) + with self._index.searcher() as searcher: + to = searcher.search(to.parse("*%s*" % query), limit=None, mask=restrict_q, + groupedby=sorting.FieldFacet('to', allow_overlap=True)).groups() + cc = searcher.search(cc.parse("*%s*" % query), limit=None, mask=restrict_q, + groupedby=sorting.FieldFacet('cc', allow_overlap=True)).groups() + bcc = searcher.search(bcc.parse("*%s*" % query), limit=None, mask=restrict_q, + groupedby=sorting.FieldFacet('bcc', allow_overlap=True)).groups() + return flatten([to, cc, bcc]) + + return [] diff --git a/service/pixelated/adapter/services/__init__.py b/service/pixelated/adapter/services/__init__.py new file mode 100644 index 00000000..2756a319 --- /dev/null +++ b/service/pixelated/adapter/services/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . diff --git a/service/pixelated/adapter/services/draft_service.py b/service/pixelated/adapter/services/draft_service.py new file mode 100644 index 00000000..ddb86c5c --- /dev/null +++ b/service/pixelated/adapter/services/draft_service.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . + + +class DraftService(object): + __slots__ = '_mailboxes' + + def __init__(self, mailboxes): + self._mailboxes = mailboxes + + def create_draft(self, input_mail): + self._drafts().add(input_mail) + return input_mail + + def update_draft(self, ident, input_mail): + new_mail = self.create_draft(input_mail) + self._drafts().remove(ident) + return new_mail + + def _drafts(self): + return self._mailboxes.drafts() diff --git a/service/pixelated/adapter/services/mail_sender.py b/service/pixelated/adapter/services/mail_sender.py new file mode 100644 index 00000000..50c17ba5 --- /dev/null +++ b/service/pixelated/adapter/services/mail_sender.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . +from StringIO import StringIO + +from twisted.internet.defer import Deferred +from twisted.mail.smtp import SMTPSenderFactory +from twisted.internet import reactor +from pixelated.support.functional import flatten + + +class MailSender(): + def __init__(self, account_email_address, smtp_client=None): + self.account_email_address = account_email_address + + def sendmail(self, mail): + recipients = flatten([mail.to, mail.cc, mail.bcc]) + + resultDeferred = Deferred() + senderFactory = SMTPSenderFactory( + fromEmail=self.account_email_address, + toEmail=recipients, + file=StringIO(mail.to_smtp_format()), + deferred=resultDeferred) + + return reactor.connectTCP('localhost', 4650, senderFactory) diff --git a/service/pixelated/adapter/services/mail_service.py b/service/pixelated/adapter/services/mail_service.py new file mode 100644 index 00000000..722b9a29 --- /dev/null +++ b/service/pixelated/adapter/services/mail_service.py @@ -0,0 +1,89 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . + + +class MailService: + __slots__ = ['leap_session', 'account', 'mailbox_name'] + + def __init__(self, mailboxes, mail_sender, tag_service, soledad_querier): + self.tag_service = tag_service + self.mailboxes = mailboxes + self.querier = soledad_querier + self.mail_sender = mail_sender + + def all_mails(self): + return self.querier.all_mails() + + def mails(self, ids): + return self.querier.mails(ids) + + def update_tags(self, mail_id, new_tags): + reserved_words = self.tag_service.extract_reserved(new_tags) + if len(reserved_words): + raise ValueError('None of the following words can be used as tags: ' + ' '.join(reserved_words)) + mail = self.mail(mail_id) + mail.update_tags(set(new_tags)) + return mail + + def mail(self, mail_id): + return self.mailboxes.mail(mail_id) + + def send(self, last_draft_ident, mail): + self.mail_sender.sendmail(mail) + if last_draft_ident: + self.mailboxes.drafts().remove(last_draft_ident) + return self.mailboxes.sent().add(mail) + + def thread(self, thread_id): + raise NotImplementedError() + + def mark_as_read(self, mail_id): + return self.mail(mail_id).mark_as_read() + + def mark_as_unread(self, mail_id): + return self.mail(mail_id).mark_as_unread() + + def tags_for_thread(self, thread): + raise NotImplementedError() + + def add_tag_to_thread(self, thread_id, tag): + raise NotImplementedError() + + def remove_tag_from_thread(self, thread_id, tag): + raise NotImplementedError() + + def delete_mail(self, mail_id): + return self.mailboxes.move_to_trash(mail_id) + + def delete_permanent(self, mail_id): + mail = self.mail(mail_id) + self.querier.remove_mail(mail) + + def save_draft(self, draft): + raise NotImplementedError() + + def draft_reply_for(self, mail_id): + raise NotImplementedError() + + def all_contacts(self, query): + raise NotImplementedError() + + def drafts(self): + raise NotImplementedError() + + def reply_all_template(self, mail_id): + mail = self.mail(mail_id) + return mail.to_reply_template() diff --git a/service/pixelated/adapter/services/mailbox.py b/service/pixelated/adapter/services/mailbox.py new file mode 100644 index 00000000..fbdbfc30 --- /dev/null +++ b/service/pixelated/adapter/services/mailbox.py @@ -0,0 +1,51 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . + + +class Mailbox: + + def __init__(self, mailbox_name, querier): + self.mailbox_name = mailbox_name + self.mailbox_tag = mailbox_name.lower() + self.querier = querier + + def mails(self): + _mails = self.querier.all_mails_by_mailbox(self.mailbox_name) + + result = [] + for mail in _mails: + result.append(mail) + return result + + def mails_by_tags(self, tags): + if 'all' in tags or self.mailbox_tag in tags: + return self.mails() + return [mail for mail in self.mails() if len(mail.tags.intersection(tags)) > 0] + + def mail(self, mail_id): + return self.querier.mail(mail_id) + + def add(self, mail): + return self.querier.create_mail(mail, self.mailbox_name) + + def remove(self, ident): + mail = self.querier.mail(ident) + mail.remove_all_tags() + self.querier.remove_mail(mail) + + @classmethod + def create(cls, mailbox_name, soledad_querier): + return Mailbox(mailbox_name, soledad_querier) diff --git a/service/pixelated/adapter/services/mailboxes.py b/service/pixelated/adapter/services/mailboxes.py new file mode 100644 index 00000000..56304dd6 --- /dev/null +++ b/service/pixelated/adapter/services/mailboxes.py @@ -0,0 +1,68 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . +from pixelated.adapter.services.mailbox import Mailbox +from pixelated.adapter.listeners.mailbox_indexer_listener import MailboxIndexerListener + + +class Mailboxes(): + + def __init__(self, account, soledad_querier): + self.account = account + self.querier = soledad_querier + for mailbox_name in account.mailboxes: + MailboxIndexerListener.listen(self.account, mailbox_name, soledad_querier) + + def _create_or_get(self, mailbox_name): + mailbox_name = mailbox_name.upper() + if mailbox_name not in self.account.mailboxes: + self.account.addMailbox(mailbox_name) + MailboxIndexerListener.listen(self.account, mailbox_name, self.querier) + return Mailbox.create(mailbox_name, self.querier) + + def inbox(self): + return self._create_or_get('INBOX') + + def drafts(self): + return self._create_or_get('DRAFTS') + + def trash(self): + return self._create_or_get('TRASH') + + def sent(self): + return self._create_or_get('SENT') + + def mailboxes(self): + return [self._create_or_get(leap_mailbox_name) for leap_mailbox_name in self.account.mailboxes] + + def mails_by_tag(self, query_tags): + mails = [] + for mailbox in self.mailboxes(): + mails.extend(mailbox.mails_by_tags(query_tags)) + + return mails + + def move_to_trash(self, mail_id): + mail = self.querier.mail(mail_id) + mail.remove_all_tags() + mail.set_mailbox(self.trash().mailbox_name) + mail.save() + return mail + + def mail(self, mail_id): + for mailbox in self.mailboxes(): + mail = mailbox.mail(mail_id) + if mail: + return mail diff --git a/service/pixelated/adapter/services/tag_service.py b/service/pixelated/adapter/services/tag_service.py new file mode 100644 index 00000000..c723b04c --- /dev/null +++ b/service/pixelated/adapter/services/tag_service.py @@ -0,0 +1,26 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . +from pixelated.adapter.model.tag import Tag + + +class TagService: + + instance = None + SPECIAL_TAGS = {Tag('inbox', True), Tag('sent', True), Tag('drafts', True), Tag('trash', True)} + + @classmethod + def extract_reserved(cls, tags): + return {tag.name for tag in cls.SPECIAL_TAGS if tag.name in tags} diff --git a/service/pixelated/adapter/soledad/__init__.py b/service/pixelated/adapter/soledad/__init__.py new file mode 100644 index 00000000..2756a319 --- /dev/null +++ b/service/pixelated/adapter/soledad/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . diff --git a/service/pixelated/adapter/soledad/soledad_querier.py b/service/pixelated/adapter/soledad/soledad_querier.py new file mode 100644 index 00000000..b319eac0 --- /dev/null +++ b/service/pixelated/adapter/soledad/soledad_querier.py @@ -0,0 +1,194 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . +import base64 +import quopri + +from cryptography.fernet import Fernet +from pixelated.adapter.model.mail import PixelatedMail +import re + + +class SoledadQuerier: + + def __init__(self, soledad): + self.soledad = soledad + + def mark_all_as_not_recent(self): + for mailbox in ['INBOX', 'DRAFTS', 'SENT', 'TRASH']: + rct = self.soledad.get_from_index('by-type-and-mbox', 'rct', mailbox) + if len(rct) == 0: + return + rct = rct[0] + rct.content['rct'] = [] + self.soledad.put_doc(rct) + + def all_mails(self): + fdocs_chash = [(fdoc, fdoc.content['chash']) for fdoc in self.soledad.get_from_index('by-type', 'flags')] + if len(fdocs_chash) == 0: + return [] + return self._build_mails_from_fdocs(fdocs_chash) + + def all_mails_by_mailbox(self, mailbox_name): + fdocs_chash = [(fdoc, fdoc.content['chash']) for fdoc in self.soledad.get_from_index('by-type-and-mbox', 'flags', mailbox_name)] + return self._build_mails_from_fdocs(fdocs_chash) + + def _build_mails_from_fdocs(self, fdocs_chash): + if len(fdocs_chash) == 0: + return [] + + fdocs_hdocs = [] + for fdoc, chash in fdocs_chash: + hdoc = self.soledad.get_from_index('by-type-and-contenthash', 'head', chash) + if len(hdoc) == 0: + continue + fdocs_hdocs.append((fdoc, hdoc[0])) + + fdocs_hdocs_bodyphash = [(f[0], f[1], f[1].content.get('body')) for f in fdocs_hdocs] + fdocs_hdocs_bdocs_parts = [] + for fdoc, hdoc, body_phash in fdocs_hdocs_bodyphash: + bdoc = self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', body_phash) + if len(bdoc) == 0: + continue + parts = self._extract_parts(hdoc.content) + fdocs_hdocs_bdocs_parts.append((fdoc, hdoc, bdoc[0], parts)) + + return [PixelatedMail.from_soledad(*raw_mail, soledad_querier=self) for raw_mail in fdocs_hdocs_bdocs_parts] + + def save_mail(self, mail): + self.soledad.put_doc(mail.fdoc) + self._update_index([mail.fdoc]) + + def create_mail(self, mail, mailbox_name): + mbox = [m for m in self.soledad.get_from_index('by-type', 'mbox') if m.content['mbox'] == 'INBOX'][0] + + uid = mbox.content['lastuid'] + 1 + new_docs = [self.soledad.create_doc(doc) for doc in mail.get_for_save(next_uid=uid, mailbox=mailbox_name)] + mbox.content['lastuid'] = uid + + self.soledad.put_doc(mbox) + self._update_index(new_docs) + + return self.mail(mail.ident) + + def mail(self, ident): + fdoc = self.soledad.get_from_index('by-type-and-contenthash', 'flags', ident)[0] + hdoc = self.soledad.get_from_index('by-type-and-contenthash', 'head', ident)[0] + parts = self._extract_parts(hdoc.content) + bdoc = self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', hdoc.content['body'])[0] + + return PixelatedMail.from_soledad(fdoc, hdoc, bdoc, parts=parts, soledad_querier=self) + + def mails(self, idents): + fdocs_chash = [(self.soledad.get_from_index('by-type-and-contenthash', 'flags', ident), ident) for ident in idents] + fdocs_chash = [(result[0], ident) for result, ident in fdocs_chash if result] + return self._build_mails_from_fdocs(fdocs_chash) + + def remove_mail(self, mail): + _mail = self.mail(mail.ident) + # FIX-ME: Must go through all the part_map phash to delete all the cdocs + self.soledad.delete_doc(_mail.fdoc) + self.soledad.delete_doc(_mail.hdoc) + self.soledad.delete_doc(_mail.bdoc) + + def idents_by_mailbox(self, mailbox_name): + return set(doc.content['chash'] for doc in self.soledad.get_from_index('by-type-and-mbox-and-deleted', 'flags', mailbox_name, '0')) + + def _update_index(self, docs): + db = self.soledad._db + + indexed_fields = db._get_indexed_fields() + if indexed_fields: + # It is expected that len(indexed_fields) is shorter than + # len(raw_doc) + getters = [(field, db._parse_index_definition(field)) + for field in indexed_fields] + for doc in docs: + db._update_indexes(doc.doc_id, doc.content, getters, db._db_handle) + + # Attachments + def attachment(self, ident, encoding): + bdoc = self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', ident)[0] + return {'content': self._try_decode(bdoc.content['raw'], encoding), + 'content-type': bdoc.content['content-type']} + + def _try_decode(self, raw, encoding): + encoding = encoding.lower() + if encoding == 'base64': + return base64.decodestring(raw) + elif encoding == 'quoted-printable': + return quopri.decodestring(raw) + else: + return str(raw) + + def _extract_parts(self, hdoc, parts=None): + if not parts: + parts = {'alternatives': [], 'attachments': []} + + if hdoc['multi']: + for part_key in hdoc.get('part_map', {}).keys(): + self._extract_parts(hdoc['part_map'][part_key], parts) + else: + headers_dict = {elem[0]: elem[1] for elem in hdoc.get('headers', [])} + if 'attachment' in headers_dict.get('Content-Disposition', ''): + parts['attachments'].append(self._extract_attachment(hdoc, headers_dict)) + else: + parts['alternatives'].append(self._extract_alternative(hdoc, headers_dict)) + return parts + + def _extract_alternative(self, hdoc, headers_dict): + bdoc = self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', hdoc['phash'])[0] + raw_content = bdoc.content['raw'] + return {'headers': headers_dict, 'content': raw_content} + + def _extract_attachment(self, hdoc, headers_dict): + content_disposition = headers_dict['Content-Disposition'] + match = re.compile('.*name=\"(.*)\".*').search(content_disposition) + filename = '' + if match: + filename = match.group(1) + return {'headers': headers_dict, 'ident': hdoc['phash'], 'name': filename} + + # Removing duplicates + def remove_duplicates(self): + for mailbox in ['INBOX', 'DRAFTS', 'SENT', 'TRASH']: + self._remove_dup_inboxes(mailbox) + self._remove_dup_recent(mailbox) + + def _remove_many(self, docs): + [self.soledad.delete_doc(doc) for doc in docs] + + def _remove_dup_inboxes(self, mailbox_name): + mailboxes = self.soledad.get_from_index('by-type-and-mbox', 'mbox', mailbox_name) + if len(mailboxes) == 0: + return + mailboxes_to_remove = sorted(mailboxes, key=lambda x: x.content['created'])[1:len(mailboxes)] + self._remove_many(mailboxes_to_remove) + + def _remove_dup_recent(self, mailbox_name): + rct = self.soledad.get_from_index('by-type-and-mbox', 'rct', mailbox_name) + if len(rct) == 0: + return + rct_to_remove = sorted(rct, key=lambda x: len(x.content['rct']), reverse=True)[1:len(rct)] + self._remove_many(rct_to_remove) + + # Search Index encryption key get/create + def get_index_masterkey(self): + index_key = self.soledad.get_from_index('by-type', 'index_key') + if len(index_key) == 0: + index_key = Fernet.generate_key() + self.soledad.create_doc(dict(type='index_key', value=index_key)) + return index_key + return str(index_key[0].content['value']) diff --git a/service/pixelated/adapter/soledad_querier.py b/service/pixelated/adapter/soledad_querier.py deleted file mode 100644 index d76ac59f..00000000 --- a/service/pixelated/adapter/soledad_querier.py +++ /dev/null @@ -1,191 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . -import base64 -import quopri - -from cryptography.fernet import Fernet -from pixelated.adapter.mail import PixelatedMail -import re - - -class SoledadQuerier: - - def __init__(self, soledad): - self.soledad = soledad - - def get_index_masterkey(self): - index_key = self.soledad.get_from_index('by-type', 'index_key') - if len(index_key) == 0: - index_key = Fernet.generate_key() - self.soledad.create_doc(dict(type='index_key', value=index_key)) - return index_key - return str(index_key[0].content['value']) - - def _remove_many(self, docs): - [self.soledad.delete_doc(doc) for doc in docs] - - def _remove_dup_inboxes(self, mailbox_name): - mailboxes = self.soledad.get_from_index('by-type-and-mbox', 'mbox', mailbox_name) - if len(mailboxes) == 0: - return - mailboxes_to_remove = sorted(mailboxes, key=lambda x: x.content['created'])[1:len(mailboxes)] - self._remove_many(mailboxes_to_remove) - - def _remove_dup_recent(self, mailbox_name): - rct = self.soledad.get_from_index('by-type-and-mbox', 'rct', mailbox_name) - if len(rct) == 0: - return - rct_to_remove = sorted(rct, key=lambda x: len(x.content['rct']), reverse=True)[1:len(rct)] - self._remove_many(rct_to_remove) - - def remove_duplicates(self): - for mailbox in ['INBOX', 'DRAFTS', 'SENT', 'TRASH']: - self._remove_dup_inboxes(mailbox) - self._remove_dup_recent(mailbox) - - def mark_all_as_not_recent(self): - for mailbox in ['INBOX', 'DRAFTS', 'SENT', 'TRASH']: - rct = self.soledad.get_from_index('by-type-and-mbox', 'rct', mailbox) - if len(rct) == 0: - return - rct = rct[0] - rct.content['rct'] = [] - self.soledad.put_doc(rct) - - def all_mails(self): - fdocs_chash = [(fdoc, fdoc.content['chash']) for fdoc in self.soledad.get_from_index('by-type', 'flags')] - if len(fdocs_chash) == 0: - return [] - return self._build_mails_from_fdocs(fdocs_chash) - - def all_mails_by_mailbox(self, mailbox_name): - fdocs_chash = [(fdoc, fdoc.content['chash']) for fdoc in self.soledad.get_from_index('by-type-and-mbox', 'flags', mailbox_name)] - return self._build_mails_from_fdocs(fdocs_chash) - - def _build_mails_from_fdocs(self, fdocs_chash): - if len(fdocs_chash) == 0: - return [] - - fdocs_hdocs = [] - for fdoc, chash in fdocs_chash: - hdoc = self.soledad.get_from_index('by-type-and-contenthash', 'head', chash) - if len(hdoc) == 0: - continue - fdocs_hdocs.append((fdoc, hdoc[0])) - - fdocs_hdocs_bodyphash = [(f[0], f[1], f[1].content.get('body')) for f in fdocs_hdocs] - fdocs_hdocs_bdocs_parts = [] - for fdoc, hdoc, body_phash in fdocs_hdocs_bodyphash: - bdoc = self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', body_phash) - if len(bdoc) == 0: - continue - parts = self._extract_parts(hdoc.content) - fdocs_hdocs_bdocs_parts.append((fdoc, hdoc, bdoc[0], parts)) - - return [PixelatedMail.from_soledad(*raw_mail, soledad_querier=self) for raw_mail in fdocs_hdocs_bdocs_parts] - - def save_mail(self, mail): - self.soledad.put_doc(mail.fdoc) - self._update_index([mail.fdoc]) - - def create_mail(self, mail, mailbox_name): - mbox = [m for m in self.soledad.get_from_index('by-type', 'mbox') if m.content['mbox'] == 'INBOX'][0] - - uid = mbox.content['lastuid'] + 1 - new_docs = [self.soledad.create_doc(doc) for doc in mail.get_for_save(next_uid=uid, mailbox=mailbox_name)] - mbox.content['lastuid'] = uid - - self.soledad.put_doc(mbox) - self._update_index(new_docs) - - return self.mail(mail.ident) - - def mail(self, ident): - fdoc = self.soledad.get_from_index('by-type-and-contenthash', 'flags', ident)[0] - hdoc = self.soledad.get_from_index('by-type-and-contenthash', 'head', ident)[0] - parts = self._extract_parts(hdoc.content) - bdoc = self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', hdoc.content['body'])[0] - - return PixelatedMail.from_soledad(fdoc, hdoc, bdoc, parts=parts, soledad_querier=self) - - def attachment(self, ident, encoding): - bdoc = self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', ident)[0] - return {'content': self._try_decode(bdoc.content['raw'], encoding), - 'content-type': bdoc.content['content-type']} - - def _try_decode(self, raw, encoding): - encoding = encoding.lower() - if encoding == 'base64': - return base64.decodestring(raw) - elif encoding == 'quoted-printable': - return quopri.decodestring(raw) - else: - return str(raw) - - def mails(self, idents): - fdocs_chash = [(self.soledad.get_from_index('by-type-and-contenthash', 'flags', ident), ident) for ident in idents] - fdocs_chash = [(result[0], ident) for result, ident in fdocs_chash if result] - return self._build_mails_from_fdocs(fdocs_chash) - - def _extract_parts(self, hdoc, parts=None): - if not parts: - parts = {'alternatives': [], 'attachments': []} - - if hdoc['multi']: - for part_key in hdoc.get('part_map', {}).keys(): - self._extract_parts(hdoc['part_map'][part_key], parts) - else: - headers_dict = {elem[0]: elem[1] for elem in hdoc.get('headers', [])} - if 'attachment' in headers_dict.get('Content-Disposition', ''): - parts['attachments'].append(self._extract_attachment(hdoc, headers_dict)) - else: - parts['alternatives'].append(self._extract_alternative(hdoc, headers_dict)) - return parts - - def _extract_alternative(self, hdoc, headers_dict): - bdoc = self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', hdoc['phash'])[0] - raw_content = bdoc.content['raw'] - return {'headers': headers_dict, 'content': raw_content} - - def _extract_attachment(self, hdoc, headers_dict): - content_disposition = headers_dict['Content-Disposition'] - match = re.compile('.*name=\"(.*)\".*').search(content_disposition) - filename = '' - if match: - filename = match.group(1) - return {'headers': headers_dict, 'ident': hdoc['phash'], 'name': filename} - - def remove_mail(self, mail): - _mail = self.mail(mail.ident) - # FIX-ME: Must go through all the part_map phash to delete all the cdocs - self.soledad.delete_doc(_mail.fdoc) - self.soledad.delete_doc(_mail.hdoc) - self.soledad.delete_doc(_mail.bdoc) - - def idents_by_mailbox(self, mailbox_name): - return set(doc.content['chash'] for doc in self.soledad.get_from_index('by-type-and-mbox-and-deleted', 'flags', mailbox_name, '0')) - - def _update_index(self, docs): - db = self.soledad._db - - indexed_fields = db._get_indexed_fields() - if indexed_fields: - # It is expected that len(indexed_fields) is shorter than - # len(raw_doc) - getters = [(field, db._parse_index_definition(field)) - for field in indexed_fields] - for doc in docs: - db._update_indexes(doc.doc_id, doc.content, getters, db._db_handle) diff --git a/service/pixelated/adapter/status.py b/service/pixelated/adapter/status.py deleted file mode 100644 index 5a11ee7b..00000000 --- a/service/pixelated/adapter/status.py +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . - - -class Status: - - SEEN = u'\\Seen' - ANSWERED = u'\\Answered' - DELETED = u'\\Deleted' - RECENT = u'\\Recent' - - FLAGS_TO_STATUSES = { - SEEN: 'read', - ANSWERED: 'replied', - RECENT: 'recent' - } - - @staticmethod - def from_flag(flag): - return Status.FLAGS_TO_STATUSES[flag] - - @staticmethod - def from_flags(flags): - return set(Status.from_flag(flag) for flag in flags if flag in Status.FLAGS_TO_STATUSES.keys()) - - @staticmethod - def to_flags(statuses): - statuses_to_flags = dict(zip(Status.FLAGS_TO_STATUSES.values(), Status.FLAGS_TO_STATUSES.keys())) - return [statuses_to_flags[status] for status in statuses] diff --git a/service/pixelated/adapter/tag.py b/service/pixelated/adapter/tag.py deleted file mode 100644 index d75022f9..00000000 --- a/service/pixelated/adapter/tag.py +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . - -import json - - -class Tag: - - @classmethod - def from_dict(cls, tag_dict): - tag = Tag(tag_dict['name'], tag_dict['default']) - tag.mails = set(tag_dict['mails']) - return tag - - @classmethod - def from_json_string(cls, json_string): - tag_dict = json.loads(json_string) - tag_dict['mails'] = set(tag_dict['mails']) - return Tag.from_dict(tag_dict) - - @property - def total(self): - return len(self.mails) - - def __init__(self, name, default=False): - self.name = name.lower() - self.ident = self.name.__hash__() - self.default = default - self.mails = set() - - def __eq__(self, other): - return self.name == other.name - - def __hash__(self): - return self.name.__hash__() - - def increment(self, mail_ident): - self.mails.add(mail_ident) - - def decrement(self, mail_ident): - self.mails.discard(mail_ident) - - def as_dict(self): - return { - 'name': self.name, - 'default': self.default, - 'ident': self.ident, - 'counts': {'total': self.total, - 'read': 0, - 'starred': 0, - 'replied': 0}, - 'mails': list(self.mails) - } - - def as_json_string(self): - tag_dict = self.as_dict() - return json.dumps(tag_dict) - - def __repr__(self): - return self.name diff --git a/service/pixelated/adapter/tag_service.py b/service/pixelated/adapter/tag_service.py deleted file mode 100644 index 54b88622..00000000 --- a/service/pixelated/adapter/tag_service.py +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . -from pixelated.adapter.tag import Tag - - -class TagService: - - instance = None - SPECIAL_TAGS = {Tag('inbox', True), Tag('sent', True), Tag('drafts', True), Tag('trash', True)} - - @classmethod - def extract_reserved(cls, tags): - return {tag.name for tag in cls.SPECIAL_TAGS if tag.name in tags} diff --git a/service/pixelated/config/app_factory.py b/service/pixelated/config/app_factory.py index 45a0ea84..035a16b2 100644 --- a/service/pixelated/config/app_factory.py +++ b/service/pixelated/config/app_factory.py @@ -22,19 +22,19 @@ from twisted.internet import ssl from twisted.web import resource from twisted.web.util import redirectTo from pixelated.config.routes import setup_routes -from pixelated.adapter.mail_service import MailService -from pixelated.adapter.mail import InputMail -from pixelated.adapter.mail_sender import MailSender -from pixelated.adapter.mailboxes import Mailboxes -from pixelated.adapter.soledad_querier import SoledadQuerier +from pixelated.adapter.services.mail_service import MailService +from pixelated.adapter.model.mail import InputMail +from pixelated.adapter.services.mail_sender import MailSender +from pixelated.adapter.services.mailboxes import Mailboxes +from pixelated.adapter.soledad.soledad_querier import SoledadQuerier from pixelated.adapter.search import SearchEngine -from pixelated.adapter.draft_service import DraftService -from pixelated.adapter.mailbox_indexer_listener import MailboxIndexerListener +from pixelated.adapter.services.draft_service import DraftService +from pixelated.adapter.listeners.mailbox_indexer_listener import MailboxIndexerListener import pixelated.bitmask_libraries.session as LeapSession from pixelated.bitmask_libraries.leap_srp import LeapAuthException from requests.exceptions import ConnectionError from pixelated.controllers import * -from pixelated.adapter.tag_service import TagService +from pixelated.adapter.services.tag_service import TagService from leap.common.events import ( register, unregister, diff --git a/service/pixelated/controllers/mails_controller.py b/service/pixelated/controllers/mails_controller.py index eba97784..50256fa5 100644 --- a/service/pixelated/controllers/mails_controller.py +++ b/service/pixelated/controllers/mails_controller.py @@ -16,7 +16,7 @@ import json -from pixelated.adapter.mail import InputMail +from pixelated.adapter.model.mail import InputMail from pixelated.controllers import respond_json diff --git a/service/setup.py b/service/setup.py index 60625013..a68c0a62 100644 --- a/service/setup.py +++ b/service/setup.py @@ -16,6 +16,7 @@ # along with Pixelated. If not, see . import sys + if 'develop' in sys.argv: sys.argv.append('--always-unzip') @@ -64,6 +65,7 @@ def data_files(): def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() + setup(name='pixelated-user-agent', version='0.1', description='API to serve the pixelated front-end requests', @@ -71,7 +73,20 @@ setup(name='pixelated-user-agent', author='Thoughtworks', author_email='pixelated-team@thoughtworks.com', url='http://pixelated-project.github.io', - packages=['pixelated', 'pixelated.adapter', 'pixelated.bitmask_libraries', 'pixelated.config', 'pixelated.certificates', 'pixelated.support', 'pixelated.controllers'], + packages=[ + 'pixelated', + 'pixelated.adapter', + 'pixelated.adapter.listeners', + 'pixelated.adapter.model', + 'pixelated.adapter.search', + 'pixelated.adapter.services', + 'pixelated.adapter.soledad', + 'pixelated.bitmask_libraries', + 'pixelated.config', + 'pixelated.certificates', + 'pixelated.support', + 'pixelated.controllers' + ], test_suite='nose.collector', install_requires=[ 'pyasn1==0.1.7', @@ -96,4 +111,4 @@ setup(name='pixelated-user-agent', }, data_files=data_files(), include_package_data=True - ) +) diff --git a/service/test/integration/contacts_test.py b/service/test/integration/contacts_test.py index 85d6cd3d..925e5e02 100644 --- a/service/test/integration/contacts_test.py +++ b/service/test/integration/contacts_test.py @@ -1,3 +1,18 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . from nose.twistedtools import deferred from test.support.integration import SoledadTestBase, MailBuilder diff --git a/service/test/integration/mark_as_read_unread_test.py b/service/test/integration/mark_as_read_unread_test.py index 03da404f..86a48e62 100644 --- a/service/test/integration/mark_as_read_unread_test.py +++ b/service/test/integration/mark_as_read_unread_test.py @@ -15,7 +15,7 @@ # along with Pixelated. If not, see . from test.support.integration import * -from pixelated.adapter.status import Status +from pixelated.adapter.model.status import Status class MarkAsReadUnreadTest(SoledadTestBase): diff --git a/service/test/support/integration/app_test_client.py b/service/test/support/integration/app_test_client.py index 82b34d69..4d444f8d 100644 --- a/service/test/support/integration/app_test_client.py +++ b/service/test/support/integration/app_test_client.py @@ -24,15 +24,15 @@ from leap.mail.imap.account import SoledadBackedAccount from leap.soledad.client import Soledad from mock import MagicMock, Mock import os -from pixelated.adapter.draft_service import DraftService -from pixelated.adapter.mail_service import MailService -from pixelated.adapter.mailboxes import Mailboxes -from pixelated.adapter.soledad_querier import SoledadQuerier -from pixelated.adapter.tag_service import TagService +from pixelated.adapter.services.draft_service import DraftService +from pixelated.adapter.services.mail_service import MailService +from pixelated.adapter.services.mailboxes import Mailboxes +from pixelated.adapter.soledad.soledad_querier import SoledadQuerier +from pixelated.adapter.services.tag_service import TagService from pixelated.controllers import FeaturesController, HomeController, MailsController, TagsController, \ SyncInfoController, AttachmentsController, ContactsController import pixelated.runserver -from pixelated.adapter.mail import PixelatedMail +from pixelated.adapter.model.mail import PixelatedMail from pixelated.adapter.search import SearchEngine from test.support.integration.model import MailBuilder diff --git a/service/test/support/integration/model.py b/service/test/support/integration/model.py index e05d74bb..ea5dcad0 100644 --- a/service/test/support/integration/model.py +++ b/service/test/support/integration/model.py @@ -15,8 +15,8 @@ # along with Pixelated. If not, see . import json -from pixelated.adapter.mail import InputMail -from pixelated.adapter.status import Status +from pixelated.adapter.model.mail import InputMail +from pixelated.adapter.model.status import Status class MailBuilder: diff --git a/service/test/support/test_helper.py b/service/test/support/test_helper.py index 881e1e5c..b2d3cf43 100644 --- a/service/test/support/test_helper.py +++ b/service/test/support/test_helper.py @@ -15,7 +15,7 @@ # along with Pixelated. If not, see . from datetime import datetime -from pixelated.adapter.mail import InputMail +from pixelated.adapter.model.mail import InputMail LEAP_FLAGS = ['\\Seen', diff --git a/service/test/unit/adapter/draft_service_test.py b/service/test/unit/adapter/draft_service_test.py index b9f1c267..baa07ce0 100644 --- a/service/test/unit/adapter/draft_service_test.py +++ b/service/test/unit/adapter/draft_service_test.py @@ -1,7 +1,7 @@ import unittest -from pixelated.adapter.mail import InputMail -from pixelated.adapter.draft_service import DraftService +from pixelated.adapter.model.mail import InputMail +from pixelated.adapter.services.draft_service import DraftService import test.support.test_helper as test_helper from mockito import * diff --git a/service/test/unit/adapter/mail_service_test.py b/service/test/unit/adapter/mail_service_test.py index e5085724..6f80c043 100644 --- a/service/test/unit/adapter/mail_service_test.py +++ b/service/test/unit/adapter/mail_service_test.py @@ -15,7 +15,7 @@ # along with Pixelated. If not, see . import unittest -from pixelated.adapter.mail_service import MailService +from pixelated.adapter.services.mail_service import MailService from mockito import * diff --git a/service/test/unit/adapter/mail_test.py b/service/test/unit/adapter/mail_test.py index 13eef91e..e8fd8755 100644 --- a/service/test/unit/adapter/mail_test.py +++ b/service/test/unit/adapter/mail_test.py @@ -16,7 +16,7 @@ import unittest import pixelated.support.date -from pixelated.adapter.mail import PixelatedMail, InputMail +from pixelated.adapter.model.mail import PixelatedMail, InputMail from mockito import * from test.support import test_helper import dateutil.parser as dateparser diff --git a/service/test/unit/adapter/mailbox_indexer_listener_test.py b/service/test/unit/adapter/mailbox_indexer_listener_test.py index 291739e0..65ba8966 100644 --- a/service/test/unit/adapter/mailbox_indexer_listener_test.py +++ b/service/test/unit/adapter/mailbox_indexer_listener_test.py @@ -16,7 +16,7 @@ import unittest from mockito import * -from pixelated.adapter.mailbox_indexer_listener import MailboxIndexerListener +from pixelated.adapter.listeners.mailbox_indexer_listener import MailboxIndexerListener class MailboxListenerTest(unittest.TestCase): diff --git a/service/test/unit/adapter/mailbox_test.py b/service/test/unit/adapter/mailbox_test.py index df46d02d..9725f418 100644 --- a/service/test/unit/adapter/mailbox_test.py +++ b/service/test/unit/adapter/mailbox_test.py @@ -15,8 +15,8 @@ # along with Pixelated. If not, see . import unittest -from pixelated.adapter.mail import PixelatedMail -from pixelated.adapter.mailbox import Mailbox +from pixelated.adapter.model.mail import PixelatedMail +from pixelated.adapter.services.mailbox import Mailbox from mockito import * from test.support import test_helper diff --git a/service/test/unit/adapter/mailboxes_test.py b/service/test/unit/adapter/mailboxes_test.py index cbed4577..542877d1 100644 --- a/service/test/unit/adapter/mailboxes_test.py +++ b/service/test/unit/adapter/mailboxes_test.py @@ -16,8 +16,8 @@ import unittest from mockito import * -from pixelated.adapter.mailbox import Mailbox -from pixelated.adapter.mailboxes import Mailboxes +from pixelated.adapter.services.mailbox import Mailbox +from pixelated.adapter.services.mailboxes import Mailboxes class PixelatedMailboxesTest(unittest.TestCase): diff --git a/service/test/unit/adapter/soledad_querier_test.py b/service/test/unit/adapter/soledad_querier_test.py index 561c4cb8..2cc23750 100644 --- a/service/test/unit/adapter/soledad_querier_test.py +++ b/service/test/unit/adapter/soledad_querier_test.py @@ -18,7 +18,7 @@ import json import base64 import quopri -from pixelated.adapter.soledad_querier import SoledadQuerier +from pixelated.adapter.soledad.soledad_querier import SoledadQuerier from mockito import mock, when, any import os diff --git a/service/test/unit/adapter/test_status.py b/service/test/unit/adapter/test_status.py index e9ead384..5cd0fa1e 100644 --- a/service/test/unit/adapter/test_status.py +++ b/service/test/unit/adapter/test_status.py @@ -15,7 +15,7 @@ # along with Pixelated. If not, see . import unittest -from pixelated.adapter.status import Status +from pixelated.adapter.model.status import Status class TestStatus(unittest.TestCase): diff --git a/service/test/unit/adapter/test_tag.py b/service/test/unit/adapter/test_tag.py index fc14ff49..a4fa819e 100644 --- a/service/test/unit/adapter/test_tag.py +++ b/service/test/unit/adapter/test_tag.py @@ -15,7 +15,7 @@ # along with Pixelated. If not, see . import unittest -from pixelated.adapter.tag import Tag +from pixelated.adapter.model.tag import Tag class TestTag(unittest.TestCase): -- cgit v1.2.3