summaryrefslogtreecommitdiff
path: root/service/src/pixelated/adapter
diff options
context:
space:
mode:
authorKali Kaneko <kali@leap.se>2017-07-25 11:40:11 -0400
committerKali Kaneko <kali@leap.se>2017-07-25 11:40:29 -0400
commit91e4481c450eb7eb928debc1cb7fa59bdb63dd7b (patch)
tree8fd7e6e77b6df669c33d96b7edad6db3cbe14dfe /service/src/pixelated/adapter
parente4f755309d4cf5cfb6b0bcc62ed73d6070956ab5 (diff)
[pkg] packaging and path changes
- move all the pixelated python package under src/ - move the pixelated_www package under the leap namespace - allow to set globally the static folder - add hours and minutes to the timestamp in package version, to allow for several releases a day.
Diffstat (limited to 'service/src/pixelated/adapter')
-rw-r--r--service/src/pixelated/adapter/__init__.py15
-rw-r--r--service/src/pixelated/adapter/listeners/__init__.py15
-rw-r--r--service/src/pixelated/adapter/listeners/mailbox_indexer_listener.py66
-rw-r--r--service/src/pixelated/adapter/mailstore/__init__.py20
-rw-r--r--service/src/pixelated/adapter/mailstore/body_parser.py69
-rw-r--r--service/src/pixelated/adapter/mailstore/leap_attachment_store.py81
-rw-r--r--service/src/pixelated/adapter/mailstore/leap_mailstore.py406
-rw-r--r--service/src/pixelated/adapter/mailstore/mailstore.py60
-rw-r--r--service/src/pixelated/adapter/mailstore/maintenance/__init__.py100
-rw-r--r--service/src/pixelated/adapter/mailstore/searchable_mailstore.py81
-rw-r--r--service/src/pixelated/adapter/model/__init__.py15
-rw-r--r--service/src/pixelated/adapter/model/mail.py224
-rw-r--r--service/src/pixelated/adapter/model/status.py42
-rw-r--r--service/src/pixelated/adapter/model/tag.py73
-rw-r--r--service/src/pixelated/adapter/search/__init__.py209
-rw-r--r--service/src/pixelated/adapter/search/contacts.py56
-rw-r--r--service/src/pixelated/adapter/search/index_storage_key.py42
-rw-r--r--service/src/pixelated/adapter/services/__init__.py15
-rw-r--r--service/src/pixelated/adapter/services/draft_service.py40
-rw-r--r--service/src/pixelated/adapter/services/feedback_service.py36
-rw-r--r--service/src/pixelated/adapter/services/mail_sender.py106
-rw-r--r--service/src/pixelated/adapter/services/mail_service.py172
-rw-r--r--service/src/pixelated/adapter/services/tag_service.py23
-rw-r--r--service/src/pixelated/adapter/welcome_mail.py30
24 files changed, 1996 insertions, 0 deletions
diff --git a/service/src/pixelated/adapter/__init__.py b/service/src/pixelated/adapter/__init__.py
new file mode 100644
index 00000000..2756a319
--- /dev/null
+++ b/service/src/pixelated/adapter/__init__.py
@@ -0,0 +1,15 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
diff --git a/service/src/pixelated/adapter/listeners/__init__.py b/service/src/pixelated/adapter/listeners/__init__.py
new file mode 100644
index 00000000..2756a319
--- /dev/null
+++ b/service/src/pixelated/adapter/listeners/__init__.py
@@ -0,0 +1,15 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
diff --git a/service/src/pixelated/adapter/listeners/mailbox_indexer_listener.py b/service/src/pixelated/adapter/listeners/mailbox_indexer_listener.py
new file mode 100644
index 00000000..9cc522cf
--- /dev/null
+++ b/service/src/pixelated/adapter/listeners/mailbox_indexer_listener.py
@@ -0,0 +1,66 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PCULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from twisted.internet import defer
+from twisted.logger import Logger
+
+logger = Logger()
+
+
+class MailboxIndexerListener(object):
+ """ Listens for new mails, keeping the index updated """
+
+ @classmethod
+ @defer.inlineCallbacks
+ def listen(cls, account, mailbox_name, mail_store, search_engine):
+ listener = MailboxIndexerListener(mailbox_name, mail_store, search_engine)
+ mail_collection = yield account.get_collection_by_mailbox(mailbox_name)
+ mail_collection.addListener(listener)
+
+ defer.returnValue(listener)
+
+ def __init__(self, mailbox_name, mail_store, search_engine):
+ self.mailbox_name = mailbox_name
+ self.mail_store = mail_store
+ self.search_engine = search_engine
+
+ @defer.inlineCallbacks
+ def notify_new(self):
+ try:
+ indexed_idents = set(self.search_engine.search('tag:' + self.mailbox_name.lower(), all_mails=True))
+ soledad_idents = yield self.mail_store.get_mailbox_mail_ids(self.mailbox_name)
+ soledad_idents = set(soledad_idents)
+
+ missing_idents = soledad_idents.difference(indexed_idents)
+
+ self.search_engine.index_mails((yield self.mail_store.get_mails(missing_idents, include_body=True)))
+ except Exception, e: # this is a event handler, don't let exceptions escape
+ logger.error(e)
+
+ def __eq__(self, other):
+ return other and other.mailbox_name == self.mailbox_name
+
+ def __hash__(self):
+ return self.mailbox_name.__hash__()
+
+ def __repr__(self):
+ return 'MailboxListener: ' + self.mailbox_name
+
+
+@defer.inlineCallbacks
+def listen_all_mailboxes(account, search_engine, mail_store):
+ mailboxes = yield account.list_all_mailbox_names()
+ for mailbox_name in mailboxes:
+ yield MailboxIndexerListener.listen(account, mailbox_name, mail_store, search_engine)
diff --git a/service/src/pixelated/adapter/mailstore/__init__.py b/service/src/pixelated/adapter/mailstore/__init__.py
new file mode 100644
index 00000000..978df45d
--- /dev/null
+++ b/service/src/pixelated/adapter/mailstore/__init__.py
@@ -0,0 +1,20 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+from pixelated.adapter.mailstore.mailstore import MailStore, underscore_uuid
+from pixelated.adapter.mailstore.leap_mailstore import LeapMailStore
+
+__all__ = ['MailStore', 'LeapMailStore', 'underscore_uuid']
diff --git a/service/src/pixelated/adapter/mailstore/body_parser.py b/service/src/pixelated/adapter/mailstore/body_parser.py
new file mode 100644
index 00000000..8cac75cf
--- /dev/null
+++ b/service/src/pixelated/adapter/mailstore/body_parser.py
@@ -0,0 +1,69 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+import re
+from email.parser import Parser
+
+from twisted.logger import Logger
+
+logger = Logger()
+
+
+def _parse_charset_header(content_type_and_charset_header, default_charset='us-ascii'):
+ try:
+ return re.compile('.*charset="?([a-zA-Z0-9-]+)"?', re.MULTILINE | re.DOTALL).match(content_type_and_charset_header).group(1)
+ except:
+ return default_charset
+
+
+class BodyParser(object):
+
+ def __init__(self, content, content_type='text/plain; charset="us-ascii"', content_transfer_encoding=None, charset=None):
+ self._content = content
+ self._content_type = content_type
+ self._content_transfer_encoding = content_transfer_encoding
+ self._charset = charset
+
+ def parsed_content(self):
+ charset = self._charset or _parse_charset_header(self._content_type)
+ text = self._serialize_for_parser(charset)
+
+ decoded_body = self._parse_and_decode(text)
+ return unicode(decoded_body, charset, errors='replace')
+
+ def _parse_and_decode(self, text):
+ parsed_body = Parser().parsestr(text)
+ decoded_body = self._unwrap_content_transfer_encoding(parsed_body)
+ return decoded_body
+
+ def _unwrap_content_transfer_encoding(self, parsed_body):
+ return parsed_body.get_payload(decode=True)
+
+ def _serialize_for_parser(self, charset):
+ text = u'Content-Type: %s\n' % self._content_type
+ if self._content_transfer_encoding is not None:
+ text += u'Content-Transfer-Encoding: %s\n' % self._content_transfer_encoding
+
+ text += u'\n'
+ encoded_text = text.encode(charset)
+ if isinstance(self._content, unicode):
+ try:
+ return encoded_text + self._content.encode(charset)
+ except UnicodeError, e:
+ logger.warn('Failed to encode content for charset %s. Ignoring invalid chars: %s' % (charset, e))
+ return encoded_text + self._content.encode(charset, 'ignore')
+ else:
+ return encoded_text + self._content
diff --git a/service/src/pixelated/adapter/mailstore/leap_attachment_store.py b/service/src/pixelated/adapter/mailstore/leap_attachment_store.py
new file mode 100644
index 00000000..b297f9e6
--- /dev/null
+++ b/service/src/pixelated/adapter/mailstore/leap_attachment_store.py
@@ -0,0 +1,81 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+
+import quopri
+import base64
+from email import encoders
+from leap.bitmask.mail.adaptors.soledad import SoledadMailAdaptor, ContentDocWrapper
+from twisted.internet import defer
+from email.mime.nonmultipart import MIMENonMultipart
+from email.mime.multipart import MIMEMultipart
+from leap.bitmask.mail.mail import Message
+
+
+class LeapAttachmentStore(object):
+
+ def __init__(self, soledad):
+ self.soledad = soledad
+
+ @defer.inlineCallbacks
+ def get_mail_attachment(self, attachment_id):
+ results = yield self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', attachment_id) if attachment_id else []
+ if results:
+ content = ContentDocWrapper(**results[0].content)
+ defer.returnValue({'content-type': content.content_type, 'content': self._try_decode(
+ content.raw, content.content_transfer_encoding)})
+ else:
+ raise ValueError('No attachment with id %s found!' % attachment_id)
+
+ @defer.inlineCallbacks
+ def add_attachment(self, content, content_type):
+ cdoc = self._attachment_to_cdoc(content, content_type)
+ attachment_id = cdoc.phash
+ try:
+ yield self.get_mail_attachment(attachment_id)
+ except ValueError:
+ yield self.soledad.create_doc(cdoc.serialize(), doc_id=attachment_id)
+ defer.returnValue(attachment_id)
+
+ def _try_decode(self, raw, encoding):
+ encoding = encoding.lower()
+ if encoding == 'base64':
+ data = base64.decodestring(raw)
+ elif encoding == 'quoted-printable':
+ data = quopri.decodestring(raw)
+ else:
+ data = str(raw)
+
+ return bytearray(data)
+
+ def _attachment_to_cdoc(self, content, content_type, encoder=encoders.encode_base64):
+ major, sub = content_type.split('/')
+ attachment = MIMENonMultipart(major, sub)
+ attachment.set_payload(content)
+ encoder(attachment)
+ attachment.add_header('Content-Disposition', 'attachment', filename='does_not_matter.txt')
+
+ pseudo_mail = MIMEMultipart()
+ pseudo_mail.attach(attachment)
+
+ tmp_mail = SoledadMailAdaptor().get_msg_from_string(MessageClass=Message, raw_msg=pseudo_mail.as_string())
+
+ cdoc = tmp_mail.get_wrapper().cdocs[1]
+ return cdoc
+
+ def _calc_attachment_id_(self, content, content_type, encoder=encoders.encode_base64):
+ cdoc = self._attachment_to_cdoc(content, content_type, encoder)
+ return cdoc.phash
diff --git a/service/src/pixelated/adapter/mailstore/leap_mailstore.py b/service/src/pixelated/adapter/mailstore/leap_mailstore.py
new file mode 100644
index 00000000..288223dd
--- /dev/null
+++ b/service/src/pixelated/adapter/mailstore/leap_mailstore.py
@@ -0,0 +1,406 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+import re
+from email.header import decode_header
+from uuid import uuid4
+
+from leap.bitmask.mail.adaptors.soledad import SoledadMailAdaptor
+from leap.bitmask.mail.mail import Message
+from twisted.internet import defer
+from twisted.internet.defer import FirstError, DeferredList
+
+from pixelated.adapter.mailstore.body_parser import BodyParser
+from pixelated.adapter.mailstore.mailstore import MailStore, underscore_uuid
+from pixelated.adapter.model.mail import Mail, InputMail
+from pixelated.support.functional import to_unicode
+from pixelated.support import date
+
+
+class AttachmentInfo(object):
+ def __init__(self, ident, name, encoding=None, ctype='application/octet-stream', size=0):
+ self.ident = ident
+ self.name = name
+ self.encoding = encoding
+ self.ctype = ctype
+ self.size = size
+
+ def __repr__(self):
+ return 'AttachmentInfo[%s, %s, %s]' % (self.ident, self.name, self.encoding)
+
+ def __str__(self):
+ return 'AttachmentInfo[%s, %s, %s]' % (self.ident, self.name, self.encoding)
+
+ def as_dict(self):
+ return {'ident': self.ident, 'name': self.name, 'encoding': self.encoding, 'size': self.size, 'content-type': self.ctype}
+
+
+class LeapMail(Mail):
+
+ def __init__(self, mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None, attachments=[]):
+ self._mail_id = mail_id
+ self._mailbox_name = mailbox_name
+ self._headers = headers if headers is not None else {}
+ self._body = to_unicode(body)
+ self.tags = set(tags) # TODO test that asserts copy
+ self._flags = set(flags) # TODO test that asserts copy
+ self._attachments = attachments
+
+ @property
+ def headers(self):
+ cpy = dict(self._headers)
+ for name in set(self._headers.keys()).intersection(['To', 'Cc', 'Bcc']):
+ cpy[name] = [address.strip() for address in (self._headers[name].split(',') if self._headers[name] else [])]
+
+ return cpy
+
+ @property
+ def ident(self):
+ return self._mail_id
+
+ @property
+ def mail_id(self):
+ return self._mail_id
+
+ @property
+ def body(self):
+ return self._body
+
+ @property
+ def flags(self):
+ return self._flags
+
+ @property
+ def mailbox_name(self):
+ return self._mailbox_name
+
+ @property
+ def security_casing(self):
+ casing = dict(imprints=self._signature_information(), locks=[])
+ if self._encrypted() == "decrypted":
+ casing["locks"] = [{"state": "valid"}]
+ return casing
+
+ def _encrypted(self):
+ return self.headers.get("X-Leap-Encryption", "false")
+
+ def _signature_information(self):
+ signature = self.headers.get("X-Leap-Signature", None)
+ if signature is None or signature.startswith("could not verify"):
+ return [{"state": "no_signature_information"}]
+ else:
+ if signature.startswith("valid"):
+ return [{"state": "valid", "seal": {"validity": "valid"}}]
+ else:
+ return []
+
+ @property
+ def raw(self):
+ result = u''
+ for k, v in self._headers.items():
+ content, encoding = decode_header(v)[0]
+ if encoding:
+ result += '%s: %s\n' % (k, unicode(content, encoding=encoding))
+ else:
+ result += '%s: %s\n' % (k, v)
+ result += '\n'
+
+ if self._body:
+ result = result + self._body
+
+ return result
+
+ def _remove_duplicates(self, values):
+ return list(set(values))
+
+ def _decoded_header_utf_8(self, header_value):
+ if isinstance(header_value, list):
+ return self._remove_duplicates([self._decoded_header_utf_8(v) for v in header_value])
+ elif header_value is not None:
+ def encode_chunk(content, encoding):
+ return unicode(content.strip(), encoding=encoding or 'ascii', errors='ignore')
+
+ try:
+ encoded_chunks = [encode_chunk(content, encoding) for content, encoding in decode_header(header_value)]
+ return ' '.join(encoded_chunks) # decode_header strips whitespaces on all chunks, joining over ' ' is only a workaround, not a proper fix
+ except UnicodeEncodeError:
+ return unicode(header_value.encode('ascii', errors='ignore'))
+
+ def as_dict(self):
+ return {
+ 'header': {k.lower(): self._decoded_header_utf_8(v) for k, v in self.headers.items()},
+ 'ident': self._mail_id,
+ 'tags': self.tags,
+ 'status': list(self.status),
+ 'body': self._body,
+ 'security_casing': self.security_casing,
+ 'textPlainBody': self._body,
+ 'mailbox': self._mailbox_name.lower(),
+ 'attachments': [attachment.as_dict() for attachment in self._attachments]
+ }
+
+ @staticmethod
+ def from_dict(mail_dict):
+ # TODO: implement this method and also write tests for it
+ headers = {key.capitalize(): value for key, value in mail_dict.get('header', {}).items()}
+ headers['Date'] = date.mail_date_now()
+ body = mail_dict.get('body', '')
+ tags = set(mail_dict.get('tags', []))
+ status = set(mail_dict.get('status', []))
+ attachments = []
+
+ # mail_id, mailbox_name, headers=None, tags=set(), flags=set(), body=None, attachments=[]
+ return LeapMail(None, None, headers, tags, set(), body, attachments)
+
+
+def _extract_filename(headers, default_filename='UNNAMED'):
+ content_disposition = headers.get('Content-Disposition') or headers.get('content-disposition', '')
+ filename = _extract_filename_from_name_header_part(content_disposition)
+ if not filename:
+ filename = headers.get('Content-Description', '')
+ if not filename:
+ content_type = headers.get('Content-Type', '')
+ filename = _extract_filename_from_name_header_part(content_type)
+
+ if not filename:
+ filename = default_filename
+
+ return filename
+
+
+def _extract_filename_from_name_header_part(header_value):
+ match = re.compile('.*name=\"?(.*[^\"\'])').search(header_value)
+ filename = ''
+ if match:
+ filename = match.group(1)
+ return filename
+
+
+class LeapMailStore(MailStore):
+ __slots__ = ('soledad')
+
+ def __init__(self, soledad):
+ self.soledad = soledad
+
+ @defer.inlineCallbacks
+ def get_mail(self, mail_id, include_body=False):
+ message = yield self._fetch_msg_from_soledad(mail_id)
+ if not _is_empty_message(message):
+ leap_mail = yield self._leap_message_to_leap_mail(mail_id, message, include_body)
+ else:
+ leap_mail = None
+
+ defer.returnValue(leap_mail)
+
+ @defer.inlineCallbacks
+ def get_mails(self, mail_ids, gracefully_ignore_errors=False, include_body=False):
+ deferreds = []
+ for mail_id in mail_ids:
+ deferreds.append(self.get_mail(mail_id, include_body=include_body))
+
+ if gracefully_ignore_errors:
+ results = yield DeferredList(deferreds, consumeErrors=True)
+ defer.returnValue([mail for ok, mail in results if ok and mail is not None])
+ else:
+ result = yield defer.gatherResults(deferreds, consumeErrors=True)
+ defer.returnValue(result)
+
+ @defer.inlineCallbacks
+ def update_mail(self, mail):
+ message = yield self._fetch_msg_from_soledad(mail.mail_id)
+ message.get_wrapper().set_tags(tuple(mail.tags))
+ message.get_wrapper().set_flags(tuple(mail.flags))
+ yield self._update_mail(message) # TODO assert this is yielded (otherwise asynchronous)
+
+ @defer.inlineCallbacks
+ def all_mails(self, gracefully_ignore_errors=False):
+ mdocs = yield self.soledad.get_from_index('by-type', 'meta')
+
+ mail_ids = map(lambda doc: doc.doc_id, mdocs)
+
+ mails = yield self.get_mails(mail_ids, gracefully_ignore_errors=gracefully_ignore_errors, include_body=True)
+ defer.returnValue(mails)
+
+ @defer.inlineCallbacks
+ def add_mailbox(self, mailbox_name):
+ mailbox = yield self._get_or_create_mailbox(mailbox_name)
+ defer.returnValue(mailbox)
+
+ @defer.inlineCallbacks
+ def get_mailbox_names(self):
+ mbox_map = set((yield self._mailbox_uuid_to_name_map()).values())
+
+ defer.returnValue(mbox_map.union({'INBOX'}))
+
+ @defer.inlineCallbacks
+ def _mailbox_uuid_to_name_map(self):
+ map = {}
+ mbox_docs = yield self.soledad.get_from_index('by-type', 'mbox')
+ for doc in mbox_docs:
+ map[underscore_uuid(doc.content.get('uuid'))] = doc.content.get('mbox')
+
+ defer.returnValue(map)
+
+ @defer.inlineCallbacks
+ def add_mail(self, mailbox_name, raw_msg):
+ mailbox = yield self._get_or_create_mailbox(mailbox_name)
+ message = SoledadMailAdaptor().get_msg_from_string(Message, raw_msg)
+ message.get_wrapper().set_mbox_uuid(mailbox.uuid)
+
+ yield SoledadMailAdaptor().create_msg(self.soledad, message)
+
+ # add behavious from insert_mdoc_id from mail.py
+ mail = yield self._leap_message_to_leap_mail(message.get_wrapper().mdoc.doc_id, message, include_body=True) # TODO test that asserts include_body
+ defer.returnValue(mail)
+
+ @defer.inlineCallbacks
+ def delete_mail(self, mail_id):
+ message = yield self._fetch_msg_from_soledad(mail_id)
+ if message and message.get_wrapper().mdoc.doc_id:
+ yield message.get_wrapper().delete(self.soledad)
+ defer.returnValue(True)
+ defer.returnValue(False)
+
+ @defer.inlineCallbacks
+ def get_mailbox_mail_ids(self, mailbox_name):
+ mailbox = yield self._get_or_create_mailbox(mailbox_name)
+ fdocs = yield self.soledad.get_from_index('by-type-and-mbox-uuid', 'flags', underscore_uuid(mailbox.uuid))
+
+ mail_ids = map(lambda doc: _fdoc_id_to_mdoc_id(doc.doc_id), fdocs)
+
+ defer.returnValue(mail_ids)
+
+ @defer.inlineCallbacks
+ def delete_mailbox(self, mailbox_name):
+ mbx_wrapper = yield self._get_or_create_mailbox(mailbox_name)
+ yield SoledadMailAdaptor().delete_mbox(self.soledad, mbx_wrapper)
+
+ @defer.inlineCallbacks
+ def copy_mail_to_mailbox(self, mail_id, mailbox_name):
+ message = yield self._fetch_msg_from_soledad(mail_id, load_body=True)
+ mailbox = yield self._get_or_create_mailbox(mailbox_name)
+ copy_wrapper = yield message.get_wrapper().copy(self.soledad, mailbox.uuid)
+
+ leap_message = Message(copy_wrapper)
+
+ mail = yield self._leap_message_to_leap_mail(copy_wrapper.mdoc.doc_id, leap_message, include_body=False)
+
+ defer.returnValue(mail)
+
+ @defer.inlineCallbacks
+ def move_mail_to_mailbox(self, mail_id, mailbox_name):
+ mail_copy = yield self.copy_mail_to_mailbox(mail_id, mailbox_name)
+ yield self.delete_mail(mail_id)
+ defer.returnValue(mail_copy)
+
+ def _update_mail(self, message):
+ return message.get_wrapper().update(self.soledad)
+
+ @defer.inlineCallbacks
+ def _leap_message_to_leap_mail(self, mail_id, message, include_body):
+ if include_body:
+ # TODO use body from message if available
+ body = yield self._raw_message_body(message)
+ else:
+ body = None
+
+ # fetch mailbox name by mbox_uuid
+ mbox_uuid = message.get_wrapper().fdoc.mbox_uuid
+ mbox_name = yield self._mailbox_name_from_uuid(mbox_uuid)
+ attachments = self._extract_attachment_info_from(message)
+ mail = LeapMail(mail_id, mbox_name, message.get_wrapper().hdoc.headers, set(message.get_tags()), set(message.get_flags()), body=body, attachments=attachments) # TODO assert flags are passed on
+
+ defer.returnValue(mail)
+
+ @defer.inlineCallbacks
+ def _raw_message_body(self, message):
+ content_doc = (yield message.get_wrapper().get_body(self.soledad))
+ parser = BodyParser('', content_type='text/plain', content_transfer_encoding='UTF-8')
+ # It fix the problem when leap doesn'r found body_phash and returns empty string
+ if not isinstance(content_doc, str):
+ parser = BodyParser(content_doc.raw, content_type=content_doc.content_type,
+ content_transfer_encoding=content_doc.content_transfer_encoding, charset=content_doc.charset)
+
+ defer.returnValue(parser.parsed_content())
+
+ @defer.inlineCallbacks
+ def _mailbox_name_from_uuid(self, uuid):
+ map = (yield self._mailbox_uuid_to_name_map())
+ defer.returnValue(map.get(uuid, ''))
+
+ @defer.inlineCallbacks
+ def _get_or_create_mailbox(self, mailbox_name):
+ mailbox_name_upper = mailbox_name.upper()
+ mbx = yield SoledadMailAdaptor().get_or_create_mbox(self.soledad, mailbox_name_upper)
+ if mbx.uuid is None:
+ mbx.uuid = str(uuid4())
+ yield mbx.update(self.soledad)
+ defer.returnValue(mbx)
+
+ def _fetch_msg_from_soledad(self, mail_id, load_body=False):
+ return SoledadMailAdaptor().get_msg_from_mdoc_id(Message, self.soledad, mail_id, get_cdocs=load_body)
+
+ @defer.inlineCallbacks
+ def _dump_soledad(self):
+ gen, docs = yield self.soledad.get_all_docs()
+ for doc in docs:
+ print '\n%s\n' % doc
+
+ def _extract_attachment_info_from(self, message):
+ wrapper = message.get_wrapper()
+ part_maps = wrapper.hdoc.part_map
+ return self._extract_part_map(part_maps)
+
+ def _is_attachment(self, part_map, headers):
+ disposition = headers.get('Content-Disposition') or headers.get('content-disposition')
+ content_type = part_map['ctype']
+
+ if 'multipart' in content_type:
+ return False
+
+ if 'text/plain' == content_type and ((disposition == 'inline') or (disposition is None)):
+ return False
+
+ return True
+
+ def _create_attachment_info_from(self, part_map, headers):
+ ident = part_map['phash']
+ name = _extract_filename(headers)
+ encoding = headers.get('Content-Transfer-Encoding', None)
+ ctype = part_map.get('ctype') or headers.get('Content-Type')
+ size = part_map.get('size', 0)
+
+ return AttachmentInfo(ident, name, encoding, ctype, size)
+
+ def _extract_part_map(self, part_maps):
+ result = []
+
+ for nr, part_map in part_maps.items():
+ if 'headers' in part_map and 'phash' in part_map:
+ headers = {header[0]: header[1] for header in part_map['headers']}
+ if self._is_attachment(part_map, headers):
+ result.append(self._create_attachment_info_from(part_map, headers))
+ if 'part_map' in part_map:
+ result += self._extract_part_map(part_map['part_map'])
+
+ return result
+
+
+def _is_empty_message(message):
+ return (message is None) or (message.get_wrapper().mdoc.doc_id is None)
+
+
+def _fdoc_id_to_mdoc_id(fdoc_id):
+ return 'M' + fdoc_id[1:]
diff --git a/service/src/pixelated/adapter/mailstore/mailstore.py b/service/src/pixelated/adapter/mailstore/mailstore.py
new file mode 100644
index 00000000..fbd7fc9e
--- /dev/null
+++ b/service/src/pixelated/adapter/mailstore/mailstore.py
@@ -0,0 +1,60 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+
+class MailStore(object):
+ def get_mail(self, mail_id):
+ pass
+
+ def get_mail_attachment(self, attachment_id):
+ pass
+
+ def get_mails(self, mail_ids, gracefully_ignore_errors=False, include_body=False):
+ pass
+
+ def all_mails(self):
+ pass
+
+ def delete_mail(self, mail_id):
+ pass
+
+ def update_mail(self, mail):
+ pass
+
+ def add_mail(self, mailbox_name, mail):
+ pass
+
+ def get_mailbox_names(self):
+ pass
+
+ def add_mailbox(self, mailbox_name):
+ pass
+
+ def delete_mailbox(self, mailbox_name):
+ pass
+
+ def get_mailbox_mail_ids(self, mailbox_name):
+ pass
+
+ def copy_mail_to_mailbox(self, mail_id, mailbox_name):
+ pass
+
+ def move_mail_to_mailbox(self, mail_id, mailbox_name):
+ pass
+
+
+def underscore_uuid(uuid):
+ return uuid.replace('-', '_')
diff --git a/service/src/pixelated/adapter/mailstore/maintenance/__init__.py b/service/src/pixelated/adapter/mailstore/maintenance/__init__.py
new file mode 100644
index 00000000..9a1007cc
--- /dev/null
+++ b/service/src/pixelated/adapter/mailstore/maintenance/__init__.py
@@ -0,0 +1,100 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from leap.bitmask.keymanager import documents as leap_doc
+from leap.bitmask.keymanager.keys import OpenPGPKey
+
+from twisted.internet import defer
+from twisted.logger import Logger
+
+
+TYPE_OPENPGP_KEY = 'OpenPGPKey'
+TYPE_OPENPGP_ACTIVE = 'OpenPGPKey-active'
+
+KEY_DOC_TYPES = {TYPE_OPENPGP_ACTIVE, TYPE_OPENPGP_KEY}
+
+logger = Logger()
+
+
+def _is_key_doc(doc):
+ return doc.content.get(leap_doc.KEY_TYPE_KEY, None) in KEY_DOC_TYPES
+
+
+def _is_private_key_doc(doc):
+ return _is_key_doc(doc) and doc.content.get(leap_doc.KEY_PRIVATE_KEY, False)
+
+
+def _is_active_key_doc(doc):
+ return _is_key_doc(doc) and doc.content.get(leap_doc.KEY_TYPE_KEY, None) == TYPE_OPENPGP_ACTIVE
+
+
+def _is_public_key(doc):
+ return _is_key_doc(doc) and not doc.content.get(leap_doc.KEY_PRIVATE_KEY, False)
+
+
+def _key_fingerprint(doc):
+ return doc.content.get(leap_doc.KEY_FINGERPRINT_KEY, None)
+
+
+def _address(doc):
+ return doc.content.get(leap_doc.KEY_ADDRESS_KEY, None)
+
+
+class SoledadMaintenance(object):
+ def __init__(self, soledad):
+ self._soledad = soledad
+
+ @defer.inlineCallbacks
+ def repair(self):
+ _, docs = yield self._soledad.get_all_docs()
+
+ private_key_fingerprints = self._key_fingerprints_with_private_key(docs)
+
+ for doc in docs:
+ if _is_key_doc(doc) and _key_fingerprint(doc) not in private_key_fingerprints:
+ logger.warn('Deleting doc %s for key %s of <%s>' % (doc.doc_id, _key_fingerprint(doc), _address(doc)))
+ yield self._soledad.delete_doc(doc)
+
+ yield self._repair_missing_active_docs(docs, private_key_fingerprints)
+
+ @defer.inlineCallbacks
+ def _repair_missing_active_docs(self, docs, private_key_fingerprints):
+ missing = self._missing_active_docs(docs, private_key_fingerprints)
+ for fingerprint in missing:
+ emails = self._emails_for_key_fingerprint(docs, fingerprint)
+ for email in emails:
+ logger.warn('Re-creating active doc for key %s, email %s' % (fingerprint, email))
+ yield self._soledad.create_doc_from_json(OpenPGPKey(email, fingerprint=fingerprint, private=False).get_active_json())
+
+ def _key_fingerprints_with_private_key(self, docs):
+ return [doc.content[leap_doc.KEY_FINGERPRINT_KEY] for doc in docs if _is_private_key_doc(doc)]
+
+ def _missing_active_docs(self, docs, private_key_fingerprints):
+ active_doc_ids = self._active_docs_for_key_fingerprint(docs)
+
+ return set([private_key_fingerprint for private_key_fingerprint in private_key_fingerprints if private_key_fingerprint not in active_doc_ids])
+
+ def _emails_for_key_fingerprint(self, docs, fingerprint):
+ for doc in docs:
+ if _is_private_key_doc(doc) and _key_fingerprint(doc) == fingerprint:
+ email = _address(doc)
+ if email is None:
+ return []
+ if isinstance(email, list):
+ return email
+ return [email]
+
+ def _active_docs_for_key_fingerprint(self, docs):
+ return [doc.content[leap_doc.KEY_FINGERPRINT_KEY] for doc in docs if _is_active_key_doc(doc) and _is_public_key(doc)]
diff --git a/service/src/pixelated/adapter/mailstore/searchable_mailstore.py b/service/src/pixelated/adapter/mailstore/searchable_mailstore.py
new file mode 100644
index 00000000..07e99ba7
--- /dev/null
+++ b/service/src/pixelated/adapter/mailstore/searchable_mailstore.py
@@ -0,0 +1,81 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from twisted.internet import defer
+from types import FunctionType
+from pixelated.adapter.mailstore import MailStore
+
+
+class SearchableMailStore(object): # implementes MailStore
+
+ def __init__(self, delegate, search_engine):
+ self._delegate = delegate
+ self._search_engine = search_engine
+
+ @classmethod
+ def _create_delegator(cls, method_name):
+ def delegator(self, *args, **kw):
+ return getattr(self._delegate, method_name)(*args, **kw)
+
+ setattr(cls, method_name, delegator)
+
+ @defer.inlineCallbacks
+ def add_mail(self, mailbox_name, mail):
+ stored_mail = yield self._delegate.add_mail(mailbox_name, mail)
+ self._search_engine.index_mail(stored_mail)
+ defer.returnValue(stored_mail)
+
+ @defer.inlineCallbacks
+ def delete_mail(self, mail_id):
+ removed = yield self._delegate.delete_mail(mail_id)
+ self._search_engine.remove_from_index(mail_id)
+ defer.returnValue(removed)
+
+ @defer.inlineCallbacks
+ def update_mail(self, mail):
+ yield self._delegate.update_mail(mail)
+ self._search_engine.index_mail(mail)
+
+ @defer.inlineCallbacks
+ def move_mail_to_mailbox(self, mail_id, mailbox_name):
+ moved_mail = yield self._delegate.move_mail_to_mailbox(mail_id, mailbox_name)
+ self._search_engine.remove_from_index(mail_id)
+ self._search_engine.index_mail(moved_mail)
+ defer.returnValue(moved_mail)
+
+ @defer.inlineCallbacks
+ def copy_mail_to_mailbox(self, mail_id, mailbox_name):
+ copied_mail = yield self._delegate.copy_mail_to_mailbox(mail_id, mailbox_name)
+ self._search_engine.index_mail(copied_mail)
+ defer.returnValue(copied_mail)
+
+ def delete_mailbox(self, mailbox_name):
+ raise NotImplementedError()
+
+ def __getattr__(self, name):
+ """
+ Acts like method missing. If a method of MailStore is not implemented in this class,
+ a delegate method is created.
+
+ :param name: attribute name
+ :return: method or attribute
+ """
+ methods = ([key for key, value in MailStore.__dict__.items() if type(value) == FunctionType])
+
+ if name in methods:
+ SearchableMailStore._create_delegator(name)
+ return super(SearchableMailStore, self).__getattribute__(name)
+ else:
+ raise NotImplementedError('No attribute %s' % name)
diff --git a/service/src/pixelated/adapter/model/__init__.py b/service/src/pixelated/adapter/model/__init__.py
new file mode 100644
index 00000000..2756a319
--- /dev/null
+++ b/service/src/pixelated/adapter/model/__init__.py
@@ -0,0 +1,15 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
diff --git a/service/src/pixelated/adapter/model/mail.py b/service/src/pixelated/adapter/model/mail.py
new file mode 100644
index 00000000..b6a8beb0
--- /dev/null
+++ b/service/src/pixelated/adapter/model/mail.py
@@ -0,0 +1,224 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PCULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+import re
+import binascii
+from email.mime.text import MIMEText
+from email.header import Header
+from hashlib import sha256
+from email.MIMEMultipart import MIMEMultipart
+from email.mime.nonmultipart import MIMENonMultipart
+
+from twisted.logger import Logger
+
+from leap.bitmask.mail import walk
+
+from pixelated.adapter.model.status import Status
+from pixelated.support import date
+
+
+logger = Logger()
+
+
+class Mail(object):
+ @property
+ def from_sender(self):
+ return self.headers['From']
+
+ @property
+ def to(self):
+ return self.headers['To']
+
+ @property
+ def cc(self):
+ return self.headers['Cc']
+
+ @property
+ def bcc(self):
+ return self.headers['Bcc']
+
+ @property
+ def subject(self):
+ return self.headers['Subject']
+
+ @property
+ def date(self):
+ return self.headers['Date']
+
+ @property
+ def status(self):
+ return Status.from_flags(self.flags)
+
+ @property
+ def flags(self):
+ return self.fdoc.content.get('flags')
+
+ @property
+ def mailbox_name(self):
+ # FIXME mbox is no longer available, instead we now have mbox_uuid
+ return self.fdoc.content.get('mbox', 'INBOX')
+
+ def _encode_header_value_list(self, header_value_list):
+ encoded_header_list = [self._encode_header_value(v) for v in header_value_list]
+ return ', '.join(encoded_header_list)
+
+ def _encode_header_value(self, header_value):
+ if isinstance(header_value, unicode):
+ return str(Header(header_value, 'utf-8'))
+ return str(header_value)
+
+ def _add_message_content(self, mime_multipart, body_to_use=None):
+ body_to_use = body_to_use or self.body
+ if isinstance(body_to_use, list):
+ for part in body_to_use:
+ mime_multipart.attach(MIMEText(part['raw'], part['content-type']))
+ else:
+ mime_multipart.attach(MIMEText(body_to_use, 'plain', self._charset()))
+
+ def _add_body(self, mime):
+ body_to_use = getattr(self, 'body', None) or getattr(self, 'text_plain_body', None)
+ self._add_message_content(mime, body_to_use)
+ self._add_attachments(mime)
+
+ def _generate_mime_multipart(self):
+ mime = MIMEMultipart()
+ self._add_headers(mime)
+ self._add_body(mime)
+ return mime
+
+ @property
+ def _mime_multipart(self):
+ self._mime = self._mime or self._generate_mime_multipart()
+ return self._mime
+
+ def _add_headers(self, mime):
+ for key, value in self.headers.items():
+ if isinstance(value, list):
+ mime[str(key)] = self._encode_header_value_list(value)
+ else:
+ mime[str(key)] = self._encode_header_value(value)
+
+ def _add_attachments(self, mime):
+ for attachment in getattr(self, '_attachments', []):
+ major, sub = attachment['content-type'].split('/')
+ attachment_mime = MIMENonMultipart(major, sub)
+ base64_attachment_file = binascii.b2a_base64(attachment['raw'])
+ attachment_mime.set_payload(base64_attachment_file)
+ attachment_mime['Content-Disposition'] = 'attachment; filename="%s"' % attachment['name']
+ attachment_mime['Content-Transfer-Encoding'] = 'base64'
+ mime.attach(attachment_mime)
+
+ def _charset(self):
+ content_type = self.headers.get('content_type', {})
+ if 'charset' in content_type:
+ return self._parse_charset_header(content_type)
+ return 'utf-8'
+
+ def _parse_charset_header(self, charset_header, default_charset='utf-8'):
+ try:
+ return re.compile('.*charset=([a-zA-Z0-9-]+)', re.MULTILINE | re.DOTALL).match(charset_header).group(1)
+ except:
+ return default_charset
+
+ @property
+ def raw(self):
+ return self._mime_multipart.as_string()
+
+ def _get_chash(self):
+ return sha256(self.raw).hexdigest()
+
+
+class InputMail(Mail):
+ def __init__(self):
+ self._raw_message = None
+ self._fd = None
+ self._hd = None
+ self._bd = None
+ self._chash = None
+ self._mime = None
+ self.headers = {}
+ self.body = ''
+ self._status = []
+ self._attachments = []
+
+ @property
+ def ident(self):
+ return self._get_chash()
+
+ def _get_body_phash(self):
+ return walk.get_body_phash(self._mime_multipart)
+
+ def _add_predefined_headers(self, mime_multipart):
+ for header in ['To', 'Cc', 'Bcc']:
+ if self.headers.get(header):
+ mime_multipart[header] = ", ".join(self.headers[header])
+ for header in ['Subject', 'From']:
+ if self.headers.get(header):
+ mime_multipart[header] = self.headers[header]
+ mime_multipart['Date'] = self.headers['Date']
+
+ def to_mime_multipart(self):
+ mime = MIMEMultipart()
+ self._add_predefined_headers(mime)
+ self._add_body(mime)
+ return mime
+
+ def to_smtp_format(self):
+ mime_multipart = self.to_mime_multipart()
+ return mime_multipart.as_string()
+
+ @staticmethod
+ def delivery_error_template(delivery_address):
+ return InputMail.from_dict({
+ 'body': "Mail undelivered for %s" % delivery_address,
+ 'header': {
+ 'bcc': [],
+ 'cc': [],
+ 'subject': "Mail undelivered for %s" % delivery_address
+ }
+ })
+
+ @staticmethod
+ def from_dict(mail_dict, from_address):
+ input_mail = InputMail()
+ input_mail.headers = {key.capitalize(): value for key, value in mail_dict.get('header', {}).items()}
+
+ input_mail.headers['Date'] = date.mail_date_now()
+ input_mail.headers['From'] = from_address
+
+ input_mail.body = mail_dict.get('body', '')
+ input_mail.tags = set(mail_dict.get('tags', []))
+ input_mail._status = set(mail_dict.get('status', []))
+ input_mail._attachments = mail_dict.get('attachments', [])
+ return input_mail
+
+ @staticmethod
+ def from_python_mail(mail):
+ input_mail = InputMail()
+ input_mail.headers = {unicode(key.capitalize()): unicode(value) for key, value in mail.items()}
+ input_mail.headers[u'Date'] = unicode(date.mail_date_now())
+ input_mail.headers[u'To'] = [u'']
+
+ for payload in mail.get_payload():
+ input_mail._mime_multipart.attach(payload)
+ if payload.get_content_type() == 'text/plain':
+ content_charset = payload.get_content_charset()
+ try:
+ input_mail.body = unicode(
+ payload.get_payload(decode=True), content_charset)
+ except TypeError:
+ input_mail.body = unicode(payload.get_payload(decode=True))
+ input_mail._mime = input_mail.to_mime_multipart()
+ return input_mail
diff --git a/service/src/pixelated/adapter/model/status.py b/service/src/pixelated/adapter/model/status.py
new file mode 100644
index 00000000..5a11ee7b
--- /dev/null
+++ b/service/src/pixelated/adapter/model/status.py
@@ -0,0 +1,42 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+
+class Status:
+
+ SEEN = u'\\Seen'
+ ANSWERED = u'\\Answered'
+ DELETED = u'\\Deleted'
+ RECENT = u'\\Recent'
+
+ FLAGS_TO_STATUSES = {
+ SEEN: 'read',
+ ANSWERED: 'replied',
+ RECENT: 'recent'
+ }
+
+ @staticmethod
+ def from_flag(flag):
+ return Status.FLAGS_TO_STATUSES[flag]
+
+ @staticmethod
+ def from_flags(flags):
+ return set(Status.from_flag(flag) for flag in flags if flag in Status.FLAGS_TO_STATUSES.keys())
+
+ @staticmethod
+ def to_flags(statuses):
+ statuses_to_flags = dict(zip(Status.FLAGS_TO_STATUSES.values(), Status.FLAGS_TO_STATUSES.keys()))
+ return [statuses_to_flags[status] for status in statuses]
diff --git a/service/src/pixelated/adapter/model/tag.py b/service/src/pixelated/adapter/model/tag.py
new file mode 100644
index 00000000..ca62a1fe
--- /dev/null
+++ b/service/src/pixelated/adapter/model/tag.py
@@ -0,0 +1,73 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+import json
+
+
+class Tag(object):
+
+ @classmethod
+ def from_dict(cls, tag_dict):
+ tag = Tag(tag_dict['name'], tag_dict['default'])
+ tag.mails = set(tag_dict['mails'])
+ return tag
+
+ @classmethod
+ def from_json_string(cls, json_string):
+ tag_dict = json.loads(json_string)
+ tag_dict['mails'] = set(tag_dict['mails'])
+ return Tag.from_dict(tag_dict)
+
+ @property
+ def total(self):
+ return len(self.mails)
+
+ def __init__(self, name, default=False):
+ self.name = name.lower()
+ self.ident = self.name.__hash__()
+ self.default = default
+ self.mails = set()
+
+ def __eq__(self, other):
+ return self.name == other.name
+
+ def __hash__(self):
+ return self.name.__hash__()
+
+ def increment(self, mail_ident):
+ self.mails.add(mail_ident)
+
+ def decrement(self, mail_ident):
+ self.mails.discard(mail_ident)
+
+ def as_dict(self):
+ return {
+ 'name': self.name,
+ 'default': self.default,
+ 'ident': self.ident,
+ 'counts': {'total': self.total,
+ 'read': 0,
+ 'starred': 0,
+ 'replied': 0},
+ 'mails': list(self.mails)
+ }
+
+ def as_json_string(self):
+ tag_dict = self.as_dict()
+ return json.dumps(tag_dict)
+
+ def __repr__(self):
+ return self.name
diff --git a/service/src/pixelated/adapter/search/__init__.py b/service/src/pixelated/adapter/search/__init__.py
new file mode 100644
index 00000000..e7a4e8c6
--- /dev/null
+++ b/service/src/pixelated/adapter/search/__init__.py
@@ -0,0 +1,209 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from pixelated.support.encrypted_file_storage import EncryptedFileStorage
+
+import os
+import re
+import dateutil.parser
+import time
+from pixelated.adapter.model.status import Status
+from pixelated.adapter.search.contacts import contacts_suggestions
+from whoosh.index import FileIndex
+from whoosh.fields import Schema, ID, KEYWORD, TEXT, NUMERIC, NGRAMWORDS
+from whoosh.qparser import QueryParser
+from whoosh.qparser import MultifieldParser
+from whoosh.writing import AsyncWriter
+from whoosh import sorting
+from pixelated.support.functional import unique, to_unicode
+import traceback
+from pixelated.support import date
+
+
+class SearchEngine(object):
+ DEFAULT_INDEX_HOME = os.path.join(os.environ['HOME'], '.leap')
+ DEFAULT_TAGS = ['inbox', 'sent', 'drafts', 'trash']
+
+ def __init__(self, key, user_home=DEFAULT_INDEX_HOME):
+ self.key = key
+ self.index_folder = os.path.join(user_home, 'search_index')
+ if not os.path.exists(self.index_folder):
+ os.makedirs(self.index_folder)
+ self._index = self._create_index()
+
+ def _add_to_tags(self, tags, group, skip_default_tags, count_type, query=None):
+ query_matcher = re.compile(".*%s.*" % query.lower()) if query else re.compile(".*")
+
+ for tag, count in group.iteritems():
+
+ if skip_default_tags and tag in self.DEFAULT_TAGS or not query_matcher.match(tag):
+ continue
+
+ if not tags.get(tag):
+ tags[tag] = {'ident': tag, 'name': tag, 'default': False, 'counts': {'total': 0, 'read': 0},
+ 'mails': []}
+ tags[tag]['counts'][count_type] += count
+
+ def _search_tag_groups(self, is_filtering_tags):
+ seen = None
+ query_parser = QueryParser('tag', self._index.schema)
+ options = {'limit': None, 'groupedby': sorting.FieldFacet('tag', allow_overlap=True), 'maptype': sorting.Count}
+
+ with self._index.searcher() as searcher:
+ total = searcher.search(query_parser.parse('*'), **options).groups()
+ if not is_filtering_tags:
+ seen = searcher.search(query_parser.parse("* AND flags:%s" % Status.SEEN), **options).groups()
+ return seen, total
+
+ def _init_tags_defaults(self):
+ tags = {}
+ for default_tag in self.DEFAULT_TAGS:
+ tags[default_tag] = {
+ 'ident': default_tag,
+ 'name': default_tag,
+ 'default': True,
+ 'counts': {
+ 'total': 0,
+ 'read': 0
+ },
+ 'mails': []
+ }
+ return tags
+
+ def _build_tags(self, seen, total, skip_default_tags, query):
+ tags = {}
+ if not skip_default_tags:
+ tags = self._init_tags_defaults()
+ self._add_to_tags(tags, total, skip_default_tags, count_type='total', query=query)
+ if seen:
+ self._add_to_tags(tags, seen, skip_default_tags, count_type='read')
+ return tags.values()
+
+ def tags(self, query, skip_default_tags):
+ is_filtering_tags = True if query else False
+ seen, total = self._search_tag_groups(is_filtering_tags=is_filtering_tags)
+ return self._build_tags(seen, total, skip_default_tags, query)
+
+ def _mail_schema(self):
+ return Schema(
+ ident=ID(stored=True, unique=True),
+ sender=ID(stored=False),
+ to=KEYWORD(stored=False, commas=True),
+ cc=KEYWORD(stored=False, commas=True),
+ bcc=KEYWORD(stored=False, commas=True),
+ subject=NGRAMWORDS(stored=False),
+ date=NUMERIC(stored=False, sortable=True, bits=64, signed=False),
+ body=NGRAMWORDS(stored=False),
+ tag=KEYWORD(stored=True, commas=True),
+ flags=KEYWORD(stored=True, commas=True),
+ raw=TEXT(stored=False))
+
+ def _create_index(self):
+ storage = EncryptedFileStorage(self.index_folder, self.key)
+ return FileIndex.create(storage, self._mail_schema(), indexname='mails')
+
+ def index_mail(self, mail):
+ if mail is not None:
+ with AsyncWriter(self._index) as writer:
+ self._index_mail(writer, mail)
+
+ def _index_mail(self, writer, mail):
+ mdict = mail.as_dict()
+ header = mdict['header']
+ tags = set(mdict.get('tags', {}))
+ tags.add(mail.mailbox_name.lower())
+
+ index_data = {
+ 'sender': self._empty_string_to_none(header.get('from', '')),
+ 'subject': self._empty_string_to_none(header.get('subject', '')),
+ 'date': self._format_utc_integer(header.get('date', date.mail_date_now())),
+ 'to': self._format_recipient(header, 'to'),
+ 'cc': self._format_recipient(header, 'cc'),
+ 'bcc': self._format_recipient(header, 'bcc'),
+ 'tag': u','.join(unique(tags)),
+ 'body': to_unicode(mdict.get('textPlainBody', mdict.get('body', ''))),
+ 'ident': unicode(mdict['ident']),
+ 'flags': unicode(','.join(unique(mail.flags))),
+ 'raw': unicode(mail.raw)
+ }
+
+ writer.update_document(**index_data)
+
+ def _format_utc_integer(self, date):
+ timetuple = dateutil.parser.parse(date).utctimetuple()
+ return time.strftime('%s', timetuple)
+
+ def _format_recipient(self, headers, name):
+ list = headers.get(name, [''])
+ return u','.join(list) if list else u''
+
+ def _empty_string_to_none(self, field_value):
+ if not field_value:
+ return None
+ else:
+ return field_value
+
+ def index_mails(self, mails, callback=None):
+ try:
+ with AsyncWriter(self._index) as writer:
+ for mail in mails:
+ self._index_mail(writer, mail)
+ if callback:
+ callback()
+ except Exception, e:
+ traceback.print_exc(e)
+ raise
+
+ def _search_with_options(self, options, query):
+ with self._index.searcher() as searcher:
+ query = QueryParser('raw', self._index.schema).parse(query)
+ results = searcher.search(query, **options)
+ return results
+
+ def search(self, query, window=25, page=1, all_mails=False):
+ query = self.prepare_query(query)
+ return self._search_all_mails(query) if all_mails else self._paginated_search_mails(query, window, page)
+
+ def _search_all_mails(self, query):
+ with self._index.searcher() as searcher:
+ sorting_facet = sorting.FieldFacet('date', reverse=True)
+ results = searcher.search(query, sortedby=sorting_facet, reverse=True, limit=None)
+ return unique([mail['ident'] for mail in results])
+
+ def _paginated_search_mails(self, query, window, page):
+ page = int(page) if page is not None and int(page) > 1 else 1
+ window = int(window) if window is not None else 25
+
+ with self._index.searcher() as searcher:
+ tags_facet = sorting.FieldFacet('tag', allow_overlap=True, maptype=sorting.Count)
+ sorting_facet = sorting.FieldFacet('date', reverse=True)
+ results = searcher.search_page(query, page, pagelen=window, groupedby=tags_facet, sortedby=sorting_facet)
+ return unique([mail['ident'] for mail in results]), sum(results.results.groups().values())
+
+ def prepare_query(self, query):
+ query = (
+ query
+ .replace('-in:', 'AND NOT tag:')
+ .replace('in:all', '*')
+ )
+ return MultifieldParser(['body', 'subject', 'raw'], self._index.schema).parse(query)
+
+ def remove_from_index(self, mail_id):
+ with AsyncWriter(self._index) as writer:
+ writer.delete_by_term('ident', mail_id)
+
+ def contacts(self, query):
+ with self._index.searcher() as searcher:
+ return contacts_suggestions(query, searcher)
diff --git a/service/src/pixelated/adapter/search/contacts.py b/service/src/pixelated/adapter/search/contacts.py
new file mode 100644
index 00000000..733489b0
--- /dev/null
+++ b/service/src/pixelated/adapter/search/contacts.py
@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from email.utils import parseaddr
+from pixelated.support.functional import flatten
+from whoosh.qparser import QueryParser
+from whoosh import sorting
+from whoosh.query import Term
+
+
+def address_duplication_filter(contacts):
+ contacts_by_mail = dict()
+
+ for contact in contacts:
+ mail_address = extract_mail_address(contact)
+ current = contacts_by_mail.get(mail_address, '')
+ current = contact if len(contact) > len(current) else current
+ contacts_by_mail[mail_address] = current
+ return contacts_by_mail.values()
+
+
+def extract_mail_address(text):
+ return parseaddr(text)[1]
+
+
+def contacts_suggestions(query, searcher):
+ return address_duplication_filter(search_addresses(searcher, query)) if query else []
+
+
+def search_addresses(searcher, query):
+ restrict_q = Term("tag", "drafts") | Term("tag", "trash")
+ results = []
+ for field in ['to', 'cc', 'bcc', 'sender']:
+ query_parser = QueryParser(field, searcher.schema)
+ results.append(
+ searcher.search(
+ query_parser.parse("*%s* OR *%s*" % (query.title(), query)),
+ limit=None,
+ mask=restrict_q,
+ groupedby=sorting.FieldFacet(
+ field,
+ allow_overlap=True),
+ terms=True).matched_terms())
+ return [address[1] for address in flatten(results)]
diff --git a/service/src/pixelated/adapter/search/index_storage_key.py b/service/src/pixelated/adapter/search/index_storage_key.py
new file mode 100644
index 00000000..b2761849
--- /dev/null
+++ b/service/src/pixelated/adapter/search/index_storage_key.py
@@ -0,0 +1,42 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+import base64
+from twisted.internet import defer
+import os
+
+
+class SearchIndexStorageKey(object):
+ __slots__ = '_soledad'
+
+ def __init__(self, soledad):
+ self._soledad = soledad
+
+ @defer.inlineCallbacks
+ def get_or_create_key(self):
+ docs = yield self._soledad.get_from_index('by-type', 'index_key')
+
+ if len(docs):
+ key = docs[0].content['value']
+ else:
+ key = self._new_index_key()
+ yield self._store_key_in_soledad(key)
+ defer.returnValue(key)
+
+ def _new_index_key(self):
+ return os.urandom(64) # 32 for encryption, 32 for hmac
+
+ def _store_key_in_soledad(self, index_key):
+ return self._soledad.create_doc(dict(type='index_key', value=base64.encodestring(index_key)))
diff --git a/service/src/pixelated/adapter/services/__init__.py b/service/src/pixelated/adapter/services/__init__.py
new file mode 100644
index 00000000..2756a319
--- /dev/null
+++ b/service/src/pixelated/adapter/services/__init__.py
@@ -0,0 +1,15 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
diff --git a/service/src/pixelated/adapter/services/draft_service.py b/service/src/pixelated/adapter/services/draft_service.py
new file mode 100644
index 00000000..504d92db
--- /dev/null
+++ b/service/src/pixelated/adapter/services/draft_service.py
@@ -0,0 +1,40 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from twisted.internet import defer
+
+
+class DraftService(object):
+ __slots__ = '_mail_store'
+
+ def __init__(self, mail_store):
+ self._mail_store = mail_store
+
+ @defer.inlineCallbacks
+ def create_draft(self, input_mail):
+ mail = yield self._mail_store.add_mail('DRAFTS', input_mail.raw)
+ defer.returnValue(mail)
+
+ @defer.inlineCallbacks
+ def update_draft(self, ident, input_mail):
+ removed = yield self._mail_store.delete_mail(ident)
+ if removed:
+ new_draft = yield self.create_draft(input_mail)
+ defer.returnValue(new_draft)
+
+ def process_draft(self, ident, input_mail):
+ if ident:
+ return self.update_draft(ident, input_mail)
+ return self.create_draft(input_mail)
diff --git a/service/src/pixelated/adapter/services/feedback_service.py b/service/src/pixelated/adapter/services/feedback_service.py
new file mode 100644
index 00000000..0cc595eb
--- /dev/null
+++ b/service/src/pixelated/adapter/services/feedback_service.py
@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import requests
+
+
+class FeedbackService(object):
+ FEEDBACK_URL = os.environ.get('FEEDBACK_URL')
+
+ def __init__(self, leap_session):
+ self.leap_session = leap_session
+
+ def open_ticket(self, feedback):
+ account_mail = self.leap_session.account_email()
+ data = {
+ "ticket[comments_attributes][0][body]": feedback,
+ "ticket[subject]": "Feedback user-agent from {0}".format(account_mail),
+ "ticket[email]": account_mail,
+ "ticket[regarding_user]": account_mail
+ }
+
+ return requests.post(self.FEEDBACK_URL, data=data, verify=False)
diff --git a/service/src/pixelated/adapter/services/mail_sender.py b/service/src/pixelated/adapter/services/mail_sender.py
new file mode 100644
index 00000000..063ea156
--- /dev/null
+++ b/service/src/pixelated/adapter/services/mail_sender.py
@@ -0,0 +1,106 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from StringIO import StringIO
+from email.utils import parseaddr
+from copy import deepcopy
+from leap.bitmask.mail.outgoing.service import OutgoingMail
+
+from twisted.internet.defer import Deferred, fail
+from twisted.mail.smtp import SMTPSenderFactory
+from twisted.internet import reactor, defer
+from pixelated.support.functional import flatten
+from twisted.mail.smtp import User
+
+from twisted.logger import Logger
+
+
+logger = Logger()
+
+
+class SMTPDownException(Exception):
+ def __init__(self):
+ Exception.__init__(self, "Couldn't send mail now, try again later.")
+
+
+NOT_NEEDED = None
+
+
+class MailSenderException(Exception):
+
+ def __init__(self, message, email_error_map):
+ super(MailSenderException, self).__init__(message, email_error_map)
+ self.email_error_map = email_error_map
+
+
+class MailSender(object):
+
+ def __init__(self, smtp_config, keymanager):
+ self._smtp_config = smtp_config
+ self._keymanager = keymanager
+
+ @defer.inlineCallbacks
+ def sendmail(self, mail):
+ # message is changed in sending, but should be saved unaltered
+ mail = deepcopy(mail)
+
+ recipients = flatten([mail.to, mail.cc, mail.bcc])
+
+ results = yield self._send_mail_to_all_recipients(mail, recipients)
+ all_succeeded = reduce(lambda a, b: a and b, [r[0] for r in results])
+
+ if not all_succeeded:
+ error_map = self._build_error_map(recipients, results)
+ raise MailSenderException('Failed to send mail to all recipients', error_map)
+
+ defer.returnValue(all_succeeded)
+
+ def _send_mail_to_all_recipients(self, mail, recipients):
+ outgoing_mail = self._create_outgoing_mail()
+ bccs = mail.bcc
+ deferreds = []
+
+ for recipient in recipients:
+ logger.info('_send_mail_to_all_recipients: Sending mail to recipient %s' % recipient)
+ self._define_bcc_field(mail, recipient, bccs)
+ smtp_recipient = self._create_twisted_smtp_recipient(recipient)
+ logger.info('_send_mail_to_all_recipients: Sending mail to smtp_recipient %s' % smtp_recipient)
+ deferreds.append(outgoing_mail.send_message(mail.to_smtp_format(), smtp_recipient))
+
+ return defer.DeferredList(deferreds, fireOnOneErrback=False, consumeErrors=True)
+
+ def _define_bcc_field(self, mail, recipient, bccs):
+ if recipient in bccs:
+ mail.headers['Bcc'] = [recipient]
+ else:
+ mail.headers['Bcc'] = []
+
+ def _build_error_map(self, recipients, results):
+ error_map = {}
+ for email, error in [(recipients[idx], r[1]) for idx, r in enumerate(results)]:
+ error_map[email] = error
+ return error_map
+
+ def _create_outgoing_mail(self):
+ return OutgoingMail(str(self._smtp_config.account_email),
+ self._keymanager,
+ self._smtp_config.cert_path,
+ self._smtp_config.cert_path,
+ str(self._smtp_config.remote_smtp_host),
+ int(self._smtp_config.remote_smtp_port))
+
+ def _create_twisted_smtp_recipient(self, recipient):
+ # TODO: Better is fix Twisted instead
+ return User(str(recipient), NOT_NEEDED, NOT_NEEDED, NOT_NEEDED)
diff --git a/service/src/pixelated/adapter/services/mail_service.py b/service/src/pixelated/adapter/services/mail_service.py
new file mode 100644
index 00000000..e5343997
--- /dev/null
+++ b/service/src/pixelated/adapter/services/mail_service.py
@@ -0,0 +1,172 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from email import encoders
+from email.mime.nonmultipart import MIMENonMultipart
+from email.mime.multipart import MIMEMultipart
+from leap.bitmask.mail.mail import Message
+
+from twisted.internet import defer
+
+from pixelated.adapter.model.mail import InputMail
+from pixelated.adapter.model.status import Status
+from pixelated.adapter.services.tag_service import extract_reserved_tags
+from leap.bitmask.mail.adaptors.soledad import SoledadMailAdaptor
+
+
+class MailService(object):
+
+ def __init__(self, mail_sender, mail_store, search_engine, account_email, attachment_store):
+ self.mail_store = mail_store
+ self.search_engine = search_engine
+ self.mail_sender = mail_sender
+ self.account_email = account_email
+ self.attachment_store = attachment_store
+
+ @defer.inlineCallbacks
+ def all_mails(self):
+ mails = yield self.mail_store.all_mails(gracefully_ignore_errors=True)
+ defer.returnValue(mails)
+
+ def save_attachment(self, content, content_type):
+ return self.attachment_store.add_attachment(content, content_type)
+
+ @defer.inlineCallbacks
+ def mails(self, query, window_size, page):
+ mail_ids, total = self.search_engine.search(query, window_size, page)
+
+ try:
+ mails = yield self.mail_store.get_mails(mail_ids)
+ defer.returnValue((mails, total))
+ except Exception, e:
+ import traceback
+ traceback.print_exc()
+ raise
+
+ @defer.inlineCallbacks
+ def update_tags(self, mail_id, new_tags):
+ new_tags = self._filter_white_space_tags(new_tags)
+ reserved_words = extract_reserved_tags(new_tags)
+ if len(reserved_words):
+ raise ValueError('None of the following words can be used as tags: ' + ' '.join(reserved_words))
+ new_tags = self._favor_existing_tags_casing(new_tags)
+ mail = yield self.mail(mail_id)
+ mail.tags = set(new_tags)
+ yield self.mail_store.update_mail(mail)
+
+ defer.returnValue(mail)
+
+ def _filter_white_space_tags(self, tags):
+ return [tag.strip() for tag in tags if not tag.isspace()]
+
+ def _favor_existing_tags_casing(self, new_tags):
+ current_tags = [tag['name'] for tag in self.search_engine.tags(query='', skip_default_tags=True)]
+ current_tags_lower = [tag.lower() for tag in current_tags]
+
+ def _use_current_casing(new_tag_lower):
+ return current_tags[current_tags_lower.index(new_tag_lower)]
+
+ return [_use_current_casing(new_tag.lower()) if new_tag.lower() in current_tags_lower else new_tag for new_tag in new_tags]
+
+ def mail(self, mail_id):
+ return self.mail_store.get_mail(mail_id, include_body=True)
+
+ def attachment(self, attachment_id):
+ return self.attachment_store.get_mail_attachment(attachment_id)
+
+ @defer.inlineCallbacks
+ def mail_exists(self, mail_id):
+ try:
+ mail = yield self.mail_store.get_mail(mail_id, include_body=False)
+ defer.returnValue(mail is not None)
+ except Exception, e:
+ defer.returnValue(False)
+
+ @defer.inlineCallbacks
+ def send_mail(self, content_dict):
+ mail = InputMail.from_dict(content_dict, self.account_email)
+ draft_id = content_dict.get('ident')
+ self._deduplicate_recipients(mail)
+ yield self.mail_sender.sendmail(mail)
+
+ sent_mail = yield self.move_to_sent(draft_id, mail)
+ defer.returnValue(sent_mail)
+
+ def _deduplicate_recipients(self, mail):
+ self._remove_canonical(mail)
+ self._remove_duplicates_form_cc_and_to(mail)
+
+ def _remove_canonical(self, mail):
+ mail.headers['To'] = map(self._remove_canonical_recipient, mail.to)
+ mail.headers['Cc'] = map(self._remove_canonical_recipient, mail.cc)
+ mail.headers['Bcc'] = map(self._remove_canonical_recipient, mail.bcc)
+
+ def _remove_duplicates_form_cc_and_to(self, mail):
+ mail.headers['To'] = list(set(self._remove_duplicates(mail.to)).difference(set(mail.bcc)))
+ mail.headers['Cc'] = list((set(self._remove_duplicates(mail.cc)).difference(set(mail.bcc)).difference(set(mail.to))))
+ mail.headers['Bcc'] = self._remove_duplicates(mail.bcc)
+
+ def _remove_duplicates(self, recipient):
+ return list(set(recipient))
+
+ # TODO removing canocical should, be added back later
+ def _remove_canonical_recipient(self, recipient):
+ return recipient.split('<')[1][0:-1] if '<' in recipient else recipient
+
+ @defer.inlineCallbacks
+ def move_to_sent(self, last_draft_ident, mail):
+ if last_draft_ident:
+ try:
+ yield self.mail_store.delete_mail(last_draft_ident)
+ except Exception as error:
+ pass
+ sent_mail = yield self.mail_store.add_mail('SENT', mail.raw)
+ sent_mail.flags.add(Status.SEEN)
+ yield self.mail_store.update_mail(sent_mail)
+ defer.returnValue(sent_mail)
+
+ @defer.inlineCallbacks
+ def mark_as_read(self, mail_id):
+ mail = yield self.mail(mail_id)
+ mail.flags.add(Status.SEEN)
+ yield self.mail_store.update_mail(mail)
+
+ @defer.inlineCallbacks
+ def mark_as_unread(self, mail_id):
+ mail = yield self.mail(mail_id)
+ mail.flags.remove(Status.SEEN)
+ yield self.mail_store.update_mail(mail)
+
+ @defer.inlineCallbacks
+ def delete_mail(self, mail_id):
+ mail = yield self.mail(mail_id)
+ if mail is not None:
+ if mail.mailbox_name.upper() in (u'TRASH', u'DRAFTS'):
+ yield self.mail_store.delete_mail(mail_id)
+ else:
+ yield self.mail_store.move_mail_to_mailbox(mail_id, 'TRASH')
+
+ @defer.inlineCallbacks
+ def recover_mail(self, mail_id):
+ yield self.mail_store.move_mail_to_mailbox(mail_id, 'INBOX')
+
+ @defer.inlineCallbacks
+ def archive_mail(self, mail_id):
+ yield self.mail_store.add_mailbox('ARCHIVE')
+ yield self.mail_store.move_mail_to_mailbox(mail_id, 'ARCHIVE')
+
+ @defer.inlineCallbacks
+ def delete_permanent(self, mail_id):
+ yield self.mail_store.delete_mail(mail_id)
diff --git a/service/src/pixelated/adapter/services/tag_service.py b/service/src/pixelated/adapter/services/tag_service.py
new file mode 100644
index 00000000..c51da625
--- /dev/null
+++ b/service/src/pixelated/adapter/services/tag_service.py
@@ -0,0 +1,23 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from pixelated.adapter.model.tag import Tag
+
+SPECIAL_TAGS = {Tag('inbox', True), Tag('sent', True), Tag('drafts', True), Tag('trash', True), Tag('ALL', True)}
+
+
+def extract_reserved_tags(tags):
+ tags = [tag.lower() for tag in tags]
+ return {tag.name for tag in SPECIAL_TAGS if tag.name in tags}
diff --git a/service/src/pixelated/adapter/welcome_mail.py b/service/src/pixelated/adapter/welcome_mail.py
new file mode 100644
index 00000000..8d3cdd7a
--- /dev/null
+++ b/service/src/pixelated/adapter/welcome_mail.py
@@ -0,0 +1,30 @@
+#
+# Copyright (c) 2015 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+import pkg_resources
+from email import message_from_file
+from pixelated.adapter.model.mail import InputMail
+
+
+def add_welcome_mail(mail_store, language='en-US'):
+ welcome_mail = pkg_resources.resource_filename(
+ 'pixelated.assets',
+ 'welcome.mail.%s' % (language))
+
+ with open(welcome_mail) as mail_template_file:
+ mail_template = message_from_file(mail_template_file)
+
+ input_mail = InputMail.from_python_mail(mail_template)
+ mail_store.add_mail('INBOX', input_mail.raw)