summaryrefslogtreecommitdiff
path: root/service/pixelated/adapter
diff options
context:
space:
mode:
Diffstat (limited to 'service/pixelated/adapter')
-rw-r--r--service/pixelated/adapter/model/mail.py60
-rw-r--r--service/pixelated/adapter/search/__init__.py27
-rw-r--r--service/pixelated/adapter/services/mail_sender.py29
-rw-r--r--service/pixelated/adapter/services/mail_service.py17
-rw-r--r--service/pixelated/adapter/services/mailbox.py6
-rw-r--r--service/pixelated/adapter/services/mailboxes.py2
-rw-r--r--service/pixelated/adapter/services/tag_service.py12
-rw-r--r--service/pixelated/adapter/soledad/soledad_facade_mixin.py17
-rw-r--r--service/pixelated/adapter/soledad/soledad_writer_mixin.py9
9 files changed, 102 insertions, 77 deletions
diff --git a/service/pixelated/adapter/model/mail.py b/service/pixelated/adapter/model/mail.py
index 96f2c81c..f23c2708 100644
--- a/service/pixelated/adapter/model/mail.py
+++ b/service/pixelated/adapter/model/mail.py
@@ -16,6 +16,7 @@
import json
from uuid import uuid4
from email.mime.text import MIMEText
+from email.header import decode_header
from leap.mail.imap.fields import fields
import leap.mail.walk as walk
@@ -26,6 +27,10 @@ from email.MIMEMultipart import MIMEMultipart
from pycryptopp.hash import sha256
import re
from pixelated.support.functional import compact
+import logging
+
+
+logger = logging.getLogger(__name__)
class Mail(object):
@@ -82,7 +87,7 @@ class Mail(object):
def _parse_charset_header(self, charset_header, default_charset='utf-8'):
try:
- return re.compile('.*charset=(.*);').match(charset_header).group(1)
+ return re.compile('.*charset=([a-zA-Z0-9-]+)', re.MULTILINE | re.DOTALL).match(charset_header).group(1)
except:
return default_charset
@@ -232,14 +237,22 @@ class PixelatedMail(Mail):
encoding = part['headers'].get('Content-Transfer-Encoding', '')
content_type = self._parse_charset_header(part['headers'].get('Content-Type'))
- decoding_map = {
- 'quoted-printable': lambda content, content_type: unicode(content.decode('quopri'), content_type),
- 'base64': lambda content, content_type: content.decode('base64').decode('utf-8')
- }
- if encoding:
- return decoding_map[encoding](part['content'], content_type)
- else:
- return part['content']
+ try:
+ decoding_map = {
+ 'quoted-printable': lambda content, content_type: unicode(content.decode('quopri'), content_type),
+ 'base64': lambda content, content_type: content.decode('base64').decode('utf-8'),
+ '7bit': lambda content, content_type: content.encode(content_type),
+ '8bit': lambda content, content_type: content.encode(content_type)
+ }
+ if encoding:
+ return decoding_map[encoding](part['content'], content_type)
+ else:
+ return part['content']
+ except Exception:
+ logger.error('Failed to decode mail part with:')
+ logger.error('Content-Transfer-Encoding: %s' % encoding)
+ logger.error('Content-Type: %s' % part['headers'].get('Content-Type'))
+ raise
@property
def alternatives(self):
@@ -269,14 +282,14 @@ class PixelatedMail(Mail):
hdoc_headers = self.hdoc.content['headers']
for header in ['To', 'Cc', 'Bcc']:
- header_value = hdoc_headers.get(header)
+ header_value = self._decode_header(hdoc_headers.get(header))
if not header_value:
continue
_headers[header] = header_value if type(header_value) is list else header_value.split(',')
- _headers[header] = map(lambda x: x.strip(), compact(_headers[header]))
+ _headers[header] = [head.strip() for head in compact(_headers[header])]
for header in ['From', 'Subject']:
- _headers[header] = hdoc_headers.get(header)
+ _headers[header] = self._decode_header(hdoc_headers.get(header))
_headers['Date'] = self._get_date()
@@ -290,18 +303,34 @@ class PixelatedMail(Mail):
return _headers
+ def _decode_header(self, header):
+ if not header:
+ return None
+ if isinstance(header, list):
+ return [decode_header(entry)[0][0] for entry in header]
+ else:
+ return decode_header(header)[0][0]
+
def _get_date(self):
date = self.hdoc.content.get('date', None)
if not date:
- date = self.hdoc.content['received'].split(";")[-1].strip()
+ received = self.hdoc.content.get('received', None)
+ if received:
+ date = received.split(";")[-1].strip()
+ else:
+ # we can't get a date for this mail, so lets just use now
+ logger.warning('Encountered a mail with missing date and received header fields. Subject %s' % self.hdoc.content.get('subject', None))
+ date = pixelated.support.date.iso_now()
return dateparser.parse(date).isoformat()
@property
def security_casing(self):
casing = {"imprints": [], "locks": []}
casing["imprints"] = self.signature_information
- if self.encrypted:
+ if self.encrypted == "true":
casing["locks"] = [{"state": "valid"}]
+ elif self.encrypted == "fail":
+ casing["locks"] = [{"state": "failure"}]
return casing
@property
@@ -377,8 +406,7 @@ class PixelatedMail(Mail):
@property
def encrypted(self):
- return self.hdoc.content["headers"].get("OpenPGP", None) is not None or \
- self.hdoc.content["headers"].get("X-Pixelated-encryption-status", "false") == "true"
+ return self.hdoc.content["headers"].get("X-Pixelated-encryption-status", "false")
@property
def bounced(self):
diff --git a/service/pixelated/adapter/search/__init__.py b/service/pixelated/adapter/search/__init__.py
index 0b1a1034..91eff4c3 100644
--- a/service/pixelated/adapter/search/__init__.py
+++ b/service/pixelated/adapter/search/__init__.py
@@ -17,10 +17,11 @@
from pixelated.support.encrypted_file_storage import EncryptedFileStorage
import os
+import re
from pixelated.adapter.model.status import Status
from pixelated.adapter.search.contacts import contacts_suggestions
from whoosh.index import FileIndex
-from whoosh.fields import *
+from whoosh.fields import Schema, ID, KEYWORD, TEXT, NUMERIC
from whoosh.qparser import QueryParser
from whoosh.qparser import MultifieldParser
from whoosh import sorting
@@ -116,8 +117,9 @@ class SearchEngine(object):
return FileIndex.create(storage, self._mail_schema(), indexname='mails')
def index_mail(self, mail):
- with self._index.writer() as writer:
- self._index_mail(writer, mail)
+ with self._write_lock:
+ with self._index.writer() as writer:
+ self._index_mail(writer, mail)
def _index_mail(self, writer, mail):
mdict = mail.as_dict()
@@ -125,23 +127,30 @@ class SearchEngine(object):
tags = mdict.get('tags', [])
tags.append(mail.mailbox_name.lower())
bounced = mail.bounced if mail.bounced else ['']
+
index_data = {
- 'sender': unicode(header.get('from', '')),
- 'subject': unicode(header.get('subject', '')),
+ 'sender': self._unicode_header_field(header.get('from', '')),
+ 'subject': self._unicode_header_field(header.get('subject', '')),
'date': milliseconds(header.get('date', '')),
- 'to': u','.join(header.get('to', [''])),
- 'cc': u','.join(header.get('cc', [''])),
- 'bcc': u','.join(header.get('bcc', [''])),
+ 'to': u','.join([h.decode('utf-8') for h in header.get('to', [''])]),
+ 'cc': u','.join([h.decode('utf-8') for h in header.get('cc', [''])]),
+ 'bcc': u','.join([h.decode('utf-8') for h in header.get('bcc', [''])]),
'tag': u','.join(unique(tags)),
'bounced': u','.join(bounced),
'body': unicode(mdict['textPlainBody']),
'ident': unicode(mdict['ident']),
'flags': unicode(','.join(unique(mail.flags))),
- 'raw': unicode(mail.raw)
+ 'raw': unicode(mail.raw.decode('utf-8'))
}
writer.update_document(**index_data)
+ def _unicode_header_field(self, field_value):
+ if not field_value:
+ return None
+
+ return unicode(field_value.decode('utf-8'))
+
def index_mails(self, mails, callback=None):
try:
with self._write_lock:
diff --git a/service/pixelated/adapter/services/mail_sender.py b/service/pixelated/adapter/services/mail_sender.py
index 9f42fbbc..bbcc1721 100644
--- a/service/pixelated/adapter/services/mail_sender.py
+++ b/service/pixelated/adapter/services/mail_sender.py
@@ -14,7 +14,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
from StringIO import StringIO
-import re
+from email.utils import parseaddr
from twisted.internet.defer import Deferred, fail
from twisted.mail.smtp import SMTPSenderFactory
@@ -28,35 +28,22 @@ class SMTPDownException(Exception):
class MailSender(object):
+
def __init__(self, account_email_address, ensure_smtp_is_running_cb):
self.ensure_smtp_is_running_cb = ensure_smtp_is_running_cb
self.account_email_address = account_email_address
- def recepients_normalizer(self, mail_list):
- return set(mail_list)
-
- def get_email_addresses(self, mail_list):
- clean_mail_list = []
- for mail_address in mail_list:
- if "<" in mail_address:
- match = re.search(r'<(.*)>', mail_address)
- clean_mail_list.append(match.group(1))
- else:
- clean_mail_list.append(mail_address)
- return self.recepients_normalizer(clean_mail_list)
-
def sendmail(self, mail):
if self.ensure_smtp_is_running_cb():
recipients = flatten([mail.to, mail.cc, mail.bcc])
- normalized_recipients = self.get_email_addresses(recipients)
- resultDeferred = Deferred()
- senderFactory = SMTPSenderFactory(
+ result_deferred = Deferred()
+ sender_factory = SMTPSenderFactory(
fromEmail=self.account_email_address,
- toEmail=normalized_recipients,
+ toEmail=set([parseaddr(recipient)[1] for recipient in recipients]),
file=StringIO(mail.to_smtp_format()),
- deferred=resultDeferred)
+ deferred=result_deferred)
- reactor.connectTCP('localhost', 4650, senderFactory)
+ reactor.connectTCP('localhost', 4650, sender_factory)
- return resultDeferred
+ return result_deferred
return fail(SMTPDownException())
diff --git a/service/pixelated/adapter/services/mail_service.py b/service/pixelated/adapter/services/mail_service.py
index 5ef0a188..03889f82 100644
--- a/service/pixelated/adapter/services/mail_service.py
+++ b/service/pixelated/adapter/services/mail_service.py
@@ -14,13 +14,12 @@
# You should have received a copy of the GNU Affero General Public License
# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
from pixelated.adapter.model.mail import InputMail
+from pixelated.adapter.services.tag_service import extract_reserved_tags
-class MailService:
- __slots__ = ['leap_session', 'account', 'mailbox_name']
+class MailService(object):
- def __init__(self, mailboxes, mail_sender, tag_service, soledad_querier, search_engine):
- self.tag_service = tag_service
+ def __init__(self, mailboxes, mail_sender, soledad_querier, search_engine):
self.mailboxes = mailboxes
self.querier = soledad_querier
self.search_engine = search_engine
@@ -36,7 +35,7 @@ class MailService:
def update_tags(self, mail_id, new_tags):
new_tags = self._filter_white_space_tags(new_tags)
- reserved_words = self.tag_service.extract_reserved(new_tags)
+ reserved_words = extract_reserved_tags(new_tags)
if len(reserved_words):
raise ValueError('None of the following words can be used as tags: ' + ' '.join(reserved_words))
new_tags = self._favor_existing_tags_casing(new_tags)
@@ -47,16 +46,16 @@ class MailService:
return mail
def _filter_white_space_tags(self, tags):
- return filter(bool, map(lambda e: e.strip(), tags))
+ return [tag.strip() for tag in tags if not tag.isspace()]
def _favor_existing_tags_casing(self, new_tags):
- current_tags = map(lambda tag: tag['name'], self.search_engine.tags(query='', skip_default_tags=True))
- current_tags_lower = map(lambda tag: tag.lower(), current_tags)
+ current_tags = [tag['name'] for tag in self.search_engine.tags(query='', skip_default_tags=True)]
+ current_tags_lower = [tag.lower() for tag in current_tags]
def _use_current_casing(new_tag_lower):
return current_tags[current_tags_lower.index(new_tag_lower)]
- return map(lambda new_tag: _use_current_casing(new_tag.lower()) if new_tag.lower() in current_tags_lower else new_tag, new_tags)
+ return [_use_current_casing(new_tag.lower()) if new_tag.lower() in current_tags_lower else new_tag for new_tag in new_tags]
def mail(self, mail_id):
return self.querier.mail(mail_id)
diff --git a/service/pixelated/adapter/services/mailbox.py b/service/pixelated/adapter/services/mailbox.py
index f934abcc..a4029d78 100644
--- a/service/pixelated/adapter/services/mailbox.py
+++ b/service/pixelated/adapter/services/mailbox.py
@@ -15,7 +15,7 @@
# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
-class Mailbox:
+class Mailbox(object):
def __init__(self, mailbox_name, querier, search_engine):
self.mailbox_name = mailbox_name
@@ -23,6 +23,10 @@ class Mailbox:
self.search_engine = search_engine
self.querier = querier
+ @property
+ def fresh(self):
+ return self.querier.get_lastuid(self.mailbox_name) == 0
+
def mail(self, mail_id):
return self.querier.mail(mail_id)
diff --git a/service/pixelated/adapter/services/mailboxes.py b/service/pixelated/adapter/services/mailboxes.py
index c761255c..a7a3a591 100644
--- a/service/pixelated/adapter/services/mailboxes.py
+++ b/service/pixelated/adapter/services/mailboxes.py
@@ -17,7 +17,7 @@ from pixelated.adapter.services.mailbox import Mailbox
from pixelated.adapter.listeners.mailbox_indexer_listener import MailboxIndexerListener
-class Mailboxes():
+class Mailboxes(object):
def __init__(self, account, soledad_querier, search_engine):
self.account = account
diff --git a/service/pixelated/adapter/services/tag_service.py b/service/pixelated/adapter/services/tag_service.py
index 601392bb..c51da625 100644
--- a/service/pixelated/adapter/services/tag_service.py
+++ b/service/pixelated/adapter/services/tag_service.py
@@ -15,13 +15,9 @@
# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
from pixelated.adapter.model.tag import Tag
+SPECIAL_TAGS = {Tag('inbox', True), Tag('sent', True), Tag('drafts', True), Tag('trash', True), Tag('ALL', True)}
-class TagService:
- instance = None
- SPECIAL_TAGS = {Tag('inbox', True), Tag('sent', True), Tag('drafts', True), Tag('trash', True), Tag('ALL', True)}
-
- @classmethod
- def extract_reserved(cls, tags):
- tags = map(lambda tag: tag.lower(), tags)
- return {tag.name for tag in cls.SPECIAL_TAGS if tag.name in tags}
+def extract_reserved_tags(tags):
+ tags = [tag.lower() for tag in tags]
+ return {tag.name for tag in SPECIAL_TAGS if tag.name in tags}
diff --git a/service/pixelated/adapter/soledad/soledad_facade_mixin.py b/service/pixelated/adapter/soledad/soledad_facade_mixin.py
index 1df038ea..280fc81e 100644
--- a/service/pixelated/adapter/soledad/soledad_facade_mixin.py
+++ b/service/pixelated/adapter/soledad/soledad_facade_mixin.py
@@ -21,25 +21,25 @@ class SoledadDbFacadeMixin(object):
return self.soledad.get_from_index('by-type', 'flags')
def get_all_flags_by_mbox(self, mbox):
- return self.soledad.get_from_index('by-type-and-mbox', 'flags', mbox)
+ return self.soledad.get_from_index('by-type-and-mbox', 'flags', mbox) if mbox else []
def get_content_by_phash(self, phash):
- content = self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', phash)
+ content = self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', phash) if phash else []
if len(content):
return content[0]
def get_flags_by_chash(self, chash):
- flags = self.soledad.get_from_index('by-type-and-contenthash', 'flags', chash)
+ flags = self.soledad.get_from_index('by-type-and-contenthash', 'flags', chash) if chash else []
if len(flags):
return flags[0]
def get_header_by_chash(self, chash):
- header = self.soledad.get_from_index('by-type-and-contenthash', 'head', chash)
+ header = self.soledad.get_from_index('by-type-and-contenthash', 'head', chash) if chash else []
if len(header):
return header[0]
def get_recent_by_mbox(self, mbox):
- return self.soledad.get_from_index('by-type-and-mbox', 'rct', mbox)
+ return self.soledad.get_from_index('by-type-and-mbox', 'rct', mbox) if mbox else []
def put_doc(self, doc):
return self.soledad.put_doc(doc)
@@ -51,13 +51,16 @@ class SoledadDbFacadeMixin(object):
return self.soledad.delete_doc(doc)
def idents_by_mailbox(self, mbox):
- return set(doc.content['chash'] for doc in self.soledad.get_from_index('by-type-and-mbox-and-deleted', 'flags', mbox, '0'))
+ return set(doc.content['chash'] for doc in self.soledad.get_from_index('by-type-and-mbox-and-deleted', 'flags', mbox, '0')) if mbox else set()
def get_all_mbox(self):
return self.soledad.get_from_index('by-type', 'mbox')
def get_mbox(self, mbox):
- return self.soledad.get_from_index('by-type-and-mbox', 'mbox', mbox)
+ return self.soledad.get_from_index('by-type-and-mbox', 'mbox', mbox) if mbox else []
+
+ def get_lastuid(self, mbox_doc):
+ return mbox_doc.content['lastuid']
def get_search_index_masterkey(self):
return self.soledad.get_from_index('by-type', 'index_key')
diff --git a/service/pixelated/adapter/soledad/soledad_writer_mixin.py b/service/pixelated/adapter/soledad/soledad_writer_mixin.py
index 869f7c07..9c5eb47a 100644
--- a/service/pixelated/adapter/soledad/soledad_writer_mixin.py
+++ b/service/pixelated/adapter/soledad/soledad_writer_mixin.py
@@ -13,7 +13,6 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
-from pixelated.adapter.model.mail import PixelatedMail
from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin
@@ -32,13 +31,13 @@ class SoledadWriterMixin(SoledadDbFacadeMixin, object):
self.put_doc(mail.fdoc)
def create_mail(self, mail, mailbox_name):
- mbox = self.get_mbox(mailbox_name)[0]
- uid = mbox.content['lastuid'] + 1
+ mbox_doc = self.get_mbox(mailbox_name)[0]
+ uid = self.get_lastuid(mbox_doc)
[self.create_doc(doc) for doc in mail.get_for_save(next_uid=uid, mailbox=mailbox_name)]
- mbox.content['lastuid'] = uid
- self.put_doc(mbox)
+ mbox_doc.content['lastuid'] = uid + 1
+ self.put_doc(mbox_doc)
return self.mail(mail.ident)
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-11 05:49:51 +0000