diff options
Diffstat (limited to 'service/pixelated')
22 files changed, 280 insertions, 175 deletions
diff --git a/service/pixelated/adapter/model/mail.py b/service/pixelated/adapter/model/mail.py index 96f2c81c..f23c2708 100644 --- a/service/pixelated/adapter/model/mail.py +++ b/service/pixelated/adapter/model/mail.py @@ -16,6 +16,7 @@ import json from uuid import uuid4 from email.mime.text import MIMEText +from email.header import decode_header from leap.mail.imap.fields import fields import leap.mail.walk as walk @@ -26,6 +27,10 @@ from email.MIMEMultipart import MIMEMultipart from pycryptopp.hash import sha256 import re from pixelated.support.functional import compact +import logging + + +logger = logging.getLogger(__name__) class Mail(object): @@ -82,7 +87,7 @@ class Mail(object): def _parse_charset_header(self, charset_header, default_charset='utf-8'): try: - return re.compile('.*charset=(.*);').match(charset_header).group(1) + return re.compile('.*charset=([a-zA-Z0-9-]+)', re.MULTILINE | re.DOTALL).match(charset_header).group(1) except: return default_charset @@ -232,14 +237,22 @@ class PixelatedMail(Mail): encoding = part['headers'].get('Content-Transfer-Encoding', '') content_type = self._parse_charset_header(part['headers'].get('Content-Type')) - decoding_map = { - 'quoted-printable': lambda content, content_type: unicode(content.decode('quopri'), content_type), - 'base64': lambda content, content_type: content.decode('base64').decode('utf-8') - } - if encoding: - return decoding_map[encoding](part['content'], content_type) - else: - return part['content'] + try: + decoding_map = { + 'quoted-printable': lambda content, content_type: unicode(content.decode('quopri'), content_type), + 'base64': lambda content, content_type: content.decode('base64').decode('utf-8'), + '7bit': lambda content, content_type: content.encode(content_type), + '8bit': lambda content, content_type: content.encode(content_type) + } + if encoding: + return decoding_map[encoding](part['content'], content_type) + else: + return part['content'] + except Exception: + logger.error('Failed to decode mail part with:') + logger.error('Content-Transfer-Encoding: %s' % encoding) + logger.error('Content-Type: %s' % part['headers'].get('Content-Type')) + raise @property def alternatives(self): @@ -269,14 +282,14 @@ class PixelatedMail(Mail): hdoc_headers = self.hdoc.content['headers'] for header in ['To', 'Cc', 'Bcc']: - header_value = hdoc_headers.get(header) + header_value = self._decode_header(hdoc_headers.get(header)) if not header_value: continue _headers[header] = header_value if type(header_value) is list else header_value.split(',') - _headers[header] = map(lambda x: x.strip(), compact(_headers[header])) + _headers[header] = [head.strip() for head in compact(_headers[header])] for header in ['From', 'Subject']: - _headers[header] = hdoc_headers.get(header) + _headers[header] = self._decode_header(hdoc_headers.get(header)) _headers['Date'] = self._get_date() @@ -290,18 +303,34 @@ class PixelatedMail(Mail): return _headers + def _decode_header(self, header): + if not header: + return None + if isinstance(header, list): + return [decode_header(entry)[0][0] for entry in header] + else: + return decode_header(header)[0][0] + def _get_date(self): date = self.hdoc.content.get('date', None) if not date: - date = self.hdoc.content['received'].split(";")[-1].strip() + received = self.hdoc.content.get('received', None) + if received: + date = received.split(";")[-1].strip() + else: + # we can't get a date for this mail, so lets just use now + logger.warning('Encountered a mail with missing date and received header fields. Subject %s' % self.hdoc.content.get('subject', None)) + date = pixelated.support.date.iso_now() return dateparser.parse(date).isoformat() @property def security_casing(self): casing = {"imprints": [], "locks": []} casing["imprints"] = self.signature_information - if self.encrypted: + if self.encrypted == "true": casing["locks"] = [{"state": "valid"}] + elif self.encrypted == "fail": + casing["locks"] = [{"state": "failure"}] return casing @property @@ -377,8 +406,7 @@ class PixelatedMail(Mail): @property def encrypted(self): - return self.hdoc.content["headers"].get("OpenPGP", None) is not None or \ - self.hdoc.content["headers"].get("X-Pixelated-encryption-status", "false") == "true" + return self.hdoc.content["headers"].get("X-Pixelated-encryption-status", "false") @property def bounced(self): diff --git a/service/pixelated/adapter/search/__init__.py b/service/pixelated/adapter/search/__init__.py index 0b1a1034..91eff4c3 100644 --- a/service/pixelated/adapter/search/__init__.py +++ b/service/pixelated/adapter/search/__init__.py @@ -17,10 +17,11 @@ from pixelated.support.encrypted_file_storage import EncryptedFileStorage import os +import re from pixelated.adapter.model.status import Status from pixelated.adapter.search.contacts import contacts_suggestions from whoosh.index import FileIndex -from whoosh.fields import * +from whoosh.fields import Schema, ID, KEYWORD, TEXT, NUMERIC from whoosh.qparser import QueryParser from whoosh.qparser import MultifieldParser from whoosh import sorting @@ -116,8 +117,9 @@ class SearchEngine(object): return FileIndex.create(storage, self._mail_schema(), indexname='mails') def index_mail(self, mail): - with self._index.writer() as writer: - self._index_mail(writer, mail) + with self._write_lock: + with self._index.writer() as writer: + self._index_mail(writer, mail) def _index_mail(self, writer, mail): mdict = mail.as_dict() @@ -125,23 +127,30 @@ class SearchEngine(object): tags = mdict.get('tags', []) tags.append(mail.mailbox_name.lower()) bounced = mail.bounced if mail.bounced else [''] + index_data = { - 'sender': unicode(header.get('from', '')), - 'subject': unicode(header.get('subject', '')), + 'sender': self._unicode_header_field(header.get('from', '')), + 'subject': self._unicode_header_field(header.get('subject', '')), 'date': milliseconds(header.get('date', '')), - 'to': u','.join(header.get('to', [''])), - 'cc': u','.join(header.get('cc', [''])), - 'bcc': u','.join(header.get('bcc', [''])), + 'to': u','.join([h.decode('utf-8') for h in header.get('to', [''])]), + 'cc': u','.join([h.decode('utf-8') for h in header.get('cc', [''])]), + 'bcc': u','.join([h.decode('utf-8') for h in header.get('bcc', [''])]), 'tag': u','.join(unique(tags)), 'bounced': u','.join(bounced), 'body': unicode(mdict['textPlainBody']), 'ident': unicode(mdict['ident']), 'flags': unicode(','.join(unique(mail.flags))), - 'raw': unicode(mail.raw) + 'raw': unicode(mail.raw.decode('utf-8')) } writer.update_document(**index_data) + def _unicode_header_field(self, field_value): + if not field_value: + return None + + return unicode(field_value.decode('utf-8')) + def index_mails(self, mails, callback=None): try: with self._write_lock: diff --git a/service/pixelated/adapter/services/mail_sender.py b/service/pixelated/adapter/services/mail_sender.py index 9f42fbbc..bbcc1721 100644 --- a/service/pixelated/adapter/services/mail_sender.py +++ b/service/pixelated/adapter/services/mail_sender.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. from StringIO import StringIO -import re +from email.utils import parseaddr from twisted.internet.defer import Deferred, fail from twisted.mail.smtp import SMTPSenderFactory @@ -28,35 +28,22 @@ class SMTPDownException(Exception): class MailSender(object): + def __init__(self, account_email_address, ensure_smtp_is_running_cb): self.ensure_smtp_is_running_cb = ensure_smtp_is_running_cb self.account_email_address = account_email_address - def recepients_normalizer(self, mail_list): - return set(mail_list) - - def get_email_addresses(self, mail_list): - clean_mail_list = [] - for mail_address in mail_list: - if "<" in mail_address: - match = re.search(r'<(.*)>', mail_address) - clean_mail_list.append(match.group(1)) - else: - clean_mail_list.append(mail_address) - return self.recepients_normalizer(clean_mail_list) - def sendmail(self, mail): if self.ensure_smtp_is_running_cb(): recipients = flatten([mail.to, mail.cc, mail.bcc]) - normalized_recipients = self.get_email_addresses(recipients) - resultDeferred = Deferred() - senderFactory = SMTPSenderFactory( + result_deferred = Deferred() + sender_factory = SMTPSenderFactory( fromEmail=self.account_email_address, - toEmail=normalized_recipients, + toEmail=set([parseaddr(recipient)[1] for recipient in recipients]), file=StringIO(mail.to_smtp_format()), - deferred=resultDeferred) + deferred=result_deferred) - reactor.connectTCP('localhost', 4650, senderFactory) + reactor.connectTCP('localhost', 4650, sender_factory) - return resultDeferred + return result_deferred return fail(SMTPDownException()) diff --git a/service/pixelated/adapter/services/mail_service.py b/service/pixelated/adapter/services/mail_service.py index 5ef0a188..03889f82 100644 --- a/service/pixelated/adapter/services/mail_service.py +++ b/service/pixelated/adapter/services/mail_service.py @@ -14,13 +14,12 @@ # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. from pixelated.adapter.model.mail import InputMail +from pixelated.adapter.services.tag_service import extract_reserved_tags -class MailService: - __slots__ = ['leap_session', 'account', 'mailbox_name'] +class MailService(object): - def __init__(self, mailboxes, mail_sender, tag_service, soledad_querier, search_engine): - self.tag_service = tag_service + def __init__(self, mailboxes, mail_sender, soledad_querier, search_engine): self.mailboxes = mailboxes self.querier = soledad_querier self.search_engine = search_engine @@ -36,7 +35,7 @@ class MailService: def update_tags(self, mail_id, new_tags): new_tags = self._filter_white_space_tags(new_tags) - reserved_words = self.tag_service.extract_reserved(new_tags) + reserved_words = extract_reserved_tags(new_tags) if len(reserved_words): raise ValueError('None of the following words can be used as tags: ' + ' '.join(reserved_words)) new_tags = self._favor_existing_tags_casing(new_tags) @@ -47,16 +46,16 @@ class MailService: return mail def _filter_white_space_tags(self, tags): - return filter(bool, map(lambda e: e.strip(), tags)) + return [tag.strip() for tag in tags if not tag.isspace()] def _favor_existing_tags_casing(self, new_tags): - current_tags = map(lambda tag: tag['name'], self.search_engine.tags(query='', skip_default_tags=True)) - current_tags_lower = map(lambda tag: tag.lower(), current_tags) + current_tags = [tag['name'] for tag in self.search_engine.tags(query='', skip_default_tags=True)] + current_tags_lower = [tag.lower() for tag in current_tags] def _use_current_casing(new_tag_lower): return current_tags[current_tags_lower.index(new_tag_lower)] - return map(lambda new_tag: _use_current_casing(new_tag.lower()) if new_tag.lower() in current_tags_lower else new_tag, new_tags) + return [_use_current_casing(new_tag.lower()) if new_tag.lower() in current_tags_lower else new_tag for new_tag in new_tags] def mail(self, mail_id): return self.querier.mail(mail_id) diff --git a/service/pixelated/adapter/services/mailbox.py b/service/pixelated/adapter/services/mailbox.py index f934abcc..a4029d78 100644 --- a/service/pixelated/adapter/services/mailbox.py +++ b/service/pixelated/adapter/services/mailbox.py @@ -15,7 +15,7 @@ # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. -class Mailbox: +class Mailbox(object): def __init__(self, mailbox_name, querier, search_engine): self.mailbox_name = mailbox_name @@ -23,6 +23,10 @@ class Mailbox: self.search_engine = search_engine self.querier = querier + @property + def fresh(self): + return self.querier.get_lastuid(self.mailbox_name) == 0 + def mail(self, mail_id): return self.querier.mail(mail_id) diff --git a/service/pixelated/adapter/services/mailboxes.py b/service/pixelated/adapter/services/mailboxes.py index c761255c..a7a3a591 100644 --- a/service/pixelated/adapter/services/mailboxes.py +++ b/service/pixelated/adapter/services/mailboxes.py @@ -17,7 +17,7 @@ from pixelated.adapter.services.mailbox import Mailbox from pixelated.adapter.listeners.mailbox_indexer_listener import MailboxIndexerListener -class Mailboxes(): +class Mailboxes(object): def __init__(self, account, soledad_querier, search_engine): self.account = account diff --git a/service/pixelated/adapter/services/tag_service.py b/service/pixelated/adapter/services/tag_service.py index 601392bb..c51da625 100644 --- a/service/pixelated/adapter/services/tag_service.py +++ b/service/pixelated/adapter/services/tag_service.py @@ -15,13 +15,9 @@ # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. from pixelated.adapter.model.tag import Tag +SPECIAL_TAGS = {Tag('inbox', True), Tag('sent', True), Tag('drafts', True), Tag('trash', True), Tag('ALL', True)} -class TagService: - instance = None - SPECIAL_TAGS = {Tag('inbox', True), Tag('sent', True), Tag('drafts', True), Tag('trash', True), Tag('ALL', True)} - - @classmethod - def extract_reserved(cls, tags): - tags = map(lambda tag: tag.lower(), tags) - return {tag.name for tag in cls.SPECIAL_TAGS if tag.name in tags} +def extract_reserved_tags(tags): + tags = [tag.lower() for tag in tags] + return {tag.name for tag in SPECIAL_TAGS if tag.name in tags} diff --git a/service/pixelated/adapter/soledad/soledad_facade_mixin.py b/service/pixelated/adapter/soledad/soledad_facade_mixin.py index 1df038ea..280fc81e 100644 --- a/service/pixelated/adapter/soledad/soledad_facade_mixin.py +++ b/service/pixelated/adapter/soledad/soledad_facade_mixin.py @@ -21,25 +21,25 @@ class SoledadDbFacadeMixin(object): return self.soledad.get_from_index('by-type', 'flags') def get_all_flags_by_mbox(self, mbox): - return self.soledad.get_from_index('by-type-and-mbox', 'flags', mbox) + return self.soledad.get_from_index('by-type-and-mbox', 'flags', mbox) if mbox else [] def get_content_by_phash(self, phash): - content = self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', phash) + content = self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', phash) if phash else [] if len(content): return content[0] def get_flags_by_chash(self, chash): - flags = self.soledad.get_from_index('by-type-and-contenthash', 'flags', chash) + flags = self.soledad.get_from_index('by-type-and-contenthash', 'flags', chash) if chash else [] if len(flags): return flags[0] def get_header_by_chash(self, chash): - header = self.soledad.get_from_index('by-type-and-contenthash', 'head', chash) + header = self.soledad.get_from_index('by-type-and-contenthash', 'head', chash) if chash else [] if len(header): return header[0] def get_recent_by_mbox(self, mbox): - return self.soledad.get_from_index('by-type-and-mbox', 'rct', mbox) + return self.soledad.get_from_index('by-type-and-mbox', 'rct', mbox) if mbox else [] def put_doc(self, doc): return self.soledad.put_doc(doc) @@ -51,13 +51,16 @@ class SoledadDbFacadeMixin(object): return self.soledad.delete_doc(doc) def idents_by_mailbox(self, mbox): - return set(doc.content['chash'] for doc in self.soledad.get_from_index('by-type-and-mbox-and-deleted', 'flags', mbox, '0')) + return set(doc.content['chash'] for doc in self.soledad.get_from_index('by-type-and-mbox-and-deleted', 'flags', mbox, '0')) if mbox else set() def get_all_mbox(self): return self.soledad.get_from_index('by-type', 'mbox') def get_mbox(self, mbox): - return self.soledad.get_from_index('by-type-and-mbox', 'mbox', mbox) + return self.soledad.get_from_index('by-type-and-mbox', 'mbox', mbox) if mbox else [] + + def get_lastuid(self, mbox_doc): + return mbox_doc.content['lastuid'] def get_search_index_masterkey(self): return self.soledad.get_from_index('by-type', 'index_key') diff --git a/service/pixelated/adapter/soledad/soledad_writer_mixin.py b/service/pixelated/adapter/soledad/soledad_writer_mixin.py index 869f7c07..9c5eb47a 100644 --- a/service/pixelated/adapter/soledad/soledad_writer_mixin.py +++ b/service/pixelated/adapter/soledad/soledad_writer_mixin.py @@ -13,7 +13,6 @@ # # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. -from pixelated.adapter.model.mail import PixelatedMail from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin @@ -32,13 +31,13 @@ class SoledadWriterMixin(SoledadDbFacadeMixin, object): self.put_doc(mail.fdoc) def create_mail(self, mail, mailbox_name): - mbox = self.get_mbox(mailbox_name)[0] - uid = mbox.content['lastuid'] + 1 + mbox_doc = self.get_mbox(mailbox_name)[0] + uid = self.get_lastuid(mbox_doc) [self.create_doc(doc) for doc in mail.get_for_save(next_uid=uid, mailbox=mailbox_name)] - mbox.content['lastuid'] = uid - self.put_doc(mbox) + mbox_doc.content['lastuid'] = uid + 1 + self.put_doc(mbox_doc) return self.mail(mail.ident) diff --git a/service/pixelated/bitmask_libraries/certs.py b/service/pixelated/bitmask_libraries/certs.py index ed597ca8..4ee28a19 100644 --- a/service/pixelated/bitmask_libraries/certs.py +++ b/service/pixelated/bitmask_libraries/certs.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. import os +import requests +import json from leap.common import ca_bundle @@ -46,7 +48,17 @@ class LeapCertificate(object): def _local_server_cert(self): cert_file = os.path.join(self._certs_home, '%s.ca.crt' % self._server_name) - if os.path.isfile(cert_file): - return cert_file - else: - return None + if not os.path.isfile(cert_file): + self._download_server_cert(cert_file) + + return cert_file + + def _download_server_cert(self, cert_file_name): + response = requests.get('https://%s/provider.json' % self._server_name) + provider_data = json.loads(response.content) + ca_cert_uri = str(provider_data['ca_cert_uri']) + + response = requests.get(ca_cert_uri) + with open(cert_file_name, 'w') as file: + file.write(response.content) + file.close diff --git a/service/pixelated/bitmask_libraries/session.py b/service/pixelated/bitmask_libraries/session.py index 9f21fbe6..b23d964f 100644 --- a/service/pixelated/bitmask_libraries/session.py +++ b/service/pixelated/bitmask_libraries/session.py @@ -14,7 +14,6 @@ # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. import errno -import logging import traceback import sys diff --git a/service/pixelated/bitmask_libraries/smtp.py b/service/pixelated/bitmask_libraries/smtp.py index d5236e8e..d4f68f94 100644 --- a/service/pixelated/bitmask_libraries/smtp.py +++ b/service/pixelated/bitmask_libraries/smtp.py @@ -55,7 +55,6 @@ class LeapSmtp(object): if not os.path.exists(os.path.dirname(cert_path)): os.makedirs(os.path.dirname(cert_path)) - session = requests.session() cert_url = '%s/%s/cert' % (self._provider.api_uri, self._provider.api_version) cookies = {"_session_id": self._srp_session.session_id} @@ -94,7 +93,7 @@ class LeapSmtp(object): if not self._smtp_service: try: self.start() - except Exception as e: + except: logger.warning("Couldn't start the SMTP server now, will try again when the user tries to use it") return False return True diff --git a/service/pixelated/bitmask_libraries/soledad.py b/service/pixelated/bitmask_libraries/soledad.py index e6607bde..1c46f2ab 100644 --- a/service/pixelated/bitmask_libraries/soledad.py +++ b/service/pixelated/bitmask_libraries/soledad.py @@ -18,7 +18,7 @@ import errno import os from leap.keymanager import KeyManager from leap.soledad.client import Soledad -from leap.soledad.common.crypto import WrongMac, UnknownMacMethod, MacMethods +from leap.soledad.common.crypto import WrongMac, UnknownMacMethod from .certs import which_bundle @@ -69,7 +69,7 @@ class SoledadSession(object): return Soledad(self.leap_srp_session.uuid, unicode(encryption_passphrase), secrets, local_db, server_url, which_bundle(self.provider), self.leap_srp_session.token, defer_encryption=False) - except (WrongMac, UnknownMacMethod, MacMethods), e: + except (WrongMac, UnknownMacMethod), e: raise SoledadWrongPassphraseException(e) def _leap_path(self): diff --git a/service/pixelated/certificates/try.pixelated-project.org.ca.crt b/service/pixelated/certificates/try.pixelated-project.org.ca.crt deleted file mode 100644 index 52f20468..00000000 --- a/service/pixelated/certificates/try.pixelated-project.org.ca.crt +++ /dev/null @@ -1,34 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFnzCCA4egAwIBAgIBATANBgkqhkiG9w0BAQ0FADBiMRUwEwYDVQQKDAxMRUFQ -X0V4YW1wbGUxKjAoBgNVBAsMIWh0dHBzOi8vdHJ5LnBpeGVsYXRlZC1wcm9qZWN0 -Lm9yZzEdMBsGA1UEAwwUTEVBUF9FeGFtcGxlIFJvb3QgQ0EwHhcNMTQxMjA4MDAw -MDAwWhcNMjQxMjA4MDAwMDAwWjBiMRUwEwYDVQQKDAxMRUFQX0V4YW1wbGUxKjAo -BgNVBAsMIWh0dHBzOi8vdHJ5LnBpeGVsYXRlZC1wcm9qZWN0Lm9yZzEdMBsGA1UE -AwwUTEVBUF9FeGFtcGxlIFJvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw -ggIKAoICAQDFkKhtf99Ybl+iLPE/G5XNCuco4rOeF4Br1ezPLSOj34jEwgrVapHj -xliFTwRE0Hkbohlh2CHDm8+slzgo7v2BN3XUmWy3D6XqfJwAS7UT4SbkLvZ/XFuH -hUiGxvBk5OSu6oi/qT0mmJaNk4CjbSzxQ2VOoLFpguhgQ5SKMHb3nYpmOg+gxWUz -tACLqV/33DeJb1bhrqKkfo1WIJ0mAJMq+re1vFYe0J/TdOmTxpXULhAlreg1QDuY -K1Tm7IWaflxLEVsUG3c8JItR2ksKPs54DxpJdILvauO2oAHdHH+FPnmtiF0fDaJQ -lNa4GbAOhAe063+vrTgfX+9BREhCU8Pn28ZIcOKyv+qqdgBSZtNyb21Lj+eCRlTA -8/TIbb29bMDhNJKxarayrfLwe4tx9In0EAhoXYBPvrf11OGuY+wcf2xxXHNMMHwb -NLr909JYBbHI10VvNUgviEir0h0DYvCAH+nUYi0cateUJG6qCE+cndiVF5VQ4/7y -7UuGo8rT2nYz06JU7QlKyfTkphc1PkXdCjdhKF4jfsVt84TQGlRr5jzBhz47PQqj -eZ87IQCkhzYD+XnaD2gT++B9i9wezn8uJo4Kjf+CRU51aKHw5U9DNTdXaAmyfwJ6 -WV1udBbY96gBqyPpGoaSadsG1WlJKYliksh983Bus4negYAIet+NKQIDAQABo2Aw -XjAdBgNVHQ4EFgQU2mxnw5M2EwE6ibGfBeSF6hwcasYwDgYDVR0PAQH/BAQDAgIE -MAwGA1UdEwQFMAMBAf8wHwYDVR0jBBgwFoAU2mxnw5M2EwE6ibGfBeSF6hwcasYw -DQYJKoZIhvcNAQENBQADggIBAGvJMfLvi1tiYDUIQpNE9G9haX2aIwVK0GEMcymW -g+/59RjhrlprE6yygzzNQ0H4C5L8EYEslEmQ/YJxl2/MzSMhR4AoYocoW5fJkdQk -P88UQ8fb7DArs6OEnC5AMxG1izFL4CfgqHffcL5DaEPi/i/Xf1M5ZyEZF/Azqn1C -CVH9VggPpN+ivKy8BBrgabaZpSYs9iVwsRxH+Gorv9mIaxhO2iIwZKk5v9PWISbi -ukGFT8XkLH9EmWZbZj5IYJkYP0R5q4ljDjd9CUQ9vWoxhgmhV1X3PW/gp+1cZ3ci -PAgcab6kyfsMjJm0NWnTUmbIcpKapWRP8naxNKboNgNL+GaKv0YgfgmB2gkIjsBM -yvQ3fdSgs/903JsZmGXEcVzqWDK+5rfZrR7PBwCUyiJmZSR//Xcppts6b/Uh9BSx -8dXDYTySliP8ncY3O5R/SGv6CHtrVwkjEZwXeghCqYaZjVPHCl3JDdK98SyFCbT5 -iPF9+mfL0lHUzr8tZCkITel0/ci5L7o7PRSFl6z4Vii/tqF+MpM/jdTPCvw91CjQ -WTFI7iuy+nvjxxB20mUyOl4rP9wfwq+5HprqyaeVZHrp9AvaknOc/nd/UEiYHl5d -hvg2snkn8lort2HmB1MylrsI9Q4dbtzdugm/dM7+n0Se8HF9lljpIUnocqCnYFtt -+G0O ------END CERTIFICATE----- - diff --git a/service/pixelated/config/__init__.py b/service/pixelated/config/__init__.py index f9c43153..2045354e 100644 --- a/service/pixelated/config/__init__.py +++ b/service/pixelated/config/__init__.py @@ -25,7 +25,7 @@ from pixelated.config.dispatcher import config_dispatcher from pixelated.config.events_server import init_events_server from pixelated.config.loading_page import loading from pixelated.config.register import register -from pixelated.config.debug import init_debugger +from pixelated.config.logging_setup import init_logging from pixelated.config.leap_cert import init_leap_cert from pixelated.config.soledad import init_soledad_and_user_key from twisted.internet import reactor @@ -36,13 +36,14 @@ import pixelated.support.ext_protobuf import pixelated.support.ext_sqlcipher import pixelated.support.ext_esmtp_sender_factory import pixelated.support.ext_fetch +import pixelated.support.ext_keymanager_fetch_key def initialize(): args = parse_args() app = App() - init_debugger(args) + init_logging(args) init_leap_cert(args) if args.register: diff --git a/service/pixelated/config/app_factory.py b/service/pixelated/config/app_factory.py index f63b49ed..f20b1229 100644 --- a/service/pixelated/config/app_factory.py +++ b/service/pixelated/config/app_factory.py @@ -33,7 +33,6 @@ from pixelated.adapter.listeners.mailbox_indexer_listener import MailboxIndexerL import pixelated.bitmask_libraries.session as LeapSession from pixelated.bitmask_libraries.leap_srp import LeapAuthException from requests.exceptions import ConnectionError -from pixelated.adapter.services.tag_service import TagService from leap.common.events import ( register, unregister, @@ -100,14 +99,13 @@ def init_app(app, leap_home, leap_session): soledad_querier = SoledadQuerier(soledad=leap_session.account._soledad) - tag_service = TagService() search_engine = SearchEngine(soledad_querier, agent_home=leap_home) pixelated_mail_sender = MailSender(leap_session.account_email(), lambda: leap_session.smtp.ensure_running()) pixelated_mailboxes = Mailboxes(leap_session.account, soledad_querier, search_engine) draft_service = DraftService(pixelated_mailboxes) - mail_service = MailService(pixelated_mailboxes, pixelated_mail_sender, tag_service, soledad_querier, search_engine) + mail_service = MailService(pixelated_mailboxes, pixelated_mail_sender, soledad_querier, search_engine) MailboxIndexerListener.SEARCH_ENGINE = search_engine InputMail.FROM_EMAIL_ADDRESS = leap_session.account_email() diff --git a/service/pixelated/config/debug.py b/service/pixelated/config/debug.py deleted file mode 100644 index d91d3a34..00000000 --- a/service/pixelated/config/debug.py +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. - -import logging -import sys -import os -from twisted.python import log - - -def init_debugger(args): - debug_enabled = args.debug or os.environ.get('DEBUG', False) - log.startLogging(sys.stdout) - - if debug_enabled: - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', - datefmt='%m-%d %H:%M', - filename='/tmp/leap.log', - filemode='w') # define a Handler which writes INFO messages or higher to the sys.stderr - - console = logging.StreamHandler() - console.setLevel(logging.DEBUG) - formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') - console.setFormatter(formatter) - logging.getLogger('').addHandler(console) - - return debug_enabled diff --git a/service/pixelated/config/logging_setup.py b/service/pixelated/config/logging_setup.py new file mode 100644 index 00000000..a15413a0 --- /dev/null +++ b/service/pixelated/config/logging_setup.py @@ -0,0 +1,65 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + +import logging +import socket +import sys +import os +from twisted.python import log +from twisted.python import util + + +def init_logging(args): + debug_enabled = args.debug or os.environ.get('DEBUG', False) + + logging.basicConfig(level=logging.DEBUG if debug_enabled else logging.WARNING, + format='[%(asctime)s] ' + socket.gethostname() + ' %(name)-12s %(levelname)-8s %(message)s', + datefmt='%m-%d %H:%M:%S', + filemode='a') + + if debug_enabled: + init_debugger() + + log.startLogging(sys.stdout) + + +def init_debugger(): + formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') + console = logging.StreamHandler() + console.setLevel(logging.DEBUG) + console.setFormatter(formatter) + logging.getLogger('').addHandler(console) + + +class PixelatedLogObserver(log.FileLogObserver): + + """ FileLogObserver with a customized format """ + def emit(self, event): + text = log.textFromEventDict(event) + + if text is None: + return + + self.timeFormat = '[%Y-%m-%d %H:%M:%S]' + time_str = self.formatTime(event['time']) + + fmt_dict = {'text': text.replace('\n', '\n\t')} + msg_str = log._safeFormat('%(text)s\n', fmt_dict) + + logging.debug(str(event)) + + util.untilConcludes(self.write, time_str + ' ' + socket.gethostname() + ' ' + msg_str) + util.untilConcludes(self.flush) diff --git a/service/pixelated/resources/sync_info_resource.py b/service/pixelated/resources/sync_info_resource.py index 5aa94218..791c5add 100644 --- a/service/pixelated/resources/sync_info_resource.py +++ b/service/pixelated/resources/sync_info_resource.py @@ -32,7 +32,7 @@ class SyncInfoResource(Resource): return self.current / float(self.total) def set_sync_info(self, soledad_sync_status): - self.current, self.total = map(int, soledad_sync_status.content.split('/')) + self.current, self.total = [int(x) for x in soledad_sync_status.content.split('/')] def render_GET(self, request): _sync_info = { diff --git a/service/pixelated/support/ext_fetch.py b/service/pixelated/support/ext_fetch.py index ab0def9f..2db5dd1d 100644 --- a/service/pixelated/support/ext_fetch.py +++ b/service/pixelated/support/ext_fetch.py @@ -1,14 +1,35 @@ import leap.mail.imap.fetch as fetch -def mark_as_encrypted(f): +def mark_as_encrypted_inline(f): def w(*args, **kwargs): - msg, was_decrypted = f(*args) - msg.add_header('X-Pixelated-encryption-status', 'true' if was_decrypted else 'false') - return msg, was_decrypted + msg, valid_sign = f(*args) + is_encrypted = fetch.PGP_BEGIN in args[1].as_string() and fetch.PGP_END in args[1].as_string() + decrypted_successfully = fetch.PGP_BEGIN not in msg.as_string() and fetch.PGP_END not in msg.as_string() + + if not is_encrypted: + encrypted = 'false' + else: + if decrypted_successfully: + encrypted = 'true' + else: + encrypted = 'fail' + + msg.add_header('X-Pixelated-encryption-status', encrypted) + return msg, valid_sign + + return w + + +def mark_as_encrypted_multipart(f): + + def w(*args, **kwargs): + msg, valid_sign = f(*args) + msg.add_header('X-Pixelated-encryption-status', 'true') + return msg, valid_sign return w -fetch.LeapIncomingMail._maybe_decrypt_inline_encrypted_msg = mark_as_encrypted(fetch.LeapIncomingMail._maybe_decrypt_inline_encrypted_msg) -fetch.LeapIncomingMail._decrypt_multipart_encrypted_msg = mark_as_encrypted(fetch.LeapIncomingMail._decrypt_multipart_encrypted_msg) +fetch.LeapIncomingMail._maybe_decrypt_inline_encrypted_msg = mark_as_encrypted_inline(fetch.LeapIncomingMail._maybe_decrypt_inline_encrypted_msg) +fetch.LeapIncomingMail._decrypt_multipart_encrypted_msg = mark_as_encrypted_multipart(fetch.LeapIncomingMail._decrypt_multipart_encrypted_msg) diff --git a/service/pixelated/support/ext_keymanager_fetch_key.py b/service/pixelated/support/ext_keymanager_fetch_key.py new file mode 100644 index 00000000..d39d1f96 --- /dev/null +++ b/service/pixelated/support/ext_keymanager_fetch_key.py @@ -0,0 +1,60 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import leap.keymanager +import requests +import logging +from leap.keymanager.errors import KeyNotFound +from leap.keymanager.openpgp import OpenPGPKey + + +logger = logging.getLogger(__name__) + + +def patched_fetch_keys_from_server(self, address): + """ + Fetch keys bound to C{address} from nickserver and insert them in + local database. + + Instead of raising a KeyNotFound only for 404 responses, this implementation + raises a KeyNotFound exception for all problems. + + For original see: https://github.com/leapcode/keymanager/blob/develop/src/leap/keymanager/__init__.py + + :param address: The address bound to the keys. + :type address: str + + :raise KeyNotFound: If the key was not found on nickserver. + """ + # request keys from the nickserver + res = None + try: + res = self._get(self._nickserver_uri, {'address': address}) + res.raise_for_status() + server_keys = res.json() + # insert keys in local database + if self.OPENPGP_KEY in server_keys: + self._wrapper_map[OpenPGPKey].put_ascii_key( + server_keys['openpgp']) + except requests.exceptions.HTTPError as e: + logger.warning("HTTP error retrieving key: %r" % (e,)) + logger.warning("%s" % (res.content,)) + raise KeyNotFound(address) + except Exception as e: + logger.warning("Error retrieving key: %r" % (e,)) + raise KeyNotFound(address) + + +leap.keymanager.KeyManager._fetch_keys_from_server = patched_fetch_keys_from_server diff --git a/service/pixelated/support/ext_protobuf.py b/service/pixelated/support/ext_protobuf.py index 06d7bcea..548f5fd6 100644 --- a/service/pixelated/support/ext_protobuf.py +++ b/service/pixelated/support/ext_protobuf.py @@ -28,9 +28,8 @@ if _platform == 'darwin': try: func(*args, **kwargs) pass - except Exception as e: - if e.strerror == 'Socket is not connected': - pass + except: + pass return wrapper |