From 62b0cd6301b7097dfa2776b677ab3c7d27f60d7b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 26 Dec 2013 14:10:14 -0400 Subject: Split the near-2k loc file into more handy modules. ...aaaand not a single fuck was given that day! --- src/leap/mail/imap/parser.py | 93 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 src/leap/mail/imap/parser.py (limited to 'src/leap/mail/imap/parser.py') diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py new file mode 100644 index 0000000..1ae19c0 --- /dev/null +++ b/src/leap/mail/imap/parser.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# parser.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Mail parser mixins. +""" +import cStringIO +import StringIO +import re + +from email.parser import Parser + + +class MailParser(object): + """ + Mixin with utility methods to parse raw messages. + """ + def __init__(self): + """ + Initializes the mail parser. + """ + self._parser = Parser() + + def _get_parsed_msg(self, raw): + """ + Return a parsed Message. + + :param raw: the raw string to parse + :type raw: basestring, or StringIO object + """ + msg = self._get_parser_fun(raw)(raw, True) + return msg + + def _get_parser_fun(self, o): + """ + Retunn the proper parser function for an object. + + :param o: object + :type o: object + :param parser: an instance of email.parser.Parser + :type parser: email.parser.Parser + """ + if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): + return self._parser.parse + if isinstance(o, basestring): + return self._parser.parsestr + # fallback + return self._parser.parsestr + + def _stringify(self, o): + """ + Return a string object. + + :param o: object + :type o: object + """ + if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): + return o.getvalue() + else: + return o + + +class MBoxParser(object): + """ + Utility function to parse mailbox names. + """ + INBOX_NAME = "INBOX" + INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + + def _parse_mailbox_name(self, name): + """ + :param name: the name of the mailbox + :type name: unicode + + :rtype: unicode + """ + if self.INBOX_RE.match(name): + # ensure inital INBOX is uppercase + return self.INBOX_NAME + name[len(self.INBOX_NAME):] + return name -- cgit v1.2.3 From 25a0aea875fd0d67238beed1237f7239474673ec Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 27 Dec 2013 02:06:44 -0400 Subject: First stage of the storage schema rewrite. * Separates between flags, docs, body and attachment docs. * Implement IMessageCopier interface: move and have fun! This little change is known to push forward our beloved architect emotional rollercoster. * Message deduplication. * It also fixes a hidden bug that was rendering the multipart mime interface useless (yes, the "True" parameter in the parsestr method). * Does not handle well nested attachs, includes dirty workaround that flattens them. * Includes chiiph's patch for rc2: * return deferred from addMessage * convert StringIO types to string * remove unneeded yields from the chain of deferreds in fetcher --- src/leap/mail/imap/parser.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) (limited to 'src/leap/mail/imap/parser.py') diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py index 1ae19c0..306dcf0 100644 --- a/src/leap/mail/imap/parser.py +++ b/src/leap/mail/imap/parser.py @@ -19,10 +19,14 @@ Mail parser mixins. """ import cStringIO import StringIO +import hashlib import re +from email.message import Message from email.parser import Parser +from leap.common.check import leap_assert_type + class MailParser(object): """ @@ -34,16 +38,30 @@ class MailParser(object): """ self._parser = Parser() - def _get_parsed_msg(self, raw): + def _get_parsed_msg(self, raw, headersonly=False): """ Return a parsed Message. :param raw: the raw string to parse :type raw: basestring, or StringIO object + + :param headersonly: True for parsing only the headers. + :type headersonly: bool """ - msg = self._get_parser_fun(raw)(raw, True) + msg = self._get_parser_fun(raw)(raw, headersonly=headersonly) return msg + def _get_hash(self, msg): + """ + Returns a hash of the string representation of the raw message, + suitable for indexing the inmutable pieces. + + :param msg: a Message object + :type msg: Message + """ + leap_assert_type(msg, Message) + return hashlib.sha256(msg.as_string()).hexdigest() + def _get_parser_fun(self, o): """ Retunn the proper parser function for an object. @@ -67,6 +85,8 @@ class MailParser(object): :param o: object :type o: object """ + # XXX Maybe we don't need no more, we're using + # msg.as_string() if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): return o.getvalue() else: -- cgit v1.2.3 From 4381131163161c00f00a3eb300041374aa06d370 Mon Sep 17 00:00:00 2001 From: drebs Date: Sat, 28 Dec 2013 20:09:03 -0200 Subject: Convert unicode to str when raising in IMAP server (#4830). --- src/leap/mail/imap/parser.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/leap/mail/imap/parser.py') diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py index 306dcf0..6a9ace9 100644 --- a/src/leap/mail/imap/parser.py +++ b/src/leap/mail/imap/parser.py @@ -102,6 +102,11 @@ class MBoxParser(object): def _parse_mailbox_name(self, name): """ + Return a normalized representation of the mailbox C{name}. + + This method ensures that an eventual initial 'inbox' part of a + mailbox name is made uppercase. + :param name: the name of the mailbox :type name: unicode -- cgit v1.2.3 From b2d97c9faef6037a065e2903afe5b0ab2624917e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 20 Feb 2014 02:52:17 -0400 Subject: mail parsing performance improvements Although the do_parse function is deferred to threads, we were actually waiting till its return to fire the callback of the deferred, and hence the "append ok" was being delayed. During massive appends, this was a tight loop contributing as much as 35 msec, of a total of 100 msec average. Several ineficiencies are addressed here: * use pycryptopp hash functions. * avoiding function calling overhead. * avoid duplicate call to message.as_string * make use of the string size caching capabilities. * avoiding the mail Parser initialization/method call completely, in favor of the module helper to get the object from string. Overall, these changes cut parsing to 50% of the initial timing by my measurements with line_profiler, YMMV. --- src/leap/mail/imap/parser.py | 75 +------------------------------------------- 1 file changed, 1 insertion(+), 74 deletions(-) (limited to 'src/leap/mail/imap/parser.py') diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py index 6a9ace9..4a801b0 100644 --- a/src/leap/mail/imap/parser.py +++ b/src/leap/mail/imap/parser.py @@ -15,83 +15,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -Mail parser mixins. +Mail parser mixin. """ -import cStringIO -import StringIO -import hashlib import re -from email.message import Message -from email.parser import Parser - -from leap.common.check import leap_assert_type - - -class MailParser(object): - """ - Mixin with utility methods to parse raw messages. - """ - def __init__(self): - """ - Initializes the mail parser. - """ - self._parser = Parser() - - def _get_parsed_msg(self, raw, headersonly=False): - """ - Return a parsed Message. - - :param raw: the raw string to parse - :type raw: basestring, or StringIO object - - :param headersonly: True for parsing only the headers. - :type headersonly: bool - """ - msg = self._get_parser_fun(raw)(raw, headersonly=headersonly) - return msg - - def _get_hash(self, msg): - """ - Returns a hash of the string representation of the raw message, - suitable for indexing the inmutable pieces. - - :param msg: a Message object - :type msg: Message - """ - leap_assert_type(msg, Message) - return hashlib.sha256(msg.as_string()).hexdigest() - - def _get_parser_fun(self, o): - """ - Retunn the proper parser function for an object. - - :param o: object - :type o: object - :param parser: an instance of email.parser.Parser - :type parser: email.parser.Parser - """ - if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): - return self._parser.parse - if isinstance(o, basestring): - return self._parser.parsestr - # fallback - return self._parser.parsestr - - def _stringify(self, o): - """ - Return a string object. - - :param o: object - :type o: object - """ - # XXX Maybe we don't need no more, we're using - # msg.as_string() - if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): - return o.getvalue() - else: - return o - class MBoxParser(object): """ -- cgit v1.2.3