From 4ba5d5b405e3c6a6bc997df2073ffc8ea3fa75a9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 7 Jan 2014 11:34:08 -0400 Subject: Second stage of the new year's storage rewrite. * documents of only three types: * flags * headers * content * add algorithm for walking the parsed message tree. * treat special cases like a multipart with a single part. * modify add_msg to use the walk routine * modify twisted interfaces to use the new storage schema. * tests for different multipart cases * fix multipart detection typo in the fetch This is a merge proposal for the 0.5.0-rc3. known bugs ---------- Some things are still know not to work well at this point (some cases of multipart messages do not display the bodies). IMAP server also is left in a bad internal state after a logout/login. --- src/leap/mail/imap/tests/walktree.py | 117 +++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 src/leap/mail/imap/tests/walktree.py (limited to 'src/leap/mail/imap/tests/walktree.py') diff --git a/src/leap/mail/imap/tests/walktree.py b/src/leap/mail/imap/tests/walktree.py new file mode 100644 index 0000000..1626f65 --- /dev/null +++ b/src/leap/mail/imap/tests/walktree.py @@ -0,0 +1,117 @@ +#t -*- coding: utf-8 -*- +# walktree.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the walktree module. +""" +import os +from email import parser + +from leap.mail import walk as W + +DEBUG = os.environ.get("BITMASK_MAIL_DEBUG") + +p = parser.Parser() + +# TODO pass an argument of the type of message + +################################################## +# Input from hell + +#msg = p.parse(open('rfc822.multi-signed.message')) +#msg = p.parse(open('rfc822.plain.message')) +msg = p.parse(open('rfc822.multi-minimal.message')) +DO_CHECK = False +################################################# + +parts = W.get_parts(msg) + +if DEBUG: + def trim(item): + item = item[:10] + [trim(part["phash"]) for part in parts if part.get('phash', None)] + +raw_docs = list(W.get_raw_docs(msg, parts)) + +body_phash_fun = [W.get_body_phash_simple, + W.get_body_phash_multi][int(msg.is_multipart())] +body_phash = body_phash_fun(W.get_payloads(msg)) +parts_map = W.walk_msg_tree(parts, body_phash=body_phash) + + +# TODO add missing headers! +expected = { + 'body': '1ddfa80485', + 'multi': True, + 'part_map': { + 1: { + 'headers': {'Content-Disposition': 'inline', + 'Content-Type': 'multipart/mixed; ' + 'boundary="z0eOaCaDLjvTGF2l"'}, + 'multi': True, + 'part_map': {1: {'ctype': 'text/plain', + 'headers': [ + ('Content-Type', + 'text/plain; charset=utf-8'), + ('Content-Disposition', + 'inline'), + ('Content-Transfer-Encoding', + 'quoted-printable')], + 'multi': False, + 'parts': 1, + 'phash': '1ddfa80485', + 'size': 206}, + 2: {'ctype': 'text/plain', + 'headers': [('Content-Type', + 'text/plain; charset=us-ascii'), + ('Content-Disposition', + 'attachment; ' + 'filename="attach.txt"')], + 'multi': False, + 'parts': 1, + 'phash': '7a94e4d769', + 'size': 133}, + 3: {'ctype': 'application/octet-stream', + 'headers': [('Content-Type', + 'application/octet-stream'), + ('Content-Disposition', + 'attachment; filename="hack.ico"'), + ('Content-Transfer-Encoding', + 'base64')], + 'multi': False, + 'parts': 1, + 'phash': 'c42cccebbd', + 'size': 12736}}}, + 2: {'ctype': 'application/pgp-signature', + 'headers': [('Content-Type', 'application/pgp-signature')], + 'multi': False, + 'parts': 1, + 'phash': '8f49fbf749', + 'size': 877}}} + +if DEBUG and DO_CHECK: + # TODO turn this into a proper unittest + assert(parts_map == expected) + print "Structure: OK" + + +import pprint +print +print "RAW DOCS" +pprint.pprint(raw_docs) +print +print "PARTS MAP" +pprint.pprint(parts_map) -- cgit v1.2.3 From 37090301e633cdf9aa0c924ea3899d4c5d4fcdb9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 29 Jan 2014 16:18:27 -0400 Subject: allow to pass file as argument --- src/leap/mail/imap/tests/walktree.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/imap/tests/walktree.py') diff --git a/src/leap/mail/imap/tests/walktree.py b/src/leap/mail/imap/tests/walktree.py index 1626f65..f3cbcb0 100644 --- a/src/leap/mail/imap/tests/walktree.py +++ b/src/leap/mail/imap/tests/walktree.py @@ -18,12 +18,14 @@ Tests for the walktree module. """ import os +import sys from email import parser from leap.mail import walk as W DEBUG = os.environ.get("BITMASK_MAIL_DEBUG") + p = parser.Parser() # TODO pass an argument of the type of message @@ -31,9 +33,17 @@ p = parser.Parser() ################################################## # Input from hell -#msg = p.parse(open('rfc822.multi-signed.message')) -#msg = p.parse(open('rfc822.plain.message')) -msg = p.parse(open('rfc822.multi-minimal.message')) +if len(sys.argv) > 1: + FILENAME = sys.argv[1] +else: + FILENAME = "rfc822.multi-minimal.message" + +""" +FILENAME = "rfc822.multi-signed.message" +FILENAME = "rfc822.plain.message" +""" + +msg = p.parse(open(FILENAME)) DO_CHECK = False ################################################# -- cgit v1.2.3 From b2d97c9faef6037a065e2903afe5b0ab2624917e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 20 Feb 2014 02:52:17 -0400 Subject: mail parsing performance improvements Although the do_parse function is deferred to threads, we were actually waiting till its return to fire the callback of the deferred, and hence the "append ok" was being delayed. During massive appends, this was a tight loop contributing as much as 35 msec, of a total of 100 msec average. Several ineficiencies are addressed here: * use pycryptopp hash functions. * avoiding function calling overhead. * avoid duplicate call to message.as_string * make use of the string size caching capabilities. * avoiding the mail Parser initialization/method call completely, in favor of the module helper to get the object from string. Overall, these changes cut parsing to 50% of the initial timing by my measurements with line_profiler, YMMV. --- src/leap/mail/imap/tests/walktree.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/leap/mail/imap/tests/walktree.py') diff --git a/src/leap/mail/imap/tests/walktree.py b/src/leap/mail/imap/tests/walktree.py index f3cbcb0..695f487 100644 --- a/src/leap/mail/imap/tests/walktree.py +++ b/src/leap/mail/imap/tests/walktree.py @@ -36,11 +36,11 @@ p = parser.Parser() if len(sys.argv) > 1: FILENAME = sys.argv[1] else: - FILENAME = "rfc822.multi-minimal.message" + FILENAME = "rfc822.multi-signed.message" """ -FILENAME = "rfc822.multi-signed.message" FILENAME = "rfc822.plain.message" +FILENAME = "rfc822.multi-minimal.message" """ msg = p.parse(open(FILENAME)) -- cgit v1.2.3