From 4ba5d5b405e3c6a6bc997df2073ffc8ea3fa75a9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 7 Jan 2014 11:34:08 -0400 Subject: Second stage of the new year's storage rewrite. * documents of only three types: * flags * headers * content * add algorithm for walking the parsed message tree. * treat special cases like a multipart with a single part. * modify add_msg to use the walk routine * modify twisted interfaces to use the new storage schema. * tests for different multipart cases * fix multipart detection typo in the fetch This is a merge proposal for the 0.5.0-rc3. known bugs ---------- Some things are still know not to work well at this point (some cases of multipart messages do not display the bodies). IMAP server also is left in a bad internal state after a logout/login. --- src/leap/mail/walk.py | 160 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 src/leap/mail/walk.py (limited to 'src/leap/mail/walk.py') diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py new file mode 100644 index 0000000..820b8c7 --- /dev/null +++ b/src/leap/mail/walk.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +# walk.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Utilities for walking along a message tree. +""" +import hashlib +import os + +from leap.mail.utils import first + +DEBUG = os.environ.get("BITMASK_MAIL_DEBUG") + +if DEBUG: + get_hash = lambda s: hashlib.sha256(s).hexdigest()[:10] +else: + get_hash = lambda s: hashlib.sha256(s).hexdigest() + + +""" +Get interesting message parts +""" +get_parts = lambda msg: [ + {'multi': part.is_multipart(), + 'ctype': part.get_content_type(), + 'size': len(part.as_string()), + 'parts': len(part.get_payload()) + if isinstance(part.get_payload(), list) + else 1, + 'headers': part.items(), + 'phash': get_hash(part.get_payload()) + if not part.is_multipart() else None} + for part in msg.walk()] + +""" +Utility lambda functions for getting the parts vector and the +payloads from the original message. +""" + +get_parts_vector = lambda parts: (x.get('parts', 1) for x in parts) +get_payloads = lambda msg: ((x.get_payload(), + dict(((str.lower(k), v) for k, v in (x.items())))) + for x in msg.walk()) + +get_body_phash_simple = lambda payloads: first( + [get_hash(payload) for payload, headers in payloads + if "text/plain" in headers.get('content-type')]) + +get_body_phash_multi = lambda payloads: (first( + [get_hash(payload) for payload, headers in payloads + if "text/plain" in headers.get('content-type')]) + or get_body_phash_simple(payloads)) + +""" +On getting the raw docs, we get also some of the headers to be able to +index the content. Here we remove any mutable part, as the the filename +in the content disposition. +""" + +get_raw_docs = lambda msg, parts: ( + {"type": "cnt", # type content they'll be + "raw": payload if not DEBUG else payload[:100], + "phash": get_hash(payload), + "content-disposition": first(headers.get( + 'content-disposition', '').split(';')), + "content-type": headers.get( + 'content-type', ''), + "content-transfer-encoding": headers.get( + 'content-transfer-type', '')} + for payload, headers in get_payloads(msg) + if not isinstance(payload, list)) + + +def walk_msg_tree(parts, body_phash=None): + """ + Take a list of interesting items of a message subparts structure, + and return a dict of dicts almost ready to be written to the content + documents that will be stored in Soledad. + + It walks down the subparts in the parsed message tree, and collapses + the leaf docuents into a wrapper document until no multipart submessages + are left. To achieve this, it iteratively calculates a wrapper vector of + all documents in the sequence that have more than one part and have unitary + documents to their right. To collapse a multipart, take as many + unitary documents as parts the submessage contains, and replace the object + in the sequence with the new wrapper document. + + :param parts: A list of dicts containing the interesting properties for + the message structure. Normally this has been generated by + doing a message walk. + :type parts: list of dicts. + :param body_phash: the payload hash of the body part, to be included + in the outer content doc for convenience. + :type body_phash: basestring or None + """ + # parts vector + pv = list(get_parts_vector(parts)) + + if len(parts) == 2: + inner_headers = parts[1].get("headers", None) + + if DEBUG: + print "parts vector: ", pv + print + + # wrappers vector + getwv = lambda pv: [True if pv[i] != 1 and pv[i + 1] == 1 else False + for i in range(len(pv) - 1)] + wv = getwv(pv) + + # do until no wrapper document is left + while any(wv): + wind = wv.index(True) # wrapper index + nsub = pv[wind] # number of subparts to pick + slic = parts[wind + 1:wind + 1 + nsub] # slice with subparts + + cwra = { + "multi": True, + "part_map": dict((index + 1, part) # content wrapper + for index, part in enumerate(slic)), + "headers": dict(parts[wind]['headers']) + } + + # remove subparts and substitue wrapper + map(lambda i: parts.remove(i), slic) + parts[wind] = cwra + + # refresh vectors for this iteration + pv = list(get_parts_vector(parts)) + wv = getwv(pv) + + outer = parts[0] + outer.pop('headers') + if not "part_map" in outer: + # we have a multipart with 1 part only, so kind of fix it + # although it would be prettier if I take this special case at + # the beginning of the walk. + pdoc = {"multi": True, + "part_map": {1: outer}} + pdoc["part_map"][1]["multi"] = False + if not pdoc["part_map"][1].get("phash", None): + pdoc["part_map"][1]["phash"] = body_phash + pdoc["part_map"][1]["headers"] = inner_headers + else: + pdoc = outer + pdoc["body"] = body_phash + return pdoc -- cgit v1.2.3 From bffdcddee55d1045be5d5c8378f712283863b6bf Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 9 Jan 2014 18:11:58 -0400 Subject: check for none --- src/leap/mail/walk.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/leap/mail/walk.py') diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 820b8c7..dc13345 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -57,11 +57,13 @@ get_payloads = lambda msg: ((x.get_payload(), get_body_phash_simple = lambda payloads: first( [get_hash(payload) for payload, headers in payloads - if "text/plain" in headers.get('content-type')]) + if payloads + and "text/plain" in headers.get('content-type')]) get_body_phash_multi = lambda payloads: (first( [get_hash(payload) for payload, headers in payloads - if "text/plain" in headers.get('content-type')]) + if payloads + and "text/plain" in headers.get('content-type')]) or get_body_phash_simple(payloads)) """ -- cgit v1.2.3 From ddb50ed05ae7141c2f9c2aece9e24681e0d5d696 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 9 Jan 2014 20:03:02 -0400 Subject: Check for none in innerheaders This was causing a bug, among other things, when saving to the Sent folder for some messages. Closes #4914 --- src/leap/mail/walk.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/walk.py') diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index dc13345..1871752 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -111,8 +111,8 @@ def walk_msg_tree(parts, body_phash=None): # parts vector pv = list(get_parts_vector(parts)) - if len(parts) == 2: - inner_headers = parts[1].get("headers", None) + inner_headers = parts[1].get("headers", None) if ( + len(parts) == 2) else None if DEBUG: print "parts vector: ", pv @@ -155,7 +155,8 @@ def walk_msg_tree(parts, body_phash=None): pdoc["part_map"][1]["multi"] = False if not pdoc["part_map"][1].get("phash", None): pdoc["part_map"][1]["phash"] = body_phash - pdoc["part_map"][1]["headers"] = inner_headers + if inner_headers: + pdoc["part_map"][1]["headers"] = inner_headers else: pdoc = outer pdoc["body"] = body_phash -- cgit v1.2.3 From cf231b4536652fadfe03169c97688a0c76606dca Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 13 Jan 2014 10:24:51 -0400 Subject: avoid failure if no content-type --- src/leap/mail/walk.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/walk.py') diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 1871752..dd3b745 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -57,13 +57,12 @@ get_payloads = lambda msg: ((x.get_payload(), get_body_phash_simple = lambda payloads: first( [get_hash(payload) for payload, headers in payloads - if payloads - and "text/plain" in headers.get('content-type')]) + if payloads]) get_body_phash_multi = lambda payloads: (first( [get_hash(payload) for payload, headers in payloads if payloads - and "text/plain" in headers.get('content-type')]) + and "text/plain" in headers.get('content-type', '')]) or get_body_phash_simple(payloads)) """ -- cgit v1.2.3 From 460539c51b431b6d16c45ecd8216ab1e0471d106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Wed, 22 Jan 2014 18:42:12 -0300 Subject: Properly parse apple mail --- src/leap/mail/walk.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'src/leap/mail/walk.py') diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index dd3b745..27d672c 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -143,6 +143,15 @@ def walk_msg_tree(parts, body_phash=None): pv = list(get_parts_vector(parts)) wv = getwv(pv) + if all(x == 1 for x in pv): + # special case in the rightmost element + main_pmap = parts[0]['part_map'] + last_part = max(main_pmap.keys()) + main_pmap[last_part]['part_map'] = {} + for partind in range(len(pv) - 1): + print partind+1, len(parts) + main_pmap[last_part]['part_map'][partind] = parts[partind+1] + outer = parts[0] outer.pop('headers') if not "part_map" in outer: -- cgit v1.2.3 From bded9833da985034a11c30b342388c397798a585 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 23 Jan 2014 10:49:30 -0400 Subject: add constants to dict keys --- src/leap/mail/walk.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) (limited to 'src/leap/mail/walk.py') diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 27d672c..856daa3 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -107,10 +107,16 @@ def walk_msg_tree(parts, body_phash=None): in the outer content doc for convenience. :type body_phash: basestring or None """ + PART_MAP = "part_map" + MULTI = "multi" + HEADERS = "headers" + PHASH = "phash" + BODY = "body" + # parts vector pv = list(get_parts_vector(parts)) - inner_headers = parts[1].get("headers", None) if ( + inner_headers = parts[1].get(HEADERS, None) if ( len(parts) == 2) else None if DEBUG: @@ -129,10 +135,10 @@ def walk_msg_tree(parts, body_phash=None): slic = parts[wind + 1:wind + 1 + nsub] # slice with subparts cwra = { - "multi": True, - "part_map": dict((index + 1, part) # content wrapper - for index, part in enumerate(slic)), - "headers": dict(parts[wind]['headers']) + MULTI: True, + PART_MAP: dict((index + 1, part) # content wrapper + for index, part in enumerate(slic)), + HEADERS: dict(parts[wind][HEADERS]) } # remove subparts and substitue wrapper @@ -153,19 +159,19 @@ def walk_msg_tree(parts, body_phash=None): main_pmap[last_part]['part_map'][partind] = parts[partind+1] outer = parts[0] - outer.pop('headers') - if not "part_map" in outer: + outer.pop(HEADERS) + if not PART_MAP in outer: # we have a multipart with 1 part only, so kind of fix it # although it would be prettier if I take this special case at # the beginning of the walk. - pdoc = {"multi": True, - "part_map": {1: outer}} - pdoc["part_map"][1]["multi"] = False - if not pdoc["part_map"][1].get("phash", None): - pdoc["part_map"][1]["phash"] = body_phash + pdoc = {MULTI: True, + PART_MAP: {1: outer}} + pdoc[PART_MAP][1][MULTI] = False + if not pdoc[PART_MAP][1].get(PHASH, None): + pdoc[PART_MAP][1][PHASH] = body_phash if inner_headers: - pdoc["part_map"][1]["headers"] = inner_headers + pdoc[PART_MAP][1][HEADERS] = inner_headers else: pdoc = outer - pdoc["body"] = body_phash + pdoc[BODY] = body_phash return pdoc -- cgit v1.2.3 From 90b870c48c0c27e3a366902c5b76986a5140258c Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 23 Jan 2014 10:49:52 -0400 Subject: add check for none in part_map special case --- src/leap/mail/walk.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'src/leap/mail/walk.py') diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 856daa3..30cb70a 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -151,12 +151,13 @@ def walk_msg_tree(parts, body_phash=None): if all(x == 1 for x in pv): # special case in the rightmost element - main_pmap = parts[0]['part_map'] - last_part = max(main_pmap.keys()) - main_pmap[last_part]['part_map'] = {} - for partind in range(len(pv) - 1): - print partind+1, len(parts) - main_pmap[last_part]['part_map'][partind] = parts[partind+1] + main_pmap = parts[0].get(PART_MAP, None) + if main_pmap is not None: + last_part = max(main_pmap.keys()) + main_pmap[last_part][PART_MAP] = {} + for partind in range(len(pv) - 1): + print partind+1, len(parts) + main_pmap[last_part][PART_MAP][partind] = parts[partind + 1] outer = parts[0] outer.pop(HEADERS) -- cgit v1.2.3 From ff28e22977db802c87f0b7be99e37c6de29183e9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 23 Jan 2014 13:32:01 -0400 Subject: Unset new flag after successful write --- src/leap/mail/walk.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) (limited to 'src/leap/mail/walk.py') diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 30cb70a..49f2c22 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -176,3 +176,36 @@ def walk_msg_tree(parts, body_phash=None): pdoc = outer pdoc[BODY] = body_phash return pdoc + +""" +Groucho Marx: Now pay particular attention to this first clause, because it's + most important. There's the party of the first part shall be + known in this contract as the party of the first part. How do you + like that, that's pretty neat eh? + +Chico Marx: No, that's no good. +Groucho Marx: What's the matter with it? + +Chico Marx: I don't know, let's hear it again. +Groucho Marx: So the party of the first part shall be known in this contract as + the party of the first part. + +Chico Marx: Well it sounds a little better this time. +Groucho Marx: Well, it grows on you. Would you like to hear it once more? + +Chico Marx: Just the first part. +Groucho Marx: All right. It says the first part of the party of the first part + shall be known in this contract as the first part of the party of + the first part, shall be known in this contract - look, why + should we quarrel about a thing like this, we'll take it right + out, eh? + +Chico Marx: Yes, it's too long anyhow. Now what have we got left? +Groucho Marx: Well I've got about a foot and a half. Now what's the matter? + +Chico Marx: I don't like the second party either. +""" + +""" +I feel you deserved it after reading the above and try to debug your problem ;) +""" -- cgit v1.2.3 From b2d97c9faef6037a065e2903afe5b0ab2624917e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 20 Feb 2014 02:52:17 -0400 Subject: mail parsing performance improvements Although the do_parse function is deferred to threads, we were actually waiting till its return to fire the callback of the deferred, and hence the "append ok" was being delayed. During massive appends, this was a tight loop contributing as much as 35 msec, of a total of 100 msec average. Several ineficiencies are addressed here: * use pycryptopp hash functions. * avoiding function calling overhead. * avoid duplicate call to message.as_string * make use of the string size caching capabilities. * avoiding the mail Parser initialization/method call completely, in favor of the module helper to get the object from string. Overall, these changes cut parsing to 50% of the initial timing by my measurements with line_profiler, YMMV. --- src/leap/mail/walk.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/walk.py') diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 49f2c22..f747377 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -17,17 +17,18 @@ """ Utilities for walking along a message tree. """ -import hashlib import os +from pycryptopp.hash import sha256 + from leap.mail.utils import first DEBUG = os.environ.get("BITMASK_MAIL_DEBUG") if DEBUG: - get_hash = lambda s: hashlib.sha256(s).hexdigest()[:10] + get_hash = lambda s: sha256.SHA256(s).hexdigest()[:10] else: - get_hash = lambda s: hashlib.sha256(s).hexdigest() + get_hash = lambda s: sha256.SHA256(s).hexdigest() """ -- cgit v1.2.3