From 62b0cd6301b7097dfa2776b677ab3c7d27f60d7b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 26 Dec 2013 14:10:14 -0400 Subject: Split the near-2k loc file into more handy modules. ...aaaand not a single fuck was given that day! --- src/leap/mail/imap/fields.py | 127 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 src/leap/mail/imap/fields.py (limited to 'src/leap/mail/imap/fields.py') diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py new file mode 100644 index 0000000..96b937e --- /dev/null +++ b/src/leap/mail/imap/fields.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +# fields.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Fields for Mailbox and Message. +""" +from leap.mail.imap.parser import MBoxParser + + +class WithMsgFields(object): + """ + Container class for class-attributes to be shared by + several message-related classes. + """ + # Internal representation of Message + DATE_KEY = "date" + HEADERS_KEY = "headers" + FLAGS_KEY = "flags" + MBOX_KEY = "mbox" + CONTENT_HASH_KEY = "chash" + RAW_KEY = "raw" + SUBJECT_KEY = "subject" + UID_KEY = "uid" + MULTIPART_KEY = "multi" + SIZE_KEY = "size" + + # Mailbox specific keys + CLOSED_KEY = "closed" + CREATED_KEY = "created" + SUBSCRIBED_KEY = "subscribed" + RW_KEY = "rw" + LAST_UID_KEY = "lastuid" + + # Document Type, for indexing + TYPE_KEY = "type" + TYPE_MBOX_VAL = "mbox" + TYPE_MESSAGE_VAL = "msg" + TYPE_FLAGS_VAL = "flags" + TYPE_HEADERS_VAL = "head" + TYPE_ATTACHMENT_VAL = "attach" + # should add also a headers val + + INBOX_VAL = "inbox" + + # Flags for SoledadDocument for indexing. + SEEN_KEY = "seen" + RECENT_KEY = "recent" + + # Flags in Mailbox and Message + SEEN_FLAG = "\\Seen" + RECENT_FLAG = "\\Recent" + ANSWERED_FLAG = "\\Answered" + FLAGGED_FLAG = "\\Flagged" # yo dawg + DELETED_FLAG = "\\Deleted" + DRAFT_FLAG = "\\Draft" + NOSELECT_FLAG = "\\Noselect" + LIST_FLAG = "List" # is this OK? (no \. ie, no system flag) + + # Fields in mail object + SUBJECT_FIELD = "Subject" + DATE_FIELD = "Date" + + # Index types + # -------------- + + TYPE_IDX = 'by-type' + TYPE_MBOX_IDX = 'by-type-and-mbox' + TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' + TYPE_SUBS_IDX = 'by-type-and-subscribed' + TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' + TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' + TYPE_HASH_IDX = 'by-type-and-hash' + + # Tomas created the `recent and seen index`, but the semantic is not too + # correct since the recent flag is volatile. + TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' + + KTYPE = TYPE_KEY + MBOX_VAL = TYPE_MBOX_VAL + HASH_VAL = CONTENT_HASH_KEY + + INDEXES = { + # generic + TYPE_IDX: [KTYPE], + TYPE_MBOX_IDX: [KTYPE, MBOX_VAL], + TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, UID_KEY], + + # mailboxes + TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'], + + # content, headers doc + TYPE_HASH_IDX: [KTYPE, HASH_VAL], + + # messages + TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], + TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'], + TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, + 'bool(recent)', 'bool(seen)'], + } + + MBOX_KEY = MBOX_VAL + + EMPTY_MBOX = { + TYPE_KEY: MBOX_KEY, + TYPE_MBOX_VAL: MBoxParser.INBOX_NAME, + SUBJECT_KEY: "", + FLAGS_KEY: [], + CLOSED_KEY: False, + SUBSCRIBED_KEY: False, + RW_KEY: 1, + LAST_UID_KEY: 0 + } + +fields = WithMsgFields # alias for convenience -- cgit v1.2.3 From 25a0aea875fd0d67238beed1237f7239474673ec Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 27 Dec 2013 02:06:44 -0400 Subject: First stage of the storage schema rewrite. * Separates between flags, docs, body and attachment docs. * Implement IMessageCopier interface: move and have fun! This little change is known to push forward our beloved architect emotional rollercoster. * Message deduplication. * It also fixes a hidden bug that was rendering the multipart mime interface useless (yes, the "True" parameter in the parsestr method). * Does not handle well nested attachs, includes dirty workaround that flattens them. * Includes chiiph's patch for rc2: * return deferred from addMessage * convert StringIO types to string * remove unneeded yields from the chain of deferreds in fetcher --- src/leap/mail/imap/fields.py | 49 +++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 14 deletions(-) (limited to 'src/leap/mail/imap/fields.py') diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py index 96b937e..40817cd 100644 --- a/src/leap/mail/imap/fields.py +++ b/src/leap/mail/imap/fields.py @@ -25,18 +25,35 @@ class WithMsgFields(object): Container class for class-attributes to be shared by several message-related classes. """ - # Internal representation of Message - DATE_KEY = "date" - HEADERS_KEY = "headers" - FLAGS_KEY = "flags" - MBOX_KEY = "mbox" + # indexing CONTENT_HASH_KEY = "chash" - RAW_KEY = "raw" - SUBJECT_KEY = "subject" + PAYLOAD_HASH_KEY = "phash" + + # Internal representation of Message + + # flags doc UID_KEY = "uid" + MBOX_KEY = "mbox" + SEEN_KEY = "seen" + RECENT_KEY = "recent" + FLAGS_KEY = "flags" MULTIPART_KEY = "multi" SIZE_KEY = "size" + # headers + HEADERS_KEY = "headers" + NUM_PARTS_KEY = "numparts" + PARTS_MAP_KEY = "partmap" + DATE_KEY = "date" + SUBJECT_KEY = "subject" + + # attachment + PART_NUMBER_KEY = "part" + RAW_KEY = "raw" + + # content + BODY_KEY = "body" + # Mailbox specific keys CLOSED_KEY = "closed" CREATED_KEY = "created" @@ -55,10 +72,6 @@ class WithMsgFields(object): INBOX_VAL = "inbox" - # Flags for SoledadDocument for indexing. - SEEN_KEY = "seen" - RECENT_KEY = "recent" - # Flags in Mailbox and Message SEEN_FLAG = "\\Seen" RECENT_FLAG = "\\Recent" @@ -82,7 +95,9 @@ class WithMsgFields(object): TYPE_SUBS_IDX = 'by-type-and-subscribed' TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' - TYPE_HASH_IDX = 'by-type-and-hash' + TYPE_C_HASH_IDX = 'by-type-and-contenthash' + TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' + TYPE_P_HASH_IDX = 'by-type-and-payloadhash' # Tomas created the `recent and seen index`, but the semantic is not too # correct since the recent flag is volatile. @@ -90,7 +105,9 @@ class WithMsgFields(object): KTYPE = TYPE_KEY MBOX_VAL = TYPE_MBOX_VAL - HASH_VAL = CONTENT_HASH_KEY + CHASH_VAL = CONTENT_HASH_KEY + PHASH_VAL = PAYLOAD_HASH_KEY + PART_VAL = PART_NUMBER_KEY INDEXES = { # generic @@ -102,7 +119,11 @@ class WithMsgFields(object): TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'], # content, headers doc - TYPE_HASH_IDX: [KTYPE, HASH_VAL], + TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL], + # attachment docs + TYPE_C_HASH_PART_IDX: [KTYPE, CHASH_VAL, PART_VAL], + # attachment payload dedup + TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL], # messages TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], -- cgit v1.2.3 From 5585ff784940dee267576d097076de66797f9188 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 2 Jan 2014 16:08:09 -0400 Subject: fix tests after rewrite --- src/leap/mail/imap/fields.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/leap/mail/imap/fields.py') diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py index 40817cd..bc536fe 100644 --- a/src/leap/mail/imap/fields.py +++ b/src/leap/mail/imap/fields.py @@ -35,6 +35,7 @@ class WithMsgFields(object): UID_KEY = "uid" MBOX_KEY = "mbox" SEEN_KEY = "seen" + DEL_KEY = "deleted" RECENT_KEY = "recent" FLAGS_KEY = "flags" MULTIPART_KEY = "multi" @@ -95,6 +96,7 @@ class WithMsgFields(object): TYPE_SUBS_IDX = 'by-type-and-subscribed' TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' + TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' TYPE_C_HASH_IDX = 'by-type-and-contenthash' TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' TYPE_P_HASH_IDX = 'by-type-and-payloadhash' @@ -128,6 +130,7 @@ class WithMsgFields(object): # messages TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'], + TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'], TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(recent)', 'bool(seen)'], } -- cgit v1.2.3 From 4ba5d5b405e3c6a6bc997df2073ffc8ea3fa75a9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 7 Jan 2014 11:34:08 -0400 Subject: Second stage of the new year's storage rewrite. * documents of only three types: * flags * headers * content * add algorithm for walking the parsed message tree. * treat special cases like a multipart with a single part. * modify add_msg to use the walk routine * modify twisted interfaces to use the new storage schema. * tests for different multipart cases * fix multipart detection typo in the fetch This is a merge proposal for the 0.5.0-rc3. known bugs ---------- Some things are still know not to work well at this point (some cases of multipart messages do not display the bodies). IMAP server also is left in a bad internal state after a logout/login. --- src/leap/mail/imap/fields.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'src/leap/mail/imap/fields.py') diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py index bc536fe..2545adf 100644 --- a/src/leap/mail/imap/fields.py +++ b/src/leap/mail/imap/fields.py @@ -43,17 +43,17 @@ class WithMsgFields(object): # headers HEADERS_KEY = "headers" - NUM_PARTS_KEY = "numparts" - PARTS_MAP_KEY = "partmap" DATE_KEY = "date" SUBJECT_KEY = "subject" - - # attachment - PART_NUMBER_KEY = "part" - RAW_KEY = "raw" + # XXX DELETE-ME + #NUM_PARTS_KEY = "numparts" # not needed?! + PARTS_MAP_KEY = "part_map" + BODY_KEY = "body" # link to phash of body # content - BODY_KEY = "body" + LINKED_FROM_KEY = "lkf" + RAW_KEY = "raw" + CTYPE_KEY = "ctype" # Mailbox specific keys CLOSED_KEY = "closed" @@ -65,11 +65,13 @@ class WithMsgFields(object): # Document Type, for indexing TYPE_KEY = "type" TYPE_MBOX_VAL = "mbox" - TYPE_MESSAGE_VAL = "msg" TYPE_FLAGS_VAL = "flags" TYPE_HEADERS_VAL = "head" - TYPE_ATTACHMENT_VAL = "attach" - # should add also a headers val + TYPE_CONTENT_VAL = "cnt" + + # XXX DEPRECATE + #TYPE_MESSAGE_VAL = "msg" + #TYPE_ATTACHMENT_VAL = "attach" INBOX_VAL = "inbox" @@ -109,7 +111,6 @@ class WithMsgFields(object): MBOX_VAL = TYPE_MBOX_VAL CHASH_VAL = CONTENT_HASH_KEY PHASH_VAL = PAYLOAD_HASH_KEY - PART_VAL = PART_NUMBER_KEY INDEXES = { # generic @@ -122,8 +123,7 @@ class WithMsgFields(object): # content, headers doc TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL], - # attachment docs - TYPE_C_HASH_PART_IDX: [KTYPE, CHASH_VAL, PART_VAL], + # attachment payload dedup TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL], -- cgit v1.2.3 From 51eaab77deedf0c923fe40cf3d346fa879bf2ae3 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 13 Jan 2014 13:20:00 -0400 Subject: Add check for uniqueness when adding mails. Check by mbox + content-hash --- src/leap/mail/imap/fields.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/leap/mail/imap/fields.py') diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py index 2545adf..70af61f 100644 --- a/src/leap/mail/imap/fields.py +++ b/src/leap/mail/imap/fields.py @@ -99,6 +99,7 @@ class WithMsgFields(object): TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' + TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash' TYPE_C_HASH_IDX = 'by-type-and-contenthash' TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' TYPE_P_HASH_IDX = 'by-type-and-payloadhash' @@ -121,6 +122,9 @@ class WithMsgFields(object): # mailboxes TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'], + # fdocs uniqueness + TYPE_MBOX_C_HASH_IDX: [KTYPE, MBOX_VAL, CHASH_VAL], + # content, headers doc TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL], -- cgit v1.2.3 From 4856f32ec75cda000fc794d0ac93990e0d1e42f6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 13 Jan 2014 17:58:02 -0400 Subject: Very limited support for SEARCH Commands. Closes: #4209 limited to HEADER Message-ID. This is a quick workaround for avoiding duplicate saves in Drafts Folder. but we'll get there! --- src/leap/mail/imap/fields.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'src/leap/mail/imap/fields.py') diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py index 70af61f..3d2ac92 100644 --- a/src/leap/mail/imap/fields.py +++ b/src/leap/mail/imap/fields.py @@ -45,13 +45,12 @@ class WithMsgFields(object): HEADERS_KEY = "headers" DATE_KEY = "date" SUBJECT_KEY = "subject" - # XXX DELETE-ME - #NUM_PARTS_KEY = "numparts" # not needed?! PARTS_MAP_KEY = "part_map" BODY_KEY = "body" # link to phash of body + MSGID_KEY = "msgid" # content - LINKED_FROM_KEY = "lkf" + LINKED_FROM_KEY = "lkf" # XXX not implemented yet! RAW_KEY = "raw" CTYPE_KEY = "ctype" @@ -69,10 +68,6 @@ class WithMsgFields(object): TYPE_HEADERS_VAL = "head" TYPE_CONTENT_VAL = "cnt" - # XXX DEPRECATE - #TYPE_MESSAGE_VAL = "msg" - #TYPE_ATTACHMENT_VAL = "attach" - INBOX_VAL = "inbox" # Flags in Mailbox and Message @@ -96,6 +91,7 @@ class WithMsgFields(object): TYPE_MBOX_IDX = 'by-type-and-mbox' TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' TYPE_SUBS_IDX = 'by-type-and-subscribed' + TYPE_MSGID_IDX = 'by-type-and-message-id' TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' @@ -125,6 +121,9 @@ class WithMsgFields(object): # fdocs uniqueness TYPE_MBOX_C_HASH_IDX: [KTYPE, MBOX_VAL, CHASH_VAL], + # headers doc - search by msgid. + TYPE_MSGID_IDX: [KTYPE, MSGID_KEY], + # content, headers doc TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL], -- cgit v1.2.3 From 9f9701d42be385aa9a6d7e72fd10104b0025971b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 16 Jan 2014 22:01:20 -0400 Subject: Separate RECENT Flag to a mailbox document. this way we avoid a bunch of writes. --- src/leap/mail/imap/fields.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/leap/mail/imap/fields.py') diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py index 3d2ac92..bc928a1 100644 --- a/src/leap/mail/imap/fields.py +++ b/src/leap/mail/imap/fields.py @@ -60,6 +60,7 @@ class WithMsgFields(object): SUBSCRIBED_KEY = "subscribed" RW_KEY = "rw" LAST_UID_KEY = "lastuid" + RECENTFLAGS_KEY = "rct" # Document Type, for indexing TYPE_KEY = "type" @@ -67,6 +68,7 @@ class WithMsgFields(object): TYPE_FLAGS_VAL = "flags" TYPE_HEADERS_VAL = "head" TYPE_CONTENT_VAL = "cnt" + TYPE_RECENT_VAL = "rct" INBOX_VAL = "inbox" -- cgit v1.2.3 From 9ef1cd79397d811575826025b924c615e6ce2aa4 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 17 Jan 2014 02:51:31 -0400 Subject: Add a fetch_headers for mass-header fetch queries --- src/leap/mail/imap/fields.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/leap/mail/imap/fields.py') diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py index bc928a1..886ee63 100644 --- a/src/leap/mail/imap/fields.py +++ b/src/leap/mail/imap/fields.py @@ -61,6 +61,7 @@ class WithMsgFields(object): RW_KEY = "rw" LAST_UID_KEY = "lastuid" RECENTFLAGS_KEY = "rct" + HDOCS_SET_KEY = "hdocset" # Document Type, for indexing TYPE_KEY = "type" @@ -69,6 +70,7 @@ class WithMsgFields(object): TYPE_HEADERS_VAL = "head" TYPE_CONTENT_VAL = "cnt" TYPE_RECENT_VAL = "rct" + TYPE_HDOCS_SET_VAL = "hdocset" INBOX_VAL = "inbox" -- cgit v1.2.3 From 40197f87b86c20ecc3f9dfd38687f25a4158d6e7 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 31 Jan 2014 14:50:16 -0400 Subject: keep processing after decoding errors during fetch --- src/leap/mail/imap/fields.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'src/leap/mail/imap/fields.py') diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py index 886ee63..4576939 100644 --- a/src/leap/mail/imap/fields.py +++ b/src/leap/mail/imap/fields.py @@ -108,6 +108,14 @@ class WithMsgFields(object): # correct since the recent flag is volatile. TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' + # Soledad index for incoming mail, without decrypting errors. + JUST_MAIL_IDX = "just-mail" + # XXX the backward-compatible index, will be deprecated at 0.7 + JUST_MAIL_COMPAT_IDX = "just-mail-compat" + + INCOMING_KEY = "incoming" + ERROR_DECRYPTING_KEY = "errdecr" + KTYPE = TYPE_KEY MBOX_VAL = TYPE_MBOX_VAL CHASH_VAL = CONTENT_HASH_KEY @@ -140,6 +148,13 @@ class WithMsgFields(object): TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'], TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(recent)', 'bool(seen)'], + + # incoming queue + JUST_MAIL_IDX: [INCOMING_KEY, + "bool(%s)" % (ERROR_DECRYPTING_KEY,)], + + # the backward-compatible index, will be deprecated at 0.7 + JUST_MAIL_COMPAT_IDX: [INCOMING_KEY], } MBOX_KEY = MBOX_VAL -- cgit v1.2.3