First stage of the storage schema rewrite.

* Separates between flags, docs, body and attachment docs.

* Implement IMessageCopier interface: move and have fun!
  This little change is known to push forward our beloved architect
  emotional rollercoster.

* Message deduplication.

* It also fixes a hidden bug that was rendering the multipart mime
  interface useless
  (yes, the "True" parameter in the parsestr method).

* Does not handle well nested attachs, includes dirty workaround that
  flattens them.

* Includes chiiph's patch for rc2:

 * return deferred from addMessage
 * convert StringIO types to string
 * remove unneeded yields from the chain of deferreds in fetcher

1 files changed, 35 insertions, 14 deletions

diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py
index 96b937e..40817cd 100644
--- a/src/leap/mail/imap/fields.py
+++ b/src/leap/mail/imap/fields.py
@@ -25,18 +25,35 @@ class WithMsgFields(object):
     Container class for class-attributes to be shared by
     several message-related classes.
     """
-    # Internal representation of Message
-    DATE_KEY = "date"
-    HEADERS_KEY = "headers"
-    FLAGS_KEY = "flags"
-    MBOX_KEY = "mbox"
+    # indexing
     CONTENT_HASH_KEY = "chash"
-    RAW_KEY = "raw"
-    SUBJECT_KEY = "subject"
+    PAYLOAD_HASH_KEY = "phash"
+
+    # Internal representation of Message
+
+    # flags doc
     UID_KEY = "uid"
+    MBOX_KEY = "mbox"
+    SEEN_KEY = "seen"
+    RECENT_KEY = "recent"
+    FLAGS_KEY = "flags"
     MULTIPART_KEY = "multi"
     SIZE_KEY = "size"
 
+    # headers
+    HEADERS_KEY = "headers"
+    NUM_PARTS_KEY = "numparts"
+    PARTS_MAP_KEY = "partmap"
+    DATE_KEY = "date"
+    SUBJECT_KEY = "subject"
+
+    # attachment
+    PART_NUMBER_KEY = "part"
+    RAW_KEY = "raw"
+
+    # content
+    BODY_KEY = "body"
+
     # Mailbox specific keys
     CLOSED_KEY = "closed"
     CREATED_KEY = "created"
@@ -55,10 +72,6 @@ class WithMsgFields(object):
 
     INBOX_VAL = "inbox"
 
-    # Flags for SoledadDocument for indexing.
-    SEEN_KEY = "seen"
-    RECENT_KEY = "recent"
-
     # Flags in Mailbox and Message
     SEEN_FLAG = "\\Seen"
     RECENT_FLAG = "\\Recent"
@@ -82,7 +95,9 @@ class WithMsgFields(object):
     TYPE_SUBS_IDX = 'by-type-and-subscribed'
     TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
     TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
-    TYPE_HASH_IDX = 'by-type-and-hash'
+    TYPE_C_HASH_IDX = 'by-type-and-contenthash'
+    TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber'
+    TYPE_P_HASH_IDX = 'by-type-and-payloadhash'
 
     # Tomas created the `recent and seen index`, but the semantic is not too
     # correct since the recent flag is volatile.
@@ -90,7 +105,9 @@ class WithMsgFields(object):
 
     KTYPE = TYPE_KEY
     MBOX_VAL = TYPE_MBOX_VAL
-    HASH_VAL = CONTENT_HASH_KEY
+    CHASH_VAL = CONTENT_HASH_KEY
+    PHASH_VAL = PAYLOAD_HASH_KEY
+    PART_VAL = PART_NUMBER_KEY
 
     INDEXES = {
         # generic
@@ -102,7 +119,11 @@ class WithMsgFields(object):
         TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'],
 
         # content, headers doc
-        TYPE_HASH_IDX: [KTYPE, HASH_VAL],
+        TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL],
+        # attachment docs
+        TYPE_C_HASH_PART_IDX: [KTYPE, CHASH_VAL, PART_VAL],
+        # attachment payload dedup
+        TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL],
 
         # messages
         TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'],