Add check for uniqueness when adding mails.

Check by mbox + content-hash

4 files changed, 59 insertions, 3 deletions

diff --git a/mail/changes/bug_4949-check-fdoc-uniqueness b/mail/changes/bug_4949-check-fdoc-uniqueness
new file mode 100644
index 0000000..bf49d1f
--- /dev/null
+++ b/mail/changes/bug_4949-check-fdoc-uniqueness
@@ -0,0 +1,2 @@
+  o Check for flags doc uniqueness before adding a message. Avoids duplicates of
+    a single message in the same mailbox while copying or moving. Closes: #4949
diff --git a/mail/src/leap/mail/imap/fields.py b/mail/src/leap/mail/imap/fields.py
index 2545adf..70af61f 100644
--- a/mail/src/leap/mail/imap/fields.py
+++ b/mail/src/leap/mail/imap/fields.py
@@ -99,6 +99,7 @@ class WithMsgFields(object):
     TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
     TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
     TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted'
+    TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash'
     TYPE_C_HASH_IDX = 'by-type-and-contenthash'
     TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber'
     TYPE_P_HASH_IDX = 'by-type-and-payloadhash'
@@ -121,6 +122,9 @@ class WithMsgFields(object):
         # mailboxes
         TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'],
 
+        # fdocs uniqueness
+        TYPE_MBOX_C_HASH_IDX: [KTYPE, MBOX_VAL, CHASH_VAL],
+
         # content, headers doc
         TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL],
 
diff --git a/mail/src/leap/mail/imap/mailbox.py b/mail/src/leap/mail/imap/mailbox.py
index 7c01490..c9e8684 100644
--- a/mail/src/leap/mail/imap/mailbox.py
+++ b/mail/src/leap/mail/imap/mailbox.py
@@ -125,7 +125,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
     def addListener(self, listener):
         """
-        Adds a listener to the listeners queue.
+        Add a listener to the listeners queue.
         The server adds itself as a listener when there is a SELECT,
         so it can send EXIST commands.
 
@@ -137,7 +137,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
     def removeListener(self, listener):
         """
-        Removes a listener from the listeners queue.
+        Remove a listener from the listeners queue.
 
         :param listener: listener to remove
         :type listener: an object that implements IMailboxListener
@@ -146,7 +146,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
 
     def _get_mbox(self):
         """
-        Returns mailbox document.
+        Return mailbox document.
 
         :return: A SoledadDocument containing this mailbox, or None if
                  the query failed.
diff --git a/mail/src/leap/mail/imap/messages.py b/mail/src/leap/mail/imap/messages.py
index 37e4311..a3fcd87 100644
--- a/mail/src/leap/mail/imap/messages.py
+++ b/mail/src/leap/mail/imap/messages.py
@@ -1064,10 +1064,27 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
             hd[self.DATE_KEY] = date
         return hd
 
+    def _fdoc_already_exists(self, chash):
+        """
+        Check whether we can find a flags doc for this mailbox with the
+        given content-hash. It enforces that we can only have the same maessage
+        listed once for a a given mailbox.
+
+        :param chash: the content-hash to check about.
+        :type chash: basestring
+        :return: False, if it does not exist, or UID.
+        """
+        exist = self._get_fdoc_from_chash(chash)
+        if exist:
+            return exist.content.get(fields.UID_KEY, "unknown-uid")
+        else:
+            return False
+
     @deferred
     def add_msg(self, raw, subject=None, flags=None, date=None, uid=1):
         """
         Creates a new message document.
+        Here lives the magic of the leap mail. Well, in soledad, really.
 
         :param raw: the raw message
         :type raw: str
@@ -1097,6 +1114,14 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
         # parse
         msg, chash, size, multi = self._do_parse(raw)
 
+        # check for uniqueness.
+        if self._fdoc_already_exists(chash):
+            logger.warning("We already have that message in this mailbox.")
+            # note that this operation will leave holes in the UID sequence,
+            # but we're gonna change that all the same for a local-only table.
+            # so not touch it by the moment.
+            return False
+
         fd = self._populate_flags(flags, uid, chash, size, multi)
         hd = self._populate_headr(msg, chash, subject, date)
 
@@ -1156,6 +1181,31 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser):
 
     # getters
 
+    def _get_fdoc_from_chash(self, chash):
+        """
+        Return a flags document for this mailbox with a given chash.
+
+        :return: A SoledadDocument containing the Flags Document, or None if
+                 the query failed.
+        :rtype: SoledadDocument or None.
+        """
+        try:
+            query = self._soledad.get_from_index(
+                fields.TYPE_MBOX_C_HASH_IDX,
+                fields.TYPE_FLAGS_VAL, self.mbox, chash)
+            if query:
+                if len(query) > 1:
+                    logger.warning(
+                        "More than one fdoc found for this chash, "
+                        "we got a duplicate!!")
+                    # XXX we could take action, like trigger a background
+                    # process to kill dupes.
+                return query.pop()
+            else:
+                return None
+        except Exception as exc:
+            logger.exception("Unhandled error %r" % exc)
+
     def get_msg_by_uid(self, uid):
         """
         Retrieves a LeapMessage by UID.