Merge remote-tracking branch 'refs/remotes/kali/feature/regression-tests' into develop

5 files changed, 492 insertions, 23 deletions

diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index c682578..d8af0a5 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -484,9 +484,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         d.addCallback(self._close_cb)
         return d
 
-    def _expunge_cb(self, result):
-        return result
-
     def expunge(self):
         """
         Remove all messages flagged \\Deleted
@@ -494,9 +491,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
         if not self.isWriteable():
             raise imap4.ReadOnlyMailbox
         d = defer.Deferred()
-        return self._memstore.expunge(self.mbox, d)
-        self._memstore.expunge(self.mbox)
-        d.addCallback(self._expunge_cb, d)
+        self._memstore.expunge(self.mbox, d)
         return d
 
     def _bound_seq(self, messages_asked):
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
index 195cef7..f4a4522 100644
--- a/src/leap/mail/imap/memorystore.py
+++ b/src/leap/mail/imap/memorystore.py
@@ -879,30 +879,43 @@ class MemoryStore(object):
         Remove all messages flagged \\Deleted, from the Memory Store
         and from the permanent store also.
 
+        It first queues up a last write, and wait for the deferreds to be done
+        before continuing.
+
         :param mbox: the mailbox
         :type mbox: str or unicode
         :param observer: a deferred that will be fired when expunge is done
         :type observer: Deferred
-        :return: a list of UIDs
-        :rtype: list
         """
-        # TODO expunge should add itself as a callback to the ongoing
-        # writes.
         soledad_store = self._permanent_store
-        all_deleted = []
-
         try:
             # 1. Stop the writing call
             self._stop_write_loop()
             # 2. Enqueue a last write.
-            #self.write_messages(soledad_store)
-            # 3. Should wait on the writebacks to finish ???
-            # FIXME wait for this, and add all the rest of the method
-            # as a callback!!!
+            self.write_messages(soledad_store)
+            # 3. Wait on the writebacks to finish
+
+            pending_deferreds = (self._new_deferreds.get(mbox, []) +
+                                 self._dirty_deferreds.get(mbox, []))
+            d1 = defer.gatherResults(pending_deferreds, consumeErrors=True)
+            d1.addCallback(
+                self._delete_from_soledad_and_memory, mbox, observer)
         except Exception as exc:
             logger.exception(exc)
 
-        # Now, we...:
+    def _delete_from_soledad_and_memory(self, result, mbox, observer):
+        """
+        Remove all messages marked as deleted from soledad and memory.
+
+        :param result: ignored. the result of the deferredList that triggers
+                       this as a callback from `expunge`.
+        :param mbox: the mailbox
+        :type mbox: str or unicode
+        :param observer: a deferred that will be fired when expunge is done
+        :type observer: Deferred
+        """
+        all_deleted = []
+        soledad_store = self._permanent_store
 
         try:
             # 1. Delete all messages marked as deleted in soledad.
@@ -927,8 +940,7 @@ class MemoryStore(object):
             logger.exception(exc)
         finally:
             self._start_write_loop()
-        observer.callback(True)
-        return all_deleted
+        observer.callback(all_deleted)
 
     # Dump-to-disk controls.
 
diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
index b07681b..2d9b3a2 100644
--- a/src/leap/mail/imap/messageparts.py
+++ b/src/leap/mail/imap/messageparts.py
@@ -397,7 +397,9 @@ class MessagePart(object):
                 logger.warning("Could not find phash for this subpart!")
                 payload = ""
             else:
-                payload = self._get_payload_from_document(phash)
+                payload = self._get_payload_from_document_memoized(phash)
+                if payload is None:
+                    payload = self._get_payload_from_document(phash)
 
         else:
             logger.warning("Message with no part_map!")
@@ -424,13 +426,24 @@ class MessagePart(object):
 
     # TODO should memory-bound this memoize!!!
     @memoized_method
+    def _get_payload_from_document_memoized(self, phash):
+        """
+        Memoized method call around the regular method, to be able
+        to call the non-memoized method in case we got a None.
+
+        :param phash: the payload hash to retrieve by.
+        :type phash: str or unicode
+        :rtype: str or unicode or None
+        """
+        return self._get_payload_from_document(phash)
+
     def _get_payload_from_document(self, phash):
         """
         Return the message payload from the content document.
 
         :param phash: the payload hash to retrieve by.
         :type phash: str or unicode
-        :rtype: str or unicode
+        :rtype: str or unicode or None
         """
         cdocs = self._soledad.get_from_index(
             fields.TYPE_P_HASH_IDX,
diff --git a/src/leap/mail/imap/tests/getmail b/src/leap/mail/imap/tests/getmail
index 17e195c..0fb00d2 100755
--- a/src/leap/mail/imap/tests/getmail
+++ b/src/leap/mail/imap/tests/getmail
@@ -5,8 +5,6 @@
 
 # Modifications by LEAP Developers 2014 to fit
 # Bitmask configuration settings.
-
-
 """
 Simple IMAP4 client which displays the subjects of all messages in a
 particular mailbox.
diff --git a/src/leap/mail/imap/tests/regressions b/src/leap/mail/imap/tests/regressions
new file mode 100755
index 0000000..0a43398
--- /dev/null
+++ b/src/leap/mail/imap/tests/regressions
@@ -0,0 +1,451 @@
+#!/usr/bin/env python
+
+# -*- coding: utf-8 -*-
+# regressions
+# Copyright (C) 2014 LEAP
+# Copyright (c) Twisted Matrix Laboratories.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Simple Regression Tests using IMAP4 client.
+
+Iterates trough all mails under a given folder and tries to APPEND them to
+the server being tested. After FETCHING the pushed message, it compares
+the received version with the one that was saved, and exits with an error
+code if they do not match.
+"""
+import os
+import StringIO
+import sys
+
+from email.parser import Parser
+
+from twisted.internet import protocol
+from twisted.internet import ssl
+from twisted.internet import defer
+from twisted.internet import stdio
+from twisted.mail import imap4
+from twisted.protocols import basic
+from twisted.python import log
+
+
+REGRESSIONS_FOLDER = "regressions_test"
+
+parser = Parser()
+
+
+def get_msg_parts(raw):
+    """
+    Return a representation of the parts of a message suitable for
+    comparison.
+
+    :param raw: string for the message
+    :type raw: str
+    """
+    m = parser.parsestr(raw)
+    return [dict(part.items())
+            if part.is_multipart()
+            else part.get_payload()
+            for part in m.walk()]
+
+
+def compare_msg_parts(a, b):
+    """
+    Compare two sequences of parts of messages.
+
+    :param a: part sequence for message a
+    :param b: part sequence for message b
+
+    :return: True if both message sequences are equivalent.
+    :rtype: bool
+    """
+    # XXX This could be smarter and show the differences in the
+    # different parts when/where they differ.
+    #import pprint; pprint.pprint(a[0])
+    #import pprint; pprint.pprint(b[0])
+
+    def lowerkey(d):
+        return dict((k.lower(), v.replace('\r', ''))
+                    for k, v in d.iteritems())
+
+    def eq(x, y):
+        # For dicts, we compare a variation with their keys
+        # in lowercase, and \r removed from their values
+        if all(map(lambda i: isinstance(i, dict), (x, y))):
+            x, y = map(lowerkey, (x, y))
+        return x == y
+
+    compare_vector = map(lambda tup: eq(tup[0], tup[1]), zip(a, b))
+    all_match = all(compare_vector)
+
+    if not all_match:
+        print "PARTS MISMATCH!"
+        print "vector: ", compare_vector
+        index = compare_vector.index(False)
+        from pprint import pprint
+        print "Expected:"
+        pprint(a[index])
+        print ("***")
+        print "Found:"
+        pprint(b[index])
+        print
+
+
+    return all_match
+
+
+def get_fd(string):
+    """
+    Return a file descriptor with the passed string
+    as content.
+    """
+    fd = StringIO.StringIO()
+    fd.write(string)
+    fd.seek(0)
+    return fd
+
+
+class TrivialPrompter(basic.LineReceiver):
+    promptDeferred = None
+
+    def prompt(self, msg):
+        assert self.promptDeferred is None
+        self.display(msg)
+        self.promptDeferred = defer.Deferred()
+        return self.promptDeferred
+
+    def display(self, msg):
+        self.transport.write(msg)
+
+    def lineReceived(self, line):
+        if self.promptDeferred is None:
+            return
+        d, self.promptDeferred = self.promptDeferred, None
+        d.callback(line)
+
+
+class SimpleIMAP4Client(imap4.IMAP4Client):
+    """
+    A client with callbacks for greeting messages from an IMAP server.
+    """
+    greetDeferred = None
+
+    def serverGreeting(self, caps):
+        self.serverCapabilities = caps
+        if self.greetDeferred is not None:
+            d, self.greetDeferred = self.greetDeferred, None
+            d.callback(self)
+
+
+class SimpleIMAP4ClientFactory(protocol.ClientFactory):
+    usedUp = False
+    protocol = SimpleIMAP4Client
+
+    def __init__(self, username, onConn):
+        self.ctx = ssl.ClientContextFactory()
+
+        self.username = username
+        self.onConn = onConn
+
+    def buildProtocol(self, addr):
+        """
+        Initiate the protocol instance. Since we are building a simple IMAP
+        client, we don't bother checking what capabilities the server has. We
+        just add all the authenticators twisted.mail has.  Note: Gmail no
+        longer uses any of the methods below, it's been using XOAUTH since
+        2010.
+        """
+        assert not self.usedUp
+        self.usedUp = True
+
+        p = self.protocol(self.ctx)
+        p.factory = self
+        p.greetDeferred = self.onConn
+
+        p.registerAuthenticator(imap4.PLAINAuthenticator(self.username))
+        p.registerAuthenticator(imap4.LOGINAuthenticator(self.username))
+        p.registerAuthenticator(
+            imap4.CramMD5ClientAuthenticator(self.username))
+
+        return p
+
+    def clientConnectionFailed(self, connector, reason):
+        d, self.onConn = self.onConn, None
+        d.errback(reason)
+
+
+def cbServerGreeting(proto, username, password):
+    """
+    Initial callback - invoked after the server sends us its greet message.
+    """
+    # Hook up stdio
+    tp = TrivialPrompter()
+    stdio.StandardIO(tp)
+
+    # And make it easily accessible
+    proto.prompt = tp.prompt
+    proto.display = tp.display
+
+    # Try to authenticate securely
+    return proto.authenticate(
+        password).addCallback(
+        cbAuthentication,
+        proto).addErrback(
+        ebAuthentication, proto, username, password
+    )
+
+
+def ebConnection(reason):
+    """
+    Fallback error-handler. If anything goes wrong, log it and quit.
+    """
+    log.startLogging(sys.stdout)
+    log.err(reason)
+    return reason
+
+
+def cbAuthentication(result, proto):
+    """
+    Callback after authentication has succeeded.
+
+    Lists a bunch of mailboxes.
+    """
+    return proto.select(
+        REGRESSIONS_FOLDER
+    ).addCallback(
+        cbSelectMbox, proto
+    ).addErrback(
+        ebSelectMbox, proto, REGRESSIONS_FOLDER)
+
+
+def ebAuthentication(failure, proto, username, password):
+    """
+    Errback invoked when authentication fails.
+
+    If it failed because no SASL mechanisms match, offer the user the choice
+    of logging in insecurely.
+
+    If you are trying to connect to your Gmail account, you will be here!
+    """
+    failure.trap(imap4.NoSupportedAuthentication)
+    return InsecureLogin(proto, username, password)
+
+
+def InsecureLogin(proto, username, password):
+    """
+    Raise insecure-login error.
+    """
+    return proto.login(
+        username, password
+    ).addCallback(
+        cbAuthentication, proto)
+
+
+def cbSelectMbox(result, proto):
+    """
+    Callback invoked when select command finishes successfully.
+
+    If any message is in the test folder, it will flag them as deleted and
+    expunge.
+    If no messages found, it will start with the APPEND tests.
+    """
+    print "SELECT: %s EXISTS " % result.get("EXISTS", "??")
+
+    if result["EXISTS"] != 0:
+        # Flag as deleted, expunge, and do an examine again.
+        #print "There is mail here, will delete..."
+        return cbDeleteAndExpungeTestFolder(proto)
+
+    else:
+        return cbAppendNextMessage(proto)
+
+
+def ebSelectMbox(failure, proto, folder):
+    """
+    Errback invoked when the examine command fails.
+
+    Creates the folder.
+    """
+    print failure.getTraceback()
+    log.msg("Folder %r does not exist. Creating..." % (folder,))
+    return proto.create(folder).addCallback(cbAuthentication, proto)
+
+
+def cbDeleteAndExpungeTestFolder(proto):
+    """
+    Callback invoked fom cbExamineMbox when the number of messages in the
+    mailbox is not zero. It flags all messages as deleted and expunge the
+    mailbox.
+    """
+    return proto.setFlags(
+        "1:*", ("\\Deleted",)
+    ).addCallback(
+        lambda r: proto.expunge()
+    ).addCallback(
+        cbExpunge, proto)
+
+
+def cbExpunge(result, proto):
+    return proto.select(
+        REGRESSIONS_FOLDER
+    ).addCallback(
+        cbSelectMbox, proto
+    ).addErrback(ebSettingDeleted, proto)
+
+
+def ebSettingDeleted(failure, proto):
+    """
+    Report errors during deletion of messages in the mailbox.
+    """
+    print failure.getTraceback()
+
+
+def cbAppendNextMessage(proto):
+    """
+    Appends the next message in the global queue to the test folder.
+    """
+    # 1. Get the next test message from global tuple.
+    try:
+        next_sample = SAMPLES.pop()
+    except IndexError:
+        # we're done!
+        return proto.logout()
+
+    print "\nAPPEND %s" % (next_sample,)
+    raw = open(next_sample).read()
+    msg = get_fd(raw)
+    return proto.append(
+        REGRESSIONS_FOLDER, msg
+    ).addCallback(
+        lambda r: proto.examine(REGRESSIONS_FOLDER)
+    ).addCallback(
+        cbAppend, proto, raw
+    ).addErrback(
+        ebAppend, proto, raw)
+
+
+def cbAppend(result, proto, orig_msg):
+    """
+    Fetches the message right after an append.
+    """
+    # XXX keep account of highest UID
+    uid = "1:*"
+
+    return proto.fetchSpecific(
+        '%s' % uid,
+        headerType='',
+        headerArgs=['BODY.PEEK[]'],
+    ).addCallback(
+        cbCompareMessage, proto, orig_msg
+    ).addErrback(ebAppend, proto, orig_msg)
+
+
+def ebAppend(failure, proto, raw):
+    """
+    Errorback for the append operation
+    """
+    print "ERROR WHILE APPENDING!"
+    print failure.getTraceback()
+
+
+def cbPickMessage(result, proto):
+    """
+    Pick a message.
+    """
+    return proto.fetchSpecific(
+        '%s' % result,
+        headerType='',
+        headerArgs=['BODY.PEEK[]'],
+        ).addCallback(cbCompareMessage, proto)
+
+
+def cbCompareMessage(result, proto, raw):
+    """
+    Display message and compare it with the original one.
+    """
+    parts_orig = get_msg_parts(raw)
+
+    if result:
+        keys = result.keys()
+        keys.sort()
+
+    latest = max(keys)
+
+    fetched_msg = result[latest][0][2]
+    parts_fetched = get_msg_parts(fetched_msg)
+
+    equal = compare_msg_parts(
+        parts_orig,
+        parts_fetched)
+
+    if equal:
+        print "[+] MESSAGES MATCH"
+        return cbAppendNextMessage(proto)
+    else:
+        print "[-] ERROR: MESSAGES DO NOT MATCH !!!"
+        print "    ABORTING COMPARISON..."
+        # FIXME logout and print the subject ...
+        return proto.logout()
+
+
+def cbClose(result):
+    """
+    Close the connection when we finish everything.
+    """
+    from twisted.internet import reactor
+    reactor.stop()
+
+
+def main():
+    import glob
+    import sys
+
+    if len(sys.argv) != 4:
+        print "Usage: regressions <user> <pass> <samples-folder>"
+        sys.exit()
+
+    hostname = "localhost"
+    port = "1984"
+    username = sys.argv[1]
+    password = sys.argv[2]
+
+    samplesdir = sys.argv[3]
+
+    if not os.path.isdir(samplesdir):
+        print ("Could not find samples folder! "
+               "Make sure of copying mail_breaker contents there.")
+        sys.exit()
+
+    samples = glob.glob(samplesdir + '/*')
+
+    global SAMPLES
+    SAMPLES = []
+    SAMPLES += samples
+
+    onConn = defer.Deferred(
+    ).addCallback(
+        cbServerGreeting, username, password
+    ).addErrback(
+        ebConnection
+    ).addBoth(cbClose)
+
+    factory = SimpleIMAP4ClientFactory(username, onConn)
+
+    from twisted.internet import reactor
+    reactor.connectTCP(hostname, int(port), factory)
+    reactor.run()
+
+
+if __name__ == '__main__':
+    main()