From 541bd8aec1f67834c42bc2e5df14c1f73c569082 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 6 Dec 2013 17:45:21 -0400 Subject: pep8 cleanup --- src/leap/mail/imap/tests/imapclient.py | 7 +++--- src/leap/mail/imap/tests/test_imap.py | 45 ++++++++++++++++++++-------------- 2 files changed, 31 insertions(+), 21 deletions(-) (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/imapclient.py b/src/leap/mail/imap/tests/imapclient.py index 027396c..c353cee 100755 --- a/src/leap/mail/imap/tests/imapclient.py +++ b/src/leap/mail/imap/tests/imapclient.py @@ -21,7 +21,7 @@ from twisted.python import log class TrivialPrompter(basic.LineReceiver): - #from os import linesep as delimiter + # from os import linesep as delimiter promptDeferred = None @@ -42,6 +42,7 @@ class TrivialPrompter(basic.LineReceiver): class SimpleIMAP4Client(imap4.IMAP4Client): + """ Add callbacks when the client receives greeting messages from an IMAP server. @@ -98,8 +99,8 @@ def cbServerGreeting(proto, username, password): # Try to authenticate securely return proto.authenticate( password).addCallback( - cbAuthentication, proto).addErrback( - ebAuthentication, proto, username, password) + cbAuthentication, proto).addErrback( + ebAuthentication, proto, username, password) def ebConnection(reason): diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index ca73a11..7d26862 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -54,7 +54,7 @@ import twisted.cred.credentials import twisted.cred.portal -#import u1db +# import u1db from leap.common.testing.basetest import BaseLeapTest from leap.mail.imap.server import SoledadMailbox @@ -120,17 +120,19 @@ def initialize_soledad(email, gnupg_home, tempdir): return _soledad -########################################## +# # Simple LEAP IMAP4 Server for testing -########################################## +# class SimpleLEAPServer(imap4.IMAP4Server): + """ A Simple IMAP4 Server with mailboxes backed by Soledad. This should be pretty close to the real LeapIMAP4Server that we will be instantiating as a service, minus the authentication bits. """ + def __init__(self, *args, **kw): soledad = kw.pop('soledad', None) @@ -153,7 +155,7 @@ class SimpleLEAPServer(imap4.IMAP4Server): def lineReceived(self, line): if self.timeoutTest: - #Do not send a respones + # Do not send a respones return imap4.IMAP4Server.lineReceived(self, line) @@ -168,6 +170,7 @@ class SimpleLEAPServer(imap4.IMAP4Server): class TestRealm: + """ A minimal auth realm for testing purposes only """ @@ -177,12 +180,13 @@ class TestRealm: return imap4.IAccount, self.theAccount, lambda: None -###################################### +# # Simple IMAP4 Client for testing -###################################### +# class SimpleClient(imap4.IMAP4Client): + """ A Simple IMAP4 Client to test our Soledad-LEAPServer @@ -210,6 +214,7 @@ class SimpleClient(imap4.IMAP4Client): class IMAP4HelperMixin(BaseLeapTest): + """ MixIn containing several utilities to be shared across different TestCases @@ -245,13 +250,13 @@ class IMAP4HelperMixin(BaseLeapTest): # Soledad: config info cls.gnupg_home = "%s/gnupg" % cls.tempdir cls.email = 'leap@leap.se' - #cls.db1_file = "%s/db1.u1db" % cls.tempdir - #cls.db2_file = "%s/db2.u1db" % cls.tempdir + # cls.db1_file = "%s/db1.u1db" % cls.tempdir + # cls.db2_file = "%s/db2.u1db" % cls.tempdir # open test dbs - #cls._db1 = u1db.open(cls.db1_file, create=True, - #document_factory=SoledadDocument) - #cls._db2 = u1db.open(cls.db2_file, create=True, - #document_factory=SoledadDocument) + # cls._db1 = u1db.open(cls.db1_file, create=True, + # document_factory=SoledadDocument) + # cls._db2 = u1db.open(cls.db2_file, create=True, + # document_factory=SoledadDocument) # initialize soledad by hand so we can control keys cls._soledad = initialize_soledad( @@ -261,7 +266,7 @@ class IMAP4HelperMixin(BaseLeapTest): # now we're passing the mailbox name, so we # should get this into a partial or something. - #cls.sm = SoledadMailbox("mailbox", soledad=cls._soledad) + # cls.sm = SoledadMailbox("mailbox", soledad=cls._soledad) # XXX REFACTOR --- self.server (in setUp) is initializing # a SoledadBackedAccount @@ -273,8 +278,8 @@ class IMAP4HelperMixin(BaseLeapTest): Restores the old path and home environment variables. Removes the temporal dir created for tests. """ - #cls._db1.close() - #cls._db2.close() + # cls._db1.close() + # cls._db2.close() cls._soledad.close() os.environ["PATH"] = cls.old_path @@ -328,8 +333,8 @@ class IMAP4HelperMixin(BaseLeapTest): acct.delete(mb) # FIXME add again - #for subs in acct.subscriptions: - #acct.unsubscribe(subs) + # for subs in acct.subscriptions: + # acct.unsubscribe(subs) del self.server del self.client @@ -375,9 +380,11 @@ class IMAP4HelperMixin(BaseLeapTest): # class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): + """ Tests for the MessageCollection class """ + def setUp(self): """ setUp method for each test @@ -434,6 +441,7 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): + """ Tests for the generic behavior of the LeapIMAP4Server which, right now, it's just implemented in this test file as @@ -1149,7 +1157,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): mb = SimpleLEAPServer.theAccount.getMailbox('PARTIAL/SUBTHING') self.assertEqual(1, len(mb.messages)) self.assertEqual( - ['\\SEEN',], + ['\\SEEN', ], mb.messages[1].content['flags'] ) self.assertEqual( @@ -1262,6 +1270,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase): + """ Tests for the behavior of the search_* functions in L{imap4.IMAP4Server}. """ -- cgit v1.2.3 From 44b8f5eaaaeeacbb1f9ceca1231cb53ef13f16ab Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 10 Dec 2013 21:00:18 -0400 Subject: make exceptions fail the test. right now, the exceptions were visible in the stdout, but the test was not *actually* failing. using nose deferred decorator for this. --- src/leap/mail/imap/tests/test_imap.py | 45 +++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 7 deletions(-) (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 7d26862..d78115e 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -25,6 +25,7 @@ XXX add authors from the original twisted tests. @license: GPLv3, see included LICENSE file """ # XXX review license of the original tests!!! +from nose.twistedtools import deferred try: from cStringIO import StringIO @@ -370,6 +371,7 @@ class IMAP4HelperMixin(BaseLeapTest): self.client.transport.loseConnection() self.server.transport.loseConnection() log.err(failure, "Problem with %r" % (self.function,)) + failure.trap(Exception) def loopback(self): return loopback.loopbackAsync(self.server, self.client) @@ -426,8 +428,11 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): Test that queries filter by selected mailbox """ mc = self.messages + self.assertEqual(self.messages.count(), 0) mc.add_msg('', subject="test1") + self.assertEqual(self.messages.count(), 1) mc.add_msg('', subject="test2") + self.assertEqual(self.messages.count(), 2) mc.add_msg('', subject="test3") self.assertEqual(self.messages.count(), 3) @@ -459,6 +464,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): # mailboxes operations # + @deferred(timeout=None) def testCreate(self): """ Test whether we can create mailboxes @@ -497,6 +503,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): answers.sort() self.assertEqual(mbox, [a.upper() for a in answers]) + @deferred(timeout=None) def testDelete(self): """ Test whether we can delete mailboxes @@ -545,6 +552,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): failure.Failure))) return d + @deferred(timeout=None) def testNonExistentDelete(self): """ Test what happens if we try to delete a non-existent mailbox. @@ -570,6 +578,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): 'No such mailbox')) return d + @deferred(timeout=None) def testIllegalDelete(self): """ Try deleting a mailbox with sub-folders, and \NoSelect flag set. @@ -604,6 +613,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.assertEqual(str(self.failure.value), expected)) return d + @deferred(timeout=None) def testRename(self): """ Test whether we can rename a mailbox @@ -627,6 +637,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): ['NEWNAME'])) return d + @deferred(timeout=None) def testIllegalInboxRename(self): """ Try to rename inbox. We expect it to fail. Then it would be not @@ -654,6 +665,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.stashed, failure.Failure))) return d + @deferred(timeout=None) def testHierarchicalRename(self): """ Try to rename hierarchical mailboxes @@ -680,6 +692,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): mboxes.sort() self.assertEqual(mboxes, [s.upper() for s in expected]) + @deferred(timeout=None) def testSubscribe(self): """ Test whether we can mark a mailbox as subscribed to @@ -701,6 +714,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): ['THIS/MBOX'])) return d + @deferred(timeout=None) def testUnsubscribe(self): """ Test whether we can unsubscribe from a set of mailboxes @@ -725,6 +739,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): ['THAT/MBOX'])) return d + @deferred(timeout=None) def testSelect(self): """ Try to select a mailbox @@ -764,6 +779,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): # capabilities # + @deferred(timeout=None) def testCapability(self): caps = {} @@ -779,6 +795,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(lambda _: self.assertEqual(expected, caps)) + @deferred(timeout=None) def testCapabilityWithAuth(self): caps = {} self.server.challengers[ @@ -803,6 +820,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): # authentication # + @deferred(timeout=None) def testLogout(self): """ Test log out @@ -817,6 +835,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d = self.loopback() return d.addCallback(lambda _: self.assertEqual(self.loggedOut, 1)) + @deferred(timeout=None) def testNoop(self): """ Test noop command @@ -832,6 +851,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d = self.loopback() return d.addCallback(lambda _: self.assertEqual(self.responses, [])) + @deferred(timeout=None) def testLogin(self): """ Test login @@ -848,6 +868,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.assertEqual(self.server.account, SimpleLEAPServer.theAccount) self.assertEqual(self.server.state, 'auth') + @deferred(timeout=None) def testFailedLogin(self): """ Test bad login @@ -866,6 +887,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.assertEqual(self.server.account, None) self.assertEqual(self.server.state, 'unauth') + @deferred(timeout=None) def testLoginRequiringQuoting(self): """ Test login requiring quoting @@ -890,6 +912,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): # Inspection # + @deferred(timeout=None) def testNamespace(self): """ Test retrieving namespace @@ -914,6 +937,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): [[['', '/']], [], []])) return d + @deferred(timeout=None) def testExamine(self): """ L{IMAP4Client.examine} issues an I{EXAMINE} command to the server and @@ -983,6 +1007,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d2 = self.loopback() return defer.gatherResults([d1, d2]).addCallback(lambda _: self.listed) + @deferred(timeout=None) def testList(self): """ Test List command @@ -999,22 +1024,21 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): )) return d - # XXX implement subscriptions - ''' + @deferred(timeout=None) def testLSub(self): """ Test LSub command """ - SimpleLEAPServer.theAccount.subscribe('ROOT/SUBTHINGL') + SimpleLEAPServer.theAccount.subscribe('ROOT/SUBTHINGL2') def lsub(): return self.client.lsub('root', '%') d = self._listSetup(lsub) d.addCallback(self.assertEqual, - [(SoledadMailbox.INIT_FLAGS, "/", "ROOT/SUBTHINGL")]) + [(SoledadMailbox.INIT_FLAGS, "/", "ROOT/SUBTHINGL2")]) return d - ''' + @deferred(timeout=None) def testStatus(self): """ Test Status command @@ -1046,6 +1070,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): )) return d + @deferred(timeout=None) def testFailedStatus(self): """ Test failed status command with a non-existent mailbox @@ -1085,6 +1110,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): # messages # + @deferred(timeout=None) def testFullAppend(self): """ Test appending a full message to the mailbox @@ -1125,6 +1151,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.assertEqual(open(infile).read(), mb.messages[1].content['raw']) + @deferred(timeout=None) def testPartialAppend(self): """ Test partially appending a message to the mailbox @@ -1151,10 +1178,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - return d.addCallback(self._cbTestPartialAppend, infile) + return d.addCallback( + self._cbTestPartialAppend, infile) def _cbTestPartialAppend(self, ignored, infile): mb = SimpleLEAPServer.theAccount.getMailbox('PARTIAL/SUBTHING') + self.assertEqual(1, len(mb.messages)) self.assertEqual( ['\\SEEN', ], @@ -1164,6 +1193,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): 'Right now', mb.messages[1].content['date']) self.assertEqual(open(infile).read(), mb.messages[1].content['raw']) + @deferred(timeout=None) def testCheck(self): """ Test check command @@ -1187,6 +1217,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): # Okay, that was fun + @deferred(timeout=None) def testClose(self): """ Test closing the mailbox. We expect to get deleted all messages flagged @@ -1222,9 +1253,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.assertEqual( m.messages[1].content['subject'], 'Message 2') - self.failUnless(m.closed) + @deferred(timeout=None) def testExpunge(self): """ Test expunge command -- cgit v1.2.3 From ddad3391ba8ad611a9bdaaf689b408d44eec9cc6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 11 Dec 2013 12:11:21 -0400 Subject: consume messages eagerly --- src/leap/mail/imap/tests/test_imap.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index d78115e..9989989 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -394,6 +394,8 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): MessageCollection interface in this particular TestCase """ self.messages = MessageCollection("testmbox", self._soledad._db) + for m in self.messages.get_all(): + self.messages.remove(m) def tearDown(self): """ @@ -423,6 +425,22 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): }) self.assertEqual(self.messages.count(), 0) + def testMultipleAdd(self): + """ + Add multiple messages + """ + # XXX watch out! we're serializing with a delay... + mc = self.messages + self.assertEqual(self.messages.count(), 0) + mc.add_msg('Stuff', subject="test1") + self.assertEqual(self.messages.count(), 1) + mc.add_msg('Stuff', subject="test2") + self.assertEqual(self.messages.count(), 2) + mc.add_msg('Stuff', subject="test3") + self.assertEqual(self.messages.count(), 3) + mc.add_msg('Stuff', subject="test4") + self.assertEqual(self.messages.count(), 4) + def testFilterByMailbox(self): """ Test that queries filter by selected mailbox @@ -1265,8 +1283,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): m = SimpleLEAPServer.theAccount.getMailbox(name) m.messages.add_msg('', subject="Message 1", flags=('\\Deleted', 'AnotherFlag')) + self.failUnless(m.messages.count() == 1) m.messages.add_msg('', subject="Message 2", flags=('AnotherFlag',)) + self.failUnless(m.messages.count() == 2) m.messages.add_msg('', subject="Message 3", flags=('\\Deleted',)) + self.failUnless(m.messages.count() == 3) def login(): return self.client.login('testuser', 'password-test') @@ -1292,7 +1313,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestExpunge, m) def _cbTestExpunge(self, ignored, m): - self.assertEqual(len(m.messages), 1) + # we only left 1 mssage with no deleted flag + self.assertEqual(m.messages.count(), 1) self.assertEqual( m.messages[1].content['subject'], 'Message 2') -- cgit v1.2.3 From d1719ca40f6c7838fb41915706960c822f081237 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 5 Dec 2013 11:24:23 -0400 Subject: count_foo uses expanded u1db count method. Other fixes in the commit: * Correct the semantic for the recent flag (reset) * Minor unicode fixes. * Use a field for tracking the last_uid In general, this tries to squash all the quick and naive methods that were relying on evaluating all the message objects before returning a result. Further work is still needed, planned also for 0.5 release. get_by_index needs to be indexed too. --- src/leap/mail/imap/tests/test_imap.py | 42 +++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 9989989..f87b534 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -370,8 +370,11 @@ class IMAP4HelperMixin(BaseLeapTest): def _ebGeneral(self, failure): self.client.transport.loseConnection() self.server.transport.loseConnection() - log.err(failure, "Problem with %r" % (self.function,)) - failure.trap(Exception) + # can we do something similar? + # I guess this was ok with trial, but not in noseland... + #log.err(failure, "Problem with %r" % (self.function,)) + raise failure.value + #failure.trap(Exception) def loopback(self): return loopback.loopbackAsync(self.server, self.client) @@ -393,7 +396,7 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): We override mixin method since we are only testing MessageCollection interface in this particular TestCase """ - self.messages = MessageCollection("testmbox", self._soledad._db) + self.messages = MessageCollection("testmbox", self._soledad) for m in self.messages.get_all(): self.messages.remove(m) @@ -429,7 +432,6 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): """ Add multiple messages """ - # XXX watch out! we're serializing with a delay... mc = self.messages self.assertEqual(self.messages.count(), 0) mc.add_msg('Stuff', subject="test1") @@ -440,6 +442,38 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): self.assertEqual(self.messages.count(), 3) mc.add_msg('Stuff', subject="test4") self.assertEqual(self.messages.count(), 4) + mc.add_msg('Stuff', subject="test5") + mc.add_msg('Stuff', subject="test6") + mc.add_msg('Stuff', subject="test7") + mc.add_msg('Stuff', subject="test8") + mc.add_msg('Stuff', subject="test9") + mc.add_msg('Stuff', subject="test10") + self.assertEqual(self.messages.count(), 10) + + def testRecentCount(self): + """ + Test the recent count + """ + mc = self.messages + self.assertEqual(self.messages.count_recent(), 0) + mc.add_msg('Stuff', subject="test1", uid=1) + # For the semantics defined in the RFC, we auto-add the + # recent flag by default. + self.assertEqual(self.messages.count_recent(), 1) + mc.add_msg('Stuff', subject="test2", uid=2, flags=('\\Deleted',)) + self.assertEqual(self.messages.count_recent(), 2) + mc.add_msg('Stuff', subject="test3", uid=3, flags=('\\Recent',)) + self.assertEqual(self.messages.count_recent(), 3) + mc.add_msg('Stuff', subject="test4", uid=4, + flags=('\\Deleted', '\\Recent')) + self.assertEqual(self.messages.count_recent(), 4) + + for m in mc: + msg = self.messages.get_msg_by_uid(m.get('uid')) + msg_newflags = msg.removeFlags(('\\Recent',)) + self._soledad.put_doc(msg_newflags) + + self.assertEqual(mc.count_recent(), 0) def testFilterByMailbox(self): """ -- cgit v1.2.3 From 6684735dc24ea02649b55e5fd795c7d2f5824d34 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 24 Dec 2013 09:27:43 -0200 Subject: Fix parsing of IMAP folder names (#4830). --- src/leap/mail/imap/tests/test_imap.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index f87b534..ea75854 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -521,7 +521,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test whether we can create mailboxes """ - succeed = ('testbox', 'test/box', 'test/', 'test/box/box', 'FOOBOX') + succeed = ('testbox', 'test/box', 'test/', 'test/box/box', 'foobox') fail = ('testbox', 'test/box') def cb(): @@ -553,7 +553,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): answers = ['foobox', 'testbox', 'test/box', 'test', 'test/box/box'] mbox.sort() answers.sort() - self.assertEqual(mbox, [a.upper() for a in answers]) + self.assertEqual(mbox, [a for a in answers]) @deferred(timeout=None) def testDelete(self): @@ -686,7 +686,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d.addCallback(lambda _: self.assertEqual( SimpleLEAPServer.theAccount.mailboxes, - ['NEWNAME'])) + ['newname'])) return d @deferred(timeout=None) @@ -742,7 +742,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): mboxes = SimpleLEAPServer.theAccount.mailboxes expected = ['newname', 'newname/m1', 'newname/m2'] mboxes.sort() - self.assertEqual(mboxes, [s.upper() for s in expected]) + self.assertEqual(mboxes, [s for s in expected]) @deferred(timeout=None) def testSubscribe(self): @@ -763,7 +763,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d.addCallback(lambda _: self.assertEqual( SimpleLEAPServer.theAccount.subscriptions, - ['THIS/MBOX'])) + ['this/mbox'])) return d @deferred(timeout=None) @@ -771,8 +771,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test whether we can unsubscribe from a set of mailboxes """ - SimpleLEAPServer.theAccount.subscribe('THIS/MBOX') - SimpleLEAPServer.theAccount.subscribe('THAT/MBOX') + SimpleLEAPServer.theAccount.subscribe('this/mbox') + SimpleLEAPServer.theAccount.subscribe('that/mbox') def login(): return self.client.login('testuser', 'password-test') @@ -788,7 +788,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d.addCallback(lambda _: self.assertEqual( SimpleLEAPServer.theAccount.subscriptions, - ['THAT/MBOX'])) + ['that/mbox'])) return d @deferred(timeout=None) @@ -1029,7 +1029,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestExamine) def _cbTestExamine(self, ignored): - mbox = self.server.theAccount.getMailbox('TEST-MAILBOX-E') + mbox = self.server.theAccount.getMailbox('test-mailbox-e') self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox) self.assertEqual(self.examinedArgs, { 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, @@ -1070,8 +1070,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d.addCallback(lambda listed: self.assertEqual( sortNest(listed), sortNest([ - (SoledadMailbox.INIT_FLAGS, "/", "ROOT/SUBTHINGL"), - (SoledadMailbox.INIT_FLAGS, "/", "ROOT/ANOTHER-THING") + (SoledadMailbox.INIT_FLAGS, "/", "root/subthingl"), + (SoledadMailbox.INIT_FLAGS, "/", "root/another-thing") ]) )) return d @@ -1081,13 +1081,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test LSub command """ - SimpleLEAPServer.theAccount.subscribe('ROOT/SUBTHINGL2') + SimpleLEAPServer.theAccount.subscribe('root/subthingl2') def lsub(): return self.client.lsub('root', '%') d = self._listSetup(lsub) d.addCallback(self.assertEqual, - [(SoledadMailbox.INIT_FLAGS, "/", "ROOT/SUBTHINGL2")]) + [(SoledadMailbox.INIT_FLAGS, "/", "root/subthingl2")]) return d @deferred(timeout=None) @@ -1190,7 +1190,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestFullAppend, infile) def _cbTestFullAppend(self, ignored, infile): - mb = SimpleLEAPServer.theAccount.getMailbox('ROOT/SUBTHING') + mb = SimpleLEAPServer.theAccount.getMailbox('root/subthing') self.assertEqual(1, len(mb.messages)) self.assertEqual( -- cgit v1.2.3 From 5585ff784940dee267576d097076de66797f9188 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 2 Jan 2014 16:08:09 -0400 Subject: fix tests after rewrite --- src/leap/mail/imap/tests/test_imap.py | 196 ++++++++++++++++++++++------------ 1 file changed, 126 insertions(+), 70 deletions(-) (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index ea75854..e1bed8c 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -25,7 +25,7 @@ XXX add authors from the original twisted tests. @license: GPLv3, see included LICENSE file """ # XXX review license of the original tests!!! -from nose.twistedtools import deferred +from email import parser try: from cStringIO import StringIO @@ -36,9 +36,13 @@ import os import types import tempfile import shutil +import time + +from itertools import chain from mock import Mock +from nose.twistedtools import deferred, stop_reactor from twisted.mail import imap4 @@ -58,9 +62,9 @@ import twisted.cred.portal # import u1db from leap.common.testing.basetest import BaseLeapTest -from leap.mail.imap.server import SoledadMailbox -from leap.mail.imap.server import SoledadBackedAccount -from leap.mail.imap.server import MessageCollection +from leap.mail.imap.account import SoledadBackedAccount +from leap.mail.imap.mailbox import SoledadMailbox +from leap.mail.imap.messages import MessageCollection from leap.soledad.client import Soledad from leap.soledad.client import SoledadCrypto @@ -321,6 +325,9 @@ class IMAP4HelperMixin(BaseLeapTest): for mb in self.server.theAccount.mailboxes: self.server.theAccount.delete(mb) + # email parser + self.parser = parser.Parser() + def tearDown(self): """ tearDown method called after each test. @@ -389,6 +396,7 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): """ Tests for the MessageCollection class """ + count = 0 def setUp(self): """ @@ -396,34 +404,35 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): We override mixin method since we are only testing MessageCollection interface in this particular TestCase """ - self.messages = MessageCollection("testmbox", self._soledad) - for m in self.messages.get_all(): - self.messages.remove(m) + self.messages = MessageCollection("testmbox%s" % (self.count,), + self._soledad) + MessageCollectionTestCase.count += 1 def tearDown(self): """ tearDown method for each test - Delete the message collection """ del self.messages + def wait(self): + time.sleep(2) + def testEmptyMessage(self): """ Test empty message and collection """ - em = self.messages._get_empty_msg() + em = self.messages._get_empty_doc() self.assertEqual( em, { - "date": '', "flags": [], - "headers": {}, "mbox": "inbox", - "raw": "", "recent": True, "seen": False, - "subject": "", - "type": "msg", + "deleted": False, + "multi": False, + "size": 0, + "type": "flags", "uid": 1, }) self.assertEqual(self.messages.count(), 0) @@ -432,23 +441,22 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): """ Add multiple messages """ + # TODO really profile addition mc = self.messages + print "messages", self.messages self.assertEqual(self.messages.count(), 0) - mc.add_msg('Stuff', subject="test1") - self.assertEqual(self.messages.count(), 1) - mc.add_msg('Stuff', subject="test2") - self.assertEqual(self.messages.count(), 2) - mc.add_msg('Stuff', subject="test3") - self.assertEqual(self.messages.count(), 3) - mc.add_msg('Stuff', subject="test4") + mc.add_msg('Stuff', uid=1, subject="test1") + mc.add_msg('Stuff', uid=2, subject="test2") + mc.add_msg('Stuff', uid=3, subject="test3") + mc.add_msg('Stuff', uid=4, subject="test4") + self.wait() self.assertEqual(self.messages.count(), 4) - mc.add_msg('Stuff', subject="test5") - mc.add_msg('Stuff', subject="test6") - mc.add_msg('Stuff', subject="test7") - mc.add_msg('Stuff', subject="test8") - mc.add_msg('Stuff', subject="test9") - mc.add_msg('Stuff', subject="test10") - self.assertEqual(self.messages.count(), 10) + mc.add_msg('Stuff', uid=5, subject="test5") + mc.add_msg('Stuff', uid=6, subject="test6") + mc.add_msg('Stuff', uid=7, subject="test7") + self.wait() + self.assertEqual(self.messages.count(), 7) + self.wait() def testRecentCount(self): """ @@ -456,45 +464,48 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): """ mc = self.messages self.assertEqual(self.messages.count_recent(), 0) - mc.add_msg('Stuff', subject="test1", uid=1) + mc.add_msg('Stuff', uid=1, subject="test1") # For the semantics defined in the RFC, we auto-add the # recent flag by default. + self.wait() self.assertEqual(self.messages.count_recent(), 1) - mc.add_msg('Stuff', subject="test2", uid=2, flags=('\\Deleted',)) + mc.add_msg('Stuff', subject="test2", uid=2, + flags=('\\Deleted',)) + self.wait() self.assertEqual(self.messages.count_recent(), 2) - mc.add_msg('Stuff', subject="test3", uid=3, flags=('\\Recent',)) + mc.add_msg('Stuff', subject="test3", uid=3, + flags=('\\Recent',)) + self.wait() self.assertEqual(self.messages.count_recent(), 3) mc.add_msg('Stuff', subject="test4", uid=4, flags=('\\Deleted', '\\Recent')) + self.wait() self.assertEqual(self.messages.count_recent(), 4) - for m in mc: - msg = self.messages.get_msg_by_uid(m.get('uid')) - msg_newflags = msg.removeFlags(('\\Recent',)) - self._soledad.put_doc(msg_newflags) - + for msg in mc: + msg.removeFlags(('\\Recent',)) self.assertEqual(mc.count_recent(), 0) def testFilterByMailbox(self): """ Test that queries filter by selected mailbox """ + def wait(): + time.sleep(1) + mc = self.messages self.assertEqual(self.messages.count(), 0) - mc.add_msg('', subject="test1") - self.assertEqual(self.messages.count(), 1) - mc.add_msg('', subject="test2") - self.assertEqual(self.messages.count(), 2) - mc.add_msg('', subject="test3") + mc.add_msg('', uid=1, subject="test1") + mc.add_msg('', uid=2, subject="test2") + mc.add_msg('', uid=3, subject="test3") + wait() self.assertEqual(self.messages.count(), 3) - - newmsg = mc._get_empty_msg() + newmsg = mc._get_empty_doc() newmsg['mailbox'] = "mailbox/foo" - newmsg['subject'] = "test another mailbox" mc._soledad.create_doc(newmsg) self.assertEqual(mc.count(), 3) self.assertEqual( - len(mc._soledad.get_from_index(mc.TYPE_IDX, "*")), 4) + len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4) class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): @@ -1174,16 +1185,20 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def login(): return self.client.login('testuser', 'password-test') + def wait(): + time.sleep(0.5) + def append(): return self.client.append( 'root/subthing', message, - ['\\SEEN', '\\DELETED'], + ('\\SEEN', '\\DELETED'), 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', ) d1 = self.connected.addCallback(strip(login)) d1.addCallbacks(strip(append), self._ebGeneral) + d1.addCallbacks(strip(wait), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) @@ -1191,17 +1206,31 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestFullAppend(self, ignored, infile): mb = SimpleLEAPServer.theAccount.getMailbox('root/subthing') + time.sleep(0.5) self.assertEqual(1, len(mb.messages)) + msg = mb.messages.get_msg_by_uid(1) self.assertEqual( - ['\\SEEN', '\\DELETED'], - mb.messages[1].content['flags']) + ('\\SEEN', '\\DELETED'), + msg.getFlags()) self.assertEqual( 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', - mb.messages[1].content['date']) + msg.getInternalDate()) + + parsed = self.parser.parse(open(infile)) + body = parsed.get_payload() + headers = parsed.items() + self.assertEqual( + body, + msg.getBodyFile().read()) + + msg_headers = msg.getHeaders(True, "",) + gotheaders = list(chain( + *[[(k, item) for item in v] for (k, v) in msg_headers.items()])) - self.assertEqual(open(infile).read(), mb.messages[1].content['raw']) + self.assertItemsEqual( + headers, gotheaders) @deferred(timeout=None) def testPartialAppend(self): @@ -1209,12 +1238,14 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Test partially appending a message to the mailbox """ infile = util.sibpath(__file__, 'rfc822.message') - message = open(infile) SimpleLEAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') def login(): return self.client.login('testuser', 'password-test') + def wait(): + time.sleep(1) + def append(): message = file(infile) return self.client.sendCommand( @@ -1226,6 +1257,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): ) ) d1 = self.connected.addCallback(strip(login)) + d1.addCallbacks(strip(wait), self._ebGeneral) d1.addCallbacks(strip(append), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -1235,15 +1267,20 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestPartialAppend(self, ignored, infile): mb = SimpleLEAPServer.theAccount.getMailbox('PARTIAL/SUBTHING') - + time.sleep(1) self.assertEqual(1, len(mb.messages)) + msg = mb.messages.get_msg_by_uid(1) self.assertEqual( - ['\\SEEN', ], - mb.messages[1].content['flags'] + ('\\SEEN', ), + msg.getFlags() ) + #self.assertEqual( + #'Right now', msg.getInternalDate()) + parsed = self.parser.parse(open(infile)) + body = parsed.get_payload() self.assertEqual( - 'Right now', mb.messages[1].content['date']) - self.assertEqual(open(infile).read(), mb.messages[1].content['raw']) + body, + msg.getBodyFile().read()) @deferred(timeout=None) def testCheck(self): @@ -1279,14 +1316,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.server.theAccount.addMailbox(name) m = SimpleLEAPServer.theAccount.getMailbox(name) - m.messages.add_msg('', subject="Message 1", + m.messages.add_msg('test 1', uid=1, subject="Message 1", flags=('\\Deleted', 'AnotherFlag')) - m.messages.add_msg('', subject="Message 2", flags=('AnotherFlag',)) - m.messages.add_msg('', subject="Message 3", flags=('\\Deleted',)) + m.messages.add_msg('test 2', uid=2, subject="Message 2", + flags=('AnotherFlag',)) + m.messages.add_msg('test 3', uid=3, subject="Message 3", + flags=('\\Deleted',)) def login(): return self.client.login('testuser', 'password-test') + def wait(): + time.sleep(1) + def select(): return self.client.select(name) @@ -1294,6 +1336,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.close() d = self.connected.addCallback(strip(login)) + d.addCallbacks(strip(wait), self._ebGeneral) d.addCallbacks(strip(select), self._ebGeneral) d.addCallbacks(strip(close), self._ebGeneral) d.addCallbacks(self._cbStopClient, self._ebGeneral) @@ -1302,8 +1345,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestClose(self, ignored, m): self.assertEqual(len(m.messages), 1) + messages = [msg for msg in m.messages] + self.assertFalse(messages[0] is None) self.assertEqual( - m.messages[1].content['subject'], + messages[0]._hdoc.content['subject'], 'Message 2') self.failUnless(m.closed) @@ -1315,17 +1360,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): name = 'mailbox-expunge' SimpleLEAPServer.theAccount.addMailbox(name) m = SimpleLEAPServer.theAccount.getMailbox(name) - m.messages.add_msg('', subject="Message 1", + m.messages.add_msg('test 1', uid=1, subject="Message 1", flags=('\\Deleted', 'AnotherFlag')) - self.failUnless(m.messages.count() == 1) - m.messages.add_msg('', subject="Message 2", flags=('AnotherFlag',)) - self.failUnless(m.messages.count() == 2) - m.messages.add_msg('', subject="Message 3", flags=('\\Deleted',)) - self.failUnless(m.messages.count() == 3) + m.messages.add_msg('test 2', uid=2, subject="Message 2", + flags=('AnotherFlag',)) + m.messages.add_msg('test 3', uid=3, subject="Message 3", + flags=('\\Deleted',)) def login(): return self.client.login('testuser', 'password-test') + def wait(): + time.sleep(2) + def select(): return self.client.select('mailbox-expunge') @@ -1338,6 +1385,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.results = None d1 = self.connected.addCallback(strip(login)) + d1.addCallbacks(strip(wait), self._ebGeneral) d1.addCallbacks(strip(select), self._ebGeneral) d1.addCallbacks(strip(expunge), self._ebGeneral) d1.addCallbacks(expunged, self._ebGeneral) @@ -1348,12 +1396,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestExpunge(self, ignored, m): # we only left 1 mssage with no deleted flag - self.assertEqual(m.messages.count(), 1) + self.assertEqual(len(m.messages), 1) + messages = [msg for msg in m.messages] self.assertEqual( - m.messages[1].content['subject'], + messages[0]._hdoc.content['subject'], 'Message 2') - self.assertEqual(self.results, [0, 1]) - # XXX fix this thing with the indexes... + # the uids of the deleted messages + self.assertItemsEqual(self.results, [1, 3]) class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase): @@ -1363,3 +1412,10 @@ class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase): """ # XXX coming soon to your screens! pass + + +def tearDownModule(): + """ + Tear down functions for module level + """ + stop_reactor() -- cgit v1.2.3 From e1946f1653dbbb6fcf61569bc873ab061965664e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 6 Jan 2014 04:44:05 -0400 Subject: tests infrastructure for multipart --- .../mail/imap/tests/rfc822.multi-signed.message | 238 +++++++++++++++++++++ src/leap/mail/imap/tests/rfc822.multi.message | 96 +++++++++ src/leap/mail/imap/tests/test_imap.py | 78 ++++++- 3 files changed, 407 insertions(+), 5 deletions(-) create mode 100644 src/leap/mail/imap/tests/rfc822.multi-signed.message create mode 100644 src/leap/mail/imap/tests/rfc822.multi.message (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/rfc822.multi-signed.message b/src/leap/mail/imap/tests/rfc822.multi-signed.message new file mode 100644 index 0000000..9907c2d --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi-signed.message @@ -0,0 +1,238 @@ +Date: Mon, 6 Jan 2014 04:40:47 -0400 +From: Kali Kaneko +To: penguin@example.com +Subject: signed message +Message-ID: <20140106084047.GA21317@samsara.lan> +MIME-Version: 1.0 +Content-Type: multipart/signed; micalg=pgp-sha1; + protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy" +Content-Disposition: inline +User-Agent: Mutt/1.5.21 (2012-12-30) + + +--z9ECzHErBrwFF8sy +Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l" +Content-Disposition: inline + + +--z0eOaCaDLjvTGF2l +Content-Type: text/plain; charset=utf-8 +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable + +This is an example of a signed message, +with attachments. + + +--=20 +Nihil sine chao! =E2=88=B4 + +--z0eOaCaDLjvTGF2l +Content-Type: text/plain; charset=us-ascii +Content-Disposition: attachment; filename="attach.txt" + +this is attachment in plain text. + +--z0eOaCaDLjvTGF2l +Content-Type: application/octet-stream +Content-Disposition: attachment; filename="hack.ico" +Content-Transfer-Encoding: base64 + +AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA +KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG +RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA +PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl +5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA +/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ +yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A +Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK +ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK +LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP +QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy +AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs +AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA +AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA +gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d +HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA +x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7 ++wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA +AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5 ++QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA +OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK +igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA +JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra +2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA +xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj +owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB +AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA +AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d +XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d +XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA +AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB +AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm +X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC +AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B +bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ +S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu +J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y +AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N +KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB +XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A +AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA +AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d +XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA +AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr +RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA +AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A +Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI +yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA +CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys +rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA +vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d +HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA +urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx +cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA +CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo +6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA +2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7 +OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA +UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp +qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA +lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa +WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB +AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB +AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB +AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA +ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA +AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB +AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB +AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB +AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB +AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA +tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA +AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF +wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB +AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2 +RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB +AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB +AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA +AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd +AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB +AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB +AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB +AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB +AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB +AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8 +ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2 +NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF +RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB +lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA +AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa +WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA +AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX +AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB +AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB +AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA +AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA +AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA +AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA +AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB +AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB +AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA +ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA +AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7 +LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA +AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF +NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB +AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2 +RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA +ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4 +RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi +JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2 +NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK +T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB +AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB +AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB +AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN +UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA +AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA +W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA +AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB +l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB +AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+ +WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA +AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv +RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA +AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj +AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB +AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB +AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA +AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA +AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA +dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A +AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB +AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB +AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB +AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW +pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + +--z0eOaCaDLjvTGF2l-- + +--z9ECzHErBrwFF8sy +Content-Type: application/pgp-signature + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.15 (GNU/Linux) + +iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv +kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl +vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK +PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC +w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw +sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr +BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN +QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt +mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ +jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8 +gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X +sSdfcAhT7Tno7PB/Acoh +=+okv +-----END PGP SIGNATURE----- + +--z9ECzHErBrwFF8sy-- diff --git a/src/leap/mail/imap/tests/rfc822.multi.message b/src/leap/mail/imap/tests/rfc822.multi.message new file mode 100644 index 0000000..30f74e5 --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi.message @@ -0,0 +1,96 @@ +Date: Fri, 19 May 2000 09:55:48 -0400 (EDT) +From: Doug Sauder +To: Joe Blow +Subject: Test message from PINE +Message-ID: +MIME-Version: 1.0 +Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452" + + This message is in MIME format. The first part should be readable text, + while the remaining parts are likely unreadable without MIME-aware tools. + Send mail to mime@docserver.cac.washington.edu for more info. + +---1463757054-952513540-958744548=:8452 +Content-Type: TEXT/PLAIN; charset=US-ASCII + +This is a test message from PINE MUA. + + +---1463757054-952513540-958744548=:8452 +Content-Type: APPLICATION/octet-stream; name="redball.png" +Content-Transfer-Encoding: BASE64 +Content-ID: +Content-Description: A PNG graphic file +Content-Disposition: attachment; filename="redball.png" + +iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A +AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj +AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv +AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0 +GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf +hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl +rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL +ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS +AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP +AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP +AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI +AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR +AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6 +AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO +AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8 +AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8 +LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu +MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF +BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp +6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow +8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G +ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn +OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1 +a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T +VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8 +Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f +lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/ +joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms +1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA +JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7 +vAAAAABJRU5ErkJggg== +---1463757054-952513540-958744548=:8452 +Content-Type: APPLICATION/octet-stream; name="blueball.png" +Content-Transfer-Encoding: BASE64 +Content-ID: +Content-Description: A PNG graphic file +Content-Disposition: attachment; filename="blueball.png" + +iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A +AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI +IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y +Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S +hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM +vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB +Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu +MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b +fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo +Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp +LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX +P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M +jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8 ++VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE +1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42 +YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY +mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp +Z3VldDZzO7wAAAAASUVORK5CYII= +---1463757054-952513540-958744548=:8452-- diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index e1bed8c..8c1cf20 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -357,11 +357,11 @@ class IMAP4HelperMixin(BaseLeapTest): # XXX we also should put this in a mailbox! - self._soledad.messages.add_msg('', subject="test1") - self._soledad.messages.add_msg('', subject="test2") - self._soledad.messages.add_msg('', subject="test3") + self._soledad.messages.add_msg('', uid=1, subject="test1") + self._soledad.messages.add_msg('', uid=2, subject="test2") + self._soledad.messages.add_msg('', uid=3, subject="test3") # XXX should change Flags too - self._soledad.messages.add_msg('', subject="test4") + self._soledad.messages.add_msg('', uid=4, subject="test4") def delete_all_docs(self): """ @@ -1405,10 +1405,78 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.assertItemsEqual(self.results, [1, 3]) +class StoreAndFetchTestCase(unittest.TestCase, IMAP4HelperMixin): + """ + Several tests to check that the internal storage representation + is able to render the message structures as we expect them. + """ + # TODO get rid of the fucking sleeps with a proper defer + # management. + + def setUp(self): + IMAP4HelperMixin.setUp(self) + MBOX_NAME = "multipart/SIGNED" + self.received_messages = self.received_uid = None + self.result = None + + self.server.state = 'select' + + infile = util.sibpath(__file__, 'rfc822.multi-signed.message') + raw = open(infile).read() + + self.server.theAccount.addMailbox(MBOX_NAME) + mbox = self.server.theAccount.getMailbox(MBOX_NAME) + time.sleep(1) + self.server.mbox = mbox + self.server.mbox.messages.add_msg(raw, uid=1) + time.sleep(1) + + def addListener(self, x): + pass + + def removeListener(self, x): + pass + + def _fetchWork(self, uids): + + def result(R): + self.result = R + + self.connected.addCallback( + lambda _: self.function( + uids, uid=1) # do NOT use seq numbers! + ).addCallback(result).addCallback( + self._cbStopClient).addErrback(self._ebGeneral) + + d = loopback.loopbackTCP(self.server, self.client, noisy=False) + d.addCallback(lambda x: self.assertEqual(self.result, self.expected)) + return d + + @deferred(timeout=None) + def testMultiBody(self): + """ + Test that a multipart signed message is retrieved the same + as we stored it. + """ + time.sleep(1) + self.function = self.client.fetchBody + messages = '1' + + # XXX review. This probably should give everything? + + self.expected = {1: { + 'RFC822.TEXT': 'This is an example of a signed message,\n' + 'with attachments.\n\n\n--=20\n' + 'Nihil sine chao! =E2=88=B4\n', + 'UID': '1'}} + print "test multi: fetch uid", messages + return self._fetchWork(messages) + + class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase): """ - Tests for the behavior of the search_* functions in L{imap4.IMAP4Server}. + Tests for the behavior of the search_* functions in L{imap5.IMAP4Server}. """ # XXX coming soon to your screens! pass -- cgit v1.2.3 From 4ba5d5b405e3c6a6bc997df2073ffc8ea3fa75a9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 7 Jan 2014 11:34:08 -0400 Subject: Second stage of the new year's storage rewrite. * documents of only three types: * flags * headers * content * add algorithm for walking the parsed message tree. * treat special cases like a multipart with a single part. * modify add_msg to use the walk routine * modify twisted interfaces to use the new storage schema. * tests for different multipart cases * fix multipart detection typo in the fetch This is a merge proposal for the 0.5.0-rc3. known bugs ---------- Some things are still know not to work well at this point (some cases of multipart messages do not display the bodies). IMAP server also is left in a bad internal state after a logout/login. --- .../mail/imap/tests/rfc822.multi-minimal.message | 16 +++ src/leap/mail/imap/tests/rfc822.plain.message | 66 ++++++++++++ src/leap/mail/imap/tests/walktree.py | 117 +++++++++++++++++++++ 3 files changed, 199 insertions(+) create mode 100644 src/leap/mail/imap/tests/rfc822.multi-minimal.message create mode 100644 src/leap/mail/imap/tests/rfc822.plain.message create mode 100644 src/leap/mail/imap/tests/walktree.py (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/rfc822.multi-minimal.message b/src/leap/mail/imap/tests/rfc822.multi-minimal.message new file mode 100644 index 0000000..582297c --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi-minimal.message @@ -0,0 +1,16 @@ +Content-Type: multipart/mixed; boundary="===============6203542367371144092==" +MIME-Version: 1.0 +Subject: [TEST] 010 - Inceptos cum lorem risus congue +From: testmailbitmaskspam@gmail.com +To: test_c5@dev.bitmask.net + +--===============6203542367371144092== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +Howdy from python! +The subject: [TEST] 010 - Inceptos cum lorem risus congue +Current date & time: Wed Jan 8 16:36:21 2014 +Trying to attach: [] +--===============6203542367371144092==-- diff --git a/src/leap/mail/imap/tests/rfc822.plain.message b/src/leap/mail/imap/tests/rfc822.plain.message new file mode 100644 index 0000000..fc627c3 --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.plain.message @@ -0,0 +1,66 @@ +From pyar-bounces@python.org.ar Wed Jan 8 14:46:02 2014 +Return-Path: +X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net +X-Spam-Level: ** +X-Spam-Pyzor: Reported 0 times. +X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE, + CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP, + NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled + version=3.3.2 +Delivered-To: kali@leap.se +Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33]) + (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) + (Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified)) + by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F + for ; Wed, 8 Jan 2014 18:46:02 +0000 (UTC) +Received: from pyar.usla.org.ar (unknown [190.228.30.157]) + by mx1.riseup.net (Postfix) with ESMTP id F244C533F4 + for ; Wed, 8 Jan 2014 10:46:01 -0800 (PST) +Received: from [127.0.0.1] (localhost [127.0.0.1]) + by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F + for ; Wed, 8 Jan 2014 15:46:00 -0300 (ART) +MIME-Version: 1.0 +Content-Type: text/plain; charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable +From: pyar-request@python.org.ar +To: kali@leap.se +Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 +Reply-To: pyar-request@python.org.ar +Auto-Submitted: auto-replied +Message-ID: +Date: Wed, 08 Jan 2014 15:45:59 -0300 +Precedence: bulk +X-BeenThere: pyar@python.org.ar +X-Mailman-Version: 2.1.15 +List-Id: Python Argentina +X-List-Administrivia: yes +Errors-To: pyar-bounces@python.org.ar +Sender: "pyar" +X-Virus-Scanned: clamav-milter 0.97.8 at mx1 +X-Virus-Status: Clean + +Mailing list subscription confirmation notice for mailing list pyar + +We have received a request de kaliyuga@riseup.net for subscription of +your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar +mailing list. To confirm that you want to be added to this mailing +list, simply reply to this message, keeping the Subject: header +intact. Or visit this web page: + + http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b= +3377148ac2 + + +Or include the following line -- and only the following line -- in a +message to pyar-request@python.org.ar: + + confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 + +Note that simply sending a `reply' to this message should work from +most mail readers, since that usually leaves the Subject: line in the +right form (additional "Re:" text in the Subject: is okay). + +If you do not wish to be subscribed to this list, please simply +disregard this message. If you think you are being maliciously +subscribed to the list, or have any other questions, send them to +pyar-owner@python.org.ar. diff --git a/src/leap/mail/imap/tests/walktree.py b/src/leap/mail/imap/tests/walktree.py new file mode 100644 index 0000000..1626f65 --- /dev/null +++ b/src/leap/mail/imap/tests/walktree.py @@ -0,0 +1,117 @@ +#t -*- coding: utf-8 -*- +# walktree.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the walktree module. +""" +import os +from email import parser + +from leap.mail import walk as W + +DEBUG = os.environ.get("BITMASK_MAIL_DEBUG") + +p = parser.Parser() + +# TODO pass an argument of the type of message + +################################################## +# Input from hell + +#msg = p.parse(open('rfc822.multi-signed.message')) +#msg = p.parse(open('rfc822.plain.message')) +msg = p.parse(open('rfc822.multi-minimal.message')) +DO_CHECK = False +################################################# + +parts = W.get_parts(msg) + +if DEBUG: + def trim(item): + item = item[:10] + [trim(part["phash"]) for part in parts if part.get('phash', None)] + +raw_docs = list(W.get_raw_docs(msg, parts)) + +body_phash_fun = [W.get_body_phash_simple, + W.get_body_phash_multi][int(msg.is_multipart())] +body_phash = body_phash_fun(W.get_payloads(msg)) +parts_map = W.walk_msg_tree(parts, body_phash=body_phash) + + +# TODO add missing headers! +expected = { + 'body': '1ddfa80485', + 'multi': True, + 'part_map': { + 1: { + 'headers': {'Content-Disposition': 'inline', + 'Content-Type': 'multipart/mixed; ' + 'boundary="z0eOaCaDLjvTGF2l"'}, + 'multi': True, + 'part_map': {1: {'ctype': 'text/plain', + 'headers': [ + ('Content-Type', + 'text/plain; charset=utf-8'), + ('Content-Disposition', + 'inline'), + ('Content-Transfer-Encoding', + 'quoted-printable')], + 'multi': False, + 'parts': 1, + 'phash': '1ddfa80485', + 'size': 206}, + 2: {'ctype': 'text/plain', + 'headers': [('Content-Type', + 'text/plain; charset=us-ascii'), + ('Content-Disposition', + 'attachment; ' + 'filename="attach.txt"')], + 'multi': False, + 'parts': 1, + 'phash': '7a94e4d769', + 'size': 133}, + 3: {'ctype': 'application/octet-stream', + 'headers': [('Content-Type', + 'application/octet-stream'), + ('Content-Disposition', + 'attachment; filename="hack.ico"'), + ('Content-Transfer-Encoding', + 'base64')], + 'multi': False, + 'parts': 1, + 'phash': 'c42cccebbd', + 'size': 12736}}}, + 2: {'ctype': 'application/pgp-signature', + 'headers': [('Content-Type', 'application/pgp-signature')], + 'multi': False, + 'parts': 1, + 'phash': '8f49fbf749', + 'size': 877}}} + +if DEBUG and DO_CHECK: + # TODO turn this into a proper unittest + assert(parts_map == expected) + print "Structure: OK" + + +import pprint +print +print "RAW DOCS" +pprint.pprint(raw_docs) +print +print "PARTS MAP" +pprint.pprint(parts_map) -- cgit v1.2.3 From 28236ecdf46360acd09a53130646e2a082583de6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 8 Jan 2014 22:24:46 -0400 Subject: add a quick message fetching utility for tests --- src/leap/mail/imap/tests/getmail | 282 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100755 src/leap/mail/imap/tests/getmail (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/getmail b/src/leap/mail/imap/tests/getmail new file mode 100755 index 0000000..17e195c --- /dev/null +++ b/src/leap/mail/imap/tests/getmail @@ -0,0 +1,282 @@ +#!/usr/bin/env python + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE in twisted for details. + +# Modifications by LEAP Developers 2014 to fit +# Bitmask configuration settings. + + +""" +Simple IMAP4 client which displays the subjects of all messages in a +particular mailbox. +""" + +import sys + +from twisted.internet import protocol +from twisted.internet import ssl +from twisted.internet import defer +from twisted.internet import stdio +from twisted.mail import imap4 +from twisted.protocols import basic +from twisted.python import log + + +class TrivialPrompter(basic.LineReceiver): + from os import linesep as delimiter + + promptDeferred = None + + def prompt(self, msg): + assert self.promptDeferred is None + self.display(msg) + self.promptDeferred = defer.Deferred() + return self.promptDeferred + + def display(self, msg): + self.transport.write(msg) + + def lineReceived(self, line): + if self.promptDeferred is None: + return + d, self.promptDeferred = self.promptDeferred, None + d.callback(line) + + +class SimpleIMAP4Client(imap4.IMAP4Client): + """ + A client with callbacks for greeting messages from an IMAP server. + """ + greetDeferred = None + + def serverGreeting(self, caps): + self.serverCapabilities = caps + if self.greetDeferred is not None: + d, self.greetDeferred = self.greetDeferred, None + d.callback(self) + + +class SimpleIMAP4ClientFactory(protocol.ClientFactory): + usedUp = False + + protocol = SimpleIMAP4Client + + def __init__(self, username, onConn): + self.ctx = ssl.ClientContextFactory() + + self.username = username + self.onConn = onConn + + def buildProtocol(self, addr): + """ + Initiate the protocol instance. Since we are building a simple IMAP + client, we don't bother checking what capabilities the server has. We + just add all the authenticators twisted.mail has. Note: Gmail no + longer uses any of the methods below, it's been using XOAUTH since + 2010. + """ + assert not self.usedUp + self.usedUp = True + + p = self.protocol(self.ctx) + p.factory = self + p.greetDeferred = self.onConn + + p.registerAuthenticator(imap4.PLAINAuthenticator(self.username)) + p.registerAuthenticator(imap4.LOGINAuthenticator(self.username)) + p.registerAuthenticator( + imap4.CramMD5ClientAuthenticator(self.username)) + + return p + + def clientConnectionFailed(self, connector, reason): + d, self.onConn = self.onConn, None + d.errback(reason) + + +def cbServerGreeting(proto, username, password): + """ + Initial callback - invoked after the server sends us its greet message. + """ + # Hook up stdio + tp = TrivialPrompter() + stdio.StandardIO(tp) + + # And make it easily accessible + proto.prompt = tp.prompt + proto.display = tp.display + + # Try to authenticate securely + return proto.authenticate( + password).addCallback( + cbAuthentication, + proto).addErrback( + ebAuthentication, proto, username, password + ) + + +def ebConnection(reason): + """ + Fallback error-handler. If anything goes wrong, log it and quit. + """ + log.startLogging(sys.stdout) + log.err(reason) + return reason + + +def cbAuthentication(result, proto): + """ + Callback after authentication has succeeded. + + Lists a bunch of mailboxes. + """ + return proto.list("", "*" + ).addCallback(cbMailboxList, proto + ) + + +def ebAuthentication(failure, proto, username, password): + """ + Errback invoked when authentication fails. + + If it failed because no SASL mechanisms match, offer the user the choice + of logging in insecurely. + + If you are trying to connect to your Gmail account, you will be here! + """ + failure.trap(imap4.NoSupportedAuthentication) + return InsecureLogin(proto, username, password) + + +def InsecureLogin(proto, username, password): + """ + insecure-login. + """ + return proto.login(username, password + ).addCallback(cbAuthentication, proto + ) + + +def cbMailboxList(result, proto): + """ + Callback invoked when a list of mailboxes has been retrieved. + """ + result = [e[2] for e in result] + s = '\n'.join(['%d. %s' % (n + 1, m) for (n, m) in zip(range(len(result)), result)]) + if not s: + return defer.fail(Exception("No mailboxes exist on server!")) + return proto.prompt(s + "\nWhich mailbox? [1] " + ).addCallback(cbPickMailbox, proto, result + ) + + +def cbPickMailbox(result, proto, mboxes): + """ + When the user selects a mailbox, "examine" it. + """ + mbox = mboxes[int(result or '1') - 1] + return proto.examine(mbox + ).addCallback(cbExamineMbox, proto + ) + + +def cbExamineMbox(result, proto): + """ + Callback invoked when examine command completes. + + Retrieve the subject header of every message in the mailbox. + """ + return proto.fetchSpecific('1:*', + headerType='HEADER.FIELDS', + headerArgs=['SUBJECT'], + ).addCallback(cbFetch, proto, + ) + + +def cbFetch(result, proto): + """ + Display headers. + """ + if result: + keys = result.keys() + keys.sort() + for k in keys: + proto.display('%s %s' % (k, result[k][0][2])) + else: + print "Hey, an empty mailbox!" + + return proto.prompt("\nWhich message? [1] (Q quits) " + ).addCallback(cbPickMessage, proto) + + +def cbPickMessage(result, proto): + """ + Pick a message. + """ + if result == "Q": + print "Bye!" + return proto.logout() + + return proto.fetchSpecific( + '%s' % result, + headerType='', + headerArgs=['BODY.PEEK[]'], + ).addCallback(cbShowmessage, proto) + + +def cbShowmessage(result, proto): + """ + Display message. + """ + if result: + keys = result.keys() + keys.sort() + for k in keys: + proto.display('%s %s' % (k, result[k][0][2])) + else: + print "Hey, an empty message!" + + return proto.logout() + + +def cbClose(result): + """ + Close the connection when we finish everything. + """ + from twisted.internet import reactor + reactor.stop() + + +def main(): + import sys + + if len(sys.argv) != 3: + print "Usage: getmail " + sys.exit() + + hostname = "localhost" + port = "1984" + username = sys.argv[1] + password = sys.argv[2] + + onConn = defer.Deferred( + ).addCallback(cbServerGreeting, username, password + ).addErrback(ebConnection + ).addBoth(cbClose) + + factory = SimpleIMAP4ClientFactory(username, onConn) + + from twisted.internet import reactor + if port == '993': + reactor.connectSSL( + hostname, int(port), factory, ssl.ClientContextFactory()) + else: + if not port: + port = 143 + reactor.connectTCP(hostname, int(port), factory) + reactor.run() + + +if __name__ == '__main__': + main() -- cgit v1.2.3 From 05c51ee2af6af6025b01831fd3b395cad80fe9f1 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 26 Jan 2014 18:09:51 -0400 Subject: Script for reproducible imaptest runs. --- src/leap/mail/imap/tests/.gitignore | 1 + src/leap/mail/imap/tests/leap_tests_imap.zsh | 160 +++++++++++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 src/leap/mail/imap/tests/.gitignore create mode 100755 src/leap/mail/imap/tests/leap_tests_imap.zsh (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/.gitignore b/src/leap/mail/imap/tests/.gitignore new file mode 100644 index 0000000..60baa9c --- /dev/null +++ b/src/leap/mail/imap/tests/.gitignore @@ -0,0 +1 @@ +data/* diff --git a/src/leap/mail/imap/tests/leap_tests_imap.zsh b/src/leap/mail/imap/tests/leap_tests_imap.zsh new file mode 100755 index 0000000..7ba408c --- /dev/null +++ b/src/leap/mail/imap/tests/leap_tests_imap.zsh @@ -0,0 +1,160 @@ +#!/bin/zsh +# BATCH STRESS TEST FOR IMAP ---------------------- +# http://imgs.xkcd.com/comics/science.jpg +# +# Run imaptest against a LEAP IMAP server +# for a fixed period of time, and collect output. +# +# Author: Kali Kaneko +# Date: 2014 01 26 +# +# To run, you need to have `imaptest` in your path. +# See: +# http://www.imapwiki.org/ImapTest/Installation +# +# For the tests, I'm using a 10MB file sample that +# can be downloaded from: +# http://www.dovecot.org/tmp/dovecot-crlf +# +# Want to contribute to benchmarking? +# +# 1. Create a pristine account in a bitmask provider. +# +# 2. Launch your bitmask client, with different flags +# if you desire. +# +# For example to try the nosync flag in sqlite: +# +# LEAP_SQLITE_NOSYNC=1 bitmask --debug -N --offline -l /tmp/leap.log +# +# 3. Run at several points in time (ie: just after +# launching the bitmask client. one minute after, +# ten minutes after) +# +# mkdir data +# cd data +# ../leap_tests_imap.zsh | tee sqlite_nosync_run2.log +# +# 4. Submit your results to: kali at leap dot se +# together with the logs of the bitmask run. +# +# Please provide also details about your system, and +# the type of hard disk setup you are running against. +# +# +# Edit these variables -------------------------- + +USER="test_f14@dev.bitmask.net" +MBOX="~/leap/imaptest/data/dovecot-crlf" + +HOST="localhost" +PORT="1984" + +IMAPTEST="imaptest" +GREP="/bin/grep" + +# ----------------------------------------------- +# +# These should be kept constant across benchmarking +# runs across different machines, for comparability. + +DURATION=100 +NUM_MSG=200 + + +# TODO add another function, and a cli flag, to be able +# to take several aggretates spaced in time, along a period +# of several minutes. + +imaptest_cmd() { + stdbuf -o0 ${IMAPTEST} user=${USER} pass=1234 host=${HOST} \ + port=${PORT} mbox=${MBOX} clients=1 msgs=${NUM_MSG} \ + no_pipelining 2>/dev/null +} + +stress_imap() { + mknod imap_pipe p + cat imap_pipe | tee output & + imaptest_cmd >> imap_pipe +} + +wait_and_kill() { + while : + do + sleep $DURATION + pkill -2 imaptest + rm imap_pipe + break + done +} + +print_results() { + sleep 1 + echo + echo + echo "AGGREGATED RESULTS" + echo "----------------------" + echo "\tavg\tstdev" + $GREP "avg" ./output | sed -e 's/^ *//g' -e 's/ *$//g' | \ + awk ' +function avg(data, count) { + sum=0; + for( x=0; x <= count-1; x++) { + sum += data[x]; + } + return sum/count; +} +function std_dev(data, count) { + sum=0; + for( x=0; x <= count-1; x++) { + sum += data[x]; + } + average = sum/count; + + sumsq=0; + for( x=0; x <= count-1; x++) { + sumsq += (data[x] - average)^2; + } + return sqrt(sumsq/count); +} +BEGIN { + cnt = 0 +} END { + +printf("LOGI:\t%04.2lf\t%04.2f\n", avg(array[1], NR), std_dev(array[1], NR)); +printf("LIST:\t%04.2lf\t%04.2f\n", avg(array[2], NR), std_dev(array[2], NR)); +printf("STAT:\t%04.2lf\t%04.2f\n", avg(array[3], NR), std_dev(array[3], NR)); +printf("SELE:\t%04.2lf\t%04.2f\n", avg(array[4], NR), std_dev(array[4], NR)); +printf("FETC:\t%04.2lf\t%04.2f\n", avg(array[5], NR), std_dev(array[5], NR)); +printf("FET2:\t%04.2lf\t%04.2f\n", avg(array[6], NR), std_dev(array[6], NR)); +printf("STOR:\t%04.2lf\t%04.2f\n", avg(array[7], NR), std_dev(array[7], NR)); +printf("DELE:\t%04.2lf\t%04.2f\n", avg(array[8], NR), std_dev(array[8], NR)); +printf("EXPU:\t%04.2lf\t%04.2f\n", avg(array[9], NR), std_dev(array[9], NR)); +printf("APPE:\t%04.2lf\t%04.2f\n", avg(array[10], NR), std_dev(array[10], NR)); +printf("LOGO:\t%04.2lf\t%04.2f\n", avg(array[11], NR), std_dev(array[11], NR)); + +print "" +print "TOT samples", NR; +} +{ + it = cnt++; + array[1][it] = $1; + array[2][it] = $2; + array[3][it] = $3; + array[4][it] = $4; + array[5][it] = $5; + array[6][it] = $6; + array[7][it] = $7; + array[8][it] = $8; + array[9][it] = $9; + array[10][it] = $10; + array[11][it] = $11; +}' +} + + +echo "[+] LEAP IMAP TESTS" +echo "[+] Running imaptest for $DURATION seconds with $NUM_MSG messages" +wait_and_kill & +stress_imap +print_results -- cgit v1.2.3 From 36baa3a0e93df779145515f879ed3efdab014bea Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 26 Jan 2014 20:27:53 -0400 Subject: Allow passing user and mbox as parameters Increase default testing duration to 200 secs. --- src/leap/mail/imap/tests/leap_tests_imap.zsh | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/leap_tests_imap.zsh b/src/leap/mail/imap/tests/leap_tests_imap.zsh index 7ba408c..676d1a8 100755 --- a/src/leap/mail/imap/tests/leap_tests_imap.zsh +++ b/src/leap/mail/imap/tests/leap_tests_imap.zsh @@ -41,8 +41,10 @@ # Please provide also details about your system, and # the type of hard disk setup you are running against. # -# -# Edit these variables -------------------------- + +# ------------------------------------------------ +# Edit these variables if you are too lazy to pass +# the user and mbox as parameters. Like me. USER="test_f14@dev.bitmask.net" MBOX="~/leap/imaptest/data/dovecot-crlf" @@ -50,15 +52,16 @@ MBOX="~/leap/imaptest/data/dovecot-crlf" HOST="localhost" PORT="1984" -IMAPTEST="imaptest" +# in case you have it aliased GREP="/bin/grep" +IMAPTEST="imaptest" # ----------------------------------------------- # # These should be kept constant across benchmarking # runs across different machines, for comparability. -DURATION=100 +DURATION=200 NUM_MSG=200 @@ -153,6 +156,21 @@ print "TOT samples", NR; } +{ test $1 = "--help" } && { + echo "Usage: $0 [user@provider] [/path/to/sample.mbox]" + exit 0 +} + +# If the first parameter is passed, take it as the user +{ test $1 } && { + USER=$1 +} + +# If the second parameter is passed, take it as the mbox +{ test $2 } && { + MBOX=$2 +} + echo "[+] LEAP IMAP TESTS" echo "[+] Running imaptest for $DURATION seconds with $NUM_MSG messages" wait_and_kill & -- cgit v1.2.3 From f5365ae0c2edb8b3e879f876f2f7e42b25f4616a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 27 Jan 2014 16:11:53 -0400 Subject: handle last_uid property in memory store --- src/leap/mail/imap/tests/leap_tests_imap.zsh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/leap_tests_imap.zsh b/src/leap/mail/imap/tests/leap_tests_imap.zsh index 676d1a8..8f0df9f 100755 --- a/src/leap/mail/imap/tests/leap_tests_imap.zsh +++ b/src/leap/mail/imap/tests/leap_tests_imap.zsh @@ -61,7 +61,8 @@ IMAPTEST="imaptest" # These should be kept constant across benchmarking # runs across different machines, for comparability. -DURATION=200 +#DURATION=200 +DURATION=60 NUM_MSG=200 @@ -76,7 +77,7 @@ imaptest_cmd() { } stress_imap() { - mknod imap_pipe p + mkfifo imap_pipe cat imap_pipe | tee output & imaptest_cmd >> imap_pipe } @@ -99,7 +100,7 @@ print_results() { echo "----------------------" echo "\tavg\tstdev" $GREP "avg" ./output | sed -e 's/^ *//g' -e 's/ *$//g' | \ - awk ' + gawk ' function avg(data, count) { sum=0; for( x=0; x <= count-1; x++) { -- cgit v1.2.3 From a7e0054b595822325f749b0b1df7d25cab4e6486 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 28 Jan 2014 18:39:59 -0400 Subject: docstring fixes Also some fixes for None comparisons. --- src/leap/mail/imap/tests/leap_tests_imap.zsh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/leap_tests_imap.zsh b/src/leap/mail/imap/tests/leap_tests_imap.zsh index 8f0df9f..544faca 100755 --- a/src/leap/mail/imap/tests/leap_tests_imap.zsh +++ b/src/leap/mail/imap/tests/leap_tests_imap.zsh @@ -61,8 +61,7 @@ IMAPTEST="imaptest" # These should be kept constant across benchmarking # runs across different machines, for comparability. -#DURATION=200 -DURATION=60 +DURATION=200 NUM_MSG=200 -- cgit v1.2.3 From 37090301e633cdf9aa0c924ea3899d4c5d4fcdb9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 29 Jan 2014 16:18:27 -0400 Subject: allow to pass file as argument --- src/leap/mail/imap/tests/walktree.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/walktree.py b/src/leap/mail/imap/tests/walktree.py index 1626f65..f3cbcb0 100644 --- a/src/leap/mail/imap/tests/walktree.py +++ b/src/leap/mail/imap/tests/walktree.py @@ -18,12 +18,14 @@ Tests for the walktree module. """ import os +import sys from email import parser from leap.mail import walk as W DEBUG = os.environ.get("BITMASK_MAIL_DEBUG") + p = parser.Parser() # TODO pass an argument of the type of message @@ -31,9 +33,17 @@ p = parser.Parser() ################################################## # Input from hell -#msg = p.parse(open('rfc822.multi-signed.message')) -#msg = p.parse(open('rfc822.plain.message')) -msg = p.parse(open('rfc822.multi-minimal.message')) +if len(sys.argv) > 1: + FILENAME = sys.argv[1] +else: + FILENAME = "rfc822.multi-minimal.message" + +""" +FILENAME = "rfc822.multi-signed.message" +FILENAME = "rfc822.plain.message" +""" + +msg = p.parse(open(FILENAME)) DO_CHECK = False ################################################# -- cgit v1.2.3 From 3511f7992e67bc49e9fc4771f4b2c0d9199822d7 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 5 Feb 2014 11:47:43 -0400 Subject: minimal regression tests --- src/leap/mail/imap/tests/getmail | 2 - src/leap/mail/imap/tests/regressions | 451 +++++++++++++++++++++++++++++++++++ 2 files changed, 451 insertions(+), 2 deletions(-) create mode 100755 src/leap/mail/imap/tests/regressions (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/getmail b/src/leap/mail/imap/tests/getmail index 17e195c..0fb00d2 100755 --- a/src/leap/mail/imap/tests/getmail +++ b/src/leap/mail/imap/tests/getmail @@ -5,8 +5,6 @@ # Modifications by LEAP Developers 2014 to fit # Bitmask configuration settings. - - """ Simple IMAP4 client which displays the subjects of all messages in a particular mailbox. diff --git a/src/leap/mail/imap/tests/regressions b/src/leap/mail/imap/tests/regressions new file mode 100755 index 0000000..0a43398 --- /dev/null +++ b/src/leap/mail/imap/tests/regressions @@ -0,0 +1,451 @@ +#!/usr/bin/env python + +# -*- coding: utf-8 -*- +# regressions +# Copyright (C) 2014 LEAP +# Copyright (c) Twisted Matrix Laboratories. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Simple Regression Tests using IMAP4 client. + +Iterates trough all mails under a given folder and tries to APPEND them to +the server being tested. After FETCHING the pushed message, it compares +the received version with the one that was saved, and exits with an error +code if they do not match. +""" +import os +import StringIO +import sys + +from email.parser import Parser + +from twisted.internet import protocol +from twisted.internet import ssl +from twisted.internet import defer +from twisted.internet import stdio +from twisted.mail import imap4 +from twisted.protocols import basic +from twisted.python import log + + +REGRESSIONS_FOLDER = "regressions_test" + +parser = Parser() + + +def get_msg_parts(raw): + """ + Return a representation of the parts of a message suitable for + comparison. + + :param raw: string for the message + :type raw: str + """ + m = parser.parsestr(raw) + return [dict(part.items()) + if part.is_multipart() + else part.get_payload() + for part in m.walk()] + + +def compare_msg_parts(a, b): + """ + Compare two sequences of parts of messages. + + :param a: part sequence for message a + :param b: part sequence for message b + + :return: True if both message sequences are equivalent. + :rtype: bool + """ + # XXX This could be smarter and show the differences in the + # different parts when/where they differ. + #import pprint; pprint.pprint(a[0]) + #import pprint; pprint.pprint(b[0]) + + def lowerkey(d): + return dict((k.lower(), v.replace('\r', '')) + for k, v in d.iteritems()) + + def eq(x, y): + # For dicts, we compare a variation with their keys + # in lowercase, and \r removed from their values + if all(map(lambda i: isinstance(i, dict), (x, y))): + x, y = map(lowerkey, (x, y)) + return x == y + + compare_vector = map(lambda tup: eq(tup[0], tup[1]), zip(a, b)) + all_match = all(compare_vector) + + if not all_match: + print "PARTS MISMATCH!" + print "vector: ", compare_vector + index = compare_vector.index(False) + from pprint import pprint + print "Expected:" + pprint(a[index]) + print ("***") + print "Found:" + pprint(b[index]) + print + + + return all_match + + +def get_fd(string): + """ + Return a file descriptor with the passed string + as content. + """ + fd = StringIO.StringIO() + fd.write(string) + fd.seek(0) + return fd + + +class TrivialPrompter(basic.LineReceiver): + promptDeferred = None + + def prompt(self, msg): + assert self.promptDeferred is None + self.display(msg) + self.promptDeferred = defer.Deferred() + return self.promptDeferred + + def display(self, msg): + self.transport.write(msg) + + def lineReceived(self, line): + if self.promptDeferred is None: + return + d, self.promptDeferred = self.promptDeferred, None + d.callback(line) + + +class SimpleIMAP4Client(imap4.IMAP4Client): + """ + A client with callbacks for greeting messages from an IMAP server. + """ + greetDeferred = None + + def serverGreeting(self, caps): + self.serverCapabilities = caps + if self.greetDeferred is not None: + d, self.greetDeferred = self.greetDeferred, None + d.callback(self) + + +class SimpleIMAP4ClientFactory(protocol.ClientFactory): + usedUp = False + protocol = SimpleIMAP4Client + + def __init__(self, username, onConn): + self.ctx = ssl.ClientContextFactory() + + self.username = username + self.onConn = onConn + + def buildProtocol(self, addr): + """ + Initiate the protocol instance. Since we are building a simple IMAP + client, we don't bother checking what capabilities the server has. We + just add all the authenticators twisted.mail has. Note: Gmail no + longer uses any of the methods below, it's been using XOAUTH since + 2010. + """ + assert not self.usedUp + self.usedUp = True + + p = self.protocol(self.ctx) + p.factory = self + p.greetDeferred = self.onConn + + p.registerAuthenticator(imap4.PLAINAuthenticator(self.username)) + p.registerAuthenticator(imap4.LOGINAuthenticator(self.username)) + p.registerAuthenticator( + imap4.CramMD5ClientAuthenticator(self.username)) + + return p + + def clientConnectionFailed(self, connector, reason): + d, self.onConn = self.onConn, None + d.errback(reason) + + +def cbServerGreeting(proto, username, password): + """ + Initial callback - invoked after the server sends us its greet message. + """ + # Hook up stdio + tp = TrivialPrompter() + stdio.StandardIO(tp) + + # And make it easily accessible + proto.prompt = tp.prompt + proto.display = tp.display + + # Try to authenticate securely + return proto.authenticate( + password).addCallback( + cbAuthentication, + proto).addErrback( + ebAuthentication, proto, username, password + ) + + +def ebConnection(reason): + """ + Fallback error-handler. If anything goes wrong, log it and quit. + """ + log.startLogging(sys.stdout) + log.err(reason) + return reason + + +def cbAuthentication(result, proto): + """ + Callback after authentication has succeeded. + + Lists a bunch of mailboxes. + """ + return proto.select( + REGRESSIONS_FOLDER + ).addCallback( + cbSelectMbox, proto + ).addErrback( + ebSelectMbox, proto, REGRESSIONS_FOLDER) + + +def ebAuthentication(failure, proto, username, password): + """ + Errback invoked when authentication fails. + + If it failed because no SASL mechanisms match, offer the user the choice + of logging in insecurely. + + If you are trying to connect to your Gmail account, you will be here! + """ + failure.trap(imap4.NoSupportedAuthentication) + return InsecureLogin(proto, username, password) + + +def InsecureLogin(proto, username, password): + """ + Raise insecure-login error. + """ + return proto.login( + username, password + ).addCallback( + cbAuthentication, proto) + + +def cbSelectMbox(result, proto): + """ + Callback invoked when select command finishes successfully. + + If any message is in the test folder, it will flag them as deleted and + expunge. + If no messages found, it will start with the APPEND tests. + """ + print "SELECT: %s EXISTS " % result.get("EXISTS", "??") + + if result["EXISTS"] != 0: + # Flag as deleted, expunge, and do an examine again. + #print "There is mail here, will delete..." + return cbDeleteAndExpungeTestFolder(proto) + + else: + return cbAppendNextMessage(proto) + + +def ebSelectMbox(failure, proto, folder): + """ + Errback invoked when the examine command fails. + + Creates the folder. + """ + print failure.getTraceback() + log.msg("Folder %r does not exist. Creating..." % (folder,)) + return proto.create(folder).addCallback(cbAuthentication, proto) + + +def cbDeleteAndExpungeTestFolder(proto): + """ + Callback invoked fom cbExamineMbox when the number of messages in the + mailbox is not zero. It flags all messages as deleted and expunge the + mailbox. + """ + return proto.setFlags( + "1:*", ("\\Deleted",) + ).addCallback( + lambda r: proto.expunge() + ).addCallback( + cbExpunge, proto) + + +def cbExpunge(result, proto): + return proto.select( + REGRESSIONS_FOLDER + ).addCallback( + cbSelectMbox, proto + ).addErrback(ebSettingDeleted, proto) + + +def ebSettingDeleted(failure, proto): + """ + Report errors during deletion of messages in the mailbox. + """ + print failure.getTraceback() + + +def cbAppendNextMessage(proto): + """ + Appends the next message in the global queue to the test folder. + """ + # 1. Get the next test message from global tuple. + try: + next_sample = SAMPLES.pop() + except IndexError: + # we're done! + return proto.logout() + + print "\nAPPEND %s" % (next_sample,) + raw = open(next_sample).read() + msg = get_fd(raw) + return proto.append( + REGRESSIONS_FOLDER, msg + ).addCallback( + lambda r: proto.examine(REGRESSIONS_FOLDER) + ).addCallback( + cbAppend, proto, raw + ).addErrback( + ebAppend, proto, raw) + + +def cbAppend(result, proto, orig_msg): + """ + Fetches the message right after an append. + """ + # XXX keep account of highest UID + uid = "1:*" + + return proto.fetchSpecific( + '%s' % uid, + headerType='', + headerArgs=['BODY.PEEK[]'], + ).addCallback( + cbCompareMessage, proto, orig_msg + ).addErrback(ebAppend, proto, orig_msg) + + +def ebAppend(failure, proto, raw): + """ + Errorback for the append operation + """ + print "ERROR WHILE APPENDING!" + print failure.getTraceback() + + +def cbPickMessage(result, proto): + """ + Pick a message. + """ + return proto.fetchSpecific( + '%s' % result, + headerType='', + headerArgs=['BODY.PEEK[]'], + ).addCallback(cbCompareMessage, proto) + + +def cbCompareMessage(result, proto, raw): + """ + Display message and compare it with the original one. + """ + parts_orig = get_msg_parts(raw) + + if result: + keys = result.keys() + keys.sort() + + latest = max(keys) + + fetched_msg = result[latest][0][2] + parts_fetched = get_msg_parts(fetched_msg) + + equal = compare_msg_parts( + parts_orig, + parts_fetched) + + if equal: + print "[+] MESSAGES MATCH" + return cbAppendNextMessage(proto) + else: + print "[-] ERROR: MESSAGES DO NOT MATCH !!!" + print " ABORTING COMPARISON..." + # FIXME logout and print the subject ... + return proto.logout() + + +def cbClose(result): + """ + Close the connection when we finish everything. + """ + from twisted.internet import reactor + reactor.stop() + + +def main(): + import glob + import sys + + if len(sys.argv) != 4: + print "Usage: regressions " + sys.exit() + + hostname = "localhost" + port = "1984" + username = sys.argv[1] + password = sys.argv[2] + + samplesdir = sys.argv[3] + + if not os.path.isdir(samplesdir): + print ("Could not find samples folder! " + "Make sure of copying mail_breaker contents there.") + sys.exit() + + samples = glob.glob(samplesdir + '/*') + + global SAMPLES + SAMPLES = [] + SAMPLES += samples + + onConn = defer.Deferred( + ).addCallback( + cbServerGreeting, username, password + ).addErrback( + ebConnection + ).addBoth(cbClose) + + factory = SimpleIMAP4ClientFactory(username, onConn) + + from twisted.internet import reactor + reactor.connectTCP(hostname, int(port), factory) + reactor.run() + + +if __name__ == '__main__': + main() -- cgit v1.2.3 From ee0786c57d72aa8b8da76533f33c3dd65253a878 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 6 Feb 2014 18:11:20 -0400 Subject: long-due update to unittests! So we're safe under the green lights before further rewriting. :) --- src/leap/mail/imap/tests/test_imap.py | 432 ++++++++++++++++------------------ 1 file changed, 200 insertions(+), 232 deletions(-) (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 8c1cf20..fd88440 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -43,6 +43,7 @@ from itertools import chain from mock import Mock from nose.twistedtools import deferred, stop_reactor +from unittest import skip from twisted.mail import imap4 @@ -64,11 +65,16 @@ import twisted.cred.portal from leap.common.testing.basetest import BaseLeapTest from leap.mail.imap.account import SoledadBackedAccount from leap.mail.imap.mailbox import SoledadMailbox +from leap.mail.imap.memorystore import MemoryStore from leap.mail.imap.messages import MessageCollection +from leap.mail.imap.server import LeapIMAPServer from leap.soledad.client import Soledad from leap.soledad.client import SoledadCrypto +TEST_USER = "testuser@leap.se" +TEST_PASSWD = "1234" + def strip(f): return lambda result, f=f: f() @@ -89,10 +95,10 @@ def initialize_soledad(email, gnupg_home, tempdir): """ Initializes soledad by hand - @param email: ID for the user - @param gnupg_home: path to home used by gnupg - @param tempdir: path to temporal dir - @rtype: Soledad instance + :param email: ID for the user + :param gnupg_home: path to home used by gnupg + :param tempdir: path to temporal dir + :rtype: Soledad instance """ uuid = "foobar-uuid" @@ -125,55 +131,6 @@ def initialize_soledad(email, gnupg_home, tempdir): return _soledad -# -# Simple LEAP IMAP4 Server for testing -# - -class SimpleLEAPServer(imap4.IMAP4Server): - - """ - A Simple IMAP4 Server with mailboxes backed by Soledad. - - This should be pretty close to the real LeapIMAP4Server that we - will be instantiating as a service, minus the authentication bits. - """ - - def __init__(self, *args, **kw): - - soledad = kw.pop('soledad', None) - - imap4.IMAP4Server.__init__(self, *args, **kw) - realm = TestRealm() - - # XXX Why I AM PASSING THE ACCOUNT TO - # REALM? I AM NOT USING THAT NOW, AM I??? - realm.theAccount = SoledadBackedAccount( - 'testuser', - soledad=soledad) - - portal = cred.portal.Portal(realm) - c = cred.checkers.InMemoryUsernamePasswordDatabaseDontUse() - self.checker = c - self.portal = portal - portal.registerChecker(c) - self.timeoutTest = False - - def lineReceived(self, line): - if self.timeoutTest: - # Do not send a respones - return - - imap4.IMAP4Server.lineReceived(self, line) - - _username = 'testuser' - _password = 'password-test' - - def authenticateLogin(self, username, password): - if username == self._username and password == self._password: - return imap4.IAccount, self.theAccount, lambda: None - raise cred.error.UnauthorizedLogin() - - class TestRealm: """ @@ -255,13 +212,6 @@ class IMAP4HelperMixin(BaseLeapTest): # Soledad: config info cls.gnupg_home = "%s/gnupg" % cls.tempdir cls.email = 'leap@leap.se' - # cls.db1_file = "%s/db1.u1db" % cls.tempdir - # cls.db2_file = "%s/db2.u1db" % cls.tempdir - # open test dbs - # cls._db1 = u1db.open(cls.db1_file, create=True, - # document_factory=SoledadDocument) - # cls._db2 = u1db.open(cls.db2_file, create=True, - # document_factory=SoledadDocument) # initialize soledad by hand so we can control keys cls._soledad = initialize_soledad( @@ -283,8 +233,6 @@ class IMAP4HelperMixin(BaseLeapTest): Restores the old path and home environment variables. Removes the temporal dir created for tests. """ - # cls._db1.close() - # cls._db2.close() cls._soledad.close() os.environ["PATH"] = cls.old_path @@ -301,8 +249,13 @@ class IMAP4HelperMixin(BaseLeapTest): but passing the same Soledad instance (it's costly to initialize), so we have to be sure to restore state across tests. """ + UUID = 'deadbeef', + USERID = TEST_USER + memstore = MemoryStore() + d = defer.Deferred() - self.server = SimpleLEAPServer( + self.server = LeapIMAPServer( + uuid=UUID, userid=USERID, contextFactory=self.serverCTX, # XXX do we really need this?? soledad=self._soledad) @@ -317,9 +270,10 @@ class IMAP4HelperMixin(BaseLeapTest): # I THINK we ONLY need to do it at one place now. theAccount = SoledadBackedAccount( - 'testuser', - soledad=self._soledad) - SimpleLEAPServer.theAccount = theAccount + USERID, + soledad=self._soledad, + memstore=memstore) + LeapIMAPServer.theAccount = theAccount # in case we get something from previous tests... for mb in self.server.theAccount.mailboxes: @@ -404,8 +358,9 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): We override mixin method since we are only testing MessageCollection interface in this particular TestCase """ + memstore = MemoryStore() self.messages = MessageCollection("testmbox%s" % (self.count,), - self._soledad) + self._soledad, memstore=memstore) MessageCollectionTestCase.count += 1 def tearDown(self): @@ -414,9 +369,6 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): """ del self.messages - def wait(self): - time.sleep(2) - def testEmptyMessage(self): """ Test empty message and collection @@ -425,11 +377,11 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): self.assertEqual( em, { + "chash": '', + "deleted": False, "flags": [], "mbox": "inbox", - "recent": True, "seen": False, - "deleted": False, "multi": False, "size": 0, "type": "flags", @@ -441,79 +393,100 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): """ Add multiple messages """ - # TODO really profile addition mc = self.messages - print "messages", self.messages self.assertEqual(self.messages.count(), 0) - mc.add_msg('Stuff', uid=1, subject="test1") - mc.add_msg('Stuff', uid=2, subject="test2") - mc.add_msg('Stuff', uid=3, subject="test3") - mc.add_msg('Stuff', uid=4, subject="test4") - self.wait() - self.assertEqual(self.messages.count(), 4) - mc.add_msg('Stuff', uid=5, subject="test5") - mc.add_msg('Stuff', uid=6, subject="test6") - mc.add_msg('Stuff', uid=7, subject="test7") - self.wait() - self.assertEqual(self.messages.count(), 7) - self.wait() + def add_first(): + d = defer.gatherResults([ + mc.add_msg('Stuff 1', uid=1, subject="test1"), + mc.add_msg('Stuff 2', uid=2, subject="test2"), + mc.add_msg('Stuff 3', uid=3, subject="test3"), + mc.add_msg('Stuff 4', uid=4, subject="test4")]) + return d + + def add_second(result): + d = defer.gatherResults([ + mc.add_msg('Stuff 5', uid=5, subject="test5"), + mc.add_msg('Stuff 6', uid=6, subject="test6"), + mc.add_msg('Stuff 7', uid=7, subject="test7")]) + return d + + def check_second(result): + return self.assertEqual(mc.count(), 7) + + d1 = add_first() + d1.addCallback(add_second) + d1.addCallback(check_second) + + @skip("needs update!") def testRecentCount(self): """ Test the recent count """ mc = self.messages - self.assertEqual(self.messages.count_recent(), 0) - mc.add_msg('Stuff', uid=1, subject="test1") + countrecent = mc.count_recent + eq = self.assertEqual + + self.assertEqual(countrecent(), 0) + + d = mc.add_msg('Stuff', uid=1, subject="test1") # For the semantics defined in the RFC, we auto-add the # recent flag by default. - self.wait() - self.assertEqual(self.messages.count_recent(), 1) - mc.add_msg('Stuff', subject="test2", uid=2, - flags=('\\Deleted',)) - self.wait() - self.assertEqual(self.messages.count_recent(), 2) - mc.add_msg('Stuff', subject="test3", uid=3, - flags=('\\Recent',)) - self.wait() - self.assertEqual(self.messages.count_recent(), 3) - mc.add_msg('Stuff', subject="test4", uid=4, - flags=('\\Deleted', '\\Recent')) - self.wait() - self.assertEqual(self.messages.count_recent(), 4) - - for msg in mc: - msg.removeFlags(('\\Recent',)) - self.assertEqual(mc.count_recent(), 0) + + def add2(_): + return mc.add_msg('Stuff', subject="test2", uid=2, + flags=('\\Deleted',)) + + def add3(_): + return mc.add_msg('Stuff', subject="test3", uid=3, + flags=('\\Recent',)) + + def add4(_): + return mc.add_msg('Stuff', subject="test4", uid=4, + flags=('\\Deleted', '\\Recent')) + + d.addCallback(lambda r: eq(countrecent(), 1)) + d.addCallback(add2) + d.addCallback(lambda r: eq(countrecent(), 2)) + d.addCallback(add3) + d.addCallback(lambda r: eq(countrecent(), 3)) + d.addCallback(add4) + d.addCallback(lambda r: eq(countrecent(), 4)) def testFilterByMailbox(self): """ Test that queries filter by selected mailbox """ - def wait(): - time.sleep(1) - mc = self.messages self.assertEqual(self.messages.count(), 0) - mc.add_msg('', uid=1, subject="test1") - mc.add_msg('', uid=2, subject="test2") - mc.add_msg('', uid=3, subject="test3") - wait() - self.assertEqual(self.messages.count(), 3) - newmsg = mc._get_empty_doc() - newmsg['mailbox'] = "mailbox/foo" - mc._soledad.create_doc(newmsg) - self.assertEqual(mc.count(), 3) - self.assertEqual( - len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4) + + def add_1(): + d1 = mc.add_msg('msg 1', uid=1, subject="test1") + d2 = mc.add_msg('msg 2', uid=2, subject="test2") + d3 = mc.add_msg('msg 3', uid=3, subject="test3") + d = defer.gatherResults([d1, d2, d3]) + return d + + add_1().addCallback(lambda ignored: self.assertEqual( + mc.count(), 3)) + + # XXX this has to be redone to fit memstore ------------# + #newmsg = mc._get_empty_doc() + #newmsg['mailbox'] = "mailbox/foo" + #mc._soledad.create_doc(newmsg) + #self.assertEqual(mc.count(), 3) + #self.assertEqual( + #len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4) class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): + # TODO this currently will use a memory-only store. + # create a different one for testing soledad sync. """ Tests for the generic behavior of the LeapIMAP4Server which, right now, it's just implemented in this test file as - SimpleLEAPServer. We will move the implementation, together with + LeapIMAPServer. We will move the implementation, together with authentication bits, to leap.mail.imap.server so it can be instantiated from the tac file. @@ -542,7 +515,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.result.append(0) def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def create(): for name in succeed + fail: @@ -560,7 +533,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestCreate(self, ignored, succeed, fail): self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail)) - mbox = SimpleLEAPServer.theAccount.mailboxes + mbox = LeapIMAPServer.theAccount.mailboxes answers = ['foobox', 'testbox', 'test/box', 'test', 'test/box/box'] mbox.sort() answers.sort() @@ -571,10 +544,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test whether we can delete mailboxes """ - SimpleLEAPServer.theAccount.addMailbox('delete/me') + LeapIMAPServer.theAccount.addMailbox('delete/me') def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def delete(): return self.client.delete('delete/me') @@ -586,7 +559,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d = defer.gatherResults([d1, d2]) d.addCallback( lambda _: self.assertEqual( - SimpleLEAPServer.theAccount.mailboxes, [])) + LeapIMAPServer.theAccount.mailboxes, [])) return d def testIllegalInboxDelete(self): @@ -597,7 +570,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.stashed = None def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def delete(): return self.client.delete('inbox') @@ -619,10 +592,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def testNonExistentDelete(self): """ Test what happens if we try to delete a non-existent mailbox. - We expect an error raised stating 'No such inbox' + We expect an error raised stating 'No such mailbox' """ def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def delete(): return self.client.delete('delete/me') @@ -637,8 +610,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: self.assertEqual(str(self.failure.value), - 'No such mailbox')) + d.addCallback(lambda _: self.assertTrue( + str(self.failure.value).startswith('No such mailbox'))) return d @deferred(timeout=None) @@ -649,14 +622,14 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Obs: this test will fail if SoledadMailbox returns hardcoded flags. """ - SimpleLEAPServer.theAccount.addMailbox('delete') - to_delete = SimpleLEAPServer.theAccount.getMailbox('delete') + LeapIMAPServer.theAccount.addMailbox('delete') + to_delete = LeapIMAPServer.theAccount.getMailbox('delete') to_delete.setFlags((r'\Noselect',)) to_delete.getFlags() - SimpleLEAPServer.theAccount.addMailbox('delete/me') + LeapIMAPServer.theAccount.addMailbox('delete/me') def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def delete(): return self.client.delete('delete') @@ -681,10 +654,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test whether we can rename a mailbox """ - SimpleLEAPServer.theAccount.addMailbox('oldmbox') + LeapIMAPServer.theAccount.addMailbox('oldmbox') def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def rename(): return self.client.rename('oldmbox', 'newname') @@ -696,7 +669,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d = defer.gatherResults([d1, d2]) d.addCallback(lambda _: self.assertEqual( - SimpleLEAPServer.theAccount.mailboxes, + LeapIMAPServer.theAccount.mailboxes, ['newname'])) return d @@ -709,7 +682,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.stashed = None def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def rename(): return self.client.rename('inbox', 'frotz') @@ -733,11 +706,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Try to rename hierarchical mailboxes """ - SimpleLEAPServer.theAccount.create('oldmbox/m1') - SimpleLEAPServer.theAccount.create('oldmbox/m2') + LeapIMAPServer.theAccount.create('oldmbox/m1') + LeapIMAPServer.theAccount.create('oldmbox/m2') def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def rename(): return self.client.rename('oldmbox', 'newname') @@ -750,7 +723,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestHierarchicalRename) def _cbTestHierarchicalRename(self, ignored): - mboxes = SimpleLEAPServer.theAccount.mailboxes + mboxes = LeapIMAPServer.theAccount.mailboxes expected = ['newname', 'newname/m1', 'newname/m2'] mboxes.sort() self.assertEqual(mboxes, [s for s in expected]) @@ -761,7 +734,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Test whether we can mark a mailbox as subscribed to """ def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def subscribe(): return self.client.subscribe('this/mbox') @@ -773,7 +746,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d = defer.gatherResults([d1, d2]) d.addCallback(lambda _: self.assertEqual( - SimpleLEAPServer.theAccount.subscriptions, + LeapIMAPServer.theAccount.subscriptions, ['this/mbox'])) return d @@ -782,11 +755,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test whether we can unsubscribe from a set of mailboxes """ - SimpleLEAPServer.theAccount.subscribe('this/mbox') - SimpleLEAPServer.theAccount.subscribe('that/mbox') + LeapIMAPServer.theAccount.subscribe('this/mbox') + LeapIMAPServer.theAccount.subscribe('that/mbox') def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def unsubscribe(): return self.client.unsubscribe('this/mbox') @@ -798,7 +771,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d = defer.gatherResults([d1, d2]) d.addCallback(lambda _: self.assertEqual( - SimpleLEAPServer.theAccount.subscriptions, + LeapIMAPServer.theAccount.subscriptions, ['that/mbox'])) return d @@ -811,7 +784,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.selectedArgs = None def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def select(): def selected(args): @@ -829,7 +802,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return defer.gatherResults([d1, d2]).addCallback(self._cbTestSelect) def _cbTestSelect(self, ignored): - mbox = SimpleLEAPServer.theAccount.getMailbox('TESTMAILBOX-SELECT') + mbox = LeapIMAPServer.theAccount.getMailbox('TESTMAILBOX-SELECT') self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox) self.assertEqual(self.selectedArgs, { 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, @@ -920,7 +893,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Test login """ def login(): - d = self.client.login('testuser', 'password-test') + d = self.client.login(TEST_USER, TEST_PASSWD) d.addCallback(self._cbStopClient) d1 = self.connected.addCallback( strip(login)).addErrback(self._ebGeneral) @@ -928,7 +901,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestLogin) def _cbTestLogin(self, ignored): - self.assertEqual(self.server.account, SimpleLEAPServer.theAccount) + self.assertEqual(self.server.account, LeapIMAPServer.theAccount) self.assertEqual(self.server.state, 'auth') @deferred(timeout=None) @@ -937,7 +910,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Test bad login """ def login(): - d = self.client.login('testuser', 'wrong-password') + d = self.client.login("bad_user@leap.se", TEST_PASSWD) d.addBoth(self._cbStopClient) d1 = self.connected.addCallback( @@ -947,19 +920,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestFailedLogin) def _cbTestFailedLogin(self, ignored): - self.assertEqual(self.server.account, None) self.assertEqual(self.server.state, 'unauth') + self.assertEqual(self.server.account, None) @deferred(timeout=None) def testLoginRequiringQuoting(self): """ Test login requiring quoting """ - self.server._username = '{test}user' + self.server._userid = '{test}user@leap.se' self.server._password = '{test}password' def login(): - d = self.client.login('{test}user', '{test}password') + d = self.client.login('{test}user@leap.se', '{test}password') d.addBoth(self._cbStopClient) d1 = self.connected.addCallback( @@ -968,7 +941,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestLoginRequiringQuoting) def _cbTestLoginRequiringQuoting(self, ignored): - self.assertEqual(self.server.account, SimpleLEAPServer.theAccount) + self.assertEqual(self.server.account, LeapIMAPServer.theAccount) self.assertEqual(self.server.state, 'auth') # @@ -983,7 +956,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.namespaceArgs = None def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def namespace(): def gotNamespace(args): @@ -1022,7 +995,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.examinedArgs = None def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def examine(): def examined(args): @@ -1049,15 +1022,15 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): 'READ-WRITE': False}) def _listSetup(self, f): - SimpleLEAPServer.theAccount.addMailbox('root/subthingl', - creation_ts=42) - SimpleLEAPServer.theAccount.addMailbox('root/another-thing', - creation_ts=42) - SimpleLEAPServer.theAccount.addMailbox('non-root/subthing', - creation_ts=42) + LeapIMAPServer.theAccount.addMailbox('root/subthingl', + creation_ts=42) + LeapIMAPServer.theAccount.addMailbox('root/another-thing', + creation_ts=42) + LeapIMAPServer.theAccount.addMailbox('non-root/subthing', + creation_ts=42) def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def listed(answers): self.listed = answers @@ -1092,7 +1065,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test LSub command """ - SimpleLEAPServer.theAccount.subscribe('root/subthingl2') + LeapIMAPServer.theAccount.subscribe('root/subthingl2') def lsub(): return self.client.lsub('root', '%') @@ -1106,12 +1079,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test Status command """ - SimpleLEAPServer.theAccount.addMailbox('root/subthings') + LeapIMAPServer.theAccount.addMailbox('root/subthings') # XXX FIXME ---- should populate this a little bit, # with unseen etc... def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def status(): return self.client.status( @@ -1139,7 +1112,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Test failed status command with a non-existent mailbox """ def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def status(): return self.client.status( @@ -1180,13 +1153,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ infile = util.sibpath(__file__, 'rfc822.message') message = open(infile) - SimpleLEAPServer.theAccount.addMailbox('root/subthing') + LeapIMAPServer.theAccount.addMailbox('root/subthing') def login(): - return self.client.login('testuser', 'password-test') - - def wait(): - time.sleep(0.5) + return self.client.login(TEST_USER, TEST_PASSWD) def append(): return self.client.append( @@ -1198,21 +1168,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d1 = self.connected.addCallback(strip(login)) d1.addCallbacks(strip(append), self._ebGeneral) - d1.addCallbacks(strip(wait), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) return d.addCallback(self._cbTestFullAppend, infile) def _cbTestFullAppend(self, ignored, infile): - mb = SimpleLEAPServer.theAccount.getMailbox('root/subthing') - time.sleep(0.5) + mb = LeapIMAPServer.theAccount.getMailbox('root/subthing') self.assertEqual(1, len(mb.messages)) msg = mb.messages.get_msg_by_uid(1) self.assertEqual( - ('\\SEEN', '\\DELETED'), - msg.getFlags()) + set(('\\Recent', '\\SEEN', '\\DELETED')), + set(msg.getFlags())) self.assertEqual( 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', @@ -1220,14 +1188,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): parsed = self.parser.parse(open(infile)) body = parsed.get_payload() - headers = parsed.items() + headers = dict(parsed.items()) self.assertEqual( body, msg.getBodyFile().read()) - - msg_headers = msg.getHeaders(True, "",) - gotheaders = list(chain( - *[[(k, item) for item in v] for (k, v) in msg_headers.items()])) + gotheaders = msg.getHeaders(True) self.assertItemsEqual( headers, gotheaders) @@ -1238,13 +1203,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Test partially appending a message to the mailbox """ infile = util.sibpath(__file__, 'rfc822.message') - SimpleLEAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') + LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') def login(): - return self.client.login('testuser', 'password-test') - - def wait(): - time.sleep(1) + return self.client.login(TEST_USER, TEST_PASSWD) def append(): message = file(infile) @@ -1257,7 +1219,6 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): ) ) d1 = self.connected.addCallback(strip(login)) - d1.addCallbacks(strip(wait), self._ebGeneral) d1.addCallbacks(strip(append), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -1266,16 +1227,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self._cbTestPartialAppend, infile) def _cbTestPartialAppend(self, ignored, infile): - mb = SimpleLEAPServer.theAccount.getMailbox('PARTIAL/SUBTHING') - time.sleep(1) + mb = LeapIMAPServer.theAccount.getMailbox('PARTIAL/SUBTHING') self.assertEqual(1, len(mb.messages)) msg = mb.messages.get_msg_by_uid(1) self.assertEqual( - ('\\SEEN', ), - msg.getFlags() + set(('\\SEEN', '\\Recent')), + set(msg.getFlags()) ) - #self.assertEqual( - #'Right now', msg.getInternalDate()) parsed = self.parser.parse(open(infile)) body = parsed.get_payload() self.assertEqual( @@ -1287,10 +1245,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test check command """ - SimpleLEAPServer.theAccount.addMailbox('root/subthing') + LeapIMAPServer.theAccount.addMailbox('root/subthing') def login(): - return self.client.login('testuser', 'password-test') + return self.client.login(TEST_USER, TEST_PASSWD) def select(): return self.client.select('root/subthing') @@ -1306,7 +1264,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): # Okay, that was fun - @deferred(timeout=None) + @deferred(timeout=5) def testClose(self): """ Test closing the mailbox. We expect to get deleted all messages flagged @@ -1315,29 +1273,33 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): name = 'mailbox-close' self.server.theAccount.addMailbox(name) - m = SimpleLEAPServer.theAccount.getMailbox(name) - m.messages.add_msg('test 1', uid=1, subject="Message 1", - flags=('\\Deleted', 'AnotherFlag')) - m.messages.add_msg('test 2', uid=2, subject="Message 2", - flags=('AnotherFlag',)) - m.messages.add_msg('test 3', uid=3, subject="Message 3", - flags=('\\Deleted',)) + m = LeapIMAPServer.theAccount.getMailbox(name) def login(): - return self.client.login('testuser', 'password-test') - - def wait(): - time.sleep(1) + return self.client.login(TEST_USER, TEST_PASSWD) def select(): return self.client.select(name) + def add_messages(): + d1 = m.messages.add_msg( + 'test 1', uid=1, subject="Message 1", + flags=('\\Deleted', 'AnotherFlag')) + d2 = m.messages.add_msg( + 'test 2', uid=2, subject="Message 2", + flags=('AnotherFlag',)) + d3 = m.messages.add_msg( + 'test 3', uid=3, subject="Message 3", + flags=('\\Deleted',)) + d = defer.gatherResults([d1, d2, d3]) + return d + def close(): return self.client.close() d = self.connected.addCallback(strip(login)) - d.addCallbacks(strip(wait), self._ebGeneral) d.addCallbacks(strip(select), self._ebGeneral) + d.addCallbacks(strip(add_messages), self._ebGeneral) d.addCallbacks(strip(close), self._ebGeneral) d.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -1345,37 +1307,42 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestClose(self, ignored, m): self.assertEqual(len(m.messages), 1) - messages = [msg for msg in m.messages] - self.assertFalse(messages[0] is None) + + msg = m.messages.get_msg_by_uid(2) + self.assertFalse(msg is None) self.assertEqual( - messages[0]._hdoc.content['subject'], + msg._hdoc.content['subject'], 'Message 2') self.failUnless(m.closed) - @deferred(timeout=None) + @deferred(timeout=5) def testExpunge(self): """ Test expunge command """ name = 'mailbox-expunge' - SimpleLEAPServer.theAccount.addMailbox(name) - m = SimpleLEAPServer.theAccount.getMailbox(name) - m.messages.add_msg('test 1', uid=1, subject="Message 1", - flags=('\\Deleted', 'AnotherFlag')) - m.messages.add_msg('test 2', uid=2, subject="Message 2", - flags=('AnotherFlag',)) - m.messages.add_msg('test 3', uid=3, subject="Message 3", - flags=('\\Deleted',)) + self.server.theAccount.addMailbox(name) + m = LeapIMAPServer.theAccount.getMailbox(name) def login(): - return self.client.login('testuser', 'password-test') - - def wait(): - time.sleep(2) + return self.client.login(TEST_USER, TEST_PASSWD) def select(): return self.client.select('mailbox-expunge') + def add_messages(): + d1 = m.messages.add_msg( + 'test 1', uid=1, subject="Message 1", + flags=('\\Deleted', 'AnotherFlag')) + d2 = m.messages.add_msg( + 'test 2', uid=2, subject="Message 2", + flags=('AnotherFlag',)) + d3 = m.messages.add_msg( + 'test 3', uid=3, subject="Message 3", + flags=('\\Deleted',)) + d = defer.gatherResults([d1, d2, d3]) + return d + def expunge(): return self.client.expunge() @@ -1385,8 +1352,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.results = None d1 = self.connected.addCallback(strip(login)) - d1.addCallbacks(strip(wait), self._ebGeneral) d1.addCallbacks(strip(select), self._ebGeneral) + d1.addCallbacks(strip(add_messages), self._ebGeneral) d1.addCallbacks(strip(expunge), self._ebGeneral) d1.addCallbacks(expunged, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) @@ -1397,9 +1364,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestExpunge(self, ignored, m): # we only left 1 mssage with no deleted flag self.assertEqual(len(m.messages), 1) - messages = [msg for msg in m.messages] + + msg = m.messages.get_msg_by_uid(2) self.assertEqual( - messages[0]._hdoc.content['subject'], + msg._hdoc.content['subject'], 'Message 2') # the uids of the deleted messages self.assertItemsEqual(self.results, [1, 3]) -- cgit v1.2.3 From 3490773898c0b1af76ff18b088b099b7cde677e7 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 12 Feb 2014 12:44:18 -0400 Subject: select instead of examine --- src/leap/mail/imap/tests/regressions | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/regressions b/src/leap/mail/imap/tests/regressions index 0a43398..efe3f46 100755 --- a/src/leap/mail/imap/tests/regressions +++ b/src/leap/mail/imap/tests/regressions @@ -101,7 +101,6 @@ def compare_msg_parts(a, b): pprint(b[index]) print - return all_match @@ -328,7 +327,7 @@ def cbAppendNextMessage(proto): return proto.append( REGRESSIONS_FOLDER, msg ).addCallback( - lambda r: proto.examine(REGRESSIONS_FOLDER) + lambda r: proto.select(REGRESSIONS_FOLDER) ).addCallback( cbAppend, proto, raw ).addErrback( @@ -379,6 +378,9 @@ def cbCompareMessage(result, proto, raw): if result: keys = result.keys() keys.sort() + else: + print "[-] GOT NO RESULT" + return proto.logout() latest = max(keys) -- cgit v1.2.3 From b2d97c9faef6037a065e2903afe5b0ab2624917e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 20 Feb 2014 02:52:17 -0400 Subject: mail parsing performance improvements Although the do_parse function is deferred to threads, we were actually waiting till its return to fire the callback of the deferred, and hence the "append ok" was being delayed. During massive appends, this was a tight loop contributing as much as 35 msec, of a total of 100 msec average. Several ineficiencies are addressed here: * use pycryptopp hash functions. * avoiding function calling overhead. * avoid duplicate call to message.as_string * make use of the string size caching capabilities. * avoiding the mail Parser initialization/method call completely, in favor of the module helper to get the object from string. Overall, these changes cut parsing to 50% of the initial timing by my measurements with line_profiler, YMMV. --- src/leap/mail/imap/tests/walktree.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/leap/mail/imap/tests') diff --git a/src/leap/mail/imap/tests/walktree.py b/src/leap/mail/imap/tests/walktree.py index f3cbcb0..695f487 100644 --- a/src/leap/mail/imap/tests/walktree.py +++ b/src/leap/mail/imap/tests/walktree.py @@ -36,11 +36,11 @@ p = parser.Parser() if len(sys.argv) > 1: FILENAME = sys.argv[1] else: - FILENAME = "rfc822.multi-minimal.message" + FILENAME = "rfc822.multi-signed.message" """ -FILENAME = "rfc822.multi-signed.message" FILENAME = "rfc822.plain.message" +FILENAME = "rfc822.multi-minimal.message" """ msg = p.parse(open(FILENAME)) -- cgit v1.2.3