From 4a0e23cf0b94ce1d9e4d8664532989ddd994dc88 Mon Sep 17 00:00:00 2001 From: Folker Bernitt Date: Mon, 4 May 2015 10:36:21 +0200 Subject: Changed maintenance load mails format from mbox to maildir. - Can now import mails from maildir (allows import of old mail accounts, too) - Flags currently supported: Replied, Seen - Issue #336 --- service/pixelated/maintenance.py | 38 +++++++++++++--------- service/test/mailsets/common/cur/mbox00000002:2,S | 19 +++++++++++ service/test/mailsets/common/mbox00000000 | 12 ------- service/test/mailsets/common/mbox00000001 | 22 ------------- service/test/mailsets/common/new/mbox00000000:2, | 12 +++++++ service/test/mailsets/common/new/mbox00000001 | 22 +++++++++++++ service/test/mailsets/common/tmp/.keep | 0 service/test/unit/fixtures/mailset/cur/.keep | 0 service/test/unit/fixtures/mailset/mbox00000000 | 12 ------- service/test/unit/fixtures/mailset/mbox00000001 | 12 ------- .../test/unit/fixtures/mailset/new/mbox00000000 | 12 +++++++ .../test/unit/fixtures/mailset/new/mbox00000001 | 12 +++++++ service/test/unit/fixtures/mailset/tmp/.keep | 0 service/test/unit/maintenance/test_commands.py | 30 ++++++++++++++--- 14 files changed, 125 insertions(+), 78 deletions(-) create mode 100644 service/test/mailsets/common/cur/mbox00000002:2,S delete mode 100644 service/test/mailsets/common/mbox00000000 delete mode 100644 service/test/mailsets/common/mbox00000001 create mode 100644 service/test/mailsets/common/new/mbox00000000:2, create mode 100644 service/test/mailsets/common/new/mbox00000001 create mode 100644 service/test/mailsets/common/tmp/.keep create mode 100644 service/test/unit/fixtures/mailset/cur/.keep delete mode 100644 service/test/unit/fixtures/mailset/mbox00000000 delete mode 100644 service/test/unit/fixtures/mailset/mbox00000001 create mode 100644 service/test/unit/fixtures/mailset/new/mbox00000000 create mode 100644 service/test/unit/fixtures/mailset/new/mbox00000001 create mode 100644 service/test/unit/fixtures/mailset/tmp/.keep diff --git a/service/pixelated/maintenance.py b/service/pixelated/maintenance.py index 77d46be0..b4e616a7 100644 --- a/service/pixelated/maintenance.py +++ b/service/pixelated/maintenance.py @@ -23,7 +23,7 @@ import email import re from os.path import join -from mailbox import mboxMessage +from mailbox import mboxMessage, Maildir from pixelated.config.app import App from pixelated.config import app_factory from pixelated.config.args import parser_add_default_arguments @@ -40,6 +40,7 @@ from twisted.internet.threads import deferToThread from leap.mail.imap.memorystore import MemoryStore from leap.mail.imap.soledadstore import SoledadStore +from leap.mail.imap.fields import WithMsgFields from leap.common.events import register, unregister, events_pb2 as proto # monkey patching some specifics @@ -146,20 +147,34 @@ def delete_all_mails(args): return args +def add_mail_folder(account, maildir, folder_name, deferreds): + mbx = account.getMailbox(folder_name) + for mail in maildir: + flags = (WithMsgFields.RECENT_FLAG,) if mail.get_subdir() == 'new' else () + if 'S' in mail.get_flags(): + flags = (WithMsgFields.SEEN_FLAG,) + flags + if 'R' in mail.get_flags(): + flags = (WithMsgFields.ANSWERED_FLAG,) + flags + + deferreds.append(mbx.addMessage(mail.as_string(), flags=flags, notify_on_disk=False)) + + @defer.inlineCallbacks def load_mails(args, mail_paths): leap_session, soledad = args account = leap_session.account + deferreds = [] + for path in mail_paths: - print 'Loading mails from %s' % path - for root, dirs, files in os.walk(path): - mbx = account.getMailbox('INBOX') - for file_name in files: - yield add_message_into_mailbox(join(root, file_name), mbx) + maildir = Maildir(path, factory=None) + add_mail_folder(account, maildir, 'INBOX', deferreds) + for mail_folder_name in maildir.list_folders(): + mail_folder = maildir.get_folder(mail_folder_name) + add_mail_folder(account, mail_folder, mail_folder_name, deferreds) + yield defer.DeferredList(deferreds) defer.returnValue(args) - return def flush_to_soledad(args, finalize): @@ -181,15 +196,6 @@ def flush_to_soledad(args, finalize): return args -def add_message_into_mailbox(file_path, mbx): - with open(file_path, 'r') as email_file: - m = email.message_from_file(email_file) - flags = ("\\RECENT",) - print 'Added message %s' % m.get('subject') - print m.as_string() - return mbx.addMessage(m.as_string(), flags=flags, notify_on_disk=False) - - def dump_soledad(args): leap_session, soledad = args diff --git a/service/test/mailsets/common/cur/mbox00000002:2,S b/service/test/mailsets/common/cur/mbox00000002:2,S new file mode 100644 index 00000000..2881e352 --- /dev/null +++ b/service/test/mailsets/common/cur/mbox00000002:2,S @@ -0,0 +1,19 @@ +Subject: A read mail +From: bob@try.pixelated-project.org +To: alice@try.pixelated-project.org +X-Pixelated-encryption-status: true +Date: Tue, 08 Apr 2015 18:32:05 +0000 (UTC) + +Q: How many hardware engineers does it take to change a lightbulb? +A: None. We'll fix it in software. + +Q: How many system programmers does it take to change a light bulb? +A: None. The application can work around it. + +Q: How many software engineers does it take to change a lightbulb? +A: None. We'll document it in the manual. + +Q: How many tech writers does it take to change a lightbulb? +A: None. The user can figure it out. + + diff --git a/service/test/mailsets/common/mbox00000000 b/service/test/mailsets/common/mbox00000000 deleted file mode 100644 index c3fab833..00000000 --- a/service/test/mailsets/common/mbox00000000 +++ /dev/null @@ -1,12 +0,0 @@ -From bob@try.pixelated-project.org -Subject: Hey Alice -To: alice@try.pixelated-project.org -X-TW-Pixelated-Tags: cia -X-Pixelated-encryption-status: true -Date: Tue, 01 Apr 2015 08:43:27 +0000 (UTC) - -Yesterday upon the stair -I met a man who wasn't there. -He wasn't there again today -- -I think he's from the CIA. - diff --git a/service/test/mailsets/common/mbox00000001 b/service/test/mailsets/common/mbox00000001 deleted file mode 100644 index f80a35cb..00000000 --- a/service/test/mailsets/common/mbox00000001 +++ /dev/null @@ -1,22 +0,0 @@ -From bob@try.pixelated-project.org -Subject: Garbage collection works -To: alice@try.pixelated-project.org -X-TW-Pixelated-Tags: garbage -X-Pixelated-encryption-status: true -Date: Tue, 05 Apr 2015 13:52:05 +0000 (UTC) - -=== ALL USERS PLEASE NOTE ======================== - -The garbage collector now works. In addition a new, experimental garbage -collection algorithm has been installed. With SI:%DSK-GC-QLX-BITS set to 17, -(NOT the default) the old garbage collection algorithm remains in force; when -virtual storage is filled, the machine cold boots itself. With SI:%DSK-GC- -QLX-BITS set to 23, the new garbage collector is enabled. Unlike most garbage -collectors, the new gc starts its mark phase from the mind of the user, rather -than from the obarray. This allows the garbage collection of significantly -more Qs. As the garbage collector runs, it may ask you something like "Do you -remember what SI:RDTBL-TRANS does?", and if you can't give a reasonable answer -in thirty seconds, the symbol becomes a candidate for GCing. The variable -SI:%GC-QLX-LUSER-TM governs how long the GC waits before timing out the user. - - diff --git a/service/test/mailsets/common/new/mbox00000000:2, b/service/test/mailsets/common/new/mbox00000000:2, new file mode 100644 index 00000000..a35e63f0 --- /dev/null +++ b/service/test/mailsets/common/new/mbox00000000:2, @@ -0,0 +1,12 @@ +Subject: Hey Alice +From: bob@try.pixelated-project.org +To: alice@try.pixelated-project.org +X-TW-Pixelated-Tags: cia +X-Pixelated-encryption-status: true +Date: Tue, 01 Apr 2015 08:43:27 +0000 (UTC) + +Yesterday upon the stair +I met a man who wasn't there. +He wasn't there again today -- +I think he's from the CIA. + diff --git a/service/test/mailsets/common/new/mbox00000001 b/service/test/mailsets/common/new/mbox00000001 new file mode 100644 index 00000000..e1c6ba8b --- /dev/null +++ b/service/test/mailsets/common/new/mbox00000001 @@ -0,0 +1,22 @@ +Subject: Garbage collection works +From: bob@try.pixelated-project.org +To: alice@try.pixelated-project.org +X-TW-Pixelated-Tags: garbage +X-Pixelated-encryption-status: true +Date: Tue, 05 Apr 2015 13:52:05 +0000 (UTC) + +=== ALL USERS PLEASE NOTE ======================== + +The garbage collector now works. In addition a new, experimental garbage +collection algorithm has been installed. With SI:%DSK-GC-QLX-BITS set to 17, +(NOT the default) the old garbage collection algorithm remains in force; when +virtual storage is filled, the machine cold boots itself. With SI:%DSK-GC- +QLX-BITS set to 23, the new garbage collector is enabled. Unlike most garbage +collectors, the new gc starts its mark phase from the mind of the user, rather +than from the obarray. This allows the garbage collection of significantly +more Qs. As the garbage collector runs, it may ask you something like "Do you +remember what SI:RDTBL-TRANS does?", and if you can't give a reasonable answer +in thirty seconds, the symbol becomes a candidate for GCing. The variable +SI:%GC-QLX-LUSER-TM governs how long the GC waits before timing out the user. + + diff --git a/service/test/mailsets/common/tmp/.keep b/service/test/mailsets/common/tmp/.keep new file mode 100644 index 00000000..e69de29b diff --git a/service/test/unit/fixtures/mailset/cur/.keep b/service/test/unit/fixtures/mailset/cur/.keep new file mode 100644 index 00000000..e69de29b diff --git a/service/test/unit/fixtures/mailset/mbox00000000 b/service/test/unit/fixtures/mailset/mbox00000000 deleted file mode 100644 index 3d01c203..00000000 --- a/service/test/unit/fixtures/mailset/mbox00000000 +++ /dev/null @@ -1,12 +0,0 @@ -From darby.senger@zemlak.biz -Subject: Itaque consequatur repellendus provident sunt quia. -To: carmel@murazikortiz.name -X-TW-Pixelated-Tags: nite, macro, trash -Date: Tue, 21 Apr 2015 08:43:27 +0000 (UTC) - -Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas. - -Earum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est. - -Ut recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia. - diff --git a/service/test/unit/fixtures/mailset/mbox00000001 b/service/test/unit/fixtures/mailset/mbox00000001 deleted file mode 100644 index fc76bba2..00000000 --- a/service/test/unit/fixtures/mailset/mbox00000001 +++ /dev/null @@ -1,12 +0,0 @@ -From madeline.littel@sanfordruel.com -Subject: Error illum dignissimos autem eos aspernatur. -To: phyllis@stiedemann.net -X-TW-Pixelated-Tags: instadaily, inspiration -Date: Tue, 21 Apr 2015 08:43:27 +0000 (UTC) - -Et inventore placeat aut. Sint eveniet labore perferendis nulla. Maiores rerum sunt perferendis. Voluptate iure hic et ut blanditiis ad veritatis. Labore occaecati rerum. - -Sit fugiat aliquam voluptates ipsum non. Dolor quo sapiente. Itaque sed odit velit. Qui et aspernatur et fugiat voluptas eum est. Et expedita eos rerum nisi ut eum vero. - -Ab et est cumque. Qui nostrum perferendis. Labore est tempora porro est quia deleniti consequatur. Fugit quis ipsa. - diff --git a/service/test/unit/fixtures/mailset/new/mbox00000000 b/service/test/unit/fixtures/mailset/new/mbox00000000 new file mode 100644 index 00000000..3d01c203 --- /dev/null +++ b/service/test/unit/fixtures/mailset/new/mbox00000000 @@ -0,0 +1,12 @@ +From darby.senger@zemlak.biz +Subject: Itaque consequatur repellendus provident sunt quia. +To: carmel@murazikortiz.name +X-TW-Pixelated-Tags: nite, macro, trash +Date: Tue, 21 Apr 2015 08:43:27 +0000 (UTC) + +Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas. + +Earum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est. + +Ut recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia. + diff --git a/service/test/unit/fixtures/mailset/new/mbox00000001 b/service/test/unit/fixtures/mailset/new/mbox00000001 new file mode 100644 index 00000000..fc76bba2 --- /dev/null +++ b/service/test/unit/fixtures/mailset/new/mbox00000001 @@ -0,0 +1,12 @@ +From madeline.littel@sanfordruel.com +Subject: Error illum dignissimos autem eos aspernatur. +To: phyllis@stiedemann.net +X-TW-Pixelated-Tags: instadaily, inspiration +Date: Tue, 21 Apr 2015 08:43:27 +0000 (UTC) + +Et inventore placeat aut. Sint eveniet labore perferendis nulla. Maiores rerum sunt perferendis. Voluptate iure hic et ut blanditiis ad veritatis. Labore occaecati rerum. + +Sit fugiat aliquam voluptates ipsum non. Dolor quo sapiente. Itaque sed odit velit. Qui et aspernatur et fugiat voluptas eum est. Et expedita eos rerum nisi ut eum vero. + +Ab et est cumque. Qui nostrum perferendis. Labore est tempora porro est quia deleniti consequatur. Fugit quis ipsa. + diff --git a/service/test/unit/fixtures/mailset/tmp/.keep b/service/test/unit/fixtures/mailset/tmp/.keep new file mode 100644 index 00000000..e69de29b diff --git a/service/test/unit/maintenance/test_commands.py b/service/test/unit/maintenance/test_commands.py index 6c881cb9..6f993106 100644 --- a/service/test/unit/maintenance/test_commands.py +++ b/service/test/unit/maintenance/test_commands.py @@ -19,10 +19,12 @@ import email from pixelated.maintenance import delete_all_mails, load_mails from pixelated.bitmask_libraries.session import LeapSession from leap.mail.imap.account import SoledadBackedAccount +from leap.mail.imap.fields import WithMsgFields from leap.soledad.client import Soledad from leap.soledad.common.document import SoledadDocument from mock import MagicMock, ANY from os.path import join, dirname +from twisted.internet import defer, reactor class TestCommands(unittest.TestCase): @@ -78,13 +80,33 @@ class TestCommands(unittest.TestCase): self.assertFalse(self.mailbox.called) def test_load_mails_adds_mails(self): + # given mail_root = join(dirname(__file__), '..', 'fixtures', 'mailset') + firstMailDeferred = defer.Deferred() + secondMailDeferred = defer.Deferred() + self.mailbox.addMessage.side_effect = [firstMailDeferred, secondMailDeferred] - foo = load_mails(self.args, [mail_root]) + # when + d = load_mails(self.args, [mail_root]) - self.assertTrue(self.mailbox.addMessage.called) - self.mailbox.addMessage.assert_any_call(self._mail_content(join(mail_root, 'mbox00000000')), flags=("\\RECENT",), notify_on_disk=False) - self.mailbox.addMessage.assert_any_call(self._mail_content(join(mail_root, 'mbox00000001')), flags=("\\RECENT",), notify_on_disk=False) + # then + def assert_mails_added(_): + self.assertTrue(self.mailbox.addMessage.called) + self.mailbox.addMessage.assert_any_call(self._mail_content(join(mail_root, 'new', 'mbox00000000')), flags=(WithMsgFields.RECENT_FLAG,), notify_on_disk=False) + self.mailbox.addMessage.assert_any_call(self._mail_content(join(mail_root, 'new', 'mbox00000001')), flags=(WithMsgFields.RECENT_FLAG,), notify_on_disk=False) + + def error_callack(err): + print err + self.assertTrue(False) + + d.addCallback(assert_mails_added) + d.addErrback(error_callack) + + # trigger callbacks for both mails + reactor.callLater(0, firstMailDeferred.callback, None) + reactor.callLater(0, secondMailDeferred.callback, None) + + return d def _mail_content(self, mail_file): with open(mail_file, 'r') as fp: -- cgit v1.2.3