Changed maintenance load mails format from mbox to maildir.

- Can now import mails from maildir (allows import of old mail accounts,
  too)
- Flags currently supported: Replied, Seen
- Issue #336

10 files changed, 69 insertions, 22 deletions

diff --git a/service/pixelated/maintenance.py b/service/pixelated/maintenance.py
index 77d46be0..b4e616a7 100644
--- a/service/pixelated/maintenance.py
+++ b/service/pixelated/maintenance.py
@@ -23,7 +23,7 @@ import email
 import re
 
 from os.path import join
-from mailbox import mboxMessage
+from mailbox import mboxMessage, Maildir
 from pixelated.config.app import App
 from pixelated.config import app_factory
 from pixelated.config.args import parser_add_default_arguments
@@ -40,6 +40,7 @@ from twisted.internet.threads import deferToThread
 
 from leap.mail.imap.memorystore import MemoryStore
 from leap.mail.imap.soledadstore import SoledadStore
+from leap.mail.imap.fields import WithMsgFields
 from leap.common.events import register, unregister, events_pb2 as proto
 
 # monkey patching some specifics
@@ -146,20 +147,34 @@ def delete_all_mails(args):
     return args
 
 
+def add_mail_folder(account, maildir, folder_name, deferreds):
+    mbx = account.getMailbox(folder_name)
+    for mail in maildir:
+        flags = (WithMsgFields.RECENT_FLAG,) if mail.get_subdir() == 'new' else ()
+        if 'S' in mail.get_flags():
+            flags = (WithMsgFields.SEEN_FLAG,) + flags
+        if 'R' in mail.get_flags():
+            flags = (WithMsgFields.ANSWERED_FLAG,) + flags
+
+        deferreds.append(mbx.addMessage(mail.as_string(), flags=flags, notify_on_disk=False))
+
+
 @defer.inlineCallbacks
 def load_mails(args, mail_paths):
     leap_session, soledad = args
     account = leap_session.account
 
+    deferreds = []
+
     for path in mail_paths:
-        print 'Loading mails from %s' % path
-        for root, dirs, files in os.walk(path):
-            mbx = account.getMailbox('INBOX')
-            for file_name in files:
-                yield add_message_into_mailbox(join(root, file_name), mbx)
+        maildir = Maildir(path, factory=None)
+        add_mail_folder(account, maildir, 'INBOX', deferreds)
+        for mail_folder_name in maildir.list_folders():
+            mail_folder = maildir.get_folder(mail_folder_name)
+            add_mail_folder(account, mail_folder, mail_folder_name, deferreds)
 
+    yield defer.DeferredList(deferreds)
     defer.returnValue(args)
-    return
 
 
 def flush_to_soledad(args, finalize):
@@ -181,15 +196,6 @@ def flush_to_soledad(args, finalize):
     return args
 
 
-def add_message_into_mailbox(file_path, mbx):
-    with open(file_path, 'r') as email_file:
-        m = email.message_from_file(email_file)
-        flags = ("\\RECENT",)
-        print 'Added message %s' % m.get('subject')
-        print m.as_string()
-        return mbx.addMessage(m.as_string(), flags=flags, notify_on_disk=False)
-
-
 def dump_soledad(args):
     leap_session, soledad = args
 
diff --git a/service/test/mailsets/common/cur/mbox00000002:2,S b/service/test/mailsets/common/cur/mbox00000002:2,S
new file mode 100644
index 00000000..2881e352
--- /dev/null
+++ b/service/test/mailsets/common/cur/mbox00000002:2,S
@@ -0,0 +1,19 @@
+Subject: A read mail
+From: bob@try.pixelated-project.org
+To: alice@try.pixelated-project.org
+X-Pixelated-encryption-status: true
+Date: Tue, 08 Apr 2015 18:32:05 +0000 (UTC)
+
+Q:	How many hardware engineers does it take to change a lightbulb?
+A:	None.  We'll fix it in software.
+
+Q:	How many system programmers does it take to change a light bulb?
+A:	None.  The application can work around it.
+
+Q:	How many software engineers does it take to change a lightbulb?
+A:	None.  We'll document it in the manual.
+
+Q:	How many tech writers does it take to change a lightbulb?
+A:	None.  The user can figure it out.
+
+
diff --git a/service/test/mailsets/common/mbox00000000 b/service/test/mailsets/common/new/mbox00000000:2,
index c3fab833..a35e63f0 100644
--- a/service/test/mailsets/common/mbox00000000
+++ b/service/test/mailsets/common/new/mbox00000000:2,
@@ -1,5 +1,5 @@
-From bob@try.pixelated-project.org
 Subject: Hey Alice
+From: bob@try.pixelated-project.org
 To: alice@try.pixelated-project.org
 X-TW-Pixelated-Tags: cia
 X-Pixelated-encryption-status: true
diff --git a/service/test/mailsets/common/mbox00000001 b/service/test/mailsets/common/new/mbox00000001
index f80a35cb..e1c6ba8b 100644
--- a/service/test/mailsets/common/mbox00000001
+++ b/service/test/mailsets/common/new/mbox00000001
@@ -1,5 +1,5 @@
-From bob@try.pixelated-project.org
 Subject: Garbage collection works
+From: bob@try.pixelated-project.org
 To: alice@try.pixelated-project.org
 X-TW-Pixelated-Tags: garbage
 X-Pixelated-encryption-status: true
diff --git a/service/test/mailsets/common/tmp/.keep b/service/test/mailsets/common/tmp/.keep
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/service/test/mailsets/common/tmp/.keep
diff --git a/service/test/unit/fixtures/mailset/cur/.keep b/service/test/unit/fixtures/mailset/cur/.keep
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/service/test/unit/fixtures/mailset/cur/.keep
diff --git a/service/test/unit/fixtures/mailset/mbox00000000 b/service/test/unit/fixtures/mailset/new/mbox00000000
index 3d01c203..3d01c203 100644
--- a/service/test/unit/fixtures/mailset/mbox00000000
+++ b/service/test/unit/fixtures/mailset/new/mbox00000000
diff --git a/service/test/unit/fixtures/mailset/mbox00000001 b/service/test/unit/fixtures/mailset/new/mbox00000001
index fc76bba2..fc76bba2 100644
--- a/service/test/unit/fixtures/mailset/mbox00000001
+++ b/service/test/unit/fixtures/mailset/new/mbox00000001
diff --git a/service/test/unit/fixtures/mailset/tmp/.keep b/service/test/unit/fixtures/mailset/tmp/.keep
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/service/test/unit/fixtures/mailset/tmp/.keep
diff --git a/service/test/unit/maintenance/test_commands.py b/service/test/unit/maintenance/test_commands.py
index 6c881cb9..6f993106 100644
--- a/service/test/unit/maintenance/test_commands.py
+++ b/service/test/unit/maintenance/test_commands.py
@@ -19,10 +19,12 @@ import email
 from pixelated.maintenance import delete_all_mails, load_mails
 from pixelated.bitmask_libraries.session import LeapSession
 from leap.mail.imap.account import SoledadBackedAccount
+from leap.mail.imap.fields import WithMsgFields
 from leap.soledad.client import Soledad
 from leap.soledad.common.document import SoledadDocument
 from mock import MagicMock, ANY
 from os.path import join, dirname
+from twisted.internet import defer, reactor
 
 
 class TestCommands(unittest.TestCase):
@@ -78,13 +80,33 @@ class TestCommands(unittest.TestCase):
         self.assertFalse(self.mailbox.called)
 
     def test_load_mails_adds_mails(self):
+        # given
         mail_root = join(dirname(__file__), '..', 'fixtures', 'mailset')
+        firstMailDeferred = defer.Deferred()
+        secondMailDeferred = defer.Deferred()
+        self.mailbox.addMessage.side_effect = [firstMailDeferred, secondMailDeferred]
 
-        foo = load_mails(self.args, [mail_root])
+        # when
+        d = load_mails(self.args, [mail_root])
 
-        self.assertTrue(self.mailbox.addMessage.called)
-        self.mailbox.addMessage.assert_any_call(self._mail_content(join(mail_root, 'mbox00000000')), flags=("\\RECENT",), notify_on_disk=False)
-        self.mailbox.addMessage.assert_any_call(self._mail_content(join(mail_root, 'mbox00000001')), flags=("\\RECENT",), notify_on_disk=False)
+        # then
+        def assert_mails_added(_):
+            self.assertTrue(self.mailbox.addMessage.called)
+            self.mailbox.addMessage.assert_any_call(self._mail_content(join(mail_root, 'new', 'mbox00000000')), flags=(WithMsgFields.RECENT_FLAG,), notify_on_disk=False)
+            self.mailbox.addMessage.assert_any_call(self._mail_content(join(mail_root, 'new', 'mbox00000001')), flags=(WithMsgFields.RECENT_FLAG,), notify_on_disk=False)
+
+        def error_callack(err):
+            print err
+            self.assertTrue(False)
+
+        d.addCallback(assert_mails_added)
+        d.addErrback(error_callack)
+
+        # trigger callbacks for both mails
+        reactor.callLater(0, firstMailDeferred.callback, None)
+        reactor.callLater(0, secondMailDeferred.callback, None)
+
+        return d
 
     def _mail_content(self, mail_file):
         with open(mail_file, 'r') as fp: