Added load-mails maintenance command.

- Issue #335

4 files changed, 110 insertions, 4 deletions

diff --git a/service/pixelated/maintenance.py b/service/pixelated/maintenance.py
index 9a621476..e5b5ce6d 100644
--- a/service/pixelated/maintenance.py
+++ b/service/pixelated/maintenance.py
@@ -14,11 +14,15 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
 
+import os
 from functools import partial
 import sys
 import json
 import argparse
+import email
 
+from os.path import join
+from mailbox import mboxMessage
 from pixelated.config.app import App
 from pixelated.config import app_factory
 from pixelated.config.args import parser_add_default_arguments
@@ -32,6 +36,11 @@ from pixelated.config.leap_cert import init_leap_cert
 from pixelated.config.soledad import init_soledad_and_user_key
 from twisted.internet import reactor, defer
 from twisted.internet.threads import deferToThread
+from progressbar import ProgressBar
+
+from leap.mail.imap.memorystore import MemoryStore
+from leap.mail.imap.soledadstore import SoledadStore
+from leap.common.events import register, unregister, events_pb2 as proto
 
 # monkey patching some specifics
 import pixelated.support.ext_protobuf
@@ -53,14 +62,50 @@ def delete_all_mails(args):
     return args
 
 
+@defer.inlineCallbacks
+def load_mails(args, mail_paths):
+    leap_session, soledad = args
+    account = leap_session.account
+
+    for path in mail_paths:
+        print 'Loading mails from %s' % path
+        for root, dirs, files in os.walk(path):
+            mbx = account.getMailbox('INBOX')
+            for f in files:
+                with open(join(root, f), 'r') as fp:
+                    m = email.message_from_file(fp)
+                    flags = ("\\RECENT",)
+                    r = yield mbx.addMessage(m.as_string(), flags=flags, notify_on_disk=False)
+                    print 'Added message %s' % m.get('subject')
+                    print m.as_string()
+
+    defer.returnValue(args)
+    return
+
+
+def dump_soledad(args):
+    leap_session, soledad = args
+
+    generation, docs = soledad.get_all_docs()
+
+    for doc in docs:
+        print doc
+        print '\n'
+
+    return args
+
+
 def initialize():
     parser = argparse.ArgumentParser(description='pixelated maintenance')
     parser_add_default_arguments(parser)
-    subparsers = parser.add_subparsers(help='commands')
+    subparsers = parser.add_subparsers(help='commands', dest='command')
     subparsers.add_parser('reset', help='reset account command')
     mails_parser = subparsers.add_parser('load-mails', help='load mails into account')
     mails_parser.add_argument('file', nargs='+', help='file(s) with mail data')
 
+    subparsers.add_parser('dump-soledad', help='dump the soledad database')
+    subparsers.add_parser('sync', help='sync the soledad database')
+
     args = parser.parse_args()
     app = App()
 
@@ -94,7 +139,17 @@ def initialize():
         d = deferToThread(init_soledad)
         d.addCallback(get_soledad_handle)
         d.addCallback(soledad_sync)
-        d.addCallback(delete_all_mails)
+        if args.command == 'reset':
+            d.addCallback(delete_all_mails)
+        elif args.command == 'load-mails':
+            d.addCallback(load_mails, args.file)
+        elif args.command == 'dump-soledad':
+            d.addCallback(dump_soledad)
+        elif args.command == 'sync':
+            # nothing to do here, sync is already part of the chain
+            pass
+        else:
+            print 'Unsupported command: %s' % args.command
         d.addCallback(soledad_sync)
         d.addCallback(shutdown)
         d.addErrback(shutdown_on_error)
diff --git a/service/test/unit/fixtures/mailset/mbox00000000 b/service/test/unit/fixtures/mailset/mbox00000000
new file mode 100644
index 00000000..3d01c203
--- /dev/null
+++ b/service/test/unit/fixtures/mailset/mbox00000000
@@ -0,0 +1,12 @@
+From darby.senger@zemlak.biz
+Subject: Itaque consequatur repellendus provident sunt quia.
+To: carmel@murazikortiz.name
+X-TW-Pixelated-Tags: nite, macro, trash
+Date: Tue, 21 Apr 2015 08:43:27 +0000 (UTC)
+
+Dignissimos ducimus veritatis. Est tenetur consequatur quia occaecati. Vel sit sit voluptas.
+
+Earum distinctio eos. Accusantium qui sint ut quia assumenda. Facere dignissimos inventore autem sit amet. Pariatur voluptatem sint est.
+
+Ut recusandae praesentium aspernatur. Exercitationem amet placeat deserunt quae consequatur eum. Unde doloremque suscipit quia.
+
diff --git a/service/test/unit/fixtures/mailset/mbox00000001 b/service/test/unit/fixtures/mailset/mbox00000001
new file mode 100644
index 00000000..fc76bba2
--- /dev/null
+++ b/service/test/unit/fixtures/mailset/mbox00000001
@@ -0,0 +1,12 @@
+From madeline.littel@sanfordruel.com
+Subject: Error illum dignissimos autem eos aspernatur.
+To: phyllis@stiedemann.net
+X-TW-Pixelated-Tags: instadaily, inspiration
+Date: Tue, 21 Apr 2015 08:43:27 +0000 (UTC)
+
+Et inventore placeat aut. Sint eveniet labore perferendis nulla. Maiores rerum sunt perferendis. Voluptate iure hic et ut blanditiis ad veritatis. Labore occaecati rerum.
+
+Sit fugiat aliquam voluptates ipsum non. Dolor quo sapiente. Itaque sed odit velit. Qui et aspernatur et fugiat voluptas eum est. Et expedita eos rerum nisi ut eum vero.
+
+Ab et est cumque. Qui nostrum perferendis. Labore est tempora porro est quia deleniti consequatur. Fugit quis ipsa.
+
diff --git a/service/test/unit/maintenance/test_commands.py b/service/test/unit/maintenance/test_commands.py
index a20b0383..f23655d8 100644
--- a/service/test/unit/maintenance/test_commands.py
+++ b/service/test/unit/maintenance/test_commands.py
@@ -14,12 +14,15 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
 import unittest
+import email
 
-from pixelated.maintenance import delete_all_mails
+from pixelated.maintenance import delete_all_mails, load_mails
 from pixelated.bitmask_libraries.session import LeapSession
+from leap.mail.imap.account import SoledadBackedAccount
 from leap.soledad.client import Soledad
 from leap.soledad.common.document import SoledadDocument
-from mock import MagicMock
+from mock import MagicMock, ANY
+from os.path import join, dirname
 
 
 class TestCommands(unittest.TestCase):
@@ -27,6 +30,11 @@ class TestCommands(unittest.TestCase):
         def setUp(self):
             self.leap_session = MagicMock(spec=LeapSession)
             self.soledad = MagicMock(spec=Soledad)
+            self.account = MagicMock(spec=SoledadBackedAccount)
+            self.mailbox = MagicMock()
+            self.leap_session.account = self.account
+            self.account.getMailbox.return_value = self.mailbox
+
             self.args = (self.leap_session, self.soledad)
 
         def test_delete_all_mails_supports_empty_doclist(self):
@@ -63,3 +71,22 @@ class TestCommands(unittest.TestCase):
             doc = MagicMock(spec=SoledadDocument)
             doc.content = {'type': doc_type}
             return doc
+
+        def test_load_mails_empty_path_list(self):
+            load_mails(self.args, [])
+
+            self.assertFalse(self.mailbox.called)
+
+        def test_load_mails_adds_mails(self):
+            mail_root = join(dirname(__file__), '..', 'fixtures', 'mailset')
+
+            foo = load_mails(self.args, [mail_root])
+
+            self.assertTrue(self.mailbox.addMessage.called)
+            self.mailbox.addMessage.assert_any_call(self._mail_content(join(mail_root, 'mbox00000000')), flags=("\\RECENT",), notify_on_disk=False)
+            self.mailbox.addMessage.assert_any_call(self._mail_content(join(mail_root, 'mbox00000001')), flags=("\\RECENT",), notify_on_disk=False)
+
+        def _mail_content(self, mail_file):
+            with open(mail_file, 'r') as fp:
+                m = email.message_from_file(fp)
+                return m.as_string()