Deferring TagsController.tags and fixing tests to deal with deferreds

13 files changed, 98 insertions, 70 deletions

diff --git a/service/go b/service/go
index 9b967b83..5ae8d26b 100755
--- a/service/go
+++ b/service/go
@@ -1,3 +1,9 @@
 #!/bin/bash
 
-python setup.py $*
+if [ "$1" == 'test' ] 
+then
+        nosetests --nocapture test/unit
+        nosetests --nocapture test/integration
+else
+        python setup.py $*
+fi
diff --git a/service/pixelated/adapter/search.py b/service/pixelated/adapter/search.py
index cd900f87..6c62ccc3 100644
--- a/service/pixelated/adapter/search.py
+++ b/service/pixelated/adapter/search.py
@@ -102,7 +102,7 @@ class SearchEngine(object):
             raw=TEXT(stored=False))
 
     def _create_index(self):
-        masterkey = self.soledad_querier.get_index_masterkey
+        masterkey = self.soledad_querier.get_index_masterkey()
         storage = EncryptedFileStorage(self.INDEX_FOLDER, masterkey)
         return FileIndex.create(storage, self._mail_schema(), indexname='mails')
 
diff --git a/service/pixelated/adapter/soledad_querier.py b/service/pixelated/adapter/soledad_querier.py
index 2e67a5ab..a11ff7a3 100644
--- a/service/pixelated/adapter/soledad_querier.py
+++ b/service/pixelated/adapter/soledad_querier.py
@@ -26,7 +26,6 @@ class SoledadQuerier:
     def __init__(self, soledad):
         self.soledad = soledad
 
-    @property
     def get_index_masterkey(self):
         index_key = self.soledad.get_from_index('by-type', 'index_key')
         if len(index_key) == 0:
diff --git a/service/pixelated/controllers/__init__.py b/service/pixelated/controllers/__init__.py
index 9e447d4d..e1c13515 100644
--- a/service/pixelated/controllers/__init__.py
+++ b/service/pixelated/controllers/__init__.py
@@ -22,6 +22,13 @@ def respond_json(entity, request, status_code=200):
     return json_response
 
 
+def respond_json_deferred(entity, request, status_code=200):
+    json_response = json.dumps(entity)
+    request.responseHeaders.addRawHeader(b"content-type", b"application/json")
+    request.code = status_code
+    request.write(json_response)
+    request.finish()
+
 import json
 
 from home_controller import HomeController
diff --git a/service/pixelated/controllers/tags_controller.py b/service/pixelated/controllers/tags_controller.py
index 0b9a94ac..b6741dcc 100644
--- a/service/pixelated/controllers/tags_controller.py
+++ b/service/pixelated/controllers/tags_controller.py
@@ -14,8 +14,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
 
-from flask import request
-from pixelated.controllers import respond_json
+from pixelated.controllers import respond_json_deferred
+from twisted.internet.threads import deferToThread
 
 
 class TagsController:
@@ -25,6 +25,9 @@ class TagsController:
 
     def tags(self, request):
         query = request.args.get('q', [''])[0]
-        skip_default_tags = request.args.get('skipDefaultTags')
-        tags = self._search_engine.tags(query=query, skip_default_tags=skip_default_tags)
-        return respond_json(tags, request)
+        skip_default_tags = request.args.get('skipDefaultTags', [False])[0]
+
+        d = deferToThread(lambda: self._search_engine.tags(query=query, skip_default_tags=skip_default_tags))
+        d.addCallback(lambda tags: respond_json_deferred(tags, request))
+
+        return d
diff --git a/service/test/integration/delete_mail_test.py b/service/test/integration/delete_mail_test.py
index 10c09e7a..69ab1f03 100644
--- a/service/test/integration/delete_mail_test.py
+++ b/service/test/integration/delete_mail_test.py
@@ -18,13 +18,13 @@ import unittest
 from test.support.integration_helper import MailBuilder, SoledadTestBase
 
 
-class DeleteMailTest(unittest.TestCase, SoledadTestBase):
+class DeleteMailTest(SoledadTestBase):
 
     def setUp(self):
-        self.setup_soledad()
+        SoledadTestBase.setUp(self)
 
     def tearDown(self):
-        self.teardown_soledad()
+        SoledadTestBase.tearDown(self)
 
     def test_move_mail_to_trash_when_deleting(self):
         input_mail = MailBuilder().with_subject('Mail with tags').build_input_mail()
diff --git a/service/test/integration/drafts_test.py b/service/test/integration/drafts_test.py
index 5d2118df..41fda0df 100644
--- a/service/test/integration/drafts_test.py
+++ b/service/test/integration/drafts_test.py
@@ -18,13 +18,13 @@ import unittest
 from test.support.integration_helper import MailBuilder, SoledadTestBase
 
 
-class DraftsTest(unittest.TestCase, SoledadTestBase):
+class DraftsTest(SoledadTestBase):
 
     def setUp(self):
-        self.setup_soledad()
+        SoledadTestBase.setUp(self)
 
     def tearDown(self):
-        self.teardown_soledad()
+        SoledadTestBase.tearDown(self)
 
     def test_post_sends_mail_and_deletes_previous_draft_if_it_exists(self):
         # creates one draft
diff --git a/service/test/integration/mark_as_read_unread_test.py b/service/test/integration/mark_as_read_unread_test.py
index dc21c7b7..c04cdbc6 100644
--- a/service/test/integration/mark_as_read_unread_test.py
+++ b/service/test/integration/mark_as_read_unread_test.py
@@ -19,13 +19,13 @@ from test.support.integration_helper import MailBuilder, SoledadTestBase
 from pixelated.adapter.status import Status
 
 
-class MarkAsReadUnreadTest(unittest.TestCase, SoledadTestBase):
+class MarkAsReadUnreadTest(SoledadTestBase):
 
     def setUp(self):
-        self.setup_soledad()
+        SoledadTestBase.setUp(self)
 
     def tearDown(self):
-        self.teardown_soledad()
+        SoledadTestBase.tearDown(self)
 
     def test_mark_single_as_read(self):
         input_mail = MailBuilder().build_input_mail()
diff --git a/service/test/integration/retrieve_attachment_test.py b/service/test/integration/retrieve_attachment_test.py
index 37bc3ca2..4e0e76e0 100644
--- a/service/test/integration/retrieve_attachment_test.py
+++ b/service/test/integration/retrieve_attachment_test.py
@@ -18,13 +18,13 @@ import unittest
 from test.support.integration_helper import SoledadTestBase
 
 
-class RetrieveAttachmentTest(unittest.TestCase, SoledadTestBase):
+class RetrieveAttachmentTest(SoledadTestBase):
 
     def setUp(self):
-        self.setup_soledad()
+        SoledadTestBase.setUp(self)
 
     def tearDown(self):
-        self.teardown_soledad()
+        SoledadTestBase.tearDown(self)
 
     def test_attachment_content_is_retrieved(self):
         ident = 'F4E99C1CEC4D300A4223A96CCABBE0304BDBC31C550A5A03E207A5E4C3C71A22'
diff --git a/service/test/integration/search_test.py b/service/test/integration/search_test.py
index 649f7b96..95b510e6 100644
--- a/service/test/integration/search_test.py
+++ b/service/test/integration/search_test.py
@@ -13,66 +13,80 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
-import unittest
 
+from nose.twistedtools import deferred
 from test.support.integration_helper import MailBuilder, SoledadTestBase
 
 
-class SearchTest(unittest.TestCase, SoledadTestBase):
+class SearchTest(SoledadTestBase):
 
     def setUp(self):
-        self.setup_soledad()
-
-    def tearDown(self):
-        self.teardown_soledad()
+        SoledadTestBase.setUp(self)
 
+    @deferred(timeout=5)
     def test_that_tags_returns_all_tags(self):
         input_mail = MailBuilder().with_tags(['important']).build_input_mail()
         self.add_mail_to_inbox(input_mail)
 
-        all_tags = self.get_tags()
+        d = self.get_tags()
 
-        all_tag_names = [t['name'] for t in all_tags]
-        self.assertTrue('inbox' in all_tag_names)
-        self.assertTrue('sent' in all_tag_names)
-        self.assertTrue('trash' in all_tag_names)
-        self.assertTrue('drafts' in all_tag_names)
-        self.assertTrue('important' in all_tag_names)
+        def _assert(all_tags):
+            all_tag_names = [t['name'] for t in all_tags]
+            self.assertTrue('inbox' in all_tag_names)
+            self.assertTrue('sent' in all_tag_names)
+            self.assertTrue('trash' in all_tag_names)
+            self.assertTrue('drafts' in all_tag_names)
+            self.assertTrue('important' in all_tag_names)
+        d.addCallback(_assert)
+        return d
 
+    @deferred(timeout=5)
     def test_that_tags_are_filtered_by_query(self):
         input_mail = MailBuilder().with_tags(['ateu', 'catoa', 'luat', 'zuado']).build_input_mail()
         self.add_mail_to_inbox(input_mail)
 
-        all_tags = self.get_tags(q=["at"], skipDefaultTags=["true"])
+        d = self.get_tags(q=["at"], skipDefaultTags=["true"])
+
+        def _assert(all_tags):
+            all_tag_names = [t['name'] for t in all_tags]
+            self.assertEqual(3, len(all_tag_names))
+            self.assertTrue('ateu' in all_tag_names)
+            self.assertTrue('catoa' in all_tag_names)
+            self.assertTrue('luat' in all_tag_names)
 
-        all_tag_names = [t['name'] for t in all_tags]
-        self.assertEqual(3, len(all_tag_names))
-        self.assertTrue('ateu' in all_tag_names)
-        self.assertTrue('catoa' in all_tag_names)
-        self.assertTrue('luat' in all_tag_names)
+        d.addCallback(_assert)
+        return d
 
+    @deferred(timeout=5)
     def test_that_default_tags_are_ignorable(self):
         input_mail = MailBuilder().with_tags(['sometag']).build_input_mail()
         self.add_mail_to_inbox(input_mail)
 
-        all_tags = self.get_tags(skipDefaultTags=["true"])
+        d = self.get_tags(skipDefaultTags=["true"])
 
-        all_tag_names = [t['name'] for t in all_tags]
-        self.assertEqual(1, len(all_tag_names))
-        self.assertTrue('sometag' in all_tag_names)
+        def _assert(all_tags):
+            all_tag_names = [t['name'] for t in all_tags]
+            self.assertEqual(1, len(all_tag_names))
+            self.assertTrue('sometag' in all_tag_names)
+        d.addCallback(_assert)
+        return d
 
+    @deferred(timeout=5)
     def test_tags_count(self):
         self.add_multiple_to_mailbox(num=10, mailbox='inbox', flags=['\\Recent'])
         self.add_multiple_to_mailbox(num=5, mailbox='inbox', flags=['\\Seen'])
         self.add_multiple_to_mailbox(num=3, mailbox='inbox', flags=['\\Recent'], tags=['important', 'later'])
         self.add_multiple_to_mailbox(num=1, mailbox='inbox', flags=['\\Seen'], tags=['important'])
 
-        tags_count = self.get_tags()
+        d = self.get_tags()
 
-        self.assertEqual(self.get_count(tags_count, 'inbox')['total'], 19)
-        self.assertEqual(self.get_count(tags_count, 'inbox')['read'], 6)
-        self.assertEqual(self.get_count(tags_count, 'important')['total'], 4)
-        self.assertEqual(self.get_count(tags_count, 'important')['read'], 1)
+        def _assert(tags_count):
+            self.assertEqual(self.get_count(tags_count, 'inbox')['total'], 19)
+            self.assertEqual(self.get_count(tags_count, 'inbox')['read'], 6)
+            self.assertEqual(self.get_count(tags_count, 'important')['total'], 4)
+            self.assertEqual(self.get_count(tags_count, 'important')['read'], 1)
+        d.addCallback(_assert)
+        return d
 
     def test_search_mails_different_window(self):
         input_mail = MailBuilder().build_input_mail()
diff --git a/service/test/integration/soledad_querier_test.py b/service/test/integration/soledad_querier_test.py
index 5d9bcccd..e8be431b 100644
--- a/service/test/integration/soledad_querier_test.py
+++ b/service/test/integration/soledad_querier_test.py
@@ -21,13 +21,13 @@ from test.support.integration_helper import SoledadTestBase, MailBuilder
 from leap.mail.imap.fields import WithMsgFields
 
 
-class SoledadQuerierTest(unittest.TestCase, SoledadTestBase, WithMsgFields):
+class SoledadQuerierTest(SoledadTestBase, WithMsgFields):
 
     def setUp(self):
-        self.setup_soledad()
+        SoledadTestBase.setUp(self)
 
     def tearDown(self):
-        self.teardown_soledad()
+        SoledadTestBase.tearDown(self)
 
     def _get_empty_mailbox(self):
         return copy.deepcopy(self.EMPTY_MBOX)
diff --git a/service/test/integration/tags_test.py b/service/test/integration/tags_test.py
index bd22e4b5..bb3bdbdb 100644
--- a/service/test/integration/tags_test.py
+++ b/service/test/integration/tags_test.py
@@ -19,13 +19,13 @@ import unittest
 from test.support.integration_helper import MailBuilder, SoledadTestBase
 
 
-class TagsTest(unittest.TestCase, SoledadTestBase):
+class TagsTest(SoledadTestBase):
 
     def setUp(self):
-        self.setup_soledad()
+        SoledadTestBase.setUp(self)
 
     def tearDown(self):
-        self.teardown_soledad()
+        SoledadTestBase.tearDown(self)
 
     def _tags_json(self, tags):
         return json.dumps({'newtags': tags})
diff --git a/service/test/support/integration_helper.py b/service/test/support/integration_helper.py
index 0411abea..0008d33f 100644
--- a/service/test/support/integration_helper.py
+++ b/service/test/support/integration_helper.py
@@ -14,8 +14,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
 import shutil
-from klein.resource import KleinResource
 
+from klein.resource import KleinResource
 from leap.soledad.client import Soledad
 from mockito import mock
 import os
@@ -33,7 +33,8 @@ from pixelated.controllers import *
 import pixelated.config.app_factory as app_factory
 from leap.mail.imap.account import SoledadBackedAccount
 from klein.test_resource import requestMock, _render
-
+from nose.twistedtools import stop_reactor, threaded_reactor
+from twisted.internet.error import ReactorNotRestartable
 
 soledad_test_folder = "soledad-test"
 
@@ -68,12 +69,6 @@ def initialize_soledad(tempdir):
         local_db_path,
         server_url,
         cert_file)
-    #
-    # from leap.mail.imap.fields import fields
-    #
-    # for name, expression in fields.INDEXES.items():
-    # _soledad.create_index(name, *expression)
-    #
     return _soledad
 
 
@@ -122,14 +117,16 @@ class MailBuilder:
         return InputMail.from_dict(self.mail)
 
 
-class SoledadTestBase:
-    def __init__(self):
-        pass
+import unittest
 
-    def teardown_soledad(self):
-        pass
 
-    def setup_soledad(self):
+class SoledadTestBase(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        threaded_reactor()
+
+    def setUp(self):
         self.soledad = initialize_soledad(tempdir=soledad_test_folder)
         self.mail_address = "test@pixelated.org"
 
@@ -141,6 +138,7 @@ class SoledadTestBase:
         self.app = pixelated.runserver.app
 
         self.soledad_querier = SoledadQuerier(self.soledad)
+        self.soledad_querier.get_index_masterkey = lambda: '_yg2oG_5ELM8_-sQYcsxI37WesI0dOtZQXpwAqjvhR4='
 
         self.account = SoledadBackedAccount('test', self.soledad, MagicMock())
         self.mailboxes = Mailboxes(self.account, self.soledad_querier)
@@ -206,8 +204,9 @@ class SoledadTestBase:
     def get_tags(self, **kwargs):
         request = requestMock('/tags')
         request.args = kwargs
-        _render(self.resource, request)
-        return json.loads(request.getWrittenData())
+        d = _render(self.resource, request)
+        d.addCallback(lambda _: json.loads(request.getWrittenData()))
+        return d
 
     def delete_mail(self, mail_ident):
         request = requestMock(path='/mail/' + mail_ident, method="DELETE")