better tests for getting all tags and filtering all tags. some fixes go together

5 files changed, 90 insertions, 29 deletions

diff --git a/service/pixelated/adapter/mail.py b/service/pixelated/adapter/mail.py
index 5cafa36a..ca29687e 100644
--- a/service/pixelated/adapter/mail.py
+++ b/service/pixelated/adapter/mail.py
@@ -116,7 +116,7 @@ class InputMail(Mail):
         fd[fields.MULTIPART_KEY] = True
         fd[fields.RECENT_KEY] = True
         fd[fields.TYPE_KEY] = fields.TYPE_FLAGS_VAL
-        fd[fields.FLAGS_KEY] = Status.to_flags(self.status)
+        fd[fields.FLAGS_KEY] = Status.to_flags(self._status)
         self._fd = fd
         return fd
 
@@ -170,7 +170,7 @@ class InputMail(Mail):
         input_mail.headers['Date'] = pixelated.support.date.iso_now()
         input_mail.body = mail_dict.get('body', '')
         input_mail.tags = set(mail_dict.get('tags', []))
-        input_mail.status = set(mail_dict.get('status', []))
+        input_mail._status = set(mail_dict.get('status', []))
         return input_mail
 
 
@@ -249,6 +249,8 @@ class PixelatedMail(Mail):
         return self.tags
 
     def mark_as_read(self):
+        if Status.SEEN in self.fdoc.content['flags']:
+            return
         self.fdoc.content['flags'].append(Status.SEEN)
         self.save()
         return self
diff --git a/service/pixelated/adapter/search.py b/service/pixelated/adapter/search.py
index ad5c3bbb..cd02f09b 100644
--- a/service/pixelated/adapter/search.py
+++ b/service/pixelated/adapter/search.py
@@ -1,8 +1,26 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
 import os
+from pixelated.adapter.status import Status
 import whoosh.index
 from whoosh.fields import *
 from whoosh.qparser import QueryParser
 from whoosh import sorting
+from pixelated.support.functional import unique
 
 
 class SearchEngine(object):
@@ -16,25 +34,27 @@ class SearchEngine(object):
             os.makedirs(self.INDEX_FOLDER)
         self._index = self._create_index()
 
-    def _add_to_tags(self, tags, seen, skip_default_tags, count_type):
-        for tag, count in seen.iteritems():
-            if skip_default_tags and tag in self.DEFAULT_TAGS:
+    def _add_to_tags(self, tags, group, skip_default_tags, count_type, query=None):
+        query_matcher = re.compile(query) if query else re.compile(".*")
+
+        for tag, count in group.iteritems():
+
+            if skip_default_tags and tag in self.DEFAULT_TAGS or not query_matcher.match(tag):
                 continue
+
             if not tags.get(tag):
                 tags[tag] = {'ident': tag, 'name': tag, 'default': False, 'counts': {'total': 0, 'read': 0}, 'mails': []}
             tags[tag]['counts'][count_type] += count
 
-    def _search_tag_groups(self, query):
+    def _search_tag_groups(self, is_filtering_tags):
         seen = None
-        query_string = (query + '*' if query else '*').lower()
         query_parser = QueryParser('tag', self._index.schema)
         options = {'limit': None, 'groupedby': sorting.FieldFacet('tag', allow_overlap=True), 'maptype': sorting.Count}
 
         with self._index.searcher() as searcher:
-            total = searcher.search(query_parser.parse(query_string), **options).groups()
-            if not query:
-                seen = searcher.search(query_parser.parse('* AND flags:\\Seen'), **options).groups()
-
+            total = searcher.search(query_parser.parse('*'), **options).groups()
+            if not is_filtering_tags:
+                seen = searcher.search(query_parser.parse("* AND flags:%s" % Status.SEEN), **options).groups()
         return seen, total
 
     def _init_tags_defaults(self):
@@ -52,18 +72,19 @@ class SearchEngine(object):
             }
         return tags
 
-    def _build_tags(self, seen, total, skip_default_tags):
+    def _build_tags(self, seen, total, skip_default_tags, query):
         tags = {}
         if not skip_default_tags:
             tags = self._init_tags_defaults()
-        self._add_to_tags(tags, total, skip_default_tags, count_type='total')
+        self._add_to_tags(tags, total, skip_default_tags, count_type='total', query=query)
         if seen:
             self._add_to_tags(tags, seen, skip_default_tags, count_type='read')
         return tags.values()
 
     def tags(self, query, skip_default_tags):
-        seen, total = self._search_tag_groups(query)
-        return self._build_tags(seen, total, skip_default_tags)
+        is_filtering_tags = True if query else False
+        seen, total = self._search_tag_groups(is_filtering_tags=is_filtering_tags)
+        return self._build_tags(seen, total, skip_default_tags, query)
 
     def _mail_schema(self):
         return Schema(
@@ -74,8 +95,8 @@ class SearchEngine(object):
             bcc=ID(stored=False),
             subject=TEXT(stored=False),
             body=TEXT(stored=False),
-            tag=KEYWORD(stored=False, commas=True),
-            flags=KEYWORD(stored=False, commas=True),
+            tag=KEYWORD(stored=True, commas=True),
+            flags=KEYWORD(stored=True, commas=True),
             raw=TEXT(stored=False))
 
     def _create_index(self):
@@ -90,16 +111,17 @@ class SearchEngine(object):
         header = mdict['header']
         tags = mdict.get('tags', [])
         tags.append(mail.mailbox_name.lower())
+       
         index_data = {
             'sender': unicode(header.get('from', '')),
             'subject': unicode(header.get('subject', '')),
             'to': unicode(header.get('to', '')),
             'cc': unicode(header.get('cc', '')),
             'bcc': unicode(header.get('bcc', '')),
-            'tag': u','.join(tags),
+            'tag': u','.join(unique(tags)),
             'body': unicode(mdict['body']),
             'ident': unicode(mdict['ident']),
-            'flags': unicode(','.join(mail.flags)),
+            'flags': unicode(','.join(unique(mail.flags))),
             'raw': unicode(mail.raw)
         }
 
diff --git a/service/pixelated/support/functional.py b/service/pixelated/support/functional.py
index e779f5bc..8d258635 100644
--- a/service/pixelated/support/functional.py
+++ b/service/pixelated/support/functional.py
@@ -18,3 +18,9 @@ from itertools import chain
 
 def flatten(_list):
     return list(chain.from_iterable(_list))
+
+
+def unique(_list):
+    seen = set()
+    seen_add = seen.add
+    return [ x for x in _list if not (x in seen or seen_add(x))]
diff --git a/service/test/integration/search_test.py b/service/test/integration/search_test.py
index 81d1fad2..04151b55 100644
--- a/service/test/integration/search_test.py
+++ b/service/test/integration/search_test.py
@@ -27,7 +27,7 @@ class SearchTest(unittest.TestCase, SoledadTestBase):
         self.teardown_soledad()
 
     def test_that_tags_returns_all_tags(self):
-        input_mail = MailBuilder().with_tags('important').build_input_mail()
+        input_mail = MailBuilder().with_tags(['important']).build_input_mail()
         self.add_mail_to_inbox(input_mail)
 
         all_tags = self.get_tags()
@@ -40,17 +40,19 @@ class SearchTest(unittest.TestCase, SoledadTestBase):
         self.assertTrue('important' in all_tag_names)
 
     def test_that_tags_are_filtered_by_query(self):
-        input_mail = MailBuilder().with_tags('mytag').build_input_mail()
+        input_mail = MailBuilder().with_tags(['ateu', 'atoa', 'atado', 'zuado']).build_input_mail()
         self.add_mail_to_inbox(input_mail)
 
-        all_tags = self.get_tags('?q=my&skipDefaultTags=true')
+        all_tags = self.get_tags('?q=at&skipDefaultTags=true')
 
         all_tag_names = [t['name'] for t in all_tags]
-        self.assertEqual(1, len(all_tag_names))
-        self.assertTrue('mytag' in all_tag_names)
+        self.assertEqual(3, len(all_tag_names))
+        self.assertTrue('ateu' in all_tag_names)
+        self.assertTrue('atoa' in all_tag_names)
+        self.assertTrue('atado' in all_tag_names)
 
     def test_that_default_tags_are_ignorable(self):
-        input_mail = MailBuilder().with_tags('sometag').build_input_mail()
+        input_mail = MailBuilder().with_tags(['sometag']).build_input_mail()
         self.add_mail_to_inbox(input_mail)
 
         all_tags = self.get_tags('?skipDefaultTags=true')
@@ -59,6 +61,19 @@ class SearchTest(unittest.TestCase, SoledadTestBase):
         self.assertEqual(1, len(all_tag_names))
         self.assertTrue('sometag' in all_tag_names)
 
+    def test_tags_count(self):
+        self.add_multiple_to_mailbox(num=10, mailbox='inbox', flags=['\\Recent'])
+        self.add_multiple_to_mailbox(num=5, mailbox='inbox', flags=['\\Seen'])
+        self.add_multiple_to_mailbox(num=3, mailbox='inbox', flags=['\\Recent'], tags=['important', 'later'])
+        self.add_multiple_to_mailbox(num=1, mailbox='inbox', flags=['\\Seen'], tags=['important'])
+
+        tags_count = self.get_tags()
+
+        self.assertEqual(self.get_count(tags_count, 'inbox')['total'], 19)
+        self.assertEqual(self.get_count(tags_count, 'inbox')['read'], 6)
+        self.assertEqual(self.get_count(tags_count, 'important')['total'], 4)
+        self.assertEqual(self.get_count(tags_count, 'important')['read'], 1)
+
     def test_search_mails_different_window(self):
         input_mail = MailBuilder().build_input_mail()
         input_mail2 = MailBuilder().build_input_mail()
@@ -68,7 +83,7 @@ class SearchTest(unittest.TestCase, SoledadTestBase):
         first_page = self.get_mails_by_tag('inbox', page=1, window=1)
 
         self.assertEqual(len(first_page), 1)
-
+        
     def test_search_mails_with_multiple_pages(self):
         input_mail = MailBuilder().build_input_mail()
         input_mail2 = MailBuilder().build_input_mail()
@@ -88,3 +103,9 @@ class SearchTest(unittest.TestCase, SoledadTestBase):
         self.add_mail_to_inbox(input_mail)
         page = self.get_mails_by_tag('inbox', page=0, window=1)
         self.assertEqual(page[0].ident, input_mail.ident)
+
+    def get_count(self, tags_count, mailbox):
+        for tag in tags_count:
+            if tag['name'] == mailbox:
+                return tag['counts']
+
diff --git a/service/test/support/integration_helper.py b/service/test/support/integration_helper.py
index f0f3b02a..e6d9b96f 100644
--- a/service/test/support/integration_helper.py
+++ b/service/test/support/integration_helper.py
@@ -21,6 +21,7 @@ import os
 from mock import Mock
 from pixelated.adapter.mail_service import MailService
 from pixelated.adapter.search import SearchEngine
+from pixelated.adapter.status import Status
 from pixelated.adapter.tag_service import TagService
 from pixelated.adapter.draft_service import DraftService
 from pixelated.adapter.mail import PixelatedMail, InputMail
@@ -105,7 +106,7 @@ class MailBuilder:
         self.mail['body'] = body
         return self
 
-    def with_tags(self, *tags):
+    def with_tags(self, tags):
         self.mail['tags'] = tags
         return self
 
@@ -113,8 +114,10 @@ class MailBuilder:
         self.mail['header']['subject'] = subject
         return self
 
-    def with_status(self, status):
-        self.mail['status'].append('read')
+    def with_status(self, flags):
+        for status in Status.from_flags(flags):
+            self.mail['status'].append(status)
+
         return self
 
     def with_ident(self, ident):
@@ -210,6 +213,13 @@ class SoledadTestBase:
         mail.update_tags(input_mail.tags)
         self.search_engine.index_mail(mail)
 
+    def add_multiple_to_mailbox(self, num, mailbox='', flags=[], tags=[]):
+        for _ in range(num):
+            input_mail = MailBuilder().with_status(flags).with_tags(tags).build_input_mail()
+            mail = self.mailboxes._create_or_get(mailbox).add(input_mail)
+            mail.update_tags(input_mail.tags)
+            self.search_engine.index_mail(mail)
+
 
 class ResponseMail:
     def __init__(self, mail_dict):