diff options
Diffstat (limited to 'service')
| -rw-r--r-- | service/pixelated/adapter/mail.py | 6 | ||||
| -rw-r--r-- | service/pixelated/adapter/search.py | 56 | ||||
| -rw-r--r-- | service/pixelated/support/functional.py | 6 | ||||
| -rw-r--r-- | service/test/integration/search_test.py | 35 | ||||
| -rw-r--r-- | service/test/support/integration_helper.py | 16 |
5 files changed, 90 insertions, 29 deletions
diff --git a/service/pixelated/adapter/mail.py b/service/pixelated/adapter/mail.py index 5cafa36a..ca29687e 100644 --- a/service/pixelated/adapter/mail.py +++ b/service/pixelated/adapter/mail.py @@ -116,7 +116,7 @@ class InputMail(Mail): fd[fields.MULTIPART_KEY] = True fd[fields.RECENT_KEY] = True fd[fields.TYPE_KEY] = fields.TYPE_FLAGS_VAL - fd[fields.FLAGS_KEY] = Status.to_flags(self.status) + fd[fields.FLAGS_KEY] = Status.to_flags(self._status) self._fd = fd return fd @@ -170,7 +170,7 @@ class InputMail(Mail): input_mail.headers['Date'] = pixelated.support.date.iso_now() input_mail.body = mail_dict.get('body', '') input_mail.tags = set(mail_dict.get('tags', [])) - input_mail.status = set(mail_dict.get('status', [])) + input_mail._status = set(mail_dict.get('status', [])) return input_mail @@ -249,6 +249,8 @@ class PixelatedMail(Mail): return self.tags def mark_as_read(self): + if Status.SEEN in self.fdoc.content['flags']: + return self.fdoc.content['flags'].append(Status.SEEN) self.save() return self diff --git a/service/pixelated/adapter/search.py b/service/pixelated/adapter/search.py index ad5c3bbb..cd02f09b 100644 --- a/service/pixelated/adapter/search.py +++ b/service/pixelated/adapter/search.py @@ -1,8 +1,26 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. + import os +from pixelated.adapter.status import Status import whoosh.index from whoosh.fields import * from whoosh.qparser import QueryParser from whoosh import sorting +from pixelated.support.functional import unique class SearchEngine(object): @@ -16,25 +34,27 @@ class SearchEngine(object): os.makedirs(self.INDEX_FOLDER) self._index = self._create_index() - def _add_to_tags(self, tags, seen, skip_default_tags, count_type): - for tag, count in seen.iteritems(): - if skip_default_tags and tag in self.DEFAULT_TAGS: + def _add_to_tags(self, tags, group, skip_default_tags, count_type, query=None): + query_matcher = re.compile(query) if query else re.compile(".*") + + for tag, count in group.iteritems(): + + if skip_default_tags and tag in self.DEFAULT_TAGS or not query_matcher.match(tag): continue + if not tags.get(tag): tags[tag] = {'ident': tag, 'name': tag, 'default': False, 'counts': {'total': 0, 'read': 0}, 'mails': []} tags[tag]['counts'][count_type] += count - def _search_tag_groups(self, query): + def _search_tag_groups(self, is_filtering_tags): seen = None - query_string = (query + '*' if query else '*').lower() query_parser = QueryParser('tag', self._index.schema) options = {'limit': None, 'groupedby': sorting.FieldFacet('tag', allow_overlap=True), 'maptype': sorting.Count} with self._index.searcher() as searcher: - total = searcher.search(query_parser.parse(query_string), **options).groups() - if not query: - seen = searcher.search(query_parser.parse('* AND flags:\\Seen'), **options).groups() - + total = searcher.search(query_parser.parse('*'), **options).groups() + if not is_filtering_tags: + seen = searcher.search(query_parser.parse("* AND flags:%s" % Status.SEEN), **options).groups() return seen, total def _init_tags_defaults(self): @@ -52,18 +72,19 @@ class SearchEngine(object): } return tags - def _build_tags(self, seen, total, skip_default_tags): + def _build_tags(self, seen, total, skip_default_tags, query): tags = {} if not skip_default_tags: tags = self._init_tags_defaults() - self._add_to_tags(tags, total, skip_default_tags, count_type='total') + self._add_to_tags(tags, total, skip_default_tags, count_type='total', query=query) if seen: self._add_to_tags(tags, seen, skip_default_tags, count_type='read') return tags.values() def tags(self, query, skip_default_tags): - seen, total = self._search_tag_groups(query) - return self._build_tags(seen, total, skip_default_tags) + is_filtering_tags = True if query else False + seen, total = self._search_tag_groups(is_filtering_tags=is_filtering_tags) + return self._build_tags(seen, total, skip_default_tags, query) def _mail_schema(self): return Schema( @@ -74,8 +95,8 @@ class SearchEngine(object): bcc=ID(stored=False), subject=TEXT(stored=False), body=TEXT(stored=False), - tag=KEYWORD(stored=False, commas=True), - flags=KEYWORD(stored=False, commas=True), + tag=KEYWORD(stored=True, commas=True), + flags=KEYWORD(stored=True, commas=True), raw=TEXT(stored=False)) def _create_index(self): @@ -90,16 +111,17 @@ class SearchEngine(object): header = mdict['header'] tags = mdict.get('tags', []) tags.append(mail.mailbox_name.lower()) + index_data = { 'sender': unicode(header.get('from', '')), 'subject': unicode(header.get('subject', '')), 'to': unicode(header.get('to', '')), 'cc': unicode(header.get('cc', '')), 'bcc': unicode(header.get('bcc', '')), - 'tag': u','.join(tags), + 'tag': u','.join(unique(tags)), 'body': unicode(mdict['body']), 'ident': unicode(mdict['ident']), - 'flags': unicode(','.join(mail.flags)), + 'flags': unicode(','.join(unique(mail.flags))), 'raw': unicode(mail.raw) } diff --git a/service/pixelated/support/functional.py b/service/pixelated/support/functional.py index e779f5bc..8d258635 100644 --- a/service/pixelated/support/functional.py +++ b/service/pixelated/support/functional.py @@ -18,3 +18,9 @@ from itertools import chain def flatten(_list): return list(chain.from_iterable(_list)) + + +def unique(_list): + seen = set() + seen_add = seen.add + return [ x for x in _list if not (x in seen or seen_add(x))] diff --git a/service/test/integration/search_test.py b/service/test/integration/search_test.py index 81d1fad2..04151b55 100644 --- a/service/test/integration/search_test.py +++ b/service/test/integration/search_test.py @@ -27,7 +27,7 @@ class SearchTest(unittest.TestCase, SoledadTestBase): self.teardown_soledad() def test_that_tags_returns_all_tags(self): - input_mail = MailBuilder().with_tags('important').build_input_mail() + input_mail = MailBuilder().with_tags(['important']).build_input_mail() self.add_mail_to_inbox(input_mail) all_tags = self.get_tags() @@ -40,17 +40,19 @@ class SearchTest(unittest.TestCase, SoledadTestBase): self.assertTrue('important' in all_tag_names) def test_that_tags_are_filtered_by_query(self): - input_mail = MailBuilder().with_tags('mytag').build_input_mail() + input_mail = MailBuilder().with_tags(['ateu', 'atoa', 'atado', 'zuado']).build_input_mail() self.add_mail_to_inbox(input_mail) - all_tags = self.get_tags('?q=my&skipDefaultTags=true') + all_tags = self.get_tags('?q=at&skipDefaultTags=true') all_tag_names = [t['name'] for t in all_tags] - self.assertEqual(1, len(all_tag_names)) - self.assertTrue('mytag' in all_tag_names) + self.assertEqual(3, len(all_tag_names)) + self.assertTrue('ateu' in all_tag_names) + self.assertTrue('atoa' in all_tag_names) + self.assertTrue('atado' in all_tag_names) def test_that_default_tags_are_ignorable(self): - input_mail = MailBuilder().with_tags('sometag').build_input_mail() + input_mail = MailBuilder().with_tags(['sometag']).build_input_mail() self.add_mail_to_inbox(input_mail) all_tags = self.get_tags('?skipDefaultTags=true') @@ -59,6 +61,19 @@ class SearchTest(unittest.TestCase, SoledadTestBase): self.assertEqual(1, len(all_tag_names)) self.assertTrue('sometag' in all_tag_names) + def test_tags_count(self): + self.add_multiple_to_mailbox(num=10, mailbox='inbox', flags=['\\Recent']) + self.add_multiple_to_mailbox(num=5, mailbox='inbox', flags=['\\Seen']) + self.add_multiple_to_mailbox(num=3, mailbox='inbox', flags=['\\Recent'], tags=['important', 'later']) + self.add_multiple_to_mailbox(num=1, mailbox='inbox', flags=['\\Seen'], tags=['important']) + + tags_count = self.get_tags() + + self.assertEqual(self.get_count(tags_count, 'inbox')['total'], 19) + self.assertEqual(self.get_count(tags_count, 'inbox')['read'], 6) + self.assertEqual(self.get_count(tags_count, 'important')['total'], 4) + self.assertEqual(self.get_count(tags_count, 'important')['read'], 1) + def test_search_mails_different_window(self): input_mail = MailBuilder().build_input_mail() input_mail2 = MailBuilder().build_input_mail() @@ -68,7 +83,7 @@ class SearchTest(unittest.TestCase, SoledadTestBase): first_page = self.get_mails_by_tag('inbox', page=1, window=1) self.assertEqual(len(first_page), 1) - + def test_search_mails_with_multiple_pages(self): input_mail = MailBuilder().build_input_mail() input_mail2 = MailBuilder().build_input_mail() @@ -88,3 +103,9 @@ class SearchTest(unittest.TestCase, SoledadTestBase): self.add_mail_to_inbox(input_mail) page = self.get_mails_by_tag('inbox', page=0, window=1) self.assertEqual(page[0].ident, input_mail.ident) + + def get_count(self, tags_count, mailbox): + for tag in tags_count: + if tag['name'] == mailbox: + return tag['counts'] + diff --git a/service/test/support/integration_helper.py b/service/test/support/integration_helper.py index f0f3b02a..e6d9b96f 100644 --- a/service/test/support/integration_helper.py +++ b/service/test/support/integration_helper.py @@ -21,6 +21,7 @@ import os from mock import Mock from pixelated.adapter.mail_service import MailService from pixelated.adapter.search import SearchEngine +from pixelated.adapter.status import Status from pixelated.adapter.tag_service import TagService from pixelated.adapter.draft_service import DraftService from pixelated.adapter.mail import PixelatedMail, InputMail @@ -105,7 +106,7 @@ class MailBuilder: self.mail['body'] = body return self - def with_tags(self, *tags): + def with_tags(self, tags): self.mail['tags'] = tags return self @@ -113,8 +114,10 @@ class MailBuilder: self.mail['header']['subject'] = subject return self - def with_status(self, status): - self.mail['status'].append('read') + def with_status(self, flags): + for status in Status.from_flags(flags): + self.mail['status'].append(status) + return self def with_ident(self, ident): @@ -210,6 +213,13 @@ class SoledadTestBase: mail.update_tags(input_mail.tags) self.search_engine.index_mail(mail) + def add_multiple_to_mailbox(self, num, mailbox='', flags=[], tags=[]): + for _ in range(num): + input_mail = MailBuilder().with_status(flags).with_tags(tags).build_input_mail() + mail = self.mailboxes._create_or_get(mailbox).add(input_mail) + mail.update_tags(input_mail.tags) + self.search_engine.index_mail(mail) + class ResponseMail: def __init__(self, mail_dict): |