No more tag_index - now whoosh does everythin

5 files changed, 72 insertions, 46 deletions

diff --git a/service/pixelated/adapter/mail_service.py b/service/pixelated/adapter/mail_service.py
index d26c7c8f..9e29b0c3 100644
--- a/service/pixelated/adapter/mail_service.py
+++ b/service/pixelated/adapter/mail_service.py
@@ -25,7 +25,6 @@ class MailService:
         self.mailboxes = mailboxes
         self.querier = SoledadQuerier.get_instance()
         self.mail_sender = mail_sender
-        self.tag_service.load_index(self.all_mails())
 
     def all_mails(self):
         return self.querier.all_mails()
diff --git a/service/pixelated/adapter/pixelated_mail.py b/service/pixelated/adapter/pixelated_mail.py
index 8b07e996..f701e648 100644
--- a/service/pixelated/adapter/pixelated_mail.py
+++ b/service/pixelated/adapter/pixelated_mail.py
@@ -19,7 +19,6 @@ from leap.mail.imap.fields import fields
 import leap.mail.walk as walk
 import dateutil.parser as dateparser
 from pixelated.adapter.status import Status
-from pixelated.adapter.tag_service import TagService
 import pixelated.support.date
 from email.MIMEMultipart import MIMEMultipart
 from email.MIMEText import MIMEText
@@ -152,9 +151,6 @@ class InputMail:
 
 class PixelatedMail:
 
-    def __init__(self, tag_service=TagService.get_instance()):
-        self.tag_service = tag_service
-
     @staticmethod
     def from_soledad(fdoc, hdoc, bdoc, soledad_querier=None):
         mail = PixelatedMail()
@@ -194,7 +190,11 @@ class PixelatedMail:
 
     @property
     def status(self):
-        return Status.from_flags(self.fdoc.content.get('flags'))
+        return Status.from_flags(self._flags)
+
+    @property
+    def _flags(self):
+        return self.fdoc.content.get('flags')
 
     @property
     def security_casing(self):
@@ -249,11 +249,7 @@ class PixelatedMail:
         self.update_tags(set([]))
 
     def update_tags(self, tags):
-        old_tags = self.tags
         self._persist_mail_tags(tags)
-        removed = old_tags.difference(tags)
-        added = tags.difference(old_tags)
-        self.tag_service.notify_tags_updated(added, removed, self.ident)
         return self.tags
 
     def mark_as_read(self):
diff --git a/service/pixelated/adapter/pixelated_mailbox.py b/service/pixelated/adapter/pixelated_mailbox.py
index 0b88e07d..0ad79b11 100644
--- a/service/pixelated/adapter/pixelated_mailbox.py
+++ b/service/pixelated/adapter/pixelated_mailbox.py
@@ -14,14 +14,12 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
 
-from pixelated.adapter.tag_service import TagService
 from pixelated.adapter.soledad_querier import SoledadQuerier
 
 
 class PixelatedMailbox:
 
-    def __init__(self, mailbox_name, querier, tag_service=TagService.get_instance()):
-        self.tag_service = tag_service
+    def __init__(self, mailbox_name, querier):
         self.mailbox_name = mailbox_name
         self.mailbox_tag = mailbox_name.lower()
         self.querier = querier
diff --git a/service/pixelated/adapter/search.py b/service/pixelated/adapter/search.py
index 4a2fb097..df45b76e 100644
--- a/service/pixelated/adapter/search.py
+++ b/service/pixelated/adapter/search.py
@@ -2,18 +2,69 @@ import os
 import whoosh.index
 from whoosh.fields import *
 from whoosh.qparser import QueryParser
+from whoosh import sorting
 
 
 class SearchEngine(object):
     __slots__ = '_index'
 
     INDEX_FOLDER = os.path.join(os.environ['HOME'], '.leap', 'search_index')
+    DEFAULT_TAGS = ['inbox', 'sent', 'drafts', 'trash']
 
     def __init__(self):
         if not os.path.exists(self.INDEX_FOLDER):
             os.makedirs(self.INDEX_FOLDER)
         self._index = self._create_index()
 
+    def _add_to_tags(self, tags, seen, skip_default_tags):
+        for tag, count in seen.iteritems():
+            if skip_default_tags and tag in self.DEFAULT_TAGS:
+                continue
+            if not tags.get(tag):
+                tags[tag] = {'ident': tag, 'name': tag, 'default': False, 'counts': {'total': 0, 'read': 0}, 'mails': []}
+            tags[tag]['counts']['read'] += count
+
+    def _search_tag_groups(self, query):
+        seen = None
+        query_string = (query + '*' if query else '*').lower()
+        query_parser = QueryParser('tag', self._index.schema)
+        options = {'limit': None, 'groupedby': sorting.FieldFacet('tag', allow_overlap=True), 'maptype': sorting.Count}
+
+        with self._index.searcher() as searcher:
+            total = searcher.search(query_parser.parse(query_string), **options).groups()
+            if not query:
+                seen = searcher.search(query_parser.parse('* AND flags:\\Seen'), **options).groups()
+
+        return seen, total
+
+    def _init_tags_defaults(self):
+        tags = {}
+        for default_tag in self.DEFAULT_TAGS:
+            tags[default_tag] = {
+                'ident': default_tag,
+                'name': default_tag,
+                'default': True,
+                 'counts': {
+                     'total': 0,
+                     'read': 0
+                 },
+                 'mails': []
+            }
+        return tags
+
+    def _build_tags(self, seen, total, skip_default_tags):
+        tags = {}
+        if not skip_default_tags:
+            tags = self._init_tags_defaults()
+        self._add_to_tags(tags, total, skip_default_tags)
+        if seen:
+            self._add_to_tags(tags, seen, skip_default_tags)
+        return tags.values()
+
+    def tags(self, query, skip_default_tags):
+        seen, total = self._search_tag_groups(query)
+        return self._build_tags(seen, total, skip_default_tags)
+
     def _mail_schema(self):
         return Schema(
             ident=ID(stored=True, unique=True),
@@ -23,7 +74,8 @@ class SearchEngine(object):
             bcc=ID(stored=False),
             subject=TEXT(stored=False),
             body=TEXT(stored=False),
-            tag=KEYWORD(stored=False, commas=True))
+            tag=KEYWORD(stored=False, commas=True),
+            flags=KEYWORD(stored=False, commas=True))
 
     def _create_index(self):
         return whoosh.index.create_in(self.INDEX_FOLDER, self._mail_schema(), indexname='mails')
@@ -45,8 +97,10 @@ class SearchEngine(object):
             'bcc': unicode(header.get('bcc', '')),
             'tag': u','.join(tags),
             'body': unicode(mdict['body']),
-            'ident': unicode(mdict['ident'])
+            'ident': unicode(mdict['ident']),
+            'flags': unicode(','.join(mail._flags))
         }
+
         writer.update_document(**index_data)
 
     def index_mails(self, mails):
@@ -54,13 +108,22 @@ class SearchEngine(object):
             for mail in mails:
                 self._index_mail(writer, mail)
 
+    def _search_with_options(self, options, query):
+        with self._index.searcher() as searcher:
+            query = QueryParser('body', self._index.schema).parse(query)
+            results = searcher.search(query, **options)
+        return results
+
     def search(self, query):
+        options = {'limit': 100}
+
         query = query.replace('\"', '')
         query = query.replace('-in:', 'AND NOT tag:')
         query = query.replace('in:all', '*')
+
         with self._index.searcher() as searcher:
             query = QueryParser('body', self._index.schema).parse(query)
-            results = searcher.search(query, limit=100)
+            results = searcher.search(query, **options)
             return [mail['ident'] for mail in results]
 
     def remove_from_index(self, mail_id):
diff --git a/service/pixelated/adapter/tag_service.py b/service/pixelated/adapter/tag_service.py
index f128f1bb..0a8a1b68 100644
--- a/service/pixelated/adapter/tag_service.py
+++ b/service/pixelated/adapter/tag_service.py
@@ -32,33 +32,3 @@ class TagService:
             cls.instance = TagService()
         return cls.instance
 
-    def __init__(self, tag_index=TagIndex()):
-        self.tag_index = tag_index
-
-    def load_index(self, mails):
-        if self.tag_index.empty():
-            for mail in mails:
-                self.notify_tags_updated(mail.tags, [], mail.ident)
-        for tag in self.SPECIAL_TAGS:
-            self.tag_index.add(tag)
-
-    def notify_tags_updated(self, added_tags, removed_tags, mail_ident):
-        for removed_tag in removed_tags:
-            tag = self.tag_index.get(removed_tag)
-            if not tag:
-                continue
-            tag.decrement(mail_ident)
-            if tag.total == 0:
-                self.tag_index.remove(tag.name)
-            else:
-                self.tag_index.set(tag)
-        for added_tag in added_tags:
-            tag = self.tag_index.get(added_tag) or self.tag_index.add(Tag(added_tag))
-            tag.increment(mail_ident)
-            self.tag_index.set(tag)
-
-    def all_tags(self):
-        return self.tag_index.values().union(self.SPECIAL_TAGS)
-
-    def all_custom_tags(self):
-        return self.tag_index.values().difference(self.SPECIAL_TAGS)