Added whoosh as search engine (issue #94).

6 files changed, 113 insertions, 39 deletions

diff --git a/service/pixelated/adapter/draft_service.py b/service/pixelated/adapter/draft_service.py
new file mode 100644
index 00000000..d9d6b12f
--- /dev/null
+++ b/service/pixelated/adapter/draft_service.py
@@ -0,0 +1,19 @@
+
+
+class DraftService(object):
+    __slots__ = '_mailboxes'
+
+    def __init__(self, mailboxes):
+        self._mailboxes = mailboxes
+
+    def create_draft(self, input_mail):
+        self._drafts().add(input_mail)
+        return input_mail
+
+    def update_draft(self, ident, input_mail):
+        new_mail = self.create_draft(input_mail)
+        self._drafts().remove(ident)
+        return new_mail
+
+    def _drafts(self):
+        return self._mailboxes.drafts()
diff --git a/service/pixelated/adapter/mail_service.py b/service/pixelated/adapter/mail_service.py
index 23afb41b..ad494ac8 100644
--- a/service/pixelated/adapter/mail_service.py
+++ b/service/pixelated/adapter/mail_service.py
@@ -20,18 +20,18 @@ from pixelated.adapter.soledad_querier import SoledadQuerier
 class MailService:
     __slots__ = ['leap_session', 'account', 'mailbox_name']
 
-    ALL_MAILS_QUERY = {'tags': ['all']}
-
     def __init__(self, mailboxes, mail_sender, tag_service=TagService.get_instance()):
         self.tag_service = tag_service
         self.mailboxes = mailboxes
         self.querier = SoledadQuerier.get_instance()
         self.mail_sender = mail_sender
-        self.tag_service.load_index(self.mails(MailService.ALL_MAILS_QUERY))
+        self.tag_service.load_index(self.all_mails())
+
+    def all_mails(self):
+        return self.querier.all_mails()
 
-    def mails(self, query):
-        _mails = self.mailboxes.mails_by_tag(query['tags']) if query['tags'] else self.querier.all_mails()
-        return sorted(_mails or [], key=lambda mail: mail.headers['Date'], reverse=True)
+    def mails(self, ids):
+        return self.querier.mails(ids)
 
     def update_tags(self, mail_id, new_tags):
         reserved_words = self.tag_service.extract_reserved(new_tags)
@@ -46,19 +46,7 @@ class MailService:
     def send(self, last_draft_ident, mail):
         self.mail_sender.sendmail(mail)
         self.mailboxes.drafts().remove(last_draft_ident)
-        self.mailboxes.sent().add(mail)
-
-    def create_draft(self, mail):
-        return self.mailboxes.add_draft(mail)
-
-    def update_draft(self, ident, new_version):
-        return self.mailboxes.update_draft(ident, new_version)
-
-    def send_draft(self, mail):
-        pass
-
-    def all_tags(self):
-        return self.tag_service.all_tags()
+        return self.mailboxes.sent().add(mail)
 
     def thread(self, thread_id):
         raise NotImplementedError()
diff --git a/service/pixelated/adapter/pixelated_mailbox.py b/service/pixelated/adapter/pixelated_mailbox.py
index e8c063e8..0b88e07d 100644
--- a/service/pixelated/adapter/pixelated_mailbox.py
+++ b/service/pixelated/adapter/pixelated_mailbox.py
@@ -45,13 +45,7 @@ class PixelatedMailbox:
                 return message
 
     def add(self, mail):
-        self.querier.create_mail(mail, self.mailbox_name)
-
-    def add_existing(self, mail_ident):
-        mail = self.querier.mail(mail_ident)
-        mail.remove_all_tags()
-        mail.set_mailbox(self.mailbox_name)
-        mail.save()
+        return self.querier.create_mail(mail, self.mailbox_name)
 
     def remove(self, ident):
         mail = self.querier.mail(ident)
diff --git a/service/pixelated/adapter/pixelated_mailboxes.py b/service/pixelated/adapter/pixelated_mailboxes.py
index aa39a5c4..c87b7ab3 100644
--- a/service/pixelated/adapter/pixelated_mailboxes.py
+++ b/service/pixelated/adapter/pixelated_mailboxes.py
@@ -14,12 +14,14 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
 from pixelated.adapter.pixelated_mailbox import PixelatedMailbox
+from pixelated.adapter.soledad_querier import SoledadQuerier
 
 
 class PixelatedMailBoxes():
 
     def __init__(self, account):
         self.account = account
+        self.querier = SoledadQuerier.get_instance()
 
     def _create_or_get(self, mailbox_name):
         mailbox_name = mailbox_name.upper()
@@ -51,18 +53,12 @@ class PixelatedMailBoxes():
 
         return mails
 
-    def add_draft(self, mail):
-        self.drafts().add(mail)
-        return mail
-
-    def update_draft(self, ident, new_version):
-        new_mail = self.add_draft(new_version)
-        self.drafts().remove(ident)
-        return new_mail
-
     def move_to_trash(self, mail_id):
-        new_mail_id = self.trash().add_existing(mail_id)
-        return new_mail_id
+        mail = self.querier.mail(mail_id)
+        mail.remove_all_tags()
+        mail.set_mailbox(self.trash().mailbox_name)
+        mail.save()
+        return mail
 
     def mail(self, mail_id):
         for mailbox in self.mailboxes:
diff --git a/service/pixelated/adapter/search.py b/service/pixelated/adapter/search.py
new file mode 100644
index 00000000..6c64cc40
--- /dev/null
+++ b/service/pixelated/adapter/search.py
@@ -0,0 +1,71 @@
+from whoosh.fields import *
+from whoosh.filedb.filestore import RamStorage
+from whoosh.qparser import QueryParser
+
+
+class SearchEngine(object):
+    __slots__ = '_index'
+
+    def __init__(self):
+        self._index = self._create_index()
+
+    def _mail_schema(self):
+        return Schema(
+            ident=ID(stored=True, unique=True),
+            sender=ID(stored=False),
+            to=ID(stored=False),
+            cc=ID(stored=False),
+            bcc=ID(stored=False),
+            subject=TEXT(stored=False),
+            body=TEXT(stored=False),
+            tag=KEYWORD(stored=False, commas=True))
+
+    def _create_index(self):
+        return RamStorage().create_index(self._mail_schema(), indexname='mails')
+
+    def index_mail(self, mail):
+        writer = self._index.writer()
+        self._index_mail(writer, mail)
+        writer.commit()
+
+    def _index_mail(self, writer, mail):
+        mdict = mail.as_dict()
+        header = mdict['header']
+        tags = mdict.get('tags', [])
+        tags.append(mail.mailbox_name.lower())
+        index_data = {
+            'sender': unicode(header.get('from', '')),
+            'subject': unicode(header.get('subject', '')),
+            'to': unicode(header.get('to', '')),
+            'cc': unicode(header.get('cc', '')),
+            'bcc': unicode(header.get('bcc', '')),
+            'tag': u','.join(tags),
+            'body': unicode(mdict['body']),
+            'ident': unicode(mdict['ident'])
+        }
+
+        writer.update_document(**index_data)
+
+    def index_mails(self, mails):
+        writer = self._index.writer()
+        try:
+            for mail in mails:
+                self._index_mail(writer, mail)
+        finally:
+            writer.commit()
+
+    def search(self, query):
+        query = query.replace('\"', '')
+        query = query.replace('-in:', 'AND NOT tag:')
+        query = query.replace('in:all', '*')
+        with self._index.searcher() as searcher:
+            query = QueryParser('body', self._index.schema).parse(query)
+            results = searcher.search(query)
+            return [mail['ident'] for mail in results]
+
+    def remove_from_index(self, mail_id):
+        writer = self._index.writer()
+        try:
+            writer.delete_by_term('ident', mail_id)
+        finally:
+            writer.commit()
diff --git a/service/pixelated/adapter/soledad_querier.py b/service/pixelated/adapter/soledad_querier.py
index ab8f6941..43b9891e 100644
--- a/service/pixelated/adapter/soledad_querier.py
+++ b/service/pixelated/adapter/soledad_querier.py
@@ -47,11 +47,12 @@ class SoledadQuerier:
 
     def all_mails_by_mailbox(self, mailbox_name):
         fdocs_chash = [(fdoc, fdoc.content['chash']) for fdoc in self.soledad.get_from_index('by-type-and-mbox', 'flags', mailbox_name)]
-        if len(fdocs_chash) == 0:
-            return []
         return self._build_mails_from_fdocs(fdocs_chash)
 
     def _build_mails_from_fdocs(self, fdocs_chash):
+        if len(fdocs_chash) == 0:
+            return []
+
         fdocs_hdocs = [(f[0], self.soledad.get_from_index('by-type-and-contenthash', 'head', f[1])[0]) for f in fdocs_chash]
         fdocs_hdocs_phash = [(f[0], f[1], f[1].content.get('body')) for f in fdocs_hdocs]
         fdocs_hdocs_bdocs = [(f[0], f[1], self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', f[2])[0]) for f in fdocs_hdocs_phash]
@@ -67,6 +68,7 @@ class SoledadQuerier:
         uid = self._next_uid_for_mailbox(mailbox_name)
         new_docs = [self.soledad.create_doc(doc) for doc in mail._get_for_save(next_uid=uid, mailbox=mailbox_name)]
         self._update_index(new_docs)
+        return self.mail(mail.ident)
 
     def mail(self, ident):
         fdoc = self.soledad.get_from_index('by-type-and-contenthash', 'flags', ident)[0]
@@ -75,6 +77,10 @@ class SoledadQuerier:
 
         return PixelatedMail.from_soledad(fdoc, hdoc, bdoc, soledad_querier=self)
 
+    def mails(self, idents):
+        fdocs_chash = [(self.soledad.get_from_index('by-type-and-contenthash', 'flags', ident)[0], ident) for ident in idents]
+        return self._build_mails_from_fdocs(fdocs_chash)
+
     def remove_mail(self, mail):
         _mail = self.mail(mail.ident)
         # FIX-ME: Must go through all the part_map phash to delete all the cdocs