Revert "Kill SoledadQuerier with 🔥🔥🔥"

This reverts commit 7deaefb939c833d65fd499aacb35502d6de1ac7e.

7 files changed, 400 insertions, 0 deletions

diff --git a/service/pixelated/adapter/soledad/__init__.py b/service/pixelated/adapter/soledad/__init__.py
new file mode 100644
index 00000000..2756a319
--- /dev/null
+++ b/service/pixelated/adapter/soledad/__init__.py
@@ -0,0 +1,15 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
diff --git a/service/pixelated/adapter/soledad/soledad_duplicate_removal_mixin.py b/service/pixelated/adapter/soledad/soledad_duplicate_removal_mixin.py
new file mode 100644
index 00000000..a2b4b6d7
--- /dev/null
+++ b/service/pixelated/adapter/soledad/soledad_duplicate_removal_mixin.py
@@ -0,0 +1,46 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin
+from twisted.internet import defer
+
+
+class SoledadDuplicateRemovalMixin(SoledadDbFacadeMixin, object):
+
+    @defer.inlineCallbacks
+    def remove_duplicates(self):
+        for mailbox in ['INBOX', 'DRAFTS', 'SENT', 'TRASH']:
+            yield self._remove_dup_inboxes(mailbox)
+            yield self._remove_dup_recent(mailbox)
+
+    @defer.inlineCallbacks
+    def _remove_many(self, docs):
+        [(yield self.delete_doc(doc)) for doc in docs]
+
+    @defer.inlineCallbacks
+    def _remove_dup_inboxes(self, mailbox_name):
+        mailboxes = yield self.get_mbox(mailbox_name)
+        if len(mailboxes) == 0:
+            return
+        mailboxes_to_remove = sorted(mailboxes, key=lambda x: x.content['created'])[1:len(mailboxes)]
+        yield self._remove_many(mailboxes_to_remove)
+
+    @defer.inlineCallbacks
+    def _remove_dup_recent(self, mailbox_name):
+        rct = yield self.get_recent_by_mbox(mailbox_name)
+        if len(rct) == 0:
+            return
+        rct_to_remove = sorted(rct, key=lambda x: len(x.content['rct']), reverse=True)[1:len(rct)]
+        yield self._remove_many(rct_to_remove)
diff --git a/service/pixelated/adapter/soledad/soledad_facade_mixin.py b/service/pixelated/adapter/soledad/soledad_facade_mixin.py
new file mode 100644
index 00000000..81280eaa
--- /dev/null
+++ b/service/pixelated/adapter/soledad/soledad_facade_mixin.py
@@ -0,0 +1,91 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+
+from twisted.internet import defer
+from leap.mail.mailbox_indexer import MailboxIndexer
+
+
+class SoledadDbFacadeMixin(object):
+
+    @defer.inlineCallbacks
+    def get_all_flags(self):
+        flags = yield self.soledad.get_from_index('by-type', 'flags')
+        defer.returnValue(flags)
+
+    def get_all_flags_by_mbox(self, mbox):
+        return self.soledad.get_from_index('by-type-and-mbox', 'flags', mbox) if mbox else []
+
+    @defer.inlineCallbacks
+    def get_content_by_phash(self, phash):
+        content = yield self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', phash) if phash else []
+        if len(content):
+            defer.returnValue(content[0])
+
+    @defer.inlineCallbacks
+    def get_flags_by_chash(self, chash):
+        flags = yield self.soledad.get_from_index('by-type-and-contenthash', 'flags', chash) if chash else []
+        if len(flags):
+            defer.returnValue(flags[0])
+
+    @defer.inlineCallbacks
+    def get_header_by_chash(self, chash):
+        header = yield self.soledad.get_from_index('by-type-and-contenthash', 'head', chash) if chash else []
+        if len(header):
+            defer.returnValue(header[0])
+
+    @defer.inlineCallbacks
+    def get_recent_by_mbox(self, mbox):
+        defer.returnValue(
+            (yield self.soledad.get_from_index('by-type-and-mbox', 'rct', mbox) if mbox else []))
+
+    def put_doc(self, doc):
+        return self.soledad.put_doc(doc)
+
+    def create_doc(self, doc):
+        return self.soledad.create_doc(doc)
+
+    @defer.inlineCallbacks
+    def create_docs(self, docs):
+        for doc in docs:
+            yield self.create_doc(doc)
+
+    def delete_doc(self, doc):
+        return self.soledad.delete_doc(doc)
+
+    @defer.inlineCallbacks
+    def idents_by_mailbox(self, mbox):
+        mbox_docs = (yield self.soledad.get_from_index('by-type-and-mbox-and-deleted', 'flags', mbox, '0')) if mbox else []
+        defer.returnValue(set(doc.content['chash'] for doc in mbox_docs))
+
+    def get_all_mbox(self):
+        return self.soledad.get_from_index('by-type', 'mbox')
+
+    def get_mbox(self, mbox):
+        return self.soledad.get_from_index('by-type-and-mbox', 'mbox', mbox) if mbox else []
+
+    @defer.inlineCallbacks
+    def get_lastuid(self, mbox):
+        if isinstance(mbox, str):
+            mbox = (yield defer.maybeDeferred(self.get_mbox, mbox))[0]
+
+        indexer = MailboxIndexer(self.soledad)
+        yield indexer.create_table(mbox.content['uuid'])
+        last_uuid = yield indexer.get_last_uid(mbox.content['uuid'])
+
+        defer.returnValue(last_uuid)
+
+    def get_search_index_masterkey(self):
+        return self.soledad.get_from_index('by-type', 'index_key')
diff --git a/service/pixelated/adapter/soledad/soledad_querier.py b/service/pixelated/adapter/soledad/soledad_querier.py
new file mode 100644
index 00000000..e0b215d3
--- /dev/null
+++ b/service/pixelated/adapter/soledad/soledad_querier.py
@@ -0,0 +1,29 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from pixelated.adapter.soledad.soledad_duplicate_removal_mixin import SoledadDuplicateRemovalMixin
+from pixelated.adapter.soledad.soledad_reader_mixin import SoledadReaderMixin
+from pixelated.adapter.soledad.soledad_search_key_masterkey_retrieval_mixin import SoledadSearchIndexMasterkeyRetrievalMixin
+from pixelated.adapter.soledad.soledad_writer_mixin import SoledadWriterMixin
+
+
+class SoledadQuerier(SoledadWriterMixin,
+                     SoledadReaderMixin,
+                     SoledadDuplicateRemovalMixin,
+                     SoledadSearchIndexMasterkeyRetrievalMixin,
+                     object):
+
+    def __init__(self, soledad):
+        self.soledad = soledad
diff --git a/service/pixelated/adapter/soledad/soledad_reader_mixin.py b/service/pixelated/adapter/soledad/soledad_reader_mixin.py
new file mode 100644
index 00000000..e86298dd
--- /dev/null
+++ b/service/pixelated/adapter/soledad/soledad_reader_mixin.py
@@ -0,0 +1,130 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+import base64
+import logging
+import quopri
+import re
+
+from twisted.internet import defer
+from pixelated.adapter.model.mail import PixelatedMail
+from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin
+
+
+logger = logging.getLogger(__name__)
+
+
+class SoledadReaderMixin(SoledadDbFacadeMixin, object):
+
+    @defer.inlineCallbacks
+    def all_mails(self):
+        fdocs_chash = [(fdoc, fdoc.content['chash']) for fdoc in (yield self.get_all_flags())]
+        if len(fdocs_chash) == 0:
+            defer.returnValue([])
+        defer.returnValue((yield self._build_mails_from_fdocs(fdocs_chash)))
+
+    @defer.inlineCallbacks
+    def _build_mails_from_fdocs(self, fdocs_chash):
+        if len(fdocs_chash) == 0:
+            defer.returnValue([])
+
+        fdocs_hdocs = []
+        for fdoc, chash in fdocs_chash:
+            hdoc = yield self.get_header_by_chash(chash)
+            if not hdoc:
+                continue
+            fdocs_hdocs.append((fdoc, hdoc))
+
+        fdocs_hdocs_bodyphash = [(f[0], f[1], f[1].content.get('body')) for f in fdocs_hdocs]
+        fdocs_hdocs_bdocs_parts = []
+        for fdoc, hdoc, body_phash in fdocs_hdocs_bodyphash:
+            bdoc = yield self.get_content_by_phash(body_phash)
+            if not bdoc:
+                continue
+            parts = yield self._extract_parts(hdoc.content)
+            fdocs_hdocs_bdocs_parts.append((fdoc, hdoc, bdoc, parts))
+
+        defer.returnValue([PixelatedMail.from_soledad(*raw_mail) for raw_mail in fdocs_hdocs_bdocs_parts])
+
+    def mail_exists(self, ident):
+        return self.get_flags_by_chash(ident)
+
+    @defer.inlineCallbacks
+    def mail(self, ident):
+        fdoc = yield self.get_flags_by_chash(ident)
+        hdoc = yield self.get_header_by_chash(ident)
+        bdoc = yield self.get_content_by_phash(hdoc.content['body'])
+        parts = yield self._extract_parts(hdoc.content)
+
+        mail = PixelatedMail.from_soledad(fdoc, hdoc, bdoc, parts=parts)
+        defer.returnValue(mail)
+
+    @defer.inlineCallbacks
+    def mails(self, idents):
+        fdocs_chash = [((yield self.get_flags_by_chash(ident)), ident) for ident in
+                       idents]
+        fdocs_chash = [(result, ident) for result, ident in fdocs_chash if result]
+        defer.returnValue((yield self._build_mails_from_fdocs(fdocs_chash)))
+
+    @defer.inlineCallbacks
+    def attachment(self, attachment_ident, encoding):
+        bdoc = yield self.get_content_by_phash(attachment_ident)
+        defer.returnValue({'content': self._try_decode(bdoc.content['raw'], encoding),
+                           'content-type': bdoc.content['content_type']})
+
+    def _try_decode(self, raw, encoding):
+        encoding = encoding.lower()
+        if encoding == 'base64':
+            return base64.decodestring(raw)
+        elif encoding == 'quoted-printable':
+            return quopri.decodestring(raw)
+        else:
+            return str(raw)
+
+    @defer.inlineCallbacks
+    def _extract_parts(self, hdoc, parts=None):
+        if not parts:
+            parts = {'alternatives': [], 'attachments': []}
+
+        if hdoc['multi']:
+            for part_key in hdoc.get('part_map', {}).keys():
+                yield self._extract_parts(hdoc['part_map'][part_key], parts)
+        else:
+            headers_dict = {elem[0]: elem[1] for elem in hdoc.get('headers', [])}
+            if 'attachment' in headers_dict.get('Content-Disposition', ''):
+                parts['attachments'].append(self._extract_attachment(hdoc, headers_dict))
+            else:
+                parts['alternatives'].append((yield self._extract_alternative(hdoc, headers_dict)))
+        defer.returnValue(parts)
+
+    @defer.inlineCallbacks
+    def _extract_alternative(self, hdoc, headers_dict):
+        bdoc = yield self.get_content_by_phash(hdoc['phash'])
+
+        if bdoc is None:
+            logger.warning("No BDOC content found for message!!!")
+            raw_content = ""
+        else:
+            raw_content = bdoc.content['raw']
+
+        defer.returnValue({'headers': headers_dict, 'content': raw_content})
+
+    def _extract_attachment(self, hdoc, headers_dict):
+        content_disposition = headers_dict['Content-Disposition']
+        match = re.compile('.*name=\"(.*)\".*').search(content_disposition)
+        filename = ''
+        if match:
+            filename = match.group(1)
+        return {'headers': headers_dict, 'ident': hdoc['phash'], 'name': filename}
diff --git a/service/pixelated/adapter/soledad/soledad_search_key_masterkey_retrieval_mixin.py b/service/pixelated/adapter/soledad/soledad_search_key_masterkey_retrieval_mixin.py
new file mode 100644
index 00000000..8a912bc2
--- /dev/null
+++ b/service/pixelated/adapter/soledad/soledad_search_key_masterkey_retrieval_mixin.py
@@ -0,0 +1,35 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin
+import os
+import base64
+
+
+class SoledadSearchIndexMasterkeyRetrievalMixin(SoledadDbFacadeMixin, object):
+
+    def get_index_masterkey(self):
+        deferred = self.get_search_index_masterkey()
+        deferred.addCallback(self._ensure_masterkey_exists)
+        return deferred
+
+    def _ensure_masterkey_exists(self, result):
+        index_key_doc = result[0] if result else None
+
+        if not index_key_doc:
+            new_index_key = os.urandom(64)  # 32 for encryption, 32 for hmac
+            self.create_doc(dict(type='index_key', value=base64.encodestring(new_index_key)))
+            return new_index_key
+        return base64.decodestring(index_key_doc.content['value'])
diff --git a/service/pixelated/adapter/soledad/soledad_writer_mixin.py b/service/pixelated/adapter/soledad/soledad_writer_mixin.py
new file mode 100644
index 00000000..d47e722a
--- /dev/null
+++ b/service/pixelated/adapter/soledad/soledad_writer_mixin.py
@@ -0,0 +1,54 @@
+#
+# Copyright (c) 2014 ThoughtWorks, Inc.
+#
+# Pixelated is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Pixelated is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin
+
+from twisted.internet import defer
+
+
+class SoledadWriterMixin(SoledadDbFacadeMixin, object):
+
+    @defer.inlineCallbacks
+    def mark_all_as_not_recent(self):
+        for mailbox in ['INBOX', 'DRAFTS', 'SENT', 'TRASH']:
+            rct = yield self.get_recent_by_mbox(mailbox)
+            if not rct or not rct[0].content['rct']:
+                return
+            rct = rct[0]
+            rct.content['rct'] = []
+            yield self.put_doc(rct)
+
+    def save_mail(self, mail):
+        return self.put_doc(mail.fdoc)
+
+    @defer.inlineCallbacks
+    def create_mail(self, mail, mailbox_name):
+        mbox_doc = (yield self.get_mbox(mailbox_name))[0]
+        uid = 1 + (yield self.get_lastuid(mbox_doc))
+
+        yield self.create_docs(mail.get_for_save(next_uid=uid, mailbox=mailbox_name))
+
+        # FIXME need to update meta message (mdoc)
+        # mbox_doc.content['lastuid'] = uid + 1
+        # self.put_doc(mbox_doc)
+
+        defer.returnValue((yield self.mail(mail.ident)))
+
+    @defer.inlineCallbacks
+    def remove_mail(self, mail):
+        # FIX-ME: Must go through all the part_map phash to delete all the cdocs
+        yield self.delete_doc(mail.fdoc)
+        yield self.delete_doc(mail.hdoc)
+        yield self.delete_doc(mail.bdoc)