adding encrypted file index to whoosh

6 files changed, 93 insertions, 13 deletions

diff --git a/service/pixelated/adapter/search.py b/service/pixelated/adapter/search.py
index a5897a12..cd900f87 100644
--- a/service/pixelated/adapter/search.py
+++ b/service/pixelated/adapter/search.py
@@ -13,10 +13,11 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from pixelated.support.encrypted_file_storage import EncryptedFileStorage
 
 import os
 from pixelated.adapter.status import Status
-import whoosh.index
+from whoosh.index import FileIndex
 from whoosh.fields import *
 from whoosh.qparser import QueryParser
 from whoosh import sorting
@@ -25,12 +26,11 @@ from pixelated.support.date import milliseconds
 
 
 class SearchEngine(object):
-    __slots__ = '_index'
-
     INDEX_FOLDER = os.path.join(os.environ['HOME'], '.leap', 'search_index')
     DEFAULT_TAGS = ['inbox', 'sent', 'drafts', 'trash']
 
-    def __init__(self):
+    def __init__(self, soledad_querier):
+        self.soledad_querier = soledad_querier
         if not os.path.exists(self.INDEX_FOLDER):
             os.makedirs(self.INDEX_FOLDER)
         self._index = self._create_index()
@@ -102,7 +102,9 @@ class SearchEngine(object):
             raw=TEXT(stored=False))
 
     def _create_index(self):
-        return whoosh.index.create_in(self.INDEX_FOLDER, self._mail_schema(), indexname='mails')
+        masterkey = self.soledad_querier.get_index_masterkey
+        storage = EncryptedFileStorage(self.INDEX_FOLDER, masterkey)
+        return FileIndex.create(storage, self._mail_schema(), indexname='mails')
 
     def index_mail(self, mail):
         with self._index.writer() as writer:
diff --git a/service/pixelated/adapter/soledad_querier.py b/service/pixelated/adapter/soledad_querier.py
index e36f2e1a..c1e0350e 100644
--- a/service/pixelated/adapter/soledad_querier.py
+++ b/service/pixelated/adapter/soledad_querier.py
@@ -13,17 +13,28 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
-from pixelated.adapter.mail import PixelatedMail
-import re
 import base64
 import quopri
 
+from cryptography.fernet import Fernet
+from pixelated.adapter.mail import PixelatedMail
+import re
+
 
 class SoledadQuerier:
 
     def __init__(self, soledad):
         self.soledad = soledad
 
+    @property
+    def get_index_masterkey(self):
+        index_key = self.soledad.get_from_index('by-type', 'index_key')
+        if len(index_key) == 0:
+            index_key = Fernet.generate_key()
+            self.soledad.create_doc(dict(type='index_key', value=index_key))
+            return index_key
+        return str(index_key[0].content['value'])
+
     def _remove_many(self, docs):
         [self.soledad.delete_doc(doc) for doc in docs]
 
diff --git a/service/pixelated/config/app_factory.py b/service/pixelated/config/app_factory.py
index 6a9046ee..93681fe3 100644
--- a/service/pixelated/config/app_factory.py
+++ b/service/pixelated/config/app_factory.py
@@ -96,12 +96,12 @@ def init_leap_session(app):
 
 def init_app(app):
     leap_session = init_leap_session(app)
+    soledad_querier = SoledadQuerier(soledad=leap_session.account._soledad)
 
     tag_service = TagService()
-    search_engine = SearchEngine()
+    search_engine = SearchEngine(soledad_querier)
     pixelated_mail_sender = MailSender(leap_session.account_email())
 
-    soledad_querier = SoledadQuerier(soledad=leap_session.account._soledad)
     pixelated_mailboxes = Mailboxes(leap_session.account, soledad_querier)
     draft_service = DraftService(pixelated_mailboxes)
     mail_service = MailService(pixelated_mailboxes, pixelated_mail_sender, tag_service, soledad_querier)
diff --git a/service/pixelated/controllers/attachments_controller.py b/service/pixelated/controllers/attachments_controller.py
index 68e73bd6..1d5360f7 100644
--- a/service/pixelated/controllers/attachments_controller.py
+++ b/service/pixelated/controllers/attachments_controller.py
@@ -19,6 +19,7 @@ from flask import request
 
 import io
 import re
+from twisted.web.server import NOT_DONE_YET
 
 
 class AttachmentsController:
@@ -29,9 +30,10 @@ class AttachmentsController:
     def attachment(self, request, attachment_id):
         encoding = request.args.get('encoding', [''])[0]
         attachment = self.querier.attachment(attachment_id, encoding)
-        response = send_file(io.BytesIO(attachment['content']),
-                             mimetype=self._extract_mimetype(attachment['content-type']))
-        return response
+        request.setRawHeader('Content-Type', self._extract_mimetype(attachment['content-type']))
+        request.write(io.BytesIO(attachment['content']))
+
+        return NOT_DONE_YET
 
     def _extract_mimetype(self, content_type):
         match = re.compile('([A-Za-z-]+\/[A-Za-z-]+)').search(content_type)
diff --git a/service/pixelated/support/encrypted_file_storage.py b/service/pixelated/support/encrypted_file_storage.py
new file mode 100644
index 00000000..cdf52ecf
--- /dev/null
+++ b/service/pixelated/support/encrypted_file_storage.py
@@ -0,0 +1,65 @@
+from StringIO import StringIO
+import io
+import sys
+import errno
+import os
+
+from whoosh.filedb.filestore import FileStorage, ReadOnlyError
+
+from whoosh.filedb.structfile import StructFile, BufferFile
+from cryptography.fernet import Fernet
+from whoosh.util import random_name
+
+
+class EncryptedFileStorage(FileStorage):
+    def __init__(self, path, masterkey=None):
+        self.masterkey = masterkey
+        self.f = Fernet(masterkey)
+        self._tmp_storage = self.temp_storage
+        FileStorage.__init__(self, path, supports_mmap=False)
+
+    def open_file(self, name, **kwargs):
+        def onclose(file_struct):
+            file_struct.seek(0)
+            content = file_struct.file.read()
+            encrypted_content = self.f.encrypt(content)
+            _file = open(self._fpath(name), 'w+b')
+            _file.write(encrypted_content)
+
+        return self._open_file(name, onclose=onclose)
+
+    def create_file(self, name, excl=False, mode="w+b", **kwargs):
+        def onclose(file_struct):
+            file_struct.seek(0)
+            content = file_struct.file.read()
+            encrypted_content = self.f.encrypt(content)
+            _file = open(self._fpath(name), 'w+b')
+            _file.write(encrypted_content)
+
+        f = StructFile(io.BytesIO(), name=name, onclose=onclose)
+        f.is_real = False
+        return f
+
+    def temp_storage(self, name=None):
+        name = name or "%s.tmp" % random_name()
+        path = os.path.join(self.folder, name)
+        tempstore = EncryptedFileStorage(path, self.masterkey)
+        # import pdb;pdb.set_trace()
+        return tempstore.create()
+
+    def file_length(self, name):
+        f = self._open_file(name)
+        length = len(f.file.read())
+        f.close()
+        return length
+
+    def _decrypt(self, file_content):
+        return self.f.decrypt(file_content) if len(file_content) else file_content
+
+    def _open_file(self, name, onclose=lambda x: None):
+        file_content = open(self._fpath(name), "rb").read()
+        decrypted = self._decrypt(file_content)
+        f = BufferFile(buffer(decrypted), name=name, onclose=onclose)
+        return f
+
+
diff --git a/service/test/support/integration_helper.py b/service/test/support/integration_helper.py
index 900b8049..6ab96e52 100644
--- a/service/test/support/integration_helper.py
+++ b/service/test/support/integration_helper.py
@@ -149,7 +149,7 @@ class SoledadTestBase:
         self.draft_service = DraftService(self.mailboxes)
         self.mail_service = MailService(self.mailboxes, self.mail_sender, self.tag_service,
                                         self.soledad_querier)
-        self.search_engine = SearchEngine()
+        self.search_engine = SearchEngine(self.soledad_querier)
         self.search_engine.index_mails(self.mail_service.all_mails())
 
         features_controller = FeaturesController()