summaryrefslogtreecommitdiff
path: root/service
diff options
context:
space:
mode:
authorDuda Dornelles <ddornell@thoughtworks.com>2014-11-10 15:25:41 -0200
committerDuda Dornelles <ddornell@thoughtworks.com>2014-11-10 17:54:12 -0200
commit84689864542cb9c6c356b5204f94dd7a0ea7a777 (patch)
tree268f8914c0e89d653ae46a33952ee6da47b914c1 /service
parent636bbf638b3b0fbd6de3b194fb91257de4fe5d5b (diff)
adding encrypted file index to whoosh
Diffstat (limited to 'service')
-rw-r--r--service/pixelated/adapter/search.py12
-rw-r--r--service/pixelated/adapter/soledad_querier.py15
-rw-r--r--service/pixelated/config/app_factory.py4
-rw-r--r--service/pixelated/controllers/attachments_controller.py8
-rw-r--r--service/pixelated/support/encrypted_file_storage.py65
-rw-r--r--service/test/support/integration_helper.py2
6 files changed, 93 insertions, 13 deletions
diff --git a/service/pixelated/adapter/search.py b/service/pixelated/adapter/search.py
index a5897a12..cd900f87 100644
--- a/service/pixelated/adapter/search.py
+++ b/service/pixelated/adapter/search.py
@@ -13,10 +13,11 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+from pixelated.support.encrypted_file_storage import EncryptedFileStorage
import os
from pixelated.adapter.status import Status
-import whoosh.index
+from whoosh.index import FileIndex
from whoosh.fields import *
from whoosh.qparser import QueryParser
from whoosh import sorting
@@ -25,12 +26,11 @@ from pixelated.support.date import milliseconds
class SearchEngine(object):
- __slots__ = '_index'
-
INDEX_FOLDER = os.path.join(os.environ['HOME'], '.leap', 'search_index')
DEFAULT_TAGS = ['inbox', 'sent', 'drafts', 'trash']
- def __init__(self):
+ def __init__(self, soledad_querier):
+ self.soledad_querier = soledad_querier
if not os.path.exists(self.INDEX_FOLDER):
os.makedirs(self.INDEX_FOLDER)
self._index = self._create_index()
@@ -102,7 +102,9 @@ class SearchEngine(object):
raw=TEXT(stored=False))
def _create_index(self):
- return whoosh.index.create_in(self.INDEX_FOLDER, self._mail_schema(), indexname='mails')
+ masterkey = self.soledad_querier.get_index_masterkey
+ storage = EncryptedFileStorage(self.INDEX_FOLDER, masterkey)
+ return FileIndex.create(storage, self._mail_schema(), indexname='mails')
def index_mail(self, mail):
with self._index.writer() as writer:
diff --git a/service/pixelated/adapter/soledad_querier.py b/service/pixelated/adapter/soledad_querier.py
index e36f2e1a..c1e0350e 100644
--- a/service/pixelated/adapter/soledad_querier.py
+++ b/service/pixelated/adapter/soledad_querier.py
@@ -13,17 +13,28 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
-from pixelated.adapter.mail import PixelatedMail
-import re
import base64
import quopri
+from cryptography.fernet import Fernet
+from pixelated.adapter.mail import PixelatedMail
+import re
+
class SoledadQuerier:
def __init__(self, soledad):
self.soledad = soledad
+ @property
+ def get_index_masterkey(self):
+ index_key = self.soledad.get_from_index('by-type', 'index_key')
+ if len(index_key) == 0:
+ index_key = Fernet.generate_key()
+ self.soledad.create_doc(dict(type='index_key', value=index_key))
+ return index_key
+ return str(index_key[0].content['value'])
+
def _remove_many(self, docs):
[self.soledad.delete_doc(doc) for doc in docs]
diff --git a/service/pixelated/config/app_factory.py b/service/pixelated/config/app_factory.py
index 6a9046ee..93681fe3 100644
--- a/service/pixelated/config/app_factory.py
+++ b/service/pixelated/config/app_factory.py
@@ -96,12 +96,12 @@ def init_leap_session(app):
def init_app(app):
leap_session = init_leap_session(app)
+ soledad_querier = SoledadQuerier(soledad=leap_session.account._soledad)
tag_service = TagService()
- search_engine = SearchEngine()
+ search_engine = SearchEngine(soledad_querier)
pixelated_mail_sender = MailSender(leap_session.account_email())
- soledad_querier = SoledadQuerier(soledad=leap_session.account._soledad)
pixelated_mailboxes = Mailboxes(leap_session.account, soledad_querier)
draft_service = DraftService(pixelated_mailboxes)
mail_service = MailService(pixelated_mailboxes, pixelated_mail_sender, tag_service, soledad_querier)
diff --git a/service/pixelated/controllers/attachments_controller.py b/service/pixelated/controllers/attachments_controller.py
index 68e73bd6..1d5360f7 100644
--- a/service/pixelated/controllers/attachments_controller.py
+++ b/service/pixelated/controllers/attachments_controller.py
@@ -19,6 +19,7 @@ from flask import request
import io
import re
+from twisted.web.server import NOT_DONE_YET
class AttachmentsController:
@@ -29,9 +30,10 @@ class AttachmentsController:
def attachment(self, request, attachment_id):
encoding = request.args.get('encoding', [''])[0]
attachment = self.querier.attachment(attachment_id, encoding)
- response = send_file(io.BytesIO(attachment['content']),
- mimetype=self._extract_mimetype(attachment['content-type']))
- return response
+ request.setRawHeader('Content-Type', self._extract_mimetype(attachment['content-type']))
+ request.write(io.BytesIO(attachment['content']))
+
+ return NOT_DONE_YET
def _extract_mimetype(self, content_type):
match = re.compile('([A-Za-z-]+\/[A-Za-z-]+)').search(content_type)
diff --git a/service/pixelated/support/encrypted_file_storage.py b/service/pixelated/support/encrypted_file_storage.py
new file mode 100644
index 00000000..cdf52ecf
--- /dev/null
+++ b/service/pixelated/support/encrypted_file_storage.py
@@ -0,0 +1,65 @@
+from StringIO import StringIO
+import io
+import sys
+import errno
+import os
+
+from whoosh.filedb.filestore import FileStorage, ReadOnlyError
+
+from whoosh.filedb.structfile import StructFile, BufferFile
+from cryptography.fernet import Fernet
+from whoosh.util import random_name
+
+
+class EncryptedFileStorage(FileStorage):
+ def __init__(self, path, masterkey=None):
+ self.masterkey = masterkey
+ self.f = Fernet(masterkey)
+ self._tmp_storage = self.temp_storage
+ FileStorage.__init__(self, path, supports_mmap=False)
+
+ def open_file(self, name, **kwargs):
+ def onclose(file_struct):
+ file_struct.seek(0)
+ content = file_struct.file.read()
+ encrypted_content = self.f.encrypt(content)
+ _file = open(self._fpath(name), 'w+b')
+ _file.write(encrypted_content)
+
+ return self._open_file(name, onclose=onclose)
+
+ def create_file(self, name, excl=False, mode="w+b", **kwargs):
+ def onclose(file_struct):
+ file_struct.seek(0)
+ content = file_struct.file.read()
+ encrypted_content = self.f.encrypt(content)
+ _file = open(self._fpath(name), 'w+b')
+ _file.write(encrypted_content)
+
+ f = StructFile(io.BytesIO(), name=name, onclose=onclose)
+ f.is_real = False
+ return f
+
+ def temp_storage(self, name=None):
+ name = name or "%s.tmp" % random_name()
+ path = os.path.join(self.folder, name)
+ tempstore = EncryptedFileStorage(path, self.masterkey)
+ # import pdb;pdb.set_trace()
+ return tempstore.create()
+
+ def file_length(self, name):
+ f = self._open_file(name)
+ length = len(f.file.read())
+ f.close()
+ return length
+
+ def _decrypt(self, file_content):
+ return self.f.decrypt(file_content) if len(file_content) else file_content
+
+ def _open_file(self, name, onclose=lambda x: None):
+ file_content = open(self._fpath(name), "rb").read()
+ decrypted = self._decrypt(file_content)
+ f = BufferFile(buffer(decrypted), name=name, onclose=onclose)
+ return f
+
+
diff --git a/service/test/support/integration_helper.py b/service/test/support/integration_helper.py
index 900b8049..6ab96e52 100644
--- a/service/test/support/integration_helper.py
+++ b/service/test/support/integration_helper.py
@@ -149,7 +149,7 @@ class SoledadTestBase:
self.draft_service = DraftService(self.mailboxes)
self.mail_service = MailService(self.mailboxes, self.mail_sender, self.tag_service,
self.soledad_querier)
- self.search_engine = SearchEngine()
+ self.search_engine = SearchEngine(self.soledad_querier)
self.search_engine.index_mails(self.mail_service.all_mails())
features_controller = FeaturesController()