diff options
| -rw-r--r-- | service/pixelated/adapter/search.py | 12 | ||||
| -rw-r--r-- | service/pixelated/adapter/soledad_querier.py | 15 | ||||
| -rw-r--r-- | service/pixelated/config/app_factory.py | 4 | ||||
| -rw-r--r-- | service/pixelated/controllers/attachments_controller.py | 8 | ||||
| -rw-r--r-- | service/pixelated/support/encrypted_file_storage.py | 65 | ||||
| -rw-r--r-- | service/test/support/integration_helper.py | 2 |
6 files changed, 93 insertions, 13 deletions
diff --git a/service/pixelated/adapter/search.py b/service/pixelated/adapter/search.py index a5897a12..cd900f87 100644 --- a/service/pixelated/adapter/search.py +++ b/service/pixelated/adapter/search.py @@ -13,10 +13,11 @@ # # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +from pixelated.support.encrypted_file_storage import EncryptedFileStorage import os from pixelated.adapter.status import Status -import whoosh.index +from whoosh.index import FileIndex from whoosh.fields import * from whoosh.qparser import QueryParser from whoosh import sorting @@ -25,12 +26,11 @@ from pixelated.support.date import milliseconds class SearchEngine(object): - __slots__ = '_index' - INDEX_FOLDER = os.path.join(os.environ['HOME'], '.leap', 'search_index') DEFAULT_TAGS = ['inbox', 'sent', 'drafts', 'trash'] - def __init__(self): + def __init__(self, soledad_querier): + self.soledad_querier = soledad_querier if not os.path.exists(self.INDEX_FOLDER): os.makedirs(self.INDEX_FOLDER) self._index = self._create_index() @@ -102,7 +102,9 @@ class SearchEngine(object): raw=TEXT(stored=False)) def _create_index(self): - return whoosh.index.create_in(self.INDEX_FOLDER, self._mail_schema(), indexname='mails') + masterkey = self.soledad_querier.get_index_masterkey + storage = EncryptedFileStorage(self.INDEX_FOLDER, masterkey) + return FileIndex.create(storage, self._mail_schema(), indexname='mails') def index_mail(self, mail): with self._index.writer() as writer: diff --git a/service/pixelated/adapter/soledad_querier.py b/service/pixelated/adapter/soledad_querier.py index e36f2e1a..c1e0350e 100644 --- a/service/pixelated/adapter/soledad_querier.py +++ b/service/pixelated/adapter/soledad_querier.py @@ -13,17 +13,28 @@ # # You should have received a copy of the GNU Affero General Public License # along with Pixelated. If not, see <http://www.gnu.org/licenses/>. -from pixelated.adapter.mail import PixelatedMail -import re import base64 import quopri +from cryptography.fernet import Fernet +from pixelated.adapter.mail import PixelatedMail +import re + class SoledadQuerier: def __init__(self, soledad): self.soledad = soledad + @property + def get_index_masterkey(self): + index_key = self.soledad.get_from_index('by-type', 'index_key') + if len(index_key) == 0: + index_key = Fernet.generate_key() + self.soledad.create_doc(dict(type='index_key', value=index_key)) + return index_key + return str(index_key[0].content['value']) + def _remove_many(self, docs): [self.soledad.delete_doc(doc) for doc in docs] diff --git a/service/pixelated/config/app_factory.py b/service/pixelated/config/app_factory.py index 6a9046ee..93681fe3 100644 --- a/service/pixelated/config/app_factory.py +++ b/service/pixelated/config/app_factory.py @@ -96,12 +96,12 @@ def init_leap_session(app): def init_app(app): leap_session = init_leap_session(app) + soledad_querier = SoledadQuerier(soledad=leap_session.account._soledad) tag_service = TagService() - search_engine = SearchEngine() + search_engine = SearchEngine(soledad_querier) pixelated_mail_sender = MailSender(leap_session.account_email()) - soledad_querier = SoledadQuerier(soledad=leap_session.account._soledad) pixelated_mailboxes = Mailboxes(leap_session.account, soledad_querier) draft_service = DraftService(pixelated_mailboxes) mail_service = MailService(pixelated_mailboxes, pixelated_mail_sender, tag_service, soledad_querier) diff --git a/service/pixelated/controllers/attachments_controller.py b/service/pixelated/controllers/attachments_controller.py index 68e73bd6..1d5360f7 100644 --- a/service/pixelated/controllers/attachments_controller.py +++ b/service/pixelated/controllers/attachments_controller.py @@ -19,6 +19,7 @@ from flask import request import io import re +from twisted.web.server import NOT_DONE_YET class AttachmentsController: @@ -29,9 +30,10 @@ class AttachmentsController: def attachment(self, request, attachment_id): encoding = request.args.get('encoding', [''])[0] attachment = self.querier.attachment(attachment_id, encoding) - response = send_file(io.BytesIO(attachment['content']), - mimetype=self._extract_mimetype(attachment['content-type'])) - return response + request.setRawHeader('Content-Type', self._extract_mimetype(attachment['content-type'])) + request.write(io.BytesIO(attachment['content'])) + + return NOT_DONE_YET def _extract_mimetype(self, content_type): match = re.compile('([A-Za-z-]+\/[A-Za-z-]+)').search(content_type) diff --git a/service/pixelated/support/encrypted_file_storage.py b/service/pixelated/support/encrypted_file_storage.py new file mode 100644 index 00000000..cdf52ecf --- /dev/null +++ b/service/pixelated/support/encrypted_file_storage.py @@ -0,0 +1,65 @@ +from StringIO import StringIO +import io +import sys +import errno +import os + +from whoosh.filedb.filestore import FileStorage, ReadOnlyError + +from whoosh.filedb.structfile import StructFile, BufferFile +from cryptography.fernet import Fernet +from whoosh.util import random_name + + +class EncryptedFileStorage(FileStorage): + def __init__(self, path, masterkey=None): + self.masterkey = masterkey + self.f = Fernet(masterkey) + self._tmp_storage = self.temp_storage + FileStorage.__init__(self, path, supports_mmap=False) + + def open_file(self, name, **kwargs): + def onclose(file_struct): + file_struct.seek(0) + content = file_struct.file.read() + encrypted_content = self.f.encrypt(content) + _file = open(self._fpath(name), 'w+b') + _file.write(encrypted_content) + + return self._open_file(name, onclose=onclose) + + def create_file(self, name, excl=False, mode="w+b", **kwargs): + def onclose(file_struct): + file_struct.seek(0) + content = file_struct.file.read() + encrypted_content = self.f.encrypt(content) + _file = open(self._fpath(name), 'w+b') + _file.write(encrypted_content) + + f = StructFile(io.BytesIO(), name=name, onclose=onclose) + f.is_real = False + return f + + def temp_storage(self, name=None): + name = name or "%s.tmp" % random_name() + path = os.path.join(self.folder, name) + tempstore = EncryptedFileStorage(path, self.masterkey) + # import pdb;pdb.set_trace() + return tempstore.create() + + def file_length(self, name): + f = self._open_file(name) + length = len(f.file.read()) + f.close() + return length + + def _decrypt(self, file_content): + return self.f.decrypt(file_content) if len(file_content) else file_content + + def _open_file(self, name, onclose=lambda x: None): + file_content = open(self._fpath(name), "rb").read() + decrypted = self._decrypt(file_content) + f = BufferFile(buffer(decrypted), name=name, onclose=onclose) + return f + + diff --git a/service/test/support/integration_helper.py b/service/test/support/integration_helper.py index 900b8049..6ab96e52 100644 --- a/service/test/support/integration_helper.py +++ b/service/test/support/integration_helper.py @@ -149,7 +149,7 @@ class SoledadTestBase: self.draft_service = DraftService(self.mailboxes) self.mail_service = MailService(self.mailboxes, self.mail_sender, self.tag_service, self.soledad_querier) - self.search_engine = SearchEngine() + self.search_engine = SearchEngine(self.soledad_querier) self.search_engine.index_mails(self.mail_service.all_mails()) features_controller = FeaturesController() |