From 78cc2dad9c34d1a86f5dfed71b0a2fc29c695478 Mon Sep 17 00:00:00 2001 From: kaeff Date: Tue, 8 Sep 2015 14:50:16 +0200 Subject: =?UTF-8?q?Revert=20"Kill=20SoledadQuerier=20with=20=F0=9F=94=A5?= =?UTF-8?q?=F0=9F=94=A5=F0=9F=94=A5"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 7deaefb939c833d65fd499aacb35502d6de1ac7e. --- service/pixelated/adapter/soledad/__init__.py | 15 +++ .../soledad/soledad_duplicate_removal_mixin.py | 46 ++++++++ .../adapter/soledad/soledad_facade_mixin.py | 91 +++++++++++++++ .../pixelated/adapter/soledad/soledad_querier.py | 29 +++++ .../adapter/soledad/soledad_reader_mixin.py | 130 +++++++++++++++++++++ ...soledad_search_key_masterkey_retrieval_mixin.py | 35 ++++++ .../adapter/soledad/soledad_writer_mixin.py | 54 +++++++++ 7 files changed, 400 insertions(+) create mode 100644 service/pixelated/adapter/soledad/__init__.py create mode 100644 service/pixelated/adapter/soledad/soledad_duplicate_removal_mixin.py create mode 100644 service/pixelated/adapter/soledad/soledad_facade_mixin.py create mode 100644 service/pixelated/adapter/soledad/soledad_querier.py create mode 100644 service/pixelated/adapter/soledad/soledad_reader_mixin.py create mode 100644 service/pixelated/adapter/soledad/soledad_search_key_masterkey_retrieval_mixin.py create mode 100644 service/pixelated/adapter/soledad/soledad_writer_mixin.py (limited to 'service/pixelated') diff --git a/service/pixelated/adapter/soledad/__init__.py b/service/pixelated/adapter/soledad/__init__.py new file mode 100644 index 00000000..2756a319 --- /dev/null +++ b/service/pixelated/adapter/soledad/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . diff --git a/service/pixelated/adapter/soledad/soledad_duplicate_removal_mixin.py b/service/pixelated/adapter/soledad/soledad_duplicate_removal_mixin.py new file mode 100644 index 00000000..a2b4b6d7 --- /dev/null +++ b/service/pixelated/adapter/soledad/soledad_duplicate_removal_mixin.py @@ -0,0 +1,46 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . +from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin +from twisted.internet import defer + + +class SoledadDuplicateRemovalMixin(SoledadDbFacadeMixin, object): + + @defer.inlineCallbacks + def remove_duplicates(self): + for mailbox in ['INBOX', 'DRAFTS', 'SENT', 'TRASH']: + yield self._remove_dup_inboxes(mailbox) + yield self._remove_dup_recent(mailbox) + + @defer.inlineCallbacks + def _remove_many(self, docs): + [(yield self.delete_doc(doc)) for doc in docs] + + @defer.inlineCallbacks + def _remove_dup_inboxes(self, mailbox_name): + mailboxes = yield self.get_mbox(mailbox_name) + if len(mailboxes) == 0: + return + mailboxes_to_remove = sorted(mailboxes, key=lambda x: x.content['created'])[1:len(mailboxes)] + yield self._remove_many(mailboxes_to_remove) + + @defer.inlineCallbacks + def _remove_dup_recent(self, mailbox_name): + rct = yield self.get_recent_by_mbox(mailbox_name) + if len(rct) == 0: + return + rct_to_remove = sorted(rct, key=lambda x: len(x.content['rct']), reverse=True)[1:len(rct)] + yield self._remove_many(rct_to_remove) diff --git a/service/pixelated/adapter/soledad/soledad_facade_mixin.py b/service/pixelated/adapter/soledad/soledad_facade_mixin.py new file mode 100644 index 00000000..81280eaa --- /dev/null +++ b/service/pixelated/adapter/soledad/soledad_facade_mixin.py @@ -0,0 +1,91 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . + +from twisted.internet import defer +from leap.mail.mailbox_indexer import MailboxIndexer + + +class SoledadDbFacadeMixin(object): + + @defer.inlineCallbacks + def get_all_flags(self): + flags = yield self.soledad.get_from_index('by-type', 'flags') + defer.returnValue(flags) + + def get_all_flags_by_mbox(self, mbox): + return self.soledad.get_from_index('by-type-and-mbox', 'flags', mbox) if mbox else [] + + @defer.inlineCallbacks + def get_content_by_phash(self, phash): + content = yield self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', phash) if phash else [] + if len(content): + defer.returnValue(content[0]) + + @defer.inlineCallbacks + def get_flags_by_chash(self, chash): + flags = yield self.soledad.get_from_index('by-type-and-contenthash', 'flags', chash) if chash else [] + if len(flags): + defer.returnValue(flags[0]) + + @defer.inlineCallbacks + def get_header_by_chash(self, chash): + header = yield self.soledad.get_from_index('by-type-and-contenthash', 'head', chash) if chash else [] + if len(header): + defer.returnValue(header[0]) + + @defer.inlineCallbacks + def get_recent_by_mbox(self, mbox): + defer.returnValue( + (yield self.soledad.get_from_index('by-type-and-mbox', 'rct', mbox) if mbox else [])) + + def put_doc(self, doc): + return self.soledad.put_doc(doc) + + def create_doc(self, doc): + return self.soledad.create_doc(doc) + + @defer.inlineCallbacks + def create_docs(self, docs): + for doc in docs: + yield self.create_doc(doc) + + def delete_doc(self, doc): + return self.soledad.delete_doc(doc) + + @defer.inlineCallbacks + def idents_by_mailbox(self, mbox): + mbox_docs = (yield self.soledad.get_from_index('by-type-and-mbox-and-deleted', 'flags', mbox, '0')) if mbox else [] + defer.returnValue(set(doc.content['chash'] for doc in mbox_docs)) + + def get_all_mbox(self): + return self.soledad.get_from_index('by-type', 'mbox') + + def get_mbox(self, mbox): + return self.soledad.get_from_index('by-type-and-mbox', 'mbox', mbox) if mbox else [] + + @defer.inlineCallbacks + def get_lastuid(self, mbox): + if isinstance(mbox, str): + mbox = (yield defer.maybeDeferred(self.get_mbox, mbox))[0] + + indexer = MailboxIndexer(self.soledad) + yield indexer.create_table(mbox.content['uuid']) + last_uuid = yield indexer.get_last_uid(mbox.content['uuid']) + + defer.returnValue(last_uuid) + + def get_search_index_masterkey(self): + return self.soledad.get_from_index('by-type', 'index_key') diff --git a/service/pixelated/adapter/soledad/soledad_querier.py b/service/pixelated/adapter/soledad/soledad_querier.py new file mode 100644 index 00000000..e0b215d3 --- /dev/null +++ b/service/pixelated/adapter/soledad/soledad_querier.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . +from pixelated.adapter.soledad.soledad_duplicate_removal_mixin import SoledadDuplicateRemovalMixin +from pixelated.adapter.soledad.soledad_reader_mixin import SoledadReaderMixin +from pixelated.adapter.soledad.soledad_search_key_masterkey_retrieval_mixin import SoledadSearchIndexMasterkeyRetrievalMixin +from pixelated.adapter.soledad.soledad_writer_mixin import SoledadWriterMixin + + +class SoledadQuerier(SoledadWriterMixin, + SoledadReaderMixin, + SoledadDuplicateRemovalMixin, + SoledadSearchIndexMasterkeyRetrievalMixin, + object): + + def __init__(self, soledad): + self.soledad = soledad diff --git a/service/pixelated/adapter/soledad/soledad_reader_mixin.py b/service/pixelated/adapter/soledad/soledad_reader_mixin.py new file mode 100644 index 00000000..e86298dd --- /dev/null +++ b/service/pixelated/adapter/soledad/soledad_reader_mixin.py @@ -0,0 +1,130 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . +import base64 +import logging +import quopri +import re + +from twisted.internet import defer +from pixelated.adapter.model.mail import PixelatedMail +from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin + + +logger = logging.getLogger(__name__) + + +class SoledadReaderMixin(SoledadDbFacadeMixin, object): + + @defer.inlineCallbacks + def all_mails(self): + fdocs_chash = [(fdoc, fdoc.content['chash']) for fdoc in (yield self.get_all_flags())] + if len(fdocs_chash) == 0: + defer.returnValue([]) + defer.returnValue((yield self._build_mails_from_fdocs(fdocs_chash))) + + @defer.inlineCallbacks + def _build_mails_from_fdocs(self, fdocs_chash): + if len(fdocs_chash) == 0: + defer.returnValue([]) + + fdocs_hdocs = [] + for fdoc, chash in fdocs_chash: + hdoc = yield self.get_header_by_chash(chash) + if not hdoc: + continue + fdocs_hdocs.append((fdoc, hdoc)) + + fdocs_hdocs_bodyphash = [(f[0], f[1], f[1].content.get('body')) for f in fdocs_hdocs] + fdocs_hdocs_bdocs_parts = [] + for fdoc, hdoc, body_phash in fdocs_hdocs_bodyphash: + bdoc = yield self.get_content_by_phash(body_phash) + if not bdoc: + continue + parts = yield self._extract_parts(hdoc.content) + fdocs_hdocs_bdocs_parts.append((fdoc, hdoc, bdoc, parts)) + + defer.returnValue([PixelatedMail.from_soledad(*raw_mail) for raw_mail in fdocs_hdocs_bdocs_parts]) + + def mail_exists(self, ident): + return self.get_flags_by_chash(ident) + + @defer.inlineCallbacks + def mail(self, ident): + fdoc = yield self.get_flags_by_chash(ident) + hdoc = yield self.get_header_by_chash(ident) + bdoc = yield self.get_content_by_phash(hdoc.content['body']) + parts = yield self._extract_parts(hdoc.content) + + mail = PixelatedMail.from_soledad(fdoc, hdoc, bdoc, parts=parts) + defer.returnValue(mail) + + @defer.inlineCallbacks + def mails(self, idents): + fdocs_chash = [((yield self.get_flags_by_chash(ident)), ident) for ident in + idents] + fdocs_chash = [(result, ident) for result, ident in fdocs_chash if result] + defer.returnValue((yield self._build_mails_from_fdocs(fdocs_chash))) + + @defer.inlineCallbacks + def attachment(self, attachment_ident, encoding): + bdoc = yield self.get_content_by_phash(attachment_ident) + defer.returnValue({'content': self._try_decode(bdoc.content['raw'], encoding), + 'content-type': bdoc.content['content_type']}) + + def _try_decode(self, raw, encoding): + encoding = encoding.lower() + if encoding == 'base64': + return base64.decodestring(raw) + elif encoding == 'quoted-printable': + return quopri.decodestring(raw) + else: + return str(raw) + + @defer.inlineCallbacks + def _extract_parts(self, hdoc, parts=None): + if not parts: + parts = {'alternatives': [], 'attachments': []} + + if hdoc['multi']: + for part_key in hdoc.get('part_map', {}).keys(): + yield self._extract_parts(hdoc['part_map'][part_key], parts) + else: + headers_dict = {elem[0]: elem[1] for elem in hdoc.get('headers', [])} + if 'attachment' in headers_dict.get('Content-Disposition', ''): + parts['attachments'].append(self._extract_attachment(hdoc, headers_dict)) + else: + parts['alternatives'].append((yield self._extract_alternative(hdoc, headers_dict))) + defer.returnValue(parts) + + @defer.inlineCallbacks + def _extract_alternative(self, hdoc, headers_dict): + bdoc = yield self.get_content_by_phash(hdoc['phash']) + + if bdoc is None: + logger.warning("No BDOC content found for message!!!") + raw_content = "" + else: + raw_content = bdoc.content['raw'] + + defer.returnValue({'headers': headers_dict, 'content': raw_content}) + + def _extract_attachment(self, hdoc, headers_dict): + content_disposition = headers_dict['Content-Disposition'] + match = re.compile('.*name=\"(.*)\".*').search(content_disposition) + filename = '' + if match: + filename = match.group(1) + return {'headers': headers_dict, 'ident': hdoc['phash'], 'name': filename} diff --git a/service/pixelated/adapter/soledad/soledad_search_key_masterkey_retrieval_mixin.py b/service/pixelated/adapter/soledad/soledad_search_key_masterkey_retrieval_mixin.py new file mode 100644 index 00000000..8a912bc2 --- /dev/null +++ b/service/pixelated/adapter/soledad/soledad_search_key_masterkey_retrieval_mixin.py @@ -0,0 +1,35 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . +from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin +import os +import base64 + + +class SoledadSearchIndexMasterkeyRetrievalMixin(SoledadDbFacadeMixin, object): + + def get_index_masterkey(self): + deferred = self.get_search_index_masterkey() + deferred.addCallback(self._ensure_masterkey_exists) + return deferred + + def _ensure_masterkey_exists(self, result): + index_key_doc = result[0] if result else None + + if not index_key_doc: + new_index_key = os.urandom(64) # 32 for encryption, 32 for hmac + self.create_doc(dict(type='index_key', value=base64.encodestring(new_index_key))) + return new_index_key + return base64.decodestring(index_key_doc.content['value']) diff --git a/service/pixelated/adapter/soledad/soledad_writer_mixin.py b/service/pixelated/adapter/soledad/soledad_writer_mixin.py new file mode 100644 index 00000000..d47e722a --- /dev/null +++ b/service/pixelated/adapter/soledad/soledad_writer_mixin.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see . +from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin + +from twisted.internet import defer + + +class SoledadWriterMixin(SoledadDbFacadeMixin, object): + + @defer.inlineCallbacks + def mark_all_as_not_recent(self): + for mailbox in ['INBOX', 'DRAFTS', 'SENT', 'TRASH']: + rct = yield self.get_recent_by_mbox(mailbox) + if not rct or not rct[0].content['rct']: + return + rct = rct[0] + rct.content['rct'] = [] + yield self.put_doc(rct) + + def save_mail(self, mail): + return self.put_doc(mail.fdoc) + + @defer.inlineCallbacks + def create_mail(self, mail, mailbox_name): + mbox_doc = (yield self.get_mbox(mailbox_name))[0] + uid = 1 + (yield self.get_lastuid(mbox_doc)) + + yield self.create_docs(mail.get_for_save(next_uid=uid, mailbox=mailbox_name)) + + # FIXME need to update meta message (mdoc) + # mbox_doc.content['lastuid'] = uid + 1 + # self.put_doc(mbox_doc) + + defer.returnValue((yield self.mail(mail.ident))) + + @defer.inlineCallbacks + def remove_mail(self, mail): + # FIX-ME: Must go through all the part_map phash to delete all the cdocs + yield self.delete_doc(mail.fdoc) + yield self.delete_doc(mail.hdoc) + yield self.delete_doc(mail.bdoc) -- cgit v1.2.3