From fbae105bcdee32ffd5cb1afee8450967af5c0549 Mon Sep 17 00:00:00 2001 From: Folker Bernitt Date: Wed, 9 Sep 2015 09:59:03 +0200 Subject: Remove soledad querier and last usages - Issue #432 --- service/pixelated/adapter/soledad/__init__.py | 15 --- .../soledad/soledad_duplicate_removal_mixin.py | 46 -------- .../adapter/soledad/soledad_facade_mixin.py | 91 --------------- .../pixelated/adapter/soledad/soledad_querier.py | 29 ----- .../adapter/soledad/soledad_reader_mixin.py | 130 --------------------- ...soledad_search_key_masterkey_retrieval_mixin.py | 35 ------ .../adapter/soledad/soledad_writer_mixin.py | 54 --------- 7 files changed, 400 deletions(-) delete mode 100644 service/pixelated/adapter/soledad/__init__.py delete mode 100644 service/pixelated/adapter/soledad/soledad_duplicate_removal_mixin.py delete mode 100644 service/pixelated/adapter/soledad/soledad_facade_mixin.py delete mode 100644 service/pixelated/adapter/soledad/soledad_querier.py delete mode 100644 service/pixelated/adapter/soledad/soledad_reader_mixin.py delete mode 100644 service/pixelated/adapter/soledad/soledad_search_key_masterkey_retrieval_mixin.py delete mode 100644 service/pixelated/adapter/soledad/soledad_writer_mixin.py (limited to 'service/pixelated') diff --git a/service/pixelated/adapter/soledad/__init__.py b/service/pixelated/adapter/soledad/__init__.py deleted file mode 100644 index 2756a319..00000000 --- a/service/pixelated/adapter/soledad/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . diff --git a/service/pixelated/adapter/soledad/soledad_duplicate_removal_mixin.py b/service/pixelated/adapter/soledad/soledad_duplicate_removal_mixin.py deleted file mode 100644 index a2b4b6d7..00000000 --- a/service/pixelated/adapter/soledad/soledad_duplicate_removal_mixin.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . -from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin -from twisted.internet import defer - - -class SoledadDuplicateRemovalMixin(SoledadDbFacadeMixin, object): - - @defer.inlineCallbacks - def remove_duplicates(self): - for mailbox in ['INBOX', 'DRAFTS', 'SENT', 'TRASH']: - yield self._remove_dup_inboxes(mailbox) - yield self._remove_dup_recent(mailbox) - - @defer.inlineCallbacks - def _remove_many(self, docs): - [(yield self.delete_doc(doc)) for doc in docs] - - @defer.inlineCallbacks - def _remove_dup_inboxes(self, mailbox_name): - mailboxes = yield self.get_mbox(mailbox_name) - if len(mailboxes) == 0: - return - mailboxes_to_remove = sorted(mailboxes, key=lambda x: x.content['created'])[1:len(mailboxes)] - yield self._remove_many(mailboxes_to_remove) - - @defer.inlineCallbacks - def _remove_dup_recent(self, mailbox_name): - rct = yield self.get_recent_by_mbox(mailbox_name) - if len(rct) == 0: - return - rct_to_remove = sorted(rct, key=lambda x: len(x.content['rct']), reverse=True)[1:len(rct)] - yield self._remove_many(rct_to_remove) diff --git a/service/pixelated/adapter/soledad/soledad_facade_mixin.py b/service/pixelated/adapter/soledad/soledad_facade_mixin.py deleted file mode 100644 index 81280eaa..00000000 --- a/service/pixelated/adapter/soledad/soledad_facade_mixin.py +++ /dev/null @@ -1,91 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . - -from twisted.internet import defer -from leap.mail.mailbox_indexer import MailboxIndexer - - -class SoledadDbFacadeMixin(object): - - @defer.inlineCallbacks - def get_all_flags(self): - flags = yield self.soledad.get_from_index('by-type', 'flags') - defer.returnValue(flags) - - def get_all_flags_by_mbox(self, mbox): - return self.soledad.get_from_index('by-type-and-mbox', 'flags', mbox) if mbox else [] - - @defer.inlineCallbacks - def get_content_by_phash(self, phash): - content = yield self.soledad.get_from_index('by-type-and-payloadhash', 'cnt', phash) if phash else [] - if len(content): - defer.returnValue(content[0]) - - @defer.inlineCallbacks - def get_flags_by_chash(self, chash): - flags = yield self.soledad.get_from_index('by-type-and-contenthash', 'flags', chash) if chash else [] - if len(flags): - defer.returnValue(flags[0]) - - @defer.inlineCallbacks - def get_header_by_chash(self, chash): - header = yield self.soledad.get_from_index('by-type-and-contenthash', 'head', chash) if chash else [] - if len(header): - defer.returnValue(header[0]) - - @defer.inlineCallbacks - def get_recent_by_mbox(self, mbox): - defer.returnValue( - (yield self.soledad.get_from_index('by-type-and-mbox', 'rct', mbox) if mbox else [])) - - def put_doc(self, doc): - return self.soledad.put_doc(doc) - - def create_doc(self, doc): - return self.soledad.create_doc(doc) - - @defer.inlineCallbacks - def create_docs(self, docs): - for doc in docs: - yield self.create_doc(doc) - - def delete_doc(self, doc): - return self.soledad.delete_doc(doc) - - @defer.inlineCallbacks - def idents_by_mailbox(self, mbox): - mbox_docs = (yield self.soledad.get_from_index('by-type-and-mbox-and-deleted', 'flags', mbox, '0')) if mbox else [] - defer.returnValue(set(doc.content['chash'] for doc in mbox_docs)) - - def get_all_mbox(self): - return self.soledad.get_from_index('by-type', 'mbox') - - def get_mbox(self, mbox): - return self.soledad.get_from_index('by-type-and-mbox', 'mbox', mbox) if mbox else [] - - @defer.inlineCallbacks - def get_lastuid(self, mbox): - if isinstance(mbox, str): - mbox = (yield defer.maybeDeferred(self.get_mbox, mbox))[0] - - indexer = MailboxIndexer(self.soledad) - yield indexer.create_table(mbox.content['uuid']) - last_uuid = yield indexer.get_last_uid(mbox.content['uuid']) - - defer.returnValue(last_uuid) - - def get_search_index_masterkey(self): - return self.soledad.get_from_index('by-type', 'index_key') diff --git a/service/pixelated/adapter/soledad/soledad_querier.py b/service/pixelated/adapter/soledad/soledad_querier.py deleted file mode 100644 index e0b215d3..00000000 --- a/service/pixelated/adapter/soledad/soledad_querier.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . -from pixelated.adapter.soledad.soledad_duplicate_removal_mixin import SoledadDuplicateRemovalMixin -from pixelated.adapter.soledad.soledad_reader_mixin import SoledadReaderMixin -from pixelated.adapter.soledad.soledad_search_key_masterkey_retrieval_mixin import SoledadSearchIndexMasterkeyRetrievalMixin -from pixelated.adapter.soledad.soledad_writer_mixin import SoledadWriterMixin - - -class SoledadQuerier(SoledadWriterMixin, - SoledadReaderMixin, - SoledadDuplicateRemovalMixin, - SoledadSearchIndexMasterkeyRetrievalMixin, - object): - - def __init__(self, soledad): - self.soledad = soledad diff --git a/service/pixelated/adapter/soledad/soledad_reader_mixin.py b/service/pixelated/adapter/soledad/soledad_reader_mixin.py deleted file mode 100644 index e86298dd..00000000 --- a/service/pixelated/adapter/soledad/soledad_reader_mixin.py +++ /dev/null @@ -1,130 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . -import base64 -import logging -import quopri -import re - -from twisted.internet import defer -from pixelated.adapter.model.mail import PixelatedMail -from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin - - -logger = logging.getLogger(__name__) - - -class SoledadReaderMixin(SoledadDbFacadeMixin, object): - - @defer.inlineCallbacks - def all_mails(self): - fdocs_chash = [(fdoc, fdoc.content['chash']) for fdoc in (yield self.get_all_flags())] - if len(fdocs_chash) == 0: - defer.returnValue([]) - defer.returnValue((yield self._build_mails_from_fdocs(fdocs_chash))) - - @defer.inlineCallbacks - def _build_mails_from_fdocs(self, fdocs_chash): - if len(fdocs_chash) == 0: - defer.returnValue([]) - - fdocs_hdocs = [] - for fdoc, chash in fdocs_chash: - hdoc = yield self.get_header_by_chash(chash) - if not hdoc: - continue - fdocs_hdocs.append((fdoc, hdoc)) - - fdocs_hdocs_bodyphash = [(f[0], f[1], f[1].content.get('body')) for f in fdocs_hdocs] - fdocs_hdocs_bdocs_parts = [] - for fdoc, hdoc, body_phash in fdocs_hdocs_bodyphash: - bdoc = yield self.get_content_by_phash(body_phash) - if not bdoc: - continue - parts = yield self._extract_parts(hdoc.content) - fdocs_hdocs_bdocs_parts.append((fdoc, hdoc, bdoc, parts)) - - defer.returnValue([PixelatedMail.from_soledad(*raw_mail) for raw_mail in fdocs_hdocs_bdocs_parts]) - - def mail_exists(self, ident): - return self.get_flags_by_chash(ident) - - @defer.inlineCallbacks - def mail(self, ident): - fdoc = yield self.get_flags_by_chash(ident) - hdoc = yield self.get_header_by_chash(ident) - bdoc = yield self.get_content_by_phash(hdoc.content['body']) - parts = yield self._extract_parts(hdoc.content) - - mail = PixelatedMail.from_soledad(fdoc, hdoc, bdoc, parts=parts) - defer.returnValue(mail) - - @defer.inlineCallbacks - def mails(self, idents): - fdocs_chash = [((yield self.get_flags_by_chash(ident)), ident) for ident in - idents] - fdocs_chash = [(result, ident) for result, ident in fdocs_chash if result] - defer.returnValue((yield self._build_mails_from_fdocs(fdocs_chash))) - - @defer.inlineCallbacks - def attachment(self, attachment_ident, encoding): - bdoc = yield self.get_content_by_phash(attachment_ident) - defer.returnValue({'content': self._try_decode(bdoc.content['raw'], encoding), - 'content-type': bdoc.content['content_type']}) - - def _try_decode(self, raw, encoding): - encoding = encoding.lower() - if encoding == 'base64': - return base64.decodestring(raw) - elif encoding == 'quoted-printable': - return quopri.decodestring(raw) - else: - return str(raw) - - @defer.inlineCallbacks - def _extract_parts(self, hdoc, parts=None): - if not parts: - parts = {'alternatives': [], 'attachments': []} - - if hdoc['multi']: - for part_key in hdoc.get('part_map', {}).keys(): - yield self._extract_parts(hdoc['part_map'][part_key], parts) - else: - headers_dict = {elem[0]: elem[1] for elem in hdoc.get('headers', [])} - if 'attachment' in headers_dict.get('Content-Disposition', ''): - parts['attachments'].append(self._extract_attachment(hdoc, headers_dict)) - else: - parts['alternatives'].append((yield self._extract_alternative(hdoc, headers_dict))) - defer.returnValue(parts) - - @defer.inlineCallbacks - def _extract_alternative(self, hdoc, headers_dict): - bdoc = yield self.get_content_by_phash(hdoc['phash']) - - if bdoc is None: - logger.warning("No BDOC content found for message!!!") - raw_content = "" - else: - raw_content = bdoc.content['raw'] - - defer.returnValue({'headers': headers_dict, 'content': raw_content}) - - def _extract_attachment(self, hdoc, headers_dict): - content_disposition = headers_dict['Content-Disposition'] - match = re.compile('.*name=\"(.*)\".*').search(content_disposition) - filename = '' - if match: - filename = match.group(1) - return {'headers': headers_dict, 'ident': hdoc['phash'], 'name': filename} diff --git a/service/pixelated/adapter/soledad/soledad_search_key_masterkey_retrieval_mixin.py b/service/pixelated/adapter/soledad/soledad_search_key_masterkey_retrieval_mixin.py deleted file mode 100644 index 8a912bc2..00000000 --- a/service/pixelated/adapter/soledad/soledad_search_key_masterkey_retrieval_mixin.py +++ /dev/null @@ -1,35 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . -from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin -import os -import base64 - - -class SoledadSearchIndexMasterkeyRetrievalMixin(SoledadDbFacadeMixin, object): - - def get_index_masterkey(self): - deferred = self.get_search_index_masterkey() - deferred.addCallback(self._ensure_masterkey_exists) - return deferred - - def _ensure_masterkey_exists(self, result): - index_key_doc = result[0] if result else None - - if not index_key_doc: - new_index_key = os.urandom(64) # 32 for encryption, 32 for hmac - self.create_doc(dict(type='index_key', value=base64.encodestring(new_index_key))) - return new_index_key - return base64.decodestring(index_key_doc.content['value']) diff --git a/service/pixelated/adapter/soledad/soledad_writer_mixin.py b/service/pixelated/adapter/soledad/soledad_writer_mixin.py deleted file mode 100644 index d47e722a..00000000 --- a/service/pixelated/adapter/soledad/soledad_writer_mixin.py +++ /dev/null @@ -1,54 +0,0 @@ -# -# Copyright (c) 2014 ThoughtWorks, Inc. -# -# Pixelated is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Pixelated is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Pixelated. If not, see . -from pixelated.adapter.soledad.soledad_facade_mixin import SoledadDbFacadeMixin - -from twisted.internet import defer - - -class SoledadWriterMixin(SoledadDbFacadeMixin, object): - - @defer.inlineCallbacks - def mark_all_as_not_recent(self): - for mailbox in ['INBOX', 'DRAFTS', 'SENT', 'TRASH']: - rct = yield self.get_recent_by_mbox(mailbox) - if not rct or not rct[0].content['rct']: - return - rct = rct[0] - rct.content['rct'] = [] - yield self.put_doc(rct) - - def save_mail(self, mail): - return self.put_doc(mail.fdoc) - - @defer.inlineCallbacks - def create_mail(self, mail, mailbox_name): - mbox_doc = (yield self.get_mbox(mailbox_name))[0] - uid = 1 + (yield self.get_lastuid(mbox_doc)) - - yield self.create_docs(mail.get_for_save(next_uid=uid, mailbox=mailbox_name)) - - # FIXME need to update meta message (mdoc) - # mbox_doc.content['lastuid'] = uid + 1 - # self.put_doc(mbox_doc) - - defer.returnValue((yield self.mail(mail.ident))) - - @defer.inlineCallbacks - def remove_mail(self, mail): - # FIX-ME: Must go through all the part_map phash to delete all the cdocs - yield self.delete_doc(mail.fdoc) - yield self.delete_doc(mail.hdoc) - yield self.delete_doc(mail.bdoc) -- cgit v1.2.3