From 6ede495b94501a4cbdfd985dcdf4be4f582bbb9b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 25 Nov 2014 15:04:26 +0100 Subject: Serializable Models + Soledad Adaptor --- src/leap/mail/adaptors/__init__.py | 0 src/leap/mail/adaptors/models.py | 125 ++++ src/leap/mail/adaptors/soledad.py | 723 +++++++++++++++++++++ src/leap/mail/adaptors/soledad_indexes.py | 112 ++++ src/leap/mail/adaptors/tests/__init__.py | 0 src/leap/mail/adaptors/tests/rfc822.message | 86 +++ src/leap/mail/adaptors/tests/test_models.py | 103 +++ .../mail/adaptors/tests/test_soledad_adaptor.py | 583 +++++++++++++++++ src/leap/mail/constants.py | 21 + src/leap/mail/imap/account.py | 223 +++---- src/leap/mail/imap/fields.py | 132 +--- src/leap/mail/imap/index.py | 90 --- src/leap/mail/imap/interfaces.py | 2 + src/leap/mail/imap/mailbox.py | 76 ++- src/leap/mail/imap/messages.py | 484 +++++--------- src/leap/mail/imap/parser.py | 45 -- src/leap/mail/imap/tests/test_imap.py | 2 + src/leap/mail/imap/tests/utils.py | 15 +- src/leap/mail/interfaces.py | 113 ++++ src/leap/mail/mail.py | 248 +++++++ 20 files changed, 2415 insertions(+), 768 deletions(-) create mode 100644 src/leap/mail/adaptors/__init__.py create mode 100644 src/leap/mail/adaptors/models.py create mode 100644 src/leap/mail/adaptors/soledad.py create mode 100644 src/leap/mail/adaptors/soledad_indexes.py create mode 100644 src/leap/mail/adaptors/tests/__init__.py create mode 100644 src/leap/mail/adaptors/tests/rfc822.message create mode 100644 src/leap/mail/adaptors/tests/test_models.py create mode 100644 src/leap/mail/adaptors/tests/test_soledad_adaptor.py create mode 100644 src/leap/mail/constants.py delete mode 100644 src/leap/mail/imap/index.py delete mode 100644 src/leap/mail/imap/parser.py create mode 100644 src/leap/mail/interfaces.py create mode 100644 src/leap/mail/mail.py diff --git a/src/leap/mail/adaptors/__init__.py b/src/leap/mail/adaptors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/mail/adaptors/models.py b/src/leap/mail/adaptors/models.py new file mode 100644 index 0000000..1648059 --- /dev/null +++ b/src/leap/mail/adaptors/models.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +# models.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Generic Models to be used by the Document Adaptors. +""" +import copy + + +class SerializableModel(object): + """ + A Generic document model, that can be serialized into a dictionary. + + Subclasses of this `SerializableModel` are meant to be added as class + attributes of classes inheriting from DocumentWrapper. + + A subclass __meta__ of this SerializableModel might exist, and contain info + relative to particularities of this model. + + For instance, the use of `__meta__.index` marks the existence of a primary + index in the model, which will be used to do unique queries (in which case + all the other indexed fields in the underlying document will be filled with + the default info contained in the model definition). + """ + + @classmethod + def serialize(klass): + """ + Get a dictionary representation of the public attributes in the model + class. To avoid collisions with builtin functions, any occurrence of an + attribute ended in '_' (like 'type_') will be normalized by removing + the trailing underscore. + + This classmethod is used from within the serialized method of a + DocumentWrapper instance: it provides defaults for the + empty document. + """ + assert isinstance(klass, type) + return _normalize_dict(klass.__dict__) + + +class DocumentWrapper(object): + """ + A Wrapper object that can be manipulated, passed around, and serialized in + a format that the store understands. + It is related to a SerializableModel, which must be specified as the + ``model`` class attribute. The instance of this DocumentWrapper will not + allow any other *public* attributes than those defined in the corresponding + model. + """ + # TODO we could do some very basic type checking here + # TODO set a dirty flag (on __setattr__, whenever the value is != from + # before) + # TODO we could enforce the existence of a correct "model" attribute + # in some other way (other than in the initializer) + + def __init__(self, **kwargs): + if not getattr(self, 'model', None): + raise RuntimeError( + 'DocumentWrapper class needs a model attribute') + + defaults = self.model.serialize() + + if kwargs: + values = copy.deepcopy(defaults) + values.update(kwargs) + else: + values = defaults + + for k, v in values.items(): + k = k.replace('-', '_') + setattr(self, k, v) + + def __setattr__(self, attr, value): + normalized = _normalize_dict(self.model.__dict__) + if not attr.startswith('_') and attr not in normalized: + raise RuntimeError( + "Cannot set attribute because it's not defined " + "in the model: %s" % attr) + object.__setattr__(self, attr, value) + + def serialize(self): + return _normalize_dict(self.__dict__) + + def create(self): + raise NotImplementedError() + + def update(self): + raise NotImplementedError() + + def delete(self): + raise NotImplementedError() + + @classmethod + def get_or_create(self): + raise NotImplementedError() + + @classmethod + def get_all(self): + raise NotImplementedError() + + +def _normalize_dict(_dict): + items = _dict.items() + not_callable = lambda (k, v): not callable(v) + not_private = lambda(k, v): not k.startswith('_') + for cond in not_callable, not_private: + items = filter(cond, items) + items = [(k, v) if not k.endswith('_') else (k[:-1], v) + for (k, v) in items] + items = [(k.replace('-', '_'), v) for (k, v) in items] + return dict(items) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py new file mode 100644 index 0000000..2e25f04 --- /dev/null +++ b/src/leap/mail/adaptors/soledad.py @@ -0,0 +1,723 @@ +# -*- coding: utf-8 -*- +# soledad.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Soledadad MailAdaptor module. +""" +import re +from collections import defaultdict +from email import message_from_string + +from pycryptopp.hash import sha256 +from twisted.internet import defer +from zope.interface import implements + +from leap.common.check import leap_assert, leap_assert_type + +from leap.mail import walk +from leap.mail.adaptors import soledad_indexes as indexes +from leap.mail.constants import INBOX_NAME +from leap.mail.adaptors import models +from leap.mail.imap.mailbox import normalize_mailbox +from leap.mail.utils import lowerdict, first +from leap.mail.utils import stringify_parts_map +from leap.mail.interfaces import IMailAdaptor, IMessageWrapper + + +# TODO +# [ ] Convenience function to create mail specifying subject, date, etc? + + +_MSGID_PATTERN = r"""<([\w@.]+)>""" +_MSGID_RE = re.compile(_MSGID_PATTERN) + + +class DuplicatedDocumentError(Exception): + """ + Raised when a duplicated document is detected. + """ + pass + + +class SoledadDocumentWrapper(models.DocumentWrapper): + """ + A Wrapper object that can be manipulated, passed around, and serialized in + a format that the Soledad Store understands. + + It ensures atomicity of the document operations on creation, update and + deletion. + """ + + # TODO we could also use a _dirty flag (in models) + + # We keep a dictionary with DeferredLocks, that will be + # unique to every subclass of SoledadDocumentWrapper. + _k_locks = defaultdict(defer.DeferredLock) + + @classmethod + def _get_klass_lock(cls): + """ + Get a DeferredLock that is unique for this subclass name. + Used to lock the access to indexes in the `get_or_create` call + for a particular DocumentWrapper. + """ + return cls._k_locks[cls.__name__] + + def __init__(self, **kwargs): + doc_id = kwargs.pop('doc_id', None) + self._doc_id = doc_id + self._lock = defer.DeferredLock() + super(SoledadDocumentWrapper, self).__init__(**kwargs) + + @property + def doc_id(self): + return self._doc_id + + def create(self, store): + """ + Create the documents for this wrapper. + Since this method will not check for duplication, the + responsibility of avoiding duplicates is left to the caller. + + You might be interested in using `get_or_create` classmethod + instead (that's the preferred way of creating documents from + the wrapper object). + + :return: a deferred that will fire when the underlying + Soledad document has been created. + :rtype: Deferred + """ + leap_assert(self._doc_id is None, + "This document already has a doc_id!") + + def update_doc_id(doc): + self._doc_id = doc.doc_id + return doc + d = store.create_doc(self.serialize()) + d.addCallback(update_doc_id) + return d + + def update(self, store): + """ + Update the documents for this wrapper. + + :return: a deferred that will fire when the underlying + Soledad document has been updated. + :rtype: Deferred + """ + # the deferred lock guards against revision conflicts + return self._lock.run(self._update, store) + + def _update(self, store): + leap_assert(self._doc_id is not None, + "Need to create doc before updating") + + def update_and_put_doc(doc): + doc.content.update(self.serialize()) + return store.put_doc(doc) + + d = store.get_doc(self._doc_id) + d.addCallback(update_and_put_doc) + return d + + def delete(self, store): + """ + Delete the documents for this wrapper. + + :return: a deferred that will fire when the underlying + Soledad document has been deleted. + :rtype: Deferred + """ + # the deferred lock guards against conflicts while updating + return self._lock.run(self._delete, store) + + def _delete(self, store): + leap_assert(self._doc_id is not None, + "Need to create doc before deleting") + # XXX might want to flag this DocumentWrapper to avoid + # updating it by mistake. This could go in models.DocumentWrapper + + def delete_doc(doc): + return store.delete_doc(doc) + + d = store.get_doc(self._doc_id) + d.addCallback(delete_doc) + return d + + @classmethod + def get_or_create(cls, store, index, value): + """ + Get a unique DocumentWrapper by index, or create a new one if the + matching query does not exist. + + :param index: the primary index for the model. + :type index: str + :param value: the value to query the primary index. + :type value: str + + :return: a deferred that will be fired with the SoledadDocumentWrapper + matching the index query, either existing or just created. + :rtype: Deferred + """ + return cls._get_klass_lock().run( + cls._get_or_create, store, index, value) + + @classmethod + def _get_or_create(cls, store, index, value): + assert store is not None + assert index is not None + assert value is not None + + def get_main_index(): + try: + return cls.model.__meta__.index + except AttributeError: + raise RuntimeError("The model is badly defined") + + def try_to_get_doc_from_index(indexes): + values = [] + idx_def = dict(indexes)[index] + if len(idx_def) == 1: + values = [value] + else: + main_index = get_main_index() + fields = cls.model.serialize() + for field in idx_def: + if field == main_index: + values.append(value) + else: + values.append(fields[field]) + d = store.get_from_index(index, *values) + return d + + def get_first_doc_if_any(docs): + if not docs: + return None + if len(docs) > 1: + raise DuplicatedDocumentError + return docs[0] + + def wrap_existing_or_create_new(doc): + if doc: + return cls(doc_id=doc.doc_id, **doc.content) + else: + return create_and_wrap_new_doc() + + def create_and_wrap_new_doc(): + # XXX use closure to store indexes instead of + # querying for them again. + d = store.list_indexes() + d.addCallback(get_wrapper_instance_from_index) + d.addCallback(return_wrapper_when_created) + return d + + def get_wrapper_instance_from_index(indexes): + init_values = {} + idx_def = dict(indexes)[index] + if len(idx_def) == 1: + init_value = {idx_def[0]: value} + return cls(**init_value) + main_index = get_main_index() + fields = cls.model.serialize() + for field in idx_def: + if field == main_index: + init_values[field] = value + else: + init_values[field] = fields[field] + return cls(**init_values) + + def return_wrapper_when_created(wrapper): + d = wrapper.create(store) + d.addCallback(lambda doc: wrapper) + return d + + d = store.list_indexes() + d.addCallback(try_to_get_doc_from_index) + d.addCallback(get_first_doc_if_any) + d.addCallback(wrap_existing_or_create_new) + return d + + @classmethod + def get_all(cls, store): + """ + Get a collection of wrappers around all the documents belonging + to this kind. + + For this to work, the model.__meta__ needs to include a tuple with + the index to be used for listing purposes, and which is the field to be + used to query the index. + + Note that this method only supports indexes of a single field at the + moment. It also might be too expensive to return all the documents + matching the query, so handle with care. + + class __meta__(object): + index = "name" + list_index = ("by-type", "type_") + + :return: a deferred that will be fired with an iterable containing + as many SoledadDocumentWrapper are matching the index defined + in the model as the `list_index`. + :rtype: Deferred + """ + # TODO + # [ ] extend support to indexes with n-ples + # [ ] benchmark the cost of querying and returning indexes in a big + # database. This might badly need pagination before being put to + # serious use. + return cls._get_klass_lock().run(cls._get_all, store) + + @classmethod + def _get_all(cls, store): + try: + list_index, list_attr = cls.model.__meta__.list_index + except AttributeError: + raise RuntimeError("The model is badly defined: no list_index") + try: + index_value = getattr(cls.model, list_attr) + except AttributeError: + raise RuntimeError("The model is badly defined: " + "no attribute matching list_index") + + def wrap_docs(docs): + return (cls(doc_id=doc.doc_id, **doc.content) for doc in docs) + + d = store.get_from_index(list_index, index_value) + d.addCallback(wrap_docs) + return d + + # TODO + # [ ] get_count() ??? + + def __repr__(self): + try: + idx = getattr(self, self.model.__meta__.index) + except AttributeError: + idx = "" + return "<%s: %s (%s)>" % (self.__class__.__name__, + idx, self._doc_id) + + +# +# Message documents +# + +class FlagsDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "flags" + chash = "" + + mbox = "inbox" + seen = False + deleted = False + recent = False + multi = False + flags = [] + tags = [] + size = 0 + + class __meta__(object): + index = "mbox" + + +class HeaderDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "head" + chash = "" + + date = "" + subject = "" + headers = {} + part_map = {} + body = "" # link to phash of body + msgid = "" + multi = False + + class __meta__(object): + index = "chash" + + +class ContentDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "cnt" + phash = "" + + ctype = "" # XXX index by ctype too? + lkf = [] # XXX not implemented yet! + raw = "" + + content_disposition = "" + content_transfer_encoding = "" + content_type = "" + + class __meta__(object): + index = "phash" + + +class MessageWrapper(object): + + # TODO generalize wrapper composition? + # This could benefit of a DeferredLock to create/update all the + # documents at the same time maybe, and defend against concurrent updates? + + implements(IMessageWrapper) + + def __init__(self, fdoc, hdoc, cdocs=None): + """ + Need at least a flag-document and a header-document to instantiate a + MessageWrapper. Content-documents can be retrieved lazily. + + cdocs, if any, should be a dictionary in which the keys are ascending + integers, beginning at one, and the values are dictionaries with the + content of the content-docs. + """ + self.fdoc = FlagsDocWrapper(**fdoc) + self.hdoc = HeaderDocWrapper(**hdoc) + if cdocs is None: + cdocs = {} + cdocs_keys = cdocs.keys() + assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1) + self.cdocs = dict([(key, ContentDocWrapper(**doc)) for (key, doc) in + cdocs.items()]) + + def create(self, store): + """ + Create all the parts for this message in the store. + """ + leap_assert(self.cdocs, + "Need non empty cdocs to create the " + "MessageWrapper documents") + leap_assert(self.fdoc.doc_id is None, + "Cannot create: fdoc has a doc_id") + + # TODO I think we need to tolerate the no hdoc.doc_id case, for when we + # are doing a copy to another mailbox. + leap_assert(self.hdoc.doc_id is None, + "Cannot create: hdoc has a doc_id") + d = [] + d.append(self.fdoc.create(store)) + d.append(self.hdoc.create(store)) + for cdoc in self.cdocs.values(): + if cdoc.doc_id is not None: + # we could be just linking to an existing + # content-doc. + continue + d.append(cdoc.create(store)) + return defer.gatherResults(d) + + def update(self, store): + """ + Update the only mutable parts, which are within the flags document. + """ + return self.fdoc.update(store) + + def delete(self, store): + # Eventually this would have to do the duplicate search or send for the + # garbage collector. At least the fdoc can be unlinked. + raise NotImplementedError() + +# +# Mailboxes +# + + +class MailboxWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "mbox" + mbox = INBOX_NAME + flags = [] + closed = False + subscribed = False + rw = True + + class __meta__(object): + index = "mbox" + list_index = (indexes.TYPE_IDX, 'type_') + + +# +# Soledad Adaptor +# + +# TODO make this an interface? +class SoledadIndexMixin(object): + """ + this will need a class attribute `indexes`, that is a dictionary containing + the index definitions for the underlying u1db store underlying soledad. + + It needs to be in the following format: + {'index-name': ['field1', 'field2']} + """ + # TODO could have a wrapper class for indexes, supporting introspection + # and __getattr__ + indexes = {} + + store_ready = False + _index_creation_deferreds = [] + + # TODO we might want to move this logic to soledad itself + # so that each application can pass a set of indexes for their data model. + # TODO check also the decorator used in keymanager for waiting for indexes + # to be ready. + + def initialize_store(self, store): + """ + Initialize the indexes in the database. + + :param store: store + :returns: a Deferred that will fire when the store is correctly + initialized. + :rtype: deferred + """ + # TODO I think we *should* get another deferredLock in here, but + # global to the soledad namespace, to protect from several points + # initializing soledad indexes at the same time. + + leap_assert(store, "Need a store") + leap_assert_type(self.indexes, dict) + self._index_creation_deferreds = [] + + def _on_indexes_created(ignored): + self.store_ready = True + + def _create_index(name, expression): + d = store.create_index(name, *expression) + self._index_creation_deferreds.append(d) + + def _create_indexes(db_indexes): + db_indexes = dict(db_indexes) + + for name, expression in self.indexes.items(): + if name not in db_indexes: + # The index does not yet exist. + _create_index(name, expression) + continue + + if expression == db_indexes[name]: + # The index exists and is up to date. + continue + # The index exists but the definition is not what expected, so + # we delete it and add the proper index expression. + d1 = store.delete_index(name) + d1.addCallback(lambda _: _create_index(name, expression)) + + all_created = defer.gatherResults(self._index_creation_deferreds) + all_created.addCallback(_on_indexes_created) + return all_created + + # Ask the database for currently existing indexes, and create them + # if not found. + d = store.list_indexes() + d.addCallback(_create_indexes) + return d + + +class SoledadMailAdaptor(SoledadIndexMixin): + + implements(IMailAdaptor) + store = None + + indexes = indexes.MAIL_INDEXES + + # Message handling + + def get_msg_from_string(self, MessageClass, raw_msg): + """ + Get an instance of a MessageClass initialized with a MessageWrapper + that contains all the parts obtained from parsing the raw string for + the message. + + :param MessageClass: any Message class that can be initialized passing + an instance of an IMessageWrapper implementor. + :type MessageClass: type + :param raw_msg: a string containing the raw email message. + :type raw_msg: str + :rtype: MessageClass instance. + """ + assert(MessageClass is not None) + fdoc, hdoc, cdocs = _split_into_parts(raw_msg) + return self.get_msg_from_docs( + MessageClass, fdoc, hdoc, cdocs) + + def get_msg_from_docs(self, MessageClass, fdoc, hdoc, cdocs=None): + """ + Get an instance of a MessageClass initialized with a MessageWrapper + that contains the passed part documents. + + This is not the recommended way of obtaining a message, unless you know + how to take care of ensuring the internal consistency between the part + documents, or unless you are glueing together the part documents that + have been previously generated by `get_msg_from_string`. + + :param MessageClass: any Message class that can be initialized passing + an instance of an IMessageWrapper implementor. + :type MessageClass: type + :param fdoc: a dictionary containing values from which a + FlagsDocWrapper can be initialized + :type fdoc: dict + :param hdoc: a dictionary containing values from which a + HeaderDocWrapper can be initialized + :type hdoc: dict + :param cdocs: None, or a dictionary mapping integers (1-indexed) to + dicts from where a ContentDocWrapper can be initialized. + :type cdocs: dict, or None + + :rtype: MessageClass instance. + """ + assert(MessageClass is not None) + return MessageClass(MessageWrapper(fdoc, hdoc, cdocs)) + + def create_msg(self, store, msg): + """ + :param store: an instance of soledad, or anything that behaves alike + :type store: + :param msg: a Message object. + + :return: a Deferred that is fired when all the underlying documents + have been created. + :rtype: defer.Deferred + """ + wrapper = msg.get_wrapper() + return wrapper.create(store) + + def update_msg(self, store, msg): + """ + :param msg: a Message object. + :param store: an instance of soledad, or anything that behaves alike + :type store: + :param msg: a Message object. + :return: a Deferred that is fired when all the underlying documents + have been updated (actually, it's only the fdoc that's allowed + to update). + :rtype: defer.Deferred + """ + wrapper = msg.get_wrapper() + return wrapper.update(store) + + # Mailbox handling + + def get_or_create_mbox(self, store, name): + """ + Get the mailbox with the given name, or creatre one if it does not + exist. + + :param name: the name of the mailbox + :type name: str + """ + index = indexes.TYPE_MBOX_IDX + mbox = normalize_mailbox(name) + return MailboxWrapper.get_or_create(store, index, mbox) + + def update_mbox(self, store, mbox_wrapper): + """ + Update the documents for a given mailbox. + :param mbox_wrapper: MailboxWrapper instance + :type mbox_wrapper: MailboxWrapper + :return: a Deferred that will be fired when the mailbox documents + have been updated. + :rtype: defer.Deferred + """ + return mbox_wrapper.update(store) + + def get_all_mboxes(self, store): + """ + Retrieve a list with wrappers for all the mailboxes. + + :return: a deferred that will be fired with a list of all the + MailboxWrappers found. + :rtype: defer.Deferred + """ + return MailboxWrapper.get_all(store) + + +def _split_into_parts(raw): + # TODO signal that we can delete the original message!----- + # when all the processing is done. + # TODO add the linked-from info ! + # TODO add reference to the original message? + # TODO populate Default FLAGS/TAGS (unseen?) + # TODO seed propely the content_docs with defaults?? + + msg, parts, chash, size, multi = _parse_msg(raw) + body_phash_fun = [walk.get_body_phash_simple, + walk.get_body_phash_multi][int(multi)] + body_phash = body_phash_fun(walk.get_payloads(msg)) + parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) + + fdoc = _build_flags_doc(chash, size, multi) + hdoc = _build_headers_doc(msg, chash, parts_map) + + # The MessageWrapper expects a dict, one-indexed + cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1)) + + # XXX convert each to_dicts... + return fdoc, hdoc, cdocs + + +def _parse_msg(raw): + msg = message_from_string(raw) + parts = walk.get_parts(msg) + size = len(raw) + chash = sha256.SHA256(raw).hexdigest() + multi = msg.is_multipart() + return msg, parts, chash, size, multi + + +def _build_flags_doc(chash, size, multi): + _fdoc = FlagsDocWrapper(chash=chash, size=size, multi=multi) + return _fdoc.serialize() + + +def _build_headers_doc(msg, chash, parts_map): + """ + Assemble a headers document from the original parsed message, the + content-hash, and the parts map. + + It takes into account possibly repeated headers. + """ + headers = defaultdict(list) + for k, v in msg.items(): + headers[k].append(v) + + # "fix" for repeated headers. + for k, v in headers.items(): + newline = "\n%s: " % (k,) + headers[k] = newline.join(v) + + lower_headers = lowerdict(headers) + msgid = first(_MSGID_RE.findall( + lower_headers.get('message-id', ''))) + + _hdoc = HeaderDocWrapper( + chash=chash, headers=lower_headers, msgid=msgid) + + def copy_attr(headers, key, doc): + if key in headers: + setattr(doc, key, headers[key]) + + copy_attr(lower_headers, "subject", _hdoc) + copy_attr(lower_headers, "date", _hdoc) + + hdoc = _hdoc.serialize() + # add parts map to header doc + # (body, multi, part_map) + for key in parts_map: + hdoc[key] = parts_map[key] + return stringify_parts_map(hdoc) diff --git a/src/leap/mail/adaptors/soledad_indexes.py b/src/leap/mail/adaptors/soledad_indexes.py new file mode 100644 index 0000000..f3e990d --- /dev/null +++ b/src/leap/mail/adaptors/soledad_indexes.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- +# soledad_indexes.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Soledad Indexes for Mail Documents. +""" + +# TODO +# [ ] hide most of the constants here + +# Document Type, for indexing + +TYPE = "type" +MBOX = "mbox" +FLAGS = "flags" +HEADERS = "head" +CONTENT = "cnt" +RECENT = "rct" +HDOCS_SET = "hdocset" + +INCOMING_KEY = "incoming" +ERROR_DECRYPTING_KEY = "errdecr" + +# indexing keys +CONTENT_HASH = "chash" +PAYLOAD_HASH = "phash" +MSGID = "msgid" +UID = "uid" + + +# Index types +# -------------- + +TYPE_IDX = 'by-type' +TYPE_MBOX_IDX = 'by-type-and-mbox' +#TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' +TYPE_SUBS_IDX = 'by-type-and-subscribed' +TYPE_MSGID_IDX = 'by-type-and-message-id' +TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' +TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' +TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' +TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash' +TYPE_C_HASH_IDX = 'by-type-and-contenthash' +TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' +TYPE_P_HASH_IDX = 'by-type-and-payloadhash' + +# Soledad index for incoming mail, without decrypting errors. +# and the backward-compatible index, will be deprecated at 0.7 +JUST_MAIL_IDX = "just-mail" +JUST_MAIL_COMPAT_IDX = "just-mail-compat" + +# Tomas created the `recent and seen index`, but the semantic is not too +# correct since the recent flag is volatile --- XXX review and delete. +#TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' + +# TODO +# it would be nice to measure the cost of indexing +# by many fields. + +# TODO +# make the indexes dict more readable! + +MAIL_INDEXES = { + # generic + TYPE_IDX: [TYPE], + TYPE_MBOX_IDX: [TYPE, MBOX], + + # XXX deprecate 0.4.0 + # TYPE_MBOX_UID_IDX: [TYPE, MBOX, UID], + + # mailboxes + TYPE_SUBS_IDX: [TYPE, 'bool(subscribed)'], + + # fdocs uniqueness + TYPE_MBOX_C_HASH_IDX: [TYPE, MBOX, CONTENT_HASH], + + # headers doc - search by msgid. + TYPE_MSGID_IDX: [TYPE, MSGID], + + # content, headers doc + TYPE_C_HASH_IDX: [TYPE, CONTENT_HASH], + + # attachment payload dedup + TYPE_P_HASH_IDX: [TYPE, PAYLOAD_HASH], + + # messages + TYPE_MBOX_SEEN_IDX: [TYPE, MBOX, 'bool(seen)'], + TYPE_MBOX_RECT_IDX: [TYPE, MBOX, 'bool(recent)'], + TYPE_MBOX_DEL_IDX: [TYPE, MBOX, 'bool(deleted)'], + #TYPE_MBOX_RECT_SEEN_IDX: [TYPE, MBOX, + #'bool(recent)', 'bool(seen)'], + + # incoming queue + JUST_MAIL_IDX: [INCOMING_KEY, + "bool(%s)" % (ERROR_DECRYPTING_KEY,)], + + # the backward-compatible index, will be deprecated at 0.7 + JUST_MAIL_COMPAT_IDX: [INCOMING_KEY], +} diff --git a/src/leap/mail/adaptors/tests/__init__.py b/src/leap/mail/adaptors/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/mail/adaptors/tests/rfc822.message b/src/leap/mail/adaptors/tests/rfc822.message new file mode 100644 index 0000000..ee97ab9 --- /dev/null +++ b/src/leap/mail/adaptors/tests/rfc822.message @@ -0,0 +1,86 @@ +Return-Path: +Delivered-To: exarkun@meson.dyndns.org +Received: from localhost [127.0.0.1] + by localhost with POP3 (fetchmail-6.2.1) + for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) +Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) + by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 + for ; Thu, 20 Mar 2003 14:49:27 -0500 (EST) +Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) + by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) + id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 +Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) + id 18w63j-0007VK-00 + for ; Thu, 20 Mar 2003 13:50:39 -0600 +To: twisted-commits@twistedmatrix.com +From: etrepum CVS +Reply-To: twisted-python@twistedmatrix.com +X-Mailer: CVSToys +Message-Id: +Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. +Sender: twisted-commits-admin@twistedmatrix.com +Errors-To: twisted-commits-admin@twistedmatrix.com +X-BeenThere: twisted-commits@twistedmatrix.com +X-Mailman-Version: 2.0.11 +Precedence: bulk +List-Help: +List-Post: +List-Subscribe: , + +List-Id: +List-Unsubscribe: , + +List-Archive: +Date: Thu, 20 Mar 2003 13:50:39 -0600 + +Modified files: +Twisted/twisted/python/rebuild.py 1.19 1.20 + +Log message: +rebuild now works on python versions from 2.2.0 and up. + + +ViewCVS links: +http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted + +Index: Twisted/twisted/python/rebuild.py +diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 +--- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 ++++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 +@@ -206,15 +206,27 @@ + clazz.__dict__.clear() + clazz.__getattr__ = __getattr__ + clazz.__module__ = module.__name__ ++ if newclasses: ++ import gc ++ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): ++ hasBrokenRebuild = 1 ++ gc_objects = gc.get_objects() ++ else: ++ hasBrokenRebuild = 0 + for nclass in newclasses: + ga = getattr(module, nclass.__name__) + if ga is nclass: + log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) + else: +- import gc +- for r in gc.get_referrers(nclass): +- if isinstance(r, nclass): ++ if hasBrokenRebuild: ++ for r in gc_objects: ++ if not getattr(r, '__class__', None) is nclass: ++ continue + r.__class__ = ga ++ else: ++ for r in gc.get_referrers(nclass): ++ if getattr(r, '__class__', None) is nclass: ++ r.__class__ = ga + if doLog: + log.msg('') + log.msg(' (fixing %s): ' % str(module.__name__)) + + +_______________________________________________ +Twisted-commits mailing list +Twisted-commits@twistedmatrix.com +http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits diff --git a/src/leap/mail/adaptors/tests/test_models.py b/src/leap/mail/adaptors/tests/test_models.py new file mode 100644 index 0000000..efe0bf2 --- /dev/null +++ b/src/leap/mail/adaptors/tests/test_models.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# test_models.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the leap.mail.adaptors.models module. +""" +from twisted.trial import unittest + +from leap.mail.adaptors import models + + +class SerializableModelsTestCase(unittest.TestCase): + + def test_good_serialized_model(self): + + class M(models.SerializableModel): + foo = 42 + bar = 33 + baaz_ = None + _nope = 0 + __nope = 0 + + def not_today(self): + pass + + class IgnoreMe(object): + pass + + killmeplease = lambda x: x + + serialized = M.serialize() + expected = {'foo': 42, 'bar': 33, 'baaz': None} + self.assertEqual(serialized, expected) + + +class DocumentWrapperTestCase(unittest.TestCase): + + def test_wrapper_defaults(self): + + class Wrapper(models.DocumentWrapper): + class model(models.SerializableModel): + foo = 42 + bar = 11 + + wrapper = Wrapper() + wrapper._ignored = True + serialized = wrapper.serialize() + expected = {'foo': 42, 'bar': 11} + self.assertEqual(serialized, expected) + + def test_initialized_wrapper(self): + + class Wrapper(models.DocumentWrapper): + class model(models.SerializableModel): + foo = 42 + bar_ = 11 + + wrapper = Wrapper(foo=0, bar=-1) + serialized = wrapper.serialize() + expected = {'foo': 0, 'bar': -1} + self.assertEqual(serialized, expected) + + wrapper.foo = 23 + serialized = wrapper.serialize() + expected = {'foo': 23, 'bar': -1} + self.assertEqual(serialized, expected) + + wrapper = Wrapper(foo=0) + serialized = wrapper.serialize() + expected = {'foo': 0, 'bar': 11} + self.assertEqual(serialized, expected) + + def test_invalid_initialized_wrapper(self): + + class Wrapper(models.DocumentWrapper): + class model(models.SerializableModel): + foo = 42 + getwrapper = lambda: Wrapper(bar=1) + self.assertRaises(RuntimeError, getwrapper) + + def test_no_model_wrapper(self): + + class Wrapper(models.DocumentWrapper): + pass + + def getwrapper(): + w = Wrapper() + w.foo = None + + self.assertRaises(RuntimeError, getwrapper) diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py new file mode 100644 index 0000000..657a602 --- /dev/null +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -0,0 +1,583 @@ +# -*- coding: utf-8 -*- +# test_soledad_adaptor.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the Soledad Adaptor module - leap.mail.adaptors.soledad +""" +import os +import shutil +import tempfile + +from functools import partial + +from twisted.internet import defer +from twisted.trial import unittest + +from leap.common.testing.basetest import BaseLeapTest +from leap.mail.adaptors import models +from leap.mail.adaptors.soledad import SoledadDocumentWrapper +from leap.mail.adaptors.soledad import SoledadIndexMixin +from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.soledad.client import Soledad + +TEST_USER = "testuser@leap.se" +TEST_PASSWD = "1234" + +# DEBUG +# import logging +# logging.basicConfig(level=logging.DEBUG) + + +def initialize_soledad(email, gnupg_home, tempdir): + """ + Initializes soledad by hand + + :param email: ID for the user + :param gnupg_home: path to home used by gnupg + :param tempdir: path to temporal dir + :rtype: Soledad instance + """ + + uuid = "foobar-uuid" + passphrase = u"verysecretpassphrase" + secret_path = os.path.join(tempdir, "secret.gpg") + local_db_path = os.path.join(tempdir, "soledad.u1db") + server_url = "https://provider" + cert_file = "" + + soledad = Soledad( + uuid, + passphrase, + secret_path, + local_db_path, + server_url, + cert_file, + syncable=False) + + return soledad + + +# TODO move to common module +# XXX remove duplication +class SoledadTestMixin(BaseLeapTest): + """ + It is **VERY** important that this base is added *AFTER* unittest.TestCase + """ + + def setUp(self): + self.results = [] + + self.old_path = os.environ['PATH'] + self.old_home = os.environ['HOME'] + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.home = self.tempdir + bin_tdir = os.path.join( + self.tempdir, + 'bin') + os.environ["PATH"] = bin_tdir + os.environ["HOME"] = self.tempdir + + # Soledad: config info + self.gnupg_home = "%s/gnupg" % self.tempdir + self.email = 'leap@leap.se' + + # initialize soledad by hand so we can control keys + self._soledad = initialize_soledad( + self.email, + self.gnupg_home, + self.tempdir) + + def tearDown(self): + """ + tearDown method called after each test. + """ + self.results = [] + try: + self._soledad.close() + except Exception as exc: + print "ERROR WHILE CLOSING SOLEDAD" + # logging.exception(exc) + finally: + os.environ["PATH"] = self.old_path + os.environ["HOME"] = self.old_home + # safety check + assert 'leap_tests-' in self.tempdir + shutil.rmtree(self.tempdir) + + +class CounterWrapper(SoledadDocumentWrapper): + class model(models.SerializableModel): + counter = 0 + flag = None + + +class CharacterWrapper(SoledadDocumentWrapper): + class model(models.SerializableModel): + name = "" + age = 20 + + +class ActorWrapper(SoledadDocumentWrapper): + class model(models.SerializableModel): + type_ = "actor" + name = None + + class __meta__(object): + index = "name" + list_index = ("by-type", "type_") + + +class TestAdaptor(SoledadIndexMixin): + indexes = {'by-name': ['name'], + 'by-type-and-name': ['type', 'name'], + 'by-type': ['type']} + + +class SoledadDocWrapperTestCase(unittest.TestCase, SoledadTestMixin): + """ + Tests for the SoledadDocumentWrapper. + """ + def assert_num_docs(self, num, docs): + self.assertEqual(len(docs[1]), num) + + def test_create_single(self): + + store = self._soledad + wrapper = CounterWrapper() + + def assert_one_doc(docs): + self.assertEqual(docs[0], 1) + + d = wrapper.create(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(assert_one_doc) + return d + + def test_create_many(self): + + store = self._soledad + w1 = CounterWrapper() + w2 = CounterWrapper(counter=1) + w3 = CounterWrapper(counter=2) + w4 = CounterWrapper(counter=3) + w5 = CounterWrapper(counter=4) + + d1 = [w1.create(store), + w2.create(store), + w3.create(store), + w4.create(store), + w5.create(store)] + + d = defer.gatherResults(d1) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 5)) + return d + + def test_multiple_updates(self): + + store = self._soledad + wrapper = CounterWrapper(counter=1) + MAX = 100 + + def assert_doc_id(doc): + self.assertTrue(wrapper._doc_id is not None) + return doc + + def assert_counter_initial_ok(doc): + self.assertEqual(wrapper.counter, 1) + + def increment_counter(ignored): + d1 = [] + + def record_revision(revision): + rev = int(revision.split(':')[1]) + self.results.append(rev) + + for i in list(range(MAX)): + wrapper.counter += 1 + wrapper.flag = i % 2 == 0 + d = wrapper.update(store) + d.addCallback(record_revision) + d1.append(d) + + return defer.gatherResults(d1) + + def assert_counter_final_ok(doc): + self.assertEqual(doc.content['counter'], MAX + 1) + self.assertEqual(doc.content['flag'], False) + + def assert_results_ordered_list(ignored): + self.assertEqual(self.results, sorted(range(2, MAX + 2))) + + d = wrapper.create(store) + d.addCallback(assert_doc_id) + d.addCallback(assert_counter_initial_ok) + d.addCallback(increment_counter) + d.addCallback(lambda _: store.get_doc(wrapper._doc_id)) + d.addCallback(assert_counter_final_ok) + d.addCallback(assert_results_ordered_list) + return d + + def test_delete(self): + adaptor = TestAdaptor() + store = self._soledad + + wrapper_list = [] + + def get_or_create_bob(ignored): + def add_to_list(wrapper): + wrapper_list.append(wrapper) + return wrapper + wrapper = CharacterWrapper.get_or_create( + store, 'by-name', 'bob') + wrapper.addCallback(add_to_list) + return wrapper + + def delete_bob(ignored): + wrapper = wrapper_list[0] + return wrapper.delete(store) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + # this should create bob document + d.addCallback(get_or_create_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + d.addCallback(delete_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + return d + + def test_get_or_create(self): + adaptor = TestAdaptor() + store = self._soledad + + def get_or_create_bob(ignored): + wrapper = CharacterWrapper.get_or_create( + store, 'by-name', 'bob') + return wrapper + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + # this should create bob document + d.addCallback(get_or_create_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + # this should get us bob document + d.addCallback(get_or_create_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + return d + + def test_get_or_create_multi_index(self): + adaptor = TestAdaptor() + store = self._soledad + + def get_or_create_actor_harry(ignored): + wrapper = ActorWrapper.get_or_create( + store, 'by-type-and-name', 'harrison') + return wrapper + + def create_director_harry(ignored): + wrapper = ActorWrapper(name="harrison", type="director") + return wrapper.create(store) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + # this should create harrison document + d.addCallback(get_or_create_actor_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + # this should get us harrison document + d.addCallback(get_or_create_actor_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + # create director harry, should create new doc + d.addCallback(create_director_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 2)) + + # this should get us harrison document, still 2 docs + d.addCallback(get_or_create_actor_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 2)) + return d + + def test_get_all(self): + adaptor = TestAdaptor() + store = self._soledad + actor_names = ["harry", "carrie", "mark", "david"] + + def create_some_actors(ignored): + deferreds = [] + for name in actor_names: + dw = ActorWrapper.get_or_create( + store, 'by-type-and-name', name) + deferreds.append(dw) + return defer.gatherResults(deferreds) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + d.addCallback(create_some_actors) + + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 4)) + + def assert_actor_list_is_expected(res): + got = set([actor.name for actor in res]) + expected = set(actor_names) + self.assertEqual(got, expected) + + d.addCallback(lambda _: ActorWrapper.get_all(store)) + d.addCallback(assert_actor_list_is_expected) + return d + +here = os.path.split(os.path.abspath(__file__))[0] + + +class TestMessageClass(object): + def __init__(self, wrapper): + self.wrapper = wrapper + + def get_wrapper(self): + return self.wrapper + + +class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): + """ + Tests for the SoledadMailAdaptor. + """ + + def get_adaptor(self): + adaptor = SoledadMailAdaptor() + adaptor.store = self._soledad + return adaptor + + def assert_num_docs(self, num, docs): + self.assertEqual(len(docs[1]), num) + + def test_mail_adaptor_init(self): + adaptor = self.get_adaptor() + self.assertTrue(isinstance(adaptor.indexes, dict)) + self.assertTrue(len(adaptor.indexes) != 0) + + # Messages + + def test_get_msg_from_string(self): + adaptor = self.get_adaptor() + + with open(os.path.join(here, "rfc822.message")) as f: + raw = f.read() + + msg = adaptor.get_msg_from_string(TestMessageClass, raw) + + chash = ("D27B2771C0DCCDCB468EE65A4540438" + "09DBD11588E87E951545BE0CBC321C308") + phash = ("64934534C1C80E0D4FA04BE1CCBA104" + "F07BCA5F469C86E2C0ABE1D41310B7299") + subject = ("[Twisted-commits] rebuild now works on " + "python versions from 2.2.0 and up.") + self.assertTrue(msg.wrapper.fdoc is not None) + self.assertTrue(msg.wrapper.hdoc is not None) + self.assertTrue(msg.wrapper.cdocs is not None) + self.assertEquals(len(msg.wrapper.cdocs), 1) + self.assertEquals(msg.wrapper.fdoc.chash, chash) + self.assertEquals(msg.wrapper.fdoc.size, 3834) + self.assertEquals(msg.wrapper.hdoc.chash, chash) + self.assertEqual(msg.wrapper.hdoc.headers['subject'], + subject) + self.assertEqual(msg.wrapper.hdoc.subject, subject) + self.assertEqual(msg.wrapper.cdocs[1].phash, phash) + + def test_get_msg_from_docs(self): + adaptor = self.get_adaptor() + fdoc = dict( + mbox="Foobox", + flags=('\Seen', '\Nice'), + tags=('Personal', 'TODO'), + seen=False, deleted=False, + recent=False, multi=False) + hdoc = dict( + subject="Test Msg") + cdocs = { + 1: dict( + raw='This is a test message')} + + msg = adaptor.get_msg_from_docs( + TestMessageClass, fdoc, hdoc, cdocs=cdocs) + self.assertEqual(msg.wrapper.fdoc.flags, + ('\Seen', '\Nice')) + self.assertEqual(msg.wrapper.fdoc.tags, + ('Personal', 'TODO')) + self.assertEqual(msg.wrapper.fdoc.mbox, "Foobox") + self.assertEqual(msg.wrapper.hdoc.multi, False) + self.assertEqual(msg.wrapper.hdoc.subject, + "Test Msg") + self.assertEqual(msg.wrapper.cdocs[1].raw, + "This is a test message") + + def test_create_msg(self): + adaptor = self.get_adaptor() + + with open(os.path.join(here, "rfc822.message")) as f: + raw = f.read() + msg = adaptor.get_msg_from_string(TestMessageClass, raw) + + def check_create_result(created): + self.assertEqual(len(created), 3) + for doc in created: + self.assertTrue( + doc.__class__.__name__, + "SoledadDocument") + + d = adaptor.create_msg(adaptor.store, msg) + d.addCallback(check_create_result) + return d + + def test_update_msg(self): + adaptor = self.get_adaptor() + with open(os.path.join(here, "rfc822.message")) as f: + raw = f.read() + + def assert_msg_has_doc_id(ignored, msg): + wrapper = msg.get_wrapper() + self.assertTrue(wrapper.fdoc.doc_id is not None) + + def assert_msg_has_no_flags(ignored, msg): + wrapper = msg.get_wrapper() + self.assertEqual(wrapper.fdoc.flags, []) + + def update_msg_flags(ignored, msg): + wrapper = msg.get_wrapper() + wrapper.fdoc.flags = ["This", "That"] + return wrapper.update(adaptor.store) + + def assert_msg_has_flags(ignored, msg): + wrapper = msg.get_wrapper() + self.assertEqual(wrapper.fdoc.flags, ["This", "That"]) + + def get_fdoc_and_check_flags(ignored): + def assert_doc_has_flags(doc): + self.assertEqual(doc.content['flags'], + ['This', 'That']) + wrapper = msg.get_wrapper() + d = adaptor.store.get_doc(wrapper.fdoc.doc_id) + d.addCallback(assert_doc_has_flags) + return d + + msg = adaptor.get_msg_from_string(TestMessageClass, raw) + d = adaptor.create_msg(adaptor.store, msg) + d.addCallback(lambda _: adaptor.store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 3)) + d.addCallback(assert_msg_has_doc_id, msg) + d.addCallback(assert_msg_has_no_flags, msg) + + # update it! + d.addCallback(update_msg_flags, msg) + d.addCallback(assert_msg_has_flags, msg) + d.addCallback(get_fdoc_and_check_flags) + return d + + # Mailboxes + + def test_get_or_create_mbox(self): + adaptor = self.get_adaptor() + + def get_or_create_mbox(ignored): + d = adaptor.get_or_create_mbox(adaptor.store, "Trash") + return d + + def assert_good_doc(mbox_wrapper): + self.assertTrue(mbox_wrapper.doc_id is not None) + self.assertEqual(mbox_wrapper.mbox, "Trash") + self.assertEqual(mbox_wrapper.type, "mbox") + self.assertEqual(mbox_wrapper.closed, False) + self.assertEqual(mbox_wrapper.subscribed, False) + + d = adaptor.initialize_store(adaptor.store) + d.addCallback(get_or_create_mbox) + d.addCallback(assert_good_doc) + d.addCallback(lambda _: adaptor.store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + return d + + def test_update_mbox(self): + adaptor = self.get_adaptor() + + wrapper_ref = [] + + def get_or_create_mbox(ignored): + d = adaptor.get_or_create_mbox(adaptor.store, "Trash") + return d + + def update_wrapper(wrapper, wrapper_ref): + wrapper_ref.append(wrapper) + wrapper.subscribed = True + wrapper.closed = True + d = adaptor.update_mbox(adaptor.store, wrapper) + return d + + def get_mbox_doc_and_check_flags(res, wrapper_ref): + wrapper = wrapper_ref[0] + + def assert_doc_has_flags(doc): + self.assertEqual(doc.content['subscribed'], True) + self.assertEqual(doc.content['closed'], True) + d = adaptor.store.get_doc(wrapper.doc_id) + d.addCallback(assert_doc_has_flags) + return d + + d = adaptor.initialize_store(adaptor.store) + d.addCallback(get_or_create_mbox) + d.addCallback(update_wrapper, wrapper_ref) + d.addCallback(get_mbox_doc_and_check_flags, wrapper_ref) + return d + + def test_get_all_mboxes(self): + adaptor = self.get_adaptor() + mboxes = ("Sent", "Trash", "Personal", "ListFoo") + + def get_or_create_mboxes(ignored): + d = [] + for mbox in mboxes: + d.append(adaptor.get_or_create_mbox( + adaptor.store, mbox)) + return defer.gatherResults(d) + + def get_all_mboxes(ignored): + return adaptor.get_all_mboxes(adaptor.store) + + def assert_mboxes_match_expected(wrappers): + names = [m.mbox for m in wrappers] + self.assertEqual(set(names), set(mboxes)) + + d = adaptor.initialize_store(adaptor.store) + d.addCallback(get_or_create_mboxes) + d.addCallback(get_all_mboxes) + d.addCallback(assert_mboxes_match_expected) + return d diff --git a/src/leap/mail/constants.py b/src/leap/mail/constants.py new file mode 100644 index 0000000..55bf1da --- /dev/null +++ b/src/leap/mail/constants.py @@ -0,0 +1,21 @@ +# *- coding: utf-8 -*- +# constants.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Constants for leap.mail. +""" + +INBOX_NAME = "INBOX" diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index fe466cb..7dfbbd1 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -28,10 +28,10 @@ from twisted.python import log from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type -from leap.mail.imap.index import IndexedDB + +from leap.mail.mail import Account from leap.mail.imap.fields import WithMsgFields -from leap.mail.imap.parser import MBoxParser -from leap.mail.imap.mailbox import SoledadMailbox +from leap.mail.imap.mailbox import SoledadMailbox, normalize_mailbox from leap.soledad.client import Soledad logger = logging.getLogger(__name__) @@ -39,7 +39,6 @@ logger = logging.getLogger(__name__) PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False) if PROFILE_CMD: - def _debugProfiling(result, cmdname, start): took = (time.time() - start) * 1000 log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec") @@ -47,96 +46,43 @@ if PROFILE_CMD: ####################################### -# Soledad Account +# Soledad IMAP Account ####################################### +# TODO remove MsgFields too -# TODO change name to LeapIMAPAccount, since we're using -# the memstore. -# IndexedDB should also not be here anymore. - -class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): +class IMAPAccount(WithMsgFields): """ - An implementation of IAccount and INamespacePresenteer + An implementation of an imap4 Account that is backed by Soledad Encrypted Documents. """ implements(imap4.IAccount, imap4.INamespacePresenter) - _soledad = None selected = None closed = False - _initialized = False - def __init__(self, account_name, soledad, memstore=None): + def __init__(self, user_id, store): """ - Creates a SoledadAccountIndex that keeps track of the mailboxes - and subscriptions handled by this account. + Keeps track of the mailboxes and subscriptions handled by this account. - :param acct_name: The name of the account (user id). - :type acct_name: str + :param account: The name of the account (user id). + :type account: str - :param soledad: a Soledad instance. - :type soledad: Soledad - :param memstore: a MemoryStore instance. - :type memstore: MemoryStore + :param store: a Soledad instance. + :type store: Soledad """ - leap_assert(soledad, "Need a soledad instance to initialize") - leap_assert_type(soledad, Soledad) + # XXX assert a generic store interface instead, so that we + # can plug the memory store wrapper seamlessly. + leap_assert(store, "Need a store instance to initialize") + leap_assert_type(store, Soledad) # XXX SHOULD assert too that the name matches the user/uuid with which # soledad has been initialized. + self.user_id = user_id + self.account = Account(store) - # XXX ??? why is this parsing mailbox name??? it's account... - # userid? homogenize. - self._account_name = self._parse_mailbox_name(account_name) - self._soledad = soledad - self._memstore = memstore - - self.__mailboxes = set([]) - - self._deferred_initialization = defer.Deferred() - self._initialize_storage() - - def _initialize_storage(self): - - def add_mailbox_if_none(result): - # every user should have the right to an inbox folder - # at least, so let's make one! - if not self.mailboxes: - self.addMailbox(self.INBOX_NAME) - - def finish_initialization(result): - self._initialized = True - self._deferred_initialization.callback(None) - - def load_mbox_cache(result): - d = self._load_mailboxes() - d.addCallback(lambda _: result) - return d - - d = self.initialize_db() - - d.addCallback(load_mbox_cache) - d.addCallback(add_mailbox_if_none) - d.addCallback(finish_initialization) - - def callWhenReady(self, cb): - if self._initialized: - cb(self) - return defer.succeed(None) - else: - self._deferred_initialization.addCallback(cb) - return self._deferred_initialization - - def _get_empty_mailbox(self): - """ - Returns an empty mailbox. - - :rtype: dict - """ - return copy.deepcopy(self.EMPTY_MBOX) - + # XXX should hide this in the adaptor... def _get_mailbox_by_name(self, name): """ Return an mbox document by name. @@ -146,32 +92,17 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :rtype: SoledadDocument """ - # XXX use soledadstore instead ...; def get_first_if_any(docs): return docs[0] if docs else None - d = self._soledad.get_from_index( + d = self._store.get_from_index( self.TYPE_MBOX_IDX, self.MBOX_KEY, - self._parse_mailbox_name(name)) + normalize_mailbox(name)) d.addCallback(get_first_if_any) return d - @property - def mailboxes(self): - """ - A list of the current mailboxes for this account. - :rtype: set - """ - return sorted(self.__mailboxes) - - def _load_mailboxes(self): - def update_mailboxes(db_indexes): - self.__mailboxes.update( - [doc.content[self.MBOX_KEY] for doc in db_indexes]) - d = self._soledad.get_from_index(self.TYPE_IDX, self.MBOX_KEY) - d.addCallback(update_mailboxes) - return d - + # XXX move to Account? + # XXX needed? def getMailbox(self, name): """ Return a Mailbox with that name, without selecting it. @@ -182,18 +113,28 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :returns: a a SoledadMailbox instance :rtype: SoledadMailbox """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) - if name not in self.mailboxes: + if name not in self.account.mailboxes: raise imap4.MailboxException("No such mailbox: %r" % name) - return SoledadMailbox(name, self._soledad, - memstore=self._memstore) + # XXX Does mailbox really need reference to soledad? + return SoledadMailbox(name, self._store) # # IAccount # + def _get_empty_mailbox(self): + """ + Returns an empty mailbox. + + :rtype: dict + """ + # XXX move to mailbox module + return copy.deepcopy(mailbox.EMPTY_MBOX) + + # TODO use mail.Account.add_mailbox def addMailbox(self, name, creation_ts=None): """ Add a mailbox to the account. @@ -209,7 +150,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :returns: a Deferred that will contain the document if successful. :rtype: bool """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) leap_assert(name, "Need a mailbox name to create a mailbox") @@ -232,10 +173,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): d.addCallback(lambda _: result) return d - d = self._soledad.create_doc(mbox) + d = self._store.create_doc(mbox) d.addCallback(load_mbox_cache) return d + # TODO use mail.Account.create_mailbox? + # Watch out, imap specific exceptions raised here. def create(self, pathspec): """ Create a new mailbox from the given hierarchical name. @@ -254,9 +197,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :raise MailboxException: Raised if this mailbox cannot be added. """ # TODO raise MailboxException - paths = filter( - None, - self._parse_mailbox_name(pathspec).split('/')) + paths = filter(None, normalize_mailbox(pathspec).split('/')) subs = [] sep = '/' @@ -295,6 +236,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): d1.addCallback(load_mbox_cache) return d1 + # TODO use mail.Account.get_collection_by_mailbox def select(self, name, readwrite=1): """ Selects a mailbox. @@ -307,21 +249,16 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :rtype: SoledadMailbox """ - if PROFILE_CMD: - start = time.time() - - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) if name not in self.mailboxes: logger.warning("No such mailbox!") return None self.selected = name - sm = SoledadMailbox( - name, self._soledad, self._memstore, readwrite) - if PROFILE_CMD: - _debugProfiling(None, "SELECT", start) + sm = SoledadMailbox(name, self._store, readwrite) return sm + # TODO use mail.Account.delete_mailbox def delete(self, name, force=False): """ Deletes a mailbox. @@ -338,7 +275,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :type force: bool :rtype: Deferred """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) if name not in self.mailboxes: err = imap4.MailboxException("No such mailbox: %r" % name) @@ -369,6 +306,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): # ??! -- can this be rite? # self._index.removeMailbox(name) + # TODO use mail.Account.rename_mailbox def rename(self, oldname, newname): """ Renames a mailbox. @@ -379,8 +317,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param newname: new name of the mailbox :type newname: str """ - oldname = self._parse_mailbox_name(oldname) - newname = self._parse_mailbox_name(newname) + oldname = normalize_mailbox(oldname) + newname = normalize_mailbox(newname) if oldname not in self.mailboxes: raise imap4.NoSuchMailbox(repr(oldname)) @@ -431,6 +369,32 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): inferiors.append(infname) return inferiors + # TODO use mail.Account.list_mailboxes + def listMailboxes(self, ref, wildcard): + """ + List the mailboxes. + + from rfc 3501: + returns a subset of names from the complete set + of all names available to the client. Zero or more untagged LIST + replies are returned, containing the name attributes, hierarchy + delimiter, and name. + + :param ref: reference name + :type ref: str + + :param wildcard: mailbox name with possible wildcards + :type wildcard: str + """ + # XXX use wildcard in index query + ref = self._inferiorNames(normalize_mailbox(ref)) + wildcard = imap4.wildcardToRegexp(wildcard, '/') + return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] + + # + # The rest of the methods are specific for leap.mail.imap.account.Account + # + # TODO ------------------ can we preserve the attr? # maybe add to memory store. def isSubscribed(self, name): @@ -442,6 +406,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :rtype: Deferred (will fire with bool) """ + # TODO use Flags class subscribed = self.SUBSCRIBED_KEY def is_subscribed(mbox): @@ -465,7 +430,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): def get_docs_content(docs): return [doc.content[self.MBOX_KEY] for doc in docs] - d = self._soledad.get_from_index( + d = self._store.get_from_index( self.TYPE_SUBS_IDX, self.MBOX_KEY, '1') d.addCallback(get_docs_content) return d @@ -488,7 +453,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): def update_subscribed_value(mbox): mbox.content[subscribed] = value - return self._soledad.put_doc(mbox) + return self._store.put_doc(mbox) # maybe we should store subscriptions in another # document... @@ -508,7 +473,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :type name: str :rtype: Deferred """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) def check_and_subscribe(subscriptions): if name not in subscriptions: @@ -525,7 +490,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :type name: str :rtype: Deferred """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) def check_and_unsubscribe(subscriptions): if name not in subscriptions: @@ -539,28 +504,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): def getSubscriptions(self): return self._get_subscriptions() - def listMailboxes(self, ref, wildcard): - """ - List the mailboxes. - - from rfc 3501: - returns a subset of names from the complete set - of all names available to the client. Zero or more untagged LIST - replies are returned, containing the name attributes, hierarchy - delimiter, and name. - - :param ref: reference name - :type ref: str - - :param wildcard: mailbox name with possible wildcards - :type wildcard: str - """ - # XXX use wildcard in index query - ref = self._inferiorNames( - self._parse_mailbox_name(ref)) - wildcard = imap4.wildcardToRegexp(wildcard, '/') - return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] - # # INamespacePresenter # @@ -592,4 +535,4 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): """ Representation string for this object. """ - return "" % self._account_name + return "" % self.user_id diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py index 4576939..a751c6d 100644 --- a/src/leap/mail/imap/fields.py +++ b/src/leap/mail/imap/fields.py @@ -17,7 +17,9 @@ """ Fields for Mailbox and Message. """ -from leap.mail.imap.parser import MBoxParser + +# TODO deprecate !!! (move all to constants maybe?) +# Flags -> foo class WithMsgFields(object): @@ -25,55 +27,12 @@ class WithMsgFields(object): Container class for class-attributes to be shared by several message-related classes. """ - # indexing - CONTENT_HASH_KEY = "chash" - PAYLOAD_HASH_KEY = "phash" - - # Internal representation of Message - - # flags doc - UID_KEY = "uid" - MBOX_KEY = "mbox" - SEEN_KEY = "seen" - DEL_KEY = "deleted" - RECENT_KEY = "recent" - FLAGS_KEY = "flags" - MULTIPART_KEY = "multi" - SIZE_KEY = "size" - - # headers - HEADERS_KEY = "headers" - DATE_KEY = "date" - SUBJECT_KEY = "subject" - PARTS_MAP_KEY = "part_map" - BODY_KEY = "body" # link to phash of body - MSGID_KEY = "msgid" - - # content - LINKED_FROM_KEY = "lkf" # XXX not implemented yet! - RAW_KEY = "raw" - CTYPE_KEY = "ctype" - # Mailbox specific keys - CLOSED_KEY = "closed" - CREATED_KEY = "created" - SUBSCRIBED_KEY = "subscribed" - RW_KEY = "rw" - LAST_UID_KEY = "lastuid" + CREATED_KEY = "created" # used??? + RECENTFLAGS_KEY = "rct" HDOCS_SET_KEY = "hdocset" - # Document Type, for indexing - TYPE_KEY = "type" - TYPE_MBOX_VAL = "mbox" - TYPE_FLAGS_VAL = "flags" - TYPE_HEADERS_VAL = "head" - TYPE_CONTENT_VAL = "cnt" - TYPE_RECENT_VAL = "rct" - TYPE_HDOCS_SET_VAL = "hdocset" - - INBOX_VAL = "inbox" - # Flags in Mailbox and Message SEEN_FLAG = "\\Seen" RECENT_FLAG = "\\Recent" @@ -88,86 +47,5 @@ class WithMsgFields(object): SUBJECT_FIELD = "Subject" DATE_FIELD = "Date" - # Index types - # -------------- - - TYPE_IDX = 'by-type' - TYPE_MBOX_IDX = 'by-type-and-mbox' - TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' - TYPE_SUBS_IDX = 'by-type-and-subscribed' - TYPE_MSGID_IDX = 'by-type-and-message-id' - TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' - TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' - TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' - TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash' - TYPE_C_HASH_IDX = 'by-type-and-contenthash' - TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' - TYPE_P_HASH_IDX = 'by-type-and-payloadhash' - - # Tomas created the `recent and seen index`, but the semantic is not too - # correct since the recent flag is volatile. - TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' - - # Soledad index for incoming mail, without decrypting errors. - JUST_MAIL_IDX = "just-mail" - # XXX the backward-compatible index, will be deprecated at 0.7 - JUST_MAIL_COMPAT_IDX = "just-mail-compat" - - INCOMING_KEY = "incoming" - ERROR_DECRYPTING_KEY = "errdecr" - - KTYPE = TYPE_KEY - MBOX_VAL = TYPE_MBOX_VAL - CHASH_VAL = CONTENT_HASH_KEY - PHASH_VAL = PAYLOAD_HASH_KEY - - INDEXES = { - # generic - TYPE_IDX: [KTYPE], - TYPE_MBOX_IDX: [KTYPE, MBOX_VAL], - TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, UID_KEY], - - # mailboxes - TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'], - - # fdocs uniqueness - TYPE_MBOX_C_HASH_IDX: [KTYPE, MBOX_VAL, CHASH_VAL], - - # headers doc - search by msgid. - TYPE_MSGID_IDX: [KTYPE, MSGID_KEY], - - # content, headers doc - TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL], - - # attachment payload dedup - TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL], - - # messages - TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], - TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'], - TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'], - TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, - 'bool(recent)', 'bool(seen)'], - - # incoming queue - JUST_MAIL_IDX: [INCOMING_KEY, - "bool(%s)" % (ERROR_DECRYPTING_KEY,)], - - # the backward-compatible index, will be deprecated at 0.7 - JUST_MAIL_COMPAT_IDX: [INCOMING_KEY], - } - - MBOX_KEY = MBOX_VAL - - EMPTY_MBOX = { - TYPE_KEY: MBOX_KEY, - TYPE_MBOX_VAL: MBoxParser.INBOX_NAME, - SUBJECT_KEY: "", - FLAGS_KEY: [], - CLOSED_KEY: False, - SUBSCRIBED_KEY: False, - RW_KEY: 1, - LAST_UID_KEY: 0 - } fields = WithMsgFields # alias for convenience diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py deleted file mode 100644 index ea35fff..0000000 --- a/src/leap/mail/imap/index.py +++ /dev/null @@ -1,90 +0,0 @@ -# -*- coding: utf-8 -*- -# index.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Index for SoledadBackedAccount, Mailbox and Messages. -""" -import logging - -from twisted.internet import defer - -from leap.common.check import leap_assert, leap_assert_type - -from leap.mail.imap.fields import fields - - -logger = logging.getLogger(__name__) - - -class IndexedDB(object): - """ - Methods dealing with the index. - - This is a MixIn that needs access to the soledad instance, - and also assumes that a INDEXES attribute is accessible to the instance. - - INDEXES must be a dictionary of type: - {'index-name': ['field1', 'field2']} - """ - # TODO we might want to move this to soledad itself, check - - _index_creation_deferreds = [] - index_ready = False - - def initialize_db(self): - """ - Initialize the database. - """ - leap_assert(self._soledad, - "Need a soledad attribute accesible in the instance") - leap_assert_type(self.INDEXES, dict) - self._index_creation_deferreds = [] - - def _on_indexes_created(ignored): - self.index_ready = True - - def _create_index(name, expression): - d = self._soledad.create_index(name, *expression) - self._index_creation_deferreds.append(d) - - def _create_indexes(db_indexes): - db_indexes = dict(db_indexes) - for name, expression in fields.INDEXES.items(): - if name not in db_indexes: - # The index does not yet exist. - _create_index(name, expression) - continue - - if expression == db_indexes[name]: - # The index exists and is up to date. - continue - # The index exists but the definition is not what expected, so - # we delete it and add the proper index expression. - d1 = self._soledad.delete_index(name) - d1.addCallback(lambda _: _create_index(name, expression)) - - all_created = defer.gatherResults(self._index_creation_deferreds) - all_created.addCallback(_on_indexes_created) - return all_created - - # Ask the database for currently existing indexes. - if not self._soledad: - logger.debug("NO SOLEDAD ON IMAP INITIALIZATION") - return - if self._soledad is not None: - d = self._soledad.list_indexes() - d.addCallback(_create_indexes) - return d diff --git a/src/leap/mail/imap/interfaces.py b/src/leap/mail/imap/interfaces.py index c906278..f8f25fa 100644 --- a/src/leap/mail/imap/interfaces.py +++ b/src/leap/mail/imap/interfaces.py @@ -20,6 +20,7 @@ Interfaces for the IMAP module. from zope.interface import Interface, Attribute +# TODO remove ---------------- class IMessageContainer(Interface): """ I am a container around the different documents that a message @@ -38,6 +39,7 @@ class IMessageContainer(Interface): """ +# TODO remove -------------------- class IMessageStore(Interface): """ I represent a generic storage for LEAP Messages. diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 3c1769a..ea54d33 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -1,6 +1,6 @@ # *- coding: utf-8 -*- # mailbox.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013, 2014 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,6 +18,7 @@ Soledad Mailbox. """ import copy +import re import threading import logging import StringIO @@ -27,6 +28,7 @@ import os from collections import defaultdict from twisted.internet import defer +from twisted.internet import reactor from twisted.internet.task import deferLater from twisted.python import log @@ -36,15 +38,18 @@ from zope.interface import implements from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type +from leap.mail.constants import INBOX_NAME from leap.mail.decorators import deferred_to_thread from leap.mail.utils import empty from leap.mail.imap.fields import WithMsgFields, fields from leap.mail.imap.messages import MessageCollection from leap.mail.imap.messageparts import MessageWrapper -from leap.mail.imap.parser import MBoxParser logger = logging.getLogger(__name__) +# TODO +# [ ] Restore profile_cmd instrumentation + """ If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid notifying clients of new messages. Use during stress tests. @@ -71,7 +76,9 @@ if PROFILE_CMD: d.addErrback(lambda f: log.msg(f.getTraceback())) -class SoledadMailbox(WithMsgFields, MBoxParser): +# TODO Rename to Mailbox +# TODO Remove WithMsgFields +class SoledadMailbox(WithMsgFields): """ A Soledad-backed IMAP mailbox. @@ -115,7 +122,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): _last_uid_primed = {} _known_uids_primed = {} - def __init__(self, mbox, soledad, memstore, rw=1): + # TODO pass the collection to the constructor + # TODO pass the mbox_doc too + def __init__(self, mbox, store, rw=1): """ SoledadMailbox constructor. Needs to get passed a name, plus a Soledad instance. @@ -123,30 +132,21 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :param mbox: the mailbox name :type mbox: str - :param soledad: a Soledad instance. - :type soledad: Soledad - - :param memstore: a MemoryStore instance - :type memstore: MemoryStore + :param store: + :type store: Soledad :param rw: read-and-write flag for this mailbox :type rw: int """ leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(soledad, "Need a soledad instance to initialize") + leap_assert(store, "Need a store instance to initialize") - from twisted.internet import reactor - self.reactor = reactor - - self.mbox = self._parse_mailbox_name(mbox) + self.mbox = normalize_mailbox(mbox) self.rw = rw - self._soledad = soledad - self._memstore = memstore - - self.messages = MessageCollection( - mbox=mbox, soledad=self._soledad, memstore=self._memstore) + self.store = store + self.messages = MessageCollection(mbox=mbox, soledad=store) self._uidvalidity = None # XXX careful with this get/set (it would be @@ -214,7 +214,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ return self._memstore.get_mbox_doc(self.mbox) - # XXX the memstore->soledadstore method in memstore is not complete def getFlags(self): """ Returns the flags defined for this mailbox. @@ -227,7 +226,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): flags = self.INIT_FLAGS return map(str, flags) - # XXX the memstore->soledadstore method in memstore is not complete def setFlags(self, flags): """ Sets flags for this mailbox. @@ -468,8 +466,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d = self._do_add_message(message, flags=flags, date=date, notify_on_disk=notify_on_disk) - if PROFILE_CMD: - do_profile_cmd(d, "APPEND") + #if PROFILE_CMD: + #do_profile_cmd(d, "APPEND") # XXX should review now that we're not using qtreactor. # A better place for this would be the COPY/APPEND dispatcher @@ -477,7 +475,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # to work fine for now. def notifyCallback(x): - self.reactor.callLater(0, self.notify_new) + reactor.callLater(0, self.notify_new) return x d.addCallback(notifyCallback) @@ -630,9 +628,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: deferred """ d = defer.Deferred() - self.reactor.callInThread(self._do_fetch, messages_asked, uid, d) - if PROFILE_CMD: - do_profile_cmd(d, "FETCH") + + # XXX do not need no thread... + reactor.callInThread(self._do_fetch, messages_asked, uid, d) d.addCallback(self.cb_signal_unread_to_ui) return d @@ -800,7 +798,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d.addCallback(self.__cb_signal_unread_to_ui) return result - @deferred_to_thread def _get_unseen_deferred(self): return self.getUnseenCount() @@ -897,7 +894,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: C{list} or C{Deferred} """ # TODO see if we can raise w/o interrupting flow - #:raise IllegalQueryError: Raised when query is not valid. + # :raise IllegalQueryError: Raised when query is not valid. # example query: # ['UNDELETED', 'HEADER', 'Message-ID', # '52D44F11.9060107@dev.bitmask.net'] @@ -991,7 +988,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d.addCallback(createCopy) d.addErrback(lambda f: log.msg(f.getTraceback())) - @deferred_to_thread + #@deferred_to_thread def _get_msg_copy(self, message): """ Get a copy of the fdoc for this message, and check whether @@ -1049,3 +1046,22 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ return u"" % ( self.mbox, self.messages.count()) + + +def normalize_mailbox(name): + """ + Return a normalized representation of the mailbox ``name``. + + This method ensures that an eventual initial 'inbox' part of a + mailbox name is made uppercase. + + :param name: the name of the mailbox + :type name: unicode + + :rtype: unicode + """ + _INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + if _INBOX_RE.match(name): + # ensure inital INBOX is uppercase + return INBOX_NAME + name[len(INBOX_NAME):] + return name diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index c761091..d47c8eb 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # messages.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013, 2014 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,30 +19,25 @@ LeapMessage and MessageCollection. """ import copy import logging -import re import threading import StringIO from collections import defaultdict -from email import message_from_string from functools import partial -from pycryptopp.hash import sha256 from twisted.mail import imap4 -from twisted.internet import defer, reactor +from twisted.internet import reactor from zope.interface import implements from zope.proxy import sameProxiedObjects from leap.common.check import leap_assert, leap_assert_type from leap.common.decorators import memoized_method from leap.common.mail import get_email_charset -from leap.mail import walk -from leap.mail.utils import first, find_charset, lowerdict, empty -from leap.mail.utils import stringify_parts_map -from leap.mail.decorators import deferred_to_thread +from leap.mail.adaptors import soledad_indexes as indexes +from leap.mail.constants import INBOX_NAME +from leap.mail.utils import find_charset, empty from leap.mail.imap.index import IndexedDB from leap.mail.imap.fields import fields, WithMsgFields -from leap.mail.imap.memorystore import MessageWrapper from leap.mail.imap.messageparts import MessagePart, MessagePartDoc from leap.mail.imap.parser import MBoxParser @@ -59,9 +54,6 @@ logger = logging.getLogger(__name__) # [ ] Delete incoming mail only after successful write! # [ ] Remove UID from syncable db. Store only those indexes locally. -MSGID_PATTERN = r"""<([\w@.]+)>""" -MSGID_RE = re.compile(MSGID_PATTERN) - def try_unique_query(curried): """ @@ -90,28 +82,18 @@ def try_unique_query(curried): logger.exception("Unhandled error %r" % exc) -""" -A dictionary that keeps one lock per mbox and uid. -""" -# XXX too much overhead? -fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock())) +# FIXME remove-me +#fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock())) -class LeapMessage(fields, MBoxParser): +class IMAPMessage(fields, MBoxParser): """ The main representation of a message. - - It indexes the messages in one mailbox by a combination - of uid+mailbox name. """ - # TODO this has to change. - # Should index primarily by chash, and keep a local-only - # UID table. - implements(imap4.IMessage) - def __init__(self, soledad, uid, mbox, collection=None, container=None): + def __init__(self, soledad, uid, mbox): """ Initializes a LeapMessage. @@ -129,76 +111,73 @@ class LeapMessage(fields, MBoxParser): self._soledad = soledad self._uid = int(uid) if uid is not None else None self._mbox = self._parse_mailbox_name(mbox) - self._collection = collection - self._container = container self.__chash = None self.__bdoc = None - # XXX make these properties public - - # XXX FIXME ------ the documents can be - # deferreds too.... niice. - - @property - def fdoc(self): - """ - An accessor to the flags document. - """ - if all(map(bool, (self._uid, self._mbox))): - fdoc = None - if self._container is not None: - fdoc = self._container.fdoc - if not fdoc: - fdoc = self._get_flags_doc() - if fdoc: - fdoc_content = fdoc.content - self.__chash = fdoc_content.get( - fields.CONTENT_HASH_KEY, None) - return fdoc - - @property - def hdoc(self): - """ - An accessor to the headers document. - """ - container = self._container - if container is not None: - hdoc = self._container.hdoc - if hdoc and not empty(hdoc.content): - return hdoc - hdoc = self._get_headers_doc() - - if container and not empty(hdoc.content): + # TODO collection and container are deprecated. + + # TODO move to adaptor + + #@property + #def fdoc(self): + #""" + #An accessor to the flags document. + #""" + #if all(map(bool, (self._uid, self._mbox))): + #fdoc = None + #if self._container is not None: + #fdoc = self._container.fdoc + #if not fdoc: + #fdoc = self._get_flags_doc() + #if fdoc: + #fdoc_content = fdoc.content + #self.__chash = fdoc_content.get( + #fields.CONTENT_HASH_KEY, None) + #return fdoc +# + #@property + #def hdoc(self): + #""" + #An accessor to the headers document. + #""" + #container = self._container + #if container is not None: + #hdoc = self._container.hdoc + #if hdoc and not empty(hdoc.content): + #return hdoc + #hdoc = self._get_headers_doc() +# + #if container and not empty(hdoc.content): # mem-cache it - hdoc_content = hdoc.content - chash = hdoc_content.get(fields.CONTENT_HASH_KEY) - hdocs = {chash: hdoc_content} - container.memstore.load_header_docs(hdocs) - return hdoc - - @property - def chash(self): - """ - An accessor to the content hash for this message. - """ - if not self.fdoc: - return None - if not self.__chash and self.fdoc: - self.__chash = self.fdoc.content.get( - fields.CONTENT_HASH_KEY, None) - return self.__chash - - @property - def bdoc(self): - """ - An accessor to the body document. - """ - if not self.hdoc: - return None - if not self.__bdoc: - self.__bdoc = self._get_body_doc() - return self.__bdoc + #hdoc_content = hdoc.content + #chash = hdoc_content.get(fields.CONTENT_HASH_KEY) + #hdocs = {chash: hdoc_content} + #container.memstore.load_header_docs(hdocs) + #return hdoc +# + #@property + #def chash(self): + #""" + #An accessor to the content hash for this message. + #""" + #if not self.fdoc: + #return None + #if not self.__chash and self.fdoc: + #self.__chash = self.fdoc.content.get( + #fields.CONTENT_HASH_KEY, None) + #return self.__chash + + #@property + #def bdoc(self): + #""" + #An accessor to the body document. + #""" + #if not self.hdoc: + #return None + #if not self.__bdoc: + #self.__bdoc = self._get_body_doc() + #return self.__bdoc # IMessage implementation @@ -209,8 +188,13 @@ class LeapMessage(fields, MBoxParser): :return: uid for this message :rtype: int """ + # TODO ----> return lookup in local sqlcipher table. return self._uid + # -------------------------------------------------------------- + # TODO -- from here on, all the methods should be proxied to the + # instance of leap.mail.mail.Message + def getFlags(self): """ Retrieve the flags associated with this Message. @@ -253,25 +237,24 @@ class LeapMessage(fields, MBoxParser): REMOVE = -1 SET = 0 - with fdoc_locks[mbox][uid]: - doc = self.fdoc - if not doc: - logger.warning( - "Could not find FDOC for %r:%s while setting flags!" % - (mbox, uid)) - return - current = doc.content[self.FLAGS_KEY] - if mode == APPEND: - newflags = tuple(set(tuple(current) + flags)) - elif mode == REMOVE: - newflags = tuple(set(current).difference(set(flags))) - elif mode == SET: - newflags = flags - new_fdoc = { - self.FLAGS_KEY: newflags, - self.SEEN_KEY: self.SEEN_FLAG in newflags, - self.DEL_KEY: self.DELETED_FLAG in newflags} - self._collection.memstore.update_flags(mbox, uid, new_fdoc) + doc = self.fdoc + if not doc: + logger.warning( + "Could not find FDOC for %r:%s while setting flags!" % + (mbox, uid)) + return + current = doc.content[self.FLAGS_KEY] + if mode == APPEND: + newflags = tuple(set(tuple(current) + flags)) + elif mode == REMOVE: + newflags = tuple(set(current).difference(set(flags))) + elif mode == SET: + newflags = flags + new_fdoc = { + self.FLAGS_KEY: newflags, + self.SEEN_KEY: self.SEEN_FLAG in newflags, + self.DEL_KEY: self.DELETED_FLAG in newflags} + self._collection.memstore.update_flags(mbox, uid, new_fdoc) return map(str, newflags) @@ -371,9 +354,9 @@ class LeapMessage(fields, MBoxParser): else: logger.warning("No FLAGS doc for %s:%s" % (self._mbox, self._uid)) - if not size: + #if not size: # XXX fallback, should remove when all migrated. - size = self.getBodyFile().len + #size = self.getBodyFile().len return size def getHeaders(self, negate, *names): @@ -395,6 +378,9 @@ class LeapMessage(fields, MBoxParser): # XXX refactor together with MessagePart method headers = self._get_headers() + + # XXX keep this in the imap imessage implementation, + # because the server impl. expects content-type to be present. if not headers: logger.warning("No headers found") return {str('content-type'): str('')} @@ -614,64 +600,23 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): (the u1db index) for all the headers documents for a given mailbox. We use it to prefetch massively all the headers for a mailbox. This is the second massive query, after fetching all the FLAGS, that - a MUA will do in a case where we do not have local disk cache. + a typical IMAP MUA will do in a case where we do not have local disk cache. """ HDOCS_SET_DOC = "HDOCS_SET" templates = { - # Message Level - - FLAGS_DOC: { - fields.TYPE_KEY: fields.TYPE_FLAGS_VAL, - fields.UID_KEY: 1, # XXX moe to a local table - fields.MBOX_KEY: fields.INBOX_VAL, - fields.CONTENT_HASH_KEY: "", - - fields.SEEN_KEY: False, - fields.DEL_KEY: False, - fields.FLAGS_KEY: [], - fields.MULTIPART_KEY: False, - fields.SIZE_KEY: 0 - }, - - HEADERS_DOC: { - fields.TYPE_KEY: fields.TYPE_HEADERS_VAL, - fields.CONTENT_HASH_KEY: "", - - fields.DATE_KEY: "", - fields.SUBJECT_KEY: "", - - fields.HEADERS_KEY: {}, - fields.PARTS_MAP_KEY: {}, - }, - - CONTENT_DOC: { - fields.TYPE_KEY: fields.TYPE_CONTENT_VAL, - fields.PAYLOAD_HASH_KEY: "", - fields.LINKED_FROM_KEY: [], - fields.CTYPE_KEY: "", # should index by this too - - # should only get inmutable headers parts - # (for indexing) - fields.HEADERS_KEY: {}, - fields.RAW_KEY: "", - fields.PARTS_MAP_KEY: {}, - fields.HEADERS_KEY: {}, - fields.MULTIPART_KEY: False, - }, - # Mailbox Level RECENT_DOC: { - fields.TYPE_KEY: fields.TYPE_RECENT_VAL, - fields.MBOX_KEY: fields.INBOX_VAL, + "type": indexes.RECENT, + "mbox": INBOX_NAME, fields.RECENTFLAGS_KEY: [], }, HDOCS_SET_DOC: { - fields.TYPE_KEY: fields.TYPE_HDOCS_SET_VAL, - fields.MBOX_KEY: fields.INBOX_VAL, + "type": indexes.HDOCS_SET, + "mbox": INBOX_NAME, fields.HDOCS_SET_KEY: [], } @@ -681,8 +626,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): # Different locks for wrapping both the u1db document getting/setting # and the property getting/settting in an atomic operation. - # TODO we would abstract this to a SoledadProperty class - + # TODO --- deprecate ! --- use SoledadDocumentWrapper + locks _rdoc_lock = defaultdict(lambda: threading.Lock()) _rdoc_write_lock = defaultdict(lambda: threading.Lock()) _rdoc_read_lock = defaultdict(lambda: threading.Lock()) @@ -764,81 +708,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): rdoc[fields.MBOX_KEY] = self.mbox self._soledad.create_doc(rdoc) - @deferred_to_thread - def _do_parse(self, raw): - """ - Parse raw message and return it along with - relevant information about its outer level. - - This is done in a separate thread, and the callback is passed - to `_do_add_msg` method. + # -------------------------------------------------------------------- - :param raw: the raw message - :type raw: StringIO or basestring - :return: msg, parts, chash, size, multi - :rtype: tuple - """ - msg = message_from_string(raw) - parts = walk.get_parts(msg) - size = len(raw) - chash = sha256.SHA256(raw).hexdigest() - multi = msg.is_multipart() - return msg, parts, chash, size, multi - - def _populate_flags(self, flags, uid, chash, size, multi): - """ - Return a flags doc. - - XXX Missing DOC ----------- - """ - fd = self._get_empty_doc(self.FLAGS_DOC) - - fd[self.MBOX_KEY] = self.mbox - fd[self.UID_KEY] = uid - fd[self.CONTENT_HASH_KEY] = chash - fd[self.SIZE_KEY] = size - fd[self.MULTIPART_KEY] = multi - if flags: - fd[self.FLAGS_KEY] = flags - fd[self.SEEN_KEY] = self.SEEN_FLAG in flags - fd[self.DEL_KEY] = self.DELETED_FLAG in flags - fd[self.RECENT_KEY] = True # set always by default - return fd - - def _populate_headr(self, msg, chash, subject, date): - """ - Return a headers doc. - - XXX Missing DOC ----------- - """ - headers = defaultdict(list) - for k, v in msg.items(): - headers[k].append(v) - - # "fix" for repeated headers. - for k, v in headers.items(): - newline = "\n%s: " % (k,) - headers[k] = newline.join(v) - - lower_headers = lowerdict(headers) - msgid = first(MSGID_RE.findall( - lower_headers.get('message-id', ''))) - - hd = self._get_empty_doc(self.HEADERS_DOC) - hd[self.CONTENT_HASH_KEY] = chash - hd[self.HEADERS_KEY] = headers - hd[self.MSGID_KEY] = msgid - - if not subject and self.SUBJECT_FIELD in headers: - hd[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] - else: - hd[self.SUBJECT_KEY] = subject - - if not date and self.DATE_FIELD in headers: - hd[self.DATE_KEY] = headers[self.DATE_FIELD] - else: - hd[self.DATE_KEY] = date - return hd + # ----------------------------------------------------------------------- def _fdoc_already_exists(self, chash): """ @@ -885,86 +757,41 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): flags = tuple() leap_assert_type(flags, tuple) - # TODO return soledad deferred instead - observer = defer.Deferred() - d = self._do_parse(raw) - d.addCallback(lambda result: reactor.callInThread( - self._do_add_msg, result, flags, subject, date, - notify_on_disk, observer)) - return observer + # TODO ---- proxy to MessageCollection addMessage + + #observer = defer.Deferred() + #d = self._do_parse(raw) + #d.addCallback(lambda result: reactor.callInThread( + #self._do_add_msg, result, flags, subject, date, + #notify_on_disk, observer)) + #return observer + + # TODO --------------------------------------------------- + # move this to leap.mail.adaptors.soledad - # Called in thread def _do_add_msg(self, parse_result, flags, subject, date, notify_on_disk, observer): """ - Helper that creates a new message document. - Here lives the magic of the leap mail. Well, in soledad, really. - - See `add_msg` docstring for parameter info. - - :param parse_result: a tuple with the results of `self._do_parse` - :type parse_result: tuple - :param observer: a deferred that will be fired with the message - uid when the adding succeed. - :type observer: deferred """ - # TODO signal that we can delete the original message!----- - # when all the processing is done. - - # TODO add the linked-from info ! - # TODO add reference to the original message - msg, parts, chash, size, multi = parse_result + # XXX move to SoledadAdaptor write operation ... ??? # check for uniqueness -------------------------------- # Watch out! We're reserving a UID right after this! existing_uid = self._fdoc_already_exists(chash) if existing_uid: msg = self.get_msg_by_uid(existing_uid) - - # We can say the observer that we're done - # TODO return soledad deferred instead reactor.callFromThread(observer.callback, existing_uid) msg.setFlags((fields.DELETED_FLAG,), -1) return + # TODO move UID autoincrement to MessageCollection.addMessage(mailbox) # TODO S2 -- get FUCKING UID from autoincremental table - uid = self.memstore.increment_last_soledad_uid(self.mbox) - - # We can say the observer that we're done at this point, but - # before that we should make sure it has no serious consequences - # if we're issued, for instance, a fetch command right after... - # reactor.callFromThread(observer.callback, uid) - # if we did the notify, we need to invalidate the deferred - # so not to try to fire it twice. - # observer = None - - fd = self._populate_flags(flags, uid, chash, size, multi) - hd = self._populate_headr(msg, chash, subject, date) - - body_phash_fun = [walk.get_body_phash_simple, - walk.get_body_phash_multi][int(multi)] - body_phash = body_phash_fun(walk.get_payloads(msg)) - parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) - - # add parts map to header doc - # (body, multi, part_map) - for key in parts_map: - hd[key] = parts_map[key] - del parts_map + #uid = self.memstore.increment_last_soledad_uid(self.mbox) + #self.set_recent_flag(uid) - hd = stringify_parts_map(hd) - # The MessageContainer expects a dict, one-indexed - cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1)) - - self.set_recent_flag(uid) - msg_container = MessageWrapper(fd, hd, cdocs) - - # TODO S1 -- just pass this to memstore and return that deferred. - self.memstore.create_message( - self.mbox, uid, msg_container, - observer=observer, notify_on_disk=notify_on_disk) + # ------------------------------------------------------------ # # getters: specific queries @@ -1073,6 +900,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): the query failed. :rtype: SoledadDocument or None. """ + # USED from: + # [ ] duplicated fdoc detection + # [ ] _get_uid_from_msgidCb + # FIXME ----- use deferreds. curried = partial( self._soledad.get_from_index, @@ -1205,51 +1036,52 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): if msg_container is not None: if mem_only: - msg = LeapMessage(None, uid, self.mbox, collection=self, + msg = IMAPMessage(None, uid, self.mbox, collection=self, container=msg_container) else: # We pass a reference to soledad just to be able to retrieve # missing parts that cannot be found in the container, like # the content docs after a copy. - msg = LeapMessage(self._soledad, uid, self.mbox, + msg = IMAPMessage(self._soledad, uid, self.mbox, collection=self, container=msg_container) else: - msg = LeapMessage(self._soledad, uid, self.mbox, collection=self) + msg = IMAPMessage(self._soledad, uid, self.mbox, collection=self) if not msg.does_exist(): return None return msg - def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL): - """ - Get all documents for the selected mailbox of the - passed type. By default, it returns the flag docs. - - If you want acess to the content, use __iter__ instead - - :return: a Deferred, that will fire with a list of u1db documents - :rtype: Deferred (promise of list of SoledadDocument) - """ - if _type not in fields.__dict__.values(): - raise TypeError("Wrong type passed to get_all_docs") - + # FIXME --- used where ? --------------------------------------------- + #def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL): + #""" + #Get all documents for the selected mailbox of the + #passed type. By default, it returns the flag docs. +# + #If you want acess to the content, use __iter__ instead +# + #:return: a Deferred, that will fire with a list of u1db documents + #:rtype: Deferred (promise of list of SoledadDocument) + #""" + #if _type not in fields.__dict__.values(): + #raise TypeError("Wrong type passed to get_all_docs") +# # FIXME ----- either raise or return a deferred wrapper. - if sameProxiedObjects(self._soledad, None): - logger.warning('Tried to get messages but soledad is None!') - return [] - - def get_sorted_docs(docs): - all_docs = [doc for doc in docs] + #if sameProxiedObjects(self._soledad, None): + #logger.warning('Tried to get messages but soledad is None!') + #return [] +# + #def get_sorted_docs(docs): + #all_docs = [doc for doc in docs] # inneficient, but first let's grok it and then # let's worry about efficiency. # XXX FIXINDEX -- should implement order by in soledad # FIXME ---------------------------------------------- - return sorted(all_docs, key=lambda item: item.content['uid']) - - d = self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, _type, self.mbox) - d.addCallback(get_sorted_docs) - return d + #return sorted(all_docs, key=lambda item: item.content['uid']) +# + #d = self._soledad.get_from_index( + #fields.TYPE_MBOX_IDX, _type, self.mbox) + #d.addCallback(get_sorted_docs) + #return d def all_soledad_uid_iter(self): """ @@ -1350,7 +1182,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :returns: a list of LeapMessages :rtype: list """ - return [LeapMessage(self._soledad, docid, self.mbox, collection=self) + return [IMAPMessage(self._soledad, docid, self.mbox, collection=self) for docid in self.unseen_iter()] # recent messages @@ -1384,7 +1216,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :returns: iterator of dicts with content for all messages. :rtype: iterable """ - return (LeapMessage(self._soledad, docuid, self.mbox, collection=self) + return (IMAPMessage(self._soledad, docuid, self.mbox, collection=self) for docuid in self.all_uid_iter()) def __repr__(self): diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py deleted file mode 100644 index 4a801b0..0000000 --- a/src/leap/mail/imap/parser.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- -# parser.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Mail parser mixin. -""" -import re - - -class MBoxParser(object): - """ - Utility function to parse mailbox names. - """ - INBOX_NAME = "INBOX" - INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) - - def _parse_mailbox_name(self, name): - """ - Return a normalized representation of the mailbox C{name}. - - This method ensures that an eventual initial 'inbox' part of a - mailbox name is made uppercase. - - :param name: the name of the mailbox - :type name: unicode - - :rtype: unicode - """ - if self.INBOX_RE.match(name): - # ensure inital INBOX is uppercase - return self.INBOX_NAME + name[len(self.INBOX_NAME):] - return name diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index dd4294c..5af499f 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -94,6 +94,8 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): MessageCollection interface in this particular TestCase """ super(MessageCollectionTestCase, self).setUp() + + # TODO deprecate memstore memstore = MemoryStore() self.messages = MessageCollection("testmbox%s" % (self.count,), self._soledad, memstore=memstore) diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index 9a3868c..920eeb0 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -51,6 +51,7 @@ class SimpleClient(imap4.IMAP4Client): self.transport.loseConnection() +# XXX move to common helper def initialize_soledad(email, gnupg_home, tempdir): """ Initializes soledad by hand @@ -110,9 +111,7 @@ class IMAP4HelperMixin(BaseLeapTest): """ Setup method for each test. - Initializes and run a LEAP IMAP4 Server, - but passing the same Soledad instance (it's costly to initialize), - so we have to be sure to restore state across tests. + Initializes and run a LEAP IMAP4 Server. """ self.old_path = os.environ['PATH'] self.old_home = os.environ['HOME'] @@ -172,19 +171,17 @@ class IMAP4HelperMixin(BaseLeapTest): def tearDown(self): """ tearDown method called after each test. - - Deletes all documents in the Index, and deletes - instances of server and client. """ try: self._soledad.close() + except Exception: + print "ERROR WHILE CLOSING SOLEDAD" + finally: os.environ["PATH"] = self.old_path os.environ["HOME"] = self.old_home # safety check assert 'leap_tests-' in self.tempdir shutil.rmtree(self.tempdir) - except Exception: - print "ERROR WHILE CLOSING SOLEDAD" def populateMessages(self): """ @@ -223,5 +220,3 @@ class IMAP4HelperMixin(BaseLeapTest): def loopback(self): return loopback.loopbackAsync(self.server, self.client) - - diff --git a/src/leap/mail/interfaces.py b/src/leap/mail/interfaces.py new file mode 100644 index 0000000..5838ce9 --- /dev/null +++ b/src/leap/mail/interfaces.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# interfaces.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Interfaces for the leap.mail module. +""" +from zope.interface import Interface, Attribute + + +class IMessageWrapper(Interface): + """ + I know how to access the different parts into which a given message is + splitted into. + """ + + fdoc = Attribute('A dictionaly-like containing the flags document ' + '(mutable)') + hdoc = Attribute('A dictionary-like containing the headers docuemnt ' + '(immutable)') + cdocs = Attribute('A dictionary with the content-docs, one-indexed') + + +class IMailAdaptor(Interface): + """ + I know how to store the standard representation for messages and mailboxes, + and how to update the relevant mutable parts when needed. + """ + + def initialize_store(self, store): + """ + Performs whatever initialization is needed before the store can be + used (creating indexes, sanity checks, etc). + + :param store: store + :returns: a Deferred that will fire when the store is correctly + initialized. + :rtype: deferred + """ + + # TODO is staticmethod valid with an interface? + # @staticmethod + def get_msg_from_string(self, MessageClass, raw_msg): + """ + Return IMessageWrapper implementor from a raw mail string + + :param MessageClass: an implementor of IMessage + :type raw_msg: str + :rtype: implementor of leap.mail.IMessage + """ + + # TODO is staticmethod valid with an interface? + # @staticmethod + def get_msg_from_docs(self, MessageClass, msg_wrapper): + """ + Return an IMessage implementor from its parts. + + :param MessageClass: an implementor of IMessage + :param msg_wrapper: an implementor of IMessageWrapper + :rtype: implementor of leap.mail.IMessage + """ + + # ------------------------------------------------------------------- + # XXX unsure about the following part yet ........................... + + # the idea behind these three methods is that the adaptor also offers a + # fixed interface to create the documents the first time (using + # soledad.create_docs or whatever method maps to it in a similar store, and + # also allows to update flags and tags, hiding the actual implementation of + # where the flags/tags live in behind the concrete MailWrapper in use + # by this particular adaptor. In our impl it will be put_doc(fdoc) after + # locking the getting + updating of that fdoc for atomicity. + + # 'store' must be an instance of something that offers a minimal subset of + # the document API that Soledad currently implements (create_doc, put_doc) + # I *think* store should belong to Account/Collection and be passed as + # param here instead of relying on it being an attribute of the instance. + + def create_msg_docs(self, store, msg_wrapper): + """ + :param store: The documents store + :type store: + :param msg_wrapper: + :type msg_wrapper: IMessageWrapper implementor + """ + + def update_msg_flags(self, store, msg_wrapper): + """ + :param store: The documents store + :type store: + :param msg_wrapper: + :type msg_wrapper: IMessageWrapper implementor + """ + + def update_msg_tags(self, store, msg_wrapper): + """ + :param store: The documents store + :type store: + :param msg_wrapper: + :type msg_wrapper: IMessageWrapper implementor + """ diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py new file mode 100644 index 0000000..ea9c95e --- /dev/null +++ b/src/leap/mail/mail.py @@ -0,0 +1,248 @@ +# -*- coding: utf-8 -*- +# mail.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Generic Access to Mail objects: Public LEAP Mail API. +""" +from twisted.internet import defer + +from leap.mail.constants import INBOX_NAME +from leap.mail.adaptors.soledad import SoledadMailAdaptor + + +# TODO +# [ ] Probably change the name of this module to "api" or "account", mail is +# too generic (there's also IncomingMail, and OutgoingMail + + +class Message(object): + + def __init__(self, wrapper): + """ + :param wrapper: an instance of an implementor of IMessageWrapper + """ + self._wrapper = wrapper + + def get_wrapper(self): + return self._wrapper + + # imap.IMessage methods + + def get_flags(): + """ + """ + + def get_internal_date(): + """ + """ + + # imap.IMessageParts + + def get_headers(): + """ + """ + + def get_body_file(): + """ + """ + + def get_size(): + """ + """ + + def is_multipart(): + """ + """ + + def get_subpart(part): + """ + """ + + # Custom methods. + + def get_tags(): + """ + """ + + +class MessageCollection(object): + """ + A generic collection of messages. It can be messages sharing the same + mailbox, tag, the result of a given query, or just a bunch of ids for + master documents. + + Since LEAP Mail is primarily oriented to store mail in Soledad, the default + (and, so far, only) implementation of the store is contained in this + Soledad Mail Adaptor. If you need to use a different adaptor, change the + adaptor class attribute in your Account object. + + Store is a reference to a particular instance of the message store (soledad + instance or proxy, for instance). + """ + + # TODO look at IMessageSet methods + + # Account should provide an adaptor instance when creating this collection. + adaptor = None + store = None + + def get_message_by_doc_id(self, doc_id): + # ... get from soledad etc + # ... but that should be part of adaptor/store too... :/ + fdoc, hdoc = None + return self.adaptor.from_docs(Message, fdoc=fdoc, hdoc=hdoc) + + # TODO review if this is the best place for: + + def create_docs(): + pass + + def udpate_flags(): + # 1. update the flags in the message wrapper --- stored where??? + # 2. call adaptor.update_msg(store) + pass + + def update_tags(): + # 1. update the tags in the message wrapper --- stored where??? + # 2. call adaptor.update_msg(store) + pass + + # TODO add delete methods here? + + +class Account(object): + """ + Account is the top level abstraction to access collections of messages + associated with a LEAP Mail Account. + + It primarily handles creation and access of Mailboxes, which will be the + basic collection handled by traditional MUAs, but it can also handle other + types of Collections (tag based, for instance). + + leap.mail.imap.SoledadBackedAccount partially proxies methods in this + class. + """ + + # Adaptor is passed to the returned MessageCollections, so if you want to + # use a different adaptor this is the place to change it, by subclassing + # the Account class. + + adaptor_class = SoledadMailAdaptor + store = None + mailboxes = None + + def __init__(self, store): + self.store = store + self.adaptor = self.adaptor_class() + + self.__mailboxes = set([]) + self._initialized = False + self._deferred_initialization = defer.Deferred() + + self._initialize_storage() + + def _initialize_storage(self): + + def add_mailbox_if_none(result): + # every user should have the right to an inbox folder + # at least, so let's make one! + if not self.mailboxes: + self.add_mailbox(INBOX_NAME) + + def finish_initialization(result): + self._initialized = True + self._deferred_initialization.callback(None) + + def load_mbox_cache(result): + d = self._load_mailboxes() + d.addCallback(lambda _: result) + return d + + d = self.adaptor.initialize_store(self.store) + d.addCallback(load_mbox_cache) + d.addCallback(add_mailbox_if_none) + d.addCallback(finish_initialization) + + def callWhenReady(self, cb): + # XXX this could use adaptor.store_ready instead...?? + if self._initialized: + cb(self) + return defer.succeed(None) + else: + self._deferred_initialization.addCallback(cb) + return self._deferred_initialization + + @property + def mailboxes(self): + """ + A list of the current mailboxes for this account. + :rtype: set + """ + return sorted(self.__mailboxes) + + def _load_mailboxes(self): + + def update_mailboxes(mbox_names): + self.__mailboxes.update(mbox_names) + + d = self.adaptor.get_all_mboxes(self.store) + d.addCallback(update_mailboxes) + return d + + # + # Public API Starts + # + + # XXX params for IMAP only??? + def list_mailboxes(self, ref, wildcard): + self.adaptor.get_all_mboxes(self.store) + + def add_mailbox(self, name, mbox=None): + pass + + def create_mailbox(self, pathspec): + pass + + def delete_mailbox(self, name): + pass + + def rename_mailbox(self, oldname, newname): + pass + + # FIXME yet to be decided if it belongs here... + + def get_collection_by_mailbox(self, name): + """ + :rtype: MessageCollection + """ + # imap select will use this, passing the collection to SoledadMailbox + # XXX pass adaptor to MessageCollection + pass + + def get_collection_by_docs(self, docs): + """ + :rtype: MessageCollection + """ + # get a collection of docs by a list of doc_id + # XXX pass adaptor to MessageCollection + pass + + def get_collection_by_tag(self, tag): + """ + :rtype: MessageCollection + """ + # is this a good idea? + pass -- cgit v1.2.3