summaryrefslogtreecommitdiff
path: root/src/leap
diff options
context:
space:
mode:
Diffstat (limited to 'src/leap')
-rw-r--r--src/leap/mail/adaptors/__init__.py0
-rw-r--r--src/leap/mail/adaptors/models.py125
-rw-r--r--src/leap/mail/adaptors/soledad.py723
-rw-r--r--src/leap/mail/adaptors/soledad_indexes.py112
-rw-r--r--src/leap/mail/adaptors/tests/__init__.py0
-rw-r--r--src/leap/mail/adaptors/tests/rfc822.message86
-rw-r--r--src/leap/mail/adaptors/tests/test_models.py103
-rw-r--r--src/leap/mail/adaptors/tests/test_soledad_adaptor.py583
-rw-r--r--src/leap/mail/constants.py21
-rw-r--r--src/leap/mail/imap/account.py223
-rw-r--r--src/leap/mail/imap/fields.py132
-rw-r--r--src/leap/mail/imap/index.py90
-rw-r--r--src/leap/mail/imap/interfaces.py2
-rw-r--r--src/leap/mail/imap/mailbox.py76
-rw-r--r--src/leap/mail/imap/messages.py484
-rw-r--r--src/leap/mail/imap/parser.py45
-rw-r--r--src/leap/mail/imap/tests/test_imap.py2
-rw-r--r--src/leap/mail/imap/tests/utils.py15
-rw-r--r--src/leap/mail/interfaces.py113
-rw-r--r--src/leap/mail/mail.py248
20 files changed, 2415 insertions, 768 deletions
diff --git a/src/leap/mail/adaptors/__init__.py b/src/leap/mail/adaptors/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/leap/mail/adaptors/__init__.py
diff --git a/src/leap/mail/adaptors/models.py b/src/leap/mail/adaptors/models.py
new file mode 100644
index 0000000..1648059
--- /dev/null
+++ b/src/leap/mail/adaptors/models.py
@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+# models.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Generic Models to be used by the Document Adaptors.
+"""
+import copy
+
+
+class SerializableModel(object):
+ """
+ A Generic document model, that can be serialized into a dictionary.
+
+ Subclasses of this `SerializableModel` are meant to be added as class
+ attributes of classes inheriting from DocumentWrapper.
+
+ A subclass __meta__ of this SerializableModel might exist, and contain info
+ relative to particularities of this model.
+
+ For instance, the use of `__meta__.index` marks the existence of a primary
+ index in the model, which will be used to do unique queries (in which case
+ all the other indexed fields in the underlying document will be filled with
+ the default info contained in the model definition).
+ """
+
+ @classmethod
+ def serialize(klass):
+ """
+ Get a dictionary representation of the public attributes in the model
+ class. To avoid collisions with builtin functions, any occurrence of an
+ attribute ended in '_' (like 'type_') will be normalized by removing
+ the trailing underscore.
+
+ This classmethod is used from within the serialized method of a
+ DocumentWrapper instance: it provides defaults for the
+ empty document.
+ """
+ assert isinstance(klass, type)
+ return _normalize_dict(klass.__dict__)
+
+
+class DocumentWrapper(object):
+ """
+ A Wrapper object that can be manipulated, passed around, and serialized in
+ a format that the store understands.
+ It is related to a SerializableModel, which must be specified as the
+ ``model`` class attribute. The instance of this DocumentWrapper will not
+ allow any other *public* attributes than those defined in the corresponding
+ model.
+ """
+ # TODO we could do some very basic type checking here
+ # TODO set a dirty flag (on __setattr__, whenever the value is != from
+ # before)
+ # TODO we could enforce the existence of a correct "model" attribute
+ # in some other way (other than in the initializer)
+
+ def __init__(self, **kwargs):
+ if not getattr(self, 'model', None):
+ raise RuntimeError(
+ 'DocumentWrapper class needs a model attribute')
+
+ defaults = self.model.serialize()
+
+ if kwargs:
+ values = copy.deepcopy(defaults)
+ values.update(kwargs)
+ else:
+ values = defaults
+
+ for k, v in values.items():
+ k = k.replace('-', '_')
+ setattr(self, k, v)
+
+ def __setattr__(self, attr, value):
+ normalized = _normalize_dict(self.model.__dict__)
+ if not attr.startswith('_') and attr not in normalized:
+ raise RuntimeError(
+ "Cannot set attribute because it's not defined "
+ "in the model: %s" % attr)
+ object.__setattr__(self, attr, value)
+
+ def serialize(self):
+ return _normalize_dict(self.__dict__)
+
+ def create(self):
+ raise NotImplementedError()
+
+ def update(self):
+ raise NotImplementedError()
+
+ def delete(self):
+ raise NotImplementedError()
+
+ @classmethod
+ def get_or_create(self):
+ raise NotImplementedError()
+
+ @classmethod
+ def get_all(self):
+ raise NotImplementedError()
+
+
+def _normalize_dict(_dict):
+ items = _dict.items()
+ not_callable = lambda (k, v): not callable(v)
+ not_private = lambda(k, v): not k.startswith('_')
+ for cond in not_callable, not_private:
+ items = filter(cond, items)
+ items = [(k, v) if not k.endswith('_') else (k[:-1], v)
+ for (k, v) in items]
+ items = [(k.replace('-', '_'), v) for (k, v) in items]
+ return dict(items)
diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py
new file mode 100644
index 0000000..2e25f04
--- /dev/null
+++ b/src/leap/mail/adaptors/soledad.py
@@ -0,0 +1,723 @@
+# -*- coding: utf-8 -*-
+# soledad.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledadad MailAdaptor module.
+"""
+import re
+from collections import defaultdict
+from email import message_from_string
+
+from pycryptopp.hash import sha256
+from twisted.internet import defer
+from zope.interface import implements
+
+from leap.common.check import leap_assert, leap_assert_type
+
+from leap.mail import walk
+from leap.mail.adaptors import soledad_indexes as indexes
+from leap.mail.constants import INBOX_NAME
+from leap.mail.adaptors import models
+from leap.mail.imap.mailbox import normalize_mailbox
+from leap.mail.utils import lowerdict, first
+from leap.mail.utils import stringify_parts_map
+from leap.mail.interfaces import IMailAdaptor, IMessageWrapper
+
+
+# TODO
+# [ ] Convenience function to create mail specifying subject, date, etc?
+
+
+_MSGID_PATTERN = r"""<([\w@.]+)>"""
+_MSGID_RE = re.compile(_MSGID_PATTERN)
+
+
+class DuplicatedDocumentError(Exception):
+ """
+ Raised when a duplicated document is detected.
+ """
+ pass
+
+
+class SoledadDocumentWrapper(models.DocumentWrapper):
+ """
+ A Wrapper object that can be manipulated, passed around, and serialized in
+ a format that the Soledad Store understands.
+
+ It ensures atomicity of the document operations on creation, update and
+ deletion.
+ """
+
+ # TODO we could also use a _dirty flag (in models)
+
+ # We keep a dictionary with DeferredLocks, that will be
+ # unique to every subclass of SoledadDocumentWrapper.
+ _k_locks = defaultdict(defer.DeferredLock)
+
+ @classmethod
+ def _get_klass_lock(cls):
+ """
+ Get a DeferredLock that is unique for this subclass name.
+ Used to lock the access to indexes in the `get_or_create` call
+ for a particular DocumentWrapper.
+ """
+ return cls._k_locks[cls.__name__]
+
+ def __init__(self, **kwargs):
+ doc_id = kwargs.pop('doc_id', None)
+ self._doc_id = doc_id
+ self._lock = defer.DeferredLock()
+ super(SoledadDocumentWrapper, self).__init__(**kwargs)
+
+ @property
+ def doc_id(self):
+ return self._doc_id
+
+ def create(self, store):
+ """
+ Create the documents for this wrapper.
+ Since this method will not check for duplication, the
+ responsibility of avoiding duplicates is left to the caller.
+
+ You might be interested in using `get_or_create` classmethod
+ instead (that's the preferred way of creating documents from
+ the wrapper object).
+
+ :return: a deferred that will fire when the underlying
+ Soledad document has been created.
+ :rtype: Deferred
+ """
+ leap_assert(self._doc_id is None,
+ "This document already has a doc_id!")
+
+ def update_doc_id(doc):
+ self._doc_id = doc.doc_id
+ return doc
+ d = store.create_doc(self.serialize())
+ d.addCallback(update_doc_id)
+ return d
+
+ def update(self, store):
+ """
+ Update the documents for this wrapper.
+
+ :return: a deferred that will fire when the underlying
+ Soledad document has been updated.
+ :rtype: Deferred
+ """
+ # the deferred lock guards against revision conflicts
+ return self._lock.run(self._update, store)
+
+ def _update(self, store):
+ leap_assert(self._doc_id is not None,
+ "Need to create doc before updating")
+
+ def update_and_put_doc(doc):
+ doc.content.update(self.serialize())
+ return store.put_doc(doc)
+
+ d = store.get_doc(self._doc_id)
+ d.addCallback(update_and_put_doc)
+ return d
+
+ def delete(self, store):
+ """
+ Delete the documents for this wrapper.
+
+ :return: a deferred that will fire when the underlying
+ Soledad document has been deleted.
+ :rtype: Deferred
+ """
+ # the deferred lock guards against conflicts while updating
+ return self._lock.run(self._delete, store)
+
+ def _delete(self, store):
+ leap_assert(self._doc_id is not None,
+ "Need to create doc before deleting")
+ # XXX might want to flag this DocumentWrapper to avoid
+ # updating it by mistake. This could go in models.DocumentWrapper
+
+ def delete_doc(doc):
+ return store.delete_doc(doc)
+
+ d = store.get_doc(self._doc_id)
+ d.addCallback(delete_doc)
+ return d
+
+ @classmethod
+ def get_or_create(cls, store, index, value):
+ """
+ Get a unique DocumentWrapper by index, or create a new one if the
+ matching query does not exist.
+
+ :param index: the primary index for the model.
+ :type index: str
+ :param value: the value to query the primary index.
+ :type value: str
+
+ :return: a deferred that will be fired with the SoledadDocumentWrapper
+ matching the index query, either existing or just created.
+ :rtype: Deferred
+ """
+ return cls._get_klass_lock().run(
+ cls._get_or_create, store, index, value)
+
+ @classmethod
+ def _get_or_create(cls, store, index, value):
+ assert store is not None
+ assert index is not None
+ assert value is not None
+
+ def get_main_index():
+ try:
+ return cls.model.__meta__.index
+ except AttributeError:
+ raise RuntimeError("The model is badly defined")
+
+ def try_to_get_doc_from_index(indexes):
+ values = []
+ idx_def = dict(indexes)[index]
+ if len(idx_def) == 1:
+ values = [value]
+ else:
+ main_index = get_main_index()
+ fields = cls.model.serialize()
+ for field in idx_def:
+ if field == main_index:
+ values.append(value)
+ else:
+ values.append(fields[field])
+ d = store.get_from_index(index, *values)
+ return d
+
+ def get_first_doc_if_any(docs):
+ if not docs:
+ return None
+ if len(docs) > 1:
+ raise DuplicatedDocumentError
+ return docs[0]
+
+ def wrap_existing_or_create_new(doc):
+ if doc:
+ return cls(doc_id=doc.doc_id, **doc.content)
+ else:
+ return create_and_wrap_new_doc()
+
+ def create_and_wrap_new_doc():
+ # XXX use closure to store indexes instead of
+ # querying for them again.
+ d = store.list_indexes()
+ d.addCallback(get_wrapper_instance_from_index)
+ d.addCallback(return_wrapper_when_created)
+ return d
+
+ def get_wrapper_instance_from_index(indexes):
+ init_values = {}
+ idx_def = dict(indexes)[index]
+ if len(idx_def) == 1:
+ init_value = {idx_def[0]: value}
+ return cls(**init_value)
+ main_index = get_main_index()
+ fields = cls.model.serialize()
+ for field in idx_def:
+ if field == main_index:
+ init_values[field] = value
+ else:
+ init_values[field] = fields[field]
+ return cls(**init_values)
+
+ def return_wrapper_when_created(wrapper):
+ d = wrapper.create(store)
+ d.addCallback(lambda doc: wrapper)
+ return d
+
+ d = store.list_indexes()
+ d.addCallback(try_to_get_doc_from_index)
+ d.addCallback(get_first_doc_if_any)
+ d.addCallback(wrap_existing_or_create_new)
+ return d
+
+ @classmethod
+ def get_all(cls, store):
+ """
+ Get a collection of wrappers around all the documents belonging
+ to this kind.
+
+ For this to work, the model.__meta__ needs to include a tuple with
+ the index to be used for listing purposes, and which is the field to be
+ used to query the index.
+
+ Note that this method only supports indexes of a single field at the
+ moment. It also might be too expensive to return all the documents
+ matching the query, so handle with care.
+
+ class __meta__(object):
+ index = "name"
+ list_index = ("by-type", "type_")
+
+ :return: a deferred that will be fired with an iterable containing
+ as many SoledadDocumentWrapper are matching the index defined
+ in the model as the `list_index`.
+ :rtype: Deferred
+ """
+ # TODO
+ # [ ] extend support to indexes with n-ples
+ # [ ] benchmark the cost of querying and returning indexes in a big
+ # database. This might badly need pagination before being put to
+ # serious use.
+ return cls._get_klass_lock().run(cls._get_all, store)
+
+ @classmethod
+ def _get_all(cls, store):
+ try:
+ list_index, list_attr = cls.model.__meta__.list_index
+ except AttributeError:
+ raise RuntimeError("The model is badly defined: no list_index")
+ try:
+ index_value = getattr(cls.model, list_attr)
+ except AttributeError:
+ raise RuntimeError("The model is badly defined: "
+ "no attribute matching list_index")
+
+ def wrap_docs(docs):
+ return (cls(doc_id=doc.doc_id, **doc.content) for doc in docs)
+
+ d = store.get_from_index(list_index, index_value)
+ d.addCallback(wrap_docs)
+ return d
+
+ # TODO
+ # [ ] get_count() ???
+
+ def __repr__(self):
+ try:
+ idx = getattr(self, self.model.__meta__.index)
+ except AttributeError:
+ idx = ""
+ return "<%s: %s (%s)>" % (self.__class__.__name__,
+ idx, self._doc_id)
+
+
+#
+# Message documents
+#
+
+class FlagsDocWrapper(SoledadDocumentWrapper):
+
+ class model(models.SerializableModel):
+ type_ = "flags"
+ chash = ""
+
+ mbox = "inbox"
+ seen = False
+ deleted = False
+ recent = False
+ multi = False
+ flags = []
+ tags = []
+ size = 0
+
+ class __meta__(object):
+ index = "mbox"
+
+
+class HeaderDocWrapper(SoledadDocumentWrapper):
+
+ class model(models.SerializableModel):
+ type_ = "head"
+ chash = ""
+
+ date = ""
+ subject = ""
+ headers = {}
+ part_map = {}
+ body = "" # link to phash of body
+ msgid = ""
+ multi = False
+
+ class __meta__(object):
+ index = "chash"
+
+
+class ContentDocWrapper(SoledadDocumentWrapper):
+
+ class model(models.SerializableModel):
+ type_ = "cnt"
+ phash = ""
+
+ ctype = "" # XXX index by ctype too?
+ lkf = [] # XXX not implemented yet!
+ raw = ""
+
+ content_disposition = ""
+ content_transfer_encoding = ""
+ content_type = ""
+
+ class __meta__(object):
+ index = "phash"
+
+
+class MessageWrapper(object):
+
+ # TODO generalize wrapper composition?
+ # This could benefit of a DeferredLock to create/update all the
+ # documents at the same time maybe, and defend against concurrent updates?
+
+ implements(IMessageWrapper)
+
+ def __init__(self, fdoc, hdoc, cdocs=None):
+ """
+ Need at least a flag-document and a header-document to instantiate a
+ MessageWrapper. Content-documents can be retrieved lazily.
+
+ cdocs, if any, should be a dictionary in which the keys are ascending
+ integers, beginning at one, and the values are dictionaries with the
+ content of the content-docs.
+ """
+ self.fdoc = FlagsDocWrapper(**fdoc)
+ self.hdoc = HeaderDocWrapper(**hdoc)
+ if cdocs is None:
+ cdocs = {}
+ cdocs_keys = cdocs.keys()
+ assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1)
+ self.cdocs = dict([(key, ContentDocWrapper(**doc)) for (key, doc) in
+ cdocs.items()])
+
+ def create(self, store):
+ """
+ Create all the parts for this message in the store.
+ """
+ leap_assert(self.cdocs,
+ "Need non empty cdocs to create the "
+ "MessageWrapper documents")
+ leap_assert(self.fdoc.doc_id is None,
+ "Cannot create: fdoc has a doc_id")
+
+ # TODO I think we need to tolerate the no hdoc.doc_id case, for when we
+ # are doing a copy to another mailbox.
+ leap_assert(self.hdoc.doc_id is None,
+ "Cannot create: hdoc has a doc_id")
+ d = []
+ d.append(self.fdoc.create(store))
+ d.append(self.hdoc.create(store))
+ for cdoc in self.cdocs.values():
+ if cdoc.doc_id is not None:
+ # we could be just linking to an existing
+ # content-doc.
+ continue
+ d.append(cdoc.create(store))
+ return defer.gatherResults(d)
+
+ def update(self, store):
+ """
+ Update the only mutable parts, which are within the flags document.
+ """
+ return self.fdoc.update(store)
+
+ def delete(self, store):
+ # Eventually this would have to do the duplicate search or send for the
+ # garbage collector. At least the fdoc can be unlinked.
+ raise NotImplementedError()
+
+#
+# Mailboxes
+#
+
+
+class MailboxWrapper(SoledadDocumentWrapper):
+
+ class model(models.SerializableModel):
+ type_ = "mbox"
+ mbox = INBOX_NAME
+ flags = []
+ closed = False
+ subscribed = False
+ rw = True
+
+ class __meta__(object):
+ index = "mbox"
+ list_index = (indexes.TYPE_IDX, 'type_')
+
+
+#
+# Soledad Adaptor
+#
+
+# TODO make this an interface?
+class SoledadIndexMixin(object):
+ """
+ this will need a class attribute `indexes`, that is a dictionary containing
+ the index definitions for the underlying u1db store underlying soledad.
+
+ It needs to be in the following format:
+ {'index-name': ['field1', 'field2']}
+ """
+ # TODO could have a wrapper class for indexes, supporting introspection
+ # and __getattr__
+ indexes = {}
+
+ store_ready = False
+ _index_creation_deferreds = []
+
+ # TODO we might want to move this logic to soledad itself
+ # so that each application can pass a set of indexes for their data model.
+ # TODO check also the decorator used in keymanager for waiting for indexes
+ # to be ready.
+
+ def initialize_store(self, store):
+ """
+ Initialize the indexes in the database.
+
+ :param store: store
+ :returns: a Deferred that will fire when the store is correctly
+ initialized.
+ :rtype: deferred
+ """
+ # TODO I think we *should* get another deferredLock in here, but
+ # global to the soledad namespace, to protect from several points
+ # initializing soledad indexes at the same time.
+
+ leap_assert(store, "Need a store")
+ leap_assert_type(self.indexes, dict)
+ self._index_creation_deferreds = []
+
+ def _on_indexes_created(ignored):
+ self.store_ready = True
+
+ def _create_index(name, expression):
+ d = store.create_index(name, *expression)
+ self._index_creation_deferreds.append(d)
+
+ def _create_indexes(db_indexes):
+ db_indexes = dict(db_indexes)
+
+ for name, expression in self.indexes.items():
+ if name not in db_indexes:
+ # The index does not yet exist.
+ _create_index(name, expression)
+ continue
+
+ if expression == db_indexes[name]:
+ # The index exists and is up to date.
+ continue
+ # The index exists but the definition is not what expected, so
+ # we delete it and add the proper index expression.
+ d1 = store.delete_index(name)
+ d1.addCallback(lambda _: _create_index(name, expression))
+
+ all_created = defer.gatherResults(self._index_creation_deferreds)
+ all_created.addCallback(_on_indexes_created)
+ return all_created
+
+ # Ask the database for currently existing indexes, and create them
+ # if not found.
+ d = store.list_indexes()
+ d.addCallback(_create_indexes)
+ return d
+
+
+class SoledadMailAdaptor(SoledadIndexMixin):
+
+ implements(IMailAdaptor)
+ store = None
+
+ indexes = indexes.MAIL_INDEXES
+
+ # Message handling
+
+ def get_msg_from_string(self, MessageClass, raw_msg):
+ """
+ Get an instance of a MessageClass initialized with a MessageWrapper
+ that contains all the parts obtained from parsing the raw string for
+ the message.
+
+ :param MessageClass: any Message class that can be initialized passing
+ an instance of an IMessageWrapper implementor.
+ :type MessageClass: type
+ :param raw_msg: a string containing the raw email message.
+ :type raw_msg: str
+ :rtype: MessageClass instance.
+ """
+ assert(MessageClass is not None)
+ fdoc, hdoc, cdocs = _split_into_parts(raw_msg)
+ return self.get_msg_from_docs(
+ MessageClass, fdoc, hdoc, cdocs)
+
+ def get_msg_from_docs(self, MessageClass, fdoc, hdoc, cdocs=None):
+ """
+ Get an instance of a MessageClass initialized with a MessageWrapper
+ that contains the passed part documents.
+
+ This is not the recommended way of obtaining a message, unless you know
+ how to take care of ensuring the internal consistency between the part
+ documents, or unless you are glueing together the part documents that
+ have been previously generated by `get_msg_from_string`.
+
+ :param MessageClass: any Message class that can be initialized passing
+ an instance of an IMessageWrapper implementor.
+ :type MessageClass: type
+ :param fdoc: a dictionary containing values from which a
+ FlagsDocWrapper can be initialized
+ :type fdoc: dict
+ :param hdoc: a dictionary containing values from which a
+ HeaderDocWrapper can be initialized
+ :type hdoc: dict
+ :param cdocs: None, or a dictionary mapping integers (1-indexed) to
+ dicts from where a ContentDocWrapper can be initialized.
+ :type cdocs: dict, or None
+
+ :rtype: MessageClass instance.
+ """
+ assert(MessageClass is not None)
+ return MessageClass(MessageWrapper(fdoc, hdoc, cdocs))
+
+ def create_msg(self, store, msg):
+ """
+ :param store: an instance of soledad, or anything that behaves alike
+ :type store:
+ :param msg: a Message object.
+
+ :return: a Deferred that is fired when all the underlying documents
+ have been created.
+ :rtype: defer.Deferred
+ """
+ wrapper = msg.get_wrapper()
+ return wrapper.create(store)
+
+ def update_msg(self, store, msg):
+ """
+ :param msg: a Message object.
+ :param store: an instance of soledad, or anything that behaves alike
+ :type store:
+ :param msg: a Message object.
+ :return: a Deferred that is fired when all the underlying documents
+ have been updated (actually, it's only the fdoc that's allowed
+ to update).
+ :rtype: defer.Deferred
+ """
+ wrapper = msg.get_wrapper()
+ return wrapper.update(store)
+
+ # Mailbox handling
+
+ def get_or_create_mbox(self, store, name):
+ """
+ Get the mailbox with the given name, or creatre one if it does not
+ exist.
+
+ :param name: the name of the mailbox
+ :type name: str
+ """
+ index = indexes.TYPE_MBOX_IDX
+ mbox = normalize_mailbox(name)
+ return MailboxWrapper.get_or_create(store, index, mbox)
+
+ def update_mbox(self, store, mbox_wrapper):
+ """
+ Update the documents for a given mailbox.
+ :param mbox_wrapper: MailboxWrapper instance
+ :type mbox_wrapper: MailboxWrapper
+ :return: a Deferred that will be fired when the mailbox documents
+ have been updated.
+ :rtype: defer.Deferred
+ """
+ return mbox_wrapper.update(store)
+
+ def get_all_mboxes(self, store):
+ """
+ Retrieve a list with wrappers for all the mailboxes.
+
+ :return: a deferred that will be fired with a list of all the
+ MailboxWrappers found.
+ :rtype: defer.Deferred
+ """
+ return MailboxWrapper.get_all(store)
+
+
+def _split_into_parts(raw):
+ # TODO signal that we can delete the original message!-----
+ # when all the processing is done.
+ # TODO add the linked-from info !
+ # TODO add reference to the original message?
+ # TODO populate Default FLAGS/TAGS (unseen?)
+ # TODO seed propely the content_docs with defaults??
+
+ msg, parts, chash, size, multi = _parse_msg(raw)
+ body_phash_fun = [walk.get_body_phash_simple,
+ walk.get_body_phash_multi][int(multi)]
+ body_phash = body_phash_fun(walk.get_payloads(msg))
+ parts_map = walk.walk_msg_tree(parts, body_phash=body_phash)
+
+ fdoc = _build_flags_doc(chash, size, multi)
+ hdoc = _build_headers_doc(msg, chash, parts_map)
+
+ # The MessageWrapper expects a dict, one-indexed
+ cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1))
+
+ # XXX convert each to_dicts...
+ return fdoc, hdoc, cdocs
+
+
+def _parse_msg(raw):
+ msg = message_from_string(raw)
+ parts = walk.get_parts(msg)
+ size = len(raw)
+ chash = sha256.SHA256(raw).hexdigest()
+ multi = msg.is_multipart()
+ return msg, parts, chash, size, multi
+
+
+def _build_flags_doc(chash, size, multi):
+ _fdoc = FlagsDocWrapper(chash=chash, size=size, multi=multi)
+ return _fdoc.serialize()
+
+
+def _build_headers_doc(msg, chash, parts_map):
+ """
+ Assemble a headers document from the original parsed message, the
+ content-hash, and the parts map.
+
+ It takes into account possibly repeated headers.
+ """
+ headers = defaultdict(list)
+ for k, v in msg.items():
+ headers[k].append(v)
+
+ # "fix" for repeated headers.
+ for k, v in headers.items():
+ newline = "\n%s: " % (k,)
+ headers[k] = newline.join(v)
+
+ lower_headers = lowerdict(headers)
+ msgid = first(_MSGID_RE.findall(
+ lower_headers.get('message-id', '')))
+
+ _hdoc = HeaderDocWrapper(
+ chash=chash, headers=lower_headers, msgid=msgid)
+
+ def copy_attr(headers, key, doc):
+ if key in headers:
+ setattr(doc, key, headers[key])
+
+ copy_attr(lower_headers, "subject", _hdoc)
+ copy_attr(lower_headers, "date", _hdoc)
+
+ hdoc = _hdoc.serialize()
+ # add parts map to header doc
+ # (body, multi, part_map)
+ for key in parts_map:
+ hdoc[key] = parts_map[key]
+ return stringify_parts_map(hdoc)
diff --git a/src/leap/mail/adaptors/soledad_indexes.py b/src/leap/mail/adaptors/soledad_indexes.py
new file mode 100644
index 0000000..f3e990d
--- /dev/null
+++ b/src/leap/mail/adaptors/soledad_indexes.py
@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+# soledad_indexes.py
+# Copyright (C) 2013, 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledad Indexes for Mail Documents.
+"""
+
+# TODO
+# [ ] hide most of the constants here
+
+# Document Type, for indexing
+
+TYPE = "type"
+MBOX = "mbox"
+FLAGS = "flags"
+HEADERS = "head"
+CONTENT = "cnt"
+RECENT = "rct"
+HDOCS_SET = "hdocset"
+
+INCOMING_KEY = "incoming"
+ERROR_DECRYPTING_KEY = "errdecr"
+
+# indexing keys
+CONTENT_HASH = "chash"
+PAYLOAD_HASH = "phash"
+MSGID = "msgid"
+UID = "uid"
+
+
+# Index types
+# --------------
+
+TYPE_IDX = 'by-type'
+TYPE_MBOX_IDX = 'by-type-and-mbox'
+#TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid'
+TYPE_SUBS_IDX = 'by-type-and-subscribed'
+TYPE_MSGID_IDX = 'by-type-and-message-id'
+TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
+TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
+TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted'
+TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash'
+TYPE_C_HASH_IDX = 'by-type-and-contenthash'
+TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber'
+TYPE_P_HASH_IDX = 'by-type-and-payloadhash'
+
+# Soledad index for incoming mail, without decrypting errors.
+# and the backward-compatible index, will be deprecated at 0.7
+JUST_MAIL_IDX = "just-mail"
+JUST_MAIL_COMPAT_IDX = "just-mail-compat"
+
+# Tomas created the `recent and seen index`, but the semantic is not too
+# correct since the recent flag is volatile --- XXX review and delete.
+#TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen'
+
+# TODO
+# it would be nice to measure the cost of indexing
+# by many fields.
+
+# TODO
+# make the indexes dict more readable!
+
+MAIL_INDEXES = {
+ # generic
+ TYPE_IDX: [TYPE],
+ TYPE_MBOX_IDX: [TYPE, MBOX],
+
+ # XXX deprecate 0.4.0
+ # TYPE_MBOX_UID_IDX: [TYPE, MBOX, UID],
+
+ # mailboxes
+ TYPE_SUBS_IDX: [TYPE, 'bool(subscribed)'],
+
+ # fdocs uniqueness
+ TYPE_MBOX_C_HASH_IDX: [TYPE, MBOX, CONTENT_HASH],
+
+ # headers doc - search by msgid.
+ TYPE_MSGID_IDX: [TYPE, MSGID],
+
+ # content, headers doc
+ TYPE_C_HASH_IDX: [TYPE, CONTENT_HASH],
+
+ # attachment payload dedup
+ TYPE_P_HASH_IDX: [TYPE, PAYLOAD_HASH],
+
+ # messages
+ TYPE_MBOX_SEEN_IDX: [TYPE, MBOX, 'bool(seen)'],
+ TYPE_MBOX_RECT_IDX: [TYPE, MBOX, 'bool(recent)'],
+ TYPE_MBOX_DEL_IDX: [TYPE, MBOX, 'bool(deleted)'],
+ #TYPE_MBOX_RECT_SEEN_IDX: [TYPE, MBOX,
+ #'bool(recent)', 'bool(seen)'],
+
+ # incoming queue
+ JUST_MAIL_IDX: [INCOMING_KEY,
+ "bool(%s)" % (ERROR_DECRYPTING_KEY,)],
+
+ # the backward-compatible index, will be deprecated at 0.7
+ JUST_MAIL_COMPAT_IDX: [INCOMING_KEY],
+}
diff --git a/src/leap/mail/adaptors/tests/__init__.py b/src/leap/mail/adaptors/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/leap/mail/adaptors/tests/__init__.py
diff --git a/src/leap/mail/adaptors/tests/rfc822.message b/src/leap/mail/adaptors/tests/rfc822.message
new file mode 100644
index 0000000..ee97ab9
--- /dev/null
+++ b/src/leap/mail/adaptors/tests/rfc822.message
@@ -0,0 +1,86 @@
+Return-Path: <twisted-commits-admin@twistedmatrix.com>
+Delivered-To: exarkun@meson.dyndns.org
+Received: from localhost [127.0.0.1]
+ by localhost with POP3 (fetchmail-6.2.1)
+ for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST)
+Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105])
+ by intarweb.us (Postfix) with ESMTP id 4A4A513EA4
+ for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST)
+Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com)
+ by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian))
+ id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600
+Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian))
+ id 18w63j-0007VK-00
+ for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600
+To: twisted-commits@twistedmatrix.com
+From: etrepum CVS <etrepum@twistedmatrix.com>
+Reply-To: twisted-python@twistedmatrix.com
+X-Mailer: CVSToys
+Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com>
+Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up.
+Sender: twisted-commits-admin@twistedmatrix.com
+Errors-To: twisted-commits-admin@twistedmatrix.com
+X-BeenThere: twisted-commits@twistedmatrix.com
+X-Mailman-Version: 2.0.11
+Precedence: bulk
+List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help>
+List-Post: <mailto:twisted-commits@twistedmatrix.com>
+List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
+ <mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe>
+List-Id: <twisted-commits.twistedmatrix.com>
+List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
+ <mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe>
+List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/>
+Date: Thu, 20 Mar 2003 13:50:39 -0600
+
+Modified files:
+Twisted/twisted/python/rebuild.py 1.19 1.20
+
+Log message:
+rebuild now works on python versions from 2.2.0 and up.
+
+
+ViewCVS links:
+http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted
+
+Index: Twisted/twisted/python/rebuild.py
+diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20
+--- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003
++++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003
+@@ -206,15 +206,27 @@
+ clazz.__dict__.clear()
+ clazz.__getattr__ = __getattr__
+ clazz.__module__ = module.__name__
++ if newclasses:
++ import gc
++ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2):
++ hasBrokenRebuild = 1
++ gc_objects = gc.get_objects()
++ else:
++ hasBrokenRebuild = 0
+ for nclass in newclasses:
+ ga = getattr(module, nclass.__name__)
+ if ga is nclass:
+ log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass))
+ else:
+- import gc
+- for r in gc.get_referrers(nclass):
+- if isinstance(r, nclass):
++ if hasBrokenRebuild:
++ for r in gc_objects:
++ if not getattr(r, '__class__', None) is nclass:
++ continue
+ r.__class__ = ga
++ else:
++ for r in gc.get_referrers(nclass):
++ if getattr(r, '__class__', None) is nclass:
++ r.__class__ = ga
+ if doLog:
+ log.msg('')
+ log.msg(' (fixing %s): ' % str(module.__name__))
+
+
+_______________________________________________
+Twisted-commits mailing list
+Twisted-commits@twistedmatrix.com
+http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits
diff --git a/src/leap/mail/adaptors/tests/test_models.py b/src/leap/mail/adaptors/tests/test_models.py
new file mode 100644
index 0000000..efe0bf2
--- /dev/null
+++ b/src/leap/mail/adaptors/tests/test_models.py
@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+# test_models.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the leap.mail.adaptors.models module.
+"""
+from twisted.trial import unittest
+
+from leap.mail.adaptors import models
+
+
+class SerializableModelsTestCase(unittest.TestCase):
+
+ def test_good_serialized_model(self):
+
+ class M(models.SerializableModel):
+ foo = 42
+ bar = 33
+ baaz_ = None
+ _nope = 0
+ __nope = 0
+
+ def not_today(self):
+ pass
+
+ class IgnoreMe(object):
+ pass
+
+ killmeplease = lambda x: x
+
+ serialized = M.serialize()
+ expected = {'foo': 42, 'bar': 33, 'baaz': None}
+ self.assertEqual(serialized, expected)
+
+
+class DocumentWrapperTestCase(unittest.TestCase):
+
+ def test_wrapper_defaults(self):
+
+ class Wrapper(models.DocumentWrapper):
+ class model(models.SerializableModel):
+ foo = 42
+ bar = 11
+
+ wrapper = Wrapper()
+ wrapper._ignored = True
+ serialized = wrapper.serialize()
+ expected = {'foo': 42, 'bar': 11}
+ self.assertEqual(serialized, expected)
+
+ def test_initialized_wrapper(self):
+
+ class Wrapper(models.DocumentWrapper):
+ class model(models.SerializableModel):
+ foo = 42
+ bar_ = 11
+
+ wrapper = Wrapper(foo=0, bar=-1)
+ serialized = wrapper.serialize()
+ expected = {'foo': 0, 'bar': -1}
+ self.assertEqual(serialized, expected)
+
+ wrapper.foo = 23
+ serialized = wrapper.serialize()
+ expected = {'foo': 23, 'bar': -1}
+ self.assertEqual(serialized, expected)
+
+ wrapper = Wrapper(foo=0)
+ serialized = wrapper.serialize()
+ expected = {'foo': 0, 'bar': 11}
+ self.assertEqual(serialized, expected)
+
+ def test_invalid_initialized_wrapper(self):
+
+ class Wrapper(models.DocumentWrapper):
+ class model(models.SerializableModel):
+ foo = 42
+ getwrapper = lambda: Wrapper(bar=1)
+ self.assertRaises(RuntimeError, getwrapper)
+
+ def test_no_model_wrapper(self):
+
+ class Wrapper(models.DocumentWrapper):
+ pass
+
+ def getwrapper():
+ w = Wrapper()
+ w.foo = None
+
+ self.assertRaises(RuntimeError, getwrapper)
diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py
new file mode 100644
index 0000000..657a602
--- /dev/null
+++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py
@@ -0,0 +1,583 @@
+# -*- coding: utf-8 -*-
+# test_soledad_adaptor.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the Soledad Adaptor module - leap.mail.adaptors.soledad
+"""
+import os
+import shutil
+import tempfile
+
+from functools import partial
+
+from twisted.internet import defer
+from twisted.trial import unittest
+
+from leap.common.testing.basetest import BaseLeapTest
+from leap.mail.adaptors import models
+from leap.mail.adaptors.soledad import SoledadDocumentWrapper
+from leap.mail.adaptors.soledad import SoledadIndexMixin
+from leap.mail.adaptors.soledad import SoledadMailAdaptor
+from leap.soledad.client import Soledad
+
+TEST_USER = "testuser@leap.se"
+TEST_PASSWD = "1234"
+
+# DEBUG
+# import logging
+# logging.basicConfig(level=logging.DEBUG)
+
+
+def initialize_soledad(email, gnupg_home, tempdir):
+ """
+ Initializes soledad by hand
+
+ :param email: ID for the user
+ :param gnupg_home: path to home used by gnupg
+ :param tempdir: path to temporal dir
+ :rtype: Soledad instance
+ """
+
+ uuid = "foobar-uuid"
+ passphrase = u"verysecretpassphrase"
+ secret_path = os.path.join(tempdir, "secret.gpg")
+ local_db_path = os.path.join(tempdir, "soledad.u1db")
+ server_url = "https://provider"
+ cert_file = ""
+
+ soledad = Soledad(
+ uuid,
+ passphrase,
+ secret_path,
+ local_db_path,
+ server_url,
+ cert_file,
+ syncable=False)
+
+ return soledad
+
+
+# TODO move to common module
+# XXX remove duplication
+class SoledadTestMixin(BaseLeapTest):
+ """
+ It is **VERY** important that this base is added *AFTER* unittest.TestCase
+ """
+
+ def setUp(self):
+ self.results = []
+
+ self.old_path = os.environ['PATH']
+ self.old_home = os.environ['HOME']
+ self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
+ self.home = self.tempdir
+ bin_tdir = os.path.join(
+ self.tempdir,
+ 'bin')
+ os.environ["PATH"] = bin_tdir
+ os.environ["HOME"] = self.tempdir
+
+ # Soledad: config info
+ self.gnupg_home = "%s/gnupg" % self.tempdir
+ self.email = 'leap@leap.se'
+
+ # initialize soledad by hand so we can control keys
+ self._soledad = initialize_soledad(
+ self.email,
+ self.gnupg_home,
+ self.tempdir)
+
+ def tearDown(self):
+ """
+ tearDown method called after each test.
+ """
+ self.results = []
+ try:
+ self._soledad.close()
+ except Exception as exc:
+ print "ERROR WHILE CLOSING SOLEDAD"
+ # logging.exception(exc)
+ finally:
+ os.environ["PATH"] = self.old_path
+ os.environ["HOME"] = self.old_home
+ # safety check
+ assert 'leap_tests-' in self.tempdir
+ shutil.rmtree(self.tempdir)
+
+
+class CounterWrapper(SoledadDocumentWrapper):
+ class model(models.SerializableModel):
+ counter = 0
+ flag = None
+
+
+class CharacterWrapper(SoledadDocumentWrapper):
+ class model(models.SerializableModel):
+ name = ""
+ age = 20
+
+
+class ActorWrapper(SoledadDocumentWrapper):
+ class model(models.SerializableModel):
+ type_ = "actor"
+ name = None
+
+ class __meta__(object):
+ index = "name"
+ list_index = ("by-type", "type_")
+
+
+class TestAdaptor(SoledadIndexMixin):
+ indexes = {'by-name': ['name'],
+ 'by-type-and-name': ['type', 'name'],
+ 'by-type': ['type']}
+
+
+class SoledadDocWrapperTestCase(unittest.TestCase, SoledadTestMixin):
+ """
+ Tests for the SoledadDocumentWrapper.
+ """
+ def assert_num_docs(self, num, docs):
+ self.assertEqual(len(docs[1]), num)
+
+ def test_create_single(self):
+
+ store = self._soledad
+ wrapper = CounterWrapper()
+
+ def assert_one_doc(docs):
+ self.assertEqual(docs[0], 1)
+
+ d = wrapper.create(store)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(assert_one_doc)
+ return d
+
+ def test_create_many(self):
+
+ store = self._soledad
+ w1 = CounterWrapper()
+ w2 = CounterWrapper(counter=1)
+ w3 = CounterWrapper(counter=2)
+ w4 = CounterWrapper(counter=3)
+ w5 = CounterWrapper(counter=4)
+
+ d1 = [w1.create(store),
+ w2.create(store),
+ w3.create(store),
+ w4.create(store),
+ w5.create(store)]
+
+ d = defer.gatherResults(d1)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 5))
+ return d
+
+ def test_multiple_updates(self):
+
+ store = self._soledad
+ wrapper = CounterWrapper(counter=1)
+ MAX = 100
+
+ def assert_doc_id(doc):
+ self.assertTrue(wrapper._doc_id is not None)
+ return doc
+
+ def assert_counter_initial_ok(doc):
+ self.assertEqual(wrapper.counter, 1)
+
+ def increment_counter(ignored):
+ d1 = []
+
+ def record_revision(revision):
+ rev = int(revision.split(':')[1])
+ self.results.append(rev)
+
+ for i in list(range(MAX)):
+ wrapper.counter += 1
+ wrapper.flag = i % 2 == 0
+ d = wrapper.update(store)
+ d.addCallback(record_revision)
+ d1.append(d)
+
+ return defer.gatherResults(d1)
+
+ def assert_counter_final_ok(doc):
+ self.assertEqual(doc.content['counter'], MAX + 1)
+ self.assertEqual(doc.content['flag'], False)
+
+ def assert_results_ordered_list(ignored):
+ self.assertEqual(self.results, sorted(range(2, MAX + 2)))
+
+ d = wrapper.create(store)
+ d.addCallback(assert_doc_id)
+ d.addCallback(assert_counter_initial_ok)
+ d.addCallback(increment_counter)
+ d.addCallback(lambda _: store.get_doc(wrapper._doc_id))
+ d.addCallback(assert_counter_final_ok)
+ d.addCallback(assert_results_ordered_list)
+ return d
+
+ def test_delete(self):
+ adaptor = TestAdaptor()
+ store = self._soledad
+
+ wrapper_list = []
+
+ def get_or_create_bob(ignored):
+ def add_to_list(wrapper):
+ wrapper_list.append(wrapper)
+ return wrapper
+ wrapper = CharacterWrapper.get_or_create(
+ store, 'by-name', 'bob')
+ wrapper.addCallback(add_to_list)
+ return wrapper
+
+ def delete_bob(ignored):
+ wrapper = wrapper_list[0]
+ return wrapper.delete(store)
+
+ d = adaptor.initialize_store(store)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 0))
+
+ # this should create bob document
+ d.addCallback(get_or_create_bob)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+
+ d.addCallback(delete_bob)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 0))
+ return d
+
+ def test_get_or_create(self):
+ adaptor = TestAdaptor()
+ store = self._soledad
+
+ def get_or_create_bob(ignored):
+ wrapper = CharacterWrapper.get_or_create(
+ store, 'by-name', 'bob')
+ return wrapper
+
+ d = adaptor.initialize_store(store)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 0))
+
+ # this should create bob document
+ d.addCallback(get_or_create_bob)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+
+ # this should get us bob document
+ d.addCallback(get_or_create_bob)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+ return d
+
+ def test_get_or_create_multi_index(self):
+ adaptor = TestAdaptor()
+ store = self._soledad
+
+ def get_or_create_actor_harry(ignored):
+ wrapper = ActorWrapper.get_or_create(
+ store, 'by-type-and-name', 'harrison')
+ return wrapper
+
+ def create_director_harry(ignored):
+ wrapper = ActorWrapper(name="harrison", type="director")
+ return wrapper.create(store)
+
+ d = adaptor.initialize_store(store)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 0))
+
+ # this should create harrison document
+ d.addCallback(get_or_create_actor_harry)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+
+ # this should get us harrison document
+ d.addCallback(get_or_create_actor_harry)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+
+ # create director harry, should create new doc
+ d.addCallback(create_director_harry)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 2))
+
+ # this should get us harrison document, still 2 docs
+ d.addCallback(get_or_create_actor_harry)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 2))
+ return d
+
+ def test_get_all(self):
+ adaptor = TestAdaptor()
+ store = self._soledad
+ actor_names = ["harry", "carrie", "mark", "david"]
+
+ def create_some_actors(ignored):
+ deferreds = []
+ for name in actor_names:
+ dw = ActorWrapper.get_or_create(
+ store, 'by-type-and-name', name)
+ deferreds.append(dw)
+ return defer.gatherResults(deferreds)
+
+ d = adaptor.initialize_store(store)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 0))
+
+ d.addCallback(create_some_actors)
+
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 4))
+
+ def assert_actor_list_is_expected(res):
+ got = set([actor.name for actor in res])
+ expected = set(actor_names)
+ self.assertEqual(got, expected)
+
+ d.addCallback(lambda _: ActorWrapper.get_all(store))
+ d.addCallback(assert_actor_list_is_expected)
+ return d
+
+here = os.path.split(os.path.abspath(__file__))[0]
+
+
+class TestMessageClass(object):
+ def __init__(self, wrapper):
+ self.wrapper = wrapper
+
+ def get_wrapper(self):
+ return self.wrapper
+
+
+class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin):
+ """
+ Tests for the SoledadMailAdaptor.
+ """
+
+ def get_adaptor(self):
+ adaptor = SoledadMailAdaptor()
+ adaptor.store = self._soledad
+ return adaptor
+
+ def assert_num_docs(self, num, docs):
+ self.assertEqual(len(docs[1]), num)
+
+ def test_mail_adaptor_init(self):
+ adaptor = self.get_adaptor()
+ self.assertTrue(isinstance(adaptor.indexes, dict))
+ self.assertTrue(len(adaptor.indexes) != 0)
+
+ # Messages
+
+ def test_get_msg_from_string(self):
+ adaptor = self.get_adaptor()
+
+ with open(os.path.join(here, "rfc822.message")) as f:
+ raw = f.read()
+
+ msg = adaptor.get_msg_from_string(TestMessageClass, raw)
+
+ chash = ("D27B2771C0DCCDCB468EE65A4540438"
+ "09DBD11588E87E951545BE0CBC321C308")
+ phash = ("64934534C1C80E0D4FA04BE1CCBA104"
+ "F07BCA5F469C86E2C0ABE1D41310B7299")
+ subject = ("[Twisted-commits] rebuild now works on "
+ "python versions from 2.2.0 and up.")
+ self.assertTrue(msg.wrapper.fdoc is not None)
+ self.assertTrue(msg.wrapper.hdoc is not None)
+ self.assertTrue(msg.wrapper.cdocs is not None)
+ self.assertEquals(len(msg.wrapper.cdocs), 1)
+ self.assertEquals(msg.wrapper.fdoc.chash, chash)
+ self.assertEquals(msg.wrapper.fdoc.size, 3834)
+ self.assertEquals(msg.wrapper.hdoc.chash, chash)
+ self.assertEqual(msg.wrapper.hdoc.headers['subject'],
+ subject)
+ self.assertEqual(msg.wrapper.hdoc.subject, subject)
+ self.assertEqual(msg.wrapper.cdocs[1].phash, phash)
+
+ def test_get_msg_from_docs(self):
+ adaptor = self.get_adaptor()
+ fdoc = dict(
+ mbox="Foobox",
+ flags=('\Seen', '\Nice'),
+ tags=('Personal', 'TODO'),
+ seen=False, deleted=False,
+ recent=False, multi=False)
+ hdoc = dict(
+ subject="Test Msg")
+ cdocs = {
+ 1: dict(
+ raw='This is a test message')}
+
+ msg = adaptor.get_msg_from_docs(
+ TestMessageClass, fdoc, hdoc, cdocs=cdocs)
+ self.assertEqual(msg.wrapper.fdoc.flags,
+ ('\Seen', '\Nice'))
+ self.assertEqual(msg.wrapper.fdoc.tags,
+ ('Personal', 'TODO'))
+ self.assertEqual(msg.wrapper.fdoc.mbox, "Foobox")
+ self.assertEqual(msg.wrapper.hdoc.multi, False)
+ self.assertEqual(msg.wrapper.hdoc.subject,
+ "Test Msg")
+ self.assertEqual(msg.wrapper.cdocs[1].raw,
+ "This is a test message")
+
+ def test_create_msg(self):
+ adaptor = self.get_adaptor()
+
+ with open(os.path.join(here, "rfc822.message")) as f:
+ raw = f.read()
+ msg = adaptor.get_msg_from_string(TestMessageClass, raw)
+
+ def check_create_result(created):
+ self.assertEqual(len(created), 3)
+ for doc in created:
+ self.assertTrue(
+ doc.__class__.__name__,
+ "SoledadDocument")
+
+ d = adaptor.create_msg(adaptor.store, msg)
+ d.addCallback(check_create_result)
+ return d
+
+ def test_update_msg(self):
+ adaptor = self.get_adaptor()
+ with open(os.path.join(here, "rfc822.message")) as f:
+ raw = f.read()
+
+ def assert_msg_has_doc_id(ignored, msg):
+ wrapper = msg.get_wrapper()
+ self.assertTrue(wrapper.fdoc.doc_id is not None)
+
+ def assert_msg_has_no_flags(ignored, msg):
+ wrapper = msg.get_wrapper()
+ self.assertEqual(wrapper.fdoc.flags, [])
+
+ def update_msg_flags(ignored, msg):
+ wrapper = msg.get_wrapper()
+ wrapper.fdoc.flags = ["This", "That"]
+ return wrapper.update(adaptor.store)
+
+ def assert_msg_has_flags(ignored, msg):
+ wrapper = msg.get_wrapper()
+ self.assertEqual(wrapper.fdoc.flags, ["This", "That"])
+
+ def get_fdoc_and_check_flags(ignored):
+ def assert_doc_has_flags(doc):
+ self.assertEqual(doc.content['flags'],
+ ['This', 'That'])
+ wrapper = msg.get_wrapper()
+ d = adaptor.store.get_doc(wrapper.fdoc.doc_id)
+ d.addCallback(assert_doc_has_flags)
+ return d
+
+ msg = adaptor.get_msg_from_string(TestMessageClass, raw)
+ d = adaptor.create_msg(adaptor.store, msg)
+ d.addCallback(lambda _: adaptor.store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 3))
+ d.addCallback(assert_msg_has_doc_id, msg)
+ d.addCallback(assert_msg_has_no_flags, msg)
+
+ # update it!
+ d.addCallback(update_msg_flags, msg)
+ d.addCallback(assert_msg_has_flags, msg)
+ d.addCallback(get_fdoc_and_check_flags)
+ return d
+
+ # Mailboxes
+
+ def test_get_or_create_mbox(self):
+ adaptor = self.get_adaptor()
+
+ def get_or_create_mbox(ignored):
+ d = adaptor.get_or_create_mbox(adaptor.store, "Trash")
+ return d
+
+ def assert_good_doc(mbox_wrapper):
+ self.assertTrue(mbox_wrapper.doc_id is not None)
+ self.assertEqual(mbox_wrapper.mbox, "Trash")
+ self.assertEqual(mbox_wrapper.type, "mbox")
+ self.assertEqual(mbox_wrapper.closed, False)
+ self.assertEqual(mbox_wrapper.subscribed, False)
+
+ d = adaptor.initialize_store(adaptor.store)
+ d.addCallback(get_or_create_mbox)
+ d.addCallback(assert_good_doc)
+ d.addCallback(lambda _: adaptor.store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+ return d
+
+ def test_update_mbox(self):
+ adaptor = self.get_adaptor()
+
+ wrapper_ref = []
+
+ def get_or_create_mbox(ignored):
+ d = adaptor.get_or_create_mbox(adaptor.store, "Trash")
+ return d
+
+ def update_wrapper(wrapper, wrapper_ref):
+ wrapper_ref.append(wrapper)
+ wrapper.subscribed = True
+ wrapper.closed = True
+ d = adaptor.update_mbox(adaptor.store, wrapper)
+ return d
+
+ def get_mbox_doc_and_check_flags(res, wrapper_ref):
+ wrapper = wrapper_ref[0]
+
+ def assert_doc_has_flags(doc):
+ self.assertEqual(doc.content['subscribed'], True)
+ self.assertEqual(doc.content['closed'], True)
+ d = adaptor.store.get_doc(wrapper.doc_id)
+ d.addCallback(assert_doc_has_flags)
+ return d
+
+ d = adaptor.initialize_store(adaptor.store)
+ d.addCallback(get_or_create_mbox)
+ d.addCallback(update_wrapper, wrapper_ref)
+ d.addCallback(get_mbox_doc_and_check_flags, wrapper_ref)
+ return d
+
+ def test_get_all_mboxes(self):
+ adaptor = self.get_adaptor()
+ mboxes = ("Sent", "Trash", "Personal", "ListFoo")
+
+ def get_or_create_mboxes(ignored):
+ d = []
+ for mbox in mboxes:
+ d.append(adaptor.get_or_create_mbox(
+ adaptor.store, mbox))
+ return defer.gatherResults(d)
+
+ def get_all_mboxes(ignored):
+ return adaptor.get_all_mboxes(adaptor.store)
+
+ def assert_mboxes_match_expected(wrappers):
+ names = [m.mbox for m in wrappers]
+ self.assertEqual(set(names), set(mboxes))
+
+ d = adaptor.initialize_store(adaptor.store)
+ d.addCallback(get_or_create_mboxes)
+ d.addCallback(get_all_mboxes)
+ d.addCallback(assert_mboxes_match_expected)
+ return d
diff --git a/src/leap/mail/constants.py b/src/leap/mail/constants.py
new file mode 100644
index 0000000..55bf1da
--- /dev/null
+++ b/src/leap/mail/constants.py
@@ -0,0 +1,21 @@
+# *- coding: utf-8 -*-
+# constants.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Constants for leap.mail.
+"""
+
+INBOX_NAME = "INBOX"
diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index fe466cb..7dfbbd1 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -28,10 +28,10 @@ from twisted.python import log
from zope.interface import implements
from leap.common.check import leap_assert, leap_assert_type
-from leap.mail.imap.index import IndexedDB
+
+from leap.mail.mail import Account
from leap.mail.imap.fields import WithMsgFields
-from leap.mail.imap.parser import MBoxParser
-from leap.mail.imap.mailbox import SoledadMailbox
+from leap.mail.imap.mailbox import SoledadMailbox, normalize_mailbox
from leap.soledad.client import Soledad
logger = logging.getLogger(__name__)
@@ -39,7 +39,6 @@ logger = logging.getLogger(__name__)
PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False)
if PROFILE_CMD:
-
def _debugProfiling(result, cmdname, start):
took = (time.time() - start) * 1000
log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec")
@@ -47,96 +46,43 @@ if PROFILE_CMD:
#######################################
-# Soledad Account
+# Soledad IMAP Account
#######################################
+# TODO remove MsgFields too
-# TODO change name to LeapIMAPAccount, since we're using
-# the memstore.
-# IndexedDB should also not be here anymore.
-
-class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
+class IMAPAccount(WithMsgFields):
"""
- An implementation of IAccount and INamespacePresenteer
+ An implementation of an imap4 Account
that is backed by Soledad Encrypted Documents.
"""
implements(imap4.IAccount, imap4.INamespacePresenter)
- _soledad = None
selected = None
closed = False
- _initialized = False
- def __init__(self, account_name, soledad, memstore=None):
+ def __init__(self, user_id, store):
"""
- Creates a SoledadAccountIndex that keeps track of the mailboxes
- and subscriptions handled by this account.
+ Keeps track of the mailboxes and subscriptions handled by this account.
- :param acct_name: The name of the account (user id).
- :type acct_name: str
+ :param account: The name of the account (user id).
+ :type account: str
- :param soledad: a Soledad instance.
- :type soledad: Soledad
- :param memstore: a MemoryStore instance.
- :type memstore: MemoryStore
+ :param store: a Soledad instance.
+ :type store: Soledad
"""
- leap_assert(soledad, "Need a soledad instance to initialize")
- leap_assert_type(soledad, Soledad)
+ # XXX assert a generic store interface instead, so that we
+ # can plug the memory store wrapper seamlessly.
+ leap_assert(store, "Need a store instance to initialize")
+ leap_assert_type(store, Soledad)
# XXX SHOULD assert too that the name matches the user/uuid with which
# soledad has been initialized.
+ self.user_id = user_id
+ self.account = Account(store)
- # XXX ??? why is this parsing mailbox name??? it's account...
- # userid? homogenize.
- self._account_name = self._parse_mailbox_name(account_name)
- self._soledad = soledad
- self._memstore = memstore
-
- self.__mailboxes = set([])
-
- self._deferred_initialization = defer.Deferred()
- self._initialize_storage()
-
- def _initialize_storage(self):
-
- def add_mailbox_if_none(result):
- # every user should have the right to an inbox folder
- # at least, so let's make one!
- if not self.mailboxes:
- self.addMailbox(self.INBOX_NAME)
-
- def finish_initialization(result):
- self._initialized = True
- self._deferred_initialization.callback(None)
-
- def load_mbox_cache(result):
- d = self._load_mailboxes()
- d.addCallback(lambda _: result)
- return d
-
- d = self.initialize_db()
-
- d.addCallback(load_mbox_cache)
- d.addCallback(add_mailbox_if_none)
- d.addCallback(finish_initialization)
-
- def callWhenReady(self, cb):
- if self._initialized:
- cb(self)
- return defer.succeed(None)
- else:
- self._deferred_initialization.addCallback(cb)
- return self._deferred_initialization
-
- def _get_empty_mailbox(self):
- """
- Returns an empty mailbox.
-
- :rtype: dict
- """
- return copy.deepcopy(self.EMPTY_MBOX)
-
+ # XXX should hide this in the adaptor...
def _get_mailbox_by_name(self, name):
"""
Return an mbox document by name.
@@ -146,32 +92,17 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:rtype: SoledadDocument
"""
- # XXX use soledadstore instead ...;
def get_first_if_any(docs):
return docs[0] if docs else None
- d = self._soledad.get_from_index(
+ d = self._store.get_from_index(
self.TYPE_MBOX_IDX, self.MBOX_KEY,
- self._parse_mailbox_name(name))
+ normalize_mailbox(name))
d.addCallback(get_first_if_any)
return d
- @property
- def mailboxes(self):
- """
- A list of the current mailboxes for this account.
- :rtype: set
- """
- return sorted(self.__mailboxes)
-
- def _load_mailboxes(self):
- def update_mailboxes(db_indexes):
- self.__mailboxes.update(
- [doc.content[self.MBOX_KEY] for doc in db_indexes])
- d = self._soledad.get_from_index(self.TYPE_IDX, self.MBOX_KEY)
- d.addCallback(update_mailboxes)
- return d
-
+ # XXX move to Account?
+ # XXX needed?
def getMailbox(self, name):
"""
Return a Mailbox with that name, without selecting it.
@@ -182,18 +113,28 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:returns: a a SoledadMailbox instance
:rtype: SoledadMailbox
"""
- name = self._parse_mailbox_name(name)
+ name = normalize_mailbox(name)
- if name not in self.mailboxes:
+ if name not in self.account.mailboxes:
raise imap4.MailboxException("No such mailbox: %r" % name)
- return SoledadMailbox(name, self._soledad,
- memstore=self._memstore)
+ # XXX Does mailbox really need reference to soledad?
+ return SoledadMailbox(name, self._store)
#
# IAccount
#
+ def _get_empty_mailbox(self):
+ """
+ Returns an empty mailbox.
+
+ :rtype: dict
+ """
+ # XXX move to mailbox module
+ return copy.deepcopy(mailbox.EMPTY_MBOX)
+
+ # TODO use mail.Account.add_mailbox
def addMailbox(self, name, creation_ts=None):
"""
Add a mailbox to the account.
@@ -209,7 +150,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:returns: a Deferred that will contain the document if successful.
:rtype: bool
"""
- name = self._parse_mailbox_name(name)
+ name = normalize_mailbox(name)
leap_assert(name, "Need a mailbox name to create a mailbox")
@@ -232,10 +173,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
d.addCallback(lambda _: result)
return d
- d = self._soledad.create_doc(mbox)
+ d = self._store.create_doc(mbox)
d.addCallback(load_mbox_cache)
return d
+ # TODO use mail.Account.create_mailbox?
+ # Watch out, imap specific exceptions raised here.
def create(self, pathspec):
"""
Create a new mailbox from the given hierarchical name.
@@ -254,9 +197,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:raise MailboxException: Raised if this mailbox cannot be added.
"""
# TODO raise MailboxException
- paths = filter(
- None,
- self._parse_mailbox_name(pathspec).split('/'))
+ paths = filter(None, normalize_mailbox(pathspec).split('/'))
subs = []
sep = '/'
@@ -295,6 +236,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
d1.addCallback(load_mbox_cache)
return d1
+ # TODO use mail.Account.get_collection_by_mailbox
def select(self, name, readwrite=1):
"""
Selects a mailbox.
@@ -307,21 +249,16 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:rtype: SoledadMailbox
"""
- if PROFILE_CMD:
- start = time.time()
-
- name = self._parse_mailbox_name(name)
+ name = normalize_mailbox(name)
if name not in self.mailboxes:
logger.warning("No such mailbox!")
return None
self.selected = name
- sm = SoledadMailbox(
- name, self._soledad, self._memstore, readwrite)
- if PROFILE_CMD:
- _debugProfiling(None, "SELECT", start)
+ sm = SoledadMailbox(name, self._store, readwrite)
return sm
+ # TODO use mail.Account.delete_mailbox
def delete(self, name, force=False):
"""
Deletes a mailbox.
@@ -338,7 +275,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:type force: bool
:rtype: Deferred
"""
- name = self._parse_mailbox_name(name)
+ name = normalize_mailbox(name)
if name not in self.mailboxes:
err = imap4.MailboxException("No such mailbox: %r" % name)
@@ -369,6 +306,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
# ??! -- can this be rite?
# self._index.removeMailbox(name)
+ # TODO use mail.Account.rename_mailbox
def rename(self, oldname, newname):
"""
Renames a mailbox.
@@ -379,8 +317,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:param newname: new name of the mailbox
:type newname: str
"""
- oldname = self._parse_mailbox_name(oldname)
- newname = self._parse_mailbox_name(newname)
+ oldname = normalize_mailbox(oldname)
+ newname = normalize_mailbox(newname)
if oldname not in self.mailboxes:
raise imap4.NoSuchMailbox(repr(oldname))
@@ -431,6 +369,32 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
inferiors.append(infname)
return inferiors
+ # TODO use mail.Account.list_mailboxes
+ def listMailboxes(self, ref, wildcard):
+ """
+ List the mailboxes.
+
+ from rfc 3501:
+ returns a subset of names from the complete set
+ of all names available to the client. Zero or more untagged LIST
+ replies are returned, containing the name attributes, hierarchy
+ delimiter, and name.
+
+ :param ref: reference name
+ :type ref: str
+
+ :param wildcard: mailbox name with possible wildcards
+ :type wildcard: str
+ """
+ # XXX use wildcard in index query
+ ref = self._inferiorNames(normalize_mailbox(ref))
+ wildcard = imap4.wildcardToRegexp(wildcard, '/')
+ return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)]
+
+ #
+ # The rest of the methods are specific for leap.mail.imap.account.Account
+ #
+
# TODO ------------------ can we preserve the attr?
# maybe add to memory store.
def isSubscribed(self, name):
@@ -442,6 +406,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:rtype: Deferred (will fire with bool)
"""
+ # TODO use Flags class
subscribed = self.SUBSCRIBED_KEY
def is_subscribed(mbox):
@@ -465,7 +430,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
def get_docs_content(docs):
return [doc.content[self.MBOX_KEY] for doc in docs]
- d = self._soledad.get_from_index(
+ d = self._store.get_from_index(
self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')
d.addCallback(get_docs_content)
return d
@@ -488,7 +453,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
def update_subscribed_value(mbox):
mbox.content[subscribed] = value
- return self._soledad.put_doc(mbox)
+ return self._store.put_doc(mbox)
# maybe we should store subscriptions in another
# document...
@@ -508,7 +473,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:type name: str
:rtype: Deferred
"""
- name = self._parse_mailbox_name(name)
+ name = normalize_mailbox(name)
def check_and_subscribe(subscriptions):
if name not in subscriptions:
@@ -525,7 +490,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:type name: str
:rtype: Deferred
"""
- name = self._parse_mailbox_name(name)
+ name = normalize_mailbox(name)
def check_and_unsubscribe(subscriptions):
if name not in subscriptions:
@@ -539,28 +504,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
def getSubscriptions(self):
return self._get_subscriptions()
- def listMailboxes(self, ref, wildcard):
- """
- List the mailboxes.
-
- from rfc 3501:
- returns a subset of names from the complete set
- of all names available to the client. Zero or more untagged LIST
- replies are returned, containing the name attributes, hierarchy
- delimiter, and name.
-
- :param ref: reference name
- :type ref: str
-
- :param wildcard: mailbox name with possible wildcards
- :type wildcard: str
- """
- # XXX use wildcard in index query
- ref = self._inferiorNames(
- self._parse_mailbox_name(ref))
- wildcard = imap4.wildcardToRegexp(wildcard, '/')
- return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)]
-
#
# INamespacePresenter
#
@@ -592,4 +535,4 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
"""
Representation string for this object.
"""
- return "<SoledadBackedAccount (%s)>" % self._account_name
+ return "<IMAPAccount (%s)>" % self.user_id
diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py
index 4576939..a751c6d 100644
--- a/src/leap/mail/imap/fields.py
+++ b/src/leap/mail/imap/fields.py
@@ -17,7 +17,9 @@
"""
Fields for Mailbox and Message.
"""
-from leap.mail.imap.parser import MBoxParser
+
+# TODO deprecate !!! (move all to constants maybe?)
+# Flags -> foo
class WithMsgFields(object):
@@ -25,55 +27,12 @@ class WithMsgFields(object):
Container class for class-attributes to be shared by
several message-related classes.
"""
- # indexing
- CONTENT_HASH_KEY = "chash"
- PAYLOAD_HASH_KEY = "phash"
-
- # Internal representation of Message
-
- # flags doc
- UID_KEY = "uid"
- MBOX_KEY = "mbox"
- SEEN_KEY = "seen"
- DEL_KEY = "deleted"
- RECENT_KEY = "recent"
- FLAGS_KEY = "flags"
- MULTIPART_KEY = "multi"
- SIZE_KEY = "size"
-
- # headers
- HEADERS_KEY = "headers"
- DATE_KEY = "date"
- SUBJECT_KEY = "subject"
- PARTS_MAP_KEY = "part_map"
- BODY_KEY = "body" # link to phash of body
- MSGID_KEY = "msgid"
-
- # content
- LINKED_FROM_KEY = "lkf" # XXX not implemented yet!
- RAW_KEY = "raw"
- CTYPE_KEY = "ctype"
-
# Mailbox specific keys
- CLOSED_KEY = "closed"
- CREATED_KEY = "created"
- SUBSCRIBED_KEY = "subscribed"
- RW_KEY = "rw"
- LAST_UID_KEY = "lastuid"
+ CREATED_KEY = "created" # used???
+
RECENTFLAGS_KEY = "rct"
HDOCS_SET_KEY = "hdocset"
- # Document Type, for indexing
- TYPE_KEY = "type"
- TYPE_MBOX_VAL = "mbox"
- TYPE_FLAGS_VAL = "flags"
- TYPE_HEADERS_VAL = "head"
- TYPE_CONTENT_VAL = "cnt"
- TYPE_RECENT_VAL = "rct"
- TYPE_HDOCS_SET_VAL = "hdocset"
-
- INBOX_VAL = "inbox"
-
# Flags in Mailbox and Message
SEEN_FLAG = "\\Seen"
RECENT_FLAG = "\\Recent"
@@ -88,86 +47,5 @@ class WithMsgFields(object):
SUBJECT_FIELD = "Subject"
DATE_FIELD = "Date"
- # Index types
- # --------------
-
- TYPE_IDX = 'by-type'
- TYPE_MBOX_IDX = 'by-type-and-mbox'
- TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid'
- TYPE_SUBS_IDX = 'by-type-and-subscribed'
- TYPE_MSGID_IDX = 'by-type-and-message-id'
- TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
- TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
- TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted'
- TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash'
- TYPE_C_HASH_IDX = 'by-type-and-contenthash'
- TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber'
- TYPE_P_HASH_IDX = 'by-type-and-payloadhash'
-
- # Tomas created the `recent and seen index`, but the semantic is not too
- # correct since the recent flag is volatile.
- TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen'
-
- # Soledad index for incoming mail, without decrypting errors.
- JUST_MAIL_IDX = "just-mail"
- # XXX the backward-compatible index, will be deprecated at 0.7
- JUST_MAIL_COMPAT_IDX = "just-mail-compat"
-
- INCOMING_KEY = "incoming"
- ERROR_DECRYPTING_KEY = "errdecr"
-
- KTYPE = TYPE_KEY
- MBOX_VAL = TYPE_MBOX_VAL
- CHASH_VAL = CONTENT_HASH_KEY
- PHASH_VAL = PAYLOAD_HASH_KEY
-
- INDEXES = {
- # generic
- TYPE_IDX: [KTYPE],
- TYPE_MBOX_IDX: [KTYPE, MBOX_VAL],
- TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, UID_KEY],
-
- # mailboxes
- TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'],
-
- # fdocs uniqueness
- TYPE_MBOX_C_HASH_IDX: [KTYPE, MBOX_VAL, CHASH_VAL],
-
- # headers doc - search by msgid.
- TYPE_MSGID_IDX: [KTYPE, MSGID_KEY],
-
- # content, headers doc
- TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL],
-
- # attachment payload dedup
- TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL],
-
- # messages
- TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'],
- TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'],
- TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'],
- TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL,
- 'bool(recent)', 'bool(seen)'],
-
- # incoming queue
- JUST_MAIL_IDX: [INCOMING_KEY,
- "bool(%s)" % (ERROR_DECRYPTING_KEY,)],
-
- # the backward-compatible index, will be deprecated at 0.7
- JUST_MAIL_COMPAT_IDX: [INCOMING_KEY],
- }
-
- MBOX_KEY = MBOX_VAL
-
- EMPTY_MBOX = {
- TYPE_KEY: MBOX_KEY,
- TYPE_MBOX_VAL: MBoxParser.INBOX_NAME,
- SUBJECT_KEY: "",
- FLAGS_KEY: [],
- CLOSED_KEY: False,
- SUBSCRIBED_KEY: False,
- RW_KEY: 1,
- LAST_UID_KEY: 0
- }
fields = WithMsgFields # alias for convenience
diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py
deleted file mode 100644
index ea35fff..0000000
--- a/src/leap/mail/imap/index.py
+++ /dev/null
@@ -1,90 +0,0 @@
-# -*- coding: utf-8 -*-
-# index.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-Index for SoledadBackedAccount, Mailbox and Messages.
-"""
-import logging
-
-from twisted.internet import defer
-
-from leap.common.check import leap_assert, leap_assert_type
-
-from leap.mail.imap.fields import fields
-
-
-logger = logging.getLogger(__name__)
-
-
-class IndexedDB(object):
- """
- Methods dealing with the index.
-
- This is a MixIn that needs access to the soledad instance,
- and also assumes that a INDEXES attribute is accessible to the instance.
-
- INDEXES must be a dictionary of type:
- {'index-name': ['field1', 'field2']}
- """
- # TODO we might want to move this to soledad itself, check
-
- _index_creation_deferreds = []
- index_ready = False
-
- def initialize_db(self):
- """
- Initialize the database.
- """
- leap_assert(self._soledad,
- "Need a soledad attribute accesible in the instance")
- leap_assert_type(self.INDEXES, dict)
- self._index_creation_deferreds = []
-
- def _on_indexes_created(ignored):
- self.index_ready = True
-
- def _create_index(name, expression):
- d = self._soledad.create_index(name, *expression)
- self._index_creation_deferreds.append(d)
-
- def _create_indexes(db_indexes):
- db_indexes = dict(db_indexes)
- for name, expression in fields.INDEXES.items():
- if name not in db_indexes:
- # The index does not yet exist.
- _create_index(name, expression)
- continue
-
- if expression == db_indexes[name]:
- # The index exists and is up to date.
- continue
- # The index exists but the definition is not what expected, so
- # we delete it and add the proper index expression.
- d1 = self._soledad.delete_index(name)
- d1.addCallback(lambda _: _create_index(name, expression))
-
- all_created = defer.gatherResults(self._index_creation_deferreds)
- all_created.addCallback(_on_indexes_created)
- return all_created
-
- # Ask the database for currently existing indexes.
- if not self._soledad:
- logger.debug("NO SOLEDAD ON IMAP INITIALIZATION")
- return
- if self._soledad is not None:
- d = self._soledad.list_indexes()
- d.addCallback(_create_indexes)
- return d
diff --git a/src/leap/mail/imap/interfaces.py b/src/leap/mail/imap/interfaces.py
index c906278..f8f25fa 100644
--- a/src/leap/mail/imap/interfaces.py
+++ b/src/leap/mail/imap/interfaces.py
@@ -20,6 +20,7 @@ Interfaces for the IMAP module.
from zope.interface import Interface, Attribute
+# TODO remove ----------------
class IMessageContainer(Interface):
"""
I am a container around the different documents that a message
@@ -38,6 +39,7 @@ class IMessageContainer(Interface):
"""
+# TODO remove --------------------
class IMessageStore(Interface):
"""
I represent a generic storage for LEAP Messages.
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 3c1769a..ea54d33 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -1,6 +1,6 @@
# *- coding: utf-8 -*-
# mailbox.py
-# Copyright (C) 2013 LEAP
+# Copyright (C) 2013, 2014 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -18,6 +18,7 @@
Soledad Mailbox.
"""
import copy
+import re
import threading
import logging
import StringIO
@@ -27,6 +28,7 @@ import os
from collections import defaultdict
from twisted.internet import defer
+from twisted.internet import reactor
from twisted.internet.task import deferLater
from twisted.python import log
@@ -36,15 +38,18 @@ from zope.interface import implements
from leap.common import events as leap_events
from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
from leap.common.check import leap_assert, leap_assert_type
+from leap.mail.constants import INBOX_NAME
from leap.mail.decorators import deferred_to_thread
from leap.mail.utils import empty
from leap.mail.imap.fields import WithMsgFields, fields
from leap.mail.imap.messages import MessageCollection
from leap.mail.imap.messageparts import MessageWrapper
-from leap.mail.imap.parser import MBoxParser
logger = logging.getLogger(__name__)
+# TODO
+# [ ] Restore profile_cmd instrumentation
+
"""
If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid
notifying clients of new messages. Use during stress tests.
@@ -71,7 +76,9 @@ if PROFILE_CMD:
d.addErrback(lambda f: log.msg(f.getTraceback()))
-class SoledadMailbox(WithMsgFields, MBoxParser):
+# TODO Rename to Mailbox
+# TODO Remove WithMsgFields
+class SoledadMailbox(WithMsgFields):
"""
A Soledad-backed IMAP mailbox.
@@ -115,7 +122,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
_last_uid_primed = {}
_known_uids_primed = {}
- def __init__(self, mbox, soledad, memstore, rw=1):
+ # TODO pass the collection to the constructor
+ # TODO pass the mbox_doc too
+ def __init__(self, mbox, store, rw=1):
"""
SoledadMailbox constructor. Needs to get passed a name, plus a
Soledad instance.
@@ -123,30 +132,21 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:param mbox: the mailbox name
:type mbox: str
- :param soledad: a Soledad instance.
- :type soledad: Soledad
-
- :param memstore: a MemoryStore instance
- :type memstore: MemoryStore
+ :param store:
+ :type store: Soledad
:param rw: read-and-write flag for this mailbox
:type rw: int
"""
leap_assert(mbox, "Need a mailbox name to initialize")
- leap_assert(soledad, "Need a soledad instance to initialize")
+ leap_assert(store, "Need a store instance to initialize")
- from twisted.internet import reactor
- self.reactor = reactor
-
- self.mbox = self._parse_mailbox_name(mbox)
+ self.mbox = normalize_mailbox(mbox)
self.rw = rw
- self._soledad = soledad
- self._memstore = memstore
-
- self.messages = MessageCollection(
- mbox=mbox, soledad=self._soledad, memstore=self._memstore)
+ self.store = store
+ self.messages = MessageCollection(mbox=mbox, soledad=store)
self._uidvalidity = None
# XXX careful with this get/set (it would be
@@ -214,7 +214,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
"""
return self._memstore.get_mbox_doc(self.mbox)
- # XXX the memstore->soledadstore method in memstore is not complete
def getFlags(self):
"""
Returns the flags defined for this mailbox.
@@ -227,7 +226,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
flags = self.INIT_FLAGS
return map(str, flags)
- # XXX the memstore->soledadstore method in memstore is not complete
def setFlags(self, flags):
"""
Sets flags for this mailbox.
@@ -468,8 +466,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
d = self._do_add_message(message, flags=flags, date=date,
notify_on_disk=notify_on_disk)
- if PROFILE_CMD:
- do_profile_cmd(d, "APPEND")
+ #if PROFILE_CMD:
+ #do_profile_cmd(d, "APPEND")
# XXX should review now that we're not using qtreactor.
# A better place for this would be the COPY/APPEND dispatcher
@@ -477,7 +475,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
# to work fine for now.
def notifyCallback(x):
- self.reactor.callLater(0, self.notify_new)
+ reactor.callLater(0, self.notify_new)
return x
d.addCallback(notifyCallback)
@@ -630,9 +628,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:rtype: deferred
"""
d = defer.Deferred()
- self.reactor.callInThread(self._do_fetch, messages_asked, uid, d)
- if PROFILE_CMD:
- do_profile_cmd(d, "FETCH")
+
+ # XXX do not need no thread...
+ reactor.callInThread(self._do_fetch, messages_asked, uid, d)
d.addCallback(self.cb_signal_unread_to_ui)
return d
@@ -800,7 +798,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
d.addCallback(self.__cb_signal_unread_to_ui)
return result
- @deferred_to_thread
def _get_unseen_deferred(self):
return self.getUnseenCount()
@@ -897,7 +894,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:rtype: C{list} or C{Deferred}
"""
# TODO see if we can raise w/o interrupting flow
- #:raise IllegalQueryError: Raised when query is not valid.
+ # :raise IllegalQueryError: Raised when query is not valid.
# example query:
# ['UNDELETED', 'HEADER', 'Message-ID',
# '52D44F11.9060107@dev.bitmask.net']
@@ -991,7 +988,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
d.addCallback(createCopy)
d.addErrback(lambda f: log.msg(f.getTraceback()))
- @deferred_to_thread
+ #@deferred_to_thread
def _get_msg_copy(self, message):
"""
Get a copy of the fdoc for this message, and check whether
@@ -1049,3 +1046,22 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
"""
return u"<SoledadMailbox: mbox '%s' (%s)>" % (
self.mbox, self.messages.count())
+
+
+def normalize_mailbox(name):
+ """
+ Return a normalized representation of the mailbox ``name``.
+
+ This method ensures that an eventual initial 'inbox' part of a
+ mailbox name is made uppercase.
+
+ :param name: the name of the mailbox
+ :type name: unicode
+
+ :rtype: unicode
+ """
+ _INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
+ if _INBOX_RE.match(name):
+ # ensure inital INBOX is uppercase
+ return INBOX_NAME + name[len(INBOX_NAME):]
+ return name
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index c761091..d47c8eb 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# messages.py
-# Copyright (C) 2013 LEAP
+# Copyright (C) 2013, 2014 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -19,30 +19,25 @@ LeapMessage and MessageCollection.
"""
import copy
import logging
-import re
import threading
import StringIO
from collections import defaultdict
-from email import message_from_string
from functools import partial
-from pycryptopp.hash import sha256
from twisted.mail import imap4
-from twisted.internet import defer, reactor
+from twisted.internet import reactor
from zope.interface import implements
from zope.proxy import sameProxiedObjects
from leap.common.check import leap_assert, leap_assert_type
from leap.common.decorators import memoized_method
from leap.common.mail import get_email_charset
-from leap.mail import walk
-from leap.mail.utils import first, find_charset, lowerdict, empty
-from leap.mail.utils import stringify_parts_map
-from leap.mail.decorators import deferred_to_thread
+from leap.mail.adaptors import soledad_indexes as indexes
+from leap.mail.constants import INBOX_NAME
+from leap.mail.utils import find_charset, empty
from leap.mail.imap.index import IndexedDB
from leap.mail.imap.fields import fields, WithMsgFields
-from leap.mail.imap.memorystore import MessageWrapper
from leap.mail.imap.messageparts import MessagePart, MessagePartDoc
from leap.mail.imap.parser import MBoxParser
@@ -59,9 +54,6 @@ logger = logging.getLogger(__name__)
# [ ] Delete incoming mail only after successful write!
# [ ] Remove UID from syncable db. Store only those indexes locally.
-MSGID_PATTERN = r"""<([\w@.]+)>"""
-MSGID_RE = re.compile(MSGID_PATTERN)
-
def try_unique_query(curried):
"""
@@ -90,28 +82,18 @@ def try_unique_query(curried):
logger.exception("Unhandled error %r" % exc)
-"""
-A dictionary that keeps one lock per mbox and uid.
-"""
-# XXX too much overhead?
-fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock()))
+# FIXME remove-me
+#fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock()))
-class LeapMessage(fields, MBoxParser):
+class IMAPMessage(fields, MBoxParser):
"""
The main representation of a message.
-
- It indexes the messages in one mailbox by a combination
- of uid+mailbox name.
"""
- # TODO this has to change.
- # Should index primarily by chash, and keep a local-only
- # UID table.
-
implements(imap4.IMessage)
- def __init__(self, soledad, uid, mbox, collection=None, container=None):
+ def __init__(self, soledad, uid, mbox):
"""
Initializes a LeapMessage.
@@ -129,76 +111,73 @@ class LeapMessage(fields, MBoxParser):
self._soledad = soledad
self._uid = int(uid) if uid is not None else None
self._mbox = self._parse_mailbox_name(mbox)
- self._collection = collection
- self._container = container
self.__chash = None
self.__bdoc = None
- # XXX make these properties public
-
- # XXX FIXME ------ the documents can be
- # deferreds too.... niice.
-
- @property
- def fdoc(self):
- """
- An accessor to the flags document.
- """
- if all(map(bool, (self._uid, self._mbox))):
- fdoc = None
- if self._container is not None:
- fdoc = self._container.fdoc
- if not fdoc:
- fdoc = self._get_flags_doc()
- if fdoc:
- fdoc_content = fdoc.content
- self.__chash = fdoc_content.get(
- fields.CONTENT_HASH_KEY, None)
- return fdoc
-
- @property
- def hdoc(self):
- """
- An accessor to the headers document.
- """
- container = self._container
- if container is not None:
- hdoc = self._container.hdoc
- if hdoc and not empty(hdoc.content):
- return hdoc
- hdoc = self._get_headers_doc()
-
- if container and not empty(hdoc.content):
+ # TODO collection and container are deprecated.
+
+ # TODO move to adaptor
+
+ #@property
+ #def fdoc(self):
+ #"""
+ #An accessor to the flags document.
+ #"""
+ #if all(map(bool, (self._uid, self._mbox))):
+ #fdoc = None
+ #if self._container is not None:
+ #fdoc = self._container.fdoc
+ #if not fdoc:
+ #fdoc = self._get_flags_doc()
+ #if fdoc:
+ #fdoc_content = fdoc.content
+ #self.__chash = fdoc_content.get(
+ #fields.CONTENT_HASH_KEY, None)
+ #return fdoc
+#
+ #@property
+ #def hdoc(self):
+ #"""
+ #An accessor to the headers document.
+ #"""
+ #container = self._container
+ #if container is not None:
+ #hdoc = self._container.hdoc
+ #if hdoc and not empty(hdoc.content):
+ #return hdoc
+ #hdoc = self._get_headers_doc()
+#
+ #if container and not empty(hdoc.content):
# mem-cache it
- hdoc_content = hdoc.content
- chash = hdoc_content.get(fields.CONTENT_HASH_KEY)
- hdocs = {chash: hdoc_content}
- container.memstore.load_header_docs(hdocs)
- return hdoc
-
- @property
- def chash(self):
- """
- An accessor to the content hash for this message.
- """
- if not self.fdoc:
- return None
- if not self.__chash and self.fdoc:
- self.__chash = self.fdoc.content.get(
- fields.CONTENT_HASH_KEY, None)
- return self.__chash
-
- @property
- def bdoc(self):
- """
- An accessor to the body document.
- """
- if not self.hdoc:
- return None
- if not self.__bdoc:
- self.__bdoc = self._get_body_doc()
- return self.__bdoc
+ #hdoc_content = hdoc.content
+ #chash = hdoc_content.get(fields.CONTENT_HASH_KEY)
+ #hdocs = {chash: hdoc_content}
+ #container.memstore.load_header_docs(hdocs)
+ #return hdoc
+#
+ #@property
+ #def chash(self):
+ #"""
+ #An accessor to the content hash for this message.
+ #"""
+ #if not self.fdoc:
+ #return None
+ #if not self.__chash and self.fdoc:
+ #self.__chash = self.fdoc.content.get(
+ #fields.CONTENT_HASH_KEY, None)
+ #return self.__chash
+
+ #@property
+ #def bdoc(self):
+ #"""
+ #An accessor to the body document.
+ #"""
+ #if not self.hdoc:
+ #return None
+ #if not self.__bdoc:
+ #self.__bdoc = self._get_body_doc()
+ #return self.__bdoc
# IMessage implementation
@@ -209,8 +188,13 @@ class LeapMessage(fields, MBoxParser):
:return: uid for this message
:rtype: int
"""
+ # TODO ----> return lookup in local sqlcipher table.
return self._uid
+ # --------------------------------------------------------------
+ # TODO -- from here on, all the methods should be proxied to the
+ # instance of leap.mail.mail.Message
+
def getFlags(self):
"""
Retrieve the flags associated with this Message.
@@ -253,25 +237,24 @@ class LeapMessage(fields, MBoxParser):
REMOVE = -1
SET = 0
- with fdoc_locks[mbox][uid]:
- doc = self.fdoc
- if not doc:
- logger.warning(
- "Could not find FDOC for %r:%s while setting flags!" %
- (mbox, uid))
- return
- current = doc.content[self.FLAGS_KEY]
- if mode == APPEND:
- newflags = tuple(set(tuple(current) + flags))
- elif mode == REMOVE:
- newflags = tuple(set(current).difference(set(flags)))
- elif mode == SET:
- newflags = flags
- new_fdoc = {
- self.FLAGS_KEY: newflags,
- self.SEEN_KEY: self.SEEN_FLAG in newflags,
- self.DEL_KEY: self.DELETED_FLAG in newflags}
- self._collection.memstore.update_flags(mbox, uid, new_fdoc)
+ doc = self.fdoc
+ if not doc:
+ logger.warning(
+ "Could not find FDOC for %r:%s while setting flags!" %
+ (mbox, uid))
+ return
+ current = doc.content[self.FLAGS_KEY]
+ if mode == APPEND:
+ newflags = tuple(set(tuple(current) + flags))
+ elif mode == REMOVE:
+ newflags = tuple(set(current).difference(set(flags)))
+ elif mode == SET:
+ newflags = flags
+ new_fdoc = {
+ self.FLAGS_KEY: newflags,
+ self.SEEN_KEY: self.SEEN_FLAG in newflags,
+ self.DEL_KEY: self.DELETED_FLAG in newflags}
+ self._collection.memstore.update_flags(mbox, uid, new_fdoc)
return map(str, newflags)
@@ -371,9 +354,9 @@ class LeapMessage(fields, MBoxParser):
else:
logger.warning("No FLAGS doc for %s:%s" % (self._mbox,
self._uid))
- if not size:
+ #if not size:
# XXX fallback, should remove when all migrated.
- size = self.getBodyFile().len
+ #size = self.getBodyFile().len
return size
def getHeaders(self, negate, *names):
@@ -395,6 +378,9 @@ class LeapMessage(fields, MBoxParser):
# XXX refactor together with MessagePart method
headers = self._get_headers()
+
+ # XXX keep this in the imap imessage implementation,
+ # because the server impl. expects content-type to be present.
if not headers:
logger.warning("No headers found")
return {str('content-type'): str('')}
@@ -614,64 +600,23 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
(the u1db index) for all the headers documents for a given mailbox.
We use it to prefetch massively all the headers for a mailbox.
This is the second massive query, after fetching all the FLAGS, that
- a MUA will do in a case where we do not have local disk cache.
+ a typical IMAP MUA will do in a case where we do not have local disk cache.
"""
HDOCS_SET_DOC = "HDOCS_SET"
templates = {
- # Message Level
-
- FLAGS_DOC: {
- fields.TYPE_KEY: fields.TYPE_FLAGS_VAL,
- fields.UID_KEY: 1, # XXX moe to a local table
- fields.MBOX_KEY: fields.INBOX_VAL,
- fields.CONTENT_HASH_KEY: "",
-
- fields.SEEN_KEY: False,
- fields.DEL_KEY: False,
- fields.FLAGS_KEY: [],
- fields.MULTIPART_KEY: False,
- fields.SIZE_KEY: 0
- },
-
- HEADERS_DOC: {
- fields.TYPE_KEY: fields.TYPE_HEADERS_VAL,
- fields.CONTENT_HASH_KEY: "",
-
- fields.DATE_KEY: "",
- fields.SUBJECT_KEY: "",
-
- fields.HEADERS_KEY: {},
- fields.PARTS_MAP_KEY: {},
- },
-
- CONTENT_DOC: {
- fields.TYPE_KEY: fields.TYPE_CONTENT_VAL,
- fields.PAYLOAD_HASH_KEY: "",
- fields.LINKED_FROM_KEY: [],
- fields.CTYPE_KEY: "", # should index by this too
-
- # should only get inmutable headers parts
- # (for indexing)
- fields.HEADERS_KEY: {},
- fields.RAW_KEY: "",
- fields.PARTS_MAP_KEY: {},
- fields.HEADERS_KEY: {},
- fields.MULTIPART_KEY: False,
- },
-
# Mailbox Level
RECENT_DOC: {
- fields.TYPE_KEY: fields.TYPE_RECENT_VAL,
- fields.MBOX_KEY: fields.INBOX_VAL,
+ "type": indexes.RECENT,
+ "mbox": INBOX_NAME,
fields.RECENTFLAGS_KEY: [],
},
HDOCS_SET_DOC: {
- fields.TYPE_KEY: fields.TYPE_HDOCS_SET_VAL,
- fields.MBOX_KEY: fields.INBOX_VAL,
+ "type": indexes.HDOCS_SET,
+ "mbox": INBOX_NAME,
fields.HDOCS_SET_KEY: [],
}
@@ -681,8 +626,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
# Different locks for wrapping both the u1db document getting/setting
# and the property getting/settting in an atomic operation.
- # TODO we would abstract this to a SoledadProperty class
-
+ # TODO --- deprecate ! --- use SoledadDocumentWrapper + locks
_rdoc_lock = defaultdict(lambda: threading.Lock())
_rdoc_write_lock = defaultdict(lambda: threading.Lock())
_rdoc_read_lock = defaultdict(lambda: threading.Lock())
@@ -764,81 +708,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
rdoc[fields.MBOX_KEY] = self.mbox
self._soledad.create_doc(rdoc)
- @deferred_to_thread
- def _do_parse(self, raw):
- """
- Parse raw message and return it along with
- relevant information about its outer level.
-
- This is done in a separate thread, and the callback is passed
- to `_do_add_msg` method.
+ # --------------------------------------------------------------------
- :param raw: the raw message
- :type raw: StringIO or basestring
- :return: msg, parts, chash, size, multi
- :rtype: tuple
- """
- msg = message_from_string(raw)
- parts = walk.get_parts(msg)
- size = len(raw)
- chash = sha256.SHA256(raw).hexdigest()
- multi = msg.is_multipart()
- return msg, parts, chash, size, multi
-
- def _populate_flags(self, flags, uid, chash, size, multi):
- """
- Return a flags doc.
-
- XXX Missing DOC -----------
- """
- fd = self._get_empty_doc(self.FLAGS_DOC)
-
- fd[self.MBOX_KEY] = self.mbox
- fd[self.UID_KEY] = uid
- fd[self.CONTENT_HASH_KEY] = chash
- fd[self.SIZE_KEY] = size
- fd[self.MULTIPART_KEY] = multi
- if flags:
- fd[self.FLAGS_KEY] = flags
- fd[self.SEEN_KEY] = self.SEEN_FLAG in flags
- fd[self.DEL_KEY] = self.DELETED_FLAG in flags
- fd[self.RECENT_KEY] = True # set always by default
- return fd
-
- def _populate_headr(self, msg, chash, subject, date):
- """
- Return a headers doc.
-
- XXX Missing DOC -----------
- """
- headers = defaultdict(list)
- for k, v in msg.items():
- headers[k].append(v)
-
- # "fix" for repeated headers.
- for k, v in headers.items():
- newline = "\n%s: " % (k,)
- headers[k] = newline.join(v)
-
- lower_headers = lowerdict(headers)
- msgid = first(MSGID_RE.findall(
- lower_headers.get('message-id', '')))
-
- hd = self._get_empty_doc(self.HEADERS_DOC)
- hd[self.CONTENT_HASH_KEY] = chash
- hd[self.HEADERS_KEY] = headers
- hd[self.MSGID_KEY] = msgid
-
- if not subject and self.SUBJECT_FIELD in headers:
- hd[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
- else:
- hd[self.SUBJECT_KEY] = subject
-
- if not date and self.DATE_FIELD in headers:
- hd[self.DATE_KEY] = headers[self.DATE_FIELD]
- else:
- hd[self.DATE_KEY] = date
- return hd
+ # -----------------------------------------------------------------------
def _fdoc_already_exists(self, chash):
"""
@@ -885,86 +757,41 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
flags = tuple()
leap_assert_type(flags, tuple)
- # TODO return soledad deferred instead
- observer = defer.Deferred()
- d = self._do_parse(raw)
- d.addCallback(lambda result: reactor.callInThread(
- self._do_add_msg, result, flags, subject, date,
- notify_on_disk, observer))
- return observer
+ # TODO ---- proxy to MessageCollection addMessage
+
+ #observer = defer.Deferred()
+ #d = self._do_parse(raw)
+ #d.addCallback(lambda result: reactor.callInThread(
+ #self._do_add_msg, result, flags, subject, date,
+ #notify_on_disk, observer))
+ #return observer
+
+ # TODO ---------------------------------------------------
+ # move this to leap.mail.adaptors.soledad
- # Called in thread
def _do_add_msg(self, parse_result, flags, subject,
date, notify_on_disk, observer):
"""
- Helper that creates a new message document.
- Here lives the magic of the leap mail. Well, in soledad, really.
-
- See `add_msg` docstring for parameter info.
-
- :param parse_result: a tuple with the results of `self._do_parse`
- :type parse_result: tuple
- :param observer: a deferred that will be fired with the message
- uid when the adding succeed.
- :type observer: deferred
"""
- # TODO signal that we can delete the original message!-----
- # when all the processing is done.
-
- # TODO add the linked-from info !
- # TODO add reference to the original message
-
msg, parts, chash, size, multi = parse_result
+ # XXX move to SoledadAdaptor write operation ... ???
# check for uniqueness --------------------------------
# Watch out! We're reserving a UID right after this!
existing_uid = self._fdoc_already_exists(chash)
if existing_uid:
msg = self.get_msg_by_uid(existing_uid)
-
- # We can say the observer that we're done
- # TODO return soledad deferred instead
reactor.callFromThread(observer.callback, existing_uid)
msg.setFlags((fields.DELETED_FLAG,), -1)
return
+ # TODO move UID autoincrement to MessageCollection.addMessage(mailbox)
# TODO S2 -- get FUCKING UID from autoincremental table
- uid = self.memstore.increment_last_soledad_uid(self.mbox)
-
- # We can say the observer that we're done at this point, but
- # before that we should make sure it has no serious consequences
- # if we're issued, for instance, a fetch command right after...
- # reactor.callFromThread(observer.callback, uid)
- # if we did the notify, we need to invalidate the deferred
- # so not to try to fire it twice.
- # observer = None
-
- fd = self._populate_flags(flags, uid, chash, size, multi)
- hd = self._populate_headr(msg, chash, subject, date)
-
- body_phash_fun = [walk.get_body_phash_simple,
- walk.get_body_phash_multi][int(multi)]
- body_phash = body_phash_fun(walk.get_payloads(msg))
- parts_map = walk.walk_msg_tree(parts, body_phash=body_phash)
-
- # add parts map to header doc
- # (body, multi, part_map)
- for key in parts_map:
- hd[key] = parts_map[key]
- del parts_map
+ #uid = self.memstore.increment_last_soledad_uid(self.mbox)
+ #self.set_recent_flag(uid)
- hd = stringify_parts_map(hd)
- # The MessageContainer expects a dict, one-indexed
- cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1))
-
- self.set_recent_flag(uid)
- msg_container = MessageWrapper(fd, hd, cdocs)
-
- # TODO S1 -- just pass this to memstore and return that deferred.
- self.memstore.create_message(
- self.mbox, uid, msg_container,
- observer=observer, notify_on_disk=notify_on_disk)
+ # ------------------------------------------------------------
#
# getters: specific queries
@@ -1073,6 +900,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
the query failed.
:rtype: SoledadDocument or None.
"""
+ # USED from:
+ # [ ] duplicated fdoc detection
+ # [ ] _get_uid_from_msgidCb
+
# FIXME ----- use deferreds.
curried = partial(
self._soledad.get_from_index,
@@ -1205,51 +1036,52 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
if msg_container is not None:
if mem_only:
- msg = LeapMessage(None, uid, self.mbox, collection=self,
+ msg = IMAPMessage(None, uid, self.mbox, collection=self,
container=msg_container)
else:
# We pass a reference to soledad just to be able to retrieve
# missing parts that cannot be found in the container, like
# the content docs after a copy.
- msg = LeapMessage(self._soledad, uid, self.mbox,
+ msg = IMAPMessage(self._soledad, uid, self.mbox,
collection=self, container=msg_container)
else:
- msg = LeapMessage(self._soledad, uid, self.mbox, collection=self)
+ msg = IMAPMessage(self._soledad, uid, self.mbox, collection=self)
if not msg.does_exist():
return None
return msg
- def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL):
- """
- Get all documents for the selected mailbox of the
- passed type. By default, it returns the flag docs.
-
- If you want acess to the content, use __iter__ instead
-
- :return: a Deferred, that will fire with a list of u1db documents
- :rtype: Deferred (promise of list of SoledadDocument)
- """
- if _type not in fields.__dict__.values():
- raise TypeError("Wrong type passed to get_all_docs")
-
+ # FIXME --- used where ? ---------------------------------------------
+ #def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL):
+ #"""
+ #Get all documents for the selected mailbox of the
+ #passed type. By default, it returns the flag docs.
+#
+ #If you want acess to the content, use __iter__ instead
+#
+ #:return: a Deferred, that will fire with a list of u1db documents
+ #:rtype: Deferred (promise of list of SoledadDocument)
+ #"""
+ #if _type not in fields.__dict__.values():
+ #raise TypeError("Wrong type passed to get_all_docs")
+#
# FIXME ----- either raise or return a deferred wrapper.
- if sameProxiedObjects(self._soledad, None):
- logger.warning('Tried to get messages but soledad is None!')
- return []
-
- def get_sorted_docs(docs):
- all_docs = [doc for doc in docs]
+ #if sameProxiedObjects(self._soledad, None):
+ #logger.warning('Tried to get messages but soledad is None!')
+ #return []
+#
+ #def get_sorted_docs(docs):
+ #all_docs = [doc for doc in docs]
# inneficient, but first let's grok it and then
# let's worry about efficiency.
# XXX FIXINDEX -- should implement order by in soledad
# FIXME ----------------------------------------------
- return sorted(all_docs, key=lambda item: item.content['uid'])
-
- d = self._soledad.get_from_index(
- fields.TYPE_MBOX_IDX, _type, self.mbox)
- d.addCallback(get_sorted_docs)
- return d
+ #return sorted(all_docs, key=lambda item: item.content['uid'])
+#
+ #d = self._soledad.get_from_index(
+ #fields.TYPE_MBOX_IDX, _type, self.mbox)
+ #d.addCallback(get_sorted_docs)
+ #return d
def all_soledad_uid_iter(self):
"""
@@ -1350,7 +1182,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
:returns: a list of LeapMessages
:rtype: list
"""
- return [LeapMessage(self._soledad, docid, self.mbox, collection=self)
+ return [IMAPMessage(self._soledad, docid, self.mbox, collection=self)
for docid in self.unseen_iter()]
# recent messages
@@ -1384,7 +1216,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
:returns: iterator of dicts with content for all messages.
:rtype: iterable
"""
- return (LeapMessage(self._soledad, docuid, self.mbox, collection=self)
+ return (IMAPMessage(self._soledad, docuid, self.mbox, collection=self)
for docuid in self.all_uid_iter())
def __repr__(self):
diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py
deleted file mode 100644
index 4a801b0..0000000
--- a/src/leap/mail/imap/parser.py
+++ /dev/null
@@ -1,45 +0,0 @@
-# -*- coding: utf-8 -*-
-# parser.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-Mail parser mixin.
-"""
-import re
-
-
-class MBoxParser(object):
- """
- Utility function to parse mailbox names.
- """
- INBOX_NAME = "INBOX"
- INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
-
- def _parse_mailbox_name(self, name):
- """
- Return a normalized representation of the mailbox C{name}.
-
- This method ensures that an eventual initial 'inbox' part of a
- mailbox name is made uppercase.
-
- :param name: the name of the mailbox
- :type name: unicode
-
- :rtype: unicode
- """
- if self.INBOX_RE.match(name):
- # ensure inital INBOX is uppercase
- return self.INBOX_NAME + name[len(self.INBOX_NAME):]
- return name
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index dd4294c..5af499f 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -94,6 +94,8 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
MessageCollection interface in this particular TestCase
"""
super(MessageCollectionTestCase, self).setUp()
+
+ # TODO deprecate memstore
memstore = MemoryStore()
self.messages = MessageCollection("testmbox%s" % (self.count,),
self._soledad, memstore=memstore)
diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py
index 9a3868c..920eeb0 100644
--- a/src/leap/mail/imap/tests/utils.py
+++ b/src/leap/mail/imap/tests/utils.py
@@ -51,6 +51,7 @@ class SimpleClient(imap4.IMAP4Client):
self.transport.loseConnection()
+# XXX move to common helper
def initialize_soledad(email, gnupg_home, tempdir):
"""
Initializes soledad by hand
@@ -110,9 +111,7 @@ class IMAP4HelperMixin(BaseLeapTest):
"""
Setup method for each test.
- Initializes and run a LEAP IMAP4 Server,
- but passing the same Soledad instance (it's costly to initialize),
- so we have to be sure to restore state across tests.
+ Initializes and run a LEAP IMAP4 Server.
"""
self.old_path = os.environ['PATH']
self.old_home = os.environ['HOME']
@@ -172,19 +171,17 @@ class IMAP4HelperMixin(BaseLeapTest):
def tearDown(self):
"""
tearDown method called after each test.
-
- Deletes all documents in the Index, and deletes
- instances of server and client.
"""
try:
self._soledad.close()
+ except Exception:
+ print "ERROR WHILE CLOSING SOLEDAD"
+ finally:
os.environ["PATH"] = self.old_path
os.environ["HOME"] = self.old_home
# safety check
assert 'leap_tests-' in self.tempdir
shutil.rmtree(self.tempdir)
- except Exception:
- print "ERROR WHILE CLOSING SOLEDAD"
def populateMessages(self):
"""
@@ -223,5 +220,3 @@ class IMAP4HelperMixin(BaseLeapTest):
def loopback(self):
return loopback.loopbackAsync(self.server, self.client)
-
-
diff --git a/src/leap/mail/interfaces.py b/src/leap/mail/interfaces.py
new file mode 100644
index 0000000..5838ce9
--- /dev/null
+++ b/src/leap/mail/interfaces.py
@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+# interfaces.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Interfaces for the leap.mail module.
+"""
+from zope.interface import Interface, Attribute
+
+
+class IMessageWrapper(Interface):
+ """
+ I know how to access the different parts into which a given message is
+ splitted into.
+ """
+
+ fdoc = Attribute('A dictionaly-like containing the flags document '
+ '(mutable)')
+ hdoc = Attribute('A dictionary-like containing the headers docuemnt '
+ '(immutable)')
+ cdocs = Attribute('A dictionary with the content-docs, one-indexed')
+
+
+class IMailAdaptor(Interface):
+ """
+ I know how to store the standard representation for messages and mailboxes,
+ and how to update the relevant mutable parts when needed.
+ """
+
+ def initialize_store(self, store):
+ """
+ Performs whatever initialization is needed before the store can be
+ used (creating indexes, sanity checks, etc).
+
+ :param store: store
+ :returns: a Deferred that will fire when the store is correctly
+ initialized.
+ :rtype: deferred
+ """
+
+ # TODO is staticmethod valid with an interface?
+ # @staticmethod
+ def get_msg_from_string(self, MessageClass, raw_msg):
+ """
+ Return IMessageWrapper implementor from a raw mail string
+
+ :param MessageClass: an implementor of IMessage
+ :type raw_msg: str
+ :rtype: implementor of leap.mail.IMessage
+ """
+
+ # TODO is staticmethod valid with an interface?
+ # @staticmethod
+ def get_msg_from_docs(self, MessageClass, msg_wrapper):
+ """
+ Return an IMessage implementor from its parts.
+
+ :param MessageClass: an implementor of IMessage
+ :param msg_wrapper: an implementor of IMessageWrapper
+ :rtype: implementor of leap.mail.IMessage
+ """
+
+ # -------------------------------------------------------------------
+ # XXX unsure about the following part yet ...........................
+
+ # the idea behind these three methods is that the adaptor also offers a
+ # fixed interface to create the documents the first time (using
+ # soledad.create_docs or whatever method maps to it in a similar store, and
+ # also allows to update flags and tags, hiding the actual implementation of
+ # where the flags/tags live in behind the concrete MailWrapper in use
+ # by this particular adaptor. In our impl it will be put_doc(fdoc) after
+ # locking the getting + updating of that fdoc for atomicity.
+
+ # 'store' must be an instance of something that offers a minimal subset of
+ # the document API that Soledad currently implements (create_doc, put_doc)
+ # I *think* store should belong to Account/Collection and be passed as
+ # param here instead of relying on it being an attribute of the instance.
+
+ def create_msg_docs(self, store, msg_wrapper):
+ """
+ :param store: The documents store
+ :type store:
+ :param msg_wrapper:
+ :type msg_wrapper: IMessageWrapper implementor
+ """
+
+ def update_msg_flags(self, store, msg_wrapper):
+ """
+ :param store: The documents store
+ :type store:
+ :param msg_wrapper:
+ :type msg_wrapper: IMessageWrapper implementor
+ """
+
+ def update_msg_tags(self, store, msg_wrapper):
+ """
+ :param store: The documents store
+ :type store:
+ :param msg_wrapper:
+ :type msg_wrapper: IMessageWrapper implementor
+ """
diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py
new file mode 100644
index 0000000..ea9c95e
--- /dev/null
+++ b/src/leap/mail/mail.py
@@ -0,0 +1,248 @@
+# -*- coding: utf-8 -*-
+# mail.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Generic Access to Mail objects: Public LEAP Mail API.
+"""
+from twisted.internet import defer
+
+from leap.mail.constants import INBOX_NAME
+from leap.mail.adaptors.soledad import SoledadMailAdaptor
+
+
+# TODO
+# [ ] Probably change the name of this module to "api" or "account", mail is
+# too generic (there's also IncomingMail, and OutgoingMail
+
+
+class Message(object):
+
+ def __init__(self, wrapper):
+ """
+ :param wrapper: an instance of an implementor of IMessageWrapper
+ """
+ self._wrapper = wrapper
+
+ def get_wrapper(self):
+ return self._wrapper
+
+ # imap.IMessage methods
+
+ def get_flags():
+ """
+ """
+
+ def get_internal_date():
+ """
+ """
+
+ # imap.IMessageParts
+
+ def get_headers():
+ """
+ """
+
+ def get_body_file():
+ """
+ """
+
+ def get_size():
+ """
+ """
+
+ def is_multipart():
+ """
+ """
+
+ def get_subpart(part):
+ """
+ """
+
+ # Custom methods.
+
+ def get_tags():
+ """
+ """
+
+
+class MessageCollection(object):
+ """
+ A generic collection of messages. It can be messages sharing the same
+ mailbox, tag, the result of a given query, or just a bunch of ids for
+ master documents.
+
+ Since LEAP Mail is primarily oriented to store mail in Soledad, the default
+ (and, so far, only) implementation of the store is contained in this
+ Soledad Mail Adaptor. If you need to use a different adaptor, change the
+ adaptor class attribute in your Account object.
+
+ Store is a reference to a particular instance of the message store (soledad
+ instance or proxy, for instance).
+ """
+
+ # TODO look at IMessageSet methods
+
+ # Account should provide an adaptor instance when creating this collection.
+ adaptor = None
+ store = None
+
+ def get_message_by_doc_id(self, doc_id):
+ # ... get from soledad etc
+ # ... but that should be part of adaptor/store too... :/
+ fdoc, hdoc = None
+ return self.adaptor.from_docs(Message, fdoc=fdoc, hdoc=hdoc)
+
+ # TODO review if this is the best place for:
+
+ def create_docs():
+ pass
+
+ def udpate_flags():
+ # 1. update the flags in the message wrapper --- stored where???
+ # 2. call adaptor.update_msg(store)
+ pass
+
+ def update_tags():
+ # 1. update the tags in the message wrapper --- stored where???
+ # 2. call adaptor.update_msg(store)
+ pass
+
+ # TODO add delete methods here?
+
+
+class Account(object):
+ """
+ Account is the top level abstraction to access collections of messages
+ associated with a LEAP Mail Account.
+
+ It primarily handles creation and access of Mailboxes, which will be the
+ basic collection handled by traditional MUAs, but it can also handle other
+ types of Collections (tag based, for instance).
+
+ leap.mail.imap.SoledadBackedAccount partially proxies methods in this
+ class.
+ """
+
+ # Adaptor is passed to the returned MessageCollections, so if you want to
+ # use a different adaptor this is the place to change it, by subclassing
+ # the Account class.
+
+ adaptor_class = SoledadMailAdaptor
+ store = None
+ mailboxes = None
+
+ def __init__(self, store):
+ self.store = store
+ self.adaptor = self.adaptor_class()
+
+ self.__mailboxes = set([])
+ self._initialized = False
+ self._deferred_initialization = defer.Deferred()
+
+ self._initialize_storage()
+
+ def _initialize_storage(self):
+
+ def add_mailbox_if_none(result):
+ # every user should have the right to an inbox folder
+ # at least, so let's make one!
+ if not self.mailboxes:
+ self.add_mailbox(INBOX_NAME)
+
+ def finish_initialization(result):
+ self._initialized = True
+ self._deferred_initialization.callback(None)
+
+ def load_mbox_cache(result):
+ d = self._load_mailboxes()
+ d.addCallback(lambda _: result)
+ return d
+
+ d = self.adaptor.initialize_store(self.store)
+ d.addCallback(load_mbox_cache)
+ d.addCallback(add_mailbox_if_none)
+ d.addCallback(finish_initialization)
+
+ def callWhenReady(self, cb):
+ # XXX this could use adaptor.store_ready instead...??
+ if self._initialized:
+ cb(self)
+ return defer.succeed(None)
+ else:
+ self._deferred_initialization.addCallback(cb)
+ return self._deferred_initialization
+
+ @property
+ def mailboxes(self):
+ """
+ A list of the current mailboxes for this account.
+ :rtype: set
+ """
+ return sorted(self.__mailboxes)
+
+ def _load_mailboxes(self):
+
+ def update_mailboxes(mbox_names):
+ self.__mailboxes.update(mbox_names)
+
+ d = self.adaptor.get_all_mboxes(self.store)
+ d.addCallback(update_mailboxes)
+ return d
+
+ #
+ # Public API Starts
+ #
+
+ # XXX params for IMAP only???
+ def list_mailboxes(self, ref, wildcard):
+ self.adaptor.get_all_mboxes(self.store)
+
+ def add_mailbox(self, name, mbox=None):
+ pass
+
+ def create_mailbox(self, pathspec):
+ pass
+
+ def delete_mailbox(self, name):
+ pass
+
+ def rename_mailbox(self, oldname, newname):
+ pass
+
+ # FIXME yet to be decided if it belongs here...
+
+ def get_collection_by_mailbox(self, name):
+ """
+ :rtype: MessageCollection
+ """
+ # imap select will use this, passing the collection to SoledadMailbox
+ # XXX pass adaptor to MessageCollection
+ pass
+
+ def get_collection_by_docs(self, docs):
+ """
+ :rtype: MessageCollection
+ """
+ # get a collection of docs by a list of doc_id
+ # XXX pass adaptor to MessageCollection
+ pass
+
+ def get_collection_by_tag(self, tag):
+ """
+ :rtype: MessageCollection
+ """
+ # is this a good idea?
+ pass
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-29 11:26:31 +0000