summaryrefslogtreecommitdiff
path: root/src/leap
diff options
context:
space:
mode:
authorKali Kaneko <kali@leap.se>2015-08-31 14:21:50 -0400
committerKali Kaneko <kali@leap.se>2015-08-31 14:21:50 -0400
commit974511645819ef2559fb0c064fdea4bcc926e6d8 (patch)
treeedecfd6c6a3b9bdadaea45de150990a40acaca96 /src/leap
parent67b738c2f5922dbc5cf105e9617ce6d1d5e4e8e3 (diff)
parent43c920f38d1c114e9044d2da15d4a05d5faab79d (diff)
Merge tag '0.4.0rc2' into debian/0.4.0rc2
Tag leap.mail version 0.4.0rc2
Diffstat (limited to 'src/leap')
-rw-r--r--src/leap/mail/_version.py1
-rw-r--r--src/leap/mail/adaptors/__init__.py0
-rw-r--r--src/leap/mail/adaptors/models.py123
-rw-r--r--src/leap/mail/adaptors/soledad.py1248
-rw-r--r--src/leap/mail/adaptors/soledad_indexes.py106
-rw-r--r--src/leap/mail/adaptors/tests/__init__.py0
l---------src/leap/mail/adaptors/tests/rfc822.message1
-rw-r--r--src/leap/mail/adaptors/tests/test_models.py106
-rw-r--r--src/leap/mail/adaptors/tests/test_soledad_adaptor.py529
-rw-r--r--src/leap/mail/constants.py52
-rw-r--r--src/leap/mail/imap/account.py519
-rw-r--r--src/leap/mail/imap/fetch.py655
-rw-r--r--src/leap/mail/imap/fields.py173
-rw-r--r--src/leap/mail/imap/index.py69
-rw-r--r--src/leap/mail/imap/interfaces.py94
-rw-r--r--src/leap/mail/imap/mailbox.py841
-rw-r--r--src/leap/mail/imap/memorystore.py1333
-rw-r--r--src/leap/mail/imap/messageparts.py586
-rw-r--r--src/leap/mail/imap/messages.py1368
-rw-r--r--src/leap/mail/imap/parser.py45
-rw-r--r--src/leap/mail/imap/server.py430
-rw-r--r--src/leap/mail/imap/service/imap-server.tac36
-rw-r--r--src/leap/mail/imap/service/imap.py161
-rw-r--r--src/leap/mail/imap/soledadstore.py620
-rw-r--r--src/leap/mail/imap/tests/__init__.py20
-rwxr-xr-xsrc/leap/mail/imap/tests/regressions_mime_struct (renamed from src/leap/mail/imap/tests/regressions)20
l---------[-rw-r--r--]src/leap/mail/imap/tests/rfc822.message87
l---------[-rw-r--r--]src/leap/mail/imap/tests/rfc822.multi-minimal.message17
l---------src/leap/mail/imap/tests/rfc822.multi-nested.message1
l---------[-rw-r--r--]src/leap/mail/imap/tests/rfc822.multi-signed.message239
l---------[-rw-r--r--]src/leap/mail/imap/tests/rfc822.multi.message97
l---------[-rw-r--r--]src/leap/mail/imap/tests/rfc822.plain.message67
-rwxr-xr-xsrc/leap/mail/imap/tests/stress_tests_imap.zsh (renamed from src/leap/mail/imap/tests/leap_tests_imap.zsh)0
-rw-r--r--src/leap/mail/imap/tests/test_imap.py676
-rw-r--r--src/leap/mail/imap/tests/test_imap_store_fetch.py71
-rw-r--r--src/leap/mail/imap/tests/utils.py227
-rw-r--r--src/leap/mail/imap/tests/walktree.py4
-rw-r--r--src/leap/mail/incoming/__init__.py0
-rw-r--r--src/leap/mail/incoming/service.py766
-rw-r--r--src/leap/mail/incoming/tests/__init__.py0
-rw-r--r--src/leap/mail/incoming/tests/test_incoming_mail.py260
-rw-r--r--src/leap/mail/interfaces.py116
-rw-r--r--src/leap/mail/mail.py1097
-rw-r--r--src/leap/mail/mailbox_indexer.py324
-rw-r--r--src/leap/mail/messageflow.py200
-rw-r--r--src/leap/mail/outgoing/__init__.py0
-rw-r--r--src/leap/mail/outgoing/service.py467
-rw-r--r--src/leap/mail/outgoing/tests/__init__.py0
-rw-r--r--src/leap/mail/outgoing/tests/test_outgoing.py256
-rw-r--r--src/leap/mail/plugins/__init__.py3
-rw-r--r--src/leap/mail/plugins/soledad_sync_hooks.py19
-rw-r--r--src/leap/mail/smtp/README.rst41
-rw-r--r--src/leap/mail/smtp/__init__.py18
-rw-r--r--src/leap/mail/smtp/gateway.py526
-rw-r--r--src/leap/mail/smtp/rfc3156.py2
-rw-r--r--src/leap/mail/smtp/tests/__init__.py386
-rw-r--r--src/leap/mail/smtp/tests/test_gateway.py263
-rw-r--r--src/leap/mail/sync_hooks.py120
-rw-r--r--src/leap/mail/tests/__init__.py331
-rw-r--r--src/leap/mail/tests/common.py99
-rw-r--r--src/leap/mail/tests/rfc822.message86
-rw-r--r--src/leap/mail/tests/rfc822.multi-minimal.message16
-rw-r--r--src/leap/mail/tests/rfc822.multi-nested.message619
-rw-r--r--src/leap/mail/tests/rfc822.multi-signed.message238
-rw-r--r--src/leap/mail/tests/rfc822.multi.message96
-rw-r--r--src/leap/mail/tests/rfc822.plain.message66
-rw-r--r--src/leap/mail/tests/test_mail.py399
-rw-r--r--src/leap/mail/tests/test_mailbox_indexer.py250
-rw-r--r--src/leap/mail/utils.py54
-rw-r--r--src/leap/mail/walk.py173
70 files changed, 9749 insertions, 8164 deletions
diff --git a/src/leap/mail/_version.py b/src/leap/mail/_version.py
index fe18f6c..d32052b 100644
--- a/src/leap/mail/_version.py
+++ b/src/leap/mail/_version.py
@@ -1,4 +1,3 @@
-
# This file was generated by the `freeze_debianver` command in setup.py
# Using 'versioneer.py' (0.7+) from
# revision-control system data, or from the parent directory name of an
diff --git a/src/leap/mail/adaptors/__init__.py b/src/leap/mail/adaptors/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/leap/mail/adaptors/__init__.py
diff --git a/src/leap/mail/adaptors/models.py b/src/leap/mail/adaptors/models.py
new file mode 100644
index 0000000..49460f7
--- /dev/null
+++ b/src/leap/mail/adaptors/models.py
@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+# models.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Generic Models to be used by the Document Adaptors.
+"""
+import copy
+
+
+class SerializableModel(object):
+ """
+ A Generic document model, that can be serialized into a dictionary.
+
+ Subclasses of this `SerializableModel` are meant to be added as class
+ attributes of classes inheriting from DocumentWrapper.
+
+ A subclass __meta__ of this SerializableModel might exist, and contain info
+ relative to particularities of this model.
+
+ For instance, the use of `__meta__.index` marks the existence of a primary
+ index in the model, which will be used to do unique queries (in which case
+ all the other indexed fields in the underlying document will be filled with
+ the default info contained in the model definition).
+ """
+
+ @classmethod
+ def serialize(klass):
+ """
+ Get a dictionary representation of the public attributes in the model
+ class. To avoid collisions with builtin functions, any occurrence of an
+ attribute ended in '_' (like 'type_') will be normalized by removing
+ the trailing underscore.
+
+ This classmethod is used from within the serialized method of a
+ DocumentWrapper instance: it provides defaults for the
+ empty document.
+ """
+ assert isinstance(klass, type)
+ return _normalize_dict(klass.__dict__)
+
+
+class DocumentWrapper(object):
+ """
+ A Wrapper object that can be manipulated, passed around, and serialized in
+ a format that the store understands.
+ It is related to a SerializableModel, which must be specified as the
+ ``model`` class attribute. The instance of this DocumentWrapper will not
+ allow any other *public* attributes than those defined in the corresponding
+ model.
+ """
+ # TODO we could do some very basic type checking here
+ # TODO set a dirty flag (on __setattr__, whenever the value is != from
+ # before)
+ # TODO we could enforce the existence of a correct "model" attribute
+ # in some other way (other than in the initializer)
+
+ def __init__(self, **kwargs):
+ if not getattr(self, 'model', None):
+ raise RuntimeError(
+ 'DocumentWrapper class needs a model attribute')
+
+ defaults = self.model.serialize()
+
+ if kwargs:
+ values = copy.deepcopy(defaults)
+ values.update(_normalize_dict(kwargs))
+ else:
+ values = defaults
+
+ for k, v in values.items():
+ k = k.replace('-', '_')
+ setattr(self, k, v)
+
+ def __setattr__(self, attr, value):
+ normalized = _normalize_dict(self.model.__dict__)
+ if not attr.startswith('_') and attr not in normalized:
+ raise RuntimeError(
+ "Cannot set attribute because it's not defined "
+ "in the model %s: %s" % (self.__class__, attr))
+ object.__setattr__(self, attr, value)
+
+ def serialize(self):
+ return _normalize_dict(self.__dict__)
+
+ def create(self):
+ raise NotImplementedError()
+
+ def update(self):
+ raise NotImplementedError()
+
+ def delete(self):
+ raise NotImplementedError()
+
+ @classmethod
+ def get_or_create(self):
+ raise NotImplementedError()
+
+ @classmethod
+ def get_all(self):
+ raise NotImplementedError()
+
+
+def _normalize_dict(_dict):
+ items = _dict.items()
+ items = filter(lambda (k, v): not callable(v), items)
+ items = filter(lambda (k, v): not k.startswith('_'), items)
+ items = [(k, v) if not k.endswith('_') else (k[:-1], v)
+ for (k, v) in items]
+ items = [(k.replace('-', '_'), v) for (k, v) in items]
+ return dict(items)
diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py
new file mode 100644
index 0000000..d114707
--- /dev/null
+++ b/src/leap/mail/adaptors/soledad.py
@@ -0,0 +1,1248 @@
+# soledad.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledadad MailAdaptor module.
+"""
+import logging
+import re
+
+from collections import defaultdict
+from email import message_from_string
+
+from pycryptopp.hash import sha256
+from twisted.internet import defer
+from twisted.python import log
+from zope.interface import implements
+import u1db
+
+from leap.common.check import leap_assert, leap_assert_type
+
+from leap.mail import constants
+from leap.mail import walk
+from leap.mail.adaptors import soledad_indexes as indexes
+from leap.mail.constants import INBOX_NAME
+from leap.mail.adaptors import models
+from leap.mail.imap.mailbox import normalize_mailbox
+from leap.mail.utils import lowerdict, first
+from leap.mail.utils import stringify_parts_map
+from leap.mail.interfaces import IMailAdaptor, IMessageWrapper
+
+from leap.soledad.common.document import SoledadDocument
+
+
+logger = logging.getLogger(__name__)
+
+# TODO
+# [ ] Convenience function to create mail specifying subject, date, etc?
+
+
+_MSGID_PATTERN = r"""<([\w@.]+)>"""
+_MSGID_RE = re.compile(_MSGID_PATTERN)
+
+
+class DuplicatedDocumentError(Exception):
+ """
+ Raised when a duplicated document is detected.
+ """
+ pass
+
+
+def cleanup_deferred_locks():
+ """
+ Need to use this from within trial to cleanup the reactor before
+ each run.
+ """
+ SoledadDocumentWrapper._k_locks = defaultdict(defer.DeferredLock)
+
+
+class SoledadDocumentWrapper(models.DocumentWrapper):
+ """
+ A Wrapper object that can be manipulated, passed around, and serialized in
+ a format that the Soledad Store understands.
+
+ It ensures atomicity of the document operations on creation, update and
+ deletion.
+ """
+ # TODO we could also use a _dirty flag (in models)
+ # TODO add a get_count() method ??? -- that is extended over u1db.
+
+ # We keep a dictionary with DeferredLocks, that will be
+ # unique to every subclass of SoledadDocumentWrapper.
+ _k_locks = defaultdict(defer.DeferredLock)
+
+ @classmethod
+ def _get_klass_lock(cls):
+ """
+ Get a DeferredLock that is unique for this subclass name.
+ Used to lock the access to indexes in the `get_or_create` call
+ for a particular DocumentWrapper.
+ """
+ return cls._k_locks[cls.__name__]
+
+ def __init__(self, doc_id=None, future_doc_id=None, **kwargs):
+ self._doc_id = doc_id
+ self._future_doc_id = future_doc_id
+ self._lock = defer.DeferredLock()
+ super(SoledadDocumentWrapper, self).__init__(**kwargs)
+
+ @property
+ def doc_id(self):
+ return self._doc_id
+
+ @property
+ def future_doc_id(self):
+ return self._future_doc_id
+
+ def set_future_doc_id(self, doc_id):
+ self._future_doc_id = doc_id
+
+ def create(self, store, is_copy=False):
+ """
+ Create the documents for this wrapper.
+ Since this method will not check for duplication, the
+ responsibility of avoiding duplicates is left to the caller.
+
+ You might be interested in using `get_or_create` classmethod
+ instead (that's the preferred way of creating documents from
+ the wrapper object).
+
+ :return: a deferred that will fire when the underlying
+ Soledad document has been created.
+ :rtype: Deferred
+ """
+ leap_assert(self._doc_id is None,
+ "This document already has a doc_id!")
+
+ def update_doc_id(doc):
+ self._doc_id = doc.doc_id
+ self.set_future_doc_id(None)
+ return doc
+
+ def update_wrapper(failure):
+ # In the case of some copies (for instance, from one folder to
+ # another and back to the original folder), the document that we
+ # want to insert already exists. In this case, putting it
+ # and overwriting the document with that doc_id is the right thing
+ # to do.
+ failure.trap(u1db.errors.RevisionConflict)
+ self._doc_id = self.future_doc_id
+ self._future_doc_id = None
+ return self.update(store)
+
+ if self.future_doc_id is None:
+ d = store.create_doc(self.serialize())
+ else:
+ d = store.create_doc(self.serialize(),
+ doc_id=self.future_doc_id)
+ d.addCallback(update_doc_id)
+
+ if is_copy:
+ d.addErrback(update_wrapper)
+ else:
+ d.addErrback(self._catch_revision_conflict, self.future_doc_id)
+ return d
+
+ def update(self, store):
+ """
+ Update the documents for this wrapper.
+
+ :return: a deferred that will fire when the underlying
+ Soledad document has been updated.
+ :rtype: Deferred
+ """
+ # the deferred lock guards against revision conflicts
+ return self._lock.run(self._update, store)
+
+ def _update(self, store):
+ leap_assert(self._doc_id is not None,
+ "Need to create doc before updating")
+
+ def update_and_put_doc(doc):
+ doc.content.update(self.serialize())
+ d = store.put_doc(doc)
+ d.addErrback(self._catch_revision_conflict, doc.doc_id)
+ return d
+
+ d = store.get_doc(self._doc_id)
+ d.addCallback(update_and_put_doc)
+ return d
+
+ def _catch_revision_conflict(self, failure, doc_id):
+ # XXX We can have some RevisionConflicts if we try
+ # to put the docs that are already there.
+ # This can happen right now when creating/saving the cdocs
+ # during a copy. Instead of catching and ignoring this
+ # error, we should mark them in the copy so there is no attempt to
+ # create/update them.
+ failure.trap(u1db.errors.RevisionConflict)
+ logger.debug("Got conflict while putting %s" % doc_id)
+
+ def delete(self, store):
+ """
+ Delete the documents for this wrapper.
+
+ :return: a deferred that will fire when the underlying
+ Soledad document has been deleted.
+ :rtype: Deferred
+ """
+ # the deferred lock guards against conflicts while updating
+ return self._lock.run(self._delete, store)
+
+ def _delete(self, store):
+ leap_assert(self._doc_id is not None,
+ "Need to create doc before deleting")
+ # XXX might want to flag this DocumentWrapper to avoid
+ # updating it by mistake. This could go in models.DocumentWrapper
+
+ def delete_doc(doc):
+ return store.delete_doc(doc)
+
+ d = store.get_doc(self._doc_id)
+ d.addCallback(delete_doc)
+ return d
+
+ @classmethod
+ def get_or_create(cls, store, index, value):
+ """
+ Get a unique DocumentWrapper by index, or create a new one if the
+ matching query does not exist.
+
+ :param index: the primary index for the model.
+ :type index: str
+ :param value: the value to query the primary index.
+ :type value: str
+
+ :return: a deferred that will be fired with the SoledadDocumentWrapper
+ matching the index query, either existing or just created.
+ :rtype: Deferred
+ """
+ return cls._get_klass_lock().run(
+ cls._get_or_create, store, index, value)
+
+ @classmethod
+ def _get_or_create(cls, store, index, value):
+ # TODO shorten this method.
+ assert store is not None
+ assert index is not None
+ assert value is not None
+
+ def get_main_index():
+ try:
+ return cls.model.__meta__.index
+ except AttributeError:
+ raise RuntimeError("The model is badly defined")
+
+ # TODO separate into another method?
+ def try_to_get_doc_from_index(indexes):
+ values = []
+ idx_def = dict(indexes)[index]
+ if len(idx_def) == 1:
+ values = [value]
+ else:
+ main_index = get_main_index()
+ fields = cls.model.serialize()
+ for field in idx_def:
+ if field == main_index:
+ values.append(value)
+ else:
+ values.append(fields[field])
+ d = store.get_from_index(index, *values)
+ return d
+
+ def get_first_doc_if_any(docs):
+ if not docs:
+ return None
+ if len(docs) > 1:
+ raise DuplicatedDocumentError
+ return docs[0]
+
+ def wrap_existing_or_create_new(doc):
+ if doc:
+ return cls(doc_id=doc.doc_id, **doc.content)
+ else:
+ return create_and_wrap_new_doc()
+
+ def create_and_wrap_new_doc():
+ # XXX use closure to store indexes instead of
+ # querying for them again.
+ d = store.list_indexes()
+ d.addCallback(get_wrapper_instance_from_index)
+ d.addCallback(return_wrapper_when_created)
+ return d
+
+ def get_wrapper_instance_from_index(indexes):
+ init_values = {}
+ idx_def = dict(indexes)[index]
+ if len(idx_def) == 1:
+ init_value = {idx_def[0]: value}
+ return cls(**init_value)
+ main_index = get_main_index()
+ fields = cls.model.serialize()
+ for field in idx_def:
+ if field == main_index:
+ init_values[field] = value
+ else:
+ init_values[field] = fields[field]
+ return cls(**init_values)
+
+ def return_wrapper_when_created(wrapper):
+ d = wrapper.create(store)
+ d.addCallback(lambda doc: wrapper)
+ return d
+
+ d = store.list_indexes()
+ d.addCallback(try_to_get_doc_from_index)
+ d.addCallback(get_first_doc_if_any)
+ d.addCallback(wrap_existing_or_create_new)
+ return d
+
+ @classmethod
+ def get_all(cls, store):
+ """
+ Get a collection of wrappers around all the documents belonging
+ to this kind.
+
+ For this to work, the model.__meta__ needs to include a tuple with
+ the index to be used for listing purposes, and which is the field to be
+ used to query the index.
+
+ Note that this method only supports indexes of a single field at the
+ moment. It also might be too expensive to return all the documents
+ matching the query, so handle with care.
+
+ class __meta__(object):
+ index = "name"
+ list_index = ("by-type", "type_")
+
+ :return: a deferred that will be fired with an iterable containing
+ as many SoledadDocumentWrapper are matching the index defined
+ in the model as the `list_index`.
+ :rtype: Deferred
+ """
+ # TODO LIST (get_all)
+ # [ ] extend support to indexes with n-ples
+ # [ ] benchmark the cost of querying and returning indexes in a big
+ # database. This might badly need pagination before being put to
+ # serious use.
+ return cls._get_klass_lock().run(cls._get_all, store)
+
+ @classmethod
+ def _get_all(cls, store):
+ try:
+ list_index, list_attr = cls.model.__meta__.list_index
+ except AttributeError:
+ raise RuntimeError("The model is badly defined: no list_index")
+ try:
+ index_value = getattr(cls.model, list_attr)
+ except AttributeError:
+ raise RuntimeError("The model is badly defined: "
+ "no attribute matching list_index")
+
+ def wrap_docs(docs):
+ return (cls(doc_id=doc.doc_id, **doc.content) for doc in docs)
+
+ d = store.get_from_index(list_index, index_value)
+ d.addCallback(wrap_docs)
+ return d
+
+ def __repr__(self):
+ try:
+ idx = getattr(self, self.model.__meta__.index)
+ except AttributeError:
+ idx = ""
+ return "<%s: %s (%s)>" % (self.__class__.__name__,
+ idx, self._doc_id)
+
+
+#
+# Message documents
+#
+
+class FlagsDocWrapper(SoledadDocumentWrapper):
+
+ class model(models.SerializableModel):
+ type_ = "flags"
+ chash = ""
+
+ mbox_uuid = ""
+ seen = False
+ deleted = False
+ recent = False
+ flags = []
+ tags = []
+ size = 0
+ multi = False
+
+ class __meta__(object):
+ index = "mbox"
+
+ def set_mbox_uuid(self, mbox_uuid):
+ # XXX raise error if already created, should use copy instead
+ mbox_uuid = mbox_uuid.replace('-', '_')
+ new_id = constants.FDOCID.format(mbox_uuid=mbox_uuid, chash=self.chash)
+ self._future_doc_id = new_id
+ self.mbox_uuid = mbox_uuid
+
+ def get_flags(self):
+ """
+ Get the flags for this message (as a tuple of strings, not unicode).
+ """
+ return map(str, self.flags)
+
+
+class HeaderDocWrapper(SoledadDocumentWrapper):
+
+ class model(models.SerializableModel):
+ type_ = "head"
+ chash = ""
+
+ date = ""
+ subject = ""
+ headers = {}
+ part_map = {}
+ body = "" # link to phash of body
+ msgid = ""
+ multi = False
+
+ class __meta__(object):
+ index = "chash"
+
+
+class ContentDocWrapper(SoledadDocumentWrapper):
+
+ class model(models.SerializableModel):
+ type_ = "cnt"
+ phash = ""
+
+ ctype = "" # XXX index by ctype too?
+ lkf = [] # XXX not implemented yet!
+ raw = ""
+
+ content_disposition = ""
+ content_transfer_encoding = ""
+ content_type = ""
+
+ class __meta__(object):
+ index = "phash"
+
+
+class MetaMsgDocWrapper(SoledadDocumentWrapper):
+
+ class model(models.SerializableModel):
+ type_ = "meta"
+ fdoc = ""
+ hdoc = ""
+ cdocs = []
+
+ def set_mbox_uuid(self, mbox_uuid):
+ # XXX raise error if already created, should use copy instead
+ mbox_uuid = mbox_uuid.replace('-', '_')
+ chash = re.findall(constants.FDOCID_CHASH_RE, self.fdoc)[0]
+ new_id = constants.METAMSGID.format(mbox_uuid=mbox_uuid, chash=chash)
+ new_fdoc_id = constants.FDOCID.format(mbox_uuid=mbox_uuid, chash=chash)
+ self._future_doc_id = new_id
+ self.fdoc = new_fdoc_id
+
+
+class MessageWrapper(object):
+
+ # This could benefit of a DeferredLock to create/update all the
+ # documents at the same time maybe, and defend against concurrent updates?
+
+ implements(IMessageWrapper)
+
+ def __init__(self, mdoc, fdoc, hdoc, cdocs=None, is_copy=False):
+ """
+ Need at least a metamsg-document, a flag-document and a header-document
+ to instantiate a MessageWrapper. Content-documents can be retrieved
+ lazily.
+
+ cdocs, if any, should be a dictionary in which the keys are ascending
+ integers, beginning at one, and the values are dictionaries with the
+ content of the content-docs.
+
+ is_copy, if set to True, will only attempt to create mdoc and fdoc
+ (because hdoc and cdocs are supposed to exist already)
+ """
+ self._is_copy = is_copy
+
+ def get_doc_wrapper(doc, cls):
+ if isinstance(doc, SoledadDocument):
+ doc_id = doc.doc_id
+ doc = doc.content
+ else:
+ doc_id = None
+ if not doc:
+ doc = {}
+ return cls(doc_id=doc_id, **doc)
+
+ self.mdoc = get_doc_wrapper(mdoc, MetaMsgDocWrapper)
+
+ self.fdoc = get_doc_wrapper(fdoc, FlagsDocWrapper)
+ self.fdoc.set_future_doc_id(self.mdoc.fdoc)
+
+ self.hdoc = get_doc_wrapper(hdoc, HeaderDocWrapper)
+ self.hdoc.set_future_doc_id(self.mdoc.hdoc)
+
+ if cdocs is None:
+ cdocs = {}
+ cdocs_keys = cdocs.keys()
+ assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1)
+ self.cdocs = dict([
+ (key, get_doc_wrapper(doc, ContentDocWrapper))
+ for (key, doc) in cdocs.items()])
+ for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()):
+ if cdoc.raw == "":
+ log.msg("Empty raw field in cdoc %s" % doc_id)
+ cdoc.set_future_doc_id(doc_id)
+
+ def create(self, store, notify_just_mdoc=False, pending_inserts_dict={}):
+ """
+ Create all the parts for this message in the store.
+
+ :param store: an instance of Soledad
+
+ :param notify_just_mdoc:
+ if set to True, this method will return *only* the deferred
+ corresponding to the creation of the meta-message document.
+ Be warned that in that case there will be no record of failures
+ when creating the other part-documents.
+
+ Otherwise, this method will return a deferred that will wait for
+ the creation of all the part documents.
+
+ Setting this flag to True is mostly a convenient workaround for the
+ fact that massive serial appends will take too much time, and in
+ most of the cases the MUA will only switch to the mailbox where the
+ appends have happened after a certain time, which in most of the
+ times will be enough to have all the queued insert operations
+ finished.
+ :type notify_just_mdoc: bool
+ :param pending_inserts_dict:
+ a dictionary with the pending inserts ids.
+ :type pending_inserts_dict: dict
+
+ :return: a deferred whose callback will be called when either all the
+ part documents have been written, or just the metamsg-doc,
+ depending on the value of the notify_just_mdoc flag
+ :rtype: defer.Deferred
+ """
+ leap_assert(self.cdocs,
+ "Need non empty cdocs to create the "
+ "MessageWrapper documents")
+ leap_assert(self.mdoc.doc_id is None,
+ "Cannot create: mdoc has a doc_id")
+ leap_assert(self.fdoc.doc_id is None,
+ "Cannot create: fdoc has a doc_id")
+
+ def unblock_pending_insert(result):
+ if pending_inserts_dict:
+ ci_headers = lowerdict(self.hdoc.headers)
+ msgid = ci_headers.get('message-id', None)
+ try:
+ d = pending_inserts_dict[msgid]
+ d.callback(msgid)
+ except KeyError:
+ pass
+ return result
+
+ # TODO check that the doc_ids in the mdoc are coherent
+ self.d = []
+
+ mdoc_created = self.mdoc.create(store, is_copy=self._is_copy)
+ fdoc_created = self.fdoc.create(store, is_copy=self._is_copy)
+
+ mdoc_created.addErrback(lambda f: log.err(f))
+ fdoc_created.addErrback(lambda f: log.err(f))
+
+ self.d.append(mdoc_created)
+ self.d.append(fdoc_created)
+
+ if not self._is_copy:
+ if self.hdoc.doc_id is None:
+ self.d.append(self.hdoc.create(store))
+ for cdoc in self.cdocs.values():
+ if cdoc.doc_id is not None:
+ # we could be just linking to an existing
+ # content-doc.
+ continue
+ self.d.append(cdoc.create(store))
+
+ def log_all_inserted(result):
+ log.msg("All parts inserted for msg!")
+ return result
+
+ self.all_inserted_d = defer.gatherResults(self.d, consumeErrors=True)
+ self.all_inserted_d.addCallback(log_all_inserted)
+ self.all_inserted_d.addCallback(unblock_pending_insert)
+ self.all_inserted_d.addErrback(lambda failure: log.err(failure))
+
+ if notify_just_mdoc:
+ return mdoc_created
+ else:
+ return self.all_inserted_d
+
+ def update(self, store):
+ """
+ Update the only mutable parts, which are within the flags document.
+ """
+ return self.fdoc.update(store)
+
+ def delete(self, store):
+ # TODO
+ # Eventually this would have to do the duplicate search or send for the
+ # garbage collector. At least mdoc and t the mdoc and fdoc can be
+ # unlinked.
+ d = []
+ if self.mdoc.doc_id:
+ d.append(self.mdoc.delete(store))
+ d.append(self.fdoc.delete(store))
+ return defer.gatherResults(d)
+
+ def copy(self, store, new_mbox_uuid):
+ """
+ Return a copy of this MessageWrapper in a new mailbox.
+
+ :param store: an instance of Soledad, or anything that behaves alike.
+ :param new_mbox_uuid: the uuid of the mailbox where we are copying this
+ message to.
+ :type new_mbox_uuid: str
+ :rtype: MessageWrapper
+ """
+ new_mdoc = self.mdoc.serialize()
+ new_fdoc = self.fdoc.serialize()
+
+ # the future doc_ids is properly set because we modified
+ # the pointers in mdoc, which has precedence.
+ new_wrapper = MessageWrapper(new_mdoc, new_fdoc, None, None,
+ is_copy=True)
+ new_wrapper.hdoc = self.hdoc
+ new_wrapper.cdocs = self.cdocs
+ new_wrapper.set_mbox_uuid(new_mbox_uuid)
+
+ # XXX could flag so that it only creates mdoc/fdoc...
+
+ d = new_wrapper.create(store)
+ d.addCallback(lambda result: new_wrapper)
+ d.addErrback(lambda failure: log.err(failure))
+ return d
+
+ def set_mbox_uuid(self, mbox_uuid):
+ """
+ Set the mailbox for this wrapper.
+ This method should only be used before the Documents for the
+ MessageWrapper have been created, will raise otherwise.
+ """
+ mbox_uuid = mbox_uuid.replace('-', '_')
+ self.mdoc.set_mbox_uuid(mbox_uuid)
+ self.fdoc.set_mbox_uuid(mbox_uuid)
+
+ def set_flags(self, flags):
+ # TODO serialize the get + update
+ if flags is None:
+ flags = tuple()
+ leap_assert_type(flags, tuple)
+ self.fdoc.flags = list(flags)
+ self.fdoc.deleted = "\\Deleted" in flags
+ self.fdoc.seen = "\\Seen" in flags
+ self.fdoc.recent = "\\Recent" in flags
+
+ def set_tags(self, tags):
+ # TODO serialize the get + update
+ if tags is None:
+ tags = tuple()
+ leap_assert_type(tags, tuple)
+ self.fdoc.tags = list(tags)
+
+ def set_date(self, date):
+ # XXX assert valid date format
+ self.hdoc.date = date
+
+ def get_subpart_dict(self, index):
+ """
+ :param index: the part to lookup, 1-indexed
+ :type index: int
+ :rtype: dict
+ """
+ return self.hdoc.part_map[str(index)]
+
+ def get_subpart_indexes(self):
+ return self.hdoc.part_map.keys()
+
+ def get_body(self, store):
+ """
+ :rtype: deferred
+ """
+ body_phash = self.hdoc.body
+ if not body_phash:
+ if self.cdocs:
+ return self.cdocs[1]
+ d = store.get_doc('C-' + body_phash)
+ d.addCallback(lambda doc: ContentDocWrapper(**doc.content))
+ return d
+
+
+#
+# Mailboxes
+#
+
+
+class MailboxWrapper(SoledadDocumentWrapper):
+
+ class model(models.SerializableModel):
+ type_ = "mbox"
+ mbox = INBOX_NAME
+ uuid = None
+ flags = []
+ recent = []
+ created = 1
+ closed = False
+ subscribed = False
+
+ class __meta__(object):
+ index = "mbox"
+ list_index = (indexes.TYPE_IDX, 'type_')
+
+
+#
+# Soledad Adaptor
+#
+
+class SoledadIndexMixin(object):
+ """
+ This will need a class attribute `indexes`, that is a dictionary containing
+ the index definitions for the underlying u1db store underlying soledad.
+
+ It needs to be in the following format:
+ {'index-name': ['field1', 'field2']}
+
+ You can also add a class attribute `wait_for_indexes` to any class
+ inheriting from this Mixin, that should be a list of strings representing
+ the methods that need to wait until the indexes have been initialized
+ before being able to work properly.
+ """
+ # TODO move this mixin to soledad itself
+ # so that each application can pass a set of indexes for their data model.
+
+ # TODO could have a wrapper class for indexes, supporting introspection
+ # and __getattr__
+
+ # TODO make this an interface?
+
+ indexes = {}
+ wait_for_indexes = []
+ store_ready = False
+
+ def initialize_store(self, store):
+ """
+ Initialize the indexes in the database.
+
+ :param store: store
+ :returns: a Deferred that will fire when the store is correctly
+ initialized.
+ :rtype: deferred
+ """
+ # TODO I think we *should* get another deferredLock in here, but
+ # global to the soledad namespace, to protect from several points
+ # initializing soledad indexes at the same time.
+ self._wait_for_indexes()
+
+ d = self._init_indexes(store)
+ d.addCallback(self._restore_waiting_methods)
+ return d
+
+ def _init_indexes(self, store):
+ """
+ Initialize the database indexes.
+ """
+ leap_assert(store, "Cannot init indexes with null soledad")
+ leap_assert_type(self.indexes, dict)
+
+ def _create_index(name, expression):
+ return store.create_index(name, *expression)
+
+ def init_idexes(indexes):
+ deferreds = []
+ db_indexes = dict(indexes)
+ # Loop through the indexes we expect to find.
+ for name, expression in self.indexes.items():
+ if name not in db_indexes:
+ # The index does not yet exist.
+ d = _create_index(name, expression)
+ deferreds.append(d)
+ elif expression != db_indexes[name]:
+ # The index exists but the definition is not what expected,
+ # so we delete it and add the proper index expression.
+ d = store.delete_index(name)
+ d.addCallback(
+ lambda _: _create_index(name, *expression))
+ deferreds.append(d)
+ return defer.gatherResults(deferreds, consumeErrors=True)
+
+ def store_ready(whatever):
+ self.store_ready = True
+ return whatever
+
+ self.deferred_indexes = store.list_indexes()
+ self.deferred_indexes.addCallback(init_idexes)
+ self.deferred_indexes.addCallback(store_ready)
+ return self.deferred_indexes
+
+ def _wait_for_indexes(self):
+ """
+ Make the marked methods to wait for the indexes to be ready.
+ Heavily based on
+ http://blogs.fluidinfo.com/terry/2009/05/11/a-mixin-class-allowing-python-__init__-methods-to-work-with-twisted-deferreds/
+
+ :param methods: methods that need to wait for the indexes to be ready
+ :type methods: tuple(str)
+ """
+ leap_assert_type(self.wait_for_indexes, list)
+ methods = self.wait_for_indexes
+
+ self.waiting = []
+ self.stored = {}
+
+ def makeWrapper(method):
+ def wrapper(*args, **kw):
+ d = defer.Deferred()
+ d.addCallback(lambda _: self.stored[method](*args, **kw))
+ self.waiting.append(d)
+ return d
+ return wrapper
+
+ for method in methods:
+ self.stored[method] = getattr(self, method)
+ setattr(self, method, makeWrapper(method))
+
+ def _restore_waiting_methods(self, _):
+ for method in self.stored:
+ setattr(self, method, self.stored[method])
+ for d in self.waiting:
+ d.callback(None)
+
+
+class SoledadMailAdaptor(SoledadIndexMixin):
+
+ implements(IMailAdaptor)
+ store = None
+
+ indexes = indexes.MAIL_INDEXES
+ wait_for_indexes = ['get_or_create_mbox', 'update_mbox', 'get_all_mboxes']
+
+ mboxwrapper_klass = MailboxWrapper
+
+ def __init__(self):
+ SoledadIndexMixin.__init__(self)
+
+ # Message handling
+
+ def get_msg_from_string(self, MessageClass, raw_msg):
+ """
+ Get an instance of a MessageClass initialized with a MessageWrapper
+ that contains all the parts obtained from parsing the raw string for
+ the message.
+
+ :param MessageClass: any Message class that can be initialized passing
+ an instance of an IMessageWrapper implementor.
+ :type MessageClass: type
+ :param raw_msg: a string containing the raw email message.
+ :type raw_msg: str
+ :rtype: MessageClass instance.
+ """
+ assert(MessageClass is not None)
+ mdoc, fdoc, hdoc, cdocs = _split_into_parts(raw_msg)
+ return self.get_msg_from_docs(
+ MessageClass, mdoc, fdoc, hdoc, cdocs)
+
+ def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None,
+ uid=None):
+ """
+ Get an instance of a MessageClass initialized with a MessageWrapper
+ that contains the passed part documents.
+
+ This is not the recommended way of obtaining a message, unless you know
+ how to take care of ensuring the internal consistency between the part
+ documents, or unless you are glueing together the part documents that
+ have been previously generated by `get_msg_from_string`.
+
+ :param MessageClass: any Message class that can be initialized passing
+ an instance of an IMessageWrapper implementor.
+ :type MessageClass: type
+ :param fdoc: a dictionary containing values from which a
+ FlagsDocWrapper can be initialized
+ :type fdoc: dict
+ :param hdoc: a dictionary containing values from which a
+ HeaderDocWrapper can be initialized
+ :type hdoc: dict
+ :param cdocs: None, or a dictionary mapping integers (1-indexed) to
+ dicts from where a ContentDocWrapper can be initialized.
+ :type cdocs: dict, or None
+
+ :rtype: MessageClass instance.
+ """
+ assert(MessageClass is not None)
+ return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs), uid=uid)
+
+ def get_msg_from_mdoc_id(self, MessageClass, store, mdoc_id,
+ uid=None, get_cdocs=False):
+
+ def wrap_meta_doc(doc):
+ cls = MetaMsgDocWrapper
+ return cls(doc_id=doc.doc_id, **doc.content)
+
+ def get_part_docs_from_mdoc_wrapper(wrapper):
+ d_docs = []
+ d_docs.append(store.get_doc(wrapper.fdoc))
+ d_docs.append(store.get_doc(wrapper.hdoc))
+ for cdoc in wrapper.cdocs:
+ d_docs.append(store.get_doc(cdoc))
+
+ def add_mdoc(doc_list):
+ return [wrapper.serialize()] + doc_list
+
+ d = defer.gatherResults(d_docs)
+ d.addCallback(add_mdoc)
+ return d
+
+ def get_parts_doc_from_mdoc_id():
+ mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0]
+ chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0]
+
+ def _get_fdoc_id_from_mdoc_id():
+ return constants.FDOCID.format(mbox_uuid=mbox, chash=chash)
+
+ def _get_hdoc_id_from_mdoc_id():
+ return constants.HDOCID.format(mbox_uuid=mbox, chash=chash)
+
+ d_docs = []
+ fdoc_id = _get_fdoc_id_from_mdoc_id()
+ hdoc_id = _get_hdoc_id_from_mdoc_id()
+
+ d_docs.append(store.get_doc(mdoc_id))
+ d_docs.append(store.get_doc(fdoc_id))
+ d_docs.append(store.get_doc(hdoc_id))
+
+ d = defer.gatherResults(d_docs)
+ return d
+
+ if get_cdocs:
+ d = store.get_doc(mdoc_id)
+ d.addCallback(wrap_meta_doc)
+ d.addCallback(get_part_docs_from_mdoc_wrapper)
+ else:
+ d = get_parts_doc_from_mdoc_id()
+
+ d.addCallback(self._get_msg_from_variable_doc_list,
+ msg_class=MessageClass, uid=uid)
+ return d
+
+ def _get_msg_from_variable_doc_list(self, doc_list, msg_class, uid=None):
+ if len(doc_list) == 3:
+ mdoc, fdoc, hdoc = doc_list
+ cdocs = None
+ elif len(doc_list) > 3:
+ # XXX is this case used?
+ mdoc, fdoc, hdoc = doc_list[:3]
+ cdocs = dict(enumerate(doc_list[3:], 1))
+ return self.get_msg_from_docs(
+ msg_class, mdoc, fdoc, hdoc, cdocs, uid=uid)
+
+ def get_flags_from_mdoc_id(self, store, mdoc_id):
+ """
+ # XXX stuff here...
+ """
+ mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0]
+ chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0]
+
+ def _get_fdoc_id_from_mdoc_id():
+ return constants.FDOCID.format(mbox_uuid=mbox, chash=chash)
+
+ fdoc_id = _get_fdoc_id_from_mdoc_id()
+
+ def wrap_fdoc(doc):
+ if not doc:
+ return
+ cls = FlagsDocWrapper
+ return cls(doc_id=doc.doc_id, **doc.content)
+
+ def get_flags(fdoc_wrapper):
+ if not fdoc_wrapper:
+ return []
+ return fdoc_wrapper.get_flags()
+
+ d = store.get_doc(fdoc_id)
+ d.addCallback(wrap_fdoc)
+ d.addCallback(get_flags)
+ return d
+
+ def create_msg(self, store, msg):
+ """
+ :param store: an instance of soledad, or anything that behaves alike
+ :param msg: a Message object.
+
+ :return: a Deferred that is fired when all the underlying documents
+ have been created.
+ :rtype: defer.Deferred
+ """
+ wrapper = msg.get_wrapper()
+ return wrapper.create(store)
+
+ def update_msg(self, store, msg):
+ """
+ :param msg: a Message object.
+ :param store: an instance of soledad, or anything that behaves alike
+ :return: a Deferred that is fired when all the underlying documents
+ have been updated (actually, it's only the fdoc that's allowed
+ to update).
+ :rtype: defer.Deferred
+ """
+ wrapper = msg.get_wrapper()
+ return wrapper.update(store)
+
+ # batch deletion
+
+ def del_all_flagged_messages(self, store, mbox_uuid):
+ """
+ Delete all messages flagged as deleted.
+ """
+ def err(failure):
+ log.err(failure)
+
+ def delete_fdoc_and_mdoc_flagged(fdocs):
+ # low level here, not using the wrappers...
+ # get meta doc ids from the flag doc ids
+ fdoc_ids = [doc.doc_id for doc in fdocs]
+ mdoc_ids = map(lambda s: "M" + s[1:], fdoc_ids)
+
+ def delete_all_docs(mdocs, fdocs):
+ mdocs = list(mdocs)
+ doc_ids = [m.doc_id for m in mdocs]
+ _d = []
+ docs = mdocs + fdocs
+ for doc in docs:
+ _d.append(store.delete_doc(doc))
+ d = defer.gatherResults(_d)
+ # return the mdocs ids only
+ d.addCallback(lambda _: doc_ids)
+ return d
+
+ d = store.get_docs(mdoc_ids)
+ d.addCallback(delete_all_docs, fdocs)
+ d.addErrback(err)
+ return d
+
+ type_ = FlagsDocWrapper.model.type_
+ uuid = mbox_uuid.replace('-', '_')
+ deleted_index = indexes.TYPE_MBOX_DEL_IDX
+
+ d = store.get_from_index(deleted_index, type_, uuid, "1")
+ d.addCallbacks(delete_fdoc_and_mdoc_flagged, err)
+ return d
+
+ # count messages
+
+ def get_count_unseen(self, store, mbox_uuid):
+ """
+ Get the number of unseen messages for a given mailbox.
+
+ :param store: instance of Soledad.
+ :param mbox_uuid: the uuid for this mailbox.
+ :rtype: int
+ """
+ type_ = FlagsDocWrapper.model.type_
+ uuid = mbox_uuid.replace('-', '_')
+
+ unseen_index = indexes.TYPE_MBOX_SEEN_IDX
+
+ d = store.get_count_from_index(unseen_index, type_, uuid, "0")
+ d.addErrback(self._errback)
+ return d
+
+ def get_count_recent(self, store, mbox_uuid):
+ """
+ Get the number of recent messages for a given mailbox.
+
+ :param store: instance of Soledad.
+ :param mbox_uuid: the uuid for this mailbox.
+ :rtype: int
+ """
+ type_ = FlagsDocWrapper.model.type_
+ uuid = mbox_uuid.replace('-', '_')
+
+ recent_index = indexes.TYPE_MBOX_RECENT_IDX
+
+ d = store.get_count_from_index(recent_index, type_, uuid, "1")
+ d.addErrback(self._errback)
+ return d
+
+ # search api
+
+ def get_mdoc_id_from_msgid(self, store, mbox_uuid, msgid):
+ """
+ Get the UID for a message with the passed msgid (the one in the headers
+ msg-id).
+ This is used by the MUA to retrieve the recently saved draft.
+ """
+ type_ = HeaderDocWrapper.model.type_
+ uuid = mbox_uuid.replace('-', '_')
+
+ msgid_index = indexes.TYPE_MSGID_IDX
+
+ def get_mdoc_id(hdoc):
+ if not hdoc:
+ log.msg("Could not find a HDOC with MSGID %s" % msgid)
+ return None
+ hdoc = hdoc[0]
+ mdoc_id = hdoc.doc_id.replace("H-", "M-%s-" % uuid)
+ return mdoc_id
+
+ d = store.get_from_index(msgid_index, type_, msgid)
+ d.addCallback(get_mdoc_id)
+ return d
+
+ # Mailbox handling
+
+ def get_or_create_mbox(self, store, name):
+ """
+ Get the mailbox with the given name, or create one if it does not
+ exist.
+
+ :param store: instance of Soledad
+ :param name: the name of the mailbox
+ :type name: str
+ """
+ index = indexes.TYPE_MBOX_IDX
+ mbox = normalize_mailbox(name)
+ return MailboxWrapper.get_or_create(store, index, mbox)
+
+ def update_mbox(self, store, mbox_wrapper):
+ """
+ Update the documents for a given mailbox.
+ :param mbox_wrapper: MailboxWrapper instance
+ :type mbox_wrapper: MailboxWrapper
+ :return: a Deferred that will be fired when the mailbox documents
+ have been updated.
+ :rtype: defer.Deferred
+ """
+ leap_assert_type(mbox_wrapper, SoledadDocumentWrapper)
+ return mbox_wrapper.update(store)
+
+ def delete_mbox(self, store, mbox_wrapper):
+ leap_assert_type(mbox_wrapper, SoledadDocumentWrapper)
+ return mbox_wrapper.delete(store)
+
+ def get_all_mboxes(self, store):
+ """
+ Retrieve a list with wrappers for all the mailboxes.
+
+ :return: a deferred that will be fired with a list of all the
+ MailboxWrappers found.
+ :rtype: defer.Deferred
+ """
+ return MailboxWrapper.get_all(store)
+
+ def _errback(self, failure):
+ log.err(failure)
+
+
+def _split_into_parts(raw):
+ # TODO signal that we can delete the original message!-----
+ # when all the processing is done.
+ # TODO add the linked-from info !
+ # TODO add reference to the original message?
+ # TODO populate Default FLAGS/TAGS (unseen?)
+ # TODO seed propely the content_docs with defaults??
+
+ msg, parts, chash, multi = _parse_msg(raw)
+ size = len(msg.as_string())
+
+ body_phash = walk.get_body_phash(msg)
+
+ parts_map = walk.walk_msg_tree(parts, body_phash=body_phash)
+ cdocs_list = list(walk.get_raw_docs(msg, parts))
+ cdocs_phashes = [c['phash'] for c in cdocs_list]
+
+ mdoc = _build_meta_doc(chash, cdocs_phashes)
+ fdoc = _build_flags_doc(chash, size, multi)
+ hdoc = _build_headers_doc(msg, chash, body_phash, parts_map)
+
+ # The MessageWrapper expects a dict, one-indexed
+ cdocs = dict(enumerate(cdocs_list, 1))
+
+ return mdoc, fdoc, hdoc, cdocs
+
+
+def _parse_msg(raw):
+ msg = message_from_string(raw)
+ parts = walk.get_parts(msg)
+ chash = sha256.SHA256(raw).hexdigest()
+ multi = msg.is_multipart()
+ return msg, parts, chash, multi
+
+
+def _build_meta_doc(chash, cdocs_phashes):
+ _mdoc = MetaMsgDocWrapper()
+ # FIXME passing the inbox name because we don't have the uuid at this
+ # point.
+
+ _mdoc.fdoc = constants.FDOCID.format(mbox_uuid=INBOX_NAME, chash=chash)
+ _mdoc.hdoc = constants.HDOCID.format(chash=chash)
+ _mdoc.cdocs = [constants.CDOCID.format(phash=p) for p in cdocs_phashes]
+ return _mdoc.serialize()
+
+
+def _build_flags_doc(chash, size, multi):
+ _fdoc = FlagsDocWrapper(chash=chash, size=size, multi=multi)
+ return _fdoc.serialize()
+
+
+def _build_headers_doc(msg, chash, body_phash, parts_map):
+ """
+ Assemble a headers document from the original parsed message, the
+ content-hash, and the parts map.
+
+ It takes into account possibly repeated headers.
+ """
+ headers = defaultdict(list)
+ for k, v in msg.items():
+ headers[k].append(v)
+ # "fix" for repeated headers (as in "Received:"
+ for k, v in headers.items():
+ newline = "\n%s: " % (k.lower(),)
+ headers[k] = newline.join(v)
+
+ lower_headers = lowerdict(dict(headers))
+ msgid = first(_MSGID_RE.findall(
+ lower_headers.get('message-id', '')))
+
+ _hdoc = HeaderDocWrapper(
+ chash=chash, headers=headers, body=body_phash,
+ msgid=msgid)
+
+ def copy_attr(headers, key, doc):
+ if key in headers:
+ setattr(doc, key, headers[key])
+
+ copy_attr(lower_headers, "subject", _hdoc)
+ copy_attr(lower_headers, "date", _hdoc)
+
+ hdoc = _hdoc.serialize()
+ # add parts map to header doc
+ # (body, multi, part_map)
+ for key in parts_map:
+ hdoc[key] = parts_map[key]
+ return stringify_parts_map(hdoc)
diff --git a/src/leap/mail/adaptors/soledad_indexes.py b/src/leap/mail/adaptors/soledad_indexes.py
new file mode 100644
index 0000000..eec7d28
--- /dev/null
+++ b/src/leap/mail/adaptors/soledad_indexes.py
@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+# soledad_indexes.py
+# Copyright (C) 2013, 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledad Indexes for Mail Documents.
+"""
+
+# TODO
+# [ ] hide most of the constants here
+
+# Document Type, for indexing
+
+TYPE = "type"
+MBOX = "mbox"
+MBOX_UUID = "mbox_uuid"
+FLAGS = "flags"
+HEADERS = "head"
+CONTENT = "cnt"
+RECENT = "rct"
+HDOCS_SET = "hdocset"
+
+INCOMING_KEY = "incoming"
+ERROR_DECRYPTING_KEY = "errdecr"
+
+# indexing keys
+CONTENT_HASH = "chash"
+PAYLOAD_HASH = "phash"
+MSGID = "msgid"
+UID = "uid"
+
+
+# Index types
+# --------------
+
+TYPE_IDX = 'by-type'
+TYPE_MBOX_IDX = 'by-type-and-mbox'
+TYPE_MBOX_UUID_IDX = 'by-type-and-mbox-uuid'
+TYPE_SUBS_IDX = 'by-type-and-subscribed'
+TYPE_MSGID_IDX = 'by-type-and-message-id'
+TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
+TYPE_MBOX_RECENT_IDX = 'by-type-and-mbox-and-recent'
+TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted'
+TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash'
+TYPE_C_HASH_IDX = 'by-type-and-contenthash'
+TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber'
+TYPE_P_HASH_IDX = 'by-type-and-payloadhash'
+
+# Soledad index for incoming mail, without decrypting errors.
+# and the backward-compatible index, will be deprecated at 0.7
+JUST_MAIL_IDX = "just-mail"
+JUST_MAIL_COMPAT_IDX = "just-mail-compat"
+
+
+# TODO
+# it would be nice to measure the cost of indexing
+# by many fields.
+
+# TODO
+# make the indexes dict more readable!
+
+MAIL_INDEXES = {
+ # generic
+ TYPE_IDX: [TYPE],
+ TYPE_MBOX_IDX: [TYPE, MBOX],
+ TYPE_MBOX_UUID_IDX: [TYPE, MBOX_UUID],
+
+ # XXX deprecate 0.4.0
+ # TYPE_MBOX_UID_IDX: [TYPE, MBOX, UID],
+
+ # mailboxes
+ TYPE_SUBS_IDX: [TYPE, 'bool(subscribed)'],
+
+ # fdocs uniqueness
+ TYPE_MBOX_C_HASH_IDX: [TYPE, MBOX, CONTENT_HASH],
+
+ # headers doc - search by msgid.
+ TYPE_MSGID_IDX: [TYPE, MSGID],
+
+ # content, headers doc
+ TYPE_C_HASH_IDX: [TYPE, CONTENT_HASH],
+
+ # attachment payload dedup
+ TYPE_P_HASH_IDX: [TYPE, PAYLOAD_HASH],
+
+ # messages
+ TYPE_MBOX_SEEN_IDX: [TYPE, MBOX_UUID, 'bool(seen)'],
+ TYPE_MBOX_RECENT_IDX: [TYPE, MBOX_UUID, 'bool(recent)'],
+ TYPE_MBOX_DEL_IDX: [TYPE, MBOX_UUID, 'bool(deleted)'],
+
+ # incoming queue
+ JUST_MAIL_IDX: ["bool(%s)" % (INCOMING_KEY,),
+ "bool(%s)" % (ERROR_DECRYPTING_KEY,)],
+}
diff --git a/src/leap/mail/adaptors/tests/__init__.py b/src/leap/mail/adaptors/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/leap/mail/adaptors/tests/__init__.py
diff --git a/src/leap/mail/adaptors/tests/rfc822.message b/src/leap/mail/adaptors/tests/rfc822.message
new file mode 120000
index 0000000..b19cc28
--- /dev/null
+++ b/src/leap/mail/adaptors/tests/rfc822.message
@@ -0,0 +1 @@
+../../tests/rfc822.message \ No newline at end of file
diff --git a/src/leap/mail/adaptors/tests/test_models.py b/src/leap/mail/adaptors/tests/test_models.py
new file mode 100644
index 0000000..b82cfad
--- /dev/null
+++ b/src/leap/mail/adaptors/tests/test_models.py
@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+# test_models.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the leap.mail.adaptors.models module.
+"""
+from twisted.trial import unittest
+
+from leap.mail.adaptors import models
+
+
+class SerializableModelsTestCase(unittest.TestCase):
+
+ def test_good_serialized_model(self):
+
+ class M(models.SerializableModel):
+ foo = 42
+ bar = 33
+ baaz_ = None
+ _nope = 0
+ __nope = 0
+
+ def not_today(self):
+ pass
+
+ class IgnoreMe(object):
+ pass
+
+ def killmeplease(x):
+ return x
+
+ serialized = M.serialize()
+ expected = {'foo': 42, 'bar': 33, 'baaz': None}
+ self.assertEqual(serialized, expected)
+
+
+class DocumentWrapperTestCase(unittest.TestCase):
+
+ def test_wrapper_defaults(self):
+
+ class Wrapper(models.DocumentWrapper):
+ class model(models.SerializableModel):
+ foo = 42
+ bar = 11
+
+ wrapper = Wrapper()
+ wrapper._ignored = True
+ serialized = wrapper.serialize()
+ expected = {'foo': 42, 'bar': 11}
+ self.assertEqual(serialized, expected)
+
+ def test_initialized_wrapper(self):
+
+ class Wrapper(models.DocumentWrapper):
+ class model(models.SerializableModel):
+ foo = 42
+ bar_ = 11
+
+ wrapper = Wrapper(foo=0, bar=-1)
+ serialized = wrapper.serialize()
+ expected = {'foo': 0, 'bar': -1}
+ self.assertEqual(serialized, expected)
+
+ wrapper.foo = 23
+ serialized = wrapper.serialize()
+ expected = {'foo': 23, 'bar': -1}
+ self.assertEqual(serialized, expected)
+
+ wrapper = Wrapper(foo=0)
+ serialized = wrapper.serialize()
+ expected = {'foo': 0, 'bar': 11}
+ self.assertEqual(serialized, expected)
+
+ def test_invalid_initialized_wrapper(self):
+
+ class Wrapper(models.DocumentWrapper):
+ class model(models.SerializableModel):
+ foo = 42
+
+ def getwrapper():
+ return Wrapper(bar=1)
+ self.assertRaises(RuntimeError, getwrapper)
+
+ def test_no_model_wrapper(self):
+
+ class Wrapper(models.DocumentWrapper):
+ pass
+
+ def getwrapper():
+ w = Wrapper()
+ w.foo = None
+
+ self.assertRaises(RuntimeError, getwrapper)
diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py
new file mode 100644
index 0000000..61e387c
--- /dev/null
+++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py
@@ -0,0 +1,529 @@
+# -*- coding: utf-8 -*-
+# test_soledad_adaptor.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the Soledad Adaptor module - leap.mail.adaptors.soledad
+"""
+import os
+from functools import partial
+
+from twisted.internet import defer
+
+from leap.mail.adaptors import models
+from leap.mail.adaptors.soledad import SoledadDocumentWrapper
+from leap.mail.adaptors.soledad import SoledadIndexMixin
+from leap.mail.adaptors.soledad import SoledadMailAdaptor
+from leap.mail.tests.common import SoledadTestMixin
+
+from email.MIMEMultipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+# DEBUG
+# import logging
+# logging.basicConfig(level=logging.DEBUG)
+
+
+class CounterWrapper(SoledadDocumentWrapper):
+ class model(models.SerializableModel):
+ counter = 0
+ flag = None
+
+
+class CharacterWrapper(SoledadDocumentWrapper):
+ class model(models.SerializableModel):
+ name = ""
+ age = 20
+
+
+class ActorWrapper(SoledadDocumentWrapper):
+ class model(models.SerializableModel):
+ type_ = "actor"
+ name = None
+
+ class __meta__(object):
+ index = "name"
+ list_index = ("by-type", "type_")
+
+
+class TestAdaptor(SoledadIndexMixin):
+ indexes = {'by-name': ['name'],
+ 'by-type-and-name': ['type', 'name'],
+ 'by-type': ['type']}
+
+
+class SoledadDocWrapperTestCase(SoledadTestMixin):
+ """
+ Tests for the SoledadDocumentWrapper.
+ """
+ def assert_num_docs(self, num, docs):
+ self.assertEqual(len(docs[1]), num)
+
+ def test_create_single(self):
+
+ store = self._soledad
+ wrapper = CounterWrapper()
+
+ def assert_one_doc(docs):
+ self.assertEqual(docs[0], 1)
+
+ d = wrapper.create(store)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(assert_one_doc)
+ return d
+
+ def test_create_many(self):
+
+ store = self._soledad
+ w1 = CounterWrapper()
+ w2 = CounterWrapper(counter=1)
+ w3 = CounterWrapper(counter=2)
+ w4 = CounterWrapper(counter=3)
+ w5 = CounterWrapper(counter=4)
+
+ d1 = [w1.create(store),
+ w2.create(store),
+ w3.create(store),
+ w4.create(store),
+ w5.create(store)]
+
+ d = defer.gatherResults(d1)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 5))
+ return d
+
+ def test_multiple_updates(self):
+
+ store = self._soledad
+ wrapper = CounterWrapper(counter=1)
+ MAX = 100
+
+ def assert_doc_id(doc):
+ self.assertTrue(wrapper._doc_id is not None)
+ return doc
+
+ def assert_counter_initial_ok(doc):
+ self.assertEqual(wrapper.counter, 1)
+
+ def increment_counter(ignored):
+ d1 = []
+
+ def record_revision(revision):
+ rev = int(revision.split(':')[1])
+ self.results.append(rev)
+
+ for i in list(range(MAX)):
+ wrapper.counter += 1
+ wrapper.flag = i % 2 == 0
+ d = wrapper.update(store)
+ d.addCallback(record_revision)
+ d1.append(d)
+
+ return defer.gatherResults(d1)
+
+ def assert_counter_final_ok(doc):
+ self.assertEqual(doc.content['counter'], MAX + 1)
+ self.assertEqual(doc.content['flag'], False)
+
+ def assert_results_ordered_list(ignored):
+ self.assertEqual(self.results, sorted(range(2, MAX + 2)))
+
+ d = wrapper.create(store)
+ d.addCallback(assert_doc_id)
+ d.addCallback(assert_counter_initial_ok)
+ d.addCallback(increment_counter)
+ d.addCallback(lambda _: store.get_doc(wrapper._doc_id))
+ d.addCallback(assert_counter_final_ok)
+ d.addCallback(assert_results_ordered_list)
+ return d
+
+ def test_delete(self):
+ adaptor = TestAdaptor()
+ store = self._soledad
+
+ wrapper_list = []
+
+ def get_or_create_bob(ignored):
+ def add_to_list(wrapper):
+ wrapper_list.append(wrapper)
+ return wrapper
+ wrapper = CharacterWrapper.get_or_create(
+ store, 'by-name', 'bob')
+ wrapper.addCallback(add_to_list)
+ return wrapper
+
+ def delete_bob(ignored):
+ wrapper = wrapper_list[0]
+ return wrapper.delete(store)
+
+ d = adaptor.initialize_store(store)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 0))
+
+ # this should create bob document
+ d.addCallback(get_or_create_bob)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+
+ d.addCallback(delete_bob)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 0))
+ return d
+
+ def test_get_or_create(self):
+ adaptor = TestAdaptor()
+ store = self._soledad
+
+ def get_or_create_bob(ignored):
+ wrapper = CharacterWrapper.get_or_create(
+ store, 'by-name', 'bob')
+ return wrapper
+
+ d = adaptor.initialize_store(store)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 0))
+
+ # this should create bob document
+ d.addCallback(get_or_create_bob)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+
+ # this should get us bob document
+ d.addCallback(get_or_create_bob)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+ return d
+
+ def test_get_or_create_multi_index(self):
+ adaptor = TestAdaptor()
+ store = self._soledad
+
+ def get_or_create_actor_harry(ignored):
+ wrapper = ActorWrapper.get_or_create(
+ store, 'by-type-and-name', 'harrison')
+ return wrapper
+
+ def create_director_harry(ignored):
+ wrapper = ActorWrapper(name="harrison", type="director")
+ return wrapper.create(store)
+
+ d = adaptor.initialize_store(store)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 0))
+
+ # this should create harrison document
+ d.addCallback(get_or_create_actor_harry)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+
+ # this should get us harrison document
+ d.addCallback(get_or_create_actor_harry)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+
+ # create director harry, should create new doc
+ d.addCallback(create_director_harry)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 2))
+
+ # this should get us harrison document, still 2 docs
+ d.addCallback(get_or_create_actor_harry)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 2))
+ return d
+
+ def test_get_all(self):
+ adaptor = TestAdaptor()
+ store = self._soledad
+ actor_names = ["harry", "carrie", "mark", "david"]
+
+ def create_some_actors(ignored):
+ deferreds = []
+ for name in actor_names:
+ dw = ActorWrapper.get_or_create(
+ store, 'by-type-and-name', name)
+ deferreds.append(dw)
+ return defer.gatherResults(deferreds)
+
+ d = adaptor.initialize_store(store)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 0))
+
+ d.addCallback(create_some_actors)
+
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 4))
+
+ def assert_actor_list_is_expected(res):
+ got = set([actor.name for actor in res])
+ expected = set(actor_names)
+ self.assertEqual(got, expected)
+
+ d.addCallback(lambda _: ActorWrapper.get_all(store))
+ d.addCallback(assert_actor_list_is_expected)
+ return d
+
+HERE = os.path.split(os.path.abspath(__file__))[0]
+
+
+class TestMessageClass(object):
+ def __init__(self, wrapper, uid):
+ self.wrapper = wrapper
+ self.uid = uid
+
+ def get_wrapper(self):
+ return self.wrapper
+
+
+class SoledadMailAdaptorTestCase(SoledadTestMixin):
+ """
+ Tests for the SoledadMailAdaptor.
+ """
+
+ def get_adaptor(self):
+ adaptor = SoledadMailAdaptor()
+ adaptor.store = self._soledad
+ return adaptor
+
+ def assert_num_docs(self, num, docs):
+ self.assertEqual(len(docs[1]), num)
+
+ def test_mail_adaptor_init(self):
+ adaptor = self.get_adaptor()
+ self.assertTrue(isinstance(adaptor.indexes, dict))
+ self.assertTrue(len(adaptor.indexes) != 0)
+
+ # Messages
+
+ def test_get_msg_from_string(self):
+ adaptor = self.get_adaptor()
+
+ with open(os.path.join(HERE, "rfc822.message")) as f:
+ raw = f.read()
+
+ msg = adaptor.get_msg_from_string(TestMessageClass, raw)
+
+ chash = ("D27B2771C0DCCDCB468EE65A4540438"
+ "09DBD11588E87E951545BE0CBC321C308")
+ phash = ("64934534C1C80E0D4FA04BE1CCBA104"
+ "F07BCA5F469C86E2C0ABE1D41310B7299")
+ subject = ("[Twisted-commits] rebuild now works on "
+ "python versions from 2.2.0 and up.")
+ self.assertTrue(msg.wrapper.fdoc is not None)
+ self.assertTrue(msg.wrapper.hdoc is not None)
+ self.assertTrue(msg.wrapper.cdocs is not None)
+ self.assertEquals(len(msg.wrapper.cdocs), 1)
+ self.assertEquals(msg.wrapper.fdoc.chash, chash)
+ self.assertEquals(msg.wrapper.fdoc.size, 3837)
+ self.assertEquals(msg.wrapper.hdoc.chash, chash)
+ self.assertEqual(dict(msg.wrapper.hdoc.headers)['Subject'],
+ subject)
+ self.assertEqual(msg.wrapper.hdoc.subject, subject)
+ self.assertEqual(msg.wrapper.cdocs[1].phash, phash)
+
+ def test_get_msg_from_string_multipart(self):
+ msg = MIMEMultipart()
+ msg['Subject'] = 'Test multipart mail'
+ msg.attach(MIMEText(u'a utf8 message', _charset='utf-8'))
+ adaptor = self.get_adaptor()
+
+ msg = adaptor.get_msg_from_string(TestMessageClass, msg.as_string())
+
+ self.assertEqual(
+ 'base64', msg.wrapper.cdocs[1].content_transfer_encoding)
+ self.assertEqual(
+ 'text/plain; charset="utf-8"', msg.wrapper.cdocs[1].content_type)
+ self.assertEqual(
+ 'YSB1dGY4IG1lc3NhZ2U=\n', msg.wrapper.cdocs[1].raw)
+
+ def test_get_msg_from_docs(self):
+ adaptor = self.get_adaptor()
+ mdoc = dict(
+ fdoc="F-Foobox-deadbeef",
+ hdoc="H-deadbeef",
+ cdocs=["C-deadabad"])
+ fdoc = dict(
+ mbox_uuid="Foobox",
+ flags=('\Seen', '\Nice'),
+ tags=('Personal', 'TODO'),
+ seen=False, deleted=False,
+ recent=False, multi=False)
+ hdoc = dict(
+ chash="deadbeef",
+ subject="Test Msg")
+ cdocs = {
+ 1: dict(
+ raw='This is a test message')}
+
+ msg = adaptor.get_msg_from_docs(
+ TestMessageClass, mdoc, fdoc, hdoc, cdocs=cdocs)
+ self.assertEqual(msg.wrapper.fdoc.flags,
+ ('\Seen', '\Nice'))
+ self.assertEqual(msg.wrapper.fdoc.tags,
+ ('Personal', 'TODO'))
+ self.assertEqual(msg.wrapper.fdoc.mbox_uuid, "Foobox")
+ self.assertEqual(msg.wrapper.hdoc.multi, False)
+ self.assertEqual(msg.wrapper.hdoc.subject,
+ "Test Msg")
+ self.assertEqual(msg.wrapper.cdocs[1].raw,
+ "This is a test message")
+
+ def test_get_msg_from_metamsg_doc_id(self):
+ # TODO complete-me!
+ pass
+
+ test_get_msg_from_metamsg_doc_id.skip = "Not yet implemented"
+
+ def test_create_msg(self):
+ adaptor = self.get_adaptor()
+
+ with open(os.path.join(HERE, "rfc822.message")) as f:
+ raw = f.read()
+ msg = adaptor.get_msg_from_string(TestMessageClass, raw)
+
+ def check_create_result(created):
+ # that's one mdoc, one hdoc, one fdoc, one cdoc
+ self.assertEqual(len(created), 4)
+ for doc in created:
+ self.assertTrue(
+ doc.__class__.__name__,
+ "SoledadDocument")
+
+ d = adaptor.create_msg(adaptor.store, msg)
+ d.addCallback(check_create_result)
+ return d
+
+ def test_update_msg(self):
+ adaptor = self.get_adaptor()
+ with open(os.path.join(HERE, "rfc822.message")) as f:
+ raw = f.read()
+
+ def assert_msg_has_doc_id(ignored, msg):
+ wrapper = msg.get_wrapper()
+ self.assertTrue(wrapper.fdoc.doc_id is not None)
+
+ def assert_msg_has_no_flags(ignored, msg):
+ wrapper = msg.get_wrapper()
+ self.assertEqual(wrapper.fdoc.flags, [])
+
+ def update_msg_flags(ignored, msg):
+ wrapper = msg.get_wrapper()
+ wrapper.fdoc.flags = ["This", "That"]
+ return wrapper.update(adaptor.store)
+
+ def assert_msg_has_flags(ignored, msg):
+ wrapper = msg.get_wrapper()
+ self.assertEqual(wrapper.fdoc.flags, ["This", "That"])
+
+ def get_fdoc_and_check_flags(ignored):
+ def assert_doc_has_flags(doc):
+ self.assertEqual(doc.content['flags'],
+ ['This', 'That'])
+ wrapper = msg.get_wrapper()
+ d = adaptor.store.get_doc(wrapper.fdoc.doc_id)
+ d.addCallback(assert_doc_has_flags)
+ return d
+
+ msg = adaptor.get_msg_from_string(TestMessageClass, raw)
+ d = adaptor.create_msg(adaptor.store, msg)
+ d.addCallback(lambda _: adaptor.store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 4))
+ d.addCallback(assert_msg_has_doc_id, msg)
+ d.addCallback(assert_msg_has_no_flags, msg)
+
+ # update it!
+ d.addCallback(update_msg_flags, msg)
+ d.addCallback(assert_msg_has_flags, msg)
+ d.addCallback(get_fdoc_and_check_flags)
+ return d
+
+ # Mailboxes
+
+ def test_get_or_create_mbox(self):
+ adaptor = self.get_adaptor()
+
+ def get_or_create_mbox(ignored):
+ d = adaptor.get_or_create_mbox(adaptor.store, "Trash")
+ return d
+
+ def assert_good_doc(mbox_wrapper):
+ self.assertTrue(mbox_wrapper.doc_id is not None)
+ self.assertEqual(mbox_wrapper.mbox, "Trash")
+ self.assertEqual(mbox_wrapper.type, "mbox")
+ self.assertEqual(mbox_wrapper.closed, False)
+ self.assertEqual(mbox_wrapper.subscribed, False)
+
+ d = adaptor.initialize_store(adaptor.store)
+ d.addCallback(get_or_create_mbox)
+ d.addCallback(assert_good_doc)
+ d.addCallback(lambda _: adaptor.store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+ return d
+
+ def test_update_mbox(self):
+ adaptor = self.get_adaptor()
+
+ wrapper_ref = []
+
+ def get_or_create_mbox(ignored):
+ d = adaptor.get_or_create_mbox(adaptor.store, "Trash")
+ return d
+
+ def update_wrapper(wrapper, wrapper_ref):
+ wrapper_ref.append(wrapper)
+ wrapper.subscribed = True
+ wrapper.closed = True
+ d = adaptor.update_mbox(adaptor.store, wrapper)
+ return d
+
+ def get_mbox_doc_and_check_flags(res, wrapper_ref):
+ wrapper = wrapper_ref[0]
+
+ def assert_doc_has_flags(doc):
+ self.assertEqual(doc.content['subscribed'], True)
+ self.assertEqual(doc.content['closed'], True)
+ d = adaptor.store.get_doc(wrapper.doc_id)
+ d.addCallback(assert_doc_has_flags)
+ return d
+
+ d = adaptor.initialize_store(adaptor.store)
+ d.addCallback(get_or_create_mbox)
+ d.addCallback(update_wrapper, wrapper_ref)
+ d.addCallback(get_mbox_doc_and_check_flags, wrapper_ref)
+ return d
+
+ def test_get_all_mboxes(self):
+ adaptor = self.get_adaptor()
+ mboxes = ("Sent", "Trash", "Personal", "ListFoo")
+
+ def get_or_create_mboxes(ignored):
+ d = []
+ for mbox in mboxes:
+ d.append(adaptor.get_or_create_mbox(
+ adaptor.store, mbox))
+ return defer.gatherResults(d)
+
+ def get_all_mboxes(ignored):
+ return adaptor.get_all_mboxes(adaptor.store)
+
+ def assert_mboxes_match_expected(wrappers):
+ names = [m.mbox for m in wrappers]
+ self.assertEqual(set(names), set(mboxes))
+
+ d = adaptor.initialize_store(adaptor.store)
+ d.addCallback(get_or_create_mboxes)
+ d.addCallback(get_all_mboxes)
+ d.addCallback(assert_mboxes_match_expected)
+ return d
diff --git a/src/leap/mail/constants.py b/src/leap/mail/constants.py
new file mode 100644
index 0000000..4ef42cb
--- /dev/null
+++ b/src/leap/mail/constants.py
@@ -0,0 +1,52 @@
+# *- coding: utf-8 -*-
+# constants.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Constants for leap.mail.
+"""
+
+INBOX_NAME = "INBOX"
+
+# Regular expressions for the identifiers to be used in the Message Data Layer.
+
+METAMSGID = "M-{mbox_uuid}-{chash}"
+METAMSGID_RE = "M\-{mbox_uuid}\-[0-9a-fA-F]+"
+METAMSGID_CHASH_RE = "M\-\w+\-([0-9a-fA-F]+)"
+METAMSGID_MBOX_RE = "M\-(\w+)\-[0-9a-fA-F]+"
+
+FDOCID = "F-{mbox_uuid}-{chash}"
+FDOCID_RE = "F\-{mbox_uuid}\-[0-9a-fA-F]+"
+FDOCID_CHASH_RE = "F\-\w+\-([0-9a-fA-F]+)"
+
+HDOCID = "H-{chash}"
+HDOCID_RE = "H\-[0-9a-fA-F]+"
+
+CDOCID = "C-{phash}"
+CDOCID_RE = "C\-[0-9a-fA-F]+"
+
+
+class MessageFlags(object):
+ """
+ Flags used in Message and Mailbox.
+ """
+ SEEN_FLAG = "\\Seen"
+ RECENT_FLAG = "\\Recent"
+ ANSWERED_FLAG = "\\Answered"
+ FLAGGED_FLAG = "\\Flagged" # yo dawg
+ DELETED_FLAG = "\\Deleted"
+ DRAFT_FLAG = "\\Draft"
+ NOSELECT_FLAG = "\\Noselect"
+ LIST_FLAG = "List" # is this OK? (no \. ie, no system flag)
diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index 70ed13b..cc56fff 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# account.py
-# Copyright (C) 2013 LEAP
+# Copyright (C) 2013-2015 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -15,22 +15,23 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
-Soledad Backed Account.
+Soledad Backed IMAP Account.
"""
-import copy
import logging
import os
import time
+from functools import partial
+from twisted.internet import defer
from twisted.mail import imap4
from twisted.python import log
from zope.interface import implements
from leap.common.check import leap_assert, leap_assert_type
-from leap.mail.imap.index import IndexedDB
-from leap.mail.imap.fields import WithMsgFields
-from leap.mail.imap.parser import MBoxParser
-from leap.mail.imap.mailbox import SoledadMailbox
+
+from leap.mail.constants import MessageFlags
+from leap.mail.mail import Account
+from leap.mail.imap.mailbox import IMAPMailbox, normalize_mailbox
from leap.soledad.client import Soledad
logger = logging.getLogger(__name__)
@@ -38,7 +39,6 @@ logger = logging.getLogger(__name__)
PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False)
if PROFILE_CMD:
-
def _debugProfiling(result, cmdname, start):
took = (time.time() - start) * 1000
log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec")
@@ -46,107 +46,72 @@ if PROFILE_CMD:
#######################################
-# Soledad Account
+# Soledad IMAP Account
#######################################
-
-# TODO change name to LeapIMAPAccount, since we're using
-# the memstore.
-# IndexedDB should also not be here anymore.
-
-class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
+class IMAPAccount(object):
"""
- An implementation of IAccount and INamespacePresenteer
+ An implementation of an imap4 Account
that is backed by Soledad Encrypted Documents.
"""
implements(imap4.IAccount, imap4.INamespacePresenter)
- _soledad = None
selected = None
- closed = False
- def __init__(self, account_name, soledad, memstore=None):
+ def __init__(self, user_id, store, d=defer.Deferred()):
"""
- Creates a SoledadAccountIndex that keeps track of the mailboxes
- and subscriptions handled by this account.
-
- :param acct_name: The name of the account (user id).
- :type acct_name: str
-
- :param soledad: a Soledad instance.
- :type soledad: Soledad
- :param memstore: a MemoryStore instance.
- :type memstore: MemoryStore
- """
- leap_assert(soledad, "Need a soledad instance to initialize")
- leap_assert_type(soledad, Soledad)
-
- # XXX SHOULD assert too that the name matches the user/uuid with which
- # soledad has been initialized.
-
- # XXX ??? why is this parsing mailbox name??? it's account...
- # userid? homogenize.
- self._account_name = self._parse_mailbox_name(account_name)
- self._soledad = soledad
- self._memstore = memstore
-
- self.__mailboxes = set([])
-
- self.initialize_db()
+ Keeps track of the mailboxes and subscriptions handled by this account.
- # every user should have the right to an inbox folder
- # at least, so let's make one!
- self._load_mailboxes()
+ The account is not ready to be used, since the store needs to be
+ initialized and we also need to do some initialization routines.
+ You can either pass a deferred to this constructor, or use
+ `callWhenReady` method.
- if not self.mailboxes:
- self.addMailbox(self.INBOX_NAME)
+ :param user_id: The name of the account (user id, in the form
+ user@provider).
+ :type user_id: str
- def _get_empty_mailbox(self):
- """
- Returns an empty mailbox.
+ :param store: a Soledad instance.
+ :type store: Soledad
- :rtype: dict
+ :param d: a deferred that will be fired with this IMAPAccount instance
+ when the account is ready to be used.
+ :type d: defer.Deferred
"""
- return copy.deepcopy(self.EMPTY_MBOX)
+ leap_assert(store, "Need a store instance to initialize")
+ leap_assert_type(store, Soledad)
- def _get_mailbox_by_name(self, name):
- """
- Return an mbox document by name.
+ # TODO assert too that the name matches the user/uuid with which
+ # soledad has been initialized. Although afaik soledad doesn't know
+ # about user_id, only the client backend.
- :param name: the name of the mailbox
- :type name: str
+ self.user_id = user_id
+ self.account = Account(store, ready_cb=lambda: d.callback(self))
- :rtype: SoledadDocument
+ def end_session(self):
"""
- # XXX use soledadstore instead ...;
- doc = self._soledad.get_from_index(
- self.TYPE_MBOX_IDX, self.MBOX_KEY,
- self._parse_mailbox_name(name))
- return doc[0] if doc else None
+ Used to mark when the session has closed, and we should not allow any
+ more commands from the client.
- @property
- def mailboxes(self):
- """
- A list of the current mailboxes for this account.
- :rtype: set
+ Right now it's called from the client backend.
"""
- return sorted(self.__mailboxes)
-
- def _load_mailboxes(self):
- self.__mailboxes.update(
- [doc.content[self.MBOX_KEY]
- for doc in self._soledad.get_from_index(
- self.TYPE_IDX, self.MBOX_KEY)])
+ # TODO move its use to the service shutdown in leap.mail
+ self.account.end_session()
@property
- def subscriptions(self):
+ def session_ended(self):
+ return self.account.session_ended
+
+ def callWhenReady(self, cb, *args, **kw):
"""
- A list of the current subscriptions for this account.
+ Execute callback when the account is ready to be used.
+ XXX note that this callback will be called with a first ignored
+ parameter.
"""
- return [doc.content[self.MBOX_KEY]
- for doc in self._soledad.get_from_index(
- self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')]
+ # TODO ignore the first parameter and change tests accordingly.
+ d = self.account.callWhenReady(cb, *args, **kw)
+ return d
def getMailbox(self, name):
"""
@@ -155,16 +120,27 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:param name: name of the mailbox
:type name: str
- :returns: a a SoledadMailbox instance
- :rtype: SoledadMailbox
+ :returns: an IMAPMailbox instance
+ :rtype: IMAPMailbox
"""
- name = self._parse_mailbox_name(name)
+ name = normalize_mailbox(name)
- if name not in self.mailboxes:
- raise imap4.MailboxException("No such mailbox: %r" % name)
+ def check_it_exists(mailboxes):
+ if name not in mailboxes:
+ raise imap4.MailboxException("No such mailbox: %r" % name)
+ return True
- return SoledadMailbox(name, self._soledad,
- memstore=self._memstore)
+ d = self.account.list_all_mailbox_names()
+ d.addCallback(check_it_exists)
+ d.addCallback(lambda _: self.account.get_collection_by_mailbox(name))
+ d.addCallback(self._return_mailbox_from_collection)
+ return d
+
+ def _return_mailbox_from_collection(self, collection, readwrite=1):
+ if collection is None:
+ return None
+ mbox = IMAPMailbox(collection, rw=readwrite)
+ return mbox
#
# IAccount
@@ -182,61 +158,76 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
one is provided.
:type creation_ts: int
- :returns: True if successful
- :rtype: bool
+ :returns: a Deferred that will contain the document if successful.
+ :rtype: defer.Deferred
"""
- name = self._parse_mailbox_name(name)
+ name = normalize_mailbox(name)
+ # FIXME --- return failure instead of AssertionError
+ # See AccountTestCase...
leap_assert(name, "Need a mailbox name to create a mailbox")
- if name in self.mailboxes:
- raise imap4.MailboxCollision(repr(name))
-
- if creation_ts is None:
- # by default, we pass an int value
- # taken from the current time
- # we make sure to take enough decimals to get a unique
- # mailbox-uidvalidity.
- creation_ts = int(time.time() * 10E2)
+ def check_it_does_not_exist(mailboxes):
+ if name in mailboxes:
+ raise imap4.MailboxCollision, repr(name)
+ return mailboxes
- mbox = self._get_empty_mailbox()
- mbox[self.MBOX_KEY] = name
- mbox[self.CREATED_KEY] = creation_ts
-
- doc = self._soledad.create_doc(mbox)
- self._load_mailboxes()
- return bool(doc)
+ d = self.account.list_all_mailbox_names()
+ d.addCallback(check_it_does_not_exist)
+ d.addCallback(lambda _: self.account.add_mailbox(
+ name, creation_ts=creation_ts))
+ d.addCallback(lambda _: self.account.get_collection_by_mailbox(name))
+ d.addCallback(self._return_mailbox_from_collection)
+ return d
def create(self, pathspec):
"""
Create a new mailbox from the given hierarchical name.
- :param pathspec: The full hierarchical name of a new mailbox to create.
- If any of the inferior hierarchical names to this one
- do not exist, they are created as well.
+ :param pathspec:
+ The full hierarchical name of a new mailbox to create.
+ If any of the inferior hierarchical names to this one
+ do not exist, they are created as well.
:type pathspec: str
- :return: A true value if the creation succeeds.
- :rtype: bool
+ :return:
+ A deferred that will fire with a true value if the creation
+ succeeds. The deferred might fail with a MailboxException
+ if the mailbox cannot be added.
+ :rtype: Deferred
- :raise MailboxException: Raised if this mailbox cannot be added.
"""
- # TODO raise MailboxException
- paths = filter(
- None,
- self._parse_mailbox_name(pathspec).split('/'))
- for accum in range(1, len(paths)):
- try:
- self.addMailbox('/'.join(paths[:accum]))
- except imap4.MailboxCollision:
- pass
- try:
- self.addMailbox('/'.join(paths))
- except imap4.MailboxCollision:
+ def pass_on_collision(failure):
+ failure.trap(imap4.MailboxCollision)
+ return True
+
+ def handle_collision(failure):
+ failure.trap(imap4.MailboxCollision)
if not pathspec.endswith('/'):
- return False
- self._load_mailboxes()
- return True
+ return defer.succeed(False)
+ else:
+ return defer.succeed(True)
+
+ def all_good(result):
+ return all(result)
+
+ paths = filter(None, normalize_mailbox(pathspec).split('/'))
+ subs = []
+ sep = '/'
+
+ for accum in range(1, len(paths)):
+ partial_path = sep.join(paths[:accum])
+ d = self.addMailbox(partial_path)
+ d.addErrback(pass_on_collision)
+ subs.append(d)
+
+ df = self.addMailbox(sep.join(paths))
+ df.addErrback(handle_collision)
+ subs.append(df)
+
+ d1 = defer.gatherResults(subs)
+ d1.addCallback(all_good)
+ return d1
def select(self, name, readwrite=1):
"""
@@ -248,65 +239,87 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:param readwrite: 1 for readwrite permissions.
:type readwrite: int
- :rtype: SoledadMailbox
+ :rtype: IMAPMailbox
"""
- if PROFILE_CMD:
- start = time.time()
+ name = normalize_mailbox(name)
- name = self._parse_mailbox_name(name)
- if name not in self.mailboxes:
- logger.warning("No such mailbox!")
- return None
- self.selected = name
+ def check_it_exists(mailboxes):
+ if name not in mailboxes:
+ logger.warning("SELECT: No such mailbox!")
+ return None
+ return name
+
+ def set_selected(_):
+ self.selected = name
- sm = SoledadMailbox(
- name, self._soledad, self._memstore, readwrite)
- if PROFILE_CMD:
- _debugProfiling(None, "SELECT", start)
- return sm
+ def get_collection(name):
+ if name is None:
+ return None
+ return self.account.get_collection_by_mailbox(name)
+
+ d = self.account.list_all_mailbox_names()
+ d.addCallback(check_it_exists)
+ d.addCallback(get_collection)
+ d.addCallback(partial(
+ self._return_mailbox_from_collection, readwrite=readwrite))
+ return d
def delete(self, name, force=False):
"""
Deletes a mailbox.
- Right now it does not purge the messages, but just removes the mailbox
- name from the mailboxes list!!!
-
:param name: the mailbox to be deleted
:type name: str
- :param force: if True, it will not check for noselect flag or inferior
- names. use with care.
+ :param force:
+ if True, it will not check for noselect flag or inferior
+ names. use with care.
:type force: bool
+ :rtype: Deferred
"""
- name = self._parse_mailbox_name(name)
+ name = normalize_mailbox(name)
+ _mboxes = None
+
+ def check_it_exists(mailboxes):
+ global _mboxes
+ _mboxes = mailboxes
+ if name not in mailboxes:
+ raise imap4.MailboxException("No such mailbox: %r" % name)
- if name not in self.mailboxes:
- raise imap4.MailboxException("No such mailbox: %r" % name)
- mbox = self.getMailbox(name)
+ def get_mailbox(_):
+ return self.getMailbox(name)
- if force is False:
+ def destroy_mailbox(mbox):
+ return mbox.destroy()
+
+ def check_can_be_deleted(mbox):
+ global _mboxes
# See if this box is flagged \Noselect
- # XXX use mbox.flags instead?
mbox_flags = mbox.getFlags()
- if self.NOSELECT_FLAG in mbox_flags:
+ if MessageFlags.NOSELECT_FLAG in mbox_flags:
# Check for hierarchically inferior mailboxes with this one
# as part of their root.
- for others in self.mailboxes:
+ for others in _mboxes:
if others != name and others.startswith(name):
- raise imap4.MailboxException, (
+ raise imap4.MailboxException(
"Hierarchically inferior mailboxes "
"exist and \\Noselect is set")
- self.__mailboxes.discard(name)
- mbox.destroy()
+ return mbox
- # XXX FIXME --- not honoring the inferior names...
+ d = self.account.list_all_mailbox_names()
+ d.addCallback(check_it_exists)
+ d.addCallback(get_mailbox)
+ if not force:
+ d.addCallback(check_can_be_deleted)
+ d.addCallback(destroy_mailbox)
+ return d
+ # FIXME --- not honoring the inferior names...
# if there are no hierarchically inferior names, we will
# delete it from our ken.
+ # XXX is this right?
# if self._inferiorNames(name) > 1:
- # ??! -- can this be rite?
- # self._index.removeMailbox(name)
+ # self._index.removeMailbox(name)
def rename(self, oldname, newname):
"""
@@ -318,27 +331,31 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:param newname: new name of the mailbox
:type newname: str
"""
- oldname = self._parse_mailbox_name(oldname)
- newname = self._parse_mailbox_name(newname)
+ oldname = normalize_mailbox(oldname)
+ newname = normalize_mailbox(newname)
+
+ def rename_inferiors((inferiors, mailboxes)):
+ rename_deferreds = []
+ inferiors = [
+ (o, o.replace(oldname, newname, 1)) for o in inferiors]
- if oldname not in self.mailboxes:
- raise imap4.NoSuchMailbox(repr(oldname))
+ for (old, new) in inferiors:
+ if new in mailboxes:
+ raise imap4.MailboxCollision(repr(new))
- inferiors = self._inferiorNames(oldname)
- inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors]
+ for (old, new) in inferiors:
+ d = self.account.rename_mailbox(old, new)
+ rename_deferreds.append(d)
- for (old, new) in inferiors:
- if new in self.mailboxes:
- raise imap4.MailboxCollision(repr(new))
+ d1 = defer.gatherResults(rename_deferreds, consumeErrors=True)
+ return d1
- for (old, new) in inferiors:
- self._memstore.rename_fdocs_mailbox(old, new)
- mbox = self._get_mailbox_by_name(old)
- mbox.content[self.MBOX_KEY] = new
- self.__mailboxes.discard(old)
- self._soledad.put_doc(mbox)
+ d1 = self._inferiorNames(oldname)
+ d2 = self.account.list_all_mailbox_names()
- self._load_mailboxes()
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(rename_inferiors)
+ return d
def _inferiorNames(self, name):
"""
@@ -348,54 +365,87 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:rtype: list
"""
# XXX use wildcard query instead
- inferiors = []
- for infname in self.mailboxes:
- if infname.startswith(name):
- inferiors.append(infname)
- return inferiors
+ def filter_inferiors(mailboxes):
+ inferiors = []
+ for infname in mailboxes:
+ if infname.startswith(name):
+ inferiors.append(infname)
+ return inferiors
- def isSubscribed(self, name):
+ d = self.account.list_all_mailbox_names()
+ d.addCallback(filter_inferiors)
+ return d
+
+ def listMailboxes(self, ref, wildcard):
"""
- Returns True if user is subscribed to this mailbox.
+ List the mailboxes.
- :param name: the mailbox to be checked.
- :type name: str
+ from rfc 3501:
+ returns a subset of names from the complete set
+ of all names available to the client. Zero or more untagged LIST
+ replies are returned, containing the name attributes, hierarchy
+ delimiter, and name.
- :rtype: bool
+ :param ref: reference name
+ :type ref: str
+
+ :param wildcard: mailbox name with possible wildcards
+ :type wildcard: str
"""
- mbox = self._get_mailbox_by_name(name)
- return mbox.content.get('subscribed', False)
+ wildcard = imap4.wildcardToRegexp(wildcard, '/')
+
+ def get_list(mboxes, mboxes_names):
+ return zip(mboxes_names, mboxes)
+
+ def filter_inferiors(ref):
+ mboxes = [mbox for mbox in ref if wildcard.match(mbox)]
+ mbox_d = defer.gatherResults([self.getMailbox(m) for m in mboxes])
- def _set_subscription(self, name, value):
+ mbox_d.addCallback(get_list, mboxes)
+ return mbox_d
+
+ d = self._inferiorNames(normalize_mailbox(ref))
+ d.addCallback(filter_inferiors)
+ return d
+
+ #
+ # The rest of the methods are specific for leap.mail.imap.account.Account
+ #
+
+ def isSubscribed(self, name):
"""
- Sets the subscription value for a given mailbox
+ Returns True if user is subscribed to this mailbox.
- :param name: the mailbox
+ :param name: the mailbox to be checked.
:type name: str
- :param value: the boolean value
- :type value: bool
+ :rtype: Deferred (will fire with bool)
"""
- # maybe we should store subscriptions in another
- # document...
- if name not in self.mailboxes:
- self.addMailbox(name)
- mbox = self._get_mailbox_by_name(name)
+ name = normalize_mailbox(name)
- if mbox:
- mbox.content[self.SUBSCRIBED_KEY] = value
- self._soledad.put_doc(mbox)
+ def get_subscribed(mbox):
+ return mbox.collection.get_mbox_attr("subscribed")
+
+ d = self.getMailbox(name)
+ d.addCallback(get_subscribed)
+ return d
def subscribe(self, name):
"""
- Subscribe to this mailbox
+ Subscribe to this mailbox if not already subscribed.
:param name: name of the mailbox
:type name: str
+ :rtype: Deferred
"""
- name = self._parse_mailbox_name(name)
- if name not in self.subscriptions:
- self._set_subscription(name, True)
+ name = normalize_mailbox(name)
+
+ def set_subscribed(mbox):
+ return mbox.collection.set_mbox_attr("subscribed", True)
+
+ d = self.getMailbox(name)
+ d.addCallback(set_subscribed)
+ return d
def unsubscribe(self, name):
"""
@@ -403,34 +453,27 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:param name: name of the mailbox
:type name: str
+ :rtype: Deferred
"""
- name = self._parse_mailbox_name(name)
- if name not in self.subscriptions:
- raise imap4.MailboxException(
- "Not currently subscribed to %r" % name)
- self._set_subscription(name, False)
+ # TODO should raise MailboxException if attempted to unsubscribe
+ # from a mailbox that is not currently subscribed.
+ # TODO factor out with subscribe method.
+ name = normalize_mailbox(name)
- def listMailboxes(self, ref, wildcard):
- """
- List the mailboxes.
+ def set_unsubscribed(mbox):
+ return mbox.collection.set_mbox_attr("subscribed", False)
- from rfc 3501:
- returns a subset of names from the complete set
- of all names available to the client. Zero or more untagged LIST
- replies are returned, containing the name attributes, hierarchy
- delimiter, and name.
+ d = self.getMailbox(name)
+ d.addCallback(set_unsubscribed)
+ return d
- :param ref: reference name
- :type ref: str
+ def getSubscriptions(self):
+ def get_subscribed(mailboxes):
+ return [x.mbox for x in mailboxes if x.subscribed]
- :param wildcard: mailbox name with possible wildcards
- :type wildcard: str
- """
- # XXX use wildcard in index query
- ref = self._inferiorNames(
- self._parse_mailbox_name(ref))
- wildcard = imap4.wildcardToRegexp(wildcard, '/')
- return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)]
+ d = self.account.get_all_mailboxes()
+ d.addCallback(get_subscribed)
+ return d
#
# INamespacePresenter
@@ -445,22 +488,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
def getOtherNamespaces(self):
return None
- # extra, for convenience
-
- def deleteAllMessages(self, iknowhatiamdoing=False):
- """
- Deletes all messages from all mailboxes.
- Danger! high voltage!
-
- :param iknowhatiamdoing: confirmation parameter, needs to be True
- to proceed.
- """
- if iknowhatiamdoing is True:
- for mbox in self.mailboxes:
- self.delete(mbox, force=True)
-
def __repr__(self):
"""
Representation string for this object.
"""
- return "<SoledadBackedAccount (%s)>" % self._account_name
+ return "<IMAPAccount (%s)>" % self.user_id
diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
deleted file mode 100644
index 0a97752..0000000
--- a/src/leap/mail/imap/fetch.py
+++ /dev/null
@@ -1,655 +0,0 @@
-# -*- coding: utf-8 -*-
-# fetch.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-Incoming mail fetcher.
-"""
-import copy
-import logging
-import threading
-import time
-import sys
-import traceback
-import warnings
-
-from email.parser import Parser
-from email.generator import Generator
-from email.utils import parseaddr
-from StringIO import StringIO
-
-from twisted.python import log
-from twisted.internet import defer
-from twisted.internet.task import LoopingCall
-from twisted.internet.task import deferLater
-from u1db import errors as u1db_errors
-from zope.proxy import sameProxiedObjects
-
-from leap.common import events as leap_events
-from leap.common.check import leap_assert, leap_assert_type
-from leap.common.events.events_pb2 import IMAP_FETCHED_INCOMING
-from leap.common.events.events_pb2 import IMAP_MSG_PROCESSING
-from leap.common.events.events_pb2 import IMAP_MSG_DECRYPTED
-from leap.common.events.events_pb2 import IMAP_MSG_SAVED_LOCALLY
-from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING
-from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
-from leap.common.events.events_pb2 import SOLEDAD_INVALID_AUTH_TOKEN
-from leap.common.mail import get_email_charset
-from leap.keymanager import errors as keymanager_errors
-from leap.keymanager.openpgp import OpenPGPKey
-from leap.mail.decorators import deferred_to_thread
-from leap.mail.imap.fields import fields
-from leap.mail.utils import json_loads, empty, first
-from leap.soledad.client import Soledad
-from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
-from leap.soledad.common.errors import InvalidAuthTokenError
-
-
-logger = logging.getLogger(__name__)
-
-MULTIPART_ENCRYPTED = "multipart/encrypted"
-MULTIPART_SIGNED = "multipart/signed"
-PGP_BEGIN = "-----BEGIN PGP MESSAGE-----"
-PGP_END = "-----END PGP MESSAGE-----"
-
-
-class MalformedMessage(Exception):
- """
- Raised when a given message is not well formed.
- """
- pass
-
-
-class LeapIncomingMail(object):
- """
- Fetches and process mail from the incoming pool.
-
- This object has public methods start_loop and stop that will
- actually initiate a LoopingCall with check_period recurrency.
- The LoopingCall itself will invoke the fetch method each time
- that the check_period expires.
-
- This loop will sync the soledad db with the remote server and
- process all the documents found tagged as incoming mail.
- """
-
- RECENT_FLAG = "\\Recent"
- CONTENT_KEY = "content"
-
- LEAP_SIGNATURE_HEADER = 'X-Leap-Signature'
- """
- Header added to messages when they are decrypted by the IMAP fetcher,
- which states the validity of an eventual signature that might be included
- in the encrypted blob.
- """
- LEAP_SIGNATURE_VALID = 'valid'
- LEAP_SIGNATURE_INVALID = 'invalid'
- LEAP_SIGNATURE_COULD_NOT_VERIFY = 'could not verify'
-
- fetching_lock = threading.Lock()
-
- def __init__(self, keymanager, soledad, imap_account,
- check_period, userid):
-
- """
- Initialize LeapIncomingMail..
-
- :param keymanager: a keymanager instance
- :type keymanager: keymanager.KeyManager
-
- :param soledad: a soledad instance
- :type soledad: Soledad
-
- :param imap_account: the account to fetch periodically
- :type imap_account: SoledadBackedAccount
-
- :param check_period: the period to fetch new mail, in seconds.
- :type check_period: int
- """
-
- leap_assert(keymanager, "need a keymanager to initialize")
- leap_assert_type(soledad, Soledad)
- leap_assert(check_period, "need a period to check incoming mail")
- leap_assert_type(check_period, int)
- leap_assert(userid, "need a userid to initialize")
-
- self._keymanager = keymanager
- self._soledad = soledad
- self.imapAccount = imap_account
- self._inbox = self.imapAccount.getMailbox('inbox')
- self._userid = userid
-
- self._loop = None
- self._check_period = check_period
-
- # initialize a mail parser only once
- self._parser = Parser()
-
- @property
- def _pkey(self):
- if sameProxiedObjects(self._keymanager, None):
- logger.warning('tried to get key, but null keymanager found')
- return None
- return self._keymanager.get_key(self._userid, OpenPGPKey, private=True)
-
- #
- # Public API: fetch, start_loop, stop.
- #
-
- def fetch(self):
- """
- Fetch incoming mail, to be called periodically.
-
- Calls a deferred that will execute the fetch callback
- in a separate thread
- """
- def syncSoledadCallback(result):
- # FIXME this needs a matching change in mx!!!
- # --> need to add ERROR_DECRYPTING_KEY = False
- # as default.
- try:
- doclist = self._soledad.get_from_index(
- fields.JUST_MAIL_IDX, "*", "0")
- except u1db_errors.InvalidGlobbing:
- # It looks like we are a dealing with an outdated
- # mx. Fallback to the version of the index
- warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!",
- DeprecationWarning)
- doclist = self._soledad.get_from_index(
- fields.JUST_MAIL_COMPAT_IDX, "*")
- self._process_doclist(doclist)
-
- logger.debug("fetching mail for: %s %s" % (
- self._soledad.uuid, self._userid))
- if not self.fetching_lock.locked():
- d1 = self._sync_soledad()
- d = defer.gatherResults([d1], consumeErrors=True)
- d.addCallbacks(syncSoledadCallback, self._errback)
- d.addCallbacks(self._signal_fetch_to_ui, self._errback)
- return d
- else:
- logger.debug("Already fetching mail.")
-
- def start_loop(self):
- """
- Starts a loop to fetch mail.
- """
- if self._loop is None:
- self._loop = LoopingCall(self.fetch)
- self._loop.start(self._check_period)
- else:
- logger.warning("Tried to start an already running fetching loop.")
-
- def stop(self):
- # XXX change the name to stop_loop, for consistency.
- """
- Stops the loop that fetches mail.
- """
- if self._loop and self._loop.running is True:
- self._loop.stop()
- self._loop = None
-
- #
- # Private methods.
- #
-
- # synchronize incoming mail
-
- def _errback(self, failure):
- logger.exception(failure.value)
- traceback.print_tb(*sys.exc_info())
-
- @deferred_to_thread
- def _sync_soledad(self):
- """
- Synchronize with remote soledad.
-
- :returns: a list of LeapDocuments, or None.
- :rtype: iterable or None
- """
- with self.fetching_lock:
- try:
- log.msg('FETCH: syncing soledad...')
- self._soledad.sync()
- log.msg('FETCH soledad SYNCED.')
- except InvalidAuthTokenError:
- # if the token is invalid, send an event so the GUI can
- # disable mail and show an error message.
- leap_events.signal(SOLEDAD_INVALID_AUTH_TOKEN)
-
- def _signal_fetch_to_ui(self, doclist):
- """
- Send leap events to ui.
-
- :param doclist: iterable with msg documents.
- :type doclist: iterable.
- :returns: doclist
- :rtype: iterable
- """
- doclist = first(doclist) # gatherResults pass us a list
- if doclist:
- fetched_ts = time.mktime(time.gmtime())
- num_mails = len(doclist) if doclist is not None else 0
- if num_mails != 0:
- log.msg("there are %s mails" % (num_mails,))
- leap_events.signal(
- IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts))
- return doclist
-
- def _signal_unread_to_ui(self, *args):
- """
- Sends unread event to ui.
- """
- leap_events.signal(
- IMAP_UNREAD_MAIL, str(self._inbox.getUnseenCount()))
-
- # process incoming mail.
-
- def _process_doclist(self, doclist):
- """
- Iterates through the doclist, checks if each doc
- looks like a message, and yields a deferred that will decrypt and
- process the message.
-
- :param doclist: iterable with msg documents.
- :type doclist: iterable.
- :returns: a list of deferreds for individual messages.
- """
- log.msg('processing doclist')
- if not doclist:
- logger.debug("no docs found")
- return
- num_mails = len(doclist)
-
- for index, doc in enumerate(doclist):
- logger.debug("processing doc %d of %d" % (index + 1, num_mails))
- leap_events.signal(
- IMAP_MSG_PROCESSING, str(index), str(num_mails))
-
- keys = doc.content.keys()
-
- # TODO Compatibility check with the index in pre-0.6 mx
- # that does not write the ERROR_DECRYPTING_KEY
- # This should be removed in 0.7
-
- has_errors = doc.content.get(fields.ERROR_DECRYPTING_KEY, None)
- if has_errors is None:
- warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!",
- DeprecationWarning)
- if has_errors:
- logger.debug("skipping msg with decrypting errors...")
-
- if self._is_msg(keys) and not has_errors:
- # Evaluating to bool of has_errors is intentional here.
- # We don't mind at this point if it's None or False.
-
- # Ok, this looks like a legit msg, and with no errors.
- # Let's process it!
-
- d1 = self._decrypt_doc(doc)
- d = defer.gatherResults([d1], consumeErrors=True)
- d.addCallbacks(self._add_message_locally, self._errback)
-
- #
- # operations on individual messages
- #
-
- @deferred_to_thread
- def _decrypt_doc(self, doc):
- """
- Decrypt the contents of a document.
-
- :param doc: A document containing an encrypted message.
- :type doc: SoledadDocument
-
- :return: A tuple containing the document and the decrypted message.
- :rtype: (SoledadDocument, str)
- """
- log.msg('decrypting msg')
- success = False
-
- try:
- decrdata = self._keymanager.decrypt(
- doc.content[ENC_JSON_KEY],
- self._pkey)
- success = True
- except Exception as exc:
- # XXX move this to errback !!!
- logger.error("Error while decrypting msg: %r" % (exc,))
- decrdata = ""
- leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0")
-
- data = self._process_decrypted_doc((doc, decrdata))
- return (doc, data)
-
- def _process_decrypted_doc(self, msgtuple):
- """
- Process a document containing a succesfully decrypted message.
-
- :param msgtuple: a tuple consisting of a SoledadDocument
- instance containing the incoming message
- and data, the json-encoded, decrypted content of the
- incoming message
- :type msgtuple: (SoledadDocument, str)
- :return: a SoledadDocument and the processed data.
- :rtype: (doc, data)
- """
- log.msg('processing decrypted doc')
- doc, data = msgtuple
-
- from twisted.internet import reactor
-
- # XXX turn this into an errBack for each one of
- # the deferreds that would process an individual document
- try:
- msg = json_loads(data)
- except UnicodeError as exc:
- logger.error("Error while decrypting %s" % (doc.doc_id,))
- logger.exception(exc)
-
- # we flag the message as "with decrypting errors",
- # to avoid further decryption attempts during sync
- # cycles until we're prepared to deal with that.
- # What is the same, when Ivan deals with it...
- # A new decrypting attempt event could be triggered by a
- # future a library upgrade, or a cli flag to the client,
- # we just `defer` that for now... :)
- doc.content[fields.ERROR_DECRYPTING_KEY] = True
- deferLater(reactor, 0, self._update_incoming_message, doc)
-
- # FIXME this is just a dirty hack to delay the proper
- # deferred organization here...
- # and remember, boys, do not do this at home.
- return []
-
- if not isinstance(msg, dict):
- defer.returnValue(False)
- if not msg.get(fields.INCOMING_KEY, False):
- defer.returnValue(False)
-
- # ok, this is an incoming message
- rawmsg = msg.get(self.CONTENT_KEY, None)
- if not rawmsg:
- return False
- return self._maybe_decrypt_msg(rawmsg)
-
- @deferred_to_thread
- def _update_incoming_message(self, doc):
- """
- Do a put for a soledad document. This probably has been called only
- in the case that we've needed to update the ERROR_DECRYPTING_KEY
- flag in an incoming message, to get it out of the decrypting queue.
-
- :param doc: the SoledadDocument to update
- :type doc: SoledadDocument
- """
- log.msg("Updating SoledadDoc %s" % (doc.doc_id))
- self._soledad.put_doc(doc)
-
- @deferred_to_thread
- def _delete_incoming_message(self, doc):
- """
- Delete document.
-
- :param doc: the SoledadDocument to delete
- :type doc: SoledadDocument
- """
- log.msg("Deleting Incoming message: %s" % (doc.doc_id,))
- self._soledad.delete_doc(doc)
-
- def _maybe_decrypt_msg(self, data):
- """
- Tries to decrypt a gpg message if data looks like one.
-
- :param data: the text to be decrypted.
- :type data: str
- :return: data, possibly descrypted.
- :rtype: str
- """
- leap_assert_type(data, str)
- log.msg('maybe decrypting doc')
-
- # parse the original message
- encoding = get_email_charset(data)
- msg = self._parser.parsestr(data)
-
- # try to obtain sender public key
- senderPubkey = None
- fromHeader = msg.get('from', None)
- if (fromHeader is not None
- and (msg.get_content_type() == MULTIPART_ENCRYPTED
- or msg.get_content_type() == MULTIPART_SIGNED)):
- _, senderAddress = parseaddr(fromHeader)
- try:
- senderPubkey = self._keymanager.get_key_from_cache(
- senderAddress, OpenPGPKey)
- except keymanager_errors.KeyNotFound:
- pass
-
- valid_sig = False # we will add a header saying if sig is valid
- decrypt_multi = self._decrypt_multipart_encrypted_msg
- decrypt_inline = self._maybe_decrypt_inline_encrypted_msg
-
- if msg.get_content_type() == MULTIPART_ENCRYPTED:
- decrmsg, valid_sig = decrypt_multi(
- msg, encoding, senderPubkey)
- else:
- decrmsg, valid_sig = decrypt_inline(
- msg, encoding, senderPubkey)
-
- # add x-leap-signature header
- if senderPubkey is None:
- decrmsg.add_header(
- self.LEAP_SIGNATURE_HEADER,
- self.LEAP_SIGNATURE_COULD_NOT_VERIFY)
- else:
- decrmsg.add_header(
- self.LEAP_SIGNATURE_HEADER,
- self.LEAP_SIGNATURE_VALID if valid_sig else
- self.LEAP_SIGNATURE_INVALID,
- pubkey=senderPubkey.key_id)
-
- return decrmsg.as_string()
-
- def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderPubkey):
- """
- Decrypt a message with content-type 'multipart/encrypted'.
-
- :param msg: The original encrypted message.
- :type msg: Message
- :param encoding: The encoding of the email message.
- :type encoding: str
- :param senderPubkey: The key of the sender of the message.
- :type senderPubkey: OpenPGPKey
-
- :return: A unitary tuple containing a decrypted message.
- :rtype: (Message)
- """
- log.msg('decrypting multipart encrypted msg')
- msg = copy.deepcopy(msg)
- self._msg_multipart_sanity_check(msg)
-
- # parse message and get encrypted content
- pgpencmsg = msg.get_payload()[1]
- encdata = pgpencmsg.get_payload()
-
- # decrypt or fail gracefully
- try:
- decrdata, valid_sig = self._decrypt_and_verify_data(
- encdata, senderPubkey)
- except keymanager_errors.DecryptError as e:
- logger.warning('Failed to decrypt encrypted message (%s). '
- 'Storing message without modifications.' % str(e))
- # Bailing out!
- return (msg, False)
-
- decrmsg = self._parser.parsestr(decrdata)
- # remove original message's multipart/encrypted content-type
- del(msg['content-type'])
-
- # replace headers back in original message
- for hkey, hval in decrmsg.items():
- try:
- # this will raise KeyError if header is not present
- msg.replace_header(hkey, hval)
- except KeyError:
- msg[hkey] = hval
-
- # all ok, replace payload by unencrypted payload
- msg.set_payload(decrmsg.get_payload())
- return (msg, valid_sig)
-
- def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding,
- senderPubkey):
- """
- Possibly decrypt an inline OpenPGP encrypted message.
-
- :param origmsg: The original, possibly encrypted message.
- :type origmsg: Message
- :param encoding: The encoding of the email message.
- :type encoding: str
- :param senderPubkey: The key of the sender of the message.
- :type senderPubkey: OpenPGPKey
-
- :return: A tuple containing a decrypted message and
- a bool indicating whether the signature is valid.
- :rtype: (Message, bool)
- """
- log.msg('maybe decrypting inline encrypted msg')
- # serialize the original message
- buf = StringIO()
- g = Generator(buf)
- g.flatten(origmsg)
- data = buf.getvalue()
- # handle exactly one inline PGP message
- valid_sig = False
- if PGP_BEGIN in data:
- begin = data.find(PGP_BEGIN)
- end = data.find(PGP_END)
- pgp_message = data[begin:end + len(PGP_END)]
- try:
- decrdata, valid_sig = self._decrypt_and_verify_data(
- pgp_message, senderPubkey)
- # replace encrypted by decrypted content
- data = data.replace(pgp_message, decrdata)
- except keymanager_errors.DecryptError:
- logger.warning('Failed to decrypt potential inline encrypted '
- 'message. Storing message as is...')
-
- # if message is not encrypted, return raw data
- if isinstance(data, unicode):
- data = data.encode(encoding, 'replace')
- return (self._parser.parsestr(data), valid_sig)
-
- def _decrypt_and_verify_data(self, data, senderPubkey):
- """
- Decrypt C{data} using our private key and attempt to verify a
- signature using C{senderPubkey}.
-
- :param data: The text to be decrypted.
- :type data: unicode
- :param senderPubkey: The public key of the sender of the message.
- :type senderPubkey: OpenPGPKey
-
- :return: The decrypted data and a boolean stating whether the
- signature could be verified.
- :rtype: (str, bool)
-
- :raise DecryptError: Raised if failed to decrypt.
- """
- log.msg('decrypting and verifying data')
- valid_sig = False
- try:
- decrdata = self._keymanager.decrypt(
- data, self._pkey,
- verify=senderPubkey)
- if senderPubkey is not None:
- valid_sig = True
- except keymanager_errors.InvalidSignature:
- decrdata = self._keymanager.decrypt(
- data, self._pkey)
- return (decrdata, valid_sig)
-
- def _add_message_locally(self, result):
- """
- Adds a message to local inbox and delete it from the incoming db
- in soledad.
-
- # XXX this comes from a gatherresult...
- :param msgtuple: a tuple consisting of a SoledadDocument
- instance containing the incoming message
- and data, the json-encoded, decrypted content of the
- incoming message
- :type msgtuple: (SoledadDocument, str)
- """
- from twisted.internet import reactor
- msgtuple = first(result)
-
- doc, data = msgtuple
- log.msg('adding message %s to local db' % (doc.doc_id,))
-
- if isinstance(data, list):
- if empty(data):
- return False
- data = data[0]
-
- def msgSavedCallback(result):
- if not empty(result):
- leap_events.signal(IMAP_MSG_SAVED_LOCALLY)
- deferLater(reactor, 0, self._delete_incoming_message, doc)
- leap_events.signal(IMAP_MSG_DELETED_INCOMING)
-
- d = self._inbox.addMessage(data, flags=(self.RECENT_FLAG,),
- notify_on_disk=True)
- d.addCallbacks(msgSavedCallback, self._errback)
-
- #
- # helpers
- #
-
- def _msg_multipart_sanity_check(self, msg):
- """
- Performs a sanity check against a multipart encrypted msg
-
- :param msg: The original encrypted message.
- :type msg: Message
- """
- # sanity check
- payload = msg.get_payload()
- if len(payload) != 2:
- raise MalformedMessage(
- 'Multipart/encrypted messages should have exactly 2 body '
- 'parts (instead of %d).' % len(payload))
- if payload[0].get_content_type() != 'application/pgp-encrypted':
- raise MalformedMessage(
- "Multipart/encrypted messages' first body part should "
- "have content type equal to 'application/pgp-encrypted' "
- "(instead of %s)." % payload[0].get_content_type())
- if payload[1].get_content_type() != 'application/octet-stream':
- raise MalformedMessage(
- "Multipart/encrypted messages' second body part should "
- "have content type equal to 'octet-stream' (instead of "
- "%s)." % payload[1].get_content_type())
-
- def _is_msg(self, keys):
- """
- Checks if the keys of a dictionary match the signature
- of the document type we use for messages.
-
- :param keys: iterable containing the strings to match.
- :type keys: iterable of strings.
- :rtype: bool
- """
- return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys
diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py
deleted file mode 100644
index 4576939..0000000
--- a/src/leap/mail/imap/fields.py
+++ /dev/null
@@ -1,173 +0,0 @@
-# -*- coding: utf-8 -*-
-# fields.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-Fields for Mailbox and Message.
-"""
-from leap.mail.imap.parser import MBoxParser
-
-
-class WithMsgFields(object):
- """
- Container class for class-attributes to be shared by
- several message-related classes.
- """
- # indexing
- CONTENT_HASH_KEY = "chash"
- PAYLOAD_HASH_KEY = "phash"
-
- # Internal representation of Message
-
- # flags doc
- UID_KEY = "uid"
- MBOX_KEY = "mbox"
- SEEN_KEY = "seen"
- DEL_KEY = "deleted"
- RECENT_KEY = "recent"
- FLAGS_KEY = "flags"
- MULTIPART_KEY = "multi"
- SIZE_KEY = "size"
-
- # headers
- HEADERS_KEY = "headers"
- DATE_KEY = "date"
- SUBJECT_KEY = "subject"
- PARTS_MAP_KEY = "part_map"
- BODY_KEY = "body" # link to phash of body
- MSGID_KEY = "msgid"
-
- # content
- LINKED_FROM_KEY = "lkf" # XXX not implemented yet!
- RAW_KEY = "raw"
- CTYPE_KEY = "ctype"
-
- # Mailbox specific keys
- CLOSED_KEY = "closed"
- CREATED_KEY = "created"
- SUBSCRIBED_KEY = "subscribed"
- RW_KEY = "rw"
- LAST_UID_KEY = "lastuid"
- RECENTFLAGS_KEY = "rct"
- HDOCS_SET_KEY = "hdocset"
-
- # Document Type, for indexing
- TYPE_KEY = "type"
- TYPE_MBOX_VAL = "mbox"
- TYPE_FLAGS_VAL = "flags"
- TYPE_HEADERS_VAL = "head"
- TYPE_CONTENT_VAL = "cnt"
- TYPE_RECENT_VAL = "rct"
- TYPE_HDOCS_SET_VAL = "hdocset"
-
- INBOX_VAL = "inbox"
-
- # Flags in Mailbox and Message
- SEEN_FLAG = "\\Seen"
- RECENT_FLAG = "\\Recent"
- ANSWERED_FLAG = "\\Answered"
- FLAGGED_FLAG = "\\Flagged" # yo dawg
- DELETED_FLAG = "\\Deleted"
- DRAFT_FLAG = "\\Draft"
- NOSELECT_FLAG = "\\Noselect"
- LIST_FLAG = "List" # is this OK? (no \. ie, no system flag)
-
- # Fields in mail object
- SUBJECT_FIELD = "Subject"
- DATE_FIELD = "Date"
-
- # Index types
- # --------------
-
- TYPE_IDX = 'by-type'
- TYPE_MBOX_IDX = 'by-type-and-mbox'
- TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid'
- TYPE_SUBS_IDX = 'by-type-and-subscribed'
- TYPE_MSGID_IDX = 'by-type-and-message-id'
- TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
- TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
- TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted'
- TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash'
- TYPE_C_HASH_IDX = 'by-type-and-contenthash'
- TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber'
- TYPE_P_HASH_IDX = 'by-type-and-payloadhash'
-
- # Tomas created the `recent and seen index`, but the semantic is not too
- # correct since the recent flag is volatile.
- TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen'
-
- # Soledad index for incoming mail, without decrypting errors.
- JUST_MAIL_IDX = "just-mail"
- # XXX the backward-compatible index, will be deprecated at 0.7
- JUST_MAIL_COMPAT_IDX = "just-mail-compat"
-
- INCOMING_KEY = "incoming"
- ERROR_DECRYPTING_KEY = "errdecr"
-
- KTYPE = TYPE_KEY
- MBOX_VAL = TYPE_MBOX_VAL
- CHASH_VAL = CONTENT_HASH_KEY
- PHASH_VAL = PAYLOAD_HASH_KEY
-
- INDEXES = {
- # generic
- TYPE_IDX: [KTYPE],
- TYPE_MBOX_IDX: [KTYPE, MBOX_VAL],
- TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, UID_KEY],
-
- # mailboxes
- TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'],
-
- # fdocs uniqueness
- TYPE_MBOX_C_HASH_IDX: [KTYPE, MBOX_VAL, CHASH_VAL],
-
- # headers doc - search by msgid.
- TYPE_MSGID_IDX: [KTYPE, MSGID_KEY],
-
- # content, headers doc
- TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL],
-
- # attachment payload dedup
- TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL],
-
- # messages
- TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'],
- TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'],
- TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'],
- TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL,
- 'bool(recent)', 'bool(seen)'],
-
- # incoming queue
- JUST_MAIL_IDX: [INCOMING_KEY,
- "bool(%s)" % (ERROR_DECRYPTING_KEY,)],
-
- # the backward-compatible index, will be deprecated at 0.7
- JUST_MAIL_COMPAT_IDX: [INCOMING_KEY],
- }
-
- MBOX_KEY = MBOX_VAL
-
- EMPTY_MBOX = {
- TYPE_KEY: MBOX_KEY,
- TYPE_MBOX_VAL: MBoxParser.INBOX_NAME,
- SUBJECT_KEY: "",
- FLAGS_KEY: [],
- CLOSED_KEY: False,
- SUBSCRIBED_KEY: False,
- RW_KEY: 1,
- LAST_UID_KEY: 0
- }
-
-fields = WithMsgFields # alias for convenience
diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py
deleted file mode 100644
index 5f0919a..0000000
--- a/src/leap/mail/imap/index.py
+++ /dev/null
@@ -1,69 +0,0 @@
-# -*- coding: utf-8 -*-
-# index.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-Index for SoledadBackedAccount, Mailbox and Messages.
-"""
-import logging
-
-from leap.common.check import leap_assert, leap_assert_type
-
-from leap.mail.imap.fields import fields
-
-
-logger = logging.getLogger(__name__)
-
-
-class IndexedDB(object):
- """
- Methods dealing with the index.
-
- This is a MixIn that needs access to the soledad instance,
- and also assumes that a INDEXES attribute is accessible to the instance.
-
- INDEXES must be a dictionary of type:
- {'index-name': ['field1', 'field2']}
- """
- # TODO we might want to move this to soledad itself, check
-
- def initialize_db(self):
- """
- Initialize the database.
- """
- leap_assert(self._soledad,
- "Need a soledad attribute accesible in the instance")
- leap_assert_type(self.INDEXES, dict)
-
- # Ask the database for currently existing indexes.
- if not self._soledad:
- logger.debug("NO SOLEDAD ON IMAP INITIALIZATION")
- return
- db_indexes = dict()
- if self._soledad is not None:
- db_indexes = dict(self._soledad.list_indexes())
- for name, expression in fields.INDEXES.items():
- if name not in db_indexes:
- # The index does not yet exist.
- self._soledad.create_index(name, *expression)
- continue
-
- if expression == db_indexes[name]:
- # The index exists and is up to date.
- continue
- # The index exists but the definition is not what expected, so we
- # delete it and add the proper index expression.
- self._soledad.delete_index(name)
- self._soledad.create_index(name, *expression)
diff --git a/src/leap/mail/imap/interfaces.py b/src/leap/mail/imap/interfaces.py
deleted file mode 100644
index c906278..0000000
--- a/src/leap/mail/imap/interfaces.py
+++ /dev/null
@@ -1,94 +0,0 @@
-# -*- coding: utf-8 -*-
-# interfaces.py
-# Copyright (C) 2014 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-Interfaces for the IMAP module.
-"""
-from zope.interface import Interface, Attribute
-
-
-class IMessageContainer(Interface):
- """
- I am a container around the different documents that a message
- is split into.
- """
- fdoc = Attribute('The flags document for this message, if any.')
- hdoc = Attribute('The headers document for this message, if any.')
- cdocs = Attribute('The dict of content documents for this message, '
- 'if any.')
-
- def walk(self):
- """
- Return an iterator to the docs for all the parts.
-
- :rtype: iterator
- """
-
-
-class IMessageStore(Interface):
- """
- I represent a generic storage for LEAP Messages.
- """
-
- def create_message(self, mbox, uid, message):
- """
- Put the passed message into this IMessageStore.
-
- :param mbox: the mbox this message belongs.
- :param uid: the UID that identifies this message in this mailbox.
- :param message: a IMessageContainer implementor.
- """
-
- def put_message(self, mbox, uid, message):
- """
- Put the passed message into this IMessageStore.
-
- :param mbox: the mbox this message belongs.
- :param uid: the UID that identifies this message in this mailbox.
- :param message: a IMessageContainer implementor.
- """
-
- def remove_message(self, mbox, uid):
- """
- Remove the given message from this IMessageStore.
-
- :param mbox: the mbox this message belongs.
- :param uid: the UID that identifies this message in this mailbox.
- """
-
- def get_message(self, mbox, uid):
- """
- Get a IMessageContainer for the given mbox and uid combination.
-
- :param mbox: the mbox this message belongs.
- :param uid: the UID that identifies this message in this mailbox.
- :return: IMessageContainer
- """
-
-
-class IMessageStoreWriter(Interface):
- """
- I represent a storage that is able to write its contents to another
- different IMessageStore.
- """
-
- def write_messages(self, store):
- """
- Write the documents in this IMessageStore to a different
- storage. Usually this will be done from a MemoryStorage to a DbStorage.
-
- :param store: another IMessageStore implementor.
- """
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 34cf535..c52a2e3 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -1,6 +1,6 @@
# *- coding: utf-8 -*-
# mailbox.py
-# Copyright (C) 2013 LEAP
+# Copyright (C) 2013-2015 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -15,36 +15,38 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
-Soledad Mailbox.
+IMAP Mailbox.
"""
-import copy
-import threading
+import re
import logging
-import StringIO
-import cStringIO
import os
+import cStringIO
+import StringIO
+import time
from collections import defaultdict
+from email.utils import formatdate
from twisted.internet import defer
-from twisted.internet.task import deferLater
+from twisted.internet import reactor
from twisted.python import log
from twisted.mail import imap4
from zope.interface import implements
-from leap.common import events as leap_events
-from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
-from leap.common.check import leap_assert, leap_assert_type
-from leap.mail.decorators import deferred_to_thread
-from leap.mail.utils import empty
-from leap.mail.imap.fields import WithMsgFields, fields
-from leap.mail.imap.messages import MessageCollection
-from leap.mail.imap.messageparts import MessageWrapper
-from leap.mail.imap.parser import MBoxParser
+from leap.common.check import leap_assert
+from leap.common.check import leap_assert_type
+from leap.mail.constants import INBOX_NAME, MessageFlags
+from leap.mail.imap.messages import IMAPMessage
logger = logging.getLogger(__name__)
+# TODO LIST
+# [ ] Restore profile_cmd instrumentation
+# [ ] finish the implementation of IMailboxListener
+# [ ] implement the rest of ISearchableMailbox
+
+
"""
If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid
notifying clients of new messages. Use during stress tests.
@@ -53,7 +55,6 @@ NOTIFY_NEW = not os.environ.get('LEAP_SKIPNOTIFY', False)
PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False)
if PROFILE_CMD:
- import time
def _debugProfiling(result, cmdname, start):
took = (time.time() - start) * 1000
@@ -70,33 +71,32 @@ if PROFILE_CMD:
d.addCallback(_debugProfiling, name, time.time())
d.addErrback(lambda f: log.msg(f.getTraceback()))
+INIT_FLAGS = (MessageFlags.SEEN_FLAG, MessageFlags.ANSWERED_FLAG,
+ MessageFlags.FLAGGED_FLAG, MessageFlags.DELETED_FLAG,
+ MessageFlags.DRAFT_FLAG, MessageFlags.RECENT_FLAG,
+ MessageFlags.LIST_FLAG)
-class SoledadMailbox(WithMsgFields, MBoxParser):
+
+class IMAPMailbox(object):
"""
A Soledad-backed IMAP mailbox.
Implements the high-level method needed for the Mailbox interfaces.
- The low-level database methods are contained in MessageCollection class,
- which we instantiate and make accessible in the `messages` attribute.
+ The low-level database methods are contained in the generic
+ MessageCollection class. We receive an instance of it and it is made
+ accessible in the `collection` attribute.
"""
implements(
imap4.IMailbox,
imap4.IMailboxInfo,
- imap4.ICloseableMailbox,
imap4.ISearchableMailbox,
+ # XXX I think we do not need to implement CloseableMailbox, do we?
+ # We could remove ourselves from the collectionListener, although I
+ # think it simply will be garbage collected.
+ # imap4.ICloseableMailbox
imap4.IMessageCopier)
- # XXX should finish the implementation of IMailboxListener
- # XXX should completely implement ISearchableMailbox too
-
- messages = None
- _closed = False
-
- INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG,
- WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG,
- WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG,
- WithMsgFields.LIST_FLAG)
- flags = None
+ init_flags = INIT_FLAGS
CMD_MSG = "MESSAGES"
CMD_RECENT = "RECENT"
@@ -104,65 +104,25 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
CMD_UIDVALIDITY = "UIDVALIDITY"
CMD_UNSEEN = "UNSEEN"
- # FIXME we should turn this into a datastructure with limited capacity
+ # TODO we should turn this into a datastructure with limited capacity
_listeners = defaultdict(set)
- next_uid_lock = threading.Lock()
- last_uid_lock = threading.Lock()
-
- # TODO unify all the `primed` dicts
- _fdoc_primed = {}
- _last_uid_primed = {}
- _known_uids_primed = {}
-
- def __init__(self, mbox, soledad, memstore, rw=1):
+ def __init__(self, collection, rw=1):
"""
- SoledadMailbox constructor. Needs to get passed a name, plus a
- Soledad instance.
-
- :param mbox: the mailbox name
- :type mbox: str
-
- :param soledad: a Soledad instance.
- :type soledad: Soledad
-
- :param memstore: a MemoryStore instance
- :type memstore: MemoryStore
+ :param collection: instance of MessageCollection
+ :type collection: MessageCollection
:param rw: read-and-write flag for this mailbox
:type rw: int
"""
- leap_assert(mbox, "Need a mailbox name to initialize")
- leap_assert(soledad, "Need a soledad instance to initialize")
-
- from twisted.internet import reactor
- self.reactor = reactor
-
- self.mbox = self._parse_mailbox_name(mbox)
self.rw = rw
-
- self._soledad = soledad
- self._memstore = memstore
-
- self.messages = MessageCollection(
- mbox=mbox, soledad=self._soledad, memstore=self._memstore)
-
self._uidvalidity = None
+ self.collection = collection
+ self.collection.addListener(self)
- # XXX careful with this get/set (it would be
- # hitting db unconditionally, move to memstore too)
- # Now it's returning a fixed amount of flags from mem
- # as a workaround.
- if not self.getFlags():
- self.setFlags(self.INIT_FLAGS)
-
- if self._memstore:
- self.prime_known_uids_to_memstore()
- self.prime_last_uid_to_memstore()
- self.prime_flag_docs_to_memstore()
-
- # purge memstore from empty fdocs.
- self._memstore.purge_fdoc_store(mbox)
+ @property
+ def mbox_name(self):
+ return self.collection.mbox_name
@property
def listeners(self):
@@ -175,11 +135,17 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:rtype: set
"""
- return self._listeners[self.mbox]
+ return self._listeners[self.mbox_name]
+
+ def get_imap_message(self, message):
+ d = defer.Deferred()
+ IMAPMessage(message, store=self.collection.store, d=d)
+ return d
- # TODO this grows too crazily when many instances are fired, like
+ # FIXME this grows too crazily when many instances are fired, like
# during imaptest stress testing. Should have a queue of limited size
# instead.
+
def addListener(self, listener):
"""
Add a listener to the listeners queue.
@@ -192,8 +158,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
if not NOTIFY_NEW:
return
- logger.debug('adding mailbox listener: %s' % listener)
- self.listeners.add(listener)
+ listeners = self.listeners
+ logger.debug('adding mailbox listener: %s. Total: %s' % (
+ listener, len(listeners)))
+ listeners.add(listener)
def removeListener(self, listener):
"""
@@ -204,17 +172,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
"""
self.listeners.remove(listener)
- def _get_mbox_doc(self):
- """
- Return mailbox document.
-
- :return: A SoledadDocument containing this mailbox, or None if
- the query failed.
- :rtype: SoledadDocument or None.
- """
- return self._memstore.get_mbox_doc(self.mbox)
-
- # XXX the memstore->soledadstore method in memstore is not complete
def getFlags(self):
"""
Returns the flags defined for this mailbox.
@@ -222,12 +179,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:returns: tuple of flags for this mailbox
:rtype: tuple of str
"""
- flags = self._memstore.get_mbox_flags(self.mbox)
+ flags = self.collection.mbox_wrapper.flags
if not flags:
- flags = self.INIT_FLAGS
- return map(str, flags)
+ flags = self.init_flags
+ flags_str = map(str, flags)
+ return flags_str
- # XXX the memstore->soledadstore method in memstore is not complete
def setFlags(self, flags):
"""
Sets flags for this mailbox.
@@ -236,87 +193,10 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:type flags: tuple of str
"""
# XXX this is setting (overriding) old flags.
+ # Better pass a mode flag
leap_assert(isinstance(flags, tuple),
"flags expected to be a tuple")
- self._memstore.set_mbox_flags(self.mbox, flags)
-
- # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG.
-
- def _get_closed(self):
- """
- Return the closed attribute for this mailbox.
-
- :return: True if the mailbox is closed
- :rtype: bool
- """
- return self._memstore.get_mbox_closed(self.mbox)
-
- def _set_closed(self, closed):
- """
- Set the closed attribute for this mailbox.
-
- :param closed: the state to be set
- :type closed: bool
- """
- self._memstore.set_mbox_closed(self.mbox, closed)
-
- closed = property(
- _get_closed, _set_closed, doc="Closed attribute.")
-
- def _get_last_uid(self):
- """
- Return the last uid for this mailbox.
- If we have a memory store, the last UID will be the highest
- recorded UID in the message store, or a counter cached from
- the mailbox document in soledad if this is higher.
-
- :return: the last uid for messages in this mailbox
- :rtype: int
- """
- last = self._memstore.get_last_uid(self.mbox)
- logger.debug("last uid for %s: %s (from memstore)" % (
- repr(self.mbox), last))
- return last
-
- last_uid = property(
- _get_last_uid, doc="Last_UID attribute.")
-
- def prime_last_uid_to_memstore(self):
- """
- Prime memstore with last_uid value
- """
- primed = self._last_uid_primed.get(self.mbox, False)
- if not primed:
- mbox = self._get_mbox_doc()
- if mbox is None:
- # memory-only store
- return
- last = mbox.content.get('lastuid', 0)
- logger.info("Priming Soledad last_uid to %s" % (last,))
- self._memstore.set_last_soledad_uid(self.mbox, last)
- self._last_uid_primed[self.mbox] = True
-
- def prime_known_uids_to_memstore(self):
- """
- Prime memstore with the set of all known uids.
-
- We do this to be able to filter the requests efficiently.
- """
- primed = self._known_uids_primed.get(self.mbox, False)
- if not primed:
- known_uids = self.messages.all_soledad_uid_iter()
- self._memstore.set_known_uids(self.mbox, known_uids)
- self._known_uids_primed[self.mbox] = True
-
- def prime_flag_docs_to_memstore(self):
- """
- Prime memstore with all the flags documents.
- """
- primed = self._fdoc_primed.get(self.mbox, False)
- if not primed:
- all_flag_docs = self.messages.get_all_soledad_flag_docs()
- self._memstore.load_flag_docs(self.mbox, all_flag_docs)
- self._fdoc_primed[self.mbox] = True
+ return self.collection.set_mbox_attr("flags", flags)
def getUIDValidity(self):
"""
@@ -325,14 +205,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:return: unique validity identifier
:rtype: int
"""
- if self._uidvalidity is None:
- mbox = self._get_mbox_doc()
- if mbox is None:
- return 0
- self._uidvalidity = mbox.content.get(self.CREATED_KEY, 1)
- return self._uidvalidity
+ return self.collection.get_mbox_attr("created")
- def getUID(self, message):
+ def getUID(self, message_number):
"""
Return the UID of a message in the mailbox
@@ -340,14 +215,15 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
but in the future will be useful to get absolute UIDs from
message sequence numbers.
- :param message: the message uid
+ :param message: the message sequence number.
:type message: int
:rtype: int
+ :return: the UID of the message.
"""
- msg = self.messages.get_msg_by_uid(message)
- if msg is not None:
- return msg.getUID()
+ # TODO support relative sequences. The (imap) message should
+ # receive a sequence number attribute: a deferred is not expected
+ return message_number
def getUIDNext(self):
"""
@@ -355,23 +231,20 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
mailbox. Currently it returns the higher UID incremented by
one.
- We increment the next uid *each* time this function gets called.
- In this way, there will be gaps if the message with the allocated
- uid cannot be saved. But that is preferable to having race conditions
- if we get to parallel message adding.
-
- :rtype: int
+ :return: deferred with int
+ :rtype: Deferred
"""
- with self.next_uid_lock:
- return self.last_uid + 1
+ d = self.collection.get_uid_next()
+ return d
def getMessageCount(self):
"""
Returns the total count of messages in this mailbox.
- :rtype: int
+ :return: deferred with int
+ :rtype: Deferred
"""
- return self.messages.count()
+ return self.collection.count()
def getUnseenCount(self):
"""
@@ -380,7 +253,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:return: count of messages flagged `unseen`
:rtype: int
"""
- return self.messages.count_unseen()
+ return self.collection.count_unseen()
def getRecentCount(self):
"""
@@ -389,7 +262,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:return: count of messages flagged `recent`
:rtype: int
"""
- return self.messages.count_recent()
+ return self.collection.count_recent()
def isWriteable(self):
"""
@@ -398,6 +271,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:return: 1 if mailbox is read-writeable, 0 otherwise.
:rtype: int
"""
+ # XXX We don't need to store it in the mbox doc, do we?
+ # return int(self.collection.get_mbox_attr('rw'))
return self.rw
def getHierarchicalDelimiter(self):
@@ -417,19 +292,26 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:type names: iter
"""
r = {}
+ maybe = defer.maybeDeferred
if self.CMD_MSG in names:
- r[self.CMD_MSG] = self.getMessageCount()
+ r[self.CMD_MSG] = maybe(self.getMessageCount)
if self.CMD_RECENT in names:
- r[self.CMD_RECENT] = self.getRecentCount()
+ r[self.CMD_RECENT] = maybe(self.getRecentCount)
if self.CMD_UIDNEXT in names:
- r[self.CMD_UIDNEXT] = self.last_uid + 1
+ r[self.CMD_UIDNEXT] = maybe(self.getUIDNext)
if self.CMD_UIDVALIDITY in names:
- r[self.CMD_UIDVALIDITY] = self.getUIDValidity()
+ r[self.CMD_UIDVALIDITY] = maybe(self.getUIDValidity)
if self.CMD_UNSEEN in names:
- r[self.CMD_UNSEEN] = self.getUnseenCount()
- return defer.succeed(r)
+ r[self.CMD_UNSEEN] = maybe(self.getUnseenCount)
+
+ def as_a_dict(values):
+ return dict(zip(r.keys(), values))
- def addMessage(self, message, flags, date=None, notify_on_disk=False):
+ d = defer.gatherResults(r.values())
+ d.addCallback(as_a_dict)
+ return d
+
+ def addMessage(self, message, flags, date=None, notify_just_mdoc=True):
"""
Adds a message to this mailbox.
@@ -440,51 +322,69 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:type flags: list of str
:param date: timestamp
- :type date: str
-
- :return: a deferred that evals to None
- """
+ :type date: str, or None
+
+ :param notify_just_mdoc:
+ boolean passed to the wrapper.create method, to indicate whether
+ we're insterested in being notified right after the mdoc has been
+ written (as it's the first doc to be written, and quite small, this
+ is faster, though potentially unsafe).
+ Setting it to True improves a *lot* the responsiveness of the
+ APPENDS: we just need to be notified when the mdoc is saved, and
+ let's just expect that the other parts are doing just fine. This
+ will not catch any errors when the inserts of the other parts
+ fail, but on the other hand allows us to return very quickly,
+ which seems a good compromise given that we have to serialize the
+ appends.
+ However, some operations like the saving of drafts need to wait for
+ all the parts to be saved, so if some heuristics are met down in
+ the call chain a Draft message will unconditionally set this flag
+ to False, and therefore ignoring the setting of this flag here.
+ :type notify_just_mdoc: bool
+
+ :return: a deferred that will be triggered with the UID of the added
+ message.
+ """
+ # TODO should raise ReadOnlyMailbox if not rw.
# TODO have a look at the cases for internal date in the rfc
+ # XXX we could treat the message as an IMessage from here
+
+ # TODO change notify_just_mdoc to something more meaningful, like
+ # fast_insert_notify?
+
+ # TODO notify_just_mdoc *sometimes* make the append tests fail.
+ # have to find a better solution for this. A workaround could probably
+ # be to have a list of the ongoing deferreds related to append, so that
+ # we queue for later all the requests having to do with these.
+
+ # A better solution will probably involve implementing MULTIAPPEND
+ # extension or patching imap server to support pipelining.
+
if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)):
message = message.getvalue()
- # XXX we could treat the message as an IMessage from here
leap_assert_type(message, basestring)
+
if flags is None:
flags = tuple()
else:
flags = tuple(str(flag) for flag in flags)
- d = self._do_add_message(message, flags=flags, date=date,
- notify_on_disk=notify_on_disk)
- if PROFILE_CMD:
- do_profile_cmd(d, "APPEND")
-
- # XXX should review now that we're not using qtreactor.
- # A better place for this would be the COPY/APPEND dispatcher
- # in server.py, but qtreactor hangs when I do that, so this seems
- # to work fine for now.
-
- def notifyCallback(x):
- self.reactor.callLater(0, self.notify_new)
- return x
+ if date is None:
+ date = formatdate(time.time())
- d.addCallback(notifyCallback)
- d.addErrback(lambda f: log.msg(f.getTraceback()))
- return d
-
- def _do_add_message(self, message, flags, date, notify_on_disk=False):
- """
- Calls to the messageCollection add_msg method.
- Invoked from addMessage.
- """
- d = self.messages.add_msg(message, flags=flags, date=date,
- notify_on_disk=notify_on_disk)
+ d = self.collection.add_msg(message, flags, date=date,
+ notify_just_mdoc=notify_just_mdoc)
+ d.addErrback(lambda failure: log.err(failure))
return d
def notify_new(self, *args):
"""
Notify of new messages to all the listeners.
+ This will be called indirectly by the underlying collection, that will
+ notify this IMAPMailbox whenever there are changes in the number of
+ messages in the collection, since we have added ourselves to the
+ collection listeners.
:param args: ignored.
"""
@@ -493,26 +393,36 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
def cbNotifyNew(result):
exists, recent = result
- for l in self.listeners:
- l.newMessages(exists, recent)
+ for listener in self.listeners:
+ listener.newMessages(exists, recent)
+
d = self._get_notify_count()
d.addCallback(cbNotifyNew)
- d.addCallback(self.cb_signal_unread_to_ui)
+ d.addCallback(self.collection.cb_signal_unread_to_ui)
+ d.addErrback(lambda failure: log.err(failure))
- @deferred_to_thread
def _get_notify_count(self):
"""
Get message count and recent count for this mailbox
Executed in a separate thread. Called from notify_new.
- :return: number of messages and number of recent messages.
- :rtype: tuple
+ :return: a deferred that will fire with a tuple, with number of
+ messages and number of recent messages.
+ :rtype: Deferred
"""
- exists = self.getMessageCount()
- recent = self.getRecentCount()
- logger.debug("NOTIFY (%r): there are %s messages, %s recent" % (
- self.mbox, exists, recent))
- return exists, recent
+ d_exists = defer.maybeDeferred(self.getMessageCount)
+ d_recent = defer.maybeDeferred(self.getRecentCount)
+ d_list = [d_exists, d_recent]
+
+ def log_num_msg(result):
+ exists, recent = tuple(result)
+ logger.debug("NOTIFY (%r): there are %s messages, %s recent" % (
+ self.mbox_name, exists, recent))
+ return result
+
+ d = defer.gatherResults(d_list)
+ d.addCallback(log_num_msg)
+ return d
# commands, do not rename methods
@@ -522,31 +432,21 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
Should cleanup resources, and set the \\Noselect flag
on the mailbox.
+
"""
- # XXX this will overwrite all the existing flags!
+ # XXX this will overwrite all the existing flags
# should better simply addFlag
- self.setFlags((self.NOSELECT_FLAG,))
- self.deleteAllDocs()
+ self.setFlags((MessageFlags.NOSELECT_FLAG,))
- # XXX removing the mailbox in situ for now,
- # we should postpone the removal
+ def remove_mbox(_):
+ uuid = self.collection.mbox_uuid
+ d = self.collection.mbox_wrapper.delete(self.collection.store)
+ d.addCallback(
+ lambda _: self.collection.mbox_indexer.delete_table(uuid))
+ return d
- # XXX move to memory store??
- mbox_doc = self._get_mbox_doc()
- if mbox_doc is None:
- # memory-only store!
- return
- self._soledad.delete_doc(self._get_mbox_doc())
-
- def _close_cb(self, result):
- self.closed = True
-
- def close(self):
- """
- Expunge and mark as closed
- """
- d = self.expunge()
- d.addCallback(self._close_cb)
+ d = self.deleteAllDocs()
+ d.addCallback(remove_mbox)
return d
def expunge(self):
@@ -555,11 +455,35 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
"""
if not self.isWriteable():
raise imap4.ReadOnlyMailbox
- d = defer.Deferred()
- self._memstore.expunge(self.mbox, d)
+ return self.collection.delete_all_flagged()
+
+ def _get_message_fun(self, uid):
+ """
+ Return the proper method to get a message for this mailbox, depending
+ on the passed uid flag.
+
+ :param uid: If true, the IDs specified in the query are UIDs;
+ otherwise they are message sequence IDs.
+ :type uid: bool
+ :rtype: callable
+ """
+ get_message_fun = [
+ self.collection.get_message_by_sequence_number,
+ self.collection.get_message_by_uid][uid]
+ return get_message_fun
+
+ def _get_messages_range(self, messages_asked, uid=True):
+
+ def get_range(messages_asked):
+ return self._filter_msg_seq(messages_asked)
+
+ d = defer.maybeDeferred(self._bound_seq, messages_asked, uid)
+ if uid:
+ d.addCallback(get_range)
+ d.addErrback(lambda f: log.err(f))
return d
- def _bound_seq(self, messages_asked):
+ def _bound_seq(self, messages_asked, uid):
"""
Put an upper bound to a messages sequence if this is open.
@@ -567,15 +491,27 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:type messages_asked: MessageSet
:rtype: MessageSet
"""
+
+ def set_last_uid(last_uid):
+ messages_asked.last = last_uid
+ return messages_asked
+
+ def set_last_seq(all_uid):
+ messages_asked.last = len(all_uid)
+ return messages_asked
+
if not messages_asked.last:
try:
iter(messages_asked)
except TypeError:
# looks like we cannot iterate
- try:
- messages_asked.last = self.last_uid
- except ValueError:
- pass
+ if uid:
+ d = self.collection.get_last_uid()
+ d.addCallback(set_last_uid)
+ else:
+ d = self.collection.all_uid_iter()
+ d.addCallback(set_last_seq)
+ return d
return messages_asked
def _filter_msg_seq(self, messages_asked):
@@ -587,10 +523,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:type messages_asked: MessageSet
:rtype: set
"""
- set_asked = set(messages_asked)
- set_exist = set(self.messages.all_uid_iter())
- seq_messg = set_asked.intersection(set_exist)
- return seq_messg
+ # TODO we could pass the asked sequence to the indexer
+ # all_uid_iter, and bound the sql query instead.
+ def filter_by_asked(all_msg_uid):
+ set_asked = set(messages_asked)
+ set_exist = set(all_msg_uid)
+ return set_asked.intersection(set_exist)
+
+ d = self.collection.all_uid_iter()
+ d.addCallback(filter_by_asked)
+ return d
def fetch(self, messages_asked, uid):
"""
@@ -607,54 +549,48 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
otherwise.
:type uid: bool
- :rtype: deferred
- """
- d = defer.Deferred()
- self.reactor.callInThread(self._do_fetch, messages_asked, uid, d)
- if PROFILE_CMD:
- do_profile_cmd(d, "FETCH")
- d.addCallback(self.cb_signal_unread_to_ui)
+ :rtype: deferred with a generator that yields...
+ """
+ get_msg_fun = self._get_message_fun(uid)
+ getimapmsg = self.get_imap_message
+
+ def get_imap_messages_for_range(msg_range):
+
+ def _get_imap_msg(messages):
+ d_imapmsg = []
+ for msg in messages:
+ d_imapmsg.append(getimapmsg(msg))
+ return defer.gatherResults(d_imapmsg, consumeErrors=True)
+
+ def _zip_msgid(imap_messages):
+ zipped = zip(
+ list(msg_range), imap_messages)
+ return (item for item in zipped)
+
+ # XXX not called??
+ def _unset_recent(sequence):
+ reactor.callLater(0, self.unset_recent_flags, sequence)
+ return sequence
+
+ d_msg = []
+ for msgid in msg_range:
+ # XXX We want cdocs because we "probably" are asked for the
+ # body. We should be smarter at do_FETCH and pass a parameter
+ # to this method in order not to prefetch cdocs if they're not
+ # going to be used.
+ d_msg.append(get_msg_fun(msgid, get_cdocs=True))
+
+ d = defer.gatherResults(d_msg, consumeErrors=True)
+ d.addCallback(_get_imap_msg)
+ d.addCallback(_zip_msgid)
+ d.addErrback(lambda failure: log.err(failure))
+ return d
+
+ d = self._get_messages_range(messages_asked, uid)
+ d.addCallback(get_imap_messages_for_range)
+ d.addErrback(lambda failure: log.err(failure))
return d
- # called in thread
- def _do_fetch(self, messages_asked, uid, d):
- """
- :param messages_asked: IDs of the messages to retrieve information
- about
- :type messages_asked: MessageSet
-
- :param uid: If true, the IDs are UIDs. They are message sequence IDs
- otherwise.
- :type uid: bool
- :param d: deferred whose callback will be called with result.
- :type d: Deferred
-
- :rtype: A tuple of two-tuples of message sequence numbers and
- LeapMessage
- """
- # For the moment our UID is sequential, so we
- # can treat them all the same.
- # Change this to the flag that twisted expects when we
- # switch to content-hash based index + local UID table.
-
- sequence = False
- # sequence = True if uid == 0 else False
-
- messages_asked = self._bound_seq(messages_asked)
- seq_messg = self._filter_msg_seq(messages_asked)
- getmsg = lambda uid: self.messages.get_msg_by_uid(uid)
-
- # for sequence numbers (uid = 0)
- if sequence:
- logger.debug("Getting msg by index: INEFFICIENT call!")
- raise NotImplementedError
- else:
- got_msg = ((msgid, getmsg(msgid)) for msgid in seq_messg)
- result = ((msgid, msg) for msgid, msg in got_msg
- if msg is not None)
- self.reactor.callLater(0, self.unset_recent_flags, seq_messg)
- self.reactor.callFromThread(d.callback, result)
-
def fetch_flags(self, messages_asked, uid):
"""
A fast method to fetch all flags, tricking just the
@@ -679,13 +615,23 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
MessagePart.
:rtype: tuple
"""
+ # is_sequence = True if uid == 0 else False
+ # XXX FIXME -----------------------------------------------------
+ # imap/tests, or muas like mutt, it will choke until we implement
+ # sequence numbers. This is an easy hack meanwhile.
+ is_sequence = False
+ # ---------------------------------------------------------------
+
+ if is_sequence:
+ raise NotImplementedError(
+ "FETCH FLAGS NOT IMPLEMENTED FOR MESSAGE SEQUENCE NUMBERS YET")
+
d = defer.Deferred()
- self.reactor.callInThread(self._do_fetch_flags, messages_asked, uid, d)
+ reactor.callLater(0, self._do_fetch_flags, messages_asked, uid, d)
if PROFILE_CMD:
do_profile_cmd(d, "FETCH-ALL-FLAGS")
return d
- # called in thread
def _do_fetch_flags(self, messages_asked, uid, d):
"""
:param messages_asked: IDs of the messages to retrieve information
@@ -698,8 +644,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:param d: deferred whose callback will be called with result.
:type d: Deferred
- :rtype: A tuple of two-tuples of message sequence numbers and
- flagsPart
+ :rtype: A generator that yields two-tuples of message sequence numbers
+ and flagsPart
"""
class flagsPart(object):
def __init__(self, uid, flags):
@@ -712,13 +658,28 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
def getFlags(self):
return map(str, self.flags)
- messages_asked = self._bound_seq(messages_asked)
- seq_messg = self._filter_msg_seq(messages_asked)
-
- all_flags = self._memstore.all_flags(self.mbox)
- result = ((msgid, flagsPart(
- msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg)
- self.reactor.callFromThread(d.callback, result)
+ def pack_flags(result):
+ _uid, _flags = result
+ return _uid, flagsPart(_uid, _flags)
+
+ def get_flags_for_seq(sequence):
+ d_all_flags = []
+ for msgid in sequence:
+ # TODO implement sequence numbers here too
+ d_flags_per_uid = self.collection.get_flags_by_uid(msgid)
+ d_flags_per_uid.addCallback(pack_flags)
+ d_all_flags.append(d_flags_per_uid)
+ gotflags = defer.gatherResults(d_all_flags)
+ gotflags.addCallback(get_uid_flag_generator)
+ return gotflags
+
+ def get_uid_flag_generator(result):
+ generator = (item for item in result)
+ d.callback(generator)
+
+ d_seq = self._get_messages_range(messages_asked, uid)
+ d_seq.addCallback(get_flags_for_seq)
+ return d_seq
def fetch_headers(self, messages_asked, uid):
"""
@@ -744,7 +705,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
MessagePart.
:rtype: tuple
"""
- # TODO how often is thunderbird doing this?
+ # TODO implement sequences
+ is_sequence = True if uid == 0 else False
+ if is_sequence:
+ raise NotImplementedError(
+ "FETCH HEADERS NOT IMPLEMENTED FOR SEQUENCE NUMBER YET")
class headersPart(object):
def __init__(self, uid, headers):
@@ -769,29 +734,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
for msgid in seq_messg)
return result
- def cb_signal_unread_to_ui(self, result):
- """
- Sends unread event to ui.
- Used as a callback in several commands.
-
- :param result: ignored
- """
- d = self._get_unseen_deferred()
- d.addCallback(self.__cb_signal_unread_to_ui)
- return result
-
- @deferred_to_thread
- def _get_unseen_deferred(self):
- return self.getUnseenCount()
-
- def __cb_signal_unread_to_ui(self, unseen):
- """
- Send the unread signal to UI.
- :param unseen: number of unseen messages.
- :type unseen: int
- """
- leap_events.signal(IMAP_UNREAD_MAIL, str(unseen))
-
def store(self, messages_asked, flags, mode, uid):
"""
Sets the flags of one or more messages.
@@ -826,17 +768,18 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
raise imap4.ReadOnlyMailbox
d = defer.Deferred()
- self.reactor.callLater(0, self._do_store, messages_asked, flags,
- mode, uid, d)
+ reactor.callLater(0, self._do_store, messages_asked, flags,
+ mode, uid, d)
if PROFILE_CMD:
do_profile_cmd(d, "STORE")
- d.addCallback(self.cb_signal_unread_to_ui)
- d.addErrback(lambda f: log.msg(f.getTraceback()))
+
+ d.addCallback(self.collection.cb_signal_unread_to_ui)
+ d.addErrback(lambda f: log.err(f))
return d
def _do_store(self, messages_asked, flags, mode, uid, observer):
"""
- Helper method, invoke set_flags method in the MessageCollection.
+ Helper method, invoke set_flags method in the IMAPMessageCollection.
See the documentation for the `store` method for the parameters.
@@ -845,14 +788,31 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
done.
:type observer: deferred
"""
- # XXX implement also sequence (uid = 0)
- # XXX we should prevent client from setting Recent flag?
+ # TODO we should prevent client from setting Recent flag
+ get_msg_fun = self._get_message_fun(uid)
leap_assert(not isinstance(flags, basestring),
"flags cannot be a string")
flags = tuple(flags)
- messages_asked = self._bound_seq(messages_asked)
- seq_messg = self._filter_msg_seq(messages_asked)
- self.messages.set_flags(self.mbox, seq_messg, flags, mode, observer)
+
+ def set_flags_for_seq(sequence):
+ def return_result_dict(list_of_flags):
+ result = dict(zip(list(sequence), list_of_flags))
+ observer.callback(result)
+ return result
+
+ d_all_set = []
+ for msgid in sequence:
+ d = get_msg_fun(msgid)
+ d.addCallback(lambda msg: self.collection.update_flags(
+ msg, flags, mode))
+ d_all_set.append(d)
+ got_flags_setted = defer.gatherResults(d_all_set)
+ got_flags_setted.addCallback(return_result_dict)
+ return got_flags_setted
+
+ d_seq = self._get_messages_range(messages_asked, uid)
+ d_seq.addCallback(set_flags_for_seq)
+ return d_seq
# ISearchableMailbox
@@ -877,23 +837,24 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:rtype: C{list} or C{Deferred}
"""
# TODO see if we can raise w/o interrupting flow
- #:raise IllegalQueryError: Raised when query is not valid.
+ # :raise IllegalQueryError: Raised when query is not valid.
# example query:
# ['UNDELETED', 'HEADER', 'Message-ID',
+ # XXX fixme, does not exist
# '52D44F11.9060107@dev.bitmask.net']
# TODO hardcoding for now! -- we'll support generic queries later on
- # but doing a quickfix for avoiding duplicat saves in the draft folder.
- # See issue #4209
+ # but doing a quickfix for avoiding duplicate saves in the draft
+ # folder. # See issue #4209
if len(query) > 2:
if query[1] == 'HEADER' and query[2].lower() == "message-id":
msgid = str(query[3]).strip()
logger.debug("Searching for %s" % (msgid,))
- d = self.messages._get_uid_from_msgid(str(msgid))
- d1 = defer.gatherResults([d])
- # we want a list, so return it all the same
- return d1
+
+ d = self.collection.get_uid_from_msgid(str(msgid))
+ d.addCallback(lambda result: [result])
+ return d
# nothing implemented for any other query
logger.warning("Cannot process query: %s" % (query,))
@@ -911,94 +872,19 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
uid when the copy succeed.
:rtype: Deferred
"""
- d = defer.Deferred()
if PROFILE_CMD:
do_profile_cmd(d, "COPY")
# A better place for this would be the COPY/APPEND dispatcher
# in server.py, but qtreactor hangs when I do that, so this seems
# to work fine for now.
- d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new))
- deferLater(self.reactor, 0, self._do_copy, message, d)
- return d
-
- def _do_copy(self, message, observer):
- """
- Call invoked from the deferLater in `copy`. This will
- copy the flags and header documents, and pass them to the
- `create_message` method in the MemoryStore, together with
- the observer deferred that we've been passed along.
-
- :param message: an IMessage implementor
- :type message: LeapMessage
- :param observer: the deferred that will fire with the
- UID of the message
- :type observer: Deferred
- """
- memstore = self._memstore
-
- def createCopy(result):
- exist, new_fdoc = result
- if exist:
- # Should we signal error on the callback?
- logger.warning("Destination message already exists!")
-
- # XXX I'm not sure if we should raise the
- # errback. This actually rases an ugly warning
- # in some muas like thunderbird.
- # UID 0 seems a good convention for no uid.
- observer.callback(0)
- else:
- mbox = self.mbox
- uid_next = memstore.increment_last_soledad_uid(mbox)
-
- new_fdoc[self.UID_KEY] = uid_next
- new_fdoc[self.MBOX_KEY] = mbox
-
- flags = list(new_fdoc[self.FLAGS_KEY])
- flags.append(fields.RECENT_FLAG)
- new_fdoc[self.FLAGS_KEY] = tuple(set(flags))
-
- # FIXME set recent!
-
- self._memstore.create_message(
- self.mbox, uid_next,
- MessageWrapper(new_fdoc),
- observer=observer,
- notify_on_disk=False)
-
- d = self._get_msg_copy(message)
- d.addCallback(createCopy)
- d.addErrback(lambda f: log.msg(f.getTraceback()))
-
- @deferred_to_thread
- def _get_msg_copy(self, message):
- """
- Get a copy of the fdoc for this message, and check whether
- it already exists.
-
- :param message: an IMessage implementor
- :type message: LeapMessage
- :return: exist, new_fdoc
- :rtype: tuple
- """
- # XXX for clarity, this could be delegated to a
- # MessageCollection mixin that implements copy too, and
- # moved out of here.
- msg = message
- memstore = self._memstore
-
- if empty(msg.fdoc):
- logger.warning("Tried to copy a MSG with no fdoc")
- return
- new_fdoc = copy.deepcopy(msg.fdoc.content)
- fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY]
+ # d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new))
+ # deferLater(self.reactor, 0, self._do_copy, message, d)
+ # return d
- dest_fdoc = memstore.get_fdoc_from_chash(
- fdoc_chash, self.mbox)
-
- exist = not empty(dest_fdoc)
- return exist, new_fdoc
+ d = self.collection.copy_msg(message.message,
+ self.collection.mbox_uuid)
+ return d
# convenience fun
@@ -1006,19 +892,42 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
"""
Delete all docs in this mailbox
"""
- docs = self.messages.get_all_docs()
- for doc in docs:
- self.messages._soledad.delete_doc(doc)
+ # FIXME not implemented
+ return self.collection.delete_all_docs()
def unset_recent_flags(self, uid_seq):
"""
Unset Recent flag for a sequence of UIDs.
"""
- self.messages.unset_recent_flags(uid_seq)
+ # FIXME not implemented
+ return self.collection.unset_recent_flags(uid_seq)
def __repr__(self):
"""
Representation string for this mailbox.
"""
- return u"<SoledadMailbox: mbox '%s' (%s)>" % (
- self.mbox, self.messages.count())
+ return u"<IMAPMailbox: mbox '%s' (%s)>" % (
+ self.mbox_name, self.collection.count())
+
+
+_INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
+
+
+def normalize_mailbox(name):
+ """
+ Return a normalized representation of the mailbox ``name``.
+
+ This method ensures that an eventual initial 'inbox' part of a
+ mailbox name is made uppercase.
+
+ :param name: the name of the mailbox
+ :type name: unicode
+
+ :rtype: unicode
+ """
+ # XXX maybe it would make sense to normalize common folders too:
+ # trash, sent, drafts, etc...
+ if _INBOX_RE.match(name):
+ # ensure inital INBOX is uppercase
+ return INBOX_NAME + name[len(INBOX_NAME):]
+ return name
diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py
deleted file mode 100644
index e075394..0000000
--- a/src/leap/mail/imap/memorystore.py
+++ /dev/null
@@ -1,1333 +0,0 @@
-# -*- coding: utf-8 -*-
-# memorystore.py
-# Copyright (C) 2014 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-In-memory transient store for a LEAPIMAPServer.
-"""
-import contextlib
-import logging
-import threading
-import weakref
-
-from collections import defaultdict
-from copy import copy
-
-from enum import Enum
-from twisted.internet import defer
-from twisted.internet import reactor
-from twisted.internet.task import LoopingCall
-from twisted.python import log
-from zope.interface import implements
-
-from leap.common.check import leap_assert_type
-from leap.mail import size
-from leap.mail.utils import empty, phash_iter
-from leap.mail.messageflow import MessageProducer
-from leap.mail.imap import interfaces
-from leap.mail.imap.fields import fields
-from leap.mail.imap.messageparts import MessagePartType, MessagePartDoc
-from leap.mail.imap.messageparts import RecentFlagsDoc
-from leap.mail.imap.messageparts import MessageWrapper
-from leap.mail.imap.messageparts import ReferenciableDict
-
-from leap.mail.decorators import deferred_to_thread
-
-logger = logging.getLogger(__name__)
-
-
-# The default period to do writebacks to the permanent
-# soledad storage, in seconds.
-SOLEDAD_WRITE_PERIOD = 15
-
-FDOC = MessagePartType.fdoc.name
-HDOC = MessagePartType.hdoc.name
-CDOCS = MessagePartType.cdocs.name
-DOCS_ID = MessagePartType.docs_id.name
-
-
-@contextlib.contextmanager
-def set_bool_flag(obj, att):
- """
- Set a boolean flag to True while we're doing our thing.
- Just to let the world know.
- """
- setattr(obj, att, True)
- try:
- yield True
- except RuntimeError as exc:
- logger.exception(exc)
- finally:
- setattr(obj, att, False)
-
-
-DirtyState = Enum("DirtyState", "none dirty new")
-
-
-class MemoryStore(object):
- """
- An in-memory store to where we can write the different parts that
- we split the messages into and buffer them until we write them to the
- permanent storage.
-
- It uses MessageWrapper instances to represent the message-parts, which are
- indexed by mailbox name and UID.
-
- It also can be passed a permanent storage as a paremeter (any implementor
- of IMessageStore, in this case a SoledadStore). In this case, a periodic
- dump of the messages stored in memory will be done. The period of the
- writes to the permanent storage is controled by the write_period parameter
- in the constructor.
- """
- implements(interfaces.IMessageStore,
- interfaces.IMessageStoreWriter)
-
- # TODO We will want to index by chash when we transition to local-only
- # UIDs.
-
- WRITING_FLAG = "_writing"
- _last_uid_lock = threading.Lock()
- _fdoc_docid_lock = threading.Lock()
-
- def __init__(self, permanent_store=None,
- write_period=SOLEDAD_WRITE_PERIOD):
- """
- Initialize a MemoryStore.
-
- :param permanent_store: a IMessageStore implementor to dump
- messages to.
- :type permanent_store: IMessageStore
- :param write_period: the interval to dump messages to disk, in seconds.
- :type write_period: int
- """
- self.reactor = reactor
-
- self._permanent_store = permanent_store
- self._write_period = write_period
-
- if permanent_store is None:
- self._mbox_closed = defaultdict(lambda: False)
-
- # Internal Storage: messages
- """
- flags document store.
- _fdoc_store[mbox][uid] = { 'content': 'aaa' }
- """
- self._fdoc_store = defaultdict(lambda: defaultdict(
- lambda: ReferenciableDict({})))
-
- # Sizes
- """
- {'mbox, uid': <int>}
- """
- self._sizes = {}
-
- # Internal Storage: payload-hash
- """
- fdocs:doc-id store, stores document IDs for putting
- the dirty flags-docs.
- """
- self._fdoc_id_store = defaultdict(lambda: defaultdict(
- lambda: ''))
-
- # Internal Storage: content-hash:hdoc
- """
- hdoc-store keeps references to
- the header-documents indexed by content-hash.
-
- {'chash': { dict-stuff }
- }
- """
- self._hdoc_store = defaultdict(lambda: ReferenciableDict({}))
-
- # Internal Storage: payload-hash:cdoc
- """
- content-docs stored by payload-hash
- {'phash': { dict-stuff } }
- """
- self._cdoc_store = defaultdict(lambda: ReferenciableDict({}))
-
- # Internal Storage: content-hash:fdoc
- """
- chash-fdoc-store keeps references to
- the flag-documents indexed by content-hash.
-
- {'chash': {'mbox-a': weakref.proxy(dict),
- 'mbox-b': weakref.proxy(dict)}
- }
- """
- self._chash_fdoc_store = defaultdict(lambda: defaultdict(lambda: None))
-
- # Internal Storage: recent-flags store
- """
- recent-flags store keeps one dict per mailbox,
- with the document-id of the u1db document
- and the set of the UIDs that have the recent flag.
-
- {'mbox-a': {'doc_id': 'deadbeef',
- 'set': {1,2,3,4}
- }
- }
- """
- # TODO this will have to transition to content-hash
- # indexes after we move to local-only UIDs.
-
- self._rflags_store = defaultdict(
- lambda: {'doc_id': None, 'set': set([])})
-
- """
- last-uid store keeps the count of the highest UID
- per mailbox.
-
- {'mbox-a': 42,
- 'mbox-b': 23}
- """
- self._last_uid = defaultdict(lambda: 0)
-
- """
- known-uids keeps a count of the uids that soledad knows for a given
- mailbox
-
- {'mbox-a': set([1,2,3])}
- """
- self._known_uids = defaultdict(set)
-
- """
- mbox-flags is a dict containing flags for each mailbox. this is
- modified from mailbox.getFlags / mailbox.setFlags
- """
- self._mbox_flags = defaultdict(set)
-
- # New and dirty flags, to set MessageWrapper State.
- self._new = set([])
- self._new_queue = set([])
- self._new_deferreds = {}
-
- self._dirty = set([])
- self._dirty_queue = set([])
- self._dirty_deferreds = {}
-
- self._rflags_dirty = set([])
-
- # Flag for signaling we're busy writing to the disk storage.
- setattr(self, self.WRITING_FLAG, False)
-
- if self._permanent_store is not None:
- # this producer spits its messages to the permanent store
- # consumer using a queue. We will use that to put
- # our messages to be written.
- self.producer = MessageProducer(permanent_store,
- period=0.1)
- # looping call for dumping to SoledadStore
- self._write_loop = LoopingCall(self.write_messages,
- permanent_store)
-
- # We can start the write loop right now, why wait?
- self._start_write_loop()
- else:
- # We have a memory-only store.
- self.producer = None
- self._write_loop = None
-
- def _start_write_loop(self):
- """
- Start loop for writing to disk database.
- """
- if self._write_loop is None:
- return
- if not self._write_loop.running:
- self._write_loop.start(self._write_period, now=True)
-
- def _stop_write_loop(self):
- """
- Stop loop for writing to disk database.
- """
- if self._write_loop is None:
- return
- if self._write_loop.running:
- self._write_loop.stop()
-
- # IMessageStore
-
- # XXX this would work well for whole message operations.
- # We would have to add a put_flags operation to modify only
- # the flags doc (and set the dirty flag accordingly)
-
- def create_message(self, mbox, uid, message, observer,
- notify_on_disk=True):
- """
- Create the passed message into this MemoryStore.
-
- By default we consider that any message is a new message.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param uid: the UID for the message
- :type uid: int
- :param message: a message to be added
- :type message: MessageWrapper
- :param observer: the deferred that will fire with the
- UID of the message. If notify_on_disk is True,
- this will happen when the message is written to
- Soledad. Otherwise it will fire as soon as we've
- added the message to the memory store.
- :type observer: Deferred
- :param notify_on_disk: whether the `observer` deferred should
- wait until the message is written to disk to
- be fired.
- :type notify_on_disk: bool
- """
- log.msg("Adding new doc to memstore %r (%r)" % (mbox, uid))
- key = mbox, uid
-
- self._add_message(mbox, uid, message, notify_on_disk)
- self._new.add(key)
-
- if observer is not None:
- if notify_on_disk:
- # We store this deferred so we can keep track of the pending
- # operations internally.
- # TODO this should fire with the UID !!! -- change that in
- # the soledad store code.
- self._new_deferreds[key] = observer
-
- else:
- # Caller does not care, just fired and forgot, so we pass
- # a defer that will inmediately have its callback triggered.
- self.reactor.callFromThread(observer.callback, uid)
-
- def put_message(self, mbox, uid, message, notify_on_disk=True):
- """
- Put an existing message.
-
- This will also set the dirty flag on the MemoryStore.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param uid: the UID for the message
- :type uid: int
- :param message: a message to be added
- :type message: MessageWrapper
- :param notify_on_disk: whether the deferred that is returned should
- wait until the message is written to disk to
- be fired.
- :type notify_on_disk: bool
-
- :return: a Deferred. if notify_on_disk is True, will be fired
- when written to the db on disk.
- Otherwise will fire inmediately
- :rtype: Deferred
- """
- key = mbox, uid
- d = defer.Deferred()
- d.addCallback(lambda result: log.msg("message PUT save: %s" % result))
-
- self._dirty.add(key)
- self._dirty_deferreds[key] = d
- self._add_message(mbox, uid, message, notify_on_disk)
- return d
-
- def _add_message(self, mbox, uid, message, notify_on_disk=True):
- """
- Helper method, called by both create_message and put_message.
- See those for parameter documentation.
- """
- msg_dict = message.as_dict()
-
- fdoc = msg_dict.get(FDOC, None)
- if fdoc is not None:
- fdoc_store = self._fdoc_store[mbox][uid]
- fdoc_store.update(fdoc)
- chash_fdoc_store = self._chash_fdoc_store
-
- # content-hash indexing
- chash = fdoc.get(fields.CONTENT_HASH_KEY)
- chash_fdoc_store[chash][mbox] = weakref.proxy(
- self._fdoc_store[mbox][uid])
-
- hdoc = msg_dict.get(HDOC, None)
- if hdoc is not None:
- chash = hdoc.get(fields.CONTENT_HASH_KEY)
- hdoc_store = self._hdoc_store[chash]
- hdoc_store.update(hdoc)
-
- cdocs = message.cdocs
- for cdoc in cdocs.values():
- phash = cdoc.get(fields.PAYLOAD_HASH_KEY, None)
- if not phash:
- continue
- cdoc_store = self._cdoc_store[phash]
- cdoc_store.update(cdoc)
-
- # Update memory store size
- # XXX this should use [mbox][uid]
- # TODO --- this has to be deferred to thread,
- # TODO add hdoc and cdocs sizes too
- # it's slowing things down here.
- # key = mbox, uid
- # self._sizes[key] = size.get_size(self._fdoc_store[key])
-
- def purge_fdoc_store(self, mbox):
- """
- Purge the empty documents from a fdoc store.
- Called during initialization of the SoledadMailbox
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- """
- # XXX This is really a workaround until I find the conditions
- # that are making the empty items remain there.
- # This happens, for instance, after running several times
- # the regression test, that issues a store deleted + expunge + select
- # The items are being correclty deleted, but in succesive appends
- # the empty items with previously deleted uids reappear as empty
- # documents. I suspect it's a timing condition with a previously
- # evaluated sequence being used after the items has been removed.
-
- for uid, value in self._fdoc_store[mbox].items():
- if empty(value):
- del self._fdoc_store[mbox][uid]
-
- def get_docid_for_fdoc(self, mbox, uid):
- """
- Return Soledad document id for the flags-doc for a given mbox and uid,
- or None of no flags document could be found.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param uid: the message UID
- :type uid: int
- :rtype: unicode or None
- """
- with self._fdoc_docid_lock:
- doc_id = self._fdoc_id_store[mbox][uid]
-
- if empty(doc_id):
- fdoc = self._permanent_store.get_flags_doc(mbox, uid)
- if empty(fdoc) or empty(fdoc.content):
- return None
- doc_id = fdoc.doc_id
- self._fdoc_id_store[mbox][uid] = doc_id
-
- return doc_id
-
- def get_message(self, mbox, uid, dirtystate=DirtyState.none,
- flags_only=False):
- """
- Get a MessageWrapper for the given mbox and uid combination.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param uid: the message UID
- :type uid: int
- :param dirtystate: DirtyState enum: one of `dirty`, `new`
- or `none` (default)
- :type dirtystate: enum
- :param flags_only: whether the message should carry only a reference
- to the flags document.
- :type flags_only: bool
- :
-
- :return: MessageWrapper or None
- """
- if dirtystate == DirtyState.dirty:
- flags_only = True
-
- key = mbox, uid
-
- fdoc = self._fdoc_store[mbox][uid]
- if empty(fdoc):
- return None
-
- new, dirty = False, False
- if dirtystate == DirtyState.none:
- new, dirty = self._get_new_dirty_state(key)
- if dirtystate == DirtyState.dirty:
- new, dirty = False, True
- if dirtystate == DirtyState.new:
- new, dirty = True, False
-
- if flags_only:
- return MessageWrapper(fdoc=fdoc,
- new=new, dirty=dirty,
- memstore=weakref.proxy(self))
- else:
- chash = fdoc.get(fields.CONTENT_HASH_KEY)
- hdoc = self._hdoc_store[chash]
- if empty(hdoc):
- hdoc = self._permanent_store.get_headers_doc(chash)
- if empty(hdoc):
- return None
- if not empty(hdoc.content):
- self._hdoc_store[chash] = hdoc.content
- hdoc = hdoc.content
- cdocs = None
-
- pmap = hdoc.get(fields.PARTS_MAP_KEY, None)
- if new and pmap is not None:
- # take the different cdocs for write...
- cdoc_store = self._cdoc_store
- cdocs_list = phash_iter(hdoc)
- cdocs = dict(enumerate(
- [cdoc_store[phash] for phash in cdocs_list], 1))
-
- return MessageWrapper(fdoc=fdoc, hdoc=hdoc, cdocs=cdocs,
- new=new, dirty=dirty,
- memstore=weakref.proxy(self))
-
- def remove_message(self, mbox, uid):
- """
- Remove a Message from this MemoryStore.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param uid: the message UID
- :type uid: int
- """
- # XXX For the moment we are only removing the flags and headers
- # docs. The rest we leave there polluting your hard disk,
- # until we think about a good way of deorphaning.
-
- # XXX implement elijah's idea of using a PUT document as a
- # token to ensure consistency in the removal.
-
- try:
- del self._fdoc_store[mbox][uid]
- except KeyError:
- pass
-
- try:
- key = mbox, uid
- self._new.discard(key)
- self._dirty.discard(key)
- if key in self._sizes:
- del self._sizes[key]
- self._known_uids[mbox].discard(uid)
- except KeyError:
- pass
- except Exception as exc:
- logger.error("error while removing message!")
- logger.exception(exc)
- try:
- with self._fdoc_docid_lock:
- del self._fdoc_id_store[mbox][uid]
- except KeyError:
- pass
- except Exception as exc:
- logger.error("error while removing message!")
- logger.exception(exc)
-
- # IMessageStoreWriter
-
- @deferred_to_thread
- def write_messages(self, store):
- """
- Write the message documents in this MemoryStore to a different store.
-
- :param store: the IMessageStore to write to
- :rtype: False if queue is not empty, None otherwise.
- """
- # For now, we pass if the queue is not empty, to avoid duplicate
- # queuing.
- # We would better use a flag to know when we've already enqueued an
- # item.
-
- # XXX this could return the deferred for all the enqueued operations
-
- if not self.producer.is_queue_empty():
- return False
-
- if any(map(lambda i: not empty(i), (self._new, self._dirty))):
- logger.info("Writing messages to Soledad...")
-
- # TODO change for lock, and make the property access
- # is accquired
- with set_bool_flag(self, self.WRITING_FLAG):
- for rflags_doc_wrapper in self.all_rdocs_iter():
- self.producer.push(rflags_doc_wrapper,
- state=self.producer.STATE_DIRTY)
- for msg_wrapper in self.all_new_msg_iter():
- self.producer.push(msg_wrapper,
- state=self.producer.STATE_NEW)
- for msg_wrapper in self.all_dirty_msg_iter():
- self.producer.push(msg_wrapper,
- state=self.producer.STATE_DIRTY)
-
- # MemoryStore specific methods.
-
- def get_uids(self, mbox):
- """
- Get all uids for a given mbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :rtype: list
- """
- return self._fdoc_store[mbox].keys()
-
- def get_soledad_known_uids(self, mbox):
- """
- Get all uids that soledad knows about, from the memory cache.
- :param mbox: the mailbox
- :type mbox: str or unicode
- :rtype: list
- """
- return self._known_uids.get(mbox, [])
-
- # last_uid
-
- def get_last_uid(self, mbox):
- """
- Return the highest UID for a given mbox.
- It will be the highest between the highest uid in the message store for
- the mailbox, and the soledad integer cache.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :rtype: int
- """
- uids = self.get_uids(mbox)
- last_mem_uid = uids and max(uids) or 0
- last_soledad_uid = self.get_last_soledad_uid(mbox)
- return max(last_mem_uid, last_soledad_uid)
-
- def get_last_soledad_uid(self, mbox):
- """
- Get last uid for a given mbox from the soledad integer cache.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- """
- return self._last_uid.get(mbox, 0)
-
- def set_last_soledad_uid(self, mbox, value):
- """
- Set last uid for a given mbox in the soledad integer cache.
- SoledadMailbox should prime this value during initialization.
- Other methods (during message adding) SHOULD call
- `increment_last_soledad_uid` instead.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param value: the value to set
- :type value: int
- """
- # can be long???
- # leap_assert_type(value, int)
- logger.info("setting last soledad uid for %s to %s" %
- (mbox, value))
- # if we already have a value here, don't do anything
- with self._last_uid_lock:
- if not self._last_uid.get(mbox, None):
- self._last_uid[mbox] = value
-
- def set_known_uids(self, mbox, value):
- """
- Set the value fo the known-uids set for this mbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param value: a sequence of integers to be added to the set.
- :type value: tuple
- """
- current = self._known_uids[mbox]
- self._known_uids[mbox] = current.union(set(value))
-
- def increment_last_soledad_uid(self, mbox):
- """
- Increment by one the soledad integer cache for the last_uid for
- this mbox, and fire a defer-to-thread to update the soledad value.
- The caller should lock the call tho this method.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- """
- with self._last_uid_lock:
- self._last_uid[mbox] += 1
- value = self._last_uid[mbox]
- self.reactor.callInThread(self.write_last_uid, mbox, value)
- return value
-
- def write_last_uid(self, mbox, value):
- """
- Increment the soledad integer cache for the highest uid value.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param value: the value to set
- :type value: int
- """
- leap_assert_type(value, int)
- if self._permanent_store:
- self._permanent_store.write_last_uid(mbox, value)
-
- def load_flag_docs(self, mbox, flag_docs):
- """
- Load the flag documents for the given mbox.
- Used during initial flag docs prefetch.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param flag_docs: a dict with the content for the flag docs, indexed
- by uid.
- :type flag_docs: dict
- """
- # We can do direct assignments cause we know this will only
- # be called during initialization of the mailbox.
- # TODO could hook here a sanity-check
- # for duplicates
-
- fdoc_store = self._fdoc_store[mbox]
- chash_fdoc_store = self._chash_fdoc_store
- for uid in flag_docs:
- rdict = ReferenciableDict(flag_docs[uid])
- fdoc_store[uid] = rdict
- # populate chash dict too, to avoid fdoc duplication
- chash = flag_docs[uid]["chash"]
- chash_fdoc_store[chash][mbox] = weakref.proxy(
- self._fdoc_store[mbox][uid])
-
- def update_flags(self, mbox, uid, fdoc):
- """
- Update the flag document for a given mbox and uid combination,
- and set the dirty flag.
- We could use put_message, but this is faster.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param uid: the uid of the message
- :type uid: int
-
- :param fdoc: a dict with the content for the flag docs
- :type fdoc: dict
- """
- key = mbox, uid
- self._fdoc_store[mbox][uid].update(fdoc)
- self._dirty.add(key)
-
- def load_header_docs(self, header_docs):
- """
- Load the flag documents for the given mbox.
- Used during header docs prefetch, and during cache after
- a read from soledad if the hdoc property in message did not
- find its value in here.
-
- :param flag_docs: a dict with the content for the flag docs.
- :type flag_docs: dict
- """
- hdoc_store = self._hdoc_store
- for chash in header_docs:
- hdoc_store[chash] = ReferenciableDict(header_docs[chash])
-
- def all_flags(self, mbox):
- """
- Return a dictionary with all the flags for a given mbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :rtype: dict
- """
- fdict = {}
- uids = self.get_uids(mbox)
- fstore = self._fdoc_store[mbox]
-
- for uid in uids:
- try:
- fdict[uid] = fstore[uid][fields.FLAGS_KEY]
- except KeyError:
- continue
- return fdict
-
- def all_headers(self, mbox):
- """
- Return a dictionary with all the header docs for a given mbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :rtype: dict
- """
- headers_dict = {}
- uids = self.get_uids(mbox)
- fdoc_store = self._fdoc_store[mbox]
- hdoc_store = self._hdoc_store
-
- for uid in uids:
- try:
- chash = fdoc_store[uid][fields.CONTENT_HASH_KEY]
- hdoc = hdoc_store[chash]
- if not empty(hdoc):
- headers_dict[uid] = hdoc
- except KeyError:
- continue
- return headers_dict
-
- # Counting sheeps...
-
- def count_new_mbox(self, mbox):
- """
- Count the new messages by mailbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :return: number of new messages
- :rtype: int
- """
- return len([(m, uid) for m, uid in self._new if mbox == mbox])
-
- # XXX used at all?
- def count_new(self):
- """
- Count all the new messages in the MemoryStore.
-
- :rtype: int
- """
- return len(self._new)
-
- def count(self, mbox):
- """
- Return the count of messages for a given mbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :return: number of messages
- :rtype: int
- """
- return len(self._fdoc_store[mbox])
-
- def unseen_iter(self, mbox):
- """
- Get an iterator for the message UIDs with no `seen` flag
- for a given mailbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :return: iterator through unseen message doc UIDs
- :rtype: iterable
- """
- fdocs = self._fdoc_store[mbox]
-
- return [uid for uid, value
- in fdocs.items()
- if fields.SEEN_FLAG not in value.get(fields.FLAGS_KEY, [])]
-
- def get_cdoc_from_phash(self, phash):
- """
- Return a content-document by its payload-hash.
-
- :param phash: the payload hash to check against
- :type phash: str or unicode
- :rtype: MessagePartDoc
- """
- doc = self._cdoc_store.get(phash, None)
-
- # XXX return None for consistency?
-
- # XXX have to keep a mapping between phash and its linkage
- # info, to know if this payload is been already saved or not.
- # We will be able to get this from the linkage-docs,
- # not yet implemented.
- new = True
- dirty = False
- return MessagePartDoc(
- new=new, dirty=dirty, store="mem",
- part=MessagePartType.cdoc,
- content=doc,
- doc_id=None)
-
- def get_fdoc_from_chash(self, chash, mbox):
- """
- Return a flags-document by its content-hash and a given mailbox.
- Used during content-duplication detection while copying or adding a
- message.
-
- :param chash: the content hash to check against
- :type chash: str or unicode
- :param mbox: the mailbox
- :type mbox: str or unicode
-
- :return: MessagePartDoc. It will return None if the flags document
- has empty content or it is flagged as \\Deleted.
- """
- fdoc = self._chash_fdoc_store[chash][mbox]
-
- # a couple of special cases.
- # 1. We might have a doc with empty content...
- if empty(fdoc):
- return None
-
- # 2. ...Or the message could exist, but being flagged for deletion.
- # We want to create a new one in this case.
- # Hmmm what if the deletion is un-done?? We would end with a
- # duplicate...
- if fdoc and fields.DELETED_FLAG in fdoc.get(fields.FLAGS_KEY, []):
- return None
-
- uid = fdoc[fields.UID_KEY]
- key = mbox, uid
- new = key in self._new
- dirty = key in self._dirty
-
- return MessagePartDoc(
- new=new, dirty=dirty, store="mem",
- part=MessagePartType.fdoc,
- content=fdoc,
- doc_id=None)
-
- def iter_fdoc_keys(self):
- """
- Return a generator through all the mbox, uid keys in the flags-doc
- store.
- """
- fdoc_store = self._fdoc_store
- for mbox in fdoc_store:
- for uid in fdoc_store[mbox]:
- yield mbox, uid
-
- def all_new_msg_iter(self):
- """
- Return generator that iterates through all new messages.
-
- :return: generator of MessageWrappers
- :rtype: generator
- """
- gm = self.get_message
- # need to freeze, set can change during iteration
- new = [gm(*key, dirtystate=DirtyState.new) for key in tuple(self._new)]
- # move content from new set to the queue
- self._new_queue.update(self._new)
- self._new.difference_update(self._new)
- return new
-
- def all_dirty_msg_iter(self):
- """
- Return generator that iterates through all dirty messages.
-
- :return: generator of MessageWrappers
- :rtype: generator
- """
- gm = self.get_message
- # need to freeze, set can change during iteration
- dirty = [gm(*key, flags_only=True, dirtystate=DirtyState.dirty)
- for key in tuple(self._dirty)]
- # move content from new and dirty sets to the queue
-
- self._dirty_queue.update(self._dirty)
- self._dirty.difference_update(self._dirty)
- return dirty
-
- def all_deleted_uid_iter(self, mbox):
- """
- Return a list with the UIDs for all messags
- with deleted flag in a given mailbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :return: list of integers
- :rtype: list
- """
- # This *needs* to return a fixed sequence. Otherwise the dictionary len
- # will change during iteration, when we modify it
- fdocs = self._fdoc_store[mbox]
- return [uid for uid, value
- in fdocs.items()
- if fields.DELETED_FLAG in value.get(fields.FLAGS_KEY, [])]
-
- # new, dirty flags
-
- def _get_new_dirty_state(self, key):
- """
- Return `new` and `dirty` flags for a given message.
-
- :param key: the key for the message, in the form mbox, uid
- :type key: tuple
- :return: tuple of bools
- :rtype: tuple
- """
- # TODO change indexing of sets to [mbox][key] too.
- # XXX should return *first* the news, and *then* the dirty...
-
- # TODO should query in queues too , true?
- #
- return map(lambda _set: key in _set, (self._new, self._dirty))
-
- def set_new_queued(self, key):
- """
- Add the key value to the `new-queue` set.
-
- :param key: the key for the message, in the form mbox, uid
- :type key: tuple
- """
- self._new_queue.add(key)
-
- def unset_new_queued(self, key):
- """
- Remove the key value from the `new-queue` set.
-
- :param key: the key for the message, in the form mbox, uid
- :type key: tuple
- """
- self._new_queue.discard(key)
- deferreds = self._new_deferreds
- d = deferreds.get(key, None)
- if d:
- # XXX use a namedtuple for passing the result
- # when we check it in the other side.
- d.callback('%s, ok' % str(key))
- deferreds.pop(key)
-
- def set_dirty_queued(self, key):
- """
- Add the key value to the `dirty-queue` set.
-
- :param key: the key for the message, in the form mbox, uid
- :type key: tuple
- """
- self._dirty_queue.add(key)
-
- def unset_dirty_queued(self, key):
- """
- Remove the key value from the `dirty-queue` set.
-
- :param key: the key for the message, in the form mbox, uid
- :type key: tuple
- """
- self._dirty_queue.discard(key)
- deferreds = self._dirty_deferreds
- d = deferreds.get(key, None)
- if d:
- # XXX use a namedtuple for passing the result
- # when we check it in the other side.
- d.callback('%s, ok' % str(key))
- deferreds.pop(key)
-
- # Recent Flags
-
- def set_recent_flag(self, mbox, uid):
- """
- Set the `Recent` flag for a given mailbox and UID.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param uid: the message UID
- :type uid: int
- """
- self._rflags_dirty.add(mbox)
- self._rflags_store[mbox]['set'].add(uid)
-
- # TODO --- nice but unused
- def unset_recent_flag(self, mbox, uid):
- """
- Unset the `Recent` flag for a given mailbox and UID.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param uid: the message UID
- :type uid: int
- """
- self._rflags_store[mbox]['set'].discard(uid)
-
- def set_recent_flags(self, mbox, value):
- """
- Set the value for the set of the recent flags.
- Used from the property in the MessageCollection.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param value: a sequence of flags to set
- :type value: sequence
- """
- self._rflags_dirty.add(mbox)
- self._rflags_store[mbox]['set'] = set(value)
-
- def load_recent_flags(self, mbox, flags_doc):
- """
- Load the passed flags document in the recent flags store, for a given
- mailbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param flags_doc: A dictionary containing the `doc_id` of the Soledad
- flags-document for this mailbox, and the `set`
- of uids marked with that flag.
- """
- self._rflags_store[mbox] = flags_doc
-
- def get_recent_flags(self, mbox):
- """
- Return the set of UIDs with the `Recent` flag for this mailbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :rtype: set, or None
- """
- rflag_for_mbox = self._rflags_store.get(mbox, None)
- if not rflag_for_mbox:
- return None
- return self._rflags_store[mbox]['set']
-
- def all_rdocs_iter(self):
- """
- Return an iterator through all in-memory recent flag dicts, wrapped
- under a RecentFlagsDoc namedtuple.
- Used for saving to disk.
-
- :return: a generator of RecentFlagDoc
- :rtype: generator
- """
- # XXX use enums
- DOC_ID = "doc_id"
- SET = "set"
-
- rflags_store = self._rflags_store
-
- def get_rdoc(mbox, rdict):
- mbox_rflag_set = rdict[SET]
- recent_set = copy(mbox_rflag_set)
- # zero it!
- mbox_rflag_set.difference_update(mbox_rflag_set)
- return RecentFlagsDoc(
- doc_id=rflags_store[mbox][DOC_ID],
- content={
- fields.TYPE_KEY: fields.TYPE_RECENT_VAL,
- fields.MBOX_KEY: mbox,
- fields.RECENTFLAGS_KEY: list(recent_set)
- })
-
- return (get_rdoc(mbox, rdict) for mbox, rdict in rflags_store.items()
- if not empty(rdict[SET]))
-
- # Methods that mirror the IMailbox interface
-
- def remove_all_deleted(self, mbox):
- """
- Remove all messages flagged \\Deleted from this Memory Store only.
- Called from `expunge`
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :return: a list of UIDs
- :rtype: list
- """
- mem_deleted = self.all_deleted_uid_iter(mbox)
- for uid in mem_deleted:
- self.remove_message(mbox, uid)
- return mem_deleted
-
- def stop_and_flush(self):
- """
- Stop the write loop and trigger a write to the producer.
- """
- self._stop_write_loop()
- if self._permanent_store is not None:
- # XXX we should check if we did get a True value on this
- # operation. If we got False we should retry! (queue was not empty)
- self.write_messages(self._permanent_store)
- self.producer.flush()
-
- def expunge(self, mbox, observer):
- """
- Remove all messages flagged \\Deleted, from the Memory Store
- and from the permanent store also.
-
- It first queues up a last write, and wait for the deferreds to be done
- before continuing.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param observer: a deferred that will be fired when expunge is done
- :type observer: Deferred
- """
- soledad_store = self._permanent_store
- if soledad_store is None:
- # just-in memory store, easy then.
- self._delete_from_memory(mbox, observer)
- return
-
- # We have a soledad storage.
- try:
- # Stop and trigger last write
- self.stop_and_flush()
- # Wait on the writebacks to finish
-
- # XXX what if pending deferreds is empty?
- pending_deferreds = (self._new_deferreds.get(mbox, []) +
- self._dirty_deferreds.get(mbox, []))
- d1 = defer.gatherResults(pending_deferreds, consumeErrors=True)
- d1.addCallback(
- self._delete_from_soledad_and_memory, mbox, observer)
- except Exception as exc:
- logger.exception(exc)
-
- def _delete_from_memory(self, mbox, observer):
- """
- Remove all messages marked as deleted from soledad and memory.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param observer: a deferred that will be fired when expunge is done
- :type observer: Deferred
- """
- mem_deleted = self.remove_all_deleted(mbox)
- observer.callback(mem_deleted)
-
- def _delete_from_soledad_and_memory(self, result, mbox, observer):
- """
- Remove all messages marked as deleted from soledad and memory.
-
- :param result: ignored. the result of the deferredList that triggers
- this as a callback from `expunge`.
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param observer: a deferred that will be fired when expunge is done
- :type observer: Deferred
- """
- all_deleted = []
- soledad_store = self._permanent_store
-
- try:
- # 1. Delete all messages marked as deleted in soledad.
- logger.debug("DELETING FROM SOLEDAD ALL FOR %r" % (mbox,))
- sol_deleted = soledad_store.remove_all_deleted(mbox)
-
- try:
- self._known_uids[mbox].difference_update(set(sol_deleted))
- except Exception as exc:
- logger.exception(exc)
-
- # 2. Delete all messages marked as deleted in memory.
- logger.debug("DELETING FROM MEM ALL FOR %r" % (mbox,))
- mem_deleted = self.remove_all_deleted(mbox)
-
- all_deleted = set(mem_deleted).union(set(sol_deleted))
- logger.debug("deleted %r" % all_deleted)
- except Exception as exc:
- logger.exception(exc)
- finally:
- self._start_write_loop()
-
- observer.callback(all_deleted)
-
- # Mailbox documents and attributes
-
- # This could be also be cached in memstore, but proxying directly
- # to soledad since it's not too performance-critical.
-
- def get_mbox_doc(self, mbox):
- """
- Return the soledad document for a given mailbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :rtype: SoledadDocument or None.
- """
- if self.permanent_store is not None:
- return self.permanent_store.get_mbox_document(mbox)
- else:
- return None
-
- def get_mbox_closed(self, mbox):
- """
- Return the closed attribute for a given mailbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :rtype: bool
- """
- if self.permanent_store is not None:
- return self.permanent_store.get_mbox_closed(mbox)
- else:
- return self._mbox_closed[mbox]
-
- def set_mbox_closed(self, mbox, closed):
- """
- Set the closed attribute for a given mailbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- """
- if self.permanent_store is not None:
- self.permanent_store.set_mbox_closed(mbox, closed)
- else:
- self._mbox_closed[mbox] = closed
-
- def get_mbox_flags(self, mbox):
- """
- Get the flags for a given mbox.
- :rtype: list
- """
- return sorted(self._mbox_flags[mbox])
-
- def set_mbox_flags(self, mbox, flags):
- """
- Set the mbox flags
- """
- self._mbox_flags[mbox] = set(flags)
- # TODO
- # This should write to the permanent store!!!
-
- # Rename flag-documents
-
- def rename_fdocs_mailbox(self, old_mbox, new_mbox):
- """
- Change the mailbox name for all flag documents in a given mailbox.
- Used from account.rename
-
- :param old_mbox: name for the old mbox
- :type old_mbox: str or unicode
- :param new_mbox: name for the new mbox
- :type new_mbox: str or unicode
- """
- fs = self._fdoc_store
- keys = fs[old_mbox].keys()
- for k in keys:
- fdoc = fs[old_mbox][k]
- fdoc['mbox'] = new_mbox
- fs[new_mbox][k] = fdoc
- fs[old_mbox].pop(k)
- self._dirty.add((new_mbox, k))
-
- # Dump-to-disk controls.
-
- @property
- def is_writing(self):
- """
- Property that returns whether the store is currently writing its
- internal state to a permanent storage.
-
- Used to evaluate whether the CHECK command can inform that the field
- is clear to proceed, or waiting for the write operations to complete
- is needed instead.
-
- :rtype: bool
- """
- # FIXME this should return a deferred !!!
- # XXX ----- can fire when all new + dirty deferreds
- # are done (gatherResults)
- return getattr(self, self.WRITING_FLAG)
-
- @property
- def permanent_store(self):
- return self._permanent_store
-
- # Memory management.
-
- def get_size(self):
- """
- Return the size of the internal storage.
- Use for calculating the limit beyond which we should flush the store.
-
- :rtype: int
- """
- return reduce(lambda x, y: x + y, self._sizes, 0)
diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py
deleted file mode 100644
index fb1d75a..0000000
--- a/src/leap/mail/imap/messageparts.py
+++ /dev/null
@@ -1,586 +0,0 @@
-# messageparts.py
-# Copyright (C) 2014 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-MessagePart implementation. Used from LeapMessage.
-"""
-import logging
-import StringIO
-import weakref
-
-from collections import namedtuple
-
-from enum import Enum
-from zope.interface import implements
-from twisted.mail import imap4
-
-from leap.common.decorators import memoized_method
-from leap.common.mail import get_email_charset
-from leap.mail.imap import interfaces
-from leap.mail.imap.fields import fields
-from leap.mail.utils import empty, first, find_charset
-
-MessagePartType = Enum("MessagePartType", "hdoc fdoc cdoc cdocs docs_id")
-
-
-logger = logging.getLogger(__name__)
-
-
-"""
-A MessagePartDoc is a light wrapper around the dictionary-like
-data that we pass along for message parts. It can be used almost everywhere
-that you would expect a SoledadDocument, since it has a dict under the
-`content` attribute.
-
-We also keep some metadata on it, relative in part to the message as a whole,
-and sometimes to a part in particular only.
-
-* `new` indicates that the document has just been created. SoledadStore
- should just create a new doc for all the related message parts.
-* `store` indicates the type of store a given MessagePartDoc lives in.
- We currently use this to indicate that the document comes from memeory,
- but we should probably get rid of it as soon as we extend the use of the
- SoledadStore interface along LeapMessage, MessageCollection and Mailbox.
-* `part` is one of the MessagePartType enums.
-
-* `dirty` indicates that, while we already have the document in Soledad,
- we have modified its state in memory, so we need to put_doc instead while
- dumping the MemoryStore contents.
- `dirty` attribute would only apply to flags-docs and linkage-docs.
-* `doc_id` is the identifier for the document in the u1db database, if any.
-
-"""
-
-MessagePartDoc = namedtuple(
- 'MessagePartDoc',
- ['new', 'dirty', 'part', 'store', 'content', 'doc_id'])
-
-"""
-A RecentFlagsDoc is used to send the recent-flags document payload to the
-SoledadWriter during dumps.
-"""
-RecentFlagsDoc = namedtuple(
- 'RecentFlagsDoc',
- ['content', 'doc_id'])
-
-
-class ReferenciableDict(dict):
- """
- A dict that can be weak-referenced.
-
- Some builtin objects are not weak-referenciable unless
- subclassed. So we do.
-
- Used to return pointers to the items in the MemoryStore.
- """
-
-
-class MessageWrapper(object):
- """
- A simple nested dictionary container around the different message subparts.
- """
- implements(interfaces.IMessageContainer)
-
- FDOC = "fdoc"
- HDOC = "hdoc"
- CDOCS = "cdocs"
- DOCS_ID = "docs_id"
-
- # Using slots to limit some the memory use,
- # Add your attribute here.
-
- __slots__ = ["_dict", "_new", "_dirty", "_storetype", "memstore"]
-
- def __init__(self, fdoc=None, hdoc=None, cdocs=None,
- from_dict=None, memstore=None,
- new=True, dirty=False, docs_id={}):
- """
- Initialize a MessageWrapper.
- """
- # TODO add optional reference to original message in the incoming
- self._dict = {}
- self.memstore = memstore
-
- self._new = new
- self._dirty = dirty
-
- self._storetype = "mem"
-
- if from_dict is not None:
- self.from_dict(from_dict)
- else:
- if fdoc is not None:
- self._dict[self.FDOC] = ReferenciableDict(fdoc)
- if hdoc is not None:
- self._dict[self.HDOC] = ReferenciableDict(hdoc)
- if cdocs is not None:
- self._dict[self.CDOCS] = ReferenciableDict(cdocs)
-
- # This will keep references to the doc_ids to be able to put
- # messages to soledad. It will be populated during the walk() to avoid
- # the overhead of reading from the db.
-
- # XXX it really *only* make sense for the FDOC, the other parts
- # should not be "dirty", just new...!!!
- self._dict[self.DOCS_ID] = docs_id
-
- # properties
-
- # TODO Could refactor new and dirty properties together.
-
- def _get_new(self):
- """
- Get the value for the `new` flag.
-
- :rtype: bool
- """
- return self._new
-
- def _set_new(self, value=False):
- """
- Set the value for the `new` flag, and propagate it
- to the memory store if any.
-
- :param value: the value to set
- :type value: bool
- """
- self._new = value
- if self.memstore:
- mbox = self.fdoc.content.get('mbox', None)
- uid = self.fdoc.content.get('uid', None)
- if not mbox or not uid:
- logger.warning("Malformed fdoc")
- return
- key = mbox, uid
- fun = [self.memstore.unset_new_queued,
- self.memstore.set_new_queued][int(value)]
- fun(key)
- else:
- logger.warning("Could not find a memstore referenced from this "
- "MessageWrapper. The value for new will not be "
- "propagated")
-
- new = property(_get_new, _set_new,
- doc="The `new` flag for this MessageWrapper")
-
- def _get_dirty(self):
- """
- Get the value for the `dirty` flag.
-
- :rtype: bool
- """
- return self._dirty
-
- def _set_dirty(self, value=True):
- """
- Set the value for the `dirty` flag, and propagate it
- to the memory store if any.
-
- :param value: the value to set
- :type value: bool
- """
- self._dirty = value
- if self.memstore:
- mbox = self.fdoc.content.get('mbox', None)
- uid = self.fdoc.content.get('uid', None)
- if not mbox or not uid:
- logger.warning("Malformed fdoc")
- return
- key = mbox, uid
- fun = [self.memstore.unset_dirty_queued,
- self.memstore.set_dirty_queued][int(value)]
- fun(key)
- else:
- logger.warning("Could not find a memstore referenced from this "
- "MessageWrapper. The value for new will not be "
- "propagated")
-
- dirty = property(_get_dirty, _set_dirty)
-
- # IMessageContainer
-
- @property
- def fdoc(self):
- """
- Return a MessagePartDoc wrapping around a weak reference to
- the flags-document in this MemoryStore, if any.
-
- :rtype: MessagePartDoc
- """
- _fdoc = self._dict.get(self.FDOC, None)
- if _fdoc:
- content_ref = weakref.proxy(_fdoc)
- else:
- logger.warning("NO FDOC!!!")
- content_ref = {}
-
- return MessagePartDoc(new=self.new, dirty=self.dirty,
- store=self._storetype,
- part=MessagePartType.fdoc,
- content=content_ref,
- doc_id=self._dict[self.DOCS_ID].get(
- self.FDOC, None))
-
- @property
- def hdoc(self):
- """
- Return a MessagePartDoc wrapping around a weak reference to
- the headers-document in this MemoryStore, if any.
-
- :rtype: MessagePartDoc
- """
- _hdoc = self._dict.get(self.HDOC, None)
- if _hdoc:
- content_ref = weakref.proxy(_hdoc)
- else:
- content_ref = {}
- return MessagePartDoc(new=self.new, dirty=self.dirty,
- store=self._storetype,
- part=MessagePartType.hdoc,
- content=content_ref,
- doc_id=self._dict[self.DOCS_ID].get(
- self.HDOC, None))
-
- @property
- def cdocs(self):
- """
- Return a weak reference to a zero-indexed dict containing
- the content-documents, or an empty dict if none found.
- If you want access to the MessagePartDoc for the individual
- parts, use the generator returned by `walk` instead.
-
- :rtype: dict
- """
- _cdocs = self._dict.get(self.CDOCS, None)
- if _cdocs:
- return weakref.proxy(_cdocs)
- else:
- return {}
-
- def walk(self):
- """
- Generator that iterates through all the parts, returning
- MessagePartDoc. Used for writing to SoledadStore.
-
- :rtype: generator
- """
- if self._dirty:
- try:
- mbox = self.fdoc.content[fields.MBOX_KEY]
- uid = self.fdoc.content[fields.UID_KEY]
- docid_dict = self._dict[self.DOCS_ID]
- docid_dict[self.FDOC] = self.memstore.get_docid_for_fdoc(
- mbox, uid)
- except Exception as exc:
- logger.debug("Error while walking message...")
- logger.exception(exc)
-
- if not empty(self.fdoc.content) and 'uid' in self.fdoc.content:
- yield self.fdoc
- if not empty(self.hdoc.content):
- yield self.hdoc
- for cdoc in self.cdocs.values():
- if not empty(cdoc):
- content_ref = weakref.proxy(cdoc)
- yield MessagePartDoc(new=self.new, dirty=self.dirty,
- store=self._storetype,
- part=MessagePartType.cdoc,
- content=content_ref,
- doc_id=None)
-
- # i/o
-
- def as_dict(self):
- """
- Return a dict representation of the parts contained.
-
- :rtype: dict
- """
- return self._dict
-
- def from_dict(self, msg_dict):
- """
- Populate MessageWrapper parts from a dictionary.
- It expects the same format that we use in a
- MessageWrapper.
-
-
- :param msg_dict: a dictionary containing the parts to populate
- the MessageWrapper from
- :type msg_dict: dict
- """
- fdoc, hdoc, cdocs = map(
- lambda part: msg_dict.get(part, None),
- [self.FDOC, self.HDOC, self.CDOCS])
-
- for t, doc in ((self.FDOC, fdoc), (self.HDOC, hdoc),
- (self.CDOCS, cdocs)):
- self._dict[t] = ReferenciableDict(doc) if doc else None
-
-
-class MessagePart(object):
- """
- IMessagePart implementor, to be passed to several methods
- of the IMAP4Server.
- It takes a subpart message and is able to find
- the inner parts.
-
- See the interface documentation.
- """
-
- implements(imap4.IMessagePart)
-
- def __init__(self, soledad, part_map):
- """
- Initializes the MessagePart.
-
- :param soledad: Soledad instance.
- :type soledad: Soledad
- :param part_map: a dictionary containing the parts map for this
- message
- :type part_map: dict
- """
- # TODO
- # It would be good to pass the uid/mailbox also
- # for references while debugging.
-
- # We have a problem on bulk moves, and is
- # that when the fetch on the new mailbox is done
- # the parts maybe are not complete.
- # So we should be able to fail with empty
- # docs until we solve that. The ideal would be
- # to gather the results of the deferred operations
- # to signal the operation is complete.
- #leap_assert(part_map, "part map dict cannot be null")
-
- self._soledad = soledad
- self._pmap = part_map
-
- def getSize(self):
- """
- Return the total size, in octets, of this message part.
-
- :return: size of the message, in octets
- :rtype: int
- """
- if empty(self._pmap):
- return 0
- size = self._pmap.get('size', None)
- if size is None:
- logger.error("Message part cannot find size in the partmap")
- size = 0
- return size
-
- def getBodyFile(self):
- """
- Retrieve a file object containing only the body of this message.
-
- :return: file-like object opened for reading
- :rtype: StringIO
- """
- fd = StringIO.StringIO()
- if not empty(self._pmap):
- multi = self._pmap.get('multi')
- if not multi:
- phash = self._pmap.get("phash", None)
- else:
- pmap = self._pmap.get('part_map')
- first_part = pmap.get('1', None)
- if not empty(first_part):
- phash = first_part['phash']
- else:
- phash = None
-
- if phash is None:
- logger.warning("Could not find phash for this subpart!")
- payload = ""
- else:
- payload = self._get_payload_from_document_memoized(phash)
- if empty(payload):
- payload = self._get_payload_from_document(phash)
-
- else:
- logger.warning("Message with no part_map!")
- payload = ""
-
- if payload:
- content_type = self._get_ctype_from_document(phash)
- charset = find_charset(content_type)
- if charset is None:
- charset = self._get_charset(payload)
- try:
- if isinstance(payload, unicode):
- payload = payload.encode(charset)
- except UnicodeError as exc:
- logger.error(
- "Unicode error, using 'replace'. {0!r}".format(exc))
- payload = payload.encode(charset, 'replace')
-
- fd.write(payload)
- fd.seek(0)
- return fd
-
- # TODO should memory-bound this memoize!!!
- @memoized_method
- def _get_payload_from_document_memoized(self, phash):
- """
- Memoized method call around the regular method, to be able
- to call the non-memoized method in case we got a None.
-
- :param phash: the payload hash to retrieve by.
- :type phash: str or unicode
- :rtype: str or unicode or None
- """
- return self._get_payload_from_document(phash)
-
- def _get_payload_from_document(self, phash):
- """
- Return the message payload from the content document.
-
- :param phash: the payload hash to retrieve by.
- :type phash: str or unicode
- :rtype: str or unicode or None
- """
- cdocs = self._soledad.get_from_index(
- fields.TYPE_P_HASH_IDX,
- fields.TYPE_CONTENT_VAL, str(phash))
-
- cdoc = first(cdocs)
- if cdoc is None:
- logger.warning(
- "Could not find the content doc "
- "for phash %s" % (phash,))
- payload = ""
- else:
- payload = cdoc.content.get(fields.RAW_KEY, "")
- return payload
-
- # TODO should memory-bound this memoize!!!
- @memoized_method
- def _get_ctype_from_document(self, phash):
- """
- Reeturn the content-type from the content document.
-
- :param phash: the payload hash to retrieve by.
- :type phash: str or unicode
- :rtype: str or unicode
- """
- cdocs = self._soledad.get_from_index(
- fields.TYPE_P_HASH_IDX,
- fields.TYPE_CONTENT_VAL, str(phash))
-
- cdoc = first(cdocs)
- if not cdoc:
- logger.warning(
- "Could not find the content doc "
- "for phash %s" % (phash,))
- ctype = cdoc.content.get('ctype', "")
- return ctype
-
- @memoized_method
- def _get_charset(self, stuff):
- # TODO put in a common class with LeapMessage
- """
- Gets (guesses?) the charset of a payload.
-
- :param stuff: the stuff to guess about.
- :type stuff: str or unicode
- :return: charset
- :rtype: unicode
- """
- # XXX existential doubt 2. shouldn't we make the scope
- # of the decorator somewhat more persistent?
- # ah! yes! and put memory bounds.
- return get_email_charset(stuff)
-
- def getHeaders(self, negate, *names):
- """
- Retrieve a group of message headers.
-
- :param names: The names of the headers to retrieve or omit.
- :type names: tuple of str
-
- :param negate: If True, indicates that the headers listed in names
- should be omitted from the return value, rather
- than included.
- :type negate: bool
-
- :return: A mapping of header field names to header field values
- :rtype: dict
- """
- # XXX refactor together with MessagePart method
- if not self._pmap:
- logger.warning("No pmap in Subpart!")
- return {}
- headers = dict(self._pmap.get("headers", []))
-
- names = map(lambda s: s.upper(), names)
- if negate:
- cond = lambda key: key.upper() not in names
- else:
- cond = lambda key: key.upper() in names
-
- # default to most likely standard
- charset = find_charset(headers, "utf-8")
- headers2 = dict()
- for key, value in headers.items():
- # twisted imap server expects *some* headers to be lowercase
- # We could use a CaseInsensitiveDict here...
- if key.lower() == "content-type":
- key = key.lower()
-
- if not isinstance(key, str):
- key = key.encode(charset, 'replace')
- if not isinstance(value, str):
- value = value.encode(charset, 'replace')
-
- # filter original dict by negate-condition
- if cond(key):
- headers2[key] = value
- return headers2
-
- def isMultipart(self):
- """
- Return True if this message is multipart.
- """
- if empty(self._pmap):
- logger.warning("Could not get part map!")
- return False
- multi = self._pmap.get("multi", False)
- return multi
-
- def getSubPart(self, part):
- """
- Retrieve a MIME submessage
-
- :type part: C{int}
- :param part: The number of the part to retrieve, indexed from 0.
- :raise IndexError: Raised if the specified part does not exist.
- :raise TypeError: Raised if this message is not multipart.
- :rtype: Any object implementing C{IMessagePart}.
- :return: The specified sub-part.
- """
- if not self.isMultipart():
- raise TypeError
-
- sub_pmap = self._pmap.get("part_map", {})
- try:
- part_map = sub_pmap[str(part + 1)]
- except KeyError:
- logger.debug("getSubpart for %s: KeyError" % (part,))
- raise IndexError
-
- # XXX check for validity
- return MessagePart(self._soledad, part_map)
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index 0356600..d1c7b93 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# messages.py
-# Copyright (C) 2013 LEAP
+# imap/messages.py
+# Copyright (C) 2013-2015 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -15,189 +15,62 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
-LeapMessage and MessageCollection.
+IMAPMessage implementation.
"""
-import copy
import logging
-import re
-import threading
-import StringIO
-
-from collections import defaultdict
-from email import message_from_string
-from functools import partial
-
-from pycryptopp.hash import sha256
from twisted.mail import imap4
from twisted.internet import defer
from zope.interface import implements
-from zope.proxy import sameProxiedObjects
-
-from leap.common.check import leap_assert, leap_assert_type
-from leap.common.decorators import memoized_method
-from leap.common.mail import get_email_charset
-from leap.mail import walk
-from leap.mail.utils import first, find_charset, lowerdict, empty
-from leap.mail.utils import stringify_parts_map
-from leap.mail.decorators import deferred_to_thread
-from leap.mail.imap.index import IndexedDB
-from leap.mail.imap.fields import fields, WithMsgFields
-from leap.mail.imap.memorystore import MessageWrapper
-from leap.mail.imap.messageparts import MessagePart, MessagePartDoc
-from leap.mail.imap.parser import MBoxParser
-
-logger = logging.getLogger(__name__)
-
-# TODO ------------------------------------------------------------
-
-# [ ] Add ref to incoming message during add_msg
-# [ ] Add linked-from info.
-# * Need a new type of documents: linkage info.
-# * HDOCS are linked from FDOCs (ref to chash)
-# * CDOCS are linked from HDOCS (ref to chash)
-
-# [ ] Delete incoming mail only after successful write!
-# [ ] Remove UID from syncable db. Store only those indexes locally.
-
-MSGID_PATTERN = r"""<([\w@.]+)>"""
-MSGID_RE = re.compile(MSGID_PATTERN)
-
-def try_unique_query(curried):
- """
- Try to execute a query that is expected to have a
- single outcome, and log a warning if more than one document found.
+from leap.mail.utils import find_charset, CaseInsensitiveDict
- :param curried: a curried function
- :type curried: callable
- """
- leap_assert(callable(curried), "A callable is expected")
- try:
- query = curried()
- if query:
- if len(query) > 1:
- # TODO we could take action, like trigger a background
- # process to kill dupes.
- name = getattr(curried, 'expected', 'doc')
- logger.warning(
- "More than one %s found for this mbox, "
- "we got a duplicate!!" % (name,))
- return query.pop()
- else:
- return None
- except Exception as exc:
- logger.exception("Unhandled error %r" % exc)
+logger = logging.getLogger(__name__)
-"""
-A dictionary that keeps one lock per mbox and uid.
-"""
-# XXX too much overhead?
-fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock()))
+# TODO
+# [ ] Add ref to incoming message during add_msg.
-class LeapMessage(fields, MBoxParser):
+class IMAPMessage(object):
"""
- The main representation of a message.
-
- It indexes the messages in one mailbox by a combination
- of uid+mailbox name.
+ The main representation of a message as seen by the IMAP Server.
+ This class implements the semantics specific to IMAP specification.
"""
-
- # TODO this has to change.
- # Should index primarily by chash, and keep a local-only
- # UID table.
-
implements(imap4.IMessage)
- def __init__(self, soledad, uid, mbox, collection=None, container=None):
- """
- Initializes a LeapMessage.
-
- :param soledad: a Soledad instance
- :type soledad: Soledad
- :param uid: the UID for the message.
- :type uid: int or basestring
- :param mbox: the mbox this message belongs to
- :type mbox: str or unicode
- :param collection: a reference to the parent collection object
- :type collection: MessageCollection
- :param container: a IMessageContainer implementor instance
- :type container: IMessageContainer
+ def __init__(self, message, prefetch_body=True,
+ store=None, d=defer.Deferred()):
"""
- self._soledad = soledad
- self._uid = int(uid) if uid is not None else None
- self._mbox = self._parse_mailbox_name(mbox)
- self._collection = collection
- self._container = container
+ Get an IMAPMessage. A mail.Message is needed, since many of the methods
+ are proxied to that object.
- self.__chash = None
- self.__bdoc = None
- from twisted.internet import reactor
- self.reactor = reactor
+ If you do not need to prefetch the body of the message, you can set
+ `prefetch_body` to False, but the current imap server implementation
+ expect the getBodyFile method to return inmediately.
- # XXX make these properties public
+ When the prefetch_body option is used, a deferred is also expected as a
+ parameter, and this will fire when the deferred initialization has
+ taken place, with this instance of IMAPMessage as a parameter.
- @property
- def fdoc(self):
+ :param message: the abstract message
+ :type message: mail.Message
+ :param prefetch_body: Whether to prefetch the content doc for the body.
+ :type prefetch_body: bool
+ :param store: an instance of soledad, or anything that behaves like it.
+ :param d: an optional deferred, that will be fired with the instance of
+ the IMAPMessage being initialized
+ :type d: defer.Deferred
"""
- An accessor to the flags document.
- """
- if all(map(bool, (self._uid, self._mbox))):
- fdoc = None
- if self._container is not None:
- fdoc = self._container.fdoc
- if not fdoc:
- fdoc = self._get_flags_doc()
- if fdoc:
- fdoc_content = fdoc.content
- self.__chash = fdoc_content.get(
- fields.CONTENT_HASH_KEY, None)
- return fdoc
-
- @property
- def hdoc(self):
- """
- An accessor to the headers document.
- """
- container = self._container
- if container is not None:
- hdoc = self._container.hdoc
- if hdoc and not empty(hdoc.content):
- return hdoc
- hdoc = self._get_headers_doc()
-
- if container and not empty(hdoc.content):
- # mem-cache it
- hdoc_content = hdoc.content
- chash = hdoc_content.get(fields.CONTENT_HASH_KEY)
- hdocs = {chash: hdoc_content}
- container.memstore.load_header_docs(hdocs)
- return hdoc
+ # TODO substitute the use of the deferred initialization by a factory
+ # function, maybe.
- @property
- def chash(self):
- """
- An accessor to the content hash for this message.
- """
- if not self.fdoc:
- return None
- if not self.__chash and self.fdoc:
- self.__chash = self.fdoc.content.get(
- fields.CONTENT_HASH_KEY, None)
- return self.__chash
-
- @property
- def bdoc(self):
- """
- An accessor to the body document.
- """
- if not self.hdoc:
- return None
- if not self.__bdoc:
- self.__bdoc = self._get_body_doc()
- return self.__bdoc
+ self.message = message
+ self.__body_fd = None
+ self.store = store
+ if prefetch_body:
+ gotbody = self.__prefetch_body_file()
+ gotbody.addCallback(lambda _: d.callback(self))
# IMessage implementation
@@ -208,7 +81,7 @@ class LeapMessage(fields, MBoxParser):
:return: uid for this message
:rtype: int
"""
- return self._uid
+ return self.message.get_uid()
def getFlags(self):
"""
@@ -217,62 +90,7 @@ class LeapMessage(fields, MBoxParser):
:return: The flags, represented as strings
:rtype: tuple
"""
- uid = self._uid
-
- flags = set([])
- fdoc = self.fdoc
- if fdoc:
- flags = set(fdoc.content.get(self.FLAGS_KEY, None))
-
- msgcol = self._collection
-
- # We treat the recent flag specially: gotten from
- # a mailbox-level document.
- if msgcol and uid in msgcol.recent_flags:
- flags.add(fields.RECENT_FLAG)
- if flags:
- flags = map(str, flags)
- return tuple(flags)
-
- # setFlags not in the interface spec but we use it with store command.
-
- def setFlags(self, flags, mode):
- """
- Sets the flags for this message
-
- :param flags: the flags to update in the message.
- :type flags: tuple of str
- :param mode: the mode for setting. 1 is append, -1 is remove, 0 set.
- :type mode: int
- """
- leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
- mbox, uid = self._mbox, self._uid
-
- APPEND = 1
- REMOVE = -1
- SET = 0
-
- with fdoc_locks[mbox][uid]:
- doc = self.fdoc
- if not doc:
- logger.warning(
- "Could not find FDOC for %r:%s while setting flags!" %
- (mbox, uid))
- return
- current = doc.content[self.FLAGS_KEY]
- if mode == APPEND:
- newflags = tuple(set(tuple(current) + flags))
- elif mode == REMOVE:
- newflags = tuple(set(current).difference(set(flags)))
- elif mode == SET:
- newflags = flags
- new_fdoc = {
- self.FLAGS_KEY: newflags,
- self.SEEN_KEY: self.SEEN_FLAG in newflags,
- self.DEL_KEY: self.DELETED_FLAG in newflags}
- self._collection.memstore.update_flags(mbox, uid, new_fdoc)
-
- return map(str, newflags)
+ return self.message.get_flags()
def getInternalDate(self):
"""
@@ -289,72 +107,27 @@ class LeapMessage(fields, MBoxParser):
:return: An RFC822-formatted date string.
:rtype: str
"""
- date = self.hdoc.content.get(fields.DATE_KEY, '')
- return date
+ return self.message.get_internal_date()
#
# IMessagePart
#
- # XXX we should implement this interface too for the subparts
- # so we allow nested parts...
-
- def getBodyFile(self):
+ def getBodyFile(self, store=None):
"""
Retrieve a file object containing only the body of this message.
:return: file-like object opened for reading
- :rtype: StringIO
+ :rtype: a deferred that will fire with a StringIO object.
"""
- def write_fd(body):
- fd.write(body)
+ if self.__body_fd is not None:
+ fd = self.__body_fd
fd.seek(0)
return fd
- # TODO refactor with getBodyFile in MessagePart
-
- fd = StringIO.StringIO()
-
- if self.bdoc is not None:
- bdoc_content = self.bdoc.content
- if empty(bdoc_content):
- logger.warning("No BDOC content found for message!!!")
- return write_fd("")
-
- body = bdoc_content.get(self.RAW_KEY, "")
- content_type = bdoc_content.get('content-type', "")
- charset = find_charset(content_type)
- if charset is None:
- charset = self._get_charset(body)
- try:
- if isinstance(body, unicode):
- body = body.encode(charset)
- except UnicodeError as exc:
- logger.error(
- "Unicode error, using 'replace'. {0!r}".format(exc))
- logger.debug("Attempted to encode with: %s" % charset)
- body = body.encode(charset, 'replace')
- finally:
- return write_fd(body)
-
- # We are still returning funky characters from here.
- else:
- logger.warning("No BDOC found for message.")
- return write_fd("")
-
- @memoized_method
- def _get_charset(self, stuff):
- """
- Gets (guesses?) the charset of a payload.
-
- :param stuff: the stuff to guess about.
- :type stuff: basestring
- :returns: charset
- """
- # XXX shouldn't we make the scope
- # of the decorator somewhat more persistent?
- # ah! yes! and put memory bounds.
- return get_email_charset(stuff)
+ if store is None:
+ store = self.store
+ return self.message.get_body_file(store)
def getSize(self):
"""
@@ -363,17 +136,7 @@ class LeapMessage(fields, MBoxParser):
:return: size of the message, in octets
:rtype: int
"""
- size = None
- if self.fdoc is not None:
- fdoc_content = self.fdoc.content
- size = fdoc_content.get(self.SIZE_KEY, False)
- else:
- logger.warning("No FLAGS doc for %s:%s" % (self._mbox,
- self._uid))
- if not size:
- # XXX fallback, should remove when all migrated.
- size = self.getBodyFile().len
- return size
+ return self.message.get_size()
def getHeaders(self, negate, *names):
"""
@@ -390,74 +153,14 @@ class LeapMessage(fields, MBoxParser):
:return: A mapping of header field names to header field values
:rtype: dict
"""
- # TODO split in smaller methods
- # XXX refactor together with MessagePart method
-
- headers = self._get_headers()
- if not headers:
- logger.warning("No headers found")
- return {str('content-type'): str('')}
-
- names = map(lambda s: s.upper(), names)
- if negate:
- cond = lambda key: key.upper() not in names
- else:
- cond = lambda key: key.upper() in names
-
- if isinstance(headers, list):
- headers = dict(headers)
-
- # default to most likely standard
- charset = find_charset(headers, "utf-8")
- headers2 = dict()
- for key, value in headers.items():
- # twisted imap server expects *some* headers to be lowercase
- # We could use a CaseInsensitiveDict here...
- if key.lower() == "content-type":
- key = key.lower()
-
- if not isinstance(key, str):
- key = key.encode(charset, 'replace')
- if not isinstance(value, str):
- value = value.encode(charset, 'replace')
-
- if value.endswith(";"):
- # bastards
- value = value[:-1]
-
- # filter original dict by negate-condition
- if cond(key):
- headers2[key] = value
- return headers2
-
- def _get_headers(self):
- """
- Return the headers dict for this message.
- """
- if self.hdoc is not None:
- hdoc_content = self.hdoc.content
- headers = hdoc_content.get(self.HEADERS_KEY, {})
- return headers
-
- else:
- logger.warning(
- "No HEADERS doc for msg %s:%s" % (
- self._mbox,
- self._uid))
+ headers = self.message.get_headers()
+ return _format_headers(headers, negate, *names)
def isMultipart(self):
"""
Return True if this message is multipart.
"""
- if self.fdoc:
- fdoc_content = self.fdoc.content
- is_multipart = fdoc_content.get(self.MULTIPART_KEY, False)
- return is_multipart
- else:
- logger.warning(
- "No FLAGS doc for msg %s:%s" % (
- self._mbox,
- self._uid))
+ return self.message.is_multipart()
def getSubPart(self, part):
"""
@@ -470,913 +173,82 @@ class LeapMessage(fields, MBoxParser):
:rtype: Any object implementing C{IMessagePart}.
:return: The specified sub-part.
"""
- if not self.isMultipart():
- raise TypeError
- try:
- pmap_dict = self._get_part_from_parts_map(part + 1)
- except KeyError:
- raise IndexError
- return MessagePart(self._soledad, pmap_dict)
-
- #
- # accessors
- #
-
- def _get_part_from_parts_map(self, part):
- """
- Get a part map from the headers doc
-
- :raises: KeyError if key does not exist
- :rtype: dict
- """
- if not self.hdoc:
- logger.warning("Tried to get part but no HDOC found!")
- return None
-
- hdoc_content = self.hdoc.content
- pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {})
-
- # remember, lads, soledad is using strings in its keys,
- # not integers!
- return pmap[str(part)]
-
- # XXX moved to memory store
- # move the rest too. ------------------------------------------
- def _get_flags_doc(self):
- """
- Return the document that keeps the flags for this
- message.
- """
- result = {}
- try:
- flag_docs = self._soledad.get_from_index(
- fields.TYPE_MBOX_UID_IDX,
- fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid))
- result = first(flag_docs)
- except Exception as exc:
- # ugh! Something's broken down there!
- logger.warning("ERROR while getting flags for UID: %s" % self._uid)
- logger.exception(exc)
- finally:
- return result
-
- # TODO move to soledadstore instead of accessing soledad directly
- def _get_headers_doc(self):
- """
- Return the document that keeps the headers for this
- message.
- """
- head_docs = self._soledad.get_from_index(
- fields.TYPE_C_HASH_IDX,
- fields.TYPE_HEADERS_VAL, str(self.chash))
- return first(head_docs)
-
- # TODO move to soledadstore instead of accessing soledad directly
- def _get_body_doc(self):
- """
- Return the document that keeps the body for this
- message.
- """
- hdoc_content = self.hdoc.content
- body_phash = hdoc_content.get(
- fields.BODY_KEY, None)
- if not body_phash:
- logger.warning("No body phash for this document!")
- return None
-
- # XXX get from memstore too...
- # if memstore: memstore.get_phrash
- # memstore should keep a dict with weakrefs to the
- # phash doc...
-
- if self._container is not None:
- bdoc = self._container.memstore.get_cdoc_from_phash(body_phash)
- if not empty(bdoc) and not empty(bdoc.content):
- return bdoc
-
- # no memstore, or no body doc found there
- if self._soledad:
- body_docs = self._soledad.get_from_index(
- fields.TYPE_P_HASH_IDX,
- fields.TYPE_CONTENT_VAL, str(body_phash))
- return first(body_docs)
- else:
- logger.error("No phash in container, and no soledad found!")
-
- def __getitem__(self, key):
- """
- Return an item from the content of the flags document,
- for convenience.
-
- :param key: The key
- :type key: str
-
- :return: The content value indexed by C{key} or None
- :rtype: str
- """
- return self.fdoc.content.get(key, None)
-
- def does_exist(self):
- """
- Return True if there is actually a flags document for this
- UID and mbox.
- """
- return not empty(self.fdoc)
-
-
-class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
- """
- A collection of messages, surprisingly.
-
- It is tied to a selected mailbox name that is passed to its constructor.
- Implements a filter query over the messages contained in a soledad
- database.
- """
-
- # XXX this should be able to produce a MessageSet methinks
- # could validate these kinds of objects turning them
- # into a template for the class.
- FLAGS_DOC = "FLAGS"
- HEADERS_DOC = "HEADERS"
- CONTENT_DOC = "CONTENT"
- """
- RECENT_DOC is a document that stores a list of the UIDs
- with the recent flag for this mailbox. It deserves a special treatment
- because:
- (1) it cannot be set by the user
- (2) it's a flag that we set inmediately after a fetch, which is quite
- often.
- (3) we need to be able to set/unset it in batches without doing a single
- write for each element in the sequence.
- """
- RECENT_DOC = "RECENT"
- """
- HDOCS_SET_DOC is a document that stores a set of the Document-IDs
- (the u1db index) for all the headers documents for a given mailbox.
- We use it to prefetch massively all the headers for a mailbox.
- This is the second massive query, after fetching all the FLAGS, that
- a MUA will do in a case where we do not have local disk cache.
- """
- HDOCS_SET_DOC = "HDOCS_SET"
-
- templates = {
-
- # Message Level
-
- FLAGS_DOC: {
- fields.TYPE_KEY: fields.TYPE_FLAGS_VAL,
- fields.UID_KEY: 1, # XXX moe to a local table
- fields.MBOX_KEY: fields.INBOX_VAL,
- fields.CONTENT_HASH_KEY: "",
-
- fields.SEEN_KEY: False,
- fields.DEL_KEY: False,
- fields.FLAGS_KEY: [],
- fields.MULTIPART_KEY: False,
- fields.SIZE_KEY: 0
- },
-
- HEADERS_DOC: {
- fields.TYPE_KEY: fields.TYPE_HEADERS_VAL,
- fields.CONTENT_HASH_KEY: "",
-
- fields.DATE_KEY: "",
- fields.SUBJECT_KEY: "",
-
- fields.HEADERS_KEY: {},
- fields.PARTS_MAP_KEY: {},
- },
-
- CONTENT_DOC: {
- fields.TYPE_KEY: fields.TYPE_CONTENT_VAL,
- fields.PAYLOAD_HASH_KEY: "",
- fields.LINKED_FROM_KEY: [],
- fields.CTYPE_KEY: "", # should index by this too
-
- # should only get inmutable headers parts
- # (for indexing)
- fields.HEADERS_KEY: {},
- fields.RAW_KEY: "",
- fields.PARTS_MAP_KEY: {},
- fields.HEADERS_KEY: {},
- fields.MULTIPART_KEY: False,
- },
-
- # Mailbox Level
-
- RECENT_DOC: {
- fields.TYPE_KEY: fields.TYPE_RECENT_VAL,
- fields.MBOX_KEY: fields.INBOX_VAL,
- fields.RECENTFLAGS_KEY: [],
- },
-
- HDOCS_SET_DOC: {
- fields.TYPE_KEY: fields.TYPE_HDOCS_SET_VAL,
- fields.MBOX_KEY: fields.INBOX_VAL,
- fields.HDOCS_SET_KEY: [],
- }
-
-
- }
-
- # Different locks for wrapping both the u1db document getting/setting
- # and the property getting/settting in an atomic operation.
-
- # TODO we would abstract this to a SoledadProperty class
-
- _rdoc_lock = defaultdict(lambda: threading.Lock())
- _rdoc_write_lock = defaultdict(lambda: threading.Lock())
- _rdoc_read_lock = defaultdict(lambda: threading.Lock())
- _rdoc_property_lock = defaultdict(lambda: threading.Lock())
-
- _initialized = {}
-
- def __init__(self, mbox=None, soledad=None, memstore=None):
- """
- Constructor for MessageCollection.
-
- On initialization, we ensure that we have a document for
- storing the recent flags. The nature of this flag make us wanting
- to store the set of the UIDs with this flag at the level of the
- MessageCollection for each mailbox, instead of treating them
- as a property of each message.
-
- We are passed an instance of MemoryStore, the same for the
- SoledadBackedAccount, that we use as a read cache and a buffer
- for writes.
-
- :param mbox: the name of the mailbox. It is the name
- with which we filter the query over the
- messages database.
- :type mbox: str
- :param soledad: Soledad database
- :type soledad: Soledad instance
- :param memstore: a MemoryStore instance
- :type memstore: MemoryStore
- """
- leap_assert(mbox, "Need a mailbox name to initialize")
- leap_assert(mbox.strip() != "", "mbox cannot be blank space")
- leap_assert(isinstance(mbox, (str, unicode)),
- "mbox needs to be a string")
- leap_assert(soledad, "Need a soledad instance to initialize")
-
- # okay, all in order, keep going...
-
- self.mbox = self._parse_mailbox_name(mbox)
-
- # XXX get a SoledadStore passed instead
- self._soledad = soledad
- self.memstore = memstore
-
- self.__rflags = None
-
- if not self._initialized.get(mbox, False):
- try:
- self.initialize_db()
- # ensure that we have a recent-flags doc
- self._get_or_create_rdoc()
- except Exception:
- logger.debug("Error initializing %r" % (mbox,))
- else:
- self._initialized[mbox] = True
-
- from twisted.internet import reactor
- self.reactor = reactor
-
- def _get_empty_doc(self, _type=FLAGS_DOC):
- """
- Returns an empty doc for storing different message parts.
- Defaults to returning a template for a flags document.
- :return: a dict with the template
- :rtype: dict
- """
- if _type not in self.templates.keys():
- raise TypeError("Improper type passed to _get_empty_doc")
- return copy.deepcopy(self.templates[_type])
-
- def _get_or_create_rdoc(self):
- """
- Try to retrieve the recent-flags doc for this MessageCollection,
- and create one if not found.
- """
- # XXX should move this to memstore too
- with self._rdoc_write_lock[self.mbox]:
- rdoc = self._get_recent_doc_from_soledad()
- if rdoc is None:
- rdoc = self._get_empty_doc(self.RECENT_DOC)
- if self.mbox != fields.INBOX_VAL:
- rdoc[fields.MBOX_KEY] = self.mbox
- self._soledad.create_doc(rdoc)
-
- @deferred_to_thread
- def _do_parse(self, raw):
- """
- Parse raw message and return it along with
- relevant information about its outer level.
-
- This is done in a separate thread, and the callback is passed
- to `_do_add_msg` method.
-
- :param raw: the raw message
- :type raw: StringIO or basestring
- :return: msg, parts, chash, size, multi
- :rtype: tuple
- """
- msg = message_from_string(raw)
- parts = walk.get_parts(msg)
- size = len(raw)
- chash = sha256.SHA256(raw).hexdigest()
- multi = msg.is_multipart()
- return msg, parts, chash, size, multi
-
- def _populate_flags(self, flags, uid, chash, size, multi):
- """
- Return a flags doc.
-
- XXX Missing DOC -----------
- """
- fd = self._get_empty_doc(self.FLAGS_DOC)
-
- fd[self.MBOX_KEY] = self.mbox
- fd[self.UID_KEY] = uid
- fd[self.CONTENT_HASH_KEY] = chash
- fd[self.SIZE_KEY] = size
- fd[self.MULTIPART_KEY] = multi
- if flags:
- fd[self.FLAGS_KEY] = flags
- fd[self.SEEN_KEY] = self.SEEN_FLAG in flags
- fd[self.DEL_KEY] = self.DELETED_FLAG in flags
- fd[self.RECENT_KEY] = True # set always by default
- return fd
-
- def _populate_headr(self, msg, chash, subject, date):
- """
- Return a headers doc.
-
- XXX Missing DOC -----------
- """
- headers = defaultdict(list)
- for k, v in msg.items():
- headers[k].append(v)
-
- # "fix" for repeated headers.
- for k, v in headers.items():
- newline = "\n%s: " % (k,)
- headers[k] = newline.join(v)
+ subpart = self.message.get_subpart(part + 1)
+ return IMAPMessagePart(subpart)
- lower_headers = lowerdict(headers)
- msgid = first(MSGID_RE.findall(
- lower_headers.get('message-id', '')))
-
- hd = self._get_empty_doc(self.HEADERS_DOC)
- hd[self.CONTENT_HASH_KEY] = chash
- hd[self.HEADERS_KEY] = headers
- hd[self.MSGID_KEY] = msgid
-
- if not subject and self.SUBJECT_FIELD in headers:
- hd[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
- else:
- hd[self.SUBJECT_KEY] = subject
-
- if not date and self.DATE_FIELD in headers:
- hd[self.DATE_KEY] = headers[self.DATE_FIELD]
- else:
- hd[self.DATE_KEY] = date
- return hd
-
- def _fdoc_already_exists(self, chash):
- """
- Check whether we can find a flags doc for this mailbox with the
- given content-hash. It enforces that we can only have the same maessage
- listed once for a a given mailbox.
-
- :param chash: the content-hash to check about.
- :type chash: basestring
- :return: False, if it does not exist, or UID.
- """
- exist = False
- exist = self.memstore.get_fdoc_from_chash(chash, self.mbox)
-
- if not exist:
- exist = self._get_fdoc_from_chash(chash)
- if exist and exist.content is not None:
- return exist.content.get(fields.UID_KEY, "unknown-uid")
- else:
- return False
-
- def add_msg(self, raw, subject=None, flags=None, date=None,
- notify_on_disk=False):
- """
- Creates a new message document.
-
- :param raw: the raw message
- :type raw: str
-
- :param subject: subject of the message.
- :type subject: str
-
- :param flags: flags
- :type flags: list
-
- :param date: the received date for the message
- :type date: str
-
- :return: a deferred that will be fired with the message
- uid when the adding succeed.
- :rtype: deferred
- """
- if flags is None:
- flags = tuple()
- leap_assert_type(flags, tuple)
-
- observer = defer.Deferred()
- d = self._do_parse(raw)
- d.addCallback(lambda result: self.reactor.callInThread(
- self._do_add_msg, result, flags, subject, date,
- notify_on_disk, observer))
- return observer
-
- # Called in thread
- def _do_add_msg(self, parse_result, flags, subject,
- date, notify_on_disk, observer):
- """
- Helper that creates a new message document.
- Here lives the magic of the leap mail. Well, in soledad, really.
-
- See `add_msg` docstring for parameter info.
-
- :param parse_result: a tuple with the results of `self._do_parse`
- :type parse_result: tuple
- :param observer: a deferred that will be fired with the message
- uid when the adding succeed.
- :type observer: deferred
- """
- # TODO signal that we can delete the original message!-----
- # when all the processing is done.
-
- # TODO add the linked-from info !
- # TODO add reference to the original message
-
- msg, parts, chash, size, multi = parse_result
-
- # check for uniqueness --------------------------------
- # Watch out! We're reserving a UID right after this!
- existing_uid = self._fdoc_already_exists(chash)
- if existing_uid:
- msg = self.get_msg_by_uid(existing_uid)
-
- # We can say the observer that we're done
- self.reactor.callFromThread(observer.callback, existing_uid)
- msg.setFlags((fields.DELETED_FLAG,), -1)
- return
-
- # XXX get FUCKING UID from autoincremental table
- uid = self.memstore.increment_last_soledad_uid(self.mbox)
-
- # We can say the observer that we're done at this point, but
- # before that we should make sure it has no serious consequences
- # if we're issued, for instance, a fetch command right after...
- # self.reactor.callFromThread(observer.callback, uid)
- # if we did the notify, we need to invalidate the deferred
- # so not to try to fire it twice.
- # observer = None
-
- fd = self._populate_flags(flags, uid, chash, size, multi)
- hd = self._populate_headr(msg, chash, subject, date)
-
- body_phash_fun = [walk.get_body_phash_simple,
- walk.get_body_phash_multi][int(multi)]
- body_phash = body_phash_fun(walk.get_payloads(msg))
- parts_map = walk.walk_msg_tree(parts, body_phash=body_phash)
-
- # add parts map to header doc
- # (body, multi, part_map)
- for key in parts_map:
- hd[key] = parts_map[key]
- del parts_map
-
- hd = stringify_parts_map(hd)
-
- # The MessageContainer expects a dict, one-indexed
- cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1))
-
- self.set_recent_flag(uid)
- msg_container = MessageWrapper(fd, hd, cdocs)
- self.memstore.create_message(
- self.mbox, uid, msg_container,
- observer=observer, notify_on_disk=notify_on_disk)
-
- #
- # getters: specific queries
- #
-
- # recent flags
-
- def _get_recent_flags(self):
- """
- An accessor for the recent-flags set for this mailbox.
- """
- # XXX check if we should remove this
- if self.__rflags is not None:
- return self.__rflags
-
- if self.memstore is not None:
- with self._rdoc_lock[self.mbox]:
- rflags = self.memstore.get_recent_flags(self.mbox)
- if not rflags:
- # not loaded in the memory store yet.
- # let's fetch them from soledad...
- rdoc = self._get_recent_doc_from_soledad()
- if rdoc is None:
- return set([])
- rflags = set(rdoc.content.get(
- fields.RECENTFLAGS_KEY, []))
- # ...and cache them now.
- self.memstore.load_recent_flags(
- self.mbox,
- {'doc_id': rdoc.doc_id, 'set': rflags})
- return rflags
-
- def _set_recent_flags(self, value):
- """
- Setter for the recent-flags set for this mailbox.
- """
- if self.memstore is not None:
- self.memstore.set_recent_flags(self.mbox, value)
-
- recent_flags = property(
- _get_recent_flags, _set_recent_flags,
- doc="Set of UIDs with the recent flag for this mailbox.")
-
- def _get_recent_doc_from_soledad(self):
- """
- Get recent-flags document from Soledad for this mailbox.
- :rtype: SoledadDocument or None
- """
- curried = partial(
- self._soledad.get_from_index,
- fields.TYPE_MBOX_IDX,
- fields.TYPE_RECENT_VAL, self.mbox)
- curried.expected = "rdoc"
- with self._rdoc_read_lock[self.mbox]:
- return try_unique_query(curried)
-
- # Property-set modification (protected by a different
- # lock to give atomicity to the read/write operation)
-
- def unset_recent_flags(self, uids):
- """
- Unset Recent flag for a sequence of uids.
-
- :param uids: the uids to unset
- :type uid: sequence
- """
- with self._rdoc_property_lock[self.mbox]:
- self.recent_flags.difference_update(
- set(uids))
-
- # Individual flags operations
-
- def unset_recent_flag(self, uid):
- """
- Unset Recent flag for a given uid.
-
- :param uid: the uid to unset
- :type uid: int
- """
- with self._rdoc_property_lock[self.mbox]:
- self.recent_flags.difference_update(
- set([uid]))
-
- @deferred_to_thread
- def set_recent_flag(self, uid):
- """
- Set Recent flag for a given uid.
-
- :param uid: the uid to set
- :type uid: int
- """
- with self._rdoc_property_lock[self.mbox]:
- self.recent_flags = self.recent_flags.union(
- set([uid]))
-
- # individual doc getters, message layer.
-
- def _get_fdoc_from_chash(self, chash):
- """
- Return a flags document for this mailbox with a given chash.
-
- :return: A SoledadDocument containing the Flags Document, or None if
- the query failed.
- :rtype: SoledadDocument or None.
- """
- curried = partial(
- self._soledad.get_from_index,
- fields.TYPE_MBOX_C_HASH_IDX,
- fields.TYPE_FLAGS_VAL, self.mbox, chash)
- curried.expected = "fdoc"
- fdoc = try_unique_query(curried)
- if fdoc is not None:
- return fdoc
- else:
- # probably this should be the other way round,
- # ie, try fist on memstore...
- cf = self.memstore._chash_fdoc_store
- fdoc = cf[chash][self.mbox]
- # hey, I just needed to wrap fdoc thing into
- # a "content" attribute, look a better way...
- if not empty(fdoc):
- return MessagePartDoc(
- new=None, dirty=None, part=None,
- store=None, doc_id=None,
- content=fdoc)
-
- def _get_uid_from_msgidCb(self, msgid):
- hdoc = None
- curried = partial(
- self._soledad.get_from_index,
- fields.TYPE_MSGID_IDX,
- fields.TYPE_HEADERS_VAL, msgid)
- curried.expected = "hdoc"
- hdoc = try_unique_query(curried)
-
- # XXX this is only a quick hack to avoid regression
- # on the "multiple copies of the draft" issue, but
- # this is currently broken since it's not efficient to
- # look for this. Should lookup better.
- # FIXME!
-
- if hdoc is not None:
- hdoc_dict = hdoc.content
-
- else:
- hdocstore = self.memstore._hdoc_store
- match = [x for _, x in hdocstore.items() if x['msgid'] == msgid]
- hdoc_dict = first(match)
-
- if hdoc_dict is None:
- logger.warning("Could not find hdoc for msgid %s"
- % (msgid,))
- return None
- msg_chash = hdoc_dict.get(fields.CONTENT_HASH_KEY)
-
- fdoc = self._get_fdoc_from_chash(msg_chash)
- if not fdoc:
- logger.warning("Could not find fdoc for msgid %s"
- % (msgid,))
- return None
- return fdoc.content.get(fields.UID_KEY, None)
-
- @deferred_to_thread
- def _get_uid_from_msgid(self, msgid):
- """
- Return a UID for a given message-id.
-
- It first gets the headers-doc for that msg-id, and
- it found it queries the flags doc for the current mailbox
- for the matching content-hash.
-
- :return: A UID, or None
- """
- # We need to wait a little bit, cause in some of the cases
- # the query is received right after we've saved the document,
- # and we cannot find it otherwise. This seems to be enough.
-
- # XXX do a deferLater instead ??
- # XXX is this working?
- return self._get_uid_from_msgidCb(msgid)
-
- @deferred_to_thread
- def set_flags(self, mbox, messages, flags, mode, observer):
- """
- Set flags for a sequence of messages.
-
- :param mbox: the mbox this message belongs to
- :type mbox: str or unicode
- :param messages: the messages to iterate through
- :type messages: sequence
- :flags: the flags to be set
- :type flags: tuple
- :param mode: the mode for setting. 1 is append, -1 is remove, 0 set.
- :type mode: int
- :param observer: a deferred that will be called with the dictionary
- mapping UIDs to flags after the operation has been
- done.
- :type observer: deferred
- """
- reactor = self.reactor
- getmsg = self.get_msg_by_uid
-
- def set_flags(uid, flags, mode):
- msg = getmsg(uid, mem_only=True, flags_only=True)
- if msg is not None:
- return uid, msg.setFlags(flags, mode)
-
- setted_flags = [set_flags(uid, flags, mode) for uid in messages]
- result = dict(filter(None, setted_flags))
-
- reactor.callFromThread(observer.callback, result)
-
- # getters: generic for a mailbox
-
- def get_msg_by_uid(self, uid, mem_only=False, flags_only=False):
- """
- Retrieves a LeapMessage by UID.
- This is used primarity in the Mailbox fetch and store methods.
-
- :param uid: the message uid to query by
- :type uid: int
- :param mem_only: a flag that indicates whether this Message should
- pass a reference to soledad to retrieve missing pieces
- or not.
- :type mem_only: bool
- :param flags_only: whether the message should carry only a reference
- to the flags document.
- :type flags_only: bool
-
- :return: A LeapMessage instance matching the query,
- or None if not found.
- :rtype: LeapMessage
- """
- msg_container = self.memstore.get_message(
- self.mbox, uid, flags_only=flags_only)
-
- if msg_container is not None:
- if mem_only:
- msg = LeapMessage(None, uid, self.mbox, collection=self,
- container=msg_container)
- else:
- # We pass a reference to soledad just to be able to retrieve
- # missing parts that cannot be found in the container, like
- # the content docs after a copy.
- msg = LeapMessage(self._soledad, uid, self.mbox,
- collection=self, container=msg_container)
- else:
- msg = LeapMessage(self._soledad, uid, self.mbox, collection=self)
-
- if not msg.does_exist():
- return None
- return msg
-
- def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL):
- """
- Get all documents for the selected mailbox of the
- passed type. By default, it returns the flag docs.
-
- If you want acess to the content, use __iter__ instead
-
- :return: a list of u1db documents
- :rtype: list of SoledadDocument
- """
- if _type not in fields.__dict__.values():
- raise TypeError("Wrong type passed to get_all_docs")
-
- if sameProxiedObjects(self._soledad, None):
- logger.warning('Tried to get messages but soledad is None!')
- return []
-
- all_docs = [doc for doc in self._soledad.get_from_index(
- fields.TYPE_MBOX_IDX,
- _type, self.mbox)]
-
- # inneficient, but first let's grok it and then
- # let's worry about efficiency.
- # XXX FIXINDEX -- should implement order by in soledad
- # FIXME ----------------------------------------------
- return sorted(all_docs, key=lambda item: item.content['uid'])
-
- def all_soledad_uid_iter(self):
- """
- Return an iterator through the UIDs of all messages, sorted in
- ascending order.
- """
- db_uids = set([doc.content[self.UID_KEY] for doc in
- self._soledad.get_from_index(
- fields.TYPE_MBOX_IDX,
- fields.TYPE_FLAGS_VAL, self.mbox)
- if not empty(doc)])
- return db_uids
-
- def all_uid_iter(self):
- """
- Return an iterator through the UIDs of all messages, from memory.
- """
- mem_uids = self.memstore.get_uids(self.mbox)
- soledad_known_uids = self.memstore.get_soledad_known_uids(
- self.mbox)
- combined = tuple(set(mem_uids).union(soledad_known_uids))
- return combined
-
- def get_all_soledad_flag_docs(self):
- """
- Return a dict with the content of all the flag documents
- in soledad store for the given mbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :rtype: dict
- """
- # XXX we really could return a reduced version with
- # just {'uid': (flags-tuple,) since the prefetch is
- # only oriented to get the flag tuples.
- all_docs = [(
- doc.content[self.UID_KEY],
- dict(doc.content))
- for doc in
- self._soledad.get_from_index(
- fields.TYPE_MBOX_IDX,
- fields.TYPE_FLAGS_VAL, self.mbox)
- if not empty(doc.content)]
- all_flags = dict(all_docs)
- return all_flags
-
- def all_headers(self):
- """
- Return a dict with all the header documents for this
- mailbox.
-
- :rtype: dict
- """
- return self.memstore.all_headers(self.mbox)
-
- def count(self):
- """
- Return the count of messages for this mailbox.
-
- :rtype: int
- """
- return self.memstore.count(self.mbox)
-
- # unseen messages
-
- def unseen_iter(self):
- """
- Get an iterator for the message UIDs with no `seen` flag
- for this mailbox.
-
- :return: iterator through unseen message doc UIDs
- :rtype: iterable
- """
- return self.memstore.unseen_iter(self.mbox)
-
- def count_unseen(self):
- """
- Count all messages with the `Unseen` flag.
-
- :returns: count
- :rtype: int
- """
- return len(list(self.unseen_iter()))
-
- def get_unseen(self):
- """
- Get all messages with the `Unseen` flag
-
- :returns: a list of LeapMessages
- :rtype: list
- """
- return [LeapMessage(self._soledad, docid, self.mbox, collection=self)
- for docid in self.unseen_iter()]
+ def __prefetch_body_file(self):
+ def assign_body_fd(fd):
+ self.__body_fd = fd
+ return fd
+ d = self.getBodyFile()
+ d.addCallback(assign_body_fd)
+ return d
- # recent messages
- # XXX take it from memstore
- def count_recent(self):
- """
- Count all messages with the `Recent` flag.
- It just retrieves the length of the recent_flags set,
- which is stored in a specific type of document for
- this collection.
+class IMAPMessagePart(object):
- :returns: count
- :rtype: int
- """
- return len(self.recent_flags)
+ def __init__(self, message_part):
+ self.message_part = message_part
- def __len__(self):
- """
- Returns the number of messages on this mailbox.
+ def getBodyFile(self, store=None):
+ return self.message_part.get_body_file()
- :rtype: int
- """
- return self.count()
-
- def __iter__(self):
- """
- Returns an iterator over all messages.
+ def getSize(self):
+ return self.message_part.get_size()
- :returns: iterator of dicts with content for all messages.
- :rtype: iterable
- """
- return (LeapMessage(self._soledad, docuid, self.mbox, collection=self)
- for docuid in self.all_uid_iter())
+ def getHeaders(self, negate, *names):
+ headers = self.message_part.get_headers()
+ return _format_headers(headers, negate, *names)
- def __repr__(self):
- """
- Representation string for this object.
- """
- return u"<MessageCollection: mbox '%s' (%s)>" % (
- self.mbox, self.count())
+ def isMultipart(self):
+ return self.message_part.is_multipart()
- # XXX should implement __eq__ also !!!
- # use chash...
+ def getSubPart(self, part):
+ subpart = self.message_part.get_subpart(part + 1)
+ return IMAPMessagePart(subpart)
+
+
+def _format_headers(headers, negate, *names):
+ # current server impl. expects content-type to be present, so if for
+ # some reason we do not have headers, we have to return at least that
+ # one
+ if not headers:
+ logger.warning("No headers found")
+ return {str('content-type'): str('')}
+
+ names = map(lambda s: s.upper(), names)
+
+ if negate:
+ def cond(key):
+ return key.upper() not in names
+ else:
+ def cond(key):
+ return key.upper() in names
+
+ if isinstance(headers, list):
+ headers = dict(headers)
+
+ # default to most likely standard
+ charset = find_charset(headers, "utf-8")
+
+ # We will return a copy of the headers dictionary that
+ # will allow case-insensitive lookups. In some parts of the twisted imap
+ # server code the keys are expected to be in lower case, and in this way
+ # we avoid having to convert them.
+
+ _headers = CaseInsensitiveDict()
+ for key, value in headers.items():
+ if not isinstance(key, str):
+ key = key.encode(charset, 'replace')
+ if not isinstance(value, str):
+ value = value.encode(charset, 'replace')
+
+ if value.endswith(";"):
+ # bastards
+ value = value[:-1]
+
+ # filter original dict by negate-condition
+ if cond(key):
+ _headers[key] = value
+
+ return _headers
diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py
deleted file mode 100644
index 4a801b0..0000000
--- a/src/leap/mail/imap/parser.py
+++ /dev/null
@@ -1,45 +0,0 @@
-# -*- coding: utf-8 -*-
-# parser.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-Mail parser mixin.
-"""
-import re
-
-
-class MBoxParser(object):
- """
- Utility function to parse mailbox names.
- """
- INBOX_NAME = "INBOX"
- INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
-
- def _parse_mailbox_name(self, name):
- """
- Return a normalized representation of the mailbox C{name}.
-
- This method ensures that an eventual initial 'inbox' part of a
- mailbox name is made uppercase.
-
- :param name: the name of the mailbox
- :type name: unicode
-
- :rtype: unicode
- """
- if self.INBOX_RE.match(name):
- # ensure inital INBOX is uppercase
- return self.INBOX_NAME + name[len(self.INBOX_NAME):]
- return name
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index fe56ea6..39f483f 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -15,18 +15,19 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
-Leap IMAP4 Server Implementation.
+LEAP IMAP4 Server Implementation.
"""
+import StringIO
from copy import copy
from twisted import cred
+from twisted.internet import reactor
from twisted.internet.defer import maybeDeferred
from twisted.mail import imap4
from twisted.python import log
-from leap.common import events as leap_events
from leap.common.check import leap_assert, leap_assert_type
-from leap.common.events.events_pb2 import IMAP_CLIENT_LOGIN
+from leap.common.events import emit, catalog
from leap.soledad.client import Soledad
# imports for LITERAL+ patch
@@ -35,9 +36,41 @@ from twisted.mail.imap4 import IllegalClientResponse
from twisted.mail.imap4 import LiteralString, LiteralFile
-class LeapIMAPServer(imap4.IMAP4Server):
+def _getContentType(msg):
"""
- An IMAP4 Server with mailboxes backed by soledad
+ Return a two-tuple of the main and subtype of the given message.
+ """
+ attrs = None
+ mm = msg.getHeaders(False, 'content-type').get('content-type', None)
+ if mm:
+ mm = ''.join(mm.splitlines())
+ mimetype = mm.split(';')
+ if mimetype:
+ type = mimetype[0].split('/', 1)
+ if len(type) == 1:
+ major = type[0]
+ minor = None
+ elif len(type) == 2:
+ major, minor = type
+ else:
+ major = minor = None
+ # XXX patched ---------------------------------------------
+ attrs = dict(x.strip().split('=', 1) for x in mimetype[1:])
+ # XXX patched ---------------------------------------------
+ else:
+ major = minor = None
+ else:
+ major = minor = None
+ return major, minor, attrs
+
+# Monkey-patch _getContentType to avoid bug that passes lower-case boundary in
+# BODYSTRUCTURE response.
+imap4._getContentType = _getContentType
+
+
+class LEAPIMAPServer(imap4.IMAP4Server):
+ """
+ An IMAP4 Server with a LEAP Storage Backend.
"""
def __init__(self, *args, **kwargs):
# pop extraneous arguments
@@ -59,8 +92,87 @@ class LeapIMAPServer(imap4.IMAP4Server):
# populate the test account properly (and only once
# per session)
- from twisted.internet import reactor
- self.reactor = reactor
+ #############################################################
+ #
+ # Twisted imap4 patch to workaround bad mime rendering in TB.
+ # See https://leap.se/code/issues/6773
+ # and https://bugzilla.mozilla.org/show_bug.cgi?id=149771
+ # Still unclear if this is a thunderbird bug.
+ # TODO send this patch upstream
+ #
+ #############################################################
+
+ def spew_body(self, part, id, msg, _w=None, _f=None):
+ if _w is None:
+ _w = self.transport.write
+ for p in part.part:
+ if msg.isMultipart():
+ msg = msg.getSubPart(p)
+ elif p > 0:
+ # Non-multipart messages have an implicit first part but no
+ # other parts - reject any request for any other part.
+ raise TypeError("Requested subpart of non-multipart message")
+
+ if part.header:
+ hdrs = msg.getHeaders(part.header.negate, *part.header.fields)
+ hdrs = imap4._formatHeaders(hdrs)
+ # PATCHED ##########################################
+ _w(str(part) + ' ' + imap4._literal(hdrs + "\r\n"))
+ # PATCHED ##########################################
+ elif part.text:
+ _w(str(part) + ' ')
+ _f()
+ return imap4.FileProducer(
+ msg.getBodyFile()
+ ).beginProducing(self.transport)
+ elif part.mime:
+ hdrs = imap4._formatHeaders(msg.getHeaders(True))
+
+ # PATCHED ##########################################
+ _w(str(part) + ' ' + imap4._literal(hdrs + "\r\n"))
+ # END PATCHED ######################################
+
+ elif part.empty:
+ _w(str(part) + ' ')
+ _f()
+ if part.part:
+ # PATCHED #############################################
+ # implement partial FETCH
+ # TODO implement boundary checks
+ # TODO see if there's a more efficient way, without
+ # copying the original content into a new buffer.
+ fd = msg.getBodyFile()
+ begin = getattr(part, "partialBegin", None)
+ _len = getattr(part, "partialLength", None)
+ if begin is not None and _len is not None:
+ _fd = StringIO.StringIO()
+ fd.seek(part.partialBegin)
+ _fd.write(fd.read(part.partialLength))
+ _fd.seek(0)
+ else:
+ _fd = fd
+ return imap4.FileProducer(
+ _fd
+ # END PATCHED #########################3
+ ).beginProducing(self.transport)
+ else:
+ mf = imap4.IMessageFile(msg, None)
+ if mf is not None:
+ return imap4.FileProducer(
+ mf.open()).beginProducing(self.transport)
+ return imap4.MessageProducer(
+ msg, None, self._scheduler).beginProducing(self.transport)
+
+ else:
+ _w('BODY ' +
+ imap4.collapseNestedLists([imap4.getBodyStructure(msg)]))
+
+ ##################################################################
+ #
+ # END Twisted imap4 patch to workaround bad mime rendering in TB.
+ # #6773
+ #
+ ##################################################################
def lineReceived(self, line):
"""
@@ -69,7 +181,7 @@ class LeapIMAPServer(imap4.IMAP4Server):
:param line: the line from the server, without the line delimiter.
:type line: str
"""
- if self.theAccount.closed is True and self.state != "unauth":
+ if self.theAccount.session_ended is True and self.state != "unauth":
log.msg("Closing the session. State: unauth")
self.state = "unauth"
@@ -82,6 +194,20 @@ class LeapIMAPServer(imap4.IMAP4Server):
log.msg('rcv (%s): %s' % (self.state, msg))
imap4.IMAP4Server.lineReceived(self, line)
+ def close_server_connection(self):
+ """
+ Send a BYE command so that the MUA at least knows that we're closing
+ the connection.
+ """
+ self.sendLine(
+ '* BYE LEAP IMAP Proxy is shutting down; '
+ 'so long and thanks for all the fish')
+ self.transport.loseConnection()
+ if self.mbox:
+ self.mbox.removeListener(self)
+ self.mbox = None
+ self.state = 'unauth'
+
def authenticateLogin(self, username, password):
"""
Lookup the account with the given parameters, and deny
@@ -98,7 +224,7 @@ class LeapIMAPServer(imap4.IMAP4Server):
# bad username, reject.
raise cred.error.UnauthorizedLogin()
# any dummy password is allowed so far. use realm instead!
- leap_events.signal(IMAP_CLIENT_LOGIN, "1")
+ emit(catalog.IMAP_CLIENT_LOGIN, "1")
return imap4.IAccount, self.theAccount, lambda: None
def do_FETCH(self, tag, messages, query, uid=0):
@@ -147,12 +273,15 @@ class LeapIMAPServer(imap4.IMAP4Server):
"""
Notify new messages to listeners.
"""
- self.reactor.callFromThread(self.mbox.notify_new)
+ reactor.callFromThread(self.mbox.notify_new)
def _cbSelectWork(self, mbox, cmdName, tag):
"""
- Callback for selectWork, patched to avoid conformance errors due to
- incomplete UIDVALIDITY line.
+ Callback for selectWork
+
+ * patched to avoid conformance errors due to incomplete UIDVALIDITY
+ line.
+ * patched to accept deferreds for messagecount and recent count
"""
if mbox is None:
self.sendNegativeResponse(tag, 'No such mailbox')
@@ -161,12 +290,22 @@ class LeapIMAPServer(imap4.IMAP4Server):
self.sendNegativeResponse(tag, 'Mailbox cannot be selected')
return
+ d1 = defer.maybeDeferred(mbox.getMessageCount)
+ d2 = defer.maybeDeferred(mbox.getRecentCount)
+ return defer.gatherResults([d1, d2]).addCallback(
+ self.__cbSelectWork, mbox, cmdName, tag)
+
+ def __cbSelectWork(self, ((msg_count, recent_count)), mbox, cmdName, tag):
flags = mbox.getFlags()
- self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS')
- self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT')
self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags))
# Patched -------------------------------------------------------
+ # accept deferreds for the count
+ self.sendUntaggedResponse(str(msg_count) + ' EXISTS')
+ self.sendUntaggedResponse(str(recent_count) + ' RECENT')
+ # ----------------------------------------------------------------
+
+ # Patched -------------------------------------------------------
# imaptest was complaining about the incomplete line, we're adding
# "UIDs valid" here.
self.sendPositiveResponse(
@@ -188,8 +327,7 @@ class LeapIMAPServer(imap4.IMAP4Server):
a deferred, the client will only be informed of success (or failure)
when the deferred's callback (or errback) is invoked.
"""
- # TODO return the output of _memstore.is_writing
- # XXX and that should return a deferred!
+ # TODO implement a collection of ongoing deferreds?
return None
#############################################################
@@ -311,21 +449,247 @@ class LeapIMAPServer(imap4.IMAP4Server):
return self._fileLiteral(size, literalPlus)
#############################
- # Need to override the command table after patching
- # arg_astring and arg_literal
+ # --------------------------------- isSubscribed patch
+ # TODO -- send patch upstream.
+ # There is a bug in twisted implementation:
+ # in cbListWork, it's assumed that account.isSubscribed IS a callable,
+ # although in the interface documentation it's stated that it can be
+ # a deferred.
+
+ def _listWork(self, tag, ref, mbox, sub, cmdName):
+ mbox = self._parseMbox(mbox)
+ mailboxes = maybeDeferred(self.account.listMailboxes, ref, mbox)
+ mailboxes.addCallback(self._cbSubscribed)
+ mailboxes.addCallback(
+ self._cbListWork, tag, sub, cmdName,
+ ).addErrback(self._ebListWork, tag)
+
+ def _cbSubscribed(self, mailboxes):
+ subscribed = [
+ maybeDeferred(self.account.isSubscribed, name)
+ for (name, box) in mailboxes]
+
+ def get_mailboxes_and_subs(result):
+ subscribed = [i[0] for i, yes in zip(mailboxes, result) if yes]
+ return mailboxes, subscribed
+
+ d = defer.gatherResults(subscribed)
+ d.addCallback(get_mailboxes_and_subs)
+ return d
+
+ def _cbListWork(self, mailboxes_subscribed, tag, sub, cmdName):
+ mailboxes, subscribed = mailboxes_subscribed
+
+ for (name, box) in mailboxes:
+ if not sub or name in subscribed:
+ flags = box.getFlags()
+ delim = box.getHierarchicalDelimiter()
+ resp = (imap4.DontQuoteMe(cmdName),
+ map(imap4.DontQuoteMe, flags),
+ delim, name.encode('imap4-utf-7'))
+ self.sendUntaggedResponse(
+ imap4.collapseNestedLists(resp))
+ self.sendPositiveResponse(tag, '%s completed' % (cmdName,))
+ # -------------------- end isSubscribed patch -----------
+
+ # TODO subscribe method had also to be changed to accomodate deferred
+ def do_SUBSCRIBE(self, tag, name):
+ name = self._parseMbox(name)
+
+ def _subscribeCb(_):
+ self.sendPositiveResponse(tag, 'Subscribed')
+
+ def _subscribeEb(failure):
+ m = failure.value
+ log.err()
+ if failure.check(imap4.MailboxException):
+ self.sendNegativeResponse(tag, str(m))
+ else:
+ self.sendBadResponse(
+ tag,
+ "Server error encountered while subscribing to mailbox")
+
+ d = self.account.subscribe(name)
+ d.addCallbacks(_subscribeCb, _subscribeEb)
+ return d
+
+ auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring)
+ select_SUBSCRIBE = auth_SUBSCRIBE
+
+ def do_UNSUBSCRIBE(self, tag, name):
+ # unsubscribe method had also to be changed to accomodate
+ # deferred
+ name = self._parseMbox(name)
+
+ def _unsubscribeCb(_):
+ self.sendPositiveResponse(tag, 'Unsubscribed')
+
+ def _unsubscribeEb(failure):
+ m = failure.value
+ log.err()
+ if failure.check(imap4.MailboxException):
+ self.sendNegativeResponse(tag, str(m))
+ else:
+ self.sendBadResponse(
+ tag,
+ "Server error encountered while unsubscribing "
+ "from mailbox")
+
+ d = self.account.unsubscribe(name)
+ d.addCallbacks(_unsubscribeCb, _unsubscribeEb)
+ return d
+
+ auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring)
+ select_UNSUBSCRIBE = auth_UNSUBSCRIBE
+
+ def do_RENAME(self, tag, oldname, newname):
+ oldname, newname = [self._parseMbox(n) for n in oldname, newname]
+ if oldname.lower() == 'inbox' or newname.lower() == 'inbox':
+ self.sendNegativeResponse(
+ tag,
+ 'You cannot rename the inbox, or '
+ 'rename another mailbox to inbox.')
+ return
+ def _renameCb(_):
+ self.sendPositiveResponse(tag, 'Mailbox renamed')
+
+ def _renameEb(failure):
+ m = failure.value
+ if failure.check(TypeError):
+ self.sendBadResponse(tag, 'Invalid command syntax')
+ elif failure.check(imap4.MailboxException):
+ self.sendNegativeResponse(tag, str(m))
+ else:
+ log.err()
+ self.sendBadResponse(
+ tag,
+ "Server error encountered while "
+ "renaming mailbox")
+
+ d = self.account.rename(oldname, newname)
+ d.addCallbacks(_renameCb, _renameEb)
+ return d
+
+ auth_RENAME = (do_RENAME, arg_astring, arg_astring)
+ select_RENAME = auth_RENAME
+
+ def do_CREATE(self, tag, name):
+ name = self._parseMbox(name)
+
+ def _createCb(result):
+ if result:
+ self.sendPositiveResponse(tag, 'Mailbox created')
+ else:
+ self.sendNegativeResponse(tag, 'Mailbox not created')
+
+ def _createEb(failure):
+ c = failure.value
+ if failure.check(imap4.MailboxException):
+ self.sendNegativeResponse(tag, str(c))
+ else:
+ log.err()
+ self.sendBadResponse(
+ tag, "Server error encountered while creating mailbox")
+
+ d = self.account.create(name)
+ d.addCallbacks(_createCb, _createEb)
+ return d
+
+ auth_CREATE = (do_CREATE, arg_astring)
+ select_CREATE = auth_CREATE
+
+ def do_DELETE(self, tag, name):
+ name = self._parseMbox(name)
+ if name.lower() == 'inbox':
+ self.sendNegativeResponse(tag, 'You cannot delete the inbox')
+ return
+
+ def _deleteCb(result):
+ self.sendPositiveResponse(tag, 'Mailbox deleted')
+
+ def _deleteEb(failure):
+ m = failure.value
+ if failure.check(imap4.MailboxException):
+ self.sendNegativeResponse(tag, str(m))
+ else:
+ print "SERVER: other error"
+ log.err()
+ self.sendBadResponse(
+ tag,
+ "Server error encountered while deleting mailbox")
+
+ d = self.account.delete(name)
+ d.addCallbacks(_deleteCb, _deleteEb)
+ return d
+
+ auth_DELETE = (do_DELETE, arg_astring)
+ select_DELETE = auth_DELETE
+
+ # -----------------------------------------------------------------------
+ # Patched just to allow __cbAppend to receive a deferred from messageCount
+ # TODO format and send upstream.
+ def do_APPEND(self, tag, mailbox, flags, date, message):
+ mailbox = self._parseMbox(mailbox)
+ maybeDeferred(self.account.select, mailbox).addCallback(
+ self._cbAppendGotMailbox, tag, flags, date, message).addErrback(
+ self._ebAppendGotMailbox, tag)
+
+ def __ebAppend(self, failure, tag):
+ self.sendBadResponse(tag, 'APPEND failed: ' + str(failure.value))
+
+ def _cbAppendGotMailbox(self, mbox, tag, flags, date, message):
+ if not mbox:
+ self.sendNegativeResponse(tag, '[TRYCREATE] No such mailbox')
+ return
+
+ d = mbox.addMessage(message, flags, date)
+ d.addCallback(self.__cbAppend, tag, mbox)
+ d.addErrback(self.__ebAppend, tag)
+
+ def _ebAppendGotMailbox(self, failure, tag):
+ self.sendBadResponse(
+ tag, "Server error encountered while opening mailbox.")
+ log.err(failure)
+
+ def __cbAppend(self, result, tag, mbox):
+
+ # XXX patched ---------------------------------
+ def send_response(count):
+ self.sendUntaggedResponse('%d EXISTS' % count)
+ self.sendPositiveResponse(tag, 'APPEND complete')
+
+ d = mbox.getMessageCount()
+ d.addCallback(send_response)
+ return d
+ # XXX patched ---------------------------------
+
+ # -----------------------------------------------------------------------
+
+ auth_APPEND = (do_APPEND, arg_astring, imap4.IMAP4Server.opt_plist,
+ imap4.IMAP4Server.opt_datetime, arg_literal)
+ select_APPEND = auth_APPEND
+
+ # Need to override the command table after patching
+ # arg_astring and arg_literal, except on the methods that we are already
+ # overriding.
+
+ # TODO --------------------------------------------
+ # Check if we really need to override these
+ # methods, or we can monkeypatch.
+ # do_DELETE = imap4.IMAP4Server.do_DELETE
+ # do_CREATE = imap4.IMAP4Server.do_CREATE
+ # do_RENAME = imap4.IMAP4Server.do_RENAME
+ # do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE
+ # do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE
+ # do_APPEND = imap4.IMAP4Server.do_APPEND
+ # -------------------------------------------------
do_LOGIN = imap4.IMAP4Server.do_LOGIN
- do_CREATE = imap4.IMAP4Server.do_CREATE
- do_DELETE = imap4.IMAP4Server.do_DELETE
- do_RENAME = imap4.IMAP4Server.do_RENAME
- do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE
- do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE
do_STATUS = imap4.IMAP4Server.do_STATUS
- do_APPEND = imap4.IMAP4Server.do_APPEND
do_COPY = imap4.IMAP4Server.do_COPY
_selectWork = imap4.IMAP4Server._selectWork
- _listWork = imap4.IMAP4Server._listWork
+
arg_plist = imap4.IMAP4Server.arg_plist
arg_seqset = imap4.IMAP4Server.arg_seqset
opt_plist = imap4.IMAP4Server.opt_plist
@@ -342,8 +706,15 @@ class LeapIMAPServer(imap4.IMAP4Server):
auth_EXAMINE = (_selectWork, arg_astring, 0, 'EXAMINE')
select_EXAMINE = auth_EXAMINE
- auth_DELETE = (do_DELETE, arg_astring)
- select_DELETE = auth_DELETE
+ # TODO -----------------------------------------------
+ # re-add if we stop overriding DELETE
+ # auth_DELETE = (do_DELETE, arg_astring)
+ # select_DELETE = auth_DELETE
+ # auth_APPEND = (do_APPEND, arg_astring, opt_plist, opt_datetime,
+ # arg_literal)
+ # select_APPEND = auth_APPEND
+
+ # ----------------------------------------------------
auth_RENAME = (do_RENAME, arg_astring, arg_astring)
select_RENAME = auth_RENAME
@@ -363,13 +734,8 @@ class LeapIMAPServer(imap4.IMAP4Server):
auth_STATUS = (do_STATUS, arg_astring, arg_plist)
select_STATUS = auth_STATUS
- auth_APPEND = (do_APPEND, arg_astring, opt_plist, opt_datetime,
- arg_literal)
- select_APPEND = auth_APPEND
-
select_COPY = (do_COPY, arg_seqset, arg_astring)
-
#############################################################
# END of Twisted imap4 patch to support LITERAL+ extension
#############################################################
diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac
index 651f71b..2045757 100644
--- a/src/leap/mail/imap/service/imap-server.tac
+++ b/src/leap/mail/imap/service/imap-server.tac
@@ -23,9 +23,9 @@ For now, and for debugging/testing purposes, you need
to pass a config file with the following structure:
[leap_mail]
-userid = "user@provider"
-uuid = "deadbeefdeadabad"
-passwd = "supersecret" # optional, will get prompted if not found.
+userid = 'user@provider'
+uuid = 'deadbeefdeadabad'
+passwd = 'supersecret' # optional, will get prompted if not found.
"""
import ConfigParser
import getpass
@@ -53,38 +53,17 @@ def initialize_soledad(uuid, email, passwd,
:param tempdir: path to temporal dir
:rtype: Soledad instance
"""
- # XXX TODO unify with an authoritative source of mocks
- # for soledad (or partial initializations).
- # This is copied from the imap tests.
-
server_url = "http://provider"
cert_file = ""
- class Mock(object):
- def __init__(self, return_value=None):
- self._return = return_value
-
- def __call__(self, *args, **kwargs):
- return self._return
-
- class MockSharedDB(object):
-
- get_doc = Mock()
- put_doc = Mock()
- lock = Mock(return_value=('atoken', 300))
- unlock = Mock(return_value=True)
-
- def __call__(self):
- return self
-
- Soledad._shared_db = MockSharedDB()
soledad = Soledad(
uuid,
passwd,
secrets,
localdb,
server_url,
- cert_file)
+ cert_file,
+ syncable=False)
return soledad
@@ -95,9 +74,9 @@ def initialize_soledad(uuid, email, passwd,
print "[+] Running LEAP IMAP Service"
-bmconf = os.environ.get("LEAP_MAIL_CONF", "")
+bmconf = os.environ.get("LEAP_MAIL_CONFIG", "")
if not bmconf:
- print ("[-] Please set LEAP_MAIL_CONF environment variable "
+ print ("[-] Please set LEAP_MAIL_CONFIG environment variable "
"pointing to your config.")
sys.exit(1)
@@ -131,6 +110,7 @@ tempdir = "/tmp/"
# Ad-hoc soledad/keymanager initialization.
+print "[~] user:", userid
soledad = initialize_soledad(uuid, userid, passwd, secrets,
localdb, gnupg_home, tempdir)
km_args = (userid, "https://localhost", soledad)
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 10ba32a..c3ae59a 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# imap.py
-# Copyright (C) 2013 LEAP
+# Copyright (C) 2013-2015 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -15,58 +15,27 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
-Imap service initialization
+IMAP service initialization
"""
import logging
import os
-import time
-from twisted.internet import defer, threads
-from twisted.internet.protocol import ServerFactory
+from collections import defaultdict
+
+from twisted.internet import reactor
from twisted.internet.error import CannotListenError
+from twisted.internet.protocol import ServerFactory
from twisted.mail import imap4
from twisted.python import log
-logger = logging.getLogger(__name__)
+from leap.common.events import emit, catalog
+from leap.common.check import leap_check
+from leap.mail.imap.account import IMAPAccount
+from leap.mail.imap.server import LEAPIMAPServer
-from leap.common import events as leap_events
-from leap.common.check import leap_assert, leap_assert_type, leap_check
-from leap.keymanager import KeyManager
-from leap.mail.imap.account import SoledadBackedAccount
-from leap.mail.imap.fetch import LeapIncomingMail
-from leap.mail.imap.memorystore import MemoryStore
-from leap.mail.imap.server import LeapIMAPServer
-from leap.mail.imap.soledadstore import SoledadStore
-from leap.soledad.client import Soledad
-
-# The default port in which imap service will run
-IMAP_PORT = 1984
+# TODO: leave only an implementor of IService in here
-# The period between succesive checks of the incoming mail
-# queue (in seconds)
-INCOMING_CHECK_PERIOD = 60
-
-from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED
-from leap.common.events.events_pb2 import IMAP_SERVICE_FAILED_TO_START
-
-######################################################
-# Temporary workaround for RecursionLimit when using
-# qt4reactor. Do remove when we move to poll or select
-# reactor, which do not show those problems. See #4974
-import resource
-import sys
-
-try:
- sys.setrecursionlimit(10**7)
-except Exception:
- print "Error setting recursion limit"
-try:
- # Increase max stack size from 8MB to 256MB
- resource.setrlimit(resource.RLIMIT_STACK, (2**28, -1))
-except Exception:
- print "Error setting stack size"
-
-######################################################
+logger = logging.getLogger(__name__)
DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None)
if DO_MANHOLE:
@@ -81,6 +50,9 @@ if DO_PROFILE:
pr = cProfile.Profile()
pr.enable()
+# The default port in which imap service will run
+IMAP_PORT = 1984
+
class IMAPAuthRealm(object):
"""
@@ -97,6 +69,7 @@ class LeapIMAPFactory(ServerFactory):
Factory for a IMAP4 server with soledad remote sync and gpg-decryption
capabilities.
"""
+ protocol = LEAPIMAPServer
def __init__(self, uuid, userid, soledad):
"""
@@ -114,14 +87,10 @@ class LeapIMAPFactory(ServerFactory):
self._uuid = uuid
self._userid = userid
self._soledad = soledad
- self._memstore = MemoryStore(
- permanent_store=SoledadStore(soledad))
- theAccount = SoledadBackedAccount(
- uuid, soledad=soledad,
- memstore=self._memstore)
+ theAccount = IMAPAccount(uuid, soledad)
self.theAccount = theAccount
-
+ self._connections = defaultdict()
# XXX how to pass the store along?
def buildProtocol(self, addr):
@@ -131,91 +100,66 @@ class LeapIMAPFactory(ServerFactory):
:param addr: remote ip address
:type addr: str
"""
- imapProtocol = LeapIMAPServer(
+ # TODO should reject anything from addr != localhost,
+ # just in case.
+ log.msg("Building protocol for connection %s" % addr)
+ imapProtocol = self.protocol(
uuid=self._uuid,
userid=self._userid,
soledad=self._soledad)
imapProtocol.theAccount = self.theAccount
imapProtocol.factory = self
+
+ self._connections[addr] = imapProtocol
return imapProtocol
- def doStop(self, cv=None):
+ def stopFactory(self):
+ # say bye!
+ for conn, proto in self._connections.items():
+ log.msg("Closing connections for %s" % conn)
+ proto.close_server_connection()
+
+ def doStop(self):
"""
Stops imap service (fetcher, factory and port).
-
- :param cv: A condition variable to which we can signal when imap
- indeed stops.
- :type cv: threading.Condition
- :return: a Deferred that stops and flushes the in memory store data to
- disk in another thread.
- :rtype: Deferred
"""
+ # mark account as unusable, so any imap command will fail
+ # with unauth state.
+ self.theAccount.end_session()
+
+ # TODO should wait for all the pending deferreds,
+ # the twisted way!
if DO_PROFILE:
log.msg("Stopping PROFILING")
pr.disable()
pr.dump_stats(PROFILE_DAT)
- ServerFactory.doStop(self)
-
- if cv is not None:
- def _stop_imap_cb():
- logger.debug('Stopping in memory store.')
- self._memstore.stop_and_flush()
- while not self._memstore.producer.is_queue_empty():
- logger.debug('Waiting for queue to be empty.')
- # TODO use a gatherResults over the new/dirty
- # deferred list,
- # as in memorystore's expunge() method.
- time.sleep(1)
- # notify that service has stopped
- logger.debug('Notifying that service has stopped.')
- cv.acquire()
- cv.notify()
- cv.release()
-
- return threads.deferToThread(_stop_imap_cb)
+ return ServerFactory.doStop(self)
-def run_service(*args, **kwargs):
+def run_service(store, **kwargs):
"""
Main entry point to run the service from the client.
- :returns: the LoopingCall instance that will have to be stoppped
- before shutting down the client, the port as returned by
- the reactor when starts listening, and the factory for
- the protocol.
- """
- from twisted.internet import reactor
- # it looks like qtreactor does not honor this,
- # but other reactors should.
- reactor.suggestThreadPoolSize(20)
+ :param store: a soledad instance
- leap_assert(len(args) == 2)
- soledad, keymanager = args
- leap_assert_type(soledad, Soledad)
- leap_assert_type(keymanager, KeyManager)
+ :returns: the port as returned by the reactor when starts listening, and
+ the factory for the protocol.
+ """
+ leap_check(store, "store cannot be None")
+ # XXX this can also be a ProxiedObject, FIXME
+ # leap_assert_type(store, Soledad)
port = kwargs.get('port', IMAP_PORT)
- check_period = kwargs.get('check_period', INCOMING_CHECK_PERIOD)
userid = kwargs.get('userid', None)
leap_check(userid is not None, "need an user id")
- offline = kwargs.get('offline', False)
- uuid = soledad._get_uuid()
- factory = LeapIMAPFactory(uuid, userid, soledad)
+ uuid = store.uuid
+ factory = LeapIMAPFactory(uuid, userid, store)
try:
tport = reactor.listenTCP(port, factory,
interface="localhost")
- if not offline:
- fetcher = LeapIncomingMail(
- keymanager,
- soledad,
- factory.theAccount,
- check_period,
- userid)
- else:
- fetcher = None
except CannotListenError:
logger.error("IMAP Service failed to start: "
"cannot listen in port %s" % (port,))
@@ -223,7 +167,6 @@ def run_service(*args, **kwargs):
logger.error("Error launching IMAP service: %r" % (exc,))
else:
# all good.
- # (the caller has still to call fetcher.start_loop)
if DO_MANHOLE:
# TODO get pass from env var.too.
@@ -235,8 +178,10 @@ def run_service(*args, **kwargs):
reactor.listenTCP(manhole.MANHOLE_PORT, manhole_factory,
interface="127.0.0.1")
logger.debug("IMAP4 Server is RUNNING in port %s" % (port,))
- leap_events.signal(IMAP_SERVICE_STARTED, str(port))
- return fetcher, tport, factory
+ emit(catalog.IMAP_SERVICE_STARTED, str(port))
+
+ # FIXME -- change service signature
+ return tport, factory
# not ok, signal error.
- leap_events.signal(IMAP_SERVICE_FAILED_TO_START, str(port))
+ emit(catalog.IMAP_SERVICE_FAILED_TO_START, str(port))
diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py
deleted file mode 100644
index f3de8eb..0000000
--- a/src/leap/mail/imap/soledadstore.py
+++ /dev/null
@@ -1,620 +0,0 @@
-# -*- coding: utf-8 -*-
-# soledadstore.py
-# Copyright (C) 2014 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-A MessageStore that writes to Soledad.
-"""
-import logging
-import threading
-
-from collections import defaultdict
-from itertools import chain
-
-from u1db import errors as u1db_errors
-from twisted.python import log
-from zope.interface import implements
-
-from leap.common.check import leap_assert_type, leap_assert
-from leap.mail.decorators import deferred_to_thread
-from leap.mail.imap.messageparts import MessagePartType
-from leap.mail.imap.messageparts import MessageWrapper
-from leap.mail.imap.messageparts import RecentFlagsDoc
-from leap.mail.imap.fields import fields
-from leap.mail.imap.interfaces import IMessageStore
-from leap.mail.messageflow import IMessageConsumer
-from leap.mail.utils import first, empty, accumulator_queue
-
-logger = logging.getLogger(__name__)
-
-
-# TODO
-# [ ] Implement a retry queue?
-# [ ] Consider journaling of operations.
-
-
-class ContentDedup(object):
- """
- Message deduplication.
-
- We do a query for the content hashes before writing to our beloved
- sqlcipher backend of Soledad. This means, by now, that:
-
- 1. We will not store the same body/attachment twice, only the hash of it.
- 2. We will not store the same message header twice, only the hash of it.
-
- The first case is useful if you are always receiving the same old memes
- from unwary friends that still have not discovered that 4chan is the
- generator of the internet. The second will save your day if you have
- initiated session with the same account in two different machines. I also
- wonder why would you do that, but let's respect each other choices, like
- with the religious celebrations, and assume that one day we'll be able
- to run Bitmask in completely free phones. Yes, I mean that, the whole GSM
- Stack.
- """
- # TODO refactor using unique_query
-
- def _header_does_exist(self, doc):
- """
- Check whether we already have a header document for this
- content hash in our database.
-
- :param doc: tentative header for document
- :type doc: dict
- :returns: True if it exists, False otherwise.
- """
- if not doc:
- return False
- chash = doc[fields.CONTENT_HASH_KEY]
- header_docs = self._soledad.get_from_index(
- fields.TYPE_C_HASH_IDX,
- fields.TYPE_HEADERS_VAL, str(chash))
- if not header_docs:
- return False
-
- # FIXME enable only to debug this problem.
- #if len(header_docs) != 1:
- #logger.warning("Found more than one copy of chash %s!"
- #% (chash,))
-
- #logger.debug("Found header doc with that hash! Skipping save!")
- return True
-
- def _content_does_exist(self, doc):
- """
- Check whether we already have a content document for a payload
- with this hash in our database.
-
- :param doc: tentative content for document
- :type doc: dict
- :returns: True if it exists, False otherwise.
- """
- if not doc:
- return False
- phash = doc[fields.PAYLOAD_HASH_KEY]
- attach_docs = self._soledad.get_from_index(
- fields.TYPE_P_HASH_IDX,
- fields.TYPE_CONTENT_VAL, str(phash))
- if not attach_docs:
- return False
-
- # FIXME enable only to debug this problem
- #if len(attach_docs) != 1:
- #logger.warning("Found more than one copy of phash %s!"
- #% (phash,))
- #logger.debug("Found attachment doc with that hash! Skipping save!")
- return True
-
-
-class MsgWriteError(Exception):
- """
- Raised if any exception is found while saving message parts.
- """
- pass
-
-
-"""
-A lock per document.
-"""
-# TODO should bound the space of this!!!
-# http://stackoverflow.com/a/2437645/1157664
-# Setting this to twice the number of threads in the threadpool
-# should be safe.
-put_locks = defaultdict(lambda: threading.Lock())
-mbox_doc_locks = defaultdict(lambda: threading.Lock())
-
-
-class SoledadStore(ContentDedup):
- """
- This will create docs in the local Soledad database.
- """
- _remove_lock = threading.Lock()
-
- implements(IMessageConsumer, IMessageStore)
-
- def __init__(self, soledad):
- """
- Initialize the permanent store that writes to Soledad database.
-
- :param soledad: the soledad instance
- :type soledad: Soledad
- """
- from twisted.internet import reactor
- self.reactor = reactor
-
- self._soledad = soledad
-
- self._CREATE_DOC_FUN = self._soledad.create_doc
- self._PUT_DOC_FUN = self._soledad.put_doc
- self._GET_DOC_FUN = self._soledad.get_doc
-
- # we instantiate an accumulator to batch the notifications
- self.docs_notify_queue = accumulator_queue(
- lambda item: reactor.callFromThread(self._unset_new_dirty, item),
- 20)
-
- # IMessageStore
-
- # -------------------------------------------------------------------
- # We are not yet using this interface, but it would make sense
- # to implement it.
-
- def create_message(self, mbox, uid, message):
- """
- Create the passed message into this SoledadStore.
-
- :param mbox: the mbox this message belongs.
- :type mbox: str or unicode
- :param uid: the UID that identifies this message in this mailbox.
- :type uid: int
- :param message: a IMessageContainer implementor.
- """
- raise NotImplementedError()
-
- def put_message(self, mbox, uid, message):
- """
- Put the passed existing message into this SoledadStore.
-
- :param mbox: the mbox this message belongs.
- :type mbox: str or unicode
- :param uid: the UID that identifies this message in this mailbox.
- :type uid: int
- :param message: a IMessageContainer implementor.
- """
- raise NotImplementedError()
-
- def remove_message(self, mbox, uid):
- """
- Remove the given message from this SoledadStore.
-
- :param mbox: the mbox this message belongs.
- :type mbox: str or unicode
- :param uid: the UID that identifies this message in this mailbox.
- :type uid: int
- """
- raise NotImplementedError()
-
- def get_message(self, mbox, uid):
- """
- Get a IMessageContainer for the given mbox and uid combination.
-
- :param mbox: the mbox this message belongs.
- :type mbox: str or unicode
- :param uid: the UID that identifies this message in this mailbox.
- :type uid: int
- """
- raise NotImplementedError()
-
- # IMessageConsumer
-
- # TODO should handle the delete case
- # TODO should handle errors better
- # TODO could generalize this method into a generic consumer
- # and only implement `process` here
-
- def consume(self, queue):
- """
- Creates a new document in soledad db.
-
- :param queue: a tuple of queues to get item from, with content of the
- document to be inserted.
- :type queue: tuple of Queues
- """
- new, dirty = queue
- while not new.empty():
- doc_wrapper = new.get()
- self.reactor.callInThread(self._consume_doc, doc_wrapper,
- self.docs_notify_queue)
- while not dirty.empty():
- doc_wrapper = dirty.get()
- self.reactor.callInThread(self._consume_doc, doc_wrapper,
- self.docs_notify_queue)
-
- # Queue empty, flush the notifications queue.
- self.docs_notify_queue(None, flush=True)
-
- def _unset_new_dirty(self, doc_wrapper):
- """
- Unset the `new` and `dirty` flags for this document wrapper in the
- memory store.
-
- :param doc_wrapper: a MessageWrapper instance
- :type doc_wrapper: MessageWrapper
- """
- if isinstance(doc_wrapper, MessageWrapper):
- # XXX still needed for debug quite often
- #logger.info("unsetting new flag!")
- doc_wrapper.new = False
- doc_wrapper.dirty = False
-
- @deferred_to_thread
- def _consume_doc(self, doc_wrapper, notify_queue):
- """
- Consume each document wrapper in a separate thread.
- We pass an instance of an accumulator that handles the notifications
- to the memorystore when the write has been done.
-
- :param doc_wrapper: a MessageWrapper or RecentFlagsDoc instance
- :type doc_wrapper: MessageWrapper or RecentFlagsDoc
- :param notify_queue: a callable that handles the writeback
- notifications to the memstore.
- :type notify_queue: callable
- """
- def queueNotifyBack(failed, doc_wrapper):
- if failed:
- log.msg("There was an error writing the mesage...")
- else:
- notify_queue(doc_wrapper)
-
- def doSoledadCalls(items):
- # we prime the generator, that should return the
- # message or flags wrapper item in the first place.
- try:
- doc_wrapper = items.next()
- except StopIteration:
- pass
- else:
- failed = self._soledad_write_document_parts(items)
- queueNotifyBack(failed, doc_wrapper)
-
- doSoledadCalls(self._iter_wrapper_subparts(doc_wrapper))
-
- #
- # SoledadStore specific methods.
- #
-
- def _soledad_write_document_parts(self, items):
- """
- Write the document parts to soledad in a separate thread.
-
- :param items: the iterator through the different document wrappers
- payloads.
- :type items: iterator
- :return: whether the write was successful or not
- :rtype: bool
- """
- failed = False
- for item, call in items:
- if empty(item):
- continue
- try:
- self._try_call(call, item)
- except Exception as exc:
- logger.debug("ITEM WAS: %s" % repr(item))
- if hasattr(item, 'content'):
- logger.debug("ITEM CONTENT WAS: %s" %
- repr(item.content))
- logger.exception(exc)
- failed = True
- continue
- return failed
-
- def _iter_wrapper_subparts(self, doc_wrapper):
- """
- Return an iterator that will yield the doc_wrapper in the first place,
- followed by the subparts item and the proper call type for every
- item in the queue, if any.
-
- :param doc_wrapper: a MessageWrapper or RecentFlagsDoc instance
- :type doc_wrapper: MessageWrapper or RecentFlagsDoc
- """
- if isinstance(doc_wrapper, MessageWrapper):
- return chain((doc_wrapper,),
- self._get_calls_for_msg_parts(doc_wrapper))
- elif isinstance(doc_wrapper, RecentFlagsDoc):
- return chain((doc_wrapper,),
- self._get_calls_for_rflags_doc(doc_wrapper))
- else:
- logger.warning("CANNOT PROCESS ITEM!")
- return (i for i in [])
-
- def _try_call(self, call, item):
- """
- Try to invoke a given call with item as a parameter.
-
- :param call: the function to call
- :type call: callable
- :param item: the payload to pass to the call as argument
- :type item: object
- """
- if call is None:
- return
-
- if call == self._PUT_DOC_FUN:
- doc_id = item.doc_id
- if doc_id is None:
- logger.warning("BUG! Dirty doc but has no doc_id!")
- return
- with put_locks[doc_id]:
- doc = self._GET_DOC_FUN(doc_id)
-
- if doc is None:
- logger.warning("BUG! Dirty doc but could not "
- "find document %s" % (doc_id,))
- return
-
- doc.content = dict(item.content)
-
- item = doc
- try:
- call(item)
- except u1db_errors.RevisionConflict as exc:
- logger.exception("Error: %r" % (exc,))
- raise exc
- except Exception as exc:
- logger.exception("Error: %r" % (exc,))
- raise exc
-
- else:
- try:
- call(item)
- except u1db_errors.RevisionConflict as exc:
- logger.exception("Error: %r" % (exc,))
- raise exc
- except Exception as exc:
- logger.exception("Error: %r" % (exc,))
- raise exc
-
- def _get_calls_for_msg_parts(self, msg_wrapper):
- """
- Generator that return the proper call type for a given item.
-
- :param msg_wrapper: A MessageWrapper
- :type msg_wrapper: IMessageContainer
- :return: a generator of tuples with recent-flags doc payload
- and callable
- :rtype: generator
- """
- call = None
-
- if msg_wrapper.new:
- call = self._CREATE_DOC_FUN
-
- # item is expected to be a MessagePartDoc
- for item in msg_wrapper.walk():
- if item.part == MessagePartType.fdoc:
- yield dict(item.content), call
-
- elif item.part == MessagePartType.hdoc:
- if not self._header_does_exist(item.content):
- yield dict(item.content), call
-
- elif item.part == MessagePartType.cdoc:
- if not self._content_does_exist(item.content):
- yield dict(item.content), call
-
- # For now, the only thing that will be dirty is
- # the flags doc.
-
- elif msg_wrapper.dirty:
- call = self._PUT_DOC_FUN
- # item is expected to be a MessagePartDoc
- for item in msg_wrapper.walk():
- # XXX FIXME Give error if dirty and not doc_id !!!
- doc_id = item.doc_id # defend!
- if not doc_id:
- logger.warning("Dirty item but no doc_id!")
- continue
-
- if item.part == MessagePartType.fdoc:
- #logger.debug("PUT dirty fdoc")
- yield item, call
-
- # XXX also for linkage-doc !!!
- else:
- logger.error("Cannot delete documents yet from the queue...!")
-
- def _get_calls_for_rflags_doc(self, rflags_wrapper):
- """
- We always put these documents.
-
- :param rflags_wrapper: A wrapper around recent flags doc.
- :type rflags_wrapper: RecentFlagsWrapper
- :return: a tuple with recent-flags doc payload and callable
- :rtype: tuple
- """
- call = self._PUT_DOC_FUN
-
- payload = rflags_wrapper.content
- if payload:
- logger.debug("Saving RFLAGS to Soledad...")
- yield rflags_wrapper, call
-
- # Mbox documents and attributes
-
- def get_mbox_document(self, mbox):
- """
- Return mailbox document.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :return: A SoledadDocument containing this mailbox, or None if
- the query failed.
- :rtype: SoledadDocument or None.
- """
- with mbox_doc_locks[mbox]:
- return self._get_mbox_document(mbox)
-
- def _get_mbox_document(self, mbox):
- """
- Helper for returning the mailbox document.
- """
- try:
- query = self._soledad.get_from_index(
- fields.TYPE_MBOX_IDX,
- fields.TYPE_MBOX_VAL, mbox)
- if query:
- return query.pop()
- else:
- logger.error("Could not find mbox document for %r" %
- (mbox,))
- except Exception as exc:
- logger.exception("Unhandled error %r" % exc)
-
- def get_mbox_closed(self, mbox):
- """
- Return the closed attribute for a given mailbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :rtype: bool
- """
- mbox_doc = self.get_mbox_document()
- return mbox_doc.content.get(fields.CLOSED_KEY, False)
-
- def set_mbox_closed(self, mbox, closed):
- """
- Set the closed attribute for a given mailbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param closed: the value to be set
- :type closed: bool
- """
- leap_assert(isinstance(closed, bool), "closed needs to be boolean")
- with mbox_doc_locks[mbox]:
- mbox_doc = self._get_mbox_document(mbox)
- if mbox_doc is None:
- logger.error(
- "Could not find mbox document for %r" % (mbox,))
- return
- mbox_doc.content[fields.CLOSED_KEY] = closed
- self._soledad.put_doc(mbox_doc)
-
- def write_last_uid(self, mbox, value):
- """
- Write the `last_uid` integer to the proper mailbox document
- in Soledad.
- This is called from the deferred triggered by
- memorystore.increment_last_soledad_uid, which is expected to
- run in a separate thread.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param value: the value to set
- :type value: int
- """
- leap_assert_type(value, int)
- key = fields.LAST_UID_KEY
-
- # XXX use accumulator to reduce number of hits
- with mbox_doc_locks[mbox]:
- mbox_doc = self._get_mbox_document(mbox)
- old_val = mbox_doc.content[key]
- if value > old_val:
- mbox_doc.content[key] = value
- try:
- self._soledad.put_doc(mbox_doc)
- except Exception as exc:
- logger.error("Error while setting last_uid for %r"
- % (mbox,))
- logger.exception(exc)
-
- def get_flags_doc(self, mbox, uid):
- """
- Return the SoledadDocument for the given mbox and uid.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :param uid: the UID for the message
- :type uid: int
- :rtype: SoledadDocument or None
- """
- result = None
- try:
- flag_docs = self._soledad.get_from_index(
- fields.TYPE_MBOX_UID_IDX,
- fields.TYPE_FLAGS_VAL, mbox, str(uid))
- if len(flag_docs) != 1:
- logger.warning("More than one flag doc for %r:%s" %
- (mbox, uid))
- result = first(flag_docs)
- except Exception as exc:
- # ugh! Something's broken down there!
- logger.warning("ERROR while getting flags for UID: %s" % uid)
- logger.exception(exc)
- finally:
- return result
-
- def get_headers_doc(self, chash):
- """
- Return the document that keeps the headers for a message
- indexed by its content-hash.
-
- :param chash: the content-hash to retrieve the document from.
- :type chash: str or unicode
- :rtype: SoledadDocument or None
- """
- head_docs = self._soledad.get_from_index(
- fields.TYPE_C_HASH_IDX,
- fields.TYPE_HEADERS_VAL, str(chash))
- return first(head_docs)
-
- # deleted messages
-
- def deleted_iter(self, mbox):
- """
- Get an iterator for the the doc_id for SoledadDocuments for messages
- with \\Deleted flag for a given mailbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- :return: iterator through deleted message docs
- :rtype: iterable
- """
- return [doc.doc_id for doc in self._soledad.get_from_index(
- fields.TYPE_MBOX_DEL_IDX,
- fields.TYPE_FLAGS_VAL, mbox, '1')]
-
- def remove_all_deleted(self, mbox):
- """
- Remove from Soledad all messages flagged as deleted for a given
- mailbox.
-
- :param mbox: the mailbox
- :type mbox: str or unicode
- """
- deleted = []
- for doc_id in self.deleted_iter(mbox):
- with self._remove_lock:
- doc = self._soledad.get_doc(doc_id)
- if doc is not None:
- self._soledad.delete_doc(doc)
- try:
- deleted.append(doc.content[fields.UID_KEY])
- except TypeError:
- # empty content
- pass
- return deleted
diff --git a/src/leap/mail/imap/tests/__init__.py b/src/leap/mail/imap/tests/__init__.py
index f3d5ca6..5cf60ed 100644
--- a/src/leap/mail/imap/tests/__init__.py
+++ b/src/leap/mail/imap/tests/__init__.py
@@ -1,4 +1,4 @@
-#-*- encoding: utf-8 -*-
+# -*- encoding: utf-8 -*-
"""
leap/email/imap/tests/__init__.py
----------------------------------
@@ -10,13 +10,6 @@ code, using twisted.trial, for testing leap_mx.
@copyright: © 2013 Kali Kaneko, see COPYLEFT file
"""
-__all__ = ['test_imap']
-
-
-def run():
- """xxx fill me in"""
- pass
-
import os
import u1db
@@ -25,11 +18,18 @@ from leap.common.testing.basetest import BaseLeapTest
from leap.soledad.client import Soledad
from leap.soledad.common.document import SoledadDocument
+__all__ = ['test_imap']
+
-#-----------------------------------------------------------------------------
+def run():
+ """xxx fill me in"""
+ pass
+
+# -----------------------------------------------------------------------------
# Some tests inherit from BaseSoledadTest in order to have a working Soledad
# instance in each test.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+
class BaseSoledadIMAPTest(BaseLeapTest):
"""
diff --git a/src/leap/mail/imap/tests/regressions b/src/leap/mail/imap/tests/regressions_mime_struct
index efe3f46..0332664 100755
--- a/src/leap/mail/imap/tests/regressions
+++ b/src/leap/mail/imap/tests/regressions_mime_struct
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-# regressions
+# regression_mime_struct
# Copyright (C) 2014 LEAP
# Copyright (c) Twisted Matrix Laboratories.
#
@@ -18,7 +18,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
-Simple Regression Tests using IMAP4 client.
+Simple Regression Tests for checking MIME struct handling using IMAP4 client.
Iterates trough all mails under a given folder and tries to APPEND them to
the server being tested. After FETCHING the pushed message, it compares
@@ -40,7 +40,9 @@ from twisted.protocols import basic
from twisted.python import log
-REGRESSIONS_FOLDER = "regressions_test"
+REGRESSIONS_FOLDER = os.environ.get(
+ "REGRESSIONS_FOLDER", "regressions_test")
+print "[+] Using regressions folder:", REGRESSIONS_FOLDER
parser = Parser()
@@ -263,7 +265,7 @@ def cbSelectMbox(result, proto):
if result["EXISTS"] != 0:
# Flag as deleted, expunge, and do an examine again.
- #print "There is mail here, will delete..."
+ print "There is mail here, will delete..."
return cbDeleteAndExpungeTestFolder(proto)
else:
@@ -276,11 +278,15 @@ def ebSelectMbox(failure, proto, folder):
Creates the folder.
"""
- print failure.getTraceback()
+ log.err(failure)
log.msg("Folder %r does not exist. Creating..." % (folder,))
return proto.create(folder).addCallback(cbAuthentication, proto)
+def ebExpunge(failure):
+ log.err(failure)
+
+
def cbDeleteAndExpungeTestFolder(proto):
"""
Callback invoked fom cbExamineMbox when the number of messages in the
@@ -292,7 +298,9 @@ def cbDeleteAndExpungeTestFolder(proto):
).addCallback(
lambda r: proto.expunge()
).addCallback(
- cbExpunge, proto)
+ cbExpunge, proto
+ ).addErrback(
+ ebExpunge)
def cbExpunge(result, proto):
diff --git a/src/leap/mail/imap/tests/rfc822.message b/src/leap/mail/imap/tests/rfc822.message
index ee97ab9..b19cc28 100644..120000
--- a/src/leap/mail/imap/tests/rfc822.message
+++ b/src/leap/mail/imap/tests/rfc822.message
@@ -1,86 +1 @@
-Return-Path: <twisted-commits-admin@twistedmatrix.com>
-Delivered-To: exarkun@meson.dyndns.org
-Received: from localhost [127.0.0.1]
- by localhost with POP3 (fetchmail-6.2.1)
- for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST)
-Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105])
- by intarweb.us (Postfix) with ESMTP id 4A4A513EA4
- for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST)
-Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com)
- by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian))
- id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600
-Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian))
- id 18w63j-0007VK-00
- for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600
-To: twisted-commits@twistedmatrix.com
-From: etrepum CVS <etrepum@twistedmatrix.com>
-Reply-To: twisted-python@twistedmatrix.com
-X-Mailer: CVSToys
-Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com>
-Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up.
-Sender: twisted-commits-admin@twistedmatrix.com
-Errors-To: twisted-commits-admin@twistedmatrix.com
-X-BeenThere: twisted-commits@twistedmatrix.com
-X-Mailman-Version: 2.0.11
-Precedence: bulk
-List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help>
-List-Post: <mailto:twisted-commits@twistedmatrix.com>
-List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
- <mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe>
-List-Id: <twisted-commits.twistedmatrix.com>
-List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
- <mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe>
-List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/>
-Date: Thu, 20 Mar 2003 13:50:39 -0600
-
-Modified files:
-Twisted/twisted/python/rebuild.py 1.19 1.20
-
-Log message:
-rebuild now works on python versions from 2.2.0 and up.
-
-
-ViewCVS links:
-http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted
-
-Index: Twisted/twisted/python/rebuild.py
-diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20
---- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003
-+++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003
-@@ -206,15 +206,27 @@
- clazz.__dict__.clear()
- clazz.__getattr__ = __getattr__
- clazz.__module__ = module.__name__
-+ if newclasses:
-+ import gc
-+ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2):
-+ hasBrokenRebuild = 1
-+ gc_objects = gc.get_objects()
-+ else:
-+ hasBrokenRebuild = 0
- for nclass in newclasses:
- ga = getattr(module, nclass.__name__)
- if ga is nclass:
- log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass))
- else:
-- import gc
-- for r in gc.get_referrers(nclass):
-- if isinstance(r, nclass):
-+ if hasBrokenRebuild:
-+ for r in gc_objects:
-+ if not getattr(r, '__class__', None) is nclass:
-+ continue
- r.__class__ = ga
-+ else:
-+ for r in gc.get_referrers(nclass):
-+ if getattr(r, '__class__', None) is nclass:
-+ r.__class__ = ga
- if doLog:
- log.msg('')
- log.msg(' (fixing %s): ' % str(module.__name__))
-
-
-_______________________________________________
-Twisted-commits mailing list
-Twisted-commits@twistedmatrix.com
-http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits
+../../tests/rfc822.message \ No newline at end of file
diff --git a/src/leap/mail/imap/tests/rfc822.multi-minimal.message b/src/leap/mail/imap/tests/rfc822.multi-minimal.message
index 582297c..e0aa678 100644..120000
--- a/src/leap/mail/imap/tests/rfc822.multi-minimal.message
+++ b/src/leap/mail/imap/tests/rfc822.multi-minimal.message
@@ -1,16 +1 @@
-Content-Type: multipart/mixed; boundary="===============6203542367371144092=="
-MIME-Version: 1.0
-Subject: [TEST] 010 - Inceptos cum lorem risus congue
-From: testmailbitmaskspam@gmail.com
-To: test_c5@dev.bitmask.net
-
---===============6203542367371144092==
-Content-Type: text/plain; charset="us-ascii"
-MIME-Version: 1.0
-Content-Transfer-Encoding: 7bit
-
-Howdy from python!
-The subject: [TEST] 010 - Inceptos cum lorem risus congue
-Current date & time: Wed Jan 8 16:36:21 2014
-Trying to attach: []
---===============6203542367371144092==--
+../../tests/rfc822.multi-minimal.message \ No newline at end of file
diff --git a/src/leap/mail/imap/tests/rfc822.multi-nested.message b/src/leap/mail/imap/tests/rfc822.multi-nested.message
new file mode 120000
index 0000000..306d0de
--- /dev/null
+++ b/src/leap/mail/imap/tests/rfc822.multi-nested.message
@@ -0,0 +1 @@
+../../tests/rfc822.multi-nested.message \ No newline at end of file
diff --git a/src/leap/mail/imap/tests/rfc822.multi-signed.message b/src/leap/mail/imap/tests/rfc822.multi-signed.message
index 9907c2d..4172244 100644..120000
--- a/src/leap/mail/imap/tests/rfc822.multi-signed.message
+++ b/src/leap/mail/imap/tests/rfc822.multi-signed.message
@@ -1,238 +1 @@
-Date: Mon, 6 Jan 2014 04:40:47 -0400
-From: Kali Kaneko <kali@leap.se>
-To: penguin@example.com
-Subject: signed message
-Message-ID: <20140106084047.GA21317@samsara.lan>
-MIME-Version: 1.0
-Content-Type: multipart/signed; micalg=pgp-sha1;
- protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy"
-Content-Disposition: inline
-User-Agent: Mutt/1.5.21 (2012-12-30)
-
-
---z9ECzHErBrwFF8sy
-Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l"
-Content-Disposition: inline
-
-
---z0eOaCaDLjvTGF2l
-Content-Type: text/plain; charset=utf-8
-Content-Disposition: inline
-Content-Transfer-Encoding: quoted-printable
-
-This is an example of a signed message,
-with attachments.
-
-
---=20
-Nihil sine chao! =E2=88=B4
-
---z0eOaCaDLjvTGF2l
-Content-Type: text/plain; charset=us-ascii
-Content-Disposition: attachment; filename="attach.txt"
-
-this is attachment in plain text.
-
---z0eOaCaDLjvTGF2l
-Content-Type: application/octet-stream
-Content-Disposition: attachment; filename="hack.ico"
-Content-Transfer-Encoding: base64
-
-AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA
-KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG
-RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA
-PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl
-5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA
-/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ
-yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A
-Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK
-ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK
-LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP
-QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy
-AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs
-AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA
-AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA
-gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d
-HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA
-x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7
-+wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA
-AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5
-+QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA
-OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK
-igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA
-JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra
-2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA
-xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj
-owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB
-AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA
-AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d
-XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d
-XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA
-AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB
-AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm
-X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC
-AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B
-bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ
-S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu
-J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y
-AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N
-KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB
-XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A
-AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA
-AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d
-XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA
-AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr
-RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA
-AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A
-Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI
-yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA
-CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys
-rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA
-vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d
-HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA
-urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx
-cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA
-CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo
-6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA
-2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7
-OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA
-UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp
-qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA
-lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa
-WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB
-AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB
-AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB
-AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA
-ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA
-AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB
-AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB
-AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB
-AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB
-AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA
-tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA
-AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF
-wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB
-AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2
-RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB
-AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB
-AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA
-AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd
-AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB
-AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB
-AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB
-AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB
-AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB
-AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8
-ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2
-NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF
-RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB
-lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA
-AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa
-WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA
-AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX
-AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB
-AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB
-AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA
-AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA
-AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA
-AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA
-AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB
-AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB
-AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA
-ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA
-AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7
-LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA
-AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF
-NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB
-AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2
-RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA
-ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4
-RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi
-JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2
-NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK
-T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB
-AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB
-AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB
-AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN
-UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA
-AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA
-W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA
-AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB
-l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB
-AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+
-WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA
-AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv
-RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA
-AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj
-AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB
-AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB
-AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA
-AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA
-AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA
-dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A
-AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB
-AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB
-AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB
-AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW
-pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
-
---z0eOaCaDLjvTGF2l--
-
---z9ECzHErBrwFF8sy
-Content-Type: application/pgp-signature
-
------BEGIN PGP SIGNATURE-----
-Version: GnuPG v1.4.15 (GNU/Linux)
-
-iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv
-kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl
-vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK
-PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC
-w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw
-sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr
-BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN
-QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt
-mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ
-jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8
-gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X
-sSdfcAhT7Tno7PB/Acoh
-=+okv
------END PGP SIGNATURE-----
-
---z9ECzHErBrwFF8sy--
+../../tests/rfc822.multi-signed.message \ No newline at end of file
diff --git a/src/leap/mail/imap/tests/rfc822.multi.message b/src/leap/mail/imap/tests/rfc822.multi.message
index 30f74e5..62057d2 100644..120000
--- a/src/leap/mail/imap/tests/rfc822.multi.message
+++ b/src/leap/mail/imap/tests/rfc822.multi.message
@@ -1,96 +1 @@
-Date: Fri, 19 May 2000 09:55:48 -0400 (EDT)
-From: Doug Sauder <doug@penguin.example.com>
-To: Joe Blow <blow@example.com>
-Subject: Test message from PINE
-Message-ID: <Pine.LNX.4.21.0005190951410.8452-102000@penguin.example.com>
-MIME-Version: 1.0
-Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452"
-
- This message is in MIME format. The first part should be readable text,
- while the remaining parts are likely unreadable without MIME-aware tools.
- Send mail to mime@docserver.cac.washington.edu for more info.
-
----1463757054-952513540-958744548=:8452
-Content-Type: TEXT/PLAIN; charset=US-ASCII
-
-This is a test message from PINE MUA.
-
-
----1463757054-952513540-958744548=:8452
-Content-Type: APPLICATION/octet-stream; name="redball.png"
-Content-Transfer-Encoding: BASE64
-Content-ID: <Pine.LNX.4.21.0005190955480.8452@penguin.example.com>
-Content-Description: A PNG graphic file
-Content-Disposition: attachment; filename="redball.png"
-
-iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A
-AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj
-AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv
-AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0
-GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf
-hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl
-rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL
-ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS
-AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP
-AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP
-AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI
-AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR
-AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6
-AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO
-AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8
-AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8
-LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu
-MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF
-BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp
-6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow
-8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G
-ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn
-OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1
-a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T
-VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8
-Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f
-lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/
-joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms
-1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA
-JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7
-vAAAAABJRU5ErkJggg==
----1463757054-952513540-958744548=:8452
-Content-Type: APPLICATION/octet-stream; name="blueball.png"
-Content-Transfer-Encoding: BASE64
-Content-ID: <Pine.LNX.4.21.0005190955481.8452@penguin.example.com>
-Content-Description: A PNG graphic file
-Content-Disposition: attachment; filename="blueball.png"
-
-iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A
-AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI
-IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y
-Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S
-hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM
-vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB
-Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu
-MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b
-fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo
-Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp
-LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX
-P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M
-jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8
-+VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE
-1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42
-YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY
-mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp
-Z3VldDZzO7wAAAAASUVORK5CYII=
----1463757054-952513540-958744548=:8452--
+../../tests/rfc822.multi.message \ No newline at end of file
diff --git a/src/leap/mail/imap/tests/rfc822.plain.message b/src/leap/mail/imap/tests/rfc822.plain.message
index fc627c3..5bab0e8 100644..120000
--- a/src/leap/mail/imap/tests/rfc822.plain.message
+++ b/src/leap/mail/imap/tests/rfc822.plain.message
@@ -1,66 +1 @@
-From pyar-bounces@python.org.ar Wed Jan 8 14:46:02 2014
-Return-Path: <pyar-bounces@python.org.ar>
-X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net
-X-Spam-Level: **
-X-Spam-Pyzor: Reported 0 times.
-X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE,
- CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP,
- NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled
- version=3.3.2
-Delivered-To: kali@leap.se
-Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33])
- (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits))
- (Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified))
- by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F
- for <kali@leap.se>; Wed, 8 Jan 2014 18:46:02 +0000 (UTC)
-Received: from pyar.usla.org.ar (unknown [190.228.30.157])
- by mx1.riseup.net (Postfix) with ESMTP id F244C533F4
- for <kali@leap.se>; Wed, 8 Jan 2014 10:46:01 -0800 (PST)
-Received: from [127.0.0.1] (localhost [127.0.0.1])
- by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F
- for <kali@leap.se>; Wed, 8 Jan 2014 15:46:00 -0300 (ART)
-MIME-Version: 1.0
-Content-Type: text/plain; charset="iso-8859-1"
-Content-Transfer-Encoding: quoted-printable
-From: pyar-request@python.org.ar
-To: kali@leap.se
-Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2
-Reply-To: pyar-request@python.org.ar
-Auto-Submitted: auto-replied
-Message-ID: <mailman.245.1389206759.1579.pyar@python.org.ar>
-Date: Wed, 08 Jan 2014 15:45:59 -0300
-Precedence: bulk
-X-BeenThere: pyar@python.org.ar
-X-Mailman-Version: 2.1.15
-List-Id: Python Argentina <pyar.python.org.ar>
-X-List-Administrivia: yes
-Errors-To: pyar-bounces@python.org.ar
-Sender: "pyar" <pyar-bounces@python.org.ar>
-X-Virus-Scanned: clamav-milter 0.97.8 at mx1
-X-Virus-Status: Clean
-
-Mailing list subscription confirmation notice for mailing list pyar
-
-We have received a request de kaliyuga@riseup.net for subscription of
-your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar
-mailing list. To confirm that you want to be added to this mailing
-list, simply reply to this message, keeping the Subject: header
-intact. Or visit this web page:
-
- http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b=
-3377148ac2
-
-
-Or include the following line -- and only the following line -- in a
-message to pyar-request@python.org.ar:
-
- confirm 0e47e4342e4d42508e8c283175b05b3377148ac2
-
-Note that simply sending a `reply' to this message should work from
-most mail readers, since that usually leaves the Subject: line in the
-right form (additional "Re:" text in the Subject: is okay).
-
-If you do not wish to be subscribed to this list, please simply
-disregard this message. If you think you are being maliciously
-subscribed to the list, or have any other questions, send them to
-pyar-owner@python.org.ar.
+../../tests/rfc822.plain.message \ No newline at end of file
diff --git a/src/leap/mail/imap/tests/leap_tests_imap.zsh b/src/leap/mail/imap/tests/stress_tests_imap.zsh
index 544faca..544faca 100755
--- a/src/leap/mail/imap/tests/leap_tests_imap.zsh
+++ b/src/leap/mail/imap/tests/stress_tests_imap.zsh
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index 631a2c1..62c3c41 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -17,7 +17,7 @@
"""
Test case for leap.email.imap.server
TestCases taken from twisted tests and modified to make them work
-against SoledadBackedAccount.
+against our implementation of the IMAPAccount.
@authors: Kali Kaneko, <kali@leap.se>
XXX add authors from the original twisted tests.
@@ -25,31 +25,20 @@ XXX add authors from the original twisted tests.
@license: GPLv3, see included LICENSE file
"""
# XXX review license of the original tests!!!
-
-try:
- from cStringIO import StringIO
-except ImportError:
- from StringIO import StringIO
-
import os
+import string
import types
from twisted.mail import imap4
from twisted.internet import defer
-from twisted.trial import unittest
from twisted.python import util
from twisted.python import failure
from twisted import cred
-
-# import u1db
-
-from leap.mail.imap.mailbox import SoledadMailbox
-from leap.mail.imap.memorystore import MemoryStore
-from leap.mail.imap.messages import MessageCollection
-from leap.mail.imap.server import LeapIMAPServer
+from leap.mail.imap.mailbox import IMAPMailbox
+from leap.mail.imap.messages import CaseInsensitiveDict
from leap.mail.imap.tests.utils import IMAP4HelperMixin
@@ -73,7 +62,6 @@ def sortNest(l):
class TestRealm:
-
"""
A minimal auth realm for testing purposes only
"""
@@ -82,153 +70,21 @@ class TestRealm:
def requestAvatar(self, avatarId, mind, *interfaces):
return imap4.IAccount, self.theAccount, lambda: None
-
#
# TestCases
#
-class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
-
- """
- Tests for the MessageCollection class
- """
- count = 0
-
- def setUp(self):
- """
- setUp method for each test
- We override mixin method since we are only testing
- MessageCollection interface in this particular TestCase
- """
- super(MessageCollectionTestCase, self).setUp()
- memstore = MemoryStore()
- self.messages = MessageCollection("testmbox%s" % (self.count,),
- self._soledad, memstore=memstore)
- MessageCollectionTestCase.count += 1
-
- def tearDown(self):
- """
- tearDown method for each test
- """
- del self.messages
-
- def testEmptyMessage(self):
- """
- Test empty message and collection
- """
- em = self.messages._get_empty_doc()
- self.assertEqual(
- em,
- {
- "chash": '',
- "deleted": False,
- "flags": [],
- "mbox": "inbox",
- "seen": False,
- "multi": False,
- "size": 0,
- "type": "flags",
- "uid": 1,
- })
- self.assertEqual(self.messages.count(), 0)
-
- def testMultipleAdd(self):
- """
- Add multiple messages
- """
- mc = self.messages
- self.assertEqual(self.messages.count(), 0)
-
- def add_first():
- d = defer.gatherResults([
- mc.add_msg('Stuff 1', subject="test1"),
- mc.add_msg('Stuff 2', subject="test2"),
- mc.add_msg('Stuff 3', subject="test3"),
- mc.add_msg('Stuff 4', subject="test4")])
- return d
-
- def add_second(result):
- d = defer.gatherResults([
- mc.add_msg('Stuff 5', subject="test5"),
- mc.add_msg('Stuff 6', subject="test6"),
- mc.add_msg('Stuff 7', subject="test7")])
- return d
-
- def check_second(result):
- return self.assertEqual(mc.count(), 7)
-
- d1 = add_first()
- d1.addCallback(add_second)
- d1.addCallback(check_second)
-
- def testRecentCount(self):
- """
- Test the recent count
- """
- mc = self.messages
- countrecent = mc.count_recent
- eq = self.assertEqual
-
- self.assertEqual(countrecent(), 0)
-
- d = mc.add_msg('Stuff', subject="test1")
- # For the semantics defined in the RFC, we auto-add the
- # recent flag by default.
-
- def add2(_):
- return mc.add_msg('Stuff', subject="test2",
- flags=('\\Deleted',))
-
- def add3(_):
- return mc.add_msg('Stuff', subject="test3",
- flags=('\\Recent',))
-
- def add4(_):
- return mc.add_msg('Stuff', subject="test4",
- flags=('\\Deleted', '\\Recent'))
-
- d.addCallback(lambda r: eq(countrecent(), 1))
- d.addCallback(add2)
- d.addCallback(lambda r: eq(countrecent(), 2))
- d.addCallback(add3)
- d.addCallback(lambda r: eq(countrecent(), 3))
- d.addCallback(add4)
- d.addCallback(lambda r: eq(countrecent(), 4))
-
- def testFilterByMailbox(self):
- """
- Test that queries filter by selected mailbox
- """
- mc = self.messages
- self.assertEqual(self.messages.count(), 0)
-
- def add_1():
- d1 = mc.add_msg('msg 1', subject="test1")
- d2 = mc.add_msg('msg 2', subject="test2")
- d3 = mc.add_msg('msg 3', subject="test3")
- d = defer.gatherResults([d1, d2, d3])
- return d
-
- add_1().addCallback(lambda ignored: self.assertEqual(
- mc.count(), 3))
-
- # XXX this has to be redone to fit memstore ------------#
- #newmsg = mc._get_empty_doc()
- #newmsg['mailbox'] = "mailbox/foo"
- #mc._soledad.create_doc(newmsg)
- #self.assertEqual(mc.count(), 3)
- #self.assertEqual(
- #len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4)
+# DEBUG ---
+# from twisted.internet.base import DelayedCall
+# DelayedCall.debug = True
-class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
- # TODO this currently will use a memory-only store.
- # create a different one for testing soledad sync.
+class LEAPIMAP4ServerTestCase(IMAP4HelperMixin):
"""
- Tests for the generic behavior of the LeapIMAP4Server
+ Tests for the generic behavior of the LEAPIMAP4Server
which, right now, it's just implemented in this test file as
- LeapIMAPServer. We will move the implementation, together with
+ LEAPIMAPServer. We will move the implementation, together with
authentication bits, to leap.mail.imap.server so it can be instantiated
from the tac file.
@@ -248,6 +104,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
"""
succeed = ('testbox', 'test/box', 'test/', 'test/box/box', 'foobox')
fail = ('testbox', 'test/box')
+ acc = self.server.theAccount
def cb():
self.result.append(1)
@@ -259,46 +116,54 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
return self.client.login(TEST_USER, TEST_PASSWD)
def create():
+ create_deferreds = []
for name in succeed + fail:
d = self.client.create(name)
d.addCallback(strip(cb)).addErrback(eb)
- d.addCallbacks(self._cbStopClient, self._ebGeneral)
+ create_deferreds.append(d)
+ dd = defer.gatherResults(create_deferreds)
+ dd.addCallbacks(self._cbStopClient, self._ebGeneral)
+ return dd
self.result = []
- d1 = self.connected.addCallback(strip(login)).addCallback(
- strip(create))
+ d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(create))
d2 = self.loopback()
- d = defer.gatherResults([d1, d2])
+ d = defer.gatherResults([d1, d2], consumeErrors=True)
+ d.addCallback(lambda _: acc.account.list_all_mailbox_names())
return d.addCallback(self._cbTestCreate, succeed, fail)
- def _cbTestCreate(self, ignored, succeed, fail):
+ def _cbTestCreate(self, mailboxes, succeed, fail):
self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail))
- mboxes = list(LeapIMAPServer.theAccount.mailboxes)
- answers = ([u'INBOX', u'foobox', 'test', u'test/box',
- u'test/box/box', 'testbox'])
- self.assertEqual(mboxes, [a for a in answers])
+ answers = ([u'INBOX', u'testbox', u'test/box', u'test',
+ u'test/box/box', 'foobox'])
+ self.assertEqual(sorted(mailboxes), sorted([a for a in answers]))
def testDelete(self):
"""
Test whether we can delete mailboxes
"""
- LeapIMAPServer.theAccount.addMailbox('delete/me')
+ def add_mailbox():
+ return self.server.theAccount.addMailbox('test-delete/me')
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
def delete():
- return self.client.delete('delete/me')
+ return self.client.delete('test-delete/me')
- d1 = self.connected.addCallback(strip(login))
+ acc = self.server.theAccount.account
+
+ d1 = self.connected.addCallback(add_mailbox)
+ d1.addCallback(strip(login))
d1.addCallbacks(strip(delete), self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
d = defer.gatherResults([d1, d2])
- d.addCallback(
- lambda _: self.assertEqual(
- LeapIMAPServer.theAccount.mailboxes, ['INBOX']))
+ d.addCallback(lambda _: acc.list_all_mailbox_names())
+ d.addCallback(lambda mboxes: self.assertEqual(
+ mboxes, ['INBOX']))
return d
def testIllegalInboxDelete(self):
@@ -357,24 +222,34 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
Try deleting a mailbox with sub-folders, and \NoSelect flag set.
An exception is expected.
"""
- LeapIMAPServer.theAccount.addMailbox('delete')
- to_delete = LeapIMAPServer.theAccount.getMailbox('delete')
- to_delete.setFlags((r'\Noselect',))
- to_delete.getFlags()
- LeapIMAPServer.theAccount.addMailbox('delete/me')
+ acc = self.server.theAccount
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
- def delete():
+ def create_mailboxes():
+ d1 = acc.addMailbox('delete')
+ d2 = acc.addMailbox('delete/me')
+ d = defer.gatherResults([d1, d2])
+ return d
+
+ def get_noselect_mailbox(mboxes):
+ mbox = mboxes[0]
+ return mbox.setFlags((r'\Noselect',))
+
+ def delete_mbox(ignored):
return self.client.delete('delete')
def deleteFailed(failure):
self.failure = failure
self.failure = None
+
d1 = self.connected.addCallback(strip(login))
- d1.addCallback(strip(delete)).addErrback(deleteFailed)
+ d1.addCallback(strip(create_mailboxes))
+ d1.addCallback(get_noselect_mailbox)
+
+ d1.addCallback(delete_mbox).addErrback(deleteFailed)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
d = defer.gatherResults([d1, d2])
@@ -386,11 +261,15 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
self.assertEqual(str(self.failure.value), expected))
return d
+ # FIXME --- this test sometimes FAILS (timing issue).
+ # Some of the deferreds used in the rename op is not waiting for the
+ # operations properly
def testRename(self):
"""
Test whether we can rename a mailbox
"""
- LeapIMAPServer.theAccount.addMailbox('oldmbox')
+ def create_mbox():
+ return self.server.theAccount.addMailbox('oldmbox')
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
@@ -398,15 +277,16 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def rename():
return self.client.rename('oldmbox', 'newname')
- d1 = self.connected.addCallback(strip(login))
+ d1 = self.connected.addCallback(strip(create_mbox))
+ d1.addCallback(strip(login))
d1.addCallbacks(strip(rename), self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
d = defer.gatherResults([d1, d2])
d.addCallback(lambda _:
- self.assertEqual(
- LeapIMAPServer.theAccount.mailboxes,
- ['INBOX', 'newname']))
+ self.server.theAccount.account.list_all_mailbox_names())
+ d.addCallback(lambda mboxes:
+ self.assertItemsEqual(mboxes, ['INBOX', 'newname']))
return d
def testIllegalInboxRename(self):
@@ -440,8 +320,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
"""
Try to rename hierarchical mailboxes
"""
- LeapIMAPServer.theAccount.create('oldmbox/m1')
- LeapIMAPServer.theAccount.create('oldmbox/m2')
+ acc = self.server.theAccount
+
+ def add_mailboxes():
+ return defer.gatherResults([
+ acc.addMailbox('oldmbox/m1'),
+ acc.addMailbox('oldmbox/m2')])
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
@@ -449,45 +333,65 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def rename():
return self.client.rename('oldmbox', 'newname')
- d1 = self.connected.addCallback(strip(login))
+ d1 = self.connected.addCallback(strip(add_mailboxes))
+ d1.addCallback(strip(login))
d1.addCallbacks(strip(rename), self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
d = defer.gatherResults([d1, d2])
+ d.addCallback(lambda _: acc.account.list_all_mailbox_names())
return d.addCallback(self._cbTestHierarchicalRename)
- def _cbTestHierarchicalRename(self, ignored):
- mboxes = LeapIMAPServer.theAccount.mailboxes
- expected = ['INBOX', 'newname', 'newname/m1', 'newname/m2']
- self.assertEqual(mboxes, [s for s in expected])
+ def _cbTestHierarchicalRename(self, mailboxes):
+ expected = ['INBOX', 'newname/m1', 'newname/m2']
+ self.assertEqual(sorted(mailboxes), sorted([s for s in expected]))
def testSubscribe(self):
"""
Test whether we can mark a mailbox as subscribed to
"""
+ acc = self.server.theAccount
+
+ def add_mailbox():
+ return acc.addMailbox('this/mbox')
+
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
def subscribe():
return self.client.subscribe('this/mbox')
- d1 = self.connected.addCallback(strip(login))
+ def get_subscriptions(ignored):
+ return self.server.theAccount.getSubscriptions()
+
+ d1 = self.connected.addCallback(strip(add_mailbox))
+ d1.addCallback(strip(login))
d1.addCallbacks(strip(subscribe), self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
d = defer.gatherResults([d1, d2])
- d.addCallback(lambda _:
- self.assertEqual(
- LeapIMAPServer.theAccount.subscriptions,
- ['this/mbox']))
+ d.addCallback(get_subscriptions)
+ d.addCallback(lambda subscriptions:
+ self.assertEqual(subscriptions,
+ ['this/mbox']))
return d
def testUnsubscribe(self):
"""
Test whether we can unsubscribe from a set of mailboxes
"""
- LeapIMAPServer.theAccount.subscribe('this/mbox')
- LeapIMAPServer.theAccount.subscribe('that/mbox')
+ acc = self.server.theAccount
+
+ def add_mailboxes():
+ return defer.gatherResults([
+ acc.addMailbox('this/mbox'),
+ acc.addMailbox('that/mbox')])
+
+ def dc1():
+ return acc.subscribe('this/mbox')
+
+ def dc2():
+ return acc.subscribe('that/mbox')
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
@@ -495,24 +399,35 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def unsubscribe():
return self.client.unsubscribe('this/mbox')
- d1 = self.connected.addCallback(strip(login))
+ def get_subscriptions(ignored):
+ return acc.getSubscriptions()
+
+ d1 = self.connected.addCallback(strip(add_mailboxes))
+ d1.addCallback(strip(login))
+ d1.addCallback(strip(dc1))
+ d1.addCallback(strip(dc2))
d1.addCallbacks(strip(unsubscribe), self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
d = defer.gatherResults([d1, d2])
- d.addCallback(lambda _:
- self.assertEqual(
- LeapIMAPServer.theAccount.subscriptions,
- ['that/mbox']))
+ d.addCallback(get_subscriptions)
+ d.addCallback(lambda subscriptions:
+ self.assertEqual(subscriptions,
+ ['that/mbox']))
return d
def testSelect(self):
"""
Try to select a mailbox
"""
- self.server.theAccount.addMailbox('TESTMAILBOX-SELECT', creation_ts=42)
+ mbox_name = "TESTMAILBOXSELECT"
self.selectedArgs = None
+ acc = self.server.theAccount
+
+ def add_mailbox():
+ return acc.addMailbox(mbox_name, creation_ts=42)
+
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
@@ -520,29 +435,26 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def selected(args):
self.selectedArgs = args
self._cbStopClient(None)
- d = self.client.select('TESTMAILBOX-SELECT')
+ d = self.client.select(mbox_name)
d.addCallback(selected)
return d
- d1 = self.connected.addCallback(strip(login))
+ d1 = self.connected.addCallback(strip(add_mailbox))
+ d1.addCallback(strip(login))
d1.addCallback(strip(select))
- d1.addErrback(self._ebGeneral)
+ # d1.addErrback(self._ebGeneral)
d2 = self.loopback()
- return defer.gatherResults([d1, d2]).addCallback(self._cbTestSelect)
+
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(self._cbTestSelect)
+ return d
def _cbTestSelect(self, ignored):
- mbox = LeapIMAPServer.theAccount.getMailbox('TESTMAILBOX-SELECT')
- self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox)
- # XXX UIDVALIDITY should be "42" if the creation_ts is passed along
- # to the memory store. However, the current state of the account
- # implementation is incomplete and we're writing to soledad store
- # directly there. We should handle the UIDVALIDITY timestamping
- # mechanism in a separate test suite.
+ self.assertTrue(self.selectedArgs is not None)
self.assertEqual(self.selectedArgs, {
- 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 0,
- # 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42,
+ 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42,
'FLAGS': ('\\Seen', '\\Answered', '\\Flagged',
'\\Deleted', '\\Draft', '\\Recent', 'List'),
'READ-WRITE': True
@@ -560,13 +472,16 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
caps.update(c)
self.server.transport.loseConnection()
return self.client.getCapabilities().addCallback(gotCaps)
- d1 = self.connected.addCallback(
+
+ d1 = self.connected
+ d1.addCallback(
strip(getCaps)).addErrback(self._ebGeneral)
+
d = defer.gatherResults([self.loopback(), d1])
expected = {'IMAP4rev1': None, 'NAMESPACE': None, 'LITERAL+': None,
'IDLE': None}
-
- return d.addCallback(lambda _: self.assertEqual(expected, caps))
+ d.addCallback(lambda _: self.assertEqual(expected, caps))
+ return d
def testCapabilityWithAuth(self):
caps = {}
@@ -587,7 +502,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
'IDLE': None, 'LITERAL+': None,
'AUTH': ['CRAM-MD5']}
- return d.addCallback(lambda _: self.assertEqual(expCap, caps))
+ d.addCallback(lambda _: self.assertEqual(expCap, caps))
+ return d
#
# authentication
@@ -635,7 +551,6 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
return d.addCallback(self._cbTestLogin)
def _cbTestLogin(self, ignored):
- self.assertEqual(self.server.account, LeapIMAPServer.theAccount)
self.assertEqual(self.server.state, 'auth')
def testFailedLogin(self):
@@ -673,7 +588,6 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
return d.addCallback(self._cbTestLoginRequiringQuoting)
def _cbTestLoginRequiringQuoting(self, ignored):
- self.assertEqual(self.server.account, LeapIMAPServer.theAccount)
self.assertEqual(self.server.state, 'auth')
#
@@ -720,11 +634,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
for details.
"""
# TODO implement the IMAP4ClientExamineTests testcase.
-
- self.server.theAccount.addMailbox('test-mailbox-e',
- creation_ts=42)
+ mbox_name = "test_mailbox_e"
+ acc = self.server.theAccount
self.examinedArgs = None
+ def add_mailbox():
+ return acc.addMailbox(mbox_name, creation_ts=42)
+
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
@@ -732,11 +648,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def examined(args):
self.examinedArgs = args
self._cbStopClient(None)
- d = self.client.examine('test-mailbox-e')
+ d = self.client.examine(mbox_name)
d.addCallback(examined)
return d
- d1 = self.connected.addCallback(strip(login))
+ d1 = self.connected.addCallback(strip(add_mailbox))
+ d1.addCallback(strip(login))
d1.addCallback(strip(examine))
d1.addErrback(self._ebGeneral)
d2 = self.loopback()
@@ -744,28 +661,24 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
return d.addCallback(self._cbTestExamine)
def _cbTestExamine(self, ignored):
- mbox = self.server.theAccount.getMailbox('test-mailbox-e')
- self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox)
-
- # XXX UIDVALIDITY should be "42" if the creation_ts is passed along
- # to the memory store. However, the current state of the account
- # implementation is incomplete and we're writing to soledad store
- # directly there. We should handle the UIDVALIDITY timestamping
- # mechanism in a separate test suite.
self.assertEqual(self.examinedArgs, {
- 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 0,
- # 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42,
+ 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42,
'FLAGS': ('\\Seen', '\\Answered', '\\Flagged',
'\\Deleted', '\\Draft', '\\Recent', 'List'),
'READ-WRITE': False})
- def _listSetup(self, f):
- LeapIMAPServer.theAccount.addMailbox('root/subthingl',
- creation_ts=42)
- LeapIMAPServer.theAccount.addMailbox('root/another-thing',
- creation_ts=42)
- LeapIMAPServer.theAccount.addMailbox('non-root/subthing',
- creation_ts=42)
+ def _listSetup(self, f, f2=None):
+
+ acc = self.server.theAccount
+
+ def dc1():
+ return acc.addMailbox('root_subthing', creation_ts=42)
+
+ def dc2():
+ return acc.addMailbox('root_another_thing', creation_ts=42)
+
+ def dc3():
+ return acc.addMailbox('non_root_subthing', creation_ts=42)
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
@@ -775,6 +688,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
self.listed = None
d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(dc1))
+ d1.addCallback(strip(dc2))
+ d1.addCallback(strip(dc3))
+
+ if f2 is not None:
+ d1.addCallback(f2)
+
d1.addCallbacks(strip(f), self._ebGeneral)
d1.addCallbacks(listed, self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
@@ -787,12 +707,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
"""
def list():
return self.client.list('root', '%')
+
d = self._listSetup(list)
d.addCallback(lambda listed: self.assertEqual(
sortNest(listed),
sortNest([
- (SoledadMailbox.INIT_FLAGS, "/", "root/subthingl"),
- (SoledadMailbox.INIT_FLAGS, "/", "root/another-thing")
+ (IMAPMailbox.init_flags, "/", "root_subthing"),
+ (IMAPMailbox.init_flags, "/", "root_another_thing")
])
))
return d
@@ -801,20 +722,29 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
"""
Test LSub command
"""
- LeapIMAPServer.theAccount.subscribe('root/subthingl2')
+ acc = self.server.theAccount
+
+ def subs_mailbox():
+ # why not client.subscribe instead?
+ return acc.subscribe('root_subthing')
def lsub():
return self.client.lsub('root', '%')
- d = self._listSetup(lsub)
+
+ d = self._listSetup(lsub, strip(subs_mailbox))
d.addCallback(self.assertEqual,
- [(SoledadMailbox.INIT_FLAGS, "/", "root/subthingl2")])
+ [(IMAPMailbox.init_flags, "/", "root_subthing")])
return d
def testStatus(self):
"""
Test Status command
"""
- LeapIMAPServer.theAccount.addMailbox('root/subthings')
+ acc = self.server.theAccount
+
+ def add_mailbox():
+ return acc.addMailbox('root_subthings')
+
# XXX FIXME ---- should populate this a little bit,
# with unseen etc...
@@ -823,13 +753,15 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def status():
return self.client.status(
- 'root/subthings', 'MESSAGES', 'UIDNEXT', 'UNSEEN')
+ 'root_subthings', 'MESSAGES', 'UIDNEXT', 'UNSEEN')
def statused(result):
self.statused = result
self.statused = None
- d1 = self.connected.addCallback(strip(login))
+
+ d1 = self.connected.addCallback(strip(add_mailbox))
+ d1.addCallback(strip(login))
d1.addCallbacks(strip(status), self._ebGeneral)
d1.addCallbacks(statused, self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
@@ -886,56 +818,86 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
"""
infile = util.sibpath(__file__, 'rfc822.message')
message = open(infile)
- LeapIMAPServer.theAccount.addMailbox('root/subthing')
+ acc = self.server.theAccount
+ mailbox_name = "appendmbox/subthing"
+
+ def add_mailbox():
+ return acc.addMailbox(mailbox_name)
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
def append():
return self.client.append(
- 'root/subthing',
- message,
+ mailbox_name, message,
('\\SEEN', '\\DELETED'),
'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)',
)
- d1 = self.connected.addCallback(strip(login))
+ d1 = self.connected.addCallback(strip(add_mailbox))
+ d1.addCallback(strip(login))
d1.addCallbacks(strip(append), self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
d = defer.gatherResults([d1, d2])
+
+ d.addCallback(lambda _: acc.getMailbox(mailbox_name))
+ d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True))
return d.addCallback(self._cbTestFullAppend, infile)
- def _cbTestFullAppend(self, ignored, infile):
- mb = LeapIMAPServer.theAccount.getMailbox('root/subthing')
- self.assertEqual(1, len(mb.messages))
+ def _cbTestFullAppend(self, fetched, infile):
+ fetched = list(fetched)
+ self.assertTrue(len(fetched) == 1)
+ self.assertTrue(len(fetched[0]) == 2)
+ uid, msg = fetched[0]
+ parsed = self.parser.parse(open(infile))
+ expected_body = parsed.get_payload()
+ expected_headers = CaseInsensitiveDict(parsed.items())
- msg = mb.messages.get_msg_by_uid(1)
- self.assertEqual(
- set(('\\Recent', '\\SEEN', '\\DELETED')),
- set(msg.getFlags()))
+ def assert_flags(flags):
+ self.assertEqual(
+ set(('\\SEEN', '\\DELETED')),
+ set(flags))
- self.assertEqual(
- 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)',
- msg.getInternalDate())
+ def assert_date(date):
+ self.assertEqual(
+ 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)',
+ date)
- parsed = self.parser.parse(open(infile))
- body = parsed.get_payload()
- headers = dict(parsed.items())
- self.assertEqual(
- body,
- msg.getBodyFile().read())
- gotheaders = msg.getHeaders(True)
+ def assert_body(body):
+ gotbody = body.read()
+ self.assertEqual(expected_body, gotbody)
+
+ def assert_headers(headers):
+ self.assertItemsEqual(map(string.lower, expected_headers), headers)
+
+ d = defer.maybeDeferred(msg.getFlags)
+ d.addCallback(assert_flags)
- self.assertItemsEqual(
- headers, gotheaders)
+ d.addCallback(lambda _: defer.maybeDeferred(msg.getInternalDate))
+ d.addCallback(assert_date)
+
+ d.addCallback(
+ lambda _: defer.maybeDeferred(
+ msg.getBodyFile, self._soledad))
+ d.addCallback(assert_body)
+
+ d.addCallback(lambda _: defer.maybeDeferred(msg.getHeaders, True))
+ d.addCallback(assert_headers)
+
+ return d
def testPartialAppend(self):
"""
Test partially appending a message to the mailbox
"""
+ # TODO this test sometimes will fail because of the notify_just_mdoc
infile = util.sibpath(__file__, 'rfc822.message')
- LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING')
+
+ acc = self.server.theAccount
+
+ def add_mailbox():
+ return acc.addMailbox('PARTIAL/SUBTHING')
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
@@ -950,33 +912,47 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
(), self.client._IMAP4Client__cbContinueAppend, message
)
)
- d1 = self.connected.addCallback(strip(login))
+ d1 = self.connected.addCallback(strip(add_mailbox))
+ d1.addCallback(strip(login))
d1.addCallbacks(strip(append), self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
d = defer.gatherResults([d1, d2])
+
+ d.addCallback(lambda _: acc.getMailbox("PARTIAL/SUBTHING"))
+ d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True))
return d.addCallback(
self._cbTestPartialAppend, infile)
- def _cbTestPartialAppend(self, ignored, infile):
- mb = LeapIMAPServer.theAccount.getMailbox('PARTIAL/SUBTHING')
- self.assertEqual(1, len(mb.messages))
- msg = mb.messages.get_msg_by_uid(1)
- self.assertEqual(
- set(('\\SEEN', '\\Recent')),
- set(msg.getFlags())
- )
+ def _cbTestPartialAppend(self, fetched, infile):
+ fetched = list(fetched)
+ self.assertTrue(len(fetched) == 1)
+ self.assertTrue(len(fetched[0]) == 2)
+ uid, msg = fetched[0]
parsed = self.parser.parse(open(infile))
- body = parsed.get_payload()
- self.assertEqual(
- body,
- msg.getBodyFile().read())
+ expected_body = parsed.get_payload()
+
+ def assert_flags(flags):
+ self.assertEqual(
+ set((['\\SEEN'])), set(flags))
+
+ def assert_body(body):
+ gotbody = body.read()
+ self.assertEqual(expected_body, gotbody)
+
+ d = defer.maybeDeferred(msg.getFlags)
+ d.addCallback(assert_flags)
+
+ d.addCallback(lambda _: defer.maybeDeferred(msg.getBodyFile))
+ d.addCallback(assert_body)
+ return d
def testCheck(self):
"""
Test check command
"""
- LeapIMAPServer.theAccount.addMailbox('root/subthing')
+ def add_mailbox():
+ return self.server.theAccount.addMailbox('root/subthing')
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
@@ -987,89 +963,51 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
def check():
return self.client.check()
- d = self.connected.addCallback(strip(login))
+ d = self.connected.addCallbacks(
+ strip(add_mailbox), self._ebGeneral)
+ d.addCallbacks(lambda _: login(), self._ebGeneral)
d.addCallbacks(strip(select), self._ebGeneral)
d.addCallbacks(strip(check), self._ebGeneral)
d.addCallbacks(self._cbStopClient, self._ebGeneral)
- return self.loopback()
-
- # Okay, that was fun
-
- def testClose(self):
- """
- Test closing the mailbox. We expect to get deleted all messages flagged
- as such.
- """
- name = 'mailbox-close'
- self.server.theAccount.addMailbox(name)
-
- m = LeapIMAPServer.theAccount.getMailbox(name)
-
- def login():
- return self.client.login(TEST_USER, TEST_PASSWD)
-
- def select():
- return self.client.select(name)
-
- def add_messages():
- d1 = m.messages.add_msg(
- 'test 1', subject="Message 1",
- flags=('\\Deleted', 'AnotherFlag'))
- d2 = m.messages.add_msg(
- 'test 2', subject="Message 2",
- flags=('AnotherFlag',))
- d3 = m.messages.add_msg(
- 'test 3', subject="Message 3",
- flags=('\\Deleted',))
- d = defer.gatherResults([d1, d2, d3])
- return d
-
- def close():
- return self.client.close()
-
- d = self.connected.addCallback(strip(login))
- d.addCallbacks(strip(select), self._ebGeneral)
- d.addCallbacks(strip(add_messages), self._ebGeneral)
- d.addCallbacks(strip(close), self._ebGeneral)
- d.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
- return defer.gatherResults([d, d2]).addCallback(self._cbTestClose, m)
-
- def _cbTestClose(self, ignored, m):
- self.assertEqual(len(m.messages), 1)
- msg = m.messages.get_msg_by_uid(2)
- self.assertTrue(msg is not None)
+ return defer.gatherResults([d, d2])
- self.assertEqual(
- dict(msg.hdoc.content)['subject'],
- 'Message 2')
- self.failUnless(m.closed)
+ # Okay, that was much fun indeed
def testExpunge(self):
"""
Test expunge command
"""
- name = 'mailbox-expunge'
- self.server.theAccount.addMailbox(name)
- m = LeapIMAPServer.theAccount.getMailbox(name)
+ acc = self.server.theAccount
+ mailbox_name = 'mailboxexpunge'
+
+ def add_mailbox():
+ return acc.addMailbox(mailbox_name)
def login():
return self.client.login(TEST_USER, TEST_PASSWD)
def select():
- return self.client.select('mailbox-expunge')
+ return self.client.select(mailbox_name)
+
+ def save_mailbox(mailbox):
+ self.mailbox = mailbox
+
+ def get_mailbox():
+ d = acc.getMailbox(mailbox_name)
+ d.addCallback(save_mailbox)
+ return d
def add_messages():
- d1 = m.messages.add_msg(
- 'test 1', subject="Message 1",
- flags=('\\Deleted', 'AnotherFlag'))
- d2 = m.messages.add_msg(
- 'test 2', subject="Message 2",
- flags=('AnotherFlag',))
- d3 = m.messages.add_msg(
- 'test 3', subject="Message 3",
- flags=('\\Deleted',))
- d = defer.gatherResults([d1, d2, d3])
+ d = self.mailbox.addMessage(
+ 'test 1', flags=('\\Deleted', 'AnotherFlag'),
+ notify_just_mdoc=False)
+ d.addCallback(lambda _: self.mailbox.addMessage(
+ 'test 2', flags=('AnotherFlag',),
+ notify_just_mdoc=False))
+ d.addCallback(lambda _: self.mailbox.addMessage(
+ 'test 3', flags=('\\Deleted',),
+ notify_just_mdoc=False))
return d
def expunge():
@@ -1080,47 +1018,41 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
self.results = results
self.results = None
- d1 = self.connected.addCallback(strip(login))
- d1.addCallbacks(strip(select), self._ebGeneral)
+ d1 = self.connected.addCallback(strip(add_mailbox))
+ d1.addCallback(strip(login))
+ d1.addCallback(strip(get_mailbox))
d1.addCallbacks(strip(add_messages), self._ebGeneral)
+ d1.addCallbacks(strip(select), self._ebGeneral)
d1.addCallbacks(strip(expunge), self._ebGeneral)
d1.addCallbacks(expunged, self._ebGeneral)
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
d = defer.gatherResults([d1, d2])
- return d.addCallback(self._cbTestExpunge, m)
+ d.addCallback(lambda _: self.mailbox.getMessageCount())
+ return d.addCallback(self._cbTestExpunge)
- def _cbTestExpunge(self, ignored, m):
+ def _cbTestExpunge(self, count):
# we only left 1 mssage with no deleted flag
- self.assertEqual(len(m.messages), 1)
- msg = m.messages.get_msg_by_uid(2)
-
- msg = list(m.messages)[0]
- self.assertTrue(msg is not None)
-
- self.assertEqual(
- msg.hdoc.content['subject'],
- 'Message 2')
-
+ self.assertEqual(count, 1)
# the uids of the deleted messages
self.assertItemsEqual(self.results, [1, 3])
-class AccountTestCase(IMAP4HelperMixin, unittest.TestCase):
+class AccountTestCase(IMAP4HelperMixin):
"""
Test the Account.
"""
def _create_empty_mailbox(self):
- LeapIMAPServer.theAccount.addMailbox('')
+ return self.server.theAccount.addMailbox('')
def _create_one_mailbox(self):
- LeapIMAPServer.theAccount.addMailbox('one')
+ return self.server.theAccount.addMailbox('one')
def test_illegalMailboxCreate(self):
self.assertRaises(AssertionError, self._create_empty_mailbox)
-class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase):
+class IMAP4ServerSearchTestCase(IMAP4HelperMixin):
"""
Tests for the behavior of the search_* functions in L{imap5.IMAP4Server}.
"""
diff --git a/src/leap/mail/imap/tests/test_imap_store_fetch.py b/src/leap/mail/imap/tests/test_imap_store_fetch.py
deleted file mode 100644
index 6da8581..0000000
--- a/src/leap/mail/imap/tests/test_imap_store_fetch.py
+++ /dev/null
@@ -1,71 +0,0 @@
-from twisted.protocols import loopback
-from twisted.python import util
-
-from leap.mail.imap.tests.utils import IMAP4HelperMixin
-
-TEST_USER = "testuser@leap.se"
-TEST_PASSWD = "1234"
-
-
-class StoreAndFetchTestCase(IMAP4HelperMixin):
- """
- Several tests to check that the internal storage representation
- is able to render the message structures as we expect them.
- """
-
- def setUp(self):
- IMAP4HelperMixin.setUp(self)
- self.received_messages = self.received_uid = None
- self.result = None
-
- def addListener(self, x):
- pass
-
- def removeListener(self, x):
- pass
-
- def _addSignedMessage(self, _):
- self.server.state = 'select'
- infile = util.sibpath(__file__, 'rfc822.multi-signed.message')
- raw = open(infile).read()
- MBOX_NAME = "multipart/SIGNED"
-
- self.server.theAccount.addMailbox(MBOX_NAME)
- mbox = self.server.theAccount.getMailbox(MBOX_NAME)
- self.server.mbox = mbox
- # return a deferred that will fire with UID
- return self.server.mbox.messages.add_msg(raw)
-
- def _fetchWork(self, uids):
-
- def result(R):
- self.result = R
-
- self.connected.addCallback(
- self._addSignedMessage).addCallback(
- lambda uid: self.function(
- uids, uid=uid) # do NOT use seq numbers!
- ).addCallback(result).addCallback(
- self._cbStopClient).addErrback(self._ebGeneral)
-
- d = loopback.loopbackTCP(self.server, self.client, noisy=False)
- d.addCallback(lambda x: self.assertEqual(self.result, self.expected))
- return d
-
- def testMultiBody(self):
- """
- Test that a multipart signed message is retrieved the same
- as we stored it.
- """
- self.function = self.client.fetchBody
- messages = '1'
-
- # XXX review. This probably should give everything?
-
- self.expected = {1: {
- 'RFC822.TEXT': 'This is an example of a signed message,\n'
- 'with attachments.\n\n\n--=20\n'
- 'Nihil sine chao! =E2=88=B4\n',
- 'UID': '1'}}
- # print "test multi: fetch uid", messages
- return self._fetchWork(messages)
diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py
index 0932bd4..a34538b 100644
--- a/src/leap/mail/imap/tests/utils.py
+++ b/src/leap/mail/imap/tests/utils.py
@@ -1,30 +1,44 @@
-import os
-import tempfile
-import shutil
-
+# -*- coding: utf-8 -*-
+# utils.py
+# Copyright (C) 2014, 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Common utilities for testing Soledad IMAP Server.
+"""
from email import parser
from mock import Mock
from twisted.mail import imap4
from twisted.internet import defer
from twisted.protocols import loopback
+from twisted.python import log
-from leap.common.testing.basetest import BaseLeapTest
-from leap.mail.imap.account import SoledadBackedAccount
-from leap.mail.imap.memorystore import MemoryStore
-from leap.mail.imap.server import LeapIMAPServer
-from leap.soledad.client import Soledad
+from leap.mail.adaptors import soledad as soledad_adaptor
+from leap.mail.imap.account import IMAPAccount
+from leap.mail.imap.server import LEAPIMAPServer
+from leap.mail.tests.common import SoledadTestMixin
TEST_USER = "testuser@leap.se"
TEST_PASSWD = "1234"
+
#
# Simple IMAP4 Client for testing
#
-
class SimpleClient(imap4.IMAP4Client):
-
"""
A Simple IMAP4 Client to test our
Soledad-LEAPServer
@@ -51,161 +65,59 @@ class SimpleClient(imap4.IMAP4Client):
self.transport.loseConnection()
-def initialize_soledad(email, gnupg_home, tempdir):
- """
- Initializes soledad by hand
-
- :param email: ID for the user
- :param gnupg_home: path to home used by gnupg
- :param tempdir: path to temporal dir
- :rtype: Soledad instance
- """
-
- uuid = "foobar-uuid"
- passphrase = u"verysecretpassphrase"
- secret_path = os.path.join(tempdir, "secret.gpg")
- local_db_path = os.path.join(tempdir, "soledad.u1db")
- server_url = "http://provider"
- cert_file = ""
-
- class MockSharedDB(object):
-
- get_doc = Mock(return_value=None)
- put_doc = Mock()
- lock = Mock(return_value=('atoken', 300))
- unlock = Mock(return_value=True)
-
- def __call__(self):
- return self
-
- Soledad._shared_db = MockSharedDB()
-
- _soledad = Soledad(
- uuid,
- passphrase,
- secret_path,
- local_db_path,
- server_url,
- cert_file)
-
- return _soledad
-
-
-# XXX this is not properly a mixin, since helper already inherits from
-# uniittest.Testcase
-class IMAP4HelperMixin(BaseLeapTest):
+class IMAP4HelperMixin(SoledadTestMixin):
"""
MixIn containing several utilities to be shared across
different TestCases
"""
-
serverCTX = None
clientCTX = None
- # setUpClass cannot be a classmethod in trial, see:
- # https://twistedmatrix.com/trac/ticket/1870
-
def setUp(self):
- """
- Setup method for each test.
-
- Initializes and run a LEAP IMAP4 Server,
- but passing the same Soledad instance (it's costly to initialize),
- so we have to be sure to restore state across tests.
- """
- self.old_path = os.environ['PATH']
- self.old_home = os.environ['HOME']
- self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
- self.home = self.tempdir
- bin_tdir = os.path.join(
- self.tempdir,
- 'bin')
- os.environ["PATH"] = bin_tdir
- os.environ["HOME"] = self.tempdir
-
- # Soledad: config info
- self.gnupg_home = "%s/gnupg" % self.tempdir
- self.email = 'leap@leap.se'
-
- # initialize soledad by hand so we can control keys
- self._soledad = initialize_soledad(
- self.email,
- self.gnupg_home,
- self.tempdir)
- UUID = 'deadbeef',
- USERID = TEST_USER
- memstore = MemoryStore()
-
- ###########
- d = defer.Deferred()
- self.server = LeapIMAPServer(
- uuid=UUID, userid=USERID,
- contextFactory=self.serverCTX,
- # XXX do we really need this??
- soledad=self._soledad)
+ soledad_adaptor.cleanup_deferred_locks()
- self.client = SimpleClient(d, contextFactory=self.clientCTX)
- self.connected = d
-
- # XXX REVIEW-ME.
- # We're adding theAccount here to server
- # but it was also passed to initialization
- # as it was passed to realm.
- # I THINK we ONLY need to do it at one place now.
-
- theAccount = SoledadBackedAccount(
- USERID,
- soledad=self._soledad,
- memstore=memstore)
- LeapIMAPServer.theAccount = theAccount
-
- # in case we get something from previous tests...
- for mb in self.server.theAccount.mailboxes:
- self.server.theAccount.delete(mb)
+ UUID = 'deadbeef',
+ USERID = TEST_USER
- # email parser
- self.parser = parser.Parser()
+ def setup_server(account):
+ self.server = LEAPIMAPServer(
+ uuid=UUID, userid=USERID,
+ contextFactory=self.serverCTX,
+ soledad=self._soledad)
+ self.server.theAccount = account
+
+ d_server_ready = defer.Deferred()
+ self.client = SimpleClient(
+ d_server_ready, contextFactory=self.clientCTX)
+ self.connected = d_server_ready
+
+ def setup_account(_):
+ self.parser = parser.Parser()
+
+ # XXX this should be fixed in soledad.
+ # Soledad sync makes trial block forever. The sync it's mocked to
+ # fix this problem. _mock_soledad_get_from_index can be used from
+ # the tests to provide documents.
+ # TODO see here, possibly related?
+ # -- http://www.pythoneye.com/83_20424875/
+ self._soledad.sync = Mock()
+
+ d = defer.Deferred()
+ self.acc = IMAPAccount(USERID, self._soledad, d=d)
+ return d
+
+ d = super(IMAP4HelperMixin, self).setUp()
+ d.addCallback(setup_account)
+ d.addCallback(setup_server)
+ return d
def tearDown(self):
- """
- tearDown method called after each test.
-
- Deletes all documents in the Index, and deletes
- instances of server and client.
- """
- try:
- self._soledad.close()
- os.environ["PATH"] = self.old_path
- os.environ["HOME"] = self.old_home
- # safety check
- assert 'leap_tests-' in self.tempdir
- shutil.rmtree(self.tempdir)
- except Exception:
- print "ERROR WHILE CLOSING SOLEDAD"
-
- def populateMessages(self):
- """
- Populates soledad instance with several simple messages
- """
- # XXX we should encapsulate this thru SoledadBackedAccount
- # instead.
-
- # XXX we also should put this in a mailbox!
-
- self._soledad.messages.add_msg('', subject="test1")
- self._soledad.messages.add_msg('', subject="test2")
- self._soledad.messages.add_msg('', subject="test3")
- # XXX should change Flags too
- self._soledad.messages.add_msg('', subject="test4")
-
- def delete_all_docs(self):
- """
- Deletes all the docs in the testing instance of the
- SoledadBackedAccount.
- """
- self.server.theAccount.deleteAllMessages(
- iknowhatiamdoing=True)
+ SoledadTestMixin.tearDown(self)
+ del self._soledad
+ del self.client
+ del self.server
+ del self.connected
def _cbStopClient(self, ignore):
self.client.transport.loseConnection()
@@ -213,13 +125,8 @@ class IMAP4HelperMixin(BaseLeapTest):
def _ebGeneral(self, failure):
self.client.transport.loseConnection()
self.server.transport.loseConnection()
- # can we do something similar?
- # I guess this was ok with trial, but not in noseland...
- # log.err(failure, "Problem with %r" % (self.function,))
- raise failure.value
- # failure.trap(Exception)
+ if hasattr(self, 'function'):
+ log.err(failure, "Problem with %r" % (self.function,))
def loopback(self):
return loopback.loopbackAsync(self.server, self.client)
-
-
diff --git a/src/leap/mail/imap/tests/walktree.py b/src/leap/mail/imap/tests/walktree.py
index 695f487..f259a55 100644
--- a/src/leap/mail/imap/tests/walktree.py
+++ b/src/leap/mail/imap/tests/walktree.py
@@ -1,4 +1,4 @@
-#t -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
# walktree.py
# Copyright (C) 2013 LEAP
#
@@ -19,6 +19,7 @@ Tests for the walktree module.
"""
import os
import sys
+import pprint
from email import parser
from leap.mail import walk as W
@@ -118,7 +119,6 @@ if DEBUG and DO_CHECK:
print "Structure: OK"
-import pprint
print
print "RAW DOCS"
pprint.pprint(raw_docs)
diff --git a/src/leap/mail/incoming/__init__.py b/src/leap/mail/incoming/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/leap/mail/incoming/__init__.py
diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py
new file mode 100644
index 0000000..2bc6751
--- /dev/null
+++ b/src/leap/mail/incoming/service.py
@@ -0,0 +1,766 @@
+# -*- coding: utf-8 -*-
+# service.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Incoming mail fetcher.
+"""
+import copy
+import logging
+import shlex
+import time
+import traceback
+import warnings
+
+from email.parser import Parser
+from email.generator import Generator
+from email.utils import parseaddr
+from email.utils import formatdate
+from StringIO import StringIO
+from urlparse import urlparse
+
+from twisted.application.service import Service
+from twisted.python import log
+from twisted.internet import defer, reactor
+from twisted.internet.task import LoopingCall
+from twisted.internet.task import deferLater
+from u1db import errors as u1db_errors
+
+from leap.common.events import emit, catalog
+from leap.common.check import leap_assert, leap_assert_type
+from leap.common.mail import get_email_charset
+from leap.keymanager import errors as keymanager_errors
+from leap.keymanager.openpgp import OpenPGPKey
+from leap.mail.adaptors import soledad_indexes as fields
+from leap.mail.utils import json_loads, empty, first
+from leap.soledad.client import Soledad
+from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
+from leap.soledad.common.errors import InvalidAuthTokenError
+
+
+logger = logging.getLogger(__name__)
+
+MULTIPART_ENCRYPTED = "multipart/encrypted"
+MULTIPART_SIGNED = "multipart/signed"
+PGP_BEGIN = "-----BEGIN PGP MESSAGE-----"
+PGP_END = "-----END PGP MESSAGE-----"
+
+# The period between succesive checks of the incoming mail
+# queue (in seconds)
+INCOMING_CHECK_PERIOD = 60
+
+
+class MalformedMessage(Exception):
+ """
+ Raised when a given message is not well formed.
+ """
+ pass
+
+
+class IncomingMail(Service):
+ """
+ Fetches and process mail from the incoming pool.
+
+ This object implements IService interface, has public methods
+ startService and stopService that will actually initiate a
+ LoopingCall with check_period recurrency.
+ The LoopingCall itself will invoke the fetch method each time
+ that the check_period expires.
+
+ This loop will sync the soledad db with the remote server and
+ process all the documents found tagged as incoming mail.
+ """
+ # TODO implements IService?
+
+ name = "IncomingMail"
+
+ RECENT_FLAG = "\\Recent"
+ CONTENT_KEY = "content"
+
+ LEAP_SIGNATURE_HEADER = 'X-Leap-Signature'
+ """
+ Header added to messages when they are decrypted by the fetcher,
+ which states the validity of an eventual signature that might be included
+ in the encrypted blob.
+ """
+ LEAP_SIGNATURE_VALID = 'valid'
+ LEAP_SIGNATURE_INVALID = 'invalid'
+ LEAP_SIGNATURE_COULD_NOT_VERIFY = 'could not verify'
+
+ def __init__(self, keymanager, soledad, inbox, userid,
+ check_period=INCOMING_CHECK_PERIOD):
+
+ """
+ Initialize IncomingMail..
+
+ :param keymanager: a keymanager instance
+ :type keymanager: keymanager.KeyManager
+
+ :param soledad: a soledad instance
+ :type soledad: Soledad
+
+ :param inbox: the collection for the inbox where the new emails will be
+ stored
+ :type inbox: MessageCollection
+
+ :param check_period: the period to fetch new mail, in seconds.
+ :type check_period: int
+ """
+
+ leap_assert(keymanager, "need a keymanager to initialize")
+ leap_assert_type(soledad, Soledad)
+ leap_assert(check_period, "need a period to check incoming mail")
+ leap_assert_type(check_period, int)
+ leap_assert(userid, "need a userid to initialize")
+
+ self._keymanager = keymanager
+ self._soledad = soledad
+ self._inbox_collection = inbox
+ self._userid = userid
+
+ self._listeners = []
+ self._loop = None
+ self._check_period = check_period
+
+ # initialize a mail parser only once
+ self._parser = Parser()
+
+ def add_listener(self, listener):
+ """
+ Add a listener to inbox insertions.
+
+ This listener function will be called for each message added to the
+ inbox with its uid as parameter. This function should not be blocking
+ or it will block the incoming queue.
+
+ :param listener: the listener function
+ :type listener: callable
+ """
+ self._listeners.append(listener)
+
+ #
+ # Public API: fetch, start_loop, stop.
+ #
+
+ def fetch(self):
+ """
+ Fetch incoming mail, to be called periodically.
+
+ Calls a deferred that will execute the fetch callback
+ in a separate thread
+ """
+ def _sync_errback(failure):
+ log.err(failure)
+
+ def syncSoledadCallback(_):
+ # XXX this should be moved to adaptors
+ d = self._soledad.get_from_index(
+ fields.JUST_MAIL_IDX, "1", "0")
+ d.addCallback(self._process_doclist)
+ d.addErrback(_sync_errback)
+ return d
+
+ logger.debug("fetching mail for: %s %s" % (
+ self._soledad.uuid, self._userid))
+ d = self._sync_soledad()
+ d.addCallbacks(syncSoledadCallback, self._errback)
+ d.addCallbacks(self._signal_fetch_to_ui, self._errback)
+ return d
+
+ def startService(self):
+ """
+ Starts a loop to fetch mail.
+ """
+ Service.startService(self)
+ if self._loop is None:
+ self._loop = LoopingCall(self.fetch)
+ return self._loop.start(self._check_period)
+ else:
+ logger.warning("Tried to start an already running fetching loop.")
+
+ def stopService(self):
+ """
+ Stops the loop that fetches mail.
+ """
+ if self._loop and self._loop.running is True:
+ self._loop.stop()
+ self._loop = None
+ Service.stopService(self)
+
+ #
+ # Private methods.
+ #
+
+ # synchronize incoming mail
+
+ def _errback(self, failure):
+ log.err(failure)
+
+ def _sync_soledad(self):
+ """
+ Synchronize with remote soledad.
+
+ :returns: a list of LeapDocuments, or None.
+ :rtype: iterable or None
+ """
+ def _log_synced(result):
+ log.msg('FETCH soledad SYNCED.')
+ return result
+ try:
+ log.msg('FETCH: syncing soledad...')
+ d = self._soledad.sync()
+ d.addCallback(_log_synced)
+ return d
+ # TODO is this still raised? or should we do failure.trap
+ # instead?
+ except InvalidAuthTokenError:
+ # if the token is invalid, send an event so the GUI can
+ # disable mail and show an error message.
+ emit(catalog.SOLEDAD_INVALID_AUTH_TOKEN)
+
+ def _signal_fetch_to_ui(self, doclist):
+ """
+ Send leap events to ui.
+
+ :param doclist: iterable with msg documents.
+ :type doclist: iterable.
+ :returns: doclist
+ :rtype: iterable
+ """
+ if doclist:
+ fetched_ts = time.mktime(time.gmtime())
+ num_mails = len(doclist) if doclist is not None else 0
+ if num_mails != 0:
+ log.msg("there are %s mails" % (num_mails,))
+ emit(catalog.MAIL_FETCHED_INCOMING,
+ str(num_mails), str(fetched_ts))
+ return doclist
+
+ def _signal_unread_to_ui(self, *args):
+ """
+ Sends unread event to ui.
+ """
+ emit(catalog.MAIL_UNREAD_MESSAGES,
+ str(self._inbox_collection.count_unseen()))
+
+ # process incoming mail.
+
+ def _process_doclist(self, doclist):
+ """
+ Iterates through the doclist, checks if each doc
+ looks like a message, and yields a deferred that will decrypt and
+ process the message.
+
+ :param doclist: iterable with msg documents.
+ :type doclist: iterable.
+ :returns: a list of deferreds for individual messages.
+ """
+ log.msg('processing doclist')
+ if not doclist:
+ logger.debug("no docs found")
+ return
+ num_mails = len(doclist)
+
+ deferreds = []
+ for index, doc in enumerate(doclist):
+ logger.debug("processing doc %d of %d" % (index + 1, num_mails))
+ emit(catalog.MAIL_MSG_PROCESSING,
+ str(index), str(num_mails))
+
+ keys = doc.content.keys()
+
+ # TODO Compatibility check with the index in pre-0.6 mx
+ # that does not write the ERROR_DECRYPTING_KEY
+ # This should be removed in 0.7
+
+ has_errors = doc.content.get(fields.ERROR_DECRYPTING_KEY, None)
+ if has_errors is None:
+ warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!",
+ DeprecationWarning)
+
+ if has_errors:
+ logger.debug("skipping msg with decrypting errors...")
+ elif self._is_msg(keys):
+ d = self._decrypt_doc(doc)
+ d.addCallback(self._extract_keys)
+ d.addCallbacks(self._add_message_locally, self._errback)
+ deferreds.append(d)
+ d = defer.gatherResults(deferreds, consumeErrors=True)
+ d.addCallback(lambda _: doclist)
+ return d
+
+ #
+ # operations on individual messages
+ #
+
+ def _decrypt_doc(self, doc):
+ """
+ Decrypt the contents of a document.
+
+ :param doc: A document containing an encrypted message.
+ :type doc: SoledadDocument
+
+ :return: A Deferred that will be fired with the document and the
+ decrypted message.
+ :rtype: SoledadDocument, str
+ """
+ log.msg('decrypting msg')
+
+ def process_decrypted(res):
+ if isinstance(res, tuple):
+ decrdata, _ = res
+ success = True
+ else:
+ decrdata = ""
+ success = False
+
+ emit(catalog.MAIL_MSG_DECRYPTED, "1" if success else "0")
+ return self._process_decrypted_doc(doc, decrdata)
+
+ d = self._keymanager.decrypt(
+ doc.content[ENC_JSON_KEY],
+ self._userid, OpenPGPKey)
+ d.addErrback(self._errback)
+ d.addCallback(process_decrypted)
+ d.addCallback(lambda data: (doc, data))
+ return d
+
+ def _process_decrypted_doc(self, doc, data):
+ """
+ Process a document containing a succesfully decrypted message.
+
+ :param doc: the incoming message
+ :type doc: SoledadDocument
+ :param data: the json-encoded, decrypted content of the incoming
+ message
+ :type data: str
+
+ :return: a Deferred that will be fired with an str of the proccessed
+ data.
+ :rtype: Deferred
+ """
+ log.msg('processing decrypted doc')
+
+ # XXX turn this into an errBack for each one of
+ # the deferreds that would process an individual document
+ try:
+ msg = json_loads(data)
+ except UnicodeError as exc:
+ logger.error("Error while decrypting %s" % (doc.doc_id,))
+ logger.exception(exc)
+
+ # we flag the message as "with decrypting errors",
+ # to avoid further decryption attempts during sync
+ # cycles until we're prepared to deal with that.
+ # What is the same, when Ivan deals with it...
+ # A new decrypting attempt event could be triggered by a
+ # future a library upgrade, or a cli flag to the client,
+ # we just `defer` that for now... :)
+ doc.content[fields.ERROR_DECRYPTING_KEY] = True
+ deferLater(reactor, 0, self._update_incoming_message, doc)
+
+ # FIXME this is just a dirty hack to delay the proper
+ # deferred organization here...
+ # and remember, boys, do not do this at home.
+ return []
+
+ if not isinstance(msg, dict):
+ defer.returnValue(False)
+ if not msg.get(fields.INCOMING_KEY, False):
+ defer.returnValue(False)
+
+ # ok, this is an incoming message
+ rawmsg = msg.get(self.CONTENT_KEY, None)
+ if not rawmsg:
+ return ""
+ return self._maybe_decrypt_msg(rawmsg)
+
+ def _update_incoming_message(self, doc):
+ """
+ Do a put for a soledad document. This probably has been called only
+ in the case that we've needed to update the ERROR_DECRYPTING_KEY
+ flag in an incoming message, to get it out of the decrypting queue.
+
+ :param doc: the SoledadDocument to update
+ :type doc: SoledadDocument
+ """
+ log.msg("Updating Incoming MSG: SoledadDoc %s" % (doc.doc_id))
+ return self._soledad.put_doc(doc)
+
+ def _delete_incoming_message(self, doc):
+ """
+ Delete document.
+
+ :param doc: the SoledadDocument to delete
+ :type doc: SoledadDocument
+ """
+ log.msg("Deleting Incoming message: %s" % (doc.doc_id,))
+ return self._soledad.delete_doc(doc)
+
+ def _maybe_decrypt_msg(self, data):
+ """
+ Tries to decrypt a gpg message if data looks like one.
+
+ :param data: the text to be decrypted.
+ :type data: str
+
+ :return: a Deferred that will be fired with an str of data, possibly
+ decrypted.
+ :rtype: Deferred
+ """
+ leap_assert_type(data, str)
+ log.msg('maybe decrypting doc')
+
+ # parse the original message
+ encoding = get_email_charset(data)
+ msg = self._parser.parsestr(data)
+
+ fromHeader = msg.get('from', None)
+ senderAddress = None
+ if (fromHeader is not None and
+ (msg.get_content_type() == MULTIPART_ENCRYPTED or
+ msg.get_content_type() == MULTIPART_SIGNED)):
+ senderAddress = parseaddr(fromHeader)[1]
+
+ def add_leap_header(ret):
+ decrmsg, signkey = ret
+ if (senderAddress is None or signkey is None or
+ isinstance(signkey, keymanager_errors.KeyNotFound)):
+ decrmsg.add_header(
+ self.LEAP_SIGNATURE_HEADER,
+ self.LEAP_SIGNATURE_COULD_NOT_VERIFY)
+ elif isinstance(signkey, keymanager_errors.InvalidSignature):
+ decrmsg.add_header(
+ self.LEAP_SIGNATURE_HEADER,
+ self.LEAP_SIGNATURE_INVALID)
+ else:
+ decrmsg.add_header(
+ self.LEAP_SIGNATURE_HEADER,
+ self.LEAP_SIGNATURE_VALID,
+ pubkey=signkey.key_id)
+ return decrmsg.as_string()
+
+ if msg.get_content_type() == MULTIPART_ENCRYPTED:
+ d = self._decrypt_multipart_encrypted_msg(
+ msg, encoding, senderAddress)
+ else:
+ d = self._maybe_decrypt_inline_encrypted_msg(
+ msg, encoding, senderAddress)
+ d.addCallback(add_leap_header)
+ return d
+
+ def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderAddress):
+ """
+ Decrypt a message with content-type 'multipart/encrypted'.
+
+ :param msg: The original encrypted message.
+ :type msg: Message
+ :param encoding: The encoding of the email message.
+ :type encoding: str
+ :param senderAddress: The email address of the sender of the message.
+ :type senderAddress: str
+
+ :return: A Deferred that will be fired with a tuple containing a
+ decrypted Message and the signing OpenPGPKey if the signature
+ is valid or InvalidSignature or KeyNotFound.
+ :rtype: Deferred
+ """
+ log.msg('decrypting multipart encrypted msg')
+ msg = copy.deepcopy(msg)
+ self._msg_multipart_sanity_check(msg)
+
+ # parse message and get encrypted content
+ pgpencmsg = msg.get_payload()[1]
+ encdata = pgpencmsg.get_payload()
+
+ # decrypt or fail gracefully
+ def build_msg(res):
+ decrdata, signkey = res
+
+ decrmsg = self._parser.parsestr(decrdata)
+ # remove original message's multipart/encrypted content-type
+ del(msg['content-type'])
+
+ # replace headers back in original message
+ for hkey, hval in decrmsg.items():
+ try:
+ # this will raise KeyError if header is not present
+ msg.replace_header(hkey, hval)
+ except KeyError:
+ msg[hkey] = hval
+
+ # all ok, replace payload by unencrypted payload
+ msg.set_payload(decrmsg.get_payload())
+ return (msg, signkey)
+
+ d = self._keymanager.decrypt(
+ encdata, self._userid, OpenPGPKey,
+ verify=senderAddress)
+ d.addCallbacks(build_msg, self._decryption_error, errbackArgs=(msg,))
+ return d
+
+ def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding,
+ senderAddress):
+ """
+ Possibly decrypt an inline OpenPGP encrypted message.
+
+ :param origmsg: The original, possibly encrypted message.
+ :type origmsg: Message
+ :param encoding: The encoding of the email message.
+ :type encoding: str
+ :param senderAddress: The email address of the sender of the message.
+ :type senderAddress: str
+
+ :return: A Deferred that will be fired with a tuple containing a
+ decrypted Message and the signing OpenPGPKey if the signature
+ is valid or InvalidSignature or KeyNotFound.
+ :rtype: Deferred
+ """
+ log.msg('maybe decrypting inline encrypted msg')
+ # serialize the original message
+ buf = StringIO()
+ g = Generator(buf)
+ g.flatten(origmsg)
+ data = buf.getvalue()
+
+ def decrypted_data(res):
+ decrdata, signkey = res
+ return data.replace(pgp_message, decrdata), signkey
+
+ def encode_and_return(res):
+ data, signkey = res
+ if isinstance(data, unicode):
+ data = data.encode(encoding, 'replace')
+ return (self._parser.parsestr(data), signkey)
+
+ # handle exactly one inline PGP message
+ if PGP_BEGIN in data:
+ begin = data.find(PGP_BEGIN)
+ end = data.find(PGP_END)
+ pgp_message = data[begin:end + len(PGP_END)]
+ d = self._keymanager.decrypt(
+ pgp_message, self._userid, OpenPGPKey,
+ verify=senderAddress)
+ d.addCallbacks(decrypted_data, self._decryption_error,
+ errbackArgs=(data,))
+ else:
+ d = defer.succeed((data, None))
+ d.addCallback(encode_and_return)
+ return d
+
+ def _decryption_error(self, failure, msg):
+ """
+ Check for known decryption errors
+ """
+ if failure.check(keymanager_errors.DecryptError):
+ logger.warning('Failed to decrypt encrypted message (%s). '
+ 'Storing message without modifications.'
+ % str(failure.value))
+ return (msg, None)
+ elif failure.check(keymanager_errors.KeyNotFound):
+ logger.error('Failed to find private key for decryption (%s). '
+ 'Storing message without modifications.'
+ % str(failure.value))
+ return (msg, None)
+ else:
+ return failure
+
+ def _extract_keys(self, msgtuple):
+ """
+ Retrieve attached keys to the mesage and parse message headers for an
+ *OpenPGP* header as described on the `IETF draft
+ <http://tools.ietf.org/html/draft-josefsson-openpgp-mailnews-header-06>`
+ only urls with https and the same hostname than the email are supported
+ for security reasons.
+
+ :param msgtuple: a tuple consisting of a SoledadDocument
+ instance containing the incoming message
+ and data, the json-encoded, decrypted content of the
+ incoming message
+ :type msgtuple: (SoledadDocument, str)
+
+ :return: A Deferred that will be fired with msgtuple when key
+ extraction finishes
+ :rtype: Deferred
+ """
+ OpenPGP_HEADER = 'OpenPGP'
+ doc, data = msgtuple
+
+ # XXX the parsing of the message is done in mailbox.addMessage, maybe
+ # we should do it in this module so we don't need to parse it again
+ # here
+ msg = self._parser.parsestr(data)
+ _, fromAddress = parseaddr(msg['from'])
+
+ header = msg.get(OpenPGP_HEADER, None)
+ dh = defer.succeed(None)
+ if header is not None:
+ dh = self._extract_openpgp_header(header, fromAddress)
+
+ da = defer.succeed(None)
+ if msg.is_multipart():
+ da = self._extract_attached_key(msg.get_payload(), fromAddress)
+
+ d = defer.gatherResults([dh, da])
+ d.addCallback(lambda _: msgtuple)
+ return d
+
+ def _extract_openpgp_header(self, header, address):
+ """
+ Import keys from the OpenPGP header
+
+ :param header: OpenPGP header string
+ :type header: str
+ :param address: email address in the from header
+ :type address: str
+
+ :return: A Deferred that will be fired when header extraction is done
+ :rtype: Deferred
+ """
+ d = defer.succeed(None)
+ fields = dict([f.strip(' ').split('=') for f in header.split(';')])
+ if 'url' in fields:
+ url = shlex.split(fields['url'])[0] # remove quotations
+ urlparts = urlparse(url)
+ addressHostname = address.split('@')[1]
+ if (
+ urlparts.scheme == 'https' and
+ urlparts.hostname == addressHostname
+ ):
+ def fetch_error(failure):
+ if failure.check(keymanager_errors.KeyNotFound):
+ logger.warning("Url from OpenPGP header %s failed"
+ % (url,))
+ elif failure.check(keymanager_errors.KeyAttributesDiffer):
+ logger.warning("Key from OpenPGP header url %s didn't "
+ "match the from address %s"
+ % (url, address))
+ else:
+ return failure
+
+ d = self._keymanager.fetch_key(address, url, OpenPGPKey)
+ d.addCallback(
+ lambda _:
+ logger.info("Imported key from header %s" % (url,)))
+ d.addErrback(fetch_error)
+ else:
+ logger.debug("No valid url on OpenPGP header %s" % (url,))
+ else:
+ logger.debug("There is no url on the OpenPGP header: %s"
+ % (header,))
+ return d
+
+ def _extract_attached_key(self, attachments, address):
+ """
+ Import keys from the attachments
+
+ :param attachments: email attachment list
+ :type attachments: list(email.Message)
+ :param address: email address in the from header
+ :type address: str
+
+ :return: A Deferred that will be fired when all the keys are stored
+ :rtype: Deferred
+ """
+ MIME_KEY = "application/pgp-keys"
+
+ deferreds = []
+ for attachment in attachments:
+ if MIME_KEY == attachment.get_content_type():
+ logger.debug("Add key from attachment")
+ d = self._keymanager.put_raw_key(
+ attachment.get_payload(),
+ OpenPGPKey,
+ address=address)
+ deferreds.append(d)
+ return defer.gatherResults(deferreds)
+
+ def _add_message_locally(self, msgtuple):
+ """
+ Adds a message to local inbox and delete it from the incoming db
+ in soledad.
+
+ :param msgtuple: a tuple consisting of a SoledadDocument
+ instance containing the incoming message
+ and data, the json-encoded, decrypted content of the
+ incoming message
+ :type msgtuple: (SoledadDocument, str)
+
+ :return: A Deferred that will be fired when the messages is stored
+ :rtype: Defferred
+ """
+ doc, raw_data = msgtuple
+ insertion_date = formatdate(time.time())
+ log.msg('adding message %s to local db' % (doc.doc_id,))
+
+ def msgSavedCallback(result):
+ if empty(result):
+ return
+
+ for listener in self._listeners:
+ listener(result)
+
+ def signal_deleted(doc_id):
+ emit(catalog.MAIL_MSG_DELETED_INCOMING)
+ return doc_id
+
+ emit(catalog.MAIL_MSG_SAVED_LOCALLY)
+ d = self._delete_incoming_message(doc)
+ d.addCallback(signal_deleted)
+ return d
+
+ d = self._inbox_collection.add_msg(
+ raw_data, (self.RECENT_FLAG,), date=insertion_date)
+ d.addCallbacks(msgSavedCallback, self._errback)
+ return d
+
+ #
+ # helpers
+ #
+
+ def _msg_multipart_sanity_check(self, msg):
+ """
+ Performs a sanity check against a multipart encrypted msg
+
+ :param msg: The original encrypted message.
+ :type msg: Message
+ """
+ # sanity check
+ payload = msg.get_payload()
+ if len(payload) != 2:
+ raise MalformedMessage(
+ 'Multipart/encrypted messages should have exactly 2 body '
+ 'parts (instead of %d).' % len(payload))
+ if payload[0].get_content_type() != 'application/pgp-encrypted':
+ raise MalformedMessage(
+ "Multipart/encrypted messages' first body part should "
+ "have content type equal to 'application/pgp-encrypted' "
+ "(instead of %s)." % payload[0].get_content_type())
+ if payload[1].get_content_type() != 'application/octet-stream':
+ raise MalformedMessage(
+ "Multipart/encrypted messages' second body part should "
+ "have content type equal to 'octet-stream' (instead of "
+ "%s)." % payload[1].get_content_type())
+
+ def _is_msg(self, keys):
+ """
+ Checks if the keys of a dictionary match the signature
+ of the document type we use for messages.
+
+ :param keys: iterable containing the strings to match.
+ :type keys: iterable of strings.
+ :rtype: bool
+ """
+ return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys
diff --git a/src/leap/mail/incoming/tests/__init__.py b/src/leap/mail/incoming/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/leap/mail/incoming/tests/__init__.py
diff --git a/src/leap/mail/incoming/tests/test_incoming_mail.py b/src/leap/mail/incoming/tests/test_incoming_mail.py
new file mode 100644
index 0000000..f43f746
--- /dev/null
+++ b/src/leap/mail/incoming/tests/test_incoming_mail.py
@@ -0,0 +1,260 @@
+# -*- coding: utf-8 -*-
+# test_incoming_mail.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Test case for leap.mail.incoming.service
+
+@authors: Ruben Pollan, <meskio@sindominio.net>
+
+@license: GPLv3, see included LICENSE file
+"""
+
+import json
+
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from email.parser import Parser
+from mock import Mock
+from twisted.internet import defer
+
+from leap.keymanager.openpgp import OpenPGPKey
+from leap.mail.adaptors import soledad_indexes as fields
+from leap.mail.constants import INBOX_NAME
+from leap.mail.imap.account import IMAPAccount
+from leap.mail.incoming.service import IncomingMail
+from leap.mail.smtp.rfc3156 import MultipartEncrypted, PGPEncrypted
+from leap.mail.tests import (
+ TestCaseWithKeyManager,
+ ADDRESS,
+ ADDRESS_2,
+)
+from leap.soledad.common.document import SoledadDocument
+from leap.soledad.common.crypto import (
+ EncryptionSchemes,
+ ENC_JSON_KEY,
+ ENC_SCHEME_KEY,
+)
+
+# TODO: add some tests for encrypted, unencrypted, signed and unsgined messages
+
+
+class IncomingMailTestCase(TestCaseWithKeyManager):
+ """
+ Tests for the incoming mail parser
+ """
+ NICKSERVER = "http://domain"
+ BODY = """
+Governments of the Industrial World, you weary giants of flesh and steel, I
+come from Cyberspace, the new home of Mind. On behalf of the future, I ask
+you of the past to leave us alone. You are not welcome among us. You have
+no sovereignty where we gather.
+ """
+ EMAIL = """from: Test from SomeDomain <%(from)s>
+to: %(to)s
+subject: independence of cyberspace
+
+%(body)s
+ """ % {
+ "from": ADDRESS_2,
+ "to": ADDRESS,
+ "body": BODY
+ }
+
+ def setUp(self):
+ def getInbox(_):
+ d = defer.Deferred()
+ theAccount = IMAPAccount(ADDRESS, self._soledad, d=d)
+ d.addCallback(
+ lambda _: theAccount.getMailbox(INBOX_NAME))
+ return d
+
+ def setUpFetcher(inbox):
+ # Soledad sync makes trial block forever. The sync it's mocked to
+ # fix this problem. _mock_soledad_get_from_index can be used from
+ # the tests to provide documents.
+ # TODO ---- see here http://www.pythoneye.com/83_20424875/
+ self._soledad.sync = Mock(return_value=defer.succeed(None))
+
+ self.fetcher = IncomingMail(
+ self._km,
+ self._soledad,
+ inbox.collection,
+ ADDRESS)
+
+ # The messages don't exist on soledad will fail on deletion
+ self.fetcher._delete_incoming_message = Mock(
+ return_value=defer.succeed(None))
+
+ d = super(IncomingMailTestCase, self).setUp()
+ d.addCallback(getInbox)
+ d.addCallback(setUpFetcher)
+ return d
+
+ def tearDown(self):
+ del self.fetcher
+ return super(IncomingMailTestCase, self).tearDown()
+
+ def testExtractOpenPGPHeader(self):
+ """
+ Test the OpenPGP header key extraction
+ """
+ KEYURL = "https://leap.se/key.txt"
+ OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,)
+
+ message = Parser().parsestr(self.EMAIL)
+ message.add_header("OpenPGP", OpenPGP)
+ self.fetcher._keymanager.fetch_key = Mock(
+ return_value=defer.succeed(None))
+
+ def fetch_key_called(ret):
+ self.fetcher._keymanager.fetch_key.assert_called_once_with(
+ ADDRESS_2, KEYURL, OpenPGPKey)
+
+ d = self._create_incoming_email(message.as_string())
+ d.addCallback(
+ lambda email:
+ self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]))
+ d.addCallback(lambda _: self.fetcher.fetch())
+ d.addCallback(fetch_key_called)
+ return d
+
+ def testExtractOpenPGPHeaderInvalidUrl(self):
+ """
+ Test the OpenPGP header key extraction
+ """
+ KEYURL = "https://someotherdomain.com/key.txt"
+ OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,)
+
+ message = Parser().parsestr(self.EMAIL)
+ message.add_header("OpenPGP", OpenPGP)
+ self.fetcher._keymanager.fetch_key = Mock()
+
+ def fetch_key_called(ret):
+ self.assertFalse(self.fetcher._keymanager.fetch_key.called)
+
+ d = self._create_incoming_email(message.as_string())
+ d.addCallback(
+ lambda email:
+ self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]))
+ d.addCallback(lambda _: self.fetcher.fetch())
+ d.addCallback(fetch_key_called)
+ return d
+
+ def testExtractAttachedKey(self):
+ """
+ Test the OpenPGP header key extraction
+ """
+ KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..."
+
+ message = MIMEMultipart()
+ message.add_header("from", ADDRESS_2)
+ key = MIMEApplication("", "pgp-keys")
+ key.set_payload(KEY)
+ message.attach(key)
+ self.fetcher._keymanager.put_raw_key = Mock(
+ return_value=defer.succeed(None))
+ self.fetcher._keymanager.fetch_key = Mock()
+
+ def put_raw_key_called(_):
+ self.fetcher._keymanager.put_raw_key.assert_called_once_with(
+ KEY, OpenPGPKey, address=ADDRESS_2)
+
+ d = self._do_fetch(message.as_string())
+ d.addCallback(put_raw_key_called)
+ return d
+
+ def testDecryptEmail(self):
+ self.fetcher._decryption_error = Mock()
+
+ def create_encrypted_message(encstr):
+ message = Parser().parsestr(self.EMAIL)
+ newmsg = MultipartEncrypted('application/pgp-encrypted')
+ for hkey, hval in message.items():
+ newmsg.add_header(hkey, hval)
+
+ encmsg = MIMEApplication(
+ encstr, _subtype='octet-stream', _encoder=lambda x: x)
+ encmsg.add_header('content-disposition', 'attachment',
+ filename='msg.asc')
+ # create meta message
+ metamsg = PGPEncrypted()
+ metamsg.add_header('Content-Disposition', 'attachment')
+ # attach pgp message parts to new message
+ newmsg.attach(metamsg)
+ newmsg.attach(encmsg)
+ return newmsg
+
+ def decryption_error_not_called(_):
+ self.assertFalse(self.fetcher._decyption_error.called,
+ "There was some errors with decryption")
+
+ d = self._km.encrypt(
+ self.EMAIL,
+ ADDRESS, OpenPGPKey, sign=ADDRESS_2)
+ d.addCallback(create_encrypted_message)
+ d.addCallback(
+ lambda message:
+ self._do_fetch(message.as_string()))
+ return d
+
+ def testListener(self):
+ self.called = False
+
+ def listener(uid):
+ self.called = True
+
+ def listener_called(_):
+ self.assertTrue(self.called)
+
+ self.fetcher.add_listener(listener)
+ d = self._do_fetch(self.EMAIL)
+ d.addCallback(listener_called)
+ return d
+
+ def _do_fetch(self, message):
+ d = self._create_incoming_email(message)
+ d.addCallback(
+ lambda email:
+ self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]))
+ d.addCallback(lambda _: self.fetcher.fetch())
+ return d
+
+ def _create_incoming_email(self, email_str):
+ email = SoledadDocument()
+ data = json.dumps(
+ {"incoming": True, "content": email_str},
+ ensure_ascii=False)
+
+ def set_email_content(encr_data):
+ email.content = {
+ fields.INCOMING_KEY: True,
+ fields.ERROR_DECRYPTING_KEY: False,
+ ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY,
+ ENC_JSON_KEY: encr_data
+ }
+ return email
+ d = self._km.encrypt(data, ADDRESS, OpenPGPKey, fetch_remote=False)
+ d.addCallback(set_email_content)
+ return d
+
+ def _mock_soledad_get_from_index(self, index_name, value):
+ get_from_index = self._soledad.get_from_index
+
+ def soledad_mock(idx_name, *key_values):
+ if index_name == idx_name:
+ return defer.succeed(value)
+ return get_from_index(idx_name, *key_values)
+ self.fetcher._soledad.get_from_index = Mock(side_effect=soledad_mock)
diff --git a/src/leap/mail/interfaces.py b/src/leap/mail/interfaces.py
new file mode 100644
index 0000000..899400f
--- /dev/null
+++ b/src/leap/mail/interfaces.py
@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+# interfaces.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Interfaces for the leap.mail module.
+"""
+from zope.interface import Interface, Attribute
+
+
+class IMessageWrapper(Interface):
+ """
+ I know how to access the different parts into which a given message is
+ splitted into.
+ """
+
+ fdoc = Attribute('A dictionaly-like containing the flags document '
+ '(mutable)')
+ hdoc = Attribute('A dictionary-like containing the headers docuemnt '
+ '(immutable)')
+ cdocs = Attribute('A dictionary with the content-docs, one-indexed')
+
+
+# TODO [ ] Catch up with the actual implementation!
+# Lot of stuff added there ...
+
+class IMailAdaptor(Interface):
+ """
+ I know how to store the standard representation for messages and mailboxes,
+ and how to update the relevant mutable parts when needed.
+ """
+
+ def initialize_store(self, store):
+ """
+ Performs whatever initialization is needed before the store can be
+ used (creating indexes, sanity checks, etc).
+
+ :param store: store
+ :returns: a Deferred that will fire when the store is correctly
+ initialized.
+ :rtype: deferred
+ """
+
+ # TODO is staticmethod valid with an interface?
+ # @staticmethod
+ def get_msg_from_string(self, MessageClass, raw_msg):
+ """
+ Return IMessageWrapper implementor from a raw mail string
+
+ :param MessageClass: an implementor of IMessage
+ :type raw_msg: str
+ :rtype: implementor of leap.mail.IMessage
+ """
+
+ # TODO is staticmethod valid with an interface?
+ # @staticmethod
+ def get_msg_from_docs(self, MessageClass, msg_wrapper):
+ """
+ Return an IMessage implementor from its parts.
+
+ :param MessageClass: an implementor of IMessage
+ :param msg_wrapper: an implementor of IMessageWrapper
+ :rtype: implementor of leap.mail.IMessage
+ """
+
+ # -------------------------------------------------------------------
+ # XXX unsure about the following part yet ...........................
+
+ # the idea behind these three methods is that the adaptor also offers a
+ # fixed interface to create the documents the first time (using
+ # soledad.create_docs or whatever method maps to it in a similar store, and
+ # also allows to update flags and tags, hiding the actual implementation of
+ # where the flags/tags live in behind the concrete MailWrapper in use
+ # by this particular adaptor. In our impl it will be put_doc(fdoc) after
+ # locking the getting + updating of that fdoc for atomicity.
+
+ # 'store' must be an instance of something that offers a minimal subset of
+ # the document API that Soledad currently implements (create_doc, put_doc)
+ # I *think* store should belong to Account/Collection and be passed as
+ # param here instead of relying on it being an attribute of the instance.
+
+ def create_msg_docs(self, store, msg_wrapper):
+ """
+ :param store: The documents store
+ :type store:
+ :param msg_wrapper:
+ :type msg_wrapper: IMessageWrapper implementor
+ """
+
+ def update_msg_flags(self, store, msg_wrapper):
+ """
+ :param store: The documents store
+ :type store:
+ :param msg_wrapper:
+ :type msg_wrapper: IMessageWrapper implementor
+ """
+
+ def update_msg_tags(self, store, msg_wrapper):
+ """
+ :param store: The documents store
+ :type store:
+ :param msg_wrapper:
+ :type msg_wrapper: IMessageWrapper implementor
+ """
diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py
new file mode 100644
index 0000000..540a493
--- /dev/null
+++ b/src/leap/mail/mail.py
@@ -0,0 +1,1097 @@
+# -*- coding: utf-8 -*-
+# mail.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Generic Access to Mail objects: Public LEAP Mail API.
+"""
+import itertools
+import uuid
+import logging
+import StringIO
+import time
+import weakref
+
+from twisted.internet import defer
+from twisted.python import log
+
+from leap.common.check import leap_assert_type
+from leap.common.events import emit, catalog
+from leap.common.mail import get_email_charset
+
+from leap.mail.adaptors.soledad import SoledadMailAdaptor
+from leap.mail.constants import INBOX_NAME
+from leap.mail.constants import MessageFlags
+from leap.mail.mailbox_indexer import MailboxIndexer
+from leap.mail.plugins import soledad_sync_hooks
+from leap.mail.utils import find_charset, CaseInsensitiveDict
+from leap.mail.utils import lowerdict
+
+logger = logging.getLogger(name=__name__)
+
+
+# TODO LIST
+# [ ] Probably change the name of this module to "api" or "account", mail is
+# too generic (there's also IncomingMail, and OutgoingMail
+# [ ] Profile add_msg.
+
+def _get_mdoc_id(mbox, chash):
+ """
+ Get the doc_id for the metamsg document.
+ """
+ return "M+{mbox}+{chash}".format(mbox=mbox, chash=chash)
+
+
+def _write_and_rewind(payload):
+ fd = StringIO.StringIO()
+ fd.write(payload)
+ fd.seek(0)
+ return fd
+
+
+def _encode_payload(payload, ctype=""):
+ """
+ Properly encode an unicode payload (which can be string or unicode) as a
+ string.
+
+ :param payload: the payload to encode. currently soledad returns unicode
+ strings.
+ :type payload: basestring
+ :param ctype: optional, the content of the content-type header for this
+ payload.
+ :type ctype: str
+ :rtype: str
+ """
+ # TODO Related, it's proposed that we're able to pass
+ # the encoding to the soledad documents. Better to store the charset there?
+ # FIXME -----------------------------------------------
+ # this need a dedicated test-suite
+ charset = find_charset(ctype)
+
+ # XXX get from mail headers if not multipart!
+ # Beware also that we should pass the proper encoding to
+ # soledad when it's creating the documents.
+ # if not charset:
+ # charset = get_email_charset(payload)
+ # -----------------------------------------------------
+
+ if not charset:
+ charset = "utf-8"
+
+ try:
+ if isinstance(payload, unicode):
+ payload = payload.encode(charset)
+ except UnicodeError as exc:
+ logger.error(
+ "Unicode error, using 'replace'. {0!r}".format(exc))
+ payload = payload.encode(charset, 'replace')
+ return payload
+
+
+def _unpack_headers(headers_dict):
+ """
+ Take a "packed" dict containing headers (with repeated keys represented as
+ line breaks inside each value, preceded by the header key) and return a
+ list of tuples in which each repeated key has a different tuple.
+ """
+ headers_l = headers_dict.items()
+ for i, (k, v) in enumerate(headers_l):
+ splitted = v.split(k.lower() + ": ")
+ if len(splitted) != 1:
+ inner = zip(
+ itertools.cycle([k]),
+ map(lambda l: l.rstrip('\n'), splitted))
+ headers_l = headers_l[:i] + inner + headers_l[i + 1:]
+ return headers_l
+
+
+def _get_index_for_cdoc(part_map, cdocs_dict):
+ """
+ Get, if possible, the index for a given content-document matching the phash
+ of the passed part_map.
+
+ This is used when we are initializing a MessagePart, because we just pass a
+ reference to the parent message cdocs container and we need to iterate
+ through the cdocs to figure out which content-doc matches the phash of the
+ part we're currently rendering.
+
+ It is also used when recursing through a nested multipart message, because
+ in the initialization of the child MessagePart we pass a dictionary only
+ for the referenced cdoc.
+
+ :param part_map: a dict describing the mapping of the parts for the current
+ message-part.
+ :param cdocs: a dict of content-documents, 0-indexed.
+ :rtype: int
+ """
+ phash = part_map.get('phash', None)
+ if phash:
+ for i, cdoc_wrapper in cdocs_dict.items():
+ if cdoc_wrapper.phash == phash:
+ return i
+ return None
+
+
+class MessagePart(object):
+ # TODO This class should be better abstracted from the data model.
+ # TODO support arbitrarily nested multiparts (right now we only support
+ # the trivial case)
+
+ def __init__(self, part_map, cdocs={}, nested=False):
+ """
+ :param part_map: a dictionary mapping the subparts for
+ this MessagePart (1-indexed).
+ :type part_map: dict
+
+ The format for the part_map is as follows:
+
+ {u'ctype': u'text/plain',
+ u'headers': [[u'Content-Type', u'text/plain; charset="utf-8"'],
+ [u'Content-Transfer-Encoding', u'8bit']],
+ u'multi': False,
+ u'parts': 1,
+ u'phash': u'02D82B29F6BB0C8612D1C',
+ u'size': 132}
+
+ :param cdocs: optional, a reference to the top-level dict of wrappers
+ for content-docs (1-indexed).
+ """
+ self._pmap = part_map
+ self._cdocs = cdocs
+ self._nested = nested
+
+ index = _get_index_for_cdoc(part_map, self._cdocs) or 1
+ self._index = index
+
+ def get_size(self):
+ """
+ Size of the body, in octets.
+ """
+ total = self._pmap['size']
+ _h = self.get_headers()
+ headers = len(
+ '\n'.join(["%s: %s" % (k, v) for k, v in dict(_h).items()]))
+ # have to subtract 2 blank lines
+ return total - headers - 2
+
+ def get_body_file(self):
+ payload = ""
+ pmap = self._pmap
+ multi = pmap.get('multi')
+ if not multi:
+ payload = self._get_payload(self._index)
+ else:
+ # XXX uh, multi also... should recurse.
+ # This needs to be implemented in a more general and elegant way.
+ raise NotImplementedError
+ if payload:
+ payload = _encode_payload(payload)
+
+ return _write_and_rewind(payload)
+
+ def get_headers(self):
+ return CaseInsensitiveDict(self._pmap.get("headers", []))
+
+ def is_multipart(self):
+ return self._pmap.get("multi", False)
+
+ def get_subpart(self, part):
+ if not self.is_multipart():
+ raise TypeError
+
+ sub_pmap = self._pmap.get("part_map", {})
+
+ # XXX BUG --- workaround. Subparts with more than 1 subparts
+ # need to get the requested index for the subpart decremented.
+ # Off-by-one error, should investigate which is the real reason and
+ # fix it, this is only a quick workaround.
+ num_parts = self._pmap.get("parts", 0)
+ if num_parts > 1:
+ part = part - 1
+ # -------------------------------------------------------------
+
+ try:
+ part_map = sub_pmap[str(part)]
+ except KeyError:
+ log.msg("getSubpart for %s: KeyError" % (part,))
+ raise IndexError
+
+ cdoc_index = _get_index_for_cdoc(part_map, self._cdocs)
+ cdoc = self._cdocs.get(cdoc_index, {})
+
+ return MessagePart(part_map, cdocs={1: cdoc}, nested=True)
+
+ def _get_payload(self, index):
+ cdoc_wrapper = self._cdocs.get(index, None)
+ if cdoc_wrapper:
+ return cdoc_wrapper.raw
+ return ""
+
+
+class Message(object):
+ """
+ Represents a single message, and gives access to all its attributes.
+ """
+
+ def __init__(self, wrapper, uid=None):
+ """
+ :param wrapper: an instance of an implementor of IMessageWrapper
+ :param uid:
+ :type uid: int
+ """
+ self._wrapper = wrapper
+ self._uid = uid
+
+ def get_wrapper(self):
+ """
+ Get the wrapper for this message.
+ """
+ return self._wrapper
+
+ def get_uid(self):
+ """
+ Get the (optional) UID.
+ """
+ return self._uid
+
+ # imap.IMessage methods
+
+ def get_flags(self):
+ """
+ Get flags for this message.
+ :rtype: tuple
+ """
+ return self._wrapper.fdoc.get_flags()
+
+ def get_internal_date(self):
+ """
+ Retrieve the date internally associated with this message
+
+ According to the spec, this is NOT the date and time in the
+ RFC-822 header, but rather a date and time that reflects when the
+ message was received.
+
+ * In SMTP, date and time of final delivery.
+ * In COPY, internal date/time of the source message.
+ * In APPEND, date/time specified.
+
+ :return: An RFC822-formatted date string.
+ :rtype: str
+ """
+ return self._wrapper.hdoc.date
+
+ # imap.IMessageParts
+
+ def get_headers(self):
+ """
+ Get the raw headers document.
+ """
+ return CaseInsensitiveDict(self._wrapper.hdoc.headers)
+
+ def get_body_file(self, store):
+ """
+ Get a file descriptor with the body content.
+ """
+ def write_and_rewind_if_found(cdoc):
+ payload = cdoc.raw if cdoc else ""
+ # XXX pass ctype from headers if not multipart?
+ if payload:
+ payload = _encode_payload(payload, ctype=cdoc.content_type)
+ return _write_and_rewind(payload)
+
+ d = defer.maybeDeferred(self._wrapper.get_body, store)
+ d.addCallback(write_and_rewind_if_found)
+ return d
+
+ def get_size(self):
+ """
+ Size of the whole message, in octets (including headers).
+ """
+ total = self._wrapper.fdoc.size
+ return total
+
+ def is_multipart(self):
+ """
+ Return True if this message is multipart.
+ """
+ return self._wrapper.fdoc.multi
+
+ def get_subpart(self, part):
+ """
+ :param part: The number of the part to retrieve, indexed from 1.
+ :type part: int
+ :rtype: MessagePart
+ """
+ if not self.is_multipart():
+ raise TypeError
+ try:
+ subpart_dict = self._wrapper.get_subpart_dict(part)
+ except KeyError:
+ raise IndexError
+
+ return MessagePart(
+ subpart_dict, cdocs=self._wrapper.cdocs)
+
+ # Custom methods.
+
+ def get_tags(self):
+ """
+ Get the tags for this message.
+ """
+ return tuple(self._wrapper.fdoc.tags)
+
+
+class Flagsmode(object):
+ """
+ Modes for setting the flags/tags.
+ """
+ APPEND = 1
+ REMOVE = -1
+ SET = 0
+
+
+class MessageCollection(object):
+ """
+ A generic collection of messages. It can be messages sharing the same
+ mailbox, tag, the result of a given query, or just a bunch of ids for
+ master documents.
+
+ Since LEAP Mail is primarily oriented to store mail in Soledad, the default
+ (and, so far, only) implementation of the store is contained in the
+ Soledad Mail Adaptor, which is passed to every collection on creation by
+ the root Account object. If you need to use a different adaptor, change the
+ adaptor class attribute in your Account object.
+
+ Store is a reference to a particular instance of the message store (soledad
+ instance or proxy, for instance).
+ """
+
+ # TODO LIST
+ # [ ] look at IMessageSet methods
+ # [ ] make constructor with a per-instance deferredLock to use on
+ # creation/deletion?
+ # [ ] instead of a mailbox, we could pass an arbitrary container with
+ # pointers to different doc_ids (type: foo)
+ # [ ] To guarantee synchronicity of the documents sent together during a
+ # sync, we could get hold of a deferredLock that inhibits
+ # synchronization while we are updating (think more about this!)
+ # [ ] review the serveral count_ methods. I think it's better to patch
+ # server to accept deferreds.
+ # [ ] Use inheritance for the mailbox-collection instead of handling the
+ # special cases everywhere?
+ # [ ] or maybe a mailbox_only decorator...
+
+ # Account should provide an adaptor instance when creating this collection.
+ adaptor = None
+ store = None
+ messageklass = Message
+
+ _pending_inserts = dict()
+
+ def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None):
+ """
+ Constructor for a MessageCollection.
+ """
+ self.adaptor = adaptor
+ self.store = store
+
+ # XXX think about what to do when there is no mbox passed to
+ # the initialization. We could still get the MetaMsg by index, instead
+ # of by doc_id. See get_message_by_content_hash
+ self.mbox_indexer = mbox_indexer
+ self.mbox_wrapper = mbox_wrapper
+ self._listeners = set([])
+
+ def is_mailbox_collection(self):
+ """
+ Return True if this collection represents a Mailbox.
+ :rtype: bool
+ """
+ return bool(self.mbox_wrapper)
+
+ @property
+ def mbox_name(self):
+ # TODO raise instead?
+ if self.mbox_wrapper is None:
+ return None
+ return self.mbox_wrapper.mbox
+
+ @property
+ def mbox_uuid(self):
+ # TODO raise instead?
+ if self.mbox_wrapper is None:
+ return None
+ return self.mbox_wrapper.uuid
+
+ def get_mbox_attr(self, attr):
+ if self.mbox_wrapper is None:
+ raise RuntimeError("This is not a mailbox collection")
+ return getattr(self.mbox_wrapper, attr)
+
+ def set_mbox_attr(self, attr, value):
+ if self.mbox_wrapper is None:
+ raise RuntimeError("This is not a mailbox collection")
+ setattr(self.mbox_wrapper, attr, value)
+ return self.mbox_wrapper.update(self.store)
+
+ # Get messages
+
+ def get_message_by_content_hash(self, chash, get_cdocs=False):
+ """
+ Retrieve a message by its content hash.
+ :rtype: Deferred
+ """
+ if not self.is_mailbox_collection():
+ # TODO instead of getting the metamsg by chash, in this case we
+ # should query by (meta) index or use the internal collection of
+ # pointers-to-docs.
+ raise NotImplementedError()
+
+ metamsg_id = _get_mdoc_id(self.mbox_name, chash)
+
+ return self.adaptor.get_msg_from_mdoc_id(
+ self.messageklass, self.store,
+ metamsg_id, get_cdocs=get_cdocs)
+
+ def get_message_by_sequence_number(self, msn, get_cdocs=False):
+ """
+ Retrieve a message by its Message Sequence Number.
+ :rtype: Deferred
+ """
+ def get_uid_for_msn(all_uid):
+ return all_uid[msn - 1]
+ d = self.all_uid_iter()
+ d.addCallback(get_uid_for_msn)
+ d.addCallback(
+ lambda uid: self.get_message_by_uid(
+ uid, get_cdocs=get_cdocs))
+ d.addErrback(lambda f: log.err(f))
+ return d
+
+ def get_message_by_uid(self, uid, absolute=True, get_cdocs=False):
+ """
+ Retrieve a message by its Unique Identifier.
+
+ If this is a Mailbox collection, that is the message UID, unique for a
+ given mailbox, or a relative sequence number depending on the absolute
+ flag. For now, only absolute identifiers are supported.
+ :rtype: Deferred
+ """
+ # TODO deprecate absolute flag, it doesn't make sense UID and
+ # !absolute. use _by_sequence_number instead.
+ if not absolute:
+ raise NotImplementedError("Does not support relative ids yet")
+
+ get_doc_fun = self.mbox_indexer.get_doc_id_from_uid
+
+ def get_msg_from_mdoc_id(doc_id):
+ if doc_id is None:
+ return None
+ return self.adaptor.get_msg_from_mdoc_id(
+ self.messageklass, self.store,
+ doc_id, uid=uid, get_cdocs=get_cdocs)
+
+ def cleanup_and_get_doc_after_pending_insert(result):
+ for key in result:
+ self._pending_inserts.pop(key, None)
+ return get_doc_fun(self.mbox_uuid, uid)
+
+ if not self._pending_inserts:
+ d = get_doc_fun(self.mbox_uuid, uid)
+ else:
+ d = defer.gatherResults(self._pending_inserts.values())
+ d.addCallback(cleanup_and_get_doc_after_pending_insert)
+ d.addCallback(get_msg_from_mdoc_id)
+ return d
+
+ def get_flags_by_uid(self, uid, absolute=True):
+ # TODO use sequence numbers
+ if not absolute:
+ raise NotImplementedError("Does not support relative ids yet")
+
+ def get_flags_from_mdoc_id(doc_id):
+ if doc_id is None: # XXX needed? or bug?
+ return None
+ return self.adaptor.get_flags_from_mdoc_id(
+ self.store, doc_id)
+
+ def wrap_in_tuple(flags):
+ return (uid, flags)
+
+ d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_uuid, uid)
+ d.addCallback(get_flags_from_mdoc_id)
+ d.addCallback(wrap_in_tuple)
+ return d
+
+ def count(self):
+ """
+ Count the messages in this collection.
+ :return: a Deferred that will fire with the integer for the count.
+ :rtype: Deferred
+ """
+ if not self.is_mailbox_collection():
+ raise NotImplementedError()
+
+ d = self.mbox_indexer.count(self.mbox_uuid)
+ return d
+
+ def count_recent(self):
+ """
+ Count the recent messages in this collection.
+ :return: a Deferred that will fire with the integer for the count.
+ :rtype: Deferred
+ """
+ if not self.is_mailbox_collection():
+ raise NotImplementedError()
+ return self.adaptor.get_count_recent(self.store, self.mbox_uuid)
+
+ def count_unseen(self):
+ """
+ Count the unseen messages in this collection.
+ :return: a Deferred that will fire with the integer for the count.
+ :rtype: Deferred
+ """
+ if not self.is_mailbox_collection():
+ raise NotImplementedError()
+ return self.adaptor.get_count_unseen(self.store, self.mbox_uuid)
+
+ def get_uid_next(self):
+ """
+ Get the next integer beyond the highest UID count for this mailbox.
+
+ :return: a Deferred that will fire with the integer for the next uid.
+ :rtype: Deferred
+ """
+ return self.mbox_indexer.get_next_uid(self.mbox_uuid)
+
+ def get_last_uid(self):
+ """
+ Get the last UID for this mailbox.
+ """
+ return self.mbox_indexer.get_last_uid(self.mbox_uuid)
+
+ def all_uid_iter(self):
+ """
+ Iterator through all the uids for this collection.
+ """
+ return self.mbox_indexer.all_uid_iter(self.mbox_uuid)
+
+ def get_uid_from_msgid(self, msgid):
+ """
+ Return the UID(s) of the matching msg-ids for this mailbox collection.
+ """
+ if not self.is_mailbox_collection():
+ raise NotImplementedError()
+
+ def get_uid(mdoc_id):
+ if not mdoc_id:
+ return None
+ d = self.mbox_indexer.get_uid_from_doc_id(
+ self.mbox_uuid, mdoc_id)
+ return d
+
+ d = self.adaptor.get_mdoc_id_from_msgid(
+ self.store, self.mbox_uuid, msgid)
+ d.addCallback(get_uid)
+ return d
+
+ # Manipulate messages
+
+ def add_msg(self, raw_msg, flags=tuple(), tags=tuple(), date="",
+ notify_just_mdoc=False):
+ """
+ Add a message to this collection.
+
+ :param raw_msg: the raw message
+ :param flags: tuple of flags for this message
+ :param tags: tuple of tags for this message
+ :param date:
+ formatted date, it will be used to retrieve the internal
+ date for this message. According to the spec, this is NOT the date
+ and time in the RFC-822 header, but rather a date and time that
+ reflects when the message was received.
+ :type date: str
+ :param notify_just_mdoc:
+ boolean passed to the wrapper.create method, to indicate whether
+ we're insterested in being notified right after the mdoc has been
+ written (as it's the first doc to be written, and quite small, this
+ is faster, though potentially unsafe), or on the contrary we want
+ to wait untill all the parts have been written.
+ Used by the imap mailbox implementation to get faster responses.
+ This will be ignored (and set to False) if a heuristic for a Draft
+ message is met, which currently is a specific mozilla header.
+ :type notify_just_mdoc: bool
+
+ :returns: a deferred that will fire with the UID of the inserted
+ message.
+ :rtype: deferred
+ """
+ # TODO watch out if the use of this method in IMAP COPY/APPEND is
+ # passing the right date.
+ # XXX mdoc ref is a leaky abstraction here. generalize.
+ leap_assert_type(flags, tuple)
+ leap_assert_type(date, str)
+
+ msg = self.adaptor.get_msg_from_string(Message, raw_msg)
+ wrapper = msg.get_wrapper()
+
+ headers = lowerdict(msg.get_headers())
+ moz_draft_hdr = "X-Mozilla-Draft-Info"
+ if moz_draft_hdr.lower() in headers:
+ log.msg("Setting fast notify to False, Draft detected")
+ notify_just_mdoc = False
+
+ if notify_just_mdoc:
+ msgid = headers.get('message-id')
+ if msgid:
+ self._pending_inserts[msgid] = defer.Deferred()
+
+ if not self.is_mailbox_collection():
+ raise NotImplementedError()
+
+ else:
+ mbox_id = self.mbox_uuid
+ wrapper.set_mbox_uuid(mbox_id)
+ wrapper.set_flags(flags)
+ wrapper.set_tags(tags)
+ wrapper.set_date(date)
+
+ def insert_mdoc_id(_, wrapper):
+ doc_id = wrapper.mdoc.doc_id
+ if not doc_id:
+ # --- BUG -----------------------------------------
+ # XXX watch out, sometimes mdoc doesn't have doc_id
+ # but it has future_id. Should be solved already.
+ logger.error("BUG: (please report) Null doc_id for "
+ "document %s" %
+ (wrapper.mdoc.serialize(),))
+ return defer.succeed("mdoc_id not inserted")
+ # XXX BUG -----------------------------------------
+
+ # XXX BUG sometimes the table is not yet created,
+ # so workaround is to make sure we always check for it before
+ # inserting the doc. I should debug into the real cause.
+ d = self.mbox_indexer.create_table(self.mbox_uuid)
+ d.addBoth(lambda _: self.mbox_indexer.insert_doc(
+ self.mbox_uuid, doc_id))
+ return d
+
+ d = wrapper.create(
+ self.store,
+ notify_just_mdoc=notify_just_mdoc,
+ pending_inserts_dict=self._pending_inserts)
+ d.addCallback(insert_mdoc_id, wrapper)
+ d.addCallback(self.cb_signal_unread_to_ui)
+ d.addCallback(self.notify_new_to_listeners)
+ d.addErrback(lambda failure: log.err(failure))
+
+ return d
+
+ # Listeners
+
+ def addListener(self, listener):
+ self._listeners.add(listener)
+
+ def removeListener(self, listener):
+ self._listeners.remove(listener)
+
+ def notify_new_to_listeners(self, result):
+ for listener in self._listeners:
+ listener.notify_new()
+ return result
+
+ def cb_signal_unread_to_ui(self, result):
+ """
+ Sends an unread event to ui, passing *only* the number of unread
+ messages if *this* is the inbox. This event is catched, for instance,
+ in the Bitmask client that displays a message with the number of unread
+ mails in the INBOX.
+
+ Used as a callback in several commands.
+
+ :param result: ignored
+ """
+ # TODO it might make sense to modify the event so that
+ # it receives both the mailbox name AND the number of unread messages.
+ if self.mbox_name.lower() == "inbox":
+ d = defer.maybeDeferred(self.count_unseen)
+ d.addCallback(self.__cb_signal_unread_to_ui)
+ return result
+
+ def __cb_signal_unread_to_ui(self, unseen):
+ """
+ Send the unread signal to UI.
+ :param unseen: number of unseen messages.
+ :type unseen: int
+ """
+ # TODO change name of the signal, independent from imap now.
+ emit(catalog.MAIL_UNREAD_MESSAGES, str(unseen))
+
+ def copy_msg(self, msg, new_mbox_uuid):
+ """
+ Copy the message to another collection. (it only makes sense for
+ mailbox collections)
+ """
+ # TODO should CHECK first if the mdoc is present in the mailbox
+ # WITH a Deleted flag... and just simply remove the flag...
+ # Another option is to delete the previous mdoc if it already exists
+ # (so we get a new UID)
+
+ if not self.is_mailbox_collection():
+ raise NotImplementedError()
+
+ def delete_mdoc_entry_and_insert(failure, mbox_uuid, doc_id):
+ d = self.mbox_indexer.delete_doc_by_hash(mbox_uuid, doc_id)
+ d.addCallback(lambda _: self.mbox_indexer.insert_doc(
+ new_mbox_uuid, doc_id))
+ return d
+
+ def insert_copied_mdoc_id(wrapper_new_msg):
+ # XXX FIXME -- since this is already saved, the future_doc_id
+ # should be already copied into the doc_id!
+ # Investigate why we are not receiving the already saved doc_id
+ doc_id = wrapper_new_msg.mdoc.doc_id
+ if not doc_id:
+ doc_id = wrapper_new_msg.mdoc._future_doc_id
+
+ def insert_conditionally(uid, mbox_uuid, doc_id):
+ indexer = self.mbox_indexer
+ if uid:
+ d = indexer.delete_doc_by_hash(mbox_uuid, doc_id)
+ d.addCallback(lambda _: indexer.insert_doc(
+ new_mbox_uuid, doc_id))
+ return d
+ else:
+ d = indexer.insert_doc(mbox_uuid, doc_id)
+ return d
+
+ def log_result(result):
+ return result
+
+ def insert_doc(_, mbox_uuid, doc_id):
+ d = self.mbox_indexer.get_uid_from_doc_id(mbox_uuid, doc_id)
+ d.addCallback(insert_conditionally, mbox_uuid, doc_id)
+ d.addErrback(lambda err: log.failure(err))
+ d.addCallback(log_result)
+ return d
+
+ d = self.mbox_indexer.create_table(new_mbox_uuid)
+ d.addBoth(insert_doc, new_mbox_uuid, doc_id)
+ return d
+
+ wrapper = msg.get_wrapper()
+
+ d = wrapper.copy(self.store, new_mbox_uuid)
+ d.addCallback(insert_copied_mdoc_id)
+ d.addCallback(self.notify_new_to_listeners)
+ return d
+
+ def delete_msg(self, msg):
+ """
+ Delete this message.
+ """
+ wrapper = msg.get_wrapper()
+
+ def delete_mdoc_id(_, wrapper):
+ doc_id = wrapper.mdoc.doc_id
+ return self.mbox_indexer.delete_doc_by_hash(
+ self.mbox_uuid, doc_id)
+ d = wrapper.delete(self.store)
+ d.addCallback(delete_mdoc_id, wrapper)
+ return d
+
+ def delete_all_flagged(self):
+ """
+ Delete all messages flagged as \\Deleted.
+ Used from IMAPMailbox.expunge()
+ """
+ def get_uid_list(hashes):
+ d = []
+ for h in hashes:
+ d.append(self.mbox_indexer.get_uid_from_doc_id(
+ self.mbox_uuid, h))
+ return defer.gatherResults(d), hashes
+
+ def delete_uid_entries((uids, hashes)):
+ d = []
+ for h in hashes:
+ d.append(self.mbox_indexer.delete_doc_by_hash(
+ self.mbox_uuid, h))
+
+ def return_uids_when_deleted(ignored):
+ return uids
+
+ all_deleted = defer.gatherResults(d).addCallback(
+ return_uids_when_deleted)
+ return all_deleted
+
+ mdocs_deleted = self.adaptor.del_all_flagged_messages(
+ self.store, self.mbox_uuid)
+ mdocs_deleted.addCallback(get_uid_list)
+ mdocs_deleted.addCallback(delete_uid_entries)
+ mdocs_deleted.addErrback(lambda f: log.err(f))
+ return mdocs_deleted
+
+ # TODO should add a delete-by-uid to collection?
+
+ def delete_all_docs(self):
+ def del_all_uid(uid_list):
+ deferreds = []
+ for uid in uid_list:
+ d = self.get_message_by_uid(uid)
+ d.addCallback(lambda msg: msg.delete())
+ deferreds.append(d)
+ return defer.gatherResults(deferreds)
+
+ d = self.all_uid_iter()
+ d.addCallback(del_all_uid)
+ return d
+
+ def update_flags(self, msg, flags, mode):
+ """
+ Update flags for a given message.
+ """
+ wrapper = msg.get_wrapper()
+ current = wrapper.fdoc.flags
+ newflags = map(str, self._update_flags_or_tags(current, flags, mode))
+ wrapper.fdoc.flags = newflags
+
+ wrapper.fdoc.seen = MessageFlags.SEEN_FLAG in newflags
+ wrapper.fdoc.deleted = MessageFlags.DELETED_FLAG in newflags
+
+ d = self.adaptor.update_msg(self.store, msg)
+ d.addCallback(lambda _: newflags)
+ return d
+
+ def update_tags(self, msg, tags, mode):
+ """
+ Update tags for a given message.
+ """
+ wrapper = msg.get_wrapper()
+ current = wrapper.fdoc.tags
+ newtags = self._update_flags_or_tags(current, tags, mode)
+
+ wrapper.fdoc.tags = newtags
+ d = self.adaptor.update_msg(self.store, msg)
+ d.addCallback(newtags)
+ return d
+
+ def _update_flags_or_tags(self, old, new, mode):
+ if mode == Flagsmode.APPEND:
+ final = list((set(tuple(old) + new)))
+ elif mode == Flagsmode.REMOVE:
+ final = list(set(old).difference(set(new)))
+ elif mode == Flagsmode.SET:
+ final = new
+ return final
+
+
+class Account(object):
+ """
+ Account is the top level abstraction to access collections of messages
+ associated with a LEAP Mail Account.
+
+ It primarily handles creation and access of Mailboxes, which will be the
+ basic collection handled by traditional MUAs, but it can also handle other
+ types of Collections (tag based, for instance).
+
+ leap.mail.imap.IMAPAccount partially proxies methods in this
+ class.
+ """
+
+ # Adaptor is passed to the returned MessageCollections, so if you want to
+ # use a different adaptor this is the place to change it, by subclassing
+ # the Account class.
+
+ adaptor_class = SoledadMailAdaptor
+
+ # This is a mapping to collection instances so that we always
+ # return a reference to them instead of creating new ones. However, being a
+ # dictionary of weakrefs values, they automagically vanish from the dict
+ # when no hard refs is left to them (so they can be garbage collected)
+ # This is important because the different wrappers rely on several
+ # kinds of deferredLocks that are kept as class or instance variables
+ _collection_mapping = weakref.WeakValueDictionary()
+
+ def __init__(self, store, ready_cb=None):
+ self.store = store
+ self.adaptor = self.adaptor_class()
+ self.mbox_indexer = MailboxIndexer(self.store)
+
+ # This flag is only used from the imap service for the moment.
+ # In the future, we should prevent any public method to continue if
+ # this is set to True. Also, it would be good to plug to the
+ # authentication layer.
+ self.session_ended = False
+
+ self.deferred_initialization = defer.Deferred()
+ self._ready_cb = ready_cb
+
+ self._init_d = self._initialize_storage()
+ self._initialize_sync_hooks()
+
+ def _initialize_storage(self):
+
+ def add_mailbox_if_none(mboxes):
+ if not mboxes:
+ return self.add_mailbox(INBOX_NAME)
+
+ def finish_initialization(result):
+ self.deferred_initialization.callback(None)
+ if self._ready_cb is not None:
+ self._ready_cb()
+
+ d = self.adaptor.initialize_store(self.store)
+ d.addCallback(lambda _: self.list_all_mailbox_names())
+ d.addCallback(add_mailbox_if_none)
+ d.addCallback(finish_initialization)
+ return d
+
+ def callWhenReady(self, cb, *args, **kw):
+ """
+ Execute the callback when the initialization of the Account is ready.
+ Note that the callback will receive a first meaningless parameter.
+ """
+ # TODO this should ignore the first parameter explicitely
+ # lambda _: cb(*args, **kw)
+ self.deferred_initialization.addCallback(cb, *args, **kw)
+ return self.deferred_initialization
+
+ # Sync hooks
+
+ def _initialize_sync_hooks(self):
+ soledad_sync_hooks.post_sync_uid_reindexer.set_account(self)
+
+ def _teardown_sync_hooks(self):
+ soledad_sync_hooks.post_sync_uid_reindexer.set_account(None)
+
+ #
+ # Public API Starts
+ #
+
+ def list_all_mailbox_names(self):
+
+ def filter_names(mboxes):
+ return [m.mbox for m in mboxes]
+
+ d = self.get_all_mailboxes()
+ d.addCallback(filter_names)
+ return d
+
+ def get_all_mailboxes(self):
+ d = self.adaptor.get_all_mboxes(self.store)
+ return d
+
+ def add_mailbox(self, name, creation_ts=None):
+
+ if creation_ts is None:
+ # by default, we pass an int value
+ # taken from the current time
+ # we make sure to take enough decimals to get a unique
+ # mailbox-uidvalidity.
+ creation_ts = int(time.time() * 10E2)
+
+ def set_creation_ts(wrapper):
+ wrapper.created = creation_ts
+ d = wrapper.update(self.store)
+ d.addCallback(lambda _: wrapper)
+ return d
+
+ def create_uuid(wrapper):
+ if not wrapper.uuid:
+ wrapper.uuid = str(uuid.uuid4())
+ d = wrapper.update(self.store)
+ d.addCallback(lambda _: wrapper)
+ return d
+ return wrapper
+
+ def create_uid_table_cb(wrapper):
+ d = self.mbox_indexer.create_table(wrapper.uuid)
+ d.addCallback(lambda _: wrapper)
+ return d
+
+ d = self.adaptor.get_or_create_mbox(self.store, name)
+ d.addCallback(set_creation_ts)
+ d.addCallback(create_uuid)
+ d.addCallback(create_uid_table_cb)
+ return d
+
+ def delete_mailbox(self, name):
+
+ def delete_uid_table_cb(wrapper):
+ d = self.mbox_indexer.delete_table(wrapper.uuid)
+ d.addCallback(lambda _: wrapper)
+ return d
+
+ d = self.adaptor.get_or_create_mbox(self.store, name)
+ d.addCallback(delete_uid_table_cb)
+ d.addCallback(
+ lambda wrapper: self.adaptor.delete_mbox(self.store, wrapper))
+ return d
+
+ def rename_mailbox(self, oldname, newname):
+
+ def _rename_mbox(wrapper):
+ wrapper.mbox = newname
+ d = wrapper.update(self.store)
+ d.addCallback(lambda result: wrapper)
+ return d
+
+ d = self.adaptor.get_or_create_mbox(self.store, oldname)
+ d.addCallback(_rename_mbox)
+ return d
+
+ # Get Collections
+
+ def get_collection_by_mailbox(self, name):
+ """
+ :rtype: deferred
+ :return: a deferred that will fire with a MessageCollection
+ """
+ collection = self._collection_mapping.get(name, None)
+ if collection:
+ return defer.succeed(collection)
+
+ # imap select will use this, passing the collection to SoledadMailbox
+ def get_collection_for_mailbox(mbox_wrapper):
+ collection = MessageCollection(
+ self.adaptor, self.store, self.mbox_indexer, mbox_wrapper)
+ self._collection_mapping[name] = collection
+ return collection
+
+ d = self.adaptor.get_or_create_mbox(self.store, name)
+ d.addCallback(get_collection_for_mailbox)
+ return d
+
+ def get_collection_by_docs(self, docs):
+ """
+ :rtype: MessageCollection
+ """
+ # get a collection of docs by a list of doc_id
+ # get.docs(...) --> it should be a generator. does it behave in the
+ # threadpool?
+ raise NotImplementedError()
+
+ def get_collection_by_tag(self, tag):
+ """
+ :rtype: MessageCollection
+ """
+ raise NotImplementedError()
+
+ # Session handling
+
+ def end_session(self):
+ self._teardown_sync_hooks()
+ self.session_ended = True
diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py
new file mode 100644
index 0000000..08e5f10
--- /dev/null
+++ b/src/leap/mail/mailbox_indexer.py
@@ -0,0 +1,324 @@
+# -*- coding: utf-8 -*-
+# mailbox_indexer.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Local tables to store the message Unique Identifiers for a given mailbox.
+"""
+import re
+import uuid
+
+from leap.mail.constants import METAMSGID_RE
+
+
+def _maybe_first_query_item(thing):
+ """
+ Return the first item the returned query result, or None
+ if empty.
+ """
+ try:
+ return thing[0][0]
+ except (TypeError, IndexError):
+ return None
+
+
+class WrongMetaDocIDError(Exception):
+ pass
+
+
+def sanitize(mailbox_uuid):
+ return mailbox_uuid.replace("-", "_")
+
+
+def check_good_uuid(mailbox_uuid):
+ """
+ Check that the passed mailbox identifier is a valid UUID.
+ :param mailbox_uuid: the uuid to check
+ :type mailbox_uuid: str
+ :return: None
+ :raises: AssertionError if a wrong uuid was passed.
+ """
+ try:
+ uuid.UUID(str(mailbox_uuid))
+ except (AttributeError, ValueError):
+ raise AssertionError(
+ "the mbox_id is not a valid uuid: %s" % mailbox_uuid)
+
+
+class MailboxIndexer(object):
+ """
+ This class contains the commands needed to create, modify and alter the
+ local-only UID tables for a given mailbox.
+
+ Its purpouse is to keep a local-only index with the messages in each
+ mailbox, mainly to satisfy the demands of the IMAP specification, but
+ useful too for any effective listing of the messages in a mailbox.
+
+ Since the incoming mail can be processed at any time in any replica, it's
+ preferred not to attempt to maintain a global chronological global index.
+
+ These indexes are Message Attributes needed for the IMAP specification (rfc
+ 3501), although they can be useful for other non-imap store
+ implementations.
+ """
+ # The uids are expected to be 32-bits values, but the ROWIDs in sqlite
+ # are 64-bit values. I *don't* think it really matters for any
+ # practical use, but it's good to remmeber we've got that difference going
+ # on.
+
+ store = None
+ table_preffix = "leapmail_uid_"
+
+ def __init__(self, store):
+ self.store = store
+
+ def _query(self, *args, **kw):
+ assert self.store is not None
+ return self.store.raw_sqlcipher_query(*args, **kw)
+
+ def _operation(self, *args, **kw):
+ assert self.store is not None
+ return self.store.raw_sqlcipher_operation(*args, **kw)
+
+ def create_table(self, mailbox_uuid):
+ """
+ Create the UID table for a given mailbox.
+ :param mailbox: the mailbox identifier.
+ :type mailbox: str
+ :rtype: Deferred
+ """
+ check_good_uuid(mailbox_uuid)
+ sql = ("CREATE TABLE if not exists {preffix}{name}( "
+ "uid INTEGER PRIMARY KEY AUTOINCREMENT, "
+ "hash TEXT UNIQUE NOT NULL)".format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid)))
+ return self._operation(sql)
+
+ def delete_table(self, mailbox_uuid):
+ """
+ Delete the UID table for a given mailbox.
+ :param mailbox: the mailbox name
+ :type mailbox: str
+ :rtype: Deferred
+ """
+ check_good_uuid(mailbox_uuid)
+ sql = ("DROP TABLE if exists {preffix}{name}".format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid)))
+ return self._operation(sql)
+
+ def insert_doc(self, mailbox_uuid, doc_id):
+ """
+ Insert the doc_id for a MetaMsg in the UID table for a given mailbox.
+
+ The doc_id must be in the format:
+
+ M-<mailbox>-<content-hash-of-the-message>
+
+ :param mailbox: the mailbox name
+ :type mailbox: str
+ :param doc_id: the doc_id for the MetaMsg
+ :type doc_id: str
+ :return: a deferred that will fire with the uid of the newly inserted
+ document.
+ :rtype: Deferred
+ """
+ check_good_uuid(mailbox_uuid)
+ assert doc_id
+ mailbox_uuid = mailbox_uuid.replace('-', '_')
+
+ if not re.findall(METAMSGID_RE.format(mbox_uuid=mailbox_uuid), doc_id):
+ raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id")
+
+ def get_rowid(result):
+ return _maybe_first_query_item(result)
+
+ sql = ("INSERT INTO {preffix}{name} VALUES ("
+ "NULL, ?)".format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid)))
+ values = (doc_id,)
+
+ sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} "
+ "LIMIT 1;").format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid))
+
+ d = self._operation(sql, values)
+ d.addCallback(lambda _: self._query(sql_last))
+ d.addCallback(get_rowid)
+ d.addErrback(lambda f: f.printTraceback())
+ return d
+
+ def delete_doc_by_uid(self, mailbox_uuid, uid):
+ """
+ Delete the entry for a MetaMsg in the UID table for a given mailbox.
+
+ :param mailbox_uuid: the mailbox uuid
+ :type mailbox: str
+ :param uid: the UID of the message.
+ :type uid: int
+ :rtype: Deferred
+ """
+ check_good_uuid(mailbox_uuid)
+ assert uid
+ sql = ("DELETE FROM {preffix}{name} "
+ "WHERE uid=?".format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid)))
+ values = (uid,)
+ return self._query(sql, values)
+
+ def delete_doc_by_hash(self, mailbox_uuid, doc_id):
+ """
+ Delete the entry for a MetaMsg in the UID table for a given mailbox.
+
+ The doc_id must be in the format:
+
+ M-<mailbox_uuid>-<content-hash-of-the-message>
+
+ :param mailbox_uuid: the mailbox uuid
+ :type mailbox: str
+ :param doc_id: the doc_id for the MetaMsg
+ :type doc_id: str
+ :return: a deferred that will fire when the deletion has succed.
+ :rtype: Deferred
+ """
+ check_good_uuid(mailbox_uuid)
+ assert doc_id
+ sql = ("DELETE FROM {preffix}{name} "
+ "WHERE hash=?".format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid)))
+ values = (doc_id,)
+ return self._query(sql, values)
+
+ def get_doc_id_from_uid(self, mailbox_uuid, uid):
+ """
+ Get the doc_id for a MetaMsg in the UID table for a given mailbox.
+
+ :param mailbox_uuid: the mailbox uuid
+ :type mailbox: str
+ :param uid: the uid for the MetaMsg for this mailbox
+ :type uid: int
+ :rtype: Deferred
+ """
+ check_good_uuid(mailbox_uuid)
+ mailbox_uuid = mailbox_uuid.replace('-', '_')
+
+ def get_hash(result):
+ return _maybe_first_query_item(result)
+
+ sql = ("SELECT hash from {preffix}{name} "
+ "WHERE uid=?".format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid)))
+ values = (uid,)
+ d = self._query(sql, values)
+ d.addCallback(get_hash)
+ return d
+
+ def get_uid_from_doc_id(self, mailbox_uuid, doc_id):
+ check_good_uuid(mailbox_uuid)
+ mailbox_uuid = mailbox_uuid.replace('-', '_')
+
+ def get_uid(result):
+ return _maybe_first_query_item(result)
+
+ sql = ("SELECT uid from {preffix}{name} "
+ "WHERE hash=?".format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid)))
+ values = (doc_id,)
+ d = self._query(sql, values)
+ d.addCallback(get_uid)
+ return d
+
+ def get_doc_ids_from_uids(self, mailbox_uuid, uids):
+ # For IMAP relative numbering /sequences.
+ # XXX dereference the range (n,*)
+ raise NotImplementedError()
+
+ def count(self, mailbox_uuid):
+ """
+ Get the number of entries in the UID table for a given mailbox.
+
+ :param mailbox_uuid: the mailbox uuid
+ :type mailbox_uuid: str
+ :return: a deferred that will fire with an integer returning the count.
+ :rtype: Deferred
+ """
+ check_good_uuid(mailbox_uuid)
+
+ def get_count(result):
+ return _maybe_first_query_item(result)
+
+ sql = ("SELECT Count(*) FROM {preffix}{name};".format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid)))
+ d = self._query(sql)
+ d.addCallback(get_count)
+ d.addErrback(lambda _: 0)
+ return d
+
+ def get_next_uid(self, mailbox_uuid):
+ """
+ Get the next integer beyond the highest UID count for a given mailbox.
+
+ This is expected by the IMAP implementation. There are no guarantees
+ that a document to be inserted in the future gets the returned UID: the
+ only thing that can be assured is that it will be equal or greater than
+ the value returned.
+
+ :param mailbox_uuid: the mailbox uuid
+ :type mailbox: str
+ :return: a deferred that will fire with an integer returning the next
+ uid.
+ :rtype: Deferred
+ """
+ check_good_uuid(mailbox_uuid)
+ d = self.get_last_uid(mailbox_uuid)
+ d.addCallback(lambda uid: uid + 1)
+ return d
+
+ def get_last_uid(self, mailbox_uuid):
+ """
+ Get the highest UID for a given mailbox.
+ """
+ check_good_uuid(mailbox_uuid)
+ sql = ("SELECT MAX(rowid) FROM {preffix}{name} "
+ "LIMIT 1;").format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid))
+
+ def getit(result):
+ rowid = _maybe_first_query_item(result)
+ if not rowid:
+ rowid = 0
+ return rowid
+
+ d = self._query(sql)
+ d.addCallback(getit)
+ return d
+
+ def all_uid_iter(self, mailbox_uuid):
+ """
+ Get a sequence of all the uids in this mailbox.
+
+ :param mailbox_uuid: the mailbox uuid
+ :type mailbox_uuid: str
+ """
+ check_good_uuid(mailbox_uuid)
+
+ sql = ("SELECT uid from {preffix}{name} ").format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid))
+
+ def get_results(result):
+ return [x[0] for x in result]
+
+ d = self._query(sql)
+ d.addCallback(get_results)
+ return d
diff --git a/src/leap/mail/messageflow.py b/src/leap/mail/messageflow.py
deleted file mode 100644
index c8f224c..0000000
--- a/src/leap/mail/messageflow.py
+++ /dev/null
@@ -1,200 +0,0 @@
-# -*- coding: utf-8 -*-
-# messageflow.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-Message Producers and Consumers for flow control.
-"""
-import Queue
-
-from twisted.internet.task import LoopingCall
-
-from zope.interface import Interface, implements
-
-
-class IMessageConsumer(Interface):
- """
- I consume messages from a queue.
- """
-
- def consume(self, queue):
- """
- Consumes the passed item.
-
- :param item: a queue where we put the object to be consumed.
- :type item: object
- """
- # TODO we could add an optional type to be passed
- # for doing type check.
-
- # TODO in case of errors, we could return the object to
- # the queue, maybe wrapped in an object with a retries attribute.
-
-
-class IMessageProducer(Interface):
- """
- I produce messages and put them in a store to be consumed by other
- entities.
- """
-
- def push(self, item, state=None):
- """
- Push a new item in the queue.
- """
-
- def start(self):
- """
- Start producing items.
- """
-
- def stop(self):
- """
- Stop producing items.
- """
-
- def flush(self):
- """
- Flush queued messages to consumer.
- """
-
-
-class DummyMsgConsumer(object):
-
- implements(IMessageConsumer)
-
- def consume(self, queue):
- """
- Just prints the passed item.
- """
- if not queue.empty():
- print "got item %s" % queue.get()
-
-
-class MessageProducer(object):
- """
- A Producer class that we can use to temporarily buffer the production
- of messages so that different objects can consume them.
-
- This is useful for serializing the consumption of the messages stream
- in the case of an slow resource (db), or for returning early from a
- deferred chain and leave further processing detached from the calling loop,
- as in the case of smtp.
- """
- implements(IMessageProducer)
-
- # TODO this can be seen as a first step towards properly implementing
- # components that implement IPushProducer / IConsumer interfaces.
- # However, I need to think more about how to pause the streaming.
- # In any case, the differential rate between message production
- # and consumption is not likely (?) to consume huge amounts of memory in
- # our current settings, so the need to pause the stream is not urgent now.
-
- # TODO use enum
- STATE_NEW = 1
- STATE_DIRTY = 2
-
- def __init__(self, consumer, queue=Queue.Queue, period=1):
- """
- Initializes the MessageProducer
-
- :param consumer: an instance of a IMessageConsumer that will consume
- the new messages.
- :param queue: any queue implementation to be used as the temporary
- buffer for new items. Default is a FIFO Queue.
- :param period: the period to check for new items, in seconds.
- """
- # XXX should assert it implements IConsumer / IMailConsumer
- # it should implement a `consume` method
- self._consumer = consumer
-
- self._queue_new = queue()
- self._queue_dirty = queue()
- self._period = period
-
- self._loop = LoopingCall(self._check_for_new)
-
- # private methods
-
- def _check_for_new(self):
- """
- Check for new items in the internal queue, and calls the consume
- method in the consumer.
-
- If the queue is found empty, the loop is stopped. It will be started
- again after the addition of new items.
- """
- self._consumer.consume((self._queue_new, self._queue_dirty))
- if self.is_queue_empty():
- self.stop()
-
- def is_queue_empty(self):
- """
- Return True if queue is empty, False otherwise.
- """
- new = self._queue_new
- dirty = self._queue_dirty
- return new.empty() and dirty.empty()
-
- # public methods: IMessageProducer
-
- def push(self, item, state=None):
- """
- Push a new item in the queue.
-
- If the queue was empty, we will start the loop again.
- """
- # XXX this might raise if the queue does not accept any new
- # items. what to do then?
- queue = self._queue_new
-
- if state == self.STATE_NEW:
- queue = self._queue_new
- if state == self.STATE_DIRTY:
- queue = self._queue_dirty
-
- queue.put(item)
- self.start()
-
- def start(self):
- """
- Start polling for new items.
- """
- if not self._loop.running:
- self._loop.start(self._period, now=True)
-
- def stop(self):
- """
- Stop polling for new items.
- """
- if self._loop.running:
- self._loop.stop()
-
- def flush(self):
- """
- Flush queued messages to consumer.
- """
- self._check_for_new()
-
-
-if __name__ == "__main__":
- from twisted.internet import reactor
- producer = MessageProducer(DummyMsgConsumer())
- producer.start()
-
- for delay, item in ((2, 1), (3, 2), (4, 3),
- (6, 4), (7, 5), (8, 6), (8.2, 7),
- (15, 'a'), (16, 'b'), (17, 'c')):
- reactor.callLater(delay, producer.put, item)
- reactor.run()
diff --git a/src/leap/mail/outgoing/__init__.py b/src/leap/mail/outgoing/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/leap/mail/outgoing/__init__.py
diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py
new file mode 100644
index 0000000..838a908
--- /dev/null
+++ b/src/leap/mail/outgoing/service.py
@@ -0,0 +1,467 @@
+# -*- coding: utf-8 -*-
+# outgoing/service.py
+# Copyright (C) 2013-2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+import re
+from StringIO import StringIO
+from copy import deepcopy
+from email.parser import Parser
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+from OpenSSL import SSL
+
+from twisted.mail import smtp
+from twisted.internet import reactor
+from twisted.internet import defer
+from twisted.protocols.amp import ssl
+from twisted.python import log
+
+from leap.common.check import leap_assert_type, leap_assert
+from leap.common.events import emit, catalog
+from leap.keymanager.openpgp import OpenPGPKey
+from leap.keymanager.errors import KeyNotFound, KeyAddressMismatch
+from leap.mail import __version__
+from leap.mail.utils import validate_address
+from leap.mail.smtp.rfc3156 import MultipartEncrypted
+from leap.mail.smtp.rfc3156 import MultipartSigned
+from leap.mail.smtp.rfc3156 import encode_base64_rec
+from leap.mail.smtp.rfc3156 import RFC3156CompliantGenerator
+from leap.mail.smtp.rfc3156 import PGPSignature
+from leap.mail.smtp.rfc3156 import PGPEncrypted
+
+# TODO
+# [ ] rename this module to something else, service should be the implementor
+# of IService
+
+
+class SSLContextFactory(ssl.ClientContextFactory):
+ def __init__(self, cert, key):
+ self.cert = cert
+ self.key = key
+
+ def getContext(self):
+ # FIXME -- we should use sslv23 to allow for tlsv1.2
+ # and, if possible, explicitely disable sslv3 clientside.
+ # Servers should avoid sslv3
+ self.method = SSL.TLSv1_METHOD # SSLv23_METHOD
+ ctx = ssl.ClientContextFactory.getContext(self)
+ ctx.use_certificate_file(self.cert)
+ ctx.use_privatekey_file(self.key)
+ return ctx
+
+
+class OutgoingMail:
+ """
+ A service for handling encrypted outgoing mail.
+ """
+
+ def __init__(self, from_address, keymanager, cert, key, host, port):
+ """
+ Initialize the mail service.
+
+ :param from_address: The sender address.
+ :type from_address: str
+ :param keymanager: A KeyManager for retrieving recipient's keys.
+ :type keymanager: leap.common.keymanager.KeyManager
+ :param cert: The client certificate for SSL authentication.
+ :type cert: str
+ :param key: The client private key for SSL authentication.
+ :type key: str
+ :param host: The hostname of the remote SMTP server.
+ :type host: str
+ :param port: The port of the remote SMTP server.
+ :type port: int
+ """
+
+ # assert params
+ leap_assert_type(from_address, str)
+ leap_assert('@' in from_address)
+
+ # XXX it can be a zope.proxy too
+ # leap_assert_type(keymanager, KeyManager)
+
+ leap_assert_type(host, str)
+ leap_assert(host != '')
+ leap_assert_type(port, int)
+ leap_assert(port is not 0)
+ leap_assert_type(cert, unicode)
+ leap_assert(cert != '')
+ leap_assert_type(key, unicode)
+ leap_assert(key != '')
+
+ self._port = port
+ self._host = host
+ self._key = key
+ self._cert = cert
+ self._from_address = from_address
+ self._keymanager = keymanager
+
+ def send_message(self, raw, recipient):
+ """
+ Sends a message to a recipient. Maybe encrypts and signs.
+
+ :param raw: The raw message
+ :type raw: str
+ :param recipient: The recipient for the message
+ :type recipient: smtp.User
+ :return: a deferred which delivers the message when fired
+ """
+ d = self._maybe_encrypt_and_sign(raw, recipient)
+ d.addCallback(self._route_msg)
+ d.addErrback(self.sendError)
+ return d
+
+ def sendSuccess(self, smtp_sender_result):
+ """
+ Callback for a successful send.
+
+ :param smtp_sender_result: The result from the ESMTPSender from
+ _route_msg
+ :type smtp_sender_result: tuple(int, list(tuple))
+ """
+ dest_addrstr = smtp_sender_result[1][0][0]
+ log.msg('Message sent to %s' % dest_addrstr)
+ emit(catalog.SMTP_SEND_MESSAGE_SUCCESS, dest_addrstr)
+
+ def sendError(self, failure):
+ """
+ Callback for an unsuccessfull send.
+
+ :param e: The result from the last errback.
+ :type e: anything
+ """
+ # XXX: need to get the address from the exception to send signal
+ # emit(catalog.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr)
+ err = failure.value
+ log.err(err)
+ raise err
+
+ def _route_msg(self, encrypt_and_sign_result):
+ """
+ Sends the msg using the ESMTPSenderFactory.
+
+ :param encrypt_and_sign_result: A tuple containing the 'maybe'
+ encrypted message and the recipient
+ :type encrypt_and_sign_result: tuple
+ """
+ message, recipient = encrypt_and_sign_result
+ log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port))
+ msg = message.as_string(False)
+
+ # we construct a defer to pass to the ESMTPSenderFactory
+ d = defer.Deferred()
+ d.addCallbacks(self.sendSuccess, self.sendError)
+ # we don't pass an ssl context factory to the ESMTPSenderFactory
+ # because ssl will be handled by reactor.connectSSL() below.
+ factory = smtp.ESMTPSenderFactory(
+ "", # username is blank, no client auth here
+ "", # password is blank, no client auth here
+ self._from_address,
+ recipient.dest.addrstr,
+ StringIO(msg),
+ d,
+ heloFallback=True,
+ requireAuthentication=False,
+ requireTransportSecurity=True)
+ factory.domain = __version__
+ emit(catalog.SMTP_SEND_MESSAGE_START, recipient.dest.addrstr)
+ reactor.connectSSL(
+ self._host, self._port, factory,
+ contextFactory=SSLContextFactory(self._cert, self._key))
+
+ def _maybe_encrypt_and_sign(self, raw, recipient):
+ """
+ Attempt to encrypt and sign the outgoing message.
+
+ The behaviour of this method depends on:
+
+ 1. the original message's content-type, and
+ 2. the availability of the recipient's public key.
+
+ If the original message's content-type is "multipart/encrypted", then
+ the original message is not altered. For any other content-type, the
+ method attempts to fetch the recipient's public key. If the
+ recipient's public key is available, the message is encrypted and
+ signed; otherwise it is only signed.
+
+ Note that, if the C{encrypted_only} configuration is set to True and
+ the recipient's public key is not available, then the recipient
+ address would have been rejected in SMTPDelivery.validateTo().
+
+ The following table summarizes the overall behaviour of the gateway:
+
+ +---------------------------------------------------+----------------+
+ | content-type | rcpt pubkey | enforce encr. | action |
+ +---------------------+-------------+---------------+----------------+
+ | multipart/encrypted | any | any | pass |
+ | other | available | any | encrypt + sign |
+ | other | unavailable | yes | reject |
+ | other | unavailable | no | sign |
+ +---------------------+-------------+---------------+----------------+
+
+ :param raw: The raw message
+ :type raw: str
+ :param recipient: The recipient for the message
+ :type: recipient: smtp.User
+
+ :return: A Deferred that will be fired with a MIMEMultipart message
+ and the original recipient Message
+ :rtype: Deferred
+ """
+ # pass if the original message's content-type is "multipart/encrypted"
+ origmsg = Parser().parsestr(raw)
+
+ if origmsg.get_content_type() == 'multipart/encrypted':
+ return defer.success((origmsg, recipient))
+
+ from_address = validate_address(self._from_address)
+ username, domain = from_address.split('@')
+ to_address = validate_address(recipient.dest.addrstr)
+
+ def maybe_encrypt_and_sign(message):
+ d = self._encrypt_and_sign(message, to_address, from_address)
+ d.addCallbacks(signal_encrypt_sign,
+ if_key_not_found_send_unencrypted,
+ errbackArgs=(message,))
+ return d
+
+ def signal_encrypt_sign(newmsg):
+ emit(catalog.SMTP_END_ENCRYPT_AND_SIGN,
+ "%s,%s" % (self._from_address, to_address))
+ return newmsg, recipient
+
+ def if_key_not_found_send_unencrypted(failure, message):
+ failure.trap(KeyNotFound, KeyAddressMismatch)
+
+ log.msg('Will send unencrypted message to %s.' % to_address)
+ emit(catalog.SMTP_START_SIGN, self._from_address)
+ d = self._sign(message, from_address)
+ d.addCallback(signal_sign)
+ return d
+
+ def signal_sign(newmsg):
+ emit(catalog.SMTP_END_SIGN, self._from_address)
+ return newmsg, recipient
+
+ log.msg("Will encrypt the message with %s and sign with %s."
+ % (to_address, from_address))
+ emit(catalog.SMTP_START_ENCRYPT_AND_SIGN,
+ "%s,%s" % (self._from_address, to_address))
+ d = self._maybe_attach_key(origmsg, from_address, to_address)
+ d.addCallback(maybe_encrypt_and_sign)
+ return d
+
+ def _maybe_attach_key(self, origmsg, from_address, to_address):
+ filename = "%s-email-key.asc" % (from_address,)
+
+ def attach_if_address_hasnt_encrypted(to_key):
+ # if the sign_used flag is true that means that we got an encrypted
+ # email from this address, because we conly check signatures on
+ # encrypted emails. In this case we don't attach.
+ # XXX: this might not be true some time in the future
+ if to_key.sign_used:
+ return origmsg
+ return get_key_and_attach(None)
+
+ def get_key_and_attach(_):
+ d = self._keymanager.get_key(from_address, OpenPGPKey,
+ fetch_remote=False)
+ d.addCallback(attach_key)
+ return d
+
+ def attach_key(from_key):
+ msg = origmsg
+ if not origmsg.is_multipart():
+ msg = MIMEMultipart()
+ for h, v in origmsg.items():
+ msg.add_header(h, v)
+ msg.attach(MIMEText(origmsg.get_payload()))
+
+ keymsg = MIMEApplication(from_key.key_data, _subtype='pgp-keys',
+ _encoder=lambda x: x)
+ keymsg.add_header('content-disposition', 'attachment',
+ filename=filename)
+ msg.attach(keymsg)
+ return msg
+
+ d = self._keymanager.get_key(to_address, OpenPGPKey,
+ fetch_remote=False)
+ d.addCallbacks(attach_if_address_hasnt_encrypted, get_key_and_attach)
+ d.addErrback(lambda _: origmsg)
+ return d
+
+ def _encrypt_and_sign(self, origmsg, encrypt_address, sign_address):
+ """
+ Create an RFC 3156 compliang PGP encrypted and signed message using
+ C{encrypt_address} to encrypt and C{sign_address} to sign.
+
+ :param origmsg: The original message
+ :type origmsg: email.message.Message
+ :param encrypt_address: The address used to encrypt the message.
+ :type encrypt_address: str
+ :param sign_address: The address used to sign the message.
+ :type sign_address: str
+
+ :return: A Deferred with the MultipartEncrypted message
+ :rtype: Deferred
+ """
+ # create new multipart/encrypted message with 'pgp-encrypted' protocol
+
+ def encrypt(res):
+ newmsg, origmsg = res
+ d = self._keymanager.encrypt(
+ origmsg.as_string(unixfrom=False),
+ encrypt_address, OpenPGPKey, sign=sign_address)
+ d.addCallback(lambda encstr: (newmsg, encstr))
+ return d
+
+ def create_encrypted_message(res):
+ newmsg, encstr = res
+ encmsg = MIMEApplication(
+ encstr, _subtype='octet-stream', _encoder=lambda x: x)
+ encmsg.add_header('content-disposition', 'attachment',
+ filename='msg.asc')
+ # create meta message
+ metamsg = PGPEncrypted()
+ metamsg.add_header('Content-Disposition', 'attachment')
+ # attach pgp message parts to new message
+ newmsg.attach(metamsg)
+ newmsg.attach(encmsg)
+ return newmsg
+
+ d = self._fix_headers(
+ origmsg,
+ MultipartEncrypted('application/pgp-encrypted'),
+ sign_address)
+ d.addCallback(encrypt)
+ d.addCallback(create_encrypted_message)
+ return d
+
+ def _sign(self, origmsg, sign_address):
+ """
+ Create an RFC 3156 compliant PGP signed MIME message using
+ C{sign_address}.
+
+ :param origmsg: The original message
+ :type origmsg: email.message.Message
+ :param sign_address: The address used to sign the message.
+ :type sign_address: str
+
+ :return: A Deferred with the MultipartSigned message.
+ :rtype: Deferred
+ """
+ # apply base64 content-transfer-encoding
+ encode_base64_rec(origmsg)
+ # get message text with headers and replace \n for \r\n
+ fp = StringIO()
+ g = RFC3156CompliantGenerator(
+ fp, mangle_from_=False, maxheaderlen=76)
+ g.flatten(origmsg)
+ msgtext = re.sub('\r?\n', '\r\n', fp.getvalue())
+ # make sure signed message ends with \r\n as per OpenPGP stantard.
+ if origmsg.is_multipart():
+ if not msgtext.endswith("\r\n"):
+ msgtext += "\r\n"
+
+ def create_signed_message(res):
+ (msg, _), signature = res
+ sigmsg = PGPSignature(signature)
+ # attach original message and signature to new message
+ msg.attach(origmsg)
+ msg.attach(sigmsg)
+ return msg
+
+ dh = self._fix_headers(
+ origmsg,
+ MultipartSigned('application/pgp-signature', 'pgp-sha512'),
+ sign_address)
+ ds = self._keymanager.sign(
+ msgtext, sign_address, OpenPGPKey, digest_algo='SHA512',
+ clearsign=False, detach=True, binary=False)
+ d = defer.gatherResults([dh, ds])
+ d.addCallback(create_signed_message)
+ return d
+
+ def _fix_headers(self, msg, newmsg, sign_address):
+ """
+ Move some headers from C{origmsg} to C{newmsg}, delete unwanted
+ headers from C{origmsg} and add new headers to C{newms}.
+
+ Outgoing messages are either encrypted and signed or just signed
+ before being sent. Because of that, they are packed inside new
+ messages and some manipulation has to be made on their headers.
+
+ Allowed headers for passing through:
+
+ - From
+ - Date
+ - To
+ - Subject
+ - Reply-To
+ - References
+ - In-Reply-To
+ - Cc
+
+ Headers to be added:
+
+ - Message-ID (i.e. should not use origmsg's Message-Id)
+ - Received (this is added automatically by twisted smtp API)
+ - OpenPGP (see #4447)
+
+ Headers to be deleted:
+
+ - User-Agent
+
+ :param msg: The original message.
+ :type msg: email.message.Message
+ :param newmsg: The new message being created.
+ :type newmsg: email.message.Message
+ :param sign_address: The address used to sign C{newmsg}
+ :type sign_address: str
+
+ :return: A Deferred with a touple:
+ (new Message with the unencrypted headers,
+ original Message with headers removed)
+ :rtype: Deferred
+ """
+ origmsg = deepcopy(msg)
+ # move headers from origmsg to newmsg
+ headers = origmsg.items()
+ passthrough = [
+ 'from', 'date', 'to', 'subject', 'reply-to', 'references',
+ 'in-reply-to', 'cc'
+ ]
+ headers = filter(lambda x: x[0].lower() in passthrough, headers)
+ for hkey, hval in headers:
+ newmsg.add_header(hkey, hval)
+ del (origmsg[hkey])
+ # add a new message-id to newmsg
+ newmsg.add_header('Message-Id', smtp.messageid())
+ # delete user-agent from origmsg
+ del (origmsg['user-agent'])
+
+ def add_openpgp_header(signkey):
+ username, domain = sign_address.split('@')
+ newmsg.add_header(
+ 'OpenPGP', 'id=%s' % signkey.key_id,
+ url='https://%s/key/%s' % (domain, username),
+ preference='signencrypt')
+ return newmsg, origmsg
+
+ d = self._keymanager.get_key(sign_address, OpenPGPKey, private=True)
+ d.addCallback(add_openpgp_header)
+ return d
diff --git a/src/leap/mail/outgoing/tests/__init__.py b/src/leap/mail/outgoing/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/leap/mail/outgoing/tests/__init__.py
diff --git a/src/leap/mail/outgoing/tests/test_outgoing.py b/src/leap/mail/outgoing/tests/test_outgoing.py
new file mode 100644
index 0000000..2376da9
--- /dev/null
+++ b/src/leap/mail/outgoing/tests/test_outgoing.py
@@ -0,0 +1,256 @@
+# -*- coding: utf-8 -*-
+# test_gateway.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+SMTP gateway tests.
+"""
+
+import re
+from StringIO import StringIO
+from email.parser import Parser
+from datetime import datetime
+from twisted.internet.defer import fail
+from twisted.mail.smtp import User
+
+from mock import Mock
+
+from leap.mail.smtp.gateway import SMTPFactory
+from leap.mail.smtp.rfc3156 import RFC3156CompliantGenerator
+from leap.mail.outgoing.service import OutgoingMail
+from leap.mail.tests import (
+ TestCaseWithKeyManager,
+ ADDRESS,
+ ADDRESS_2,
+ PUBLIC_KEY_2,
+)
+from leap.keymanager import openpgp, errors
+
+
+BEGIN_PUBLIC_KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----"
+
+
+class TestOutgoingMail(TestCaseWithKeyManager):
+ EMAIL_DATA = ['HELO gateway.leap.se',
+ 'MAIL FROM: <%s>' % ADDRESS_2,
+ 'RCPT TO: <%s>' % ADDRESS,
+ 'DATA',
+ 'From: User <%s>' % ADDRESS_2,
+ 'To: Leap <%s>' % ADDRESS,
+ 'Date: ' + datetime.now().strftime('%c'),
+ 'Subject: test message',
+ '',
+ 'This is a secret message.',
+ 'Yours,',
+ 'A.',
+ '',
+ '.',
+ 'QUIT']
+
+ def setUp(self):
+ self.lines = [line for line in self.EMAIL_DATA[4:12]]
+ self.lines.append('') # add a trailing newline
+ self.raw = '\r\n'.join(self.lines)
+ self.expected_body = '\r\n'.join(self.EMAIL_DATA[9:12]) + "\r\n"
+ self.fromAddr = ADDRESS_2
+
+ def init_outgoing_and_proto(_):
+ self.outgoing_mail = OutgoingMail(
+ self.fromAddr, self._km, self._config['cert'],
+ self._config['key'], self._config['host'],
+ self._config['port'])
+ self.proto = SMTPFactory(
+ u'anotheruser@leap.se',
+ self._km,
+ self._config['encrypted_only'],
+ self.outgoing_mail).buildProtocol(('127.0.0.1', 0))
+ self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS_2)
+
+ d = TestCaseWithKeyManager.setUp(self)
+ d.addCallback(init_outgoing_and_proto)
+ return d
+
+ def test_message_encrypt(self):
+ """
+ Test if message gets encrypted to destination email.
+ """
+ def check_decryption(res):
+ decrypted, _ = res
+ self.assertEqual(
+ '\n' + self.expected_body,
+ decrypted,
+ 'Decrypted text differs from plaintext.')
+
+ d = self._set_sign_used(ADDRESS)
+ d.addCallback(
+ lambda _:
+ self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest))
+ d.addCallback(self._assert_encrypted)
+ d.addCallback(lambda message: self._km.decrypt(
+ message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey))
+ d.addCallback(check_decryption)
+ return d
+
+ def test_message_encrypt_sign(self):
+ """
+ Test if message gets encrypted to destination email and signed with
+ sender key.
+ '"""
+ def check_decryption_and_verify(res):
+ decrypted, signkey = res
+ self.assertEqual(
+ '\n' + self.expected_body,
+ decrypted,
+ 'Decrypted text differs from plaintext.')
+ self.assertTrue(ADDRESS_2 in signkey.address,
+ "Verification failed")
+
+ d = self._set_sign_used(ADDRESS)
+ d.addCallback(
+ lambda _:
+ self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest))
+ d.addCallback(self._assert_encrypted)
+ d.addCallback(lambda message: self._km.decrypt(
+ message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey,
+ verify=ADDRESS_2))
+ d.addCallback(check_decryption_and_verify)
+ return d
+
+ def test_message_sign(self):
+ """
+ Test if message is signed with sender key.
+ """
+ # mock the key fetching
+ self._km._fetch_keys_from_server = Mock(
+ return_value=fail(errors.KeyNotFound()))
+ recipient = User('ihavenopubkey@nonleap.se',
+ 'gateway.leap.se', self.proto, ADDRESS)
+ self.outgoing_mail = OutgoingMail(
+ self.fromAddr, self._km, self._config['cert'], self._config['key'],
+ self._config['host'], self._config['port'])
+
+ def check_signed(res):
+ message, _ = res
+ self.assertTrue('Content-Type' in message)
+ self.assertEqual('multipart/signed', message.get_content_type())
+ self.assertEqual('application/pgp-signature',
+ message.get_param('protocol'))
+ self.assertEqual('pgp-sha512', message.get_param('micalg'))
+ # assert content of message
+ body = (message.get_payload(0)
+ .get_payload(0)
+ .get_payload(decode=True))
+ self.assertEqual(self.expected_body,
+ body)
+ # assert content of signature
+ self.assertTrue(
+ message.get_payload(1).get_payload().startswith(
+ '-----BEGIN PGP SIGNATURE-----\n'),
+ 'Message does not start with signature header.')
+ self.assertTrue(
+ message.get_payload(1).get_payload().endswith(
+ '-----END PGP SIGNATURE-----\n'),
+ 'Message does not end with signature footer.')
+ return message
+
+ def verify(message):
+ # replace EOL before verifying (according to rfc3156)
+ fp = StringIO()
+ g = RFC3156CompliantGenerator(
+ fp, mangle_from_=False, maxheaderlen=76)
+ g.flatten(message.get_payload(0))
+ signed_text = re.sub('\r?\n', '\r\n',
+ fp.getvalue())
+
+ def assert_verify(key):
+ self.assertTrue(ADDRESS_2 in key.address,
+ 'Signature could not be verified.')
+
+ d = self._km.verify(
+ signed_text, ADDRESS_2, openpgp.OpenPGPKey,
+ detached_sig=message.get_payload(1).get_payload())
+ d.addCallback(assert_verify)
+ return d
+
+ d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, recipient)
+ d.addCallback(check_signed)
+ d.addCallback(verify)
+ return d
+
+ def test_attach_key(self):
+ d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest)
+ d.addCallback(self._assert_encrypted)
+ d.addCallback(self._check_headers, self.lines[:4])
+ d.addCallback(lambda message: self._km.decrypt(
+ message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey))
+ d.addCallback(lambda (decrypted, _):
+ self._check_key_attachment(Parser().parsestr(decrypted)))
+ return d
+
+ def test_attach_key_not_known(self):
+ address = "someunknownaddress@somewhere.com"
+ lines = self.lines
+ lines[1] = "To: <%s>" % (address,)
+ raw = '\r\n'.join(lines)
+ dest = User(address, 'gateway.leap.se', self.proto, ADDRESS_2)
+
+ d = self.outgoing_mail._maybe_encrypt_and_sign(raw, dest)
+ d.addCallback(lambda (message, _):
+ self._check_headers(message, lines[:4]))
+ d.addCallback(self._check_key_attachment)
+ return d
+
+ def _check_headers(self, message, headers):
+ msgstr = message.as_string(unixfrom=False)
+ for header in headers:
+ self.assertTrue(header in msgstr,
+ "Missing header: %s" % (header,))
+ return message
+
+ def _check_key_attachment(self, message):
+ for payload in message.get_payload():
+ if payload.is_multipart():
+ return self._check_key_attachment(payload)
+ if 'application/pgp-keys' == payload.get_content_type():
+ keylines = PUBLIC_KEY_2.split('\n')
+ key = BEGIN_PUBLIC_KEY + '\n\n' + '\n'.join(keylines[4:-1])
+ self.assertTrue(key in payload.get_payload(decode=True),
+ "Key attachment don't match")
+ return
+ self.fail("No public key attachment found")
+
+ def _set_sign_used(self, address):
+ def set_sign(key):
+ key.sign_used = True
+ return self._km.put_key(key, address)
+
+ d = self._km.get_key(address, openpgp.OpenPGPKey, fetch_remote=False)
+ d.addCallback(set_sign)
+ return d
+
+ def _assert_encrypted(self, res):
+ message, _ = res
+ self.assertTrue('Content-Type' in message)
+ self.assertEqual('multipart/encrypted', message.get_content_type())
+ self.assertEqual('application/pgp-encrypted',
+ message.get_param('protocol'))
+ self.assertEqual(2, len(message.get_payload()))
+ self.assertEqual('application/pgp-encrypted',
+ message.get_payload(0).get_content_type())
+ self.assertEqual('application/octet-stream',
+ message.get_payload(1).get_content_type())
+ return message
diff --git a/src/leap/mail/plugins/__init__.py b/src/leap/mail/plugins/__init__.py
new file mode 100644
index 0000000..ddb8691
--- /dev/null
+++ b/src/leap/mail/plugins/__init__.py
@@ -0,0 +1,3 @@
+from twisted.plugin import pluginPackagePaths
+__path__.extend(pluginPackagePaths(__name__))
+__all__ = []
diff --git a/src/leap/mail/plugins/soledad_sync_hooks.py b/src/leap/mail/plugins/soledad_sync_hooks.py
new file mode 100644
index 0000000..9d48126
--- /dev/null
+++ b/src/leap/mail/plugins/soledad_sync_hooks.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# soledad_sync_hooks.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from leap.mail.sync_hooks import MailProcessingPostSyncHook
+post_sync_uid_reindexer = MailProcessingPostSyncHook()
diff --git a/src/leap/mail/smtp/README.rst b/src/leap/mail/smtp/README.rst
index f625441..1d3a903 100644
--- a/src/leap/mail/smtp/README.rst
+++ b/src/leap/mail/smtp/README.rst
@@ -1,18 +1,39 @@
Leap SMTP Gateway
=================
+The Bitmask Client runs a thin SMTP gateway on the user's device, which
+intends to encrypt and sign outgoing messages to achieve point to point
+encryption.
+
+The gateway is bound to localhost and the user's MUA should be configured to
+send messages to it. After doing its thing, the gateway will relay the
+messages to the remote SMTP server.
+
Outgoing mail workflow:
- * LEAP client runs a thin SMTP proxy on the user's device, bound to
- localhost.
- * User's MUA is configured outgoing SMTP to localhost.
- * When SMTP proxy receives an email from MUA:
- * SMTP proxy queries Key Manager for the user's private key and public
- keys of all recipients.
- * Message is signed by sender and encrypted to recipients.
- * If recipient's key is missing, email goes out in cleartext (unless
- user has configured option to send only encrypted email).
- * Finally, message is gatewayed to provider's SMTP server.
+ * SMTP gateway receives a message from the MUA.
+
+ * SMTP gateway queries Key Manager for the user's private key.
+
+ * For each recipient (including addresses in "To", "Cc" anc "Bcc" fields),
+ the following happens:
+
+ - The recipient's address is validated against RFC2822.
+
+ - An attempt is made to fetch the recipient's public PGP key.
+
+ - If key is not found:
+
+ - If the gateway is configured to only send encrypted messages the
+ recipient is rejected.
+
+ - Otherwise, the message is signed and sent as plain text.
+
+ - If the key is found, the message is encrypted to the recipient and
+ signed with the sender's private PGP key.
+
+ * Finally, one message for each recipient is gatewayed to provider's SMTP
+ server.
Running tests
diff --git a/src/leap/mail/smtp/__init__.py b/src/leap/mail/smtp/__init__.py
index bbd4064..2ff14d7 100644
--- a/src/leap/mail/smtp/__init__.py
+++ b/src/leap/mail/smtp/__init__.py
@@ -22,12 +22,13 @@ import logging
from twisted.internet import reactor
from twisted.internet.error import CannotListenError
+from leap.mail.outgoing.service import OutgoingMail
-logger = logging.getLogger(__name__)
-
-from leap.common.events import proto, signal
+from leap.common.events import emit, catalog
from leap.mail.smtp.gateway import SMTPFactory
+logger = logging.getLogger(__name__)
+
def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port,
smtp_cert, smtp_key, encrypted_only):
@@ -40,7 +41,7 @@ def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port,
:param port: The port in which to run the server.
:type port: int
:param userid: The user currently logged in
- :type userid: unicode
+ :type userid: str
:param keymanager: A Key Manager from where to get recipients' public
keys.
:type keymanager: leap.common.keymanager.KeyManager
@@ -59,16 +60,17 @@ def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port,
:returns: tuple of SMTPFactory, twisted.internet.tcp.Port
"""
# configure the use of this service with twistd
- factory = SMTPFactory(userid, keymanager, smtp_host, smtp_port, smtp_cert,
- smtp_key, encrypted_only)
+ outgoing_mail = OutgoingMail(
+ userid, keymanager, smtp_cert, smtp_key, smtp_host, smtp_port)
+ factory = SMTPFactory(userid, keymanager, encrypted_only, outgoing_mail)
try:
tport = reactor.listenTCP(port, factory, interface="localhost")
- signal(proto.SMTP_SERVICE_STARTED, str(port))
+ emit(catalog.SMTP_SERVICE_STARTED, str(port))
return factory, tport
except CannotListenError:
logger.error("STMP Service failed to start: "
"cannot listen in port %s" % port)
- signal(proto.SMTP_SERVICE_FAILED_TO_START, str(port))
+ emit(catalog.SMTP_SERVICE_FAILED_TO_START, str(port))
except Exception as exc:
logger.error("Unhandled error while launching smtp gateway service")
logger.exception(exc)
diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py
index 13d3bbf..7dae907 100644
--- a/src/leap/mail/smtp/gateway.py
+++ b/src/leap/mail/smtp/gateway.py
@@ -21,49 +21,34 @@ The following classes comprise the SMTP gateway service:
* SMTPFactory - A twisted.internet.protocol.ServerFactory that provides
the SMTPDelivery protocol.
+
* SMTPDelivery - A twisted.mail.smtp.IMessageDelivery implementation. It
knows how to validate sender and receiver of messages and it generates
an EncryptedMessage for each recipient.
- * SSLContextFactory - Contains the relevant ssl information for the
- connection.
+
* EncryptedMessage - An implementation of twisted.mail.smtp.IMessage that
knows how to encrypt/sign itself before sending.
-
-
"""
-import re
-from StringIO import StringIO
-from email.Header import Header
-from email.utils import parseaddr
-from email.parser import Parser
-from email.mime.application import MIMEApplication
from zope.interface import implements
-from OpenSSL import SSL
from twisted.mail import smtp
from twisted.internet.protocol import ServerFactory
-from twisted.internet import reactor, ssl
-from twisted.internet import defer
-from twisted.internet.threads import deferToThread
from twisted.python import log
-from leap.common.check import leap_assert, leap_assert_type
-from leap.common.events import proto, signal
-from leap.keymanager import KeyManager
+from email.Header import Header
+from leap.common.check import leap_assert_type
+from leap.common.events import emit, catalog
from leap.keymanager.openpgp import OpenPGPKey
from leap.keymanager.errors import KeyNotFound
-from leap.mail import __version__
+from leap.mail.utils import validate_address
+
from leap.mail.smtp.rfc3156 import (
- MultipartSigned,
- MultipartEncrypted,
- PGPEncrypted,
- PGPSignature,
RFC3156CompliantGenerator,
- encode_base64_rec,
)
# replace email generator with a RFC 3156 compliant one.
from email import generator
+
generator.Generator = RFC3156CompliantGenerator
@@ -74,31 +59,6 @@ generator.Generator = RFC3156CompliantGenerator
LOCAL_FQDN = "bitmask.local"
-def validate_address(address):
- """
- Validate C{address} as defined in RFC 2822.
-
- :param address: The address to be validated.
- :type address: str
-
- @return: A valid address.
- @rtype: str
-
- @raise smtp.SMTPBadRcpt: Raised if C{address} is invalid.
- """
- leap_assert_type(address, str)
- # in the following, the address is parsed as described in RFC 2822 and
- # ('', '') is returned if the parse fails.
- _, address = parseaddr(address)
- if address == '':
- raise smtp.SMTPBadRcpt(address)
- return address
-
-
-#
-# SMTPFactory
-#
-
class SMTPHeloLocalhost(smtp.SMTP):
"""
An SMTP class that ensures a proper FQDN
@@ -119,45 +79,26 @@ class SMTPFactory(ServerFactory):
"""
domain = LOCAL_FQDN
- def __init__(self, userid, keymanager, host, port, cert, key,
- encrypted_only):
+ def __init__(self, userid, keymanager, encrypted_only, outgoing_mail):
"""
Initialize the SMTP factory.
:param userid: The user currently logged in
:type userid: unicode
- :param keymanager: A KeyManager for retrieving recipient's keys.
- :type keymanager: leap.common.keymanager.KeyManager
- :param host: The hostname of the remote SMTP server.
- :type host: str
- :param port: The port of the remote SMTP server.
- :type port: int
- :param cert: The client certificate for authentication.
- :type cert: str
- :param key: The client key for authentication.
- :type key: str
+ :param keymanager: A Key Manager from where to get recipients' public
+ keys.
:param encrypted_only: Whether the SMTP gateway should send unencrypted
mail or not.
:type encrypted_only: bool
+ :param outgoing_mail: The outgoing mail to send the message
+ :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail
"""
- # assert params
- leap_assert_type(keymanager, KeyManager)
- leap_assert_type(host, str)
- leap_assert(host != '')
- leap_assert_type(port, int)
- leap_assert(port is not 0)
- leap_assert_type(cert, unicode)
- leap_assert(cert != '')
- leap_assert_type(key, unicode)
- leap_assert(key != '')
+
leap_assert_type(encrypted_only, bool)
# and store them
self._userid = userid
self._km = keymanager
- self._host = host
- self._port = port
- self._cert = cert
- self._key = key
+ self._outgoing_mail = outgoing_mail
self._encrypted_only = encrypted_only
def buildProtocol(self, addr):
@@ -170,9 +111,10 @@ class SMTPFactory(ServerFactory):
@return: The protocol.
@rtype: SMTPDelivery
"""
- smtpProtocol = SMTPHeloLocalhost(SMTPDelivery(
- self._userid, self._km, self._host, self._port, self._cert,
- self._key, self._encrypted_only))
+ smtpProtocol = SMTPHeloLocalhost(
+ SMTPDelivery(
+ self._userid, self._km, self._encrypted_only,
+ self._outgoing_mail))
smtpProtocol.factory = self
return smtpProtocol
@@ -188,33 +130,23 @@ class SMTPDelivery(object):
implements(smtp.IMessageDelivery)
- def __init__(self, userid, keymanager, host, port, cert, key,
- encrypted_only):
+ def __init__(self, userid, keymanager, encrypted_only, outgoing_mail):
"""
Initialize the SMTP delivery object.
:param userid: The user currently logged in
:type userid: unicode
- :param keymanager: A KeyManager for retrieving recipient's keys.
- :type keymanager: leap.common.keymanager.KeyManager
- :param host: The hostname of the remote SMTP server.
- :type host: str
- :param port: The port of the remote SMTP server.
- :type port: int
- :param cert: The client certificate for authentication.
- :type cert: str
- :param key: The client key for authentication.
- :type key: str
+ :param keymanager: A Key Manager from where to get recipients' public
+ keys.
:param encrypted_only: Whether the SMTP gateway should send unencrypted
mail or not.
:type encrypted_only: bool
+ :param outgoing_mail: The outgoing mail to send the message
+ :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail
"""
self._userid = userid
+ self._outgoing_mail = outgoing_mail
self._km = keymanager
- self._host = host
- self._port = port
- self._cert = cert
- self._key = key
self._encrypted_only = encrypted_only
self._origin = None
@@ -242,47 +174,58 @@ class SMTPDelivery(object):
def validateTo(self, user):
"""
- Validate the address of C{user}, a recipient of the message.
+ Validate the address of a recipient of the message, possibly
+ rejecting it if the recipient key is not available.
- This method is called once for each recipient and validates the
- C{user}'s address against the RFC 2822 definition. If the
- configuration option ENCRYPTED_ONLY_KEY is True, it also asserts the
- existence of the user's key.
+ This method is called once for each recipient, i.e. for each SMTP
+ protocol line beginning with "RCPT TO:", which includes all addresses
+ in "To", "Cc" and "Bcc" MUA fields.
- In the end, it returns an encrypted message object that is able to
- send itself to the C{user}'s address.
+ The recipient's address is validated against the RFC 2822 definition.
+ If self._encrypted_only is True and no key is found for a recipient,
+ then that recipient is rejected.
+
+ The method returns an encrypted message object that is able to send
+ itself to the user's address.
:param user: The user whose address we wish to validate.
:type: twisted.mail.smtp.User
- @return: A Deferred which becomes, or a callable which takes no
- arguments and returns an object implementing IMessage. This will
- be called and the returned object used to deliver the message when
- it arrives.
+ @return: A callable which takes no arguments and returns an
+ encryptedMessage.
@rtype: no-argument callable
@raise SMTPBadRcpt: Raised if messages to the address are not to be
- accepted.
+ accepted.
"""
# try to find recipient's public key
- try:
- address = validate_address(user.dest.addrstr)
- # verify if recipient key is available in keyring
- self._km.get_key(address, OpenPGPKey) # might raise KeyNotFound
+ address = validate_address(user.dest.addrstr)
+
+ # verify if recipient key is available in keyring
+ def found(_):
log.msg("Accepting mail for %s..." % user.dest.addrstr)
- signal(proto.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, user.dest.addrstr)
- except KeyNotFound:
- # if key was not found, check config to see if will send anyway.
+ emit(catalog.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, user.dest.addrstr)
+
+ def not_found(failure):
+ failure.trap(KeyNotFound)
+
+ # if key was not found, check config to see if will send anyway
if self._encrypted_only:
- signal(proto.SMTP_RECIPIENT_REJECTED, user.dest.addrstr)
+ emit(catalog.SMTP_RECIPIENT_REJECTED, user.dest.addrstr)
raise smtp.SMTPBadRcpt(user.dest.addrstr)
log.msg("Warning: will send an unencrypted message (because "
"encrypted_only' is set to False).")
- signal(
- proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, user.dest.addrstr)
- return lambda: EncryptedMessage(
- self._origin, user, self._km, self._host, self._port, self._cert,
- self._key)
+ emit(
+ catalog.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED,
+ user.dest.addrstr)
+
+ def encrypt_func(_):
+ return lambda: EncryptedMessage(user, self._outgoing_mail)
+
+ d = self._km.get_key(address, OpenPGPKey)
+ d.addCallbacks(found, not_found)
+ d.addCallback(encrypt_func)
+ return d
def validateFrom(self, helo, origin):
"""
@@ -314,19 +257,6 @@ class SMTPDelivery(object):
# EncryptedMessage
#
-class SSLContextFactory(ssl.ClientContextFactory):
- def __init__(self, cert, key):
- self.cert = cert
- self.key = key
-
- def getContext(self):
- self.method = SSL.TLSv1_METHOD # SSLv23_METHOD
- ctx = ssl.ClientContextFactory.getContext(self)
- ctx.use_certificate_file(self.cert)
- ctx.use_privatekey_file(self.key)
- return ctx
-
-
class EncryptedMessage(object):
"""
Receive plaintext from client, encrypt it and send message to a
@@ -334,44 +264,21 @@ class EncryptedMessage(object):
"""
implements(smtp.IMessage)
- FOOTER_STRING = "I prefer encrypted email"
-
- def __init__(self, fromAddress, user, keymanager, host, port, cert, key):
+ def __init__(self, user, outgoing_mail):
"""
Initialize the encrypted message.
- :param fromAddress: The address of the sender.
- :type fromAddress: twisted.mail.smtp.Address
:param user: The recipient of this message.
:type user: twisted.mail.smtp.User
- :param keymanager: A KeyManager for retrieving recipient's keys.
- :type keymanager: leap.common.keymanager.KeyManager
- :param host: The hostname of the remote SMTP server.
- :type host: str
- :param port: The port of the remote SMTP server.
- :type port: int
- :param cert: The client certificate for authentication.
- :type cert: str
- :param key: The client key for authentication.
- :type key: str
+ :param outgoing_mail: The outgoing mail to send the message
+ :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail
"""
# assert params
leap_assert_type(user, smtp.User)
- leap_assert_type(keymanager, KeyManager)
- # and store them
- self._fromAddress = fromAddress
- self._user = user
- self._km = keymanager
- self._host = host
- self._port = port
- self._cert = cert
- self._key = key
- # initialize list for message's lines
- self.lines = []
- #
- # methods from smtp.IMessage
- #
+ self._user = user
+ self._lines = []
+ self._outgoing_mail = outgoing_mail
def lineReceived(self, line):
"""
@@ -380,7 +287,7 @@ class EncryptedMessage(object):
:param line: The received line.
:type line: str
"""
- self.lines.append(line)
+ self._lines.append(line)
def eomReceived(self):
"""
@@ -391,10 +298,10 @@ class EncryptedMessage(object):
:returns: a deferred
"""
log.msg("Message data complete.")
- self.lines.append('') # add a trailing newline
- d = deferToThread(self._maybe_encrypt_and_sign)
- d.addCallbacks(self.sendMessage, self.skipNoKeyErrBack)
- return d
+ self._lines.append('') # add a trailing newline
+ raw_mail = '\r\n'.join(self._lines)
+
+ return self._outgoing_mail.send_message(raw_mail, self._user)
def connectionLost(self):
"""
@@ -402,292 +309,7 @@ class EncryptedMessage(object):
"""
log.msg("Connection lost unexpectedly!")
log.err()
- signal(proto.SMTP_CONNECTION_LOST, self._user.dest.addrstr)
+ emit(catalog.SMTP_CONNECTION_LOST, self._user.dest.addrstr)
# unexpected loss of connection; don't save
- self.lines = []
-
- # ends IMessage implementation
-
- def skipNoKeyErrBack(self, failure):
- """
- Errback that ignores a KeyNotFound
-
- :param failure: the failure
- :type Failure: Failure
- """
- err = failure.value
- if failure.check(KeyNotFound):
- pass
- else:
- raise err
-
- def parseMessage(self):
- """
- Separate message headers from body.
- """
- parser = Parser()
- return parser.parsestr('\r\n'.join(self.lines))
-
- def sendQueued(self, r):
- """
- Callback for the queued message.
-
- :param r: The result from the last previous callback in the chain.
- :type r: anything
- """
- log.msg(r)
-
- def sendSuccess(self, r):
- """
- Callback for a successful send.
-
- :param r: The result from the last previous callback in the chain.
- :type r: anything
- """
- log.msg(r)
- signal(proto.SMTP_SEND_MESSAGE_SUCCESS, self._user.dest.addrstr)
-
- def sendError(self, failure):
- """
- Callback for an unsuccessfull send.
-
- :param e: The result from the last errback.
- :type e: anything
- """
- signal(proto.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr)
- err = failure.value
- log.err(err)
- raise err
-
- def sendMessage(self, *args):
- """
- Sends the message.
-
- :return: A deferred with callback and errback for
- this #message send.
- :rtype: twisted.internet.defer.Deferred
- """
- d = deferToThread(self._route_msg)
- d.addCallbacks(self.sendQueued, self.sendError)
- return d
-
- def _route_msg(self):
- """
- Sends the msg using the ESMTPSenderFactory.
- """
- log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port))
- msg = self._msg.as_string(False)
-
- # we construct a defer to pass to the ESMTPSenderFactory
- d = defer.Deferred()
- d.addCallbacks(self.sendSuccess, self.sendError)
- # we don't pass an ssl context factory to the ESMTPSenderFactory
- # because ssl will be handled by reactor.connectSSL() below.
- factory = smtp.ESMTPSenderFactory(
- "", # username is blank because server does not use auth.
- "", # password is blank because server does not use auth.
- self._fromAddress.addrstr,
- self._user.dest.addrstr,
- StringIO(msg),
- d,
- heloFallback=True,
- requireAuthentication=False,
- requireTransportSecurity=True)
- factory.domain = __version__
- signal(proto.SMTP_SEND_MESSAGE_START, self._user.dest.addrstr)
- reactor.connectSSL(
- self._host, self._port, factory,
- contextFactory=SSLContextFactory(self._cert, self._key))
-
- #
- # encryption methods
- #
-
- def _encrypt_and_sign(self, pubkey, signkey):
- """
- Create an RFC 3156 compliang PGP encrypted and signed message using
- C{pubkey} to encrypt and C{signkey} to sign.
- :param pubkey: The public key used to encrypt the message.
- :type pubkey: OpenPGPKey
- :param signkey: The private key used to sign the message.
- :type signkey: OpenPGPKey
- """
- # create new multipart/encrypted message with 'pgp-encrypted' protocol
- newmsg = MultipartEncrypted('application/pgp-encrypted')
- # move (almost) all headers from original message to the new message
- self._fix_headers(self._origmsg, newmsg, signkey)
- # create 'application/octet-stream' encrypted message
- encmsg = MIMEApplication(
- self._km.encrypt(self._origmsg.as_string(unixfrom=False), pubkey,
- sign=signkey),
- _subtype='octet-stream', _encoder=lambda x: x)
- encmsg.add_header('content-disposition', 'attachment',
- filename='msg.asc')
- # create meta message
- metamsg = PGPEncrypted()
- metamsg.add_header('Content-Disposition', 'attachment')
- # attach pgp message parts to new message
- newmsg.attach(metamsg)
- newmsg.attach(encmsg)
- self._msg = newmsg
-
- def _sign(self, signkey):
- """
- Create an RFC 3156 compliant PGP signed MIME message using C{signkey}.
-
- :param signkey: The private key used to sign the message.
- :type signkey: leap.common.keymanager.openpgp.OpenPGPKey
- """
- # create new multipart/signed message
- newmsg = MultipartSigned('application/pgp-signature', 'pgp-sha512')
- # move (almost) all headers from original message to the new message
- self._fix_headers(self._origmsg, newmsg, signkey)
- # apply base64 content-transfer-encoding
- encode_base64_rec(self._origmsg)
- # get message text with headers and replace \n for \r\n
- fp = StringIO()
- g = RFC3156CompliantGenerator(
- fp, mangle_from_=False, maxheaderlen=76)
- g.flatten(self._origmsg)
- msgtext = re.sub('\r?\n', '\r\n', fp.getvalue())
- # make sure signed message ends with \r\n as per OpenPGP stantard.
- if self._origmsg.is_multipart():
- if not msgtext.endswith("\r\n"):
- msgtext += "\r\n"
- # calculate signature
- signature = self._km.sign(msgtext, signkey, digest_algo='SHA512',
- clearsign=False, detach=True, binary=False)
- sigmsg = PGPSignature(signature)
- # attach original message and signature to new message
- newmsg.attach(self._origmsg)
- newmsg.attach(sigmsg)
- self._msg = newmsg
-
- def _maybe_encrypt_and_sign(self):
- """
- Attempt to encrypt and sign the outgoing message.
-
- The behaviour of this method depends on:
-
- 1. the original message's content-type, and
- 2. the availability of the recipient's public key.
-
- If the original message's content-type is "multipart/encrypted", then
- the original message is not altered. For any other content-type, the
- method attempts to fetch the recipient's public key. If the
- recipient's public key is available, the message is encrypted and
- signed; otherwise it is only signed.
-
- Note that, if the C{encrypted_only} configuration is set to True and
- the recipient's public key is not available, then the recipient
- address would have been rejected in SMTPDelivery.validateTo().
-
- The following table summarizes the overall behaviour of the gateway:
-
- +---------------------------------------------------+----------------+
- | content-type | rcpt pubkey | enforce encr. | action |
- +---------------------+-------------+---------------+----------------+
- | multipart/encrypted | any | any | pass |
- | other | available | any | encrypt + sign |
- | other | unavailable | yes | reject |
- | other | unavailable | no | sign |
- +---------------------+-------------+---------------+----------------+
- """
- # pass if the original message's content-type is "multipart/encrypted"
- self._origmsg = self.parseMessage()
- if self._origmsg.get_content_type() == 'multipart/encrypted':
- self._msg = self._origmsg
- return
-
- from_address = validate_address(self._fromAddress.addrstr)
- username, domain = from_address.split('@')
-
- # add a nice footer to the outgoing message
- if self._origmsg.get_content_type() == 'text/plain':
- self.lines.append('--')
- self.lines.append('%s - https://%s/key/%s' %
- (self.FOOTER_STRING, domain, username))
- self.lines.append('')
-
- self._origmsg = self.parseMessage()
-
- # get sender and recipient data
- signkey = self._km.get_key(from_address, OpenPGPKey, private=True)
- log.msg("Will sign the message with %s." % signkey.fingerprint)
- to_address = validate_address(self._user.dest.addrstr)
- try:
- # try to get the recipient pubkey
- pubkey = self._km.get_key(to_address, OpenPGPKey)
- log.msg("Will encrypt the message to %s." % pubkey.fingerprint)
- signal(proto.SMTP_START_ENCRYPT_AND_SIGN,
- "%s,%s" % (self._fromAddress.addrstr, to_address))
- self._encrypt_and_sign(pubkey, signkey)
- signal(proto.SMTP_END_ENCRYPT_AND_SIGN,
- "%s,%s" % (self._fromAddress.addrstr, to_address))
- except KeyNotFound:
- # at this point we _can_ send unencrypted mail, because if the
- # configuration said the opposite the address would have been
- # rejected in SMTPDelivery.validateTo().
- log.msg('Will send unencrypted message to %s.' % to_address)
- signal(proto.SMTP_START_SIGN, self._fromAddress.addrstr)
- self._sign(signkey)
- signal(proto.SMTP_END_SIGN, self._fromAddress.addrstr)
-
- def _fix_headers(self, origmsg, newmsg, signkey):
- """
- Move some headers from C{origmsg} to C{newmsg}, delete unwanted
- headers from C{origmsg} and add new headers to C{newms}.
-
- Outgoing messages are either encrypted and signed or just signed
- before being sent. Because of that, they are packed inside new
- messages and some manipulation has to be made on their headers.
-
- Allowed headers for passing through:
-
- - From
- - Date
- - To
- - Subject
- - Reply-To
- - References
- - In-Reply-To
- - Cc
-
- Headers to be added:
-
- - Message-ID (i.e. should not use origmsg's Message-Id)
- - Received (this is added automatically by twisted smtp API)
- - OpenPGP (see #4447)
-
- Headers to be deleted:
-
- - User-Agent
-
- :param origmsg: The original message.
- :type origmsg: email.message.Message
- :param newmsg: The new message being created.
- :type newmsg: email.message.Message
- :param signkey: The key used to sign C{newmsg}
- :type signkey: OpenPGPKey
- """
- # move headers from origmsg to newmsg
- headers = origmsg.items()
- passthrough = [
- 'from', 'date', 'to', 'subject', 'reply-to', 'references',
- 'in-reply-to', 'cc'
- ]
- headers = filter(lambda x: x[0].lower() in passthrough, headers)
- for hkey, hval in headers:
- newmsg.add_header(hkey, hval)
- del(origmsg[hkey])
- # add a new message-id to newmsg
- newmsg.add_header('Message-Id', smtp.messageid())
- # add openpgp header to newmsg
- username, domain = signkey.address.split('@')
- newmsg.add_header(
- 'OpenPGP', 'id=%s' % signkey.key_id,
- url='https://%s/key/%s' % (domain, username),
- preference='signencrypt')
- # delete user-agent from origmsg
- del(origmsg['user-agent'])
+ self._lines = []
diff --git a/src/leap/mail/smtp/rfc3156.py b/src/leap/mail/smtp/rfc3156.py
index 62a0675..7d7bc0f 100644
--- a/src/leap/mail/smtp/rfc3156.py
+++ b/src/leap/mail/smtp/rfc3156.py
@@ -19,9 +19,7 @@
Implements RFC 3156: MIME Security with OpenPGP.
"""
-import re
import base64
-from abc import ABCMeta, abstractmethod
from StringIO import StringIO
from twisted.python import log
diff --git a/src/leap/mail/smtp/tests/__init__.py b/src/leap/mail/smtp/tests/__init__.py
index dc24293..e69de29 100644
--- a/src/leap/mail/smtp/tests/__init__.py
+++ b/src/leap/mail/smtp/tests/__init__.py
@@ -1,386 +0,0 @@
-# -*- coding: utf-8 -*-
-# __init__.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-Base classes and keys for SMTP gateway tests.
-"""
-
-import os
-import distutils.spawn
-import shutil
-import tempfile
-from mock import Mock
-
-
-from twisted.trial import unittest
-
-
-from leap.soledad.client import Soledad
-from leap.keymanager import (
- KeyManager,
- openpgp,
-)
-
-
-from leap.common.testing.basetest import BaseLeapTest
-
-
-def _find_gpg():
- gpg_path = distutils.spawn.find_executable('gpg')
- return os.path.realpath(gpg_path) if gpg_path is not None else "/usr/bin/gpg"
-
-
-class TestCaseWithKeyManager(BaseLeapTest):
-
- GPG_BINARY_PATH = _find_gpg()
-
- def setUp(self):
- # mimic BaseLeapTest.setUpClass behaviour, because this is deprecated
- # in Twisted: http://twistedmatrix.com/trac/ticket/1870
- self.old_path = os.environ['PATH']
- self.old_home = os.environ['HOME']
- self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
- self.home = self.tempdir
- bin_tdir = os.path.join(
- self.tempdir,
- 'bin')
- os.environ["PATH"] = bin_tdir
- os.environ["HOME"] = self.tempdir
-
- # setup our own stuff
- address = 'leap@leap.se' # user's address in the form user@provider
- uuid = 'leap@leap.se'
- passphrase = u'123'
- secrets_path = os.path.join(self.tempdir, 'secret.gpg')
- local_db_path = os.path.join(self.tempdir, 'soledad.u1db')
- server_url = 'http://provider/'
- cert_file = ''
-
- self._soledad = self._soledad_instance(
- uuid, passphrase, secrets_path, local_db_path, server_url,
- cert_file)
- self._km = self._keymanager_instance(address)
-
- def _soledad_instance(self, uuid, passphrase, secrets_path, local_db_path,
- server_url, cert_file):
- """
- Return a Soledad instance for tests.
- """
- # mock key fetching and storing so Soledad doesn't fail when trying to
- # reach the server.
- Soledad._fetch_keys_from_shared_db = Mock(return_value=None)
- Soledad._assert_keys_in_shared_db = Mock(return_value=None)
-
- # instantiate soledad
- def _put_doc_side_effect(doc):
- self._doc_put = doc
-
- class MockSharedDB(object):
-
- get_doc = Mock(return_value=None)
- put_doc = Mock(side_effect=_put_doc_side_effect)
- lock = Mock(return_value=('atoken', 300))
- unlock = Mock(return_value=True)
-
- def __call__(self):
- return self
-
- Soledad._shared_db = MockSharedDB()
-
- return Soledad(
- uuid,
- passphrase,
- secrets_path=secrets_path,
- local_db_path=local_db_path,
- server_url=server_url,
- cert_file=cert_file,
- )
-
- def _keymanager_instance(self, address):
- """
- Return a Key Manager instance for tests.
- """
- self._config = {
- 'host': 'http://provider/',
- 'port': 25,
- 'username': address,
- 'password': '<password>',
- 'encrypted_only': True,
- 'cert': u'src/leap/mail/smtp/tests/cert/server.crt',
- 'key': u'src/leap/mail/smtp/tests/cert/server.key',
- }
-
- class Response(object):
- status_code = 200
- headers = {'content-type': 'application/json'}
-
- def json(self):
- return {'address': ADDRESS_2, 'openpgp': PUBLIC_KEY_2}
-
- def raise_for_status(self):
- pass
-
- nickserver_url = '' # the url of the nickserver
- km = KeyManager(address, nickserver_url, self._soledad,
- ca_cert_path='', gpgbinary=self.GPG_BINARY_PATH)
- km._fetcher.put = Mock()
- km._fetcher.get = Mock(return_value=Response())
-
- # insert test keys in key manager.
- pgp = openpgp.OpenPGPScheme(
- self._soledad, gpgbinary=self.GPG_BINARY_PATH)
- pgp.put_ascii_key(PRIVATE_KEY)
- pgp.put_ascii_key(PRIVATE_KEY_2)
-
- return km
-
- def tearDown(self):
- # mimic LeapBaseTest.tearDownClass behaviour
- os.environ["PATH"] = self.old_path
- os.environ["HOME"] = self.old_home
- # safety check
- assert 'leap_tests-' in self.tempdir
- shutil.rmtree(self.tempdir)
-
-
-# Key material for testing
-KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF"
-
-ADDRESS = 'leap@leap.se'
-
-PUBLIC_KEY = """
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1.4.10 (GNU/Linux)
-
-mQINBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz
-iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO
-zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx
-irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT
-huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs
-d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g
-wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb
-hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv
-U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H
-T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i
-Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB
-tBxMZWFwIFRlc3QgS2V5IDxsZWFwQGxlYXAuc2U+iQI3BBMBCAAhBQJQvfnZAhsD
-BQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEC9FXigk0Y3fT7EQAKH3IuRniOpb
-T/DDIgwwjz3oxB/W0DDMyPXowlhSOuM0rgGfntBpBb3boezEXwL86NPQxNGGruF5
-hkmecSiuPSvOmQlqlS95NGQp6hNG0YaKColh+Q5NTspFXCAkFch9oqUje0LdxfSP
-QfV9UpeEvGyPmk1I9EJV/YDmZ4+Djge1d7qhVZInz4Rx1NrSyF/Tc2EC0VpjQFsU
-Y9Kb2YBBR7ivG6DBc8ty0jJXi7B4WjkFcUEJviQpMF2dCLdonCehYs1PqsN1N7j+
-eFjQd+hqVMJgYuSGKjvuAEfClM6MQw7+FmFwMyLgK/Ew/DttHEDCri77SPSkOGSI
-txCzhTg6798f6mJr7WcXmHX1w1Vcib5FfZ8vTDFVhz/XgAgArdhPo9V6/1dgSSiB
-KPQ/spsco6u5imdOhckERE0lnAYvVT6KE81TKuhF/b23u7x+Wdew6kK0EQhYA7wy
-7LmlaNXc7rMBQJ9Z60CJ4JDtatBWZ0kNrt2VfdDHVdqBTOpl0CraNUjWE5YMDasr
-K2dF5IX8D3uuYtpZnxqg0KzyLg0tzL0tvOL1C2iudgZUISZNPKbS0z0v+afuAAnx
-2pTC3uezbh2Jt8SWTLhll4i0P4Ps5kZ6HQUO56O+/Z1cWovX+mQekYFmERySDR9n
-3k1uAwLilJmRmepGmvYbB8HloV8HqwgguQINBFC9+dkBEAC0I/xn1uborMgDvBtf
-H0sEhwnXBC849/32zic6udB6/3Efk9nzbSpL3FSOuXITZsZgCHPkKarnoQ2ztMcS
-sh1ke1C5gQGms75UVmM/nS+2YI4vY8OX/GC/on2vUyncqdH+bR6xH5hx4NbWpfTs
-iQHmz5C6zzS/kuabGdZyKRaZHt23WQ7JX/4zpjqbC99DjHcP9BSk7tJ8wI4bkMYD
-uFVQdT9O6HwyKGYwUU4sAQRAj7XCTGvVbT0dpgJwH4RmrEtJoHAx4Whg8mJ710E0
-GCmzf2jqkNuOw76ivgk27Kge+Hw00jmJjQhHY0yVbiaoJwcRrPKzaSjEVNgrpgP3
-lXPRGQArgESsIOTeVVHQ8fhK2YtTeCY9rIiO+L0OX2xo9HK7hfHZZWL6rqymXdyS
-fhzh/f6IPyHFWnvj7Brl7DR8heMikygcJqv+ed2yx7iLyCUJ10g12I48+aEj1aLe
-dP7lna32iY8/Z0SHQLNH6PXO9SlPcq2aFUgKqE75A/0FMk7CunzU1OWr2ZtTLNO1
-WT/13LfOhhuEq9jTyTosn0WxBjJKq18lnhzCXlaw6EAtbA7CUwsD3CTPR56aAXFK
-3I7KXOVAqggrvMe5Tpdg5drfYpI8hZovL5aAgb+7Y5ta10TcJdUhS5K3kFAWe/td
-U0cmWUMDP1UMSQ5Jg6JIQVWhSwARAQABiQIfBBgBCAAJBQJQvfnZAhsMAAoJEC9F
-Xigk0Y3fRwsP/i0ElYCyxeLpWJTwo1iCLkMKz2yX1lFVa9nT1BVTPOQwr/IAc5OX
-NdtbJ14fUsKL5pWgW8OmrXtwZm1y4euI1RPWWubG01ouzwnGzv26UcuHeqC5orZj
-cOnKtL40y8VGMm8LoicVkRJH8blPORCnaLjdOtmA3rx/v2EXrJpSa3AhOy0ZSRXk
-ZSrK68AVNwamHRoBSYyo0AtaXnkPX4+tmO8X8BPfj125IljubvwZPIW9VWR9UqCE
-VPfDR1XKegVb6VStIywF7kmrknM1C5qUY28rdZYWgKorw01hBGV4jTW0cqde3N51
-XT1jnIAa+NoXUM9uQoGYMiwrL7vNsLlyyiW5ayDyV92H/rIuiqhFgbJsHTlsm7I8
-oGheR784BagAA1NIKD1qEO9T6Kz9lzlDaeWS5AUKeXrb7ZJLI1TTCIZx5/DxjLqM
-Tt/RFBpVo9geZQrvLUqLAMwdaUvDXC2c6DaCPXTh65oCZj/hqzlJHH+RoTWWzKI+
-BjXxgUWF9EmZUBrg68DSmI+9wuDFsjZ51BcqvJwxyfxtTaWhdoYqH/UQS+D1FP3/
-diZHHlzwVwPICzM9ooNTgbrcDzyxRkIVqsVwBq7EtzcvgYUyX53yG25Giy6YQaQ2
-ZtQ/VymwFL3XdUWV6B/hU4PVAFvO3qlOtdJ6TpE+nEWgcWjCv5g7RjXX
-=MuOY
------END PGP PUBLIC KEY BLOCK-----
-"""
-
-PRIVATE_KEY = """
------BEGIN PGP PRIVATE KEY BLOCK-----
-Version: GnuPG v1.4.10 (GNU/Linux)
-
-lQcYBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz
-iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO
-zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx
-irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT
-huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs
-d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g
-wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb
-hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv
-U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H
-T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i
-Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB
-AA/+JHtlL39G1wsH9R6UEfUQJGXR9MiIiwZoKcnRB2o8+DS+OLjg0JOh8XehtuCs
-E/8oGQKtQqa5bEIstX7IZoYmYFiUQi9LOzIblmp2vxOm+HKkxa4JszWci2/ZmC3t
-KtaA4adl9XVnshoQ7pijuCMUKB3naBEOAxd8s9d/JeReGIYkJErdrnVfNk5N71Ds
-FmH5Ll3XtEDvgBUQP3nkA6QFjpsaB94FHjL3gDwum/cxzj6pCglcvHOzEhfY0Ddb
-J967FozQTaf2JW3O+w3LOqtcKWpq87B7+O61tVidQPSSuzPjCtFF0D2LC9R/Hpky
-KTMQ6CaKja4MPhjwywd4QPcHGYSqjMpflvJqi+kYIt8psUK/YswWjnr3r4fbuqVY
-VhtiHvnBHQjz135lUqWvEz4hM3Xpnxydx7aRlv5NlevK8+YIO5oFbWbGNTWsPZI5
-jpoFBpSsnR1Q5tnvtNHauvoWV+XN2qAOBTG+/nEbDYH6Ak3aaE9jrpTdYh0CotYF
-q7csANsDy3JvkAzeU6WnYpsHHaAjqOGyiZGsLej1UcXPFMosE/aUo4WQhiS8Zx2c
-zOVKOi/X5vQ2GdNT9Qolz8AriwzsvFR+bxPzyd8V6ALwDsoXvwEYinYBKK8j0OPv
-OOihSR6HVsuP9NUZNU9ewiGzte/+/r6pNXHvR7wTQ8EWLcEIAN6Zyrb0bHZTIlxt
-VWur/Ht2mIZrBaO50qmM5RD3T5oXzWXi/pjLrIpBMfeZR9DWfwQwjYzwqi7pxtYx
-nJvbMuY505rfnMoYxb4J+cpRXV8MS7Dr1vjjLVUC9KiwSbM3gg6emfd2yuA93ihv
-Pe3mffzLIiQa4mRE3wtGcioC43nWuV2K2e1KjxeFg07JhrezA/1Cak505ab/tmvP
-4YmjR5c44+yL/YcQ3HdFgs4mV+nVbptRXvRcPpolJsgxPccGNdvHhsoR4gwXMS3F
-RRPD2z6x8xeN73Q4KH3bm01swQdwFBZbWVfmUGLxvN7leCdfs9+iFJyqHiCIB6Iv
-mQfp8F0IAOwSo8JhWN+V1dwML4EkIrM8wUb4yecNLkyR6TpPH/qXx4PxVMC+vy6x
-sCtjeHIwKE+9vqnlhd5zOYh7qYXEJtYwdeDDmDbL8oks1LFfd+FyAuZXY33DLwn0
-cRYsr2OEZmaajqUB3NVmj3H4uJBN9+paFHyFSXrH68K1Fk2o3n+RSf2EiX+eICwI
-L6rqoF5sSVUghBWdNegV7qfy4anwTQwrIMGjgU5S6PKW0Dr/3iO5z3qQpGPAj5OW
-ATqPWkDICLbObPxD5cJlyyNE2wCA9VVc6/1d6w4EVwSq9h3/WTpATEreXXxTGptd
-LNiTA1nmakBYNO2Iyo3djhaqBdWjk+EIAKtVEnJH9FAVwWOvaj1RoZMA5DnDMo7e
-SnhrCXl8AL7Z1WInEaybasTJXn1uQ8xY52Ua4b8cbuEKRKzw/70NesFRoMLYoHTO
-dyeszvhoDHberpGRTciVmpMu7Hyi33rM31K9epA4ib6QbbCHnxkWOZB+Bhgj1hJ8
-xb4RBYWiWpAYcg0+DAC3w9gfxQhtUlZPIbmbrBmrVkO2GVGUj8kH6k4UV6kUHEGY
-HQWQR0HcbKcXW81ZXCCD0l7ROuEWQtTe5Jw7dJ4/QFuqZnPutXVRNOZqpl6eRShw
-7X2/a29VXBpmHA95a88rSQsL+qm7Fb3prqRmuMCtrUZgFz7HLSTuUMR867QcTGVh
-cCBUZXN0IEtleSA8bGVhcEBsZWFwLnNlPokCNwQTAQgAIQUCUL352QIbAwULCQgH
-AwUVCgkICwUWAgMBAAIeAQIXgAAKCRAvRV4oJNGN30+xEACh9yLkZ4jqW0/wwyIM
-MI896MQf1tAwzMj16MJYUjrjNK4Bn57QaQW926HsxF8C/OjT0MTRhq7heYZJnnEo
-rj0rzpkJapUveTRkKeoTRtGGigqJYfkOTU7KRVwgJBXIfaKlI3tC3cX0j0H1fVKX
-hLxsj5pNSPRCVf2A5mePg44HtXe6oVWSJ8+EcdTa0shf03NhAtFaY0BbFGPSm9mA
-QUe4rxugwXPLctIyV4uweFo5BXFBCb4kKTBdnQi3aJwnoWLNT6rDdTe4/nhY0Hfo
-alTCYGLkhio77gBHwpTOjEMO/hZhcDMi4CvxMPw7bRxAwq4u+0j0pDhkiLcQs4U4
-Ou/fH+pia+1nF5h19cNVXIm+RX2fL0wxVYc/14AIAK3YT6PVev9XYEkogSj0P7Kb
-HKOruYpnToXJBERNJZwGL1U+ihPNUyroRf29t7u8flnXsOpCtBEIWAO8Muy5pWjV
-3O6zAUCfWetAieCQ7WrQVmdJDa7dlX3Qx1XagUzqZdAq2jVI1hOWDA2rKytnReSF
-/A97rmLaWZ8aoNCs8i4NLcy9Lbzi9QtornYGVCEmTTym0tM9L/mn7gAJ8dqUwt7n
-s24dibfElky4ZZeItD+D7OZGeh0FDuejvv2dXFqL1/pkHpGBZhEckg0fZ95NbgMC
-4pSZkZnqRpr2GwfB5aFfB6sIIJ0HGARQvfnZARAAtCP8Z9bm6KzIA7wbXx9LBIcJ
-1wQvOPf99s4nOrnQev9xH5PZ820qS9xUjrlyE2bGYAhz5Cmq56ENs7THErIdZHtQ
-uYEBprO+VFZjP50vtmCOL2PDl/xgv6J9r1Mp3KnR/m0esR+YceDW1qX07IkB5s+Q
-us80v5LmmxnWcikWmR7dt1kOyV/+M6Y6mwvfQ4x3D/QUpO7SfMCOG5DGA7hVUHU/
-Tuh8MihmMFFOLAEEQI+1wkxr1W09HaYCcB+EZqxLSaBwMeFoYPJie9dBNBgps39o
-6pDbjsO+or4JNuyoHvh8NNI5iY0IR2NMlW4mqCcHEazys2koxFTYK6YD95Vz0RkA
-K4BErCDk3lVR0PH4StmLU3gmPayIjvi9Dl9saPRyu4Xx2WVi+q6spl3ckn4c4f3+
-iD8hxVp74+wa5ew0fIXjIpMoHCar/nndsse4i8glCddINdiOPPmhI9Wi3nT+5Z2t
-9omPP2dEh0CzR+j1zvUpT3KtmhVICqhO+QP9BTJOwrp81NTlq9mbUyzTtVk/9dy3
-zoYbhKvY08k6LJ9FsQYySqtfJZ4cwl5WsOhALWwOwlMLA9wkz0eemgFxStyOylzl
-QKoIK7zHuU6XYOXa32KSPIWaLy+WgIG/u2ObWtdE3CXVIUuSt5BQFnv7XVNHJllD
-Az9VDEkOSYOiSEFVoUsAEQEAAQAP/1AagnZQZyzHDEgw4QELAspYHCWLXE5aZInX
-wTUJhK31IgIXNn9bJ0hFiSpQR2xeMs9oYtRuPOu0P8oOFMn4/z374fkjZy8QVY3e
-PlL+3EUeqYtkMwlGNmVw5a/NbNuNfm5Darb7pEfbYd1gPcni4MAYw7R2SG/57GbC
-9gucvspHIfOSfBNLBthDzmK8xEKe1yD2eimfc2T7IRYb6hmkYfeds5GsqvGI6mwI
-85h4uUHWRc5JOlhVM6yX8hSWx0L60Z3DZLChmc8maWnFXd7C8eQ6P1azJJbW71Ih
-7CoK0XW4LE82vlQurSRFgTwfl7wFYszW2bOzCuhHDDtYnwH86Nsu0DC78ZVRnvxn
-E8Ke/AJgrdhIOo4UAyR+aZD2+2mKd7/waOUTUrUtTzc7i8N3YXGi/EIaNReBXaq+
-ZNOp24BlFzRp+FCF/pptDW9HjPdiV09x0DgICmeZS4Gq/4vFFIahWctg52NGebT0
-Idxngjj+xDtLaZlLQoOz0n5ByjO/Wi0ANmMv1sMKCHhGvdaSws2/PbMR2r4caj8m
-KXpIgdinM/wUzHJ5pZyF2U/qejsRj8Kw8KH/tfX4JCLhiaP/mgeTuWGDHeZQERAT
-xPmRFHaLP9/ZhvGNh6okIYtrKjWTLGoXvKLHcrKNisBLSq+P2WeFrlme1vjvJMo/
-jPwLT5o9CADQmcbKZ+QQ1ZM9v99iDZol7SAMZX43JC019sx6GK0u6xouJBcLfeB4
-OXacTgmSYdTa9RM9fbfVpti01tJ84LV2SyL/VJq/enJF4XQPSynT/tFTn1PAor6o
-tEAAd8fjKdJ6LnD5wb92SPHfQfXqI84rFEO8rUNIE/1ErT6DYifDzVCbfD2KZdoF
-cOSp7TpD77sY1bs74ocBX5ejKtd+aH99D78bJSMM4pSDZsIEwnomkBHTziubPwJb
-OwnATy0LmSMAWOw5rKbsh5nfwCiUTM20xp0t5JeXd+wPVWbpWqI2EnkCEN+RJr9i
-7dp/ymDQ+Yt5wrsN3NwoyiexPOG91WQVCADdErHsnglVZZq9Z8Wx7KwecGCUurJ2
-H6lKudv5YOxPnAzqZS5HbpZd/nRTMZh2rdXCr5m2YOuewyYjvM757AkmUpM09zJX
-MQ1S67/UX2y8/74TcRF97Ncx9HeELs92innBRXoFitnNguvcO6Esx4BTe1OdU6qR
-ER3zAmVf22Le9ciXbu24DN4mleOH+OmBx7X2PqJSYW9GAMTsRB081R6EWKH7romQ
-waxFrZ4DJzZ9ltyosEJn5F32StyLrFxpcrdLUoEaclZCv2qka7sZvi0EvovDVEBU
-e10jOx9AOwf8Gj2ufhquQ6qgVYCzbP+YrodtkFrXRS3IsljIchj1M2ffB/0bfoUs
-rtER9pLvYzCjBPg8IfGLw0o754Qbhh/ReplCRTusP/fQMybvCvfxreS3oyEriu/G
-GufRomjewZ8EMHDIgUsLcYo2UHZsfF7tcazgxMGmMvazp4r8vpgrvW/8fIN/6Adu
-tF+WjWDTvJLFJCe6O+BFJOWrssNrrra1zGtLC1s8s+Wfpe+bGPL5zpHeebGTwH1U
-22eqgJArlEKxrfarz7W5+uHZJHSjF/K9ZvunLGD0n9GOPMpji3UO3zeM8IYoWn7E
-/EWK1XbjnssNemeeTZ+sDh+qrD7BOi+vCX1IyBxbfqnQfJZvmcPWpruy1UsO+aIC
-0GY8Jr3OL69dDQ21jueJAh8EGAEIAAkFAlC9+dkCGwwACgkQL0VeKCTRjd9HCw/+
-LQSVgLLF4ulYlPCjWIIuQwrPbJfWUVVr2dPUFVM85DCv8gBzk5c121snXh9Swovm
-laBbw6ate3BmbXLh64jVE9Za5sbTWi7PCcbO/bpRy4d6oLmitmNw6cq0vjTLxUYy
-bwuiJxWREkfxuU85EKdouN062YDevH+/YResmlJrcCE7LRlJFeRlKsrrwBU3BqYd
-GgFJjKjQC1peeQ9fj62Y7xfwE9+PXbkiWO5u/Bk8hb1VZH1SoIRU98NHVcp6BVvp
-VK0jLAXuSauSczULmpRjbyt1lhaAqivDTWEEZXiNNbRyp17c3nVdPWOcgBr42hdQ
-z25CgZgyLCsvu82wuXLKJblrIPJX3Yf+si6KqEWBsmwdOWybsjygaF5HvzgFqAAD
-U0goPWoQ71PorP2XOUNp5ZLkBQp5etvtkksjVNMIhnHn8PGMuoxO39EUGlWj2B5l
-Cu8tSosAzB1pS8NcLZzoNoI9dOHrmgJmP+GrOUkcf5GhNZbMoj4GNfGBRYX0SZlQ
-GuDrwNKYj73C4MWyNnnUFyq8nDHJ/G1NpaF2hiof9RBL4PUU/f92JkceXPBXA8gL
-Mz2ig1OButwPPLFGQhWqxXAGrsS3Ny+BhTJfnfIbbkaLLphBpDZm1D9XKbAUvdd1
-RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc=
-=JTFu
------END PGP PRIVATE KEY BLOCK-----
-"""
-
-ADDRESS_2 = 'anotheruser@leap.se'
-
-PUBLIC_KEY_2 = """
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1.4.10 (GNU/Linux)
-
-mI0EUYwJXgEEAMbTKHuPJ5/Gk34l9Z06f+0WCXTDXdte1UBoDtZ1erAbudgC4MOR
-gquKqoj3Hhw0/ILqJ88GcOJmKK/bEoIAuKaqlzDF7UAYpOsPZZYmtRfPC2pTCnXq
-Z1vdeqLwTbUspqXflkCkFtfhGKMq5rH8GV5a3tXZkRWZhdNwhVXZagC3ABEBAAG0
-IWFub3RoZXJ1c2VyIDxhbm90aGVydXNlckBsZWFwLnNlPoi4BBMBAgAiBQJRjAle
-AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRB/nfpof+5XWotuA/4tLN4E
-gUr7IfLy2HkHAxzw7A4rqfMN92DIM9mZrDGaWRrOn3aVF7VU1UG7MDkHfPvp/cFw
-ezoCw4s4IoHVc/pVlOkcHSyt4/Rfh248tYEJmFCJXGHpkK83VIKYJAithNccJ6Q4
-JE/o06Mtf4uh/cA1HUL4a4ceqUhtpLJULLeKo7iNBFGMCV4BBADsyQI7GR0wSAxz
-VayLjuPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQt
-Z/hwcLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63
-yuRe94WenT1RJd6xU1aaUff4rKizuQARAQABiJ8EGAECAAkFAlGMCV4CGwwACgkQ
-f536aH/uV1rPZQQAqCzRysOlu8ez7PuiBD4SebgRqWlxa1TF1ujzfLmuPivROZ2X
-Kw5aQstxgGSjoB7tac49s0huh4X8XK+BtJBfU84JS8Jc2satlfwoyZ35LH6sDZck
-I+RS/3we6zpMfHs3vvp9xgca6ZupQxivGtxlJs294TpJorx+mFFqbV17AzQ=
-=Thdu
------END PGP PUBLIC KEY BLOCK-----
-"""
-
-PRIVATE_KEY_2 = """
------BEGIN PGP PRIVATE KEY BLOCK-----
-Version: GnuPG v1.4.10 (GNU/Linux)
-
-lQHYBFGMCV4BBADG0yh7jyefxpN+JfWdOn/tFgl0w13bXtVAaA7WdXqwG7nYAuDD
-kYKriqqI9x4cNPyC6ifPBnDiZiiv2xKCALimqpcwxe1AGKTrD2WWJrUXzwtqUwp1
-6mdb3Xqi8E21LKal35ZApBbX4RijKuax/BleWt7V2ZEVmYXTcIVV2WoAtwARAQAB
-AAP7BLuSAx7tOohnimEs74ks8l/L6dOcsFQZj2bqs4AoY3jFe7bV0tHr4llypb/8
-H3/DYvpf6DWnCjyUS1tTnXSW8JXtx01BUKaAufSmMNg9blKV6GGHlT/Whe9uVyks
-7XHk/+9mebVMNJ/kNlqq2k+uWqJohzC8WWLRK+d1tBeqDsECANZmzltPaqUsGV5X
-C3zszE3tUBgptV/mKnBtopKi+VH+t7K6fudGcG+bAcZDUoH/QVde52mIIjjIdLje
-uajJuHUCAO1mqh+vPoGv4eBLV7iBo3XrunyGXiys4a39eomhxTy3YktQanjjx+ty
-GltAGCs5PbWGO6/IRjjvd46wh53kzvsCAO0J97gsWhzLuFnkxFAJSPk7RRlyl7lI
-1XS/x0Og6j9XHCyY1OYkfBm0to3UlCfkgirzCYlTYObCofzdKFIPDmSqHbQhYW5v
-dGhlcnVzZXIgPGFub3RoZXJ1c2VyQGxlYXAuc2U+iLgEEwECACIFAlGMCV4CGwMG
-CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEH+d+mh/7ldai24D/i0s3gSBSvsh
-8vLYeQcDHPDsDiup8w33YMgz2ZmsMZpZGs6fdpUXtVTVQbswOQd8++n9wXB7OgLD
-izgigdVz+lWU6RwdLK3j9F+Hbjy1gQmYUIlcYemQrzdUgpgkCK2E1xwnpDgkT+jT
-oy1/i6H9wDUdQvhrhx6pSG2kslQst4qjnQHYBFGMCV4BBADsyQI7GR0wSAxzVayL
-juPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQtZ/hw
-cLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63yuRe
-94WenT1RJd6xU1aaUff4rKizuQARAQABAAP9EyElqJ3dq3EErXwwT4mMnbd1SrVC
-rUJrNWQZL59mm5oigS00uIyR0SvusOr+UzTtd8ysRuwHy5d/LAZsbjQStaOMBILx
-77TJveOel0a1QK0YSMF2ywZMCKvquvjli4hAtWYz/EwfuzQN3t23jc5ny+GqmqD2
-3FUxLJosFUfLNmECAO9KhVmJi+L9dswIs+2Dkjd1eiRQzNOEVffvYkGYZyKxNiXF
-UA5kvyZcB4iAN9sWCybE4WHZ9jd4myGB0MPDGxkCAP1RsXJbbuD6zS7BXe5gwunO
-2q4q7ptdSl/sJYQuTe1KNP5d/uGsvlcFfsYjpsopasPjFBIncc/2QThMKlhoEaEB
-/0mVAxpT6SrEvUbJ18z7kna24SgMPr3OnPMxPGfvNLJY/Xv/A17YfoqjmByCvsKE
-JCDjopXtmbcrZyoEZbEht9mko4ifBBgBAgAJBQJRjAleAhsMAAoJEH+d+mh/7lda
-z2UEAKgs0crDpbvHs+z7ogQ+Enm4EalpcWtUxdbo83y5rj4r0TmdlysOWkLLcYBk
-o6Ae7WnOPbNIboeF/FyvgbSQX1POCUvCXNrGrZX8KMmd+Sx+rA2XJCPkUv98Hus6
-THx7N776fcYHGumbqUMYrxrcZSbNveE6SaK8fphRam1dewM0
-=a5gs
------END PGP PRIVATE KEY BLOCK-----
-"""
diff --git a/src/leap/mail/smtp/tests/test_gateway.py b/src/leap/mail/smtp/tests/test_gateway.py
index 466677f..0b9a364 100644
--- a/src/leap/mail/smtp/tests/test_gateway.py
+++ b/src/leap/mail/smtp/tests/test_gateway.py
@@ -20,24 +20,24 @@
SMTP gateway tests.
"""
-
import re
from datetime import datetime
+
+from twisted.internet import reactor
+from twisted.internet.defer import inlineCallbacks, fail, succeed, Deferred
from twisted.test import proto_helpers
-from twisted.mail.smtp import User, Address
from mock import Mock
-
from leap.mail.smtp.gateway import (
- SMTPFactory,
- EncryptedMessage,
+ SMTPFactory
)
-from leap.mail.smtp.tests import (
+from leap.mail.tests import (
TestCaseWithKeyManager,
ADDRESS,
ADDRESS_2,
)
-from leap.keymanager import openpgp
+from leap.keymanager import openpgp, errors
+
# some regexps
IP_REGEX = "(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}" + \
@@ -71,20 +71,7 @@ class TestSmtpGateway(TestCaseWithKeyManager):
% (string, pattern))
raise self.failureException(msg)
- def test_openpgp_encrypt_decrypt(self):
- "Test if openpgp can encrypt and decrypt."
- text = "simple raw text"
- pubkey = self._km.get_key(
- ADDRESS, openpgp.OpenPGPKey, private=False)
- encrypted = self._km.encrypt(text, pubkey)
- self.assertNotEqual(
- text, encrypted, "Ciphertext is equal to plaintext.")
- privkey = self._km.get_key(
- ADDRESS, openpgp.OpenPGPKey, private=True)
- decrypted = self._km.decrypt(encrypted, privkey)
- self.assertEqual(text, decrypted,
- "Decrypted text differs from plaintext.")
-
+ @inlineCallbacks
def test_gateway_accepts_valid_email(self):
"""
Test if SMTP server responds correctly for valid interaction.
@@ -102,223 +89,99 @@ class TestSmtpGateway(TestCaseWithKeyManager):
# method...
proto = SMTPFactory(
u'anotheruser@leap.se',
- self._km, self._config['host'],
- self._config['port'],
- self._config['cert'], self._config['key'],
- self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
+ self._km,
+ self._config['encrypted_only'],
+ outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0))
# snip...
transport = proto_helpers.StringTransport()
proto.makeConnection(transport)
+ reply = ""
for i, line in enumerate(self.EMAIL_DATA):
- proto.lineReceived(line + '\r\n')
- self.assertMatch(transport.value(),
- '\r\n'.join(SMTP_ANSWERS[0:i + 1]),
- 'Did not get expected answer from gateway.')
+ reply += yield self.getReply(line + '\r\n', proto, transport)
+ self.assertMatch(reply, '\r\n'.join(SMTP_ANSWERS),
+ 'Did not get expected answer from gateway.')
proto.setTimeout(None)
- def test_message_encrypt(self):
- """
- Test if message gets encrypted to destination email.
- """
- proto = SMTPFactory(
- u'anotheruser@leap.se',
- self._km, self._config['host'],
- self._config['port'],
- self._config['cert'], self._config['key'],
- self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
- fromAddr = Address(ADDRESS_2)
- dest = User(ADDRESS, 'gateway.leap.se', proto, ADDRESS)
- m = EncryptedMessage(
- fromAddr, dest, self._km, self._config['host'],
- self._config['port'], self._config['cert'], self._config['key'])
- for line in self.EMAIL_DATA[4:12]:
- m.lineReceived(line)
- # m.eomReceived() # this includes a defer, so we avoid calling it here
- m.lines.append('') # add a trailing newline
- # we need to call the following explicitelly because it was deferred
- # inside the previous method
- m._maybe_encrypt_and_sign()
- # assert structure of encrypted message
- self.assertTrue('Content-Type' in m._msg)
- self.assertEqual('multipart/encrypted', m._msg.get_content_type())
- self.assertEqual('application/pgp-encrypted',
- m._msg.get_param('protocol'))
- self.assertEqual(2, len(m._msg.get_payload()))
- self.assertEqual('application/pgp-encrypted',
- m._msg.get_payload(0).get_content_type())
- self.assertEqual('application/octet-stream',
- m._msg.get_payload(1).get_content_type())
- privkey = self._km.get_key(
- ADDRESS, openpgp.OpenPGPKey, private=True)
- decrypted = self._km.decrypt(
- m._msg.get_payload(1).get_payload(), privkey)
- self.assertEqual(
- '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n\r\n--\r\n' +
- 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n',
- decrypted,
- 'Decrypted text differs from plaintext.')
-
- def test_message_encrypt_sign(self):
- """
- Test if message gets encrypted to destination email and signed with
- sender key.
- """
- proto = SMTPFactory(
- u'anotheruser@leap.se',
- self._km, self._config['host'],
- self._config['port'],
- self._config['cert'], self._config['key'],
- self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
- user = User(ADDRESS, 'gateway.leap.se', proto, ADDRESS)
- fromAddr = Address(ADDRESS_2)
- m = EncryptedMessage(
- fromAddr, user, self._km, self._config['host'],
- self._config['port'], self._config['cert'], self._config['key'])
- for line in self.EMAIL_DATA[4:12]:
- m.lineReceived(line)
- # trigger encryption and signing
- # m.eomReceived() # this includes a defer, so we avoid calling it here
- m.lines.append('') # add a trailing newline
- # we need to call the following explicitelly because it was deferred
- # inside the previous method
- m._maybe_encrypt_and_sign()
- # assert structure of encrypted message
- self.assertTrue('Content-Type' in m._msg)
- self.assertEqual('multipart/encrypted', m._msg.get_content_type())
- self.assertEqual('application/pgp-encrypted',
- m._msg.get_param('protocol'))
- self.assertEqual(2, len(m._msg.get_payload()))
- self.assertEqual('application/pgp-encrypted',
- m._msg.get_payload(0).get_content_type())
- self.assertEqual('application/octet-stream',
- m._msg.get_payload(1).get_content_type())
- # decrypt and verify
- privkey = self._km.get_key(
- ADDRESS, openpgp.OpenPGPKey, private=True)
- pubkey = self._km.get_key(ADDRESS_2, openpgp.OpenPGPKey)
- decrypted = self._km.decrypt(
- m._msg.get_payload(1).get_payload(), privkey, verify=pubkey)
- self.assertEqual(
- '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n\r\n--\r\n' +
- 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n',
- decrypted,
- 'Decrypted text differs from plaintext.')
-
- def test_message_sign(self):
- """
- Test if message is signed with sender key.
- """
- # mock the key fetching
- self._km.fetch_keys_from_server = Mock(return_value=[])
- proto = SMTPFactory(
- u'anotheruser@leap.se',
- self._km, self._config['host'],
- self._config['port'],
- self._config['cert'], self._config['key'],
- self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
- user = User('ihavenopubkey@nonleap.se',
- 'gateway.leap.se', proto, ADDRESS)
- fromAddr = Address(ADDRESS_2)
- m = EncryptedMessage(
- fromAddr, user, self._km, self._config['host'],
- self._config['port'], self._config['cert'], self._config['key'])
- for line in self.EMAIL_DATA[4:12]:
- m.lineReceived(line)
- # trigger signing
- # m.eomReceived() # this includes a defer, so we avoid calling it here
- m.lines.append('') # add a trailing newline
- # we need to call the following explicitelly because it was deferred
- # inside the previous method
- m._maybe_encrypt_and_sign()
- # assert structure of signed message
- self.assertTrue('Content-Type' in m._msg)
- self.assertEqual('multipart/signed', m._msg.get_content_type())
- self.assertEqual('application/pgp-signature',
- m._msg.get_param('protocol'))
- self.assertEqual('pgp-sha512', m._msg.get_param('micalg'))
- # assert content of message
- self.assertEqual(
- '\r\n'.join(self.EMAIL_DATA[9:13]) + '\r\n--\r\n' +
- 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n',
- m._msg.get_payload(0).get_payload(decode=True))
- # assert content of signature
- self.assertTrue(
- m._msg.get_payload(1).get_payload().startswith(
- '-----BEGIN PGP SIGNATURE-----\n'),
- 'Message does not start with signature header.')
- self.assertTrue(
- m._msg.get_payload(1).get_payload().endswith(
- '-----END PGP SIGNATURE-----\n'),
- 'Message does not end with signature footer.')
- # assert signature is valid
- pubkey = self._km.get_key(ADDRESS_2, openpgp.OpenPGPKey)
- # replace EOL before verifying (according to rfc3156)
- signed_text = re.sub('\r?\n', '\r\n',
- m._msg.get_payload(0).as_string())
- self.assertTrue(
- self._km.verify(signed_text,
- pubkey,
- detached_sig=m._msg.get_payload(1).get_payload()),
- 'Signature could not be verified.')
-
+ @inlineCallbacks
def test_missing_key_rejects_address(self):
"""
Test if server rejects to send unencrypted when 'encrypted_only' is
True.
"""
# remove key from key manager
- pubkey = self._km.get_key(ADDRESS, openpgp.OpenPGPKey)
+ pubkey = yield self._km.get_key(ADDRESS, openpgp.OpenPGPKey)
pgp = openpgp.OpenPGPScheme(
self._soledad, gpgbinary=self.GPG_BINARY_PATH)
- pgp.delete_key(pubkey)
+ yield pgp.delete_key(pubkey)
# mock the key fetching
- self._km.fetch_keys_from_server = Mock(return_value=[])
+ self._km._fetch_keys_from_server = Mock(
+ return_value=fail(errors.KeyNotFound()))
# prepare the SMTP factory
proto = SMTPFactory(
u'anotheruser@leap.se',
- self._km, self._config['host'],
- self._config['port'],
- self._config['cert'], self._config['key'],
- self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
+ self._km,
+ self._config['encrypted_only'],
+ outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0))
transport = proto_helpers.StringTransport()
proto.makeConnection(transport)
- proto.lineReceived(self.EMAIL_DATA[0] + '\r\n')
- proto.lineReceived(self.EMAIL_DATA[1] + '\r\n')
- proto.lineReceived(self.EMAIL_DATA[2] + '\r\n')
+ yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport)
+ yield self.getReply(self.EMAIL_DATA[1] + '\r\n', proto, transport)
+ reply = yield self.getReply(self.EMAIL_DATA[2] + '\r\n',
+ proto, transport)
# ensure the address was rejected
- lines = transport.value().rstrip().split('\n')
self.assertEqual(
- '550 Cannot receive for specified address',
- lines[-1],
+ '550 Cannot receive for specified address\r\n',
+ reply,
'Address should have been rejecetd with appropriate message.')
+ proto.setTimeout(None)
+ @inlineCallbacks
def test_missing_key_accepts_address(self):
"""
Test if server accepts to send unencrypted when 'encrypted_only' is
False.
"""
# remove key from key manager
- pubkey = self._km.get_key(ADDRESS, openpgp.OpenPGPKey)
+ pubkey = yield self._km.get_key(ADDRESS, openpgp.OpenPGPKey)
pgp = openpgp.OpenPGPScheme(
self._soledad, gpgbinary=self.GPG_BINARY_PATH)
- pgp.delete_key(pubkey)
+ yield pgp.delete_key(pubkey)
# mock the key fetching
- self._km.fetch_keys_from_server = Mock(return_value=[])
+ self._km._fetch_keys_from_server = Mock(
+ return_value=fail(errors.KeyNotFound()))
# prepare the SMTP factory with encrypted only equal to false
proto = SMTPFactory(
u'anotheruser@leap.se',
- self._km, self._config['host'],
- self._config['port'],
- self._config['cert'], self._config['key'],
- False).buildProtocol(('127.0.0.1', 0))
+ self._km,
+ False, outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0))
transport = proto_helpers.StringTransport()
proto.makeConnection(transport)
- proto.lineReceived(self.EMAIL_DATA[0] + '\r\n')
- proto.lineReceived(self.EMAIL_DATA[1] + '\r\n')
- proto.lineReceived(self.EMAIL_DATA[2] + '\r\n')
+ yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport)
+ yield self.getReply(self.EMAIL_DATA[1] + '\r\n', proto, transport)
+ reply = yield self.getReply(self.EMAIL_DATA[2] + '\r\n',
+ proto, transport)
# ensure the address was accepted
- lines = transport.value().rstrip().split('\n')
self.assertEqual(
- '250 Recipient address accepted',
- lines[-1],
+ '250 Recipient address accepted\r\n',
+ reply,
'Address should have been accepted with appropriate message.')
+ proto.setTimeout(None)
+
+ def getReply(self, line, proto, transport):
+ proto.lineReceived(line)
+
+ if line[:4] not in ['HELO', 'MAIL', 'RCPT', 'DATA']:
+ return succeed("")
+
+ def check_transport(_):
+ reply = transport.value()
+ if reply:
+ transport.clear()
+ return succeed(reply)
+
+ d = Deferred()
+ d.addCallback(check_transport)
+ reactor.callLater(0, lambda: d.callback(None))
+ return d
+
+ return check_transport(None)
diff --git a/src/leap/mail/sync_hooks.py b/src/leap/mail/sync_hooks.py
new file mode 100644
index 0000000..8efbb7c
--- /dev/null
+++ b/src/leap/mail/sync_hooks.py
@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+# sync_hooks.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledad PostSync Hooks.
+
+Process every new document of interest after every soledad synchronization,
+using the hooks that soledad exposes via plugins.
+"""
+import logging
+
+from re import compile as regex_compile
+
+from zope.interface import implements
+from twisted.internet import defer
+from twisted.plugin import IPlugin
+from twisted.python import log
+
+from leap.soledad.client.interfaces import ISoledadPostSyncPlugin
+from leap.mail import constants
+
+logger = logging.getLogger(__name__)
+
+
+def _get_doc_type_preffix(s):
+ return s[:2]
+
+
+class MailProcessingPostSyncHook(object):
+ implements(IPlugin, ISoledadPostSyncPlugin)
+
+ META_DOC_PREFFIX = _get_doc_type_preffix(constants.METAMSGID)
+ watched_doc_types = (META_DOC_PREFFIX, )
+
+ _account = None
+ _pending_docs = []
+ _processing_deferreds = []
+
+ def process_received_docs(self, doc_id_list):
+ if self._has_configured_account():
+ process_fun = self._make_uid_index
+ else:
+ self._processing_deferreds = []
+ process_fun = self._queue_doc_id
+
+ for doc_id in doc_id_list:
+ if _get_doc_type_preffix(doc_id) in self.watched_doc_types:
+ log.msg("Mail post-sync hook: processing %s" % doc_id)
+ process_fun(doc_id)
+
+ return defer.gatherResults(self._processing_deferreds)
+
+ def set_account(self, account):
+ self._account = account
+ if account:
+ self._process_queued_docs()
+
+ def _has_configured_account(self):
+ return self._account is not None
+
+ def _queue_doc_id(self, doc_id):
+ self._pending_docs.append(doc_id)
+
+ def _make_uid_index(self, mdoc_id):
+ indexer = self._account.mbox_indexer
+ mbox_uuid = _get_mbox_uuid(mdoc_id)
+ if mbox_uuid:
+ chash = _get_chash_from_mdoc(mdoc_id)
+ logger.debug("Making index table for %s:%s" % (mbox_uuid, chash))
+ index_docid = constants.METAMSGID.format(
+ mbox_uuid=mbox_uuid.replace('-', '_'),
+ chash=chash)
+ # XXX could avoid creating table if I track which ones I already
+ # have seen -- but make sure *it's already created* before
+ # inserting the index entry!.
+ d = indexer.create_table(mbox_uuid)
+ d.addBoth(lambda _: indexer.insert_doc(mbox_uuid, index_docid))
+ self._processing_deferreds.append(d)
+
+ def _process_queued_docs(self):
+ assert(self._has_configured_account())
+ pending = self._pending_docs
+ log.msg("Mail post-sync hook: processing queued docs")
+
+ def remove_pending_docs(res):
+ self._pending_docs = []
+ return res
+
+ d = self.process_received_docs(pending)
+ d.addCallback(remove_pending_docs)
+ return d
+
+
+_mbox_uuid_regex = regex_compile(constants.METAMSGID_MBOX_RE)
+_mdoc_chash_regex = regex_compile(constants.METAMSGID_CHASH_RE)
+
+
+def _get_mbox_uuid(doc_id):
+ matches = _mbox_uuid_regex.findall(doc_id)
+ if matches:
+ return matches[0].replace('_', '-')
+
+
+def _get_chash_from_mdoc(doc_id):
+ matches = _mdoc_chash_regex.findall(doc_id)
+ if matches:
+ return matches[0]
diff --git a/src/leap/mail/tests/__init__.py b/src/leap/mail/tests/__init__.py
new file mode 100644
index 0000000..de0088f
--- /dev/null
+++ b/src/leap/mail/tests/__init__.py
@@ -0,0 +1,331 @@
+# -*- coding: utf-8 -*-
+# __init__.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Base classes and keys for leap.mail tests.
+"""
+import os
+import distutils.spawn
+from mock import Mock
+from twisted.internet.defer import gatherResults
+from twisted.trial import unittest
+
+
+from leap.soledad.client import Soledad
+from leap.keymanager import KeyManager
+from leap.keymanager.openpgp import OpenPGPKey
+
+
+from leap.common.testing.basetest import BaseLeapTest
+
+
+def _find_gpg():
+ gpg_path = distutils.spawn.find_executable('gpg')
+ return (os.path.realpath(gpg_path)
+ if gpg_path is not None else "/usr/bin/gpg")
+
+
+class TestCaseWithKeyManager(unittest.TestCase, BaseLeapTest):
+
+ GPG_BINARY_PATH = _find_gpg()
+
+ def setUp(self):
+ self.setUpEnv()
+
+ # setup our own stuff
+ address = 'leap@leap.se' # user's address in the form user@provider
+ uuid = 'leap@leap.se'
+ passphrase = u'123'
+ secrets_path = os.path.join(self.tempdir, 'secret.gpg')
+ local_db_path = os.path.join(self.tempdir, 'soledad.u1db')
+ server_url = 'http://provider/'
+ cert_file = ''
+
+ self._soledad = Soledad(
+ uuid,
+ passphrase,
+ secrets_path=secrets_path,
+ local_db_path=local_db_path,
+ server_url=server_url,
+ cert_file=cert_file,
+ syncable=False
+ )
+ return self._setup_keymanager(address)
+
+ def _setup_keymanager(self, address):
+ """
+ Set up Key Manager and return a Deferred that will be fired when done.
+ """
+ self._config = {
+ 'host': 'https://provider/',
+ 'port': 25,
+ 'username': address,
+ 'password': '<password>',
+ 'encrypted_only': True,
+ 'cert': u'src/leap/mail/smtp/tests/cert/server.crt',
+ 'key': u'src/leap/mail/smtp/tests/cert/server.key',
+ }
+
+ class Response(object):
+ status_code = 200
+ headers = {'content-type': 'application/json'}
+
+ def json(self):
+ return {'address': ADDRESS_2, 'openpgp': PUBLIC_KEY_2}
+
+ def raise_for_status(self):
+ pass
+
+ nickserver_url = '' # the url of the nickserver
+ self._km = KeyManager(address, nickserver_url, self._soledad,
+ ca_cert_path='', gpgbinary=self.GPG_BINARY_PATH)
+ self._km._fetcher.put = Mock()
+ self._km._fetcher.get = Mock(return_value=Response())
+
+ d1 = self._km.put_raw_key(PRIVATE_KEY, OpenPGPKey, ADDRESS)
+ d2 = self._km.put_raw_key(PRIVATE_KEY_2, OpenPGPKey, ADDRESS_2)
+ return gatherResults([d1, d2])
+
+ def tearDown(self):
+ self.tearDownEnv()
+
+
+# Key material for testing
+KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF"
+
+ADDRESS = 'leap@leap.se'
+
+PUBLIC_KEY = """
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.10 (GNU/Linux)
+
+mQINBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz
+iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO
+zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx
+irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT
+huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs
+d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g
+wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb
+hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv
+U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H
+T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i
+Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB
+tBxMZWFwIFRlc3QgS2V5IDxsZWFwQGxlYXAuc2U+iQI3BBMBCAAhBQJQvfnZAhsD
+BQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEC9FXigk0Y3fT7EQAKH3IuRniOpb
+T/DDIgwwjz3oxB/W0DDMyPXowlhSOuM0rgGfntBpBb3boezEXwL86NPQxNGGruF5
+hkmecSiuPSvOmQlqlS95NGQp6hNG0YaKColh+Q5NTspFXCAkFch9oqUje0LdxfSP
+QfV9UpeEvGyPmk1I9EJV/YDmZ4+Djge1d7qhVZInz4Rx1NrSyF/Tc2EC0VpjQFsU
+Y9Kb2YBBR7ivG6DBc8ty0jJXi7B4WjkFcUEJviQpMF2dCLdonCehYs1PqsN1N7j+
+eFjQd+hqVMJgYuSGKjvuAEfClM6MQw7+FmFwMyLgK/Ew/DttHEDCri77SPSkOGSI
+txCzhTg6798f6mJr7WcXmHX1w1Vcib5FfZ8vTDFVhz/XgAgArdhPo9V6/1dgSSiB
+KPQ/spsco6u5imdOhckERE0lnAYvVT6KE81TKuhF/b23u7x+Wdew6kK0EQhYA7wy
+7LmlaNXc7rMBQJ9Z60CJ4JDtatBWZ0kNrt2VfdDHVdqBTOpl0CraNUjWE5YMDasr
+K2dF5IX8D3uuYtpZnxqg0KzyLg0tzL0tvOL1C2iudgZUISZNPKbS0z0v+afuAAnx
+2pTC3uezbh2Jt8SWTLhll4i0P4Ps5kZ6HQUO56O+/Z1cWovX+mQekYFmERySDR9n
+3k1uAwLilJmRmepGmvYbB8HloV8HqwgguQINBFC9+dkBEAC0I/xn1uborMgDvBtf
+H0sEhwnXBC849/32zic6udB6/3Efk9nzbSpL3FSOuXITZsZgCHPkKarnoQ2ztMcS
+sh1ke1C5gQGms75UVmM/nS+2YI4vY8OX/GC/on2vUyncqdH+bR6xH5hx4NbWpfTs
+iQHmz5C6zzS/kuabGdZyKRaZHt23WQ7JX/4zpjqbC99DjHcP9BSk7tJ8wI4bkMYD
+uFVQdT9O6HwyKGYwUU4sAQRAj7XCTGvVbT0dpgJwH4RmrEtJoHAx4Whg8mJ710E0
+GCmzf2jqkNuOw76ivgk27Kge+Hw00jmJjQhHY0yVbiaoJwcRrPKzaSjEVNgrpgP3
+lXPRGQArgESsIOTeVVHQ8fhK2YtTeCY9rIiO+L0OX2xo9HK7hfHZZWL6rqymXdyS
+fhzh/f6IPyHFWnvj7Brl7DR8heMikygcJqv+ed2yx7iLyCUJ10g12I48+aEj1aLe
+dP7lna32iY8/Z0SHQLNH6PXO9SlPcq2aFUgKqE75A/0FMk7CunzU1OWr2ZtTLNO1
+WT/13LfOhhuEq9jTyTosn0WxBjJKq18lnhzCXlaw6EAtbA7CUwsD3CTPR56aAXFK
+3I7KXOVAqggrvMe5Tpdg5drfYpI8hZovL5aAgb+7Y5ta10TcJdUhS5K3kFAWe/td
+U0cmWUMDP1UMSQ5Jg6JIQVWhSwARAQABiQIfBBgBCAAJBQJQvfnZAhsMAAoJEC9F
+Xigk0Y3fRwsP/i0ElYCyxeLpWJTwo1iCLkMKz2yX1lFVa9nT1BVTPOQwr/IAc5OX
+NdtbJ14fUsKL5pWgW8OmrXtwZm1y4euI1RPWWubG01ouzwnGzv26UcuHeqC5orZj
+cOnKtL40y8VGMm8LoicVkRJH8blPORCnaLjdOtmA3rx/v2EXrJpSa3AhOy0ZSRXk
+ZSrK68AVNwamHRoBSYyo0AtaXnkPX4+tmO8X8BPfj125IljubvwZPIW9VWR9UqCE
+VPfDR1XKegVb6VStIywF7kmrknM1C5qUY28rdZYWgKorw01hBGV4jTW0cqde3N51
+XT1jnIAa+NoXUM9uQoGYMiwrL7vNsLlyyiW5ayDyV92H/rIuiqhFgbJsHTlsm7I8
+oGheR784BagAA1NIKD1qEO9T6Kz9lzlDaeWS5AUKeXrb7ZJLI1TTCIZx5/DxjLqM
+Tt/RFBpVo9geZQrvLUqLAMwdaUvDXC2c6DaCPXTh65oCZj/hqzlJHH+RoTWWzKI+
+BjXxgUWF9EmZUBrg68DSmI+9wuDFsjZ51BcqvJwxyfxtTaWhdoYqH/UQS+D1FP3/
+diZHHlzwVwPICzM9ooNTgbrcDzyxRkIVqsVwBq7EtzcvgYUyX53yG25Giy6YQaQ2
+ZtQ/VymwFL3XdUWV6B/hU4PVAFvO3qlOtdJ6TpE+nEWgcWjCv5g7RjXX
+=MuOY
+-----END PGP PUBLIC KEY BLOCK-----
+"""
+
+PRIVATE_KEY = """
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: GnuPG v1.4.10 (GNU/Linux)
+
+lQcYBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz
+iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO
+zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx
+irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT
+huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs
+d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g
+wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb
+hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv
+U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H
+T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i
+Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB
+AA/+JHtlL39G1wsH9R6UEfUQJGXR9MiIiwZoKcnRB2o8+DS+OLjg0JOh8XehtuCs
+E/8oGQKtQqa5bEIstX7IZoYmYFiUQi9LOzIblmp2vxOm+HKkxa4JszWci2/ZmC3t
+KtaA4adl9XVnshoQ7pijuCMUKB3naBEOAxd8s9d/JeReGIYkJErdrnVfNk5N71Ds
+FmH5Ll3XtEDvgBUQP3nkA6QFjpsaB94FHjL3gDwum/cxzj6pCglcvHOzEhfY0Ddb
+J967FozQTaf2JW3O+w3LOqtcKWpq87B7+O61tVidQPSSuzPjCtFF0D2LC9R/Hpky
+KTMQ6CaKja4MPhjwywd4QPcHGYSqjMpflvJqi+kYIt8psUK/YswWjnr3r4fbuqVY
+VhtiHvnBHQjz135lUqWvEz4hM3Xpnxydx7aRlv5NlevK8+YIO5oFbWbGNTWsPZI5
+jpoFBpSsnR1Q5tnvtNHauvoWV+XN2qAOBTG+/nEbDYH6Ak3aaE9jrpTdYh0CotYF
+q7csANsDy3JvkAzeU6WnYpsHHaAjqOGyiZGsLej1UcXPFMosE/aUo4WQhiS8Zx2c
+zOVKOi/X5vQ2GdNT9Qolz8AriwzsvFR+bxPzyd8V6ALwDsoXvwEYinYBKK8j0OPv
+OOihSR6HVsuP9NUZNU9ewiGzte/+/r6pNXHvR7wTQ8EWLcEIAN6Zyrb0bHZTIlxt
+VWur/Ht2mIZrBaO50qmM5RD3T5oXzWXi/pjLrIpBMfeZR9DWfwQwjYzwqi7pxtYx
+nJvbMuY505rfnMoYxb4J+cpRXV8MS7Dr1vjjLVUC9KiwSbM3gg6emfd2yuA93ihv
+Pe3mffzLIiQa4mRE3wtGcioC43nWuV2K2e1KjxeFg07JhrezA/1Cak505ab/tmvP
+4YmjR5c44+yL/YcQ3HdFgs4mV+nVbptRXvRcPpolJsgxPccGNdvHhsoR4gwXMS3F
+RRPD2z6x8xeN73Q4KH3bm01swQdwFBZbWVfmUGLxvN7leCdfs9+iFJyqHiCIB6Iv
+mQfp8F0IAOwSo8JhWN+V1dwML4EkIrM8wUb4yecNLkyR6TpPH/qXx4PxVMC+vy6x
+sCtjeHIwKE+9vqnlhd5zOYh7qYXEJtYwdeDDmDbL8oks1LFfd+FyAuZXY33DLwn0
+cRYsr2OEZmaajqUB3NVmj3H4uJBN9+paFHyFSXrH68K1Fk2o3n+RSf2EiX+eICwI
+L6rqoF5sSVUghBWdNegV7qfy4anwTQwrIMGjgU5S6PKW0Dr/3iO5z3qQpGPAj5OW
+ATqPWkDICLbObPxD5cJlyyNE2wCA9VVc6/1d6w4EVwSq9h3/WTpATEreXXxTGptd
+LNiTA1nmakBYNO2Iyo3djhaqBdWjk+EIAKtVEnJH9FAVwWOvaj1RoZMA5DnDMo7e
+SnhrCXl8AL7Z1WInEaybasTJXn1uQ8xY52Ua4b8cbuEKRKzw/70NesFRoMLYoHTO
+dyeszvhoDHberpGRTciVmpMu7Hyi33rM31K9epA4ib6QbbCHnxkWOZB+Bhgj1hJ8
+xb4RBYWiWpAYcg0+DAC3w9gfxQhtUlZPIbmbrBmrVkO2GVGUj8kH6k4UV6kUHEGY
+HQWQR0HcbKcXW81ZXCCD0l7ROuEWQtTe5Jw7dJ4/QFuqZnPutXVRNOZqpl6eRShw
+7X2/a29VXBpmHA95a88rSQsL+qm7Fb3prqRmuMCtrUZgFz7HLSTuUMR867QcTGVh
+cCBUZXN0IEtleSA8bGVhcEBsZWFwLnNlPokCNwQTAQgAIQUCUL352QIbAwULCQgH
+AwUVCgkICwUWAgMBAAIeAQIXgAAKCRAvRV4oJNGN30+xEACh9yLkZ4jqW0/wwyIM
+MI896MQf1tAwzMj16MJYUjrjNK4Bn57QaQW926HsxF8C/OjT0MTRhq7heYZJnnEo
+rj0rzpkJapUveTRkKeoTRtGGigqJYfkOTU7KRVwgJBXIfaKlI3tC3cX0j0H1fVKX
+hLxsj5pNSPRCVf2A5mePg44HtXe6oVWSJ8+EcdTa0shf03NhAtFaY0BbFGPSm9mA
+QUe4rxugwXPLctIyV4uweFo5BXFBCb4kKTBdnQi3aJwnoWLNT6rDdTe4/nhY0Hfo
+alTCYGLkhio77gBHwpTOjEMO/hZhcDMi4CvxMPw7bRxAwq4u+0j0pDhkiLcQs4U4
+Ou/fH+pia+1nF5h19cNVXIm+RX2fL0wxVYc/14AIAK3YT6PVev9XYEkogSj0P7Kb
+HKOruYpnToXJBERNJZwGL1U+ihPNUyroRf29t7u8flnXsOpCtBEIWAO8Muy5pWjV
+3O6zAUCfWetAieCQ7WrQVmdJDa7dlX3Qx1XagUzqZdAq2jVI1hOWDA2rKytnReSF
+/A97rmLaWZ8aoNCs8i4NLcy9Lbzi9QtornYGVCEmTTym0tM9L/mn7gAJ8dqUwt7n
+s24dibfElky4ZZeItD+D7OZGeh0FDuejvv2dXFqL1/pkHpGBZhEckg0fZ95NbgMC
+4pSZkZnqRpr2GwfB5aFfB6sIIJ0HGARQvfnZARAAtCP8Z9bm6KzIA7wbXx9LBIcJ
+1wQvOPf99s4nOrnQev9xH5PZ820qS9xUjrlyE2bGYAhz5Cmq56ENs7THErIdZHtQ
+uYEBprO+VFZjP50vtmCOL2PDl/xgv6J9r1Mp3KnR/m0esR+YceDW1qX07IkB5s+Q
+us80v5LmmxnWcikWmR7dt1kOyV/+M6Y6mwvfQ4x3D/QUpO7SfMCOG5DGA7hVUHU/
+Tuh8MihmMFFOLAEEQI+1wkxr1W09HaYCcB+EZqxLSaBwMeFoYPJie9dBNBgps39o
+6pDbjsO+or4JNuyoHvh8NNI5iY0IR2NMlW4mqCcHEazys2koxFTYK6YD95Vz0RkA
+K4BErCDk3lVR0PH4StmLU3gmPayIjvi9Dl9saPRyu4Xx2WVi+q6spl3ckn4c4f3+
+iD8hxVp74+wa5ew0fIXjIpMoHCar/nndsse4i8glCddINdiOPPmhI9Wi3nT+5Z2t
+9omPP2dEh0CzR+j1zvUpT3KtmhVICqhO+QP9BTJOwrp81NTlq9mbUyzTtVk/9dy3
+zoYbhKvY08k6LJ9FsQYySqtfJZ4cwl5WsOhALWwOwlMLA9wkz0eemgFxStyOylzl
+QKoIK7zHuU6XYOXa32KSPIWaLy+WgIG/u2ObWtdE3CXVIUuSt5BQFnv7XVNHJllD
+Az9VDEkOSYOiSEFVoUsAEQEAAQAP/1AagnZQZyzHDEgw4QELAspYHCWLXE5aZInX
+wTUJhK31IgIXNn9bJ0hFiSpQR2xeMs9oYtRuPOu0P8oOFMn4/z374fkjZy8QVY3e
+PlL+3EUeqYtkMwlGNmVw5a/NbNuNfm5Darb7pEfbYd1gPcni4MAYw7R2SG/57GbC
+9gucvspHIfOSfBNLBthDzmK8xEKe1yD2eimfc2T7IRYb6hmkYfeds5GsqvGI6mwI
+85h4uUHWRc5JOlhVM6yX8hSWx0L60Z3DZLChmc8maWnFXd7C8eQ6P1azJJbW71Ih
+7CoK0XW4LE82vlQurSRFgTwfl7wFYszW2bOzCuhHDDtYnwH86Nsu0DC78ZVRnvxn
+E8Ke/AJgrdhIOo4UAyR+aZD2+2mKd7/waOUTUrUtTzc7i8N3YXGi/EIaNReBXaq+
+ZNOp24BlFzRp+FCF/pptDW9HjPdiV09x0DgICmeZS4Gq/4vFFIahWctg52NGebT0
+Idxngjj+xDtLaZlLQoOz0n5ByjO/Wi0ANmMv1sMKCHhGvdaSws2/PbMR2r4caj8m
+KXpIgdinM/wUzHJ5pZyF2U/qejsRj8Kw8KH/tfX4JCLhiaP/mgeTuWGDHeZQERAT
+xPmRFHaLP9/ZhvGNh6okIYtrKjWTLGoXvKLHcrKNisBLSq+P2WeFrlme1vjvJMo/
+jPwLT5o9CADQmcbKZ+QQ1ZM9v99iDZol7SAMZX43JC019sx6GK0u6xouJBcLfeB4
+OXacTgmSYdTa9RM9fbfVpti01tJ84LV2SyL/VJq/enJF4XQPSynT/tFTn1PAor6o
+tEAAd8fjKdJ6LnD5wb92SPHfQfXqI84rFEO8rUNIE/1ErT6DYifDzVCbfD2KZdoF
+cOSp7TpD77sY1bs74ocBX5ejKtd+aH99D78bJSMM4pSDZsIEwnomkBHTziubPwJb
+OwnATy0LmSMAWOw5rKbsh5nfwCiUTM20xp0t5JeXd+wPVWbpWqI2EnkCEN+RJr9i
+7dp/ymDQ+Yt5wrsN3NwoyiexPOG91WQVCADdErHsnglVZZq9Z8Wx7KwecGCUurJ2
+H6lKudv5YOxPnAzqZS5HbpZd/nRTMZh2rdXCr5m2YOuewyYjvM757AkmUpM09zJX
+MQ1S67/UX2y8/74TcRF97Ncx9HeELs92innBRXoFitnNguvcO6Esx4BTe1OdU6qR
+ER3zAmVf22Le9ciXbu24DN4mleOH+OmBx7X2PqJSYW9GAMTsRB081R6EWKH7romQ
+waxFrZ4DJzZ9ltyosEJn5F32StyLrFxpcrdLUoEaclZCv2qka7sZvi0EvovDVEBU
+e10jOx9AOwf8Gj2ufhquQ6qgVYCzbP+YrodtkFrXRS3IsljIchj1M2ffB/0bfoUs
+rtER9pLvYzCjBPg8IfGLw0o754Qbhh/ReplCRTusP/fQMybvCvfxreS3oyEriu/G
+GufRomjewZ8EMHDIgUsLcYo2UHZsfF7tcazgxMGmMvazp4r8vpgrvW/8fIN/6Adu
+tF+WjWDTvJLFJCe6O+BFJOWrssNrrra1zGtLC1s8s+Wfpe+bGPL5zpHeebGTwH1U
+22eqgJArlEKxrfarz7W5+uHZJHSjF/K9ZvunLGD0n9GOPMpji3UO3zeM8IYoWn7E
+/EWK1XbjnssNemeeTZ+sDh+qrD7BOi+vCX1IyBxbfqnQfJZvmcPWpruy1UsO+aIC
+0GY8Jr3OL69dDQ21jueJAh8EGAEIAAkFAlC9+dkCGwwACgkQL0VeKCTRjd9HCw/+
+LQSVgLLF4ulYlPCjWIIuQwrPbJfWUVVr2dPUFVM85DCv8gBzk5c121snXh9Swovm
+laBbw6ate3BmbXLh64jVE9Za5sbTWi7PCcbO/bpRy4d6oLmitmNw6cq0vjTLxUYy
+bwuiJxWREkfxuU85EKdouN062YDevH+/YResmlJrcCE7LRlJFeRlKsrrwBU3BqYd
+GgFJjKjQC1peeQ9fj62Y7xfwE9+PXbkiWO5u/Bk8hb1VZH1SoIRU98NHVcp6BVvp
+VK0jLAXuSauSczULmpRjbyt1lhaAqivDTWEEZXiNNbRyp17c3nVdPWOcgBr42hdQ
+z25CgZgyLCsvu82wuXLKJblrIPJX3Yf+si6KqEWBsmwdOWybsjygaF5HvzgFqAAD
+U0goPWoQ71PorP2XOUNp5ZLkBQp5etvtkksjVNMIhnHn8PGMuoxO39EUGlWj2B5l
+Cu8tSosAzB1pS8NcLZzoNoI9dOHrmgJmP+GrOUkcf5GhNZbMoj4GNfGBRYX0SZlQ
+GuDrwNKYj73C4MWyNnnUFyq8nDHJ/G1NpaF2hiof9RBL4PUU/f92JkceXPBXA8gL
+Mz2ig1OButwPPLFGQhWqxXAGrsS3Ny+BhTJfnfIbbkaLLphBpDZm1D9XKbAUvdd1
+RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc=
+=JTFu
+-----END PGP PRIVATE KEY BLOCK-----
+"""
+
+ADDRESS_2 = 'anotheruser@leap.se'
+
+PUBLIC_KEY_2 = """
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.10 (GNU/Linux)
+
+mI0EUYwJXgEEAMbTKHuPJ5/Gk34l9Z06f+0WCXTDXdte1UBoDtZ1erAbudgC4MOR
+gquKqoj3Hhw0/ILqJ88GcOJmKK/bEoIAuKaqlzDF7UAYpOsPZZYmtRfPC2pTCnXq
+Z1vdeqLwTbUspqXflkCkFtfhGKMq5rH8GV5a3tXZkRWZhdNwhVXZagC3ABEBAAG0
+IWFub3RoZXJ1c2VyIDxhbm90aGVydXNlckBsZWFwLnNlPoi4BBMBAgAiBQJRjAle
+AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRB/nfpof+5XWotuA/4tLN4E
+gUr7IfLy2HkHAxzw7A4rqfMN92DIM9mZrDGaWRrOn3aVF7VU1UG7MDkHfPvp/cFw
+ezoCw4s4IoHVc/pVlOkcHSyt4/Rfh248tYEJmFCJXGHpkK83VIKYJAithNccJ6Q4
+JE/o06Mtf4uh/cA1HUL4a4ceqUhtpLJULLeKo7iNBFGMCV4BBADsyQI7GR0wSAxz
+VayLjuPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQt
+Z/hwcLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63
+yuRe94WenT1RJd6xU1aaUff4rKizuQARAQABiJ8EGAECAAkFAlGMCV4CGwwACgkQ
+f536aH/uV1rPZQQAqCzRysOlu8ez7PuiBD4SebgRqWlxa1TF1ujzfLmuPivROZ2X
+Kw5aQstxgGSjoB7tac49s0huh4X8XK+BtJBfU84JS8Jc2satlfwoyZ35LH6sDZck
+I+RS/3we6zpMfHs3vvp9xgca6ZupQxivGtxlJs294TpJorx+mFFqbV17AzQ=
+=Thdu
+-----END PGP PUBLIC KEY BLOCK-----
+"""
+
+PRIVATE_KEY_2 = """
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: GnuPG v1.4.10 (GNU/Linux)
+
+lQHYBFGMCV4BBADG0yh7jyefxpN+JfWdOn/tFgl0w13bXtVAaA7WdXqwG7nYAuDD
+kYKriqqI9x4cNPyC6ifPBnDiZiiv2xKCALimqpcwxe1AGKTrD2WWJrUXzwtqUwp1
+6mdb3Xqi8E21LKal35ZApBbX4RijKuax/BleWt7V2ZEVmYXTcIVV2WoAtwARAQAB
+AAP7BLuSAx7tOohnimEs74ks8l/L6dOcsFQZj2bqs4AoY3jFe7bV0tHr4llypb/8
+H3/DYvpf6DWnCjyUS1tTnXSW8JXtx01BUKaAufSmMNg9blKV6GGHlT/Whe9uVyks
+7XHk/+9mebVMNJ/kNlqq2k+uWqJohzC8WWLRK+d1tBeqDsECANZmzltPaqUsGV5X
+C3zszE3tUBgptV/mKnBtopKi+VH+t7K6fudGcG+bAcZDUoH/QVde52mIIjjIdLje
+uajJuHUCAO1mqh+vPoGv4eBLV7iBo3XrunyGXiys4a39eomhxTy3YktQanjjx+ty
+GltAGCs5PbWGO6/IRjjvd46wh53kzvsCAO0J97gsWhzLuFnkxFAJSPk7RRlyl7lI
+1XS/x0Og6j9XHCyY1OYkfBm0to3UlCfkgirzCYlTYObCofzdKFIPDmSqHbQhYW5v
+dGhlcnVzZXIgPGFub3RoZXJ1c2VyQGxlYXAuc2U+iLgEEwECACIFAlGMCV4CGwMG
+CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEH+d+mh/7ldai24D/i0s3gSBSvsh
+8vLYeQcDHPDsDiup8w33YMgz2ZmsMZpZGs6fdpUXtVTVQbswOQd8++n9wXB7OgLD
+izgigdVz+lWU6RwdLK3j9F+Hbjy1gQmYUIlcYemQrzdUgpgkCK2E1xwnpDgkT+jT
+oy1/i6H9wDUdQvhrhx6pSG2kslQst4qjnQHYBFGMCV4BBADsyQI7GR0wSAxzVayL
+juPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQtZ/hw
+cLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63yuRe
+94WenT1RJd6xU1aaUff4rKizuQARAQABAAP9EyElqJ3dq3EErXwwT4mMnbd1SrVC
+rUJrNWQZL59mm5oigS00uIyR0SvusOr+UzTtd8ysRuwHy5d/LAZsbjQStaOMBILx
+77TJveOel0a1QK0YSMF2ywZMCKvquvjli4hAtWYz/EwfuzQN3t23jc5ny+GqmqD2
+3FUxLJosFUfLNmECAO9KhVmJi+L9dswIs+2Dkjd1eiRQzNOEVffvYkGYZyKxNiXF
+UA5kvyZcB4iAN9sWCybE4WHZ9jd4myGB0MPDGxkCAP1RsXJbbuD6zS7BXe5gwunO
+2q4q7ptdSl/sJYQuTe1KNP5d/uGsvlcFfsYjpsopasPjFBIncc/2QThMKlhoEaEB
+/0mVAxpT6SrEvUbJ18z7kna24SgMPr3OnPMxPGfvNLJY/Xv/A17YfoqjmByCvsKE
+JCDjopXtmbcrZyoEZbEht9mko4ifBBgBAgAJBQJRjAleAhsMAAoJEH+d+mh/7lda
+z2UEAKgs0crDpbvHs+z7ogQ+Enm4EalpcWtUxdbo83y5rj4r0TmdlysOWkLLcYBk
+o6Ae7WnOPbNIboeF/FyvgbSQX1POCUvCXNrGrZX8KMmd+Sx+rA2XJCPkUv98Hus6
+THx7N776fcYHGumbqUMYrxrcZSbNveE6SaK8fphRam1dewM0
+=a5gs
+-----END PGP PRIVATE KEY BLOCK-----
+"""
diff --git a/src/leap/mail/tests/common.py b/src/leap/mail/tests/common.py
new file mode 100644
index 0000000..6ef5d17
--- /dev/null
+++ b/src/leap/mail/tests/common.py
@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+# common.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Common utilities for testing Soledad.
+"""
+import os
+import shutil
+import tempfile
+
+from twisted.internet import defer
+from twisted.trial import unittest
+
+from leap.common.testing.basetest import BaseLeapTest
+from leap.soledad.client import Soledad
+
+# TODO move to common module, or Soledad itself
+# XXX remove duplication
+
+TEST_USER = "testuser@leap.se"
+TEST_PASSWD = "1234"
+
+
+def _initialize_soledad(email, gnupg_home, tempdir):
+ """
+ Initializes soledad by hand
+
+ :param email: ID for the user
+ :param gnupg_home: path to home used by gnupg
+ :param tempdir: path to temporal dir
+ :rtype: Soledad instance
+ """
+
+ uuid = "foobar-uuid"
+ passphrase = u"verysecretpassphrase"
+ secret_path = os.path.join(tempdir, "secret.gpg")
+ local_db_path = os.path.join(tempdir, "soledad.u1db")
+ server_url = "https://provider"
+ cert_file = ""
+
+ soledad = Soledad(
+ uuid,
+ passphrase,
+ secret_path,
+ local_db_path,
+ server_url,
+ cert_file,
+ syncable=False)
+
+ return soledad
+
+
+class SoledadTestMixin(unittest.TestCase, BaseLeapTest):
+ """
+ It is **VERY** important that this base is added *AFTER* unittest.TestCase
+ """
+
+ def setUp(self):
+ self.results = []
+
+ self.setUpEnv()
+
+ # Soledad: config info
+ self.gnupg_home = "%s/gnupg" % self.tempdir
+ self.email = 'leap@leap.se'
+
+ # initialize soledad by hand so we can control keys
+ self._soledad = _initialize_soledad(
+ self.email,
+ self.gnupg_home,
+ self.tempdir)
+
+ return defer.succeed(True)
+
+ def tearDown(self):
+ """
+ tearDown method called after each test.
+ """
+ self.results = []
+ try:
+ self._soledad.close()
+ except Exception:
+ print "ERROR WHILE CLOSING SOLEDAD"
+ # logging.exception(exc)
+ finally:
+ self.tearDownEnv()
diff --git a/src/leap/mail/tests/rfc822.message b/src/leap/mail/tests/rfc822.message
new file mode 100644
index 0000000..ee97ab9
--- /dev/null
+++ b/src/leap/mail/tests/rfc822.message
@@ -0,0 +1,86 @@
+Return-Path: <twisted-commits-admin@twistedmatrix.com>
+Delivered-To: exarkun@meson.dyndns.org
+Received: from localhost [127.0.0.1]
+ by localhost with POP3 (fetchmail-6.2.1)
+ for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST)
+Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105])
+ by intarweb.us (Postfix) with ESMTP id 4A4A513EA4
+ for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST)
+Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com)
+ by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian))
+ id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600
+Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian))
+ id 18w63j-0007VK-00
+ for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600
+To: twisted-commits@twistedmatrix.com
+From: etrepum CVS <etrepum@twistedmatrix.com>
+Reply-To: twisted-python@twistedmatrix.com
+X-Mailer: CVSToys
+Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com>
+Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up.
+Sender: twisted-commits-admin@twistedmatrix.com
+Errors-To: twisted-commits-admin@twistedmatrix.com
+X-BeenThere: twisted-commits@twistedmatrix.com
+X-Mailman-Version: 2.0.11
+Precedence: bulk
+List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help>
+List-Post: <mailto:twisted-commits@twistedmatrix.com>
+List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
+ <mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe>
+List-Id: <twisted-commits.twistedmatrix.com>
+List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
+ <mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe>
+List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/>
+Date: Thu, 20 Mar 2003 13:50:39 -0600
+
+Modified files:
+Twisted/twisted/python/rebuild.py 1.19 1.20
+
+Log message:
+rebuild now works on python versions from 2.2.0 and up.
+
+
+ViewCVS links:
+http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted
+
+Index: Twisted/twisted/python/rebuild.py
+diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20
+--- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003
++++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003
+@@ -206,15 +206,27 @@
+ clazz.__dict__.clear()
+ clazz.__getattr__ = __getattr__
+ clazz.__module__ = module.__name__
++ if newclasses:
++ import gc
++ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2):
++ hasBrokenRebuild = 1
++ gc_objects = gc.get_objects()
++ else:
++ hasBrokenRebuild = 0
+ for nclass in newclasses:
+ ga = getattr(module, nclass.__name__)
+ if ga is nclass:
+ log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass))
+ else:
+- import gc
+- for r in gc.get_referrers(nclass):
+- if isinstance(r, nclass):
++ if hasBrokenRebuild:
++ for r in gc_objects:
++ if not getattr(r, '__class__', None) is nclass:
++ continue
+ r.__class__ = ga
++ else:
++ for r in gc.get_referrers(nclass):
++ if getattr(r, '__class__', None) is nclass:
++ r.__class__ = ga
+ if doLog:
+ log.msg('')
+ log.msg(' (fixing %s): ' % str(module.__name__))
+
+
+_______________________________________________
+Twisted-commits mailing list
+Twisted-commits@twistedmatrix.com
+http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits
diff --git a/src/leap/mail/tests/rfc822.multi-minimal.message b/src/leap/mail/tests/rfc822.multi-minimal.message
new file mode 100644
index 0000000..582297c
--- /dev/null
+++ b/src/leap/mail/tests/rfc822.multi-minimal.message
@@ -0,0 +1,16 @@
+Content-Type: multipart/mixed; boundary="===============6203542367371144092=="
+MIME-Version: 1.0
+Subject: [TEST] 010 - Inceptos cum lorem risus congue
+From: testmailbitmaskspam@gmail.com
+To: test_c5@dev.bitmask.net
+
+--===============6203542367371144092==
+Content-Type: text/plain; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+
+Howdy from python!
+The subject: [TEST] 010 - Inceptos cum lorem risus congue
+Current date & time: Wed Jan 8 16:36:21 2014
+Trying to attach: []
+--===============6203542367371144092==--
diff --git a/src/leap/mail/tests/rfc822.multi-nested.message b/src/leap/mail/tests/rfc822.multi-nested.message
new file mode 100644
index 0000000..694bef5
--- /dev/null
+++ b/src/leap/mail/tests/rfc822.multi-nested.message
@@ -0,0 +1,619 @@
+From: TEST <test.bitmask@example.com>
+Content-Type: multipart/alternative;
+ boundary="Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8"
+X-Smtp-Server: smtp.example.com:test.bitmask
+Subject: test simple attachment
+X-Universally-Unique-Identifier: 0ea1b4b2-cdb8-43c3-b54c-dc88a19c6e0a
+Date: Wed, 8 Jul 2015 04:25:56 +0900
+Message-Id: <47278179-628A-43F5-95C9-BC7E1753C521@example.com>
+To: test_alpha14_001@dev.bitmask.net
+Mime-Version: 1.0 (Apple Message framework v1251.1)
+
+
+--Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8
+Content-Transfer-Encoding: 7bit
+Content-Type: text/plain;
+ charset=us-ascii
+
+this is a simple attachment
+--Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8
+Content-Type: multipart/related;
+ type="text/html";
+ boundary="Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B"
+
+
+--Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B
+Content-Transfer-Encoding: 7bit
+Content-Type: text/html;
+ charset=us-ascii
+
+<html><head></head><body style="word-wrap: break-word; -webkit-nbsp-mode: space; -webkit-line-break: after-white-space; ">this is a simple attachment<img height="286" width="300" apple-width="yes" apple-height="yes" id="fd3d0c89-709d-419f-b293-a6827f75c8d4" src="cid:163B7957-4342-485F-8FD6-D46A4A53A2C1"></body></html>
+--Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B
+Content-Transfer-Encoding: base64
+Content-Disposition: inline;
+ filename="saing_ergol.jpg"
+Content-Type: image/jpg;
+ x-mac-hide-extension=yes;
+ x-unix-mode=0600;
+ name="saint_ergol.jpg"
+Content-Id: <163B7957-4342-485F-8FD6-D46A4A53A2C1>
+
+/9j/4AAQSkZJRgABAQEAYABgAAD/4QCURXhpZgAASUkqAAgAAAACADEBAgALAAAAJgAAAGmHBAAB
+AAAAMgAAAAAAAABQaWNhc2EgMy4wAAAEAAKgBAABAAAALAEAAAOgBAABAAAAHgEAAACQBwAEAAAA
+MDIxMAWgBAABAAAAaAAAAAAAAAACAAEAAgAFAAAAhgAAAAIABwAEAAAAMDEwMAAAAAAgICAgAAD/
+7QAcUGhvdG9zaG9wIDMuMAA4QklNBAQAAAAAAAD//gAmRmlsZSB3cml0dGVuIGJ5IEFkb2JlIFBo
+b3Rvc2hvcKggNS4y/9sAQwAFAwQEBAMFBAQEBQUFBgcMCAcHBwcPCwsJDBEPEhIRDxERExYcFxMU
+GhURERghGBodHR8fHxMXIiQiHiQcHh8e/9sAQwEFBQUHBgcOCAgOHhQRFB4eHh4eHh4eHh4eHh4e
+Hh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4e/8AAEQgBHgEsAwEiAAIRAQMRAf/E
+AB0AAAEFAQEBAQAAAAAAAAAAAAUCAwQGBwgAAQn/xABQEAABAwIEAwUFBQUGBAQFAgcCAwQFARIA
+BhETFCEiBzEyQVEVI0JSYSQzYnGBCBZDcqElNIKRkqJTscHRRGOy0jVUc4PhF2SkwsPT4vDy/8QA
+FAEBAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/ANW2
+3RMhJzJOiUK4up+4Lv8A4JUt/rgjBZfmN1JRBsaDchEklXfSaQ15GkdfFX6YteR4tNtDiuuCC7gh
+6iHpt89O+vLFg+ARv8WAQ0bizSSaD1CkNo4fuL58J2/BdhXSJ+PAex64sL+PCDTEjK47vw4D1qnz
+49b1+Pw4bMrf+IOFWj/NgPdV5Wn/AIceSUuMus/5sKMRw2Y+Lo6i8OAV4kiETtL5sMGpaACofUXh
+/F+WPN1vvei0R/1YV7tRLpC4fxYBvcT2hJMztIenxXD/AP76YVvCICQnd+G/COpMNwvu/iMvh/Om
+Bkgs63QXYtk+IIv4h9FuAJm4+0XCdpdN4+Lp+v8A3xGBZYXBJ3mVt1nzW/T1w6qmperb0koNoeG8
+f+44Emo6QBJN9YoSnVv+FJUvr5p4CVxBKXEmsdxdNvwCPpT5S+mIvGSBN1d4zbOkytuLqEgH1p/1
+phDtZwg1VIt+0LbwEKXjr6j/ABKfWmGGjhQmSvE/CVqVp3Xaf8Mq+Gv0rgLEyUW4JJVT4vx3D+mv
+w4dN04IVyRv6Oq6y4fr3YQ0IXMeLkQMRtutIOr/LAqY4VNJqMgYJo2/iG3/TgGJDMyyCS5JuQEh6
+UhctlQH9a24EuJbjnFzGShyISuV3HiqRidPpUfBhSpEoBptjfKIp+7AUXgFdp8WhjiM3FYd1y5fr
+kj/+7igI0h/FZXATmSz7dIkW0omSfUYIyQGCo18x6sNHMSjFVVZ28mEESK1tcwFURL5a2692BgLM
+RBJMloRe4iIbQWQJL8Q99448kzjyVVJNtFqXCJf/ABJUAX+nOnTgGpiWeO0l0F5K5NTqVFRFZsqg
+VO46Wj4cRop8pxC7FN4+EhSEr05gDScj+C8qFStPyxMWEuldBaUUWTIhDZeJKkkVP4daENLh9MMG
+Kj5IWOy6ZFaSi7ZaNAyIfmoQlTAS28lME0/vkpvUuFuuTluYpDX/AIlL+/DDSQmkFVV1JibJqmRE
+ZKMwIkD866hWuqeGHCjdNp7SF4xJO0UUC9lGIL+VqlKH5YdSKNjY8XJLNdkfERM1ht15W1pd4cA0
+bjMSkO1TXkl7R98u54ZUCS6unwjzGuGnr6U2lVBklEBUEb7XLojSLyU0t8OBTh8oSSCaaLVNqur0
+ELlZK3q0tKlRLpri6/um4FoRIGgIiQ2JEe6QpU/g+XTrgKe79rPrU1JWR4xMbiJs8dW20LkuOo86
+euLXkyPnEGntKSknwrOfAgm5JUEh7q8lNO/lXBiJg4tsDYkGa+8gO4Jl/wCmnV3YK+8USVLZ6vh6
+PX9cA3/aCaVvtJdRa3wiil1f7sKbuH3FluP1yH+QLS8vmx401ib7lnw22eHCVk1t0RFY+rqO0BL/
+AK+eAc4p0nu778y+XoDp/rhYLOCMS4ldNNPqUuttLUfzxBj94jESbGSY9VpIj1f7sPpKOBcEPiR8
+I22l3eeAfCSJNv8AfXeK0yWG78PnhJksropxT1O6mtorDp/zwhuKyiRD4VBL8FuHiJ0VdRqadPl6
+cBToqUKPcIFwx2q9JG5tAiL5SKnQWL5GPEXzIXYgpaXzYzA0U2J7DZYLrSIjK9iRad415bZV/wAW
+Lbld4oLtBMUdtuokV1yJCV/0INRL/PAWsxHCTG48KuIvD1Y8ZF04BIF1+D/Vh3wn4MNH81/+rHkl
+FLyuDAKtuAui7DG5aYiQdP8A6cP9VmEqjcA3dJCXTbgHbv8Adho07rh/04i++Td9R/Z/iH4h/LEq
+4f5R8WAauIlbbOr5sNJIrClcXVb8v/TEhZMd3cI/F0/LiPcKYffGPwl8PV8v0r9cB5VToLr/AAl/
+3w2bckw93Z7zxB4bvy+WuEGPujITu6rb7P8AbXDpimolaQWl8v4cBFSWTLpUW3xTK0yILSSLy19M
+J4cve9ZkXSN/iK38VPOmJjtPwiSxipb0mPUQ/wDfDTdmSYbdgW9V3i/1U/PADDTWEFSQC7cK4Osv
+Ll7s/g/LFbnSeEqKaaNyYn9qVIPCXounT/1ji7pR6nUQncRdRJWdKunnWnrhjg7URVH3dtxXl4kv
+yKvfT6VwDGWnlzUWy5nb4UNzquGnfYfxYmSo7bQiT8SfUIpmIjZX+bWlcDJXejW5Wsw2SIulO4ki
+9Knp1BX60w5BOJJTpK9dmRWmC3jQ/wDudyg4AYlH+9XJyz30yt6lGHr8VydKcsNfYU2/EpmDZQSF
+Mi49VLp9DpUsWl6mok4FZAA92Nto3D0V/Lz9MQ9kSbiShuhu6SEljLqr9K0wFb2RJXiWjl7t/Akm
+5SPq86p1KlS/TDSqJCkkx3nztFUSWICYIkJD8vIeRUxMerJppCg5O1MS6QWATEvxDU9LcDHCm84J
+sotHEiRD0u0UhEiqPK6oFUqVwC3DFumqlILhHDtESaS7mNJIh8tK1DTDjRSPdpW3tUFhuG4X5pEk
+XzUpUudK4S9jfY2X9xcDQapl9pVbPFRJIfmr088KbpuvZiqyDx0u3QtUQJN4KpD+KnPUh591cAwH
+Ctm5Nr0C3SuMBmCucl8wal34YVki4dqgUk6+0+AikgK4e+xTX4sSjGYUZIERrkiJEoR3pWkdS5VD
+UuRU+XEVVGSkHpFs76atyZbkakQql5jqBVtL64B9wimQCuKxjtD1A5lfuh8k66d419cXBJwptIKc
+GaC20IkIgJj4eY618WBOX4V42PcI1+DBINseGAC8VemulcWAy6/dhaJXdI2Db/XAI6VQLYDbLwkJ
+AIkWvnzwwkTwXApkshaVvQNtwjTyxISIdolFPDbdaVn/AErjLc5zXs3tryVEoOTLjUFxVSTBK0Q6
+a3F1YDVne8QCQrAh8XgHqH64jKl96oRgIj1FcAlb+euHHAjYkmR3DddulZ4qfrjAP2jc/SBOEsoZ
+QACcO1RRVGwS4nXltAdK8vxYAjCdpEl2g9qbWCyS5XbQMWRLSL4Ww3LlTltalTQRrz6sbU3RtutC
+34hGwBIv9uM77Gsh/uLk9nHqAHHEJOHRpoo+Ov47uqlMXreWvJRS8fitLa9/08q63eeAmfcJWqBc
+RfDYHh/yw8KgFTVR0FS8/By/pga7HfAVCcroDd0j7rw/L4sRyXSGtqbghEeVKJrJW/8AqwAqHb9D
+wRBBNFUusGjwXIJLXaXCmoOtLq9/PEHedRbhrw14iToU3IIgQKiXz7ZXUrSv81MTNtuILryiICok
+vtkT1htEvr8eoUpX/Fgrl8dj3fGLrt0uoy4wHNw1/PUsBZEpBO9dN2sCbhLqMRO60fLyxNPwfNis
+N3DUnBKNt9QkyK4rFi6q/StcSmT5RNJK5suSZdV4omQ3fStfLAGrbvEGEqjaZW9JW3Ya4gbBu3BI
+urqAsKbum6hlaYEVvhE7iup5aUwDqRDtdXi+LCjttx4BEgust+a3Dd2302GWAcMRsu+L/wBOGNxM
+XBCXT8mJXwXYQqnuWlZ1D4S+IcBD3hTPpDcu6enqtH6YdNNEjJT4bbbvFcP5YYVuG5AQuIuoPhu9
+dPr9MeScEKQWmChEPSRBbd6/kXrgPJKfZ/uTEfiEerp/7Y8BCQDaBl8tvi/w4TasQEW9bcQiN3hH
+/tXD7RNTiOo9serpEOkvzwDrdEhD357iheIvw4UZWmPXaNuFH8OB0hxCjhL7xAfDfvANuAlGp8Im
+Al8JYGPVhbRi4qLIcQp8qxCJfrpX/liY7uF2P2k0xttHrH/rgE4UFRJUk3LohTIRsTchd/lgCPHN
+xSVInIbJJCVhH0W07x8OJzQkRbjsBanaJdJ9PPu0xXjUUct0k0wXXJUesSRAwVG3zqI9OFRjxxGp
+L3AuLceoS4AhtKnLnpTSuAKO+OT30yNclFLRSuciJF+tB6cQnpPBb2+/Hbt97xI3fkXThYSm4xIn
+exxCdqgincHTXnd1/wBcQwkmq6tzaYBe64TMTStH/wAutNP64CO4cOC3RHilRTESEBcpF0111tuH
+nXEF66TFUSdn94NoGswEwIbddCqBU0KmIOY2/wDYSqbYFyRQIVC4ZEF10Du8VtB1KnPvwwkxWsJy
+ViZKiKaqAmq2FyFP4yfVpgCDRwjcP2xqoip4hK9Akh/lOh0KnpiU4FYj8Cgo23CAtgXD86VAg/yx
+GbxrjhLWz90797uAuRi5EUqFrs1Guv6Urh9uiixVEk2xoN1BJR4AomgSRU7ipTy78AObrM7+GUbM
+dsi3PdrEBEVC5EmBjpQ6fniL7NeNJAlF5VdMiPbQ3rhSXHv+8Ctaa0/lwb2SFwkmJm7akO8oCljw
+efcVKVpUqaYRw6KIDcFpeIQbLKobpV53CmdbaF9MBKhJoU24i5WXXT+Ix97bp6+Gv+3BoH0W52iY
+uQUEhu6ekuX0rirTCKZHuF7sRG5VdRtZ/wDxCendiGqo3kFUl00eGFoQ3P8AeErht15Kj02187sB
+fEU00EiTU6iH8HixgvbLJIse1iFmbwIotdoN42ltCtuUK6nTrroPnjaoeWTctBtPc6RHqWSEhL8v
++WOY/wBo1qnJZgmpCPA3b5SRZxjVIQEw3QuKpnpy+LAbJ2wdozHKGWnjRMFymFUi4YSRG0ip6dVe
+eMr/AGX8llO5gddpE62kRTTVL2ULvrAhLxndypr+mM37SE3Ehn1DK+aJv3iSqZTrps2E0m2gD9zQ
+afJprSvcXPHQX/6pdnOR4GHyvl8150U0uGQZsAWclcI8x8VeeA15X3jgSTR27vFuIjaWn+L+mBzg
+nC73qZqKIh4fcjbb6eLA7JMo+norjpbKr2H1+6auESuKlfi/pixNExELtlBsiXhSURtL/wDGAU4J
+QgEU23u+nw22jiNSig1KgpbdNfD0f+3D52iGwIAJfONvhr9MOJN0doOIVQUU05lshz/pgKvk8lFz
+VUFmCagpEo2BQ1hEdf4ZXjTSmJwC+GPXJjv9VvUpduj1cxpy50wJdSDx63SU3uJJUhECUDqJGnMh
+U0KminLBSPTeNGQoILe5V6QcE2L3WpcwL3uAfVFwrH+Bds8EiEDELhtr8NfUa4fjGJF9rXRQFS0h
+JIWxDcXxFrTCQZkTRW6xsI3dSiJ9I0/I+6uByKg2L75tbum0RWVSIS+bnrgH3AppgK6CKaZJ+7Ax
+WVQO30PpxKSdPOLSUQDdTtLdEdpX8+dCuL/LEe5w2cWpuXyjhRK4myblJcSD56ajTnhi4SbjuGgT
+dQvdEo2JIhKnKo3AXTWn9cBa4yQRcpGRGA+ERHqD+h0piZ8H8uK9GLcc3VUFYE9vwCR7/VTz0rQS
+wdaERNxu6VLbiEfDgFbnXb4RHqLouwoyHpK/xYYcbZJXXn09XThO4NgkXhU8VuAfcJpqBtl1CXVg
+c4TtcESnURl/qKnd+VfriVdtq2j4bekvl/8A8cIaKIuTVtv3E+kx/wCX+H64BIbJN+pbcISJMiIL
+bvw1/wC+GnBOlHbXYM00RLqIfEJW/HTzH64mXeL8PUY/DhaRJkrb0XD8vit9MA4Y9YXdXy/zeuIb
+jb4hLrtHq6Cs/wAu/E8y6Pw4iqkmVwqI7lxeGzAQTFwKvUZkn/wyAOn/AHYrr1PfkEF1D21Ej3Ac
+popWkXop1f0xaXu8JpJoLbZXeKwekfm78AnCaKDhXrAhckN4rGAi5DuqQ/iwEG14mqSbYNxQUh4p
+mTYRBXXvIK0r54UDdmpHtXKCIL7dyYlsmNpV+A7aV7sLcTBNIx8pe64dLpEhRG9IaFpb4ueKVlLO
+0LmLMBRbaSapzyAkso2EDteI15Xjofl50wF1cOkWjRIWPFfMYC5VAhGneVNR8OuIAOnTlVBBiG4S
+hWqr3pGC4V7+8qdWBj3bF17QjX/92LZJVS4dg689sqVu7/LD/tR0xkHTYTNNuyS60hWASI+/3fuu
+dNO/ALVUYk0JymsCaKDrhh3GYXCXmFbCr04cBTgXDoUwPbQEVthNYwND0LQhp04Y4ckHaBDYo82F
+FhDZExV10r16EPOmEx7MY+PSUXcroKLqk4VeEBCSQV52c7u/y9MAo3SjaQFpH3uZJ21EnI+6MHI9
+1fi13KUx5u+dNgERcgTe4UwdKXttoqeJI7ht/rgK92ZSP33Ky/Bqq7zoLBI0BEukkzGwvFjzdq+9
+sJJrtgFq594qwJyQk5DyWoJUPAGHr4V27qZURNOPRIiEyRFU0iEtPEFa1tuxFip5u+BUXZuouQbf
+AS3uhKo6X0opbdTzxYjhWq6RIIHc12OHFIvAqN1dddK06sAnDUk5Uk2KwXDcmVywmIhQvuq0IfP+
+bATm7N8pGKrw7BBR4ncLpSOW2CVV7+jToLX88VtkNsgKd5xcgr4mrkCZkRVHUxu+7V/K7ErMUoSG
+TYpPLrnhm6zpcXBX7H2ig60Cteq3XyxnPY/2mDmjKUxF9qMkbtvHKkK/EIjutgqdaCpRYdNa0r0+
+H4cBfXs5H5Fy4q9/u0ekVtt5ILqq17kaWUIVDrXlTSuMjnc0PMkuFc3yDMHeYph4LlJqp9+x1HSq
+NCDWu5pt8jtwf7WG62TnCGYM2vAkW7RcigOH6mz4qhXkuPz2/wAQe6uAfYpl192n57LtEzeCCka0
+K1mwcrEHWOltdaD1269/ngCGTOynOHaQD7NGf5J1Cx8ouKhxgomKq4jyEj0Hl3Y3jKnZ/k3LcU1a
+QmXmrZG7cMyRI1SK3kWteeDEYs3UcERGC/iESTvLp9MSDWETSQFsdviG5Eur6eLAIdtWqloi29yn
+aXUF135YlJC3La2D6bbQ9zb03eHDUg3UXO0um23q2S6S/wBWFJDa48B3CN1+zaN3n8WAYVaqL+Hq
+IS8JMwL/AA8ywpOrlMKDukNfOgtktNf9WPo7yO+oXSSnh9z1D/uwtFOhJ0qoNCL1FHl/zwFO9k7j
+Rdjw32pdUiExRRISGnKleQ+PTExu1TbGQuUTURFIRO6NMBL/AEfH+WJziFRcu1UyYIdXUJcMBjdT
+vLmV2EybNmRigIG2IhG4RBUbtC+YRrTAQ0nEgs0SYNHMcSbsis23JpKiPkHVXvpg+CaijcUxcuvl
+L3wL2/QvXXENgK3tAVOMkRuK1UU3IKj/AJV6ta1w6qXRwyjZdQSVtAl2Y2l5W9+v64CL7LT6uJZg
+O3aKSpMLbTu8N4UpqGPJM2qavwJ7he9Fs5VSC6heLSpYdVcC2aJJp8K2RHpvT3Q7tenSg1078Smj
+wrBUI/fWimKAvBu8XfoemAkt2ai0vxithDZaIltKXfioVPDgoBEoFyVlvV1YQyIiAeu64viMSIfp
+ywsFveql4huEREeq71wDlvw9H83iwwbVQTAkztG7q/CWF2rCJEO2RfD0fDhpJwN6qafwlcfuS/7Y
+BXD7Ye7C3DAD767wlaQl8wj5a/hxOBQSDx4S4RIvuz27vDb8vy4BhJQiNXcC23wl4ht+anqP0wwk
+jaZEhfb/AD+H8q+mJwC3TtH/AIf4+kdcJuTEBEQtTLp6Qu/5YBzq2hEvF8VuEqkXUJeH/EX/ACwO
+OQtVFAjQtUO26+20qfniUagqAuh19Nt1wWjbXz1wA6QTH3q5AHu/CWzcVv5V78cO/tF5mkn3aLIx
+t5sGrQxRsTAgAituvT07u/yx3ce2SVrYNzb8I+L+uMy7YOzGBzxDnxKJoPkklCQFFYrhOvxDbSv6
+0rgOaA7ZMwNsqOstyyISSyrNNuMin0iQcrhP1L8eAHYllOSzb2kRLaJM40RLiFV0zMSQAS1v1pXz
+7sCc8ZHlss5lGJcrA9HwpKiZCJaDqVNC0rTBPsizxIZCzGM3HxpySl3CGkodo7NedRGlO8qaYDtF
+2PAzo8MiabpISUVQUNUUnw28iHnbWtMQZNRvMmkPEgW6uK1yh2mx0LXqofw64g5S7RIHOzRV9ls9
+xqyt3+g+JYlUdbtKU8OJztZRo9Vkhc2oqJCNxbtr4bfS3kWAYVUTkGk0SgbbpRcW6pJgkREAc6KJ
+20uw5Jk4kIol48zUtS2WwiCw8SlQeevPSheuILJaQTj1Xz4EE1I+5RmgIB039wqXFStcDknDpCPQ
+UFtt2tVFJHoIeDMir1UqF3Td3YCHNvHSdqjFYF3X90YCpYXeXUioFerlXBaCJYdpdofDJqK7Krlb
+dI0CHxDSuultcQ527LcU1cvgaqEggTl0k5edSqp9G8G5b82AEnIZils1rwmVI105FNJNpvpgCSQg
+Y31PfCtev1wF3kM8ZbYvUkJZ5HNm5CooIEsiJidDLQuXPAKMzZE5tMUI+YQHiVxbGxTvG0R67yvr
+pSug4qmcOyOaQdrzcyftJwo1TRSjE1ituoWlDVLTrGmObZhHMmX5NWQ3jjnC6pJhwhiJX09RGtef
+0wHZMK+Yy3tDJahoC+kEifxxJgFhKp1tr4KaXW4wrKLHJ8tHv3c+/Xi3kI+X4tspq3SeJEqVqJqj
+omZUPXvxXYftokGhtXCDBBo+ZvE3aZprHtlaNqgCnSmlx0xdf2pURm8nwXaDle9zluYK1dIQtJo4
+Itba0p6nry+bAEOzDN2X88LSPZZma9eHWIih3ghyaHXwhUx8yH0xdVe0px2ayC+Uhio4W8Paja/m
+EkDVRt8YBWt35YwXKSOXYl6xEWexMKrp2MFrVV17kipRbcqVBS6vEHiHG3ZwJrm3ITybUh8uzWds
+otxbvOJuVbJJFzopS2lbyHTADFf2hnhNFZlTIy4sbiEFxniSBUaFp0jUuf1wYaftCErk8syM8mOj
+jWx7bhdaVtJL1rTq1PT6Y5omHHD5cXbOcvQ7aWc+8J4udV3TwVCvpVNGn3I2lrz8sPqsYtOPZsYJ
+zl17JCQtjBPdJVyZBWhFQ1BEQGn/AKsB1UH7RWU49ukhmCEn4VZVIVrVmxq9Fe4/5cSm/b92XruN
+v95F7i6R/s1a0tO7TXGB5oymcW+y3HlA5eczCqVq8EymCNddGo8ycOK0tAR05DrimRXGRsxPxooy
+jBNcdkmcY5ScpEN1dBWXu0Cn1wHbDTPGVZtuxdxs3HKIyglwt3SatvfpSv8AXBU3xt6AndbS2laU
+2fLHBEfGtyBiIg1zHFtECWkQjjJsbYSLXQlDtuL8sEY6NynJJG6mO0CQg16qVoDAk1XBIp/CNVB5
+FXTzwHfLgkWjvcXNBMi94S5ImIEXy60LH33K6pKILXCPVamsQld81taYHAoiVqHGAmoQ7hEm8MRI
+bvQywg953aKDyR2WxkIriiC5CVPhry10rrgJircrxUUsLxdZIiYjr9QqOE3Ji3LrNdr0ioCZkYjp
+3cq1oVP88BxcOFAJBNsxSTH3hoCCzUyG7xJlXQcE2ixEsQrhvolbwypbS+6NvrTrp+WAD5gtTNJB
+8bHqVJS29Ud0KDyGvf7z0w/Dvickk5E91MhEWwkYGqOg8xr7umLAYpkySacS6QvK3pMw+vnzw+yj
+yJ6q7XM/utkTvuEgu+SvxU+bADuIkuNSQbAZIiO4vaA+6O3y00xMZJuCtc8MumpaX3iJDb/u88Fg
+FuhbaiFw/wCrCjcCNokZ/wCgsBW5XjEw3E0UFOq0bgMSEfTShYU3U2zFSwLbeohckI8h76UrSuLE
+bpMQIusbvwFiCq8TUZELkwHc8XwiOndzr64Ac3dPHIAKJmV5WiqVq4/49KDg+BEKQ7h+8+IhC0eQ
+/rgY3dMyblw2wmsXjJMwEh/PElJ8iqAknYSalw2isFpFTv0/LAJSj1hcKqqLApu23iSJF/W7DUgJ
+IBxIo9Q9NxLEI/0w+ahDcuJmQlb4QEsOt3gk4Ftfa4EbiEukrfXTABVpJqmaSDkHS6aolaQgBj/W
+lK64aSISMUxNBQh6htvSIR/FrUqYLTEaMk32uhD8RIiZf1pgA7FRNVJB3Zw4+7ESvQ3fw6F0lXAH
+G6xEyEhO0RPrAguL+bkXdj0rJM4mHXdvjBNMEiUIhC278sVlw8GEaOncksxaIppXWFYNofmGla4w
+ftI7YGso1eSEei6QZsEPdEsaoiurUtEqbZV08XxYDMO0gX072tlGtHi7twuuLQFyMeo1CtU8uXfi
+29tfYf8Auy4/eDI58SzSFNNdgiZqqpHQaX6d3u6+eGP2QstupntT/eBf7SnG3EuQheIrGPLn3c+/
+HXOaBTFwTlN+u2WttIk3Ipd/cVR88Bz7+y/2fuo/L6+ZnYBFupJ1cglsqkIpCX3ZUup01r640aYb
+uIsCtA1FFnQqGxTckIthp3Kp1IS11xfmjxFCPtF4gXT1qk8G0jp5VpSvLFWNw8KbfO1DBRNNJNMQ
+RkrDH8SZGXd64BjbUdtHRXoLqSFok5JyBJKhTnbzGnOnngBl9jISUfLLsWdzVR4mmBuQNIeFEaCp
+Sp39Y8vTAztleKZd7MppQTTXFJmVwEaSlxqFyup81PWmMo7D83ZwhJOAybNrIO4GSSEUHKhkQpDX
+4aV88A7+0ApJZdmIwpts1dkquo4SapuSVQ2e8RKtaa/pjqPscnonNeQY6Zi0UGyKo2mgj0ikdOVR
+rypzw1nXIcPmnK/AkCBN10veKiFxEGt1LP8AFTXAWHZw/Zrld409sAgN3ELmodypDXl+dacsAW7T
+WakzCSMM2kgbFaKjxfwkklXlqNaefpjjjtDheEy5ttg2BTeJqM2ajkbyCpW7ltRvIirXW6qn6Yum
+YO1bicvyKDGedEirIkntCwVAHI1PSiZrH4Nfp4cV6dzQ6YuDXm8vQ79FdcU3C8iZqAkr82zSvvCp
+T1wGPzYqcaF3uyEtnYTRt2redvfX19caP2F9o0XFoyOQ88Gf7nTvTVVPxR61e5UK110/6Yoz2aUX
+lXT5tY2RUXEngCijYJ9VKbIUHSg24BSCwru1VEkQTTU8IfKNOXOn1wGvZ17OEsmZ9QUmlgex7tcS
+iiYPNpWSRrTQVRUMSpuXVG/F3/Z4lEYvtVQj1DfPWsw1KHkzWAOHFal9QBNQdL6UpXTw4pnY92kw
+q8Gl2adosX7VywfUwclaS8atQdakNa/DjXshZByu2ze17SkM+Qj/ACwxVTcoCmAJEJAJDQbB00Pn
+gMwzaz9jTs1lJMPZKjZ0omIRkUTlyulQtRGq6p17vSnw4qTduK/Zk8gpCbmxeRZE4OKbQgECVvxE
+4pW6nV3/AFxrL2Jku0bND6ZUWQfw71cmDAE5Uw4NZQ613KgBeQ1540fIWT4WJhyaZLeLuU5Z+LAT
+dnckg1bnQlST/CVuAweT7Jc9LysO0fxuVoseA41BVBZUeJCtLy3CqV1a8vw+LFn7MuyEnmeGK+b4
+eOUhZ2HN+q2ZOTQQZhTW3purUtdO+tca6xcOs2vppKQZoMlJmTGFaq9RAqxQG6ulfmrrin5tfOGs
+rcxm2rkZaWHLisWwAiXYx6HKtEbeq+lOoq4AZlfsty64exzR8xdTjqSQcroA593wzIPBubdt1alb
+1Y+9k+R8szGQo54tm4YdX3iZs0yREUqioVNNFBIu6lO+uCrJ44TcTXsebeu2+ZHiUPliTICEWyQc
+1EK+VtLfF8WM17Ycms8ydoEh7IynGsvZdsc7obsU91wnSlTUpQOWlb6c68+WA6bRfOmwqsf7bQIx
+EjQcMBck0qXylSvUOHYq7qTj0Ytyool71dRFVrv6fCVLeWKeqs6THbTMF1F3hNxXcrOGptv/ACi6
+i6dfPGgx4vBjxXXNdk4JAhVaovANJUacrqEY19cApwTxNuPArGKdvUkm8EiSKny6+VcCnEgiLjfc
+tnW2V3VwAqiqXdcFUyraWBLuQlhbuicg+bR6CRJ8U/ZgqIl+aZDy0x5kUW5cPHaxxaCLZAViFNZZ
+C3T4/FUcBeoov7PSTEwIfhtMgMf8J0pzwRVU21RJQzES8IiZdReflilQT58o3auRcunYqD0E2eIr
+3fWuoULzwa450SpIyRmKIW2kszIBE/xFQ68/0wDUxnyFi8wDCSRmm66Ss2VS3Rr4Oqg201+tcFHc
+oJASiBh0/ESN4/y10xmmeGL6NnWeeMttuPUYCXtEOPvBVGg89AIfHTvxYsnrZfncuK5kykYOWrtX
+cVSUM7R08dnV3YC0gXFt99dHY6rvd33Fp5jW3COOInFwrIWq9NxLGQ+mmhDTA5u6auWQin/cfF1X
+gaQ/NTqrityEw6d2pwjlBR9cRAYvBFJUKFzHqTr7yuAtjhZZFVVS8BTSStEiMCLn5U58+7EdJ069
+n7gguTciFQSEEiG36UvxBh3Cz4NwQfcOkXUKjMCJA6fDrrzHnhTh43TVdXbnvEulsTAhFUafxB0L
+ASqqLKEIkTJdYri3SbKpDbQta30pSumPJSTxA9tA7vmBFyKto9/TQ7f6YAKuBT4VpxgCsqraDlNY
+wt0/grUK7nicZEo3ErHTZNoqQ3+6MxV/QaahgLTCSRPmiBR5oLpkBEBiBiH+LpwnMbgULXK6xijt
+FYArWkRelOWmAkYTxNwKhM0EFPESXUhvj849RDg/GCpIe8Jb7OmIiKBAJdX1rp5YAAlDpzKTOSkr
+yRH7pJRG01R8r/yxzN+0b2d5mj5hVzEw67mJekSxigAiKStPSl2vdjtAExEOoLrR6R8Q/pgBnMm4
+tEt899S0iSQ2b7it9KVp/wA8ByX2BZ4cdnMezYzsI6Qg5R4XErleKu7UtKFSlR6gpT0x1bMPk0AF
+2ma5AraILtgAejyqQHWmuOUs6tXGf+0OMy/lmKek6YPEyVMXO77q64zKlelIraaUpTHU1roWl3vy
+RQHbHbZ9XL1oZVwCm6inDoJke24ckRWkYbRDTnzqNa86jirTcojFgRPeOTZgqo7XSJZI7RpzpVJS
+71+HFkBR8lH742JkRdR8MApfnXn008sZF+0BnDKsW7jsuyjldRRJVNZVsmAEKQ99C6fFTz0wGV5t
+lJbtYzB/Z8lKxuX36pCqCzYEiV+QaUu67dCrX0xecqdk8S0ewqEXNryjGJSInK5GRJb1eeqdKd/P
+/LFGzX+7ck9SnSmDUg2F1qVhWrq9OvMCGzv/AFxfMpdsWUVAVhFwNo3SakRk2RtSSSoOpDTn0cu/
+xYC7uMxPISPKSQzICEKhamSFhHb1adWlOWvlijdrbeazXkyRmcszzGShU0hLYK1QV7br+fwlTXGI
+NO1DM0FNSycO5aqRMo6JTg3CO5tbnOhCNa89MDgzpPSkYWX2zlrGxqxC2EUQ4YCGpaqFWvO3u5+m
+Ahw6cauaRKImg3JISSIXIiVwDfUqHWvRWpVwSz5KrLxJi9Bom6cEmk5QTtqu5NOnPW2taAkJfXqr
+gN9jbOjbQz81yXXFDg+GEwVSE61oW4X/AD0prgY72U4z3bb7Urt2rl1EI0HU7K00+L1wEN6s4ILn
+bnfWuuG4yIh+tNKaYYVFGwRTO4h8Zjdb+mEGI2EQmA+HovuLnh1u3cOd3hkbhRSJQ7flp8WAmZcj
+XktNNYuLZqPZBc+hvYJbpU56aVrTWnLHRvYR2Px9PZ55ohJh0+cy224aqGKTBAgG60wqVx9/y4uP
+7PuScsw+So6WGNXQlm1sobpw2LfVK21MNdem6teQ4uj2cLLpyDZ2ALy0XGKSjm463KOnJWpiH5UH
+AO5XynFxMPc0ZsY1x7RkH5N0UbRIRIgDWvcGlBxFhk28fmjJrRg/BdbL0Au5etRutVSLpvp611rg
+J+/yJRAkig6e5fb5d2J5cQ0SbOz16OfVUr/Fz5YrQR/7vvc0sZDM5/vZGwiDbL6qP3pN1g5IV+E9
+DtHXAWPJiR25IqJIL5eeuXz1q5QO3gVfx6+nriN2eos5jMTB9Hnt5uj2cgslxN1r65wqNSv0tPp8
+6d2Bzh1IQTKamYTLfARLCJbR00xerW2rLFosSI87CprStfmw48FpFu5WJeLOXsfHtm0Pld41C0UH
+CydK1Cp0+MrvFgG27zJ8S4YxCiK7iHlIBzahfVUo18Neszr/AA9a9xYsuRs8ZZylkuHh84ZffNpm
+jWii+raq9VKVKtp1UGlaFWo0pz1wEcNXjYcxSy8UyZOsrw4wr6OI7ReAuH3255/TpxSs3yfbBD0h
+o9nmJoTdOKR2qptrumtSrSla611rSmlPLu7sBvbt8+cuECEHRFdtoLtJVJUFQt8KiZlUe/Dhk8JI
+k3ca+TbkVrls5ihMufxBtjz7sDAUWlgJyTYyUEi2gUYAok5C3ysOtL8SmTpSNbk2QBD3hEiYuWyo
+GzuHWzpErqcu/AOgUeobVsoiDS4x2leDcJA5G3S2tKcrsDpNNiWTJaPmcyNYlEhuamKytyHVqGoK
+V5h64hw75mu7dMXbbhk0FxUPZfqggudvIk7g78DO2BqtLZXVzAxlTFZpc4bKjJFYI0GtaCsnb318
+sBRcuZuyeLtqnm0GsTNOUuIbzsUAqtXmhW9QV1prXTGzwTN49iUH0bmaONS4C3VDMe8uXh8q4xMH
+2RZmCVhs2g+y88FJO102MTYuSMdR7vXSvfpg/lLsxybGuI6Wgu1ddMVCTWXbk5EgeCP3gn1eHywG
+pTWR1loCWDMkws9QfhYItEbditfQ/FX9cRf2Ymfs/Ji7NS/cbPlUwuAQ6KaacqcrsaQ9WJONFZsC
+bkSEbB+EhqXi/LzxWcnt1ISdXaKAvsu7VErrdsjqVddPxYAFNsXmWc1ul2xmqzfiSiW4ssZpKkVd
+wk9K+HTyxBBFqMZ7PXW3ESElHKQuQsXGni27+odMXDPEgm5BWJFyug4X+ztl0ekxPzK7T4fPHKXb
+xmDPmUJ5rCN8yLv492gL1AXaIkZABeKulfiwHQ0Y6byTIXaDMxjR6W7rgw9+VOVito8tMOpN9twL
+tzY2WuEgbE5cIKpH3ahQi58vLHJeR+2iciZVV7NxsdmGHdl9sakiQFb8JUrTkOnljoLJXadl/MWw
+5aPzTdbu2k248hNmVvLVMw5j+VeWA0GScPlNpBj9pkF+o9t+BJLjTxaUUr4qUx6VYltJLoLL7KA2
+++jRMf5S0p1054CK5wYtJAWhTDFzIF7x0kmsiaRacq2VMxtKnfpgjl99EzqROWhoOxUIrz4MhK6n
+rQa1pWuAlMkxFLcFHY2BJQGye8kQ+pJ0+WvlgrCPm8fxUlIPNtqQ9a6jkiAdPz+L1xmHaB2jR8I0
+XGC2J2SESFBg2MhVZlXoutrS4hrWnOmKKGX8zdoOWkpbP8qom1IRFqzvIWbnq5ipUepuf5jgNjnu
+37s1i+JTGVXeqIEQ/ZESMVSpbyvp0/FjIe2Dt2azbjhMshIpkfu26qjbbAXHwa106ufr04Mfui3h
+FUigoEPbEb769yF75AKjz5U1Byly9dcYZnXPk1JNF3cgzjmUW9ebiDFozAAfLInaRLalcn3c+WA2
+f9kKHY/uo6za+eIO5JdUkxBNYjVEaFpW6lOd1fTF67UO17KOR/7NlpV0TxVDcBq2bGSpfz7nhpXH
+Ks3mqcyFmh0hlSSdRryQa/2ogQBtidedBCytR6acqYpntK5w6cyRunssSorA/JzcSRfNpXkWA1yT
+z52idqvthpDOYHLkftCmux4xJInNLuVOqvj/AJeWMsk4V4KpLiEiusIiiLn70COnKo0UT5VpTFp2
+ZCZOJzFLZk9pLO91RfeZkqDFEen3tQ5ipWlOXLBaCy/ltSVdS2zl1eLuFFmKjxaMNIg5VqI2V6/M
+q1wGWu3jxoyViRld9mv7xVIQtuOnffSvPliG0UIkiTIAVRFJQhSUOwRGo+Kn5Y3HOGR4UocY8Tn4
+VwLVNZq1ethkGdlSOu6LhMq1oPfrdTFNmMhtUuMXbSUc7GPFInIRRmuNhANd7W3/AP5wFbh414Nz
+xSNB2KAisYCAmJXDoI1rT8+dPLD6pNU4f7YCiaaaBWJJ3g2XVqXhPX4x89Pw4GMotQnrxEXOwLIr
+lTLUkr/DQLw1pTXXTDTtq3QBVcVkLSG5AbyIyKvLly0rpgFhMELjdFEy6RtVvK4Sp3fQvpriZmCU
+ZKSRC2hWsOqKRJqVTvExr53p16afp04rpiJK2+IvDh9JqSgJbBm5WXK1JBMCI1dO/XAJSTUduEEU
+kffEIpjthbd389Mat2FZRy7mTtaistzbAHLUUFBXJot0kdOoiMq+n4MGOx/9n/MmavZUtOouo6De
+qkiJDaS6Zc9CINaVGmOlezzJbHsnyk6ThIcHLrj9gVbxVUckQ0ChVrXTb+o4CzTcKtINI6ICSuj3
+MiLlxsrEJi2R60xCvfpcI0xW+0KPWfNYxy9Uatvt6s0u4W16EkBtBHT0rUsW2RB86kpMmtlVl004
+9sSev2bUdVO/TFGmG6a6XCRLw5ZrJFwBLisQBHx6ZarK93jvwAyCasZDK8i3m2a6DMUlJybjhvHd
+MyIkqDr/AA7benFYzXKFsZNicxN4tpLOf7TQnWhiRJNkAqYifxeVnpgwjF5oLMDObjX+1JPZEhbe
++/v0S15UQ/mK27FSexKcsD7NCiKjaJzvIkJ8Mjc5jGrfrM9NenWzq0wBNlNZmkHpOXbM085ZgXT9
+pxdg8OvFJalRdPX0pT/dh/8AeaDcxr6LYsJVCBzfOpezJMj2towEAU7uoLSHzxGSdSGa3sYxkIqR
+TzJmMTRh5RsskkaESjpStdLuV/xD54hqrM4admFN6HdumSow8dBJgXDJAQCBuhGlLjL5vTAOx7VH
+MGeJoUZVf2lJZiQQYKuLwGTatOag1qPir04NS6mR835kmXzp08iF2j0mSjZJZK0STEedPpzwOzA3
+9l5ShYqAePZBjDvCSy/NsrRVKQMap0Rt+USLmVfhw8lm7KGXboOeyvVWdZ125VROy1RzpSplzrr5
+0wF93G+0kmobFoR3CbMkVkFROvKpp2lbSuFw6nCSqQsTi3LghJvxSMqqBl3aCompfzxYYyBFKMSE
+pV8wFAyWbf2qKto+mt3ViUrEpqPVXKDlBRwukIqmQNy3dPOtMBXQUWdyYx6iMoTdNcidCLlurtad
+xDTb18WDBIqOyXeoIqJoubhtIEVUFdS+MaDStP8APHm+XxXMehiKyZWmqLNvfb3+VMTEoFu0t3Gy
+GyXSYjGjYJU8+kfPAYfmPs/GNz7wmXXMim4ciT1WMJmBtbrdOYkXxelMBIpx2hRMgxhk8kwD9R28
+EbhZ2mkHmRj8AjjXs4ZNg52P2FzQTkBVuYP2TZUV0NC8NCGndTASVynnZKPXaKdp0qmiqI/afZR8
+SOheEdBwFszB2qZTyo9Z5Zm8yRzKSFr78bCsEre7kX+WKjm3t4yfHxSrmLmAeqJEO+QtiP6Uspd3
+fXBPLnZTktSMtfM46aklPeOZNbeJUir8+vg/lxXnH7PvZm5kN9BHhritJDiXCSRF52lWuAfjO1SN
+mYwVCB1tkKYkkQCJFryBKnmF5eIsYZ+0tLN5bPqUPlt+g76dl4qif8YyoO3rXwpjrpTSuBna7leW
+7O84Euxm+LR3SJqSLm5XaEdLSrrqWg8sVbs5j3WYsykXsEJFmQkifRtpCZgVA+l2tfPAezQKKb2O
+TEweikgmjJs4zwCYEVKdfdzuwhxl2ej99ZaBXbLXEInxNpoBQdyvPypVMhxenr5EjSj3MlHQTNyS
+aJtmACRiksJDXSvdVShtx10+fEyQRZzcqzhG0VIuyJBJy+VcrEkIk36V6VofxVSEBwGTTcMoi0Xe
+iKI7a4p1ETIlSM6X86/Fbpprg12aZyzNBSDptDTxsE3KRe9WvMUir00KmhUsrz01xq+aHzFskWX3
+LZi9JOHGJJVkYkZGoO4ianyaENA5/NjHcrxu7mhncBshUMbEBDirQp86ddTNO5Pn6YDcOzRNjDO3
+xJooKOhJJRVq/WudLkI0uVZOaeIty/UK40Z7m4W0Yk+bLG/uVJuq+FsIgqVDsseofBS6um5TGTSq
+y0M9YxLZm1i/aCqb8mbtYBSfXnrvtFf/AA31HpxufZ+xh42Q9koLA/lhako8NZEQkVUT53KDXk4p
++LqwFbeymfvaboU8sAnHtvdpNifikbYrdaG1cW9YFr4SxzvnXL+ZI126zM59lcUkqIvEiMLxSUHT
+3iPOlda95jXG+dsuSZ7NbJ0ojmpjCw6g/wDzJhHLkPcCievu1PzxzhP5fy+0Vaw0PmF1IzYobNjo
+w4MdeeqKlOXKvngKjsuLCT3gUtEVCK/15aa151LBXK+XXkzvu7DQh2FvtN+IXi2AuVK1p54smX+y
+uWdsvaksDpNmLVJ+SrSxURaVKmpfnTGjZHy3FyjKMF2TLbRVOOkWTJYox+Il1N11R6dz9cBJ7H8k
+wPBCmq2ay3GkqwIhWVbLktdUm6hBdb1jUNNaYub/ACyyWn2LsmLpszIVHLKPk2YEgnICFRVQUUrT
+uUpdgnGR/wDZls3x0pIKEhGOR6UHzZUC0buK1pp8NupYNSTpxKNVY1u8jicSlrd7Ey5+8QkAG4Kh
+9K24DJ89rReRTdT2yvBOJJLiIl02WNdiuPKizJVK7QfpjOZicarsmMfHsGMOiW6/XVYLbAuWVRvJ
+Dc5jUrqlTmOLX2kSxLzSUAhJOsqx7ISJzHSLO9ik+AeoAOtNOuleWMHcCK6S8goANk1CUJqkmBEF
+1S1qI1+HT64B+VdKIGTRia7SPJIREbLCco18BKUp01LT6YgmoS6pKEdpfD8o6fTyx5K5D3lgESvu
+QBQLum3S4a+tMWfsvyS+z1mVrl2JWQTeK9RG4uEREe8dfWtMBEyFk+Yz1m1jleEBEnTu4h3j0AQp
+3kX5Y7M7GuyNnkuRy8+JmnGzqrddJ6isjRcOXdtH8PPqw32UdmMHkn2dmRFgg7e0kybqul7gNsld
+bSgU/mxrPGf3xBis6TUiVd9QVEamSiNR1tCnf+WAQxW/shjKe4lnTZUhVVZdA866EWnxF4cNyDNw
+LSTJP3CzR1xIiytM1xt8VRr8WI6ScW9YrwSPu0ZZInbBJFEkLRoVKkN3z3Yr2Zc3IoO2eZLHQoxZ
+cPOt2R3Ls9O5Q6U5kl64AxlREVz2FJUHMogahDcZCSorc7lA8unAfPDyJj5VjDMXhsnUoJQ7VBEA
+2hAupVxX8tNNcVHtrzY6guz95M5dkoQnQvCYEq5RsV2lPB1U7ra89ccoXN1pOVLMWZJFzKMmYptV
+WixOTcuCLUhEi15UH5eWA6NzpmrIeW8iTSbSadFwy4xOX0kD3HKAB94oPnpU7tcNuM4QMzl11nRs
+8h2QzTwYdVqssQWsg++UEKF0GQ0xzetGx7kJCbgeEg04ZqgtsOn9CdOVa0pzRqNddfit+HDrLJst
+JBKyTZG5jCtUnr5eVAhItwh6qJ+Ihrry+mA6BdosSyFO5oGeapov0k2WXWvEikUQ1qVaVWurS+6u
+3gV2VZ6iZSS9sNm7V3N5SjnKEYigjti+bWaGssZa2mXixgjd4pPCvJLM3eYZZBdN65cXltC2DkQE
+Ff8ABg2bNnKRM7MNHy45sdr7iELCI3ICy8St9Q5VGg4DpKEgY8v3Zy+pKoKwbRgvmtd8hcKSCxDX
+poQlTpuK+mtPhxKyoz7O3eWmEp2hMWimY5BMnLpRzXQ1KVMqAX5VARxyzJZmzMmq8KWmHTIphJsg
+5jhbVTJWP0upbpTQE6DTw0w52lTMzmzNKkzlaKzA3hTRSSZhUSPoAaDrr+dK4D9ApCWJiyLaYOnq
+hdSQJswC4fUNS0rpiuhmBRR0cWg/fETAvtS/sfrQ6dbS1rS6ldfhwFcIpuZB45TCHdsUepVqUqtc
+JULkaVdeQ/TFdZTDOJZcSQLisuvw7V40myJQdda2qApStMBdWWbCd/aX1jISVJFsKkOZJPPTQh1I
+f8sSm8wzT+1oItd63qFQFmyo6d9eQldSlcU521kBSHL/ALKn2zh2qLm1OVAkHOo8ttTTo/TuwWXW
+nRcKtiRznwrUwHifcmq0ER500s1MK4C6qyDdy0QIuolREhtcpEJF5UG4hLX9MPgoo7aAgu2Xeoiu
+PSVwqj9dNNP64zSkhHrv15Jw+uV2hG+RhCSQIOeg1qNelX8sfcvyDcW68g2coKOLiEUE3izZygNC
+58lCOh0p/wAsBa8xqFDuFU09gXRKiQkW7db+KohWluH1U1CbquU1uGLxEk2kiEVfOpDcHf8ATDTR
+wT5uKC7ldRZyIqCScqBDbT01D/OmDG2m7DYvejcXSSeyuIlTurXppgMK/a6h2st2coTblF9vMVd5
+i6TciokQmPvEz7rK4yPs3i4Fj2eJTLk5h64epL77NtcIbyJXgOunSW2JV1xrn7UBFIJZdyTIOV4l
+u5kUydOiAUklUSOy7l08tNe7GV51ecXxOWxeTD9wgWyDZojsAubfpqXLuubbutcBSMvqAUw2YkjH
+QqTtW0V1A3XHWV4FStNddCT0/wAWLRHqSzTtIdZiXigcyyAlIuuPeCkKoh/eAER18VbqaYp+Wk48
+p15KO3nCNU1U0RQ+/dJDXWoKBbpQbTAf9WFuH3t2VkXyhxyDwRUckq5c3kSwD7wenT7wufy4A/Hv
+Nr2jHtph1JRMgJNBZsg9+KtfeJjQq/CKop0rprgJ2SIqFnJdBjxzZ8kNzVdMBJdJahiFp6102yuO
+hYH5aWJpxX9pOkEbVEySaBcrbQrxKlfl3KBrUbcK7N3gx+ZUnLnikyXVUTMiuFBfUhrVI606uf5+
+K3AbBmBYWPaEu8QRdRqkA63HTUmwqoRBVAQuOl1dxuoVPLw4tv7OrhTM2aprMzaN2ExXJRy2HxNt
+OSazMu+o28iHFEzFmx9GlxUWunuHus2b9yFoKpU6jYOhrrU1OfKuNA/Z49m5bhH0zvPmEeO0JKrg
+QlFLEOpAQ+betcBG/aVzYo2cEhGrINBXSJQ3hdTCVAdKVTrSn8al+KB2aN27Ro6Fszjm0kyQF7w0
+rdYuSB1WBIFCGmlwVHp88Fs+PNJ/jXzYHca7XEn8ZZ0lvXgLhkPwhWtBxZM0Q7okozJqEkg2cEIk
+8GcRvJJ23SHh6XjbbQxoA4CoquHDRw1YyEIu2br7sm6dZfWuAopyNtR56fdEY91MXfs4bt5dornS
+YRa5pRjx9mLk3bbTwUaF9ncc/FpjM4SQKC35RMJSHlFXigsSaBxjYmv/AIlqVK8h07+WNpaM3CLS
+MaOVvaaKTUl20xCmIE8ji+83EqfEkWAkPXQiAuZtZeWRXL2TLNlLUpBsdVSq2W1u59NRxTMwZsWY
+wjopQ/a0optxzmOeo8LIC48Ld6Ned5YBSeYo+Seus3yxxeZYWHEYU3LTdSeKjX+7uqhd7whrXTux
+nmaFn0obGSlJv94czSCXALtnNwPmKqJe75fXAO9pco6bQ6GW1ZibXkl7VJ2MmkRJVs6HWgkB+hCX
+PFFNwomqJCz2G4qipwal2wqQ9J6curu54Ovvbk3uqLzwSUo7AnEik7MRXSNDptur9K8vXEFJvJKR
+6DGQW2BTtKOZufEqKneSf4fPvwEGQInyqTZMzQUVX22bMjHYSAy+f0u79cdr9kmXW/Zdk9dRys64
+qNeJrvBRWFcVxMRpdTlcKfP/AG45O7KomPlM4MfaUachDpltyZKARglr4CrQK07yx2Tm6Sh04JnP
+Ciu7y3LsxinhRAEa/LUU+XoPPAEc2yEXCCUI9mzGPn19yMdJ9RtnNSvp/h53UxVp7tBl2GRoLNAt
+XUhLtnCLBddvaSbsqlQVRVTpXWnTqeAvahFtXfZ6/aSje5jlkmhQuYm95GkFo6b4UrQqVDzLAU8v
+i+zxNRb7NQNnDtdKYi5ODAuFuT6bFh50Cta1wF0zlmRGLOai5J++lYtyz41g4ikfexi1fhM6V6aa
+088AJWJmJdWMc5sc8W4ncuiizmIY6bguEhqr9oSpXQtRIe6pYlzEg0lX2Xczt1nuUnk4t7HeOxQB
+WOkkqULmr8I6lTliM4y2xyFlxBo5eKQE5HyhLDNtDM2LYFz01KhVrZ0aUtwEbMCeYMwZMSzt/Zyj
+p/lhRo8S4Yb1VrqAIop61tP1+XGJw8C+Ty+8mYsGLB5lUuLfSah7rpUlx29nQ7R93p/vxsTLNGYB
+ZDFwkOcopkiT2ym2RgQKokXLdCtPFXvrjPc1SUPlXtNGJ7Tma7tEnjmYdNmH3BGuAcONRpzOmoc+
+eAo80WRYn2EtEwq889YoEU05WWIW6rg/uu6lcMrs4mUCFfPsyS87NS6SqktHNek0ACmoCVSrSnLT
+E6cdZjdrA9zB7Ljsv5hoMsuzRAU/ctjqFltKUK/o8OEPVE5ZaThcrZGWi/az8ZFs+vO9CPoNaV59
+9vO6tcBEn2843gIzMDSUQbp5qQUYKRsZpv7KZU0Egp6+eD2SW2didx9MloxeXRzKkWXyoJ9R7I+9
+VurTp566liI3DKGVo6dpAS9ZPNNJFJpCl4bUbfen+HW7TAIo9mWT3j9bMjptKRcmLCLiU1qmaomV
+yxCdNPMsBbcrzUSxkJ+GY5Y9pSU1Gex2JktxXDOKdBERlbSgkPUOnhwf7LU89SOTmwU7Vo/L6DEi
+Zt2pElStQTrpd1VpWutalT9MZ+oxdJuJvNGR33sGHi3gNEkH61AWBRYbS6dK92terHyQkMt5Wl30
+MyZ5WnWyS1yb18moZqaiNS0rSult12mA7DNN5HtEsuj7bXTQuWXFaHFddtqWoEnURqJjSvpgPmBm
+3lJVBd2jCKN0iTUGRe5eVDfGnKqahCGlOde/BBW5o99qWIOUzuJs5aT1y7ET9aENOnCTmJCLV4Rd
+nMP5J2kVzZvMImk5C74btOfPAAHZQqcg8kpuEgItq2XHhWqm8kSpAFK7iJ/T5aYkwjNP2Oq9YzDX
+7SqNkxHTxJLoalcKawKFTXniwcGsukqJHnMmLa29JZZFVdidPi7/AE7sfFW71eTWkicSiZCIi0X9
+ipCguJDySOgnW7TzrgIgN5InfsYQmBcOUBXcoNJ5IgXG7QySKpa3euJKQvnzhBym5zEMfH7ggus2
+SeGloOm2ppqZDy8WAcemom3XjVNgnjkRJditCGkqzCpFqaZUKtCHl3a4NOPZ6IthjVoQniZdbwQc
+IJK6d4LDbWndgHeMkkDVc8HuLLq9LUoEiSc6fxk6iOtC/PFhb8PxYkojHDtDcuqpFKpXF/NSnI8V
+ZwnEuXvu20cm1SH+6+0lkhJXvGqNahp+uuJDJ0QsiUiXi/FOfdgqUr1JFT/jCYUr+umAxDtozEjJ
+dpEnIJv0EGrRrwy4E2NURSIdFNsa06NB6q/NjK2TrMk+9VHem3LwldlquTmwCWt5VrrW2nuNylLc
+aD2sSikhnV9BSG+yJVUbVeJCwnVtEwUqpTuDp0trTnjNcryTFtmiRTlDAWLv7ETp2ZEbYiL77p77
+aYATGL1bTSbdoxRXBYT3UB0I7CLQx3S7q0s8WJSojHZ0S99FIikKayRJt6rp6UGhpjSg0rW6vdcO
+JKrVEnHtBi/aqMRdWr3e6bX07+impbZ0AefrgZm4kyV4hoZizVMVG6QtiSFIi6zS1r5BrywHm6hI
+OF10DkXqKqRCBCj0Kj3q9/OlKV54GR6zy+1s5feLcEEzLxVIaUrWtPX1xYnyLcR4lRm63thJFwo5
+eD0u1OtRYaU52EFD8sV+0mL0hQcmg4Iegkwt7++mtfywGsnAySm++Js6fouUFU3gSqxIARgiPQI1
+57g/Cf642Psqdey4p+io8azDN2km2YvnVojIDZpwLsK/dq0+b4sc3qziLmPXkBB9LS3UJryPS3sF
+uFKGNNdKKhp3fFTGrZKzV+7qpMc5Ngko2WYIO3XAWkC6VdKcYGuhbwVr5DgLRIMY9lmhq8LLAPYt
+sRsF2r1baeR5kI6CN3eI1rqGBWcPbhOUsly0q19sTqqbB+3mg90SVOTVwmqPiIvPFuSnC9poCJx2
+bCbMFVHL7wjJxVSGlNSry309e/GaZ9nEcr8Y2Xee2m67UWiDCaO41WKhVNFVJYdbKpVPTATGUG6T
+zQ1IY11khNgqJbROQOOKVQHw6a9AKjQufnhrMEx7GhUF20bw00/JWRy2/wAvgRilcVjhsaXy+fpi
+hOMzoRPDskMxyUS8oahzKVnFIKukepuv1acjrQaVwGLNUtLO5HNrlm+bzSio+y3EZ0oIOP4nR+Mf
+TAJza42XCEWo8ayLWLX3OPadLwt3romXrtlX9K4qzh06dqqvnyxvXSokQqEt1pFTuxMbuikm6SCs
+Ugu8VdKkR+A1yW5ac+XIu6uGFkXDSNFpwaI75itvuQtVuDpMB58g19cARkBg+I/styuqzMUCQ40N
+pVU6iV4606dKV88QpBRQjHccm5ERtDr3REaF4Rr8uLY3kvZKAIRbUBZlHJXk6bA5tCp13FKlSvRT
+ny88V0eFbTcj7Jcg/R3VU25KI2gqlXuKnp9MBpX7N6hITGZE05hdg4SYC9bG2WEhVNEqHQSGvjHl
+jeLJRGCmBWkv3cTlzTmI542MQbLq23KN6VpzDWzz/FjmHsceN4ftCh5CQeMWylyZMzco7qCtxUEk
+ldK9PKv+rG/59mE4KTfISBuk0Y+fQdoZbegRpPEVAqFKNFadPVUi6a4A1O5kWzQ4gnbZFTKo5mYq
+tnKoo8S1eOiDRMDMdR0qXLXxYEQgyWX5PIexsZRlnoqxM1J7KRtnJgNSpSpV+OpiPixAcC4HImZk
+xZh+7sPmJJwOXn5iLpijS2qg6ULw693PDqrhQssZ1yc0bIv2MS+SlIyJkUSSdEjUqKKbR879K8qY
+CBmuPfQ3ZnP5eeNnXtKDmE13oogarGZGp3cw/g9+Ck6pmb94HMIKzXKMHmyEJZIXv25qurbzAj51
+R0HFhZTyjTtA/d/Ksr7DFeCJ4ULNh7pVxXTp3C5Dp81LsDoThUHGSJIkTym8FVdF+T1EVY5yag21
+06rdCLuPXAAIx06j8ykxUikMoxecWqTYXzY0nMe5cIh7ylacxompSleunVgZnvslmJvMTV3lvY3H
+48RIvCMV0A2PuxbnX1592Dcgj+6mZZFynMBDx6UiJTCorA8jBEuVABCuhBXnz0xVu2ftOfZOWLL+
+T34Nm7Yk1EnEc/FdmSRcxGg94FzU19MBTc0Ry6E4zRztFryjuj1N4s4aHurizMddqiQdA3V/33YI
+TD6ed5Ris7DmAIKJeoFltmyaGRuSYpczI/P4fLniY3F1m+elY3LL+Oyr7QgN6VQTWE2zkadd4HWu
+t3Pn8pYocetAxuWpbjo2RZZiUJsUOQnupII/xSqfdrXAWTPcejKGxzBknJgQuW5BgMcz9yO+5JMr
+lnFNPBWlKeIsR0pDJLGZnmuQ8vTWYTfQ4NIp4sFSUFyQ6qr2fMOvl54EKzTeSaRkNJP5TM4x5KIx
+0S0MgSED51PcpS4i9enGqZSedpTRxlYWmQzjW8ag5btn1gG5JqQ8ysu8XPxYCi5Fjch/vBIscySr
+rMyyrBMWrBgiQC5kFBtoHT4qhr48VrMyT5F2gxcFGRq7JCjdVAUTcHQhIvvCpStKHpWlK08tMaOB
+M8s/u9JZL9lu57LMiqnJuXrbbN46cFYnTbrWlS8XfTw4nNx7JYWUmG3bQo6d5xVkFF3SzQ67ZCYj
+UdLa6cudP0wGuKqChNryintHeepE3bLrZeAmznpuHWzTQtKaYVKy0a0AicsIddRTbI2ZRSqTlnr0
+kQaF5fTEFWPdOXqUbBAaiLYSKTim2aritoNdCT1PlWpYfaFMCDWQcts4D0kjFPhnkVdoq/w1dD0L
+n5lgGnsxEoSrWEYv4FZqVpFKbzpK4qcqAp161/zxKCeatlXke0Wjk49ICJdqnNmW6NedNute7XHy
+MTzZDuV2BLZi42UAVXDNyi3XQVqPItsufV8VuPswM9Ht9tocw/jUVRJUnOXgI2fT4k+nw09MBLSz
+ADY2uYCRevUySFNur7bAjZiWtLVKWeHWmFqyE4g7SiSZzacw7SJwaCazdVBcPirrZyLTuwFcOhey
+ass59iEi2ajtG9y2qCDnUtLS6PFr6YFm4btAdSC+W8tMnDkdvg1GbgVxO7TcR1HX/LAXBxCyD7g4
+ti2zEUS2IiXFQG97Y6lr0UIK6j64nyrgSMpbhnwuCLgmrrg0bS/CtSo8qcsZzIJsWyQxZOYTe3QT
+VmE+ItEhLWgLcvp59+CMgMWM3aoGT2BIKpkK5IuCZvLh8VSoNl1f88BVO2XsXzBMoFmiFZqLTF32
+xi/NIAIBLWlUraUvprz54wfMEwjJH7Pkoq2aL7O5VTtARVDpT0HT079fFjrmKeRqkZIoKMMrSQq3
+DwDdsqqrf57dbeY6YfZQMfKSaCacCxU4JL3qrSBEDEq+FMtwacsBwsKaiZrjvdKfjHwj4vT9MWmM
+i56ZS4ZSBkZLc8KiIGZIFUtaFprbppjrbL/ZjklebSevssQ6klvq8UKiwqhcXKmqSetpaeXdjbMu
+Q8XCMhj4Zgg0appDdw1oiRUHTupgPzwkMn5mUh0F2mT3TZ0kuqJqkiRXCXLnUq1w7lTshzxmRVqK
+cDIpt1PdkuQFakXrrXH6ImLMVeqwiIbSut8PrzwFdvBbKkV/DJ/GIgj1D5FgPz87QMt5iydmBLLu
+YpUEiFcXAfZvddIUDd5/y24C5XfQLFwTl88kRkGzW6OVQ6gQdAXKlde9M8dS/tYKZTd5KJOWcoFM
+L/3EhBIlyOncPRztr544/epuG32ZREB96XWPVu6eh/FpgLFIZ0mJaH9kqbDZvvquQBqsSQiR2XjX
+8FbfDiLOzDPi0PZvFLs21nDIPwuFKlQpen81vlTnit/BcOJyRLPnou11kFFE+ohWP4RHkPVy+mmA
+eScLGyXUbImInci/dKdfiOlac64fVlH14rtlgYNytUBJsZWoaDZXQfIipiKrIJ7Qi0RNp0+8AViI
+FSEumtRr6YigoRKkSl5EoJdXxXYAjKyTqQcM0yWNZuxSFBsSiIgQpUK6mttKeuHQFaQcWoXuXjlV
+NMWwgV6pl3cyrXTA5kLh2rsJ76rhQSTSAQvIit8OLXlqHFSMeO5JHaTZLtnLqTJYd1BIh+6Ee+4v
+pgBSRNWaRfaXrZbhyFdLZ92q4A6e50+WnmXdh+6UzA3eSSbACFV0ncCIW9Zcuke7EVuIyQLlGooE
+4SarkrvWiWzQx0PWvTfzwYbtYeNy+845hNoShPGiiSpBYLZvaNaqV0+f4cAFJ83hHsY+i0VxfNhL
+juJREhJW7mOleVeWOqnqjfMTeTbNv7UazWVknqCEi5sEVUb/AO6lXuqNfixyS4TubpOxeAv94RId
+RbAVLTXn82N67GnEpmjskKClIr2s3glSdteJbFdwVeSopqVppWlO/TAFuzpixksuvGwt32cBzJl0
+Xj5LwPkHCI2VoB/xPB4a9+CeXJaNzJnDJ6c68DNyhZbUbjwVyD9odCGhjUKF1HSnljzSNFePkSyL
+MLuYPLLpB7HKkCoyKDUwAlE076aKDXXSuuD8bkWPXjJqZerLryEbKJSscbREGb4gr4xPXTTv5lTA
+AuzprOSErCy04f72Novi4VzDPURSfM0SrcJqXa1ryHywWzbnDsvjcqHE+23T+BQfisvlh3aJilQu
+Yp6jdaJfDriozXadNPms6xhMsLy0jv8AEnJulhB4KNO/bpTn+WmMwtZ5kz3HfY805jTlECHbfvwS
+dEdvIQOlfDb83iwE/tDksvvpqRyyhnBCRy6RcawkWsbU1xVLvTUoNaXUEddSrgBk94WW2SuYIRtx
+Lxirw66i8UK7HaLXqLXwF3Yiw7GacxMq+bMFxdQlopq8eKSrEKd47d1x0t86YPum76GyvGJMniDR
+bMKW88eJzBKtnwU1rtLBr7uuAhw8HMSVkWTORaSUy+J22YiwERVRMfeKJqU500+X0xbofsb7VM1p
+IZfs4CFZJFsG/tAUgvpqNdOZFrjeuwfIMDl/KrHOM62XXkE0AdM1yuV2hMeQI0r+HlSmLa4zMzUA
+UYmEdPZK3ebwSYWigZeFRwfhDXvrrgAHZ12R9n/ZvDoZqYm6UkBSFMnjm0jVKvIhANOm4saek3ap
+gMo5A2zzhR+MbmKVvhDlypjOWWallXq6Aj+9WZWyopvHg+7i4oq940KvLpp3+uCj3MzqfdIM8uma
+yKCoDIyyaIiTmtC+5R9da95YCH2gdmMHnhJdyoi1bSnGEXEpo3CIiWtDPTQqVqGONu1ns+Xgc8vm
+yKEo9ZrVos0Xq1JSppV5Urrr60L9NMfoI3XEW679K9Nsklc+e7Ni65o/h+LliUlFNn1KvFxRMV9F
+ERJtzBOtOQ9351/XAc4KvGKEUggo/hFuLfbiqS0Csg6Yifg8OvL6YNRieX3b5JoLbJNW6DYhVETc
+bTvSvK2tR92VOeuJTdOYFIZltmSY2XLoRjnnEt3JFdztPVPX9a4RBNZxoy9ksUZsX3FEtJoOY1uY
+9WupI9HdXANmOWXbRV65hMrOWaH3V0qsKrYu4j8OttPpgg3j30Tlwtixyo7K0XyeYbiEO8KKVqGv
+08OJUhHvHMm2T490KLZIrn37tpCCRDzqKnT50xOVUkrFXYo2qCltpAnAgKDz0KuvxeWArIfvI0ky
+hkwP2SkhvOmaOYQVVV1+JO4KD/XA5xIE+SdTMstnMYOPIeFdFJN+JbFQdK0qFNbqa4LN2cgLImy4
+bjp+JEbN3l4RXQC6nhIK0rbTHnELJC4axK4MXe0qKhOW0CYqCdC1DcGp6FSlMBHis1QLFw+kpLMM
+3x0jtJtSIESQc/L0UrprX64ek3m1lr2WutmVRR2vuJMHINSP/wC2d+mnpiTKotUHCssu24Zumqm2
+SQTYNwbKn4daiYkQ1wYgOzdZ26Xkm0ko33xK1s7Z3igRd9lfl/LAAIwVE3DVsL/MUkiySIiSUNFq
+5E7uQ7lCrfph2PGWeZt4ko18om/Q2Wy8m8V3UtPhrYnbbjSIrJsShFey5BsDsU7bjURtK6n4/FSn
+pixshUTZC3HpTQLpAjIitwETLUe4Yx+w5Bim6ErjVZdN3prrTXBVuoV/u/eD8REf/wCMeNNbiPdg
+FvxdHw4C59zdA5Jy/wC1Jl+1ZCRbaQqHbcfpywBN3am3VUc2J9f8Y7REfm/7Ywft77dMt5UinkFl
+8wfzSokmBtDEhQP8Rf8AbGNdrfbpmbPCLlWCWOJjQZi2ftVFh96VVR0IK1pz1+mMphGbds3XknzA
+1xds1eBBPq97Qg6i86efVgDUw3fOeGls1LBLPljbEKDsyBVdIiurRLXw059RYgoFFCs8vP2SzVB8
+idzbfa601NNFLnrdrTv9cWY0XE2ynXb6eZSyiUSzJAVkS3VxHuRR8wGnd+LA9pl+YUzR7ETZrxb5
+R4V4uzE2LNJYCpXWtaeLngIGc4FjHhwblgEXLHwzlO1YlEEm5BXWqqlKeKpUp5YoppkPUQeLqEv8
+WmNVPJok0SUdrOmzM4xTbXJapjLum6vhEfIba4gTuVXQtBdiwXF4kgD8WyZiSAtFOZWU8Vw93fgK
+GbNZOMGQJseyuqSaCvTaWneNfQqYYBMt3bs6h+H5S+uDRpyGW3aqCgfZVCFbach0rh5KDTEOQ4N2
+quszPgk9q4m6nivqXhCtO/ASsqPvZc2lLKIggmJKJgQhdadvlz5fzYdm5BZ8bVGbBASFq2TA23SC
+QXU6lPmU078QFfajaQJfxOEEvEICQiNAp5d3Lzw1vfZ9hC9sQkmRJEF9x0Hxa+VPpgLVGuIeHNmU
+lCBmGPTSckyTTebZKGVvvlNBrWg0qPIcCfbkwpHrrqTy6lzxJbaU6iVOgjShV5fBgOCniU+7U6iE
+hDqIq/DyxaOyrIcpnrMCUXG/ZkSuFd4QXiloPdpy54B3svyXIZ6zGQqAunHkW4+eJtrtrq1rp9cd
+rNI2DhMvw4sWz1BNgQpgkiz2isqNtSPnXWldcTsg5Dj8r5cSho+NBNNMbVSEFR3T8y8Xng29TTE/
+cHcNvSF5kQjXl1dXdgMj7bZLL+VJOMkk4tkLiLFRsmXH7S6iRDckfSNaaUKuMbb5+zJn2Pmoly2a
+yxKx1zM1HgoOUjEqU8ddKH/LjSO3rs0eyT2MkYiHZKVaIW+8BUkypdeSapVPwVuxhKrMouP2pRaO
+FGJV4lJiszK1cy5ElRelaVLTXlgI2SVGse6Z5km4prMRLBXhnzEnmw6VMrhpTlXqHXDsJDqzaM62
+iYqIYrR6nugUf7D5OlSqVoqV5KaeHy6cSwJvCNEkF4Fk/LNDW09yNVA4y4unZO7qLl54Hyb7jp2H
+uABbwtqLp8jFWroJUOoe+GuoqcvpgLHOw8lG5fytmhRbKzlvbsk+bPLzVEipW50jTqP5K4v/AGP9
+n/HTbXPso5jkItgqSjX2UwMhkyPlUNs/CI6emKbkLIcfnTOEwSh7EHHvE1imGzYWYiZF/wAM6F5d
+w46Imm5Rskll85J6yjbLQbNgEpGXoI/wRGlNoK69ReeAUWYM0T+bXgxsWCbOFMUWJJrDwLEqhS41
+y7jUp5CPSPrhcwxfPY8WSaMiMKo6tLhFhSfTzmveVS+BCmGcsCDU0GkkzNZ6venTLrJaxlGpVKtb
+3KnmVvfdj7K5iTkJ1WSQcGhGtEuCKWEOi6v/AIdgn/Er6n5YATIZZnHMOllltEx1rZUVCjG7kgjG
+ZXV63a3iWOnyYvWS4OLy62XzNMZgB8T8dk3BaoJICHKiLVH6lit8Uqmm1QUy86KTL3kTlNNb3hF5
+Onp/154cjmLpzmBjISrxjPvodUl5KRdnawjTr/CQTpoJGNPPAXqBRmpucOdlHBobqRDEw5XACaWv
+NZTlzMv6YsbOOQvcGUu/MjWqRUFwIiFdKUtpT05YqXtQZlkC3tVeJy+ouHDPCP7U+K7XopXwp17u
+7FoZLSJUV32LFjopWgJ1UEiqOlNCKvrXAY81hyeSrqUdsItYRLbBq4gSFquN3QVwjh2dFq0ZE5Jm
+xJ4oW2ZDGuAXZiXnrQLraaU0xbmmT5BtFJM1odld1EXCPzQAi8iIKiWnP64aOBzQ2VSubSKjdsO5
+sJzAkV1O4R1T6+/zwFKjLVEmKe9DruFCIVVUXjhAlypypuh82nzYOyDxi2aIRbT92kEdpNZRs5kj
+FIhoVKdOnT3+uDUfH5gUk0lXLbMSYgO8ZXo+KvwkPnh1JrKIKkmUVNrtV7uhZsir3/W6lmArLJrF
+rSZPiZwim1btF7buXEq/8Ot3hwVBu3eq8W7YRyjpPcTQBY1lXP16qUrdyxKDLc4+ZcIoEi0R/FtJ
+GI/Qqa4Iw+VyX2l5s3xOGCv2beciqRB566UHvwEDK+V49y3SUUbAmIqkQkLDaMht0tKhUofd54vk
+ZFsY1kLZszaotx8ACFo3fNhYEJOBuMC2/hs8P9cSXCyKDcl3KwIopjcRKdIiP64BsGqI22+7EfDb
+04CZwzhlfKrdJXMkq1ZCors3KdQ3/Wvl+uMP7bf2gCi3CENkB+1XcLlauvZeaWpWXJ07j7vXHOeb
+fbE7Ie0p2YMnkhxKxGpcW6QiVBCiX8MqeHAbr2n/ALSBOUfZ+Q2N9FW5bij3pMeu2m2P8TX6YwqT
+zVOPpvjp+SueJCuoJubrBKgWW0T8QFyx5WLko8yTFyDSUTdNNpmXU8G0CrQkuWnnzw1KyCbF2l7Q
+bNXMhIEqLxUjuVLUq6mY91K4CrsmrVeSSduXMdw5Ok0yS6hEurxfgH1wfgWbVs8QbvtlNFzxjUhY
+LbpuR6SEbKfw/rg7GZJzZmtJ5+6jNr7PX94e5YBkKfeI869NK4fylBvhmGcpEgBKKLk0YbgAIKnQ
+feCfV0/T5sAdaTUe5jF49oaDuWd5fSZXqAIixMfgS+ZSvzYGLTzt8/22MPwUcLhBw3YqBfVy+Qro
+e8VfWl3PBplkFu2cbjQAbJyjwmvtXZtOKc+SNKXcta4s2X+zNGQSJzKNuGar7kU6atnJBwL2pa0X
+rqOtplTAQ4x9JL5rGZTZxyEwqkT+FZtFr2ot6jY5Tsryv88QTZtWjRr7lrJCwLi4VsSI3yqJnWpp
+qUDyArun0xO/cPgpj7cC6cpvpsnD5Y7eDdpiRUUpp8JjbTXEbKgxvtNrLQz/AG5Rd4rtIEHTGSNC
+qNe+v3a1KcqeHAVHtgyaOXYJi7UeLu30gkLtjt3ihwneTbq77O//AA4yllBvnaW8j8TVR2l0F1BQ
+qUqNMbj2zuCmYVrleJWUXYOTN2wF17so94A+/QEvTv6cUd7CxKcYl7LeLprPYziYzbPoScCWi6P+
+L0wFGjykHYKimG41ZDvKpdNgjbpdZXl5YgKrEoBXABLKdRFf4vp6UwYZOIlSFdIqImMom6FZILOg
+kf4iZUwHIRILrwtUIrh+L9aeWA1LsJ7I1s9SyTmQ308vp/emmsKSpfldWmOx4Ls/g8utEm0TDhti
+kmIri2b320HlWpeeAH7N7MlOw+AklLxJVAi6Tuu0LTutxp+yRJCShmoXhH4em3lgBMeooQKioYEp
+4blASErfXDCTVuglamG2REVpfZ+r/diUyRZiqruHt7YkJeL/ANuFAioNxJublPh6y8/8GAGO2qKa
+SqCh9KgkJCoCJCQenf3YBPct5ZdtxbO42LdtytIgUbN7breVdK4siorJ3Fx7obRtG4yL/wDp4fbv
+FkwFteBCI9RqGV3h/wDp4DDO1XsxlJSHVGGzCC8agO41jHLkEmzMR8k6J186+WMuyZ2eyUpMR0yM
+r7YRkFfZ0mLB+YvGPwkoY16yHHWM7KLR7QXMe2XfqFamkzZHcapV/nClMZTKs5RDMq8s7W/dxw7X
+QEkhfibldKpaWLogGlOX4sBcm7f3oxOXZVDjGyWzNZtdoiRFp3UD4VFfrTw4nTEe3jWDh+pmF1E+
+0rWwSNm/KPq/8NOlfAPpbgTCzAk+blMIoZjzImkS0XARCJA3Zh3UJQq0pS6vzFiaxlHrycSJq2az
+mbDoXGPCPWOgk6d9KFWmhFT6YAItDrNpZNqUPxpF/cctIreI7uTqQV7v01w7QZAsxKsvaUc9lIdC
+59IpgKUZlwa+KidK+NbT/wDOLTl5ipJUk0YSXdA1clbJz3/iXytOVjf0GndriBG5Ti0zGCXhduBa
+WrNYfeI3L5xXvXdf/nAU9FwLZBzKISL5CHfq7DP45bMqtfkrXrTRxNeJOXqjPKqjBjIzTfbXVjiO
+nszLqPfQ3H/FV+nfi0N2b5jJlEi/QTneFNxKTBJ0MIxvTkCSNO4a6d2IMSxGG9mNIlguUK5dXJC4
+6nU47IruIWLv2x1uwEl0+i4J6rmTMDx09sVSQZyaiP36tf8Aw7VtT17rtMfMy9oDJtKmMtnqDyg5
+KlD9mORE1gGvcSnylXzp5aYmzb5rmJ4q1i6pruI8th5NlaLWP1p7wUa15bn1piBl/KkO+jRdQOSc
+vSTAyLbfTRVq5d1pXSqta176VrrpX6YDXwEbLS+XxEeEq2l1D1KeEcO3dFw//wA2Gj3PFYHi/FgE
+AJCqQkd13gIrbhH/ACwkFFB3RK8tzq8Y4dVJbduTC78N/ThwfD1B4huL+bANgO4A3I2/zW4UaY+G
+z+Yit/7Y9cVnTjztwLRkq5UPpSSJQrvoOAizchGwUUvLyCiCCKAXEfTdbT4afnjj/tY7UMydoO+c
+a/OHy+oquixFYOle0R1oWnxVu8OC3aln5x2oybZoPHQ+X2zxsnvj1GkqZa0qQ08X0xT27OUQyuxi
+WiKAtXa7695s3hICJB0080y5d+AqPCx8M9SaLsF01kFWZElfc6Z6JAdTTpT4da64am5xRtGcCpwI
+iuKqgPy6l1xu1tU+QvTBLMqkXDZoEXMa+iU1CbOQVW63iWgD90pX4fTngc4h2Mw9JVOeais53CJy
+7uA0FS50FXTzL8WAcAph89Zk0Dgk3rpJxGLufeurwHnRNWml3f4cWmPa5dTzw6cyFijctwvaKkad
+yUhXuQWCtbaUr+WJmR41wnl9BWSilFY1svsNUhf+8inVPCuA1r93XXvxdY/Lco7jJYXz90TpchRz
+OzIAO5Kpe6dpV/64Co5HiWbmMllF544laQdJovB2SD2eY/dq8u5Mq4ubjLcWo0XdqMGrIk9uOlTT
+/wDDOqc0nQ0p30LzLFyy4izjY9djIPPanBCKD5WwbnzQ/u1/xKD54aOLdNJsmy7YHLdJAWD4iMbX
+LI+aS9KfgrgIcOzbpwjxSZRNgL0uAndsyIUFqfcvE9ddNaW1rXFgaLSTmPMpY2q6g2x06SICJX0/
+u7sdfTENoxR4dWPmQXUKNH2TLGJ3AqxMdUXP6a484fC0VSaO1gtu9kzu4HXtGOjdz+etvPAQ8xrM
+ZBwqpOgumJF7JlgLq2lbfszmmnrz54ocnDzCfHbiLGJREW0C/LhupdWhe6c0L4aW1HqwYm5R02zG
++bSxtSWSIYN+YnYJBUa1QdemtO7ATOE5INGW5KOd/wCx+xZYSO4L6fcrjX0+uAxjPDhOUzGSD43X
+tbqTctm11qr6hWgVfS7D+bszNV4J+LbLy8W1XVTUY/CTOQSGgrf6vp+uEvk55zJ8FLP2UWm9dcI6
+kekPfD1pl6iNbcCc4IqJxQu5A13pOVRJR4oY3cSF1DEtPmGod+ATldrLIO2slHooPbhVeiFlxKkA
++8E9fyuwCklouxsMWC+4Ilvmp/F1LlXT1xZmmXXybRBzEzCDYlIdV6QLdKgiJFRZL866d2KbduJJ
+IJtrrbi6QuIvzwHf37NXDl2KZbubOl0xQIkzTAwHxV/F/wAsaM4TFc/44ppj4CA7i05992M1/ZtE
+WnYrllMg3CTQ8Sm0J21IuWla3UxfnCJO3Y7DY7R6bx2THn69WAUrxgpWkBlb4rWxl8XL4v8APEF2
+T5NkXCNjXIS6REFfF9OrE5JqimoTUQREfAoRbOhV8vPHm7cW1qCbbcT8IkIIj1V9OrAAOOmuHLiU
+V7i6rU0Vv/fgnxT7h0hFgvcVolcBj3fWpYHvWMb1DwwJl1e9JFv0+XLUsUrtdz84yhkfcI0BcKCK
+LYljbkQiRcyENdTpQeelMAMzXOReZMxlH/urMOU0FdkZVOSVj0t2pa7Y9XUVbcXlkms5fICKOWoA
+nKRNyal9seKlbpT3tPTGLdh80nmIlynQi5SS4pQhKefkG6rSz3gt+9Pp+mN/yoo1afZoRyyeorqq
+LPHTQBJBmVB16Cry78AJyu1a5SW9jqNuARcpWqmosSr+QK3S0R8VBp64hybOSXSVjVofeRXG5llh
+qtRsCAUr986Wpzur8uBaVUWs8+kMpyXFunStkjmd/XiFbA6qos0ud3d4R6cSHD54WWnSctFO2jV8
+vti2Jzuy0qVfD3fdAX+2mAJtX0+vGkjl32VIyLRUkwep+4jIilRoJCH/ABa0wHZS0fCZYeKx8w+Q
+Ypq2yeaX4Fvyat2m22pXx615Ut6Rw5KQryPjmLaWigfrIICMblSIMkmaF3eTlSug10/F+mBqKcxO
+5iZu2z+HIou4eJRbVKOgQoNfubh0WXr9e7ASlt5y4atHeXl4mDVtcDE8SRScu6/h73O4E6aa1uw5
+MOHjl5JySz+2XaIbLldH+4wLb+IIF3EqQ4XBQ8e2buZBT2w5GSVJFWWX6X0qsXcknp90l04OSbXL
+cW1jovMjsItND7WhCMlqlugFNffW+Pn54CgS66KjWIqpFGqyUSJPLGUbDSVXr/8AOOufSHxVu8se
+aZeknqW+5j5rOrita0VkWL7hWol5ooAI6bYd1K+ddcWdim4nMyqi5jVGj2SSFebfkdvBtO5JkHnQ
+i9MWFyhKt1OFr2iMcqgjTbTjWzZJWiYU8NSIud9ad+A0S7o6f92PGJCHT4vhwsOrqLHumy7ANXKX
+9QBh0LvF1/y4909JeL8OIsrJRsM04uUfoNG9wp7qx2jfXuHAPqrN2jddd2tsopBcqqXTaP1xyd28
+dpUhmuaKPgni/sVk6UT2mi1irnQBPeEudCHn4cQO3DtWzFnGUk8ops30XBpAuK7ZoFzxWqdn31PJ
+Hq15YpeT/wB3Y2PZsXKzV6m0dXLoIn7q1RELTFbv5VwEuM9sJpKyCCyHGNl4/hpEfumwW8hcBX4K
+V7ywluxnotk6Xj3JucxNl11pMr72pN1NOpGmJ7eDnJ1psSDZAnTlqm2iXLBawSNDuSXoWtD105XY
+tsJAymWYLLPDHvsydKEPEgIL8R8bRTX+GXlgK2yiZJ20jkHkU6kpJsKai43iYLtO8FU/xjixR+X1
+pR2qWyu/WU95MJEzFMXKNC+9T/8ANDzpg7lSBFtIIPmKK8a1dvCWauiMrox3TxM1PLbKvKmLarvL
+yCD4nPsclXRJpJEdvBvh7hrp/CUwAVWDZsTXknwb7xghbZ8ErH106qUp8YYkg6WJug7iVkFFmCW4
+1VLxScZX7xOvzEHP8sRjavH1qaiJoOifEo1uMvsb2njQ5fw1R7sCXrV05kEJCEZrobW64Ztr+lm4
+Aq8S109DpgDsS4KNkEnLFmo7YtPtbE7x+0x6w+8RL+StcO5iUaxMggu7M3Ps9Ir/AISXjl//AO2W
+AWYpD2EkzGL3yaikMtGAPgVRPku3H8NNcGDnE/Zi6ikbxIxoj0F1krHr6a86/KWAlQ7cRaf2g5BR
+u0ujpW7+OxU5orV/LXA7OdwskmLta0Ykhjp0B61V2Rfdq0L6d+HTT9lhdIOTKNQSJg8EQuEminNB
+YvPp1wEe5mFNk1kE4037xJJSDnUr7Ssr0pql6/8AtwFIzatGzOVxfKIrk+hyKOmBUMhNVqoXuF66
+U6qU08WKlmJ019scDNyQRaIoJxMml4iVtG5F1X0pzxa3ZFDO9vMD9BAd32LLeEt1oQ3pkNKd9Brj
+KMyyCJSAtBhEHakfuMF3ThYrXIGdaIKV9NBt0wDEISMlNi0bLHLOH6oo77joFBa+m2pTv1qWJXaL
+Le1M0JFGw4JyAkIrtkw6Rdj0qDSndXURphrL7N5JSr4ZQ+CZiPBKumwWg2dAOqJV09a0w12XrOCz
+AkxXRBckiVepHeQqbolS8qV118NC/PAEcwQaL4J9SEeAm3YIJvT3ukiKoU32wV9Br5UxRW7wmwKq
+JommRNdki3v0qXd3YueblmMpDz60XvoRaboHMWneJFtLlot+XPXFAMiFIrunpK7owH6F9ggvG3Y/
+lklzBRRRn4yW+G6v4MXlkRNktxy52/is3iMbv9FMVbs0ZjE9lWW2xAuntsExIRMrefPnp+eLYkQp
+pCK5r2l1ERXXf564BSq3UShIoEPwnvEJFr/gww7cLJnsCFpEXwrFcI/6MRXr5HdVJB509IhcZdJe
+fnhbclF26qm8Cl3hK9W39a0LuwENInBXFfcJDcFrkrrfyqljC/2k5hqpJsYld41TcJtTUFV6iSrY
+RqFeWlBoVFKeWN7abiCVxH74riG01vDXy8WOeP2hXzdt2ipLprXOkWe4OyiRkzGvKp1E61osPPQw
+8sBmGTM0I5fVffukaDZm5QEXLmTbCb4deSotT1tP6a2+LG65dTcZhyoxi4394l0XbUU2sY90bhaJ
+dSq6g08Gn+rHO7Ru1ThxdxcJxqLR0oL6Vd//AA/r+Krf+F5Y6c7AlE22V44bZV28XIRbILPN3fSp
+zKz0RGv9MBdDkIlhJpREAim9ogCiiT0QAWcMNB0rqp5lX6YGKz2V4dAW8XIKXL+/eTJBU1V/LRHl
+1FX6eHE+byOnKNI6LFxttWC6qx8MG0xsv02lBHxFQcGsj5TiY9v7SFb2kS9ynHOQEdoe6gop06QH
+8sAHcJvJ8UOJbLtotIRIIdM/tj7XkNXBfAniZDwck9Fq+mWbVJREi4CHa/3ZidvKq1afeH+mLO4T
+Yxrfi0TQZboihxxdRkVS0Eda864oWcpKU4h4CbNeFy+SpdTc7X004qPgQD4KfiwBGWnFI9RVPLyL
+aazOqkImO9QWbOo69VS0rQKU154pq3s32euUlnY2CJXDIyzQPtMg5IetJvqNS2h+lMQMvwLpiqMM
+5hDkZd2qLn9245ztM4xLnYTtenUodfOhV6sX7LUGaL5WZfBHyuYU1RaJJNAHh4YflH09a4DNQWYx
+7eJkI2NmxFddRbL8E5O59Juy8Ltf0SHy1xbzn5ZgXCQ/Zw6zTt/3ySMtdxzXmoNK6c6DWumv0xKk
+6lAZxUXYM/3gnHSFs2/UPrQC3QG6FPgIq/DTy6sD3uXidEkpMzWZYtxt0oLGDoVW7UNa1onWuvUf
+PWtfrTAbKAiPhDHjtEPBh0cJ54BCRDZ8v+3HIn7X+eJJ9m9vHwj/AG42NQVRXVbnRcCcW3+AdbCH
+TlWuOoe0CY/d/IsxLCdqiDVTbH5jqPTSlPPH55cGIq8c7cuiuJDiVWx2GruJFeFKV1offpdgFhNL
+RJiS5rpPFCUHaRW+03LAHXv/ABjy501xfsiZX4FoSko23EyY2yO2Hul0ajaRp/8AmI6eKmKj2Pps
+SzR7ScogvHtEhTcoKASpIJEZUorr5EJbeNGOQcZZnXjknJi4UdEmLYgtSSdW6VoeuugLBbX/ABYC
+5wWWYdQxfSD/AH3iAppya49IkNf7q/T07q911aYuQRayce+kJ1mu9UVIUZZIfB/5b5Gnd8uumK63
+mm8f7H9nw7XhVWpC1QUP79uX94Yl+MPEP8mLFFPkRkGrSPeLuSZNVHUYfVZIR9fvGta1+NOtMAAz
+Q4UbOHSEzKmmmQi2mCT6Rsr/AHd8nr6fHiBmCQUIFXMle9UbCnHTZtguEh/gPaUph3tAkEyaFLII
+oO24sxJIVg/vkeZe9RrT50ueA/taNy+qzj2J7jNNgLeTVL7pePXKgp6V56qBd+mAsTTNBNGSskoz
+MXSiqbKRJQ7iSMeaLmn83dXFPk86EvmNedbb7Rwp0qpdIglIJ+L/AAqDhjNCgwj1JjJb66jYSaPC
+E/vWp9SC3Lx26YxrNeYiJukuu/4lwJbLwU/wF7lan5jb1f0wGmZ1zNIcQKmX2G3FtBJ+ld1ESS3Q
+sKf0pf8Ay4Hg6zBlJwqNhvWLBnauPzNHPhLX4rS8sCXc08Uyo6fQiJiMIqRBudRKslx0P/TWuGof
+MWZlAdLyFijVkw9mOTI7t1FYtUlO74dPF/TAbPl8XBZMZtJkAXcdMLLFvWmLVQbm6nPnpS7vxVJV
+ZZSPVUlpLbRSX9iyzMeki2x90ty/FQerzxKyEsop7MY5k4UhmGqkK+citdaYf3danp5YVmt0oSTU
+iZsWjp+gpCzB32kg6H7tauvzad+AyTNuaFk5OMF2w2pJkko0kFVgtuKv3alafNQfXFRVcM5J2zUk
+FnTl5ukm5JG7qSHkJU/Fj02soU2q5mX/ALSkt9Ru5SHp8OggV2PQTV8nxiHBgSyoi0uWO0kDIuRU
+wBuCJOLyEqTZzxac6KjV01K33TgCuSP+mFdnMaQpSc2Jmo6gkk3CDZQ7CVC73g8++nP8sO9ocgmh
+FRgkHDSRCSb8bB6XCHSBUp5aj/niz5S3I3syatpBFihIEqMixfEdxLslCqCwV9a0rpgKVnNq1aC6
+9pGbSUcr8RtJ2kgk1UAVUxoNO4+vFbj2sgvbGsWy6ijshEEk/ER+WlMWHtAGFjZD932hnJCwFQUn
+wncTkSG5On+DXEFks8dyApxZoNk+P4hql1AQqiOtREtPDgP0PyOm+QyVGIPjuecGnviRq/CNKacx
+xOcKLIbRCB/CJW7vn+mIMOs+KHY76IIqEgmoqJIjddZTWv3mH5AR4clLLiHwkmHi/wB+A89WWFUh
+URX2x+IuI8X6Dhpo6dIOCHZ8NtvvlhHn60IaYSyTRdgP2YBISttUAh/xfe1xM4URMkyR2PiExDw/
+n11wDr1ZYUkiEF1BIrSFO8/r6Yxz9p3LLybyuxzEga6AxKvEFts1SXFKheVbf89cbKkis23U11kC
+HpISJEh/y6sCcxwrObhHkavYui5SK3oIhIajz/iUwHJLtwhOspydJJbMq9jZyMwij9lZgNw2vGvn
+Wmvi0xrv7IScehDzDSNmGr94Dol1XDICJszRr1WDfSnfjnzMrdbLshLQSDORJG3h0AFyKBiND199
+WmoKUGvdSunjxrH7Ms9MFnV004N6uioSRSL5EAbEkNB5I1R+Lq+KneOA6rboorAguPS13dxqCNwd
+487vXHgWcJ1QTd7ZSRCptCjdtW0+avliSipaqQie6oRCQoeHYCuAc81nFEiicuufZokW44k1g3bQ
+r3inTl1f8sBHzFmJvDAlx6KktMKpgSUSyDdLdpzoX4efxFgFlaJzNNSCWZMyOWSEskqSfAo+8CMb
+lSvRSvmtXzLFnh4FrHtVUYu8U10ivkSW+0qHd3c6d2IuYJRuhHruXL4IWJtEvaN9qqytK06aBWnn
+TAMQuVYeGZPo6CNRkzXIl3stxNy+7QqV0I689MB5iaWcpKqZbftcvQZFacmLa9d8t3e5D4/5sU9X
+NTrPk8xi2gcBCgqSoQSlyDiQAf4y5/w0fw18WA/aHna12upk1YJaatJgxVbo3JCr/wAFmn4en41M
+A7mDNEbEGuXttCAcJiQpM1LyXvU6auXNaU5KWVKwPXuwmAkO015FIuGGf4TIUeVPscZIMAJ0ol5L
+q386Edda6emmEdmXZqUb/a2dFo6Smo10Lt+g7P3DMqjeSypV++UpTu+EcFs3Z8yCi8bKTEPGzyq7
+aiyT1+6FJVRKpFQenypyrp9MBvvx+PCvFj4Ajf3Y+15a6eXdgOef21c1DE5XjMvqMzXbyhksRitt
+EFnPpr82OSMzPh3UOEeLuUWQ8MAEY2pWleOnzjpTHQ37Y7mQLPgtmpN1ExiVK2OQuFIfnT5clPrj
+niPTkpmBbMGz6gIryQAILDr11Gtda179K178BrHZkKcRldisq5ZOU3KSj10aNu4qxUKgLJ10+JMq
+AXLCpVmom7k4l3MOnLe1NGVc2CuRMq9TR0H8o1pTEuDi3ySqpRps29VGwv0E6JWp0EfdronSneJ0
+pStC76YrmYiFmo4BZRfaibQUogdRq6jVOfDnX5g1roXPXzwB/J8g8sdRcoZthTVFTitkruKoXuXQ
+6+GivhP+fFuj5SWKCeC2cgyJdUlIwyusZvh+8b//AE1B54xxw+UXhXHBSkkiq1dIxyaihUKtY5au
+qadfUxrSnfyp5YvPZ1OSUoJxrrbMFqVaODLqqLlKlyTkOXKunIqf1wBSVzA8kG8SMbGmLfj+JEiO
+4Gbug/aGqn/ln8OKHnucYje0y2AezySUWSbEdxIJGNaKtS1+IddaY88zlKZlaydFUko9OQUFNxwZ
+VCou0g04gPSpUpTWnLXz1xWmsWwWjk0UKrIqOo83Jr/GDgB5lSmulRLzp/TAWOQJ1Fx4zruVCWeQ
+opCaSJiO+xUGtKcq/LjPOIasZtBRyw9z7wjKwT3UTLpLTw8sG7G45faS3CpaRZJgonSmm+gdNSCt
+fLn5c6Yqjt0g+lG7ZFExZm6Mm4qncVAIuQl66YDR27daJj3S7ZY3rFgW2qF5CRMl/ARUr36VrTAV
+V1IIN3jZewk0Ek2D74QVSMtUVOXkOuLhInXJCLriCJ8ghRSMdDTkSqRjWo6etaVpTWtef1xnjxN8
+3kF2ajlMkkwSarjZdQklK6hSmvnSvr+mAs2Qmb5tJycfKSX2hRUY4x6fcEI02luXw/XF57VWbeWj
+Gs2+bLuXDlLgpNsn4kpABtTUqP1xnmX8ssOMRZLLOarlMnErGBaCoppqB189KV8+/F6gZprIQ1ZJ
+03VApAVI2QBMqVEl0qag4CleV2vnXngMMlUVGxl9j4Ikvsy4F4t2nOpYNQ8SUkySlnkkajcnQsnl
+txEgdvuir6jgVmhJYaJyLxyaq7xRSqtfFrUK6U7/AKYumSE2yOUgUqlU208ktHvEa+RhzSVGvrSv
+f3fngKzm6SWlpMnLttwyxCKLnc6iVWEbalWlPDrX1xa80Ok2zTK0XHtvbUD1P2LYfvRG2lFkSLxV
+0t10xRYtwqEok4TbtlXJGK4KLUuqKqZUMq93dXu07vWmLmbqSWziM3Bk3bK0aDKKJLhqFCK4STHS
+ngrTy0wAKbavF5D3fAsmuwLlnVQxDfRqdSTEa+RU8PLvxFhE5Sbm2DHe6XbwVitttEqnShF0+HDe
+amjhGdfbrmqttU6lry0pULqCP0p5YtHYvEm/zxEhHPFUttRBZxRbnQq7g00HT6euA7n2xQh2aZLI
+ESaSfSRpCPIdPMce4xum3SQJsxIVCuG1yl/1HywaogKBkSKCQKiVhaEVtPoP0xJSZvCVvMkan8VL
+yt/TlgBDckb7d5raPyrI+L5uQ+eHTdEmqNphvEVxANl1tPXpxPJo/TQNusadEKd20udC8X5YivnH
+AK1SJV2e5TSlarkVtP1wCjeEuZKEBqD8IkiJdXp3YfBMRuTFHqHwe5G23/viIaTUgEbVKlTuqVcR
+hdtA02xWpeNbeVOnXv8APAczftC5VTY9oTqQbNkH60ta7bRg3CkSoDWil4dxdNdenFB7NZ5qnmZJ
+9LM15tZBAukZIkgbABVITTMa0qoIj01Avlx1J2y5ZaZ5yO9hUXKzV8nQKtHJDTRM7uWunPT1xyqG
+XvbMs0bOHCQzbt3wIUogPB1KgiJGY9+te+6lNcB3BkWW9t5fYzKjYGzqQapuJECuE0BqHSP+WCqo
+qKKhILouk02R2t0kTuFca6UuKmOUexfPS2Ss8o5RclJvSqqSEuuo8qsDkxPQSAD0oNKUx0a+zgqW
+X46Rat9l5JOeDaUrXVNGl2lxU8+XlgDrgniDhmgKPFut9Qb07tpsFfn/AE7sZznXNULxaSe9FT2Y
+I5JTiXqxiTOPIfGVAryJT8NOeLJPrPqSq8G2XpHgtRKi7lr0rKLHTS7XlbSlPTHOud8tqJyuVhlp
+g0Ip46dxyKUYzTTWbGJXblD5VIiKlK1KvPAXTPce1aZCnXzGSN63XFstNkK21KvBIuYqKV+5S08v
+lxj7+YmMoTb7NeX5iIQ9hPmzZiggtvpNmiojoIadFtteo9NSrimzWa4R07NzKUnnaziKqksqbqh7
+6gqHQLwrXSyg0py9cQ6ZkZSUll1lBZajIWWZtiaruRHdTdFdX3pDWnIvrzr9cAfzrmyYlM2lDz2f
++JbkZIqP2QUJrw6/OtKhpcVvpX8sQYma7I6sE0815azFNSiWqRvkX1oLCNdBIRrzGmmnLywh5x/a
+T2oUbxMbE5bfE1rRbhiOiKhohdVTSlNbq9/59+KPmSUSeTTlZywboL3UFQWw2p1KlKUrWlPLXTX8
+64D/2Q==
+
+--Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B--
+
+--Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8--
diff --git a/src/leap/mail/tests/rfc822.multi-signed.message b/src/leap/mail/tests/rfc822.multi-signed.message
new file mode 100644
index 0000000..9907c2d
--- /dev/null
+++ b/src/leap/mail/tests/rfc822.multi-signed.message
@@ -0,0 +1,238 @@
+Date: Mon, 6 Jan 2014 04:40:47 -0400
+From: Kali Kaneko <kali@leap.se>
+To: penguin@example.com
+Subject: signed message
+Message-ID: <20140106084047.GA21317@samsara.lan>
+MIME-Version: 1.0
+Content-Type: multipart/signed; micalg=pgp-sha1;
+ protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy"
+Content-Disposition: inline
+User-Agent: Mutt/1.5.21 (2012-12-30)
+
+
+--z9ECzHErBrwFF8sy
+Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l"
+Content-Disposition: inline
+
+
+--z0eOaCaDLjvTGF2l
+Content-Type: text/plain; charset=utf-8
+Content-Disposition: inline
+Content-Transfer-Encoding: quoted-printable
+
+This is an example of a signed message,
+with attachments.
+
+
+--=20
+Nihil sine chao! =E2=88=B4
+
+--z0eOaCaDLjvTGF2l
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: attachment; filename="attach.txt"
+
+this is attachment in plain text.
+
+--z0eOaCaDLjvTGF2l
+Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="hack.ico"
+Content-Transfer-Encoding: base64
+
+AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA
+KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG
+RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA
+PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl
+5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA
+/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ
+yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A
+Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK
+ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK
+LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP
+QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy
+AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs
+AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA
+AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA
+gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d
+HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA
+x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7
++wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA
+AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5
++QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA
+OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK
+igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA
+JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra
+2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA
+xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj
+owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB
+AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA
+AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d
+XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d
+XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA
+AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB
+AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm
+X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC
+AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B
+bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ
+S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu
+J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y
+AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N
+KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB
+XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A
+AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA
+AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d
+XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA
+AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr
+RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA
+AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A
+Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI
+yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA
+CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys
+rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA
+vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d
+HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA
+urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx
+cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA
+CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo
+6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA
+2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7
+OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA
+UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp
+qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA
+lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa
+WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB
+AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB
+AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB
+AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA
+ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA
+AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB
+AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB
+AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB
+AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB
+AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA
+tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA
+AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF
+wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB
+AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2
+RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB
+AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB
+AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA
+AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd
+AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB
+AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB
+AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB
+AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB
+AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB
+AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8
+ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2
+NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF
+RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB
+lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA
+AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa
+WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA
+AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX
+AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB
+AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB
+AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA
+AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA
+AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA
+AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA
+AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB
+AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB
+AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA
+ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA
+AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7
+LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA
+AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF
+NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB
+AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2
+RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA
+ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4
+RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi
+JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2
+NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK
+T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB
+AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB
+AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB
+AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN
+UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA
+AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA
+W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA
+AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB
+l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB
+AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+
+WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA
+AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv
+RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA
+AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj
+AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB
+AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB
+AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA
+AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA
+AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA
+dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A
+AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB
+AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB
+AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB
+AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW
+pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+
+--z0eOaCaDLjvTGF2l--
+
+--z9ECzHErBrwFF8sy
+Content-Type: application/pgp-signature
+
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.15 (GNU/Linux)
+
+iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv
+kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl
+vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK
+PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC
+w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw
+sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr
+BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN
+QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt
+mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ
+jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8
+gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X
+sSdfcAhT7Tno7PB/Acoh
+=+okv
+-----END PGP SIGNATURE-----
+
+--z9ECzHErBrwFF8sy--
diff --git a/src/leap/mail/tests/rfc822.multi.message b/src/leap/mail/tests/rfc822.multi.message
new file mode 100644
index 0000000..30f74e5
--- /dev/null
+++ b/src/leap/mail/tests/rfc822.multi.message
@@ -0,0 +1,96 @@
+Date: Fri, 19 May 2000 09:55:48 -0400 (EDT)
+From: Doug Sauder <doug@penguin.example.com>
+To: Joe Blow <blow@example.com>
+Subject: Test message from PINE
+Message-ID: <Pine.LNX.4.21.0005190951410.8452-102000@penguin.example.com>
+MIME-Version: 1.0
+Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452"
+
+ This message is in MIME format. The first part should be readable text,
+ while the remaining parts are likely unreadable without MIME-aware tools.
+ Send mail to mime@docserver.cac.washington.edu for more info.
+
+---1463757054-952513540-958744548=:8452
+Content-Type: TEXT/PLAIN; charset=US-ASCII
+
+This is a test message from PINE MUA.
+
+
+---1463757054-952513540-958744548=:8452
+Content-Type: APPLICATION/octet-stream; name="redball.png"
+Content-Transfer-Encoding: BASE64
+Content-ID: <Pine.LNX.4.21.0005190955480.8452@penguin.example.com>
+Content-Description: A PNG graphic file
+Content-Disposition: attachment; filename="redball.png"
+
+iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A
+AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj
+AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv
+AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0
+GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf
+hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl
+rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL
+ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS
+AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP
+AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP
+AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI
+AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR
+AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6
+AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO
+AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8
+AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8
+LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu
+MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF
+BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp
+6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow
+8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G
+ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn
+OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1
+a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T
+VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8
+Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f
+lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/
+joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms
+1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA
+JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7
+vAAAAABJRU5ErkJggg==
+---1463757054-952513540-958744548=:8452
+Content-Type: APPLICATION/octet-stream; name="blueball.png"
+Content-Transfer-Encoding: BASE64
+Content-ID: <Pine.LNX.4.21.0005190955481.8452@penguin.example.com>
+Content-Description: A PNG graphic file
+Content-Disposition: attachment; filename="blueball.png"
+
+iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A
+AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI
+IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y
+Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S
+hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM
+vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB
+Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu
+MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b
+fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo
+Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp
+LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX
+P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M
+jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8
++VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE
+1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42
+YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY
+mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp
+Z3VldDZzO7wAAAAASUVORK5CYII=
+---1463757054-952513540-958744548=:8452--
diff --git a/src/leap/mail/tests/rfc822.plain.message b/src/leap/mail/tests/rfc822.plain.message
new file mode 100644
index 0000000..fc627c3
--- /dev/null
+++ b/src/leap/mail/tests/rfc822.plain.message
@@ -0,0 +1,66 @@
+From pyar-bounces@python.org.ar Wed Jan 8 14:46:02 2014
+Return-Path: <pyar-bounces@python.org.ar>
+X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net
+X-Spam-Level: **
+X-Spam-Pyzor: Reported 0 times.
+X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE,
+ CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP,
+ NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled
+ version=3.3.2
+Delivered-To: kali@leap.se
+Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33])
+ (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits))
+ (Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified))
+ by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F
+ for <kali@leap.se>; Wed, 8 Jan 2014 18:46:02 +0000 (UTC)
+Received: from pyar.usla.org.ar (unknown [190.228.30.157])
+ by mx1.riseup.net (Postfix) with ESMTP id F244C533F4
+ for <kali@leap.se>; Wed, 8 Jan 2014 10:46:01 -0800 (PST)
+Received: from [127.0.0.1] (localhost [127.0.0.1])
+ by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F
+ for <kali@leap.se>; Wed, 8 Jan 2014 15:46:00 -0300 (ART)
+MIME-Version: 1.0
+Content-Type: text/plain; charset="iso-8859-1"
+Content-Transfer-Encoding: quoted-printable
+From: pyar-request@python.org.ar
+To: kali@leap.se
+Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2
+Reply-To: pyar-request@python.org.ar
+Auto-Submitted: auto-replied
+Message-ID: <mailman.245.1389206759.1579.pyar@python.org.ar>
+Date: Wed, 08 Jan 2014 15:45:59 -0300
+Precedence: bulk
+X-BeenThere: pyar@python.org.ar
+X-Mailman-Version: 2.1.15
+List-Id: Python Argentina <pyar.python.org.ar>
+X-List-Administrivia: yes
+Errors-To: pyar-bounces@python.org.ar
+Sender: "pyar" <pyar-bounces@python.org.ar>
+X-Virus-Scanned: clamav-milter 0.97.8 at mx1
+X-Virus-Status: Clean
+
+Mailing list subscription confirmation notice for mailing list pyar
+
+We have received a request de kaliyuga@riseup.net for subscription of
+your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar
+mailing list. To confirm that you want to be added to this mailing
+list, simply reply to this message, keeping the Subject: header
+intact. Or visit this web page:
+
+ http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b=
+3377148ac2
+
+
+Or include the following line -- and only the following line -- in a
+message to pyar-request@python.org.ar:
+
+ confirm 0e47e4342e4d42508e8c283175b05b3377148ac2
+
+Note that simply sending a `reply' to this message should work from
+most mail readers, since that usually leaves the Subject: line in the
+right form (additional "Re:" text in the Subject: is okay).
+
+If you do not wish to be subscribed to this list, please simply
+disregard this message. If you think you are being maliciously
+subscribed to the list, or have any other questions, send them to
+pyar-owner@python.org.ar.
diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py
new file mode 100644
index 0000000..9f40ffb
--- /dev/null
+++ b/src/leap/mail/tests/test_mail.py
@@ -0,0 +1,399 @@
+# -*- coding: utf-8 -*-
+# test_mail.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the mail module.
+"""
+import os
+import time
+import uuid
+
+from functools import partial
+from email.parser import Parser
+from email.Utils import formatdate
+
+from leap.mail.adaptors.soledad import SoledadMailAdaptor
+from leap.mail.mail import MessageCollection, Account, _unpack_headers
+from leap.mail.mailbox_indexer import MailboxIndexer
+from leap.mail.tests.common import SoledadTestMixin
+
+HERE = os.path.split(os.path.abspath(__file__))[0]
+
+
+def _get_raw_msg(multi=False):
+ if multi:
+ sample = "rfc822.multi.message"
+ else:
+ sample = "rfc822.message"
+ with open(os.path.join(HERE, sample)) as f:
+ raw = f.read()
+ return raw
+
+
+def _get_parsed_msg(multi=False):
+ mail_parser = Parser()
+ raw = _get_raw_msg(multi=multi)
+ return mail_parser.parsestr(raw)
+
+
+def _get_msg_time():
+ timestamp = time.mktime((2010, 12, 12, 1, 1, 1, 1, 1, 1))
+ return formatdate(timestamp)
+
+
+class CollectionMixin(object):
+
+ def get_collection(self, mbox_collection=True, mbox_name=None,
+ mbox_uuid=None):
+ """
+ Get a collection for tests.
+ """
+ adaptor = SoledadMailAdaptor()
+ store = self._soledad
+ adaptor.store = store
+
+ if mbox_collection:
+ mbox_indexer = MailboxIndexer(store)
+ mbox_name = mbox_name or "TestMbox"
+ mbox_uuid = mbox_uuid or str(uuid.uuid4())
+ else:
+ mbox_indexer = mbox_name = None
+
+ def get_collection_from_mbox_wrapper(wrapper):
+ wrapper.uuid = mbox_uuid
+ return MessageCollection(
+ adaptor, store,
+ mbox_indexer=mbox_indexer, mbox_wrapper=wrapper)
+
+ d = adaptor.initialize_store(store)
+ if mbox_collection:
+ d.addCallback(lambda _: mbox_indexer.create_table(mbox_uuid))
+ d.addCallback(lambda _: adaptor.get_or_create_mbox(store, mbox_name))
+ d.addCallback(get_collection_from_mbox_wrapper)
+ return d
+
+
+# TODO profile add_msg. Why are these tests so SLOW??!
+class MessageTestCase(SoledadTestMixin, CollectionMixin):
+ """
+ Tests for the Message class.
+ """
+ msg_flags = ('\Recent', '\Unseen', '\TestFlag')
+ msg_tags = ('important', 'todo', 'wonderful')
+ internal_date = "19-Mar-2015 19:22:21 -0500"
+
+ maxDiff = None
+
+ def _do_insert_msg(self, multi=False):
+ """
+ Inserts and return a regular message, for tests.
+ """
+ def insert_message(collection):
+ self._mbox_uuid = collection.mbox_uuid
+ return collection.add_msg(
+ raw, flags=self.msg_flags, tags=self.msg_tags,
+ date=self.internal_date)
+
+ raw = _get_raw_msg(multi=multi)
+
+ d = self.get_collection()
+ d.addCallback(insert_message)
+ return d
+
+ def get_inserted_msg(self, multi=False):
+ d = self._do_insert_msg(multi=multi)
+ d.addCallback(lambda _: self.get_collection(mbox_uuid=self._mbox_uuid))
+ d.addCallback(lambda col: col.get_message_by_uid(1))
+ return d
+
+ def test_get_flags(self):
+ d = self.get_inserted_msg()
+ d.addCallback(self._test_get_flags_cb)
+ return d
+
+ def _test_get_flags_cb(self, msg):
+ self.assertTrue(msg is not None)
+ self.assertEquals(tuple(msg.get_flags()), self.msg_flags)
+
+ def test_get_internal_date(self):
+ d = self.get_inserted_msg()
+ d.addCallback(self._test_get_internal_date_cb)
+
+ def _test_get_internal_date_cb(self, msg):
+ self.assertTrue(msg is not None)
+ self.assertDictEqual(msg.get_internal_date(),
+ self.internal_date)
+
+ def test_get_headers(self):
+ d = self.get_inserted_msg()
+ d.addCallback(self._test_get_headers_cb)
+ return d
+
+ def _test_get_headers_cb(self, msg):
+ self.assertTrue(msg is not None)
+ expected = [
+ (str(key.lower()), str(value))
+ for (key, value) in _get_parsed_msg().items()]
+ self.assertItemsEqual(_unpack_headers(msg.get_headers()), expected)
+
+ def test_get_body_file(self):
+ d = self.get_inserted_msg(multi=True)
+ d.addCallback(self._test_get_body_file_cb)
+ return d
+
+ def _test_get_body_file_cb(self, msg):
+ self.assertTrue(msg is not None)
+ orig = _get_parsed_msg(multi=True)
+ expected = orig.get_payload()[0].get_payload()
+ d = msg.get_body_file(self._soledad)
+
+ def assert_body(fd):
+ self.assertTrue(fd is not None)
+ self.assertEqual(fd.read(), expected)
+ d.addCallback(assert_body)
+ return d
+
+ def test_get_size(self):
+ d = self.get_inserted_msg()
+ d.addCallback(self._test_get_size_cb)
+ return d
+
+ def _test_get_size_cb(self, msg):
+ self.assertTrue(msg is not None)
+ expected = len(_get_parsed_msg().as_string())
+ self.assertEqual(msg.get_size(), expected)
+
+ def test_is_multipart_no(self):
+ d = self.get_inserted_msg()
+ d.addCallback(self._test_is_multipart_no_cb)
+ return d
+
+ def _test_is_multipart_no_cb(self, msg):
+ self.assertTrue(msg is not None)
+ expected = _get_parsed_msg().is_multipart()
+ self.assertEqual(msg.is_multipart(), expected)
+
+ def test_is_multipart_yes(self):
+ d = self.get_inserted_msg(multi=True)
+ d.addCallback(self._test_is_multipart_yes_cb)
+ return d
+
+ def _test_is_multipart_yes_cb(self, msg):
+ self.assertTrue(msg is not None)
+ expected = _get_parsed_msg(multi=True).is_multipart()
+ self.assertEqual(msg.is_multipart(), expected)
+
+ def test_get_subpart(self):
+ d = self.get_inserted_msg(multi=True)
+ d.addCallback(self._test_get_subpart_cb)
+ return d
+
+ def _test_get_subpart_cb(self, msg):
+ self.assertTrue(msg is not None)
+
+ def test_get_tags(self):
+ d = self.get_inserted_msg()
+ d.addCallback(self._test_get_tags_cb)
+ return d
+
+ def _test_get_tags_cb(self, msg):
+ self.assertTrue(msg is not None)
+ self.assertEquals(msg.get_tags(), self.msg_tags)
+
+
+class MessageCollectionTestCase(SoledadTestMixin, CollectionMixin):
+ """
+ Tests for the MessageCollection class.
+ """
+ _mbox_uuid = None
+
+ def assert_collection_count(self, _, expected):
+ def _assert_count(count):
+ self.assertEqual(count, expected)
+
+ d = self.get_collection()
+ d.addCallback(lambda col: col.count())
+ d.addCallback(_assert_count)
+ return d
+
+ def add_msg_to_collection(self):
+ raw = _get_raw_msg()
+
+ def add_msg_to_collection(collection):
+ # We keep the uuid in case we need to instantiate the same
+ # collection afterwards.
+ self._mbox_uuid = collection.mbox_uuid
+ d = collection.add_msg(raw, date=_get_msg_time())
+ return d
+
+ d = self.get_collection()
+ d.addCallback(add_msg_to_collection)
+ return d
+
+ def test_is_mailbox_collection(self):
+ d = self.get_collection()
+ d.addCallback(self._test_is_mailbox_collection_cb)
+ return d
+
+ def _test_is_mailbox_collection_cb(self, collection):
+ self.assertTrue(collection.is_mailbox_collection())
+
+ def test_get_uid_next(self):
+ d = self.add_msg_to_collection()
+ d.addCallback(lambda _: self.get_collection())
+ d.addCallback(lambda col: col.get_uid_next())
+ d.addCallback(self._test_get_uid_next_cb)
+
+ def _test_get_uid_next_cb(self, next_uid):
+ self.assertEqual(next_uid, 2)
+
+ def test_add_and_count_msg(self):
+ d = self.add_msg_to_collection()
+ d.addCallback(self._test_add_and_count_msg_cb)
+ return d
+
+ def _test_add_and_count_msg_cb(self, _):
+ return partial(self.assert_collection_count, expected=1)
+
+ def test_copy_msg(self):
+ # TODO ---- update when implementing messagecopier
+ # interface
+ pass
+ test_copy_msg.skip = "Not yet implemented"
+
+ def test_delete_msg(self):
+ d = self.add_msg_to_collection()
+
+ def del_msg(collection):
+ def _delete_it(msg):
+ self.assertTrue(msg is not None)
+ return collection.delete_msg(msg)
+
+ d = collection.get_message_by_uid(1)
+ d.addCallback(_delete_it)
+ return d
+
+ # We need to instantiate an mbox collection with the same uuid that
+ # the one in which we inserted the doc.
+ d.addCallback(lambda _: self.get_collection(mbox_uuid=self._mbox_uuid))
+ d.addCallback(del_msg)
+ d.addCallback(self._test_delete_msg_cb)
+ return d
+
+ def _test_delete_msg_cb(self, _):
+ return partial(self.assert_collection_count, expected=0)
+
+ def test_update_flags(self):
+ d = self.add_msg_to_collection()
+ d.addCallback(self._test_update_flags_cb)
+ return d
+
+ def _test_update_flags_cb(self, msg):
+ pass
+
+ def test_update_tags(self):
+ d = self.add_msg_to_collection()
+ d.addCallback(self._test_update_tags_cb)
+ return d
+
+ def _test_update_tags_cb(self, msg):
+ pass
+
+
+class AccountTestCase(SoledadTestMixin):
+ """
+ Tests for the Account class.
+ """
+ def get_account(self):
+ store = self._soledad
+ return Account(store)
+
+ def test_add_mailbox(self):
+ acc = self.get_account()
+ d = acc.callWhenReady(lambda _: acc.add_mailbox("TestMailbox"))
+ d.addCallback(lambda _: acc.list_all_mailbox_names())
+ d.addCallback(self._test_add_mailbox_cb)
+ return d
+
+ def _test_add_mailbox_cb(self, mboxes):
+ expected = ['INBOX', 'TestMailbox']
+ self.assertItemsEqual(mboxes, expected)
+
+ def test_delete_mailbox(self):
+ acc = self.get_account()
+ d = acc.callWhenReady(lambda _: acc.delete_mailbox("Inbox"))
+ d.addCallback(lambda _: acc.list_all_mailbox_names())
+ d.addCallback(self._test_delete_mailbox_cb)
+ return d
+
+ def _test_delete_mailbox_cb(self, mboxes):
+ expected = []
+ self.assertItemsEqual(mboxes, expected)
+
+ def test_rename_mailbox(self):
+ acc = self.get_account()
+ d = acc.callWhenReady(lambda _: acc.add_mailbox("OriginalMailbox"))
+ d.addCallback(lambda _: acc.rename_mailbox(
+ "OriginalMailbox", "RenamedMailbox"))
+ d.addCallback(lambda _: acc.list_all_mailbox_names())
+ d.addCallback(self._test_rename_mailbox_cb)
+ return d
+
+ def _test_rename_mailbox_cb(self, mboxes):
+ expected = ['INBOX', 'RenamedMailbox']
+ self.assertItemsEqual(mboxes, expected)
+
+ def test_get_all_mailboxes(self):
+ acc = self.get_account()
+ d = acc.callWhenReady(lambda _: acc.add_mailbox("OneMailbox"))
+ d.addCallback(lambda _: acc.add_mailbox("TwoMailbox"))
+ d.addCallback(lambda _: acc.add_mailbox("ThreeMailbox"))
+ d.addCallback(lambda _: acc.add_mailbox("anotherthing"))
+ d.addCallback(lambda _: acc.add_mailbox("anotherthing2"))
+ d.addCallback(lambda _: acc.get_all_mailboxes())
+ d.addCallback(self._test_get_all_mailboxes_cb)
+ return d
+
+ def _test_get_all_mailboxes_cb(self, mailboxes):
+ expected = ["INBOX", "OneMailbox", "TwoMailbox", "ThreeMailbox",
+ "anotherthing", "anotherthing2"]
+ names = [m.mbox for m in mailboxes]
+ self.assertItemsEqual(names, expected)
+
+ def test_get_collection_by_mailbox(self):
+ acc = self.get_account()
+ d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox("INBOX"))
+ d.addCallback(self._test_get_collection_by_mailbox_cb)
+ return d
+
+ def _test_get_collection_by_mailbox_cb(self, collection):
+ self.assertTrue(collection.is_mailbox_collection())
+
+ def assert_uid_next_empty_collection(uid):
+ self.assertEqual(uid, 1)
+ d = collection.get_uid_next()
+ d.addCallback(assert_uid_next_empty_collection)
+ return d
+
+ def test_get_collection_by_docs(self):
+ pass
+
+ test_get_collection_by_docs.skip = "Not yet implemented"
+
+ def test_get_collection_by_tag(self):
+ pass
+
+ test_get_collection_by_tag.skip = "Not yet implemented"
diff --git a/src/leap/mail/tests/test_mailbox_indexer.py b/src/leap/mail/tests/test_mailbox_indexer.py
new file mode 100644
index 0000000..b82fd2d
--- /dev/null
+++ b/src/leap/mail/tests/test_mailbox_indexer.py
@@ -0,0 +1,250 @@
+# -*- coding: utf-8 -*-
+# test_mailbox_indexer.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the mailbox_indexer module.
+"""
+import uuid
+from functools import partial
+
+from leap.mail import mailbox_indexer as mi
+from leap.mail.tests.common import SoledadTestMixin
+
+hash_test0 = '590c9f8430c7435807df8ba9a476e3f1295d46ef210f6efae2043a4c085a569e'
+hash_test1 = '1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014'
+hash_test2 = '60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752'
+hash_test3 = 'fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13'
+hash_test4 = 'a4e624d686e03ed2767c0abd85c14426b0b1157d2ce81d27bb4fe4f6f01d688a'
+
+
+def fmt_hash(mailbox_uuid, hash):
+ return "M-" + mailbox_uuid.replace('-', '_') + "-" + hash
+
+mbox_id = str(uuid.uuid4())
+
+
+class MailboxIndexerTestCase(SoledadTestMixin):
+ """
+ Tests for the MailboxUID class.
+ """
+ def get_mbox_uid(self):
+ m_uid = mi.MailboxIndexer(self._soledad)
+ return m_uid
+
+ def list_mail_tables_cb(self, ignored):
+ def filter_mailuid_tables(tables):
+ filtered = [
+ table[0] for table in tables if
+ table[0].startswith(mi.MailboxIndexer.table_preffix)]
+ return filtered
+
+ sql = "SELECT name FROM sqlite_master WHERE type='table';"
+ d = self._soledad.raw_sqlcipher_query(sql)
+ d.addCallback(filter_mailuid_tables)
+ return d
+
+ def select_uid_rows(self, mailbox):
+ sql = "SELECT * FROM %s%s;" % (
+ mi.MailboxIndexer.table_preffix, mailbox.replace('-', '_'))
+ d = self._soledad.raw_sqlcipher_query(sql)
+ return d
+
+ def test_create_table(self):
+ def assert_table_created(tables):
+ self.assertEqual(
+ tables, ["leapmail_uid_" + mbox_id.replace('-', '_')])
+
+ m_uid = self.get_mbox_uid()
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(self.list_mail_tables_cb)
+ d.addCallback(assert_table_created)
+ return d
+
+ def test_create_and_delete_table(self):
+ def assert_table_deleted(tables):
+ self.assertEqual(tables, [])
+
+ m_uid = self.get_mbox_uid()
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(lambda _: m_uid.delete_table(mbox_id))
+ d.addCallback(self.list_mail_tables_cb)
+ d.addCallback(assert_table_deleted)
+ return d
+
+ def test_insert_doc(self):
+ m_uid = self.get_mbox_uid()
+
+ h1 = fmt_hash(mbox_id, hash_test0)
+ h2 = fmt_hash(mbox_id, hash_test1)
+ h3 = fmt_hash(mbox_id, hash_test2)
+ h4 = fmt_hash(mbox_id, hash_test3)
+ h5 = fmt_hash(mbox_id, hash_test4)
+
+ def assert_uid_rows(rows):
+ expected = [(1, h1), (2, h2), (3, h3), (4, h4), (5, h5)]
+ self.assertEquals(rows, expected)
+
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5))
+ d.addCallback(lambda _: self.select_uid_rows(mbox_id))
+ d.addCallback(assert_uid_rows)
+ return d
+
+ def test_insert_doc_return(self):
+ m_uid = self.get_mbox_uid()
+
+ def assert_rowid(rowid, expected=None):
+ self.assertEqual(rowid, expected)
+
+ h1 = fmt_hash(mbox_id, hash_test0)
+ h2 = fmt_hash(mbox_id, hash_test1)
+ h3 = fmt_hash(mbox_id, hash_test2)
+
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1))
+ d.addCallback(partial(assert_rowid, expected=1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2))
+ d.addCallback(partial(assert_rowid, expected=2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3))
+ d.addCallback(partial(assert_rowid, expected=3))
+ return d
+
+ def test_delete_doc(self):
+ m_uid = self.get_mbox_uid()
+
+ h1 = fmt_hash(mbox_id, hash_test0)
+ h2 = fmt_hash(mbox_id, hash_test1)
+ h3 = fmt_hash(mbox_id, hash_test2)
+ h4 = fmt_hash(mbox_id, hash_test3)
+ h5 = fmt_hash(mbox_id, hash_test4)
+
+ def assert_uid_rows(rows):
+ expected = [(4, h4), (5, h5)]
+ self.assertEquals(rows, expected)
+
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5))
+
+ d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1))
+ d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 2))
+ d.addCallbacks(lambda _: m_uid.delete_doc_by_hash(mbox_id, h3))
+
+ d.addCallback(lambda _: self.select_uid_rows(mbox_id))
+ d.addCallback(assert_uid_rows)
+ return d
+
+ def test_get_doc_id_from_uid(self):
+ m_uid = self.get_mbox_uid()
+
+ h1 = fmt_hash(mbox_id, hash_test0)
+
+ def assert_doc_hash(res):
+ self.assertEqual(res, h1)
+
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1))
+ d.addCallback(lambda _: m_uid.get_doc_id_from_uid(mbox_id, 1))
+ d.addCallback(assert_doc_hash)
+ return d
+
+ def test_count(self):
+ m_uid = self.get_mbox_uid()
+
+ h1 = fmt_hash(mbox_id, hash_test0)
+ h2 = fmt_hash(mbox_id, hash_test1)
+ h3 = fmt_hash(mbox_id, hash_test2)
+ h4 = fmt_hash(mbox_id, hash_test3)
+ h5 = fmt_hash(mbox_id, hash_test4)
+
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5))
+
+ def assert_count_after_inserts(count):
+ self.assertEquals(count, 5)
+
+ d.addCallback(lambda _: m_uid.count(mbox_id))
+ d.addCallback(assert_count_after_inserts)
+
+ d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1))
+ d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 2))
+
+ def assert_count_after_deletions(count):
+ self.assertEquals(count, 3)
+
+ d.addCallback(lambda _: m_uid.count(mbox_id))
+ d.addCallback(assert_count_after_deletions)
+ return d
+
+ def test_get_next_uid(self):
+ m_uid = self.get_mbox_uid()
+
+ h1 = fmt_hash(mbox_id, hash_test0)
+ h2 = fmt_hash(mbox_id, hash_test1)
+ h3 = fmt_hash(mbox_id, hash_test2)
+ h4 = fmt_hash(mbox_id, hash_test3)
+ h5 = fmt_hash(mbox_id, hash_test4)
+
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5))
+
+ def assert_next_uid(result, expected=1):
+ self.assertEquals(result, expected)
+
+ d.addCallback(lambda _: m_uid.get_next_uid(mbox_id))
+ d.addCallback(partial(assert_next_uid, expected=6))
+ return d
+
+ def test_all_uid_iter(self):
+
+ m_uid = self.get_mbox_uid()
+
+ h1 = fmt_hash(mbox_id, hash_test0)
+ h2 = fmt_hash(mbox_id, hash_test1)
+ h3 = fmt_hash(mbox_id, hash_test2)
+ h4 = fmt_hash(mbox_id, hash_test3)
+ h5 = fmt_hash(mbox_id, hash_test4)
+
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5))
+ d.addCallback(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1))
+ d.addCallback(lambda _: m_uid.delete_doc_by_uid(mbox_id, 4))
+
+ def assert_all_uid(result, expected=[2, 3, 5]):
+ self.assertEquals(result, expected)
+
+ d.addCallback(lambda _: m_uid.all_uid_iter(mbox_id))
+ d.addCallback(partial(assert_all_uid))
+ return d
diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py
index fed24b3..64fca98 100644
--- a/src/leap/mail/utils.py
+++ b/src/leap/mail/utils.py
@@ -17,12 +17,15 @@
"""
Mail utilities.
"""
+from email.utils import parseaddr
import json
import re
import traceback
import Queue
from leap.soledad.common.document import SoledadDocument
+from leap.common.check import leap_assert_type
+from twisted.mail import smtp
CHARSET_PATTERN = r"""charset=([\w-]+)"""
@@ -42,9 +45,12 @@ def first(things):
def empty(thing):
"""
Return True if a thing is None or its length is zero.
+ If thing is a number (int, float, long), return False.
"""
if thing is None:
return True
+ if isinstance(thing, (int, float, long)):
+ return False
if isinstance(thing, SoledadDocument):
thing = thing.content
try:
@@ -224,10 +230,31 @@ def accumulator_queue(fun, lim):
return _accumulator
+def validate_address(address):
+ """
+ Validate C{address} as defined in RFC 2822.
+
+ :param address: The address to be validated.
+ :type address: str
+
+ @return: A valid address.
+ @rtype: str
+
+ @raise smtp.SMTPBadRcpt: Raised if C{address} is invalid.
+ """
+ leap_assert_type(address, str)
+ # in the following, the address is parsed as described in RFC 2822 and
+ # ('', '') is returned if the parse fails.
+ _, address = parseaddr(address)
+ if address == '':
+ raise smtp.SMTPBadRcpt(address)
+ return address
+
#
# String manipulation
#
+
class CustomJsonScanner(object):
"""
This class is a context manager definition used to monkey patch the default
@@ -273,13 +300,13 @@ class CustomJsonScanner(object):
end = s.find("\"", idx)
while not found:
try:
- if s[end-1] != "\\":
+ if s[end - 1] != "\\":
found = True
else:
- end = s.find("\"", end+1)
+ end = s.find("\"", end + 1)
except Exception:
found = True
- return s[idx:end].decode("string-escape"), end+1
+ return s[idx:end].decode("string-escape"), end + 1
def __enter__(self):
"""
@@ -325,3 +352,24 @@ def json_loads(data):
obj = json.loads(data, cls=json.JSONDecoder)
return obj
+
+
+class CaseInsensitiveDict(dict):
+ """
+ A dictionary subclass that will allow case-insenstive key lookups.
+ """
+ def __init__(self, d=None):
+ if d is None:
+ d = []
+ if isinstance(d, dict):
+ for key, value in d.items():
+ self[key] = value
+ else:
+ for key, value in d:
+ self[key] = value
+
+ def __setitem__(self, key, value):
+ super(CaseInsensitiveDict, self).__setitem__(key.lower(), value)
+
+ def __getitem__(self, key):
+ return super(CaseInsensitiveDict, self).__getitem__(key.lower())
diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py
index f747377..1c74366 100644
--- a/src/leap/mail/walk.py
+++ b/src/leap/mail/walk.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# walk.py
-# Copyright (C) 2013 LEAP
+# Copyright (C) 2013-2015 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -26,45 +26,60 @@ from leap.mail.utils import first
DEBUG = os.environ.get("BITMASK_MAIL_DEBUG")
if DEBUG:
- get_hash = lambda s: sha256.SHA256(s).hexdigest()[:10]
+ def get_hash(s):
+ return sha256.SHA256(s).hexdigest()[:10]
else:
- get_hash = lambda s: sha256.SHA256(s).hexdigest()
+ def get_hash(s):
+ return sha256.SHA256(s).hexdigest()
"""
Get interesting message parts
"""
-get_parts = lambda msg: [
- {'multi': part.is_multipart(),
- 'ctype': part.get_content_type(),
- 'size': len(part.as_string()),
- 'parts': len(part.get_payload())
- if isinstance(part.get_payload(), list)
- else 1,
- 'headers': part.items(),
- 'phash': get_hash(part.get_payload())
- if not part.is_multipart() else None}
- for part in msg.walk()]
+
+
+def get_parts(msg):
+ return [
+ {
+ 'multi': part.is_multipart(),
+ 'ctype': part.get_content_type(),
+ 'size': len(part.as_string()),
+ 'parts':
+ len(part.get_payload())
+ if isinstance(part.get_payload(), list)
+ else 1,
+ 'headers': part.items(),
+ 'phash':
+ get_hash(part.get_payload())
+ if not part.is_multipart()
+ else None
+ } for part in msg.walk()]
"""
Utility lambda functions for getting the parts vector and the
payloads from the original message.
"""
-get_parts_vector = lambda parts: (x.get('parts', 1) for x in parts)
-get_payloads = lambda msg: ((x.get_payload(),
- dict(((str.lower(k), v) for k, v in (x.items()))))
- for x in msg.walk())
-get_body_phash_simple = lambda payloads: first(
- [get_hash(payload) for payload, headers in payloads
- if payloads])
+def get_parts_vector(parts):
+ return (x.get('parts', 1) for x in parts)
+
-get_body_phash_multi = lambda payloads: (first(
- [get_hash(payload) for payload, headers in payloads
- if payloads
- and "text/plain" in headers.get('content-type', '')])
- or get_body_phash_simple(payloads))
+def get_payloads(msg):
+ return ((x.get_payload(),
+ dict(((str.lower(k), v) for k, v in (x.items()))))
+ for x in msg.walk())
+
+
+def get_body_phash(msg):
+ """
+ Find the body payload-hash for this message.
+ """
+ for part in msg.walk():
+ # XXX what other ctypes should be considered body?
+ if part.get_content_type() in ("text/plain", "text/html"):
+ # XXX avoid hashing again
+ return get_hash(part.get_payload())
"""
On getting the raw docs, we get also some of the headers to be able to
@@ -72,18 +87,51 @@ index the content. Here we remove any mutable part, as the the filename
in the content disposition.
"""
-get_raw_docs = lambda msg, parts: (
- {"type": "cnt", # type content they'll be
- "raw": payload if not DEBUG else payload[:100],
- "phash": get_hash(payload),
- "content-disposition": first(headers.get(
- 'content-disposition', '').split(';')),
- "content-type": headers.get(
- 'content-type', ''),
- "content-transfer-encoding": headers.get(
- 'content-transfer-type', '')}
- for payload, headers in get_payloads(msg)
- if not isinstance(payload, list))
+
+def get_raw_docs(msg, parts):
+ return (
+ {
+ "type": "cnt", # type content they'll be
+ "raw": payload if not DEBUG else payload[:100],
+ "phash": get_hash(payload),
+ "content-disposition": first(headers.get(
+ 'content-disposition', '').split(';')),
+ "content-type": headers.get(
+ 'content-type', ''),
+ "content-transfer-encoding": headers.get(
+ 'content-transfer-encoding', '')
+ } for payload, headers in get_payloads(msg)
+ if not isinstance(payload, list))
+
+
+"""
+Groucho Marx: Now pay particular attention to this first clause, because it's
+ most important. There's the party of the first part shall be
+ known in this contract as the party of the first part. How do you
+ like that, that's pretty neat eh?
+
+Chico Marx: No, that's no good.
+Groucho Marx: What's the matter with it?
+
+Chico Marx: I don't know, let's hear it again.
+Groucho Marx: So the party of the first part shall be known in this contract as
+ the party of the first part.
+
+Chico Marx: Well it sounds a little better this time.
+Groucho Marx: Well, it grows on you. Would you like to hear it once more?
+
+Chico Marx: Just the first part.
+Groucho Marx: All right. It says the first part of the party of the first part
+ shall be known in this contract as the first part of the party of
+ the first part, shall be known in this contract - look, why
+ should we quarrel about a thing like this, we'll take it right
+ out, eh?
+
+Chico Marx: Yes, it's too long anyhow. Now what have we got left?
+Groucho Marx: Well I've got about a foot and a half. Now what's the matter?
+
+Chico Marx: I don't like the second party either.
+"""
def walk_msg_tree(parts, body_phash=None):
@@ -93,7 +141,7 @@ def walk_msg_tree(parts, body_phash=None):
documents that will be stored in Soledad.
It walks down the subparts in the parsed message tree, and collapses
- the leaf docuents into a wrapper document until no multipart submessages
+ the leaf documents into a wrapper document until no multipart submessages
are left. To achieve this, it iteratively calculates a wrapper vector of
all documents in the sequence that have more than one part and have unitary
documents to their right. To collapse a multipart, take as many
@@ -125,8 +173,12 @@ def walk_msg_tree(parts, body_phash=None):
print
# wrappers vector
- getwv = lambda pv: [True if pv[i] != 1 and pv[i + 1] == 1 else False
- for i in range(len(pv) - 1)]
+ def getwv(pv):
+ return [
+ True if pv[i] != 1 and pv[i + 1] == 1
+ else False
+ for i in range(len(pv) - 1)
+ ]
wv = getwv(pv)
# do until no wrapper document is left
@@ -142,7 +194,7 @@ def walk_msg_tree(parts, body_phash=None):
HEADERS: dict(parts[wind][HEADERS])
}
- # remove subparts and substitue wrapper
+ # remove subparts and substitute wrapper
map(lambda i: parts.remove(i), slic)
parts[wind] = cwra
@@ -157,12 +209,12 @@ def walk_msg_tree(parts, body_phash=None):
last_part = max(main_pmap.keys())
main_pmap[last_part][PART_MAP] = {}
for partind in range(len(pv) - 1):
- print partind+1, len(parts)
+ print partind + 1, len(parts)
main_pmap[last_part][PART_MAP][partind] = parts[partind + 1]
outer = parts[0]
outer.pop(HEADERS)
- if not PART_MAP in outer:
+ if PART_MAP not in outer:
# we have a multipart with 1 part only, so kind of fix it
# although it would be prettier if I take this special case at
# the beginning of the walk.
@@ -177,36 +229,3 @@ def walk_msg_tree(parts, body_phash=None):
pdoc = outer
pdoc[BODY] = body_phash
return pdoc
-
-"""
-Groucho Marx: Now pay particular attention to this first clause, because it's
- most important. There's the party of the first part shall be
- known in this contract as the party of the first part. How do you
- like that, that's pretty neat eh?
-
-Chico Marx: No, that's no good.
-Groucho Marx: What's the matter with it?
-
-Chico Marx: I don't know, let's hear it again.
-Groucho Marx: So the party of the first part shall be known in this contract as
- the party of the first part.
-
-Chico Marx: Well it sounds a little better this time.
-Groucho Marx: Well, it grows on you. Would you like to hear it once more?
-
-Chico Marx: Just the first part.
-Groucho Marx: All right. It says the first part of the party of the first part
- shall be known in this contract as the first part of the party of
- the first part, shall be known in this contract - look, why
- should we quarrel about a thing like this, we'll take it right
- out, eh?
-
-Chico Marx: Yes, it's too long anyhow. Now what have we got left?
-Groucho Marx: Well I've got about a foot and a half. Now what's the matter?
-
-Chico Marx: I don't like the second party either.
-"""
-
-"""
-I feel you deserved it after reading the above and try to debug your problem ;)
-"""