From cd5d18ae0ef5286b3a9f12705256d68017c6ad1e Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Tue, 16 Sep 2014 10:54:09 -0500 Subject: The get_key cache now it's automagical --- src/leap/mail/imap/fetch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index 0a97752..9cd2940 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -433,7 +433,7 @@ class LeapIncomingMail(object): or msg.get_content_type() == MULTIPART_SIGNED)): _, senderAddress = parseaddr(fromHeader) try: - senderPubkey = self._keymanager.get_key_from_cache( + senderPubkey = self._keymanager.get_key( senderAddress, OpenPGPKey) except keymanager_errors.KeyNotFound: pass -- cgit v1.2.3 From cf32f8bd5685114310c8dd6ce1368f230ed643a6 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Mon, 6 Oct 2014 11:36:38 -0500 Subject: Add new leap.keymanager version to VERSION_COMPAT --- changes/VERSION_COMPAT | 1 + 1 file changed, 1 insertion(+) diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT index cc00ecf..1eadcbe 100644 --- a/changes/VERSION_COMPAT +++ b/changes/VERSION_COMPAT @@ -8,3 +8,4 @@ # # BEGIN DEPENDENCY LIST ------------------------- # leap.foo.bar>=x.y.z +leap.keymanager>=0.4.0 -- cgit v1.2.3 From f47897be1ad5351bdb91388d07c0c876c4084b11 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Mon, 6 Oct 2014 09:56:25 -0500 Subject: Update docstrings --- src/leap/mail/imap/fetch.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index 9cd2940..d78b4ee 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -31,7 +31,7 @@ from email.utils import parseaddr from StringIO import StringIO from twisted.python import log -from twisted.internet import defer +from twisted.internet import defer, reactor from twisted.internet.task import LoopingCall from twisted.internet.task import deferLater from u1db import errors as u1db_errors @@ -343,14 +343,12 @@ class LeapIncomingMail(object): and data, the json-encoded, decrypted content of the incoming message :type msgtuple: (SoledadDocument, str) - :return: a SoledadDocument and the processed data. - :rtype: (doc, data) + :return: the processed data. + :rtype: str """ log.msg('processing decrypted doc') doc, data = msgtuple - from twisted.internet import reactor - # XXX turn this into an errBack for each one of # the deferreds that would process an individual document try: @@ -382,7 +380,7 @@ class LeapIncomingMail(object): # ok, this is an incoming message rawmsg = msg.get(self.CONTENT_KEY, None) if not rawmsg: - return False + return "" return self._maybe_decrypt_msg(rawmsg) @deferred_to_thread @@ -415,7 +413,7 @@ class LeapIncomingMail(object): :param data: the text to be decrypted. :type data: str - :return: data, possibly descrypted. + :return: data, possibly decrypted. :rtype: str """ leap_assert_type(data, str) @@ -474,8 +472,9 @@ class LeapIncomingMail(object): :param senderPubkey: The key of the sender of the message. :type senderPubkey: OpenPGPKey - :return: A unitary tuple containing a decrypted message. - :rtype: (Message) + :return: A tuple containing a decrypted message and + a bool indicating whether the signature is valid. + :rtype: (Message, bool) """ log.msg('decrypting multipart encrypted msg') msg = copy.deepcopy(msg) @@ -594,7 +593,6 @@ class LeapIncomingMail(object): incoming message :type msgtuple: (SoledadDocument, str) """ - from twisted.internet import reactor msgtuple = first(result) doc, data = msgtuple -- cgit v1.2.3 From 0f6a093bfbbac2cd738c32287b2316d481ed67f3 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Thu, 9 Oct 2014 00:59:32 -0500 Subject: Get keys from OpenPGP email header --- changes/feature-3879_openpgp_header | 1 + src/leap/mail/imap/fetch.py | 89 ++++-- src/leap/mail/imap/messages.py | 4 +- src/leap/mail/imap/tests/test_imap.py | 7 +- src/leap/mail/imap/tests/test_incoming_mail.py | 158 ++++++++++ src/leap/mail/smtp/tests/__init__.py | 386 ------------------------- src/leap/mail/smtp/tests/test_gateway.py | 2 +- src/leap/mail/tests/__init__.py | 386 +++++++++++++++++++++++++ 8 files changed, 620 insertions(+), 413 deletions(-) create mode 100644 changes/feature-3879_openpgp_header create mode 100644 src/leap/mail/imap/tests/test_incoming_mail.py delete mode 100644 src/leap/mail/smtp/tests/__init__.py create mode 100644 src/leap/mail/tests/__init__.py diff --git a/changes/feature-3879_openpgp_header b/changes/feature-3879_openpgp_header new file mode 100644 index 0000000..e04c925 --- /dev/null +++ b/changes/feature-3879_openpgp_header @@ -0,0 +1 @@ +- Parse OpenPGP header and import keys from it (Closes: #3879) diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index d78b4ee..863f5fe 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -19,9 +19,9 @@ Incoming mail fetcher. """ import copy import logging +import shlex import threading import time -import sys import traceback import warnings @@ -29,6 +29,7 @@ from email.parser import Parser from email.generator import Generator from email.utils import parseaddr from StringIO import StringIO +from urlparse import urlparse from twisted.python import log from twisted.internet import defer, reactor @@ -169,7 +170,7 @@ class LeapIncomingMail(object): DeprecationWarning) doclist = self._soledad.get_from_index( fields.JUST_MAIL_COMPAT_IDX, "*") - self._process_doclist(doclist) + return self._process_doclist(doclist) logger.debug("fetching mail for: %s %s" % ( self._soledad.uuid, self._userid)) @@ -209,7 +210,7 @@ class LeapIncomingMail(object): def _errback(self, failure): logger.exception(failure.value) - traceback.print_tb(*sys.exc_info()) + traceback.print_exc() @deferred_to_thread def _sync_soledad(self): @@ -273,6 +274,7 @@ class LeapIncomingMail(object): return num_mails = len(doclist) + deferreds = [] for index, doc in enumerate(doclist): logger.debug("processing doc %d of %d" % (index + 1, num_mails)) leap_events.signal( @@ -288,19 +290,15 @@ class LeapIncomingMail(object): if has_errors is None: warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", DeprecationWarning) + if has_errors: logger.debug("skipping msg with decrypting errors...") - - if self._is_msg(keys) and not has_errors: - # Evaluating to bool of has_errors is intentional here. - # We don't mind at this point if it's None or False. - - # Ok, this looks like a legit msg, and with no errors. - # Let's process it! - - d1 = self._decrypt_doc(doc) - d = defer.gatherResults([d1], consumeErrors=True) + elif self._is_msg(keys): + d = self._decrypt_doc(doc) + d.addCallback(self._extract_keys) d.addCallbacks(self._add_message_locally, self._errback) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) # # operations on individual messages @@ -581,20 +579,77 @@ class LeapIncomingMail(object): data, self._pkey) return (decrdata, valid_sig) - def _add_message_locally(self, result): + def _extract_keys(self, msgtuple): + """ + Parse message headers for an *OpenPGP* header as described on the + `IETF draft + ` + only urls with https and the same hostname than the email are supported + for security reasons. + + :param msgtuple: a tuple consisting of a SoledadDocument + instance containing the incoming message + and data, the json-encoded, decrypted content of the + incoming message + :type msgtuple: (SoledadDocument, str) + """ + OpenPGP_HEADER = 'OpenPGP' + doc, data = msgtuple + + # XXX the parsing of the message is done in mailbox.addMessage, maybe + # we should do it in this module so we don't need to parse it again + # here + msg = self._parser.parsestr(data) + header = msg.get(OpenPGP_HEADER, None) + if header is not None: + self._extract_openpgp_header(msg, header) + + return msgtuple + + def _extract_openpgp_header(self, msg, header): + """ + Import keys from the OpenPGP header + + :param msg: parsed email + :type msg: email.Message + :param header: OpenPGP header string + :type header: str + """ + fields = dict([f.strip(' ').split('=') for f in header.split(';')]) + if 'url' in fields: + url = shlex.split(fields['url'])[0] # remove quotations + _, fromAddress = parseaddr(msg['from']) + urlparts = urlparse(url) + fromHostname = fromAddress.split('@')[1] + if (urlparts.scheme == 'https' + and urlparts.hostname == fromHostname): + try: + self._keymanager.fetch_key(fromAddress, url, OpenPGPKey) + logger.info("Imported key from header %s" % (url,)) + except keymanager_errors.KeyNotFound: + logger.warning("Url from OpenPGP header %s failed" + % (url,)) + except keymanager_errors.KeyAttributesDiffer: + logger.warning("Key from OpenPGP header url %s didn't " + "match the from address %s" + % (url, fromAddress)) + else: + logger.debug("No valid url on OpenPGP header %s" % (url,)) + else: + logger.debug("There is no url on the OpenPGP header: %s" + % (header,)) + + def _add_message_locally(self, msgtuple): """ Adds a message to local inbox and delete it from the incoming db in soledad. - # XXX this comes from a gatherresult... :param msgtuple: a tuple consisting of a SoledadDocument instance containing the incoming message and data, the json-encoded, decrypted content of the incoming message :type msgtuple: (SoledadDocument, str) """ - msgtuple = first(result) - doc, data = msgtuple log.msg('adding message %s to local db' % (doc.doc_id,)) diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 0356600..e8d64d1 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -29,7 +29,7 @@ from functools import partial from pycryptopp.hash import sha256 from twisted.mail import imap4 -from twisted.internet import defer +from twisted.internet import defer, reactor from zope.interface import implements from zope.proxy import sameProxiedObjects @@ -134,7 +134,6 @@ class LeapMessage(fields, MBoxParser): self.__chash = None self.__bdoc = None - from twisted.internet import reactor self.reactor = reactor # XXX make these properties public @@ -740,7 +739,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): else: self._initialized[mbox] = True - from twisted.internet import reactor self.reactor = reactor def _get_empty_doc(self, _type=FLAGS_DOC): diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 631a2c1..7837aaa 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -26,11 +26,6 @@ XXX add authors from the original twisted tests. """ # XXX review license of the original tests!!! -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - import os import types @@ -218,7 +213,7 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): #mc._soledad.create_doc(newmsg) #self.assertEqual(mc.count(), 3) #self.assertEqual( - #len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4) + #len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4) class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): diff --git a/src/leap/mail/imap/tests/test_incoming_mail.py b/src/leap/mail/imap/tests/test_incoming_mail.py new file mode 100644 index 0000000..5b72fac --- /dev/null +++ b/src/leap/mail/imap/tests/test_incoming_mail.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +# test_imap.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test case for leap.email.imap.fetch + +@authors: Ruben Pollan, + +@license: GPLv3, see included LICENSE file +""" + +import json + +from email.parser import Parser +from mock import Mock +from twisted.trial import unittest + +from leap.keymanager.openpgp import OpenPGPKey +from leap.mail.imap.account import SoledadBackedAccount +from leap.mail.imap.fetch import LeapIncomingMail +from leap.mail.imap.fields import fields +from leap.mail.imap.memorystore import MemoryStore +from leap.mail.imap.service.imap import INCOMING_CHECK_PERIOD +from leap.mail.tests import ( + TestCaseWithKeyManager, + ADDRESS, +) +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.crypto import ( + EncryptionSchemes, + ENC_JSON_KEY, + ENC_SCHEME_KEY, +) + + +class LeapIncomingMailTestCase(TestCaseWithKeyManager, unittest.TestCase): + """ + Tests for the incoming mail parser + """ + NICKSERVER = "http://domain" + FROM_ADDRESS = "test@somedomain.com" + BODY = """ +Governments of the Industrial World, you weary giants of flesh and steel, I +come from Cyberspace, the new home of Mind. On behalf of the future, I ask +you of the past to leave us alone. You are not welcome among us. You have +no sovereignty where we gather. + """ + EMAIL = """from: Test from SomeDomain <%(from)s> +to: %(to)s +subject: independence of cyberspace + +%(body)s + """ % { + "from": FROM_ADDRESS, + "to": ADDRESS, + "body": BODY + } + + def setUp(self): + super(LeapIncomingMailTestCase, self).setUp() + + # Soledad sync makes trial block forever. The sync it's mocked to fix + # this problem. _mock_soledad_get_from_index can be used from the tests + # to provide documents. + self._soledad.sync = Mock() + + memstore = MemoryStore() + theAccount = SoledadBackedAccount( + ADDRESS, + soledad=self._soledad, + memstore=memstore) + self.fetcher = LeapIncomingMail( + self._km, + self._soledad, + theAccount, + INCOMING_CHECK_PERIOD, + ADDRESS) + + def tearDown(self): + del self.fetcher + super(LeapIncomingMailTestCase, self).tearDown() + + def testExtractOpenPGPHeader(self): + """ + Test the OpenPGP header key extraction + """ + KEYURL = "https://somedomain.com/key.txt" + OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) + + message = Parser().parsestr(self.EMAIL) + message.add_header("OpenPGP", OpenPGP) + email = self._create_incoming_email(message.as_string()) + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]) + self.fetcher._keymanager.fetch_key = Mock() + d = self.fetcher.fetch() + + def fetch_key_called(ret): + self.fetcher._keymanager.fetch_key.assert_called_once_with( + self.FROM_ADDRESS, KEYURL, OpenPGPKey) + d.addCallback(fetch_key_called) + + return d + + def testExtractOpenPGPHeaderInvalidUrl(self): + """ + Test the OpenPGP header key extraction + """ + KEYURL = "https://someotherdomain.com/key.txt" + OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) + + message = Parser().parsestr(self.EMAIL) + message.add_header("OpenPGP", OpenPGP) + email = self._create_incoming_email(message.as_string()) + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]) + self.fetcher._keymanager.fetch_key = Mock() + d = self.fetcher.fetch() + + def fetch_key_called(ret): + self.assertFalse(self.fetcher._keymanager.fetch_key.called) + d.addCallback(fetch_key_called) + + return d + + def _create_incoming_email(self, email_str): + email = SoledadDocument() + pubkey = self._km.get_key(ADDRESS, OpenPGPKey) + data = json.dumps( + {"incoming": True, "content": email_str}, + ensure_ascii=False) + email.content = { + fields.INCOMING_KEY: True, + fields.ERROR_DECRYPTING_KEY: False, + ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, + ENC_JSON_KEY: str(self._km.encrypt(data, pubkey)) + } + return email + + def _mock_soledad_get_from_index(self, index_name, value): + get_from_index = self._soledad.get_from_index + + def soledad_mock(idx_name, *key_values): + if index_name == idx_name: + return value + return get_from_index(idx_name, *key_values) + self.fetcher._soledad.get_from_index = Mock(side_effect=soledad_mock) diff --git a/src/leap/mail/smtp/tests/__init__.py b/src/leap/mail/smtp/tests/__init__.py deleted file mode 100644 index dc24293..0000000 --- a/src/leap/mail/smtp/tests/__init__.py +++ /dev/null @@ -1,386 +0,0 @@ -# -*- coding: utf-8 -*- -# __init__.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -Base classes and keys for SMTP gateway tests. -""" - -import os -import distutils.spawn -import shutil -import tempfile -from mock import Mock - - -from twisted.trial import unittest - - -from leap.soledad.client import Soledad -from leap.keymanager import ( - KeyManager, - openpgp, -) - - -from leap.common.testing.basetest import BaseLeapTest - - -def _find_gpg(): - gpg_path = distutils.spawn.find_executable('gpg') - return os.path.realpath(gpg_path) if gpg_path is not None else "/usr/bin/gpg" - - -class TestCaseWithKeyManager(BaseLeapTest): - - GPG_BINARY_PATH = _find_gpg() - - def setUp(self): - # mimic BaseLeapTest.setUpClass behaviour, because this is deprecated - # in Twisted: http://twistedmatrix.com/trac/ticket/1870 - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir - - # setup our own stuff - address = 'leap@leap.se' # user's address in the form user@provider - uuid = 'leap@leap.se' - passphrase = u'123' - secrets_path = os.path.join(self.tempdir, 'secret.gpg') - local_db_path = os.path.join(self.tempdir, 'soledad.u1db') - server_url = 'http://provider/' - cert_file = '' - - self._soledad = self._soledad_instance( - uuid, passphrase, secrets_path, local_db_path, server_url, - cert_file) - self._km = self._keymanager_instance(address) - - def _soledad_instance(self, uuid, passphrase, secrets_path, local_db_path, - server_url, cert_file): - """ - Return a Soledad instance for tests. - """ - # mock key fetching and storing so Soledad doesn't fail when trying to - # reach the server. - Soledad._fetch_keys_from_shared_db = Mock(return_value=None) - Soledad._assert_keys_in_shared_db = Mock(return_value=None) - - # instantiate soledad - def _put_doc_side_effect(doc): - self._doc_put = doc - - class MockSharedDB(object): - - get_doc = Mock(return_value=None) - put_doc = Mock(side_effect=_put_doc_side_effect) - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) - - def __call__(self): - return self - - Soledad._shared_db = MockSharedDB() - - return Soledad( - uuid, - passphrase, - secrets_path=secrets_path, - local_db_path=local_db_path, - server_url=server_url, - cert_file=cert_file, - ) - - def _keymanager_instance(self, address): - """ - Return a Key Manager instance for tests. - """ - self._config = { - 'host': 'http://provider/', - 'port': 25, - 'username': address, - 'password': '', - 'encrypted_only': True, - 'cert': u'src/leap/mail/smtp/tests/cert/server.crt', - 'key': u'src/leap/mail/smtp/tests/cert/server.key', - } - - class Response(object): - status_code = 200 - headers = {'content-type': 'application/json'} - - def json(self): - return {'address': ADDRESS_2, 'openpgp': PUBLIC_KEY_2} - - def raise_for_status(self): - pass - - nickserver_url = '' # the url of the nickserver - km = KeyManager(address, nickserver_url, self._soledad, - ca_cert_path='', gpgbinary=self.GPG_BINARY_PATH) - km._fetcher.put = Mock() - km._fetcher.get = Mock(return_value=Response()) - - # insert test keys in key manager. - pgp = openpgp.OpenPGPScheme( - self._soledad, gpgbinary=self.GPG_BINARY_PATH) - pgp.put_ascii_key(PRIVATE_KEY) - pgp.put_ascii_key(PRIVATE_KEY_2) - - return km - - def tearDown(self): - # mimic LeapBaseTest.tearDownClass behaviour - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check - assert 'leap_tests-' in self.tempdir - shutil.rmtree(self.tempdir) - - -# Key material for testing -KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF" - -ADDRESS = 'leap@leap.se' - -PUBLIC_KEY = """ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -mQINBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz -iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO -zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx -irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT -huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs -d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g -wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb -hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv -U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H -T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i -Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB -tBxMZWFwIFRlc3QgS2V5IDxsZWFwQGxlYXAuc2U+iQI3BBMBCAAhBQJQvfnZAhsD -BQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEC9FXigk0Y3fT7EQAKH3IuRniOpb -T/DDIgwwjz3oxB/W0DDMyPXowlhSOuM0rgGfntBpBb3boezEXwL86NPQxNGGruF5 -hkmecSiuPSvOmQlqlS95NGQp6hNG0YaKColh+Q5NTspFXCAkFch9oqUje0LdxfSP -QfV9UpeEvGyPmk1I9EJV/YDmZ4+Djge1d7qhVZInz4Rx1NrSyF/Tc2EC0VpjQFsU -Y9Kb2YBBR7ivG6DBc8ty0jJXi7B4WjkFcUEJviQpMF2dCLdonCehYs1PqsN1N7j+ -eFjQd+hqVMJgYuSGKjvuAEfClM6MQw7+FmFwMyLgK/Ew/DttHEDCri77SPSkOGSI -txCzhTg6798f6mJr7WcXmHX1w1Vcib5FfZ8vTDFVhz/XgAgArdhPo9V6/1dgSSiB -KPQ/spsco6u5imdOhckERE0lnAYvVT6KE81TKuhF/b23u7x+Wdew6kK0EQhYA7wy -7LmlaNXc7rMBQJ9Z60CJ4JDtatBWZ0kNrt2VfdDHVdqBTOpl0CraNUjWE5YMDasr -K2dF5IX8D3uuYtpZnxqg0KzyLg0tzL0tvOL1C2iudgZUISZNPKbS0z0v+afuAAnx -2pTC3uezbh2Jt8SWTLhll4i0P4Ps5kZ6HQUO56O+/Z1cWovX+mQekYFmERySDR9n -3k1uAwLilJmRmepGmvYbB8HloV8HqwgguQINBFC9+dkBEAC0I/xn1uborMgDvBtf -H0sEhwnXBC849/32zic6udB6/3Efk9nzbSpL3FSOuXITZsZgCHPkKarnoQ2ztMcS -sh1ke1C5gQGms75UVmM/nS+2YI4vY8OX/GC/on2vUyncqdH+bR6xH5hx4NbWpfTs -iQHmz5C6zzS/kuabGdZyKRaZHt23WQ7JX/4zpjqbC99DjHcP9BSk7tJ8wI4bkMYD -uFVQdT9O6HwyKGYwUU4sAQRAj7XCTGvVbT0dpgJwH4RmrEtJoHAx4Whg8mJ710E0 -GCmzf2jqkNuOw76ivgk27Kge+Hw00jmJjQhHY0yVbiaoJwcRrPKzaSjEVNgrpgP3 -lXPRGQArgESsIOTeVVHQ8fhK2YtTeCY9rIiO+L0OX2xo9HK7hfHZZWL6rqymXdyS -fhzh/f6IPyHFWnvj7Brl7DR8heMikygcJqv+ed2yx7iLyCUJ10g12I48+aEj1aLe -dP7lna32iY8/Z0SHQLNH6PXO9SlPcq2aFUgKqE75A/0FMk7CunzU1OWr2ZtTLNO1 -WT/13LfOhhuEq9jTyTosn0WxBjJKq18lnhzCXlaw6EAtbA7CUwsD3CTPR56aAXFK -3I7KXOVAqggrvMe5Tpdg5drfYpI8hZovL5aAgb+7Y5ta10TcJdUhS5K3kFAWe/td -U0cmWUMDP1UMSQ5Jg6JIQVWhSwARAQABiQIfBBgBCAAJBQJQvfnZAhsMAAoJEC9F -Xigk0Y3fRwsP/i0ElYCyxeLpWJTwo1iCLkMKz2yX1lFVa9nT1BVTPOQwr/IAc5OX -NdtbJ14fUsKL5pWgW8OmrXtwZm1y4euI1RPWWubG01ouzwnGzv26UcuHeqC5orZj -cOnKtL40y8VGMm8LoicVkRJH8blPORCnaLjdOtmA3rx/v2EXrJpSa3AhOy0ZSRXk -ZSrK68AVNwamHRoBSYyo0AtaXnkPX4+tmO8X8BPfj125IljubvwZPIW9VWR9UqCE -VPfDR1XKegVb6VStIywF7kmrknM1C5qUY28rdZYWgKorw01hBGV4jTW0cqde3N51 -XT1jnIAa+NoXUM9uQoGYMiwrL7vNsLlyyiW5ayDyV92H/rIuiqhFgbJsHTlsm7I8 -oGheR784BagAA1NIKD1qEO9T6Kz9lzlDaeWS5AUKeXrb7ZJLI1TTCIZx5/DxjLqM -Tt/RFBpVo9geZQrvLUqLAMwdaUvDXC2c6DaCPXTh65oCZj/hqzlJHH+RoTWWzKI+ -BjXxgUWF9EmZUBrg68DSmI+9wuDFsjZ51BcqvJwxyfxtTaWhdoYqH/UQS+D1FP3/ -diZHHlzwVwPICzM9ooNTgbrcDzyxRkIVqsVwBq7EtzcvgYUyX53yG25Giy6YQaQ2 -ZtQ/VymwFL3XdUWV6B/hU4PVAFvO3qlOtdJ6TpE+nEWgcWjCv5g7RjXX -=MuOY ------END PGP PUBLIC KEY BLOCK----- -""" - -PRIVATE_KEY = """ ------BEGIN PGP PRIVATE KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -lQcYBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz -iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO -zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx -irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT -huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs -d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g -wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb -hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv -U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H -T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i -Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB -AA/+JHtlL39G1wsH9R6UEfUQJGXR9MiIiwZoKcnRB2o8+DS+OLjg0JOh8XehtuCs -E/8oGQKtQqa5bEIstX7IZoYmYFiUQi9LOzIblmp2vxOm+HKkxa4JszWci2/ZmC3t -KtaA4adl9XVnshoQ7pijuCMUKB3naBEOAxd8s9d/JeReGIYkJErdrnVfNk5N71Ds -FmH5Ll3XtEDvgBUQP3nkA6QFjpsaB94FHjL3gDwum/cxzj6pCglcvHOzEhfY0Ddb -J967FozQTaf2JW3O+w3LOqtcKWpq87B7+O61tVidQPSSuzPjCtFF0D2LC9R/Hpky -KTMQ6CaKja4MPhjwywd4QPcHGYSqjMpflvJqi+kYIt8psUK/YswWjnr3r4fbuqVY -VhtiHvnBHQjz135lUqWvEz4hM3Xpnxydx7aRlv5NlevK8+YIO5oFbWbGNTWsPZI5 -jpoFBpSsnR1Q5tnvtNHauvoWV+XN2qAOBTG+/nEbDYH6Ak3aaE9jrpTdYh0CotYF -q7csANsDy3JvkAzeU6WnYpsHHaAjqOGyiZGsLej1UcXPFMosE/aUo4WQhiS8Zx2c -zOVKOi/X5vQ2GdNT9Qolz8AriwzsvFR+bxPzyd8V6ALwDsoXvwEYinYBKK8j0OPv -OOihSR6HVsuP9NUZNU9ewiGzte/+/r6pNXHvR7wTQ8EWLcEIAN6Zyrb0bHZTIlxt -VWur/Ht2mIZrBaO50qmM5RD3T5oXzWXi/pjLrIpBMfeZR9DWfwQwjYzwqi7pxtYx -nJvbMuY505rfnMoYxb4J+cpRXV8MS7Dr1vjjLVUC9KiwSbM3gg6emfd2yuA93ihv -Pe3mffzLIiQa4mRE3wtGcioC43nWuV2K2e1KjxeFg07JhrezA/1Cak505ab/tmvP -4YmjR5c44+yL/YcQ3HdFgs4mV+nVbptRXvRcPpolJsgxPccGNdvHhsoR4gwXMS3F -RRPD2z6x8xeN73Q4KH3bm01swQdwFBZbWVfmUGLxvN7leCdfs9+iFJyqHiCIB6Iv -mQfp8F0IAOwSo8JhWN+V1dwML4EkIrM8wUb4yecNLkyR6TpPH/qXx4PxVMC+vy6x -sCtjeHIwKE+9vqnlhd5zOYh7qYXEJtYwdeDDmDbL8oks1LFfd+FyAuZXY33DLwn0 -cRYsr2OEZmaajqUB3NVmj3H4uJBN9+paFHyFSXrH68K1Fk2o3n+RSf2EiX+eICwI -L6rqoF5sSVUghBWdNegV7qfy4anwTQwrIMGjgU5S6PKW0Dr/3iO5z3qQpGPAj5OW -ATqPWkDICLbObPxD5cJlyyNE2wCA9VVc6/1d6w4EVwSq9h3/WTpATEreXXxTGptd -LNiTA1nmakBYNO2Iyo3djhaqBdWjk+EIAKtVEnJH9FAVwWOvaj1RoZMA5DnDMo7e -SnhrCXl8AL7Z1WInEaybasTJXn1uQ8xY52Ua4b8cbuEKRKzw/70NesFRoMLYoHTO -dyeszvhoDHberpGRTciVmpMu7Hyi33rM31K9epA4ib6QbbCHnxkWOZB+Bhgj1hJ8 -xb4RBYWiWpAYcg0+DAC3w9gfxQhtUlZPIbmbrBmrVkO2GVGUj8kH6k4UV6kUHEGY -HQWQR0HcbKcXW81ZXCCD0l7ROuEWQtTe5Jw7dJ4/QFuqZnPutXVRNOZqpl6eRShw -7X2/a29VXBpmHA95a88rSQsL+qm7Fb3prqRmuMCtrUZgFz7HLSTuUMR867QcTGVh -cCBUZXN0IEtleSA8bGVhcEBsZWFwLnNlPokCNwQTAQgAIQUCUL352QIbAwULCQgH -AwUVCgkICwUWAgMBAAIeAQIXgAAKCRAvRV4oJNGN30+xEACh9yLkZ4jqW0/wwyIM -MI896MQf1tAwzMj16MJYUjrjNK4Bn57QaQW926HsxF8C/OjT0MTRhq7heYZJnnEo -rj0rzpkJapUveTRkKeoTRtGGigqJYfkOTU7KRVwgJBXIfaKlI3tC3cX0j0H1fVKX -hLxsj5pNSPRCVf2A5mePg44HtXe6oVWSJ8+EcdTa0shf03NhAtFaY0BbFGPSm9mA -QUe4rxugwXPLctIyV4uweFo5BXFBCb4kKTBdnQi3aJwnoWLNT6rDdTe4/nhY0Hfo -alTCYGLkhio77gBHwpTOjEMO/hZhcDMi4CvxMPw7bRxAwq4u+0j0pDhkiLcQs4U4 -Ou/fH+pia+1nF5h19cNVXIm+RX2fL0wxVYc/14AIAK3YT6PVev9XYEkogSj0P7Kb -HKOruYpnToXJBERNJZwGL1U+ihPNUyroRf29t7u8flnXsOpCtBEIWAO8Muy5pWjV -3O6zAUCfWetAieCQ7WrQVmdJDa7dlX3Qx1XagUzqZdAq2jVI1hOWDA2rKytnReSF -/A97rmLaWZ8aoNCs8i4NLcy9Lbzi9QtornYGVCEmTTym0tM9L/mn7gAJ8dqUwt7n -s24dibfElky4ZZeItD+D7OZGeh0FDuejvv2dXFqL1/pkHpGBZhEckg0fZ95NbgMC -4pSZkZnqRpr2GwfB5aFfB6sIIJ0HGARQvfnZARAAtCP8Z9bm6KzIA7wbXx9LBIcJ -1wQvOPf99s4nOrnQev9xH5PZ820qS9xUjrlyE2bGYAhz5Cmq56ENs7THErIdZHtQ -uYEBprO+VFZjP50vtmCOL2PDl/xgv6J9r1Mp3KnR/m0esR+YceDW1qX07IkB5s+Q -us80v5LmmxnWcikWmR7dt1kOyV/+M6Y6mwvfQ4x3D/QUpO7SfMCOG5DGA7hVUHU/ -Tuh8MihmMFFOLAEEQI+1wkxr1W09HaYCcB+EZqxLSaBwMeFoYPJie9dBNBgps39o -6pDbjsO+or4JNuyoHvh8NNI5iY0IR2NMlW4mqCcHEazys2koxFTYK6YD95Vz0RkA -K4BErCDk3lVR0PH4StmLU3gmPayIjvi9Dl9saPRyu4Xx2WVi+q6spl3ckn4c4f3+ -iD8hxVp74+wa5ew0fIXjIpMoHCar/nndsse4i8glCddINdiOPPmhI9Wi3nT+5Z2t -9omPP2dEh0CzR+j1zvUpT3KtmhVICqhO+QP9BTJOwrp81NTlq9mbUyzTtVk/9dy3 -zoYbhKvY08k6LJ9FsQYySqtfJZ4cwl5WsOhALWwOwlMLA9wkz0eemgFxStyOylzl -QKoIK7zHuU6XYOXa32KSPIWaLy+WgIG/u2ObWtdE3CXVIUuSt5BQFnv7XVNHJllD -Az9VDEkOSYOiSEFVoUsAEQEAAQAP/1AagnZQZyzHDEgw4QELAspYHCWLXE5aZInX -wTUJhK31IgIXNn9bJ0hFiSpQR2xeMs9oYtRuPOu0P8oOFMn4/z374fkjZy8QVY3e -PlL+3EUeqYtkMwlGNmVw5a/NbNuNfm5Darb7pEfbYd1gPcni4MAYw7R2SG/57GbC -9gucvspHIfOSfBNLBthDzmK8xEKe1yD2eimfc2T7IRYb6hmkYfeds5GsqvGI6mwI -85h4uUHWRc5JOlhVM6yX8hSWx0L60Z3DZLChmc8maWnFXd7C8eQ6P1azJJbW71Ih -7CoK0XW4LE82vlQurSRFgTwfl7wFYszW2bOzCuhHDDtYnwH86Nsu0DC78ZVRnvxn -E8Ke/AJgrdhIOo4UAyR+aZD2+2mKd7/waOUTUrUtTzc7i8N3YXGi/EIaNReBXaq+ -ZNOp24BlFzRp+FCF/pptDW9HjPdiV09x0DgICmeZS4Gq/4vFFIahWctg52NGebT0 -Idxngjj+xDtLaZlLQoOz0n5ByjO/Wi0ANmMv1sMKCHhGvdaSws2/PbMR2r4caj8m -KXpIgdinM/wUzHJ5pZyF2U/qejsRj8Kw8KH/tfX4JCLhiaP/mgeTuWGDHeZQERAT -xPmRFHaLP9/ZhvGNh6okIYtrKjWTLGoXvKLHcrKNisBLSq+P2WeFrlme1vjvJMo/ -jPwLT5o9CADQmcbKZ+QQ1ZM9v99iDZol7SAMZX43JC019sx6GK0u6xouJBcLfeB4 -OXacTgmSYdTa9RM9fbfVpti01tJ84LV2SyL/VJq/enJF4XQPSynT/tFTn1PAor6o -tEAAd8fjKdJ6LnD5wb92SPHfQfXqI84rFEO8rUNIE/1ErT6DYifDzVCbfD2KZdoF -cOSp7TpD77sY1bs74ocBX5ejKtd+aH99D78bJSMM4pSDZsIEwnomkBHTziubPwJb -OwnATy0LmSMAWOw5rKbsh5nfwCiUTM20xp0t5JeXd+wPVWbpWqI2EnkCEN+RJr9i -7dp/ymDQ+Yt5wrsN3NwoyiexPOG91WQVCADdErHsnglVZZq9Z8Wx7KwecGCUurJ2 -H6lKudv5YOxPnAzqZS5HbpZd/nRTMZh2rdXCr5m2YOuewyYjvM757AkmUpM09zJX -MQ1S67/UX2y8/74TcRF97Ncx9HeELs92innBRXoFitnNguvcO6Esx4BTe1OdU6qR -ER3zAmVf22Le9ciXbu24DN4mleOH+OmBx7X2PqJSYW9GAMTsRB081R6EWKH7romQ -waxFrZ4DJzZ9ltyosEJn5F32StyLrFxpcrdLUoEaclZCv2qka7sZvi0EvovDVEBU -e10jOx9AOwf8Gj2ufhquQ6qgVYCzbP+YrodtkFrXRS3IsljIchj1M2ffB/0bfoUs -rtER9pLvYzCjBPg8IfGLw0o754Qbhh/ReplCRTusP/fQMybvCvfxreS3oyEriu/G -GufRomjewZ8EMHDIgUsLcYo2UHZsfF7tcazgxMGmMvazp4r8vpgrvW/8fIN/6Adu -tF+WjWDTvJLFJCe6O+BFJOWrssNrrra1zGtLC1s8s+Wfpe+bGPL5zpHeebGTwH1U -22eqgJArlEKxrfarz7W5+uHZJHSjF/K9ZvunLGD0n9GOPMpji3UO3zeM8IYoWn7E -/EWK1XbjnssNemeeTZ+sDh+qrD7BOi+vCX1IyBxbfqnQfJZvmcPWpruy1UsO+aIC -0GY8Jr3OL69dDQ21jueJAh8EGAEIAAkFAlC9+dkCGwwACgkQL0VeKCTRjd9HCw/+ -LQSVgLLF4ulYlPCjWIIuQwrPbJfWUVVr2dPUFVM85DCv8gBzk5c121snXh9Swovm -laBbw6ate3BmbXLh64jVE9Za5sbTWi7PCcbO/bpRy4d6oLmitmNw6cq0vjTLxUYy -bwuiJxWREkfxuU85EKdouN062YDevH+/YResmlJrcCE7LRlJFeRlKsrrwBU3BqYd -GgFJjKjQC1peeQ9fj62Y7xfwE9+PXbkiWO5u/Bk8hb1VZH1SoIRU98NHVcp6BVvp -VK0jLAXuSauSczULmpRjbyt1lhaAqivDTWEEZXiNNbRyp17c3nVdPWOcgBr42hdQ -z25CgZgyLCsvu82wuXLKJblrIPJX3Yf+si6KqEWBsmwdOWybsjygaF5HvzgFqAAD -U0goPWoQ71PorP2XOUNp5ZLkBQp5etvtkksjVNMIhnHn8PGMuoxO39EUGlWj2B5l -Cu8tSosAzB1pS8NcLZzoNoI9dOHrmgJmP+GrOUkcf5GhNZbMoj4GNfGBRYX0SZlQ -GuDrwNKYj73C4MWyNnnUFyq8nDHJ/G1NpaF2hiof9RBL4PUU/f92JkceXPBXA8gL -Mz2ig1OButwPPLFGQhWqxXAGrsS3Ny+BhTJfnfIbbkaLLphBpDZm1D9XKbAUvdd1 -RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc= -=JTFu ------END PGP PRIVATE KEY BLOCK----- -""" - -ADDRESS_2 = 'anotheruser@leap.se' - -PUBLIC_KEY_2 = """ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -mI0EUYwJXgEEAMbTKHuPJ5/Gk34l9Z06f+0WCXTDXdte1UBoDtZ1erAbudgC4MOR -gquKqoj3Hhw0/ILqJ88GcOJmKK/bEoIAuKaqlzDF7UAYpOsPZZYmtRfPC2pTCnXq -Z1vdeqLwTbUspqXflkCkFtfhGKMq5rH8GV5a3tXZkRWZhdNwhVXZagC3ABEBAAG0 -IWFub3RoZXJ1c2VyIDxhbm90aGVydXNlckBsZWFwLnNlPoi4BBMBAgAiBQJRjAle -AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRB/nfpof+5XWotuA/4tLN4E -gUr7IfLy2HkHAxzw7A4rqfMN92DIM9mZrDGaWRrOn3aVF7VU1UG7MDkHfPvp/cFw -ezoCw4s4IoHVc/pVlOkcHSyt4/Rfh248tYEJmFCJXGHpkK83VIKYJAithNccJ6Q4 -JE/o06Mtf4uh/cA1HUL4a4ceqUhtpLJULLeKo7iNBFGMCV4BBADsyQI7GR0wSAxz -VayLjuPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQt -Z/hwcLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63 -yuRe94WenT1RJd6xU1aaUff4rKizuQARAQABiJ8EGAECAAkFAlGMCV4CGwwACgkQ -f536aH/uV1rPZQQAqCzRysOlu8ez7PuiBD4SebgRqWlxa1TF1ujzfLmuPivROZ2X -Kw5aQstxgGSjoB7tac49s0huh4X8XK+BtJBfU84JS8Jc2satlfwoyZ35LH6sDZck -I+RS/3we6zpMfHs3vvp9xgca6ZupQxivGtxlJs294TpJorx+mFFqbV17AzQ= -=Thdu ------END PGP PUBLIC KEY BLOCK----- -""" - -PRIVATE_KEY_2 = """ ------BEGIN PGP PRIVATE KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -lQHYBFGMCV4BBADG0yh7jyefxpN+JfWdOn/tFgl0w13bXtVAaA7WdXqwG7nYAuDD -kYKriqqI9x4cNPyC6ifPBnDiZiiv2xKCALimqpcwxe1AGKTrD2WWJrUXzwtqUwp1 -6mdb3Xqi8E21LKal35ZApBbX4RijKuax/BleWt7V2ZEVmYXTcIVV2WoAtwARAQAB -AAP7BLuSAx7tOohnimEs74ks8l/L6dOcsFQZj2bqs4AoY3jFe7bV0tHr4llypb/8 -H3/DYvpf6DWnCjyUS1tTnXSW8JXtx01BUKaAufSmMNg9blKV6GGHlT/Whe9uVyks -7XHk/+9mebVMNJ/kNlqq2k+uWqJohzC8WWLRK+d1tBeqDsECANZmzltPaqUsGV5X -C3zszE3tUBgptV/mKnBtopKi+VH+t7K6fudGcG+bAcZDUoH/QVde52mIIjjIdLje -uajJuHUCAO1mqh+vPoGv4eBLV7iBo3XrunyGXiys4a39eomhxTy3YktQanjjx+ty -GltAGCs5PbWGO6/IRjjvd46wh53kzvsCAO0J97gsWhzLuFnkxFAJSPk7RRlyl7lI -1XS/x0Og6j9XHCyY1OYkfBm0to3UlCfkgirzCYlTYObCofzdKFIPDmSqHbQhYW5v -dGhlcnVzZXIgPGFub3RoZXJ1c2VyQGxlYXAuc2U+iLgEEwECACIFAlGMCV4CGwMG -CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEH+d+mh/7ldai24D/i0s3gSBSvsh -8vLYeQcDHPDsDiup8w33YMgz2ZmsMZpZGs6fdpUXtVTVQbswOQd8++n9wXB7OgLD -izgigdVz+lWU6RwdLK3j9F+Hbjy1gQmYUIlcYemQrzdUgpgkCK2E1xwnpDgkT+jT -oy1/i6H9wDUdQvhrhx6pSG2kslQst4qjnQHYBFGMCV4BBADsyQI7GR0wSAxzVayL -juPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQtZ/hw -cLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63yuRe -94WenT1RJd6xU1aaUff4rKizuQARAQABAAP9EyElqJ3dq3EErXwwT4mMnbd1SrVC -rUJrNWQZL59mm5oigS00uIyR0SvusOr+UzTtd8ysRuwHy5d/LAZsbjQStaOMBILx -77TJveOel0a1QK0YSMF2ywZMCKvquvjli4hAtWYz/EwfuzQN3t23jc5ny+GqmqD2 -3FUxLJosFUfLNmECAO9KhVmJi+L9dswIs+2Dkjd1eiRQzNOEVffvYkGYZyKxNiXF -UA5kvyZcB4iAN9sWCybE4WHZ9jd4myGB0MPDGxkCAP1RsXJbbuD6zS7BXe5gwunO -2q4q7ptdSl/sJYQuTe1KNP5d/uGsvlcFfsYjpsopasPjFBIncc/2QThMKlhoEaEB -/0mVAxpT6SrEvUbJ18z7kna24SgMPr3OnPMxPGfvNLJY/Xv/A17YfoqjmByCvsKE -JCDjopXtmbcrZyoEZbEht9mko4ifBBgBAgAJBQJRjAleAhsMAAoJEH+d+mh/7lda -z2UEAKgs0crDpbvHs+z7ogQ+Enm4EalpcWtUxdbo83y5rj4r0TmdlysOWkLLcYBk -o6Ae7WnOPbNIboeF/FyvgbSQX1POCUvCXNrGrZX8KMmd+Sx+rA2XJCPkUv98Hus6 -THx7N776fcYHGumbqUMYrxrcZSbNveE6SaK8fphRam1dewM0 -=a5gs ------END PGP PRIVATE KEY BLOCK----- -""" diff --git a/src/leap/mail/smtp/tests/test_gateway.py b/src/leap/mail/smtp/tests/test_gateway.py index 466677f..3635a9f 100644 --- a/src/leap/mail/smtp/tests/test_gateway.py +++ b/src/leap/mail/smtp/tests/test_gateway.py @@ -32,7 +32,7 @@ from leap.mail.smtp.gateway import ( SMTPFactory, EncryptedMessage, ) -from leap.mail.smtp.tests import ( +from leap.mail.tests import ( TestCaseWithKeyManager, ADDRESS, ADDRESS_2, diff --git a/src/leap/mail/tests/__init__.py b/src/leap/mail/tests/__init__.py new file mode 100644 index 0000000..dc24293 --- /dev/null +++ b/src/leap/mail/tests/__init__.py @@ -0,0 +1,386 @@ +# -*- coding: utf-8 -*- +# __init__.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Base classes and keys for SMTP gateway tests. +""" + +import os +import distutils.spawn +import shutil +import tempfile +from mock import Mock + + +from twisted.trial import unittest + + +from leap.soledad.client import Soledad +from leap.keymanager import ( + KeyManager, + openpgp, +) + + +from leap.common.testing.basetest import BaseLeapTest + + +def _find_gpg(): + gpg_path = distutils.spawn.find_executable('gpg') + return os.path.realpath(gpg_path) if gpg_path is not None else "/usr/bin/gpg" + + +class TestCaseWithKeyManager(BaseLeapTest): + + GPG_BINARY_PATH = _find_gpg() + + def setUp(self): + # mimic BaseLeapTest.setUpClass behaviour, because this is deprecated + # in Twisted: http://twistedmatrix.com/trac/ticket/1870 + self.old_path = os.environ['PATH'] + self.old_home = os.environ['HOME'] + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.home = self.tempdir + bin_tdir = os.path.join( + self.tempdir, + 'bin') + os.environ["PATH"] = bin_tdir + os.environ["HOME"] = self.tempdir + + # setup our own stuff + address = 'leap@leap.se' # user's address in the form user@provider + uuid = 'leap@leap.se' + passphrase = u'123' + secrets_path = os.path.join(self.tempdir, 'secret.gpg') + local_db_path = os.path.join(self.tempdir, 'soledad.u1db') + server_url = 'http://provider/' + cert_file = '' + + self._soledad = self._soledad_instance( + uuid, passphrase, secrets_path, local_db_path, server_url, + cert_file) + self._km = self._keymanager_instance(address) + + def _soledad_instance(self, uuid, passphrase, secrets_path, local_db_path, + server_url, cert_file): + """ + Return a Soledad instance for tests. + """ + # mock key fetching and storing so Soledad doesn't fail when trying to + # reach the server. + Soledad._fetch_keys_from_shared_db = Mock(return_value=None) + Soledad._assert_keys_in_shared_db = Mock(return_value=None) + + # instantiate soledad + def _put_doc_side_effect(doc): + self._doc_put = doc + + class MockSharedDB(object): + + get_doc = Mock(return_value=None) + put_doc = Mock(side_effect=_put_doc_side_effect) + lock = Mock(return_value=('atoken', 300)) + unlock = Mock(return_value=True) + + def __call__(self): + return self + + Soledad._shared_db = MockSharedDB() + + return Soledad( + uuid, + passphrase, + secrets_path=secrets_path, + local_db_path=local_db_path, + server_url=server_url, + cert_file=cert_file, + ) + + def _keymanager_instance(self, address): + """ + Return a Key Manager instance for tests. + """ + self._config = { + 'host': 'http://provider/', + 'port': 25, + 'username': address, + 'password': '', + 'encrypted_only': True, + 'cert': u'src/leap/mail/smtp/tests/cert/server.crt', + 'key': u'src/leap/mail/smtp/tests/cert/server.key', + } + + class Response(object): + status_code = 200 + headers = {'content-type': 'application/json'} + + def json(self): + return {'address': ADDRESS_2, 'openpgp': PUBLIC_KEY_2} + + def raise_for_status(self): + pass + + nickserver_url = '' # the url of the nickserver + km = KeyManager(address, nickserver_url, self._soledad, + ca_cert_path='', gpgbinary=self.GPG_BINARY_PATH) + km._fetcher.put = Mock() + km._fetcher.get = Mock(return_value=Response()) + + # insert test keys in key manager. + pgp = openpgp.OpenPGPScheme( + self._soledad, gpgbinary=self.GPG_BINARY_PATH) + pgp.put_ascii_key(PRIVATE_KEY) + pgp.put_ascii_key(PRIVATE_KEY_2) + + return km + + def tearDown(self): + # mimic LeapBaseTest.tearDownClass behaviour + os.environ["PATH"] = self.old_path + os.environ["HOME"] = self.old_home + # safety check + assert 'leap_tests-' in self.tempdir + shutil.rmtree(self.tempdir) + + +# Key material for testing +KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF" + +ADDRESS = 'leap@leap.se' + +PUBLIC_KEY = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +mQINBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz +iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO +zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx +irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT +huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs +d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g +wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb +hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv +U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H +T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i +Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB +tBxMZWFwIFRlc3QgS2V5IDxsZWFwQGxlYXAuc2U+iQI3BBMBCAAhBQJQvfnZAhsD +BQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEC9FXigk0Y3fT7EQAKH3IuRniOpb +T/DDIgwwjz3oxB/W0DDMyPXowlhSOuM0rgGfntBpBb3boezEXwL86NPQxNGGruF5 +hkmecSiuPSvOmQlqlS95NGQp6hNG0YaKColh+Q5NTspFXCAkFch9oqUje0LdxfSP +QfV9UpeEvGyPmk1I9EJV/YDmZ4+Djge1d7qhVZInz4Rx1NrSyF/Tc2EC0VpjQFsU +Y9Kb2YBBR7ivG6DBc8ty0jJXi7B4WjkFcUEJviQpMF2dCLdonCehYs1PqsN1N7j+ +eFjQd+hqVMJgYuSGKjvuAEfClM6MQw7+FmFwMyLgK/Ew/DttHEDCri77SPSkOGSI +txCzhTg6798f6mJr7WcXmHX1w1Vcib5FfZ8vTDFVhz/XgAgArdhPo9V6/1dgSSiB +KPQ/spsco6u5imdOhckERE0lnAYvVT6KE81TKuhF/b23u7x+Wdew6kK0EQhYA7wy +7LmlaNXc7rMBQJ9Z60CJ4JDtatBWZ0kNrt2VfdDHVdqBTOpl0CraNUjWE5YMDasr +K2dF5IX8D3uuYtpZnxqg0KzyLg0tzL0tvOL1C2iudgZUISZNPKbS0z0v+afuAAnx +2pTC3uezbh2Jt8SWTLhll4i0P4Ps5kZ6HQUO56O+/Z1cWovX+mQekYFmERySDR9n +3k1uAwLilJmRmepGmvYbB8HloV8HqwgguQINBFC9+dkBEAC0I/xn1uborMgDvBtf +H0sEhwnXBC849/32zic6udB6/3Efk9nzbSpL3FSOuXITZsZgCHPkKarnoQ2ztMcS +sh1ke1C5gQGms75UVmM/nS+2YI4vY8OX/GC/on2vUyncqdH+bR6xH5hx4NbWpfTs +iQHmz5C6zzS/kuabGdZyKRaZHt23WQ7JX/4zpjqbC99DjHcP9BSk7tJ8wI4bkMYD +uFVQdT9O6HwyKGYwUU4sAQRAj7XCTGvVbT0dpgJwH4RmrEtJoHAx4Whg8mJ710E0 +GCmzf2jqkNuOw76ivgk27Kge+Hw00jmJjQhHY0yVbiaoJwcRrPKzaSjEVNgrpgP3 +lXPRGQArgESsIOTeVVHQ8fhK2YtTeCY9rIiO+L0OX2xo9HK7hfHZZWL6rqymXdyS +fhzh/f6IPyHFWnvj7Brl7DR8heMikygcJqv+ed2yx7iLyCUJ10g12I48+aEj1aLe +dP7lna32iY8/Z0SHQLNH6PXO9SlPcq2aFUgKqE75A/0FMk7CunzU1OWr2ZtTLNO1 +WT/13LfOhhuEq9jTyTosn0WxBjJKq18lnhzCXlaw6EAtbA7CUwsD3CTPR56aAXFK +3I7KXOVAqggrvMe5Tpdg5drfYpI8hZovL5aAgb+7Y5ta10TcJdUhS5K3kFAWe/td +U0cmWUMDP1UMSQ5Jg6JIQVWhSwARAQABiQIfBBgBCAAJBQJQvfnZAhsMAAoJEC9F +Xigk0Y3fRwsP/i0ElYCyxeLpWJTwo1iCLkMKz2yX1lFVa9nT1BVTPOQwr/IAc5OX +NdtbJ14fUsKL5pWgW8OmrXtwZm1y4euI1RPWWubG01ouzwnGzv26UcuHeqC5orZj +cOnKtL40y8VGMm8LoicVkRJH8blPORCnaLjdOtmA3rx/v2EXrJpSa3AhOy0ZSRXk +ZSrK68AVNwamHRoBSYyo0AtaXnkPX4+tmO8X8BPfj125IljubvwZPIW9VWR9UqCE +VPfDR1XKegVb6VStIywF7kmrknM1C5qUY28rdZYWgKorw01hBGV4jTW0cqde3N51 +XT1jnIAa+NoXUM9uQoGYMiwrL7vNsLlyyiW5ayDyV92H/rIuiqhFgbJsHTlsm7I8 +oGheR784BagAA1NIKD1qEO9T6Kz9lzlDaeWS5AUKeXrb7ZJLI1TTCIZx5/DxjLqM +Tt/RFBpVo9geZQrvLUqLAMwdaUvDXC2c6DaCPXTh65oCZj/hqzlJHH+RoTWWzKI+ +BjXxgUWF9EmZUBrg68DSmI+9wuDFsjZ51BcqvJwxyfxtTaWhdoYqH/UQS+D1FP3/ +diZHHlzwVwPICzM9ooNTgbrcDzyxRkIVqsVwBq7EtzcvgYUyX53yG25Giy6YQaQ2 +ZtQ/VymwFL3XdUWV6B/hU4PVAFvO3qlOtdJ6TpE+nEWgcWjCv5g7RjXX +=MuOY +-----END PGP PUBLIC KEY BLOCK----- +""" + +PRIVATE_KEY = """ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQcYBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz +iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO +zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx +irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT +huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs +d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g +wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb +hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv +U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H +T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i +Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB +AA/+JHtlL39G1wsH9R6UEfUQJGXR9MiIiwZoKcnRB2o8+DS+OLjg0JOh8XehtuCs +E/8oGQKtQqa5bEIstX7IZoYmYFiUQi9LOzIblmp2vxOm+HKkxa4JszWci2/ZmC3t +KtaA4adl9XVnshoQ7pijuCMUKB3naBEOAxd8s9d/JeReGIYkJErdrnVfNk5N71Ds +FmH5Ll3XtEDvgBUQP3nkA6QFjpsaB94FHjL3gDwum/cxzj6pCglcvHOzEhfY0Ddb +J967FozQTaf2JW3O+w3LOqtcKWpq87B7+O61tVidQPSSuzPjCtFF0D2LC9R/Hpky +KTMQ6CaKja4MPhjwywd4QPcHGYSqjMpflvJqi+kYIt8psUK/YswWjnr3r4fbuqVY +VhtiHvnBHQjz135lUqWvEz4hM3Xpnxydx7aRlv5NlevK8+YIO5oFbWbGNTWsPZI5 +jpoFBpSsnR1Q5tnvtNHauvoWV+XN2qAOBTG+/nEbDYH6Ak3aaE9jrpTdYh0CotYF +q7csANsDy3JvkAzeU6WnYpsHHaAjqOGyiZGsLej1UcXPFMosE/aUo4WQhiS8Zx2c +zOVKOi/X5vQ2GdNT9Qolz8AriwzsvFR+bxPzyd8V6ALwDsoXvwEYinYBKK8j0OPv +OOihSR6HVsuP9NUZNU9ewiGzte/+/r6pNXHvR7wTQ8EWLcEIAN6Zyrb0bHZTIlxt +VWur/Ht2mIZrBaO50qmM5RD3T5oXzWXi/pjLrIpBMfeZR9DWfwQwjYzwqi7pxtYx +nJvbMuY505rfnMoYxb4J+cpRXV8MS7Dr1vjjLVUC9KiwSbM3gg6emfd2yuA93ihv +Pe3mffzLIiQa4mRE3wtGcioC43nWuV2K2e1KjxeFg07JhrezA/1Cak505ab/tmvP +4YmjR5c44+yL/YcQ3HdFgs4mV+nVbptRXvRcPpolJsgxPccGNdvHhsoR4gwXMS3F +RRPD2z6x8xeN73Q4KH3bm01swQdwFBZbWVfmUGLxvN7leCdfs9+iFJyqHiCIB6Iv +mQfp8F0IAOwSo8JhWN+V1dwML4EkIrM8wUb4yecNLkyR6TpPH/qXx4PxVMC+vy6x +sCtjeHIwKE+9vqnlhd5zOYh7qYXEJtYwdeDDmDbL8oks1LFfd+FyAuZXY33DLwn0 +cRYsr2OEZmaajqUB3NVmj3H4uJBN9+paFHyFSXrH68K1Fk2o3n+RSf2EiX+eICwI +L6rqoF5sSVUghBWdNegV7qfy4anwTQwrIMGjgU5S6PKW0Dr/3iO5z3qQpGPAj5OW +ATqPWkDICLbObPxD5cJlyyNE2wCA9VVc6/1d6w4EVwSq9h3/WTpATEreXXxTGptd +LNiTA1nmakBYNO2Iyo3djhaqBdWjk+EIAKtVEnJH9FAVwWOvaj1RoZMA5DnDMo7e +SnhrCXl8AL7Z1WInEaybasTJXn1uQ8xY52Ua4b8cbuEKRKzw/70NesFRoMLYoHTO +dyeszvhoDHberpGRTciVmpMu7Hyi33rM31K9epA4ib6QbbCHnxkWOZB+Bhgj1hJ8 +xb4RBYWiWpAYcg0+DAC3w9gfxQhtUlZPIbmbrBmrVkO2GVGUj8kH6k4UV6kUHEGY +HQWQR0HcbKcXW81ZXCCD0l7ROuEWQtTe5Jw7dJ4/QFuqZnPutXVRNOZqpl6eRShw +7X2/a29VXBpmHA95a88rSQsL+qm7Fb3prqRmuMCtrUZgFz7HLSTuUMR867QcTGVh +cCBUZXN0IEtleSA8bGVhcEBsZWFwLnNlPokCNwQTAQgAIQUCUL352QIbAwULCQgH +AwUVCgkICwUWAgMBAAIeAQIXgAAKCRAvRV4oJNGN30+xEACh9yLkZ4jqW0/wwyIM +MI896MQf1tAwzMj16MJYUjrjNK4Bn57QaQW926HsxF8C/OjT0MTRhq7heYZJnnEo +rj0rzpkJapUveTRkKeoTRtGGigqJYfkOTU7KRVwgJBXIfaKlI3tC3cX0j0H1fVKX +hLxsj5pNSPRCVf2A5mePg44HtXe6oVWSJ8+EcdTa0shf03NhAtFaY0BbFGPSm9mA +QUe4rxugwXPLctIyV4uweFo5BXFBCb4kKTBdnQi3aJwnoWLNT6rDdTe4/nhY0Hfo +alTCYGLkhio77gBHwpTOjEMO/hZhcDMi4CvxMPw7bRxAwq4u+0j0pDhkiLcQs4U4 +Ou/fH+pia+1nF5h19cNVXIm+RX2fL0wxVYc/14AIAK3YT6PVev9XYEkogSj0P7Kb +HKOruYpnToXJBERNJZwGL1U+ihPNUyroRf29t7u8flnXsOpCtBEIWAO8Muy5pWjV +3O6zAUCfWetAieCQ7WrQVmdJDa7dlX3Qx1XagUzqZdAq2jVI1hOWDA2rKytnReSF +/A97rmLaWZ8aoNCs8i4NLcy9Lbzi9QtornYGVCEmTTym0tM9L/mn7gAJ8dqUwt7n +s24dibfElky4ZZeItD+D7OZGeh0FDuejvv2dXFqL1/pkHpGBZhEckg0fZ95NbgMC +4pSZkZnqRpr2GwfB5aFfB6sIIJ0HGARQvfnZARAAtCP8Z9bm6KzIA7wbXx9LBIcJ +1wQvOPf99s4nOrnQev9xH5PZ820qS9xUjrlyE2bGYAhz5Cmq56ENs7THErIdZHtQ +uYEBprO+VFZjP50vtmCOL2PDl/xgv6J9r1Mp3KnR/m0esR+YceDW1qX07IkB5s+Q +us80v5LmmxnWcikWmR7dt1kOyV/+M6Y6mwvfQ4x3D/QUpO7SfMCOG5DGA7hVUHU/ +Tuh8MihmMFFOLAEEQI+1wkxr1W09HaYCcB+EZqxLSaBwMeFoYPJie9dBNBgps39o +6pDbjsO+or4JNuyoHvh8NNI5iY0IR2NMlW4mqCcHEazys2koxFTYK6YD95Vz0RkA +K4BErCDk3lVR0PH4StmLU3gmPayIjvi9Dl9saPRyu4Xx2WVi+q6spl3ckn4c4f3+ +iD8hxVp74+wa5ew0fIXjIpMoHCar/nndsse4i8glCddINdiOPPmhI9Wi3nT+5Z2t +9omPP2dEh0CzR+j1zvUpT3KtmhVICqhO+QP9BTJOwrp81NTlq9mbUyzTtVk/9dy3 +zoYbhKvY08k6LJ9FsQYySqtfJZ4cwl5WsOhALWwOwlMLA9wkz0eemgFxStyOylzl +QKoIK7zHuU6XYOXa32KSPIWaLy+WgIG/u2ObWtdE3CXVIUuSt5BQFnv7XVNHJllD +Az9VDEkOSYOiSEFVoUsAEQEAAQAP/1AagnZQZyzHDEgw4QELAspYHCWLXE5aZInX +wTUJhK31IgIXNn9bJ0hFiSpQR2xeMs9oYtRuPOu0P8oOFMn4/z374fkjZy8QVY3e +PlL+3EUeqYtkMwlGNmVw5a/NbNuNfm5Darb7pEfbYd1gPcni4MAYw7R2SG/57GbC +9gucvspHIfOSfBNLBthDzmK8xEKe1yD2eimfc2T7IRYb6hmkYfeds5GsqvGI6mwI +85h4uUHWRc5JOlhVM6yX8hSWx0L60Z3DZLChmc8maWnFXd7C8eQ6P1azJJbW71Ih +7CoK0XW4LE82vlQurSRFgTwfl7wFYszW2bOzCuhHDDtYnwH86Nsu0DC78ZVRnvxn +E8Ke/AJgrdhIOo4UAyR+aZD2+2mKd7/waOUTUrUtTzc7i8N3YXGi/EIaNReBXaq+ +ZNOp24BlFzRp+FCF/pptDW9HjPdiV09x0DgICmeZS4Gq/4vFFIahWctg52NGebT0 +Idxngjj+xDtLaZlLQoOz0n5ByjO/Wi0ANmMv1sMKCHhGvdaSws2/PbMR2r4caj8m +KXpIgdinM/wUzHJ5pZyF2U/qejsRj8Kw8KH/tfX4JCLhiaP/mgeTuWGDHeZQERAT +xPmRFHaLP9/ZhvGNh6okIYtrKjWTLGoXvKLHcrKNisBLSq+P2WeFrlme1vjvJMo/ +jPwLT5o9CADQmcbKZ+QQ1ZM9v99iDZol7SAMZX43JC019sx6GK0u6xouJBcLfeB4 +OXacTgmSYdTa9RM9fbfVpti01tJ84LV2SyL/VJq/enJF4XQPSynT/tFTn1PAor6o +tEAAd8fjKdJ6LnD5wb92SPHfQfXqI84rFEO8rUNIE/1ErT6DYifDzVCbfD2KZdoF +cOSp7TpD77sY1bs74ocBX5ejKtd+aH99D78bJSMM4pSDZsIEwnomkBHTziubPwJb +OwnATy0LmSMAWOw5rKbsh5nfwCiUTM20xp0t5JeXd+wPVWbpWqI2EnkCEN+RJr9i +7dp/ymDQ+Yt5wrsN3NwoyiexPOG91WQVCADdErHsnglVZZq9Z8Wx7KwecGCUurJ2 +H6lKudv5YOxPnAzqZS5HbpZd/nRTMZh2rdXCr5m2YOuewyYjvM757AkmUpM09zJX +MQ1S67/UX2y8/74TcRF97Ncx9HeELs92innBRXoFitnNguvcO6Esx4BTe1OdU6qR +ER3zAmVf22Le9ciXbu24DN4mleOH+OmBx7X2PqJSYW9GAMTsRB081R6EWKH7romQ +waxFrZ4DJzZ9ltyosEJn5F32StyLrFxpcrdLUoEaclZCv2qka7sZvi0EvovDVEBU +e10jOx9AOwf8Gj2ufhquQ6qgVYCzbP+YrodtkFrXRS3IsljIchj1M2ffB/0bfoUs +rtER9pLvYzCjBPg8IfGLw0o754Qbhh/ReplCRTusP/fQMybvCvfxreS3oyEriu/G +GufRomjewZ8EMHDIgUsLcYo2UHZsfF7tcazgxMGmMvazp4r8vpgrvW/8fIN/6Adu +tF+WjWDTvJLFJCe6O+BFJOWrssNrrra1zGtLC1s8s+Wfpe+bGPL5zpHeebGTwH1U +22eqgJArlEKxrfarz7W5+uHZJHSjF/K9ZvunLGD0n9GOPMpji3UO3zeM8IYoWn7E +/EWK1XbjnssNemeeTZ+sDh+qrD7BOi+vCX1IyBxbfqnQfJZvmcPWpruy1UsO+aIC +0GY8Jr3OL69dDQ21jueJAh8EGAEIAAkFAlC9+dkCGwwACgkQL0VeKCTRjd9HCw/+ +LQSVgLLF4ulYlPCjWIIuQwrPbJfWUVVr2dPUFVM85DCv8gBzk5c121snXh9Swovm +laBbw6ate3BmbXLh64jVE9Za5sbTWi7PCcbO/bpRy4d6oLmitmNw6cq0vjTLxUYy +bwuiJxWREkfxuU85EKdouN062YDevH+/YResmlJrcCE7LRlJFeRlKsrrwBU3BqYd +GgFJjKjQC1peeQ9fj62Y7xfwE9+PXbkiWO5u/Bk8hb1VZH1SoIRU98NHVcp6BVvp +VK0jLAXuSauSczULmpRjbyt1lhaAqivDTWEEZXiNNbRyp17c3nVdPWOcgBr42hdQ +z25CgZgyLCsvu82wuXLKJblrIPJX3Yf+si6KqEWBsmwdOWybsjygaF5HvzgFqAAD +U0goPWoQ71PorP2XOUNp5ZLkBQp5etvtkksjVNMIhnHn8PGMuoxO39EUGlWj2B5l +Cu8tSosAzB1pS8NcLZzoNoI9dOHrmgJmP+GrOUkcf5GhNZbMoj4GNfGBRYX0SZlQ +GuDrwNKYj73C4MWyNnnUFyq8nDHJ/G1NpaF2hiof9RBL4PUU/f92JkceXPBXA8gL +Mz2ig1OButwPPLFGQhWqxXAGrsS3Ny+BhTJfnfIbbkaLLphBpDZm1D9XKbAUvdd1 +RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc= +=JTFu +-----END PGP PRIVATE KEY BLOCK----- +""" + +ADDRESS_2 = 'anotheruser@leap.se' + +PUBLIC_KEY_2 = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +mI0EUYwJXgEEAMbTKHuPJ5/Gk34l9Z06f+0WCXTDXdte1UBoDtZ1erAbudgC4MOR +gquKqoj3Hhw0/ILqJ88GcOJmKK/bEoIAuKaqlzDF7UAYpOsPZZYmtRfPC2pTCnXq +Z1vdeqLwTbUspqXflkCkFtfhGKMq5rH8GV5a3tXZkRWZhdNwhVXZagC3ABEBAAG0 +IWFub3RoZXJ1c2VyIDxhbm90aGVydXNlckBsZWFwLnNlPoi4BBMBAgAiBQJRjAle +AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRB/nfpof+5XWotuA/4tLN4E +gUr7IfLy2HkHAxzw7A4rqfMN92DIM9mZrDGaWRrOn3aVF7VU1UG7MDkHfPvp/cFw +ezoCw4s4IoHVc/pVlOkcHSyt4/Rfh248tYEJmFCJXGHpkK83VIKYJAithNccJ6Q4 +JE/o06Mtf4uh/cA1HUL4a4ceqUhtpLJULLeKo7iNBFGMCV4BBADsyQI7GR0wSAxz +VayLjuPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQt +Z/hwcLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63 +yuRe94WenT1RJd6xU1aaUff4rKizuQARAQABiJ8EGAECAAkFAlGMCV4CGwwACgkQ +f536aH/uV1rPZQQAqCzRysOlu8ez7PuiBD4SebgRqWlxa1TF1ujzfLmuPivROZ2X +Kw5aQstxgGSjoB7tac49s0huh4X8XK+BtJBfU84JS8Jc2satlfwoyZ35LH6sDZck +I+RS/3we6zpMfHs3vvp9xgca6ZupQxivGtxlJs294TpJorx+mFFqbV17AzQ= +=Thdu +-----END PGP PUBLIC KEY BLOCK----- +""" + +PRIVATE_KEY_2 = """ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQHYBFGMCV4BBADG0yh7jyefxpN+JfWdOn/tFgl0w13bXtVAaA7WdXqwG7nYAuDD +kYKriqqI9x4cNPyC6ifPBnDiZiiv2xKCALimqpcwxe1AGKTrD2WWJrUXzwtqUwp1 +6mdb3Xqi8E21LKal35ZApBbX4RijKuax/BleWt7V2ZEVmYXTcIVV2WoAtwARAQAB +AAP7BLuSAx7tOohnimEs74ks8l/L6dOcsFQZj2bqs4AoY3jFe7bV0tHr4llypb/8 +H3/DYvpf6DWnCjyUS1tTnXSW8JXtx01BUKaAufSmMNg9blKV6GGHlT/Whe9uVyks +7XHk/+9mebVMNJ/kNlqq2k+uWqJohzC8WWLRK+d1tBeqDsECANZmzltPaqUsGV5X +C3zszE3tUBgptV/mKnBtopKi+VH+t7K6fudGcG+bAcZDUoH/QVde52mIIjjIdLje +uajJuHUCAO1mqh+vPoGv4eBLV7iBo3XrunyGXiys4a39eomhxTy3YktQanjjx+ty +GltAGCs5PbWGO6/IRjjvd46wh53kzvsCAO0J97gsWhzLuFnkxFAJSPk7RRlyl7lI +1XS/x0Og6j9XHCyY1OYkfBm0to3UlCfkgirzCYlTYObCofzdKFIPDmSqHbQhYW5v +dGhlcnVzZXIgPGFub3RoZXJ1c2VyQGxlYXAuc2U+iLgEEwECACIFAlGMCV4CGwMG +CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEH+d+mh/7ldai24D/i0s3gSBSvsh +8vLYeQcDHPDsDiup8w33YMgz2ZmsMZpZGs6fdpUXtVTVQbswOQd8++n9wXB7OgLD +izgigdVz+lWU6RwdLK3j9F+Hbjy1gQmYUIlcYemQrzdUgpgkCK2E1xwnpDgkT+jT +oy1/i6H9wDUdQvhrhx6pSG2kslQst4qjnQHYBFGMCV4BBADsyQI7GR0wSAxzVayL +juPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQtZ/hw +cLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63yuRe +94WenT1RJd6xU1aaUff4rKizuQARAQABAAP9EyElqJ3dq3EErXwwT4mMnbd1SrVC +rUJrNWQZL59mm5oigS00uIyR0SvusOr+UzTtd8ysRuwHy5d/LAZsbjQStaOMBILx +77TJveOel0a1QK0YSMF2ywZMCKvquvjli4hAtWYz/EwfuzQN3t23jc5ny+GqmqD2 +3FUxLJosFUfLNmECAO9KhVmJi+L9dswIs+2Dkjd1eiRQzNOEVffvYkGYZyKxNiXF +UA5kvyZcB4iAN9sWCybE4WHZ9jd4myGB0MPDGxkCAP1RsXJbbuD6zS7BXe5gwunO +2q4q7ptdSl/sJYQuTe1KNP5d/uGsvlcFfsYjpsopasPjFBIncc/2QThMKlhoEaEB +/0mVAxpT6SrEvUbJ18z7kna24SgMPr3OnPMxPGfvNLJY/Xv/A17YfoqjmByCvsKE +JCDjopXtmbcrZyoEZbEht9mko4ifBBgBAgAJBQJRjAleAhsMAAoJEH+d+mh/7lda +z2UEAKgs0crDpbvHs+z7ogQ+Enm4EalpcWtUxdbo83y5rj4r0TmdlysOWkLLcYBk +o6Ae7WnOPbNIboeF/FyvgbSQX1POCUvCXNrGrZX8KMmd+Sx+rA2XJCPkUv98Hus6 +THx7N776fcYHGumbqUMYrxrcZSbNveE6SaK8fphRam1dewM0 +=a5gs +-----END PGP PRIVATE KEY BLOCK----- +""" -- cgit v1.2.3 From 6102c35586bb925de2624d3164e6b9e7d8838a0e Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Mon, 3 Nov 2014 22:54:15 -0600 Subject: Discover public key via attachment --- changes/feature-5937_key_attachment | 1 + src/leap/mail/imap/fetch.py | 45 +++++++++++++++++++------- src/leap/mail/imap/tests/test_incoming_mail.py | 31 ++++++++++++++++-- 3 files changed, 63 insertions(+), 14 deletions(-) create mode 100644 changes/feature-5937_key_attachment diff --git a/changes/feature-5937_key_attachment b/changes/feature-5937_key_attachment new file mode 100644 index 0000000..08c37e0 --- /dev/null +++ b/changes/feature-5937_key_attachment @@ -0,0 +1 @@ +- Discover public keys via attachment (Closes: #5937) diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index 863f5fe..01373be 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -581,8 +581,8 @@ class LeapIncomingMail(object): def _extract_keys(self, msgtuple): """ - Parse message headers for an *OpenPGP* header as described on the - `IETF draft + Retrieve attached keys to the mesage and parse message headers for an + *OpenPGP* header as described on the `IETF draft ` only urls with https and the same hostname than the email are supported for security reasons. @@ -600,31 +600,35 @@ class LeapIncomingMail(object): # we should do it in this module so we don't need to parse it again # here msg = self._parser.parsestr(data) + _, fromAddress = parseaddr(msg['from']) + header = msg.get(OpenPGP_HEADER, None) if header is not None: - self._extract_openpgp_header(msg, header) + self._extract_openpgp_header(header, fromAddress) + + if msg.is_multipart(): + self._extract_attached_key(msg.get_payload(), fromAddress) return msgtuple - def _extract_openpgp_header(self, msg, header): + def _extract_openpgp_header(self, header, address): """ Import keys from the OpenPGP header - :param msg: parsed email - :type msg: email.Message :param header: OpenPGP header string :type header: str + :param address: email address in the from header + :type address: str """ fields = dict([f.strip(' ').split('=') for f in header.split(';')]) if 'url' in fields: url = shlex.split(fields['url'])[0] # remove quotations - _, fromAddress = parseaddr(msg['from']) urlparts = urlparse(url) - fromHostname = fromAddress.split('@')[1] + addressHostname = address.split('@')[1] if (urlparts.scheme == 'https' - and urlparts.hostname == fromHostname): + and urlparts.hostname == addressHostname): try: - self._keymanager.fetch_key(fromAddress, url, OpenPGPKey) + self._keymanager.fetch_key(address, url, OpenPGPKey) logger.info("Imported key from header %s" % (url,)) except keymanager_errors.KeyNotFound: logger.warning("Url from OpenPGP header %s failed" @@ -632,13 +636,32 @@ class LeapIncomingMail(object): except keymanager_errors.KeyAttributesDiffer: logger.warning("Key from OpenPGP header url %s didn't " "match the from address %s" - % (url, fromAddress)) + % (url, address)) else: logger.debug("No valid url on OpenPGP header %s" % (url,)) else: logger.debug("There is no url on the OpenPGP header: %s" % (header,)) + def _extract_attached_key(self, attachments, address): + """ + Import keys from the attachments + + :param attachments: email attachment list + :type attachments: list(email.Message) + :param address: email address in the from header + :type address: str + """ + MIME_KEY = "application/pgp-keys" + + for attachment in attachments: + if MIME_KEY == attachment.get_content_type(): + logger.debug("Add key from attachment") + self._keymanager.put_raw_key( + attachment.get_payload(), + OpenPGPKey, + address=address) + def _add_message_locally(self, msgtuple): """ Adds a message to local inbox and delete it from the incoming db diff --git a/src/leap/mail/imap/tests/test_incoming_mail.py b/src/leap/mail/imap/tests/test_incoming_mail.py index 5b72fac..ce6d56a 100644 --- a/src/leap/mail/imap/tests/test_incoming_mail.py +++ b/src/leap/mail/imap/tests/test_incoming_mail.py @@ -24,6 +24,8 @@ Test case for leap.email.imap.fetch import json +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart from email.parser import Parser from mock import Mock from twisted.trial import unittest @@ -105,13 +107,13 @@ subject: independence of cyberspace email = self._create_incoming_email(message.as_string()) self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]) self.fetcher._keymanager.fetch_key = Mock() - d = self.fetcher.fetch() def fetch_key_called(ret): self.fetcher._keymanager.fetch_key.assert_called_once_with( self.FROM_ADDRESS, KEYURL, OpenPGPKey) - d.addCallback(fetch_key_called) + d = self.fetcher.fetch() + d.addCallback(fetch_key_called) return d def testExtractOpenPGPHeaderInvalidUrl(self): @@ -126,12 +128,35 @@ subject: independence of cyberspace email = self._create_incoming_email(message.as_string()) self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]) self.fetcher._keymanager.fetch_key = Mock() - d = self.fetcher.fetch() def fetch_key_called(ret): self.assertFalse(self.fetcher._keymanager.fetch_key.called) + + d = self.fetcher.fetch() d.addCallback(fetch_key_called) + return d + def testExtractAttachedKey(self): + """ + Test the OpenPGP header key extraction + """ + KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." + + message = MIMEMultipart() + message.add_header("from", self.FROM_ADDRESS) + key = MIMEApplication("", "pgp-keys") + key.set_payload(KEY) + message.attach(key) + email = self._create_incoming_email(message.as_string()) + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]) + self.fetcher._keymanager.put_raw_key = Mock() + + def put_raw_key_called(ret): + self.fetcher._keymanager.put_raw_key.assert_called_once_with( + KEY, OpenPGPKey, address=self.FROM_ADDRESS) + + d = self.fetcher.fetch() + d.addCallback(put_raw_key_called) return d def _create_incoming_email(self, email_str): -- cgit v1.2.3 From 040faf2bf94db227246a00b38da8f92bfa0c8fa8 Mon Sep 17 00:00:00 2001 From: Duda Dornelles Date: Wed, 12 Nov 2014 18:27:46 -0200 Subject: Moving encrypt, sign and send logic from gateway (SMTP) to a MailService --- ..._encryptio_and_sending_out_of_encrypted_message | 1 + src/leap/mail/service.py | 384 +++++++++++++++++ src/leap/mail/smtp/__init__.py | 6 +- src/leap/mail/smtp/gateway.py | 458 ++------------------- src/leap/mail/smtp/tests/__init__.py | 0 src/leap/mail/smtp/tests/test_gateway.py | 185 +-------- src/leap/mail/tests/test_service.py | 185 +++++++++ src/leap/mail/utils.py | 23 ++ 8 files changed, 639 insertions(+), 603 deletions(-) create mode 100644 changes/feature-6357_factor_encryptio_and_sending_out_of_encrypted_message create mode 100644 src/leap/mail/service.py create mode 100644 src/leap/mail/smtp/tests/__init__.py create mode 100644 src/leap/mail/tests/test_service.py diff --git a/changes/feature-6357_factor_encryptio_and_sending_out_of_encrypted_message b/changes/feature-6357_factor_encryptio_and_sending_out_of_encrypted_message new file mode 100644 index 0000000..6b95c6a --- /dev/null +++ b/changes/feature-6357_factor_encryptio_and_sending_out_of_encrypted_message @@ -0,0 +1 @@ +- Creates a OutgoingMail class that has the logic for encrypting, signing and sending messages. Factors that logic out of EncryptedMessage so it can be used by other clients (Closes: #6357) diff --git a/src/leap/mail/service.py b/src/leap/mail/service.py new file mode 100644 index 0000000..d595067 --- /dev/null +++ b/src/leap/mail/service.py @@ -0,0 +1,384 @@ +# -*- coding: utf-8 -*- +# service.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import re +from StringIO import StringIO +from email.parser import Parser +from email.mime.application import MIMEApplication + +from OpenSSL import SSL + +from twisted.mail import smtp +from twisted.internet import reactor +from twisted.internet import defer +from twisted.internet.threads import deferToThread +from twisted.protocols.amp import ssl +from twisted.python import log + +from leap.common.check import leap_assert_type, leap_assert +from leap.common.events import proto, signal +from leap.keymanager import KeyManager +from leap.keymanager.openpgp import OpenPGPKey +from leap.keymanager.errors import KeyNotFound +from leap.mail import __version__ +from leap.mail.utils import validate_address +from leap.mail.smtp.rfc3156 import MultipartEncrypted +from leap.mail.smtp.rfc3156 import MultipartSigned +from leap.mail.smtp.rfc3156 import encode_base64_rec +from leap.mail.smtp.rfc3156 import RFC3156CompliantGenerator +from leap.mail.smtp.rfc3156 import PGPSignature +from leap.mail.smtp.rfc3156 import PGPEncrypted + + +class SSLContextFactory(ssl.ClientContextFactory): + def __init__(self, cert, key): + self.cert = cert + self.key = key + + def getContext(self): + self.method = SSL.TLSv1_METHOD # SSLv23_METHOD + ctx = ssl.ClientContextFactory.getContext(self) + ctx.use_certificate_file(self.cert) + ctx.use_privatekey_file(self.key) + return ctx + + +class OutgoingMail: + """ + A service for handling encrypted mail. + """ + + FOOTER_STRING = "I prefer encrypted email" + + def __init__(self, from_address, keymanager, cert, key, host, port): + """ + Initialize the mail service. + + :param from_address: The sender address. + :type from_address: str + :param keymanager: A KeyManager for retrieving recipient's keys. + :type keymanager: leap.common.keymanager.KeyManager + :param cert: The client certificate for SSL authentication. + :type cert: str + :param key: The client private key for SSL authentication. + :type key: str + :param host: The hostname of the remote SMTP server. + :type host: str + :param port: The port of the remote SMTP server. + :type port: int + """ + + # XXX: should we keep these checks? + # assert params + leap_assert_type(keymanager, KeyManager) + leap_assert_type(host, str) + leap_assert(host != '') + leap_assert_type(port, int) + leap_assert(port is not 0) + leap_assert_type(cert, unicode) + leap_assert(cert != '') + leap_assert_type(key, unicode) + leap_assert(key != '') + + self._port = port + self._host = host + self._key = key + self._cert = cert + self._from_address = from_address + self._keymanager = keymanager + + def send_message(self, raw, recipient): + """ + Sends a message to a recipient. Maybe encrypts and signs. + + :param raw: The raw message + :type raw: str + :param recipient: The recipient for the message + :type recipient: smtp.User + :return: a deferred which delivers the message when fired + """ + d = deferToThread(lambda: self._maybe_encrypt_and_sign(raw, recipient)) + d.addCallback(self._route_msg) + d.addErrback(self.sendError) + + return d + + def sendSuccess(self, smtp_sender_result): + """ + Callback for a successful send. + + :param smtp_sender_result: The result from the ESMTPSender from _route_msg + :type smtp_sender_result: tuple(int, list(tuple)) + """ + dest_addrstr = smtp_sender_result[1][0][0] + log.msg('Message sent to %s' % dest_addrstr) + signal(proto.SMTP_SEND_MESSAGE_SUCCESS, dest_addrstr) + + def sendError(self, failure): + """ + Callback for an unsuccessfull send. + + :param e: The result from the last errback. + :type e: anything + """ + # XXX: need to get the address from the exception to send signal + # signal(proto.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr) + err = failure.value + log.err(err) + raise err + + def _route_msg(self, encrypt_and_sign_result): + """ + Sends the msg using the ESMTPSenderFactory. + + :param encrypt_and_sign_result: A tuple containing the 'maybe' encrypted message and the recipient + :type encrypt_and_sign_result: tuple + """ + message, recipient = encrypt_and_sign_result + log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port)) + msg = message.as_string(False) + + # we construct a defer to pass to the ESMTPSenderFactory + d = defer.Deferred() + d.addCallbacks(self.sendSuccess, self.sendError) + # we don't pass an ssl context factory to the ESMTPSenderFactory + # because ssl will be handled by reactor.connectSSL() below. + factory = smtp.ESMTPSenderFactory( + "", # username is blank because client auth is done on SSL protocol level + "", # password is blank because client auth is done on SSL protocol level + self._from_address, + recipient.dest.addrstr, + StringIO(msg), + d, + heloFallback=True, + requireAuthentication=False, + requireTransportSecurity=True) + factory.domain = __version__ + signal(proto.SMTP_SEND_MESSAGE_START, recipient.dest.addrstr) + reactor.connectSSL( + self._host, self._port, factory, + contextFactory=SSLContextFactory(self._cert, self._key)) + + + def _maybe_encrypt_and_sign(self, raw, recipient): + """ + Attempt to encrypt and sign the outgoing message. + + The behaviour of this method depends on: + + 1. the original message's content-type, and + 2. the availability of the recipient's public key. + + If the original message's content-type is "multipart/encrypted", then + the original message is not altered. For any other content-type, the + method attempts to fetch the recipient's public key. If the + recipient's public key is available, the message is encrypted and + signed; otherwise it is only signed. + + Note that, if the C{encrypted_only} configuration is set to True and + the recipient's public key is not available, then the recipient + address would have been rejected in SMTPDelivery.validateTo(). + + The following table summarizes the overall behaviour of the gateway: + + +---------------------------------------------------+----------------+ + | content-type | rcpt pubkey | enforce encr. | action | + +---------------------+-------------+---------------+----------------+ + | multipart/encrypted | any | any | pass | + | other | available | any | encrypt + sign | + | other | unavailable | yes | reject | + | other | unavailable | no | sign | + +---------------------+-------------+---------------+----------------+ + + :param raw: The raw message + :type raw: str + :param recipient: The recipient for the message + :type: recipient: smtp.User + + """ + # pass if the original message's content-type is "multipart/encrypted" + lines = raw.split('\r\n') + origmsg = Parser().parsestr(raw) + + if origmsg.get_content_type() == 'multipart/encrypted': + return origmsg + + from_address = validate_address(self._from_address) + username, domain = from_address.split('@') + + # add a nice footer to the outgoing message + # XXX: footer will eventually optional or be removed + if origmsg.get_content_type() == 'text/plain': + lines.append('--') + lines.append('%s - https://%s/key/%s' % + (self.FOOTER_STRING, domain, username)) + lines.append('') + + origmsg = Parser().parsestr('\r\n'.join(lines)) + + # get sender and recipient data + signkey = self._keymanager.get_key(from_address, OpenPGPKey, private=True) + log.msg("Will sign the message with %s." % signkey.fingerprint) + to_address = validate_address(recipient.dest.addrstr) + try: + # try to get the recipient pubkey + pubkey = self._keymanager.get_key(to_address, OpenPGPKey) + log.msg("Will encrypt the message to %s." % pubkey.fingerprint) + signal(proto.SMTP_START_ENCRYPT_AND_SIGN, + "%s,%s" % (self._from_address, to_address)) + newmsg = self._encrypt_and_sign(origmsg, pubkey, signkey) + + signal(proto.SMTP_END_ENCRYPT_AND_SIGN, + "%s,%s" % (self._from_address, to_address)) + except KeyNotFound: + # at this point we _can_ send unencrypted mail, because if the + # configuration said the opposite the address would have been + # rejected in SMTPDelivery.validateTo(). + log.msg('Will send unencrypted message to %s.' % to_address) + signal(proto.SMTP_START_SIGN, self._from_address) + newmsg = self._sign(origmsg, signkey) + signal(proto.SMTP_END_SIGN, self._from_address) + return newmsg, recipient + + + def _encrypt_and_sign(self, origmsg, pubkey, signkey): + """ + Create an RFC 3156 compliang PGP encrypted and signed message using + C{pubkey} to encrypt and C{signkey} to sign. + + :param origmsg: The original message + :type origmsg: email.message.Message + :param pubkey: The public key used to encrypt the message. + :type pubkey: OpenPGPKey + :param signkey: The private key used to sign the message. + :type signkey: OpenPGPKey + :return: The encrypted and signed message + :rtype: MultipartEncrypted + """ + # create new multipart/encrypted message with 'pgp-encrypted' protocol + newmsg = MultipartEncrypted('application/pgp-encrypted') + # move (almost) all headers from original message to the new message + self._fix_headers(origmsg, newmsg, signkey) + # create 'application/octet-stream' encrypted message + encmsg = MIMEApplication( + self._keymanager.encrypt(origmsg.as_string(unixfrom=False), pubkey, + sign=signkey), + _subtype='octet-stream', _encoder=lambda x: x) + encmsg.add_header('content-disposition', 'attachment', + filename='msg.asc') + # create meta message + metamsg = PGPEncrypted() + metamsg.add_header('Content-Disposition', 'attachment') + # attach pgp message parts to new message + newmsg.attach(metamsg) + newmsg.attach(encmsg) + return newmsg + + + def _sign(self, origmsg, signkey): + """ + Create an RFC 3156 compliant PGP signed MIME message using C{signkey}. + + :param origmsg: The original message + :type origmsg: email.message.Message + :param signkey: The private key used to sign the message. + :type signkey: leap.common.keymanager.openpgp.OpenPGPKey + :return: The signed message. + :rtype: MultipartSigned + """ + # create new multipart/signed message + newmsg = MultipartSigned('application/pgp-signature', 'pgp-sha512') + # move (almost) all headers from original message to the new message + self._fix_headers(origmsg, newmsg, signkey) + # apply base64 content-transfer-encoding + encode_base64_rec(origmsg) + # get message text with headers and replace \n for \r\n + fp = StringIO() + g = RFC3156CompliantGenerator( + fp, mangle_from_=False, maxheaderlen=76) + g.flatten(origmsg) + msgtext = re.sub('\r?\n', '\r\n', fp.getvalue()) + # make sure signed message ends with \r\n as per OpenPGP stantard. + if origmsg.is_multipart(): + if not msgtext.endswith("\r\n"): + msgtext += "\r\n" + # calculate signature + signature = self._keymanager.sign(msgtext, signkey, digest_algo='SHA512', + clearsign=False, detach=True, binary=False) + sigmsg = PGPSignature(signature) + # attach original message and signature to new message + newmsg.attach(origmsg) + newmsg.attach(sigmsg) + return newmsg + + + def _fix_headers(self, origmsg, newmsg, signkey): + """ + Move some headers from C{origmsg} to C{newmsg}, delete unwanted + headers from C{origmsg} and add new headers to C{newms}. + + Outgoing messages are either encrypted and signed or just signed + before being sent. Because of that, they are packed inside new + messages and some manipulation has to be made on their headers. + + Allowed headers for passing through: + + - From + - Date + - To + - Subject + - Reply-To + - References + - In-Reply-To + - Cc + + Headers to be added: + + - Message-ID (i.e. should not use origmsg's Message-Id) + - Received (this is added automatically by twisted smtp API) + - OpenPGP (see #4447) + + Headers to be deleted: + + - User-Agent + + :param origmsg: The original message. + :type origmsg: email.message.Message + :param newmsg: The new message being created. + :type newmsg: email.message.Message + :param signkey: The key used to sign C{newmsg} + :type signkey: OpenPGPKey + """ + # move headers from origmsg to newmsg + headers = origmsg.items() + passthrough = [ + 'from', 'date', 'to', 'subject', 'reply-to', 'references', + 'in-reply-to', 'cc' + ] + headers = filter(lambda x: x[0].lower() in passthrough, headers) + for hkey, hval in headers: + newmsg.add_header(hkey, hval) + del (origmsg[hkey]) + # add a new message-id to newmsg + newmsg.add_header('Message-Id', smtp.messageid()) + # add openpgp header to newmsg + username, domain = signkey.address.split('@') + newmsg.add_header( + 'OpenPGP', 'id=%s' % signkey.key_id, + url='https://%s/key/%s' % (domain, username), + preference='signencrypt') + # delete user-agent from origmsg + del (origmsg['user-agent']) diff --git a/src/leap/mail/smtp/__init__.py b/src/leap/mail/smtp/__init__.py index bbd4064..f740f5e 100644 --- a/src/leap/mail/smtp/__init__.py +++ b/src/leap/mail/smtp/__init__.py @@ -22,6 +22,8 @@ import logging from twisted.internet import reactor from twisted.internet.error import CannotListenError +from twisted.mail import smtp +from leap.mail.service import OutgoingMail logger = logging.getLogger(__name__) @@ -59,8 +61,8 @@ def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port, :returns: tuple of SMTPFactory, twisted.internet.tcp.Port """ # configure the use of this service with twistd - factory = SMTPFactory(userid, keymanager, smtp_host, smtp_port, smtp_cert, - smtp_key, encrypted_only) + outgoing_mail = OutgoingMail(str(userid), keymanager, smtp_cert, smtp_key, smtp_host, smtp_port) + factory = SMTPFactory(userid, keymanager, encrypted_only, outgoing_mail) try: tport = reactor.listenTCP(port, factory, interface="localhost") signal(proto.SMTP_SERVICE_STARTED, str(port)) diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index 13d3bbf..b022091 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -31,39 +31,27 @@ The following classes comprise the SMTP gateway service: """ -import re -from StringIO import StringIO -from email.Header import Header -from email.utils import parseaddr -from email.parser import Parser -from email.mime.application import MIMEApplication from zope.interface import implements -from OpenSSL import SSL from twisted.mail import smtp from twisted.internet.protocol import ServerFactory -from twisted.internet import reactor, ssl -from twisted.internet import defer -from twisted.internet.threads import deferToThread from twisted.python import log -from leap.common.check import leap_assert, leap_assert_type +from email.Header import Header +from leap.common.check import leap_assert_type from leap.common.events import proto, signal -from leap.keymanager import KeyManager from leap.keymanager.openpgp import OpenPGPKey from leap.keymanager.errors import KeyNotFound -from leap.mail import __version__ +from leap.mail.utils import validate_address + from leap.mail.smtp.rfc3156 import ( - MultipartSigned, - MultipartEncrypted, - PGPEncrypted, - PGPSignature, RFC3156CompliantGenerator, - encode_base64_rec, ) +from leap.mail.service import OutgoingMail # replace email generator with a RFC 3156 compliant one. from email import generator + generator.Generator = RFC3156CompliantGenerator @@ -74,31 +62,6 @@ generator.Generator = RFC3156CompliantGenerator LOCAL_FQDN = "bitmask.local" -def validate_address(address): - """ - Validate C{address} as defined in RFC 2822. - - :param address: The address to be validated. - :type address: str - - @return: A valid address. - @rtype: str - - @raise smtp.SMTPBadRcpt: Raised if C{address} is invalid. - """ - leap_assert_type(address, str) - # in the following, the address is parsed as described in RFC 2822 and - # ('', '') is returned if the parse fails. - _, address = parseaddr(address) - if address == '': - raise smtp.SMTPBadRcpt(address) - return address - - -# -# SMTPFactory -# - class SMTPHeloLocalhost(smtp.SMTP): """ An SMTP class that ensures a proper FQDN @@ -119,45 +82,26 @@ class SMTPFactory(ServerFactory): """ domain = LOCAL_FQDN - def __init__(self, userid, keymanager, host, port, cert, key, - encrypted_only): + def __init__(self, userid, keymanager, encrypted_only, outgoing_mail): """ Initialize the SMTP factory. :param userid: The user currently logged in :type userid: unicode - :param keymanager: A KeyManager for retrieving recipient's keys. - :type keymanager: leap.common.keymanager.KeyManager - :param host: The hostname of the remote SMTP server. - :type host: str - :param port: The port of the remote SMTP server. - :type port: int - :param cert: The client certificate for authentication. - :type cert: str - :param key: The client key for authentication. - :type key: str + :param keymanager: A Key Manager from where to get recipients' public + keys. :param encrypted_only: Whether the SMTP gateway should send unencrypted mail or not. :type encrypted_only: bool + :param outgoing_mail: The outgoing mail to send the message + :type outgoing_mail: leap.mail.service.OutgoingMail """ - # assert params - leap_assert_type(keymanager, KeyManager) - leap_assert_type(host, str) - leap_assert(host != '') - leap_assert_type(port, int) - leap_assert(port is not 0) - leap_assert_type(cert, unicode) - leap_assert(cert != '') - leap_assert_type(key, unicode) - leap_assert(key != '') + leap_assert_type(encrypted_only, bool) # and store them self._userid = userid self._km = keymanager - self._host = host - self._port = port - self._cert = cert - self._key = key + self._outgoing_mail = outgoing_mail self._encrypted_only = encrypted_only def buildProtocol(self, addr): @@ -170,9 +114,7 @@ class SMTPFactory(ServerFactory): @return: The protocol. @rtype: SMTPDelivery """ - smtpProtocol = SMTPHeloLocalhost(SMTPDelivery( - self._userid, self._km, self._host, self._port, self._cert, - self._key, self._encrypted_only)) + smtpProtocol = SMTPHeloLocalhost(SMTPDelivery(self._userid, self._km, self._encrypted_only, self._outgoing_mail)) smtpProtocol.factory = self return smtpProtocol @@ -188,33 +130,23 @@ class SMTPDelivery(object): implements(smtp.IMessageDelivery) - def __init__(self, userid, keymanager, host, port, cert, key, - encrypted_only): + def __init__(self, userid, keymanager, encrypted_only, outgoing_mail): """ Initialize the SMTP delivery object. :param userid: The user currently logged in :type userid: unicode - :param keymanager: A KeyManager for retrieving recipient's keys. - :type keymanager: leap.common.keymanager.KeyManager - :param host: The hostname of the remote SMTP server. - :type host: str - :param port: The port of the remote SMTP server. - :type port: int - :param cert: The client certificate for authentication. - :type cert: str - :param key: The client key for authentication. - :type key: str + :param keymanager: A Key Manager from where to get recipients' public + keys. :param encrypted_only: Whether the SMTP gateway should send unencrypted mail or not. :type encrypted_only: bool + :param outgoing_mail: The outgoing mail to send the message + :type outgoing_mail: leap.mail.service.OutgoingMail """ self._userid = userid + self._outgoing_mail = outgoing_mail self._km = keymanager - self._host = host - self._port = port - self._cert = cert - self._key = key self._encrypted_only = encrypted_only self._origin = None @@ -280,9 +212,7 @@ class SMTPDelivery(object): "encrypted_only' is set to False).") signal( proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, user.dest.addrstr) - return lambda: EncryptedMessage( - self._origin, user, self._km, self._host, self._port, self._cert, - self._key) + return lambda: EncryptedMessage(user, self._outgoing_mail) def validateFrom(self, helo, origin): """ @@ -314,19 +244,6 @@ class SMTPDelivery(object): # EncryptedMessage # -class SSLContextFactory(ssl.ClientContextFactory): - def __init__(self, cert, key): - self.cert = cert - self.key = key - - def getContext(self): - self.method = SSL.TLSv1_METHOD # SSLv23_METHOD - ctx = ssl.ClientContextFactory.getContext(self) - ctx.use_certificate_file(self.cert) - ctx.use_privatekey_file(self.key) - return ctx - - class EncryptedMessage(object): """ Receive plaintext from client, encrypt it and send message to a @@ -334,44 +251,21 @@ class EncryptedMessage(object): """ implements(smtp.IMessage) - FOOTER_STRING = "I prefer encrypted email" - - def __init__(self, fromAddress, user, keymanager, host, port, cert, key): + def __init__(self, user, outgoing_mail): """ Initialize the encrypted message. - :param fromAddress: The address of the sender. - :type fromAddress: twisted.mail.smtp.Address :param user: The recipient of this message. :type user: twisted.mail.smtp.User - :param keymanager: A KeyManager for retrieving recipient's keys. - :type keymanager: leap.common.keymanager.KeyManager - :param host: The hostname of the remote SMTP server. - :type host: str - :param port: The port of the remote SMTP server. - :type port: int - :param cert: The client certificate for authentication. - :type cert: str - :param key: The client key for authentication. - :type key: str + :param outgoing_mail: The outgoing mail to send the message + :type outgoing_mail: leap.mail.service.OutgoingMail """ # assert params leap_assert_type(user, smtp.User) - leap_assert_type(keymanager, KeyManager) - # and store them - self._fromAddress = fromAddress - self._user = user - self._km = keymanager - self._host = host - self._port = port - self._cert = cert - self._key = key - # initialize list for message's lines - self.lines = [] - # - # methods from smtp.IMessage - # + self._user = user + self._lines = [] + self._outgoing_mail = outgoing_mail def lineReceived(self, line): """ @@ -380,7 +274,7 @@ class EncryptedMessage(object): :param line: The received line. :type line: str """ - self.lines.append(line) + self._lines.append(line) def eomReceived(self): """ @@ -391,10 +285,10 @@ class EncryptedMessage(object): :returns: a deferred """ log.msg("Message data complete.") - self.lines.append('') # add a trailing newline - d = deferToThread(self._maybe_encrypt_and_sign) - d.addCallbacks(self.sendMessage, self.skipNoKeyErrBack) - return d + self._lines.append('') # add a trailing newline + raw_mail = '\r\n'.join(self._lines) + + return self._outgoing_mail.send_message(raw_mail, self._user) def connectionLost(self): """ @@ -404,290 +298,4 @@ class EncryptedMessage(object): log.err() signal(proto.SMTP_CONNECTION_LOST, self._user.dest.addrstr) # unexpected loss of connection; don't save - self.lines = [] - - # ends IMessage implementation - - def skipNoKeyErrBack(self, failure): - """ - Errback that ignores a KeyNotFound - - :param failure: the failure - :type Failure: Failure - """ - err = failure.value - if failure.check(KeyNotFound): - pass - else: - raise err - - def parseMessage(self): - """ - Separate message headers from body. - """ - parser = Parser() - return parser.parsestr('\r\n'.join(self.lines)) - - def sendQueued(self, r): - """ - Callback for the queued message. - - :param r: The result from the last previous callback in the chain. - :type r: anything - """ - log.msg(r) - - def sendSuccess(self, r): - """ - Callback for a successful send. - - :param r: The result from the last previous callback in the chain. - :type r: anything - """ - log.msg(r) - signal(proto.SMTP_SEND_MESSAGE_SUCCESS, self._user.dest.addrstr) - - def sendError(self, failure): - """ - Callback for an unsuccessfull send. - - :param e: The result from the last errback. - :type e: anything - """ - signal(proto.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr) - err = failure.value - log.err(err) - raise err - - def sendMessage(self, *args): - """ - Sends the message. - - :return: A deferred with callback and errback for - this #message send. - :rtype: twisted.internet.defer.Deferred - """ - d = deferToThread(self._route_msg) - d.addCallbacks(self.sendQueued, self.sendError) - return d - - def _route_msg(self): - """ - Sends the msg using the ESMTPSenderFactory. - """ - log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port)) - msg = self._msg.as_string(False) - - # we construct a defer to pass to the ESMTPSenderFactory - d = defer.Deferred() - d.addCallbacks(self.sendSuccess, self.sendError) - # we don't pass an ssl context factory to the ESMTPSenderFactory - # because ssl will be handled by reactor.connectSSL() below. - factory = smtp.ESMTPSenderFactory( - "", # username is blank because server does not use auth. - "", # password is blank because server does not use auth. - self._fromAddress.addrstr, - self._user.dest.addrstr, - StringIO(msg), - d, - heloFallback=True, - requireAuthentication=False, - requireTransportSecurity=True) - factory.domain = __version__ - signal(proto.SMTP_SEND_MESSAGE_START, self._user.dest.addrstr) - reactor.connectSSL( - self._host, self._port, factory, - contextFactory=SSLContextFactory(self._cert, self._key)) - - # - # encryption methods - # - - def _encrypt_and_sign(self, pubkey, signkey): - """ - Create an RFC 3156 compliang PGP encrypted and signed message using - C{pubkey} to encrypt and C{signkey} to sign. - - :param pubkey: The public key used to encrypt the message. - :type pubkey: OpenPGPKey - :param signkey: The private key used to sign the message. - :type signkey: OpenPGPKey - """ - # create new multipart/encrypted message with 'pgp-encrypted' protocol - newmsg = MultipartEncrypted('application/pgp-encrypted') - # move (almost) all headers from original message to the new message - self._fix_headers(self._origmsg, newmsg, signkey) - # create 'application/octet-stream' encrypted message - encmsg = MIMEApplication( - self._km.encrypt(self._origmsg.as_string(unixfrom=False), pubkey, - sign=signkey), - _subtype='octet-stream', _encoder=lambda x: x) - encmsg.add_header('content-disposition', 'attachment', - filename='msg.asc') - # create meta message - metamsg = PGPEncrypted() - metamsg.add_header('Content-Disposition', 'attachment') - # attach pgp message parts to new message - newmsg.attach(metamsg) - newmsg.attach(encmsg) - self._msg = newmsg - - def _sign(self, signkey): - """ - Create an RFC 3156 compliant PGP signed MIME message using C{signkey}. - - :param signkey: The private key used to sign the message. - :type signkey: leap.common.keymanager.openpgp.OpenPGPKey - """ - # create new multipart/signed message - newmsg = MultipartSigned('application/pgp-signature', 'pgp-sha512') - # move (almost) all headers from original message to the new message - self._fix_headers(self._origmsg, newmsg, signkey) - # apply base64 content-transfer-encoding - encode_base64_rec(self._origmsg) - # get message text with headers and replace \n for \r\n - fp = StringIO() - g = RFC3156CompliantGenerator( - fp, mangle_from_=False, maxheaderlen=76) - g.flatten(self._origmsg) - msgtext = re.sub('\r?\n', '\r\n', fp.getvalue()) - # make sure signed message ends with \r\n as per OpenPGP stantard. - if self._origmsg.is_multipart(): - if not msgtext.endswith("\r\n"): - msgtext += "\r\n" - # calculate signature - signature = self._km.sign(msgtext, signkey, digest_algo='SHA512', - clearsign=False, detach=True, binary=False) - sigmsg = PGPSignature(signature) - # attach original message and signature to new message - newmsg.attach(self._origmsg) - newmsg.attach(sigmsg) - self._msg = newmsg - - def _maybe_encrypt_and_sign(self): - """ - Attempt to encrypt and sign the outgoing message. - - The behaviour of this method depends on: - - 1. the original message's content-type, and - 2. the availability of the recipient's public key. - - If the original message's content-type is "multipart/encrypted", then - the original message is not altered. For any other content-type, the - method attempts to fetch the recipient's public key. If the - recipient's public key is available, the message is encrypted and - signed; otherwise it is only signed. - - Note that, if the C{encrypted_only} configuration is set to True and - the recipient's public key is not available, then the recipient - address would have been rejected in SMTPDelivery.validateTo(). - - The following table summarizes the overall behaviour of the gateway: - - +---------------------------------------------------+----------------+ - | content-type | rcpt pubkey | enforce encr. | action | - +---------------------+-------------+---------------+----------------+ - | multipart/encrypted | any | any | pass | - | other | available | any | encrypt + sign | - | other | unavailable | yes | reject | - | other | unavailable | no | sign | - +---------------------+-------------+---------------+----------------+ - """ - # pass if the original message's content-type is "multipart/encrypted" - self._origmsg = self.parseMessage() - if self._origmsg.get_content_type() == 'multipart/encrypted': - self._msg = self._origmsg - return - - from_address = validate_address(self._fromAddress.addrstr) - username, domain = from_address.split('@') - - # add a nice footer to the outgoing message - if self._origmsg.get_content_type() == 'text/plain': - self.lines.append('--') - self.lines.append('%s - https://%s/key/%s' % - (self.FOOTER_STRING, domain, username)) - self.lines.append('') - - self._origmsg = self.parseMessage() - - # get sender and recipient data - signkey = self._km.get_key(from_address, OpenPGPKey, private=True) - log.msg("Will sign the message with %s." % signkey.fingerprint) - to_address = validate_address(self._user.dest.addrstr) - try: - # try to get the recipient pubkey - pubkey = self._km.get_key(to_address, OpenPGPKey) - log.msg("Will encrypt the message to %s." % pubkey.fingerprint) - signal(proto.SMTP_START_ENCRYPT_AND_SIGN, - "%s,%s" % (self._fromAddress.addrstr, to_address)) - self._encrypt_and_sign(pubkey, signkey) - signal(proto.SMTP_END_ENCRYPT_AND_SIGN, - "%s,%s" % (self._fromAddress.addrstr, to_address)) - except KeyNotFound: - # at this point we _can_ send unencrypted mail, because if the - # configuration said the opposite the address would have been - # rejected in SMTPDelivery.validateTo(). - log.msg('Will send unencrypted message to %s.' % to_address) - signal(proto.SMTP_START_SIGN, self._fromAddress.addrstr) - self._sign(signkey) - signal(proto.SMTP_END_SIGN, self._fromAddress.addrstr) - - def _fix_headers(self, origmsg, newmsg, signkey): - """ - Move some headers from C{origmsg} to C{newmsg}, delete unwanted - headers from C{origmsg} and add new headers to C{newms}. - - Outgoing messages are either encrypted and signed or just signed - before being sent. Because of that, they are packed inside new - messages and some manipulation has to be made on their headers. - - Allowed headers for passing through: - - - From - - Date - - To - - Subject - - Reply-To - - References - - In-Reply-To - - Cc - - Headers to be added: - - - Message-ID (i.e. should not use origmsg's Message-Id) - - Received (this is added automatically by twisted smtp API) - - OpenPGP (see #4447) - - Headers to be deleted: - - - User-Agent - - :param origmsg: The original message. - :type origmsg: email.message.Message - :param newmsg: The new message being created. - :type newmsg: email.message.Message - :param signkey: The key used to sign C{newmsg} - :type signkey: OpenPGPKey - """ - # move headers from origmsg to newmsg - headers = origmsg.items() - passthrough = [ - 'from', 'date', 'to', 'subject', 'reply-to', 'references', - 'in-reply-to', 'cc' - ] - headers = filter(lambda x: x[0].lower() in passthrough, headers) - for hkey, hval in headers: - newmsg.add_header(hkey, hval) - del(origmsg[hkey]) - # add a new message-id to newmsg - newmsg.add_header('Message-Id', smtp.messageid()) - # add openpgp header to newmsg - username, domain = signkey.address.split('@') - newmsg.add_header( - 'OpenPGP', 'id=%s' % signkey.key_id, - url='https://%s/key/%s' % (domain, username), - preference='signencrypt') - # delete user-agent from origmsg - del(origmsg['user-agent']) + self._lines = [] diff --git a/src/leap/mail/smtp/tests/__init__.py b/src/leap/mail/smtp/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/mail/smtp/tests/test_gateway.py b/src/leap/mail/smtp/tests/test_gateway.py index 3635a9f..aeace4a 100644 --- a/src/leap/mail/smtp/tests/test_gateway.py +++ b/src/leap/mail/smtp/tests/test_gateway.py @@ -20,17 +20,14 @@ SMTP gateway tests. """ - import re from datetime import datetime + from twisted.test import proto_helpers -from twisted.mail.smtp import User, Address from mock import Mock - from leap.mail.smtp.gateway import ( - SMTPFactory, - EncryptedMessage, + SMTPFactory ) from leap.mail.tests import ( TestCaseWithKeyManager, @@ -39,6 +36,7 @@ from leap.mail.tests import ( ) from leap.keymanager import openpgp + # some regexps IP_REGEX = "(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}" + \ "([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])" @@ -71,20 +69,6 @@ class TestSmtpGateway(TestCaseWithKeyManager): % (string, pattern)) raise self.failureException(msg) - def test_openpgp_encrypt_decrypt(self): - "Test if openpgp can encrypt and decrypt." - text = "simple raw text" - pubkey = self._km.get_key( - ADDRESS, openpgp.OpenPGPKey, private=False) - encrypted = self._km.encrypt(text, pubkey) - self.assertNotEqual( - text, encrypted, "Ciphertext is equal to plaintext.") - privkey = self._km.get_key( - ADDRESS, openpgp.OpenPGPKey, private=True) - decrypted = self._km.decrypt(encrypted, privkey) - self.assertEqual(text, decrypted, - "Decrypted text differs from plaintext.") - def test_gateway_accepts_valid_email(self): """ Test if SMTP server responds correctly for valid interaction. @@ -102,10 +86,8 @@ class TestSmtpGateway(TestCaseWithKeyManager): # method... proto = SMTPFactory( u'anotheruser@leap.se', - self._km, self._config['host'], - self._config['port'], - self._config['cert'], self._config['key'], - self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) + self._km, + self._config['encrypted_only'], outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) # snip... transport = proto_helpers.StringTransport() proto.makeConnection(transport) @@ -116,151 +98,6 @@ class TestSmtpGateway(TestCaseWithKeyManager): 'Did not get expected answer from gateway.') proto.setTimeout(None) - def test_message_encrypt(self): - """ - Test if message gets encrypted to destination email. - """ - proto = SMTPFactory( - u'anotheruser@leap.se', - self._km, self._config['host'], - self._config['port'], - self._config['cert'], self._config['key'], - self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) - fromAddr = Address(ADDRESS_2) - dest = User(ADDRESS, 'gateway.leap.se', proto, ADDRESS) - m = EncryptedMessage( - fromAddr, dest, self._km, self._config['host'], - self._config['port'], self._config['cert'], self._config['key']) - for line in self.EMAIL_DATA[4:12]: - m.lineReceived(line) - # m.eomReceived() # this includes a defer, so we avoid calling it here - m.lines.append('') # add a trailing newline - # we need to call the following explicitelly because it was deferred - # inside the previous method - m._maybe_encrypt_and_sign() - # assert structure of encrypted message - self.assertTrue('Content-Type' in m._msg) - self.assertEqual('multipart/encrypted', m._msg.get_content_type()) - self.assertEqual('application/pgp-encrypted', - m._msg.get_param('protocol')) - self.assertEqual(2, len(m._msg.get_payload())) - self.assertEqual('application/pgp-encrypted', - m._msg.get_payload(0).get_content_type()) - self.assertEqual('application/octet-stream', - m._msg.get_payload(1).get_content_type()) - privkey = self._km.get_key( - ADDRESS, openpgp.OpenPGPKey, private=True) - decrypted = self._km.decrypt( - m._msg.get_payload(1).get_payload(), privkey) - self.assertEqual( - '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n\r\n--\r\n' + - 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n', - decrypted, - 'Decrypted text differs from plaintext.') - - def test_message_encrypt_sign(self): - """ - Test if message gets encrypted to destination email and signed with - sender key. - """ - proto = SMTPFactory( - u'anotheruser@leap.se', - self._km, self._config['host'], - self._config['port'], - self._config['cert'], self._config['key'], - self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) - user = User(ADDRESS, 'gateway.leap.se', proto, ADDRESS) - fromAddr = Address(ADDRESS_2) - m = EncryptedMessage( - fromAddr, user, self._km, self._config['host'], - self._config['port'], self._config['cert'], self._config['key']) - for line in self.EMAIL_DATA[4:12]: - m.lineReceived(line) - # trigger encryption and signing - # m.eomReceived() # this includes a defer, so we avoid calling it here - m.lines.append('') # add a trailing newline - # we need to call the following explicitelly because it was deferred - # inside the previous method - m._maybe_encrypt_and_sign() - # assert structure of encrypted message - self.assertTrue('Content-Type' in m._msg) - self.assertEqual('multipart/encrypted', m._msg.get_content_type()) - self.assertEqual('application/pgp-encrypted', - m._msg.get_param('protocol')) - self.assertEqual(2, len(m._msg.get_payload())) - self.assertEqual('application/pgp-encrypted', - m._msg.get_payload(0).get_content_type()) - self.assertEqual('application/octet-stream', - m._msg.get_payload(1).get_content_type()) - # decrypt and verify - privkey = self._km.get_key( - ADDRESS, openpgp.OpenPGPKey, private=True) - pubkey = self._km.get_key(ADDRESS_2, openpgp.OpenPGPKey) - decrypted = self._km.decrypt( - m._msg.get_payload(1).get_payload(), privkey, verify=pubkey) - self.assertEqual( - '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n\r\n--\r\n' + - 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n', - decrypted, - 'Decrypted text differs from plaintext.') - - def test_message_sign(self): - """ - Test if message is signed with sender key. - """ - # mock the key fetching - self._km.fetch_keys_from_server = Mock(return_value=[]) - proto = SMTPFactory( - u'anotheruser@leap.se', - self._km, self._config['host'], - self._config['port'], - self._config['cert'], self._config['key'], - self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) - user = User('ihavenopubkey@nonleap.se', - 'gateway.leap.se', proto, ADDRESS) - fromAddr = Address(ADDRESS_2) - m = EncryptedMessage( - fromAddr, user, self._km, self._config['host'], - self._config['port'], self._config['cert'], self._config['key']) - for line in self.EMAIL_DATA[4:12]: - m.lineReceived(line) - # trigger signing - # m.eomReceived() # this includes a defer, so we avoid calling it here - m.lines.append('') # add a trailing newline - # we need to call the following explicitelly because it was deferred - # inside the previous method - m._maybe_encrypt_and_sign() - # assert structure of signed message - self.assertTrue('Content-Type' in m._msg) - self.assertEqual('multipart/signed', m._msg.get_content_type()) - self.assertEqual('application/pgp-signature', - m._msg.get_param('protocol')) - self.assertEqual('pgp-sha512', m._msg.get_param('micalg')) - # assert content of message - self.assertEqual( - '\r\n'.join(self.EMAIL_DATA[9:13]) + '\r\n--\r\n' + - 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n', - m._msg.get_payload(0).get_payload(decode=True)) - # assert content of signature - self.assertTrue( - m._msg.get_payload(1).get_payload().startswith( - '-----BEGIN PGP SIGNATURE-----\n'), - 'Message does not start with signature header.') - self.assertTrue( - m._msg.get_payload(1).get_payload().endswith( - '-----END PGP SIGNATURE-----\n'), - 'Message does not end with signature footer.') - # assert signature is valid - pubkey = self._km.get_key(ADDRESS_2, openpgp.OpenPGPKey) - # replace EOL before verifying (according to rfc3156) - signed_text = re.sub('\r?\n', '\r\n', - m._msg.get_payload(0).as_string()) - self.assertTrue( - self._km.verify(signed_text, - pubkey, - detached_sig=m._msg.get_payload(1).get_payload()), - 'Signature could not be verified.') - def test_missing_key_rejects_address(self): """ Test if server rejects to send unencrypted when 'encrypted_only' is @@ -276,10 +113,8 @@ class TestSmtpGateway(TestCaseWithKeyManager): # prepare the SMTP factory proto = SMTPFactory( u'anotheruser@leap.se', - self._km, self._config['host'], - self._config['port'], - self._config['cert'], self._config['key'], - self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0)) + self._km, + self._config['encrypted_only'], outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) transport = proto_helpers.StringTransport() proto.makeConnection(transport) proto.lineReceived(self.EMAIL_DATA[0] + '\r\n') @@ -307,10 +142,8 @@ class TestSmtpGateway(TestCaseWithKeyManager): # prepare the SMTP factory with encrypted only equal to false proto = SMTPFactory( u'anotheruser@leap.se', - self._km, self._config['host'], - self._config['port'], - self._config['cert'], self._config['key'], - False).buildProtocol(('127.0.0.1', 0)) + self._km, + False, outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) transport = proto_helpers.StringTransport() proto.makeConnection(transport) proto.lineReceived(self.EMAIL_DATA[0] + '\r\n') diff --git a/src/leap/mail/tests/test_service.py b/src/leap/mail/tests/test_service.py new file mode 100644 index 0000000..f0a807d --- /dev/null +++ b/src/leap/mail/tests/test_service.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- +# test_gateway.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +SMTP gateway tests. +""" + +import re +from datetime import datetime +from twisted.mail.smtp import User, Address + +from mock import Mock + +from leap.mail.smtp.gateway import SMTPFactory +from leap.mail.service import OutgoingMail +from leap.mail.tests import ( + TestCaseWithKeyManager, + ADDRESS, + ADDRESS_2, +) +from leap.keymanager import openpgp + + +class TestOutgoingMail(TestCaseWithKeyManager): + EMAIL_DATA = ['HELO gateway.leap.se', + 'MAIL FROM: <%s>' % ADDRESS_2, + 'RCPT TO: <%s>' % ADDRESS, + 'DATA', + 'From: User <%s>' % ADDRESS_2, + 'To: Leap <%s>' % ADDRESS, + 'Date: ' + datetime.now().strftime('%c'), + 'Subject: test message', + '', + 'This is a secret message.', + 'Yours,', + 'A.', + '', + '.', + 'QUIT'] + + def setUp(self): + TestCaseWithKeyManager.setUp(self) + self.lines = [line for line in self.EMAIL_DATA[4:12]] + self.lines.append('') # add a trailing newline + self.raw = '\r\n'.join(self.lines) + self.fromAddr = ADDRESS_2 + self.outgoing_mail = OutgoingMail(self.fromAddr, self._km, self._config['cert'], self._config['key'], + self._config['host'], self._config['port']) + self.proto = SMTPFactory( + u'anotheruser@leap.se', + self._km, + self._config['encrypted_only'], + self.outgoing_mail).buildProtocol(('127.0.0.1', 0)) + self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS) + + def test_openpgp_encrypt_decrypt(self): + "Test if openpgp can encrypt and decrypt." + text = "simple raw text" + pubkey = self._km.get_key( + ADDRESS, openpgp.OpenPGPKey, private=False) + encrypted = self._km.encrypt(text, pubkey) + self.assertNotEqual( + text, encrypted, "Ciphertext is equal to plaintext.") + privkey = self._km.get_key( + ADDRESS, openpgp.OpenPGPKey, private=True) + decrypted = self._km.decrypt(encrypted, privkey) + self.assertEqual(text, decrypted, + "Decrypted text differs from plaintext.") + + def test_message_encrypt(self): + """ + Test if message gets encrypted to destination email. + """ + + message, _ = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + + # assert structure of encrypted message + self.assertTrue('Content-Type' in message) + self.assertEqual('multipart/encrypted', message.get_content_type()) + self.assertEqual('application/pgp-encrypted', + message.get_param('protocol')) + self.assertEqual(2, len(message.get_payload())) + self.assertEqual('application/pgp-encrypted', + message.get_payload(0).get_content_type()) + self.assertEqual('application/octet-stream', + message.get_payload(1).get_content_type()) + privkey = self._km.get_key( + ADDRESS, openpgp.OpenPGPKey, private=True) + decrypted = self._km.decrypt( + message.get_payload(1).get_payload(), privkey) + + expected = '\n' + '\r\n'.join( + self.EMAIL_DATA[9:12]) + '\r\n\r\n--\r\n' + 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n' + self.assertEqual( + expected, + decrypted, + 'Decrypted text differs from plaintext.') + + def test_message_encrypt_sign(self): + """ + Test if message gets encrypted to destination email and signed with + sender key. + """ + message, _ = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + + # assert structure of encrypted message + self.assertTrue('Content-Type' in message) + self.assertEqual('multipart/encrypted', message.get_content_type()) + self.assertEqual('application/pgp-encrypted', + message.get_param('protocol')) + self.assertEqual(2, len(message.get_payload())) + self.assertEqual('application/pgp-encrypted', + message.get_payload(0).get_content_type()) + self.assertEqual('application/octet-stream', + message.get_payload(1).get_content_type()) + # decrypt and verify + privkey = self._km.get_key( + ADDRESS, openpgp.OpenPGPKey, private=True) + pubkey = self._km.get_key(ADDRESS_2, openpgp.OpenPGPKey) + decrypted = self._km.decrypt( + message.get_payload(1).get_payload(), privkey, verify=pubkey) + self.assertEqual( + '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n\r\n--\r\n' + + 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n', + decrypted, + 'Decrypted text differs from plaintext.') + + def test_message_sign(self): + """ + Test if message is signed with sender key. + """ + # mock the key fetching + self._km.fetch_keys_from_server = Mock(return_value=[]) + recipient = User('ihavenopubkey@nonleap.se', + 'gateway.leap.se', self.proto, ADDRESS) + self.outgoing_mail = OutgoingMail(self.fromAddr, self._km, self._config['cert'], self._config['key'], + self._config['host'], self._config['port']) + + message, _ = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, recipient) + + # assert structure of signed message + self.assertTrue('Content-Type' in message) + self.assertEqual('multipart/signed', message.get_content_type()) + self.assertEqual('application/pgp-signature', + message.get_param('protocol')) + self.assertEqual('pgp-sha512', message.get_param('micalg')) + # assert content of message + self.assertEqual( + '\r\n'.join(self.EMAIL_DATA[9:13]) + '\r\n--\r\n' + + 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n', + message.get_payload(0).get_payload(decode=True)) + # assert content of signature + self.assertTrue( + message.get_payload(1).get_payload().startswith( + '-----BEGIN PGP SIGNATURE-----\n'), + 'Message does not start with signature header.') + self.assertTrue( + message.get_payload(1).get_payload().endswith( + '-----END PGP SIGNATURE-----\n'), + 'Message does not end with signature footer.') + # assert signature is valid + pubkey = self._km.get_key(ADDRESS_2, openpgp.OpenPGPKey) + # replace EOL before verifying (according to rfc3156) + signed_text = re.sub('\r?\n', '\r\n', + message.get_payload(0).as_string()) + self.assertTrue( + self._km.verify(signed_text, + pubkey, + detached_sig=message.get_payload(1).get_payload()), + 'Signature could not be verified.') diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py index fed24b3..457097b 100644 --- a/src/leap/mail/utils.py +++ b/src/leap/mail/utils.py @@ -17,12 +17,15 @@ """ Mail utilities. """ +from email.utils import parseaddr import json import re import traceback import Queue from leap.soledad.common.document import SoledadDocument +from leap.common.check import leap_assert_type +from twisted.mail import smtp CHARSET_PATTERN = r"""charset=([\w-]+)""" @@ -224,6 +227,26 @@ def accumulator_queue(fun, lim): return _accumulator +def validate_address(address): + """ + Validate C{address} as defined in RFC 2822. + + :param address: The address to be validated. + :type address: str + + @return: A valid address. + @rtype: str + + @raise smtp.SMTPBadRcpt: Raised if C{address} is invalid. + """ + leap_assert_type(address, str) + # in the following, the address is parsed as described in RFC 2822 and + # ('', '') is returned if the parse fails. + _, address = parseaddr(address) + if address == '': + raise smtp.SMTPBadRcpt(address) + return address + # # String manipulation # -- cgit v1.2.3 From a3acaed8512c16fc1d53141d288d60eacc6ef94e Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 25 Nov 2014 11:55:49 -0200 Subject: Move SMTP gateway str assertion to inside OutgoingMail. --- src/leap/mail/service.py | 3 ++- src/leap/mail/smtp/__init__.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/leap/mail/service.py b/src/leap/mail/service.py index d595067..f6e4d11 100644 --- a/src/leap/mail/service.py +++ b/src/leap/mail/service.py @@ -81,8 +81,9 @@ class OutgoingMail: :type port: int """ - # XXX: should we keep these checks? # assert params + leap_assert_type(from_address, str) + leap_assert('@' in from_address) leap_assert_type(keymanager, KeyManager) leap_assert_type(host, str) leap_assert(host != '') diff --git a/src/leap/mail/smtp/__init__.py b/src/leap/mail/smtp/__init__.py index f740f5e..72b26ed 100644 --- a/src/leap/mail/smtp/__init__.py +++ b/src/leap/mail/smtp/__init__.py @@ -22,7 +22,6 @@ import logging from twisted.internet import reactor from twisted.internet.error import CannotListenError -from twisted.mail import smtp from leap.mail.service import OutgoingMail logger = logging.getLogger(__name__) @@ -42,7 +41,7 @@ def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port, :param port: The port in which to run the server. :type port: int :param userid: The user currently logged in - :type userid: unicode + :type userid: str :param keymanager: A Key Manager from where to get recipients' public keys. :type keymanager: leap.common.keymanager.KeyManager @@ -61,7 +60,8 @@ def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port, :returns: tuple of SMTPFactory, twisted.internet.tcp.Port """ # configure the use of this service with twistd - outgoing_mail = OutgoingMail(str(userid), keymanager, smtp_cert, smtp_key, smtp_host, smtp_port) + outgoing_mail = OutgoingMail( + userid, keymanager, smtp_cert, smtp_key, smtp_host, smtp_port) factory = SMTPFactory(userid, keymanager, encrypted_only, outgoing_mail) try: tport = reactor.listenTCP(port, factory, interface="localhost") -- cgit v1.2.3 From 87433821ad3ade5440814ffe9950e04c211a7a4c Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Fri, 2 Jan 2015 21:05:22 -0600 Subject: Port `enum` to `enum34` --- changes/bug-6601_port_enum34 | 1 + pkg/requirements.pip | 2 +- src/leap/mail/imap/memorystore.py | 10 +++++----- src/leap/mail/imap/messageparts.py | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 changes/bug-6601_port_enum34 diff --git a/changes/bug-6601_port_enum34 b/changes/bug-6601_port_enum34 new file mode 100644 index 0000000..2ca551d --- /dev/null +++ b/changes/bug-6601_port_enum34 @@ -0,0 +1 @@ +- Port `enum` to `enum34` (Closes #6601) diff --git a/pkg/requirements.pip b/pkg/requirements.pip index 17ceba6..5bd4972 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -4,4 +4,4 @@ leap.common>=0.3.7 leap.keymanager>=0.3.8 twisted # >= 12.0.3 ?? zope.proxy -enum +enum34 diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py index 5eea4ef..e075394 100644 --- a/src/leap/mail/imap/memorystore.py +++ b/src/leap/mail/imap/memorystore.py @@ -52,10 +52,10 @@ logger = logging.getLogger(__name__) # soledad storage, in seconds. SOLEDAD_WRITE_PERIOD = 15 -FDOC = MessagePartType.fdoc.key -HDOC = MessagePartType.hdoc.key -CDOCS = MessagePartType.cdocs.key -DOCS_ID = MessagePartType.docs_id.key +FDOC = MessagePartType.fdoc.name +HDOC = MessagePartType.hdoc.name +CDOCS = MessagePartType.cdocs.name +DOCS_ID = MessagePartType.docs_id.name @contextlib.contextmanager @@ -73,7 +73,7 @@ def set_bool_flag(obj, att): setattr(obj, att, False) -DirtyState = Enum("none", "dirty", "new") +DirtyState = Enum("DirtyState", "none dirty new") class MemoryStore(object): diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py index 257721c..fb1d75a 100644 --- a/src/leap/mail/imap/messageparts.py +++ b/src/leap/mail/imap/messageparts.py @@ -32,7 +32,7 @@ from leap.mail.imap import interfaces from leap.mail.imap.fields import fields from leap.mail.utils import empty, first, find_charset -MessagePartType = Enum("hdoc", "fdoc", "cdoc", "cdocs", "docs_id") +MessagePartType = Enum("MessagePartType", "hdoc fdoc cdoc cdocs docs_id") logger = logging.getLogger(__name__) -- cgit v1.2.3 From af7f5779fcc8755a95a1530efe5b833b7b60d040 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 15 Jan 2015 16:31:22 -0400 Subject: remove enum dep --- pkg/requirements.pip | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/requirements.pip b/pkg/requirements.pip index 5bd4972..64ff28c 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -4,4 +4,3 @@ leap.common>=0.3.7 leap.keymanager>=0.3.8 twisted # >= 12.0.3 ?? zope.proxy -enum34 -- cgit v1.2.3 From 891e9e048ee5cc5f72ce065354ff36eb92a17568 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 16 Jan 2015 19:16:38 -0400 Subject: bump version compat for soledad client --- changes/VERSION_COMPAT | 1 + 1 file changed, 1 insertion(+) diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT index 1eadcbe..12822ac 100644 --- a/changes/VERSION_COMPAT +++ b/changes/VERSION_COMPAT @@ -9,3 +9,4 @@ # BEGIN DEPENDENCY LIST ------------------------- # leap.foo.bar>=x.y.z leap.keymanager>=0.4.0 +leap.soledad.client>=0.7.0 -- cgit v1.2.3 From 70a88ba3a8761da2d2f7f9f5fb9d044d88de98a5 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 16 Jan 2015 19:19:53 -0400 Subject: add service-identity as a dependency for leap.mail --- pkg/requirements.pip | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/requirements.pip b/pkg/requirements.pip index 64ff28c..20f93a6 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -4,3 +4,4 @@ leap.common>=0.3.7 leap.keymanager>=0.3.8 twisted # >= 12.0.3 ?? zope.proxy +service-identity -- cgit v1.2.3 From 85e5b4d68b328eb06a52581dee300ebafc1cc763 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 16 Oct 2014 14:41:42 +0200 Subject: specify not syncable shared db --- src/leap/mail/imap/tests/utils.py | 5 +++-- src/leap/mail/tests/__init__.py | 20 ++++++++------------ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index 0932bd4..5339acf 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -65,7 +65,7 @@ def initialize_soledad(email, gnupg_home, tempdir): passphrase = u"verysecretpassphrase" secret_path = os.path.join(tempdir, "secret.gpg") local_db_path = os.path.join(tempdir, "soledad.u1db") - server_url = "http://provider" + server_url = "https://provider" cert_file = "" class MockSharedDB(object): @@ -86,7 +86,8 @@ def initialize_soledad(email, gnupg_home, tempdir): secret_path, local_db_path, server_url, - cert_file) + cert_file, + syncable=False) return _soledad diff --git a/src/leap/mail/tests/__init__.py b/src/leap/mail/tests/__init__.py index dc24293..10bc5fe 100644 --- a/src/leap/mail/tests/__init__.py +++ b/src/leap/mail/tests/__init__.py @@ -14,12 +14,9 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - - """ -Base classes and keys for SMTP gateway tests. +Base classes and keys for leap.mail tests. """ - import os import distutils.spawn import shutil @@ -27,9 +24,6 @@ import tempfile from mock import Mock -from twisted.trial import unittest - - from leap.soledad.client import Soledad from leap.keymanager import ( KeyManager, @@ -43,7 +37,7 @@ from leap.common.testing.basetest import BaseLeapTest def _find_gpg(): gpg_path = distutils.spawn.find_executable('gpg') return os.path.realpath(gpg_path) if gpg_path is not None else "/usr/bin/gpg" - + class TestCaseWithKeyManager(BaseLeapTest): @@ -100,23 +94,25 @@ class TestCaseWithKeyManager(BaseLeapTest): def __call__(self): return self - Soledad._shared_db = MockSharedDB() - - return Soledad( + soledad = Soledad( uuid, passphrase, secrets_path=secrets_path, local_db_path=local_db_path, server_url=server_url, cert_file=cert_file, + syncable=False ) + soledad._shared_db = MockSharedDB() + return soledad + def _keymanager_instance(self, address): """ Return a Key Manager instance for tests. """ self._config = { - 'host': 'http://provider/', + 'host': 'https://provider/', 'port': 25, 'username': address, 'password': '', -- cgit v1.2.3 From aa619124b9293b12d732e275c7f57b11dcb01f60 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 23 Dec 2014 10:21:06 -0400 Subject: fix typo in docs --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 4801833..d8634ea 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,7 +6,7 @@ Welcome to leap.mail's documentation! ===================================== -This is the documentation for the ``leap.imap`` module. It is a twisted package +This is the documentation for the ``leap.mail`` module. It is a twisted package that exposes two services, ``smtp`` and ``imap``, that run local proxies and interact with a remote ``LEAP`` provider that offers *a soledad syncronization endpoint* and receive the outgoing email. -- cgit v1.2.3 From 80ed7b9b85686b5d10f359114ca703dba4a5820b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 16 Oct 2014 14:51:53 +0200 Subject: adapt to soledad 0.7 async API --- src/leap/mail/imap/account.py | 253 +++++++++++++++++++++++++--------- src/leap/mail/imap/index.py | 51 +++++-- src/leap/mail/imap/mailbox.py | 57 ++++++-- src/leap/mail/imap/memorystore.py | 39 +++--- src/leap/mail/imap/messages.py | 136 ++++++++++-------- src/leap/mail/imap/server.py | 204 +++++++++++++++++++++++++-- src/leap/mail/imap/soledadstore.py | 11 +- src/leap/mail/imap/tests/test_imap.py | 185 ++++++++++++++++--------- src/leap/mail/imap/tests/utils.py | 25 ++-- 9 files changed, 698 insertions(+), 263 deletions(-) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 70ed13b..fe466cb 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -22,6 +22,7 @@ import logging import os import time +from twisted.internet import defer from twisted.mail import imap4 from twisted.python import log from zope.interface import implements @@ -65,6 +66,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): _soledad = None selected = None closed = False + _initialized = False def __init__(self, account_name, soledad, memstore=None): """ @@ -93,14 +95,39 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): self.__mailboxes = set([]) - self.initialize_db() + self._deferred_initialization = defer.Deferred() + self._initialize_storage() - # every user should have the right to an inbox folder - # at least, so let's make one! - self._load_mailboxes() + def _initialize_storage(self): - if not self.mailboxes: - self.addMailbox(self.INBOX_NAME) + def add_mailbox_if_none(result): + # every user should have the right to an inbox folder + # at least, so let's make one! + if not self.mailboxes: + self.addMailbox(self.INBOX_NAME) + + def finish_initialization(result): + self._initialized = True + self._deferred_initialization.callback(None) + + def load_mbox_cache(result): + d = self._load_mailboxes() + d.addCallback(lambda _: result) + return d + + d = self.initialize_db() + + d.addCallback(load_mbox_cache) + d.addCallback(add_mailbox_if_none) + d.addCallback(finish_initialization) + + def callWhenReady(self, cb): + if self._initialized: + cb(self) + return defer.succeed(None) + else: + self._deferred_initialization.addCallback(cb) + return self._deferred_initialization def _get_empty_mailbox(self): """ @@ -120,10 +147,14 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :rtype: SoledadDocument """ # XXX use soledadstore instead ...; - doc = self._soledad.get_from_index( + def get_first_if_any(docs): + return docs[0] if docs else None + + d = self._soledad.get_from_index( self.TYPE_MBOX_IDX, self.MBOX_KEY, self._parse_mailbox_name(name)) - return doc[0] if doc else None + d.addCallback(get_first_if_any) + return d @property def mailboxes(self): @@ -134,19 +165,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): return sorted(self.__mailboxes) def _load_mailboxes(self): - self.__mailboxes.update( - [doc.content[self.MBOX_KEY] - for doc in self._soledad.get_from_index( - self.TYPE_IDX, self.MBOX_KEY)]) - - @property - def subscriptions(self): - """ - A list of the current subscriptions for this account. - """ - return [doc.content[self.MBOX_KEY] - for doc in self._soledad.get_from_index( - self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')] + def update_mailboxes(db_indexes): + self.__mailboxes.update( + [doc.content[self.MBOX_KEY] for doc in db_indexes]) + d = self._soledad.get_from_index(self.TYPE_IDX, self.MBOX_KEY) + d.addCallback(update_mailboxes) + return d def getMailbox(self, name): """ @@ -182,7 +206,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): one is provided. :type creation_ts: int - :returns: True if successful + :returns: a Deferred that will contain the document if successful. :rtype: bool """ name = self._parse_mailbox_name(name) @@ -203,21 +227,29 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): mbox[self.MBOX_KEY] = name mbox[self.CREATED_KEY] = creation_ts - doc = self._soledad.create_doc(mbox) - self._load_mailboxes() - return bool(doc) + def load_mbox_cache(result): + d = self._load_mailboxes() + d.addCallback(lambda _: result) + return d + + d = self._soledad.create_doc(mbox) + d.addCallback(load_mbox_cache) + return d def create(self, pathspec): """ Create a new mailbox from the given hierarchical name. - :param pathspec: The full hierarchical name of a new mailbox to create. - If any of the inferior hierarchical names to this one - do not exist, they are created as well. + :param pathspec: + The full hierarchical name of a new mailbox to create. + If any of the inferior hierarchical names to this one + do not exist, they are created as well. :type pathspec: str - :return: A true value if the creation succeeds. - :rtype: bool + :return: + A deferred that will fire with a true value if the creation + succeeds. + :rtype: Deferred :raise MailboxException: Raised if this mailbox cannot be added. """ @@ -225,18 +257,43 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): paths = filter( None, self._parse_mailbox_name(pathspec).split('/')) + + subs = [] + sep = '/' + for accum in range(1, len(paths)): try: - self.addMailbox('/'.join(paths[:accum])) + partial = sep.join(paths[:accum]) + d = self.addMailbox(partial) + subs.append(d) except imap4.MailboxCollision: pass try: - self.addMailbox('/'.join(paths)) + df = self.addMailbox(sep.join(paths)) except imap4.MailboxCollision: if not pathspec.endswith('/'): - return False - self._load_mailboxes() - return True + df = defer.succeed(False) + else: + df = defer.succeed(True) + finally: + subs.append(df) + + def all_good(result): + return all(result) + + def load_mbox_cache(result): + d = self._load_mailboxes() + d.addCallback(lambda _: result) + return d + + if subs: + d1 = defer.gatherResults(subs, consumeErrors=True) + d1.addCallback(load_mbox_cache) + d1.addCallback(all_good) + else: + d1 = defer.succeed(False) + d1.addCallback(load_mbox_cache) + return d1 def select(self, name, readwrite=1): """ @@ -275,17 +332,20 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param name: the mailbox to be deleted :type name: str - :param force: if True, it will not check for noselect flag or inferior - names. use with care. + :param force: + if True, it will not check for noselect flag or inferior + names. use with care. :type force: bool + :rtype: Deferred """ name = self._parse_mailbox_name(name) if name not in self.mailboxes: - raise imap4.MailboxException("No such mailbox: %r" % name) + err = imap4.MailboxException("No such mailbox: %r" % name) + return defer.fail(err) mbox = self.getMailbox(name) - if force is False: + if not force: # See if this box is flagged \Noselect # XXX use mbox.flags instead? mbox_flags = mbox.getFlags() @@ -294,11 +354,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): # as part of their root. for others in self.mailboxes: if others != name and others.startswith(name): - raise imap4.MailboxException, ( + err = imap4.MailboxException( "Hierarchically inferior mailboxes " "exist and \\Noselect is set") + return defer.fail(err) self.__mailboxes.discard(name) - mbox.destroy() + return mbox.destroy() # XXX FIXME --- not honoring the inferior names... @@ -331,14 +392,30 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): if new in self.mailboxes: raise imap4.MailboxCollision(repr(new)) + rename_deferreds = [] + + def load_mbox_cache(result): + d = self._load_mailboxes() + d.addCallback(lambda _: result) + return d + + def update_mbox_doc_name(mbox, oldname, newname, update_deferred): + mbox.content[self.MBOX_KEY] = newname + d = self._soledad.put_doc(mbox) + d.addCallback(lambda r: update_deferred.callback(True)) + for (old, new) in inferiors: - self._memstore.rename_fdocs_mailbox(old, new) - mbox = self._get_mailbox_by_name(old) - mbox.content[self.MBOX_KEY] = new self.__mailboxes.discard(old) - self._soledad.put_doc(mbox) + self._memstore.rename_fdocs_mailbox(old, new) + + d0 = defer.Deferred() + d = self._get_mailbox_by_name(old) + d.addCallback(update_mbox_doc_name, old, new, d0) + rename_deferreds.append(d0) - self._load_mailboxes() + d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) + d1.addCallback(load_mbox_cache) + return d1 def _inferiorNames(self, name): """ @@ -354,6 +431,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): inferiors.append(infname) return inferiors + # TODO ------------------ can we preserve the attr? + # maybe add to memory store. def isSubscribed(self, name): """ Returns True if user is subscribed to this mailbox. @@ -361,10 +440,35 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param name: the mailbox to be checked. :type name: str - :rtype: bool + :rtype: Deferred (will fire with bool) + """ + subscribed = self.SUBSCRIBED_KEY + + def is_subscribed(mbox): + subs_bool = bool(mbox.content.get(subscribed, False)) + return subs_bool + + d = self._get_mailbox_by_name(name) + d.addCallback(is_subscribed) + return d + + # TODO ------------------ can we preserve the property? + # maybe add to memory store. + + def _get_subscriptions(self): """ - mbox = self._get_mailbox_by_name(name) - return mbox.content.get('subscribed', False) + Return a list of the current subscriptions for this account. + + :returns: A deferred that will fire with the subscriptions. + :rtype: Deferred + """ + def get_docs_content(docs): + return [doc.content[self.MBOX_KEY] for doc in docs] + + d = self._soledad.get_from_index( + self.TYPE_SUBS_IDX, self.MBOX_KEY, '1') + d.addCallback(get_docs_content) + return d def _set_subscription(self, name, value): """ @@ -376,26 +480,42 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param value: the boolean value :type value: bool """ + # XXX Note that this kind of operation has + # no guarantees of atomicity. We should not be accessing mbox + # documents concurrently. + + subscribed = self.SUBSCRIBED_KEY + + def update_subscribed_value(mbox): + mbox.content[subscribed] = value + return self._soledad.put_doc(mbox) + # maybe we should store subscriptions in another # document... if name not in self.mailboxes: - self.addMailbox(name) - mbox = self._get_mailbox_by_name(name) - - if mbox: - mbox.content[self.SUBSCRIBED_KEY] = value - self._soledad.put_doc(mbox) + d = self.addMailbox(name) + d.addCallback(lambda v: self._get_mailbox_by_name(name)) + else: + d = self._get_mailbox_by_name(name) + d.addCallback(update_subscribed_value) + return d def subscribe(self, name): """ - Subscribe to this mailbox + Subscribe to this mailbox if not already subscribed. :param name: name of the mailbox :type name: str + :rtype: Deferred """ name = self._parse_mailbox_name(name) - if name not in self.subscriptions: - self._set_subscription(name, True) + + def check_and_subscribe(subscriptions): + if name not in subscriptions: + return self._set_subscription(name, True) + d = self._get_subscriptions() + d.addCallback(check_and_subscribe) + return d def unsubscribe(self, name): """ @@ -403,12 +523,21 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param name: name of the mailbox :type name: str + :rtype: Deferred """ name = self._parse_mailbox_name(name) - if name not in self.subscriptions: - raise imap4.MailboxException( - "Not currently subscribed to %r" % name) - self._set_subscription(name, False) + + def check_and_unsubscribe(subscriptions): + if name not in subscriptions: + raise imap4.MailboxException( + "Not currently subscribed to %r" % name) + return self._set_subscription(name, False) + d = self._get_subscriptions() + d.addCallback(check_and_unsubscribe) + return d + + def getSubscriptions(self): + return self._get_subscriptions() def listMailboxes(self, ref, wildcard): """ diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py index 5f0919a..ea35fff 100644 --- a/src/leap/mail/imap/index.py +++ b/src/leap/mail/imap/index.py @@ -19,6 +19,8 @@ Index for SoledadBackedAccount, Mailbox and Messages. """ import logging +from twisted.internet import defer + from leap.common.check import leap_assert, leap_assert_type from leap.mail.imap.fields import fields @@ -39,6 +41,9 @@ class IndexedDB(object): """ # TODO we might want to move this to soledad itself, check + _index_creation_deferreds = [] + index_ready = False + def initialize_db(self): """ Initialize the database. @@ -46,24 +51,40 @@ class IndexedDB(object): leap_assert(self._soledad, "Need a soledad attribute accesible in the instance") leap_assert_type(self.INDEXES, dict) + self._index_creation_deferreds = [] + + def _on_indexes_created(ignored): + self.index_ready = True + + def _create_index(name, expression): + d = self._soledad.create_index(name, *expression) + self._index_creation_deferreds.append(d) + + def _create_indexes(db_indexes): + db_indexes = dict(db_indexes) + for name, expression in fields.INDEXES.items(): + if name not in db_indexes: + # The index does not yet exist. + _create_index(name, expression) + continue + + if expression == db_indexes[name]: + # The index exists and is up to date. + continue + # The index exists but the definition is not what expected, so + # we delete it and add the proper index expression. + d1 = self._soledad.delete_index(name) + d1.addCallback(lambda _: _create_index(name, expression)) + + all_created = defer.gatherResults(self._index_creation_deferreds) + all_created.addCallback(_on_indexes_created) + return all_created # Ask the database for currently existing indexes. if not self._soledad: logger.debug("NO SOLEDAD ON IMAP INITIALIZATION") return - db_indexes = dict() if self._soledad is not None: - db_indexes = dict(self._soledad.list_indexes()) - for name, expression in fields.INDEXES.items(): - if name not in db_indexes: - # The index does not yet exist. - self._soledad.create_index(name, *expression) - continue - - if expression == db_indexes[name]: - # The index exists and is up to date. - continue - # The index exists but the definition is not what expected, so we - # delete it and add the proper index expression. - self._soledad.delete_index(name) - self._soledad.create_index(name, *expression) + d = self._soledad.list_indexes() + d.addCallback(_create_indexes) + return d diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 34cf535..3c1769a 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -303,21 +303,32 @@ class SoledadMailbox(WithMsgFields, MBoxParser): We do this to be able to filter the requests efficiently. """ primed = self._known_uids_primed.get(self.mbox, False) - if not primed: - known_uids = self.messages.all_soledad_uid_iter() + # XXX handle the maybeDeferred + + def set_primed(known_uids): self._memstore.set_known_uids(self.mbox, known_uids) self._known_uids_primed[self.mbox] = True + if not primed: + d = self.messages.all_soledad_uid_iter() + d.addCallback(set_primed) + return d + def prime_flag_docs_to_memstore(self): """ Prime memstore with all the flags documents. """ primed = self._fdoc_primed.get(self.mbox, False) - if not primed: - all_flag_docs = self.messages.get_all_soledad_flag_docs() - self._memstore.load_flag_docs(self.mbox, all_flag_docs) + + def set_flag_docs(flag_docs): + self._memstore.load_flag_docs(self.mbox, flag_docs) self._fdoc_primed[self.mbox] = True + if not primed: + d = self.messages.get_all_soledad_flag_docs() + d.addCallback(set_flag_docs) + return d + def getUIDValidity(self): """ Return the unique validity identifier for this mailbox. @@ -522,21 +533,30 @@ class SoledadMailbox(WithMsgFields, MBoxParser): Should cleanup resources, and set the \\Noselect flag on the mailbox. + """ # XXX this will overwrite all the existing flags! # should better simply addFlag self.setFlags((self.NOSELECT_FLAG,)) - self.deleteAllDocs() # XXX removing the mailbox in situ for now, # we should postpone the removal - # XXX move to memory store?? - mbox_doc = self._get_mbox_doc() - if mbox_doc is None: - # memory-only store! - return - self._soledad.delete_doc(self._get_mbox_doc()) + def remove_mbox_doc(ignored): + # XXX move to memory store?? + + def _remove_mbox_doc(doc): + if doc is None: + # memory-only store! + return defer.succeed(True) + return self._soledad.delete_doc(doc) + + doc = self._get_mbox_doc() + return _remove_mbox_doc(doc) + + d = self.deleteAllDocs() + d.addCallback(remove_mbox_doc) + return d def _close_cb(self, result): self.closed = True @@ -1006,9 +1026,16 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ Delete all docs in this mailbox """ - docs = self.messages.get_all_docs() - for doc in docs: - self.messages._soledad.delete_doc(doc) + def del_all_docs(docs): + todelete = [] + for doc in docs: + d = self.messages._soledad.delete_doc(doc) + todelete.append(d) + return defer.gatherResults(todelete) + + d = self.messages.get_all_docs() + d.addCallback(del_all_docs) + return d def unset_recent_flags(self, uid_seq): """ diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py index e075394..eda5b96 100644 --- a/src/leap/mail/imap/memorystore.py +++ b/src/leap/mail/imap/memorystore.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # memorystore.py # Copyright (C) 2014 LEAP # @@ -112,8 +112,6 @@ class MemoryStore(object): :param write_period: the interval to dump messages to disk, in seconds. :type write_period: int """ - self.reactor = reactor - self._permanent_store = permanent_store self._write_period = write_period @@ -241,6 +239,7 @@ class MemoryStore(object): self.producer = None self._write_loop = None + # TODO -- remove def _start_write_loop(self): """ Start loop for writing to disk database. @@ -250,6 +249,7 @@ class MemoryStore(object): if not self._write_loop.running: self._write_loop.start(self._write_period, now=True) + # TODO -- remove def _stop_write_loop(self): """ Stop loop for writing to disk database. @@ -278,17 +278,18 @@ class MemoryStore(object): :type uid: int :param message: a message to be added :type message: MessageWrapper - :param observer: the deferred that will fire with the - UID of the message. If notify_on_disk is True, - this will happen when the message is written to - Soledad. Otherwise it will fire as soon as we've - added the message to the memory store. + :param observer: + the deferred that will fire with the UID of the message. If + notify_on_disk is True, this will happen when the message is + written to Soledad. Otherwise it will fire as soon as we've added + the message to the memory store. :type observer: Deferred - :param notify_on_disk: whether the `observer` deferred should - wait until the message is written to disk to - be fired. + :param notify_on_disk: + whether the `observer` deferred should wait until the message is + written to disk to be fired. :type notify_on_disk: bool """ + # TODO -- return a deferred log.msg("Adding new doc to memstore %r (%r)" % (mbox, uid)) key = mbox, uid @@ -306,7 +307,7 @@ class MemoryStore(object): else: # Caller does not care, just fired and forgot, so we pass # a defer that will inmediately have its callback triggered. - self.reactor.callFromThread(observer.callback, uid) + reactor.callFromThread(observer.callback, uid) def put_message(self, mbox, uid, message, notify_on_disk=True): """ @@ -442,6 +443,7 @@ class MemoryStore(object): :return: MessageWrapper or None """ + # TODO -- return deferred if dirtystate == DirtyState.dirty: flags_only = True @@ -467,6 +469,7 @@ class MemoryStore(object): chash = fdoc.get(fields.CONTENT_HASH_KEY) hdoc = self._hdoc_store[chash] if empty(hdoc): + # XXX this will be a deferred hdoc = self._permanent_store.get_headers_doc(chash) if empty(hdoc): return None @@ -531,7 +534,8 @@ class MemoryStore(object): # IMessageStoreWriter - @deferred_to_thread + # TODO -- I think we don't need this anymore. + # instead, we can have def write_messages(self, store): """ Write the message documents in this MemoryStore to a different store. @@ -657,7 +661,7 @@ class MemoryStore(object): with self._last_uid_lock: self._last_uid[mbox] += 1 value = self._last_uid[mbox] - self.reactor.callInThread(self.write_last_uid, mbox, value) + reactor.callInThread(self.write_last_uid, mbox, value) return value def write_last_uid(self, mbox, value): @@ -1077,6 +1081,7 @@ class MemoryStore(object): return None return self._rflags_store[mbox]['set'] + # XXX -- remove def all_rdocs_iter(self): """ Return an iterator through all in-memory recent flag dicts, wrapped @@ -1125,6 +1130,7 @@ class MemoryStore(object): self.remove_message(mbox, uid) return mem_deleted + # TODO -- remove def stop_and_flush(self): """ Stop the write loop and trigger a write to the producer. @@ -1180,6 +1186,7 @@ class MemoryStore(object): :type observer: Deferred """ mem_deleted = self.remove_all_deleted(mbox) + # TODO return a DeferredList observer.callback(mem_deleted) def _delete_from_soledad_and_memory(self, result, mbox, observer): @@ -1313,8 +1320,8 @@ class MemoryStore(object): :rtype: bool """ # FIXME this should return a deferred !!! - # XXX ----- can fire when all new + dirty deferreds - # are done (gatherResults) + # TODO this should be moved to soledadStore instead + # (all pending deferreds) return getattr(self, self.WRITING_FLAG) @property diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index e8d64d1..c761091 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -71,6 +71,7 @@ def try_unique_query(curried): :param curried: a curried function :type curried: callable """ + # XXX FIXME ---------- convert to deferreds leap_assert(callable(curried), "A callable is expected") try: query = curried() @@ -134,10 +135,11 @@ class LeapMessage(fields, MBoxParser): self.__chash = None self.__bdoc = None - self.reactor = reactor - # XXX make these properties public + # XXX FIXME ------ the documents can be + # deferreds too.... niice. + @property def fdoc(self): """ @@ -506,18 +508,15 @@ class LeapMessage(fields, MBoxParser): Return the document that keeps the flags for this message. """ - result = {} - try: - flag_docs = self._soledad.get_from_index( - fields.TYPE_MBOX_UID_IDX, - fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) - result = first(flag_docs) - except Exception as exc: - # ugh! Something's broken down there! - logger.warning("ERROR while getting flags for UID: %s" % self._uid) - logger.exception(exc) - finally: - return result + def get_first_if_any(docs): + result = first(docs) + return result if result else {} + + d = self._soledad.get_from_index( + fields.TYPE_MBOX_UID_IDX, + fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) + d.addCallback(get_first_if_any) + return d # TODO move to soledadstore instead of accessing soledad directly def _get_headers_doc(self): @@ -525,10 +524,11 @@ class LeapMessage(fields, MBoxParser): Return the document that keeps the headers for this message. """ - head_docs = self._soledad.get_from_index( + d = self._soledad.get_from_index( fields.TYPE_C_HASH_IDX, fields.TYPE_HEADERS_VAL, str(self.chash)) - return first(head_docs) + d.addCallback(lambda docs: first(docs)) + return d # TODO move to soledadstore instead of accessing soledad directly def _get_body_doc(self): @@ -536,6 +536,8 @@ class LeapMessage(fields, MBoxParser): Return the document that keeps the body for this message. """ + # XXX FIXME --- this might need a maybedeferred + # on the receiving side... hdoc_content = self.hdoc.content body_phash = hdoc_content.get( fields.BODY_KEY, None) @@ -554,13 +556,11 @@ class LeapMessage(fields, MBoxParser): return bdoc # no memstore, or no body doc found there - if self._soledad: - body_docs = self._soledad.get_from_index( - fields.TYPE_P_HASH_IDX, - fields.TYPE_CONTENT_VAL, str(body_phash)) - return first(body_docs) - else: - logger.error("No phash in container, and no soledad found!") + d = self._soledad.get_from_index( + fields.TYPE_P_HASH_IDX, + fields.TYPE_CONTENT_VAL, str(body_phash)) + d.addCallback(lambda docs: first(docs)) + return d def __getitem__(self, key): """ @@ -739,8 +739,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): else: self._initialized[mbox] = True - self.reactor = reactor - def _get_empty_doc(self, _type=FLAGS_DOC): """ Returns an empty doc for storing different message parts. @@ -887,9 +885,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): flags = tuple() leap_assert_type(flags, tuple) + # TODO return soledad deferred instead observer = defer.Deferred() d = self._do_parse(raw) - d.addCallback(lambda result: self.reactor.callInThread( + d.addCallback(lambda result: reactor.callInThread( self._do_add_msg, result, flags, subject, date, notify_on_disk, observer)) return observer @@ -924,17 +923,18 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): msg = self.get_msg_by_uid(existing_uid) # We can say the observer that we're done - self.reactor.callFromThread(observer.callback, existing_uid) + # TODO return soledad deferred instead + reactor.callFromThread(observer.callback, existing_uid) msg.setFlags((fields.DELETED_FLAG,), -1) return - # XXX get FUCKING UID from autoincremental table + # TODO S2 -- get FUCKING UID from autoincremental table uid = self.memstore.increment_last_soledad_uid(self.mbox) # We can say the observer that we're done at this point, but # before that we should make sure it has no serious consequences # if we're issued, for instance, a fetch command right after... - # self.reactor.callFromThread(observer.callback, uid) + # reactor.callFromThread(observer.callback, uid) # if we did the notify, we need to invalidate the deferred # so not to try to fire it twice. # observer = None @@ -960,6 +960,8 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): self.set_recent_flag(uid) msg_container = MessageWrapper(fd, hd, cdocs) + + # TODO S1 -- just pass this to memstore and return that deferred. self.memstore.create_message( self.mbox, uid, msg_container, observer=observer, notify_on_disk=notify_on_disk) @@ -1011,6 +1013,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): Get recent-flags document from Soledad for this mailbox. :rtype: SoledadDocument or None """ + # FIXME ----- use deferreds. curried = partial( self._soledad.get_from_index, fields.TYPE_MBOX_IDX, @@ -1029,6 +1032,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :param uids: the uids to unset :type uid: sequence """ + # FIXME ----- use deferreds. with self._rdoc_property_lock[self.mbox]: self.recent_flags.difference_update( set(uids)) @@ -1042,11 +1046,11 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :param uid: the uid to unset :type uid: int """ + # FIXME ----- use deferreds. with self._rdoc_property_lock[self.mbox]: self.recent_flags.difference_update( set([uid])) - @deferred_to_thread def set_recent_flag(self, uid): """ Set Recent flag for a given uid. @@ -1054,6 +1058,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :param uid: the uid to set :type uid: int """ + # FIXME ----- use deferreds. with self._rdoc_property_lock[self.mbox]: self.recent_flags = self.recent_flags.union( set([uid])) @@ -1068,6 +1073,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): the query failed. :rtype: SoledadDocument or None. """ + # FIXME ----- use deferreds. curried = partial( self._soledad.get_from_index, fields.TYPE_MBOX_C_HASH_IDX, @@ -1125,7 +1131,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): return None return fdoc.content.get(fields.UID_KEY, None) - @deferred_to_thread def _get_uid_from_msgid(self, msgid): """ Return a UID for a given message-id. @@ -1144,7 +1149,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): # XXX is this working? return self._get_uid_from_msgidCb(msgid) - @deferred_to_thread def set_flags(self, mbox, messages, flags, mode, observer): """ Set flags for a sequence of messages. @@ -1162,7 +1166,6 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): done. :type observer: deferred """ - reactor = self.reactor getmsg = self.get_msg_by_uid def set_flags(uid, flags, mode): @@ -1173,6 +1176,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): setted_flags = [set_flags(uid, flags, mode) for uid in messages] result = dict(filter(None, setted_flags)) + # TODO -- remove reactor.callFromThread(observer.callback, result) # getters: generic for a mailbox @@ -1223,37 +1227,45 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): If you want acess to the content, use __iter__ instead - :return: a list of u1db documents - :rtype: list of SoledadDocument + :return: a Deferred, that will fire with a list of u1db documents + :rtype: Deferred (promise of list of SoledadDocument) """ if _type not in fields.__dict__.values(): raise TypeError("Wrong type passed to get_all_docs") + # FIXME ----- either raise or return a deferred wrapper. if sameProxiedObjects(self._soledad, None): logger.warning('Tried to get messages but soledad is None!') return [] - all_docs = [doc for doc in self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - _type, self.mbox)] + def get_sorted_docs(docs): + all_docs = [doc for doc in docs] + # inneficient, but first let's grok it and then + # let's worry about efficiency. + # XXX FIXINDEX -- should implement order by in soledad + # FIXME ---------------------------------------------- + return sorted(all_docs, key=lambda item: item.content['uid']) - # inneficient, but first let's grok it and then - # let's worry about efficiency. - # XXX FIXINDEX -- should implement order by in soledad - # FIXME ---------------------------------------------- - return sorted(all_docs, key=lambda item: item.content['uid']) + d = self._soledad.get_from_index( + fields.TYPE_MBOX_IDX, _type, self.mbox) + d.addCallback(get_sorted_docs) + return d def all_soledad_uid_iter(self): """ Return an iterator through the UIDs of all messages, sorted in ascending order. """ - db_uids = set([doc.content[self.UID_KEY] for doc in - self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - fields.TYPE_FLAGS_VAL, self.mbox) - if not empty(doc)]) - return db_uids + # XXX FIXME ------ sorted??? + + def get_uids(docs): + return set([ + doc.content[self.UID_KEY] for doc in docs if not empty(doc)]) + + d = self._soledad.get_from_index( + fields.TYPE_MBOX_IDX, fields.TYPE_FLAGS_VAL, self.mbox) + d.addCallback(get_uids) + return d def all_uid_iter(self): """ @@ -1277,16 +1289,21 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): # XXX we really could return a reduced version with # just {'uid': (flags-tuple,) since the prefetch is # only oriented to get the flag tuples. - all_docs = [( - doc.content[self.UID_KEY], - dict(doc.content)) - for doc in - self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - fields.TYPE_FLAGS_VAL, self.mbox) - if not empty(doc.content)] - all_flags = dict(all_docs) - return all_flags + + def get_content(docs): + all_docs = [( + doc.content[self.UID_KEY], + dict(doc.content)) + for doc in docs + if not empty(doc.content)] + all_flags = dict(all_docs) + return all_flags + + d = self._soledad.get_from_index( + fields.TYPE_MBOX_IDX, + fields.TYPE_FLAGS_VAL, self.mbox) + d.addCallback(get_content) + return d def all_headers(self): """ @@ -1339,6 +1356,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): # recent messages # XXX take it from memstore + # XXX Used somewhere? def count_recent(self): """ Count all messages with the `Recent` flag. diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index fe56ea6..cf0ba74 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -20,6 +20,7 @@ Leap IMAP4 Server Implementation. from copy import copy from twisted import cred +from twisted.internet import reactor from twisted.internet.defer import maybeDeferred from twisted.mail import imap4 from twisted.python import log @@ -50,6 +51,7 @@ class LeapIMAPServer(imap4.IMAP4Server): leap_assert(uuid, "need a user in the initialization") self._userid = userid + self.reactor = reactor # initialize imap server! imap4.IMAP4Server.__init__(self, *args, **kwargs) @@ -59,9 +61,6 @@ class LeapIMAPServer(imap4.IMAP4Server): # populate the test account properly (and only once # per session) - from twisted.internet import reactor - self.reactor = reactor - def lineReceived(self, line): """ Attempt to parse a single line from the server. @@ -311,21 +310,203 @@ class LeapIMAPServer(imap4.IMAP4Server): return self._fileLiteral(size, literalPlus) ############################# + # --------------------------------- isSubscribed patch + # TODO -- send patch upstream. + # There is a bug in twisted implementation: + # in cbListWork, it's assumed that account.isSubscribed IS a callable, + # although in the interface documentation it's stated that it can be + # a deferred. + + def _listWork(self, tag, ref, mbox, sub, cmdName): + mbox = self._parseMbox(mbox) + mailboxes = maybeDeferred(self.account.listMailboxes, ref, mbox) + mailboxes.addCallback(self._cbSubscribed) + mailboxes.addCallback( + self._cbListWork, tag, sub, cmdName, + ).addErrback(self._ebListWork, tag) + + def _cbSubscribed(self, mailboxes): + subscribed = [ + maybeDeferred(self.account.isSubscribed, name) + for (name, box) in mailboxes] + + def get_mailboxes_and_subs(result): + subscribed = [i[0] for i, yes in zip(mailboxes, result) if yes] + return mailboxes, subscribed + + d = defer.gatherResults(subscribed) + d.addCallback(get_mailboxes_and_subs) + return d + + def _cbListWork(self, mailboxes_subscribed, tag, sub, cmdName): + mailboxes, subscribed = mailboxes_subscribed + + for (name, box) in mailboxes: + if not sub or name in subscribed: + flags = box.getFlags() + delim = box.getHierarchicalDelimiter() + resp = (imap4.DontQuoteMe(cmdName), + map(imap4.DontQuoteMe, flags), + delim, name.encode('imap4-utf-7')) + self.sendUntaggedResponse( + imap4.collapseNestedLists(resp)) + self.sendPositiveResponse(tag, '%s completed' % (cmdName,)) + # -------------------- end isSubscribed patch ----------- + + # TODO ---- + # subscribe method had also to be changed to accomodate + # deferred + # Revert to regular methods as soon as we implement non-deferred memory + # cache. + def do_SUBSCRIBE(self, tag, name): + name = self._parseMbox(name) + + def _subscribeCb(_): + self.sendPositiveResponse(tag, 'Subscribed') + + def _subscribeEb(failure): + m = failure.value + log.err() + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + self.sendBadResponse( + tag, + "Server error encountered while subscribing to mailbox") + + d = self.account.subscribe(name) + d.addCallbacks(_subscribeCb, _subscribeEb) + return d + + auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring) + select_SUBSCRIBE = auth_SUBSCRIBE + + def do_UNSUBSCRIBE(self, tag, name): + # unsubscribe method had also to be changed to accomodate + # deferred + name = self._parseMbox(name) + + def _unsubscribeCb(_): + self.sendPositiveResponse(tag, 'Unsubscribed') + + def _unsubscribeEb(failure): + m = failure.value + log.err() + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + self.sendBadResponse( + tag, + "Server error encountered while unsubscribing " + "from mailbox") + + d = self.account.unsubscribe(name) + d.addCallbacks(_unsubscribeCb, _unsubscribeEb) + return d + + auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring) + select_UNSUBSCRIBE = auth_UNSUBSCRIBE + + def do_RENAME(self, tag, oldname, newname): + oldname, newname = [self._parseMbox(n) for n in oldname, newname] + if oldname.lower() == 'inbox' or newname.lower() == 'inbox': + self.sendNegativeResponse( + tag, + 'You cannot rename the inbox, or ' + 'rename another mailbox to inbox.') + return + + def _renameCb(_): + self.sendPositiveResponse(tag, 'Mailbox renamed') + + def _renameEb(failure): + m = failure.value + print "rename failure!" + if failure.check(TypeError): + self.sendBadResponse(tag, 'Invalid command syntax') + elif failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + log.err() + self.sendBadResponse( + tag, + "Server error encountered while " + "renaming mailbox") + + d = self.account.rename(oldname, newname) + d.addCallbacks(_renameCb, _renameEb) + return d + + auth_RENAME = (do_RENAME, arg_astring, arg_astring) + select_RENAME = auth_RENAME + + def do_CREATE(self, tag, name): + name = self._parseMbox(name) + + def _createCb(result): + if result: + self.sendPositiveResponse(tag, 'Mailbox created') + else: + self.sendNegativeResponse(tag, 'Mailbox not created') + + def _createEb(failure): + c = failure.value + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(c)) + else: + log.err() + self.sendBadResponse( + tag, "Server error encountered while creating mailbox") + + d = self.account.create(name) + d.addCallbacks(_createCb, _createEb) + return d + + auth_CREATE = (do_CREATE, arg_astring) + select_CREATE = auth_CREATE + + def do_DELETE(self, tag, name): + name = self._parseMbox(name) + if name.lower() == 'inbox': + self.sendNegativeResponse(tag, 'You cannot delete the inbox') + return + + def _deleteCb(result): + self.sendPositiveResponse(tag, 'Mailbox deleted') + + def _deleteEb(failure): + m = failure.value + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + print "other error" + log.err() + self.sendBadResponse( + tag, + "Server error encountered while deleting mailbox") + + d = self.account.delete(name) + d.addCallbacks(_deleteCb, _deleteEb) + return d + + auth_DELETE = (do_DELETE, arg_astring) + select_DELETE = auth_DELETE + # Need to override the command table after patching # arg_astring and arg_literal + # do_DELETE = imap4.IMAP4Server.do_DELETE + # do_CREATE = imap4.IMAP4Server.do_CREATE + # do_RENAME = imap4.IMAP4Server.do_RENAME + # do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE + # do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE do_LOGIN = imap4.IMAP4Server.do_LOGIN - do_CREATE = imap4.IMAP4Server.do_CREATE - do_DELETE = imap4.IMAP4Server.do_DELETE - do_RENAME = imap4.IMAP4Server.do_RENAME - do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE - do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE do_STATUS = imap4.IMAP4Server.do_STATUS do_APPEND = imap4.IMAP4Server.do_APPEND do_COPY = imap4.IMAP4Server.do_COPY _selectWork = imap4.IMAP4Server._selectWork - _listWork = imap4.IMAP4Server._listWork + arg_plist = imap4.IMAP4Server.arg_plist arg_seqset = imap4.IMAP4Server.arg_seqset opt_plist = imap4.IMAP4Server.opt_plist @@ -342,8 +523,8 @@ class LeapIMAPServer(imap4.IMAP4Server): auth_EXAMINE = (_selectWork, arg_astring, 0, 'EXAMINE') select_EXAMINE = auth_EXAMINE - auth_DELETE = (do_DELETE, arg_astring) - select_DELETE = auth_DELETE + # auth_DELETE = (do_DELETE, arg_astring) + # select_DELETE = auth_DELETE auth_RENAME = (do_RENAME, arg_astring, arg_astring) select_RENAME = auth_RENAME @@ -369,7 +550,6 @@ class LeapIMAPServer(imap4.IMAP4Server): select_COPY = (do_COPY, arg_seqset, arg_astring) - ############################################################# # END of Twisted imap4 patch to support LITERAL+ extension ############################################################# diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py index f3de8eb..fc8ea55 100644 --- a/src/leap/mail/imap/soledadstore.py +++ b/src/leap/mail/imap/soledadstore.py @@ -40,11 +40,6 @@ from leap.mail.utils import first, empty, accumulator_queue logger = logging.getLogger(__name__) -# TODO -# [ ] Implement a retry queue? -# [ ] Consider journaling of operations. - - class ContentDedup(object): """ Message deduplication. @@ -132,6 +127,7 @@ A lock per document. # http://stackoverflow.com/a/2437645/1157664 # Setting this to twice the number of threads in the threadpool # should be safe. + put_locks = defaultdict(lambda: threading.Lock()) mbox_doc_locks = defaultdict(lambda: threading.Lock()) @@ -429,7 +425,6 @@ class SoledadStore(ContentDedup): continue if item.part == MessagePartType.fdoc: - #logger.debug("PUT dirty fdoc") yield item, call # XXX also for linkage-doc !!! @@ -479,7 +474,7 @@ class SoledadStore(ContentDedup): return query.pop() else: logger.error("Could not find mbox document for %r" % - (mbox,)) + (mbox,)) except Exception as exc: logger.exception("Unhandled error %r" % exc) @@ -552,8 +547,10 @@ class SoledadStore(ContentDedup): :type uid: int :rtype: SoledadDocument or None """ + # TODO -- inlineCallbacks result = None try: + # TODO -- yield flag_docs = self._soledad.get_from_index( fields.TYPE_MBOX_UID_IDX, fields.TYPE_FLAGS_VAL, mbox, str(uid)) diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 7837aaa..dd4294c 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -68,7 +68,6 @@ def sortNest(l): class TestRealm: - """ A minimal auth realm for testing purposes only """ @@ -83,7 +82,6 @@ class TestRealm: # class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): - """ Tests for the MessageCollection class """ @@ -254,14 +252,18 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.login(TEST_USER, TEST_PASSWD) def create(): + create_deferreds = [] for name in succeed + fail: d = self.client.create(name) d.addCallback(strip(cb)).addErrback(eb) - d.addCallbacks(self._cbStopClient, self._ebGeneral) + create_deferreds.append(d) + dd = defer.gatherResults(create_deferreds) + dd.addCallbacks(self._cbStopClient, self._ebGeneral) + return dd self.result = [] - d1 = self.connected.addCallback(strip(login)).addCallback( - strip(create)) + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(create)) d2 = self.loopback() d = defer.gatherResults([d1, d2]) return d.addCallback(self._cbTestCreate, succeed, fail) @@ -269,24 +271,27 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestCreate(self, ignored, succeed, fail): self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail)) - mboxes = list(LeapIMAPServer.theAccount.mailboxes) - answers = ([u'INBOX', u'foobox', 'test', u'test/box', - u'test/box/box', 'testbox']) - self.assertEqual(mboxes, [a for a in answers]) + mboxes = LeapIMAPServer.theAccount.mailboxes + + answers = ([u'INBOX', u'testbox', u'test/box', u'test', + u'test/box/box', 'foobox']) + self.assertEqual(sorted(mboxes), sorted([a for a in answers])) def testDelete(self): """ Test whether we can delete mailboxes """ - LeapIMAPServer.theAccount.addMailbox('delete/me') + acc = LeapIMAPServer.theAccount + d0 = lambda: acc.addMailbox('test-delete/me') def login(): return self.client.login(TEST_USER, TEST_PASSWD) def delete(): - return self.client.delete('delete/me') + return self.client.delete('test-delete/me') d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d0)) d1.addCallbacks(strip(delete), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -352,11 +357,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Try deleting a mailbox with sub-folders, and \NoSelect flag set. An exception is expected. """ - LeapIMAPServer.theAccount.addMailbox('delete') - to_delete = LeapIMAPServer.theAccount.getMailbox('delete') - to_delete.setFlags((r'\Noselect',)) - to_delete.getFlags() - LeapIMAPServer.theAccount.addMailbox('delete/me') + acc = LeapIMAPServer.theAccount + d_del0 = lambda: acc.addMailbox('delete') + d_del1 = lambda: acc.addMailbox('delete/me') + + def set_noselect_flag(): + mbox = acc.getMailbox('delete') + mbox.setFlags((r'\Noselect',)) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -369,6 +376,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.failure = None d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d_del0)) + d1.addCallback(strip(d_del1)) + d1.addCallback(strip(set_noselect_flag)) d1.addCallback(strip(delete)).addErrback(deleteFailed) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -385,7 +395,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test whether we can rename a mailbox """ - LeapIMAPServer.theAccount.addMailbox('oldmbox') + d0 = lambda: LeapIMAPServer.theAccount.addMailbox('oldmbox') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -394,6 +404,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.rename('oldmbox', 'newname') d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d0)) d1.addCallbacks(strip(rename), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -435,8 +446,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Try to rename hierarchical mailboxes """ - LeapIMAPServer.theAccount.create('oldmbox/m1') - LeapIMAPServer.theAccount.create('oldmbox/m2') + acc = LeapIMAPServer.theAccount + dc1 = lambda: acc.create('oldmbox/m1') + dc2 = lambda: acc.create('oldmbox/m2') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -445,6 +457,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.rename('oldmbox', 'newname') d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(dc1)) + d1.addCallback(strip(dc2)) d1.addCallbacks(strip(rename), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -454,7 +468,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def _cbTestHierarchicalRename(self, ignored): mboxes = LeapIMAPServer.theAccount.mailboxes expected = ['INBOX', 'newname', 'newname/m1', 'newname/m2'] - self.assertEqual(mboxes, [s for s in expected]) + self.assertEqual(sorted(mboxes), sorted([s for s in expected])) def testSubscribe(self): """ @@ -466,23 +480,28 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def subscribe(): return self.client.subscribe('this/mbox') + def get_subscriptions(ignored): + return LeapIMAPServer.theAccount.getSubscriptions() + d1 = self.connected.addCallback(strip(login)) d1.addCallbacks(strip(subscribe), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: - self.assertEqual( - LeapIMAPServer.theAccount.subscriptions, - ['this/mbox'])) + d.addCallback(get_subscriptions) + d.addCallback(lambda subscriptions: + self.assertEqual(subscriptions, + ['this/mbox'])) return d def testUnsubscribe(self): """ Test whether we can unsubscribe from a set of mailboxes """ - LeapIMAPServer.theAccount.subscribe('this/mbox') - LeapIMAPServer.theAccount.subscribe('that/mbox') + acc = LeapIMAPServer.theAccount + + dc1 = lambda: acc.subscribe('this/mbox') + dc2 = lambda: acc.subscribe('that/mbox') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -490,22 +509,28 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def unsubscribe(): return self.client.unsubscribe('this/mbox') + def get_subscriptions(ignored): + return LeapIMAPServer.theAccount.getSubscriptions() + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(dc1)) + d1.addCallback(strip(dc2)) d1.addCallbacks(strip(unsubscribe), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: - self.assertEqual( - LeapIMAPServer.theAccount.subscriptions, - ['that/mbox'])) + d.addCallback(get_subscriptions) + d.addCallback(lambda subscriptions: + self.assertEqual(subscriptions, + ['that/mbox'])) return d def testSelect(self): """ Try to select a mailbox """ - self.server.theAccount.addMailbox('TESTMAILBOX-SELECT', creation_ts=42) + acc = self.server.theAccount + d0 = lambda: acc.addMailbox('TESTMAILBOX-SELECT', creation_ts=42) self.selectedArgs = None def login(): @@ -520,6 +545,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d0)) d1.addCallback(strip(select)) d1.addErrback(self._ebGeneral) @@ -754,13 +780,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): '\\Deleted', '\\Draft', '\\Recent', 'List'), 'READ-WRITE': False}) - def _listSetup(self, f): - LeapIMAPServer.theAccount.addMailbox('root/subthingl', - creation_ts=42) - LeapIMAPServer.theAccount.addMailbox('root/another-thing', - creation_ts=42) - LeapIMAPServer.theAccount.addMailbox('non-root/subthing', - creation_ts=42) + def _listSetup(self, f, f2=None): + acc = LeapIMAPServer.theAccount + + dc1 = lambda: acc.addMailbox('root/subthing', creation_ts=42) + dc2 = lambda: acc.addMailbox('root/another-thing', creation_ts=42) + dc3 = lambda: acc.addMailbox('non-root/subthing', creation_ts=42) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -770,6 +795,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.listed = None d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(dc1)) + d1.addCallback(strip(dc2)) + d1.addCallback(strip(dc3)) + + if f2 is not None: + d1.addCallback(f2) + d1.addCallbacks(strip(f), self._ebGeneral) d1.addCallbacks(listed, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) @@ -786,7 +818,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d.addCallback(lambda listed: self.assertEqual( sortNest(listed), sortNest([ - (SoledadMailbox.INIT_FLAGS, "/", "root/subthingl"), + (SoledadMailbox.INIT_FLAGS, "/", "root/subthing"), (SoledadMailbox.INIT_FLAGS, "/", "root/another-thing") ]) )) @@ -796,20 +828,29 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test LSub command """ - LeapIMAPServer.theAccount.subscribe('root/subthingl2') + acc = LeapIMAPServer.theAccount + + def subs_mailbox(): + # why not client.subscribe instead? + return acc.subscribe('root/subthing') def lsub(): return self.client.lsub('root', '%') - d = self._listSetup(lsub) + + d = self._listSetup(lsub, strip(subs_mailbox)) d.addCallback(self.assertEqual, - [(SoledadMailbox.INIT_FLAGS, "/", "root/subthingl2")]) + [(SoledadMailbox.INIT_FLAGS, "/", "root/subthing")]) return d def testStatus(self): """ Test Status command """ - LeapIMAPServer.theAccount.addMailbox('root/subthings') + acc = LeapIMAPServer.theAccount + + def add_mailbox(): + return acc.addMailbox('root/subthings') + # XXX FIXME ---- should populate this a little bit, # with unseen etc... @@ -824,7 +865,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.statused = result self.statused = None - d1 = self.connected.addCallback(strip(login)) + + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(status), self._ebGeneral) d1.addCallbacks(statused, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) @@ -930,7 +973,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Test partially appending a message to the mailbox """ infile = util.sibpath(__file__, 'rfc822.message') - LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') + d0 = lambda: LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -946,6 +989,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): ) ) d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d0)) d1.addCallbacks(strip(append), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -995,10 +1039,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Test closing the mailbox. We expect to get deleted all messages flagged as such. """ + acc = self.server.theAccount name = 'mailbox-close' - self.server.theAccount.addMailbox(name) - m = LeapIMAPServer.theAccount.getMailbox(name) + d0 = lambda: acc.addMailbox(name) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -1006,14 +1050,17 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def select(): return self.client.select(name) + def get_mailbox(): + self.mailbox = LeapIMAPServer.theAccount.getMailbox(name) + def add_messages(): - d1 = m.messages.add_msg( + d1 = self.mailbox.messages.add_msg( 'test 1', subject="Message 1", flags=('\\Deleted', 'AnotherFlag')) - d2 = m.messages.add_msg( + d2 = self.mailbox.messages.add_msg( 'test 2', subject="Message 2", flags=('AnotherFlag',)) - d3 = m.messages.add_msg( + d3 = self.mailbox.messages.add_msg( 'test 3', subject="Message 3", flags=('\\Deleted',)) d = defer.gatherResults([d1, d2, d3]) @@ -1023,30 +1070,33 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.close() d = self.connected.addCallback(strip(login)) + d.addCallback(strip(d0)) d.addCallbacks(strip(select), self._ebGeneral) + d.addCallback(strip(get_mailbox)) d.addCallbacks(strip(add_messages), self._ebGeneral) d.addCallbacks(strip(close), self._ebGeneral) d.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() - return defer.gatherResults([d, d2]).addCallback(self._cbTestClose, m) + return defer.gatherResults([d, d2]).addCallback(self._cbTestClose) - def _cbTestClose(self, ignored, m): - self.assertEqual(len(m.messages), 1) - msg = m.messages.get_msg_by_uid(2) + def _cbTestClose(self, ignored): + self.assertEqual(len(self.mailbox.messages), 1) + msg = self.mailbox.messages.get_msg_by_uid(2) self.assertTrue(msg is not None) self.assertEqual( dict(msg.hdoc.content)['subject'], 'Message 2') - self.failUnless(m.closed) + self.failUnless(self.mailbox.closed) def testExpunge(self): """ Test expunge command """ + acc = self.server.theAccount name = 'mailbox-expunge' - self.server.theAccount.addMailbox(name) - m = LeapIMAPServer.theAccount.getMailbox(name) + + d0 = lambda: acc.addMailbox(name) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -1054,14 +1104,17 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def select(): return self.client.select('mailbox-expunge') + def get_mailbox(): + self.mailbox = LeapIMAPServer.theAccount.getMailbox(name) + def add_messages(): - d1 = m.messages.add_msg( + d1 = self.mailbox.messages.add_msg( 'test 1', subject="Message 1", flags=('\\Deleted', 'AnotherFlag')) - d2 = m.messages.add_msg( + d2 = self.mailbox.messages.add_msg( 'test 2', subject="Message 2", flags=('AnotherFlag',)) - d3 = m.messages.add_msg( + d3 = self.mailbox.messages.add_msg( 'test 3', subject="Message 3", flags=('\\Deleted',)) d = defer.gatherResults([d1, d2, d3]) @@ -1076,21 +1129,23 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.results = None d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(d0)) d1.addCallbacks(strip(select), self._ebGeneral) + d1.addCallback(strip(get_mailbox)) d1.addCallbacks(strip(add_messages), self._ebGeneral) d1.addCallbacks(strip(expunge), self._ebGeneral) d1.addCallbacks(expunged, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - return d.addCallback(self._cbTestExpunge, m) + return d.addCallback(self._cbTestExpunge) - def _cbTestExpunge(self, ignored, m): + def _cbTestExpunge(self, ignored): # we only left 1 mssage with no deleted flag - self.assertEqual(len(m.messages), 1) - msg = m.messages.get_msg_by_uid(2) + self.assertEqual(len(self.mailbox.messages), 1) + msg = self.mailbox.messages.get_msg_by_uid(2) - msg = list(m.messages)[0] + msg = list(self.mailbox.messages)[0] self.assertTrue(msg is not None) self.assertEqual( diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index 5339acf..9a3868c 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -139,31 +139,32 @@ class IMAP4HelperMixin(BaseLeapTest): ########### - d = defer.Deferred() + d_server_ready = defer.Deferred() + self.server = LeapIMAPServer( uuid=UUID, userid=USERID, contextFactory=self.serverCTX, - # XXX do we really need this?? soledad=self._soledad) - self.client = SimpleClient(d, contextFactory=self.clientCTX) - self.connected = d - - # XXX REVIEW-ME. - # We're adding theAccount here to server - # but it was also passed to initialization - # as it was passed to realm. - # I THINK we ONLY need to do it at one place now. + self.client = SimpleClient( + d_server_ready, contextFactory=self.clientCTX) theAccount = SoledadBackedAccount( USERID, soledad=self._soledad, memstore=memstore) + d_account_ready = theAccount.callWhenReady(lambda r: None) LeapIMAPServer.theAccount = theAccount + self.connected = defer.gatherResults( + [d_server_ready, d_account_ready]) + + # XXX FIXME -------------------------------------------- + # XXX this needs to be done differently, + # have to be hooked on initialization callback instead. # in case we get something from previous tests... - for mb in self.server.theAccount.mailboxes: - self.server.theAccount.delete(mb) + #for mb in self.server.theAccount.mailboxes: + #self.server.theAccount.delete(mb) # email parser self.parser = parser.Parser() -- cgit v1.2.3 From ea82f75f5465de47c4a838fbd1dfe8b2030fd842 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Tue, 9 Dec 2014 12:18:40 -0600 Subject: New keymanager async API --- src/leap/mail/imap/fetch.py | 312 +++++++++++++------------ src/leap/mail/imap/tests/test_incoming_mail.py | 38 +-- src/leap/mail/service.py | 201 +++++++++------- src/leap/mail/smtp/gateway.py | 38 +-- src/leap/mail/smtp/tests/test_gateway.py | 25 +- src/leap/mail/tests/__init__.py | 88 ++----- src/leap/mail/tests/test_service.py | 216 ++++++++--------- 7 files changed, 474 insertions(+), 444 deletions(-) diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index 01373be..dbc726a 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -36,7 +36,6 @@ from twisted.internet import defer, reactor from twisted.internet.task import LoopingCall from twisted.internet.task import deferLater from u1db import errors as u1db_errors -from zope.proxy import sameProxiedObjects from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type @@ -138,13 +137,6 @@ class LeapIncomingMail(object): # initialize a mail parser only once self._parser = Parser() - @property - def _pkey(self): - if sameProxiedObjects(self._keymanager, None): - logger.warning('tried to get key, but null keymanager found') - return None - return self._keymanager.get_key(self._userid, OpenPGPKey, private=True) - # # Public API: fetch, start_loop, stop. # @@ -312,40 +304,46 @@ class LeapIncomingMail(object): :param doc: A document containing an encrypted message. :type doc: SoledadDocument - :return: A tuple containing the document and the decrypted message. - :rtype: (SoledadDocument, str) + :return: A Deferred that will be fired with the document and the + decrypted message. + :rtype: SoledadDocument, str """ log.msg('decrypting msg') - success = False - try: - decrdata = self._keymanager.decrypt( - doc.content[ENC_JSON_KEY], - self._pkey) - success = True - except Exception as exc: - # XXX move this to errback !!! - logger.error("Error while decrypting msg: %r" % (exc,)) - decrdata = "" - leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0") + def process_decrypted(res): + if isinstance(res, tuple): + decrdata, _ = res + success = True + else: + decrdata = "" + success = False - data = self._process_decrypted_doc((doc, decrdata)) - return (doc, data) + leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0") - def _process_decrypted_doc(self, msgtuple): + data = self._process_decrypted_doc(doc, decrdata) + return doc, data + + d = self._keymanager.decrypt( + doc.content[ENC_JSON_KEY], + self._userid, OpenPGPKey) + d.addErrback(self._errback) + d.addCallback(process_decrypted) + return d + + def _process_decrypted_doc(self, doc, data): """ Process a document containing a succesfully decrypted message. - :param msgtuple: a tuple consisting of a SoledadDocument - instance containing the incoming message - and data, the json-encoded, decrypted content of the - incoming message - :type msgtuple: (SoledadDocument, str) + :param doc: the incoming message + :type doc: SoledadDocument + :param data: the json-encoded, decrypted content of the incoming + message + :type data: str + :return: the processed data. :rtype: str """ log.msg('processing decrypted doc') - doc, data = msgtuple # XXX turn this into an errBack for each one of # the deferreds that would process an individual document @@ -421,45 +419,40 @@ class LeapIncomingMail(object): encoding = get_email_charset(data) msg = self._parser.parsestr(data) - # try to obtain sender public key - senderPubkey = None fromHeader = msg.get('from', None) + senderAddress = None if (fromHeader is not None and (msg.get_content_type() == MULTIPART_ENCRYPTED or msg.get_content_type() == MULTIPART_SIGNED)): - _, senderAddress = parseaddr(fromHeader) - try: - senderPubkey = self._keymanager.get_key( - senderAddress, OpenPGPKey) - except keymanager_errors.KeyNotFound: - pass - - valid_sig = False # we will add a header saying if sig is valid - decrypt_multi = self._decrypt_multipart_encrypted_msg - decrypt_inline = self._maybe_decrypt_inline_encrypted_msg + senderAddress = parseaddr(fromHeader) + + def add_leap_header(decrmsg, signkey): + if (senderAddress is None or + isinstance(signkey, keymanager_errors.KeyNotFound)): + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_COULD_NOT_VERIFY) + elif isinstance(signkey, keymanager_errors.InvalidSignature): + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_INVALID) + else: + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_VALID, + pubkey=signkey.key_id) + return decrmsg.as_string() if msg.get_content_type() == MULTIPART_ENCRYPTED: - decrmsg, valid_sig = decrypt_multi( - msg, encoding, senderPubkey) + d = self._decrypt_multipart_encrypted_msg( + msg, encoding, senderAddress) else: - decrmsg, valid_sig = decrypt_inline( - msg, encoding, senderPubkey) - - # add x-leap-signature header - if senderPubkey is None: - decrmsg.add_header( - self.LEAP_SIGNATURE_HEADER, - self.LEAP_SIGNATURE_COULD_NOT_VERIFY) - else: - decrmsg.add_header( - self.LEAP_SIGNATURE_HEADER, - self.LEAP_SIGNATURE_VALID if valid_sig else - self.LEAP_SIGNATURE_INVALID, - pubkey=senderPubkey.key_id) - - return decrmsg.as_string() + d = self._maybe_decrypt_inline_encrypted_msg( + msg, encoding, senderAddress) + d.addCallback(add_leap_header) + return d - def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderPubkey): + def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderAddress): """ Decrypt a message with content-type 'multipart/encrypted'. @@ -467,12 +460,13 @@ class LeapIncomingMail(object): :type msg: Message :param encoding: The encoding of the email message. :type encoding: str - :param senderPubkey: The key of the sender of the message. - :type senderPubkey: OpenPGPKey + :param senderAddress: The email address of the sender of the message. + :type senderAddress: str - :return: A tuple containing a decrypted message and - a bool indicating whether the signature is valid. - :rtype: (Message, bool) + :return: A Deferred that will be fired with a tuple containing a + decrypted Message and the signing OpenPGPKey if the signature + is valid or InvalidSignature or KeyNotFound. + :rtype: Deferred """ log.msg('decrypting multipart encrypted msg') msg = copy.deepcopy(msg) @@ -483,33 +477,33 @@ class LeapIncomingMail(object): encdata = pgpencmsg.get_payload() # decrypt or fail gracefully - try: - decrdata, valid_sig = self._decrypt_and_verify_data( - encdata, senderPubkey) - except keymanager_errors.DecryptError as e: - logger.warning('Failed to decrypt encrypted message (%s). ' - 'Storing message without modifications.' % str(e)) - # Bailing out! - return (msg, False) + def build_msg(res): + decrdata, signkey = res - decrmsg = self._parser.parsestr(decrdata) - # remove original message's multipart/encrypted content-type - del(msg['content-type']) + decrmsg = self._parser.parsestr(decrdata) + # remove original message's multipart/encrypted content-type + del(msg['content-type']) - # replace headers back in original message - for hkey, hval in decrmsg.items(): - try: - # this will raise KeyError if header is not present - msg.replace_header(hkey, hval) - except KeyError: - msg[hkey] = hval + # replace headers back in original message + for hkey, hval in decrmsg.items(): + try: + # this will raise KeyError if header is not present + msg.replace_header(hkey, hval) + except KeyError: + msg[hkey] = hval + + # all ok, replace payload by unencrypted payload + msg.set_payload(decrmsg.get_payload()) + return (msg, signkey) - # all ok, replace payload by unencrypted payload - msg.set_payload(decrmsg.get_payload()) - return (msg, valid_sig) + d = self._keymanager.decrypt( + encdata, self._userid, OpenPGPKey, + verify=senderAddress) + d.addCallbacks(build_msg, self._decryption_error, errbackArgs=(msg,)) + return d def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding, - senderPubkey): + senderAddress): """ Possibly decrypt an inline OpenPGP encrypted message. @@ -517,12 +511,13 @@ class LeapIncomingMail(object): :type origmsg: Message :param encoding: The encoding of the email message. :type encoding: str - :param senderPubkey: The key of the sender of the message. - :type senderPubkey: OpenPGPKey + :param senderAddress: The email address of the sender of the message. + :type senderAddress: str - :return: A tuple containing a decrypted message and - a bool indicating whether the signature is valid. - :rtype: (Message, bool) + :return: A Deferred that will be fired with a tuple containing a + decrypted Message and the signing OpenPGPKey if the signature + is valid or InvalidSignature or KeyNotFound. + :rtype: Deferred """ log.msg('maybe decrypting inline encrypted msg') # serialize the original message @@ -530,54 +525,48 @@ class LeapIncomingMail(object): g = Generator(buf) g.flatten(origmsg) data = buf.getvalue() + + def decrypted_data(res): + decrdata, signkey = res + return data.replace(pgp_message, decrdata), signkey + + def encode_and_return(res): + data, signkey = res + if isinstance(data, unicode): + data = data.encode(encoding, 'replace') + return (self._parser.parsestr(data), signkey) + # handle exactly one inline PGP message - valid_sig = False if PGP_BEGIN in data: begin = data.find(PGP_BEGIN) end = data.find(PGP_END) pgp_message = data[begin:end + len(PGP_END)] - try: - decrdata, valid_sig = self._decrypt_and_verify_data( - pgp_message, senderPubkey) - # replace encrypted by decrypted content - data = data.replace(pgp_message, decrdata) - except keymanager_errors.DecryptError: - logger.warning('Failed to decrypt potential inline encrypted ' - 'message. Storing message as is...') - - # if message is not encrypted, return raw data - if isinstance(data, unicode): - data = data.encode(encoding, 'replace') - return (self._parser.parsestr(data), valid_sig) + d = self._keymanager.decrypt( + pgp_message, self._userid, OpenPGPKey, + verify=senderAddress) + d.addCallbacks(decrypted_data, self._decryption_error, + errbackArgs=(data,)) + else: + d = defer.succeed((data, None)) + d.addCallback(encode_and_return) + return d - def _decrypt_and_verify_data(self, data, senderPubkey): + def _decryption_error(self, failure, msg): """ - Decrypt C{data} using our private key and attempt to verify a - signature using C{senderPubkey}. - - :param data: The text to be decrypted. - :type data: unicode - :param senderPubkey: The public key of the sender of the message. - :type senderPubkey: OpenPGPKey - - :return: The decrypted data and a boolean stating whether the - signature could be verified. - :rtype: (str, bool) - - :raise DecryptError: Raised if failed to decrypt. + Check for known decryption errors """ - log.msg('decrypting and verifying data') - valid_sig = False - try: - decrdata = self._keymanager.decrypt( - data, self._pkey, - verify=senderPubkey) - if senderPubkey is not None: - valid_sig = True - except keymanager_errors.InvalidSignature: - decrdata = self._keymanager.decrypt( - data, self._pkey) - return (decrdata, valid_sig) + if failure.check(keymanager_errors.DecryptError): + logger.warning('Failed to decrypt encrypted message (%s). ' + 'Storing message without modifications.' + % str(failure.value)) + return (msg, None) + elif failure.check(keymanager_errors.KeyNotFound): + logger.error('Failed to find private key for decryption (%s). ' + 'Storing message without modifications.' + % str(failure.value)) + return (msg, None) + else: + return failure def _extract_keys(self, msgtuple): """ @@ -592,6 +581,10 @@ class LeapIncomingMail(object): and data, the json-encoded, decrypted content of the incoming message :type msgtuple: (SoledadDocument, str) + + :return: A Deferred that will be fired with msgtuple when key + extraction finishes + :rtype: Deferred """ OpenPGP_HEADER = 'OpenPGP' doc, data = msgtuple @@ -603,13 +596,17 @@ class LeapIncomingMail(object): _, fromAddress = parseaddr(msg['from']) header = msg.get(OpenPGP_HEADER, None) + dh = defer.success() if header is not None: - self._extract_openpgp_header(header, fromAddress) + dh = self._extract_openpgp_header(header, fromAddress) + da = defer.success() if msg.is_multipart(): - self._extract_attached_key(msg.get_payload(), fromAddress) + da = self._extract_attached_key(msg.get_payload(), fromAddress) - return msgtuple + d = defer.gatherResults([dh, da]) + d.addCallback(lambda _: msgtuple) + return d def _extract_openpgp_header(self, header, address): """ @@ -619,7 +616,11 @@ class LeapIncomingMail(object): :type header: str :param address: email address in the from header :type address: str + + :return: A Deferred that will be fired when header extraction is done + :rtype: Deferred """ + d = defer.success() fields = dict([f.strip(' ').split('=') for f in header.split(';')]) if 'url' in fields: url = shlex.split(fields['url'])[0] # remove quotations @@ -627,21 +628,28 @@ class LeapIncomingMail(object): addressHostname = address.split('@')[1] if (urlparts.scheme == 'https' and urlparts.hostname == addressHostname): - try: - self._keymanager.fetch_key(address, url, OpenPGPKey) - logger.info("Imported key from header %s" % (url,)) - except keymanager_errors.KeyNotFound: - logger.warning("Url from OpenPGP header %s failed" - % (url,)) - except keymanager_errors.KeyAttributesDiffer: - logger.warning("Key from OpenPGP header url %s didn't " - "match the from address %s" - % (url, address)) + def fetch_error(failure): + if failure.check(keymanager_errors.KeyNotFound): + logger.warning("Url from OpenPGP header %s failed" + % (url,)) + elif failure.check(keymanager_errors.KeyAttributesDiffer): + logger.warning("Key from OpenPGP header url %s didn't " + "match the from address %s" + % (url, address)) + else: + return failure + + d = self._keymanager.fetch_key(address, url, OpenPGPKey) + d.addCallback( + lambda _: + logger.info("Imported key from header %s" % (url,))) + d.addErrback(fetch_error) else: logger.debug("No valid url on OpenPGP header %s" % (url,)) else: logger.debug("There is no url on the OpenPGP header: %s" % (header,)) + return d def _extract_attached_key(self, attachments, address): """ @@ -651,16 +659,22 @@ class LeapIncomingMail(object): :type attachments: list(email.Message) :param address: email address in the from header :type address: str + + :return: A Deferred that will be fired when all the keys are stored + :rtype: Deferred """ MIME_KEY = "application/pgp-keys" + deferreds = [] for attachment in attachments: if MIME_KEY == attachment.get_content_type(): logger.debug("Add key from attachment") - self._keymanager.put_raw_key( + d = self._keymanager.put_raw_key( attachment.get_payload(), OpenPGPKey, address=address) + deferreds.append(d) + return defer.gatherResults(deferreds) def _add_message_locally(self, msgtuple): """ @@ -672,6 +686,9 @@ class LeapIncomingMail(object): and data, the json-encoded, decrypted content of the incoming message :type msgtuple: (SoledadDocument, str) + + :return: A Deferred that will be fired when the messages is stored + :rtype: Defferred """ doc, data = msgtuple log.msg('adding message %s to local db' % (doc.doc_id,)) @@ -690,6 +707,7 @@ class LeapIncomingMail(object): d = self._inbox.addMessage(data, flags=(self.RECENT_FLAG,), notify_on_disk=True) d.addCallbacks(msgSavedCallback, self._errback) + return d # # helpers diff --git a/src/leap/mail/imap/tests/test_incoming_mail.py b/src/leap/mail/imap/tests/test_incoming_mail.py index ce6d56a..03c0164 100644 --- a/src/leap/mail/imap/tests/test_incoming_mail.py +++ b/src/leap/mail/imap/tests/test_incoming_mail.py @@ -28,7 +28,6 @@ from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.parser import Parser from mock import Mock -from twisted.trial import unittest from leap.keymanager.openpgp import OpenPGPKey from leap.mail.imap.account import SoledadBackedAccount @@ -48,7 +47,7 @@ from leap.soledad.common.crypto import ( ) -class LeapIncomingMailTestCase(TestCaseWithKeyManager, unittest.TestCase): +class LeapIncomingMailTestCase(TestCaseWithKeyManager): """ Tests for the incoming mail parser """ @@ -147,31 +146,42 @@ subject: independence of cyberspace key = MIMEApplication("", "pgp-keys") key.set_payload(KEY) message.attach(key) - email = self._create_incoming_email(message.as_string()) - self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]) - self.fetcher._keymanager.put_raw_key = Mock() def put_raw_key_called(ret): self.fetcher._keymanager.put_raw_key.assert_called_once_with( KEY, OpenPGPKey, address=self.FROM_ADDRESS) - d = self.fetcher.fetch() + d = self.mock_fetch(message.as_string()) d.addCallback(put_raw_key_called) return d + def _mock_fetch(self, message): + self.fetcher._keymanager.fetch_key = Mock() + d = self._create_incoming_email(message) + d.addCallback( + lambda email: + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) + d.addCallback(lambda _: self.fetcher.fetch()) + return d + def _create_incoming_email(self, email_str): email = SoledadDocument() - pubkey = self._km.get_key(ADDRESS, OpenPGPKey) data = json.dumps( {"incoming": True, "content": email_str}, ensure_ascii=False) - email.content = { - fields.INCOMING_KEY: True, - fields.ERROR_DECRYPTING_KEY: False, - ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, - ENC_JSON_KEY: str(self._km.encrypt(data, pubkey)) - } - return email + + def set_email_content(pubkey): + email.content = { + fields.INCOMING_KEY: True, + fields.ERROR_DECRYPTING_KEY: False, + ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, + ENC_JSON_KEY: str(self._km.encrypt(data, pubkey)) + } + return email + + d = self._km.get_key(ADDRESS, OpenPGPKey) + d.addCallback(set_email_content) + return d def _mock_soledad_get_from_index(self, index_name, value): get_from_index = self._soledad.get_from_index diff --git a/src/leap/mail/service.py b/src/leap/mail/service.py index f6e4d11..a99f13a 100644 --- a/src/leap/mail/service.py +++ b/src/leap/mail/service.py @@ -24,7 +24,6 @@ from OpenSSL import SSL from twisted.mail import smtp from twisted.internet import reactor from twisted.internet import defer -from twisted.internet.threads import deferToThread from twisted.protocols.amp import ssl from twisted.python import log @@ -111,17 +110,17 @@ class OutgoingMail: :type recipient: smtp.User :return: a deferred which delivers the message when fired """ - d = deferToThread(lambda: self._maybe_encrypt_and_sign(raw, recipient)) + d = self._maybe_encrypt_and_sign(raw, recipient) d.addCallback(self._route_msg) d.addErrback(self.sendError) - return d def sendSuccess(self, smtp_sender_result): """ Callback for a successful send. - :param smtp_sender_result: The result from the ESMTPSender from _route_msg + :param smtp_sender_result: The result from the ESMTPSender from + _route_msg :type smtp_sender_result: tuple(int, list(tuple)) """ dest_addrstr = smtp_sender_result[1][0][0] @@ -145,7 +144,8 @@ class OutgoingMail: """ Sends the msg using the ESMTPSenderFactory. - :param encrypt_and_sign_result: A tuple containing the 'maybe' encrypted message and the recipient + :param encrypt_and_sign_result: A tuple containing the 'maybe' + encrypted message and the recipient :type encrypt_and_sign_result: tuple """ message, recipient = encrypt_and_sign_result @@ -173,7 +173,6 @@ class OutgoingMail: self._host, self._port, factory, contextFactory=SSLContextFactory(self._cert, self._key)) - def _maybe_encrypt_and_sign(self, raw, recipient): """ Attempt to encrypt and sign the outgoing message. @@ -209,16 +208,20 @@ class OutgoingMail: :param recipient: The recipient for the message :type: recipient: smtp.User + :return: A Deferred that will be fired with a MIMEMultipart message + and the original recipient Message + :rtype: Deferred """ # pass if the original message's content-type is "multipart/encrypted" lines = raw.split('\r\n') origmsg = Parser().parsestr(raw) if origmsg.get_content_type() == 'multipart/encrypted': - return origmsg + return defer.success((origmsg, recipient)) from_address = validate_address(self._from_address) username, domain = from_address.split('@') + to_address = validate_address(recipient.dest.addrstr) # add a nice footer to the outgoing message # XXX: footer will eventually optional or be removed @@ -230,80 +233,93 @@ class OutgoingMail: origmsg = Parser().parsestr('\r\n'.join(lines)) - # get sender and recipient data - signkey = self._keymanager.get_key(from_address, OpenPGPKey, private=True) - log.msg("Will sign the message with %s." % signkey.fingerprint) - to_address = validate_address(recipient.dest.addrstr) - try: - # try to get the recipient pubkey - pubkey = self._keymanager.get_key(to_address, OpenPGPKey) - log.msg("Will encrypt the message to %s." % pubkey.fingerprint) - signal(proto.SMTP_START_ENCRYPT_AND_SIGN, - "%s,%s" % (self._from_address, to_address)) - newmsg = self._encrypt_and_sign(origmsg, pubkey, signkey) - + def signal_encrypt_sign(newmsg): signal(proto.SMTP_END_ENCRYPT_AND_SIGN, "%s,%s" % (self._from_address, to_address)) - except KeyNotFound: - # at this point we _can_ send unencrypted mail, because if the - # configuration said the opposite the address would have been - # rejected in SMTPDelivery.validateTo(). - log.msg('Will send unencrypted message to %s.' % to_address) - signal(proto.SMTP_START_SIGN, self._from_address) - newmsg = self._sign(origmsg, signkey) - signal(proto.SMTP_END_SIGN, self._from_address) - return newmsg, recipient + return newmsg, recipient + def signal_sign(newmsg): + signal(proto.SMTP_END_SIGN, self._from_address) + return newmsg, recipient + + def if_key_not_found_send_unencrypted(failure): + if failure.check(KeyNotFound): + log.msg('Will send unencrypted message to %s.' % to_address) + signal(proto.SMTP_START_SIGN, self._from_address) + d = self._sign(origmsg, from_address) + d.addCallback(signal_sign) + return d + else: + return failure + + log.msg("Will encrypt the message with %s and sign with %s." + % (to_address, from_address)) + signal(proto.SMTP_START_ENCRYPT_AND_SIGN, + "%s,%s" % (self._from_address, to_address)) + d = self._encrypt_and_sign(origmsg, to_address, from_address) + d.addCallbacks(signal_encrypt_sign, if_key_not_found_send_unencrypted) + return d - def _encrypt_and_sign(self, origmsg, pubkey, signkey): + def _encrypt_and_sign(self, origmsg, encrypt_address, sign_address): """ Create an RFC 3156 compliang PGP encrypted and signed message using - C{pubkey} to encrypt and C{signkey} to sign. + C{encrypt_address} to encrypt and C{sign_address} to sign. :param origmsg: The original message :type origmsg: email.message.Message - :param pubkey: The public key used to encrypt the message. - :type pubkey: OpenPGPKey - :param signkey: The private key used to sign the message. - :type signkey: OpenPGPKey - :return: The encrypted and signed message - :rtype: MultipartEncrypted + :param encrypt_address: The address used to encrypt the message. + :type encrypt_address: str + :param sign_address: The address used to sign the message. + :type sign_address: str + + :return: A Deferred with the MultipartEncrypted message + :rtype: Deferred """ # create new multipart/encrypted message with 'pgp-encrypted' protocol - newmsg = MultipartEncrypted('application/pgp-encrypted') - # move (almost) all headers from original message to the new message - self._fix_headers(origmsg, newmsg, signkey) - # create 'application/octet-stream' encrypted message - encmsg = MIMEApplication( - self._keymanager.encrypt(origmsg.as_string(unixfrom=False), pubkey, - sign=signkey), - _subtype='octet-stream', _encoder=lambda x: x) - encmsg.add_header('content-disposition', 'attachment', - filename='msg.asc') - # create meta message - metamsg = PGPEncrypted() - metamsg.add_header('Content-Disposition', 'attachment') - # attach pgp message parts to new message - newmsg.attach(metamsg) - newmsg.attach(encmsg) - return newmsg - - - def _sign(self, origmsg, signkey): + + def encrypt(res): + newmsg, origmsg = res + d = self._keymanager.encrypt( + origmsg.as_string(unixfrom=False), + encrypt_address, OpenPGPKey, sign=sign_address) + d.addCallback(lambda encstr: (newmsg, encstr)) + return d + + def create_encrypted_message(res): + newmsg, encstr = res + encmsg = MIMEApplication( + encstr, _subtype='octet-stream', _encoder=lambda x: x) + encmsg.add_header('content-disposition', 'attachment', + filename='msg.asc') + # create meta message + metamsg = PGPEncrypted() + metamsg.add_header('Content-Disposition', 'attachment') + # attach pgp message parts to new message + newmsg.attach(metamsg) + newmsg.attach(encmsg) + return newmsg + + d = self._fix_headers( + origmsg, + MultipartEncrypted('application/pgp-encrypted'), + sign_address) + d.addCallback(encrypt) + d.addCallback(create_encrypted_message) + return d + + def _sign(self, origmsg, sign_address): """ - Create an RFC 3156 compliant PGP signed MIME message using C{signkey}. + Create an RFC 3156 compliant PGP signed MIME message using + C{sign_address}. :param origmsg: The original message :type origmsg: email.message.Message - :param signkey: The private key used to sign the message. - :type signkey: leap.common.keymanager.openpgp.OpenPGPKey - :return: The signed message. - :rtype: MultipartSigned + :param sign_address: The address used to sign the message. + :type sign_address: str + + :return: A Deferred with the MultipartSigned message. + :rtype: Deferred """ - # create new multipart/signed message - newmsg = MultipartSigned('application/pgp-signature', 'pgp-sha512') - # move (almost) all headers from original message to the new message - self._fix_headers(origmsg, newmsg, signkey) # apply base64 content-transfer-encoding encode_base64_rec(origmsg) # get message text with headers and replace \n for \r\n @@ -316,17 +332,27 @@ class OutgoingMail: if origmsg.is_multipart(): if not msgtext.endswith("\r\n"): msgtext += "\r\n" - # calculate signature - signature = self._keymanager.sign(msgtext, signkey, digest_algo='SHA512', - clearsign=False, detach=True, binary=False) - sigmsg = PGPSignature(signature) - # attach original message and signature to new message - newmsg.attach(origmsg) - newmsg.attach(sigmsg) - return newmsg + def create_signed_message(res): + (msg, _), signature = res + sigmsg = PGPSignature(signature) + # attach original message and signature to new message + msg.attach(origmsg) + msg.attach(sigmsg) + return msg + + dh = self._fix_headers( + origmsg, + MultipartSigned('application/pgp-signature', 'pgp-sha512'), + sign_address) + ds = self._keymanager.sign( + msgtext, sign_address, OpenPGPKey, digest_algo='SHA512', + clearsign=False, detach=True, binary=False) + d = defer.gatherResults([dh, ds]) + d.addCallback(create_signed_message) + return d - def _fix_headers(self, origmsg, newmsg, signkey): + def _fix_headers(self, origmsg, newmsg, sign_address): """ Move some headers from C{origmsg} to C{newmsg}, delete unwanted headers from C{origmsg} and add new headers to C{newms}. @@ -360,8 +386,13 @@ class OutgoingMail: :type origmsg: email.message.Message :param newmsg: The new message being created. :type newmsg: email.message.Message - :param signkey: The key used to sign C{newmsg} - :type signkey: OpenPGPKey + :param sign_address: The address used to sign C{newmsg} + :type sign_address: str + + :return: A Deferred with a touple: + (new Message with the unencrypted headers, + original Message with headers removed) + :rtype: Deferred """ # move headers from origmsg to newmsg headers = origmsg.items() @@ -375,11 +406,17 @@ class OutgoingMail: del (origmsg[hkey]) # add a new message-id to newmsg newmsg.add_header('Message-Id', smtp.messageid()) - # add openpgp header to newmsg - username, domain = signkey.address.split('@') - newmsg.add_header( - 'OpenPGP', 'id=%s' % signkey.key_id, - url='https://%s/key/%s' % (domain, username), - preference='signencrypt') # delete user-agent from origmsg del (origmsg['user-agent']) + + def add_openpgp_header(signkey): + username, domain = sign_address.split('@') + newmsg.add_header( + 'OpenPGP', 'id=%s' % signkey.key_id, + url='https://%s/key/%s' % (domain, username), + preference='signencrypt') + return newmsg, origmsg + + d = self._keymanager.get_key(sign_address, OpenPGPKey, private=True) + d.addCallback(add_openpgp_header) + return d diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index b022091..d58c581 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -48,7 +48,6 @@ from leap.mail.smtp.rfc3156 import ( RFC3156CompliantGenerator, ) -from leap.mail.service import OutgoingMail # replace email generator with a RFC 3156 compliant one. from email import generator @@ -197,22 +196,31 @@ class SMTPDelivery(object): accepted. """ # try to find recipient's public key - try: - address = validate_address(user.dest.addrstr) - # verify if recipient key is available in keyring - self._km.get_key(address, OpenPGPKey) # might raise KeyNotFound + address = validate_address(user.dest.addrstr) + + # verify if recipient key is available in keyring + def found(_): log.msg("Accepting mail for %s..." % user.dest.addrstr) signal(proto.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, user.dest.addrstr) - except KeyNotFound: - # if key was not found, check config to see if will send anyway. - if self._encrypted_only: - signal(proto.SMTP_RECIPIENT_REJECTED, user.dest.addrstr) - raise smtp.SMTPBadRcpt(user.dest.addrstr) - log.msg("Warning: will send an unencrypted message (because " - "encrypted_only' is set to False).") - signal( - proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, user.dest.addrstr) - return lambda: EncryptedMessage(user, self._outgoing_mail) + + def not_found(failure): + if failure.check(KeyNotFound): + # if key was not found, check config to see if will send anyway + if self._encrypted_only: + signal(proto.SMTP_RECIPIENT_REJECTED, user.dest.addrstr) + raise smtp.SMTPBadRcpt(user.dest.addrstr) + log.msg("Warning: will send an unencrypted message (because " + "encrypted_only' is set to False).") + signal( + proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, + user.dest.addrstr) + else: + return failure + + d = self._km.get_key(address, OpenPGPKey) # might raise KeyNotFound + d.addCallbacks(found, not_found) + d.addCallbac(lambda _: EncryptedMessage(user, self._outgoing_mail)) + return d def validateFrom(self, helo, origin): """ diff --git a/src/leap/mail/smtp/tests/test_gateway.py b/src/leap/mail/smtp/tests/test_gateway.py index aeace4a..8cbff8f 100644 --- a/src/leap/mail/smtp/tests/test_gateway.py +++ b/src/leap/mail/smtp/tests/test_gateway.py @@ -23,6 +23,7 @@ SMTP gateway tests. import re from datetime import datetime +from twisted.internet.defer import inlineCallbacks, fail from twisted.test import proto_helpers from mock import Mock @@ -34,7 +35,7 @@ from leap.mail.tests import ( ADDRESS, ADDRESS_2, ) -from leap.keymanager import openpgp +from leap.keymanager import openpgp, errors # some regexps @@ -87,7 +88,8 @@ class TestSmtpGateway(TestCaseWithKeyManager): proto = SMTPFactory( u'anotheruser@leap.se', self._km, - self._config['encrypted_only'], outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) + self._config['encrypted_only'], + outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) # snip... transport = proto_helpers.StringTransport() proto.makeConnection(transport) @@ -98,23 +100,26 @@ class TestSmtpGateway(TestCaseWithKeyManager): 'Did not get expected answer from gateway.') proto.setTimeout(None) + @inlineCallbacks def test_missing_key_rejects_address(self): """ Test if server rejects to send unencrypted when 'encrypted_only' is True. """ # remove key from key manager - pubkey = self._km.get_key(ADDRESS, openpgp.OpenPGPKey) + pubkey = yield self._km.get_key(ADDRESS, openpgp.OpenPGPKey) pgp = openpgp.OpenPGPScheme( self._soledad, gpgbinary=self.GPG_BINARY_PATH) - pgp.delete_key(pubkey) + yield pgp.delete_key(pubkey) # mock the key fetching - self._km.fetch_keys_from_server = Mock(return_value=[]) + self._km._fetch_keys_from_server = Mock( + return_value=fail(errors.KeyNotFound())) # prepare the SMTP factory proto = SMTPFactory( u'anotheruser@leap.se', self._km, - self._config['encrypted_only'], outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) + self._config['encrypted_only'], + outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) transport = proto_helpers.StringTransport() proto.makeConnection(transport) proto.lineReceived(self.EMAIL_DATA[0] + '\r\n') @@ -127,18 +132,20 @@ class TestSmtpGateway(TestCaseWithKeyManager): lines[-1], 'Address should have been rejecetd with appropriate message.') + @inlineCallbacks def test_missing_key_accepts_address(self): """ Test if server accepts to send unencrypted when 'encrypted_only' is False. """ # remove key from key manager - pubkey = self._km.get_key(ADDRESS, openpgp.OpenPGPKey) + pubkey = yield self._km.get_key(ADDRESS, openpgp.OpenPGPKey) pgp = openpgp.OpenPGPScheme( self._soledad, gpgbinary=self.GPG_BINARY_PATH) - pgp.delete_key(pubkey) + yield pgp.delete_key(pubkey) # mock the key fetching - self._km.fetch_keys_from_server = Mock(return_value=[]) + self._km._fetch_keys_from_server = Mock( + return_value=fail(errors.KeyNotFound())) # prepare the SMTP factory with encrypted only equal to false proto = SMTPFactory( u'anotheruser@leap.se', diff --git a/src/leap/mail/tests/__init__.py b/src/leap/mail/tests/__init__.py index 10bc5fe..b35107d 100644 --- a/src/leap/mail/tests/__init__.py +++ b/src/leap/mail/tests/__init__.py @@ -19,16 +19,14 @@ Base classes and keys for leap.mail tests. """ import os import distutils.spawn -import shutil -import tempfile from mock import Mock +from twisted.internet.defer import gatherResults +from twisted.trial import unittest from leap.soledad.client import Soledad -from leap.keymanager import ( - KeyManager, - openpgp, -) +from leap.keymanager import KeyManager +from leap.keymanager.openpgp import OpenPGPKey from leap.common.testing.basetest import BaseLeapTest @@ -39,22 +37,12 @@ def _find_gpg(): return os.path.realpath(gpg_path) if gpg_path is not None else "/usr/bin/gpg" -class TestCaseWithKeyManager(BaseLeapTest): +class TestCaseWithKeyManager(unittest.TestCase, BaseLeapTest): GPG_BINARY_PATH = _find_gpg() def setUp(self): - # mimic BaseLeapTest.setUpClass behaviour, because this is deprecated - # in Twisted: http://twistedmatrix.com/trac/ticket/1870 - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir + self.setUpEnv() # setup our own stuff address = 'leap@leap.se' # user's address in the form user@provider @@ -65,36 +53,7 @@ class TestCaseWithKeyManager(BaseLeapTest): server_url = 'http://provider/' cert_file = '' - self._soledad = self._soledad_instance( - uuid, passphrase, secrets_path, local_db_path, server_url, - cert_file) - self._km = self._keymanager_instance(address) - - def _soledad_instance(self, uuid, passphrase, secrets_path, local_db_path, - server_url, cert_file): - """ - Return a Soledad instance for tests. - """ - # mock key fetching and storing so Soledad doesn't fail when trying to - # reach the server. - Soledad._fetch_keys_from_shared_db = Mock(return_value=None) - Soledad._assert_keys_in_shared_db = Mock(return_value=None) - - # instantiate soledad - def _put_doc_side_effect(doc): - self._doc_put = doc - - class MockSharedDB(object): - - get_doc = Mock(return_value=None) - put_doc = Mock(side_effect=_put_doc_side_effect) - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) - - def __call__(self): - return self - - soledad = Soledad( + self._soledad = Soledad( uuid, passphrase, secrets_path=secrets_path, @@ -103,13 +62,11 @@ class TestCaseWithKeyManager(BaseLeapTest): cert_file=cert_file, syncable=False ) + return self._setup_keymanager(address) - soledad._shared_db = MockSharedDB() - return soledad - - def _keymanager_instance(self, address): + def _setup_keymanager(self, address): """ - Return a Key Manager instance for tests. + Set up Key Manager and return a Deferred that will be fired when done. """ self._config = { 'host': 'https://provider/', @@ -132,26 +89,17 @@ class TestCaseWithKeyManager(BaseLeapTest): pass nickserver_url = '' # the url of the nickserver - km = KeyManager(address, nickserver_url, self._soledad, - ca_cert_path='', gpgbinary=self.GPG_BINARY_PATH) - km._fetcher.put = Mock() - km._fetcher.get = Mock(return_value=Response()) - - # insert test keys in key manager. - pgp = openpgp.OpenPGPScheme( - self._soledad, gpgbinary=self.GPG_BINARY_PATH) - pgp.put_ascii_key(PRIVATE_KEY) - pgp.put_ascii_key(PRIVATE_KEY_2) + self._km = KeyManager(address, nickserver_url, self._soledad, + ca_cert_path='', gpgbinary=self.GPG_BINARY_PATH) + self._km._fetcher.put = Mock() + self._km._fetcher.get = Mock(return_value=Response()) - return km + d1 = self._km.put_raw_key(PRIVATE_KEY, OpenPGPKey, ADDRESS) + d2 = self._km.put_raw_key(PRIVATE_KEY_2, OpenPGPKey, ADDRESS_2) + return gatherResults([d1, d2]) def tearDown(self): - # mimic LeapBaseTest.tearDownClass behaviour - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check - assert 'leap_tests-' in self.tempdir - shutil.rmtree(self.tempdir) + self.tearDownEnv() # Key material for testing diff --git a/src/leap/mail/tests/test_service.py b/src/leap/mail/tests/test_service.py index f0a807d..43f354d 100644 --- a/src/leap/mail/tests/test_service.py +++ b/src/leap/mail/tests/test_service.py @@ -22,7 +22,8 @@ SMTP gateway tests. import re from datetime import datetime -from twisted.mail.smtp import User, Address +from twisted.internet.defer import fail +from twisted.mail.smtp import User from mock import Mock @@ -33,7 +34,7 @@ from leap.mail.tests import ( ADDRESS, ADDRESS_2, ) -from leap.keymanager import openpgp +from leap.keymanager import openpgp, errors class TestOutgoingMail(TestCaseWithKeyManager): @@ -54,71 +55,126 @@ class TestOutgoingMail(TestCaseWithKeyManager): 'QUIT'] def setUp(self): - TestCaseWithKeyManager.setUp(self) self.lines = [line for line in self.EMAIL_DATA[4:12]] self.lines.append('') # add a trailing newline self.raw = '\r\n'.join(self.lines) + self.expected_body = ('\r\n'.join(self.EMAIL_DATA[9:12]) + + "\r\n\r\n--\r\nI prefer encrypted email - " + "https://leap.se/key/anotheruser\r\n") self.fromAddr = ADDRESS_2 - self.outgoing_mail = OutgoingMail(self.fromAddr, self._km, self._config['cert'], self._config['key'], - self._config['host'], self._config['port']) - self.proto = SMTPFactory( - u'anotheruser@leap.se', - self._km, - self._config['encrypted_only'], - self.outgoing_mail).buildProtocol(('127.0.0.1', 0)) - self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS) - - def test_openpgp_encrypt_decrypt(self): - "Test if openpgp can encrypt and decrypt." - text = "simple raw text" - pubkey = self._km.get_key( - ADDRESS, openpgp.OpenPGPKey, private=False) - encrypted = self._km.encrypt(text, pubkey) - self.assertNotEqual( - text, encrypted, "Ciphertext is equal to plaintext.") - privkey = self._km.get_key( - ADDRESS, openpgp.OpenPGPKey, private=True) - decrypted = self._km.decrypt(encrypted, privkey) - self.assertEqual(text, decrypted, - "Decrypted text differs from plaintext.") + + def init_outgoing_and_proto(_): + self.outgoing_mail = OutgoingMail( + self.fromAddr, self._km, self._config['cert'], + self._config['key'], self._config['host'], + self._config['port']) + self.proto = SMTPFactory( + u'anotheruser@leap.se', + self._km, + self._config['encrypted_only'], + self.outgoing_mail).buildProtocol(('127.0.0.1', 0)) + self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS) + + d = TestCaseWithKeyManager.setUp(self) + d.addCallback(init_outgoing_and_proto) + return d def test_message_encrypt(self): """ Test if message gets encrypted to destination email. """ - - message, _ = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) - - # assert structure of encrypted message - self.assertTrue('Content-Type' in message) - self.assertEqual('multipart/encrypted', message.get_content_type()) - self.assertEqual('application/pgp-encrypted', - message.get_param('protocol')) - self.assertEqual(2, len(message.get_payload())) - self.assertEqual('application/pgp-encrypted', - message.get_payload(0).get_content_type()) - self.assertEqual('application/octet-stream', - message.get_payload(1).get_content_type()) - privkey = self._km.get_key( - ADDRESS, openpgp.OpenPGPKey, private=True) - decrypted = self._km.decrypt( - message.get_payload(1).get_payload(), privkey) - - expected = '\n' + '\r\n'.join( - self.EMAIL_DATA[9:12]) + '\r\n\r\n--\r\n' + 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n' - self.assertEqual( - expected, - decrypted, - 'Decrypted text differs from plaintext.') + def check_decryption(res): + decrypted, _ = res + self.assertEqual( + '\n' + self.expected_body, + decrypted, + 'Decrypted text differs from plaintext.') + + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + d.addCallback(self._assert_encrypted) + d.addCallback(lambda message: self._km.decrypt( + message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey)) + d.addCallback(check_decryption) + return d def test_message_encrypt_sign(self): """ Test if message gets encrypted to destination email and signed with sender key. - """ - message, _ = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + '""" + def check_decryption_and_verify(res): + decrypted, signkey = res + self.assertEqual( + '\n' + self.expected_body, + decrypted, + 'Decrypted text differs from plaintext.') + self.assertTrue(ADDRESS_2 in signkey.address, + "Verification failed") + + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + d.addCallback(self._assert_encrypted) + d.addCallback(lambda message: self._km.decrypt( + message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey, + verify=ADDRESS_2)) + d.addCallback(check_decryption_and_verify) + return d - # assert structure of encrypted message + def test_message_sign(self): + """ + Test if message is signed with sender key. + """ + # mock the key fetching + self._km._fetch_keys_from_server = Mock( + return_value=fail(errors.KeyNotFound())) + recipient = User('ihavenopubkey@nonleap.se', + 'gateway.leap.se', self.proto, ADDRESS) + self.outgoing_mail = OutgoingMail( + self.fromAddr, self._km, self._config['cert'], self._config['key'], + self._config['host'], self._config['port']) + + def check_signed(res): + message, _ = res + self.assertTrue('Content-Type' in message) + self.assertEqual('multipart/signed', message.get_content_type()) + self.assertEqual('application/pgp-signature', + message.get_param('protocol')) + self.assertEqual('pgp-sha512', message.get_param('micalg')) + # assert content of message + self.assertEqual(self.expected_body, + message.get_payload(0).get_payload(decode=True)) + # assert content of signature + self.assertTrue( + message.get_payload(1).get_payload().startswith( + '-----BEGIN PGP SIGNATURE-----\n'), + 'Message does not start with signature header.') + self.assertTrue( + message.get_payload(1).get_payload().endswith( + '-----END PGP SIGNATURE-----\n'), + 'Message does not end with signature footer.') + return message + + def verify(message): + # replace EOL before verifying (according to rfc3156) + signed_text = re.sub('\r?\n', '\r\n', + message.get_payload(0).as_string()) + + def assert_verify(key): + self.assertTrue(ADDRESS_2 in key.address, + 'Signature could not be verified.') + + d = self._km.verify( + signed_text, ADDRESS_2, openpgp.OpenPGPKey, + detached_sig=message.get_payload(1).get_payload()) + d.addCallback(assert_verify) + return d + + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, recipient) + d.addCallback(check_signed) + d.addCallback(verify) + return d + + def _assert_encrypted(self, res): + message, _ = res self.assertTrue('Content-Type' in message) self.assertEqual('multipart/encrypted', message.get_content_type()) self.assertEqual('application/pgp-encrypted', @@ -128,58 +184,4 @@ class TestOutgoingMail(TestCaseWithKeyManager): message.get_payload(0).get_content_type()) self.assertEqual('application/octet-stream', message.get_payload(1).get_content_type()) - # decrypt and verify - privkey = self._km.get_key( - ADDRESS, openpgp.OpenPGPKey, private=True) - pubkey = self._km.get_key(ADDRESS_2, openpgp.OpenPGPKey) - decrypted = self._km.decrypt( - message.get_payload(1).get_payload(), privkey, verify=pubkey) - self.assertEqual( - '\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n\r\n--\r\n' + - 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n', - decrypted, - 'Decrypted text differs from plaintext.') - - def test_message_sign(self): - """ - Test if message is signed with sender key. - """ - # mock the key fetching - self._km.fetch_keys_from_server = Mock(return_value=[]) - recipient = User('ihavenopubkey@nonleap.se', - 'gateway.leap.se', self.proto, ADDRESS) - self.outgoing_mail = OutgoingMail(self.fromAddr, self._km, self._config['cert'], self._config['key'], - self._config['host'], self._config['port']) - - message, _ = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, recipient) - - # assert structure of signed message - self.assertTrue('Content-Type' in message) - self.assertEqual('multipart/signed', message.get_content_type()) - self.assertEqual('application/pgp-signature', - message.get_param('protocol')) - self.assertEqual('pgp-sha512', message.get_param('micalg')) - # assert content of message - self.assertEqual( - '\r\n'.join(self.EMAIL_DATA[9:13]) + '\r\n--\r\n' + - 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n', - message.get_payload(0).get_payload(decode=True)) - # assert content of signature - self.assertTrue( - message.get_payload(1).get_payload().startswith( - '-----BEGIN PGP SIGNATURE-----\n'), - 'Message does not start with signature header.') - self.assertTrue( - message.get_payload(1).get_payload().endswith( - '-----END PGP SIGNATURE-----\n'), - 'Message does not end with signature footer.') - # assert signature is valid - pubkey = self._km.get_key(ADDRESS_2, openpgp.OpenPGPKey) - # replace EOL before verifying (according to rfc3156) - signed_text = re.sub('\r?\n', '\r\n', - message.get_payload(0).as_string()) - self.assertTrue( - self._km.verify(signed_text, - pubkey, - detached_sig=message.get_payload(1).get_payload()), - 'Signature could not be verified.') + return message -- cgit v1.2.3 From 6ede495b94501a4cbdfd985dcdf4be4f582bbb9b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 25 Nov 2014 15:04:26 +0100 Subject: Serializable Models + Soledad Adaptor --- src/leap/mail/adaptors/__init__.py | 0 src/leap/mail/adaptors/models.py | 125 ++++ src/leap/mail/adaptors/soledad.py | 723 +++++++++++++++++++++ src/leap/mail/adaptors/soledad_indexes.py | 112 ++++ src/leap/mail/adaptors/tests/__init__.py | 0 src/leap/mail/adaptors/tests/rfc822.message | 86 +++ src/leap/mail/adaptors/tests/test_models.py | 103 +++ .../mail/adaptors/tests/test_soledad_adaptor.py | 583 +++++++++++++++++ src/leap/mail/constants.py | 21 + src/leap/mail/imap/account.py | 223 +++---- src/leap/mail/imap/fields.py | 132 +--- src/leap/mail/imap/index.py | 90 --- src/leap/mail/imap/interfaces.py | 2 + src/leap/mail/imap/mailbox.py | 76 ++- src/leap/mail/imap/messages.py | 484 +++++--------- src/leap/mail/imap/parser.py | 45 -- src/leap/mail/imap/tests/test_imap.py | 2 + src/leap/mail/imap/tests/utils.py | 15 +- src/leap/mail/interfaces.py | 113 ++++ src/leap/mail/mail.py | 248 +++++++ 20 files changed, 2415 insertions(+), 768 deletions(-) create mode 100644 src/leap/mail/adaptors/__init__.py create mode 100644 src/leap/mail/adaptors/models.py create mode 100644 src/leap/mail/adaptors/soledad.py create mode 100644 src/leap/mail/adaptors/soledad_indexes.py create mode 100644 src/leap/mail/adaptors/tests/__init__.py create mode 100644 src/leap/mail/adaptors/tests/rfc822.message create mode 100644 src/leap/mail/adaptors/tests/test_models.py create mode 100644 src/leap/mail/adaptors/tests/test_soledad_adaptor.py create mode 100644 src/leap/mail/constants.py delete mode 100644 src/leap/mail/imap/index.py delete mode 100644 src/leap/mail/imap/parser.py create mode 100644 src/leap/mail/interfaces.py create mode 100644 src/leap/mail/mail.py diff --git a/src/leap/mail/adaptors/__init__.py b/src/leap/mail/adaptors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/mail/adaptors/models.py b/src/leap/mail/adaptors/models.py new file mode 100644 index 0000000..1648059 --- /dev/null +++ b/src/leap/mail/adaptors/models.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +# models.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Generic Models to be used by the Document Adaptors. +""" +import copy + + +class SerializableModel(object): + """ + A Generic document model, that can be serialized into a dictionary. + + Subclasses of this `SerializableModel` are meant to be added as class + attributes of classes inheriting from DocumentWrapper. + + A subclass __meta__ of this SerializableModel might exist, and contain info + relative to particularities of this model. + + For instance, the use of `__meta__.index` marks the existence of a primary + index in the model, which will be used to do unique queries (in which case + all the other indexed fields in the underlying document will be filled with + the default info contained in the model definition). + """ + + @classmethod + def serialize(klass): + """ + Get a dictionary representation of the public attributes in the model + class. To avoid collisions with builtin functions, any occurrence of an + attribute ended in '_' (like 'type_') will be normalized by removing + the trailing underscore. + + This classmethod is used from within the serialized method of a + DocumentWrapper instance: it provides defaults for the + empty document. + """ + assert isinstance(klass, type) + return _normalize_dict(klass.__dict__) + + +class DocumentWrapper(object): + """ + A Wrapper object that can be manipulated, passed around, and serialized in + a format that the store understands. + It is related to a SerializableModel, which must be specified as the + ``model`` class attribute. The instance of this DocumentWrapper will not + allow any other *public* attributes than those defined in the corresponding + model. + """ + # TODO we could do some very basic type checking here + # TODO set a dirty flag (on __setattr__, whenever the value is != from + # before) + # TODO we could enforce the existence of a correct "model" attribute + # in some other way (other than in the initializer) + + def __init__(self, **kwargs): + if not getattr(self, 'model', None): + raise RuntimeError( + 'DocumentWrapper class needs a model attribute') + + defaults = self.model.serialize() + + if kwargs: + values = copy.deepcopy(defaults) + values.update(kwargs) + else: + values = defaults + + for k, v in values.items(): + k = k.replace('-', '_') + setattr(self, k, v) + + def __setattr__(self, attr, value): + normalized = _normalize_dict(self.model.__dict__) + if not attr.startswith('_') and attr not in normalized: + raise RuntimeError( + "Cannot set attribute because it's not defined " + "in the model: %s" % attr) + object.__setattr__(self, attr, value) + + def serialize(self): + return _normalize_dict(self.__dict__) + + def create(self): + raise NotImplementedError() + + def update(self): + raise NotImplementedError() + + def delete(self): + raise NotImplementedError() + + @classmethod + def get_or_create(self): + raise NotImplementedError() + + @classmethod + def get_all(self): + raise NotImplementedError() + + +def _normalize_dict(_dict): + items = _dict.items() + not_callable = lambda (k, v): not callable(v) + not_private = lambda(k, v): not k.startswith('_') + for cond in not_callable, not_private: + items = filter(cond, items) + items = [(k, v) if not k.endswith('_') else (k[:-1], v) + for (k, v) in items] + items = [(k.replace('-', '_'), v) for (k, v) in items] + return dict(items) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py new file mode 100644 index 0000000..2e25f04 --- /dev/null +++ b/src/leap/mail/adaptors/soledad.py @@ -0,0 +1,723 @@ +# -*- coding: utf-8 -*- +# soledad.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Soledadad MailAdaptor module. +""" +import re +from collections import defaultdict +from email import message_from_string + +from pycryptopp.hash import sha256 +from twisted.internet import defer +from zope.interface import implements + +from leap.common.check import leap_assert, leap_assert_type + +from leap.mail import walk +from leap.mail.adaptors import soledad_indexes as indexes +from leap.mail.constants import INBOX_NAME +from leap.mail.adaptors import models +from leap.mail.imap.mailbox import normalize_mailbox +from leap.mail.utils import lowerdict, first +from leap.mail.utils import stringify_parts_map +from leap.mail.interfaces import IMailAdaptor, IMessageWrapper + + +# TODO +# [ ] Convenience function to create mail specifying subject, date, etc? + + +_MSGID_PATTERN = r"""<([\w@.]+)>""" +_MSGID_RE = re.compile(_MSGID_PATTERN) + + +class DuplicatedDocumentError(Exception): + """ + Raised when a duplicated document is detected. + """ + pass + + +class SoledadDocumentWrapper(models.DocumentWrapper): + """ + A Wrapper object that can be manipulated, passed around, and serialized in + a format that the Soledad Store understands. + + It ensures atomicity of the document operations on creation, update and + deletion. + """ + + # TODO we could also use a _dirty flag (in models) + + # We keep a dictionary with DeferredLocks, that will be + # unique to every subclass of SoledadDocumentWrapper. + _k_locks = defaultdict(defer.DeferredLock) + + @classmethod + def _get_klass_lock(cls): + """ + Get a DeferredLock that is unique for this subclass name. + Used to lock the access to indexes in the `get_or_create` call + for a particular DocumentWrapper. + """ + return cls._k_locks[cls.__name__] + + def __init__(self, **kwargs): + doc_id = kwargs.pop('doc_id', None) + self._doc_id = doc_id + self._lock = defer.DeferredLock() + super(SoledadDocumentWrapper, self).__init__(**kwargs) + + @property + def doc_id(self): + return self._doc_id + + def create(self, store): + """ + Create the documents for this wrapper. + Since this method will not check for duplication, the + responsibility of avoiding duplicates is left to the caller. + + You might be interested in using `get_or_create` classmethod + instead (that's the preferred way of creating documents from + the wrapper object). + + :return: a deferred that will fire when the underlying + Soledad document has been created. + :rtype: Deferred + """ + leap_assert(self._doc_id is None, + "This document already has a doc_id!") + + def update_doc_id(doc): + self._doc_id = doc.doc_id + return doc + d = store.create_doc(self.serialize()) + d.addCallback(update_doc_id) + return d + + def update(self, store): + """ + Update the documents for this wrapper. + + :return: a deferred that will fire when the underlying + Soledad document has been updated. + :rtype: Deferred + """ + # the deferred lock guards against revision conflicts + return self._lock.run(self._update, store) + + def _update(self, store): + leap_assert(self._doc_id is not None, + "Need to create doc before updating") + + def update_and_put_doc(doc): + doc.content.update(self.serialize()) + return store.put_doc(doc) + + d = store.get_doc(self._doc_id) + d.addCallback(update_and_put_doc) + return d + + def delete(self, store): + """ + Delete the documents for this wrapper. + + :return: a deferred that will fire when the underlying + Soledad document has been deleted. + :rtype: Deferred + """ + # the deferred lock guards against conflicts while updating + return self._lock.run(self._delete, store) + + def _delete(self, store): + leap_assert(self._doc_id is not None, + "Need to create doc before deleting") + # XXX might want to flag this DocumentWrapper to avoid + # updating it by mistake. This could go in models.DocumentWrapper + + def delete_doc(doc): + return store.delete_doc(doc) + + d = store.get_doc(self._doc_id) + d.addCallback(delete_doc) + return d + + @classmethod + def get_or_create(cls, store, index, value): + """ + Get a unique DocumentWrapper by index, or create a new one if the + matching query does not exist. + + :param index: the primary index for the model. + :type index: str + :param value: the value to query the primary index. + :type value: str + + :return: a deferred that will be fired with the SoledadDocumentWrapper + matching the index query, either existing or just created. + :rtype: Deferred + """ + return cls._get_klass_lock().run( + cls._get_or_create, store, index, value) + + @classmethod + def _get_or_create(cls, store, index, value): + assert store is not None + assert index is not None + assert value is not None + + def get_main_index(): + try: + return cls.model.__meta__.index + except AttributeError: + raise RuntimeError("The model is badly defined") + + def try_to_get_doc_from_index(indexes): + values = [] + idx_def = dict(indexes)[index] + if len(idx_def) == 1: + values = [value] + else: + main_index = get_main_index() + fields = cls.model.serialize() + for field in idx_def: + if field == main_index: + values.append(value) + else: + values.append(fields[field]) + d = store.get_from_index(index, *values) + return d + + def get_first_doc_if_any(docs): + if not docs: + return None + if len(docs) > 1: + raise DuplicatedDocumentError + return docs[0] + + def wrap_existing_or_create_new(doc): + if doc: + return cls(doc_id=doc.doc_id, **doc.content) + else: + return create_and_wrap_new_doc() + + def create_and_wrap_new_doc(): + # XXX use closure to store indexes instead of + # querying for them again. + d = store.list_indexes() + d.addCallback(get_wrapper_instance_from_index) + d.addCallback(return_wrapper_when_created) + return d + + def get_wrapper_instance_from_index(indexes): + init_values = {} + idx_def = dict(indexes)[index] + if len(idx_def) == 1: + init_value = {idx_def[0]: value} + return cls(**init_value) + main_index = get_main_index() + fields = cls.model.serialize() + for field in idx_def: + if field == main_index: + init_values[field] = value + else: + init_values[field] = fields[field] + return cls(**init_values) + + def return_wrapper_when_created(wrapper): + d = wrapper.create(store) + d.addCallback(lambda doc: wrapper) + return d + + d = store.list_indexes() + d.addCallback(try_to_get_doc_from_index) + d.addCallback(get_first_doc_if_any) + d.addCallback(wrap_existing_or_create_new) + return d + + @classmethod + def get_all(cls, store): + """ + Get a collection of wrappers around all the documents belonging + to this kind. + + For this to work, the model.__meta__ needs to include a tuple with + the index to be used for listing purposes, and which is the field to be + used to query the index. + + Note that this method only supports indexes of a single field at the + moment. It also might be too expensive to return all the documents + matching the query, so handle with care. + + class __meta__(object): + index = "name" + list_index = ("by-type", "type_") + + :return: a deferred that will be fired with an iterable containing + as many SoledadDocumentWrapper are matching the index defined + in the model as the `list_index`. + :rtype: Deferred + """ + # TODO + # [ ] extend support to indexes with n-ples + # [ ] benchmark the cost of querying and returning indexes in a big + # database. This might badly need pagination before being put to + # serious use. + return cls._get_klass_lock().run(cls._get_all, store) + + @classmethod + def _get_all(cls, store): + try: + list_index, list_attr = cls.model.__meta__.list_index + except AttributeError: + raise RuntimeError("The model is badly defined: no list_index") + try: + index_value = getattr(cls.model, list_attr) + except AttributeError: + raise RuntimeError("The model is badly defined: " + "no attribute matching list_index") + + def wrap_docs(docs): + return (cls(doc_id=doc.doc_id, **doc.content) for doc in docs) + + d = store.get_from_index(list_index, index_value) + d.addCallback(wrap_docs) + return d + + # TODO + # [ ] get_count() ??? + + def __repr__(self): + try: + idx = getattr(self, self.model.__meta__.index) + except AttributeError: + idx = "" + return "<%s: %s (%s)>" % (self.__class__.__name__, + idx, self._doc_id) + + +# +# Message documents +# + +class FlagsDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "flags" + chash = "" + + mbox = "inbox" + seen = False + deleted = False + recent = False + multi = False + flags = [] + tags = [] + size = 0 + + class __meta__(object): + index = "mbox" + + +class HeaderDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "head" + chash = "" + + date = "" + subject = "" + headers = {} + part_map = {} + body = "" # link to phash of body + msgid = "" + multi = False + + class __meta__(object): + index = "chash" + + +class ContentDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "cnt" + phash = "" + + ctype = "" # XXX index by ctype too? + lkf = [] # XXX not implemented yet! + raw = "" + + content_disposition = "" + content_transfer_encoding = "" + content_type = "" + + class __meta__(object): + index = "phash" + + +class MessageWrapper(object): + + # TODO generalize wrapper composition? + # This could benefit of a DeferredLock to create/update all the + # documents at the same time maybe, and defend against concurrent updates? + + implements(IMessageWrapper) + + def __init__(self, fdoc, hdoc, cdocs=None): + """ + Need at least a flag-document and a header-document to instantiate a + MessageWrapper. Content-documents can be retrieved lazily. + + cdocs, if any, should be a dictionary in which the keys are ascending + integers, beginning at one, and the values are dictionaries with the + content of the content-docs. + """ + self.fdoc = FlagsDocWrapper(**fdoc) + self.hdoc = HeaderDocWrapper(**hdoc) + if cdocs is None: + cdocs = {} + cdocs_keys = cdocs.keys() + assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1) + self.cdocs = dict([(key, ContentDocWrapper(**doc)) for (key, doc) in + cdocs.items()]) + + def create(self, store): + """ + Create all the parts for this message in the store. + """ + leap_assert(self.cdocs, + "Need non empty cdocs to create the " + "MessageWrapper documents") + leap_assert(self.fdoc.doc_id is None, + "Cannot create: fdoc has a doc_id") + + # TODO I think we need to tolerate the no hdoc.doc_id case, for when we + # are doing a copy to another mailbox. + leap_assert(self.hdoc.doc_id is None, + "Cannot create: hdoc has a doc_id") + d = [] + d.append(self.fdoc.create(store)) + d.append(self.hdoc.create(store)) + for cdoc in self.cdocs.values(): + if cdoc.doc_id is not None: + # we could be just linking to an existing + # content-doc. + continue + d.append(cdoc.create(store)) + return defer.gatherResults(d) + + def update(self, store): + """ + Update the only mutable parts, which are within the flags document. + """ + return self.fdoc.update(store) + + def delete(self, store): + # Eventually this would have to do the duplicate search or send for the + # garbage collector. At least the fdoc can be unlinked. + raise NotImplementedError() + +# +# Mailboxes +# + + +class MailboxWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "mbox" + mbox = INBOX_NAME + flags = [] + closed = False + subscribed = False + rw = True + + class __meta__(object): + index = "mbox" + list_index = (indexes.TYPE_IDX, 'type_') + + +# +# Soledad Adaptor +# + +# TODO make this an interface? +class SoledadIndexMixin(object): + """ + this will need a class attribute `indexes`, that is a dictionary containing + the index definitions for the underlying u1db store underlying soledad. + + It needs to be in the following format: + {'index-name': ['field1', 'field2']} + """ + # TODO could have a wrapper class for indexes, supporting introspection + # and __getattr__ + indexes = {} + + store_ready = False + _index_creation_deferreds = [] + + # TODO we might want to move this logic to soledad itself + # so that each application can pass a set of indexes for their data model. + # TODO check also the decorator used in keymanager for waiting for indexes + # to be ready. + + def initialize_store(self, store): + """ + Initialize the indexes in the database. + + :param store: store + :returns: a Deferred that will fire when the store is correctly + initialized. + :rtype: deferred + """ + # TODO I think we *should* get another deferredLock in here, but + # global to the soledad namespace, to protect from several points + # initializing soledad indexes at the same time. + + leap_assert(store, "Need a store") + leap_assert_type(self.indexes, dict) + self._index_creation_deferreds = [] + + def _on_indexes_created(ignored): + self.store_ready = True + + def _create_index(name, expression): + d = store.create_index(name, *expression) + self._index_creation_deferreds.append(d) + + def _create_indexes(db_indexes): + db_indexes = dict(db_indexes) + + for name, expression in self.indexes.items(): + if name not in db_indexes: + # The index does not yet exist. + _create_index(name, expression) + continue + + if expression == db_indexes[name]: + # The index exists and is up to date. + continue + # The index exists but the definition is not what expected, so + # we delete it and add the proper index expression. + d1 = store.delete_index(name) + d1.addCallback(lambda _: _create_index(name, expression)) + + all_created = defer.gatherResults(self._index_creation_deferreds) + all_created.addCallback(_on_indexes_created) + return all_created + + # Ask the database for currently existing indexes, and create them + # if not found. + d = store.list_indexes() + d.addCallback(_create_indexes) + return d + + +class SoledadMailAdaptor(SoledadIndexMixin): + + implements(IMailAdaptor) + store = None + + indexes = indexes.MAIL_INDEXES + + # Message handling + + def get_msg_from_string(self, MessageClass, raw_msg): + """ + Get an instance of a MessageClass initialized with a MessageWrapper + that contains all the parts obtained from parsing the raw string for + the message. + + :param MessageClass: any Message class that can be initialized passing + an instance of an IMessageWrapper implementor. + :type MessageClass: type + :param raw_msg: a string containing the raw email message. + :type raw_msg: str + :rtype: MessageClass instance. + """ + assert(MessageClass is not None) + fdoc, hdoc, cdocs = _split_into_parts(raw_msg) + return self.get_msg_from_docs( + MessageClass, fdoc, hdoc, cdocs) + + def get_msg_from_docs(self, MessageClass, fdoc, hdoc, cdocs=None): + """ + Get an instance of a MessageClass initialized with a MessageWrapper + that contains the passed part documents. + + This is not the recommended way of obtaining a message, unless you know + how to take care of ensuring the internal consistency between the part + documents, or unless you are glueing together the part documents that + have been previously generated by `get_msg_from_string`. + + :param MessageClass: any Message class that can be initialized passing + an instance of an IMessageWrapper implementor. + :type MessageClass: type + :param fdoc: a dictionary containing values from which a + FlagsDocWrapper can be initialized + :type fdoc: dict + :param hdoc: a dictionary containing values from which a + HeaderDocWrapper can be initialized + :type hdoc: dict + :param cdocs: None, or a dictionary mapping integers (1-indexed) to + dicts from where a ContentDocWrapper can be initialized. + :type cdocs: dict, or None + + :rtype: MessageClass instance. + """ + assert(MessageClass is not None) + return MessageClass(MessageWrapper(fdoc, hdoc, cdocs)) + + def create_msg(self, store, msg): + """ + :param store: an instance of soledad, or anything that behaves alike + :type store: + :param msg: a Message object. + + :return: a Deferred that is fired when all the underlying documents + have been created. + :rtype: defer.Deferred + """ + wrapper = msg.get_wrapper() + return wrapper.create(store) + + def update_msg(self, store, msg): + """ + :param msg: a Message object. + :param store: an instance of soledad, or anything that behaves alike + :type store: + :param msg: a Message object. + :return: a Deferred that is fired when all the underlying documents + have been updated (actually, it's only the fdoc that's allowed + to update). + :rtype: defer.Deferred + """ + wrapper = msg.get_wrapper() + return wrapper.update(store) + + # Mailbox handling + + def get_or_create_mbox(self, store, name): + """ + Get the mailbox with the given name, or creatre one if it does not + exist. + + :param name: the name of the mailbox + :type name: str + """ + index = indexes.TYPE_MBOX_IDX + mbox = normalize_mailbox(name) + return MailboxWrapper.get_or_create(store, index, mbox) + + def update_mbox(self, store, mbox_wrapper): + """ + Update the documents for a given mailbox. + :param mbox_wrapper: MailboxWrapper instance + :type mbox_wrapper: MailboxWrapper + :return: a Deferred that will be fired when the mailbox documents + have been updated. + :rtype: defer.Deferred + """ + return mbox_wrapper.update(store) + + def get_all_mboxes(self, store): + """ + Retrieve a list with wrappers for all the mailboxes. + + :return: a deferred that will be fired with a list of all the + MailboxWrappers found. + :rtype: defer.Deferred + """ + return MailboxWrapper.get_all(store) + + +def _split_into_parts(raw): + # TODO signal that we can delete the original message!----- + # when all the processing is done. + # TODO add the linked-from info ! + # TODO add reference to the original message? + # TODO populate Default FLAGS/TAGS (unseen?) + # TODO seed propely the content_docs with defaults?? + + msg, parts, chash, size, multi = _parse_msg(raw) + body_phash_fun = [walk.get_body_phash_simple, + walk.get_body_phash_multi][int(multi)] + body_phash = body_phash_fun(walk.get_payloads(msg)) + parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) + + fdoc = _build_flags_doc(chash, size, multi) + hdoc = _build_headers_doc(msg, chash, parts_map) + + # The MessageWrapper expects a dict, one-indexed + cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1)) + + # XXX convert each to_dicts... + return fdoc, hdoc, cdocs + + +def _parse_msg(raw): + msg = message_from_string(raw) + parts = walk.get_parts(msg) + size = len(raw) + chash = sha256.SHA256(raw).hexdigest() + multi = msg.is_multipart() + return msg, parts, chash, size, multi + + +def _build_flags_doc(chash, size, multi): + _fdoc = FlagsDocWrapper(chash=chash, size=size, multi=multi) + return _fdoc.serialize() + + +def _build_headers_doc(msg, chash, parts_map): + """ + Assemble a headers document from the original parsed message, the + content-hash, and the parts map. + + It takes into account possibly repeated headers. + """ + headers = defaultdict(list) + for k, v in msg.items(): + headers[k].append(v) + + # "fix" for repeated headers. + for k, v in headers.items(): + newline = "\n%s: " % (k,) + headers[k] = newline.join(v) + + lower_headers = lowerdict(headers) + msgid = first(_MSGID_RE.findall( + lower_headers.get('message-id', ''))) + + _hdoc = HeaderDocWrapper( + chash=chash, headers=lower_headers, msgid=msgid) + + def copy_attr(headers, key, doc): + if key in headers: + setattr(doc, key, headers[key]) + + copy_attr(lower_headers, "subject", _hdoc) + copy_attr(lower_headers, "date", _hdoc) + + hdoc = _hdoc.serialize() + # add parts map to header doc + # (body, multi, part_map) + for key in parts_map: + hdoc[key] = parts_map[key] + return stringify_parts_map(hdoc) diff --git a/src/leap/mail/adaptors/soledad_indexes.py b/src/leap/mail/adaptors/soledad_indexes.py new file mode 100644 index 0000000..f3e990d --- /dev/null +++ b/src/leap/mail/adaptors/soledad_indexes.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- +# soledad_indexes.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Soledad Indexes for Mail Documents. +""" + +# TODO +# [ ] hide most of the constants here + +# Document Type, for indexing + +TYPE = "type" +MBOX = "mbox" +FLAGS = "flags" +HEADERS = "head" +CONTENT = "cnt" +RECENT = "rct" +HDOCS_SET = "hdocset" + +INCOMING_KEY = "incoming" +ERROR_DECRYPTING_KEY = "errdecr" + +# indexing keys +CONTENT_HASH = "chash" +PAYLOAD_HASH = "phash" +MSGID = "msgid" +UID = "uid" + + +# Index types +# -------------- + +TYPE_IDX = 'by-type' +TYPE_MBOX_IDX = 'by-type-and-mbox' +#TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' +TYPE_SUBS_IDX = 'by-type-and-subscribed' +TYPE_MSGID_IDX = 'by-type-and-message-id' +TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' +TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' +TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' +TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash' +TYPE_C_HASH_IDX = 'by-type-and-contenthash' +TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' +TYPE_P_HASH_IDX = 'by-type-and-payloadhash' + +# Soledad index for incoming mail, without decrypting errors. +# and the backward-compatible index, will be deprecated at 0.7 +JUST_MAIL_IDX = "just-mail" +JUST_MAIL_COMPAT_IDX = "just-mail-compat" + +# Tomas created the `recent and seen index`, but the semantic is not too +# correct since the recent flag is volatile --- XXX review and delete. +#TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' + +# TODO +# it would be nice to measure the cost of indexing +# by many fields. + +# TODO +# make the indexes dict more readable! + +MAIL_INDEXES = { + # generic + TYPE_IDX: [TYPE], + TYPE_MBOX_IDX: [TYPE, MBOX], + + # XXX deprecate 0.4.0 + # TYPE_MBOX_UID_IDX: [TYPE, MBOX, UID], + + # mailboxes + TYPE_SUBS_IDX: [TYPE, 'bool(subscribed)'], + + # fdocs uniqueness + TYPE_MBOX_C_HASH_IDX: [TYPE, MBOX, CONTENT_HASH], + + # headers doc - search by msgid. + TYPE_MSGID_IDX: [TYPE, MSGID], + + # content, headers doc + TYPE_C_HASH_IDX: [TYPE, CONTENT_HASH], + + # attachment payload dedup + TYPE_P_HASH_IDX: [TYPE, PAYLOAD_HASH], + + # messages + TYPE_MBOX_SEEN_IDX: [TYPE, MBOX, 'bool(seen)'], + TYPE_MBOX_RECT_IDX: [TYPE, MBOX, 'bool(recent)'], + TYPE_MBOX_DEL_IDX: [TYPE, MBOX, 'bool(deleted)'], + #TYPE_MBOX_RECT_SEEN_IDX: [TYPE, MBOX, + #'bool(recent)', 'bool(seen)'], + + # incoming queue + JUST_MAIL_IDX: [INCOMING_KEY, + "bool(%s)" % (ERROR_DECRYPTING_KEY,)], + + # the backward-compatible index, will be deprecated at 0.7 + JUST_MAIL_COMPAT_IDX: [INCOMING_KEY], +} diff --git a/src/leap/mail/adaptors/tests/__init__.py b/src/leap/mail/adaptors/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/mail/adaptors/tests/rfc822.message b/src/leap/mail/adaptors/tests/rfc822.message new file mode 100644 index 0000000..ee97ab9 --- /dev/null +++ b/src/leap/mail/adaptors/tests/rfc822.message @@ -0,0 +1,86 @@ +Return-Path: +Delivered-To: exarkun@meson.dyndns.org +Received: from localhost [127.0.0.1] + by localhost with POP3 (fetchmail-6.2.1) + for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) +Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) + by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 + for ; Thu, 20 Mar 2003 14:49:27 -0500 (EST) +Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) + by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) + id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 +Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) + id 18w63j-0007VK-00 + for ; Thu, 20 Mar 2003 13:50:39 -0600 +To: twisted-commits@twistedmatrix.com +From: etrepum CVS +Reply-To: twisted-python@twistedmatrix.com +X-Mailer: CVSToys +Message-Id: +Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. +Sender: twisted-commits-admin@twistedmatrix.com +Errors-To: twisted-commits-admin@twistedmatrix.com +X-BeenThere: twisted-commits@twistedmatrix.com +X-Mailman-Version: 2.0.11 +Precedence: bulk +List-Help: +List-Post: +List-Subscribe: , + +List-Id: +List-Unsubscribe: , + +List-Archive: +Date: Thu, 20 Mar 2003 13:50:39 -0600 + +Modified files: +Twisted/twisted/python/rebuild.py 1.19 1.20 + +Log message: +rebuild now works on python versions from 2.2.0 and up. + + +ViewCVS links: +http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted + +Index: Twisted/twisted/python/rebuild.py +diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 +--- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 ++++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 +@@ -206,15 +206,27 @@ + clazz.__dict__.clear() + clazz.__getattr__ = __getattr__ + clazz.__module__ = module.__name__ ++ if newclasses: ++ import gc ++ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): ++ hasBrokenRebuild = 1 ++ gc_objects = gc.get_objects() ++ else: ++ hasBrokenRebuild = 0 + for nclass in newclasses: + ga = getattr(module, nclass.__name__) + if ga is nclass: + log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) + else: +- import gc +- for r in gc.get_referrers(nclass): +- if isinstance(r, nclass): ++ if hasBrokenRebuild: ++ for r in gc_objects: ++ if not getattr(r, '__class__', None) is nclass: ++ continue + r.__class__ = ga ++ else: ++ for r in gc.get_referrers(nclass): ++ if getattr(r, '__class__', None) is nclass: ++ r.__class__ = ga + if doLog: + log.msg('') + log.msg(' (fixing %s): ' % str(module.__name__)) + + +_______________________________________________ +Twisted-commits mailing list +Twisted-commits@twistedmatrix.com +http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits diff --git a/src/leap/mail/adaptors/tests/test_models.py b/src/leap/mail/adaptors/tests/test_models.py new file mode 100644 index 0000000..efe0bf2 --- /dev/null +++ b/src/leap/mail/adaptors/tests/test_models.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# test_models.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the leap.mail.adaptors.models module. +""" +from twisted.trial import unittest + +from leap.mail.adaptors import models + + +class SerializableModelsTestCase(unittest.TestCase): + + def test_good_serialized_model(self): + + class M(models.SerializableModel): + foo = 42 + bar = 33 + baaz_ = None + _nope = 0 + __nope = 0 + + def not_today(self): + pass + + class IgnoreMe(object): + pass + + killmeplease = lambda x: x + + serialized = M.serialize() + expected = {'foo': 42, 'bar': 33, 'baaz': None} + self.assertEqual(serialized, expected) + + +class DocumentWrapperTestCase(unittest.TestCase): + + def test_wrapper_defaults(self): + + class Wrapper(models.DocumentWrapper): + class model(models.SerializableModel): + foo = 42 + bar = 11 + + wrapper = Wrapper() + wrapper._ignored = True + serialized = wrapper.serialize() + expected = {'foo': 42, 'bar': 11} + self.assertEqual(serialized, expected) + + def test_initialized_wrapper(self): + + class Wrapper(models.DocumentWrapper): + class model(models.SerializableModel): + foo = 42 + bar_ = 11 + + wrapper = Wrapper(foo=0, bar=-1) + serialized = wrapper.serialize() + expected = {'foo': 0, 'bar': -1} + self.assertEqual(serialized, expected) + + wrapper.foo = 23 + serialized = wrapper.serialize() + expected = {'foo': 23, 'bar': -1} + self.assertEqual(serialized, expected) + + wrapper = Wrapper(foo=0) + serialized = wrapper.serialize() + expected = {'foo': 0, 'bar': 11} + self.assertEqual(serialized, expected) + + def test_invalid_initialized_wrapper(self): + + class Wrapper(models.DocumentWrapper): + class model(models.SerializableModel): + foo = 42 + getwrapper = lambda: Wrapper(bar=1) + self.assertRaises(RuntimeError, getwrapper) + + def test_no_model_wrapper(self): + + class Wrapper(models.DocumentWrapper): + pass + + def getwrapper(): + w = Wrapper() + w.foo = None + + self.assertRaises(RuntimeError, getwrapper) diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py new file mode 100644 index 0000000..657a602 --- /dev/null +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -0,0 +1,583 @@ +# -*- coding: utf-8 -*- +# test_soledad_adaptor.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the Soledad Adaptor module - leap.mail.adaptors.soledad +""" +import os +import shutil +import tempfile + +from functools import partial + +from twisted.internet import defer +from twisted.trial import unittest + +from leap.common.testing.basetest import BaseLeapTest +from leap.mail.adaptors import models +from leap.mail.adaptors.soledad import SoledadDocumentWrapper +from leap.mail.adaptors.soledad import SoledadIndexMixin +from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.soledad.client import Soledad + +TEST_USER = "testuser@leap.se" +TEST_PASSWD = "1234" + +# DEBUG +# import logging +# logging.basicConfig(level=logging.DEBUG) + + +def initialize_soledad(email, gnupg_home, tempdir): + """ + Initializes soledad by hand + + :param email: ID for the user + :param gnupg_home: path to home used by gnupg + :param tempdir: path to temporal dir + :rtype: Soledad instance + """ + + uuid = "foobar-uuid" + passphrase = u"verysecretpassphrase" + secret_path = os.path.join(tempdir, "secret.gpg") + local_db_path = os.path.join(tempdir, "soledad.u1db") + server_url = "https://provider" + cert_file = "" + + soledad = Soledad( + uuid, + passphrase, + secret_path, + local_db_path, + server_url, + cert_file, + syncable=False) + + return soledad + + +# TODO move to common module +# XXX remove duplication +class SoledadTestMixin(BaseLeapTest): + """ + It is **VERY** important that this base is added *AFTER* unittest.TestCase + """ + + def setUp(self): + self.results = [] + + self.old_path = os.environ['PATH'] + self.old_home = os.environ['HOME'] + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.home = self.tempdir + bin_tdir = os.path.join( + self.tempdir, + 'bin') + os.environ["PATH"] = bin_tdir + os.environ["HOME"] = self.tempdir + + # Soledad: config info + self.gnupg_home = "%s/gnupg" % self.tempdir + self.email = 'leap@leap.se' + + # initialize soledad by hand so we can control keys + self._soledad = initialize_soledad( + self.email, + self.gnupg_home, + self.tempdir) + + def tearDown(self): + """ + tearDown method called after each test. + """ + self.results = [] + try: + self._soledad.close() + except Exception as exc: + print "ERROR WHILE CLOSING SOLEDAD" + # logging.exception(exc) + finally: + os.environ["PATH"] = self.old_path + os.environ["HOME"] = self.old_home + # safety check + assert 'leap_tests-' in self.tempdir + shutil.rmtree(self.tempdir) + + +class CounterWrapper(SoledadDocumentWrapper): + class model(models.SerializableModel): + counter = 0 + flag = None + + +class CharacterWrapper(SoledadDocumentWrapper): + class model(models.SerializableModel): + name = "" + age = 20 + + +class ActorWrapper(SoledadDocumentWrapper): + class model(models.SerializableModel): + type_ = "actor" + name = None + + class __meta__(object): + index = "name" + list_index = ("by-type", "type_") + + +class TestAdaptor(SoledadIndexMixin): + indexes = {'by-name': ['name'], + 'by-type-and-name': ['type', 'name'], + 'by-type': ['type']} + + +class SoledadDocWrapperTestCase(unittest.TestCase, SoledadTestMixin): + """ + Tests for the SoledadDocumentWrapper. + """ + def assert_num_docs(self, num, docs): + self.assertEqual(len(docs[1]), num) + + def test_create_single(self): + + store = self._soledad + wrapper = CounterWrapper() + + def assert_one_doc(docs): + self.assertEqual(docs[0], 1) + + d = wrapper.create(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(assert_one_doc) + return d + + def test_create_many(self): + + store = self._soledad + w1 = CounterWrapper() + w2 = CounterWrapper(counter=1) + w3 = CounterWrapper(counter=2) + w4 = CounterWrapper(counter=3) + w5 = CounterWrapper(counter=4) + + d1 = [w1.create(store), + w2.create(store), + w3.create(store), + w4.create(store), + w5.create(store)] + + d = defer.gatherResults(d1) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 5)) + return d + + def test_multiple_updates(self): + + store = self._soledad + wrapper = CounterWrapper(counter=1) + MAX = 100 + + def assert_doc_id(doc): + self.assertTrue(wrapper._doc_id is not None) + return doc + + def assert_counter_initial_ok(doc): + self.assertEqual(wrapper.counter, 1) + + def increment_counter(ignored): + d1 = [] + + def record_revision(revision): + rev = int(revision.split(':')[1]) + self.results.append(rev) + + for i in list(range(MAX)): + wrapper.counter += 1 + wrapper.flag = i % 2 == 0 + d = wrapper.update(store) + d.addCallback(record_revision) + d1.append(d) + + return defer.gatherResults(d1) + + def assert_counter_final_ok(doc): + self.assertEqual(doc.content['counter'], MAX + 1) + self.assertEqual(doc.content['flag'], False) + + def assert_results_ordered_list(ignored): + self.assertEqual(self.results, sorted(range(2, MAX + 2))) + + d = wrapper.create(store) + d.addCallback(assert_doc_id) + d.addCallback(assert_counter_initial_ok) + d.addCallback(increment_counter) + d.addCallback(lambda _: store.get_doc(wrapper._doc_id)) + d.addCallback(assert_counter_final_ok) + d.addCallback(assert_results_ordered_list) + return d + + def test_delete(self): + adaptor = TestAdaptor() + store = self._soledad + + wrapper_list = [] + + def get_or_create_bob(ignored): + def add_to_list(wrapper): + wrapper_list.append(wrapper) + return wrapper + wrapper = CharacterWrapper.get_or_create( + store, 'by-name', 'bob') + wrapper.addCallback(add_to_list) + return wrapper + + def delete_bob(ignored): + wrapper = wrapper_list[0] + return wrapper.delete(store) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + # this should create bob document + d.addCallback(get_or_create_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + d.addCallback(delete_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + return d + + def test_get_or_create(self): + adaptor = TestAdaptor() + store = self._soledad + + def get_or_create_bob(ignored): + wrapper = CharacterWrapper.get_or_create( + store, 'by-name', 'bob') + return wrapper + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + # this should create bob document + d.addCallback(get_or_create_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + # this should get us bob document + d.addCallback(get_or_create_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + return d + + def test_get_or_create_multi_index(self): + adaptor = TestAdaptor() + store = self._soledad + + def get_or_create_actor_harry(ignored): + wrapper = ActorWrapper.get_or_create( + store, 'by-type-and-name', 'harrison') + return wrapper + + def create_director_harry(ignored): + wrapper = ActorWrapper(name="harrison", type="director") + return wrapper.create(store) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + # this should create harrison document + d.addCallback(get_or_create_actor_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + # this should get us harrison document + d.addCallback(get_or_create_actor_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + # create director harry, should create new doc + d.addCallback(create_director_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 2)) + + # this should get us harrison document, still 2 docs + d.addCallback(get_or_create_actor_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 2)) + return d + + def test_get_all(self): + adaptor = TestAdaptor() + store = self._soledad + actor_names = ["harry", "carrie", "mark", "david"] + + def create_some_actors(ignored): + deferreds = [] + for name in actor_names: + dw = ActorWrapper.get_or_create( + store, 'by-type-and-name', name) + deferreds.append(dw) + return defer.gatherResults(deferreds) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + d.addCallback(create_some_actors) + + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 4)) + + def assert_actor_list_is_expected(res): + got = set([actor.name for actor in res]) + expected = set(actor_names) + self.assertEqual(got, expected) + + d.addCallback(lambda _: ActorWrapper.get_all(store)) + d.addCallback(assert_actor_list_is_expected) + return d + +here = os.path.split(os.path.abspath(__file__))[0] + + +class TestMessageClass(object): + def __init__(self, wrapper): + self.wrapper = wrapper + + def get_wrapper(self): + return self.wrapper + + +class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): + """ + Tests for the SoledadMailAdaptor. + """ + + def get_adaptor(self): + adaptor = SoledadMailAdaptor() + adaptor.store = self._soledad + return adaptor + + def assert_num_docs(self, num, docs): + self.assertEqual(len(docs[1]), num) + + def test_mail_adaptor_init(self): + adaptor = self.get_adaptor() + self.assertTrue(isinstance(adaptor.indexes, dict)) + self.assertTrue(len(adaptor.indexes) != 0) + + # Messages + + def test_get_msg_from_string(self): + adaptor = self.get_adaptor() + + with open(os.path.join(here, "rfc822.message")) as f: + raw = f.read() + + msg = adaptor.get_msg_from_string(TestMessageClass, raw) + + chash = ("D27B2771C0DCCDCB468EE65A4540438" + "09DBD11588E87E951545BE0CBC321C308") + phash = ("64934534C1C80E0D4FA04BE1CCBA104" + "F07BCA5F469C86E2C0ABE1D41310B7299") + subject = ("[Twisted-commits] rebuild now works on " + "python versions from 2.2.0 and up.") + self.assertTrue(msg.wrapper.fdoc is not None) + self.assertTrue(msg.wrapper.hdoc is not None) + self.assertTrue(msg.wrapper.cdocs is not None) + self.assertEquals(len(msg.wrapper.cdocs), 1) + self.assertEquals(msg.wrapper.fdoc.chash, chash) + self.assertEquals(msg.wrapper.fdoc.size, 3834) + self.assertEquals(msg.wrapper.hdoc.chash, chash) + self.assertEqual(msg.wrapper.hdoc.headers['subject'], + subject) + self.assertEqual(msg.wrapper.hdoc.subject, subject) + self.assertEqual(msg.wrapper.cdocs[1].phash, phash) + + def test_get_msg_from_docs(self): + adaptor = self.get_adaptor() + fdoc = dict( + mbox="Foobox", + flags=('\Seen', '\Nice'), + tags=('Personal', 'TODO'), + seen=False, deleted=False, + recent=False, multi=False) + hdoc = dict( + subject="Test Msg") + cdocs = { + 1: dict( + raw='This is a test message')} + + msg = adaptor.get_msg_from_docs( + TestMessageClass, fdoc, hdoc, cdocs=cdocs) + self.assertEqual(msg.wrapper.fdoc.flags, + ('\Seen', '\Nice')) + self.assertEqual(msg.wrapper.fdoc.tags, + ('Personal', 'TODO')) + self.assertEqual(msg.wrapper.fdoc.mbox, "Foobox") + self.assertEqual(msg.wrapper.hdoc.multi, False) + self.assertEqual(msg.wrapper.hdoc.subject, + "Test Msg") + self.assertEqual(msg.wrapper.cdocs[1].raw, + "This is a test message") + + def test_create_msg(self): + adaptor = self.get_adaptor() + + with open(os.path.join(here, "rfc822.message")) as f: + raw = f.read() + msg = adaptor.get_msg_from_string(TestMessageClass, raw) + + def check_create_result(created): + self.assertEqual(len(created), 3) + for doc in created: + self.assertTrue( + doc.__class__.__name__, + "SoledadDocument") + + d = adaptor.create_msg(adaptor.store, msg) + d.addCallback(check_create_result) + return d + + def test_update_msg(self): + adaptor = self.get_adaptor() + with open(os.path.join(here, "rfc822.message")) as f: + raw = f.read() + + def assert_msg_has_doc_id(ignored, msg): + wrapper = msg.get_wrapper() + self.assertTrue(wrapper.fdoc.doc_id is not None) + + def assert_msg_has_no_flags(ignored, msg): + wrapper = msg.get_wrapper() + self.assertEqual(wrapper.fdoc.flags, []) + + def update_msg_flags(ignored, msg): + wrapper = msg.get_wrapper() + wrapper.fdoc.flags = ["This", "That"] + return wrapper.update(adaptor.store) + + def assert_msg_has_flags(ignored, msg): + wrapper = msg.get_wrapper() + self.assertEqual(wrapper.fdoc.flags, ["This", "That"]) + + def get_fdoc_and_check_flags(ignored): + def assert_doc_has_flags(doc): + self.assertEqual(doc.content['flags'], + ['This', 'That']) + wrapper = msg.get_wrapper() + d = adaptor.store.get_doc(wrapper.fdoc.doc_id) + d.addCallback(assert_doc_has_flags) + return d + + msg = adaptor.get_msg_from_string(TestMessageClass, raw) + d = adaptor.create_msg(adaptor.store, msg) + d.addCallback(lambda _: adaptor.store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 3)) + d.addCallback(assert_msg_has_doc_id, msg) + d.addCallback(assert_msg_has_no_flags, msg) + + # update it! + d.addCallback(update_msg_flags, msg) + d.addCallback(assert_msg_has_flags, msg) + d.addCallback(get_fdoc_and_check_flags) + return d + + # Mailboxes + + def test_get_or_create_mbox(self): + adaptor = self.get_adaptor() + + def get_or_create_mbox(ignored): + d = adaptor.get_or_create_mbox(adaptor.store, "Trash") + return d + + def assert_good_doc(mbox_wrapper): + self.assertTrue(mbox_wrapper.doc_id is not None) + self.assertEqual(mbox_wrapper.mbox, "Trash") + self.assertEqual(mbox_wrapper.type, "mbox") + self.assertEqual(mbox_wrapper.closed, False) + self.assertEqual(mbox_wrapper.subscribed, False) + + d = adaptor.initialize_store(adaptor.store) + d.addCallback(get_or_create_mbox) + d.addCallback(assert_good_doc) + d.addCallback(lambda _: adaptor.store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + return d + + def test_update_mbox(self): + adaptor = self.get_adaptor() + + wrapper_ref = [] + + def get_or_create_mbox(ignored): + d = adaptor.get_or_create_mbox(adaptor.store, "Trash") + return d + + def update_wrapper(wrapper, wrapper_ref): + wrapper_ref.append(wrapper) + wrapper.subscribed = True + wrapper.closed = True + d = adaptor.update_mbox(adaptor.store, wrapper) + return d + + def get_mbox_doc_and_check_flags(res, wrapper_ref): + wrapper = wrapper_ref[0] + + def assert_doc_has_flags(doc): + self.assertEqual(doc.content['subscribed'], True) + self.assertEqual(doc.content['closed'], True) + d = adaptor.store.get_doc(wrapper.doc_id) + d.addCallback(assert_doc_has_flags) + return d + + d = adaptor.initialize_store(adaptor.store) + d.addCallback(get_or_create_mbox) + d.addCallback(update_wrapper, wrapper_ref) + d.addCallback(get_mbox_doc_and_check_flags, wrapper_ref) + return d + + def test_get_all_mboxes(self): + adaptor = self.get_adaptor() + mboxes = ("Sent", "Trash", "Personal", "ListFoo") + + def get_or_create_mboxes(ignored): + d = [] + for mbox in mboxes: + d.append(adaptor.get_or_create_mbox( + adaptor.store, mbox)) + return defer.gatherResults(d) + + def get_all_mboxes(ignored): + return adaptor.get_all_mboxes(adaptor.store) + + def assert_mboxes_match_expected(wrappers): + names = [m.mbox for m in wrappers] + self.assertEqual(set(names), set(mboxes)) + + d = adaptor.initialize_store(adaptor.store) + d.addCallback(get_or_create_mboxes) + d.addCallback(get_all_mboxes) + d.addCallback(assert_mboxes_match_expected) + return d diff --git a/src/leap/mail/constants.py b/src/leap/mail/constants.py new file mode 100644 index 0000000..55bf1da --- /dev/null +++ b/src/leap/mail/constants.py @@ -0,0 +1,21 @@ +# *- coding: utf-8 -*- +# constants.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Constants for leap.mail. +""" + +INBOX_NAME = "INBOX" diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index fe466cb..7dfbbd1 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -28,10 +28,10 @@ from twisted.python import log from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type -from leap.mail.imap.index import IndexedDB + +from leap.mail.mail import Account from leap.mail.imap.fields import WithMsgFields -from leap.mail.imap.parser import MBoxParser -from leap.mail.imap.mailbox import SoledadMailbox +from leap.mail.imap.mailbox import SoledadMailbox, normalize_mailbox from leap.soledad.client import Soledad logger = logging.getLogger(__name__) @@ -39,7 +39,6 @@ logger = logging.getLogger(__name__) PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False) if PROFILE_CMD: - def _debugProfiling(result, cmdname, start): took = (time.time() - start) * 1000 log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec") @@ -47,96 +46,43 @@ if PROFILE_CMD: ####################################### -# Soledad Account +# Soledad IMAP Account ####################################### +# TODO remove MsgFields too -# TODO change name to LeapIMAPAccount, since we're using -# the memstore. -# IndexedDB should also not be here anymore. - -class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): +class IMAPAccount(WithMsgFields): """ - An implementation of IAccount and INamespacePresenteer + An implementation of an imap4 Account that is backed by Soledad Encrypted Documents. """ implements(imap4.IAccount, imap4.INamespacePresenter) - _soledad = None selected = None closed = False - _initialized = False - def __init__(self, account_name, soledad, memstore=None): + def __init__(self, user_id, store): """ - Creates a SoledadAccountIndex that keeps track of the mailboxes - and subscriptions handled by this account. + Keeps track of the mailboxes and subscriptions handled by this account. - :param acct_name: The name of the account (user id). - :type acct_name: str + :param account: The name of the account (user id). + :type account: str - :param soledad: a Soledad instance. - :type soledad: Soledad - :param memstore: a MemoryStore instance. - :type memstore: MemoryStore + :param store: a Soledad instance. + :type store: Soledad """ - leap_assert(soledad, "Need a soledad instance to initialize") - leap_assert_type(soledad, Soledad) + # XXX assert a generic store interface instead, so that we + # can plug the memory store wrapper seamlessly. + leap_assert(store, "Need a store instance to initialize") + leap_assert_type(store, Soledad) # XXX SHOULD assert too that the name matches the user/uuid with which # soledad has been initialized. + self.user_id = user_id + self.account = Account(store) - # XXX ??? why is this parsing mailbox name??? it's account... - # userid? homogenize. - self._account_name = self._parse_mailbox_name(account_name) - self._soledad = soledad - self._memstore = memstore - - self.__mailboxes = set([]) - - self._deferred_initialization = defer.Deferred() - self._initialize_storage() - - def _initialize_storage(self): - - def add_mailbox_if_none(result): - # every user should have the right to an inbox folder - # at least, so let's make one! - if not self.mailboxes: - self.addMailbox(self.INBOX_NAME) - - def finish_initialization(result): - self._initialized = True - self._deferred_initialization.callback(None) - - def load_mbox_cache(result): - d = self._load_mailboxes() - d.addCallback(lambda _: result) - return d - - d = self.initialize_db() - - d.addCallback(load_mbox_cache) - d.addCallback(add_mailbox_if_none) - d.addCallback(finish_initialization) - - def callWhenReady(self, cb): - if self._initialized: - cb(self) - return defer.succeed(None) - else: - self._deferred_initialization.addCallback(cb) - return self._deferred_initialization - - def _get_empty_mailbox(self): - """ - Returns an empty mailbox. - - :rtype: dict - """ - return copy.deepcopy(self.EMPTY_MBOX) - + # XXX should hide this in the adaptor... def _get_mailbox_by_name(self, name): """ Return an mbox document by name. @@ -146,32 +92,17 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :rtype: SoledadDocument """ - # XXX use soledadstore instead ...; def get_first_if_any(docs): return docs[0] if docs else None - d = self._soledad.get_from_index( + d = self._store.get_from_index( self.TYPE_MBOX_IDX, self.MBOX_KEY, - self._parse_mailbox_name(name)) + normalize_mailbox(name)) d.addCallback(get_first_if_any) return d - @property - def mailboxes(self): - """ - A list of the current mailboxes for this account. - :rtype: set - """ - return sorted(self.__mailboxes) - - def _load_mailboxes(self): - def update_mailboxes(db_indexes): - self.__mailboxes.update( - [doc.content[self.MBOX_KEY] for doc in db_indexes]) - d = self._soledad.get_from_index(self.TYPE_IDX, self.MBOX_KEY) - d.addCallback(update_mailboxes) - return d - + # XXX move to Account? + # XXX needed? def getMailbox(self, name): """ Return a Mailbox with that name, without selecting it. @@ -182,18 +113,28 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :returns: a a SoledadMailbox instance :rtype: SoledadMailbox """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) - if name not in self.mailboxes: + if name not in self.account.mailboxes: raise imap4.MailboxException("No such mailbox: %r" % name) - return SoledadMailbox(name, self._soledad, - memstore=self._memstore) + # XXX Does mailbox really need reference to soledad? + return SoledadMailbox(name, self._store) # # IAccount # + def _get_empty_mailbox(self): + """ + Returns an empty mailbox. + + :rtype: dict + """ + # XXX move to mailbox module + return copy.deepcopy(mailbox.EMPTY_MBOX) + + # TODO use mail.Account.add_mailbox def addMailbox(self, name, creation_ts=None): """ Add a mailbox to the account. @@ -209,7 +150,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :returns: a Deferred that will contain the document if successful. :rtype: bool """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) leap_assert(name, "Need a mailbox name to create a mailbox") @@ -232,10 +173,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): d.addCallback(lambda _: result) return d - d = self._soledad.create_doc(mbox) + d = self._store.create_doc(mbox) d.addCallback(load_mbox_cache) return d + # TODO use mail.Account.create_mailbox? + # Watch out, imap specific exceptions raised here. def create(self, pathspec): """ Create a new mailbox from the given hierarchical name. @@ -254,9 +197,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :raise MailboxException: Raised if this mailbox cannot be added. """ # TODO raise MailboxException - paths = filter( - None, - self._parse_mailbox_name(pathspec).split('/')) + paths = filter(None, normalize_mailbox(pathspec).split('/')) subs = [] sep = '/' @@ -295,6 +236,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): d1.addCallback(load_mbox_cache) return d1 + # TODO use mail.Account.get_collection_by_mailbox def select(self, name, readwrite=1): """ Selects a mailbox. @@ -307,21 +249,16 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :rtype: SoledadMailbox """ - if PROFILE_CMD: - start = time.time() - - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) if name not in self.mailboxes: logger.warning("No such mailbox!") return None self.selected = name - sm = SoledadMailbox( - name, self._soledad, self._memstore, readwrite) - if PROFILE_CMD: - _debugProfiling(None, "SELECT", start) + sm = SoledadMailbox(name, self._store, readwrite) return sm + # TODO use mail.Account.delete_mailbox def delete(self, name, force=False): """ Deletes a mailbox. @@ -338,7 +275,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :type force: bool :rtype: Deferred """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) if name not in self.mailboxes: err = imap4.MailboxException("No such mailbox: %r" % name) @@ -369,6 +306,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): # ??! -- can this be rite? # self._index.removeMailbox(name) + # TODO use mail.Account.rename_mailbox def rename(self, oldname, newname): """ Renames a mailbox. @@ -379,8 +317,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :param newname: new name of the mailbox :type newname: str """ - oldname = self._parse_mailbox_name(oldname) - newname = self._parse_mailbox_name(newname) + oldname = normalize_mailbox(oldname) + newname = normalize_mailbox(newname) if oldname not in self.mailboxes: raise imap4.NoSuchMailbox(repr(oldname)) @@ -431,6 +369,32 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): inferiors.append(infname) return inferiors + # TODO use mail.Account.list_mailboxes + def listMailboxes(self, ref, wildcard): + """ + List the mailboxes. + + from rfc 3501: + returns a subset of names from the complete set + of all names available to the client. Zero or more untagged LIST + replies are returned, containing the name attributes, hierarchy + delimiter, and name. + + :param ref: reference name + :type ref: str + + :param wildcard: mailbox name with possible wildcards + :type wildcard: str + """ + # XXX use wildcard in index query + ref = self._inferiorNames(normalize_mailbox(ref)) + wildcard = imap4.wildcardToRegexp(wildcard, '/') + return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] + + # + # The rest of the methods are specific for leap.mail.imap.account.Account + # + # TODO ------------------ can we preserve the attr? # maybe add to memory store. def isSubscribed(self, name): @@ -442,6 +406,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :rtype: Deferred (will fire with bool) """ + # TODO use Flags class subscribed = self.SUBSCRIBED_KEY def is_subscribed(mbox): @@ -465,7 +430,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): def get_docs_content(docs): return [doc.content[self.MBOX_KEY] for doc in docs] - d = self._soledad.get_from_index( + d = self._store.get_from_index( self.TYPE_SUBS_IDX, self.MBOX_KEY, '1') d.addCallback(get_docs_content) return d @@ -488,7 +453,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): def update_subscribed_value(mbox): mbox.content[subscribed] = value - return self._soledad.put_doc(mbox) + return self._store.put_doc(mbox) # maybe we should store subscriptions in another # document... @@ -508,7 +473,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :type name: str :rtype: Deferred """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) def check_and_subscribe(subscriptions): if name not in subscriptions: @@ -525,7 +490,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): :type name: str :rtype: Deferred """ - name = self._parse_mailbox_name(name) + name = normalize_mailbox(name) def check_and_unsubscribe(subscriptions): if name not in subscriptions: @@ -539,28 +504,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): def getSubscriptions(self): return self._get_subscriptions() - def listMailboxes(self, ref, wildcard): - """ - List the mailboxes. - - from rfc 3501: - returns a subset of names from the complete set - of all names available to the client. Zero or more untagged LIST - replies are returned, containing the name attributes, hierarchy - delimiter, and name. - - :param ref: reference name - :type ref: str - - :param wildcard: mailbox name with possible wildcards - :type wildcard: str - """ - # XXX use wildcard in index query - ref = self._inferiorNames( - self._parse_mailbox_name(ref)) - wildcard = imap4.wildcardToRegexp(wildcard, '/') - return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] - # # INamespacePresenter # @@ -592,4 +535,4 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): """ Representation string for this object. """ - return "" % self._account_name + return "" % self.user_id diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py index 4576939..a751c6d 100644 --- a/src/leap/mail/imap/fields.py +++ b/src/leap/mail/imap/fields.py @@ -17,7 +17,9 @@ """ Fields for Mailbox and Message. """ -from leap.mail.imap.parser import MBoxParser + +# TODO deprecate !!! (move all to constants maybe?) +# Flags -> foo class WithMsgFields(object): @@ -25,55 +27,12 @@ class WithMsgFields(object): Container class for class-attributes to be shared by several message-related classes. """ - # indexing - CONTENT_HASH_KEY = "chash" - PAYLOAD_HASH_KEY = "phash" - - # Internal representation of Message - - # flags doc - UID_KEY = "uid" - MBOX_KEY = "mbox" - SEEN_KEY = "seen" - DEL_KEY = "deleted" - RECENT_KEY = "recent" - FLAGS_KEY = "flags" - MULTIPART_KEY = "multi" - SIZE_KEY = "size" - - # headers - HEADERS_KEY = "headers" - DATE_KEY = "date" - SUBJECT_KEY = "subject" - PARTS_MAP_KEY = "part_map" - BODY_KEY = "body" # link to phash of body - MSGID_KEY = "msgid" - - # content - LINKED_FROM_KEY = "lkf" # XXX not implemented yet! - RAW_KEY = "raw" - CTYPE_KEY = "ctype" - # Mailbox specific keys - CLOSED_KEY = "closed" - CREATED_KEY = "created" - SUBSCRIBED_KEY = "subscribed" - RW_KEY = "rw" - LAST_UID_KEY = "lastuid" + CREATED_KEY = "created" # used??? + RECENTFLAGS_KEY = "rct" HDOCS_SET_KEY = "hdocset" - # Document Type, for indexing - TYPE_KEY = "type" - TYPE_MBOX_VAL = "mbox" - TYPE_FLAGS_VAL = "flags" - TYPE_HEADERS_VAL = "head" - TYPE_CONTENT_VAL = "cnt" - TYPE_RECENT_VAL = "rct" - TYPE_HDOCS_SET_VAL = "hdocset" - - INBOX_VAL = "inbox" - # Flags in Mailbox and Message SEEN_FLAG = "\\Seen" RECENT_FLAG = "\\Recent" @@ -88,86 +47,5 @@ class WithMsgFields(object): SUBJECT_FIELD = "Subject" DATE_FIELD = "Date" - # Index types - # -------------- - - TYPE_IDX = 'by-type' - TYPE_MBOX_IDX = 'by-type-and-mbox' - TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' - TYPE_SUBS_IDX = 'by-type-and-subscribed' - TYPE_MSGID_IDX = 'by-type-and-message-id' - TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' - TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' - TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' - TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash' - TYPE_C_HASH_IDX = 'by-type-and-contenthash' - TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' - TYPE_P_HASH_IDX = 'by-type-and-payloadhash' - - # Tomas created the `recent and seen index`, but the semantic is not too - # correct since the recent flag is volatile. - TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' - - # Soledad index for incoming mail, without decrypting errors. - JUST_MAIL_IDX = "just-mail" - # XXX the backward-compatible index, will be deprecated at 0.7 - JUST_MAIL_COMPAT_IDX = "just-mail-compat" - - INCOMING_KEY = "incoming" - ERROR_DECRYPTING_KEY = "errdecr" - - KTYPE = TYPE_KEY - MBOX_VAL = TYPE_MBOX_VAL - CHASH_VAL = CONTENT_HASH_KEY - PHASH_VAL = PAYLOAD_HASH_KEY - - INDEXES = { - # generic - TYPE_IDX: [KTYPE], - TYPE_MBOX_IDX: [KTYPE, MBOX_VAL], - TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, UID_KEY], - - # mailboxes - TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'], - - # fdocs uniqueness - TYPE_MBOX_C_HASH_IDX: [KTYPE, MBOX_VAL, CHASH_VAL], - - # headers doc - search by msgid. - TYPE_MSGID_IDX: [KTYPE, MSGID_KEY], - - # content, headers doc - TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL], - - # attachment payload dedup - TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL], - - # messages - TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], - TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'], - TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'], - TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, - 'bool(recent)', 'bool(seen)'], - - # incoming queue - JUST_MAIL_IDX: [INCOMING_KEY, - "bool(%s)" % (ERROR_DECRYPTING_KEY,)], - - # the backward-compatible index, will be deprecated at 0.7 - JUST_MAIL_COMPAT_IDX: [INCOMING_KEY], - } - - MBOX_KEY = MBOX_VAL - - EMPTY_MBOX = { - TYPE_KEY: MBOX_KEY, - TYPE_MBOX_VAL: MBoxParser.INBOX_NAME, - SUBJECT_KEY: "", - FLAGS_KEY: [], - CLOSED_KEY: False, - SUBSCRIBED_KEY: False, - RW_KEY: 1, - LAST_UID_KEY: 0 - } fields = WithMsgFields # alias for convenience diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py deleted file mode 100644 index ea35fff..0000000 --- a/src/leap/mail/imap/index.py +++ /dev/null @@ -1,90 +0,0 @@ -# -*- coding: utf-8 -*- -# index.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Index for SoledadBackedAccount, Mailbox and Messages. -""" -import logging - -from twisted.internet import defer - -from leap.common.check import leap_assert, leap_assert_type - -from leap.mail.imap.fields import fields - - -logger = logging.getLogger(__name__) - - -class IndexedDB(object): - """ - Methods dealing with the index. - - This is a MixIn that needs access to the soledad instance, - and also assumes that a INDEXES attribute is accessible to the instance. - - INDEXES must be a dictionary of type: - {'index-name': ['field1', 'field2']} - """ - # TODO we might want to move this to soledad itself, check - - _index_creation_deferreds = [] - index_ready = False - - def initialize_db(self): - """ - Initialize the database. - """ - leap_assert(self._soledad, - "Need a soledad attribute accesible in the instance") - leap_assert_type(self.INDEXES, dict) - self._index_creation_deferreds = [] - - def _on_indexes_created(ignored): - self.index_ready = True - - def _create_index(name, expression): - d = self._soledad.create_index(name, *expression) - self._index_creation_deferreds.append(d) - - def _create_indexes(db_indexes): - db_indexes = dict(db_indexes) - for name, expression in fields.INDEXES.items(): - if name not in db_indexes: - # The index does not yet exist. - _create_index(name, expression) - continue - - if expression == db_indexes[name]: - # The index exists and is up to date. - continue - # The index exists but the definition is not what expected, so - # we delete it and add the proper index expression. - d1 = self._soledad.delete_index(name) - d1.addCallback(lambda _: _create_index(name, expression)) - - all_created = defer.gatherResults(self._index_creation_deferreds) - all_created.addCallback(_on_indexes_created) - return all_created - - # Ask the database for currently existing indexes. - if not self._soledad: - logger.debug("NO SOLEDAD ON IMAP INITIALIZATION") - return - if self._soledad is not None: - d = self._soledad.list_indexes() - d.addCallback(_create_indexes) - return d diff --git a/src/leap/mail/imap/interfaces.py b/src/leap/mail/imap/interfaces.py index c906278..f8f25fa 100644 --- a/src/leap/mail/imap/interfaces.py +++ b/src/leap/mail/imap/interfaces.py @@ -20,6 +20,7 @@ Interfaces for the IMAP module. from zope.interface import Interface, Attribute +# TODO remove ---------------- class IMessageContainer(Interface): """ I am a container around the different documents that a message @@ -38,6 +39,7 @@ class IMessageContainer(Interface): """ +# TODO remove -------------------- class IMessageStore(Interface): """ I represent a generic storage for LEAP Messages. diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 3c1769a..ea54d33 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -1,6 +1,6 @@ # *- coding: utf-8 -*- # mailbox.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013, 2014 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,6 +18,7 @@ Soledad Mailbox. """ import copy +import re import threading import logging import StringIO @@ -27,6 +28,7 @@ import os from collections import defaultdict from twisted.internet import defer +from twisted.internet import reactor from twisted.internet.task import deferLater from twisted.python import log @@ -36,15 +38,18 @@ from zope.interface import implements from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type +from leap.mail.constants import INBOX_NAME from leap.mail.decorators import deferred_to_thread from leap.mail.utils import empty from leap.mail.imap.fields import WithMsgFields, fields from leap.mail.imap.messages import MessageCollection from leap.mail.imap.messageparts import MessageWrapper -from leap.mail.imap.parser import MBoxParser logger = logging.getLogger(__name__) +# TODO +# [ ] Restore profile_cmd instrumentation + """ If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid notifying clients of new messages. Use during stress tests. @@ -71,7 +76,9 @@ if PROFILE_CMD: d.addErrback(lambda f: log.msg(f.getTraceback())) -class SoledadMailbox(WithMsgFields, MBoxParser): +# TODO Rename to Mailbox +# TODO Remove WithMsgFields +class SoledadMailbox(WithMsgFields): """ A Soledad-backed IMAP mailbox. @@ -115,7 +122,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): _last_uid_primed = {} _known_uids_primed = {} - def __init__(self, mbox, soledad, memstore, rw=1): + # TODO pass the collection to the constructor + # TODO pass the mbox_doc too + def __init__(self, mbox, store, rw=1): """ SoledadMailbox constructor. Needs to get passed a name, plus a Soledad instance. @@ -123,30 +132,21 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :param mbox: the mailbox name :type mbox: str - :param soledad: a Soledad instance. - :type soledad: Soledad - - :param memstore: a MemoryStore instance - :type memstore: MemoryStore + :param store: + :type store: Soledad :param rw: read-and-write flag for this mailbox :type rw: int """ leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(soledad, "Need a soledad instance to initialize") + leap_assert(store, "Need a store instance to initialize") - from twisted.internet import reactor - self.reactor = reactor - - self.mbox = self._parse_mailbox_name(mbox) + self.mbox = normalize_mailbox(mbox) self.rw = rw - self._soledad = soledad - self._memstore = memstore - - self.messages = MessageCollection( - mbox=mbox, soledad=self._soledad, memstore=self._memstore) + self.store = store + self.messages = MessageCollection(mbox=mbox, soledad=store) self._uidvalidity = None # XXX careful with this get/set (it would be @@ -214,7 +214,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ return self._memstore.get_mbox_doc(self.mbox) - # XXX the memstore->soledadstore method in memstore is not complete def getFlags(self): """ Returns the flags defined for this mailbox. @@ -227,7 +226,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): flags = self.INIT_FLAGS return map(str, flags) - # XXX the memstore->soledadstore method in memstore is not complete def setFlags(self, flags): """ Sets flags for this mailbox. @@ -468,8 +466,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d = self._do_add_message(message, flags=flags, date=date, notify_on_disk=notify_on_disk) - if PROFILE_CMD: - do_profile_cmd(d, "APPEND") + #if PROFILE_CMD: + #do_profile_cmd(d, "APPEND") # XXX should review now that we're not using qtreactor. # A better place for this would be the COPY/APPEND dispatcher @@ -477,7 +475,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # to work fine for now. def notifyCallback(x): - self.reactor.callLater(0, self.notify_new) + reactor.callLater(0, self.notify_new) return x d.addCallback(notifyCallback) @@ -630,9 +628,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: deferred """ d = defer.Deferred() - self.reactor.callInThread(self._do_fetch, messages_asked, uid, d) - if PROFILE_CMD: - do_profile_cmd(d, "FETCH") + + # XXX do not need no thread... + reactor.callInThread(self._do_fetch, messages_asked, uid, d) d.addCallback(self.cb_signal_unread_to_ui) return d @@ -800,7 +798,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d.addCallback(self.__cb_signal_unread_to_ui) return result - @deferred_to_thread def _get_unseen_deferred(self): return self.getUnseenCount() @@ -897,7 +894,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: C{list} or C{Deferred} """ # TODO see if we can raise w/o interrupting flow - #:raise IllegalQueryError: Raised when query is not valid. + # :raise IllegalQueryError: Raised when query is not valid. # example query: # ['UNDELETED', 'HEADER', 'Message-ID', # '52D44F11.9060107@dev.bitmask.net'] @@ -991,7 +988,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): d.addCallback(createCopy) d.addErrback(lambda f: log.msg(f.getTraceback())) - @deferred_to_thread + #@deferred_to_thread def _get_msg_copy(self, message): """ Get a copy of the fdoc for this message, and check whether @@ -1049,3 +1046,22 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ return u"" % ( self.mbox, self.messages.count()) + + +def normalize_mailbox(name): + """ + Return a normalized representation of the mailbox ``name``. + + This method ensures that an eventual initial 'inbox' part of a + mailbox name is made uppercase. + + :param name: the name of the mailbox + :type name: unicode + + :rtype: unicode + """ + _INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + if _INBOX_RE.match(name): + # ensure inital INBOX is uppercase + return INBOX_NAME + name[len(INBOX_NAME):] + return name diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index c761091..d47c8eb 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # messages.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013, 2014 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,30 +19,25 @@ LeapMessage and MessageCollection. """ import copy import logging -import re import threading import StringIO from collections import defaultdict -from email import message_from_string from functools import partial -from pycryptopp.hash import sha256 from twisted.mail import imap4 -from twisted.internet import defer, reactor +from twisted.internet import reactor from zope.interface import implements from zope.proxy import sameProxiedObjects from leap.common.check import leap_assert, leap_assert_type from leap.common.decorators import memoized_method from leap.common.mail import get_email_charset -from leap.mail import walk -from leap.mail.utils import first, find_charset, lowerdict, empty -from leap.mail.utils import stringify_parts_map -from leap.mail.decorators import deferred_to_thread +from leap.mail.adaptors import soledad_indexes as indexes +from leap.mail.constants import INBOX_NAME +from leap.mail.utils import find_charset, empty from leap.mail.imap.index import IndexedDB from leap.mail.imap.fields import fields, WithMsgFields -from leap.mail.imap.memorystore import MessageWrapper from leap.mail.imap.messageparts import MessagePart, MessagePartDoc from leap.mail.imap.parser import MBoxParser @@ -59,9 +54,6 @@ logger = logging.getLogger(__name__) # [ ] Delete incoming mail only after successful write! # [ ] Remove UID from syncable db. Store only those indexes locally. -MSGID_PATTERN = r"""<([\w@.]+)>""" -MSGID_RE = re.compile(MSGID_PATTERN) - def try_unique_query(curried): """ @@ -90,28 +82,18 @@ def try_unique_query(curried): logger.exception("Unhandled error %r" % exc) -""" -A dictionary that keeps one lock per mbox and uid. -""" -# XXX too much overhead? -fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock())) +# FIXME remove-me +#fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock())) -class LeapMessage(fields, MBoxParser): +class IMAPMessage(fields, MBoxParser): """ The main representation of a message. - - It indexes the messages in one mailbox by a combination - of uid+mailbox name. """ - # TODO this has to change. - # Should index primarily by chash, and keep a local-only - # UID table. - implements(imap4.IMessage) - def __init__(self, soledad, uid, mbox, collection=None, container=None): + def __init__(self, soledad, uid, mbox): """ Initializes a LeapMessage. @@ -129,76 +111,73 @@ class LeapMessage(fields, MBoxParser): self._soledad = soledad self._uid = int(uid) if uid is not None else None self._mbox = self._parse_mailbox_name(mbox) - self._collection = collection - self._container = container self.__chash = None self.__bdoc = None - # XXX make these properties public - - # XXX FIXME ------ the documents can be - # deferreds too.... niice. - - @property - def fdoc(self): - """ - An accessor to the flags document. - """ - if all(map(bool, (self._uid, self._mbox))): - fdoc = None - if self._container is not None: - fdoc = self._container.fdoc - if not fdoc: - fdoc = self._get_flags_doc() - if fdoc: - fdoc_content = fdoc.content - self.__chash = fdoc_content.get( - fields.CONTENT_HASH_KEY, None) - return fdoc - - @property - def hdoc(self): - """ - An accessor to the headers document. - """ - container = self._container - if container is not None: - hdoc = self._container.hdoc - if hdoc and not empty(hdoc.content): - return hdoc - hdoc = self._get_headers_doc() - - if container and not empty(hdoc.content): + # TODO collection and container are deprecated. + + # TODO move to adaptor + + #@property + #def fdoc(self): + #""" + #An accessor to the flags document. + #""" + #if all(map(bool, (self._uid, self._mbox))): + #fdoc = None + #if self._container is not None: + #fdoc = self._container.fdoc + #if not fdoc: + #fdoc = self._get_flags_doc() + #if fdoc: + #fdoc_content = fdoc.content + #self.__chash = fdoc_content.get( + #fields.CONTENT_HASH_KEY, None) + #return fdoc +# + #@property + #def hdoc(self): + #""" + #An accessor to the headers document. + #""" + #container = self._container + #if container is not None: + #hdoc = self._container.hdoc + #if hdoc and not empty(hdoc.content): + #return hdoc + #hdoc = self._get_headers_doc() +# + #if container and not empty(hdoc.content): # mem-cache it - hdoc_content = hdoc.content - chash = hdoc_content.get(fields.CONTENT_HASH_KEY) - hdocs = {chash: hdoc_content} - container.memstore.load_header_docs(hdocs) - return hdoc - - @property - def chash(self): - """ - An accessor to the content hash for this message. - """ - if not self.fdoc: - return None - if not self.__chash and self.fdoc: - self.__chash = self.fdoc.content.get( - fields.CONTENT_HASH_KEY, None) - return self.__chash - - @property - def bdoc(self): - """ - An accessor to the body document. - """ - if not self.hdoc: - return None - if not self.__bdoc: - self.__bdoc = self._get_body_doc() - return self.__bdoc + #hdoc_content = hdoc.content + #chash = hdoc_content.get(fields.CONTENT_HASH_KEY) + #hdocs = {chash: hdoc_content} + #container.memstore.load_header_docs(hdocs) + #return hdoc +# + #@property + #def chash(self): + #""" + #An accessor to the content hash for this message. + #""" + #if not self.fdoc: + #return None + #if not self.__chash and self.fdoc: + #self.__chash = self.fdoc.content.get( + #fields.CONTENT_HASH_KEY, None) + #return self.__chash + + #@property + #def bdoc(self): + #""" + #An accessor to the body document. + #""" + #if not self.hdoc: + #return None + #if not self.__bdoc: + #self.__bdoc = self._get_body_doc() + #return self.__bdoc # IMessage implementation @@ -209,8 +188,13 @@ class LeapMessage(fields, MBoxParser): :return: uid for this message :rtype: int """ + # TODO ----> return lookup in local sqlcipher table. return self._uid + # -------------------------------------------------------------- + # TODO -- from here on, all the methods should be proxied to the + # instance of leap.mail.mail.Message + def getFlags(self): """ Retrieve the flags associated with this Message. @@ -253,25 +237,24 @@ class LeapMessage(fields, MBoxParser): REMOVE = -1 SET = 0 - with fdoc_locks[mbox][uid]: - doc = self.fdoc - if not doc: - logger.warning( - "Could not find FDOC for %r:%s while setting flags!" % - (mbox, uid)) - return - current = doc.content[self.FLAGS_KEY] - if mode == APPEND: - newflags = tuple(set(tuple(current) + flags)) - elif mode == REMOVE: - newflags = tuple(set(current).difference(set(flags))) - elif mode == SET: - newflags = flags - new_fdoc = { - self.FLAGS_KEY: newflags, - self.SEEN_KEY: self.SEEN_FLAG in newflags, - self.DEL_KEY: self.DELETED_FLAG in newflags} - self._collection.memstore.update_flags(mbox, uid, new_fdoc) + doc = self.fdoc + if not doc: + logger.warning( + "Could not find FDOC for %r:%s while setting flags!" % + (mbox, uid)) + return + current = doc.content[self.FLAGS_KEY] + if mode == APPEND: + newflags = tuple(set(tuple(current) + flags)) + elif mode == REMOVE: + newflags = tuple(set(current).difference(set(flags))) + elif mode == SET: + newflags = flags + new_fdoc = { + self.FLAGS_KEY: newflags, + self.SEEN_KEY: self.SEEN_FLAG in newflags, + self.DEL_KEY: self.DELETED_FLAG in newflags} + self._collection.memstore.update_flags(mbox, uid, new_fdoc) return map(str, newflags) @@ -371,9 +354,9 @@ class LeapMessage(fields, MBoxParser): else: logger.warning("No FLAGS doc for %s:%s" % (self._mbox, self._uid)) - if not size: + #if not size: # XXX fallback, should remove when all migrated. - size = self.getBodyFile().len + #size = self.getBodyFile().len return size def getHeaders(self, negate, *names): @@ -395,6 +378,9 @@ class LeapMessage(fields, MBoxParser): # XXX refactor together with MessagePart method headers = self._get_headers() + + # XXX keep this in the imap imessage implementation, + # because the server impl. expects content-type to be present. if not headers: logger.warning("No headers found") return {str('content-type'): str('')} @@ -614,64 +600,23 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): (the u1db index) for all the headers documents for a given mailbox. We use it to prefetch massively all the headers for a mailbox. This is the second massive query, after fetching all the FLAGS, that - a MUA will do in a case where we do not have local disk cache. + a typical IMAP MUA will do in a case where we do not have local disk cache. """ HDOCS_SET_DOC = "HDOCS_SET" templates = { - # Message Level - - FLAGS_DOC: { - fields.TYPE_KEY: fields.TYPE_FLAGS_VAL, - fields.UID_KEY: 1, # XXX moe to a local table - fields.MBOX_KEY: fields.INBOX_VAL, - fields.CONTENT_HASH_KEY: "", - - fields.SEEN_KEY: False, - fields.DEL_KEY: False, - fields.FLAGS_KEY: [], - fields.MULTIPART_KEY: False, - fields.SIZE_KEY: 0 - }, - - HEADERS_DOC: { - fields.TYPE_KEY: fields.TYPE_HEADERS_VAL, - fields.CONTENT_HASH_KEY: "", - - fields.DATE_KEY: "", - fields.SUBJECT_KEY: "", - - fields.HEADERS_KEY: {}, - fields.PARTS_MAP_KEY: {}, - }, - - CONTENT_DOC: { - fields.TYPE_KEY: fields.TYPE_CONTENT_VAL, - fields.PAYLOAD_HASH_KEY: "", - fields.LINKED_FROM_KEY: [], - fields.CTYPE_KEY: "", # should index by this too - - # should only get inmutable headers parts - # (for indexing) - fields.HEADERS_KEY: {}, - fields.RAW_KEY: "", - fields.PARTS_MAP_KEY: {}, - fields.HEADERS_KEY: {}, - fields.MULTIPART_KEY: False, - }, - # Mailbox Level RECENT_DOC: { - fields.TYPE_KEY: fields.TYPE_RECENT_VAL, - fields.MBOX_KEY: fields.INBOX_VAL, + "type": indexes.RECENT, + "mbox": INBOX_NAME, fields.RECENTFLAGS_KEY: [], }, HDOCS_SET_DOC: { - fields.TYPE_KEY: fields.TYPE_HDOCS_SET_VAL, - fields.MBOX_KEY: fields.INBOX_VAL, + "type": indexes.HDOCS_SET, + "mbox": INBOX_NAME, fields.HDOCS_SET_KEY: [], } @@ -681,8 +626,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): # Different locks for wrapping both the u1db document getting/setting # and the property getting/settting in an atomic operation. - # TODO we would abstract this to a SoledadProperty class - + # TODO --- deprecate ! --- use SoledadDocumentWrapper + locks _rdoc_lock = defaultdict(lambda: threading.Lock()) _rdoc_write_lock = defaultdict(lambda: threading.Lock()) _rdoc_read_lock = defaultdict(lambda: threading.Lock()) @@ -764,81 +708,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): rdoc[fields.MBOX_KEY] = self.mbox self._soledad.create_doc(rdoc) - @deferred_to_thread - def _do_parse(self, raw): - """ - Parse raw message and return it along with - relevant information about its outer level. - - This is done in a separate thread, and the callback is passed - to `_do_add_msg` method. + # -------------------------------------------------------------------- - :param raw: the raw message - :type raw: StringIO or basestring - :return: msg, parts, chash, size, multi - :rtype: tuple - """ - msg = message_from_string(raw) - parts = walk.get_parts(msg) - size = len(raw) - chash = sha256.SHA256(raw).hexdigest() - multi = msg.is_multipart() - return msg, parts, chash, size, multi - - def _populate_flags(self, flags, uid, chash, size, multi): - """ - Return a flags doc. - - XXX Missing DOC ----------- - """ - fd = self._get_empty_doc(self.FLAGS_DOC) - - fd[self.MBOX_KEY] = self.mbox - fd[self.UID_KEY] = uid - fd[self.CONTENT_HASH_KEY] = chash - fd[self.SIZE_KEY] = size - fd[self.MULTIPART_KEY] = multi - if flags: - fd[self.FLAGS_KEY] = flags - fd[self.SEEN_KEY] = self.SEEN_FLAG in flags - fd[self.DEL_KEY] = self.DELETED_FLAG in flags - fd[self.RECENT_KEY] = True # set always by default - return fd - - def _populate_headr(self, msg, chash, subject, date): - """ - Return a headers doc. - - XXX Missing DOC ----------- - """ - headers = defaultdict(list) - for k, v in msg.items(): - headers[k].append(v) - - # "fix" for repeated headers. - for k, v in headers.items(): - newline = "\n%s: " % (k,) - headers[k] = newline.join(v) - - lower_headers = lowerdict(headers) - msgid = first(MSGID_RE.findall( - lower_headers.get('message-id', ''))) - - hd = self._get_empty_doc(self.HEADERS_DOC) - hd[self.CONTENT_HASH_KEY] = chash - hd[self.HEADERS_KEY] = headers - hd[self.MSGID_KEY] = msgid - - if not subject and self.SUBJECT_FIELD in headers: - hd[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] - else: - hd[self.SUBJECT_KEY] = subject - - if not date and self.DATE_FIELD in headers: - hd[self.DATE_KEY] = headers[self.DATE_FIELD] - else: - hd[self.DATE_KEY] = date - return hd + # ----------------------------------------------------------------------- def _fdoc_already_exists(self, chash): """ @@ -885,86 +757,41 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): flags = tuple() leap_assert_type(flags, tuple) - # TODO return soledad deferred instead - observer = defer.Deferred() - d = self._do_parse(raw) - d.addCallback(lambda result: reactor.callInThread( - self._do_add_msg, result, flags, subject, date, - notify_on_disk, observer)) - return observer + # TODO ---- proxy to MessageCollection addMessage + + #observer = defer.Deferred() + #d = self._do_parse(raw) + #d.addCallback(lambda result: reactor.callInThread( + #self._do_add_msg, result, flags, subject, date, + #notify_on_disk, observer)) + #return observer + + # TODO --------------------------------------------------- + # move this to leap.mail.adaptors.soledad - # Called in thread def _do_add_msg(self, parse_result, flags, subject, date, notify_on_disk, observer): """ - Helper that creates a new message document. - Here lives the magic of the leap mail. Well, in soledad, really. - - See `add_msg` docstring for parameter info. - - :param parse_result: a tuple with the results of `self._do_parse` - :type parse_result: tuple - :param observer: a deferred that will be fired with the message - uid when the adding succeed. - :type observer: deferred """ - # TODO signal that we can delete the original message!----- - # when all the processing is done. - - # TODO add the linked-from info ! - # TODO add reference to the original message - msg, parts, chash, size, multi = parse_result + # XXX move to SoledadAdaptor write operation ... ??? # check for uniqueness -------------------------------- # Watch out! We're reserving a UID right after this! existing_uid = self._fdoc_already_exists(chash) if existing_uid: msg = self.get_msg_by_uid(existing_uid) - - # We can say the observer that we're done - # TODO return soledad deferred instead reactor.callFromThread(observer.callback, existing_uid) msg.setFlags((fields.DELETED_FLAG,), -1) return + # TODO move UID autoincrement to MessageCollection.addMessage(mailbox) # TODO S2 -- get FUCKING UID from autoincremental table - uid = self.memstore.increment_last_soledad_uid(self.mbox) - - # We can say the observer that we're done at this point, but - # before that we should make sure it has no serious consequences - # if we're issued, for instance, a fetch command right after... - # reactor.callFromThread(observer.callback, uid) - # if we did the notify, we need to invalidate the deferred - # so not to try to fire it twice. - # observer = None - - fd = self._populate_flags(flags, uid, chash, size, multi) - hd = self._populate_headr(msg, chash, subject, date) - - body_phash_fun = [walk.get_body_phash_simple, - walk.get_body_phash_multi][int(multi)] - body_phash = body_phash_fun(walk.get_payloads(msg)) - parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) - - # add parts map to header doc - # (body, multi, part_map) - for key in parts_map: - hd[key] = parts_map[key] - del parts_map + #uid = self.memstore.increment_last_soledad_uid(self.mbox) + #self.set_recent_flag(uid) - hd = stringify_parts_map(hd) - # The MessageContainer expects a dict, one-indexed - cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1)) - - self.set_recent_flag(uid) - msg_container = MessageWrapper(fd, hd, cdocs) - - # TODO S1 -- just pass this to memstore and return that deferred. - self.memstore.create_message( - self.mbox, uid, msg_container, - observer=observer, notify_on_disk=notify_on_disk) + # ------------------------------------------------------------ # # getters: specific queries @@ -1073,6 +900,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): the query failed. :rtype: SoledadDocument or None. """ + # USED from: + # [ ] duplicated fdoc detection + # [ ] _get_uid_from_msgidCb + # FIXME ----- use deferreds. curried = partial( self._soledad.get_from_index, @@ -1205,51 +1036,52 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): if msg_container is not None: if mem_only: - msg = LeapMessage(None, uid, self.mbox, collection=self, + msg = IMAPMessage(None, uid, self.mbox, collection=self, container=msg_container) else: # We pass a reference to soledad just to be able to retrieve # missing parts that cannot be found in the container, like # the content docs after a copy. - msg = LeapMessage(self._soledad, uid, self.mbox, + msg = IMAPMessage(self._soledad, uid, self.mbox, collection=self, container=msg_container) else: - msg = LeapMessage(self._soledad, uid, self.mbox, collection=self) + msg = IMAPMessage(self._soledad, uid, self.mbox, collection=self) if not msg.does_exist(): return None return msg - def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL): - """ - Get all documents for the selected mailbox of the - passed type. By default, it returns the flag docs. - - If you want acess to the content, use __iter__ instead - - :return: a Deferred, that will fire with a list of u1db documents - :rtype: Deferred (promise of list of SoledadDocument) - """ - if _type not in fields.__dict__.values(): - raise TypeError("Wrong type passed to get_all_docs") - + # FIXME --- used where ? --------------------------------------------- + #def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL): + #""" + #Get all documents for the selected mailbox of the + #passed type. By default, it returns the flag docs. +# + #If you want acess to the content, use __iter__ instead +# + #:return: a Deferred, that will fire with a list of u1db documents + #:rtype: Deferred (promise of list of SoledadDocument) + #""" + #if _type not in fields.__dict__.values(): + #raise TypeError("Wrong type passed to get_all_docs") +# # FIXME ----- either raise or return a deferred wrapper. - if sameProxiedObjects(self._soledad, None): - logger.warning('Tried to get messages but soledad is None!') - return [] - - def get_sorted_docs(docs): - all_docs = [doc for doc in docs] + #if sameProxiedObjects(self._soledad, None): + #logger.warning('Tried to get messages but soledad is None!') + #return [] +# + #def get_sorted_docs(docs): + #all_docs = [doc for doc in docs] # inneficient, but first let's grok it and then # let's worry about efficiency. # XXX FIXINDEX -- should implement order by in soledad # FIXME ---------------------------------------------- - return sorted(all_docs, key=lambda item: item.content['uid']) - - d = self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, _type, self.mbox) - d.addCallback(get_sorted_docs) - return d + #return sorted(all_docs, key=lambda item: item.content['uid']) +# + #d = self._soledad.get_from_index( + #fields.TYPE_MBOX_IDX, _type, self.mbox) + #d.addCallback(get_sorted_docs) + #return d def all_soledad_uid_iter(self): """ @@ -1350,7 +1182,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :returns: a list of LeapMessages :rtype: list """ - return [LeapMessage(self._soledad, docid, self.mbox, collection=self) + return [IMAPMessage(self._soledad, docid, self.mbox, collection=self) for docid in self.unseen_iter()] # recent messages @@ -1384,7 +1216,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :returns: iterator of dicts with content for all messages. :rtype: iterable """ - return (LeapMessage(self._soledad, docuid, self.mbox, collection=self) + return (IMAPMessage(self._soledad, docuid, self.mbox, collection=self) for docuid in self.all_uid_iter()) def __repr__(self): diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py deleted file mode 100644 index 4a801b0..0000000 --- a/src/leap/mail/imap/parser.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- -# parser.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Mail parser mixin. -""" -import re - - -class MBoxParser(object): - """ - Utility function to parse mailbox names. - """ - INBOX_NAME = "INBOX" - INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) - - def _parse_mailbox_name(self, name): - """ - Return a normalized representation of the mailbox C{name}. - - This method ensures that an eventual initial 'inbox' part of a - mailbox name is made uppercase. - - :param name: the name of the mailbox - :type name: unicode - - :rtype: unicode - """ - if self.INBOX_RE.match(name): - # ensure inital INBOX is uppercase - return self.INBOX_NAME + name[len(self.INBOX_NAME):] - return name diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index dd4294c..5af499f 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -94,6 +94,8 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): MessageCollection interface in this particular TestCase """ super(MessageCollectionTestCase, self).setUp() + + # TODO deprecate memstore memstore = MemoryStore() self.messages = MessageCollection("testmbox%s" % (self.count,), self._soledad, memstore=memstore) diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index 9a3868c..920eeb0 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -51,6 +51,7 @@ class SimpleClient(imap4.IMAP4Client): self.transport.loseConnection() +# XXX move to common helper def initialize_soledad(email, gnupg_home, tempdir): """ Initializes soledad by hand @@ -110,9 +111,7 @@ class IMAP4HelperMixin(BaseLeapTest): """ Setup method for each test. - Initializes and run a LEAP IMAP4 Server, - but passing the same Soledad instance (it's costly to initialize), - so we have to be sure to restore state across tests. + Initializes and run a LEAP IMAP4 Server. """ self.old_path = os.environ['PATH'] self.old_home = os.environ['HOME'] @@ -172,19 +171,17 @@ class IMAP4HelperMixin(BaseLeapTest): def tearDown(self): """ tearDown method called after each test. - - Deletes all documents in the Index, and deletes - instances of server and client. """ try: self._soledad.close() + except Exception: + print "ERROR WHILE CLOSING SOLEDAD" + finally: os.environ["PATH"] = self.old_path os.environ["HOME"] = self.old_home # safety check assert 'leap_tests-' in self.tempdir shutil.rmtree(self.tempdir) - except Exception: - print "ERROR WHILE CLOSING SOLEDAD" def populateMessages(self): """ @@ -223,5 +220,3 @@ class IMAP4HelperMixin(BaseLeapTest): def loopback(self): return loopback.loopbackAsync(self.server, self.client) - - diff --git a/src/leap/mail/interfaces.py b/src/leap/mail/interfaces.py new file mode 100644 index 0000000..5838ce9 --- /dev/null +++ b/src/leap/mail/interfaces.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# interfaces.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Interfaces for the leap.mail module. +""" +from zope.interface import Interface, Attribute + + +class IMessageWrapper(Interface): + """ + I know how to access the different parts into which a given message is + splitted into. + """ + + fdoc = Attribute('A dictionaly-like containing the flags document ' + '(mutable)') + hdoc = Attribute('A dictionary-like containing the headers docuemnt ' + '(immutable)') + cdocs = Attribute('A dictionary with the content-docs, one-indexed') + + +class IMailAdaptor(Interface): + """ + I know how to store the standard representation for messages and mailboxes, + and how to update the relevant mutable parts when needed. + """ + + def initialize_store(self, store): + """ + Performs whatever initialization is needed before the store can be + used (creating indexes, sanity checks, etc). + + :param store: store + :returns: a Deferred that will fire when the store is correctly + initialized. + :rtype: deferred + """ + + # TODO is staticmethod valid with an interface? + # @staticmethod + def get_msg_from_string(self, MessageClass, raw_msg): + """ + Return IMessageWrapper implementor from a raw mail string + + :param MessageClass: an implementor of IMessage + :type raw_msg: str + :rtype: implementor of leap.mail.IMessage + """ + + # TODO is staticmethod valid with an interface? + # @staticmethod + def get_msg_from_docs(self, MessageClass, msg_wrapper): + """ + Return an IMessage implementor from its parts. + + :param MessageClass: an implementor of IMessage + :param msg_wrapper: an implementor of IMessageWrapper + :rtype: implementor of leap.mail.IMessage + """ + + # ------------------------------------------------------------------- + # XXX unsure about the following part yet ........................... + + # the idea behind these three methods is that the adaptor also offers a + # fixed interface to create the documents the first time (using + # soledad.create_docs or whatever method maps to it in a similar store, and + # also allows to update flags and tags, hiding the actual implementation of + # where the flags/tags live in behind the concrete MailWrapper in use + # by this particular adaptor. In our impl it will be put_doc(fdoc) after + # locking the getting + updating of that fdoc for atomicity. + + # 'store' must be an instance of something that offers a minimal subset of + # the document API that Soledad currently implements (create_doc, put_doc) + # I *think* store should belong to Account/Collection and be passed as + # param here instead of relying on it being an attribute of the instance. + + def create_msg_docs(self, store, msg_wrapper): + """ + :param store: The documents store + :type store: + :param msg_wrapper: + :type msg_wrapper: IMessageWrapper implementor + """ + + def update_msg_flags(self, store, msg_wrapper): + """ + :param store: The documents store + :type store: + :param msg_wrapper: + :type msg_wrapper: IMessageWrapper implementor + """ + + def update_msg_tags(self, store, msg_wrapper): + """ + :param store: The documents store + :type store: + :param msg_wrapper: + :type msg_wrapper: IMessageWrapper implementor + """ diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py new file mode 100644 index 0000000..ea9c95e --- /dev/null +++ b/src/leap/mail/mail.py @@ -0,0 +1,248 @@ +# -*- coding: utf-8 -*- +# mail.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Generic Access to Mail objects: Public LEAP Mail API. +""" +from twisted.internet import defer + +from leap.mail.constants import INBOX_NAME +from leap.mail.adaptors.soledad import SoledadMailAdaptor + + +# TODO +# [ ] Probably change the name of this module to "api" or "account", mail is +# too generic (there's also IncomingMail, and OutgoingMail + + +class Message(object): + + def __init__(self, wrapper): + """ + :param wrapper: an instance of an implementor of IMessageWrapper + """ + self._wrapper = wrapper + + def get_wrapper(self): + return self._wrapper + + # imap.IMessage methods + + def get_flags(): + """ + """ + + def get_internal_date(): + """ + """ + + # imap.IMessageParts + + def get_headers(): + """ + """ + + def get_body_file(): + """ + """ + + def get_size(): + """ + """ + + def is_multipart(): + """ + """ + + def get_subpart(part): + """ + """ + + # Custom methods. + + def get_tags(): + """ + """ + + +class MessageCollection(object): + """ + A generic collection of messages. It can be messages sharing the same + mailbox, tag, the result of a given query, or just a bunch of ids for + master documents. + + Since LEAP Mail is primarily oriented to store mail in Soledad, the default + (and, so far, only) implementation of the store is contained in this + Soledad Mail Adaptor. If you need to use a different adaptor, change the + adaptor class attribute in your Account object. + + Store is a reference to a particular instance of the message store (soledad + instance or proxy, for instance). + """ + + # TODO look at IMessageSet methods + + # Account should provide an adaptor instance when creating this collection. + adaptor = None + store = None + + def get_message_by_doc_id(self, doc_id): + # ... get from soledad etc + # ... but that should be part of adaptor/store too... :/ + fdoc, hdoc = None + return self.adaptor.from_docs(Message, fdoc=fdoc, hdoc=hdoc) + + # TODO review if this is the best place for: + + def create_docs(): + pass + + def udpate_flags(): + # 1. update the flags in the message wrapper --- stored where??? + # 2. call adaptor.update_msg(store) + pass + + def update_tags(): + # 1. update the tags in the message wrapper --- stored where??? + # 2. call adaptor.update_msg(store) + pass + + # TODO add delete methods here? + + +class Account(object): + """ + Account is the top level abstraction to access collections of messages + associated with a LEAP Mail Account. + + It primarily handles creation and access of Mailboxes, which will be the + basic collection handled by traditional MUAs, but it can also handle other + types of Collections (tag based, for instance). + + leap.mail.imap.SoledadBackedAccount partially proxies methods in this + class. + """ + + # Adaptor is passed to the returned MessageCollections, so if you want to + # use a different adaptor this is the place to change it, by subclassing + # the Account class. + + adaptor_class = SoledadMailAdaptor + store = None + mailboxes = None + + def __init__(self, store): + self.store = store + self.adaptor = self.adaptor_class() + + self.__mailboxes = set([]) + self._initialized = False + self._deferred_initialization = defer.Deferred() + + self._initialize_storage() + + def _initialize_storage(self): + + def add_mailbox_if_none(result): + # every user should have the right to an inbox folder + # at least, so let's make one! + if not self.mailboxes: + self.add_mailbox(INBOX_NAME) + + def finish_initialization(result): + self._initialized = True + self._deferred_initialization.callback(None) + + def load_mbox_cache(result): + d = self._load_mailboxes() + d.addCallback(lambda _: result) + return d + + d = self.adaptor.initialize_store(self.store) + d.addCallback(load_mbox_cache) + d.addCallback(add_mailbox_if_none) + d.addCallback(finish_initialization) + + def callWhenReady(self, cb): + # XXX this could use adaptor.store_ready instead...?? + if self._initialized: + cb(self) + return defer.succeed(None) + else: + self._deferred_initialization.addCallback(cb) + return self._deferred_initialization + + @property + def mailboxes(self): + """ + A list of the current mailboxes for this account. + :rtype: set + """ + return sorted(self.__mailboxes) + + def _load_mailboxes(self): + + def update_mailboxes(mbox_names): + self.__mailboxes.update(mbox_names) + + d = self.adaptor.get_all_mboxes(self.store) + d.addCallback(update_mailboxes) + return d + + # + # Public API Starts + # + + # XXX params for IMAP only??? + def list_mailboxes(self, ref, wildcard): + self.adaptor.get_all_mboxes(self.store) + + def add_mailbox(self, name, mbox=None): + pass + + def create_mailbox(self, pathspec): + pass + + def delete_mailbox(self, name): + pass + + def rename_mailbox(self, oldname, newname): + pass + + # FIXME yet to be decided if it belongs here... + + def get_collection_by_mailbox(self, name): + """ + :rtype: MessageCollection + """ + # imap select will use this, passing the collection to SoledadMailbox + # XXX pass adaptor to MessageCollection + pass + + def get_collection_by_docs(self, docs): + """ + :rtype: MessageCollection + """ + # get a collection of docs by a list of doc_id + # XXX pass adaptor to MessageCollection + pass + + def get_collection_by_tag(self, tag): + """ + :rtype: MessageCollection + """ + # is this a good idea? + pass -- cgit v1.2.3 From 22f619cf2a295ecc5beff16a91de63728ac4b17d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 26 Dec 2014 18:25:36 -0400 Subject: MessageCollections + MailboxIndexer --- src/leap/mail/adaptors/soledad.py | 173 ++++++++++-- src/leap/mail/adaptors/tests/rfc822.message | 87 +----- .../mail/adaptors/tests/test_soledad_adaptor.py | 110 ++------ src/leap/mail/constants.py | 17 ++ src/leap/mail/mail.py | 296 ++++++++++++++++----- src/leap/mail/mailbox_indexer.py | 254 ++++++++++++++++++ src/leap/mail/tests/common.py | 106 ++++++++ src/leap/mail/tests/rfc822.message | 86 ++++++ src/leap/mail/tests/test_mail.py | 95 +++++++ src/leap/mail/tests/test_mailbox_indexer.py | 241 +++++++++++++++++ 10 files changed, 1204 insertions(+), 261 deletions(-) mode change 100644 => 120000 src/leap/mail/adaptors/tests/rfc822.message create mode 100644 src/leap/mail/mailbox_indexer.py create mode 100644 src/leap/mail/tests/common.py create mode 100644 src/leap/mail/tests/rfc822.message create mode 100644 src/leap/mail/tests/test_mail.py create mode 100644 src/leap/mail/tests/test_mailbox_indexer.py diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 2e25f04..0b97869 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # soledad.py # Copyright (C) 2014 LEAP # @@ -20,6 +19,7 @@ Soledadad MailAdaptor module. import re from collections import defaultdict from email import message_from_string +from functools import partial from pycryptopp.hash import sha256 from twisted.internet import defer @@ -27,6 +27,7 @@ from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type +from leap.mail import constants from leap.mail import walk from leap.mail.adaptors import soledad_indexes as indexes from leap.mail.constants import INBOX_NAME @@ -60,7 +61,6 @@ class SoledadDocumentWrapper(models.DocumentWrapper): It ensures atomicity of the document operations on creation, update and deletion. """ - # TODO we could also use a _dirty flag (in models) # We keep a dictionary with DeferredLocks, that will be @@ -79,6 +79,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): def __init__(self, **kwargs): doc_id = kwargs.pop('doc_id', None) self._doc_id = doc_id + self._future_doc_id = kwargs.pop('future_doc_id', None) self._lock = defer.DeferredLock() super(SoledadDocumentWrapper, self).__init__(**kwargs) @@ -86,6 +87,13 @@ class SoledadDocumentWrapper(models.DocumentWrapper): def doc_id(self): return self._doc_id + @property + def future_doc_id(self): + return self._future_doc_id + + def set_future_doc_id(self, doc_id): + self._future_doc_id = doc_id + def create(self, store): """ Create the documents for this wrapper. @@ -105,8 +113,14 @@ class SoledadDocumentWrapper(models.DocumentWrapper): def update_doc_id(doc): self._doc_id = doc.doc_id + self._future_doc_id = None return doc - d = store.create_doc(self.serialize()) + + if self.future_doc_id is None: + d = store.create_doc(self.serialize()) + else: + d = store.create_doc(self.serialize(), + doc_id=self.future_doc_id) d.addCallback(update_doc_id) return d @@ -333,6 +347,12 @@ class FlagsDocWrapper(SoledadDocumentWrapper): class __meta__(object): index = "mbox" + def set_mbox(self, mbox): + # XXX raise error if already created, should use copy instead + new_id = constants.FDOCID.format(mbox=mbox, chash=self.chash) + self._future_doc_id = new_id + self.mbox = mbox + class HeaderDocWrapper(SoledadDocumentWrapper): @@ -370,6 +390,23 @@ class ContentDocWrapper(SoledadDocumentWrapper): index = "phash" +class MetaMsgDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "meta" + fdoc = "" + hdoc = "" + cdocs = [] + + def set_mbox(self, mbox): + # XXX raise error if already created, should use copy instead + chash = re.findall(constants.FDOCID_CHASH_RE, self.fdoc)[0] + new_id = constants.METAMSGID.format(mbox=mbox, chash=chash) + new_fdoc_id = constants.FDOCID.format(mbox=mbox, chash=chash) + self._future_doc_id = new_id + self.fdoc = new_fdoc_id + + class MessageWrapper(object): # TODO generalize wrapper composition? @@ -378,23 +415,32 @@ class MessageWrapper(object): implements(IMessageWrapper) - def __init__(self, fdoc, hdoc, cdocs=None): + def __init__(self, mdoc, fdoc, hdoc, cdocs=None): """ - Need at least a flag-document and a header-document to instantiate a - MessageWrapper. Content-documents can be retrieved lazily. + Need at least a metamsg-document, a flag-document and a header-document + to instantiate a MessageWrapper. Content-documents can be retrieved + lazily. cdocs, if any, should be a dictionary in which the keys are ascending integers, beginning at one, and the values are dictionaries with the content of the content-docs. """ + self.mdoc = MetaMsgDocWrapper(**mdoc) + self.fdoc = FlagsDocWrapper(**fdoc) + self.fdoc.set_future_doc_id(self.mdoc.fdoc) + self.hdoc = HeaderDocWrapper(**hdoc) + self.hdoc.set_future_doc_id(self.mdoc.hdoc) + if cdocs is None: cdocs = {} cdocs_keys = cdocs.keys() assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1) self.cdocs = dict([(key, ContentDocWrapper(**doc)) for (key, doc) in cdocs.items()]) + for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()): + cdoc.set_future_doc_id(doc_id) def create(self, store): """ @@ -403,16 +449,21 @@ class MessageWrapper(object): leap_assert(self.cdocs, "Need non empty cdocs to create the " "MessageWrapper documents") + leap_assert(self.mdoc.doc_id is None, + "Cannot create: mdoc has a doc_id") leap_assert(self.fdoc.doc_id is None, "Cannot create: fdoc has a doc_id") + # TODO check that the doc_ids in the mdoc are coherent # TODO I think we need to tolerate the no hdoc.doc_id case, for when we # are doing a copy to another mailbox. - leap_assert(self.hdoc.doc_id is None, - "Cannot create: hdoc has a doc_id") + # leap_assert(self.hdoc.doc_id is None, + # "Cannot create: hdoc has a doc_id") d = [] + d.append(self.mdoc.create(store)) d.append(self.fdoc.create(store)) - d.append(self.hdoc.create(store)) + if self.hdoc.doc_id is None: + d.append(self.hdoc.create(store)) for cdoc in self.cdocs.values(): if cdoc.doc_id is not None: # we could be just linking to an existing @@ -432,6 +483,25 @@ class MessageWrapper(object): # garbage collector. At least the fdoc can be unlinked. raise NotImplementedError() + def copy(self, store, newmailbox): + """ + Return a copy of this MessageWrapper in a new mailbox. + """ + # 1. copy the fdoc, mdoc + # 2. remove the doc_id of that fdoc + # 3. create it (with new doc_id) + # 4. return new wrapper (new meta too!) + raise NotImplementedError() + + def set_mbox(self, mbox): + """ + Set the mailbox for this wrapper. + This method should only be used before the Documents for the + MessageWrapper have been created, will raise otherwise. + """ + self.mdoc.set_mbox(mbox) + self.fdoc.set_mbox(mbox) + # # Mailboxes # @@ -535,6 +605,7 @@ class SoledadMailAdaptor(SoledadIndexMixin): store = None indexes = indexes.MAIL_INDEXES + mboxwrapper_klass = MailboxWrapper # Message handling @@ -552,11 +623,11 @@ class SoledadMailAdaptor(SoledadIndexMixin): :rtype: MessageClass instance. """ assert(MessageClass is not None) - fdoc, hdoc, cdocs = _split_into_parts(raw_msg) + mdoc, fdoc, hdoc, cdocs = _split_into_parts(raw_msg) return self.get_msg_from_docs( - MessageClass, fdoc, hdoc, cdocs) + MessageClass, mdoc, fdoc, hdoc, cdocs) - def get_msg_from_docs(self, MessageClass, fdoc, hdoc, cdocs=None): + def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None): """ Get an instance of a MessageClass initialized with a MessageWrapper that contains the passed part documents. @@ -582,7 +653,62 @@ class SoledadMailAdaptor(SoledadIndexMixin): :rtype: MessageClass instance. """ assert(MessageClass is not None) - return MessageClass(MessageWrapper(fdoc, hdoc, cdocs)) + return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs)) + + def _get_msg_from_variable_doc_list(self, doc_list, msg_class): + if len(doc_list) == 2: + fdoc, hdoc = doc_list + cdocs = None + elif len(doc_list) > 2: + fdoc, hdoc = doc_list[:2] + cdocs = dict(enumerate(doc_list[2:], 1)) + return self.get_msg_from_docs(msg_class, fdoc, hdoc, cdocs) + + def get_msg_from_mdoc_id(self, MessageClass, store, doc_id, + get_cdocs=False): + metamsg_id = doc_id + + def wrap_meta_doc(doc): + cls = MetaMsgDocWrapper + return cls(doc_id=doc.doc_id, **doc.content) + + def get_part_docs_from_mdoc_wrapper(wrapper): + d_docs = [] + d_docs.append(store.get_doc(wrapper.fdoc)) + d_docs.append(store.get_doc(wrapper.hdoc)) + for cdoc in wrapper.cdocs: + d_docs.append(store.get_doc(cdoc)) + d = defer.gatherResults(d_docs) + return d + + def get_parts_doc_from_mdoc_id(): + mbox = re.findall(constants.METAMSGID_MBOX_RE, doc_id)[0] + chash = re.findall(constants.METAMSGID_CHASH_RE, doc_id)[0] + + def _get_fdoc_id_from_mdoc_id(): + return constants.FDOCID.format(mbox=mbox, chash=chash) + + def _get_hdoc_id_from_mdoc_id(): + return constants.FDOCID.format(mbox=mbox, chash=chash) + + d_docs = [] + fdoc_id = _get_fdoc_id_from_mdoc_id(doc_id) + hdoc_id = _get_hdoc_id_from_mdoc_id(doc_id) + d_docs.append(store.get_doc(fdoc_id)) + d_docs.append(store.get_doc(hdoc_id)) + d = defer.gatherResults(d_docs) + return d + + if get_cdocs: + d = store.get_doc(metamsg_id) + d.addCallback(wrap_meta_doc) + d.addCallback(get_part_docs_from_mdoc_wrapper) + else: + d = get_parts_doc_from_mdoc_id() + + d.addCallback(partial(self._get_msg_from_variable_doc_list, + msg_class=MessageClass)) + return d def create_msg(self, store, msg): """ @@ -615,7 +741,7 @@ class SoledadMailAdaptor(SoledadIndexMixin): def get_or_create_mbox(self, store, name): """ - Get the mailbox with the given name, or creatre one if it does not + Get the mailbox with the given name, or create one if it does not exist. :param name: the name of the mailbox @@ -636,6 +762,9 @@ class SoledadMailAdaptor(SoledadIndexMixin): """ return mbox_wrapper.update(store) + def delete_mbox(self, store, mbox_wrapper): + return mbox_wrapper.delete(store) + def get_all_mboxes(self, store): """ Retrieve a list with wrappers for all the mailboxes. @@ -660,15 +789,17 @@ def _split_into_parts(raw): walk.get_body_phash_multi][int(multi)] body_phash = body_phash_fun(walk.get_payloads(msg)) parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) + cdocs_list = list(walk.get_raw_docs(msg, parts)) + cdocs_phashes = [c['phash'] for c in cdocs_list] + mdoc = _build_meta_doc(chash, cdocs_phashes) fdoc = _build_flags_doc(chash, size, multi) hdoc = _build_headers_doc(msg, chash, parts_map) # The MessageWrapper expects a dict, one-indexed - cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1)) + cdocs = dict(enumerate(cdocs_list, 1)) - # XXX convert each to_dicts... - return fdoc, hdoc, cdocs + return mdoc, fdoc, hdoc, cdocs def _parse_msg(raw): @@ -680,6 +811,14 @@ def _parse_msg(raw): return msg, parts, chash, size, multi +def _build_meta_doc(chash, cdocs_phashes): + _mdoc = MetaMsgDocWrapper() + _mdoc.fdoc = constants.FDOCID.format(mbox=INBOX_NAME, chash=chash) + _mdoc.hdoc = constants.HDOCID.format(chash=chash) + _mdoc.cdocs = [constants.CDOCID.format(phash=p) for p in cdocs_phashes] + return _mdoc.serialize() + + def _build_flags_doc(chash, size, multi): _fdoc = FlagsDocWrapper(chash=chash, size=size, multi=multi) return _fdoc.serialize() diff --git a/src/leap/mail/adaptors/tests/rfc822.message b/src/leap/mail/adaptors/tests/rfc822.message deleted file mode 100644 index ee97ab9..0000000 --- a/src/leap/mail/adaptors/tests/rfc822.message +++ /dev/null @@ -1,86 +0,0 @@ -Return-Path: -Delivered-To: exarkun@meson.dyndns.org -Received: from localhost [127.0.0.1] - by localhost with POP3 (fetchmail-6.2.1) - for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) -Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) - by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 - for ; Thu, 20 Mar 2003 14:49:27 -0500 (EST) -Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) - by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) - id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 -Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) - id 18w63j-0007VK-00 - for ; Thu, 20 Mar 2003 13:50:39 -0600 -To: twisted-commits@twistedmatrix.com -From: etrepum CVS -Reply-To: twisted-python@twistedmatrix.com -X-Mailer: CVSToys -Message-Id: -Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. -Sender: twisted-commits-admin@twistedmatrix.com -Errors-To: twisted-commits-admin@twistedmatrix.com -X-BeenThere: twisted-commits@twistedmatrix.com -X-Mailman-Version: 2.0.11 -Precedence: bulk -List-Help: -List-Post: -List-Subscribe: , - -List-Id: -List-Unsubscribe: , - -List-Archive: -Date: Thu, 20 Mar 2003 13:50:39 -0600 - -Modified files: -Twisted/twisted/python/rebuild.py 1.19 1.20 - -Log message: -rebuild now works on python versions from 2.2.0 and up. - - -ViewCVS links: -http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted - -Index: Twisted/twisted/python/rebuild.py -diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 ---- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 -+++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 -@@ -206,15 +206,27 @@ - clazz.__dict__.clear() - clazz.__getattr__ = __getattr__ - clazz.__module__ = module.__name__ -+ if newclasses: -+ import gc -+ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): -+ hasBrokenRebuild = 1 -+ gc_objects = gc.get_objects() -+ else: -+ hasBrokenRebuild = 0 - for nclass in newclasses: - ga = getattr(module, nclass.__name__) - if ga is nclass: - log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) - else: -- import gc -- for r in gc.get_referrers(nclass): -- if isinstance(r, nclass): -+ if hasBrokenRebuild: -+ for r in gc_objects: -+ if not getattr(r, '__class__', None) is nclass: -+ continue - r.__class__ = ga -+ else: -+ for r in gc.get_referrers(nclass): -+ if getattr(r, '__class__', None) is nclass: -+ r.__class__ = ga - if doLog: - log.msg('') - log.msg(' (fixing %s): ' % str(module.__name__)) - - -_______________________________________________ -Twisted-commits mailing list -Twisted-commits@twistedmatrix.com -http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits diff --git a/src/leap/mail/adaptors/tests/rfc822.message b/src/leap/mail/adaptors/tests/rfc822.message new file mode 120000 index 0000000..b19cc28 --- /dev/null +++ b/src/leap/mail/adaptors/tests/rfc822.message @@ -0,0 +1 @@ +../../tests/rfc822.message \ No newline at end of file diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py index 657a602..0cca5ef 100644 --- a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -18,106 +18,22 @@ Tests for the Soledad Adaptor module - leap.mail.adaptors.soledad """ import os -import shutil -import tempfile - from functools import partial from twisted.internet import defer from twisted.trial import unittest -from leap.common.testing.basetest import BaseLeapTest from leap.mail.adaptors import models from leap.mail.adaptors.soledad import SoledadDocumentWrapper from leap.mail.adaptors.soledad import SoledadIndexMixin from leap.mail.adaptors.soledad import SoledadMailAdaptor -from leap.soledad.client import Soledad - -TEST_USER = "testuser@leap.se" -TEST_PASSWD = "1234" +from leap.mail.tests.common import SoledadTestMixin # DEBUG # import logging # logging.basicConfig(level=logging.DEBUG) -def initialize_soledad(email, gnupg_home, tempdir): - """ - Initializes soledad by hand - - :param email: ID for the user - :param gnupg_home: path to home used by gnupg - :param tempdir: path to temporal dir - :rtype: Soledad instance - """ - - uuid = "foobar-uuid" - passphrase = u"verysecretpassphrase" - secret_path = os.path.join(tempdir, "secret.gpg") - local_db_path = os.path.join(tempdir, "soledad.u1db") - server_url = "https://provider" - cert_file = "" - - soledad = Soledad( - uuid, - passphrase, - secret_path, - local_db_path, - server_url, - cert_file, - syncable=False) - - return soledad - - -# TODO move to common module -# XXX remove duplication -class SoledadTestMixin(BaseLeapTest): - """ - It is **VERY** important that this base is added *AFTER* unittest.TestCase - """ - - def setUp(self): - self.results = [] - - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir - - # Soledad: config info - self.gnupg_home = "%s/gnupg" % self.tempdir - self.email = 'leap@leap.se' - - # initialize soledad by hand so we can control keys - self._soledad = initialize_soledad( - self.email, - self.gnupg_home, - self.tempdir) - - def tearDown(self): - """ - tearDown method called after each test. - """ - self.results = [] - try: - self._soledad.close() - except Exception as exc: - print "ERROR WHILE CLOSING SOLEDAD" - # logging.exception(exc) - finally: - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check - assert 'leap_tests-' in self.tempdir - shutil.rmtree(self.tempdir) - - class CounterWrapper(SoledadDocumentWrapper): class model(models.SerializableModel): counter = 0 @@ -357,7 +273,7 @@ class SoledadDocWrapperTestCase(unittest.TestCase, SoledadTestMixin): d.addCallback(assert_actor_list_is_expected) return d -here = os.path.split(os.path.abspath(__file__))[0] +HERE = os.path.split(os.path.abspath(__file__))[0] class TestMessageClass(object): @@ -391,7 +307,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): def test_get_msg_from_string(self): adaptor = self.get_adaptor() - with open(os.path.join(here, "rfc822.message")) as f: + with open(os.path.join(HERE, "rfc822.message")) as f: raw = f.read() msg = adaptor.get_msg_from_string(TestMessageClass, raw) @@ -416,6 +332,10 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): def test_get_msg_from_docs(self): adaptor = self.get_adaptor() + mdoc = dict( + fdoc="F-Foobox-deadbeef", + hdoc="H-deadbeef", + cdocs=["C-deadabad"]) fdoc = dict( mbox="Foobox", flags=('\Seen', '\Nice'), @@ -423,13 +343,14 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): seen=False, deleted=False, recent=False, multi=False) hdoc = dict( + chash="deadbeef", subject="Test Msg") cdocs = { 1: dict( raw='This is a test message')} msg = adaptor.get_msg_from_docs( - TestMessageClass, fdoc, hdoc, cdocs=cdocs) + TestMessageClass, mdoc, fdoc, hdoc, cdocs=cdocs) self.assertEqual(msg.wrapper.fdoc.flags, ('\Seen', '\Nice')) self.assertEqual(msg.wrapper.fdoc.tags, @@ -441,15 +362,20 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): self.assertEqual(msg.wrapper.cdocs[1].raw, "This is a test message") + def test_get_msg_from_metamsg_doc_id(self): + # XXX complete-me! + self.fail() + def test_create_msg(self): adaptor = self.get_adaptor() - with open(os.path.join(here, "rfc822.message")) as f: + with open(os.path.join(HERE, "rfc822.message")) as f: raw = f.read() msg = adaptor.get_msg_from_string(TestMessageClass, raw) def check_create_result(created): - self.assertEqual(len(created), 3) + # that's one mdoc, one hdoc, one fdoc, one cdoc + self.assertEqual(len(created), 4) for doc in created: self.assertTrue( doc.__class__.__name__, @@ -461,7 +387,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): def test_update_msg(self): adaptor = self.get_adaptor() - with open(os.path.join(here, "rfc822.message")) as f: + with open(os.path.join(HERE, "rfc822.message")) as f: raw = f.read() def assert_msg_has_doc_id(ignored, msg): @@ -493,7 +419,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): msg = adaptor.get_msg_from_string(TestMessageClass, raw) d = adaptor.create_msg(adaptor.store, msg) d.addCallback(lambda _: adaptor.store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 3)) + d.addCallback(partial(self.assert_num_docs, 4)) d.addCallback(assert_msg_has_doc_id, msg) d.addCallback(assert_msg_has_no_flags, msg) diff --git a/src/leap/mail/constants.py b/src/leap/mail/constants.py index 55bf1da..bf1db7f 100644 --- a/src/leap/mail/constants.py +++ b/src/leap/mail/constants.py @@ -19,3 +19,20 @@ Constants for leap.mail. """ INBOX_NAME = "INBOX" + +# Regular expressions for the identifiers to be used in the Message Data Layer. + +METAMSGID = "M-{mbox}-{chash}" +METAMSGID_RE = "M\-{mbox}\-[0-9a-fA-F]+" +METAMSGID_CHASH_RE = "M\-\w+\-([0-9a-fA-F]+)" +METAMSGID_MBOX_RE = "M\-(\w+)\-[0-9a-fA-F]+" + +FDOCID = "F-{mbox}-{chash}" +FDOCID_RE = "F\-{mbox}\-[0-9a-fA-F]+" +FDOCID_CHASH_RE = "F\-\w+\-([0-9a-fA-F]+)" + +HDOCID = "H-{chash}" +HDOCID_RE = "H\-[0-9a-fA-F]+" + +CDOCID = "C-{phash}" +CDOCID_RE = "C\-[0-9a-fA-F]+" diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index ea9c95e..ca07f67 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -20,6 +20,7 @@ Generic Access to Mail objects: Public LEAP Mail API. from twisted.internet import defer from leap.mail.constants import INBOX_NAME +from leap.mail.mailbox_indexer import MailboxIndexer from leap.mail.adaptors.soledad import SoledadMailAdaptor @@ -27,8 +28,17 @@ from leap.mail.adaptors.soledad import SoledadMailAdaptor # [ ] Probably change the name of this module to "api" or "account", mail is # too generic (there's also IncomingMail, and OutgoingMail +def _get_mdoc_id(mbox, chash): + """ + Get the doc_id for the metamsg document. + """ + return "M+{mbox}+{chash}".format(mbox=mbox, chash=chash) + class Message(object): + """ + Represents a single message, and gives access to all its attributes. + """ def __init__(self, wrapper): """ @@ -37,45 +47,56 @@ class Message(object): self._wrapper = wrapper def get_wrapper(self): + """ + Get the wrapper for this message. + """ return self._wrapper # imap.IMessage methods - def get_flags(): + def get_flags(self): """ """ + return tuple(self._wrapper.fdoc.flags) - def get_internal_date(): + def get_internal_date(self): """ """ + return self._wrapper.fdoc.date # imap.IMessageParts - def get_headers(): + def get_headers(self): """ """ + # XXX process here? from imap.messages + return self._wrapper.hdoc.headers - def get_body_file(): + def get_body_file(self): """ """ - def get_size(): + def get_size(self): """ """ + return self._wrapper.fdoc.size - def is_multipart(): + def is_multipart(self): """ """ + return self._wrapper.fdoc.multi - def get_subpart(part): + def get_subpart(self, part): """ """ + # XXX ??? return MessagePart? # Custom methods. - def get_tags(): + def get_tags(self): """ """ + return tuple(self._wrapper.fdoc.tags) class MessageCollection(object): @@ -85,43 +106,174 @@ class MessageCollection(object): master documents. Since LEAP Mail is primarily oriented to store mail in Soledad, the default - (and, so far, only) implementation of the store is contained in this - Soledad Mail Adaptor. If you need to use a different adaptor, change the + (and, so far, only) implementation of the store is contained in the + Soledad Mail Adaptor, which is passed to every collection on creation by + the root Account object. If you need to use a different adaptor, change the adaptor class attribute in your Account object. Store is a reference to a particular instance of the message store (soledad instance or proxy, for instance). """ - # TODO look at IMessageSet methods + # TODO + # [ ] look at IMessageSet methods + # [ ] make constructor with a per-instance deferredLock to use on + # creation/deletion? + # [ ] instead of a mailbox, we could pass an arbitrary container with + # pointers to different doc_ids (type: foo) + # [ ] To guarantee synchronicity of the documents sent together during a + # sync, we could get hold of a deferredLock that inhibits + # synchronization while we are updating (think more about this!) # Account should provide an adaptor instance when creating this collection. adaptor = None store = None + messageklass = Message - def get_message_by_doc_id(self, doc_id): - # ... get from soledad etc - # ... but that should be part of adaptor/store too... :/ - fdoc, hdoc = None - return self.adaptor.from_docs(Message, fdoc=fdoc, hdoc=hdoc) + def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): + """ + """ + self.adaptor = adaptor + self.store = store - # TODO review if this is the best place for: + # TODO I have to think about what to do when there is no mbox passed to + # the initialization. We could still get the MetaMsg by index, instead + # of by doc_id. See get_message_by_content_hash + self.mbox_indexer = mbox_indexer + self.mbox_wrapper = mbox_wrapper - def create_docs(): - pass + def is_mailbox_collection(self): + """ + Return True if this collection represents a Mailbox. + :rtype: bool + """ + return bool(self.mbox_wrapper) + + # Get messages + + def get_message_by_content_hash(self, chash, get_cdocs=False): + """ + Retrieve a message by its content hash. + :rtype: Deferred + """ + + if not self.is_mailbox_collection(): + # instead of getting the metamsg by chash, query by (meta) index + # or use the internal collection of pointers-to-docs. + raise NotImplementedError() + + metamsg_id = _get_mdoc_id(self.mbox_wrapper.mbox, chash) + + return self.adaptor.get_msg_from_mdoc_id( + self.messageklass, self.store, + metamsg_id, get_cdocs=get_cdocs) + + def get_message_by_uid(self, uid, absolute=True, get_cdocs=False): + """ + Retrieve a message by its Unique Identifier. + + If this is a Mailbox collection, that is the message UID, unique for a + given mailbox, or a relative sequence number depending on the absolute + flag. For now, only absolute identifiers are supported. + :rtype: Deferred + """ + if not absolute: + raise NotImplementedError("Does not support relative ids yet") + + def get_msg_from_mdoc_id(doc_id): + return self.adaptor.get_msg_from_mdoc_id( + self.messageklass, self.store, + doc_id, get_cdocs=get_cdocs) + + d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_wrapper.mbox, uid) + d.addCallback(get_msg_from_mdoc_id) + return d + + def count(self): + """ + Count the messages in this collection. + :rtype: int + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + return self.mbox_indexer.count(self.mbox_wrapper.mbox) + + # Manipulate messages + + def add_msg(self, raw_msg): + """ + Add a message to this collection. + """ + msg = self.adaptor.get_msg_from_string(Message, raw_msg) + wrapper = msg.get_wrapper() + + if self.is_mailbox_collection(): + mbox = self.mbox_wrapper.mbox + wrapper.set_mbox(mbox) + + def insert_mdoc_id(_): + # XXX does this work? + doc_id = wrapper.mdoc.doc_id + return self.mbox_indexer.insert_doc( + self.mbox_wrapper.mbox, doc_id) + + d = wrapper.create(self.store) + d.addCallback(insert_mdoc_id) + return d + + def copy_msg(self, msg, newmailbox): + """ + Copy the message to another collection. (it only makes sense for + mailbox collections) + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + + def insert_copied_mdoc_id(wrapper): + return self.mbox_indexer.insert_doc( + newmailbox, wrapper.mdoc.doc_id) - def udpate_flags(): + wrapper = msg.get_wrapper() + d = wrapper.copy(self.store, newmailbox) + d.addCallback(insert_copied_mdoc_id) + return d + + def delete_msg(self, msg): + """ + Delete this message. + """ + wrapper = msg.get_wrapper() + + def delete_mdoc_id(_): + # XXX does this work? + doc_id = wrapper.mdoc.doc_id + return self.mbox_indexer.delete_doc_by_hash( + self.mbox_wrapper.mbox, doc_id) + d = wrapper.delete(self.store) + d.addCallback(delete_mdoc_id) + return d + + # TODO should add a delete-by-uid to collection? + + def udpate_flags(self, msg, flags, mode): + """ + Update flags for a given message. + """ + wrapper = msg.get_wrapper() # 1. update the flags in the message wrapper --- stored where??? - # 2. call adaptor.update_msg(store) + # 2. update the special flags in the wrapper (seen, etc) + # 3. call adaptor.update_msg(store) pass - def update_tags(): + def update_tags(self, msg, tags, mode): + """ + Update tags for a given message. + """ + wrapper = msg.get_wrapper() # 1. update the tags in the message wrapper --- stored where??? # 2. call adaptor.update_msg(store) pass - # TODO add delete methods here? - class Account(object): """ @@ -147,8 +299,8 @@ class Account(object): def __init__(self, store): self.store = store self.adaptor = self.adaptor_class() + self.mbox_indexer = MailboxIndexer(self.store) - self.__mailboxes = set([]) self._initialized = False self._deferred_initialization = defer.Deferred() @@ -156,23 +308,16 @@ class Account(object): def _initialize_storage(self): - def add_mailbox_if_none(result): - # every user should have the right to an inbox folder - # at least, so let's make one! - if not self.mailboxes: + def add_mailbox_if_none(mboxes): + if not mboxes: self.add_mailbox(INBOX_NAME) def finish_initialization(result): self._initialized = True self._deferred_initialization.callback(None) - def load_mbox_cache(result): - d = self._load_mailboxes() - d.addCallback(lambda _: result) - return d - d = self.adaptor.initialize_store(self.store) - d.addCallback(load_mbox_cache) + d.addCallback(self.list_all_mailbox_names) d.addCallback(add_mailbox_if_none) d.addCallback(finish_initialization) @@ -185,64 +330,83 @@ class Account(object): self._deferred_initialization.addCallback(cb) return self._deferred_initialization - @property - def mailboxes(self): - """ - A list of the current mailboxes for this account. - :rtype: set - """ - return sorted(self.__mailboxes) + # + # Public API Starts + # - def _load_mailboxes(self): + def list_all_mailbox_names(self): + def filter_names(mboxes): + return [m.name for m in mboxes] - def update_mailboxes(mbox_names): - self.__mailboxes.update(mbox_names) + d = self.get_all_mailboxes() + d.addCallback(filter_names) + return d + def get_all_mailboxes(self): d = self.adaptor.get_all_mboxes(self.store) - d.addCallback(update_mailboxes) return d - # - # Public API Starts - # + def add_mailbox(self, name): - # XXX params for IMAP only??? - def list_mailboxes(self, ref, wildcard): - self.adaptor.get_all_mboxes(self.store) - - def add_mailbox(self, name, mbox=None): - pass + def create_uid_table_cb(res): + d = self.mbox_uid.create_table(name) + d.addCallback(lambda _: res) + return d - def create_mailbox(self, pathspec): - pass + d = self.adaptor.__class__.get_or_create(name) + d.addCallback(create_uid_table_cb) + return d def delete_mailbox(self, name): - pass + def delete_uid_table_cb(res): + d = self.mbox_uid.delete_table(name) + d.addCallback(lambda _: res) + return d + + d = self.adaptor.delete_mbox(self.store) + d.addCallback(delete_uid_table_cb) + return d def rename_mailbox(self, oldname, newname): - pass + def _rename_mbox(wrapper): + wrapper.mbox = newname + return wrapper.update() - # FIXME yet to be decided if it belongs here... + def rename_uid_table_cb(res): + d = self.mbox_uid.rename_table(oldname, newname) + d.addCallback(lambda _: res) + return d + + d = self.adaptor.__class__.get_or_create(oldname) + d.addCallback(_rename_mbox) + d.addCallback(rename_uid_table_cb) + return d def get_collection_by_mailbox(self, name): """ :rtype: MessageCollection """ # imap select will use this, passing the collection to SoledadMailbox - # XXX pass adaptor to MessageCollection - pass + def get_collection_for_mailbox(mbox_wrapper): + return MessageCollection( + self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) + + mboxwrapper_klass = self.adaptor.mboxwrapper_klass + d = mboxwrapper_klass.get_or_create(name) + d.addCallback(get_collection_for_mailbox) + return d def get_collection_by_docs(self, docs): """ :rtype: MessageCollection """ # get a collection of docs by a list of doc_id - # XXX pass adaptor to MessageCollection - pass + # get.docs(...) --> it should be a generator. does it behave in the + # threadpool? + raise NotImplementedError() def get_collection_by_tag(self, tag): """ :rtype: MessageCollection """ - # is this a good idea? - pass + raise NotImplementedError() diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py new file mode 100644 index 0000000..bc298ea --- /dev/null +++ b/src/leap/mail/mailbox_indexer.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 -*- +# mailbox_indexer.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Local tables to store the message Unique Identifiers for a given mailbox. +""" +import re + +from leap.mail.constants import METAMSGID_RE + + +class WrongMetaDocIDError(Exception): + pass + + +class MailboxIndexer(object): + """ + This class contains the commands needed to create, modify and alter the + local-only UID tables for a given mailbox. + + Its purpouse is to keep a local-only index with the messages in each + mailbox, mainly to satisfy the demands of the IMAP specification, but + useful too for any effective listing of the messages in a mailbox. + + Since the incoming mail can be processed at any time in any replica, it's + preferred not to attempt to maintain a global chronological global index. + + These indexes are Message Attributes needed for the IMAP specification (rfc + 3501), although they can be useful for other non-imap store + implementations. + """ + # The uids are expected to be 32-bits values, but the ROWIDs in sqlite + # are 64-bit values. I *don't* think it really matters for any + # practical use, but it's good to remmeber we've got that difference going + # on. + + store = None + table_preffix = "leapmail_uid_" + + def __init__(self, store): + self.store = store + + def _query(self, *args, **kw): + assert self.store is not None + return self.store.raw_sqlcipher_query(*args, **kw) + + def create_table(self, mailbox): + """ + Create the UID table for a given mailbox. + :param mailbox: the mailbox name + :type mailbox: str + :rtype: Deferred + """ + assert mailbox + sql = ("CREATE TABLE if not exists {preffix}{name}( " + "uid INTEGER PRIMARY KEY AUTOINCREMENT, " + "hash TEXT UNIQUE NOT NULL)".format( + preffix=self.table_preffix, name=mailbox)) + return self._query(sql) + + def delete_table(self, mailbox): + """ + Delete the UID table for a given mailbox. + :param mailbox: the mailbox name + :type mailbox: str + :rtype: Deferred + """ + assert mailbox + sql = ("DROP TABLE if exists {preffix}{name}".format( + preffix=self.table_preffix, name=mailbox)) + return self._query(sql) + + def rename_table(self, oldmailbox, newmailbox): + """ + Delete the UID table for a given mailbox. + :param oldmailbox: the old mailbox name + :type oldmailbox: str + :param newmailbox: the new mailbox name + :type newmailbox: str + :rtype: Deferred + """ + assert oldmailbox + assert newmailbox + assert oldmailbox != newmailbox + sql = ("ALTER TABLE {preffix}{old} " + "RENAME TO {preffix}{new}".format( + preffix=self.table_preffix, + old=oldmailbox, new=newmailbox)) + return self._query(sql) + + def insert_doc(self, mailbox, doc_id): + """ + Insert the doc_id for a MetaMsg in the UID table for a given mailbox. + + The doc_id must be in the format: + + M++ + + :param mailbox: the mailbox name + :type mailbox: str + :param doc_id: the doc_id for the MetaMsg + :type doc_id: str + :return: a deferred that will fire with the uid of the newly inserted + document. + :rtype: Deferred + """ + assert mailbox + assert doc_id + + if not re.findall(METAMSGID_RE.format(mbox=mailbox), doc_id): + raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") + + def get_rowid(result): + return result[0][0] + + sql = ("INSERT INTO {preffix}{name} VALUES (" + "NULL, ?)".format( + preffix=self.table_preffix, name=mailbox)) + values = (doc_id,) + + sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " + "LIMIT 1;").format( + preffix=self.table_preffix, name=mailbox) + d = self._query(sql, values) + d.addCallback(lambda _: self._query(sql_last)) + d.addCallback(get_rowid) + return d + + def delete_doc_by_uid(self, mailbox, uid): + """ + Delete the entry for a MetaMsg in the UID table for a given mailbox. + + :param mailbox: the mailbox name + :type mailbox: str + :param uid: the UID of the message. + :type uid: int + :rtype: Deferred + """ + assert mailbox + assert uid + sql = ("DELETE FROM {preffix}{name} " + "WHERE uid=?".format( + preffix=self.table_preffix, name=mailbox)) + values = (uid,) + return self._query(sql, values) + + def delete_doc_by_hash(self, mailbox, doc_id): + """ + Delete the entry for a MetaMsg in the UID table for a given mailbox. + + The doc_id must be in the format: + + M++ + + :param mailbox: the mailbox name + :type mailbox: str + :param doc_id: the doc_id for the MetaMsg + :type doc_id: str + :return: a deferred that will fire with the uid of the newly inserted + document. + :rtype: Deferred + """ + assert mailbox + assert doc_id + sql = ("DELETE FROM {preffix}{name} " + "WHERE hash=?".format( + preffix=self.table_preffix, name=mailbox)) + values = (doc_id,) + return self._query(sql, values) + + def get_doc_id_from_uid(self, mailbox, uid): + """ + Get the doc_id for a MetaMsg in the UID table for a given mailbox. + + :param mailbox: the mailbox name + :type mailbox: str + :param uid: the uid for the MetaMsg for this mailbox + :type uid: int + :rtype: Deferred + """ + def get_hash(result): + return result[0][0] + + sql = ("SELECT hash from {preffix}{name} " + "WHERE uid=?".format( + preffix=self.table_preffix, name=mailbox)) + values = (uid,) + d = self._query(sql, values) + d.addCallback(get_hash) + return d + + def get_doc_ids_from_uids(self, mailbox, uids): + # For IMAP relative numbering /sequences. + # XXX dereference the range (n,*) + raise NotImplementedError() + + def count(self, mailbox): + """ + Get the number of entries in the UID table for a given mailbox. + + :param mailbox: the mailbox name + :type mailbox: str + :return: a deferred that will fire with an integer returning the count. + :rtype: Deferred + """ + def get_count(result): + return result[0][0] + + sql = ("SELECT Count(*) FROM {preffix}{name};".format( + preffix=self.table_preffix, name=mailbox)) + d = self._query(sql) + d.addCallback(get_count) + return d + + def get_next_uid(self, mailbox): + """ + Get the next integer beyond the highest UID count for a given mailbox. + + This is expected by the IMAP implementation. There are no guarantees + that a document to be inserted in the future gets the returned UID: the + only thing that can be assured is that it will be equal or greater than + the value returned. + + :param mailbox: the mailbox name + :type mailbox: str + :return: a deferred that will fire with an integer returning the next + uid. + :rtype: Deferred + """ + assert mailbox + + def increment(result): + return result[0][0] + 1 + + sql = ("SELECT MAX(rowid) FROM {preffix}{name} " + "LIMIT 1;").format( + preffix=self.table_preffix, name=mailbox) + + d = self._query(sql) + d.addCallback(increment) + return d diff --git a/src/leap/mail/tests/common.py b/src/leap/mail/tests/common.py new file mode 100644 index 0000000..fefa7ee --- /dev/null +++ b/src/leap/mail/tests/common.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +# common.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Common utilities for testing Soledad. +""" +import os +import shutil +import tempfile + +from leap.common.testing.basetest import BaseLeapTest +from leap.soledad.client import Soledad + +# TODO move to common module, or Soledad itself +# XXX remove duplication + +TEST_USER = "testuser@leap.se" +TEST_PASSWD = "1234" + + +def _initialize_soledad(email, gnupg_home, tempdir): + """ + Initializes soledad by hand + + :param email: ID for the user + :param gnupg_home: path to home used by gnupg + :param tempdir: path to temporal dir + :rtype: Soledad instance + """ + + uuid = "foobar-uuid" + passphrase = u"verysecretpassphrase" + secret_path = os.path.join(tempdir, "secret.gpg") + local_db_path = os.path.join(tempdir, "soledad.u1db") + server_url = "https://provider" + cert_file = "" + + soledad = Soledad( + uuid, + passphrase, + secret_path, + local_db_path, + server_url, + cert_file, + syncable=False) + + return soledad + + +class SoledadTestMixin(BaseLeapTest): + """ + It is **VERY** important that this base is added *AFTER* unittest.TestCase + """ + + def setUp(self): + self.results = [] + + self.old_path = os.environ['PATH'] + self.old_home = os.environ['HOME'] + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.home = self.tempdir + bin_tdir = os.path.join( + self.tempdir, + 'bin') + os.environ["PATH"] = bin_tdir + os.environ["HOME"] = self.tempdir + + # Soledad: config info + self.gnupg_home = "%s/gnupg" % self.tempdir + self.email = 'leap@leap.se' + + # initialize soledad by hand so we can control keys + self._soledad = _initialize_soledad( + self.email, + self.gnupg_home, + self.tempdir) + + def tearDown(self): + """ + tearDown method called after each test. + """ + self.results = [] + try: + self._soledad.close() + except Exception as exc: + print "ERROR WHILE CLOSING SOLEDAD" + # logging.exception(exc) + finally: + os.environ["PATH"] = self.old_path + os.environ["HOME"] = self.old_home + # safety check + assert 'leap_tests-' in self.tempdir + shutil.rmtree(self.tempdir) diff --git a/src/leap/mail/tests/rfc822.message b/src/leap/mail/tests/rfc822.message new file mode 100644 index 0000000..ee97ab9 --- /dev/null +++ b/src/leap/mail/tests/rfc822.message @@ -0,0 +1,86 @@ +Return-Path: +Delivered-To: exarkun@meson.dyndns.org +Received: from localhost [127.0.0.1] + by localhost with POP3 (fetchmail-6.2.1) + for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) +Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) + by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 + for ; Thu, 20 Mar 2003 14:49:27 -0500 (EST) +Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) + by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) + id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 +Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) + id 18w63j-0007VK-00 + for ; Thu, 20 Mar 2003 13:50:39 -0600 +To: twisted-commits@twistedmatrix.com +From: etrepum CVS +Reply-To: twisted-python@twistedmatrix.com +X-Mailer: CVSToys +Message-Id: +Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. +Sender: twisted-commits-admin@twistedmatrix.com +Errors-To: twisted-commits-admin@twistedmatrix.com +X-BeenThere: twisted-commits@twistedmatrix.com +X-Mailman-Version: 2.0.11 +Precedence: bulk +List-Help: +List-Post: +List-Subscribe: , + +List-Id: +List-Unsubscribe: , + +List-Archive: +Date: Thu, 20 Mar 2003 13:50:39 -0600 + +Modified files: +Twisted/twisted/python/rebuild.py 1.19 1.20 + +Log message: +rebuild now works on python versions from 2.2.0 and up. + + +ViewCVS links: +http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted + +Index: Twisted/twisted/python/rebuild.py +diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 +--- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 ++++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 +@@ -206,15 +206,27 @@ + clazz.__dict__.clear() + clazz.__getattr__ = __getattr__ + clazz.__module__ = module.__name__ ++ if newclasses: ++ import gc ++ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): ++ hasBrokenRebuild = 1 ++ gc_objects = gc.get_objects() ++ else: ++ hasBrokenRebuild = 0 + for nclass in newclasses: + ga = getattr(module, nclass.__name__) + if ga is nclass: + log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) + else: +- import gc +- for r in gc.get_referrers(nclass): +- if isinstance(r, nclass): ++ if hasBrokenRebuild: ++ for r in gc_objects: ++ if not getattr(r, '__class__', None) is nclass: ++ continue + r.__class__ = ga ++ else: ++ for r in gc.get_referrers(nclass): ++ if getattr(r, '__class__', None) is nclass: ++ r.__class__ = ga + if doLog: + log.msg('') + log.msg(' (fixing %s): ' % str(module.__name__)) + + +_______________________________________________ +Twisted-commits mailing list +Twisted-commits@twistedmatrix.com +http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py new file mode 100644 index 0000000..ce2366c --- /dev/null +++ b/src/leap/mail/tests/test_mail.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +# test_mail.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the mail module. +""" +import os +from functools import partial + +from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.mail.mail import MessageCollection +from leap.mail.mailbox_indexer import MailboxIndexer +from leap.mail.tests.common import SoledadTestMixin + +from twisted.internet import defer +from twisted.trial import unittest + +HERE = os.path.split(os.path.abspath(__file__))[0] + + +class MessageCollectionTestCase(unittest.TestCase, SoledadTestMixin): + """ + Tests for the SoledadDocumentWrapper. + """ + + def get_collection(self, mbox_collection=True): + """ + Get a collection for tests. + """ + adaptor = SoledadMailAdaptor() + store = self._soledad + adaptor.store = store + if mbox_collection: + mbox_indexer = MailboxIndexer(store) + mbox_name = "TestMbox" + else: + mbox_indexer = mbox_name = None + + def get_collection_from_mbox_wrapper(wrapper): + return MessageCollection( + adaptor, store, + mbox_indexer=mbox_indexer, mbox_wrapper=wrapper) + + d = adaptor.initialize_store(store) + if mbox_collection: + d.addCallback(lambda _: mbox_indexer.create_table(mbox_name)) + d.addCallback(lambda _: adaptor.get_or_create_mbox(store, mbox_name)) + d.addCallback(get_collection_from_mbox_wrapper) + return d + + def test_is_mailbox_collection(self): + + def assert_is_mbox_collection(collection): + self.assertTrue(collection.is_mailbox_collection()) + + d = self.get_collection() + d.addCallback(assert_is_mbox_collection) + return d + + def assert_collection_count(self, _, expected, collection): + + def _assert_count(count): + self.assertEqual(count, expected) + d = collection.count() + d.addCallback(_assert_count) + return d + + def test_add_msg(self): + + with open(os.path.join(HERE, "rfc822.message")) as f: + raw = f.read() + + def add_msg_to_collection_and_assert_count(collection): + d = collection.add_msg(raw) + d.addCallback(partial( + self.assert_collection_count, + expected=1, collection=collection)) + return d + + d = self.get_collection() + d.addCallback(add_msg_to_collection_and_assert_count) + return d diff --git a/src/leap/mail/tests/test_mailbox_indexer.py b/src/leap/mail/tests/test_mailbox_indexer.py new file mode 100644 index 0000000..47a3bdc --- /dev/null +++ b/src/leap/mail/tests/test_mailbox_indexer.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- +# test_mailbox_indexer.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the mailbox_indexer module. +""" +from functools import partial + +from twisted.trial import unittest + +from leap.mail import mailbox_indexer as mi +from leap.mail.tests.common import SoledadTestMixin + +hash_test0 = '590c9f8430c7435807df8ba9a476e3f1295d46ef210f6efae2043a4c085a569e' +hash_test1 = '1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014' +hash_test2 = '60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752' +hash_test3 = 'fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13' +hash_test4 = 'a4e624d686e03ed2767c0abd85c14426b0b1157d2ce81d27bb4fe4f6f01d688a' + + +def fmt_hash(mailbox, hash): + return "M-" + mailbox + "-" + hash + + +class MailboxIndexerTestCase(unittest.TestCase, SoledadTestMixin): + """ + Tests for the MailboxUID class. + """ + def get_mbox_uid(self): + m_uid = mi.MailboxIndexer(self._soledad) + return m_uid + + def list_mail_tables_cb(self, ignored): + def filter_mailuid_tables(tables): + filtered = [ + table[0] for table in tables if + table[0].startswith(mi.MailboxIndexer.table_preffix)] + return filtered + + sql = "SELECT name FROM sqlite_master WHERE type='table';" + d = self._soledad.raw_sqlcipher_query(sql) + d.addCallback(filter_mailuid_tables) + return d + + def select_uid_rows(self, mailbox): + sql = "SELECT * FROM %s%s;" % ( + mi.MailboxIndexer.table_preffix, mailbox) + d = self._soledad.raw_sqlcipher_query(sql) + return d + + def test_create_table(self): + def assert_table_created(tables): + self.assertEqual( + tables, ["leapmail_uid_inbox"]) + + m_uid = self.get_mbox_uid() + d = m_uid.create_table('inbox') + d.addCallback(self.list_mail_tables_cb) + d.addCallback(assert_table_created) + return d + + def test_create_and_delete_table(self): + def assert_table_deleted(tables): + self.assertEqual(tables, []) + + m_uid = self.get_mbox_uid() + d = m_uid.create_table('inbox') + d.addCallback(lambda _: m_uid.delete_table('inbox')) + d.addCallback(self.list_mail_tables_cb) + d.addCallback(assert_table_deleted) + return d + + def test_rename_table(self): + def assert_table_renamed(tables): + self.assertEqual( + tables, ["leapmail_uid_foomailbox"]) + + m_uid = self.get_mbox_uid() + d = m_uid.create_table('inbox') + d.addCallback(lambda _: m_uid.rename_table('inbox', 'foomailbox')) + d.addCallback(self.list_mail_tables_cb) + d.addCallback(assert_table_renamed) + return d + + def test_insert_doc(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + h1 = fmt_hash(mbox, hash_test0) + h2 = fmt_hash(mbox, hash_test1) + h3 = fmt_hash(mbox, hash_test2) + h4 = fmt_hash(mbox, hash_test3) + h5 = fmt_hash(mbox, hash_test4) + + def assert_uid_rows(rows): + expected = [(1, h1), (2, h2), (3, h3), (4, h4), (5, h5)] + self.assertEquals(rows, expected) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + d.addCallback(lambda _: self.select_uid_rows(mbox)) + d.addCallback(assert_uid_rows) + return d + + def test_insert_doc_return(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + def assert_rowid(rowid, expected=None): + self.assertEqual(rowid, expected) + + h1 = fmt_hash(mbox, hash_test0) + h2 = fmt_hash(mbox, hash_test1) + h3 = fmt_hash(mbox, hash_test2) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(partial(assert_rowid, expected=1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(partial(assert_rowid, expected=2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(partial(assert_rowid, expected=3)) + return d + + def test_delete_doc(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + h1 = fmt_hash(mbox, hash_test0) + h2 = fmt_hash(mbox, hash_test1) + h3 = fmt_hash(mbox, hash_test2) + h4 = fmt_hash(mbox, hash_test3) + h5 = fmt_hash(mbox, hash_test4) + + def assert_uid_rows(rows): + expected = [(4, h4), (5, h5)] + self.assertEquals(rows, expected) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 1)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 2)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_hash(mbox, h3)) + + d.addCallback(lambda _: self.select_uid_rows(mbox)) + d.addCallback(assert_uid_rows) + return d + + def test_get_doc_id_from_uid(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + h1 = fmt_hash(mbox, hash_test0) + + def assert_doc_hash(res): + self.assertEqual(res, h1) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(lambda _: m_uid.get_doc_id_from_uid(mbox, 1)) + d.addCallback(assert_doc_hash) + return d + + def test_count(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + h1 = fmt_hash(mbox, hash_test0) + h2 = fmt_hash(mbox, hash_test1) + h3 = fmt_hash(mbox, hash_test2) + h4 = fmt_hash(mbox, hash_test3) + h5 = fmt_hash(mbox, hash_test4) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + + def assert_count_after_inserts(count): + self.assertEquals(count, 5) + + d.addCallback(lambda _: m_uid.count(mbox)) + d.addCallback(assert_count_after_inserts) + + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 1)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 2)) + + def assert_count_after_deletions(count): + self.assertEquals(count, 3) + + d.addCallback(lambda _: m_uid.count(mbox)) + d.addCallback(assert_count_after_deletions) + return d + + def test_get_next_uid(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + h1 = fmt_hash(mbox, hash_test0) + h2 = fmt_hash(mbox, hash_test1) + h3 = fmt_hash(mbox, hash_test2) + h4 = fmt_hash(mbox, hash_test3) + h5 = fmt_hash(mbox, hash_test4) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + + def assert_next_uid(result, expected=1): + self.assertEquals(result, expected) + + d.addCallback(lambda _: m_uid.get_next_uid(mbox)) + d.addCallback(partial(assert_next_uid, expected=6)) + return d -- cgit v1.2.3 From 4fa3d27225f44d9598d24d9e59a97baaa9bbb90e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 2 Jan 2015 10:58:43 -0400 Subject: make outgoing a new submodule --- src/leap/mail/outgoing/__init__.py | 0 src/leap/mail/outgoing/service.py | 429 ++++++++++++++++++++++++++ src/leap/mail/outgoing/tests/__init__.py | 0 src/leap/mail/outgoing/tests/test_outgoing.py | 187 +++++++++++ src/leap/mail/service.py | 422 ------------------------- src/leap/mail/smtp/__init__.py | 2 +- src/leap/mail/smtp/gateway.py | 6 +- src/leap/mail/tests/test_service.py | 187 ----------- src/leap/mail/walk.py | 64 ++-- 9 files changed, 650 insertions(+), 647 deletions(-) create mode 100644 src/leap/mail/outgoing/__init__.py create mode 100644 src/leap/mail/outgoing/service.py create mode 100644 src/leap/mail/outgoing/tests/__init__.py create mode 100644 src/leap/mail/outgoing/tests/test_outgoing.py delete mode 100644 src/leap/mail/service.py delete mode 100644 src/leap/mail/tests/test_service.py diff --git a/src/leap/mail/outgoing/__init__.py b/src/leap/mail/outgoing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py new file mode 100644 index 0000000..b70b3b1 --- /dev/null +++ b/src/leap/mail/outgoing/service.py @@ -0,0 +1,429 @@ +# -*- coding: utf-8 -*- +# outgoing/service.py +# Copyright (C) 2013-2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import re +from StringIO import StringIO +from email.parser import Parser +from email.mime.application import MIMEApplication + +from OpenSSL import SSL + +from twisted.mail import smtp +from twisted.internet import reactor +from twisted.internet import defer +from twisted.protocols.amp import ssl +from twisted.python import log + +from leap.common.check import leap_assert_type, leap_assert +from leap.common.events import proto, signal +from leap.keymanager import KeyManager +from leap.keymanager.openpgp import OpenPGPKey +from leap.keymanager.errors import KeyNotFound +from leap.mail import __version__ +from leap.mail.utils import validate_address +from leap.mail.smtp.rfc3156 import MultipartEncrypted +from leap.mail.smtp.rfc3156 import MultipartSigned +from leap.mail.smtp.rfc3156 import encode_base64_rec +from leap.mail.smtp.rfc3156 import RFC3156CompliantGenerator +from leap.mail.smtp.rfc3156 import PGPSignature +from leap.mail.smtp.rfc3156 import PGPEncrypted + +# TODO +# [ ] rename this module to something else, service should be the implementor +# of IService + + +class SSLContextFactory(ssl.ClientContextFactory): + def __init__(self, cert, key): + self.cert = cert + self.key = key + + def getContext(self): + # FIXME -- we should use sslv23 to allow for tlsv1.2 + # and, if possible, explicitely disable sslv3 clientside. + # Servers should avoid sslv3 + self.method = SSL.TLSv1_METHOD # SSLv23_METHOD + ctx = ssl.ClientContextFactory.getContext(self) + ctx.use_certificate_file(self.cert) + ctx.use_privatekey_file(self.key) + return ctx + + +class OutgoingMail: + """ + A service for handling encrypted outgoing mail. + """ + + FOOTER_STRING = "I prefer encrypted email" + + def __init__(self, from_address, keymanager, cert, key, host, port): + """ + Initialize the mail service. + + :param from_address: The sender address. + :type from_address: str + :param keymanager: A KeyManager for retrieving recipient's keys. + :type keymanager: leap.common.keymanager.KeyManager + :param cert: The client certificate for SSL authentication. + :type cert: str + :param key: The client private key for SSL authentication. + :type key: str + :param host: The hostname of the remote SMTP server. + :type host: str + :param port: The port of the remote SMTP server. + :type port: int + """ + + # assert params + leap_assert_type(from_address, str) + leap_assert('@' in from_address) + leap_assert_type(keymanager, KeyManager) + leap_assert_type(host, str) + leap_assert(host != '') + leap_assert_type(port, int) + leap_assert(port is not 0) + leap_assert_type(cert, unicode) + leap_assert(cert != '') + leap_assert_type(key, unicode) + leap_assert(key != '') + + self._port = port + self._host = host + self._key = key + self._cert = cert + self._from_address = from_address + self._keymanager = keymanager + + def send_message(self, raw, recipient): + """ + Sends a message to a recipient. Maybe encrypts and signs. + + :param raw: The raw message + :type raw: str + :param recipient: The recipient for the message + :type recipient: smtp.User + :return: a deferred which delivers the message when fired + """ + d = self._maybe_encrypt_and_sign(raw, recipient) + d.addCallback(self._route_msg) + d.addErrback(self.sendError) + return d + + def sendSuccess(self, smtp_sender_result): + """ + Callback for a successful send. + + :param smtp_sender_result: The result from the ESMTPSender from + _route_msg + :type smtp_sender_result: tuple(int, list(tuple)) + """ + dest_addrstr = smtp_sender_result[1][0][0] + log.msg('Message sent to %s' % dest_addrstr) + signal(proto.SMTP_SEND_MESSAGE_SUCCESS, dest_addrstr) + + def sendError(self, failure): + """ + Callback for an unsuccessfull send. + + :param e: The result from the last errback. + :type e: anything + """ + # XXX: need to get the address from the exception to send signal + # signal(proto.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr) + err = failure.value + log.err(err) + raise err + + def _route_msg(self, encrypt_and_sign_result): + """ + Sends the msg using the ESMTPSenderFactory. + + :param encrypt_and_sign_result: A tuple containing the 'maybe' + encrypted message and the recipient + :type encrypt_and_sign_result: tuple + """ + message, recipient = encrypt_and_sign_result + log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port)) + msg = message.as_string(False) + + # we construct a defer to pass to the ESMTPSenderFactory + d = defer.Deferred() + d.addCallbacks(self.sendSuccess, self.sendError) + # we don't pass an ssl context factory to the ESMTPSenderFactory + # because ssl will be handled by reactor.connectSSL() below. + factory = smtp.ESMTPSenderFactory( + "", # username is blank because client auth is done on SSL protocol level + "", # password is blank because client auth is done on SSL protocol level + self._from_address, + recipient.dest.addrstr, + StringIO(msg), + d, + heloFallback=True, + requireAuthentication=False, + requireTransportSecurity=True) + factory.domain = __version__ + signal(proto.SMTP_SEND_MESSAGE_START, recipient.dest.addrstr) + reactor.connectSSL( + self._host, self._port, factory, + contextFactory=SSLContextFactory(self._cert, self._key)) + + def _maybe_encrypt_and_sign(self, raw, recipient): + """ + Attempt to encrypt and sign the outgoing message. + + The behaviour of this method depends on: + + 1. the original message's content-type, and + 2. the availability of the recipient's public key. + + If the original message's content-type is "multipart/encrypted", then + the original message is not altered. For any other content-type, the + method attempts to fetch the recipient's public key. If the + recipient's public key is available, the message is encrypted and + signed; otherwise it is only signed. + + Note that, if the C{encrypted_only} configuration is set to True and + the recipient's public key is not available, then the recipient + address would have been rejected in SMTPDelivery.validateTo(). + + The following table summarizes the overall behaviour of the gateway: + + +---------------------------------------------------+----------------+ + | content-type | rcpt pubkey | enforce encr. | action | + +---------------------+-------------+---------------+----------------+ + | multipart/encrypted | any | any | pass | + | other | available | any | encrypt + sign | + | other | unavailable | yes | reject | + | other | unavailable | no | sign | + +---------------------+-------------+---------------+----------------+ + + :param raw: The raw message + :type raw: str + :param recipient: The recipient for the message + :type: recipient: smtp.User + + :return: A Deferred that will be fired with a MIMEMultipart message + and the original recipient Message + :rtype: Deferred + """ + # pass if the original message's content-type is "multipart/encrypted" + lines = raw.split('\r\n') + origmsg = Parser().parsestr(raw) + + if origmsg.get_content_type() == 'multipart/encrypted': + return defer.success((origmsg, recipient)) + + from_address = validate_address(self._from_address) + username, domain = from_address.split('@') + to_address = validate_address(recipient.dest.addrstr) + + # add a nice footer to the outgoing message + # XXX: footer will eventually optional or be removed + if origmsg.get_content_type() == 'text/plain': + lines.append('--') + lines.append('%s - https://%s/key/%s' % + (self.FOOTER_STRING, domain, username)) + lines.append('') + + origmsg = Parser().parsestr('\r\n'.join(lines)) + + def signal_encrypt_sign(newmsg): + signal(proto.SMTP_END_ENCRYPT_AND_SIGN, + "%s,%s" % (self._from_address, to_address)) + return newmsg, recipient + + def signal_sign(newmsg): + signal(proto.SMTP_END_SIGN, self._from_address) + return newmsg, recipient + + def if_key_not_found_send_unencrypted(failure): + if failure.check(KeyNotFound): + log.msg('Will send unencrypted message to %s.' % to_address) + signal(proto.SMTP_START_SIGN, self._from_address) + d = self._sign(origmsg, from_address) + d.addCallback(signal_sign) + return d + else: + return failure + + log.msg("Will encrypt the message with %s and sign with %s." + % (to_address, from_address)) + signal(proto.SMTP_START_ENCRYPT_AND_SIGN, + "%s,%s" % (self._from_address, to_address)) + d = self._encrypt_and_sign(origmsg, to_address, from_address) + d.addCallbacks(signal_encrypt_sign, if_key_not_found_send_unencrypted) + return d + + def _encrypt_and_sign(self, origmsg, encrypt_address, sign_address): + """ + Create an RFC 3156 compliang PGP encrypted and signed message using + C{encrypt_address} to encrypt and C{sign_address} to sign. + + :param origmsg: The original message + :type origmsg: email.message.Message + :param encrypt_address: The address used to encrypt the message. + :type encrypt_address: str + :param sign_address: The address used to sign the message. + :type sign_address: str + + :return: A Deferred with the MultipartEncrypted message + :rtype: Deferred + """ + # create new multipart/encrypted message with 'pgp-encrypted' protocol + + def encrypt(res): + newmsg, origmsg = res + d = self._keymanager.encrypt( + origmsg.as_string(unixfrom=False), + encrypt_address, OpenPGPKey, sign=sign_address) + d.addCallback(lambda encstr: (newmsg, encstr)) + return d + + def create_encrypted_message(res): + newmsg, encstr = res + encmsg = MIMEApplication( + encstr, _subtype='octet-stream', _encoder=lambda x: x) + encmsg.add_header('content-disposition', 'attachment', + filename='msg.asc') + # create meta message + metamsg = PGPEncrypted() + metamsg.add_header('Content-Disposition', 'attachment') + # attach pgp message parts to new message + newmsg.attach(metamsg) + newmsg.attach(encmsg) + return newmsg + + d = self._fix_headers( + origmsg, + MultipartEncrypted('application/pgp-encrypted'), + sign_address) + d.addCallback(encrypt) + d.addCallback(create_encrypted_message) + return d + + def _sign(self, origmsg, sign_address): + """ + Create an RFC 3156 compliant PGP signed MIME message using + C{sign_address}. + + :param origmsg: The original message + :type origmsg: email.message.Message + :param sign_address: The address used to sign the message. + :type sign_address: str + + :return: A Deferred with the MultipartSigned message. + :rtype: Deferred + """ + # apply base64 content-transfer-encoding + encode_base64_rec(origmsg) + # get message text with headers and replace \n for \r\n + fp = StringIO() + g = RFC3156CompliantGenerator( + fp, mangle_from_=False, maxheaderlen=76) + g.flatten(origmsg) + msgtext = re.sub('\r?\n', '\r\n', fp.getvalue()) + # make sure signed message ends with \r\n as per OpenPGP stantard. + if origmsg.is_multipart(): + if not msgtext.endswith("\r\n"): + msgtext += "\r\n" + + def create_signed_message(res): + (msg, _), signature = res + sigmsg = PGPSignature(signature) + # attach original message and signature to new message + msg.attach(origmsg) + msg.attach(sigmsg) + return msg + + dh = self._fix_headers( + origmsg, + MultipartSigned('application/pgp-signature', 'pgp-sha512'), + sign_address) + ds = self._keymanager.sign( + msgtext, sign_address, OpenPGPKey, digest_algo='SHA512', + clearsign=False, detach=True, binary=False) + d = defer.gatherResults([dh, ds]) + d.addCallback(create_signed_message) + return d + + def _fix_headers(self, origmsg, newmsg, sign_address): + """ + Move some headers from C{origmsg} to C{newmsg}, delete unwanted + headers from C{origmsg} and add new headers to C{newms}. + + Outgoing messages are either encrypted and signed or just signed + before being sent. Because of that, they are packed inside new + messages and some manipulation has to be made on their headers. + + Allowed headers for passing through: + + - From + - Date + - To + - Subject + - Reply-To + - References + - In-Reply-To + - Cc + + Headers to be added: + + - Message-ID (i.e. should not use origmsg's Message-Id) + - Received (this is added automatically by twisted smtp API) + - OpenPGP (see #4447) + + Headers to be deleted: + + - User-Agent + + :param origmsg: The original message. + :type origmsg: email.message.Message + :param newmsg: The new message being created. + :type newmsg: email.message.Message + :param sign_address: The address used to sign C{newmsg} + :type sign_address: str + + :return: A Deferred with a touple: + (new Message with the unencrypted headers, + original Message with headers removed) + :rtype: Deferred + """ + # move headers from origmsg to newmsg + headers = origmsg.items() + passthrough = [ + 'from', 'date', 'to', 'subject', 'reply-to', 'references', + 'in-reply-to', 'cc' + ] + headers = filter(lambda x: x[0].lower() in passthrough, headers) + for hkey, hval in headers: + newmsg.add_header(hkey, hval) + del (origmsg[hkey]) + # add a new message-id to newmsg + newmsg.add_header('Message-Id', smtp.messageid()) + # delete user-agent from origmsg + del (origmsg['user-agent']) + + def add_openpgp_header(signkey): + username, domain = sign_address.split('@') + newmsg.add_header( + 'OpenPGP', 'id=%s' % signkey.key_id, + url='https://%s/key/%s' % (domain, username), + preference='signencrypt') + return newmsg, origmsg + + d = self._keymanager.get_key(sign_address, OpenPGPKey, private=True) + d.addCallback(add_openpgp_header) + return d diff --git a/src/leap/mail/outgoing/tests/__init__.py b/src/leap/mail/outgoing/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/mail/outgoing/tests/test_outgoing.py b/src/leap/mail/outgoing/tests/test_outgoing.py new file mode 100644 index 0000000..fa50c30 --- /dev/null +++ b/src/leap/mail/outgoing/tests/test_outgoing.py @@ -0,0 +1,187 @@ +# -*- coding: utf-8 -*- +# test_gateway.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +SMTP gateway tests. +""" + +import re +from datetime import datetime +from twisted.internet.defer import fail +from twisted.mail.smtp import User + +from mock import Mock + +from leap.mail.smtp.gateway import SMTPFactory +from leap.mail.outgoing.service import OutgoingMail +from leap.mail.tests import ( + TestCaseWithKeyManager, + ADDRESS, + ADDRESS_2, +) +from leap.keymanager import openpgp, errors + + +class TestOutgoingMail(TestCaseWithKeyManager): + EMAIL_DATA = ['HELO gateway.leap.se', + 'MAIL FROM: <%s>' % ADDRESS_2, + 'RCPT TO: <%s>' % ADDRESS, + 'DATA', + 'From: User <%s>' % ADDRESS_2, + 'To: Leap <%s>' % ADDRESS, + 'Date: ' + datetime.now().strftime('%c'), + 'Subject: test message', + '', + 'This is a secret message.', + 'Yours,', + 'A.', + '', + '.', + 'QUIT'] + + def setUp(self): + self.lines = [line for line in self.EMAIL_DATA[4:12]] + self.lines.append('') # add a trailing newline + self.raw = '\r\n'.join(self.lines) + self.expected_body = ('\r\n'.join(self.EMAIL_DATA[9:12]) + + "\r\n\r\n--\r\nI prefer encrypted email - " + "https://leap.se/key/anotheruser\r\n") + self.fromAddr = ADDRESS_2 + + def init_outgoing_and_proto(_): + self.outgoing_mail = OutgoingMail( + self.fromAddr, self._km, self._config['cert'], + self._config['key'], self._config['host'], + self._config['port']) + self.proto = SMTPFactory( + u'anotheruser@leap.se', + self._km, + self._config['encrypted_only'], + self.outgoing_mail).buildProtocol(('127.0.0.1', 0)) + self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS) + + d = TestCaseWithKeyManager.setUp(self) + d.addCallback(init_outgoing_and_proto) + return d + + def test_message_encrypt(self): + """ + Test if message gets encrypted to destination email. + """ + def check_decryption(res): + decrypted, _ = res + self.assertEqual( + '\n' + self.expected_body, + decrypted, + 'Decrypted text differs from plaintext.') + + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + d.addCallback(self._assert_encrypted) + d.addCallback(lambda message: self._km.decrypt( + message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey)) + d.addCallback(check_decryption) + return d + + def test_message_encrypt_sign(self): + """ + Test if message gets encrypted to destination email and signed with + sender key. + '""" + def check_decryption_and_verify(res): + decrypted, signkey = res + self.assertEqual( + '\n' + self.expected_body, + decrypted, + 'Decrypted text differs from plaintext.') + self.assertTrue(ADDRESS_2 in signkey.address, + "Verification failed") + + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + d.addCallback(self._assert_encrypted) + d.addCallback(lambda message: self._km.decrypt( + message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey, + verify=ADDRESS_2)) + d.addCallback(check_decryption_and_verify) + return d + + def test_message_sign(self): + """ + Test if message is signed with sender key. + """ + # mock the key fetching + self._km._fetch_keys_from_server = Mock( + return_value=fail(errors.KeyNotFound())) + recipient = User('ihavenopubkey@nonleap.se', + 'gateway.leap.se', self.proto, ADDRESS) + self.outgoing_mail = OutgoingMail( + self.fromAddr, self._km, self._config['cert'], self._config['key'], + self._config['host'], self._config['port']) + + def check_signed(res): + message, _ = res + self.assertTrue('Content-Type' in message) + self.assertEqual('multipart/signed', message.get_content_type()) + self.assertEqual('application/pgp-signature', + message.get_param('protocol')) + self.assertEqual('pgp-sha512', message.get_param('micalg')) + # assert content of message + self.assertEqual(self.expected_body, + message.get_payload(0).get_payload(decode=True)) + # assert content of signature + self.assertTrue( + message.get_payload(1).get_payload().startswith( + '-----BEGIN PGP SIGNATURE-----\n'), + 'Message does not start with signature header.') + self.assertTrue( + message.get_payload(1).get_payload().endswith( + '-----END PGP SIGNATURE-----\n'), + 'Message does not end with signature footer.') + return message + + def verify(message): + # replace EOL before verifying (according to rfc3156) + signed_text = re.sub('\r?\n', '\r\n', + message.get_payload(0).as_string()) + + def assert_verify(key): + self.assertTrue(ADDRESS_2 in key.address, + 'Signature could not be verified.') + + d = self._km.verify( + signed_text, ADDRESS_2, openpgp.OpenPGPKey, + detached_sig=message.get_payload(1).get_payload()) + d.addCallback(assert_verify) + return d + + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, recipient) + d.addCallback(check_signed) + d.addCallback(verify) + return d + + def _assert_encrypted(self, res): + message, _ = res + self.assertTrue('Content-Type' in message) + self.assertEqual('multipart/encrypted', message.get_content_type()) + self.assertEqual('application/pgp-encrypted', + message.get_param('protocol')) + self.assertEqual(2, len(message.get_payload())) + self.assertEqual('application/pgp-encrypted', + message.get_payload(0).get_content_type()) + self.assertEqual('application/octet-stream', + message.get_payload(1).get_content_type()) + return message diff --git a/src/leap/mail/service.py b/src/leap/mail/service.py deleted file mode 100644 index a99f13a..0000000 --- a/src/leap/mail/service.py +++ /dev/null @@ -1,422 +0,0 @@ -# -*- coding: utf-8 -*- -# service.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import re -from StringIO import StringIO -from email.parser import Parser -from email.mime.application import MIMEApplication - -from OpenSSL import SSL - -from twisted.mail import smtp -from twisted.internet import reactor -from twisted.internet import defer -from twisted.protocols.amp import ssl -from twisted.python import log - -from leap.common.check import leap_assert_type, leap_assert -from leap.common.events import proto, signal -from leap.keymanager import KeyManager -from leap.keymanager.openpgp import OpenPGPKey -from leap.keymanager.errors import KeyNotFound -from leap.mail import __version__ -from leap.mail.utils import validate_address -from leap.mail.smtp.rfc3156 import MultipartEncrypted -from leap.mail.smtp.rfc3156 import MultipartSigned -from leap.mail.smtp.rfc3156 import encode_base64_rec -from leap.mail.smtp.rfc3156 import RFC3156CompliantGenerator -from leap.mail.smtp.rfc3156 import PGPSignature -from leap.mail.smtp.rfc3156 import PGPEncrypted - - -class SSLContextFactory(ssl.ClientContextFactory): - def __init__(self, cert, key): - self.cert = cert - self.key = key - - def getContext(self): - self.method = SSL.TLSv1_METHOD # SSLv23_METHOD - ctx = ssl.ClientContextFactory.getContext(self) - ctx.use_certificate_file(self.cert) - ctx.use_privatekey_file(self.key) - return ctx - - -class OutgoingMail: - """ - A service for handling encrypted mail. - """ - - FOOTER_STRING = "I prefer encrypted email" - - def __init__(self, from_address, keymanager, cert, key, host, port): - """ - Initialize the mail service. - - :param from_address: The sender address. - :type from_address: str - :param keymanager: A KeyManager for retrieving recipient's keys. - :type keymanager: leap.common.keymanager.KeyManager - :param cert: The client certificate for SSL authentication. - :type cert: str - :param key: The client private key for SSL authentication. - :type key: str - :param host: The hostname of the remote SMTP server. - :type host: str - :param port: The port of the remote SMTP server. - :type port: int - """ - - # assert params - leap_assert_type(from_address, str) - leap_assert('@' in from_address) - leap_assert_type(keymanager, KeyManager) - leap_assert_type(host, str) - leap_assert(host != '') - leap_assert_type(port, int) - leap_assert(port is not 0) - leap_assert_type(cert, unicode) - leap_assert(cert != '') - leap_assert_type(key, unicode) - leap_assert(key != '') - - self._port = port - self._host = host - self._key = key - self._cert = cert - self._from_address = from_address - self._keymanager = keymanager - - def send_message(self, raw, recipient): - """ - Sends a message to a recipient. Maybe encrypts and signs. - - :param raw: The raw message - :type raw: str - :param recipient: The recipient for the message - :type recipient: smtp.User - :return: a deferred which delivers the message when fired - """ - d = self._maybe_encrypt_and_sign(raw, recipient) - d.addCallback(self._route_msg) - d.addErrback(self.sendError) - return d - - def sendSuccess(self, smtp_sender_result): - """ - Callback for a successful send. - - :param smtp_sender_result: The result from the ESMTPSender from - _route_msg - :type smtp_sender_result: tuple(int, list(tuple)) - """ - dest_addrstr = smtp_sender_result[1][0][0] - log.msg('Message sent to %s' % dest_addrstr) - signal(proto.SMTP_SEND_MESSAGE_SUCCESS, dest_addrstr) - - def sendError(self, failure): - """ - Callback for an unsuccessfull send. - - :param e: The result from the last errback. - :type e: anything - """ - # XXX: need to get the address from the exception to send signal - # signal(proto.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr) - err = failure.value - log.err(err) - raise err - - def _route_msg(self, encrypt_and_sign_result): - """ - Sends the msg using the ESMTPSenderFactory. - - :param encrypt_and_sign_result: A tuple containing the 'maybe' - encrypted message and the recipient - :type encrypt_and_sign_result: tuple - """ - message, recipient = encrypt_and_sign_result - log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port)) - msg = message.as_string(False) - - # we construct a defer to pass to the ESMTPSenderFactory - d = defer.Deferred() - d.addCallbacks(self.sendSuccess, self.sendError) - # we don't pass an ssl context factory to the ESMTPSenderFactory - # because ssl will be handled by reactor.connectSSL() below. - factory = smtp.ESMTPSenderFactory( - "", # username is blank because client auth is done on SSL protocol level - "", # password is blank because client auth is done on SSL protocol level - self._from_address, - recipient.dest.addrstr, - StringIO(msg), - d, - heloFallback=True, - requireAuthentication=False, - requireTransportSecurity=True) - factory.domain = __version__ - signal(proto.SMTP_SEND_MESSAGE_START, recipient.dest.addrstr) - reactor.connectSSL( - self._host, self._port, factory, - contextFactory=SSLContextFactory(self._cert, self._key)) - - def _maybe_encrypt_and_sign(self, raw, recipient): - """ - Attempt to encrypt and sign the outgoing message. - - The behaviour of this method depends on: - - 1. the original message's content-type, and - 2. the availability of the recipient's public key. - - If the original message's content-type is "multipart/encrypted", then - the original message is not altered. For any other content-type, the - method attempts to fetch the recipient's public key. If the - recipient's public key is available, the message is encrypted and - signed; otherwise it is only signed. - - Note that, if the C{encrypted_only} configuration is set to True and - the recipient's public key is not available, then the recipient - address would have been rejected in SMTPDelivery.validateTo(). - - The following table summarizes the overall behaviour of the gateway: - - +---------------------------------------------------+----------------+ - | content-type | rcpt pubkey | enforce encr. | action | - +---------------------+-------------+---------------+----------------+ - | multipart/encrypted | any | any | pass | - | other | available | any | encrypt + sign | - | other | unavailable | yes | reject | - | other | unavailable | no | sign | - +---------------------+-------------+---------------+----------------+ - - :param raw: The raw message - :type raw: str - :param recipient: The recipient for the message - :type: recipient: smtp.User - - :return: A Deferred that will be fired with a MIMEMultipart message - and the original recipient Message - :rtype: Deferred - """ - # pass if the original message's content-type is "multipart/encrypted" - lines = raw.split('\r\n') - origmsg = Parser().parsestr(raw) - - if origmsg.get_content_type() == 'multipart/encrypted': - return defer.success((origmsg, recipient)) - - from_address = validate_address(self._from_address) - username, domain = from_address.split('@') - to_address = validate_address(recipient.dest.addrstr) - - # add a nice footer to the outgoing message - # XXX: footer will eventually optional or be removed - if origmsg.get_content_type() == 'text/plain': - lines.append('--') - lines.append('%s - https://%s/key/%s' % - (self.FOOTER_STRING, domain, username)) - lines.append('') - - origmsg = Parser().parsestr('\r\n'.join(lines)) - - def signal_encrypt_sign(newmsg): - signal(proto.SMTP_END_ENCRYPT_AND_SIGN, - "%s,%s" % (self._from_address, to_address)) - return newmsg, recipient - - def signal_sign(newmsg): - signal(proto.SMTP_END_SIGN, self._from_address) - return newmsg, recipient - - def if_key_not_found_send_unencrypted(failure): - if failure.check(KeyNotFound): - log.msg('Will send unencrypted message to %s.' % to_address) - signal(proto.SMTP_START_SIGN, self._from_address) - d = self._sign(origmsg, from_address) - d.addCallback(signal_sign) - return d - else: - return failure - - log.msg("Will encrypt the message with %s and sign with %s." - % (to_address, from_address)) - signal(proto.SMTP_START_ENCRYPT_AND_SIGN, - "%s,%s" % (self._from_address, to_address)) - d = self._encrypt_and_sign(origmsg, to_address, from_address) - d.addCallbacks(signal_encrypt_sign, if_key_not_found_send_unencrypted) - return d - - def _encrypt_and_sign(self, origmsg, encrypt_address, sign_address): - """ - Create an RFC 3156 compliang PGP encrypted and signed message using - C{encrypt_address} to encrypt and C{sign_address} to sign. - - :param origmsg: The original message - :type origmsg: email.message.Message - :param encrypt_address: The address used to encrypt the message. - :type encrypt_address: str - :param sign_address: The address used to sign the message. - :type sign_address: str - - :return: A Deferred with the MultipartEncrypted message - :rtype: Deferred - """ - # create new multipart/encrypted message with 'pgp-encrypted' protocol - - def encrypt(res): - newmsg, origmsg = res - d = self._keymanager.encrypt( - origmsg.as_string(unixfrom=False), - encrypt_address, OpenPGPKey, sign=sign_address) - d.addCallback(lambda encstr: (newmsg, encstr)) - return d - - def create_encrypted_message(res): - newmsg, encstr = res - encmsg = MIMEApplication( - encstr, _subtype='octet-stream', _encoder=lambda x: x) - encmsg.add_header('content-disposition', 'attachment', - filename='msg.asc') - # create meta message - metamsg = PGPEncrypted() - metamsg.add_header('Content-Disposition', 'attachment') - # attach pgp message parts to new message - newmsg.attach(metamsg) - newmsg.attach(encmsg) - return newmsg - - d = self._fix_headers( - origmsg, - MultipartEncrypted('application/pgp-encrypted'), - sign_address) - d.addCallback(encrypt) - d.addCallback(create_encrypted_message) - return d - - def _sign(self, origmsg, sign_address): - """ - Create an RFC 3156 compliant PGP signed MIME message using - C{sign_address}. - - :param origmsg: The original message - :type origmsg: email.message.Message - :param sign_address: The address used to sign the message. - :type sign_address: str - - :return: A Deferred with the MultipartSigned message. - :rtype: Deferred - """ - # apply base64 content-transfer-encoding - encode_base64_rec(origmsg) - # get message text with headers and replace \n for \r\n - fp = StringIO() - g = RFC3156CompliantGenerator( - fp, mangle_from_=False, maxheaderlen=76) - g.flatten(origmsg) - msgtext = re.sub('\r?\n', '\r\n', fp.getvalue()) - # make sure signed message ends with \r\n as per OpenPGP stantard. - if origmsg.is_multipart(): - if not msgtext.endswith("\r\n"): - msgtext += "\r\n" - - def create_signed_message(res): - (msg, _), signature = res - sigmsg = PGPSignature(signature) - # attach original message and signature to new message - msg.attach(origmsg) - msg.attach(sigmsg) - return msg - - dh = self._fix_headers( - origmsg, - MultipartSigned('application/pgp-signature', 'pgp-sha512'), - sign_address) - ds = self._keymanager.sign( - msgtext, sign_address, OpenPGPKey, digest_algo='SHA512', - clearsign=False, detach=True, binary=False) - d = defer.gatherResults([dh, ds]) - d.addCallback(create_signed_message) - return d - - def _fix_headers(self, origmsg, newmsg, sign_address): - """ - Move some headers from C{origmsg} to C{newmsg}, delete unwanted - headers from C{origmsg} and add new headers to C{newms}. - - Outgoing messages are either encrypted and signed or just signed - before being sent. Because of that, they are packed inside new - messages and some manipulation has to be made on their headers. - - Allowed headers for passing through: - - - From - - Date - - To - - Subject - - Reply-To - - References - - In-Reply-To - - Cc - - Headers to be added: - - - Message-ID (i.e. should not use origmsg's Message-Id) - - Received (this is added automatically by twisted smtp API) - - OpenPGP (see #4447) - - Headers to be deleted: - - - User-Agent - - :param origmsg: The original message. - :type origmsg: email.message.Message - :param newmsg: The new message being created. - :type newmsg: email.message.Message - :param sign_address: The address used to sign C{newmsg} - :type sign_address: str - - :return: A Deferred with a touple: - (new Message with the unencrypted headers, - original Message with headers removed) - :rtype: Deferred - """ - # move headers from origmsg to newmsg - headers = origmsg.items() - passthrough = [ - 'from', 'date', 'to', 'subject', 'reply-to', 'references', - 'in-reply-to', 'cc' - ] - headers = filter(lambda x: x[0].lower() in passthrough, headers) - for hkey, hval in headers: - newmsg.add_header(hkey, hval) - del (origmsg[hkey]) - # add a new message-id to newmsg - newmsg.add_header('Message-Id', smtp.messageid()) - # delete user-agent from origmsg - del (origmsg['user-agent']) - - def add_openpgp_header(signkey): - username, domain = sign_address.split('@') - newmsg.add_header( - 'OpenPGP', 'id=%s' % signkey.key_id, - url='https://%s/key/%s' % (domain, username), - preference='signencrypt') - return newmsg, origmsg - - d = self._keymanager.get_key(sign_address, OpenPGPKey, private=True) - d.addCallback(add_openpgp_header) - return d diff --git a/src/leap/mail/smtp/__init__.py b/src/leap/mail/smtp/__init__.py index 72b26ed..24402b4 100644 --- a/src/leap/mail/smtp/__init__.py +++ b/src/leap/mail/smtp/__init__.py @@ -22,7 +22,7 @@ import logging from twisted.internet import reactor from twisted.internet.error import CannotListenError -from leap.mail.service import OutgoingMail +from leap.mail.outgoing.service import OutgoingMail logger = logging.getLogger(__name__) diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index d58c581..222ef3f 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -93,7 +93,7 @@ class SMTPFactory(ServerFactory): mail or not. :type encrypted_only: bool :param outgoing_mail: The outgoing mail to send the message - :type outgoing_mail: leap.mail.service.OutgoingMail + :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail """ leap_assert_type(encrypted_only, bool) @@ -141,7 +141,7 @@ class SMTPDelivery(object): mail or not. :type encrypted_only: bool :param outgoing_mail: The outgoing mail to send the message - :type outgoing_mail: leap.mail.service.OutgoingMail + :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail """ self._userid = userid self._outgoing_mail = outgoing_mail @@ -266,7 +266,7 @@ class EncryptedMessage(object): :param user: The recipient of this message. :type user: twisted.mail.smtp.User :param outgoing_mail: The outgoing mail to send the message - :type outgoing_mail: leap.mail.service.OutgoingMail + :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail """ # assert params leap_assert_type(user, smtp.User) diff --git a/src/leap/mail/tests/test_service.py b/src/leap/mail/tests/test_service.py deleted file mode 100644 index 43f354d..0000000 --- a/src/leap/mail/tests/test_service.py +++ /dev/null @@ -1,187 +0,0 @@ -# -*- coding: utf-8 -*- -# test_gateway.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -SMTP gateway tests. -""" - -import re -from datetime import datetime -from twisted.internet.defer import fail -from twisted.mail.smtp import User - -from mock import Mock - -from leap.mail.smtp.gateway import SMTPFactory -from leap.mail.service import OutgoingMail -from leap.mail.tests import ( - TestCaseWithKeyManager, - ADDRESS, - ADDRESS_2, -) -from leap.keymanager import openpgp, errors - - -class TestOutgoingMail(TestCaseWithKeyManager): - EMAIL_DATA = ['HELO gateway.leap.se', - 'MAIL FROM: <%s>' % ADDRESS_2, - 'RCPT TO: <%s>' % ADDRESS, - 'DATA', - 'From: User <%s>' % ADDRESS_2, - 'To: Leap <%s>' % ADDRESS, - 'Date: ' + datetime.now().strftime('%c'), - 'Subject: test message', - '', - 'This is a secret message.', - 'Yours,', - 'A.', - '', - '.', - 'QUIT'] - - def setUp(self): - self.lines = [line for line in self.EMAIL_DATA[4:12]] - self.lines.append('') # add a trailing newline - self.raw = '\r\n'.join(self.lines) - self.expected_body = ('\r\n'.join(self.EMAIL_DATA[9:12]) + - "\r\n\r\n--\r\nI prefer encrypted email - " - "https://leap.se/key/anotheruser\r\n") - self.fromAddr = ADDRESS_2 - - def init_outgoing_and_proto(_): - self.outgoing_mail = OutgoingMail( - self.fromAddr, self._km, self._config['cert'], - self._config['key'], self._config['host'], - self._config['port']) - self.proto = SMTPFactory( - u'anotheruser@leap.se', - self._km, - self._config['encrypted_only'], - self.outgoing_mail).buildProtocol(('127.0.0.1', 0)) - self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS) - - d = TestCaseWithKeyManager.setUp(self) - d.addCallback(init_outgoing_and_proto) - return d - - def test_message_encrypt(self): - """ - Test if message gets encrypted to destination email. - """ - def check_decryption(res): - decrypted, _ = res - self.assertEqual( - '\n' + self.expected_body, - decrypted, - 'Decrypted text differs from plaintext.') - - d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) - d.addCallback(self._assert_encrypted) - d.addCallback(lambda message: self._km.decrypt( - message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey)) - d.addCallback(check_decryption) - return d - - def test_message_encrypt_sign(self): - """ - Test if message gets encrypted to destination email and signed with - sender key. - '""" - def check_decryption_and_verify(res): - decrypted, signkey = res - self.assertEqual( - '\n' + self.expected_body, - decrypted, - 'Decrypted text differs from plaintext.') - self.assertTrue(ADDRESS_2 in signkey.address, - "Verification failed") - - d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) - d.addCallback(self._assert_encrypted) - d.addCallback(lambda message: self._km.decrypt( - message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey, - verify=ADDRESS_2)) - d.addCallback(check_decryption_and_verify) - return d - - def test_message_sign(self): - """ - Test if message is signed with sender key. - """ - # mock the key fetching - self._km._fetch_keys_from_server = Mock( - return_value=fail(errors.KeyNotFound())) - recipient = User('ihavenopubkey@nonleap.se', - 'gateway.leap.se', self.proto, ADDRESS) - self.outgoing_mail = OutgoingMail( - self.fromAddr, self._km, self._config['cert'], self._config['key'], - self._config['host'], self._config['port']) - - def check_signed(res): - message, _ = res - self.assertTrue('Content-Type' in message) - self.assertEqual('multipart/signed', message.get_content_type()) - self.assertEqual('application/pgp-signature', - message.get_param('protocol')) - self.assertEqual('pgp-sha512', message.get_param('micalg')) - # assert content of message - self.assertEqual(self.expected_body, - message.get_payload(0).get_payload(decode=True)) - # assert content of signature - self.assertTrue( - message.get_payload(1).get_payload().startswith( - '-----BEGIN PGP SIGNATURE-----\n'), - 'Message does not start with signature header.') - self.assertTrue( - message.get_payload(1).get_payload().endswith( - '-----END PGP SIGNATURE-----\n'), - 'Message does not end with signature footer.') - return message - - def verify(message): - # replace EOL before verifying (according to rfc3156) - signed_text = re.sub('\r?\n', '\r\n', - message.get_payload(0).as_string()) - - def assert_verify(key): - self.assertTrue(ADDRESS_2 in key.address, - 'Signature could not be verified.') - - d = self._km.verify( - signed_text, ADDRESS_2, openpgp.OpenPGPKey, - detached_sig=message.get_payload(1).get_payload()) - d.addCallback(assert_verify) - return d - - d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, recipient) - d.addCallback(check_signed) - d.addCallback(verify) - return d - - def _assert_encrypted(self, res): - message, _ = res - self.assertTrue('Content-Type' in message) - self.assertEqual('multipart/encrypted', message.get_content_type()) - self.assertEqual('application/pgp-encrypted', - message.get_param('protocol')) - self.assertEqual(2, len(message.get_payload())) - self.assertEqual('application/pgp-encrypted', - message.get_payload(0).get_content_type()) - self.assertEqual('application/octet-stream', - message.get_payload(1).get_content_type()) - return message diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index f747377..5172837 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -85,6 +85,35 @@ get_raw_docs = lambda msg, parts: ( for payload, headers in get_payloads(msg) if not isinstance(payload, list)) +""" +Groucho Marx: Now pay particular attention to this first clause, because it's + most important. There's the party of the first part shall be + known in this contract as the party of the first part. How do you + like that, that's pretty neat eh? + +Chico Marx: No, that's no good. +Groucho Marx: What's the matter with it? + +Chico Marx: I don't know, let's hear it again. +Groucho Marx: So the party of the first part shall be known in this contract as + the party of the first part. + +Chico Marx: Well it sounds a little better this time. +Groucho Marx: Well, it grows on you. Would you like to hear it once more? + +Chico Marx: Just the first part. +Groucho Marx: All right. It says the first part of the party of the first part + shall be known in this contract as the first part of the party of + the first part, shall be known in this contract - look, why + should we quarrel about a thing like this, we'll take it right + out, eh? + +Chico Marx: Yes, it's too long anyhow. Now what have we got left? +Groucho Marx: Well I've got about a foot and a half. Now what's the matter? + +Chico Marx: I don't like the second party either. +""" + def walk_msg_tree(parts, body_phash=None): """ @@ -162,7 +191,7 @@ def walk_msg_tree(parts, body_phash=None): outer = parts[0] outer.pop(HEADERS) - if not PART_MAP in outer: + if PART_MAP not in outer: # we have a multipart with 1 part only, so kind of fix it # although it would be prettier if I take this special case at # the beginning of the walk. @@ -177,36 +206,3 @@ def walk_msg_tree(parts, body_phash=None): pdoc = outer pdoc[BODY] = body_phash return pdoc - -""" -Groucho Marx: Now pay particular attention to this first clause, because it's - most important. There's the party of the first part shall be - known in this contract as the party of the first part. How do you - like that, that's pretty neat eh? - -Chico Marx: No, that's no good. -Groucho Marx: What's the matter with it? - -Chico Marx: I don't know, let's hear it again. -Groucho Marx: So the party of the first part shall be known in this contract as - the party of the first part. - -Chico Marx: Well it sounds a little better this time. -Groucho Marx: Well, it grows on you. Would you like to hear it once more? - -Chico Marx: Just the first part. -Groucho Marx: All right. It says the first part of the party of the first part - shall be known in this contract as the first part of the party of - the first part, shall be known in this contract - look, why - should we quarrel about a thing like this, we'll take it right - out, eh? - -Chico Marx: Yes, it's too long anyhow. Now what have we got left? -Groucho Marx: Well I've got about a foot and a half. Now what's the matter? - -Chico Marx: I don't like the second party either. -""" - -""" -I feel you deserved it after reading the above and try to debug your problem ;) -""" -- cgit v1.2.3 From fa60b76ce9cdf6684945c6bc724f10818104166b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 1 Jan 2015 18:21:44 -0400 Subject: cleanup imap implementation --- src/leap/mail/adaptors/soledad.py | 13 +- src/leap/mail/constants.py | 14 + src/leap/mail/imap/account.py | 306 +++----- src/leap/mail/imap/fields.py | 51 -- src/leap/mail/imap/interfaces.py | 96 --- src/leap/mail/imap/mailbox.py | 472 ++++--------- src/leap/mail/imap/memorystore.py | 1340 ------------------------------------ src/leap/mail/imap/messageparts.py | 586 ---------------- src/leap/mail/imap/messages.py | 1007 +++++---------------------- src/leap/mail/imap/soledadstore.py | 617 ----------------- src/leap/mail/mail.py | 93 ++- src/leap/mail/messageflow.py | 200 ------ 12 files changed, 522 insertions(+), 4273 deletions(-) delete mode 100644 src/leap/mail/imap/fields.py delete mode 100644 src/leap/mail/imap/interfaces.py delete mode 100644 src/leap/mail/imap/memorystore.py delete mode 100644 src/leap/mail/imap/messageparts.py delete mode 100644 src/leap/mail/imap/soledadstore.py delete mode 100644 src/leap/mail/messageflow.py diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 0b97869..bf8f7e9 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -513,9 +513,13 @@ class MailboxWrapper(SoledadDocumentWrapper): type_ = "mbox" mbox = INBOX_NAME flags = [] + recent = [] + created = 1 closed = False subscribed = False - rw = True + + # I think we don't need to store this one. + # rw = True class __meta__(object): index = "mbox" @@ -655,6 +659,7 @@ class SoledadMailAdaptor(SoledadIndexMixin): assert(MessageClass is not None) return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs)) + # XXX pass UID too? def _get_msg_from_variable_doc_list(self, doc_list, msg_class): if len(doc_list) == 2: fdoc, hdoc = doc_list @@ -664,12 +669,14 @@ class SoledadMailAdaptor(SoledadIndexMixin): cdocs = dict(enumerate(doc_list[2:], 1)) return self.get_msg_from_docs(msg_class, fdoc, hdoc, cdocs) + # XXX pass UID too ? def get_msg_from_mdoc_id(self, MessageClass, store, doc_id, get_cdocs=False): metamsg_id = doc_id def wrap_meta_doc(doc): cls = MetaMsgDocWrapper + # XXX pass UID? return cls(doc_id=doc.doc_id, **doc.content) def get_part_docs_from_mdoc_wrapper(wrapper): @@ -692,8 +699,8 @@ class SoledadMailAdaptor(SoledadIndexMixin): return constants.FDOCID.format(mbox=mbox, chash=chash) d_docs = [] - fdoc_id = _get_fdoc_id_from_mdoc_id(doc_id) - hdoc_id = _get_hdoc_id_from_mdoc_id(doc_id) + fdoc_id = _get_fdoc_id_from_mdoc_id() + hdoc_id = _get_hdoc_id_from_mdoc_id() d_docs.append(store.get_doc(fdoc_id)) d_docs.append(store.get_doc(hdoc_id)) d = defer.gatherResults(d_docs) diff --git a/src/leap/mail/constants.py b/src/leap/mail/constants.py index bf1db7f..d76e652 100644 --- a/src/leap/mail/constants.py +++ b/src/leap/mail/constants.py @@ -36,3 +36,17 @@ HDOCID_RE = "H\-[0-9a-fA-F]+" CDOCID = "C-{phash}" CDOCID_RE = "C\-[0-9a-fA-F]+" + + +class MessageFlags(object): + """ + Flags used in Message and Mailbox. + """ + SEEN_FLAG = "\\Seen" + RECENT_FLAG = "\\Recent" + ANSWERED_FLAG = "\\Answered" + FLAGGED_FLAG = "\\Flagged" # yo dawg + DELETED_FLAG = "\\Deleted" + DRAFT_FLAG = "\\Draft" + NOSELECT_FLAG = "\\Noselect" + LIST_FLAG = "List" # is this OK? (no \. ie, no system flag) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 7dfbbd1..0baf078 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # account.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,12 +15,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -Soledad Backed Account. +Soledad Backed IMAP Account. """ -import copy import logging import os import time +from functools import partial from twisted.internet import defer from twisted.mail import imap4 @@ -29,9 +29,9 @@ from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type +from leap.mail.constants import MessageFlags from leap.mail.mail import Account -from leap.mail.imap.fields import WithMsgFields -from leap.mail.imap.mailbox import SoledadMailbox, normalize_mailbox +from leap.mail.imap.mailbox import IMAPMailbox, normalize_mailbox from leap.soledad.client import Soledad logger = logging.getLogger(__name__) @@ -49,9 +49,10 @@ if PROFILE_CMD: # Soledad IMAP Account ####################################### -# TODO remove MsgFields too +# XXX watchout, account needs to be ready... so we should maybe return +# a deferred to the IMAP service when it's initialized -class IMAPAccount(WithMsgFields): +class IMAPAccount(object): """ An implementation of an imap4 Account that is backed by Soledad Encrypted Documents. @@ -72,37 +73,20 @@ class IMAPAccount(WithMsgFields): :param store: a Soledad instance. :type store: Soledad """ - # XXX assert a generic store interface instead, so that we - # can plug the memory store wrapper seamlessly. leap_assert(store, "Need a store instance to initialize") leap_assert_type(store, Soledad) - # XXX SHOULD assert too that the name matches the user/uuid with which + # TODO assert too that the name matches the user/uuid with which # soledad has been initialized. self.user_id = user_id self.account = Account(store) - # XXX should hide this in the adaptor... - def _get_mailbox_by_name(self, name): - """ - Return an mbox document by name. - - :param name: the name of the mailbox - :type name: str - - :rtype: SoledadDocument - """ - def get_first_if_any(docs): - return docs[0] if docs else None - - d = self._store.get_from_index( - self.TYPE_MBOX_IDX, self.MBOX_KEY, - normalize_mailbox(name)) - d.addCallback(get_first_if_any) - return d + def _return_mailbox_from_collection(self, collection, readwrite=1): + if collection is None: + return None + return IMAPMailbox(collection, rw=readwrite) - # XXX move to Account? - # XXX needed? + # XXX Where's this used from? -- self.delete... def getMailbox(self, name): """ Return a Mailbox with that name, without selecting it. @@ -110,31 +94,25 @@ class IMAPAccount(WithMsgFields): :param name: name of the mailbox :type name: str - :returns: a a SoledadMailbox instance - :rtype: SoledadMailbox + :returns: an IMAPMailbox instance + :rtype: IMAPMailbox """ name = normalize_mailbox(name) - if name not in self.account.mailboxes: - raise imap4.MailboxException("No such mailbox: %r" % name) + def check_it_exists(mailboxes): + if name not in mailboxes: + raise imap4.MailboxException("No such mailbox: %r" % name) - # XXX Does mailbox really need reference to soledad? - return SoledadMailbox(name, self._store) + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_exists) + d.addCallback(lambda _: self.account.get_collection_by_mailbox, name) + d.addCallbacK(self._return_mailbox_from_collection) + return d # # IAccount # - def _get_empty_mailbox(self): - """ - Returns an empty mailbox. - - :rtype: dict - """ - # XXX move to mailbox module - return copy.deepcopy(mailbox.EMPTY_MBOX) - - # TODO use mail.Account.add_mailbox def addMailbox(self, name, creation_ts=None): """ Add a mailbox to the account. @@ -154,8 +132,9 @@ class IMAPAccount(WithMsgFields): leap_assert(name, "Need a mailbox name to create a mailbox") - if name in self.mailboxes: - raise imap4.MailboxCollision(repr(name)) + def check_it_does_not_exist(mailboxes): + if name in mailboxes: + raise imap4.MailboxCollision(repr(name)) if creation_ts is None: # by default, we pass an int value @@ -164,21 +143,18 @@ class IMAPAccount(WithMsgFields): # mailbox-uidvalidity. creation_ts = int(time.time() * 10E2) - mbox = self._get_empty_mailbox() - mbox[self.MBOX_KEY] = name - mbox[self.CREATED_KEY] = creation_ts - - def load_mbox_cache(result): - d = self._load_mailboxes() - d.addCallback(lambda _: result) + def set_mbox_creation_ts(collection): + d = collection.set_mbox_attr("created") + d.addCallback(lambda _: collection) return d - d = self._store.create_doc(mbox) - d.addCallback(load_mbox_cache) + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_does_not_exist) + d.addCallback(lambda _: self.account.get_collection_by_mailbox, name) + d.addCallback(set_mbox_creation_ts) + d.addCallback(self._return_mailbox_from_collection) return d - # TODO use mail.Account.create_mailbox? - # Watch out, imap specific exceptions raised here. def create(self, pathspec): """ Create a new mailbox from the given hierarchical name. @@ -204,9 +180,10 @@ class IMAPAccount(WithMsgFields): for accum in range(1, len(paths)): try: - partial = sep.join(paths[:accum]) - d = self.addMailbox(partial) + partial_path = sep.join(paths[:accum]) + d = self.addMailbox(partial_path) subs.append(d) + # XXX should this be handled by the deferred? except imap4.MailboxCollision: pass try: @@ -222,21 +199,13 @@ class IMAPAccount(WithMsgFields): def all_good(result): return all(result) - def load_mbox_cache(result): - d = self._load_mailboxes() - d.addCallback(lambda _: result) - return d - if subs: d1 = defer.gatherResults(subs, consumeErrors=True) - d1.addCallback(load_mbox_cache) d1.addCallback(all_good) else: d1 = defer.succeed(False) - d1.addCallback(load_mbox_cache) return d1 - # TODO use mail.Account.get_collection_by_mailbox def select(self, name, readwrite=1): """ Selects a mailbox. @@ -250,15 +219,28 @@ class IMAPAccount(WithMsgFields): :rtype: SoledadMailbox """ name = normalize_mailbox(name) - if name not in self.mailboxes: - logger.warning("No such mailbox!") - return None - self.selected = name - sm = SoledadMailbox(name, self._store, readwrite) - return sm + def check_it_exists(mailboxes): + if name not in mailboxes: + logger.warning("SELECT: No such mailbox!") + return None + return name + + def set_selected(_): + self.selected = name + + def get_collection(name): + if name is None: + return None + return self.account.get_collection_by_mailbox(name) + + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_exists) + d.addCallback(get_collection) + d.addCallback(partial( + self._return_mailbox_from_collection, readwrite=readwrite)) + return d - # TODO use mail.Account.delete_mailbox def delete(self, name, force=False): """ Deletes a mailbox. @@ -276,37 +258,52 @@ class IMAPAccount(WithMsgFields): :rtype: Deferred """ name = normalize_mailbox(name) + _mboxes = [] - if name not in self.mailboxes: - err = imap4.MailboxException("No such mailbox: %r" % name) - return defer.fail(err) - mbox = self.getMailbox(name) + def check_it_exists(mailboxes): + # FIXME works? -- pass variable ref to outer scope + _mboxes = mailboxes + if name not in mailboxes: + err = imap4.MailboxException("No such mailbox: %r" % name) + return defer.fail(err) - if not force: + def get_mailbox(_): + return self.getMailbox(name) + + def destroy_mailbox(mbox): + return mbox.destroy() + + def check_can_be_deleted(mbox): # See if this box is flagged \Noselect - # XXX use mbox.flags instead? mbox_flags = mbox.getFlags() - if self.NOSELECT_FLAG in mbox_flags: + if MessageFlags.NOSELECT_FLAG in mbox_flags: # Check for hierarchically inferior mailboxes with this one # as part of their root. - for others in self.mailboxes: + for others in _mboxes: if others != name and others.startswith(name): err = imap4.MailboxException( "Hierarchically inferior mailboxes " "exist and \\Noselect is set") return defer.fail(err) - self.__mailboxes.discard(name) - return mbox.destroy() + return mbox - # XXX FIXME --- not honoring the inferior names... + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_exists) + d.addCallback(get_mailbox) + if not force: + d.addCallback(check_can_be_deleted) + d.addCallback(destroy_mailbox) + return d + # FIXME --- not honoring the inferior names... # if there are no hierarchically inferior names, we will # delete it from our ken. + # XXX is this right? # if self._inferiorNames(name) > 1: - # ??! -- can this be rite? - # self._index.removeMailbox(name) + # self._index.removeMailbox(name) # TODO use mail.Account.rename_mailbox + # TODO finish conversion to deferreds def rename(self, oldname, newname): """ Renames a mailbox. @@ -320,6 +317,9 @@ class IMAPAccount(WithMsgFields): oldname = normalize_mailbox(oldname) newname = normalize_mailbox(newname) + # FIXME check that scope works (test) + _mboxes = [] + if oldname not in self.mailboxes: raise imap4.NoSuchMailbox(repr(oldname)) @@ -327,34 +327,19 @@ class IMAPAccount(WithMsgFields): inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors] for (old, new) in inferiors: - if new in self.mailboxes: + if new in _mboxes: raise imap4.MailboxCollision(repr(new)) rename_deferreds = [] - def load_mbox_cache(result): - d = self._load_mailboxes() - d.addCallback(lambda _: result) - return d - - def update_mbox_doc_name(mbox, oldname, newname, update_deferred): - mbox.content[self.MBOX_KEY] = newname - d = self._soledad.put_doc(mbox) - d.addCallback(lambda r: update_deferred.callback(True)) - for (old, new) in inferiors: - self.__mailboxes.discard(old) - self._memstore.rename_fdocs_mailbox(old, new) - - d0 = defer.Deferred() - d = self._get_mailbox_by_name(old) - d.addCallback(update_mbox_doc_name, old, new, d0) - rename_deferreds.append(d0) + d = self.account.rename_mailbox(old, new) + rename_deferreds.append(d) d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) - d1.addCallback(load_mbox_cache) return d1 + # FIXME use deferreds (list_all_mailbox_names, etc) def _inferiorNames(self, name): """ Return hierarchically inferior mailboxes. @@ -387,16 +372,15 @@ class IMAPAccount(WithMsgFields): :type wildcard: str """ # XXX use wildcard in index query - ref = self._inferiorNames(normalize_mailbox(ref)) + # TODO get deferreds wildcard = imap4.wildcardToRegexp(wildcard, '/') + ref = self._inferiorNames(normalize_mailbox(ref)) return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] # # The rest of the methods are specific for leap.mail.imap.account.Account # - # TODO ------------------ can we preserve the attr? - # maybe add to memory store. def isSubscribed(self, name): """ Returns True if user is subscribed to this mailbox. @@ -406,63 +390,13 @@ class IMAPAccount(WithMsgFields): :rtype: Deferred (will fire with bool) """ - # TODO use Flags class - subscribed = self.SUBSCRIBED_KEY - - def is_subscribed(mbox): - subs_bool = bool(mbox.content.get(subscribed, False)) - return subs_bool - - d = self._get_mailbox_by_name(name) - d.addCallback(is_subscribed) - return d - - # TODO ------------------ can we preserve the property? - # maybe add to memory store. - - def _get_subscriptions(self): - """ - Return a list of the current subscriptions for this account. - - :returns: A deferred that will fire with the subscriptions. - :rtype: Deferred - """ - def get_docs_content(docs): - return [doc.content[self.MBOX_KEY] for doc in docs] - - d = self._store.get_from_index( - self.TYPE_SUBS_IDX, self.MBOX_KEY, '1') - d.addCallback(get_docs_content) - return d - - def _set_subscription(self, name, value): - """ - Sets the subscription value for a given mailbox - - :param name: the mailbox - :type name: str - - :param value: the boolean value - :type value: bool - """ - # XXX Note that this kind of operation has - # no guarantees of atomicity. We should not be accessing mbox - # documents concurrently. - - subscribed = self.SUBSCRIBED_KEY + name = normalize_mailbox(name) - def update_subscribed_value(mbox): - mbox.content[subscribed] = value - return self._store.put_doc(mbox) + def get_subscribed(mbox): + return mbox.get_mbox_attr("subscribed") - # maybe we should store subscriptions in another - # document... - if name not in self.mailboxes: - d = self.addMailbox(name) - d.addCallback(lambda v: self._get_mailbox_by_name(name)) - else: - d = self._get_mailbox_by_name(name) - d.addCallback(update_subscribed_value) + d = self.getMailbox(name) + d.addCallback(get_subscribed) return d def subscribe(self, name): @@ -475,11 +409,11 @@ class IMAPAccount(WithMsgFields): """ name = normalize_mailbox(name) - def check_and_subscribe(subscriptions): - if name not in subscriptions: - return self._set_subscription(name, True) - d = self._get_subscriptions() - d.addCallback(check_and_subscribe) + def set_subscribed(mbox): + return mbox.set_mbox_attr("subscribed", True) + + d = self.getMailbox(name) + d.addCallback(set_subscribed) return d def unsubscribe(self, name): @@ -492,17 +426,17 @@ class IMAPAccount(WithMsgFields): """ name = normalize_mailbox(name) - def check_and_unsubscribe(subscriptions): - if name not in subscriptions: - raise imap4.MailboxException( - "Not currently subscribed to %r" % name) - return self._set_subscription(name, False) - d = self._get_subscriptions() - d.addCallback(check_and_unsubscribe) + def set_unsubscribed(mbox): + return mbox.set_mbox_attr("subscribed", False) + + d = self.getMailbox(name) + d.addCallback(set_unsubscribed) return d + # TODO -- get__all_mboxes, return tuple + # with ... name? and subscribed bool... def getSubscriptions(self): - return self._get_subscriptions() + raise NotImplementedError() # # INamespacePresenter @@ -517,20 +451,6 @@ class IMAPAccount(WithMsgFields): def getOtherNamespaces(self): return None - # extra, for convenience - - def deleteAllMessages(self, iknowhatiamdoing=False): - """ - Deletes all messages from all mailboxes. - Danger! high voltage! - - :param iknowhatiamdoing: confirmation parameter, needs to be True - to proceed. - """ - if iknowhatiamdoing is True: - for mbox in self.mailboxes: - self.delete(mbox, force=True) - def __repr__(self): """ Representation string for this object. diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py deleted file mode 100644 index a751c6d..0000000 --- a/src/leap/mail/imap/fields.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- -# fields.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Fields for Mailbox and Message. -""" - -# TODO deprecate !!! (move all to constants maybe?) -# Flags -> foo - - -class WithMsgFields(object): - """ - Container class for class-attributes to be shared by - several message-related classes. - """ - # Mailbox specific keys - CREATED_KEY = "created" # used??? - - RECENTFLAGS_KEY = "rct" - HDOCS_SET_KEY = "hdocset" - - # Flags in Mailbox and Message - SEEN_FLAG = "\\Seen" - RECENT_FLAG = "\\Recent" - ANSWERED_FLAG = "\\Answered" - FLAGGED_FLAG = "\\Flagged" # yo dawg - DELETED_FLAG = "\\Deleted" - DRAFT_FLAG = "\\Draft" - NOSELECT_FLAG = "\\Noselect" - LIST_FLAG = "List" # is this OK? (no \. ie, no system flag) - - # Fields in mail object - SUBJECT_FIELD = "Subject" - DATE_FIELD = "Date" - - -fields = WithMsgFields # alias for convenience diff --git a/src/leap/mail/imap/interfaces.py b/src/leap/mail/imap/interfaces.py deleted file mode 100644 index f8f25fa..0000000 --- a/src/leap/mail/imap/interfaces.py +++ /dev/null @@ -1,96 +0,0 @@ -# -*- coding: utf-8 -*- -# interfaces.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Interfaces for the IMAP module. -""" -from zope.interface import Interface, Attribute - - -# TODO remove ---------------- -class IMessageContainer(Interface): - """ - I am a container around the different documents that a message - is split into. - """ - fdoc = Attribute('The flags document for this message, if any.') - hdoc = Attribute('The headers document for this message, if any.') - cdocs = Attribute('The dict of content documents for this message, ' - 'if any.') - - def walk(self): - """ - Return an iterator to the docs for all the parts. - - :rtype: iterator - """ - - -# TODO remove -------------------- -class IMessageStore(Interface): - """ - I represent a generic storage for LEAP Messages. - """ - - def create_message(self, mbox, uid, message): - """ - Put the passed message into this IMessageStore. - - :param mbox: the mbox this message belongs. - :param uid: the UID that identifies this message in this mailbox. - :param message: a IMessageContainer implementor. - """ - - def put_message(self, mbox, uid, message): - """ - Put the passed message into this IMessageStore. - - :param mbox: the mbox this message belongs. - :param uid: the UID that identifies this message in this mailbox. - :param message: a IMessageContainer implementor. - """ - - def remove_message(self, mbox, uid): - """ - Remove the given message from this IMessageStore. - - :param mbox: the mbox this message belongs. - :param uid: the UID that identifies this message in this mailbox. - """ - - def get_message(self, mbox, uid): - """ - Get a IMessageContainer for the given mbox and uid combination. - - :param mbox: the mbox this message belongs. - :param uid: the UID that identifies this message in this mailbox. - :return: IMessageContainer - """ - - -class IMessageStoreWriter(Interface): - """ - I represent a storage that is able to write its contents to another - different IMessageStore. - """ - - def write_messages(self, store): - """ - Write the documents in this IMessageStore to a different - storage. Usually this will be done from a MemoryStorage to a DbStorage. - - :param store: another IMessageStore implementor. - """ diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index ea54d33..faeba9d 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -1,6 +1,6 @@ # *- coding: utf-8 -*- # mailbox.py -# Copyright (C) 2013, 2014 LEAP +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,11 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -Soledad Mailbox. +IMAP Mailbox. """ -import copy import re -import threading import logging import StringIO import cStringIO @@ -29,7 +27,6 @@ from collections import defaultdict from twisted.internet import defer from twisted.internet import reactor -from twisted.internet.task import deferLater from twisted.python import log from twisted.mail import imap4 @@ -38,17 +35,15 @@ from zope.interface import implements from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type -from leap.mail.constants import INBOX_NAME -from leap.mail.decorators import deferred_to_thread -from leap.mail.utils import empty -from leap.mail.imap.fields import WithMsgFields, fields -from leap.mail.imap.messages import MessageCollection -from leap.mail.imap.messageparts import MessageWrapper +from leap.mail.constants import INBOX_NAME, MessageFlags logger = logging.getLogger(__name__) -# TODO +# TODO LIST # [ ] Restore profile_cmd instrumentation +# [ ] finish the implementation of IMailboxListener +# [ ] implement the rest of ISearchableMailbox + """ If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid @@ -75,16 +70,20 @@ if PROFILE_CMD: d.addCallback(_debugProfiling, name, time.time()) d.addErrback(lambda f: log.msg(f.getTraceback())) +INIT_FLAGS = (MessageFlags.SEEN_FLAG, MessageFlags.ANSWERED_FLAG, + MessageFlags.FLAGGED_FLAG, MessageFlags.DELETED_FLAG, + MessageFlags.DRAFT_FLAG, MessageFlags.RECENT_FLAG, + MessageFlags.LIST_FLAG) + -# TODO Rename to Mailbox -# TODO Remove WithMsgFields -class SoledadMailbox(WithMsgFields): +class IMAPMailbox(object): """ A Soledad-backed IMAP mailbox. Implements the high-level method needed for the Mailbox interfaces. - The low-level database methods are contained in MessageCollection class, - which we instantiate and make accessible in the `messages` attribute. + The low-level database methods are contained in IMAPMessageCollection + class, which we instantiate and make accessible in the `messages` + attribute. """ implements( imap4.IMailbox, @@ -93,17 +92,7 @@ class SoledadMailbox(WithMsgFields): imap4.ISearchableMailbox, imap4.IMessageCopier) - # XXX should finish the implementation of IMailboxListener - # XXX should completely implement ISearchableMailbox too - - messages = None - _closed = False - - INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG, - WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG, - WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG, - WithMsgFields.LIST_FLAG) - flags = None + init_flags = INIT_FLAGS CMD_MSG = "MESSAGES" CMD_RECENT = "RECENT" @@ -111,58 +100,31 @@ class SoledadMailbox(WithMsgFields): CMD_UIDVALIDITY = "UIDVALIDITY" CMD_UNSEEN = "UNSEEN" - # FIXME we should turn this into a datastructure with limited capacity + # TODO we should turn this into a datastructure with limited capacity _listeners = defaultdict(set) - next_uid_lock = threading.Lock() - last_uid_lock = threading.Lock() - - # TODO unify all the `primed` dicts - _fdoc_primed = {} - _last_uid_primed = {} - _known_uids_primed = {} - - # TODO pass the collection to the constructor - # TODO pass the mbox_doc too - def __init__(self, mbox, store, rw=1): + def __init__(self, collection, rw=1): """ SoledadMailbox constructor. Needs to get passed a name, plus a Soledad instance. - :param mbox: the mailbox name - :type mbox: str - - :param store: - :type store: Soledad + :param collection: instance of IMAPMessageCollection + :type collection: IMAPMessageCollection :param rw: read-and-write flag for this mailbox :type rw: int """ - leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(store, "Need a store instance to initialize") - - self.mbox = normalize_mailbox(mbox) self.rw = rw - self.store = store - - self.messages = MessageCollection(mbox=mbox, soledad=store) self._uidvalidity = None + self.collection = collection - # XXX careful with this get/set (it would be - # hitting db unconditionally, move to memstore too) - # Now it's returning a fixed amount of flags from mem - # as a workaround. if not self.getFlags(): - self.setFlags(self.INIT_FLAGS) - - if self._memstore: - self.prime_known_uids_to_memstore() - self.prime_last_uid_to_memstore() - self.prime_flag_docs_to_memstore() + self.setFlags(self.init_flags) - # purge memstore from empty fdocs. - self._memstore.purge_fdoc_store(mbox) + @property + def mbox_name(self): + return self.collection.mbox_name @property def listeners(self): @@ -175,11 +137,12 @@ class SoledadMailbox(WithMsgFields): :rtype: set """ - return self._listeners[self.mbox] + return self._listeners[self.mbox_name] - # TODO this grows too crazily when many instances are fired, like + # FIXME this grows too crazily when many instances are fired, like # during imaptest stress testing. Should have a queue of limited size # instead. + def addListener(self, listener): """ Add a listener to the listeners queue. @@ -204,16 +167,6 @@ class SoledadMailbox(WithMsgFields): """ self.listeners.remove(listener) - def _get_mbox_doc(self): - """ - Return mailbox document. - - :return: A SoledadDocument containing this mailbox, or None if - the query failed. - :rtype: SoledadDocument or None. - """ - return self._memstore.get_mbox_doc(self.mbox) - def getFlags(self): """ Returns the flags defined for this mailbox. @@ -221,10 +174,11 @@ class SoledadMailbox(WithMsgFields): :returns: tuple of flags for this mailbox :rtype: tuple of str """ - flags = self._memstore.get_mbox_flags(self.mbox) + flags = self.collection.mbox_wrapper.flags if not flags: - flags = self.INIT_FLAGS - return map(str, flags) + flags = self.init_flags + flags_str = map(str, flags) + return flags_str def setFlags(self, flags): """ @@ -234,98 +188,31 @@ class SoledadMailbox(WithMsgFields): :type flags: tuple of str """ # XXX this is setting (overriding) old flags. + # Better pass a mode flag leap_assert(isinstance(flags, tuple), "flags expected to be a tuple") - self._memstore.set_mbox_flags(self.mbox, flags) - - # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG. + return self.collection.set_mbox_attr("flags", flags) - def _get_closed(self): + @property + def is_closed(self): """ Return the closed attribute for this mailbox. :return: True if the mailbox is closed :rtype: bool """ - return self._memstore.get_mbox_closed(self.mbox) + return self.collection.get_mbox_attr("closed") - def _set_closed(self, closed): + def set_closed(self, closed): """ Set the closed attribute for this mailbox. :param closed: the state to be set :type closed: bool - """ - self._memstore.set_mbox_closed(self.mbox, closed) - - closed = property( - _get_closed, _set_closed, doc="Closed attribute.") - - def _get_last_uid(self): - """ - Return the last uid for this mailbox. - If we have a memory store, the last UID will be the highest - recorded UID in the message store, or a counter cached from - the mailbox document in soledad if this is higher. - - :return: the last uid for messages in this mailbox - :rtype: int - """ - last = self._memstore.get_last_uid(self.mbox) - logger.debug("last uid for %s: %s (from memstore)" % ( - repr(self.mbox), last)) - return last - - last_uid = property( - _get_last_uid, doc="Last_UID attribute.") - def prime_last_uid_to_memstore(self): - """ - Prime memstore with last_uid value - """ - primed = self._last_uid_primed.get(self.mbox, False) - if not primed: - mbox = self._get_mbox_doc() - if mbox is None: - # memory-only store - return - last = mbox.content.get('lastuid', 0) - logger.info("Priming Soledad last_uid to %s" % (last,)) - self._memstore.set_last_soledad_uid(self.mbox, last) - self._last_uid_primed[self.mbox] = True - - def prime_known_uids_to_memstore(self): - """ - Prime memstore with the set of all known uids. - - We do this to be able to filter the requests efficiently. - """ - primed = self._known_uids_primed.get(self.mbox, False) - # XXX handle the maybeDeferred - - def set_primed(known_uids): - self._memstore.set_known_uids(self.mbox, known_uids) - self._known_uids_primed[self.mbox] = True - - if not primed: - d = self.messages.all_soledad_uid_iter() - d.addCallback(set_primed) - return d - - def prime_flag_docs_to_memstore(self): - """ - Prime memstore with all the flags documents. + :rtype: Deferred """ - primed = self._fdoc_primed.get(self.mbox, False) - - def set_flag_docs(flag_docs): - self._memstore.load_flag_docs(self.mbox, flag_docs) - self._fdoc_primed[self.mbox] = True - - if not primed: - d = self.messages.get_all_soledad_flag_docs() - d.addCallback(set_flag_docs) - return d + return self.collection.set_mbox_attr("closed", closed) def getUIDValidity(self): """ @@ -334,12 +221,7 @@ class SoledadMailbox(WithMsgFields): :return: unique validity identifier :rtype: int """ - if self._uidvalidity is None: - mbox = self._get_mbox_doc() - if mbox is None: - return 0 - self._uidvalidity = mbox.content.get(self.CREATED_KEY, 1) - return self._uidvalidity + return self.collection.get_mbox_attr("created") def getUID(self, message): """ @@ -354,9 +236,9 @@ class SoledadMailbox(WithMsgFields): :rtype: int """ - msg = self.messages.get_msg_by_uid(message) - if msg is not None: - return msg.getUID() + d = self.collection.get_msg_by_uid(message) + d.addCallback(lambda m: m.getUID()) + return d def getUIDNext(self): """ @@ -364,23 +246,20 @@ class SoledadMailbox(WithMsgFields): mailbox. Currently it returns the higher UID incremented by one. - We increment the next uid *each* time this function gets called. - In this way, there will be gaps if the message with the allocated - uid cannot be saved. But that is preferable to having race conditions - if we get to parallel message adding. - - :rtype: int + :return: deferred with int + :rtype: Deferred """ - with self.next_uid_lock: - return self.last_uid + 1 + d = self.collection.get_uid_next() + return d def getMessageCount(self): """ Returns the total count of messages in this mailbox. - :rtype: int + :return: deferred with int + :rtype: Deferred """ - return self.messages.count() + return self.collection.count() def getUnseenCount(self): """ @@ -389,7 +268,7 @@ class SoledadMailbox(WithMsgFields): :return: count of messages flagged `unseen` :rtype: int """ - return self.messages.count_unseen() + return self.collection.count_unseen() def getRecentCount(self): """ @@ -398,7 +277,7 @@ class SoledadMailbox(WithMsgFields): :return: count of messages flagged `recent` :rtype: int """ - return self.messages.count_recent() + return self.collection.count_recent() def isWriteable(self): """ @@ -407,6 +286,8 @@ class SoledadMailbox(WithMsgFields): :return: 1 if mailbox is read-writeable, 0 otherwise. :rtype: int """ + # XXX We don't need to store it in the mbox doc, do we? + # return int(self.collection.get_mbox_attr('rw')) return self.rw def getHierarchicalDelimiter(self): @@ -431,14 +312,14 @@ class SoledadMailbox(WithMsgFields): if self.CMD_RECENT in names: r[self.CMD_RECENT] = self.getRecentCount() if self.CMD_UIDNEXT in names: - r[self.CMD_UIDNEXT] = self.last_uid + 1 + r[self.CMD_UIDNEXT] = self.getUIDNext() if self.CMD_UIDVALIDITY in names: r[self.CMD_UIDVALIDITY] = self.getUIDValidity() if self.CMD_UNSEEN in names: r[self.CMD_UNSEEN] = self.getUnseenCount() return defer.succeed(r) - def addMessage(self, message, flags, date=None, notify_on_disk=False): + def addMessage(self, message, flags, date=None): """ Adds a message to this mailbox. @@ -464,10 +345,8 @@ class SoledadMailbox(WithMsgFields): else: flags = tuple(str(flag) for flag in flags) - d = self._do_add_message(message, flags=flags, date=date, - notify_on_disk=notify_on_disk) - #if PROFILE_CMD: - #do_profile_cmd(d, "APPEND") + # if PROFILE_CMD: + # do_profile_cmd(d, "APPEND") # XXX should review now that we're not using qtreactor. # A better place for this would be the COPY/APPEND dispatcher @@ -478,19 +357,11 @@ class SoledadMailbox(WithMsgFields): reactor.callLater(0, self.notify_new) return x + d = self.collection.add_message(flags=flags, date=date) d.addCallback(notifyCallback) d.addErrback(lambda f: log.msg(f.getTraceback())) return d - def _do_add_message(self, message, flags, date, notify_on_disk=False): - """ - Calls to the messageCollection add_msg method. - Invoked from addMessage. - """ - d = self.messages.add_msg(message, flags=flags, date=date, - notify_on_disk=notify_on_disk) - return d - def notify_new(self, *args): """ Notify of new messages to all the listeners. @@ -502,26 +373,34 @@ class SoledadMailbox(WithMsgFields): def cbNotifyNew(result): exists, recent = result - for l in self.listeners: - l.newMessages(exists, recent) + for listener in self.listeners: + listener.newMessages(exists, recent) + d = self._get_notify_count() d.addCallback(cbNotifyNew) d.addCallback(self.cb_signal_unread_to_ui) - @deferred_to_thread def _get_notify_count(self): """ Get message count and recent count for this mailbox Executed in a separate thread. Called from notify_new. - :return: number of messages and number of recent messages. - :rtype: tuple + :return: a deferred that will fire with a tuple, with number of + messages and number of recent messages. + :rtype: Deferred """ - exists = self.getMessageCount() - recent = self.getRecentCount() - logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( - self.mbox, exists, recent)) - return exists, recent + d_exists = self.getMessageCount() + d_recent = self.getRecentCount() + d_list = [d_exists, d_recent] + + def log_num_msg(result): + exists, recent = result + logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( + self.mbox_name, exists, recent)) + + d = defer.gatherResults(d_list) + d.addCallback(log_num_msg) + return d # commands, do not rename methods @@ -533,27 +412,18 @@ class SoledadMailbox(WithMsgFields): on the mailbox. """ - # XXX this will overwrite all the existing flags! + # XXX this will overwrite all the existing flags # should better simply addFlag - self.setFlags((self.NOSELECT_FLAG,)) - - # XXX removing the mailbox in situ for now, - # we should postpone the removal - - def remove_mbox_doc(ignored): - # XXX move to memory store?? + self.setFlags((MessageFlags.NOSELECT_FLAG,)) - def _remove_mbox_doc(doc): - if doc is None: - # memory-only store! - return defer.succeed(True) - return self._soledad.delete_doc(doc) - - doc = self._get_mbox_doc() - return _remove_mbox_doc(doc) + def remove_mbox(_): + # FIXME collection does not have a delete_mbox method, + # it's in account. + # XXX should take care of deleting the uid table too. + return self.collection.delete_mbox(self.mbox_name) d = self.deleteAllDocs() - d.addCallback(remove_mbox_doc) + d.addCallback(remove_mbox) return d def _close_cb(self, result): @@ -574,9 +444,11 @@ class SoledadMailbox(WithMsgFields): if not self.isWriteable(): raise imap4.ReadOnlyMailbox d = defer.Deferred() - self._memstore.expunge(self.mbox, d) + # FIXME actually broken. + # Iterate through index, and do a expunge. return d + # FIXME -- get last_uid from mbox_indexer def _bound_seq(self, messages_asked): """ Put an upper bound to a messages sequence if this is open. @@ -596,6 +468,7 @@ class SoledadMailbox(WithMsgFields): pass return messages_asked + # TODO -- needed? --- we can get the sequence from the indexer. def _filter_msg_seq(self, messages_asked): """ Filter a message sequence returning only the ones that do exist in the @@ -627,29 +500,6 @@ class SoledadMailbox(WithMsgFields): :rtype: deferred """ - d = defer.Deferred() - - # XXX do not need no thread... - reactor.callInThread(self._do_fetch, messages_asked, uid, d) - d.addCallback(self.cb_signal_unread_to_ui) - return d - - # called in thread - def _do_fetch(self, messages_asked, uid, d): - """ - :param messages_asked: IDs of the messages to retrieve information - about - :type messages_asked: MessageSet - - :param uid: If true, the IDs are UIDs. They are message sequence IDs - otherwise. - :type uid: bool - :param d: deferred whose callback will be called with result. - :type d: Deferred - - :rtype: A tuple of two-tuples of message sequence numbers and - LeapMessage - """ # For the moment our UID is sequential, so we # can treat them all the same. # Change this to the flag that twisted expects when we @@ -660,18 +510,23 @@ class SoledadMailbox(WithMsgFields): messages_asked = self._bound_seq(messages_asked) seq_messg = self._filter_msg_seq(messages_asked) - getmsg = lambda uid: self.messages.get_msg_by_uid(uid) + getmsg = self.collection.get_msg_by_uid # for sequence numbers (uid = 0) if sequence: logger.debug("Getting msg by index: INEFFICIENT call!") + # TODO --- implement sequences in mailbox indexer raise NotImplementedError else: got_msg = ((msgid, getmsg(msgid)) for msgid in seq_messg) result = ((msgid, msg) for msgid, msg in got_msg if msg is not None) - self.reactor.callLater(0, self.unset_recent_flags, seq_messg) - self.reactor.callFromThread(d.callback, result) + reactor.callLater(0, self.unset_recent_flags, seq_messg) + + # TODO -- call signal_to_ui + # d.addCallback(self.cb_signal_unread_to_ui) + + return result def fetch_flags(self, messages_asked, uid): """ @@ -698,12 +553,11 @@ class SoledadMailbox(WithMsgFields): :rtype: tuple """ d = defer.Deferred() - self.reactor.callInThread(self._do_fetch_flags, messages_asked, uid, d) + reactor.callLater(0, self._do_fetch_flags, messages_asked, uid, d) if PROFILE_CMD: do_profile_cmd(d, "FETCH-ALL-FLAGS") return d - # called in thread def _do_fetch_flags(self, messages_asked, uid, d): """ :param messages_asked: IDs of the messages to retrieve information @@ -733,10 +587,11 @@ class SoledadMailbox(WithMsgFields): messages_asked = self._bound_seq(messages_asked) seq_messg = self._filter_msg_seq(messages_asked) - all_flags = self._memstore.all_flags(self.mbox) + # FIXME use deferreds here + all_flags = self.collection.get_all_flags(self.mbox_name) result = ((msgid, flagsPart( msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg) - self.reactor.callFromThread(d.callback, result) + d.callback(result) def fetch_headers(self, messages_asked, uid): """ @@ -843,8 +698,8 @@ class SoledadMailbox(WithMsgFields): raise imap4.ReadOnlyMailbox d = defer.Deferred() - self.reactor.callLater(0, self._do_store, messages_asked, flags, - mode, uid, d) + reactor.callLater(0, self._do_store, messages_asked, flags, + mode, uid, d) if PROFILE_CMD: do_profile_cmd(d, "STORE") d.addCallback(self.cb_signal_unread_to_ui) @@ -853,7 +708,7 @@ class SoledadMailbox(WithMsgFields): def _do_store(self, messages_asked, flags, mode, uid, observer): """ - Helper method, invoke set_flags method in the MessageCollection. + Helper method, invoke set_flags method in the IMAPMessageCollection. See the documentation for the `store` method for the parameters. @@ -869,7 +724,8 @@ class SoledadMailbox(WithMsgFields): flags = tuple(flags) messages_asked = self._bound_seq(messages_asked) seq_messg = self._filter_msg_seq(messages_asked) - self.messages.set_flags(self.mbox, seq_messg, flags, mode, observer) + self.collection.set_flags( + self.mbox_name, seq_messg, flags, mode, observer) # ISearchableMailbox @@ -908,6 +764,7 @@ class SoledadMailbox(WithMsgFields): msgid = str(query[3]).strip() logger.debug("Searching for %s" % (msgid,)) d = self.messages._get_uid_from_msgid(str(msgid)) + # XXX remove gatherResults d1 = defer.gatherResults([d]) # we want a list, so return it all the same return d1 @@ -928,94 +785,18 @@ class SoledadMailbox(WithMsgFields): uid when the copy succeed. :rtype: Deferred """ - d = defer.Deferred() if PROFILE_CMD: do_profile_cmd(d, "COPY") # A better place for this would be the COPY/APPEND dispatcher # in server.py, but qtreactor hangs when I do that, so this seems # to work fine for now. - d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) - deferLater(self.reactor, 0, self._do_copy, message, d) - return d + #d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) + #deferLater(self.reactor, 0, self._do_copy, message, d) + #return d - def _do_copy(self, message, observer): - """ - Call invoked from the deferLater in `copy`. This will - copy the flags and header documents, and pass them to the - `create_message` method in the MemoryStore, together with - the observer deferred that we've been passed along. - - :param message: an IMessage implementor - :type message: LeapMessage - :param observer: the deferred that will fire with the - UID of the message - :type observer: Deferred - """ - memstore = self._memstore - - def createCopy(result): - exist, new_fdoc = result - if exist: - # Should we signal error on the callback? - logger.warning("Destination message already exists!") - - # XXX I'm not sure if we should raise the - # errback. This actually rases an ugly warning - # in some muas like thunderbird. - # UID 0 seems a good convention for no uid. - observer.callback(0) - else: - mbox = self.mbox - uid_next = memstore.increment_last_soledad_uid(mbox) - - new_fdoc[self.UID_KEY] = uid_next - new_fdoc[self.MBOX_KEY] = mbox - - flags = list(new_fdoc[self.FLAGS_KEY]) - flags.append(fields.RECENT_FLAG) - new_fdoc[self.FLAGS_KEY] = tuple(set(flags)) - - # FIXME set recent! - - self._memstore.create_message( - self.mbox, uid_next, - MessageWrapper(new_fdoc), - observer=observer, - notify_on_disk=False) - - d = self._get_msg_copy(message) - d.addCallback(createCopy) - d.addErrback(lambda f: log.msg(f.getTraceback())) - - #@deferred_to_thread - def _get_msg_copy(self, message): - """ - Get a copy of the fdoc for this message, and check whether - it already exists. - - :param message: an IMessage implementor - :type message: LeapMessage - :return: exist, new_fdoc - :rtype: tuple - """ - # XXX for clarity, this could be delegated to a - # MessageCollection mixin that implements copy too, and - # moved out of here. - msg = message - memstore = self._memstore - - if empty(msg.fdoc): - logger.warning("Tried to copy a MSG with no fdoc") - return - new_fdoc = copy.deepcopy(msg.fdoc.content) - fdoc_chash = new_fdoc[fields.CONTENT_HASH_KEY] - - dest_fdoc = memstore.get_fdoc_from_chash( - fdoc_chash, self.mbox) - - exist = not empty(dest_fdoc) - return exist, new_fdoc + # FIXME not implemented !!! --- + return self.collection.copy_msg(message, self.mbox_name) # convenience fun @@ -1023,29 +804,25 @@ class SoledadMailbox(WithMsgFields): """ Delete all docs in this mailbox """ - def del_all_docs(docs): - todelete = [] - for doc in docs: - d = self.messages._soledad.delete_doc(doc) - todelete.append(d) - return defer.gatherResults(todelete) - - d = self.messages.get_all_docs() - d.addCallback(del_all_docs) - return d + # FIXME not implemented + return self.collection.delete_all_docs() def unset_recent_flags(self, uid_seq): """ Unset Recent flag for a sequence of UIDs. """ - self.messages.unset_recent_flags(uid_seq) + # FIXME not implemented + return self.collection.unset_recent_flags(uid_seq) def __repr__(self): """ Representation string for this mailbox. """ - return u"" % ( - self.mbox, self.messages.count()) + return u"" % ( + self.mbox_name, self.messages.count()) + + +_INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) def normalize_mailbox(name): @@ -1060,7 +837,8 @@ def normalize_mailbox(name): :rtype: unicode """ - _INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + # XXX maybe it would make sense to normalize common folders too: + # trash, sent, drafts, etc... if _INBOX_RE.match(name): # ensure inital INBOX is uppercase return INBOX_NAME + name[len(INBOX_NAME):] diff --git a/src/leap/mail/imap/memorystore.py b/src/leap/mail/imap/memorystore.py deleted file mode 100644 index eda5b96..0000000 --- a/src/leap/mail/imap/memorystore.py +++ /dev/null @@ -1,1340 +0,0 @@ - -# memorystore.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -In-memory transient store for a LEAPIMAPServer. -""" -import contextlib -import logging -import threading -import weakref - -from collections import defaultdict -from copy import copy - -from enum import Enum -from twisted.internet import defer -from twisted.internet import reactor -from twisted.internet.task import LoopingCall -from twisted.python import log -from zope.interface import implements - -from leap.common.check import leap_assert_type -from leap.mail import size -from leap.mail.utils import empty, phash_iter -from leap.mail.messageflow import MessageProducer -from leap.mail.imap import interfaces -from leap.mail.imap.fields import fields -from leap.mail.imap.messageparts import MessagePartType, MessagePartDoc -from leap.mail.imap.messageparts import RecentFlagsDoc -from leap.mail.imap.messageparts import MessageWrapper -from leap.mail.imap.messageparts import ReferenciableDict - -from leap.mail.decorators import deferred_to_thread - -logger = logging.getLogger(__name__) - - -# The default period to do writebacks to the permanent -# soledad storage, in seconds. -SOLEDAD_WRITE_PERIOD = 15 - -FDOC = MessagePartType.fdoc.name -HDOC = MessagePartType.hdoc.name -CDOCS = MessagePartType.cdocs.name -DOCS_ID = MessagePartType.docs_id.name - - -@contextlib.contextmanager -def set_bool_flag(obj, att): - """ - Set a boolean flag to True while we're doing our thing. - Just to let the world know. - """ - setattr(obj, att, True) - try: - yield True - except RuntimeError as exc: - logger.exception(exc) - finally: - setattr(obj, att, False) - - -DirtyState = Enum("DirtyState", "none dirty new") - - -class MemoryStore(object): - """ - An in-memory store to where we can write the different parts that - we split the messages into and buffer them until we write them to the - permanent storage. - - It uses MessageWrapper instances to represent the message-parts, which are - indexed by mailbox name and UID. - - It also can be passed a permanent storage as a paremeter (any implementor - of IMessageStore, in this case a SoledadStore). In this case, a periodic - dump of the messages stored in memory will be done. The period of the - writes to the permanent storage is controled by the write_period parameter - in the constructor. - """ - implements(interfaces.IMessageStore, - interfaces.IMessageStoreWriter) - - # TODO We will want to index by chash when we transition to local-only - # UIDs. - - WRITING_FLAG = "_writing" - _last_uid_lock = threading.Lock() - _fdoc_docid_lock = threading.Lock() - - def __init__(self, permanent_store=None, - write_period=SOLEDAD_WRITE_PERIOD): - """ - Initialize a MemoryStore. - - :param permanent_store: a IMessageStore implementor to dump - messages to. - :type permanent_store: IMessageStore - :param write_period: the interval to dump messages to disk, in seconds. - :type write_period: int - """ - self._permanent_store = permanent_store - self._write_period = write_period - - if permanent_store is None: - self._mbox_closed = defaultdict(lambda: False) - - # Internal Storage: messages - """ - flags document store. - _fdoc_store[mbox][uid] = { 'content': 'aaa' } - """ - self._fdoc_store = defaultdict(lambda: defaultdict( - lambda: ReferenciableDict({}))) - - # Sizes - """ - {'mbox, uid': } - """ - self._sizes = {} - - # Internal Storage: payload-hash - """ - fdocs:doc-id store, stores document IDs for putting - the dirty flags-docs. - """ - self._fdoc_id_store = defaultdict(lambda: defaultdict( - lambda: '')) - - # Internal Storage: content-hash:hdoc - """ - hdoc-store keeps references to - the header-documents indexed by content-hash. - - {'chash': { dict-stuff } - } - """ - self._hdoc_store = defaultdict(lambda: ReferenciableDict({})) - - # Internal Storage: payload-hash:cdoc - """ - content-docs stored by payload-hash - {'phash': { dict-stuff } } - """ - self._cdoc_store = defaultdict(lambda: ReferenciableDict({})) - - # Internal Storage: content-hash:fdoc - """ - chash-fdoc-store keeps references to - the flag-documents indexed by content-hash. - - {'chash': {'mbox-a': weakref.proxy(dict), - 'mbox-b': weakref.proxy(dict)} - } - """ - self._chash_fdoc_store = defaultdict(lambda: defaultdict(lambda: None)) - - # Internal Storage: recent-flags store - """ - recent-flags store keeps one dict per mailbox, - with the document-id of the u1db document - and the set of the UIDs that have the recent flag. - - {'mbox-a': {'doc_id': 'deadbeef', - 'set': {1,2,3,4} - } - } - """ - # TODO this will have to transition to content-hash - # indexes after we move to local-only UIDs. - - self._rflags_store = defaultdict( - lambda: {'doc_id': None, 'set': set([])}) - - """ - last-uid store keeps the count of the highest UID - per mailbox. - - {'mbox-a': 42, - 'mbox-b': 23} - """ - self._last_uid = defaultdict(lambda: 0) - - """ - known-uids keeps a count of the uids that soledad knows for a given - mailbox - - {'mbox-a': set([1,2,3])} - """ - self._known_uids = defaultdict(set) - - """ - mbox-flags is a dict containing flags for each mailbox. this is - modified from mailbox.getFlags / mailbox.setFlags - """ - self._mbox_flags = defaultdict(set) - - # New and dirty flags, to set MessageWrapper State. - self._new = set([]) - self._new_queue = set([]) - self._new_deferreds = {} - - self._dirty = set([]) - self._dirty_queue = set([]) - self._dirty_deferreds = {} - - self._rflags_dirty = set([]) - - # Flag for signaling we're busy writing to the disk storage. - setattr(self, self.WRITING_FLAG, False) - - if self._permanent_store is not None: - # this producer spits its messages to the permanent store - # consumer using a queue. We will use that to put - # our messages to be written. - self.producer = MessageProducer(permanent_store, - period=0.1) - # looping call for dumping to SoledadStore - self._write_loop = LoopingCall(self.write_messages, - permanent_store) - - # We can start the write loop right now, why wait? - self._start_write_loop() - else: - # We have a memory-only store. - self.producer = None - self._write_loop = None - - # TODO -- remove - def _start_write_loop(self): - """ - Start loop for writing to disk database. - """ - if self._write_loop is None: - return - if not self._write_loop.running: - self._write_loop.start(self._write_period, now=True) - - # TODO -- remove - def _stop_write_loop(self): - """ - Stop loop for writing to disk database. - """ - if self._write_loop is None: - return - if self._write_loop.running: - self._write_loop.stop() - - # IMessageStore - - # XXX this would work well for whole message operations. - # We would have to add a put_flags operation to modify only - # the flags doc (and set the dirty flag accordingly) - - def create_message(self, mbox, uid, message, observer, - notify_on_disk=True): - """ - Create the passed message into this MemoryStore. - - By default we consider that any message is a new message. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the UID for the message - :type uid: int - :param message: a message to be added - :type message: MessageWrapper - :param observer: - the deferred that will fire with the UID of the message. If - notify_on_disk is True, this will happen when the message is - written to Soledad. Otherwise it will fire as soon as we've added - the message to the memory store. - :type observer: Deferred - :param notify_on_disk: - whether the `observer` deferred should wait until the message is - written to disk to be fired. - :type notify_on_disk: bool - """ - # TODO -- return a deferred - log.msg("Adding new doc to memstore %r (%r)" % (mbox, uid)) - key = mbox, uid - - self._add_message(mbox, uid, message, notify_on_disk) - self._new.add(key) - - if observer is not None: - if notify_on_disk: - # We store this deferred so we can keep track of the pending - # operations internally. - # TODO this should fire with the UID !!! -- change that in - # the soledad store code. - self._new_deferreds[key] = observer - - else: - # Caller does not care, just fired and forgot, so we pass - # a defer that will inmediately have its callback triggered. - reactor.callFromThread(observer.callback, uid) - - def put_message(self, mbox, uid, message, notify_on_disk=True): - """ - Put an existing message. - - This will also set the dirty flag on the MemoryStore. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the UID for the message - :type uid: int - :param message: a message to be added - :type message: MessageWrapper - :param notify_on_disk: whether the deferred that is returned should - wait until the message is written to disk to - be fired. - :type notify_on_disk: bool - - :return: a Deferred. if notify_on_disk is True, will be fired - when written to the db on disk. - Otherwise will fire inmediately - :rtype: Deferred - """ - key = mbox, uid - d = defer.Deferred() - d.addCallback(lambda result: log.msg("message PUT save: %s" % result)) - - self._dirty.add(key) - self._dirty_deferreds[key] = d - self._add_message(mbox, uid, message, notify_on_disk) - return d - - def _add_message(self, mbox, uid, message, notify_on_disk=True): - """ - Helper method, called by both create_message and put_message. - See those for parameter documentation. - """ - msg_dict = message.as_dict() - - fdoc = msg_dict.get(FDOC, None) - if fdoc is not None: - fdoc_store = self._fdoc_store[mbox][uid] - fdoc_store.update(fdoc) - chash_fdoc_store = self._chash_fdoc_store - - # content-hash indexing - chash = fdoc.get(fields.CONTENT_HASH_KEY) - chash_fdoc_store[chash][mbox] = weakref.proxy( - self._fdoc_store[mbox][uid]) - - hdoc = msg_dict.get(HDOC, None) - if hdoc is not None: - chash = hdoc.get(fields.CONTENT_HASH_KEY) - hdoc_store = self._hdoc_store[chash] - hdoc_store.update(hdoc) - - cdocs = message.cdocs - for cdoc in cdocs.values(): - phash = cdoc.get(fields.PAYLOAD_HASH_KEY, None) - if not phash: - continue - cdoc_store = self._cdoc_store[phash] - cdoc_store.update(cdoc) - - # Update memory store size - # XXX this should use [mbox][uid] - # TODO --- this has to be deferred to thread, - # TODO add hdoc and cdocs sizes too - # it's slowing things down here. - # key = mbox, uid - # self._sizes[key] = size.get_size(self._fdoc_store[key]) - - def purge_fdoc_store(self, mbox): - """ - Purge the empty documents from a fdoc store. - Called during initialization of the SoledadMailbox - - :param mbox: the mailbox - :type mbox: str or unicode - """ - # XXX This is really a workaround until I find the conditions - # that are making the empty items remain there. - # This happens, for instance, after running several times - # the regression test, that issues a store deleted + expunge + select - # The items are being correclty deleted, but in succesive appends - # the empty items with previously deleted uids reappear as empty - # documents. I suspect it's a timing condition with a previously - # evaluated sequence being used after the items has been removed. - - for uid, value in self._fdoc_store[mbox].items(): - if empty(value): - del self._fdoc_store[mbox][uid] - - def get_docid_for_fdoc(self, mbox, uid): - """ - Return Soledad document id for the flags-doc for a given mbox and uid, - or None of no flags document could be found. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the message UID - :type uid: int - :rtype: unicode or None - """ - with self._fdoc_docid_lock: - doc_id = self._fdoc_id_store[mbox][uid] - - if empty(doc_id): - fdoc = self._permanent_store.get_flags_doc(mbox, uid) - if empty(fdoc) or empty(fdoc.content): - return None - doc_id = fdoc.doc_id - self._fdoc_id_store[mbox][uid] = doc_id - - return doc_id - - def get_message(self, mbox, uid, dirtystate=DirtyState.none, - flags_only=False): - """ - Get a MessageWrapper for the given mbox and uid combination. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the message UID - :type uid: int - :param dirtystate: DirtyState enum: one of `dirty`, `new` - or `none` (default) - :type dirtystate: enum - :param flags_only: whether the message should carry only a reference - to the flags document. - :type flags_only: bool - : - - :return: MessageWrapper or None - """ - # TODO -- return deferred - if dirtystate == DirtyState.dirty: - flags_only = True - - key = mbox, uid - - fdoc = self._fdoc_store[mbox][uid] - if empty(fdoc): - return None - - new, dirty = False, False - if dirtystate == DirtyState.none: - new, dirty = self._get_new_dirty_state(key) - if dirtystate == DirtyState.dirty: - new, dirty = False, True - if dirtystate == DirtyState.new: - new, dirty = True, False - - if flags_only: - return MessageWrapper(fdoc=fdoc, - new=new, dirty=dirty, - memstore=weakref.proxy(self)) - else: - chash = fdoc.get(fields.CONTENT_HASH_KEY) - hdoc = self._hdoc_store[chash] - if empty(hdoc): - # XXX this will be a deferred - hdoc = self._permanent_store.get_headers_doc(chash) - if empty(hdoc): - return None - if not empty(hdoc.content): - self._hdoc_store[chash] = hdoc.content - hdoc = hdoc.content - cdocs = None - - pmap = hdoc.get(fields.PARTS_MAP_KEY, None) - if new and pmap is not None: - # take the different cdocs for write... - cdoc_store = self._cdoc_store - cdocs_list = phash_iter(hdoc) - cdocs = dict(enumerate( - [cdoc_store[phash] for phash in cdocs_list], 1)) - - return MessageWrapper(fdoc=fdoc, hdoc=hdoc, cdocs=cdocs, - new=new, dirty=dirty, - memstore=weakref.proxy(self)) - - def remove_message(self, mbox, uid): - """ - Remove a Message from this MemoryStore. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the message UID - :type uid: int - """ - # XXX For the moment we are only removing the flags and headers - # docs. The rest we leave there polluting your hard disk, - # until we think about a good way of deorphaning. - - # XXX implement elijah's idea of using a PUT document as a - # token to ensure consistency in the removal. - - try: - del self._fdoc_store[mbox][uid] - except KeyError: - pass - - try: - key = mbox, uid - self._new.discard(key) - self._dirty.discard(key) - if key in self._sizes: - del self._sizes[key] - self._known_uids[mbox].discard(uid) - except KeyError: - pass - except Exception as exc: - logger.error("error while removing message!") - logger.exception(exc) - try: - with self._fdoc_docid_lock: - del self._fdoc_id_store[mbox][uid] - except KeyError: - pass - except Exception as exc: - logger.error("error while removing message!") - logger.exception(exc) - - # IMessageStoreWriter - - # TODO -- I think we don't need this anymore. - # instead, we can have - def write_messages(self, store): - """ - Write the message documents in this MemoryStore to a different store. - - :param store: the IMessageStore to write to - :rtype: False if queue is not empty, None otherwise. - """ - # For now, we pass if the queue is not empty, to avoid duplicate - # queuing. - # We would better use a flag to know when we've already enqueued an - # item. - - # XXX this could return the deferred for all the enqueued operations - - if not self.producer.is_queue_empty(): - return False - - if any(map(lambda i: not empty(i), (self._new, self._dirty))): - logger.info("Writing messages to Soledad...") - - # TODO change for lock, and make the property access - # is accquired - with set_bool_flag(self, self.WRITING_FLAG): - for rflags_doc_wrapper in self.all_rdocs_iter(): - self.producer.push(rflags_doc_wrapper, - state=self.producer.STATE_DIRTY) - for msg_wrapper in self.all_new_msg_iter(): - self.producer.push(msg_wrapper, - state=self.producer.STATE_NEW) - for msg_wrapper in self.all_dirty_msg_iter(): - self.producer.push(msg_wrapper, - state=self.producer.STATE_DIRTY) - - # MemoryStore specific methods. - - def get_uids(self, mbox): - """ - Get all uids for a given mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: list - """ - return self._fdoc_store[mbox].keys() - - def get_soledad_known_uids(self, mbox): - """ - Get all uids that soledad knows about, from the memory cache. - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: list - """ - return self._known_uids.get(mbox, []) - - # last_uid - - def get_last_uid(self, mbox): - """ - Return the highest UID for a given mbox. - It will be the highest between the highest uid in the message store for - the mailbox, and the soledad integer cache. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: int - """ - uids = self.get_uids(mbox) - last_mem_uid = uids and max(uids) or 0 - last_soledad_uid = self.get_last_soledad_uid(mbox) - return max(last_mem_uid, last_soledad_uid) - - def get_last_soledad_uid(self, mbox): - """ - Get last uid for a given mbox from the soledad integer cache. - - :param mbox: the mailbox - :type mbox: str or unicode - """ - return self._last_uid.get(mbox, 0) - - def set_last_soledad_uid(self, mbox, value): - """ - Set last uid for a given mbox in the soledad integer cache. - SoledadMailbox should prime this value during initialization. - Other methods (during message adding) SHOULD call - `increment_last_soledad_uid` instead. - - :param mbox: the mailbox - :type mbox: str or unicode - :param value: the value to set - :type value: int - """ - # can be long??? - # leap_assert_type(value, int) - logger.info("setting last soledad uid for %s to %s" % - (mbox, value)) - # if we already have a value here, don't do anything - with self._last_uid_lock: - if not self._last_uid.get(mbox, None): - self._last_uid[mbox] = value - - def set_known_uids(self, mbox, value): - """ - Set the value fo the known-uids set for this mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :param value: a sequence of integers to be added to the set. - :type value: tuple - """ - current = self._known_uids[mbox] - self._known_uids[mbox] = current.union(set(value)) - - def increment_last_soledad_uid(self, mbox): - """ - Increment by one the soledad integer cache for the last_uid for - this mbox, and fire a defer-to-thread to update the soledad value. - The caller should lock the call tho this method. - - :param mbox: the mailbox - :type mbox: str or unicode - """ - with self._last_uid_lock: - self._last_uid[mbox] += 1 - value = self._last_uid[mbox] - reactor.callInThread(self.write_last_uid, mbox, value) - return value - - def write_last_uid(self, mbox, value): - """ - Increment the soledad integer cache for the highest uid value. - - :param mbox: the mailbox - :type mbox: str or unicode - :param value: the value to set - :type value: int - """ - leap_assert_type(value, int) - if self._permanent_store: - self._permanent_store.write_last_uid(mbox, value) - - def load_flag_docs(self, mbox, flag_docs): - """ - Load the flag documents for the given mbox. - Used during initial flag docs prefetch. - - :param mbox: the mailbox - :type mbox: str or unicode - :param flag_docs: a dict with the content for the flag docs, indexed - by uid. - :type flag_docs: dict - """ - # We can do direct assignments cause we know this will only - # be called during initialization of the mailbox. - # TODO could hook here a sanity-check - # for duplicates - - fdoc_store = self._fdoc_store[mbox] - chash_fdoc_store = self._chash_fdoc_store - for uid in flag_docs: - rdict = ReferenciableDict(flag_docs[uid]) - fdoc_store[uid] = rdict - # populate chash dict too, to avoid fdoc duplication - chash = flag_docs[uid]["chash"] - chash_fdoc_store[chash][mbox] = weakref.proxy( - self._fdoc_store[mbox][uid]) - - def update_flags(self, mbox, uid, fdoc): - """ - Update the flag document for a given mbox and uid combination, - and set the dirty flag. - We could use put_message, but this is faster. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the uid of the message - :type uid: int - - :param fdoc: a dict with the content for the flag docs - :type fdoc: dict - """ - key = mbox, uid - self._fdoc_store[mbox][uid].update(fdoc) - self._dirty.add(key) - - def load_header_docs(self, header_docs): - """ - Load the flag documents for the given mbox. - Used during header docs prefetch, and during cache after - a read from soledad if the hdoc property in message did not - find its value in here. - - :param flag_docs: a dict with the content for the flag docs. - :type flag_docs: dict - """ - hdoc_store = self._hdoc_store - for chash in header_docs: - hdoc_store[chash] = ReferenciableDict(header_docs[chash]) - - def all_flags(self, mbox): - """ - Return a dictionary with all the flags for a given mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: dict - """ - fdict = {} - uids = self.get_uids(mbox) - fstore = self._fdoc_store[mbox] - - for uid in uids: - try: - fdict[uid] = fstore[uid][fields.FLAGS_KEY] - except KeyError: - continue - return fdict - - def all_headers(self, mbox): - """ - Return a dictionary with all the header docs for a given mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: dict - """ - headers_dict = {} - uids = self.get_uids(mbox) - fdoc_store = self._fdoc_store[mbox] - hdoc_store = self._hdoc_store - - for uid in uids: - try: - chash = fdoc_store[uid][fields.CONTENT_HASH_KEY] - hdoc = hdoc_store[chash] - if not empty(hdoc): - headers_dict[uid] = hdoc - except KeyError: - continue - return headers_dict - - # Counting sheeps... - - def count_new_mbox(self, mbox): - """ - Count the new messages by mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: number of new messages - :rtype: int - """ - return len([(m, uid) for m, uid in self._new if mbox == mbox]) - - # XXX used at all? - def count_new(self): - """ - Count all the new messages in the MemoryStore. - - :rtype: int - """ - return len(self._new) - - def count(self, mbox): - """ - Return the count of messages for a given mbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: number of messages - :rtype: int - """ - return len(self._fdoc_store[mbox]) - - def unseen_iter(self, mbox): - """ - Get an iterator for the message UIDs with no `seen` flag - for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: iterator through unseen message doc UIDs - :rtype: iterable - """ - fdocs = self._fdoc_store[mbox] - - return [uid for uid, value - in fdocs.items() - if fields.SEEN_FLAG not in value.get(fields.FLAGS_KEY, [])] - - def get_cdoc_from_phash(self, phash): - """ - Return a content-document by its payload-hash. - - :param phash: the payload hash to check against - :type phash: str or unicode - :rtype: MessagePartDoc - """ - doc = self._cdoc_store.get(phash, None) - - # XXX return None for consistency? - - # XXX have to keep a mapping between phash and its linkage - # info, to know if this payload is been already saved or not. - # We will be able to get this from the linkage-docs, - # not yet implemented. - new = True - dirty = False - return MessagePartDoc( - new=new, dirty=dirty, store="mem", - part=MessagePartType.cdoc, - content=doc, - doc_id=None) - - def get_fdoc_from_chash(self, chash, mbox): - """ - Return a flags-document by its content-hash and a given mailbox. - Used during content-duplication detection while copying or adding a - message. - - :param chash: the content hash to check against - :type chash: str or unicode - :param mbox: the mailbox - :type mbox: str or unicode - - :return: MessagePartDoc. It will return None if the flags document - has empty content or it is flagged as \\Deleted. - """ - fdoc = self._chash_fdoc_store[chash][mbox] - - # a couple of special cases. - # 1. We might have a doc with empty content... - if empty(fdoc): - return None - - # 2. ...Or the message could exist, but being flagged for deletion. - # We want to create a new one in this case. - # Hmmm what if the deletion is un-done?? We would end with a - # duplicate... - if fdoc and fields.DELETED_FLAG in fdoc.get(fields.FLAGS_KEY, []): - return None - - uid = fdoc[fields.UID_KEY] - key = mbox, uid - new = key in self._new - dirty = key in self._dirty - - return MessagePartDoc( - new=new, dirty=dirty, store="mem", - part=MessagePartType.fdoc, - content=fdoc, - doc_id=None) - - def iter_fdoc_keys(self): - """ - Return a generator through all the mbox, uid keys in the flags-doc - store. - """ - fdoc_store = self._fdoc_store - for mbox in fdoc_store: - for uid in fdoc_store[mbox]: - yield mbox, uid - - def all_new_msg_iter(self): - """ - Return generator that iterates through all new messages. - - :return: generator of MessageWrappers - :rtype: generator - """ - gm = self.get_message - # need to freeze, set can change during iteration - new = [gm(*key, dirtystate=DirtyState.new) for key in tuple(self._new)] - # move content from new set to the queue - self._new_queue.update(self._new) - self._new.difference_update(self._new) - return new - - def all_dirty_msg_iter(self): - """ - Return generator that iterates through all dirty messages. - - :return: generator of MessageWrappers - :rtype: generator - """ - gm = self.get_message - # need to freeze, set can change during iteration - dirty = [gm(*key, flags_only=True, dirtystate=DirtyState.dirty) - for key in tuple(self._dirty)] - # move content from new and dirty sets to the queue - - self._dirty_queue.update(self._dirty) - self._dirty.difference_update(self._dirty) - return dirty - - def all_deleted_uid_iter(self, mbox): - """ - Return a list with the UIDs for all messags - with deleted flag in a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: list of integers - :rtype: list - """ - # This *needs* to return a fixed sequence. Otherwise the dictionary len - # will change during iteration, when we modify it - fdocs = self._fdoc_store[mbox] - return [uid for uid, value - in fdocs.items() - if fields.DELETED_FLAG in value.get(fields.FLAGS_KEY, [])] - - # new, dirty flags - - def _get_new_dirty_state(self, key): - """ - Return `new` and `dirty` flags for a given message. - - :param key: the key for the message, in the form mbox, uid - :type key: tuple - :return: tuple of bools - :rtype: tuple - """ - # TODO change indexing of sets to [mbox][key] too. - # XXX should return *first* the news, and *then* the dirty... - - # TODO should query in queues too , true? - # - return map(lambda _set: key in _set, (self._new, self._dirty)) - - def set_new_queued(self, key): - """ - Add the key value to the `new-queue` set. - - :param key: the key for the message, in the form mbox, uid - :type key: tuple - """ - self._new_queue.add(key) - - def unset_new_queued(self, key): - """ - Remove the key value from the `new-queue` set. - - :param key: the key for the message, in the form mbox, uid - :type key: tuple - """ - self._new_queue.discard(key) - deferreds = self._new_deferreds - d = deferreds.get(key, None) - if d: - # XXX use a namedtuple for passing the result - # when we check it in the other side. - d.callback('%s, ok' % str(key)) - deferreds.pop(key) - - def set_dirty_queued(self, key): - """ - Add the key value to the `dirty-queue` set. - - :param key: the key for the message, in the form mbox, uid - :type key: tuple - """ - self._dirty_queue.add(key) - - def unset_dirty_queued(self, key): - """ - Remove the key value from the `dirty-queue` set. - - :param key: the key for the message, in the form mbox, uid - :type key: tuple - """ - self._dirty_queue.discard(key) - deferreds = self._dirty_deferreds - d = deferreds.get(key, None) - if d: - # XXX use a namedtuple for passing the result - # when we check it in the other side. - d.callback('%s, ok' % str(key)) - deferreds.pop(key) - - # Recent Flags - - def set_recent_flag(self, mbox, uid): - """ - Set the `Recent` flag for a given mailbox and UID. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the message UID - :type uid: int - """ - self._rflags_dirty.add(mbox) - self._rflags_store[mbox]['set'].add(uid) - - # TODO --- nice but unused - def unset_recent_flag(self, mbox, uid): - """ - Unset the `Recent` flag for a given mailbox and UID. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the message UID - :type uid: int - """ - self._rflags_store[mbox]['set'].discard(uid) - - def set_recent_flags(self, mbox, value): - """ - Set the value for the set of the recent flags. - Used from the property in the MessageCollection. - - :param mbox: the mailbox - :type mbox: str or unicode - :param value: a sequence of flags to set - :type value: sequence - """ - self._rflags_dirty.add(mbox) - self._rflags_store[mbox]['set'] = set(value) - - def load_recent_flags(self, mbox, flags_doc): - """ - Load the passed flags document in the recent flags store, for a given - mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :param flags_doc: A dictionary containing the `doc_id` of the Soledad - flags-document for this mailbox, and the `set` - of uids marked with that flag. - """ - self._rflags_store[mbox] = flags_doc - - def get_recent_flags(self, mbox): - """ - Return the set of UIDs with the `Recent` flag for this mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: set, or None - """ - rflag_for_mbox = self._rflags_store.get(mbox, None) - if not rflag_for_mbox: - return None - return self._rflags_store[mbox]['set'] - - # XXX -- remove - def all_rdocs_iter(self): - """ - Return an iterator through all in-memory recent flag dicts, wrapped - under a RecentFlagsDoc namedtuple. - Used for saving to disk. - - :return: a generator of RecentFlagDoc - :rtype: generator - """ - # XXX use enums - DOC_ID = "doc_id" - SET = "set" - - rflags_store = self._rflags_store - - def get_rdoc(mbox, rdict): - mbox_rflag_set = rdict[SET] - recent_set = copy(mbox_rflag_set) - # zero it! - mbox_rflag_set.difference_update(mbox_rflag_set) - return RecentFlagsDoc( - doc_id=rflags_store[mbox][DOC_ID], - content={ - fields.TYPE_KEY: fields.TYPE_RECENT_VAL, - fields.MBOX_KEY: mbox, - fields.RECENTFLAGS_KEY: list(recent_set) - }) - - return (get_rdoc(mbox, rdict) for mbox, rdict in rflags_store.items() - if not empty(rdict[SET])) - - # Methods that mirror the IMailbox interface - - def remove_all_deleted(self, mbox): - """ - Remove all messages flagged \\Deleted from this Memory Store only. - Called from `expunge` - - :param mbox: the mailbox - :type mbox: str or unicode - :return: a list of UIDs - :rtype: list - """ - mem_deleted = self.all_deleted_uid_iter(mbox) - for uid in mem_deleted: - self.remove_message(mbox, uid) - return mem_deleted - - # TODO -- remove - def stop_and_flush(self): - """ - Stop the write loop and trigger a write to the producer. - """ - self._stop_write_loop() - if self._permanent_store is not None: - # XXX we should check if we did get a True value on this - # operation. If we got False we should retry! (queue was not empty) - self.write_messages(self._permanent_store) - self.producer.flush() - - def expunge(self, mbox, observer): - """ - Remove all messages flagged \\Deleted, from the Memory Store - and from the permanent store also. - - It first queues up a last write, and wait for the deferreds to be done - before continuing. - - :param mbox: the mailbox - :type mbox: str or unicode - :param observer: a deferred that will be fired when expunge is done - :type observer: Deferred - """ - soledad_store = self._permanent_store - if soledad_store is None: - # just-in memory store, easy then. - self._delete_from_memory(mbox, observer) - return - - # We have a soledad storage. - try: - # Stop and trigger last write - self.stop_and_flush() - # Wait on the writebacks to finish - - # XXX what if pending deferreds is empty? - pending_deferreds = (self._new_deferreds.get(mbox, []) + - self._dirty_deferreds.get(mbox, [])) - d1 = defer.gatherResults(pending_deferreds, consumeErrors=True) - d1.addCallback( - self._delete_from_soledad_and_memory, mbox, observer) - except Exception as exc: - logger.exception(exc) - - def _delete_from_memory(self, mbox, observer): - """ - Remove all messages marked as deleted from soledad and memory. - - :param mbox: the mailbox - :type mbox: str or unicode - :param observer: a deferred that will be fired when expunge is done - :type observer: Deferred - """ - mem_deleted = self.remove_all_deleted(mbox) - # TODO return a DeferredList - observer.callback(mem_deleted) - - def _delete_from_soledad_and_memory(self, result, mbox, observer): - """ - Remove all messages marked as deleted from soledad and memory. - - :param result: ignored. the result of the deferredList that triggers - this as a callback from `expunge`. - :param mbox: the mailbox - :type mbox: str or unicode - :param observer: a deferred that will be fired when expunge is done - :type observer: Deferred - """ - all_deleted = [] - soledad_store = self._permanent_store - - try: - # 1. Delete all messages marked as deleted in soledad. - logger.debug("DELETING FROM SOLEDAD ALL FOR %r" % (mbox,)) - sol_deleted = soledad_store.remove_all_deleted(mbox) - - try: - self._known_uids[mbox].difference_update(set(sol_deleted)) - except Exception as exc: - logger.exception(exc) - - # 2. Delete all messages marked as deleted in memory. - logger.debug("DELETING FROM MEM ALL FOR %r" % (mbox,)) - mem_deleted = self.remove_all_deleted(mbox) - - all_deleted = set(mem_deleted).union(set(sol_deleted)) - logger.debug("deleted %r" % all_deleted) - except Exception as exc: - logger.exception(exc) - finally: - self._start_write_loop() - - observer.callback(all_deleted) - - # Mailbox documents and attributes - - # This could be also be cached in memstore, but proxying directly - # to soledad since it's not too performance-critical. - - def get_mbox_doc(self, mbox): - """ - Return the soledad document for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: SoledadDocument or None. - """ - if self.permanent_store is not None: - return self.permanent_store.get_mbox_document(mbox) - else: - return None - - def get_mbox_closed(self, mbox): - """ - Return the closed attribute for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: bool - """ - if self.permanent_store is not None: - return self.permanent_store.get_mbox_closed(mbox) - else: - return self._mbox_closed[mbox] - - def set_mbox_closed(self, mbox, closed): - """ - Set the closed attribute for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - """ - if self.permanent_store is not None: - self.permanent_store.set_mbox_closed(mbox, closed) - else: - self._mbox_closed[mbox] = closed - - def get_mbox_flags(self, mbox): - """ - Get the flags for a given mbox. - :rtype: list - """ - return sorted(self._mbox_flags[mbox]) - - def set_mbox_flags(self, mbox, flags): - """ - Set the mbox flags - """ - self._mbox_flags[mbox] = set(flags) - # TODO - # This should write to the permanent store!!! - - # Rename flag-documents - - def rename_fdocs_mailbox(self, old_mbox, new_mbox): - """ - Change the mailbox name for all flag documents in a given mailbox. - Used from account.rename - - :param old_mbox: name for the old mbox - :type old_mbox: str or unicode - :param new_mbox: name for the new mbox - :type new_mbox: str or unicode - """ - fs = self._fdoc_store - keys = fs[old_mbox].keys() - for k in keys: - fdoc = fs[old_mbox][k] - fdoc['mbox'] = new_mbox - fs[new_mbox][k] = fdoc - fs[old_mbox].pop(k) - self._dirty.add((new_mbox, k)) - - # Dump-to-disk controls. - - @property - def is_writing(self): - """ - Property that returns whether the store is currently writing its - internal state to a permanent storage. - - Used to evaluate whether the CHECK command can inform that the field - is clear to proceed, or waiting for the write operations to complete - is needed instead. - - :rtype: bool - """ - # FIXME this should return a deferred !!! - # TODO this should be moved to soledadStore instead - # (all pending deferreds) - return getattr(self, self.WRITING_FLAG) - - @property - def permanent_store(self): - return self._permanent_store - - # Memory management. - - def get_size(self): - """ - Return the size of the internal storage. - Use for calculating the limit beyond which we should flush the store. - - :rtype: int - """ - return reduce(lambda x, y: x + y, self._sizes, 0) diff --git a/src/leap/mail/imap/messageparts.py b/src/leap/mail/imap/messageparts.py deleted file mode 100644 index fb1d75a..0000000 --- a/src/leap/mail/imap/messageparts.py +++ /dev/null @@ -1,586 +0,0 @@ -# messageparts.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -MessagePart implementation. Used from LeapMessage. -""" -import logging -import StringIO -import weakref - -from collections import namedtuple - -from enum import Enum -from zope.interface import implements -from twisted.mail import imap4 - -from leap.common.decorators import memoized_method -from leap.common.mail import get_email_charset -from leap.mail.imap import interfaces -from leap.mail.imap.fields import fields -from leap.mail.utils import empty, first, find_charset - -MessagePartType = Enum("MessagePartType", "hdoc fdoc cdoc cdocs docs_id") - - -logger = logging.getLogger(__name__) - - -""" -A MessagePartDoc is a light wrapper around the dictionary-like -data that we pass along for message parts. It can be used almost everywhere -that you would expect a SoledadDocument, since it has a dict under the -`content` attribute. - -We also keep some metadata on it, relative in part to the message as a whole, -and sometimes to a part in particular only. - -* `new` indicates that the document has just been created. SoledadStore - should just create a new doc for all the related message parts. -* `store` indicates the type of store a given MessagePartDoc lives in. - We currently use this to indicate that the document comes from memeory, - but we should probably get rid of it as soon as we extend the use of the - SoledadStore interface along LeapMessage, MessageCollection and Mailbox. -* `part` is one of the MessagePartType enums. - -* `dirty` indicates that, while we already have the document in Soledad, - we have modified its state in memory, so we need to put_doc instead while - dumping the MemoryStore contents. - `dirty` attribute would only apply to flags-docs and linkage-docs. -* `doc_id` is the identifier for the document in the u1db database, if any. - -""" - -MessagePartDoc = namedtuple( - 'MessagePartDoc', - ['new', 'dirty', 'part', 'store', 'content', 'doc_id']) - -""" -A RecentFlagsDoc is used to send the recent-flags document payload to the -SoledadWriter during dumps. -""" -RecentFlagsDoc = namedtuple( - 'RecentFlagsDoc', - ['content', 'doc_id']) - - -class ReferenciableDict(dict): - """ - A dict that can be weak-referenced. - - Some builtin objects are not weak-referenciable unless - subclassed. So we do. - - Used to return pointers to the items in the MemoryStore. - """ - - -class MessageWrapper(object): - """ - A simple nested dictionary container around the different message subparts. - """ - implements(interfaces.IMessageContainer) - - FDOC = "fdoc" - HDOC = "hdoc" - CDOCS = "cdocs" - DOCS_ID = "docs_id" - - # Using slots to limit some the memory use, - # Add your attribute here. - - __slots__ = ["_dict", "_new", "_dirty", "_storetype", "memstore"] - - def __init__(self, fdoc=None, hdoc=None, cdocs=None, - from_dict=None, memstore=None, - new=True, dirty=False, docs_id={}): - """ - Initialize a MessageWrapper. - """ - # TODO add optional reference to original message in the incoming - self._dict = {} - self.memstore = memstore - - self._new = new - self._dirty = dirty - - self._storetype = "mem" - - if from_dict is not None: - self.from_dict(from_dict) - else: - if fdoc is not None: - self._dict[self.FDOC] = ReferenciableDict(fdoc) - if hdoc is not None: - self._dict[self.HDOC] = ReferenciableDict(hdoc) - if cdocs is not None: - self._dict[self.CDOCS] = ReferenciableDict(cdocs) - - # This will keep references to the doc_ids to be able to put - # messages to soledad. It will be populated during the walk() to avoid - # the overhead of reading from the db. - - # XXX it really *only* make sense for the FDOC, the other parts - # should not be "dirty", just new...!!! - self._dict[self.DOCS_ID] = docs_id - - # properties - - # TODO Could refactor new and dirty properties together. - - def _get_new(self): - """ - Get the value for the `new` flag. - - :rtype: bool - """ - return self._new - - def _set_new(self, value=False): - """ - Set the value for the `new` flag, and propagate it - to the memory store if any. - - :param value: the value to set - :type value: bool - """ - self._new = value - if self.memstore: - mbox = self.fdoc.content.get('mbox', None) - uid = self.fdoc.content.get('uid', None) - if not mbox or not uid: - logger.warning("Malformed fdoc") - return - key = mbox, uid - fun = [self.memstore.unset_new_queued, - self.memstore.set_new_queued][int(value)] - fun(key) - else: - logger.warning("Could not find a memstore referenced from this " - "MessageWrapper. The value for new will not be " - "propagated") - - new = property(_get_new, _set_new, - doc="The `new` flag for this MessageWrapper") - - def _get_dirty(self): - """ - Get the value for the `dirty` flag. - - :rtype: bool - """ - return self._dirty - - def _set_dirty(self, value=True): - """ - Set the value for the `dirty` flag, and propagate it - to the memory store if any. - - :param value: the value to set - :type value: bool - """ - self._dirty = value - if self.memstore: - mbox = self.fdoc.content.get('mbox', None) - uid = self.fdoc.content.get('uid', None) - if not mbox or not uid: - logger.warning("Malformed fdoc") - return - key = mbox, uid - fun = [self.memstore.unset_dirty_queued, - self.memstore.set_dirty_queued][int(value)] - fun(key) - else: - logger.warning("Could not find a memstore referenced from this " - "MessageWrapper. The value for new will not be " - "propagated") - - dirty = property(_get_dirty, _set_dirty) - - # IMessageContainer - - @property - def fdoc(self): - """ - Return a MessagePartDoc wrapping around a weak reference to - the flags-document in this MemoryStore, if any. - - :rtype: MessagePartDoc - """ - _fdoc = self._dict.get(self.FDOC, None) - if _fdoc: - content_ref = weakref.proxy(_fdoc) - else: - logger.warning("NO FDOC!!!") - content_ref = {} - - return MessagePartDoc(new=self.new, dirty=self.dirty, - store=self._storetype, - part=MessagePartType.fdoc, - content=content_ref, - doc_id=self._dict[self.DOCS_ID].get( - self.FDOC, None)) - - @property - def hdoc(self): - """ - Return a MessagePartDoc wrapping around a weak reference to - the headers-document in this MemoryStore, if any. - - :rtype: MessagePartDoc - """ - _hdoc = self._dict.get(self.HDOC, None) - if _hdoc: - content_ref = weakref.proxy(_hdoc) - else: - content_ref = {} - return MessagePartDoc(new=self.new, dirty=self.dirty, - store=self._storetype, - part=MessagePartType.hdoc, - content=content_ref, - doc_id=self._dict[self.DOCS_ID].get( - self.HDOC, None)) - - @property - def cdocs(self): - """ - Return a weak reference to a zero-indexed dict containing - the content-documents, or an empty dict if none found. - If you want access to the MessagePartDoc for the individual - parts, use the generator returned by `walk` instead. - - :rtype: dict - """ - _cdocs = self._dict.get(self.CDOCS, None) - if _cdocs: - return weakref.proxy(_cdocs) - else: - return {} - - def walk(self): - """ - Generator that iterates through all the parts, returning - MessagePartDoc. Used for writing to SoledadStore. - - :rtype: generator - """ - if self._dirty: - try: - mbox = self.fdoc.content[fields.MBOX_KEY] - uid = self.fdoc.content[fields.UID_KEY] - docid_dict = self._dict[self.DOCS_ID] - docid_dict[self.FDOC] = self.memstore.get_docid_for_fdoc( - mbox, uid) - except Exception as exc: - logger.debug("Error while walking message...") - logger.exception(exc) - - if not empty(self.fdoc.content) and 'uid' in self.fdoc.content: - yield self.fdoc - if not empty(self.hdoc.content): - yield self.hdoc - for cdoc in self.cdocs.values(): - if not empty(cdoc): - content_ref = weakref.proxy(cdoc) - yield MessagePartDoc(new=self.new, dirty=self.dirty, - store=self._storetype, - part=MessagePartType.cdoc, - content=content_ref, - doc_id=None) - - # i/o - - def as_dict(self): - """ - Return a dict representation of the parts contained. - - :rtype: dict - """ - return self._dict - - def from_dict(self, msg_dict): - """ - Populate MessageWrapper parts from a dictionary. - It expects the same format that we use in a - MessageWrapper. - - - :param msg_dict: a dictionary containing the parts to populate - the MessageWrapper from - :type msg_dict: dict - """ - fdoc, hdoc, cdocs = map( - lambda part: msg_dict.get(part, None), - [self.FDOC, self.HDOC, self.CDOCS]) - - for t, doc in ((self.FDOC, fdoc), (self.HDOC, hdoc), - (self.CDOCS, cdocs)): - self._dict[t] = ReferenciableDict(doc) if doc else None - - -class MessagePart(object): - """ - IMessagePart implementor, to be passed to several methods - of the IMAP4Server. - It takes a subpart message and is able to find - the inner parts. - - See the interface documentation. - """ - - implements(imap4.IMessagePart) - - def __init__(self, soledad, part_map): - """ - Initializes the MessagePart. - - :param soledad: Soledad instance. - :type soledad: Soledad - :param part_map: a dictionary containing the parts map for this - message - :type part_map: dict - """ - # TODO - # It would be good to pass the uid/mailbox also - # for references while debugging. - - # We have a problem on bulk moves, and is - # that when the fetch on the new mailbox is done - # the parts maybe are not complete. - # So we should be able to fail with empty - # docs until we solve that. The ideal would be - # to gather the results of the deferred operations - # to signal the operation is complete. - #leap_assert(part_map, "part map dict cannot be null") - - self._soledad = soledad - self._pmap = part_map - - def getSize(self): - """ - Return the total size, in octets, of this message part. - - :return: size of the message, in octets - :rtype: int - """ - if empty(self._pmap): - return 0 - size = self._pmap.get('size', None) - if size is None: - logger.error("Message part cannot find size in the partmap") - size = 0 - return size - - def getBodyFile(self): - """ - Retrieve a file object containing only the body of this message. - - :return: file-like object opened for reading - :rtype: StringIO - """ - fd = StringIO.StringIO() - if not empty(self._pmap): - multi = self._pmap.get('multi') - if not multi: - phash = self._pmap.get("phash", None) - else: - pmap = self._pmap.get('part_map') - first_part = pmap.get('1', None) - if not empty(first_part): - phash = first_part['phash'] - else: - phash = None - - if phash is None: - logger.warning("Could not find phash for this subpart!") - payload = "" - else: - payload = self._get_payload_from_document_memoized(phash) - if empty(payload): - payload = self._get_payload_from_document(phash) - - else: - logger.warning("Message with no part_map!") - payload = "" - - if payload: - content_type = self._get_ctype_from_document(phash) - charset = find_charset(content_type) - if charset is None: - charset = self._get_charset(payload) - try: - if isinstance(payload, unicode): - payload = payload.encode(charset) - except UnicodeError as exc: - logger.error( - "Unicode error, using 'replace'. {0!r}".format(exc)) - payload = payload.encode(charset, 'replace') - - fd.write(payload) - fd.seek(0) - return fd - - # TODO should memory-bound this memoize!!! - @memoized_method - def _get_payload_from_document_memoized(self, phash): - """ - Memoized method call around the regular method, to be able - to call the non-memoized method in case we got a None. - - :param phash: the payload hash to retrieve by. - :type phash: str or unicode - :rtype: str or unicode or None - """ - return self._get_payload_from_document(phash) - - def _get_payload_from_document(self, phash): - """ - Return the message payload from the content document. - - :param phash: the payload hash to retrieve by. - :type phash: str or unicode - :rtype: str or unicode or None - """ - cdocs = self._soledad.get_from_index( - fields.TYPE_P_HASH_IDX, - fields.TYPE_CONTENT_VAL, str(phash)) - - cdoc = first(cdocs) - if cdoc is None: - logger.warning( - "Could not find the content doc " - "for phash %s" % (phash,)) - payload = "" - else: - payload = cdoc.content.get(fields.RAW_KEY, "") - return payload - - # TODO should memory-bound this memoize!!! - @memoized_method - def _get_ctype_from_document(self, phash): - """ - Reeturn the content-type from the content document. - - :param phash: the payload hash to retrieve by. - :type phash: str or unicode - :rtype: str or unicode - """ - cdocs = self._soledad.get_from_index( - fields.TYPE_P_HASH_IDX, - fields.TYPE_CONTENT_VAL, str(phash)) - - cdoc = first(cdocs) - if not cdoc: - logger.warning( - "Could not find the content doc " - "for phash %s" % (phash,)) - ctype = cdoc.content.get('ctype', "") - return ctype - - @memoized_method - def _get_charset(self, stuff): - # TODO put in a common class with LeapMessage - """ - Gets (guesses?) the charset of a payload. - - :param stuff: the stuff to guess about. - :type stuff: str or unicode - :return: charset - :rtype: unicode - """ - # XXX existential doubt 2. shouldn't we make the scope - # of the decorator somewhat more persistent? - # ah! yes! and put memory bounds. - return get_email_charset(stuff) - - def getHeaders(self, negate, *names): - """ - Retrieve a group of message headers. - - :param names: The names of the headers to retrieve or omit. - :type names: tuple of str - - :param negate: If True, indicates that the headers listed in names - should be omitted from the return value, rather - than included. - :type negate: bool - - :return: A mapping of header field names to header field values - :rtype: dict - """ - # XXX refactor together with MessagePart method - if not self._pmap: - logger.warning("No pmap in Subpart!") - return {} - headers = dict(self._pmap.get("headers", [])) - - names = map(lambda s: s.upper(), names) - if negate: - cond = lambda key: key.upper() not in names - else: - cond = lambda key: key.upper() in names - - # default to most likely standard - charset = find_charset(headers, "utf-8") - headers2 = dict() - for key, value in headers.items(): - # twisted imap server expects *some* headers to be lowercase - # We could use a CaseInsensitiveDict here... - if key.lower() == "content-type": - key = key.lower() - - if not isinstance(key, str): - key = key.encode(charset, 'replace') - if not isinstance(value, str): - value = value.encode(charset, 'replace') - - # filter original dict by negate-condition - if cond(key): - headers2[key] = value - return headers2 - - def isMultipart(self): - """ - Return True if this message is multipart. - """ - if empty(self._pmap): - logger.warning("Could not get part map!") - return False - multi = self._pmap.get("multi", False) - return multi - - def getSubPart(self, part): - """ - Retrieve a MIME submessage - - :type part: C{int} - :param part: The number of the part to retrieve, indexed from 0. - :raise IndexError: Raised if the specified part does not exist. - :raise TypeError: Raised if this message is not multipart. - :rtype: Any object implementing C{IMessagePart}. - :return: The specified sub-part. - """ - if not self.isMultipart(): - raise TypeError - - sub_pmap = self._pmap.get("part_map", {}) - try: - part_map = sub_pmap[str(part + 1)] - except KeyError: - logger.debug("getSubpart for %s: KeyError" % (part,)) - raise IndexError - - # XXX check for validity - return MessagePart(self._soledad, part_map) diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index d47c8eb..7e0f973 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# messages.py -# Copyright (C) 2013, 2014 LEAP +# imap/messages.py +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,85 +15,41 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -LeapMessage and MessageCollection. +IMAPMessage and IMAPMessageCollection. """ -import copy import logging -import threading -import StringIO - -from collections import defaultdict -from functools import partial - +# import StringIO from twisted.mail import imap4 -from twisted.internet import reactor from zope.interface import implements -from zope.proxy import sameProxiedObjects from leap.common.check import leap_assert, leap_assert_type from leap.common.decorators import memoized_method from leap.common.mail import get_email_charset -from leap.mail.adaptors import soledad_indexes as indexes -from leap.mail.constants import INBOX_NAME -from leap.mail.utils import find_charset, empty -from leap.mail.imap.index import IndexedDB -from leap.mail.imap.fields import fields, WithMsgFields -from leap.mail.imap.messageparts import MessagePart, MessagePartDoc -from leap.mail.imap.parser import MBoxParser - -logger = logging.getLogger(__name__) - -# TODO ------------------------------------------------------------ -# [ ] Add ref to incoming message during add_msg -# [ ] Add linked-from info. -# * Need a new type of documents: linkage info. -# * HDOCS are linked from FDOCs (ref to chash) -# * CDOCS are linked from HDOCS (ref to chash) +from leap.mail.utils import find_charset -# [ ] Delete incoming mail only after successful write! -# [ ] Remove UID from syncable db. Store only those indexes locally. +from leap.mail.imap.messageparts import MessagePart +# from leap.mail.imap.messagepargs import MessagePartDoc +logger = logging.getLogger(__name__) -def try_unique_query(curried): - """ - Try to execute a query that is expected to have a - single outcome, and log a warning if more than one document found. - - :param curried: a curried function - :type curried: callable - """ - # XXX FIXME ---------- convert to deferreds - leap_assert(callable(curried), "A callable is expected") - try: - query = curried() - if query: - if len(query) > 1: - # TODO we could take action, like trigger a background - # process to kill dupes. - name = getattr(curried, 'expected', 'doc') - logger.warning( - "More than one %s found for this mbox, " - "we got a duplicate!!" % (name,)) - return query.pop() - else: - return None - except Exception as exc: - logger.exception("Unhandled error %r" % exc) - +# TODO ------------------------------------------------------------ -# FIXME remove-me -#fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock())) +# [ ] Add ref to incoming message during add_msg. +# [ ] Delete incoming mail only after successful write. -class IMAPMessage(fields, MBoxParser): +class IMAPMessage(object): """ The main representation of a message. """ implements(imap4.IMessage) - def __init__(self, soledad, uid, mbox): + # TODO ---- see what should we pass here instead + # where's UID added to the message? + # def __init__(self, soledad, uid, mbox): + def __init__(self, message, collection): """ Initializes a LeapMessage. @@ -103,81 +59,14 @@ class IMAPMessage(fields, MBoxParser): :type uid: int or basestring :param mbox: the mbox this message belongs to :type mbox: str or unicode - :param collection: a reference to the parent collection object - :type collection: MessageCollection - :param container: a IMessageContainer implementor instance - :type container: IMessageContainer """ - self._soledad = soledad - self._uid = int(uid) if uid is not None else None - self._mbox = self._parse_mailbox_name(mbox) - - self.__chash = None - self.__bdoc = None - - # TODO collection and container are deprecated. - - # TODO move to adaptor - - #@property - #def fdoc(self): - #""" - #An accessor to the flags document. - #""" - #if all(map(bool, (self._uid, self._mbox))): - #fdoc = None - #if self._container is not None: - #fdoc = self._container.fdoc - #if not fdoc: - #fdoc = self._get_flags_doc() - #if fdoc: - #fdoc_content = fdoc.content - #self.__chash = fdoc_content.get( - #fields.CONTENT_HASH_KEY, None) - #return fdoc -# - #@property - #def hdoc(self): - #""" - #An accessor to the headers document. - #""" - #container = self._container - #if container is not None: - #hdoc = self._container.hdoc - #if hdoc and not empty(hdoc.content): - #return hdoc - #hdoc = self._get_headers_doc() -# - #if container and not empty(hdoc.content): - # mem-cache it - #hdoc_content = hdoc.content - #chash = hdoc_content.get(fields.CONTENT_HASH_KEY) - #hdocs = {chash: hdoc_content} - #container.memstore.load_header_docs(hdocs) - #return hdoc -# - #@property - #def chash(self): - #""" - #An accessor to the content hash for this message. - #""" - #if not self.fdoc: - #return None - #if not self.__chash and self.fdoc: - #self.__chash = self.fdoc.content.get( - #fields.CONTENT_HASH_KEY, None) - #return self.__chash - - #@property - #def bdoc(self): - #""" - #An accessor to the body document. - #""" - #if not self.hdoc: - #return None - #if not self.__bdoc: - #self.__bdoc = self._get_body_doc() - #return self.__bdoc + #self._uid = int(uid) if uid is not None else None + #self._mbox = normalize_mailbox(mbox) + + self.message = message + + # TODO maybe not needed, see setFlags below + self.collection = collection # IMessage implementation @@ -188,12 +77,7 @@ class IMAPMessage(fields, MBoxParser): :return: uid for this message :rtype: int """ - # TODO ----> return lookup in local sqlcipher table. - return self._uid - - # -------------------------------------------------------------- - # TODO -- from here on, all the methods should be proxied to the - # instance of leap.mail.mail.Message + return self.message.get_uid() def getFlags(self): """ @@ -202,24 +86,14 @@ class IMAPMessage(fields, MBoxParser): :return: The flags, represented as strings :rtype: tuple """ - uid = self._uid - - flags = set([]) - fdoc = self.fdoc - if fdoc: - flags = set(fdoc.content.get(self.FLAGS_KEY, None)) + return self.message.get_flags() - msgcol = self._collection + # setFlags not in the interface spec but we use it with store command. - # We treat the recent flag specially: gotten from - # a mailbox-level document. - if msgcol and uid in msgcol.recent_flags: - flags.add(fields.RECENT_FLAG) - if flags: - flags = map(str, flags) - return tuple(flags) + # XXX if we can move it to a collection method, we don't need to pass + # collection to the IMAPMessage - # setFlags not in the interface spec but we use it with store command. + # lookup method? IMAPMailbox? def setFlags(self, flags, mode): """ @@ -231,32 +105,11 @@ class IMAPMessage(fields, MBoxParser): :type mode: int """ leap_assert(isinstance(flags, tuple), "flags need to be a tuple") - mbox, uid = self._mbox, self._uid - - APPEND = 1 - REMOVE = -1 - SET = 0 - - doc = self.fdoc - if not doc: - logger.warning( - "Could not find FDOC for %r:%s while setting flags!" % - (mbox, uid)) - return - current = doc.content[self.FLAGS_KEY] - if mode == APPEND: - newflags = tuple(set(tuple(current) + flags)) - elif mode == REMOVE: - newflags = tuple(set(current).difference(set(flags))) - elif mode == SET: - newflags = flags - new_fdoc = { - self.FLAGS_KEY: newflags, - self.SEEN_KEY: self.SEEN_FLAG in newflags, - self.DEL_KEY: self.DELETED_FLAG in newflags} - self._collection.memstore.update_flags(mbox, uid, new_fdoc) - - return map(str, newflags) + # XXX + # return new flags + # map to str + #self.message.set_flags(flags, mode) + self.collection.update_flags(self.message, flags, mode) def getInternalDate(self): """ @@ -273,8 +126,7 @@ class IMAPMessage(fields, MBoxParser): :return: An RFC822-formatted date string. :rtype: str """ - date = self.hdoc.content.get(fields.DATE_KEY, '') - return date + return self.message.get_internal_date() # # IMessagePart @@ -290,42 +142,40 @@ class IMAPMessage(fields, MBoxParser): :return: file-like object opened for reading :rtype: StringIO """ - def write_fd(body): - fd.write(body) - fd.seek(0) - return fd - + #def write_fd(body): + #fd.write(body) + #fd.seek(0) + #return fd +# # TODO refactor with getBodyFile in MessagePart - - fd = StringIO.StringIO() - - if self.bdoc is not None: - bdoc_content = self.bdoc.content - if empty(bdoc_content): - logger.warning("No BDOC content found for message!!!") - return write_fd("") - - body = bdoc_content.get(self.RAW_KEY, "") - content_type = bdoc_content.get('content-type', "") - charset = find_charset(content_type) - if charset is None: - charset = self._get_charset(body) - try: - if isinstance(body, unicode): - body = body.encode(charset) - except UnicodeError as exc: - logger.error( - "Unicode error, using 'replace'. {0!r}".format(exc)) - logger.debug("Attempted to encode with: %s" % charset) - body = body.encode(charset, 'replace') - finally: - return write_fd(body) - - # We are still returning funky characters from here. - else: - logger.warning("No BDOC found for message.") - return write_fd("") - +# + #fd = StringIO.StringIO() +# + #if self.bdoc is not None: + #bdoc_content = self.bdoc.content + #if empty(bdoc_content): + #logger.warning("No BDOC content found for message!!!") + #return write_fd("") +# + #body = bdoc_content.get(self.RAW_KEY, "") + #content_type = bdoc_content.get('content-type', "") + #charset = find_charset(content_type) + #if charset is None: + #charset = self._get_charset(body) + #try: + #if isinstance(body, unicode): + #body = body.encode(charset) + #except UnicodeError as exc: + #logger.error( + #"Unicode error, using 'replace'. {0!r}".format(exc)) + #logger.debug("Attempted to encode with: %s" % charset) + #body = body.encode(charset, 'replace') + #finally: + #return write_fd(body) + + return self.message.get_body_file() + + # TODO move to mail.mail @memoized_method def _get_charset(self, stuff): """ @@ -337,7 +187,7 @@ class IMAPMessage(fields, MBoxParser): """ # XXX shouldn't we make the scope # of the decorator somewhat more persistent? - # ah! yes! and put memory bounds. + # and put memory bounds. return get_email_charset(stuff) def getSize(self): @@ -347,17 +197,11 @@ class IMAPMessage(fields, MBoxParser): :return: size of the message, in octets :rtype: int """ - size = None - if self.fdoc is not None: - fdoc_content = self.fdoc.content - size = fdoc_content.get(self.SIZE_KEY, False) - else: - logger.warning("No FLAGS doc for %s:%s" % (self._mbox, - self._uid)) - #if not size: - # XXX fallback, should remove when all migrated. - #size = self.getBodyFile().len - return size + #size = None + #fdoc_content = self.fdoc.content + #size = fdoc_content.get(self.SIZE_KEY, False) + #return size + return self.message.get_size() def getHeaders(self, negate, *names): """ @@ -374,10 +218,10 @@ class IMAPMessage(fields, MBoxParser): :return: A mapping of header field names to header field values :rtype: dict """ - # TODO split in smaller methods + # TODO split in smaller methods -- format_headers()? # XXX refactor together with MessagePart method - headers = self._get_headers() + headers = self.message.get_headers() # XXX keep this in the imap imessage implementation, # because the server impl. expects content-type to be present. @@ -417,34 +261,15 @@ class IMAPMessage(fields, MBoxParser): headers2[key] = value return headers2 - def _get_headers(self): - """ - Return the headers dict for this message. - """ - if self.hdoc is not None: - hdoc_content = self.hdoc.content - headers = hdoc_content.get(self.HEADERS_KEY, {}) - return headers - - else: - logger.warning( - "No HEADERS doc for msg %s:%s" % ( - self._mbox, - self._uid)) - def isMultipart(self): """ Return True if this message is multipart. """ - if self.fdoc: - fdoc_content = self.fdoc.content - is_multipart = fdoc_content.get(self.MULTIPART_KEY, False) - return is_multipart - else: - logger.warning( - "No FLAGS doc for msg %s:%s" % ( - self._mbox, - self._uid)) + #fdoc_content = self.fdoc.content + #is_multipart = fdoc_content.get(self.MULTIPART_KEY, False) + #return is_multipart + + return self.message.fdoc.is_multi def getSubPart(self, part): """ @@ -463,12 +288,16 @@ class IMAPMessage(fields, MBoxParser): pmap_dict = self._get_part_from_parts_map(part + 1) except KeyError: raise IndexError + + # TODO move access to adaptor ---- return MessagePart(self._soledad, pmap_dict) # # accessors # + # FIXME + # -- move to wrapper/adaptor def _get_part_from_parts_map(self, part): """ Get a part map from the headers doc @@ -476,100 +305,44 @@ class IMAPMessage(fields, MBoxParser): :raises: KeyError if key does not exist :rtype: dict """ - if not self.hdoc: - logger.warning("Tried to get part but no HDOC found!") - return None - - hdoc_content = self.hdoc.content - pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {}) + raise NotImplementedError() + #hdoc_content = self.hdoc.content + #pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {}) +# # remember, lads, soledad is using strings in its keys, # not integers! - return pmap[str(part)] + #return pmap[str(part)] - # XXX moved to memory store - # move the rest too. ------------------------------------------ - def _get_flags_doc(self): - """ - Return the document that keeps the flags for this - message. - """ - def get_first_if_any(docs): - result = first(docs) - return result if result else {} - - d = self._soledad.get_from_index( - fields.TYPE_MBOX_UID_IDX, - fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) - d.addCallback(get_first_if_any) - return d - - # TODO move to soledadstore instead of accessing soledad directly - def _get_headers_doc(self): - """ - Return the document that keeps the headers for this - message. - """ - d = self._soledad.get_from_index( - fields.TYPE_C_HASH_IDX, - fields.TYPE_HEADERS_VAL, str(self.chash)) - d.addCallback(lambda docs: first(docs)) - return d - - # TODO move to soledadstore instead of accessing soledad directly + # TODO move to wrapper/adaptor def _get_body_doc(self): """ Return the document that keeps the body for this message. """ - # XXX FIXME --- this might need a maybedeferred - # on the receiving side... - hdoc_content = self.hdoc.content - body_phash = hdoc_content.get( - fields.BODY_KEY, None) - if not body_phash: - logger.warning("No body phash for this document!") - return None - - # XXX get from memstore too... - # if memstore: memstore.get_phrash - # memstore should keep a dict with weakrefs to the - # phash doc... - - if self._container is not None: - bdoc = self._container.memstore.get_cdoc_from_phash(body_phash) - if not empty(bdoc) and not empty(bdoc.content): - return bdoc - + # FIXME + # -- just get the body and retrieve the cdoc P- + #hdoc_content = self.hdoc.content + #body_phash = hdoc_content.get( + #fields.BODY_KEY, None) + #if not body_phash: + #logger.warning("No body phash for this document!") + #return None +# + #if self._container is not None: + #bdoc = self._container.memstore.get_cdoc_from_phash(body_phash) + #if not empty(bdoc) and not empty(bdoc.content): + #return bdoc +# # no memstore, or no body doc found there - d = self._soledad.get_from_index( - fields.TYPE_P_HASH_IDX, - fields.TYPE_CONTENT_VAL, str(body_phash)) - d.addCallback(lambda docs: first(docs)) - return d - - def __getitem__(self, key): - """ - Return an item from the content of the flags document, - for convenience. - - :param key: The key - :type key: str - - :return: The content value indexed by C{key} or None - :rtype: str - """ - return self.fdoc.content.get(key, None) - - def does_exist(self): - """ - Return True if there is actually a flags document for this - UID and mbox. - """ - return not empty(self.fdoc) + #d = self._soledad.get_from_index( + #fields.TYPE_P_HASH_IDX, + #fields.TYPE_CONTENT_VAL, str(body_phash)) + #d.addCallback(lambda docs: first(docs)) + #return d -class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): +class IMAPMessageCollection(object): """ A collection of messages, surprisingly. @@ -578,9 +351,15 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): database. """ - # XXX this should be able to produce a MessageSet methinks - # could validate these kinds of objects turning them - # into a template for the class. + messageklass = IMAPMessage + + # TODO + # [ ] Add RECENT flags docs to mailbox-doc attributes (list-of-uids) + # [ ] move Query for all the headers documents to Collection + + # TODO this should be able to produce a MessageSet methinks + # TODO --- reimplement, review and prune documentation below. + FLAGS_DOC = "FLAGS" HEADERS_DOC = "HEADERS" CONTENT_DOC = "CONTENT" @@ -604,145 +383,40 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): """ HDOCS_SET_DOC = "HDOCS_SET" - templates = { - - # Mailbox Level - - RECENT_DOC: { - "type": indexes.RECENT, - "mbox": INBOX_NAME, - fields.RECENTFLAGS_KEY: [], - }, - - HDOCS_SET_DOC: { - "type": indexes.HDOCS_SET, - "mbox": INBOX_NAME, - fields.HDOCS_SET_KEY: [], - } - - - } - - # Different locks for wrapping both the u1db document getting/setting - # and the property getting/settting in an atomic operation. - - # TODO --- deprecate ! --- use SoledadDocumentWrapper + locks - _rdoc_lock = defaultdict(lambda: threading.Lock()) - _rdoc_write_lock = defaultdict(lambda: threading.Lock()) - _rdoc_read_lock = defaultdict(lambda: threading.Lock()) - _rdoc_property_lock = defaultdict(lambda: threading.Lock()) - - _initialized = {} - - def __init__(self, mbox=None, soledad=None, memstore=None): - """ - Constructor for MessageCollection. - - On initialization, we ensure that we have a document for - storing the recent flags. The nature of this flag make us wanting - to store the set of the UIDs with this flag at the level of the - MessageCollection for each mailbox, instead of treating them - as a property of each message. - - We are passed an instance of MemoryStore, the same for the - SoledadBackedAccount, that we use as a read cache and a buffer - for writes. - - :param mbox: the name of the mailbox. It is the name - with which we filter the query over the - messages database. - :type mbox: str - :param soledad: Soledad database - :type soledad: Soledad instance - :param memstore: a MemoryStore instance - :type memstore: MemoryStore + def __init__(self, collection): """ - leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(mbox.strip() != "", "mbox cannot be blank space") - leap_assert(isinstance(mbox, (str, unicode)), - "mbox needs to be a string") - leap_assert(soledad, "Need a soledad instance to initialize") + Constructor for IMAPMessageCollection. - # okay, all in order, keep going... - - self.mbox = self._parse_mailbox_name(mbox) - - # XXX get a SoledadStore passed instead - self._soledad = soledad - self.memstore = memstore - - self.__rflags = None - - if not self._initialized.get(mbox, False): - try: - self.initialize_db() - # ensure that we have a recent-flags doc - self._get_or_create_rdoc() - except Exception: - logger.debug("Error initializing %r" % (mbox,)) - else: - self._initialized[mbox] = True - - def _get_empty_doc(self, _type=FLAGS_DOC): - """ - Returns an empty doc for storing different message parts. - Defaults to returning a template for a flags document. - :return: a dict with the template - :rtype: dict - """ - if _type not in self.templates.keys(): - raise TypeError("Improper type passed to _get_empty_doc") - return copy.deepcopy(self.templates[_type]) - - def _get_or_create_rdoc(self): - """ - Try to retrieve the recent-flags doc for this MessageCollection, - and create one if not found. + :param collection: an instance of a MessageCollection + :type collection: MessageCollection """ - # XXX should move this to memstore too - with self._rdoc_write_lock[self.mbox]: - rdoc = self._get_recent_doc_from_soledad() - if rdoc is None: - rdoc = self._get_empty_doc(self.RECENT_DOC) - if self.mbox != fields.INBOX_VAL: - rdoc[fields.MBOX_KEY] = self.mbox - self._soledad.create_doc(rdoc) - - # -------------------------------------------------------------------- + leap_assert( + collection.is_mailbox_collection(), + "Need a mailbox name to initialize") + mbox_name = collection.mbox_name + leap_assert(mbox_name.strip() != "", "mbox cannot be blank space") + leap_assert(isinstance(mbox_name, (str, unicode)), + "mbox needs to be a string") + self.collection = collection - # ----------------------------------------------------------------------- + # XXX this has to be done in IMAPAccount + # (Where the collection must be instantiated and passed to us) + # self.mbox = normalize_mailbox(mbox) - def _fdoc_already_exists(self, chash): + @property + def mbox_name(self): """ - Check whether we can find a flags doc for this mailbox with the - given content-hash. It enforces that we can only have the same maessage - listed once for a a given mailbox. - - :param chash: the content-hash to check about. - :type chash: basestring - :return: False, if it does not exist, or UID. + Return the string that identifies this mailbox. """ - exist = False - exist = self.memstore.get_fdoc_from_chash(chash, self.mbox) + return self.collection.mbox_name - if not exist: - exist = self._get_fdoc_from_chash(chash) - if exist and exist.content is not None: - return exist.content.get(fields.UID_KEY, "unknown-uid") - else: - return False - - def add_msg(self, raw, subject=None, flags=None, date=None, - notify_on_disk=False): + def add_msg(self, raw, flags=None, date=None): """ Creates a new message document. :param raw: the raw message :type raw: str - :param subject: subject of the message. - :type subject: str - :param flags: flags :type flags: list @@ -756,212 +430,30 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): if flags is None: flags = tuple() leap_assert_type(flags, tuple) + return self.collection.add_msg(raw, flags, date) - # TODO ---- proxy to MessageCollection addMessage - - #observer = defer.Deferred() - #d = self._do_parse(raw) - #d.addCallback(lambda result: reactor.callInThread( - #self._do_add_msg, result, flags, subject, date, - #notify_on_disk, observer)) - #return observer - - # TODO --------------------------------------------------- - # move this to leap.mail.adaptors.soledad - - def _do_add_msg(self, parse_result, flags, subject, - date, notify_on_disk, observer): - """ - """ - msg, parts, chash, size, multi = parse_result - - # XXX move to SoledadAdaptor write operation ... ??? - # check for uniqueness -------------------------------- - # Watch out! We're reserving a UID right after this! - existing_uid = self._fdoc_already_exists(chash) - if existing_uid: - msg = self.get_msg_by_uid(existing_uid) - reactor.callFromThread(observer.callback, existing_uid) - msg.setFlags((fields.DELETED_FLAG,), -1) - return - - # TODO move UID autoincrement to MessageCollection.addMessage(mailbox) - # TODO S2 -- get FUCKING UID from autoincremental table - #uid = self.memstore.increment_last_soledad_uid(self.mbox) - #self.set_recent_flag(uid) - - - # ------------------------------------------------------------ - - # - # getters: specific queries - # - - # recent flags - - def _get_recent_flags(self): - """ - An accessor for the recent-flags set for this mailbox. + def get_msg_by_uid(self, uid, absolute=True): """ - # XXX check if we should remove this - if self.__rflags is not None: - return self.__rflags - - if self.memstore is not None: - with self._rdoc_lock[self.mbox]: - rflags = self.memstore.get_recent_flags(self.mbox) - if not rflags: - # not loaded in the memory store yet. - # let's fetch them from soledad... - rdoc = self._get_recent_doc_from_soledad() - if rdoc is None: - return set([]) - rflags = set(rdoc.content.get( - fields.RECENTFLAGS_KEY, [])) - # ...and cache them now. - self.memstore.load_recent_flags( - self.mbox, - {'doc_id': rdoc.doc_id, 'set': rflags}) - return rflags - - def _set_recent_flags(self, value): - """ - Setter for the recent-flags set for this mailbox. - """ - if self.memstore is not None: - self.memstore.set_recent_flags(self.mbox, value) - - recent_flags = property( - _get_recent_flags, _set_recent_flags, - doc="Set of UIDs with the recent flag for this mailbox.") - - def _get_recent_doc_from_soledad(self): - """ - Get recent-flags document from Soledad for this mailbox. - :rtype: SoledadDocument or None - """ - # FIXME ----- use deferreds. - curried = partial( - self._soledad.get_from_index, - fields.TYPE_MBOX_IDX, - fields.TYPE_RECENT_VAL, self.mbox) - curried.expected = "rdoc" - with self._rdoc_read_lock[self.mbox]: - return try_unique_query(curried) - - # Property-set modification (protected by a different - # lock to give atomicity to the read/write operation) - - def unset_recent_flags(self, uids): - """ - Unset Recent flag for a sequence of uids. - - :param uids: the uids to unset - :type uid: sequence - """ - # FIXME ----- use deferreds. - with self._rdoc_property_lock[self.mbox]: - self.recent_flags.difference_update( - set(uids)) - - # Individual flags operations - - def unset_recent_flag(self, uid): - """ - Unset Recent flag for a given uid. - - :param uid: the uid to unset - :type uid: int - """ - # FIXME ----- use deferreds. - with self._rdoc_property_lock[self.mbox]: - self.recent_flags.difference_update( - set([uid])) - - def set_recent_flag(self, uid): - """ - Set Recent flag for a given uid. + Retrieves a IMAPMessage by UID. + This is used primarity in the Mailbox fetch and store methods. - :param uid: the uid to set + :param uid: the message uid to query by :type uid: int - """ - # FIXME ----- use deferreds. - with self._rdoc_property_lock[self.mbox]: - self.recent_flags = self.recent_flags.union( - set([uid])) - - # individual doc getters, message layer. - def _get_fdoc_from_chash(self, chash): + :rtype: IMAPMessage """ - Return a flags document for this mailbox with a given chash. + def make_imap_msg(msg): + kls = self.messageklass + # TODO --- remove ref to collection + return kls(msg, self.collection) - :return: A SoledadDocument containing the Flags Document, or None if - the query failed. - :rtype: SoledadDocument or None. - """ - # USED from: - # [ ] duplicated fdoc detection - # [ ] _get_uid_from_msgidCb - - # FIXME ----- use deferreds. - curried = partial( - self._soledad.get_from_index, - fields.TYPE_MBOX_C_HASH_IDX, - fields.TYPE_FLAGS_VAL, self.mbox, chash) - curried.expected = "fdoc" - fdoc = try_unique_query(curried) - if fdoc is not None: - return fdoc - else: - # probably this should be the other way round, - # ie, try fist on memstore... - cf = self.memstore._chash_fdoc_store - fdoc = cf[chash][self.mbox] - # hey, I just needed to wrap fdoc thing into - # a "content" attribute, look a better way... - if not empty(fdoc): - return MessagePartDoc( - new=None, dirty=None, part=None, - store=None, doc_id=None, - content=fdoc) - - def _get_uid_from_msgidCb(self, msgid): - hdoc = None - curried = partial( - self._soledad.get_from_index, - fields.TYPE_MSGID_IDX, - fields.TYPE_HEADERS_VAL, msgid) - curried.expected = "hdoc" - hdoc = try_unique_query(curried) - - # XXX this is only a quick hack to avoid regression - # on the "multiple copies of the draft" issue, but - # this is currently broken since it's not efficient to - # look for this. Should lookup better. - # FIXME! - - if hdoc is not None: - hdoc_dict = hdoc.content + d = self.collection.get_msg_by_uid(uid, absolute=absolute) + d.addCalback(make_imap_msg) + return d - else: - hdocstore = self.memstore._hdoc_store - match = [x for _, x in hdocstore.items() if x['msgid'] == msgid] - hdoc_dict = first(match) - - if hdoc_dict is None: - logger.warning("Could not find hdoc for msgid %s" - % (msgid,)) - return None - msg_chash = hdoc_dict.get(fields.CONTENT_HASH_KEY) - - fdoc = self._get_fdoc_from_chash(msg_chash) - if not fdoc: - logger.warning("Could not find fdoc for msgid %s" - % (msgid,)) - return None - return fdoc.content.get(fields.UID_KEY, None) + # TODO -- move this to collection too + # Used for the Search (Drafts) queries? def _get_uid_from_msgid(self, msgid): """ Return a UID for a given message-id. @@ -972,15 +464,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :return: A UID, or None """ - # We need to wait a little bit, cause in some of the cases - # the query is received right after we've saved the document, - # and we cannot find it otherwise. This seems to be enough. - - # XXX do a deferLater instead ?? - # XXX is this working? return self._get_uid_from_msgidCb(msgid) - def set_flags(self, mbox, messages, flags, mode, observer): + # TODO handle deferreds + def set_flags(self, messages, flags, mode): """ Set flags for a sequence of messages. @@ -1000,142 +487,27 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): getmsg = self.get_msg_by_uid def set_flags(uid, flags, mode): - msg = getmsg(uid, mem_only=True, flags_only=True) + msg = getmsg(uid) if msg is not None: + # XXX IMAPMessage needs access to the collection + # to be able to set flags. Better if we make use + # of collection... here. return uid, msg.setFlags(flags, mode) setted_flags = [set_flags(uid, flags, mode) for uid in messages] result = dict(filter(None, setted_flags)) + # XXX return gatherResults or something + return result - # TODO -- remove - reactor.callFromThread(observer.callback, result) - - # getters: generic for a mailbox - - def get_msg_by_uid(self, uid, mem_only=False, flags_only=False): - """ - Retrieves a LeapMessage by UID. - This is used primarity in the Mailbox fetch and store methods. - - :param uid: the message uid to query by - :type uid: int - :param mem_only: a flag that indicates whether this Message should - pass a reference to soledad to retrieve missing pieces - or not. - :type mem_only: bool - :param flags_only: whether the message should carry only a reference - to the flags document. - :type flags_only: bool - - :return: A LeapMessage instance matching the query, - or None if not found. - :rtype: LeapMessage - """ - msg_container = self.memstore.get_message( - self.mbox, uid, flags_only=flags_only) - - if msg_container is not None: - if mem_only: - msg = IMAPMessage(None, uid, self.mbox, collection=self, - container=msg_container) - else: - # We pass a reference to soledad just to be able to retrieve - # missing parts that cannot be found in the container, like - # the content docs after a copy. - msg = IMAPMessage(self._soledad, uid, self.mbox, - collection=self, container=msg_container) - else: - msg = IMAPMessage(self._soledad, uid, self.mbox, collection=self) - - if not msg.does_exist(): - return None - return msg - - # FIXME --- used where ? --------------------------------------------- - #def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL): - #""" - #Get all documents for the selected mailbox of the - #passed type. By default, it returns the flag docs. -# - #If you want acess to the content, use __iter__ instead -# - #:return: a Deferred, that will fire with a list of u1db documents - #:rtype: Deferred (promise of list of SoledadDocument) - #""" - #if _type not in fields.__dict__.values(): - #raise TypeError("Wrong type passed to get_all_docs") -# - # FIXME ----- either raise or return a deferred wrapper. - #if sameProxiedObjects(self._soledad, None): - #logger.warning('Tried to get messages but soledad is None!') - #return [] -# - #def get_sorted_docs(docs): - #all_docs = [doc for doc in docs] - # inneficient, but first let's grok it and then - # let's worry about efficiency. - # XXX FIXINDEX -- should implement order by in soledad - # FIXME ---------------------------------------------- - #return sorted(all_docs, key=lambda item: item.content['uid']) -# - #d = self._soledad.get_from_index( - #fields.TYPE_MBOX_IDX, _type, self.mbox) - #d.addCallback(get_sorted_docs) - #return d - - def all_soledad_uid_iter(self): - """ - Return an iterator through the UIDs of all messages, sorted in - ascending order. - """ - # XXX FIXME ------ sorted??? - - def get_uids(docs): - return set([ - doc.content[self.UID_KEY] for doc in docs if not empty(doc)]) - - d = self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, fields.TYPE_FLAGS_VAL, self.mbox) - d.addCallback(get_uids) - return d - - def all_uid_iter(self): - """ - Return an iterator through the UIDs of all messages, from memory. + def count(self): """ - mem_uids = self.memstore.get_uids(self.mbox) - soledad_known_uids = self.memstore.get_soledad_known_uids( - self.mbox) - combined = tuple(set(mem_uids).union(soledad_known_uids)) - return combined + Return the count of messages for this mailbox. - def get_all_soledad_flag_docs(self): + :rtype: int """ - Return a dict with the content of all the flag documents - in soledad store for the given mbox. + return self.collection.count() - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: dict - """ - # XXX we really could return a reduced version with - # just {'uid': (flags-tuple,) since the prefetch is - # only oriented to get the flag tuples. - - def get_content(docs): - all_docs = [( - doc.content[self.UID_KEY], - dict(doc.content)) - for doc in docs - if not empty(doc.content)] - all_flags = dict(all_docs) - return all_flags - - d = self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - fields.TYPE_FLAGS_VAL, self.mbox) - d.addCallback(get_content) - return d + # headers query def all_headers(self): """ @@ -1144,15 +516,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :rtype: dict """ - return self.memstore.all_headers(self.mbox) - - def count(self): - """ - Return the count of messages for this mailbox. - - :rtype: int - """ - return self.memstore.count(self.mbox) + # Use self.collection.mbox_indexer + # and derive all the doc_ids for the hdocs + raise NotImplementedError() # unseen messages @@ -1164,7 +530,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :return: iterator through unseen message doc UIDs :rtype: iterable """ - return self.memstore.unseen_iter(self.mbox) + raise NotImplementedError() def count_unseen(self): """ @@ -1182,13 +548,12 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :returns: a list of LeapMessages :rtype: list """ - return [IMAPMessage(self._soledad, docid, self.mbox, collection=self) - for docid in self.unseen_iter()] + raise NotImplementedError() + #return [self.messageklass(self._soledad, doc_id, self.mbox) + #for doc_id in self.unseen_iter()] # recent messages - # XXX take it from memstore - # XXX Used somewhere? def count_recent(self): """ Count all messages with the `Recent` flag. @@ -1199,32 +564,22 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser): :returns: count :rtype: int """ - return len(self.recent_flags) + raise NotImplementedError() + + # magic def __len__(self): """ Returns the number of messages on this mailbox. - :rtype: int """ return self.count() - def __iter__(self): - """ - Returns an iterator over all messages. - - :returns: iterator of dicts with content for all messages. - :rtype: iterable - """ - return (IMAPMessage(self._soledad, docuid, self.mbox, collection=self) - for docuid in self.all_uid_iter()) - def __repr__(self): """ Representation string for this object. """ - return u"" % ( - self.mbox, self.count()) + return u"" % ( + self.mbox_name, self.count()) - # XXX should implement __eq__ also !!! - # use chash... + # TODO implement __iter__ ? diff --git a/src/leap/mail/imap/soledadstore.py b/src/leap/mail/imap/soledadstore.py deleted file mode 100644 index fc8ea55..0000000 --- a/src/leap/mail/imap/soledadstore.py +++ /dev/null @@ -1,617 +0,0 @@ -# -*- coding: utf-8 -*- -# soledadstore.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -A MessageStore that writes to Soledad. -""" -import logging -import threading - -from collections import defaultdict -from itertools import chain - -from u1db import errors as u1db_errors -from twisted.python import log -from zope.interface import implements - -from leap.common.check import leap_assert_type, leap_assert -from leap.mail.decorators import deferred_to_thread -from leap.mail.imap.messageparts import MessagePartType -from leap.mail.imap.messageparts import MessageWrapper -from leap.mail.imap.messageparts import RecentFlagsDoc -from leap.mail.imap.fields import fields -from leap.mail.imap.interfaces import IMessageStore -from leap.mail.messageflow import IMessageConsumer -from leap.mail.utils import first, empty, accumulator_queue - -logger = logging.getLogger(__name__) - - -class ContentDedup(object): - """ - Message deduplication. - - We do a query for the content hashes before writing to our beloved - sqlcipher backend of Soledad. This means, by now, that: - - 1. We will not store the same body/attachment twice, only the hash of it. - 2. We will not store the same message header twice, only the hash of it. - - The first case is useful if you are always receiving the same old memes - from unwary friends that still have not discovered that 4chan is the - generator of the internet. The second will save your day if you have - initiated session with the same account in two different machines. I also - wonder why would you do that, but let's respect each other choices, like - with the religious celebrations, and assume that one day we'll be able - to run Bitmask in completely free phones. Yes, I mean that, the whole GSM - Stack. - """ - # TODO refactor using unique_query - - def _header_does_exist(self, doc): - """ - Check whether we already have a header document for this - content hash in our database. - - :param doc: tentative header for document - :type doc: dict - :returns: True if it exists, False otherwise. - """ - if not doc: - return False - chash = doc[fields.CONTENT_HASH_KEY] - header_docs = self._soledad.get_from_index( - fields.TYPE_C_HASH_IDX, - fields.TYPE_HEADERS_VAL, str(chash)) - if not header_docs: - return False - - # FIXME enable only to debug this problem. - #if len(header_docs) != 1: - #logger.warning("Found more than one copy of chash %s!" - #% (chash,)) - - #logger.debug("Found header doc with that hash! Skipping save!") - return True - - def _content_does_exist(self, doc): - """ - Check whether we already have a content document for a payload - with this hash in our database. - - :param doc: tentative content for document - :type doc: dict - :returns: True if it exists, False otherwise. - """ - if not doc: - return False - phash = doc[fields.PAYLOAD_HASH_KEY] - attach_docs = self._soledad.get_from_index( - fields.TYPE_P_HASH_IDX, - fields.TYPE_CONTENT_VAL, str(phash)) - if not attach_docs: - return False - - # FIXME enable only to debug this problem - #if len(attach_docs) != 1: - #logger.warning("Found more than one copy of phash %s!" - #% (phash,)) - #logger.debug("Found attachment doc with that hash! Skipping save!") - return True - - -class MsgWriteError(Exception): - """ - Raised if any exception is found while saving message parts. - """ - pass - - -""" -A lock per document. -""" -# TODO should bound the space of this!!! -# http://stackoverflow.com/a/2437645/1157664 -# Setting this to twice the number of threads in the threadpool -# should be safe. - -put_locks = defaultdict(lambda: threading.Lock()) -mbox_doc_locks = defaultdict(lambda: threading.Lock()) - - -class SoledadStore(ContentDedup): - """ - This will create docs in the local Soledad database. - """ - _remove_lock = threading.Lock() - - implements(IMessageConsumer, IMessageStore) - - def __init__(self, soledad): - """ - Initialize the permanent store that writes to Soledad database. - - :param soledad: the soledad instance - :type soledad: Soledad - """ - from twisted.internet import reactor - self.reactor = reactor - - self._soledad = soledad - - self._CREATE_DOC_FUN = self._soledad.create_doc - self._PUT_DOC_FUN = self._soledad.put_doc - self._GET_DOC_FUN = self._soledad.get_doc - - # we instantiate an accumulator to batch the notifications - self.docs_notify_queue = accumulator_queue( - lambda item: reactor.callFromThread(self._unset_new_dirty, item), - 20) - - # IMessageStore - - # ------------------------------------------------------------------- - # We are not yet using this interface, but it would make sense - # to implement it. - - def create_message(self, mbox, uid, message): - """ - Create the passed message into this SoledadStore. - - :param mbox: the mbox this message belongs. - :type mbox: str or unicode - :param uid: the UID that identifies this message in this mailbox. - :type uid: int - :param message: a IMessageContainer implementor. - """ - raise NotImplementedError() - - def put_message(self, mbox, uid, message): - """ - Put the passed existing message into this SoledadStore. - - :param mbox: the mbox this message belongs. - :type mbox: str or unicode - :param uid: the UID that identifies this message in this mailbox. - :type uid: int - :param message: a IMessageContainer implementor. - """ - raise NotImplementedError() - - def remove_message(self, mbox, uid): - """ - Remove the given message from this SoledadStore. - - :param mbox: the mbox this message belongs. - :type mbox: str or unicode - :param uid: the UID that identifies this message in this mailbox. - :type uid: int - """ - raise NotImplementedError() - - def get_message(self, mbox, uid): - """ - Get a IMessageContainer for the given mbox and uid combination. - - :param mbox: the mbox this message belongs. - :type mbox: str or unicode - :param uid: the UID that identifies this message in this mailbox. - :type uid: int - """ - raise NotImplementedError() - - # IMessageConsumer - - # TODO should handle the delete case - # TODO should handle errors better - # TODO could generalize this method into a generic consumer - # and only implement `process` here - - def consume(self, queue): - """ - Creates a new document in soledad db. - - :param queue: a tuple of queues to get item from, with content of the - document to be inserted. - :type queue: tuple of Queues - """ - new, dirty = queue - while not new.empty(): - doc_wrapper = new.get() - self.reactor.callInThread(self._consume_doc, doc_wrapper, - self.docs_notify_queue) - while not dirty.empty(): - doc_wrapper = dirty.get() - self.reactor.callInThread(self._consume_doc, doc_wrapper, - self.docs_notify_queue) - - # Queue empty, flush the notifications queue. - self.docs_notify_queue(None, flush=True) - - def _unset_new_dirty(self, doc_wrapper): - """ - Unset the `new` and `dirty` flags for this document wrapper in the - memory store. - - :param doc_wrapper: a MessageWrapper instance - :type doc_wrapper: MessageWrapper - """ - if isinstance(doc_wrapper, MessageWrapper): - # XXX still needed for debug quite often - #logger.info("unsetting new flag!") - doc_wrapper.new = False - doc_wrapper.dirty = False - - @deferred_to_thread - def _consume_doc(self, doc_wrapper, notify_queue): - """ - Consume each document wrapper in a separate thread. - We pass an instance of an accumulator that handles the notifications - to the memorystore when the write has been done. - - :param doc_wrapper: a MessageWrapper or RecentFlagsDoc instance - :type doc_wrapper: MessageWrapper or RecentFlagsDoc - :param notify_queue: a callable that handles the writeback - notifications to the memstore. - :type notify_queue: callable - """ - def queueNotifyBack(failed, doc_wrapper): - if failed: - log.msg("There was an error writing the mesage...") - else: - notify_queue(doc_wrapper) - - def doSoledadCalls(items): - # we prime the generator, that should return the - # message or flags wrapper item in the first place. - try: - doc_wrapper = items.next() - except StopIteration: - pass - else: - failed = self._soledad_write_document_parts(items) - queueNotifyBack(failed, doc_wrapper) - - doSoledadCalls(self._iter_wrapper_subparts(doc_wrapper)) - - # - # SoledadStore specific methods. - # - - def _soledad_write_document_parts(self, items): - """ - Write the document parts to soledad in a separate thread. - - :param items: the iterator through the different document wrappers - payloads. - :type items: iterator - :return: whether the write was successful or not - :rtype: bool - """ - failed = False - for item, call in items: - if empty(item): - continue - try: - self._try_call(call, item) - except Exception as exc: - logger.debug("ITEM WAS: %s" % repr(item)) - if hasattr(item, 'content'): - logger.debug("ITEM CONTENT WAS: %s" % - repr(item.content)) - logger.exception(exc) - failed = True - continue - return failed - - def _iter_wrapper_subparts(self, doc_wrapper): - """ - Return an iterator that will yield the doc_wrapper in the first place, - followed by the subparts item and the proper call type for every - item in the queue, if any. - - :param doc_wrapper: a MessageWrapper or RecentFlagsDoc instance - :type doc_wrapper: MessageWrapper or RecentFlagsDoc - """ - if isinstance(doc_wrapper, MessageWrapper): - return chain((doc_wrapper,), - self._get_calls_for_msg_parts(doc_wrapper)) - elif isinstance(doc_wrapper, RecentFlagsDoc): - return chain((doc_wrapper,), - self._get_calls_for_rflags_doc(doc_wrapper)) - else: - logger.warning("CANNOT PROCESS ITEM!") - return (i for i in []) - - def _try_call(self, call, item): - """ - Try to invoke a given call with item as a parameter. - - :param call: the function to call - :type call: callable - :param item: the payload to pass to the call as argument - :type item: object - """ - if call is None: - return - - if call == self._PUT_DOC_FUN: - doc_id = item.doc_id - if doc_id is None: - logger.warning("BUG! Dirty doc but has no doc_id!") - return - with put_locks[doc_id]: - doc = self._GET_DOC_FUN(doc_id) - - if doc is None: - logger.warning("BUG! Dirty doc but could not " - "find document %s" % (doc_id,)) - return - - doc.content = dict(item.content) - - item = doc - try: - call(item) - except u1db_errors.RevisionConflict as exc: - logger.exception("Error: %r" % (exc,)) - raise exc - except Exception as exc: - logger.exception("Error: %r" % (exc,)) - raise exc - - else: - try: - call(item) - except u1db_errors.RevisionConflict as exc: - logger.exception("Error: %r" % (exc,)) - raise exc - except Exception as exc: - logger.exception("Error: %r" % (exc,)) - raise exc - - def _get_calls_for_msg_parts(self, msg_wrapper): - """ - Generator that return the proper call type for a given item. - - :param msg_wrapper: A MessageWrapper - :type msg_wrapper: IMessageContainer - :return: a generator of tuples with recent-flags doc payload - and callable - :rtype: generator - """ - call = None - - if msg_wrapper.new: - call = self._CREATE_DOC_FUN - - # item is expected to be a MessagePartDoc - for item in msg_wrapper.walk(): - if item.part == MessagePartType.fdoc: - yield dict(item.content), call - - elif item.part == MessagePartType.hdoc: - if not self._header_does_exist(item.content): - yield dict(item.content), call - - elif item.part == MessagePartType.cdoc: - if not self._content_does_exist(item.content): - yield dict(item.content), call - - # For now, the only thing that will be dirty is - # the flags doc. - - elif msg_wrapper.dirty: - call = self._PUT_DOC_FUN - # item is expected to be a MessagePartDoc - for item in msg_wrapper.walk(): - # XXX FIXME Give error if dirty and not doc_id !!! - doc_id = item.doc_id # defend! - if not doc_id: - logger.warning("Dirty item but no doc_id!") - continue - - if item.part == MessagePartType.fdoc: - yield item, call - - # XXX also for linkage-doc !!! - else: - logger.error("Cannot delete documents yet from the queue...!") - - def _get_calls_for_rflags_doc(self, rflags_wrapper): - """ - We always put these documents. - - :param rflags_wrapper: A wrapper around recent flags doc. - :type rflags_wrapper: RecentFlagsWrapper - :return: a tuple with recent-flags doc payload and callable - :rtype: tuple - """ - call = self._PUT_DOC_FUN - - payload = rflags_wrapper.content - if payload: - logger.debug("Saving RFLAGS to Soledad...") - yield rflags_wrapper, call - - # Mbox documents and attributes - - def get_mbox_document(self, mbox): - """ - Return mailbox document. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: A SoledadDocument containing this mailbox, or None if - the query failed. - :rtype: SoledadDocument or None. - """ - with mbox_doc_locks[mbox]: - return self._get_mbox_document(mbox) - - def _get_mbox_document(self, mbox): - """ - Helper for returning the mailbox document. - """ - try: - query = self._soledad.get_from_index( - fields.TYPE_MBOX_IDX, - fields.TYPE_MBOX_VAL, mbox) - if query: - return query.pop() - else: - logger.error("Could not find mbox document for %r" % - (mbox,)) - except Exception as exc: - logger.exception("Unhandled error %r" % exc) - - def get_mbox_closed(self, mbox): - """ - Return the closed attribute for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :rtype: bool - """ - mbox_doc = self.get_mbox_document() - return mbox_doc.content.get(fields.CLOSED_KEY, False) - - def set_mbox_closed(self, mbox, closed): - """ - Set the closed attribute for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :param closed: the value to be set - :type closed: bool - """ - leap_assert(isinstance(closed, bool), "closed needs to be boolean") - with mbox_doc_locks[mbox]: - mbox_doc = self._get_mbox_document(mbox) - if mbox_doc is None: - logger.error( - "Could not find mbox document for %r" % (mbox,)) - return - mbox_doc.content[fields.CLOSED_KEY] = closed - self._soledad.put_doc(mbox_doc) - - def write_last_uid(self, mbox, value): - """ - Write the `last_uid` integer to the proper mailbox document - in Soledad. - This is called from the deferred triggered by - memorystore.increment_last_soledad_uid, which is expected to - run in a separate thread. - - :param mbox: the mailbox - :type mbox: str or unicode - :param value: the value to set - :type value: int - """ - leap_assert_type(value, int) - key = fields.LAST_UID_KEY - - # XXX use accumulator to reduce number of hits - with mbox_doc_locks[mbox]: - mbox_doc = self._get_mbox_document(mbox) - old_val = mbox_doc.content[key] - if value > old_val: - mbox_doc.content[key] = value - try: - self._soledad.put_doc(mbox_doc) - except Exception as exc: - logger.error("Error while setting last_uid for %r" - % (mbox,)) - logger.exception(exc) - - def get_flags_doc(self, mbox, uid): - """ - Return the SoledadDocument for the given mbox and uid. - - :param mbox: the mailbox - :type mbox: str or unicode - :param uid: the UID for the message - :type uid: int - :rtype: SoledadDocument or None - """ - # TODO -- inlineCallbacks - result = None - try: - # TODO -- yield - flag_docs = self._soledad.get_from_index( - fields.TYPE_MBOX_UID_IDX, - fields.TYPE_FLAGS_VAL, mbox, str(uid)) - if len(flag_docs) != 1: - logger.warning("More than one flag doc for %r:%s" % - (mbox, uid)) - result = first(flag_docs) - except Exception as exc: - # ugh! Something's broken down there! - logger.warning("ERROR while getting flags for UID: %s" % uid) - logger.exception(exc) - finally: - return result - - def get_headers_doc(self, chash): - """ - Return the document that keeps the headers for a message - indexed by its content-hash. - - :param chash: the content-hash to retrieve the document from. - :type chash: str or unicode - :rtype: SoledadDocument or None - """ - head_docs = self._soledad.get_from_index( - fields.TYPE_C_HASH_IDX, - fields.TYPE_HEADERS_VAL, str(chash)) - return first(head_docs) - - # deleted messages - - def deleted_iter(self, mbox): - """ - Get an iterator for the the doc_id for SoledadDocuments for messages - with \\Deleted flag for a given mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - :return: iterator through deleted message docs - :rtype: iterable - """ - return [doc.doc_id for doc in self._soledad.get_from_index( - fields.TYPE_MBOX_DEL_IDX, - fields.TYPE_FLAGS_VAL, mbox, '1')] - - def remove_all_deleted(self, mbox): - """ - Remove from Soledad all messages flagged as deleted for a given - mailbox. - - :param mbox: the mailbox - :type mbox: str or unicode - """ - deleted = [] - for doc_id in self.deleted_iter(mbox): - with self._remove_lock: - doc = self._soledad.get_doc(doc_id) - if doc is not None: - self._soledad.delete_doc(doc) - try: - deleted.append(doc.content[fields.UID_KEY]) - except TypeError: - # empty content - pass - return deleted diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index ca07f67..482b64d 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -20,6 +20,7 @@ Generic Access to Mail objects: Public LEAP Mail API. from twisted.internet import defer from leap.mail.constants import INBOX_NAME +from leap.mail.constants import MessageFlags from leap.mail.mailbox_indexer import MailboxIndexer from leap.mail.adaptors.soledad import SoledadMailAdaptor @@ -61,6 +62,18 @@ class Message(object): def get_internal_date(self): """ + Retrieve the date internally associated with this message + + According to the spec, this is NOT the date and time in the + RFC-822 header, but rather a date and time that reflects when the + message was received. + + * In SMTP, date and time of final delivery. + * In COPY, internal date/time of the source message. + * In APPEND, date/time specified. + + :return: An RFC822-formatted date string. + :rtype: str """ return self._wrapper.fdoc.date @@ -99,6 +112,15 @@ class Message(object): return tuple(self._wrapper.fdoc.tags) +class Flagsmode(object): + """ + Modes for setting the flags/tags. + """ + APPEND = 1 + REMOVE = -1 + SET = 0 + + class MessageCollection(object): """ A generic collection of messages. It can be messages sharing the same @@ -132,6 +154,7 @@ class MessageCollection(object): def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): """ + Constructore for a MessageCollection. """ self.adaptor = adaptor self.store = store @@ -149,6 +172,20 @@ class MessageCollection(object): """ return bool(self.mbox_wrapper) + @property + def mbox_name(self): + wrapper = getattr(self, "mbox_wrapper", None) + if not wrapper: + return None + return wrapper.mbox + + def get_mbox_attr(self, attr): + return getattr(self.mbox_wrapper, attr) + + def set_mbox_attr(self, attr, value): + setattr(self.mbox_wrapper, attr, value) + return self.mbox_wrapper.update(self.store) + # Get messages def get_message_by_content_hash(self, chash, get_cdocs=False): @@ -162,7 +199,7 @@ class MessageCollection(object): # or use the internal collection of pointers-to-docs. raise NotImplementedError() - metamsg_id = _get_mdoc_id(self.mbox_wrapper.mbox, chash) + metamsg_id = _get_mdoc_id(self.mbox_name, chash) return self.adaptor.get_msg_from_mdoc_id( self.messageklass, self.store, @@ -181,25 +218,37 @@ class MessageCollection(object): raise NotImplementedError("Does not support relative ids yet") def get_msg_from_mdoc_id(doc_id): + # XXX pass UID? return self.adaptor.get_msg_from_mdoc_id( self.messageklass, self.store, doc_id, get_cdocs=get_cdocs) - d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_wrapper.mbox, uid) + d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_name, uid) d.addCallback(get_msg_from_mdoc_id) return d def count(self): """ Count the messages in this collection. - :rtype: int + :return: a Deferred that will fire with the integer for the count. + :rtype: Deferred """ if not self.is_mailbox_collection(): raise NotImplementedError() - return self.mbox_indexer.count(self.mbox_wrapper.mbox) + return self.mbox_indexer.count(self.mbox_name) + + def get_uid_next(self): + """ + Get the next integer beyond the highest UID count for this mailbox. + + :return: a Deferred that will fire with the integer for the next uid. + :rtype: Deferred + """ + return self.mbox_indexer.get_uid_next(self.mbox_name) # Manipulate messages + # TODO pass flags, date too... def add_msg(self, raw_msg): """ Add a message to this collection. @@ -208,14 +257,14 @@ class MessageCollection(object): wrapper = msg.get_wrapper() if self.is_mailbox_collection(): - mbox = self.mbox_wrapper.mbox + mbox = self.mbox_name wrapper.set_mbox(mbox) def insert_mdoc_id(_): # XXX does this work? doc_id = wrapper.mdoc.doc_id return self.mbox_indexer.insert_doc( - self.mbox_wrapper.mbox, doc_id) + self.mbox_name, doc_id) d = wrapper.create(self.store) d.addCallback(insert_mdoc_id) @@ -248,31 +297,45 @@ class MessageCollection(object): # XXX does this work? doc_id = wrapper.mdoc.doc_id return self.mbox_indexer.delete_doc_by_hash( - self.mbox_wrapper.mbox, doc_id) + self.mbox_name, doc_id) d = wrapper.delete(self.store) d.addCallback(delete_mdoc_id) return d # TODO should add a delete-by-uid to collection? + def _update_flags_or_tags(self, old, new, mode): + if mode == Flagsmode.APPEND: + final = list((set(tuple(old) + new))) + elif mode == Flagsmode.REMOVE: + final = list(set(old).difference(set(new))) + elif mode == Flagsmode.SET: + final = new + return final + def udpate_flags(self, msg, flags, mode): """ Update flags for a given message. """ wrapper = msg.get_wrapper() - # 1. update the flags in the message wrapper --- stored where??? - # 2. update the special flags in the wrapper (seen, etc) - # 3. call adaptor.update_msg(store) - pass + current = wrapper.fdoc.flags + newflags = self._update_flags_or_tags(current, flags, mode) + wrapper.fdoc.flags = newflags + + wrapper.fdoc.seen = MessageFlags.SEEN_FLAG in newflags + wrapper.fdoc.deleted = MessageFlags.DELETED_FLAG in newflags + + return self.adaptor.update_msg(self.store, msg) def update_tags(self, msg, tags, mode): """ Update tags for a given message. """ wrapper = msg.get_wrapper() - # 1. update the tags in the message wrapper --- stored where??? - # 2. call adaptor.update_msg(store) - pass + current = wrapper.fdoc.tags + newtags = self._update_flags_or_tags(current, tags, mode) + wrapper.fdoc.tags = newtags + return self.adaptor.update_msg(self.store, msg) class Account(object): @@ -382,6 +445,8 @@ class Account(object): d.addCallback(rename_uid_table_cb) return d + # Get Collections + def get_collection_by_mailbox(self, name): """ :rtype: MessageCollection diff --git a/src/leap/mail/messageflow.py b/src/leap/mail/messageflow.py deleted file mode 100644 index c8f224c..0000000 --- a/src/leap/mail/messageflow.py +++ /dev/null @@ -1,200 +0,0 @@ -# -*- coding: utf-8 -*- -# messageflow.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Message Producers and Consumers for flow control. -""" -import Queue - -from twisted.internet.task import LoopingCall - -from zope.interface import Interface, implements - - -class IMessageConsumer(Interface): - """ - I consume messages from a queue. - """ - - def consume(self, queue): - """ - Consumes the passed item. - - :param item: a queue where we put the object to be consumed. - :type item: object - """ - # TODO we could add an optional type to be passed - # for doing type check. - - # TODO in case of errors, we could return the object to - # the queue, maybe wrapped in an object with a retries attribute. - - -class IMessageProducer(Interface): - """ - I produce messages and put them in a store to be consumed by other - entities. - """ - - def push(self, item, state=None): - """ - Push a new item in the queue. - """ - - def start(self): - """ - Start producing items. - """ - - def stop(self): - """ - Stop producing items. - """ - - def flush(self): - """ - Flush queued messages to consumer. - """ - - -class DummyMsgConsumer(object): - - implements(IMessageConsumer) - - def consume(self, queue): - """ - Just prints the passed item. - """ - if not queue.empty(): - print "got item %s" % queue.get() - - -class MessageProducer(object): - """ - A Producer class that we can use to temporarily buffer the production - of messages so that different objects can consume them. - - This is useful for serializing the consumption of the messages stream - in the case of an slow resource (db), or for returning early from a - deferred chain and leave further processing detached from the calling loop, - as in the case of smtp. - """ - implements(IMessageProducer) - - # TODO this can be seen as a first step towards properly implementing - # components that implement IPushProducer / IConsumer interfaces. - # However, I need to think more about how to pause the streaming. - # In any case, the differential rate between message production - # and consumption is not likely (?) to consume huge amounts of memory in - # our current settings, so the need to pause the stream is not urgent now. - - # TODO use enum - STATE_NEW = 1 - STATE_DIRTY = 2 - - def __init__(self, consumer, queue=Queue.Queue, period=1): - """ - Initializes the MessageProducer - - :param consumer: an instance of a IMessageConsumer that will consume - the new messages. - :param queue: any queue implementation to be used as the temporary - buffer for new items. Default is a FIFO Queue. - :param period: the period to check for new items, in seconds. - """ - # XXX should assert it implements IConsumer / IMailConsumer - # it should implement a `consume` method - self._consumer = consumer - - self._queue_new = queue() - self._queue_dirty = queue() - self._period = period - - self._loop = LoopingCall(self._check_for_new) - - # private methods - - def _check_for_new(self): - """ - Check for new items in the internal queue, and calls the consume - method in the consumer. - - If the queue is found empty, the loop is stopped. It will be started - again after the addition of new items. - """ - self._consumer.consume((self._queue_new, self._queue_dirty)) - if self.is_queue_empty(): - self.stop() - - def is_queue_empty(self): - """ - Return True if queue is empty, False otherwise. - """ - new = self._queue_new - dirty = self._queue_dirty - return new.empty() and dirty.empty() - - # public methods: IMessageProducer - - def push(self, item, state=None): - """ - Push a new item in the queue. - - If the queue was empty, we will start the loop again. - """ - # XXX this might raise if the queue does not accept any new - # items. what to do then? - queue = self._queue_new - - if state == self.STATE_NEW: - queue = self._queue_new - if state == self.STATE_DIRTY: - queue = self._queue_dirty - - queue.put(item) - self.start() - - def start(self): - """ - Start polling for new items. - """ - if not self._loop.running: - self._loop.start(self._period, now=True) - - def stop(self): - """ - Stop polling for new items. - """ - if self._loop.running: - self._loop.stop() - - def flush(self): - """ - Flush queued messages to consumer. - """ - self._check_for_new() - - -if __name__ == "__main__": - from twisted.internet import reactor - producer = MessageProducer(DummyMsgConsumer()) - producer.start() - - for delay, item in ((2, 1), (3, 2), (4, 3), - (6, 4), (7, 5), (8, 6), (8.2, 7), - (15, 'a'), (16, 'b'), (17, 'c')): - reactor.callLater(delay, producer.put, item) - reactor.run() -- cgit v1.2.3 From 84888155b09b3af6a755262b28728de2f851c8cb Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 4 Jan 2015 03:37:18 -0400 Subject: tests for mail.mail module: Message --- src/leap/mail/adaptors/models.py | 2 +- src/leap/mail/adaptors/soledad.py | 131 ++++++++--- src/leap/mail/imap/messages.py | 172 +++----------- src/leap/mail/imap/tests/rfc822.message | 87 +------ .../mail/imap/tests/rfc822.multi-minimal.message | 17 +- .../mail/imap/tests/rfc822.multi-signed.message | 239 +------------------ src/leap/mail/imap/tests/rfc822.multi.message | 97 +------- src/leap/mail/imap/tests/rfc822.plain.message | 67 +----- src/leap/mail/mail.py | 165 +++++++++++-- src/leap/mail/mailbox_indexer.py | 22 +- src/leap/mail/tests/rfc822.multi-minimal.message | 16 ++ src/leap/mail/tests/rfc822.multi-signed.message | 238 +++++++++++++++++++ src/leap/mail/tests/rfc822.multi.message | 96 ++++++++ src/leap/mail/tests/rfc822.plain.message | 66 ++++++ src/leap/mail/tests/test_mail.py | 255 +++++++++++++++++++-- src/leap/mail/walk.py | 18 +- 16 files changed, 955 insertions(+), 733 deletions(-) mode change 100644 => 120000 src/leap/mail/imap/tests/rfc822.message mode change 100644 => 120000 src/leap/mail/imap/tests/rfc822.multi-minimal.message mode change 100644 => 120000 src/leap/mail/imap/tests/rfc822.multi-signed.message mode change 100644 => 120000 src/leap/mail/imap/tests/rfc822.multi.message mode change 100644 => 120000 src/leap/mail/imap/tests/rfc822.plain.message create mode 100644 src/leap/mail/tests/rfc822.multi-minimal.message create mode 100644 src/leap/mail/tests/rfc822.multi-signed.message create mode 100644 src/leap/mail/tests/rfc822.multi.message create mode 100644 src/leap/mail/tests/rfc822.plain.message diff --git a/src/leap/mail/adaptors/models.py b/src/leap/mail/adaptors/models.py index 1648059..88e0e4e 100644 --- a/src/leap/mail/adaptors/models.py +++ b/src/leap/mail/adaptors/models.py @@ -89,7 +89,7 @@ class DocumentWrapper(object): if not attr.startswith('_') and attr not in normalized: raise RuntimeError( "Cannot set attribute because it's not defined " - "in the model: %s" % attr) + "in the model %s: %s" % (self.__class__, attr)) object.__setattr__(self, attr, value) def serialize(self): diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index bf8f7e9..f0808af 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -23,6 +23,7 @@ from functools import partial from pycryptopp.hash import sha256 from twisted.internet import defer +from twisted.python import util from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type @@ -37,6 +38,8 @@ from leap.mail.utils import lowerdict, first from leap.mail.utils import stringify_parts_map from leap.mail.interfaces import IMailAdaptor, IMessageWrapper +from leap.soledad.common.document import SoledadDocument + # TODO # [ ] Convenience function to create mail specifying subject, date, etc? @@ -339,10 +342,10 @@ class FlagsDocWrapper(SoledadDocumentWrapper): seen = False deleted = False recent = False - multi = False flags = [] tags = [] size = 0 + multi = False class __meta__(object): index = "mbox" @@ -409,7 +412,6 @@ class MetaMsgDocWrapper(SoledadDocumentWrapper): class MessageWrapper(object): - # TODO generalize wrapper composition? # This could benefit of a DeferredLock to create/update all the # documents at the same time maybe, and defend against concurrent updates? @@ -425,11 +427,19 @@ class MessageWrapper(object): integers, beginning at one, and the values are dictionaries with the content of the content-docs. """ + if isinstance(mdoc, SoledadDocument): + mdoc = mdoc.content + if not mdoc: + mdoc = {} self.mdoc = MetaMsgDocWrapper(**mdoc) + if isinstance(fdoc, SoledadDocument): + fdoc = fdoc.content self.fdoc = FlagsDocWrapper(**fdoc) self.fdoc.set_future_doc_id(self.mdoc.fdoc) + if isinstance(hdoc, SoledadDocument): + hdoc = hdoc.content self.hdoc = HeaderDocWrapper(**hdoc) self.hdoc.set_future_doc_id(self.mdoc.hdoc) @@ -502,6 +512,43 @@ class MessageWrapper(object): self.mdoc.set_mbox(mbox) self.fdoc.set_mbox(mbox) + def set_flags(self, flags): + # TODO serialize the get + update + if flags is None: + flags = tuple() + leap_assert_type(flags, tuple) + self.fdoc.flags = list(flags) + + def set_tags(self, tags): + # TODO serialize the get + update + if tags is None: + tags = tuple() + leap_assert_type(tags, tuple) + self.fdoc.tags = list(tags) + + def set_date(self, date): + # XXX assert valid date format + self.hdoc.date = date + + def get_subpart_dict(self, index): + """ + :param index: index, 1-indexed + :type index: int + """ + return self.hdoc.part_map[str(index)] + + def get_body(self, store): + """ + :rtype: deferred + """ + body_phash = self.hdoc.body + if not body_phash: + return None + d = store.get_doc('C-' + body_phash) + d.addCallback(lambda doc: ContentDocWrapper(**doc.content)) + return d + + # # Mailboxes # @@ -631,7 +678,8 @@ class SoledadMailAdaptor(SoledadIndexMixin): return self.get_msg_from_docs( MessageClass, mdoc, fdoc, hdoc, cdocs) - def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None): + def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None, + uid=None): """ Get an instance of a MessageClass initialized with a MessageWrapper that contains the passed part documents. @@ -657,26 +705,24 @@ class SoledadMailAdaptor(SoledadIndexMixin): :rtype: MessageClass instance. """ assert(MessageClass is not None) - return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs)) + return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs), uid=uid) - # XXX pass UID too? - def _get_msg_from_variable_doc_list(self, doc_list, msg_class): - if len(doc_list) == 2: - fdoc, hdoc = doc_list + def _get_msg_from_variable_doc_list(self, doc_list, msg_class, uid=None): + if len(doc_list) == 3: + mdoc, fdoc, hdoc = doc_list cdocs = None - elif len(doc_list) > 2: - fdoc, hdoc = doc_list[:2] - cdocs = dict(enumerate(doc_list[2:], 1)) - return self.get_msg_from_docs(msg_class, fdoc, hdoc, cdocs) + elif len(doc_list) > 3: + fdoc, hdoc = doc_list[:3] + cdocs = dict(enumerate(doc_list[3:], 1)) + return self.get_msg_from_docs( + msg_class, mdoc, fdoc, hdoc, cdocs, uid=None) - # XXX pass UID too ? def get_msg_from_mdoc_id(self, MessageClass, store, doc_id, - get_cdocs=False): + uid=None, get_cdocs=False): metamsg_id = doc_id def wrap_meta_doc(doc): cls = MetaMsgDocWrapper - # XXX pass UID? return cls(doc_id=doc.doc_id, **doc.content) def get_part_docs_from_mdoc_wrapper(wrapper): @@ -685,7 +731,12 @@ class SoledadMailAdaptor(SoledadIndexMixin): d_docs.append(store.get_doc(wrapper.hdoc)) for cdoc in wrapper.cdocs: d_docs.append(store.get_doc(cdoc)) + + def add_mdoc(doc_list): + return [wrapper.serialize()] + doc_list + d = defer.gatherResults(d_docs) + d.addCallback(add_mdoc) return d def get_parts_doc_from_mdoc_id(): @@ -696,25 +747,31 @@ class SoledadMailAdaptor(SoledadIndexMixin): return constants.FDOCID.format(mbox=mbox, chash=chash) def _get_hdoc_id_from_mdoc_id(): - return constants.FDOCID.format(mbox=mbox, chash=chash) + return constants.HDOCID.format(mbox=mbox, chash=chash) d_docs = [] fdoc_id = _get_fdoc_id_from_mdoc_id() hdoc_id = _get_hdoc_id_from_mdoc_id() + d_docs.append(store.get_doc(fdoc_id)) d_docs.append(store.get_doc(hdoc_id)) + d = defer.gatherResults(d_docs) return d + def add_mdoc_id_placeholder(docs_list): + return [None] + docs_list + if get_cdocs: d = store.get_doc(metamsg_id) d.addCallback(wrap_meta_doc) d.addCallback(get_part_docs_from_mdoc_wrapper) else: d = get_parts_doc_from_mdoc_id() + d.addCallback(add_mdoc_id_placeholder) d.addCallback(partial(self._get_msg_from_variable_doc_list, - msg_class=MessageClass)) + msg_class=MessageClass, uid=uid)) return d def create_msg(self, store, msg): @@ -791,17 +848,17 @@ def _split_into_parts(raw): # TODO populate Default FLAGS/TAGS (unseen?) # TODO seed propely the content_docs with defaults?? - msg, parts, chash, size, multi = _parse_msg(raw) - body_phash_fun = [walk.get_body_phash_simple, - walk.get_body_phash_multi][int(multi)] - body_phash = body_phash_fun(walk.get_payloads(msg)) + msg, parts, chash, multi = _parse_msg(raw) + size = len(msg.as_string()) + body_phash = walk.get_body_phash(msg) + parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) cdocs_list = list(walk.get_raw_docs(msg, parts)) cdocs_phashes = [c['phash'] for c in cdocs_list] mdoc = _build_meta_doc(chash, cdocs_phashes) fdoc = _build_flags_doc(chash, size, multi) - hdoc = _build_headers_doc(msg, chash, parts_map) + hdoc = _build_headers_doc(msg, chash, body_phash, parts_map) # The MessageWrapper expects a dict, one-indexed cdocs = dict(enumerate(cdocs_list, 1)) @@ -812,10 +869,9 @@ def _split_into_parts(raw): def _parse_msg(raw): msg = message_from_string(raw) parts = walk.get_parts(msg) - size = len(raw) chash = sha256.SHA256(raw).hexdigest() multi = msg.is_multipart() - return msg, parts, chash, size, multi + return msg, parts, chash, multi def _build_meta_doc(chash, cdocs_phashes): @@ -831,28 +887,31 @@ def _build_flags_doc(chash, size, multi): return _fdoc.serialize() -def _build_headers_doc(msg, chash, parts_map): +def _build_headers_doc(msg, chash, body_phash, parts_map): """ Assemble a headers document from the original parsed message, the content-hash, and the parts map. It takes into account possibly repeated headers. """ - headers = defaultdict(list) - for k, v in msg.items(): - headers[k].append(v) - - # "fix" for repeated headers. - for k, v in headers.items(): - newline = "\n%s: " % (k,) - headers[k] = newline.join(v) - - lower_headers = lowerdict(headers) + headers = msg.items() + + # TODO move this manipulation to IMAP + #headers = defaultdict(list) + #for k, v in msg.items(): + #headers[k].append(v) + ## "fix" for repeated headers. + #for k, v in headers.items(): + #newline = "\n%s: " % (k,) + #headers[k] = newline.join(v) + + lower_headers = lowerdict(dict(headers)) msgid = first(_MSGID_RE.findall( lower_headers.get('message-id', ''))) _hdoc = HeaderDocWrapper( - chash=chash, headers=lower_headers, msgid=msgid) + chash=chash, headers=headers, body=body_phash, + msgid=msgid) def copy_attr(headers, key, doc): if key in headers: diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 7e0f973..883da35 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -46,28 +46,12 @@ class IMAPMessage(object): implements(imap4.IMessage) - # TODO ---- see what should we pass here instead - # where's UID added to the message? - # def __init__(self, soledad, uid, mbox): - def __init__(self, message, collection): + def __init__(self, message): """ Initializes a LeapMessage. - - :param soledad: a Soledad instance - :type soledad: Soledad - :param uid: the UID for the message. - :type uid: int or basestring - :param mbox: the mbox this message belongs to - :type mbox: str or unicode """ - #self._uid = int(uid) if uid is not None else None - #self._mbox = normalize_mailbox(mbox) - self.message = message - # TODO maybe not needed, see setFlags below - self.collection = collection - # IMessage implementation def getUID(self): @@ -95,21 +79,21 @@ class IMAPMessage(object): # lookup method? IMAPMailbox? - def setFlags(self, flags, mode): - """ - Sets the flags for this message - - :param flags: the flags to update in the message. - :type flags: tuple of str - :param mode: the mode for setting. 1 is append, -1 is remove, 0 set. - :type mode: int - """ - leap_assert(isinstance(flags, tuple), "flags need to be a tuple") + #def setFlags(self, flags, mode): + #""" + #Sets the flags for this message +# + #:param flags: the flags to update in the message. + #:type flags: tuple of str + #:param mode: the mode for setting. 1 is append, -1 is remove, 0 set. + #:type mode: int + #""" + #leap_assert(isinstance(flags, tuple), "flags need to be a tuple") # XXX # return new flags # map to str #self.message.set_flags(flags, mode) - self.collection.update_flags(self.message, flags, mode) + #self.collection.update_flags(self.message, flags, mode) def getInternalDate(self): """ @@ -132,9 +116,6 @@ class IMAPMessage(object): # IMessagePart # - # XXX we should implement this interface too for the subparts - # so we allow nested parts... - def getBodyFile(self): """ Retrieve a file object containing only the body of this message. @@ -142,53 +123,25 @@ class IMAPMessage(object): :return: file-like object opened for reading :rtype: StringIO """ - #def write_fd(body): - #fd.write(body) - #fd.seek(0) - #return fd -# # TODO refactor with getBodyFile in MessagePart -# - #fd = StringIO.StringIO() -# - #if self.bdoc is not None: - #bdoc_content = self.bdoc.content - #if empty(bdoc_content): - #logger.warning("No BDOC content found for message!!!") - #return write_fd("") -# - #body = bdoc_content.get(self.RAW_KEY, "") - #content_type = bdoc_content.get('content-type', "") - #charset = find_charset(content_type) - #if charset is None: - #charset = self._get_charset(body) - #try: - #if isinstance(body, unicode): - #body = body.encode(charset) - #except UnicodeError as exc: - #logger.error( - #"Unicode error, using 'replace'. {0!r}".format(exc)) - #logger.debug("Attempted to encode with: %s" % charset) - #body = body.encode(charset, 'replace') - #finally: - #return write_fd(body) - return self.message.get_body_file() + #body = bdoc_content.get(self.RAW_KEY, "") + #content_type = bdoc_content.get('content-type', "") + #charset = find_charset(content_type) + #if charset is None: + #charset = self._get_charset(body) + #try: + #if isinstance(body, unicode): + #body = body.encode(charset) + #except UnicodeError as exc: + #logger.error( + #"Unicode error, using 'replace'. {0!r}".format(exc)) + #logger.debug("Attempted to encode with: %s" % charset) + #body = body.encode(charset, 'replace') + #finally: + #return write_fd(body) - # TODO move to mail.mail - @memoized_method - def _get_charset(self, stuff): - """ - Gets (guesses?) the charset of a payload. - - :param stuff: the stuff to guess about. - :type stuff: basestring - :returns: charset - """ - # XXX shouldn't we make the scope - # of the decorator somewhat more persistent? - # and put memory bounds. - return get_email_charset(stuff) + return self.message.get_body_file() def getSize(self): """ @@ -197,10 +150,6 @@ class IMAPMessage(object): :return: size of the message, in octets :rtype: int """ - #size = None - #fdoc_content = self.fdoc.content - #size = fdoc_content.get(self.SIZE_KEY, False) - #return size return self.message.get_size() def getHeaders(self, negate, *names): @@ -265,11 +214,7 @@ class IMAPMessage(object): """ Return True if this message is multipart. """ - #fdoc_content = self.fdoc.content - #is_multipart = fdoc_content.get(self.MULTIPART_KEY, False) - #return is_multipart - - return self.message.fdoc.is_multi + return self.message.is_multipart() def getSubPart(self, part): """ @@ -282,64 +227,7 @@ class IMAPMessage(object): :rtype: Any object implementing C{IMessagePart}. :return: The specified sub-part. """ - if not self.isMultipart(): - raise TypeError - try: - pmap_dict = self._get_part_from_parts_map(part + 1) - except KeyError: - raise IndexError - - # TODO move access to adaptor ---- - return MessagePart(self._soledad, pmap_dict) - - # - # accessors - # - - # FIXME - # -- move to wrapper/adaptor - def _get_part_from_parts_map(self, part): - """ - Get a part map from the headers doc - - :raises: KeyError if key does not exist - :rtype: dict - """ - raise NotImplementedError() - - #hdoc_content = self.hdoc.content - #pmap = hdoc_content.get(fields.PARTS_MAP_KEY, {}) -# - # remember, lads, soledad is using strings in its keys, - # not integers! - #return pmap[str(part)] - - # TODO move to wrapper/adaptor - def _get_body_doc(self): - """ - Return the document that keeps the body for this - message. - """ - # FIXME - # -- just get the body and retrieve the cdoc P- - #hdoc_content = self.hdoc.content - #body_phash = hdoc_content.get( - #fields.BODY_KEY, None) - #if not body_phash: - #logger.warning("No body phash for this document!") - #return None -# - #if self._container is not None: - #bdoc = self._container.memstore.get_cdoc_from_phash(body_phash) - #if not empty(bdoc) and not empty(bdoc.content): - #return bdoc -# - # no memstore, or no body doc found there - #d = self._soledad.get_from_index( - #fields.TYPE_P_HASH_IDX, - #fields.TYPE_CONTENT_VAL, str(body_phash)) - #d.addCallback(lambda docs: first(docs)) - #return d + return self.message.get_subpart(part) class IMAPMessageCollection(object): diff --git a/src/leap/mail/imap/tests/rfc822.message b/src/leap/mail/imap/tests/rfc822.message deleted file mode 100644 index ee97ab9..0000000 --- a/src/leap/mail/imap/tests/rfc822.message +++ /dev/null @@ -1,86 +0,0 @@ -Return-Path: -Delivered-To: exarkun@meson.dyndns.org -Received: from localhost [127.0.0.1] - by localhost with POP3 (fetchmail-6.2.1) - for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) -Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) - by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 - for ; Thu, 20 Mar 2003 14:49:27 -0500 (EST) -Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) - by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) - id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 -Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) - id 18w63j-0007VK-00 - for ; Thu, 20 Mar 2003 13:50:39 -0600 -To: twisted-commits@twistedmatrix.com -From: etrepum CVS -Reply-To: twisted-python@twistedmatrix.com -X-Mailer: CVSToys -Message-Id: -Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. -Sender: twisted-commits-admin@twistedmatrix.com -Errors-To: twisted-commits-admin@twistedmatrix.com -X-BeenThere: twisted-commits@twistedmatrix.com -X-Mailman-Version: 2.0.11 -Precedence: bulk -List-Help: -List-Post: -List-Subscribe: , - -List-Id: -List-Unsubscribe: , - -List-Archive: -Date: Thu, 20 Mar 2003 13:50:39 -0600 - -Modified files: -Twisted/twisted/python/rebuild.py 1.19 1.20 - -Log message: -rebuild now works on python versions from 2.2.0 and up. - - -ViewCVS links: -http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted - -Index: Twisted/twisted/python/rebuild.py -diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 ---- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 -+++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 -@@ -206,15 +206,27 @@ - clazz.__dict__.clear() - clazz.__getattr__ = __getattr__ - clazz.__module__ = module.__name__ -+ if newclasses: -+ import gc -+ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): -+ hasBrokenRebuild = 1 -+ gc_objects = gc.get_objects() -+ else: -+ hasBrokenRebuild = 0 - for nclass in newclasses: - ga = getattr(module, nclass.__name__) - if ga is nclass: - log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) - else: -- import gc -- for r in gc.get_referrers(nclass): -- if isinstance(r, nclass): -+ if hasBrokenRebuild: -+ for r in gc_objects: -+ if not getattr(r, '__class__', None) is nclass: -+ continue - r.__class__ = ga -+ else: -+ for r in gc.get_referrers(nclass): -+ if getattr(r, '__class__', None) is nclass: -+ r.__class__ = ga - if doLog: - log.msg('') - log.msg(' (fixing %s): ' % str(module.__name__)) - - -_______________________________________________ -Twisted-commits mailing list -Twisted-commits@twistedmatrix.com -http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits diff --git a/src/leap/mail/imap/tests/rfc822.message b/src/leap/mail/imap/tests/rfc822.message new file mode 120000 index 0000000..b19cc28 --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.message @@ -0,0 +1 @@ +../../tests/rfc822.message \ No newline at end of file diff --git a/src/leap/mail/imap/tests/rfc822.multi-minimal.message b/src/leap/mail/imap/tests/rfc822.multi-minimal.message deleted file mode 100644 index 582297c..0000000 --- a/src/leap/mail/imap/tests/rfc822.multi-minimal.message +++ /dev/null @@ -1,16 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============6203542367371144092==" -MIME-Version: 1.0 -Subject: [TEST] 010 - Inceptos cum lorem risus congue -From: testmailbitmaskspam@gmail.com -To: test_c5@dev.bitmask.net - ---===============6203542367371144092== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -Howdy from python! -The subject: [TEST] 010 - Inceptos cum lorem risus congue -Current date & time: Wed Jan 8 16:36:21 2014 -Trying to attach: [] ---===============6203542367371144092==-- diff --git a/src/leap/mail/imap/tests/rfc822.multi-minimal.message b/src/leap/mail/imap/tests/rfc822.multi-minimal.message new file mode 120000 index 0000000..e0aa678 --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi-minimal.message @@ -0,0 +1 @@ +../../tests/rfc822.multi-minimal.message \ No newline at end of file diff --git a/src/leap/mail/imap/tests/rfc822.multi-signed.message b/src/leap/mail/imap/tests/rfc822.multi-signed.message deleted file mode 100644 index 9907c2d..0000000 --- a/src/leap/mail/imap/tests/rfc822.multi-signed.message +++ /dev/null @@ -1,238 +0,0 @@ -Date: Mon, 6 Jan 2014 04:40:47 -0400 -From: Kali Kaneko -To: penguin@example.com -Subject: signed message -Message-ID: <20140106084047.GA21317@samsara.lan> -MIME-Version: 1.0 -Content-Type: multipart/signed; micalg=pgp-sha1; - protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy" -Content-Disposition: inline -User-Agent: Mutt/1.5.21 (2012-12-30) - - ---z9ECzHErBrwFF8sy -Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l" -Content-Disposition: inline - - ---z0eOaCaDLjvTGF2l -Content-Type: text/plain; charset=utf-8 -Content-Disposition: inline -Content-Transfer-Encoding: quoted-printable - -This is an example of a signed message, -with attachments. - - ---=20 -Nihil sine chao! =E2=88=B4 - ---z0eOaCaDLjvTGF2l -Content-Type: text/plain; charset=us-ascii -Content-Disposition: attachment; filename="attach.txt" - -this is attachment in plain text. - ---z0eOaCaDLjvTGF2l -Content-Type: application/octet-stream -Content-Disposition: attachment; filename="hack.ico" -Content-Transfer-Encoding: base64 - -AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA -KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG -RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA -PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl -5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA -/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ -yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A -Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK -ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK -LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP -QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy -AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs -AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA -AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA -gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d -HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA -x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7 -+wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA -AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5 -+QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA -OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK -igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA -JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra -2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA -xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj -owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB -AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA -AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d -XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d -XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA -AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB -AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm -X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC -AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B -bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ -S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu -J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y -AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N -KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB -XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A -AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA -AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d -XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA -AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr -RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA -AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A -Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI -yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA -CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys -rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA -vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d -HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA -urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx -cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA -CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo -6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA -2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7 -OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA -UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp -qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA -lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa -WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB -AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB -AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB -AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA -ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA -AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB -AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB -AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB -AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB -AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA -tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA -AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF -wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB -AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2 -RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB -AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB -AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA -AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd -AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB -AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB -AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB -AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB -AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB -AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8 -ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2 -NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF -RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB -lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA -AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa -WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA -AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX -AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB -AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB -AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA -AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA -AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA -AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA -AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB -AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB -AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA -ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA -AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7 -LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA -AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF -NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB -AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2 -RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA -ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4 -RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi -JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2 -NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK -T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB -AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB -AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB -AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN -UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA -AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA -W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA -AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB -l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB -AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+ -WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA -AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv -RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA -AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj -AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB -AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB -AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA -AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA -AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA -dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A -AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB -AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB -AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB -AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW -pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= - ---z0eOaCaDLjvTGF2l-- - ---z9ECzHErBrwFF8sy -Content-Type: application/pgp-signature - ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.15 (GNU/Linux) - -iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv -kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl -vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK -PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC -w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw -sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr -BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN -QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt -mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ -jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8 -gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X -sSdfcAhT7Tno7PB/Acoh -=+okv ------END PGP SIGNATURE----- - ---z9ECzHErBrwFF8sy-- diff --git a/src/leap/mail/imap/tests/rfc822.multi-signed.message b/src/leap/mail/imap/tests/rfc822.multi-signed.message new file mode 120000 index 0000000..4172244 --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi-signed.message @@ -0,0 +1 @@ +../../tests/rfc822.multi-signed.message \ No newline at end of file diff --git a/src/leap/mail/imap/tests/rfc822.multi.message b/src/leap/mail/imap/tests/rfc822.multi.message deleted file mode 100644 index 30f74e5..0000000 --- a/src/leap/mail/imap/tests/rfc822.multi.message +++ /dev/null @@ -1,96 +0,0 @@ -Date: Fri, 19 May 2000 09:55:48 -0400 (EDT) -From: Doug Sauder -To: Joe Blow -Subject: Test message from PINE -Message-ID: -MIME-Version: 1.0 -Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452" - - This message is in MIME format. The first part should be readable text, - while the remaining parts are likely unreadable without MIME-aware tools. - Send mail to mime@docserver.cac.washington.edu for more info. - ----1463757054-952513540-958744548=:8452 -Content-Type: TEXT/PLAIN; charset=US-ASCII - -This is a test message from PINE MUA. - - ----1463757054-952513540-958744548=:8452 -Content-Type: APPLICATION/octet-stream; name="redball.png" -Content-Transfer-Encoding: BASE64 -Content-ID: -Content-Description: A PNG graphic file -Content-Disposition: attachment; filename="redball.png" - -iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A -AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj -AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv -AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0 -GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf -hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl -rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL -ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS -AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP -AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP -AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI -AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR -AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6 -AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO -AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8 -AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8 -LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu -MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF -BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp -6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow -8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G -ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn -OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1 -a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T -VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8 -Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f -lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/ -joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms -1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA -JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7 -vAAAAABJRU5ErkJggg== ----1463757054-952513540-958744548=:8452 -Content-Type: APPLICATION/octet-stream; name="blueball.png" -Content-Transfer-Encoding: BASE64 -Content-ID: -Content-Description: A PNG graphic file -Content-Disposition: attachment; filename="blueball.png" - -iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A -AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI -IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y -Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S -hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM -vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB -Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu -MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b -fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo -Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp -LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX -P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M -jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8 -+VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE -1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42 -YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY -mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp -Z3VldDZzO7wAAAAASUVORK5CYII= ----1463757054-952513540-958744548=:8452-- diff --git a/src/leap/mail/imap/tests/rfc822.multi.message b/src/leap/mail/imap/tests/rfc822.multi.message new file mode 120000 index 0000000..62057d2 --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi.message @@ -0,0 +1 @@ +../../tests/rfc822.multi.message \ No newline at end of file diff --git a/src/leap/mail/imap/tests/rfc822.plain.message b/src/leap/mail/imap/tests/rfc822.plain.message deleted file mode 100644 index fc627c3..0000000 --- a/src/leap/mail/imap/tests/rfc822.plain.message +++ /dev/null @@ -1,66 +0,0 @@ -From pyar-bounces@python.org.ar Wed Jan 8 14:46:02 2014 -Return-Path: -X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net -X-Spam-Level: ** -X-Spam-Pyzor: Reported 0 times. -X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE, - CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP, - NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled - version=3.3.2 -Delivered-To: kali@leap.se -Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33]) - (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) - (Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified)) - by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F - for ; Wed, 8 Jan 2014 18:46:02 +0000 (UTC) -Received: from pyar.usla.org.ar (unknown [190.228.30.157]) - by mx1.riseup.net (Postfix) with ESMTP id F244C533F4 - for ; Wed, 8 Jan 2014 10:46:01 -0800 (PST) -Received: from [127.0.0.1] (localhost [127.0.0.1]) - by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F - for ; Wed, 8 Jan 2014 15:46:00 -0300 (ART) -MIME-Version: 1.0 -Content-Type: text/plain; charset="iso-8859-1" -Content-Transfer-Encoding: quoted-printable -From: pyar-request@python.org.ar -To: kali@leap.se -Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 -Reply-To: pyar-request@python.org.ar -Auto-Submitted: auto-replied -Message-ID: -Date: Wed, 08 Jan 2014 15:45:59 -0300 -Precedence: bulk -X-BeenThere: pyar@python.org.ar -X-Mailman-Version: 2.1.15 -List-Id: Python Argentina -X-List-Administrivia: yes -Errors-To: pyar-bounces@python.org.ar -Sender: "pyar" -X-Virus-Scanned: clamav-milter 0.97.8 at mx1 -X-Virus-Status: Clean - -Mailing list subscription confirmation notice for mailing list pyar - -We have received a request de kaliyuga@riseup.net for subscription of -your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar -mailing list. To confirm that you want to be added to this mailing -list, simply reply to this message, keeping the Subject: header -intact. Or visit this web page: - - http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b= -3377148ac2 - - -Or include the following line -- and only the following line -- in a -message to pyar-request@python.org.ar: - - confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 - -Note that simply sending a `reply' to this message should work from -most mail readers, since that usually leaves the Subject: line in the -right form (additional "Re:" text in the Subject: is okay). - -If you do not wish to be subscribed to this list, please simply -disregard this message. If you think you are being maliciously -subscribed to the list, or have any other questions, send them to -pyar-owner@python.org.ar. diff --git a/src/leap/mail/imap/tests/rfc822.plain.message b/src/leap/mail/imap/tests/rfc822.plain.message new file mode 120000 index 0000000..5bab0e8 --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.plain.message @@ -0,0 +1 @@ +../../tests/rfc822.plain.message \ No newline at end of file diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 482b64d..55e50f7 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -17,15 +17,24 @@ """ Generic Access to Mail objects: Public LEAP Mail API. """ +import logging +import StringIO + from twisted.internet import defer +from leap.common.check import leap_assert_type +from leap.common.mail import get_email_charset + +from leap.mail.adaptors.soledad import SoledadMailAdaptor from leap.mail.constants import INBOX_NAME from leap.mail.constants import MessageFlags from leap.mail.mailbox_indexer import MailboxIndexer -from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.mail.utils import empty, find_charset + +logger = logging.getLogger(name=__name__) -# TODO +# TODO LIST # [ ] Probably change the name of this module to "api" or "account", mail is # too generic (there's also IncomingMail, and OutgoingMail @@ -36,16 +45,92 @@ def _get_mdoc_id(mbox, chash): return "M+{mbox}+{chash}".format(mbox=mbox, chash=chash) +def _write_and_rewind(payload): + fd = StringIO.StringIO() + fd.write(payload) + fd.seek(0) + return fd + + +class MessagePart(object): + + # TODO pass cdocs in init + + def __init__(self, part_map, cdocs={}): + self._pmap = part_map + self._cdocs = cdocs + + def get_size(self): + return self._pmap['size'] + + def get_body_file(self): + pmap = self._pmap + multi = pmap.get('multi') + if not multi: + phash = pmap.get("phash") + else: + pmap_ = pmap.get('part_map') + first_part = pmap_.get('1', None) + if not empty(first_part): + phash = first_part['phash'] + else: + phash = "" + + payload = self._get_payload(phash) + + if payload: + # FIXME + # content_type = self._get_ctype_from_document(phash) + # charset = find_charset(content_type) + charset = None + if charset is None: + charset = get_email_charset(payload) + try: + if isinstance(payload, unicode): + payload = payload.encode(charset) + except UnicodeError as exc: + logger.error( + "Unicode error, using 'replace'. {0!r}".format(exc)) + payload = payload.encode(charset, 'replace') + + return _write_and_rewind(payload) + + def get_headers(self): + return self._pmap.get("headers", []) + + def is_multipart(self): + multi = self._pmap.get("multi", False) + return multi + + def get_subpart(self, part): + if not self.is_multipart(): + raise TypeError + + sub_pmap = self._pmap.get("part_map", {}) + try: + part_map = sub_pmap[str(part + 1)] + except KeyError: + logger.debug("getSubpart for %s: KeyError" % (part,)) + raise IndexError + return MessagePart(self._soledad, part_map) + + def _get_payload(self, phash): + return self._cdocs.get(phash, "") + + class Message(object): """ Represents a single message, and gives access to all its attributes. """ - def __init__(self, wrapper): + def __init__(self, wrapper, uid=None): """ :param wrapper: an instance of an implementor of IMessageWrapper + :param uid: + :type uid: int """ self._wrapper = wrapper + self._uid = uid def get_wrapper(self): """ @@ -53,10 +138,18 @@ class Message(object): """ return self._wrapper + def get_uid(self): + """ + Get the (optional) UID. + """ + return self._uid + # imap.IMessage methods def get_flags(self): """ + Get flags for this message. + :rtype: tuple """ return tuple(self._wrapper.fdoc.flags) @@ -81,28 +174,50 @@ class Message(object): def get_headers(self): """ + Get the raw headers document. """ - # XXX process here? from imap.messages - return self._wrapper.hdoc.headers + return [tuple(item) for item in self._wrapper.hdoc.headers] - def get_body_file(self): + def get_body_file(self, store): """ """ + def write_and_rewind_if_found(cdoc): + if not cdoc: + return None + return _write_and_rewind(cdoc.raw) + + d = defer.maybeDeferred(self._wrapper.get_body, store) + d.addCallback(write_and_rewind_if_found) + return d def get_size(self): """ + Size, in octets. """ return self._wrapper.fdoc.size def is_multipart(self): """ + Return True if this message is multipart. """ return self._wrapper.fdoc.multi def get_subpart(self, part): """ - """ - # XXX ??? return MessagePart? + :param part: The number of the part to retrieve, indexed from 0. + :type part: int + :rtype: MessagePart + """ + if not self.is_multipart(): + raise TypeError + part_index = part + 1 + try: + subpart_dict = self._wrapper.get_subpart_dict( + part_index) + except KeyError: + raise TypeError + # XXX pass cdocs + return MessagePart(subpart_dict) # Custom methods. @@ -137,7 +252,7 @@ class MessageCollection(object): instance or proxy, for instance). """ - # TODO + # TODO LIST # [ ] look at IMessageSet methods # [ ] make constructor with a per-instance deferredLock to use on # creation/deletion? @@ -159,7 +274,7 @@ class MessageCollection(object): self.adaptor = adaptor self.store = store - # TODO I have to think about what to do when there is no mbox passed to + # XXX I have to think about what to do when there is no mbox passed to # the initialization. We could still get the MetaMsg by index, instead # of by doc_id. See get_message_by_content_hash self.mbox_indexer = mbox_indexer @@ -218,10 +333,11 @@ class MessageCollection(object): raise NotImplementedError("Does not support relative ids yet") def get_msg_from_mdoc_id(doc_id): - # XXX pass UID? + if doc_id is None: + return None return self.adaptor.get_msg_from_mdoc_id( self.messageklass, self.store, - doc_id, get_cdocs=get_cdocs) + doc_id, uid=uid, get_cdocs=get_cdocs) d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_name, uid) d.addCallback(get_msg_from_mdoc_id) @@ -248,26 +364,37 @@ class MessageCollection(object): # Manipulate messages - # TODO pass flags, date too... - def add_msg(self, raw_msg): + def add_msg(self, raw_msg, flags=None, tags=None, date=None): """ Add a message to this collection. """ + if not flags: + flags = tuple() + if not tags: + tags = tuple() + leap_assert_type(flags, tuple) + leap_assert_type(date, str) + msg = self.adaptor.get_msg_from_string(Message, raw_msg) wrapper = msg.get_wrapper() - if self.is_mailbox_collection(): + if not self.is_mailbox_collection(): + raise NotImplementedError() + + else: mbox = self.mbox_name + wrapper.set_flags(flags) + wrapper.set_tags(tags) + wrapper.set_date(date) wrapper.set_mbox(mbox) - def insert_mdoc_id(_): - # XXX does this work? + def insert_mdoc_id(_, wrapper): doc_id = wrapper.mdoc.doc_id return self.mbox_indexer.insert_doc( self.mbox_name, doc_id) d = wrapper.create(self.store) - d.addCallback(insert_mdoc_id) + d.addCallback(insert_mdoc_id, wrapper) return d def copy_msg(self, msg, newmailbox): @@ -338,6 +465,8 @@ class MessageCollection(object): return self.adaptor.update_msg(self.store, msg) +# TODO -------------------- split into account object? + class Account(object): """ Account is the top level abstraction to access collections of messages diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index bc298ea..1ceaec0 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -22,6 +22,17 @@ import re from leap.mail.constants import METAMSGID_RE +def _maybe_first_query_item(thing): + """ + Return the first item the returned query result, or None + if empty. + """ + try: + return thing[0][0] + except IndexError: + return None + + class WrongMetaDocIDError(Exception): pass @@ -124,7 +135,7 @@ class MailboxIndexer(object): raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") def get_rowid(result): - return result[0][0] + return _maybe_first_query_item(result) sql = ("INSERT INTO {preffix}{name} VALUES (" "NULL, ?)".format( @@ -192,7 +203,7 @@ class MailboxIndexer(object): :rtype: Deferred """ def get_hash(result): - return result[0][0] + return _maybe_first_query_item(result) sql = ("SELECT hash from {preffix}{name} " "WHERE uid=?".format( @@ -217,7 +228,7 @@ class MailboxIndexer(object): :rtype: Deferred """ def get_count(result): - return result[0][0] + return _maybe_first_query_item(result) sql = ("SELECT Count(*) FROM {preffix}{name};".format( preffix=self.table_preffix, name=mailbox)) @@ -243,7 +254,10 @@ class MailboxIndexer(object): assert mailbox def increment(result): - return result[0][0] + 1 + uid = _maybe_first_query_item(result) + if uid is None: + return None + return uid + 1 sql = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( diff --git a/src/leap/mail/tests/rfc822.multi-minimal.message b/src/leap/mail/tests/rfc822.multi-minimal.message new file mode 100644 index 0000000..582297c --- /dev/null +++ b/src/leap/mail/tests/rfc822.multi-minimal.message @@ -0,0 +1,16 @@ +Content-Type: multipart/mixed; boundary="===============6203542367371144092==" +MIME-Version: 1.0 +Subject: [TEST] 010 - Inceptos cum lorem risus congue +From: testmailbitmaskspam@gmail.com +To: test_c5@dev.bitmask.net + +--===============6203542367371144092== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +Howdy from python! +The subject: [TEST] 010 - Inceptos cum lorem risus congue +Current date & time: Wed Jan 8 16:36:21 2014 +Trying to attach: [] +--===============6203542367371144092==-- diff --git a/src/leap/mail/tests/rfc822.multi-signed.message b/src/leap/mail/tests/rfc822.multi-signed.message new file mode 100644 index 0000000..9907c2d --- /dev/null +++ b/src/leap/mail/tests/rfc822.multi-signed.message @@ -0,0 +1,238 @@ +Date: Mon, 6 Jan 2014 04:40:47 -0400 +From: Kali Kaneko +To: penguin@example.com +Subject: signed message +Message-ID: <20140106084047.GA21317@samsara.lan> +MIME-Version: 1.0 +Content-Type: multipart/signed; micalg=pgp-sha1; + protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy" +Content-Disposition: inline +User-Agent: Mutt/1.5.21 (2012-12-30) + + +--z9ECzHErBrwFF8sy +Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l" +Content-Disposition: inline + + +--z0eOaCaDLjvTGF2l +Content-Type: text/plain; charset=utf-8 +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable + +This is an example of a signed message, +with attachments. + + +--=20 +Nihil sine chao! =E2=88=B4 + +--z0eOaCaDLjvTGF2l +Content-Type: text/plain; charset=us-ascii +Content-Disposition: attachment; filename="attach.txt" + +this is attachment in plain text. + +--z0eOaCaDLjvTGF2l +Content-Type: application/octet-stream +Content-Disposition: attachment; filename="hack.ico" +Content-Transfer-Encoding: base64 + +AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA +KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG +RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA +PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl +5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA +/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ +yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A +Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK +ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK +LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP +QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy +AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs +AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA +AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA +gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d +HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA +x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7 ++wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA +AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5 ++QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA +OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK +igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA +JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra +2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA +xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj +owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB +AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA +AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d +XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d +XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA +AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB +AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm +X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC +AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B +bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ +S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu +J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y +AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N +KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB +XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A +AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA +AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d +XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA +AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr +RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA +AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A +Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI +yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA +CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys +rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA +vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d +HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA +urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx +cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA +CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo +6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA +2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7 +OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA +UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp +qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA +lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa +WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB +AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB +AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB +AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA +ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA +AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB +AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB +AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB +AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB +AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA +tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA +AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF +wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB +AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2 +RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB +AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB +AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA +AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd +AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB +AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB +AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB +AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB +AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB +AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8 +ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2 +NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF +RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB +lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA +AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa +WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA +AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX +AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB +AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB +AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA +AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA +AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA +AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA +AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB +AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB +AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA +ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA +AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7 +LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA +AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF +NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB +AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2 +RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA +ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4 +RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi +JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2 +NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK +T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB +AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB +AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB +AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN +UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA +AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA +W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA +AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB +l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB +AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+ +WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA +AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv +RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA +AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj +AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB +AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB +AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA +AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA +AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA +dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A +AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB +AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB +AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB +AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW +pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + +--z0eOaCaDLjvTGF2l-- + +--z9ECzHErBrwFF8sy +Content-Type: application/pgp-signature + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.15 (GNU/Linux) + +iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv +kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl +vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK +PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC +w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw +sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr +BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN +QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt +mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ +jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8 +gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X +sSdfcAhT7Tno7PB/Acoh +=+okv +-----END PGP SIGNATURE----- + +--z9ECzHErBrwFF8sy-- diff --git a/src/leap/mail/tests/rfc822.multi.message b/src/leap/mail/tests/rfc822.multi.message new file mode 100644 index 0000000..30f74e5 --- /dev/null +++ b/src/leap/mail/tests/rfc822.multi.message @@ -0,0 +1,96 @@ +Date: Fri, 19 May 2000 09:55:48 -0400 (EDT) +From: Doug Sauder +To: Joe Blow +Subject: Test message from PINE +Message-ID: +MIME-Version: 1.0 +Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452" + + This message is in MIME format. The first part should be readable text, + while the remaining parts are likely unreadable without MIME-aware tools. + Send mail to mime@docserver.cac.washington.edu for more info. + +---1463757054-952513540-958744548=:8452 +Content-Type: TEXT/PLAIN; charset=US-ASCII + +This is a test message from PINE MUA. + + +---1463757054-952513540-958744548=:8452 +Content-Type: APPLICATION/octet-stream; name="redball.png" +Content-Transfer-Encoding: BASE64 +Content-ID: +Content-Description: A PNG graphic file +Content-Disposition: attachment; filename="redball.png" + +iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A +AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj +AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv +AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0 +GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf +hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl +rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL +ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS +AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP +AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP +AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI +AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR +AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6 +AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO +AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8 +AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8 +LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu +MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF +BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp +6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow +8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G +ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn +OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1 +a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T +VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8 +Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f +lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/ +joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms +1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA +JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7 +vAAAAABJRU5ErkJggg== +---1463757054-952513540-958744548=:8452 +Content-Type: APPLICATION/octet-stream; name="blueball.png" +Content-Transfer-Encoding: BASE64 +Content-ID: +Content-Description: A PNG graphic file +Content-Disposition: attachment; filename="blueball.png" + +iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A +AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI +IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y +Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S +hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM +vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB +Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu +MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b +fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo +Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp +LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX +P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M +jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8 ++VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE +1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42 +YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY +mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp +Z3VldDZzO7wAAAAASUVORK5CYII= +---1463757054-952513540-958744548=:8452-- diff --git a/src/leap/mail/tests/rfc822.plain.message b/src/leap/mail/tests/rfc822.plain.message new file mode 100644 index 0000000..fc627c3 --- /dev/null +++ b/src/leap/mail/tests/rfc822.plain.message @@ -0,0 +1,66 @@ +From pyar-bounces@python.org.ar Wed Jan 8 14:46:02 2014 +Return-Path: +X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net +X-Spam-Level: ** +X-Spam-Pyzor: Reported 0 times. +X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE, + CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP, + NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled + version=3.3.2 +Delivered-To: kali@leap.se +Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33]) + (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) + (Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified)) + by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F + for ; Wed, 8 Jan 2014 18:46:02 +0000 (UTC) +Received: from pyar.usla.org.ar (unknown [190.228.30.157]) + by mx1.riseup.net (Postfix) with ESMTP id F244C533F4 + for ; Wed, 8 Jan 2014 10:46:01 -0800 (PST) +Received: from [127.0.0.1] (localhost [127.0.0.1]) + by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F + for ; Wed, 8 Jan 2014 15:46:00 -0300 (ART) +MIME-Version: 1.0 +Content-Type: text/plain; charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable +From: pyar-request@python.org.ar +To: kali@leap.se +Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 +Reply-To: pyar-request@python.org.ar +Auto-Submitted: auto-replied +Message-ID: +Date: Wed, 08 Jan 2014 15:45:59 -0300 +Precedence: bulk +X-BeenThere: pyar@python.org.ar +X-Mailman-Version: 2.1.15 +List-Id: Python Argentina +X-List-Administrivia: yes +Errors-To: pyar-bounces@python.org.ar +Sender: "pyar" +X-Virus-Scanned: clamav-milter 0.97.8 at mx1 +X-Virus-Status: Clean + +Mailing list subscription confirmation notice for mailing list pyar + +We have received a request de kaliyuga@riseup.net for subscription of +your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar +mailing list. To confirm that you want to be added to this mailing +list, simply reply to this message, keeping the Subject: header +intact. Or visit this web page: + + http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b= +3377148ac2 + + +Or include the following line -- and only the following line -- in a +message to pyar-request@python.org.ar: + + confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 + +Note that simply sending a `reply' to this message should work from +most mail readers, since that usually leaves the Subject: line in the +right form (additional "Re:" text in the Subject: is okay). + +If you do not wish to be subscribed to this list, please simply +disregard this message. If you think you are being maliciously +subscribed to the list, or have any other questions, send them to +pyar-owner@python.org.ar. diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py index ce2366c..cb97be5 100644 --- a/src/leap/mail/tests/test_mail.py +++ b/src/leap/mail/tests/test_mail.py @@ -17,24 +17,48 @@ """ Tests for the mail module. """ +import time import os from functools import partial +from email.parser import Parser +from email.Utils import formatdate + +from twisted.python import util from leap.mail.adaptors.soledad import SoledadMailAdaptor from leap.mail.mail import MessageCollection from leap.mail.mailbox_indexer import MailboxIndexer from leap.mail.tests.common import SoledadTestMixin -from twisted.internet import defer +# from twisted.internet import defer from twisted.trial import unittest HERE = os.path.split(os.path.abspath(__file__))[0] -class MessageCollectionTestCase(unittest.TestCase, SoledadTestMixin): - """ - Tests for the SoledadDocumentWrapper. - """ +def _get_raw_msg(multi=False): + if multi: + sample = "rfc822.multi.message" + else: + sample = "rfc822.message" + with open(os.path.join(HERE, sample)) as f: + raw = f.read() + return raw + + +def _get_parsed_msg(multi=False): + mail_parser = Parser() + raw = _get_raw_msg(multi=multi) + return mail_parser.parsestr(raw) + + +def _get_msg_time(): + timestamp = time.mktime((2010, 12, 12, 1, 1, 1, 1, 1, 1)) + return formatdate(timestamp) + + + +class CollectionMixin(object): def get_collection(self, mbox_collection=True): """ @@ -61,35 +85,224 @@ class MessageCollectionTestCase(unittest.TestCase, SoledadTestMixin): d.addCallback(get_collection_from_mbox_wrapper) return d - def test_is_mailbox_collection(self): - def assert_is_mbox_collection(collection): - self.assertTrue(collection.is_mailbox_collection()) +class MessageTestCase(unittest.TestCase, SoledadTestMixin, CollectionMixin): + """ + Tests for the Message class. + """ + msg_flags = ('\Recent', '\Unseen', '\TestFlag') + msg_tags = ('important', 'todo', 'wonderful') + internal_date = "19-Mar-2015 19:22:21 -0500" + maxDiff = None + + def _do_insert_msg(self, multi=False): + """ + Inserts and return a regular message, for tests. + """ + raw = _get_raw_msg(multi=multi) d = self.get_collection() - d.addCallback(assert_is_mbox_collection) + d.addCallback(lambda col: col.add_msg( + raw, flags=self.msg_flags, tags=self.msg_tags, + date=self.internal_date)) + return d + + def get_inserted_msg(self, multi=False): + d = self._do_insert_msg(multi=multi) + d.addCallback(lambda _: self.get_collection()) + d.addCallback(lambda col: col.get_message_by_uid(1)) + return d + + def test_get_flags(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_flags_cb) + return d + + def _test_get_flags_cb(self, msg): + self.assertTrue(msg is not None) + self.assertEquals(msg.get_flags(), self.msg_flags) + + def test_get_internal_date(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_internal_date_cb) + + def _test_get_internal_date_cb(self, msg): + self.assertTrue(msg is not None) + self.assertDictEqual(msg.get_internal_date(), + self.internal_date) + + def test_get_headers(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_headers_cb) + return d + + def _test_get_headers_cb(self, msg): + self.assertTrue(msg is not None) + expected = _get_parsed_msg().items() + self.assertEqual(msg.get_headers(), expected) + + def test_get_body_file(self): + d = self.get_inserted_msg(multi=True) + d.addCallback(self._test_get_body_file_cb) + return d + + def _test_get_body_file_cb(self, msg): + self.assertTrue(msg is not None) + orig = _get_parsed_msg(multi=True) + expected = orig.get_payload()[0].get_payload() + d = msg.get_body_file(self._soledad) + + def assert_body(fd): + self.assertTrue(fd is not None) + self.assertEqual(fd.read(), expected) + d.addCallback(assert_body) + return d + + def test_get_size(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_size_cb) + return d + + def _test_get_size_cb(self, msg): + self.assertTrue(msg is not None) + expected = len(_get_parsed_msg().as_string()) + self.assertEqual(msg.get_size(), expected) + + def test_is_multipart_no(self): + d = self.get_inserted_msg() + d.addCallback(self._test_is_multipart_no_cb) + return d + + def _test_is_multipart_no_cb(self, msg): + self.assertTrue(msg is not None) + expected = _get_parsed_msg().is_multipart() + self.assertEqual(msg.is_multipart(), expected) + + def test_is_multipart_yes(self): + d = self.get_inserted_msg(multi=True) + d.addCallback(self._test_is_multipart_yes_cb) + return d + + def _test_is_multipart_yes_cb(self, msg): + self.assertTrue(msg is not None) + expected = _get_parsed_msg(multi=True).is_multipart() + self.assertEqual(msg.is_multipart(), expected) + + def test_get_subpart(self): + d = self.get_inserted_msg(multi=True) + d.addCallback(self._test_get_subpart_cb) + return d + + def _test_get_subpart_cb(self, msg): + self.assertTrue(msg is not None) + + def test_get_tags(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_tags_cb) return d - def assert_collection_count(self, _, expected, collection): + def _test_get_tags_cb(self, msg): + self.assertTrue(msg is not None) + self.assertEquals(msg.get_tags(), self.msg_tags) + +class MessageCollectionTestCase(unittest.TestCase, + SoledadTestMixin, CollectionMixin): + """ + Tests for the MessageCollection class. + """ + def assert_collection_count(self, _, expected): def _assert_count(count): self.assertEqual(count, expected) - d = collection.count() + + d = self.get_collection() + d.addCallback(lambda col: col.count()) d.addCallback(_assert_count) return d - def test_add_msg(self): + def add_msg_to_collection(self): + raw = _get_raw_msg() - with open(os.path.join(HERE, "rfc822.message")) as f: - raw = f.read() - - def add_msg_to_collection_and_assert_count(collection): - d = collection.add_msg(raw) - d.addCallback(partial( - self.assert_collection_count, - expected=1, collection=collection)) + def add_msg_to_collection(collection): + d = collection.add_msg(raw, date=_get_msg_time()) return d + d = self.get_collection() + d.addCallback(add_msg_to_collection) + return d + def test_is_mailbox_collection(self): d = self.get_collection() - d.addCallback(add_msg_to_collection_and_assert_count) + d.addCallback(self._test_is_mailbox_collection_cb) + return d + + def _test_is_mailbox_collection_cb(self, collection): + self.assertTrue(collection.is_mailbox_collection()) + + def test_get_uid_next(self): + d = self.add_msg_to_collection() + d.addCallback(lambda _: self.get_collection()) + d.addCallback(lambda col: col.get_uid_next()) + d.addCallback(self._test_get_uid_next_cb) + + def _test_get_uid_next_cb(self, next_uid): + self.assertEqual(next_uid, 2) + + def test_add_and_count_msg(self): + d = self.add_msg_to_collection() + d.addCallback(self._test_add_and_count_msg_cb) + return d + + def _test_add_and_count_msg_cb(self, _): + return partial(self.assert_collection_count, expected=1) + + def test_coppy_msg(self): + self.fail() + + def test_delete_msg(self): + self.fail() + + def test_update_flags(self): + d = self.add_msg_to_collection() + d.addCallback(self._test_update_flags_cb) + return d + + def _test_update_flags_cb(self, msg): + pass + + def test_update_tags(self): + d = self.add_msg_to_collection() + d.addCallback(self._test_update_tags_cb) return d + + def _test_update_tags_cb(self, msg): + pass + + +class AccountTestCase(unittest.TestCase, SoledadTestMixin): + """ + Tests for the Account class. + """ + + def test_add_mailbox(self): + self.fail() + + def test_delete_mailbox(self): + self.fail() + + def test_rename_mailbox(self): + self.fail() + + def test_list_all_mailbox_names(self): + self.fail() + + def test_get_all_mailboxes(self): + self.fail() + + def test_get_collection_by_docs(self): + self.fail() + + def test_get_collection_by_mailbox(self): + self.fail() + + def test_get_collection_by_tag(self): + self.fail() diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 5172837..8653a5f 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # walk.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -56,15 +56,15 @@ get_payloads = lambda msg: ((x.get_payload(), dict(((str.lower(k), v) for k, v in (x.items())))) for x in msg.walk()) -get_body_phash_simple = lambda payloads: first( - [get_hash(payload) for payload, headers in payloads - if payloads]) -get_body_phash_multi = lambda payloads: (first( - [get_hash(payload) for payload, headers in payloads - if payloads - and "text/plain" in headers.get('content-type', '')]) - or get_body_phash_simple(payloads)) +def get_body_phash(msg): + """ + Find the body payload-hash for this message. + """ + for part in msg.walk(): + if part.get_content_type() == "text/plain": + # XXX avoid hashing again + return get_hash(part.get_payload()) """ On getting the raw docs, we get also some of the headers to be able to -- cgit v1.2.3 From ce6008a326dfc8020b341904308115917eabd7de Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 6 Jan 2015 01:31:26 -0400 Subject: tests for mail.mail module: MessageCollection --- src/leap/mail/adaptors/soledad.py | 44 +++++++++++++++++++++++---------------- src/leap/mail/mail.py | 5 ++--- src/leap/mail/tests/test_mail.py | 18 +++++++++++++++- 3 files changed, 45 insertions(+), 22 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index f0808af..522d2d3 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -290,7 +290,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): in the model as the `list_index`. :rtype: Deferred """ - # TODO + # TODO LIST (get_all) # [ ] extend support to indexes with n-ples # [ ] benchmark the cost of querying and returning indexes in a big # database. This might badly need pagination before being put to @@ -428,19 +428,28 @@ class MessageWrapper(object): content of the content-docs. """ if isinstance(mdoc, SoledadDocument): + mdoc_id = mdoc.doc_id mdoc = mdoc.content + else: + mdoc_id = None if not mdoc: mdoc = {} - self.mdoc = MetaMsgDocWrapper(**mdoc) + self.mdoc = MetaMsgDocWrapper(doc_id=mdoc_id, **mdoc) if isinstance(fdoc, SoledadDocument): + fdoc_id = fdoc.doc_id fdoc = fdoc.content - self.fdoc = FlagsDocWrapper(**fdoc) + else: + fdoc_id = None + self.fdoc = FlagsDocWrapper(doc_id=fdoc_id, **fdoc) self.fdoc.set_future_doc_id(self.mdoc.fdoc) if isinstance(hdoc, SoledadDocument): + hdoc_id = hdoc.doc_id hdoc = hdoc.content - self.hdoc = HeaderDocWrapper(**hdoc) + else: + hdoc_id = None + self.hdoc = HeaderDocWrapper(doc_id=hdoc_id, **hdoc) self.hdoc.set_future_doc_id(self.mdoc.hdoc) if cdocs is None: @@ -489,9 +498,15 @@ class MessageWrapper(object): return self.fdoc.update(store) def delete(self, store): + # TODO # Eventually this would have to do the duplicate search or send for the - # garbage collector. At least the fdoc can be unlinked. - raise NotImplementedError() + # garbage collector. At least mdoc and t the mdoc and fdoc can be + # unlinked. + d = [] + if self.mdoc.doc_id: + d.append(self.mdoc.delete(store)) + d.append(self.fdoc.delete(store)) + return defer.gatherResults(d) def copy(self, store, newmailbox): """ @@ -565,9 +580,6 @@ class MailboxWrapper(SoledadDocumentWrapper): closed = False subscribed = False - # I think we don't need to store this one. - # rw = True - class __meta__(object): index = "mbox" list_index = (indexes.TYPE_IDX, 'type_') @@ -717,9 +729,8 @@ class SoledadMailAdaptor(SoledadIndexMixin): return self.get_msg_from_docs( msg_class, mdoc, fdoc, hdoc, cdocs, uid=None) - def get_msg_from_mdoc_id(self, MessageClass, store, doc_id, + def get_msg_from_mdoc_id(self, MessageClass, store, mdoc_id, uid=None, get_cdocs=False): - metamsg_id = doc_id def wrap_meta_doc(doc): cls = MetaMsgDocWrapper @@ -740,8 +751,8 @@ class SoledadMailAdaptor(SoledadIndexMixin): return d def get_parts_doc_from_mdoc_id(): - mbox = re.findall(constants.METAMSGID_MBOX_RE, doc_id)[0] - chash = re.findall(constants.METAMSGID_CHASH_RE, doc_id)[0] + mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0] + chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0] def _get_fdoc_id_from_mdoc_id(): return constants.FDOCID.format(mbox=mbox, chash=chash) @@ -753,22 +764,19 @@ class SoledadMailAdaptor(SoledadIndexMixin): fdoc_id = _get_fdoc_id_from_mdoc_id() hdoc_id = _get_hdoc_id_from_mdoc_id() + d_docs.append(store.get_doc(mdoc_id)) d_docs.append(store.get_doc(fdoc_id)) d_docs.append(store.get_doc(hdoc_id)) d = defer.gatherResults(d_docs) return d - def add_mdoc_id_placeholder(docs_list): - return [None] + docs_list - if get_cdocs: - d = store.get_doc(metamsg_id) + d = store.get_doc(mdoc_id) d.addCallback(wrap_meta_doc) d.addCallback(get_part_docs_from_mdoc_wrapper) else: d = get_parts_doc_from_mdoc_id() - d.addCallback(add_mdoc_id_placeholder) d.addCallback(partial(self._get_msg_from_variable_doc_list, msg_class=MessageClass, uid=uid)) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 55e50f7..671642a 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -420,13 +420,12 @@ class MessageCollection(object): """ wrapper = msg.get_wrapper() - def delete_mdoc_id(_): - # XXX does this work? + def delete_mdoc_id(_, wrapper): doc_id = wrapper.mdoc.doc_id return self.mbox_indexer.delete_doc_by_hash( self.mbox_name, doc_id) d = wrapper.delete(self.store) - d.addCallback(delete_mdoc_id) + d.addCallback(delete_mdoc_id, wrapper) return d # TODO should add a delete-by-uid to collection? diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py index cb97be5..d11df40 100644 --- a/src/leap/mail/tests/test_mail.py +++ b/src/leap/mail/tests/test_mail.py @@ -259,7 +259,23 @@ class MessageCollectionTestCase(unittest.TestCase, self.fail() def test_delete_msg(self): - self.fail() + d = self.add_msg_to_collection() + + def del_msg(collection): + def _delete_it(msg): + return collection.delete_msg(msg) + + d = collection.get_message_by_uid(1) + d.addCallback(_delete_it) + return d + + d.addCallback(lambda _: self.get_collection()) + d.addCallback(del_msg) + d.addCallback(self._test_delete_msg_cb) + return d + + def _test_delete_msg_cb(self, _): + return partial(self.assert_collection_count, expected=0) def test_update_flags(self): d = self.add_msg_to_collection() -- cgit v1.2.3 From 9e2cf40db51889ec43cd4e27b55c7f14e0436c01 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 6 Jan 2015 02:19:02 -0400 Subject: tests for mail.mail module: Account --- src/leap/mail/adaptors/soledad.py | 19 +++++---- src/leap/mail/mail.py | 46 ++++++++++++--------- src/leap/mail/mailbox_indexer.py | 2 +- src/leap/mail/tests/test_mail.py | 84 ++++++++++++++++++++++++++++++++------- 4 files changed, 110 insertions(+), 41 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 522d2d3..389307f 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -625,14 +625,11 @@ class SoledadIndexMixin(object): leap_assert(store, "Need a store") leap_assert_type(self.indexes, dict) - self._index_creation_deferreds = [] - def _on_indexes_created(ignored): - self.store_ready = True + self._index_creation_deferreds = [] def _create_index(name, expression): - d = store.create_index(name, *expression) - self._index_creation_deferreds.append(d) + return store.create_index(name, *expression) def _create_indexes(db_indexes): db_indexes = dict(db_indexes) @@ -640,7 +637,8 @@ class SoledadIndexMixin(object): for name, expression in self.indexes.items(): if name not in db_indexes: # The index does not yet exist. - _create_index(name, expression) + d = _create_index(name, expression) + self._index_creation_deferreds.append(d) continue if expression == db_indexes[name]: @@ -650,11 +648,16 @@ class SoledadIndexMixin(object): # we delete it and add the proper index expression. d1 = store.delete_index(name) d1.addCallback(lambda _: _create_index(name, expression)) + self._index_creation_deferreds.append(d1) - all_created = defer.gatherResults(self._index_creation_deferreds) + all_created = defer.gatherResults( + self._index_creation_deferreds, consumeErrors=True) all_created.addCallback(_on_indexes_created) return all_created + def _on_indexes_created(ignored): + self.store_ready = True + # Ask the database for currently existing indexes, and create them # if not found. d = store.list_indexes() @@ -832,9 +835,11 @@ class SoledadMailAdaptor(SoledadIndexMixin): have been updated. :rtype: defer.Deferred """ + leap_assert_type(mbox_wrapper, SoledadDocumentWrapper) return mbox_wrapper.update(store) def delete_mbox(self, store, mbox_wrapper): + leap_assert_type(mbox_wrapper, SoledadDocumentWrapper) return mbox_wrapper.delete(store) def get_all_mboxes(self, store): diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 671642a..0c9b7a3 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -37,6 +37,9 @@ logger = logging.getLogger(name=__name__) # TODO LIST # [ ] Probably change the name of this module to "api" or "account", mail is # too generic (there's also IncomingMail, and OutgoingMail +# [ ] Change the doc_ids scheme for part-docs: use mailbox UID validity +# identifier, instead of name! (renames are broken!) +# [ ] Profile add_msg. def _get_mdoc_id(mbox, chash): """ @@ -360,7 +363,7 @@ class MessageCollection(object): :return: a Deferred that will fire with the integer for the next uid. :rtype: Deferred """ - return self.mbox_indexer.get_uid_next(self.mbox_name) + return self.mbox_indexer.get_next_uid(self.mbox_name) # Manipulate messages @@ -464,8 +467,6 @@ class MessageCollection(object): return self.adaptor.update_msg(self.store, msg) -# TODO -------------------- split into account object? - class Account(object): """ Account is the top level abstraction to access collections of messages @@ -485,7 +486,6 @@ class Account(object): adaptor_class = SoledadMailAdaptor store = None - mailboxes = None def __init__(self, store): self.store = store @@ -508,17 +508,17 @@ class Account(object): self._deferred_initialization.callback(None) d = self.adaptor.initialize_store(self.store) - d.addCallback(self.list_all_mailbox_names) + d.addCallback(lambda _: self.list_all_mailbox_names()) d.addCallback(add_mailbox_if_none) d.addCallback(finish_initialization) - def callWhenReady(self, cb): - # XXX this could use adaptor.store_ready instead...?? + def callWhenReady(self, cb, *args, **kw): + # use adaptor.store_ready instead? if self._initialized: - cb(self) + cb(self, *args, **kw) return defer.succeed(None) else: - self._deferred_initialization.addCallback(cb) + self._deferred_initialization.addCallback(cb, *args, **kw) return self._deferred_initialization # @@ -527,7 +527,7 @@ class Account(object): def list_all_mailbox_names(self): def filter_names(mboxes): - return [m.name for m in mboxes] + return [m.mbox for m in mboxes] d = self.get_all_mailboxes() d.addCallback(filter_names) @@ -540,35 +540,44 @@ class Account(object): def add_mailbox(self, name): def create_uid_table_cb(res): - d = self.mbox_uid.create_table(name) + d = self.mbox_indexer.create_table(name) d.addCallback(lambda _: res) return d - d = self.adaptor.__class__.get_or_create(name) + d = self.adaptor.get_or_create_mbox(self.store, name) d.addCallback(create_uid_table_cb) return d def delete_mailbox(self, name): def delete_uid_table_cb(res): - d = self.mbox_uid.delete_table(name) + d = self.mbox_indexer.delete_table(name) d.addCallback(lambda _: res) return d - d = self.adaptor.delete_mbox(self.store) + d = self.adaptor.get_or_create_mbox(self.store, name) + d.addCallback( + lambda wrapper: self.adaptor.delete_mbox(self.store, wrapper)) d.addCallback(delete_uid_table_cb) return d def rename_mailbox(self, oldname, newname): + # TODO incomplete/wrong!!! + # Should rename also ALL of the document ids that are pointing + # to the old mailbox!!! + + # TODO part-docs identifiers should have the UID_validity of the + # mailbox embedded, instead of the name! (so they can survive a rename) + def _rename_mbox(wrapper): wrapper.mbox = newname - return wrapper.update() + return wrapper.update(self.store) def rename_uid_table_cb(res): - d = self.mbox_uid.rename_table(oldname, newname) + d = self.mbox_indexer.rename_table(oldname, newname) d.addCallback(lambda _: res) return d - d = self.adaptor.__class__.get_or_create(oldname) + d = self.adaptor.get_or_create_mbox(self.store, oldname) d.addCallback(_rename_mbox) d.addCallback(rename_uid_table_cb) return d @@ -585,7 +594,8 @@ class Account(object): self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) mboxwrapper_klass = self.adaptor.mboxwrapper_klass - d = mboxwrapper_klass.get_or_create(name) + #d = mboxwrapper_klass.get_or_create(name) + d = self.adaptor.get_or_create_mbox(self.store, name) d.addCallback(get_collection_for_mailbox) return d diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 1ceaec0..e5b813f 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -256,7 +256,7 @@ class MailboxIndexer(object): def increment(result): uid = _maybe_first_query_item(result) if uid is None: - return None + return 1 return uid + 1 sql = ("SELECT MAX(rowid) FROM {preffix}{name} " diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py index d11df40..2c4b919 100644 --- a/src/leap/mail/tests/test_mail.py +++ b/src/leap/mail/tests/test_mail.py @@ -23,10 +23,8 @@ from functools import partial from email.parser import Parser from email.Utils import formatdate -from twisted.python import util - from leap.mail.adaptors.soledad import SoledadMailAdaptor -from leap.mail.mail import MessageCollection +from leap.mail.mail import MessageCollection, Account from leap.mail.mailbox_indexer import MailboxIndexer from leap.mail.tests.common import SoledadTestMixin @@ -57,7 +55,6 @@ def _get_msg_time(): return formatdate(timestamp) - class CollectionMixin(object): def get_collection(self, mbox_collection=True): @@ -86,6 +83,7 @@ class CollectionMixin(object): return d +# TODO profile add_msg. Why are these tests so SLOW??! class MessageTestCase(unittest.TestCase, SoledadTestMixin, CollectionMixin): """ Tests for the Message class. @@ -256,7 +254,9 @@ class MessageCollectionTestCase(unittest.TestCase, return partial(self.assert_collection_count, expected=1) def test_coppy_msg(self): - self.fail() + # TODO ---- update when implementing messagecopier + # interface + self.fail("Not Yet Implemented") def test_delete_msg(self): d = self.add_msg_to_collection() @@ -298,27 +298,81 @@ class AccountTestCase(unittest.TestCase, SoledadTestMixin): """ Tests for the Account class. """ + def get_account(self): + store = self._soledad + return Account(store) def test_add_mailbox(self): - self.fail() + acc = self.get_account() + d = acc.callWhenReady(lambda _: acc.add_mailbox("TestMailbox")) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(self._test_add_mailbox_cb) + return d + + def _test_add_mailbox_cb(self, mboxes): + expected = ['INBOX', 'TestMailbox'] + self.assertItemsEqual(mboxes, expected) def test_delete_mailbox(self): - self.fail() + acc = self.get_account() + d = acc.callWhenReady(lambda _: acc.delete_mailbox("Inbox")) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(self._test_delete_mailbox_cb) + return d + + def _test_delete_mailbox_cb(self, mboxes): + expected = [] + self.assertItemsEqual(mboxes, expected) def test_rename_mailbox(self): - self.fail() + acc = self.get_account() + d = acc.callWhenReady(lambda _: acc.add_mailbox("TestMailbox")) + d = acc.callWhenReady(lambda _: acc.rename_mailbox( + "TestMailbox", "RenamedMailbox")) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(self._test_rename_mailbox_cb) + return d - def test_list_all_mailbox_names(self): - self.fail() + def _test_rename_mailbox_cb(self, mboxes): + expected = ['INBOX', 'RenamedMailbox'] + self.assertItemsEqual(mboxes, expected) def test_get_all_mailboxes(self): - self.fail() + acc = self.get_account() + d = acc.callWhenReady(lambda _: acc.add_mailbox("OneMailbox")) + d.addCallback(lambda _: acc.add_mailbox("TwoMailbox")) + d.addCallback(lambda _: acc.add_mailbox("ThreeMailbox")) + d.addCallback(lambda _: acc.add_mailbox("anotherthing")) + d.addCallback(lambda _: acc.add_mailbox("anotherthing2")) + d.addCallback(lambda _: acc.get_all_mailboxes()) + d.addCallback(self._test_get_all_mailboxes_cb) + return d - def test_get_collection_by_docs(self): - self.fail() + def _test_get_all_mailboxes_cb(self, mailboxes): + expected = ["INBOX", "OneMailbox", "TwoMailbox", "ThreeMailbox", + "anotherthing", "anotherthing2"] + names = [m.mbox for m in mailboxes] + self.assertItemsEqual(names, expected) def test_get_collection_by_mailbox(self): - self.fail() + acc = self.get_account() + d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox("INBOX")) + d.addCallback(self._test_get_collection_by_mailbox_cb) + return d + + def _test_get_collection_by_mailbox_cb(self, collection): + self.assertTrue(collection.is_mailbox_collection()) + + def assert_uid_next_empty_collection(uid): + self.assertEqual(uid, 1) + d = collection.get_uid_next() + d.addCallback(assert_uid_next_empty_collection) + return d + + # XXX not yet implemented + + def test_get_collection_by_docs(self): + self.fail("Not Yet Implemented") def test_get_collection_by_tag(self): - self.fail() + self.fail("Not Yet Implemented") -- cgit v1.2.3 From f7030295a936cc5da33f50411b5ff60ae0eec7cc Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 12 Jan 2015 20:47:29 -0400 Subject: Use mailbox uuids The previous implementation is naive, since it imposes a burden when renaming mailboxes. We're using uuids in the local uid tables instead, which is more cryptic but way more efficient. * receive mbox uuid instead of name * use mailbox uuid in identifiers --- src/leap/mail/adaptors/soledad.py | 29 ++-- .../mail/adaptors/tests/test_soledad_adaptor.py | 11 +- src/leap/mail/constants.py | 8 +- src/leap/mail/imap/mailbox.py | 12 +- src/leap/mail/imap/tests/test_imap.py | 26 ++- src/leap/mail/mail.py | 57 +++++-- src/leap/mail/mailbox_indexer.py | 101 ++++++------ src/leap/mail/tests/test_mailbox_indexer.py | 182 ++++++++++----------- 8 files changed, 235 insertions(+), 191 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 389307f..c5cfce0 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -338,7 +338,7 @@ class FlagsDocWrapper(SoledadDocumentWrapper): type_ = "flags" chash = "" - mbox = "inbox" + mbox_uuid = "" seen = False deleted = False recent = False @@ -350,11 +350,12 @@ class FlagsDocWrapper(SoledadDocumentWrapper): class __meta__(object): index = "mbox" - def set_mbox(self, mbox): + def set_mbox_uuid(self, mbox_uuid): # XXX raise error if already created, should use copy instead - new_id = constants.FDOCID.format(mbox=mbox, chash=self.chash) + mbox_uuid = mbox_uuid.replace('-', '_') + new_id = constants.FDOCID.format(mbox_uuid=mbox_uuid, chash=self.chash) self._future_doc_id = new_id - self.mbox = mbox + self.mbox_uuid = mbox_uuid class HeaderDocWrapper(SoledadDocumentWrapper): @@ -401,11 +402,12 @@ class MetaMsgDocWrapper(SoledadDocumentWrapper): hdoc = "" cdocs = [] - def set_mbox(self, mbox): + def set_mbox_uuid(self, mbox_uuid): # XXX raise error if already created, should use copy instead + mbox_uuid = mbox_uuid.replace('-', '_') chash = re.findall(constants.FDOCID_CHASH_RE, self.fdoc)[0] - new_id = constants.METAMSGID.format(mbox=mbox, chash=chash) - new_fdoc_id = constants.FDOCID.format(mbox=mbox, chash=chash) + new_id = constants.METAMSGID.format(mbox_uuid=mbox_uuid, chash=chash) + new_fdoc_id = constants.FDOCID.format(mbox_uuid=mbox_uuid, chash=chash) self._future_doc_id = new_id self.fdoc = new_fdoc_id @@ -518,14 +520,15 @@ class MessageWrapper(object): # 4. return new wrapper (new meta too!) raise NotImplementedError() - def set_mbox(self, mbox): + def set_mbox_uuid(self, mbox_uuid): """ Set the mailbox for this wrapper. This method should only be used before the Documents for the MessageWrapper have been created, will raise otherwise. """ - self.mdoc.set_mbox(mbox) - self.fdoc.set_mbox(mbox) + mbox_uuid = mbox.uuid.replace('-', '_') + self.mdoc.set_mbox_uuid(mbox_uuid) + self.fdoc.set_mbox_uuid(mbox_uuid) def set_flags(self, flags): # TODO serialize the get + update @@ -574,6 +577,7 @@ class MailboxWrapper(SoledadDocumentWrapper): class model(models.SerializableModel): type_ = "mbox" mbox = INBOX_NAME + uuid = None flags = [] recent = [] created = 1 @@ -889,7 +893,10 @@ def _parse_msg(raw): def _build_meta_doc(chash, cdocs_phashes): _mdoc = MetaMsgDocWrapper() - _mdoc.fdoc = constants.FDOCID.format(mbox=INBOX_NAME, chash=chash) + # FIXME passing the inbox name because we don't have the uuid at this + # point. + + _mdoc.fdoc = constants.FDOCID.format(mbox_uuid=INBOX_NAME, chash=chash) _mdoc.hdoc = constants.HDOCID.format(chash=chash) _mdoc.cdocs = [constants.CDOCID.format(phash=p) for p in cdocs_phashes] return _mdoc.serialize() diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py index 0cca5ef..7bdeef5 100644 --- a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -21,7 +21,6 @@ import os from functools import partial from twisted.internet import defer -from twisted.trial import unittest from leap.mail.adaptors import models from leap.mail.adaptors.soledad import SoledadDocumentWrapper @@ -62,7 +61,7 @@ class TestAdaptor(SoledadIndexMixin): 'by-type': ['type']} -class SoledadDocWrapperTestCase(unittest.TestCase, SoledadTestMixin): +class SoledadDocWrapperTestCase(SoledadTestMixin): """ Tests for the SoledadDocumentWrapper. """ @@ -284,7 +283,7 @@ class TestMessageClass(object): return self.wrapper -class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): +class SoledadMailAdaptorTestCase(SoledadTestMixin): """ Tests for the SoledadMailAdaptor. """ @@ -337,7 +336,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): hdoc="H-deadbeef", cdocs=["C-deadabad"]) fdoc = dict( - mbox="Foobox", + mbox_uuid="Foobox", flags=('\Seen', '\Nice'), tags=('Personal', 'TODO'), seen=False, deleted=False, @@ -355,7 +354,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): ('\Seen', '\Nice')) self.assertEqual(msg.wrapper.fdoc.tags, ('Personal', 'TODO')) - self.assertEqual(msg.wrapper.fdoc.mbox, "Foobox") + self.assertEqual(msg.wrapper.fdoc.mbox_uuid, "Foobox") self.assertEqual(msg.wrapper.hdoc.multi, False) self.assertEqual(msg.wrapper.hdoc.subject, "Test Msg") @@ -363,7 +362,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): "This is a test message") def test_get_msg_from_metamsg_doc_id(self): - # XXX complete-me! + # TODO complete-me! self.fail() def test_create_msg(self): diff --git a/src/leap/mail/constants.py b/src/leap/mail/constants.py index d76e652..4ef42cb 100644 --- a/src/leap/mail/constants.py +++ b/src/leap/mail/constants.py @@ -22,13 +22,13 @@ INBOX_NAME = "INBOX" # Regular expressions for the identifiers to be used in the Message Data Layer. -METAMSGID = "M-{mbox}-{chash}" -METAMSGID_RE = "M\-{mbox}\-[0-9a-fA-F]+" +METAMSGID = "M-{mbox_uuid}-{chash}" +METAMSGID_RE = "M\-{mbox_uuid}\-[0-9a-fA-F]+" METAMSGID_CHASH_RE = "M\-\w+\-([0-9a-fA-F]+)" METAMSGID_MBOX_RE = "M\-(\w+)\-[0-9a-fA-F]+" -FDOCID = "F-{mbox}-{chash}" -FDOCID_RE = "F\-{mbox}\-[0-9a-fA-F]+" +FDOCID = "F-{mbox_uuid}-{chash}" +FDOCID_RE = "F\-{mbox_uuid}\-[0-9a-fA-F]+" FDOCID_CHASH_RE = "F\-\w+\-([0-9a-fA-F]+)" HDOCID = "H-{chash}" diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index faeba9d..f2cbf75 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -236,6 +236,7 @@ class IMAPMailbox(object): :rtype: int """ + # TODO --- return the uid if it has it!!! d = self.collection.get_msg_by_uid(message) d.addCallback(lambda m: m.getUID()) return d @@ -357,7 +358,7 @@ class IMAPMailbox(object): reactor.callLater(0, self.notify_new) return x - d = self.collection.add_message(flags=flags, date=date) + d = self.collection.add_msg(message, flags=flags, date=date) d.addCallback(notifyCallback) d.addErrback(lambda f: log.msg(f.getTraceback())) return d @@ -389,14 +390,15 @@ class IMAPMailbox(object): messages and number of recent messages. :rtype: Deferred """ - d_exists = self.getMessageCount() - d_recent = self.getRecentCount() + d_exists = defer.maybeDeferred(self.getMessageCount) + d_recent = defer.maybeDeferred(self.getRecentCount) d_list = [d_exists, d_recent] def log_num_msg(result): - exists, recent = result + exists, recent = tuple(result) logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( self.mbox_name, exists, recent)) + return result d = defer.gatherResults(d_list) d.addCallback(log_num_msg) @@ -654,7 +656,7 @@ class IMAPMailbox(object): return result def _get_unseen_deferred(self): - return self.getUnseenCount() + return defer.maybeDeferred(self.getUnseenCount) def __cb_signal_unread_to_ui(self, unseen): """ diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 5af499f..dbb823f 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -926,31 +926,39 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ infile = util.sibpath(__file__, 'rfc822.message') message = open(infile) - LeapIMAPServer.theAccount.addMailbox('root/subthing') + acc = self.server.theAccount + mailbox_name = "root_subthing" + + def add_mailbox(): + return acc.addMailbox(mailbox_name) def login(): return self.client.login(TEST_USER, TEST_PASSWD) def append(): return self.client.append( - 'root/subthing', - message, + mailbox_name, message, ('\\SEEN', '\\DELETED'), 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', ) - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(append), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - return d.addCallback(self._cbTestFullAppend, infile) + d.addCallback(lambda _: acc.getMailbox(mailbox_name)) - def _cbTestFullAppend(self, ignored, infile): - mb = LeapIMAPServer.theAccount.getMailbox('root/subthing') - self.assertEqual(1, len(mb.messages)) + def print_mb(mb): + print "MB ----", mb + return mb + d.addCallback(print_mb) + d.addCallback(lambda mb: mb.collection.get_message_by_uid(1)) + return d.addCallback(self._cbTestFullAppend, infile) - msg = mb.messages.get_msg_by_uid(1) + def _cbTestFullAppend(self, msg, infile): + # TODO --- move to deferreds self.assertEqual( set(('\\Recent', '\\SEEN', '\\DELETED')), set(msg.getFlags())) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 0c9b7a3..b2caa33 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -17,6 +17,7 @@ """ Generic Access to Mail objects: Public LEAP Mail API. """ +import uuid import logging import StringIO @@ -283,6 +284,20 @@ class MessageCollection(object): self.mbox_indexer = mbox_indexer self.mbox_wrapper = mbox_wrapper + # TODO need to initialize count here because imap server does not + # expect a defered for the count. caller should return the deferred for + # prime_count (ie, initialize) when returning the collection + # TODO should increment and decrement when adding/deleting. + # TODO recent count should also be static. + + if not count: + count = 0 + self._count = count + + #def initialize(self): + #d = self.prime_count() + #return d + def is_mailbox_collection(self): """ Return True if this collection represents a Mailbox. @@ -297,6 +312,13 @@ class MessageCollection(object): return None return wrapper.mbox + @property + def mbox_uuid(self): + wrapper = getattr(self, "mbox_wrapper", None) + if not wrapper: + return None + return wrapper.mbox_uuid + def get_mbox_attr(self, attr): return getattr(self.mbox_wrapper, attr) @@ -385,16 +407,16 @@ class MessageCollection(object): raise NotImplementedError() else: - mbox = self.mbox_name + mbox_id = self.mbox_uuid wrapper.set_flags(flags) wrapper.set_tags(tags) wrapper.set_date(date) - wrapper.set_mbox(mbox) + wrapper.set_mbox_uuid(mbox_id) def insert_mdoc_id(_, wrapper): doc_id = wrapper.mdoc.doc_id return self.mbox_indexer.insert_doc( - self.mbox_name, doc_id) + self.mbox_uuid, doc_id) d = wrapper.create(self.store) d.addCallback(insert_mdoc_id, wrapper) @@ -410,7 +432,7 @@ class MessageCollection(object): def insert_copied_mdoc_id(wrapper): return self.mbox_indexer.insert_doc( - newmailbox, wrapper.mdoc.doc_id) + newmailbox_uuid, wrapper.mdoc.doc_id) wrapper = msg.get_wrapper() d = wrapper.copy(self.store, newmailbox) @@ -539,25 +561,32 @@ class Account(object): def add_mailbox(self, name): - def create_uid_table_cb(res): - d = self.mbox_indexer.create_table(name) - d.addCallback(lambda _: res) + def create_uuid(wrapper): + if not wrapper.uuid: + wrapper.uuid = uuid.uuid4() + return wrapper.update(self.store) + + def create_uid_table_cb(wrapper): + d = self.mbox_indexer.create_table(wrapper.uuid) + d.addCallback(lambda _: wrapper) return d d = self.adaptor.get_or_create_mbox(self.store, name) + d.addCallback(create_uuid) d.addCallback(create_uid_table_cb) return d def delete_mailbox(self, name): - def delete_uid_table_cb(res): - d = self.mbox_indexer.delete_table(name) - d.addCallback(lambda _: res) + + def delete_uid_table_cb(wrapper): + d = self.mbox_indexer.delete_table(wrapper.uuid) + d.addCallback(lambda _: wrapper) return d d = self.adaptor.get_or_create_mbox(self.store, name) + d.addCallback(delete_uid_table_cb) d.addCallback( lambda wrapper: self.adaptor.delete_mbox(self.store, wrapper)) - d.addCallback(delete_uid_table_cb) return d def rename_mailbox(self, oldname, newname): @@ -572,14 +601,8 @@ class Account(object): wrapper.mbox = newname return wrapper.update(self.store) - def rename_uid_table_cb(res): - d = self.mbox_indexer.rename_table(oldname, newname) - d.addCallback(lambda _: res) - return d - d = self.adaptor.get_or_create_mbox(self.store, oldname) d.addCallback(_rename_mbox) - d.addCallback(rename_uid_table_cb) return d # Get Collections diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index e5b813f..6155a7a 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -18,6 +18,7 @@ Local tables to store the message Unique Identifiers for a given mailbox. """ import re +import uuid from leap.mail.constants import METAMSGID_RE @@ -37,6 +38,25 @@ class WrongMetaDocIDError(Exception): pass +def sanitize(mailbox_id): + return mailbox_id.replace("-", "_") + + +def check_good_uuid(mailbox_id): + """ + Check that the passed mailbox identifier is a valid UUID. + :param mailbox_id: the uuid to check + :type mailbox_id: str + :return: None + :raises: AssertionError if a wrong uuid was passed. + """ + try: + uuid.UUID(str(mailbox_id)) + except (AttributeError, ValueError): + raise AssertionError( + "the mbox_id is not a valid uuid: %s" % mailbox_id) + + class MailboxIndexer(object): """ This class contains the commands needed to create, modify and alter the @@ -68,51 +88,33 @@ class MailboxIndexer(object): assert self.store is not None return self.store.raw_sqlcipher_query(*args, **kw) - def create_table(self, mailbox): + def create_table(self, mailbox_id): """ Create the UID table for a given mailbox. - :param mailbox: the mailbox name + :param mailbox: the mailbox identifier. :type mailbox: str :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) sql = ("CREATE TABLE if not exists {preffix}{name}( " "uid INTEGER PRIMARY KEY AUTOINCREMENT, " "hash TEXT UNIQUE NOT NULL)".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) return self._query(sql) - def delete_table(self, mailbox): + def delete_table(self, mailbox_id): """ Delete the UID table for a given mailbox. :param mailbox: the mailbox name :type mailbox: str :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) sql = ("DROP TABLE if exists {preffix}{name}".format( - preffix=self.table_preffix, name=mailbox)) - return self._query(sql) - - def rename_table(self, oldmailbox, newmailbox): - """ - Delete the UID table for a given mailbox. - :param oldmailbox: the old mailbox name - :type oldmailbox: str - :param newmailbox: the new mailbox name - :type newmailbox: str - :rtype: Deferred - """ - assert oldmailbox - assert newmailbox - assert oldmailbox != newmailbox - sql = ("ALTER TABLE {preffix}{old} " - "RENAME TO {preffix}{new}".format( - preffix=self.table_preffix, - old=oldmailbox, new=newmailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) return self._query(sql) - def insert_doc(self, mailbox, doc_id): + def insert_doc(self, mailbox_id, doc_id): """ Insert the doc_id for a MetaMsg in the UID table for a given mailbox. @@ -128,10 +130,11 @@ class MailboxIndexer(object): document. :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) assert doc_id + mailbox_id = mailbox_id.replace('-', '_') - if not re.findall(METAMSGID_RE.format(mbox=mailbox), doc_id): + if not re.findall(METAMSGID_RE.format(mbox=mailbox_id), doc_id): raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") def get_rowid(result): @@ -139,44 +142,44 @@ class MailboxIndexer(object): sql = ("INSERT INTO {preffix}{name} VALUES (" "NULL, ?)".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) values = (doc_id,) sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( - preffix=self.table_preffix, name=mailbox) + preffix=self.table_preffix, name=sanitize(mailbox_id)) d = self._query(sql, values) d.addCallback(lambda _: self._query(sql_last)) d.addCallback(get_rowid) return d - def delete_doc_by_uid(self, mailbox, uid): + def delete_doc_by_uid(self, mailbox_id, uid): """ Delete the entry for a MetaMsg in the UID table for a given mailbox. - :param mailbox: the mailbox name + :param mailbox_id: the mailbox uuid :type mailbox: str :param uid: the UID of the message. :type uid: int :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) assert uid sql = ("DELETE FROM {preffix}{name} " "WHERE uid=?".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) values = (uid,) return self._query(sql, values) - def delete_doc_by_hash(self, mailbox, doc_id): + def delete_doc_by_hash(self, mailbox_id, doc_id): """ Delete the entry for a MetaMsg in the UID table for a given mailbox. The doc_id must be in the format: - M++ + M-- - :param mailbox: the mailbox name + :param mailbox_id: the mailbox uuid :type mailbox: str :param doc_id: the doc_id for the MetaMsg :type doc_id: str @@ -184,30 +187,32 @@ class MailboxIndexer(object): document. :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) assert doc_id sql = ("DELETE FROM {preffix}{name} " "WHERE hash=?".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) values = (doc_id,) return self._query(sql, values) - def get_doc_id_from_uid(self, mailbox, uid): + def get_doc_id_from_uid(self, mailbox_id, uid): """ Get the doc_id for a MetaMsg in the UID table for a given mailbox. - :param mailbox: the mailbox name + :param mailbox: the mailbox uuid :type mailbox: str :param uid: the uid for the MetaMsg for this mailbox :type uid: int :rtype: Deferred """ + check_good_uuid(mailbox_id) + def get_hash(result): return _maybe_first_query_item(result) sql = ("SELECT hash from {preffix}{name} " "WHERE uid=?".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) values = (uid,) d = self._query(sql, values) d.addCallback(get_hash) @@ -218,7 +223,7 @@ class MailboxIndexer(object): # XXX dereference the range (n,*) raise NotImplementedError() - def count(self, mailbox): + def count(self, mailbox_id): """ Get the number of entries in the UID table for a given mailbox. @@ -227,16 +232,18 @@ class MailboxIndexer(object): :return: a deferred that will fire with an integer returning the count. :rtype: Deferred """ + check_good_uuid(mailbox_id) + def get_count(result): return _maybe_first_query_item(result) sql = ("SELECT Count(*) FROM {preffix}{name};".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) d = self._query(sql) d.addCallback(get_count) return d - def get_next_uid(self, mailbox): + def get_next_uid(self, mailbox_id): """ Get the next integer beyond the highest UID count for a given mailbox. @@ -251,7 +258,7 @@ class MailboxIndexer(object): uid. :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) def increment(result): uid = _maybe_first_query_item(result) @@ -261,7 +268,7 @@ class MailboxIndexer(object): sql = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( - preffix=self.table_preffix, name=mailbox) + preffix=self.table_preffix, name=sanitize(mailbox_id)) d = self._query(sql) d.addCallback(increment) diff --git a/src/leap/mail/tests/test_mailbox_indexer.py b/src/leap/mail/tests/test_mailbox_indexer.py index 47a3bdc..2edf1d8 100644 --- a/src/leap/mail/tests/test_mailbox_indexer.py +++ b/src/leap/mail/tests/test_mailbox_indexer.py @@ -17,10 +17,9 @@ """ Tests for the mailbox_indexer module. """ +import uuid from functools import partial -from twisted.trial import unittest - from leap.mail import mailbox_indexer as mi from leap.mail.tests.common import SoledadTestMixin @@ -31,11 +30,13 @@ hash_test3 = 'fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13' hash_test4 = 'a4e624d686e03ed2767c0abd85c14426b0b1157d2ce81d27bb4fe4f6f01d688a' -def fmt_hash(mailbox, hash): - return "M-" + mailbox + "-" + hash +def fmt_hash(mailbox_uuid, hash): + return "M-" + mailbox_uuid.replace('-', '_') + "-" + hash + +mbox_id = str(uuid.uuid4()) -class MailboxIndexerTestCase(unittest.TestCase, SoledadTestMixin): +class MailboxIndexerTestCase(SoledadTestMixin): """ Tests for the MailboxUID class. """ @@ -57,17 +58,17 @@ class MailboxIndexerTestCase(unittest.TestCase, SoledadTestMixin): def select_uid_rows(self, mailbox): sql = "SELECT * FROM %s%s;" % ( - mi.MailboxIndexer.table_preffix, mailbox) + mi.MailboxIndexer.table_preffix, mailbox.replace('-', '_')) d = self._soledad.raw_sqlcipher_query(sql) return d def test_create_table(self): def assert_table_created(tables): self.assertEqual( - tables, ["leapmail_uid_inbox"]) + tables, ["leapmail_uid_" + mbox_id.replace('-', '_')]) m_uid = self.get_mbox_uid() - d = m_uid.create_table('inbox') + d = m_uid.create_table(mbox_id) d.addCallback(self.list_mail_tables_cb) d.addCallback(assert_table_created) return d @@ -77,165 +78,162 @@ class MailboxIndexerTestCase(unittest.TestCase, SoledadTestMixin): self.assertEqual(tables, []) m_uid = self.get_mbox_uid() - d = m_uid.create_table('inbox') - d.addCallback(lambda _: m_uid.delete_table('inbox')) + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.delete_table(mbox_id)) d.addCallback(self.list_mail_tables_cb) d.addCallback(assert_table_deleted) return d - def test_rename_table(self): - def assert_table_renamed(tables): - self.assertEqual( - tables, ["leapmail_uid_foomailbox"]) - - m_uid = self.get_mbox_uid() - d = m_uid.create_table('inbox') - d.addCallback(lambda _: m_uid.rename_table('inbox', 'foomailbox')) - d.addCallback(self.list_mail_tables_cb) - d.addCallback(assert_table_renamed) - return d + #def test_rename_table(self): + #def assert_table_renamed(tables): + #self.assertEqual( + #tables, ["leapmail_uid_foomailbox"]) +# + #m_uid = self.get_mbox_uid() + #d = m_uid.create_table('inbox') + #d.addCallback(lambda _: m_uid.rename_table('inbox', 'foomailbox')) + #d.addCallback(self.list_mail_tables_cb) + #d.addCallback(assert_table_renamed) + #return d def test_insert_doc(self): m_uid = self.get_mbox_uid() - mbox = 'foomailbox' - h1 = fmt_hash(mbox, hash_test0) - h2 = fmt_hash(mbox, hash_test1) - h3 = fmt_hash(mbox, hash_test2) - h4 = fmt_hash(mbox, hash_test3) - h5 = fmt_hash(mbox, hash_test4) + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) def assert_uid_rows(rows): expected = [(1, h1), (2, h2), (3, h3), (4, h4), (5, h5)] self.assertEquals(rows, expected) - d = m_uid.create_table(mbox) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) - d.addCallback(lambda _: self.select_uid_rows(mbox)) + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) + d.addCallback(lambda _: self.select_uid_rows(mbox_id)) d.addCallback(assert_uid_rows) return d def test_insert_doc_return(self): m_uid = self.get_mbox_uid() - mbox = 'foomailbox' def assert_rowid(rowid, expected=None): self.assertEqual(rowid, expected) - h1 = fmt_hash(mbox, hash_test0) - h2 = fmt_hash(mbox, hash_test1) - h3 = fmt_hash(mbox, hash_test2) + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) - d = m_uid.create_table(mbox) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) d.addCallback(partial(assert_rowid, expected=1)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) d.addCallback(partial(assert_rowid, expected=2)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) d.addCallback(partial(assert_rowid, expected=3)) return d def test_delete_doc(self): m_uid = self.get_mbox_uid() - mbox = 'foomailbox' - h1 = fmt_hash(mbox, hash_test0) - h2 = fmt_hash(mbox, hash_test1) - h3 = fmt_hash(mbox, hash_test2) - h4 = fmt_hash(mbox, hash_test3) - h5 = fmt_hash(mbox, hash_test4) + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) def assert_uid_rows(rows): expected = [(4, h4), (5, h5)] self.assertEquals(rows, expected) - d = m_uid.create_table(mbox) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) - d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 1)) - d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 2)) - d.addCallbacks(lambda _: m_uid.delete_doc_by_hash(mbox, h3)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 2)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_hash(mbox_id, h3)) - d.addCallback(lambda _: self.select_uid_rows(mbox)) + d.addCallback(lambda _: self.select_uid_rows(mbox_id)) d.addCallback(assert_uid_rows) return d def test_get_doc_id_from_uid(self): m_uid = self.get_mbox_uid() - mbox = 'foomailbox' + #mbox = 'foomailbox' - h1 = fmt_hash(mbox, hash_test0) + h1 = fmt_hash(mbox_id, hash_test0) def assert_doc_hash(res): self.assertEqual(res, h1) - d = m_uid.create_table(mbox) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) - d.addCallback(lambda _: m_uid.get_doc_id_from_uid(mbox, 1)) + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.get_doc_id_from_uid(mbox_id, 1)) d.addCallback(assert_doc_hash) return d def test_count(self): m_uid = self.get_mbox_uid() - mbox = 'foomailbox' + #mbox = 'foomailbox' - h1 = fmt_hash(mbox, hash_test0) - h2 = fmt_hash(mbox, hash_test1) - h3 = fmt_hash(mbox, hash_test2) - h4 = fmt_hash(mbox, hash_test3) - h5 = fmt_hash(mbox, hash_test4) + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) - d = m_uid.create_table(mbox) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) def assert_count_after_inserts(count): self.assertEquals(count, 5) - d.addCallback(lambda _: m_uid.count(mbox)) + d.addCallback(lambda _: m_uid.count(mbox_id)) d.addCallback(assert_count_after_inserts) - d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 1)) - d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 2)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 2)) def assert_count_after_deletions(count): self.assertEquals(count, 3) - d.addCallback(lambda _: m_uid.count(mbox)) + d.addCallback(lambda _: m_uid.count(mbox_id)) d.addCallback(assert_count_after_deletions) return d def test_get_next_uid(self): m_uid = self.get_mbox_uid() - mbox = 'foomailbox' + #mbox = 'foomailbox' - h1 = fmt_hash(mbox, hash_test0) - h2 = fmt_hash(mbox, hash_test1) - h3 = fmt_hash(mbox, hash_test2) - h4 = fmt_hash(mbox, hash_test3) - h5 = fmt_hash(mbox, hash_test4) + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) - d = m_uid.create_table(mbox) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) - d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) def assert_next_uid(result, expected=1): self.assertEquals(result, expected) - d.addCallback(lambda _: m_uid.get_next_uid(mbox)) + d.addCallback(lambda _: m_uid.get_next_uid(mbox_id)) d.addCallback(partial(assert_next_uid, expected=6)) return d -- cgit v1.2.3 From 9c40103a3c3dcdb3e4c4edae9f466f1701e022fc Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 7 Jan 2015 12:12:24 -0400 Subject: Complete IMAP implementation, update tests --- src/leap/mail/adaptors/soledad.py | 183 +++++-- src/leap/mail/adaptors/soledad_indexes.py | 15 +- .../mail/adaptors/tests/test_soledad_adaptor.py | 7 +- src/leap/mail/imap/account.py | 174 ++++--- src/leap/mail/imap/mailbox.py | 116 +++-- src/leap/mail/imap/messages.py | 13 +- src/leap/mail/imap/server.py | 30 +- src/leap/mail/imap/service/imap.py | 98 +--- src/leap/mail/imap/tests/leap_tests_imap.zsh | 178 ------- src/leap/mail/imap/tests/stress_tests_imap.zsh | 178 +++++++ src/leap/mail/imap/tests/test_imap.py | 539 +++++++++++---------- src/leap/mail/imap/tests/utils.py | 212 +++----- src/leap/mail/mail.py | 114 ++++- src/leap/mail/mailbox_indexer.py | 70 ++- src/leap/mail/tests/common.py | 17 +- src/leap/mail/tests/test_mailbox_indexer.py | 41 +- 16 files changed, 1098 insertions(+), 887 deletions(-) delete mode 100755 src/leap/mail/imap/tests/leap_tests_imap.zsh create mode 100755 src/leap/mail/imap/tests/stress_tests_imap.zsh diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index c5cfce0..d99f677 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -23,7 +23,6 @@ from functools import partial from pycryptopp.hash import sha256 from twisted.internet import defer -from twisted.python import util from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type @@ -56,6 +55,14 @@ class DuplicatedDocumentError(Exception): pass +def cleanup_deferred_locks(): + """ + Need to use this from within trial to cleanup the reactor before + each run. + """ + SoledadDocumentWrapper._k_locks = defaultdict(defer.DeferredLock) + + class SoledadDocumentWrapper(models.DocumentWrapper): """ A Wrapper object that can be manipulated, passed around, and serialized in @@ -526,7 +533,7 @@ class MessageWrapper(object): This method should only be used before the Documents for the MessageWrapper have been created, will raise otherwise. """ - mbox_uuid = mbox.uuid.replace('-', '_') + mbox_uuid = mbox_uuid.replace('-', '_') self.mdoc.set_mbox_uuid(mbox_uuid) self.fdoc.set_mbox_uuid(mbox_uuid) @@ -536,6 +543,9 @@ class MessageWrapper(object): flags = tuple() leap_assert_type(flags, tuple) self.fdoc.flags = list(flags) + self.fdoc.deleted = "\\Deleted" in flags + self.fdoc.seen = "\\Seen" in flags + self.fdoc.recent = "\\Recent" in flags def set_tags(self, tags): # TODO serialize the get + update @@ -593,26 +603,30 @@ class MailboxWrapper(SoledadDocumentWrapper): # Soledad Adaptor # -# TODO make this an interface? class SoledadIndexMixin(object): """ - this will need a class attribute `indexes`, that is a dictionary containing + This will need a class attribute `indexes`, that is a dictionary containing the index definitions for the underlying u1db store underlying soledad. It needs to be in the following format: {'index-name': ['field1', 'field2']} + + You can also add a class attribute `wait_for_indexes` to any class + inheriting from this Mixin, that should be a list of strings representing + the methods that need to wait until the indexes have been initialized + before being able to work properly. """ + # TODO move this mixin to soledad itself + # so that each application can pass a set of indexes for their data model. + # TODO could have a wrapper class for indexes, supporting introspection # and __getattr__ - indexes = {} - store_ready = False - _index_creation_deferreds = [] + # TODO make this an interface? - # TODO we might want to move this logic to soledad itself - # so that each application can pass a set of indexes for their data model. - # TODO check also the decorator used in keymanager for waiting for indexes - # to be ready. + indexes = {} + wait_for_indexes = [] + store_ready = False def initialize_store(self, store): """ @@ -626,47 +640,81 @@ class SoledadIndexMixin(object): # TODO I think we *should* get another deferredLock in here, but # global to the soledad namespace, to protect from several points # initializing soledad indexes at the same time. + self._wait_for_indexes() - leap_assert(store, "Need a store") - leap_assert_type(self.indexes, dict) + d = self._init_indexes(store) + d.addCallback(self._restore_waiting_methods) + return d - self._index_creation_deferreds = [] + def _init_indexes(self, store): + """ + Initialize the database indexes. + """ + leap_assert(store, "Cannot init indexes with null soledad") + leap_assert_type(self.indexes, dict) def _create_index(name, expression): return store.create_index(name, *expression) - def _create_indexes(db_indexes): - db_indexes = dict(db_indexes) - + def init_idexes(indexes): + deferreds = [] + db_indexes = dict(indexes) + # Loop through the indexes we expect to find. for name, expression in self.indexes.items(): if name not in db_indexes: # The index does not yet exist. d = _create_index(name, expression) - self._index_creation_deferreds.append(d) - continue - - if expression == db_indexes[name]: - # The index exists and is up to date. - continue - # The index exists but the definition is not what expected, so - # we delete it and add the proper index expression. - d1 = store.delete_index(name) - d1.addCallback(lambda _: _create_index(name, expression)) - self._index_creation_deferreds.append(d1) - - all_created = defer.gatherResults( - self._index_creation_deferreds, consumeErrors=True) - all_created.addCallback(_on_indexes_created) - return all_created - - def _on_indexes_created(ignored): + deferreds.append(d) + elif expression != db_indexes[name]: + # The index exists but the definition is not what expected, + # so we delete it and add the proper index expression. + d = store.delete_index(name) + d.addCallback( + lambda _: _create_index(name, *expression)) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + def store_ready(whatever): self.store_ready = True + return whatever - # Ask the database for currently existing indexes, and create them - # if not found. - d = store.list_indexes() - d.addCallback(_create_indexes) - return d + self.deferred_indexes = store.list_indexes() + self.deferred_indexes.addCallback(init_idexes) + self.deferred_indexes.addCallback(store_ready) + return self.deferred_indexes + + def _wait_for_indexes(self): + """ + Make the marked methods to wait for the indexes to be ready. + Heavily based on + http://blogs.fluidinfo.com/terry/2009/05/11/a-mixin-class-allowing-python-__init__-methods-to-work-with-twisted-deferreds/ + + :param methods: methods that need to wait for the indexes to be ready + :type methods: tuple(str) + """ + leap_assert_type(self.wait_for_indexes, list) + methods = self.wait_for_indexes + + self.waiting = [] + self.stored = {} + + def makeWrapper(method): + def wrapper(*args, **kw): + d = defer.Deferred() + d.addCallback(lambda _: self.stored[method](*args, **kw)) + self.waiting.append(d) + return d + return wrapper + + for method in methods: + self.stored[method] = getattr(self, method) + setattr(self, method, makeWrapper(method)) + + def _restore_waiting_methods(self, _): + for method in self.stored: + setattr(self, method, self.stored[method]) + for d in self.waiting: + d.callback(None) class SoledadMailAdaptor(SoledadIndexMixin): @@ -675,8 +723,18 @@ class SoledadMailAdaptor(SoledadIndexMixin): store = None indexes = indexes.MAIL_INDEXES + wait_for_indexes = ['get_or_create_mbox', 'update_mbox', 'get_all_mboxes'] + mboxwrapper_klass = MailboxWrapper + def __init__(self): + SoledadIndexMixin.__init__(self) + + mboxwrapper_klass = MailboxWrapper + + def __init__(self): + SoledadIndexMixin.__init__(self) + # Message handling def get_msg_from_string(self, MessageClass, raw_msg): @@ -762,10 +820,10 @@ class SoledadMailAdaptor(SoledadIndexMixin): chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0] def _get_fdoc_id_from_mdoc_id(): - return constants.FDOCID.format(mbox=mbox, chash=chash) + return constants.FDOCID.format(mbox_uuid=mbox, chash=chash) def _get_hdoc_id_from_mdoc_id(): - return constants.HDOCID.format(mbox=mbox, chash=chash) + return constants.HDOCID.format(mbox_uuid=mbox, chash=chash) d_docs = [] fdoc_id = _get_fdoc_id_from_mdoc_id() @@ -816,6 +874,47 @@ class SoledadMailAdaptor(SoledadIndexMixin): wrapper = msg.get_wrapper() return wrapper.update(store) + # batch deletion + + def del_all_flagged_messages(self, store, mbox_uuid): + """ + Delete all messages flagged as deleted. + """ + def err(f): + print "ERROR GETTING FROM INDEX" + f.printTraceback() + + def delete_fdoc_and_mdoc_flagged(fdocs): + # low level here, not using the wrappers... + # get meta doc ids from the flag doc ids + fdoc_ids = [doc.doc_id for doc in fdocs] + mdoc_ids = map(lambda s: "M" + s[1:], fdoc_ids) + + def delete_all_docs(mdocs, fdocs): + mdocs = list(mdocs) + doc_ids = [m.doc_id for m in mdocs] + _d = [] + docs = mdocs + fdocs + for doc in docs: + _d.append(store.delete_doc(doc)) + d = defer.gatherResults(_d) + # return the mdocs ids only + d.addCallback(lambda _: doc_ids) + return d + + d = store.get_docs(mdoc_ids) + d.addCallback(delete_all_docs, fdocs) + d.addErrback(err) + return d + + type_ = FlagsDocWrapper.model.type_ + uuid = mbox_uuid.replace('-', '_') + deleted_index = indexes.TYPE_MBOX_DEL_IDX + + d = store.get_from_index(deleted_index, type_, uuid, "1") + d.addCallbacks(delete_fdoc_and_mdoc_flagged, err) + return d + # Mailbox handling def get_or_create_mbox(self, store, name): diff --git a/src/leap/mail/adaptors/soledad_indexes.py b/src/leap/mail/adaptors/soledad_indexes.py index f3e990d..856dfb4 100644 --- a/src/leap/mail/adaptors/soledad_indexes.py +++ b/src/leap/mail/adaptors/soledad_indexes.py @@ -25,6 +25,7 @@ Soledad Indexes for Mail Documents. TYPE = "type" MBOX = "mbox" +MBOX_UUID = "mbox_uuid" FLAGS = "flags" HEADERS = "head" CONTENT = "cnt" @@ -46,7 +47,7 @@ UID = "uid" TYPE_IDX = 'by-type' TYPE_MBOX_IDX = 'by-type-and-mbox' -#TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' +TYPE_MBOX_UUID_IDX = 'by-type-and-mbox-uuid' TYPE_SUBS_IDX = 'by-type-and-subscribed' TYPE_MSGID_IDX = 'by-type-and-message-id' TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' @@ -62,9 +63,6 @@ TYPE_P_HASH_IDX = 'by-type-and-payloadhash' JUST_MAIL_IDX = "just-mail" JUST_MAIL_COMPAT_IDX = "just-mail-compat" -# Tomas created the `recent and seen index`, but the semantic is not too -# correct since the recent flag is volatile --- XXX review and delete. -#TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' # TODO # it would be nice to measure the cost of indexing @@ -77,6 +75,7 @@ MAIL_INDEXES = { # generic TYPE_IDX: [TYPE], TYPE_MBOX_IDX: [TYPE, MBOX], + TYPE_MBOX_UUID_IDX: [TYPE, MBOX_UUID], # XXX deprecate 0.4.0 # TYPE_MBOX_UID_IDX: [TYPE, MBOX, UID], @@ -97,11 +96,9 @@ MAIL_INDEXES = { TYPE_P_HASH_IDX: [TYPE, PAYLOAD_HASH], # messages - TYPE_MBOX_SEEN_IDX: [TYPE, MBOX, 'bool(seen)'], - TYPE_MBOX_RECT_IDX: [TYPE, MBOX, 'bool(recent)'], - TYPE_MBOX_DEL_IDX: [TYPE, MBOX, 'bool(deleted)'], - #TYPE_MBOX_RECT_SEEN_IDX: [TYPE, MBOX, - #'bool(recent)', 'bool(seen)'], + TYPE_MBOX_SEEN_IDX: [TYPE, MBOX_UUID, 'bool(seen)'], + TYPE_MBOX_RECT_IDX: [TYPE, MBOX_UUID, 'bool(recent)'], + TYPE_MBOX_DEL_IDX: [TYPE, MBOX_UUID, 'bool(deleted)'], # incoming queue JUST_MAIL_IDX: [INCOMING_KEY, diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py index 7bdeef5..3dc79fe 100644 --- a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -276,8 +276,9 @@ HERE = os.path.split(os.path.abspath(__file__))[0] class TestMessageClass(object): - def __init__(self, wrapper): + def __init__(self, wrapper, uid): self.wrapper = wrapper + self.uid = uid def get_wrapper(self): return self.wrapper @@ -322,9 +323,9 @@ class SoledadMailAdaptorTestCase(SoledadTestMixin): self.assertTrue(msg.wrapper.cdocs is not None) self.assertEquals(len(msg.wrapper.cdocs), 1) self.assertEquals(msg.wrapper.fdoc.chash, chash) - self.assertEquals(msg.wrapper.fdoc.size, 3834) + self.assertEquals(msg.wrapper.fdoc.size, 3837) self.assertEquals(msg.wrapper.hdoc.chash, chash) - self.assertEqual(msg.wrapper.hdoc.headers['subject'], + self.assertEqual(dict(msg.wrapper.hdoc.headers)['Subject'], subject) self.assertEqual(msg.wrapper.hdoc.subject, subject) self.assertEqual(msg.wrapper.cdocs[1].phash, phash) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 0baf078..dfc0d62 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -63,7 +63,7 @@ class IMAPAccount(object): selected = None closed = False - def __init__(self, user_id, store): + def __init__(self, user_id, store, d=None): """ Keeps track of the mailboxes and subscriptions handled by this account. @@ -72,21 +72,31 @@ class IMAPAccount(object): :param store: a Soledad instance. :type store: Soledad + + :param d: a deferred that will be fired with this IMAPAccount instance + when the account is ready to be used. + :type d: defer.Deferred """ leap_assert(store, "Need a store instance to initialize") leap_assert_type(store, Soledad) # TODO assert too that the name matches the user/uuid with which - # soledad has been initialized. + # soledad has been initialized. Although afaik soledad doesn't know + # about user_id, only the client backend. + self.user_id = user_id - self.account = Account(store) + self.account = Account(store, ready_cb=lambda: d.callback(self)) def _return_mailbox_from_collection(self, collection, readwrite=1): if collection is None: return None - return IMAPMailbox(collection, rw=readwrite) + mbox = IMAPMailbox(collection, rw=readwrite) + return mbox + + def callWhenReady(self, cb, *args, **kw): + d = self.account.callWhenReady(cb, *args, **kw) + return d - # XXX Where's this used from? -- self.delete... def getMailbox(self, name): """ Return a Mailbox with that name, without selecting it. @@ -102,11 +112,12 @@ class IMAPAccount(object): def check_it_exists(mailboxes): if name not in mailboxes: raise imap4.MailboxException("No such mailbox: %r" % name) + return True d = self.account.list_all_mailbox_names() d.addCallback(check_it_exists) - d.addCallback(lambda _: self.account.get_collection_by_mailbox, name) - d.addCallbacK(self._return_mailbox_from_collection) + d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) + d.addCallback(self._return_mailbox_from_collection) return d # @@ -130,12 +141,9 @@ class IMAPAccount(object): """ name = normalize_mailbox(name) + # FIXME --- return failure instead of AssertionError + # See AccountTestCase... leap_assert(name, "Need a mailbox name to create a mailbox") - - def check_it_does_not_exist(mailboxes): - if name in mailboxes: - raise imap4.MailboxCollision(repr(name)) - if creation_ts is None: # by default, we pass an int value # taken from the current time @@ -143,14 +151,20 @@ class IMAPAccount(object): # mailbox-uidvalidity. creation_ts = int(time.time() * 10E2) + def check_it_does_not_exist(mailboxes): + if name in mailboxes: + raise imap4.MailboxCollision, repr(name) + return mailboxes + def set_mbox_creation_ts(collection): - d = collection.set_mbox_attr("created") + d = collection.set_mbox_attr("created", creation_ts) d.addCallback(lambda _: collection) return d d = self.account.list_all_mailbox_names() d.addCallback(check_it_does_not_exist) - d.addCallback(lambda _: self.account.get_collection_by_mailbox, name) + d.addCallback(lambda _: self.account.add_mailbox(name)) + d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) d.addCallback(set_mbox_creation_ts) d.addCallback(self._return_mailbox_from_collection) return d @@ -172,39 +186,40 @@ class IMAPAccount(object): :raise MailboxException: Raised if this mailbox cannot be added. """ - # TODO raise MailboxException paths = filter(None, normalize_mailbox(pathspec).split('/')) - subs = [] sep = '/' + def pass_on_collision(failure): + failure.trap(imap4.MailboxCollision) + return True + for accum in range(1, len(paths)): - try: - partial_path = sep.join(paths[:accum]) - d = self.addMailbox(partial_path) - subs.append(d) - # XXX should this be handled by the deferred? - except imap4.MailboxCollision: - pass - try: - df = self.addMailbox(sep.join(paths)) - except imap4.MailboxCollision: + partial_path = sep.join(paths[:accum]) + d = self.addMailbox(partial_path) + d.addErrback(pass_on_collision) + subs.append(d) + + def handle_collision(failure): + failure.trap(imap4.MailboxCollision) if not pathspec.endswith('/'): - df = defer.succeed(False) + return defer.succeed(False) else: - df = defer.succeed(True) - finally: - subs.append(df) + return defer.succeed(True) + + df = self.addMailbox(sep.join(paths)) + df.addErrback(handle_collision) + subs.append(df) def all_good(result): return all(result) if subs: - d1 = defer.gatherResults(subs, consumeErrors=True) + d1 = defer.gatherResults(subs) d1.addCallback(all_good) + return d1 else: - d1 = defer.succeed(False) - return d1 + return defer.succeed(False) def select(self, name, readwrite=1): """ @@ -216,7 +231,7 @@ class IMAPAccount(object): :param readwrite: 1 for readwrite permissions. :type readwrite: int - :rtype: SoledadMailbox + :rtype: IMAPMailbox """ name = normalize_mailbox(name) @@ -245,9 +260,6 @@ class IMAPAccount(object): """ Deletes a mailbox. - Right now it does not purge the messages, but just removes the mailbox - name from the mailboxes list!!! - :param name: the mailbox to be deleted :type name: str @@ -258,10 +270,10 @@ class IMAPAccount(object): :rtype: Deferred """ name = normalize_mailbox(name) - _mboxes = [] + _mboxes = None def check_it_exists(mailboxes): - # FIXME works? -- pass variable ref to outer scope + global _mboxes _mboxes = mailboxes if name not in mailboxes: err = imap4.MailboxException("No such mailbox: %r" % name) @@ -274,6 +286,7 @@ class IMAPAccount(object): return mbox.destroy() def check_can_be_deleted(mbox): + global _mboxes # See if this box is flagged \Noselect mbox_flags = mbox.getFlags() if MessageFlags.NOSELECT_FLAG in mbox_flags: @@ -317,29 +330,27 @@ class IMAPAccount(object): oldname = normalize_mailbox(oldname) newname = normalize_mailbox(newname) - # FIXME check that scope works (test) - _mboxes = [] - - if oldname not in self.mailboxes: - raise imap4.NoSuchMailbox(repr(oldname)) - - inferiors = self._inferiorNames(oldname) - inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors] + def rename_inferiors(inferiors_result): + inferiors, mailboxes = inferiors_result + rename_deferreds = [] + inferiors = [ + (o, o.replace(oldname, newname, 1)) for o in inferiors] - for (old, new) in inferiors: - if new in _mboxes: - raise imap4.MailboxCollision(repr(new)) + for (old, new) in inferiors: + if new in mailboxes: + raise imap4.MailboxCollision(repr(new)) - rename_deferreds = [] + for (old, new) in inferiors: + d = self.account.rename_mailbox(old, new) + rename_deferreds.append(d) - for (old, new) in inferiors: - d = self.account.rename_mailbox(old, new) - rename_deferreds.append(d) + d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) + return d1 - d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) - return d1 + d = self._inferiorNames(oldname) + d.addCallback(rename_inferiors) + return d - # FIXME use deferreds (list_all_mailbox_names, etc) def _inferiorNames(self, name): """ Return hierarchically inferior mailboxes. @@ -348,13 +359,17 @@ class IMAPAccount(object): :rtype: list """ # XXX use wildcard query instead - inferiors = [] - for infname in self.mailboxes: - if infname.startswith(name): - inferiors.append(infname) - return inferiors + def filter_inferiors(mailboxes): + inferiors = [] + for infname in mailboxes: + if infname.startswith(name): + inferiors.append(infname) + return inferiors + + d = self.account.list_all_mailbox_names() + d.addCallback(filter_inferiors) + return d - # TODO use mail.Account.list_mailboxes def listMailboxes(self, ref, wildcard): """ List the mailboxes. @@ -371,11 +386,21 @@ class IMAPAccount(object): :param wildcard: mailbox name with possible wildcards :type wildcard: str """ - # XXX use wildcard in index query - # TODO get deferreds wildcard = imap4.wildcardToRegexp(wildcard, '/') - ref = self._inferiorNames(normalize_mailbox(ref)) - return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] + + def get_list(mboxes, mboxes_names): + return zip(mboxes_names, mboxes) + + def filter_inferiors(ref): + mboxes = [mbox for mbox in ref if wildcard.match(mbox)] + mbox_d = defer.gatherResults([self.getMailbox(m) for m in mboxes]) + + mbox_d.addCallback(get_list, mboxes) + return mbox_d + + d = self._inferiorNames(normalize_mailbox(ref)) + d.addCallback(filter_inferiors) + return d # # The rest of the methods are specific for leap.mail.imap.account.Account @@ -393,7 +418,7 @@ class IMAPAccount(object): name = normalize_mailbox(name) def get_subscribed(mbox): - return mbox.get_mbox_attr("subscribed") + return mbox.collection.get_mbox_attr("subscribed") d = self.getMailbox(name) d.addCallback(get_subscribed) @@ -410,7 +435,7 @@ class IMAPAccount(object): name = normalize_mailbox(name) def set_subscribed(mbox): - return mbox.set_mbox_attr("subscribed", True) + return mbox.collection.set_mbox_attr("subscribed", True) d = self.getMailbox(name) d.addCallback(set_subscribed) @@ -427,16 +452,19 @@ class IMAPAccount(object): name = normalize_mailbox(name) def set_unsubscribed(mbox): - return mbox.set_mbox_attr("subscribed", False) + return mbox.collection.set_mbox_attr("subscribed", False) d = self.getMailbox(name) d.addCallback(set_unsubscribed) return d - # TODO -- get__all_mboxes, return tuple - # with ... name? and subscribed bool... def getSubscriptions(self): - raise NotImplementedError() + def get_subscribed(mailboxes): + return [x.mbox for x in mailboxes if x.subscribed] + + d = self.account.get_all_mailboxes() + d.addCallback(get_subscribed) + return d # # INamespacePresenter diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index f2cbf75..e1eb6bf 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -21,9 +21,11 @@ import re import logging import StringIO import cStringIO +import time import os from collections import defaultdict +from email.utils import formatdate from twisted.internet import defer from twisted.internet import reactor @@ -36,6 +38,7 @@ from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type from leap.mail.constants import INBOX_NAME, MessageFlags +from leap.mail.imap.messages import IMAPMessage logger = logging.getLogger(__name__) @@ -88,7 +91,7 @@ class IMAPMailbox(object): implements( imap4.IMailbox, imap4.IMailboxInfo, - imap4.ICloseableMailbox, + #imap4.ICloseableMailbox, imap4.ISearchableMailbox, imap4.IMessageCopier) @@ -105,8 +108,7 @@ class IMAPMailbox(object): def __init__(self, collection, rw=1): """ - SoledadMailbox constructor. Needs to get passed a name, plus a - Soledad instance. + SoledadMailbox constructor. :param collection: instance of IMAPMessageCollection :type collection: IMAPMessageCollection @@ -115,6 +117,7 @@ class IMAPMailbox(object): :type rw: int """ self.rw = rw + self.closed = False self._uidvalidity = None self.collection = collection @@ -139,6 +142,11 @@ class IMAPMailbox(object): """ return self._listeners[self.mbox_name] + def get_imap_message(self, message): + msg = IMAPMessage(message) + msg.store = self.collection.store + return msg + # FIXME this grows too crazily when many instances are fired, like # during imaptest stress testing. Should have a queue of limited size # instead. @@ -308,17 +316,24 @@ class IMAPMailbox(object): :type names: iter """ r = {} + maybe = defer.maybeDeferred if self.CMD_MSG in names: - r[self.CMD_MSG] = self.getMessageCount() + r[self.CMD_MSG] = maybe(self.getMessageCount) if self.CMD_RECENT in names: - r[self.CMD_RECENT] = self.getRecentCount() + r[self.CMD_RECENT] = maybe(self.getRecentCount) if self.CMD_UIDNEXT in names: - r[self.CMD_UIDNEXT] = self.getUIDNext() + r[self.CMD_UIDNEXT] = maybe(self.getUIDNext) if self.CMD_UIDVALIDITY in names: - r[self.CMD_UIDVALIDITY] = self.getUIDValidity() + r[self.CMD_UIDVALIDITY] = maybe(self.getUIDValidity) if self.CMD_UNSEEN in names: - r[self.CMD_UNSEEN] = self.getUnseenCount() - return defer.succeed(r) + r[self.CMD_UNSEEN] = maybe(self.getUnseenCount) + + def as_a_dict(values): + return dict(zip(r.keys(), values)) + + d = defer.gatherResults(r.values()) + d.addCallback(as_a_dict) + return d def addMessage(self, message, flags, date=None): """ @@ -341,11 +356,15 @@ class IMAPMailbox(object): # XXX we could treat the message as an IMessage from here leap_assert_type(message, basestring) + if flags is None: flags = tuple() else: flags = tuple(str(flag) for flag in flags) + if date is None: + date = formatdate(time.time()) + # if PROFILE_CMD: # do_profile_cmd(d, "APPEND") @@ -419,10 +438,11 @@ class IMAPMailbox(object): self.setFlags((MessageFlags.NOSELECT_FLAG,)) def remove_mbox(_): - # FIXME collection does not have a delete_mbox method, - # it's in account. - # XXX should take care of deleting the uid table too. - return self.collection.delete_mbox(self.mbox_name) + uuid = self.collection.mbox_uuid + d = self.collection.mbox_wrapper.delete(self.collection.store) + d.addCallback( + lambda _: self.collection.mbox_indexer.delete_table(uuid)) + return d d = self.deleteAllDocs() d.addCallback(remove_mbox) @@ -431,13 +451,14 @@ class IMAPMailbox(object): def _close_cb(self, result): self.closed = True - def close(self): - """ - Expunge and mark as closed - """ - d = self.expunge() - d.addCallback(self._close_cb) - return d + # TODO server already calls expunge for closing + #def close(self): + #""" + #Expunge and mark as closed + #""" + #d = self.expunge() + #d.addCallback(self._close_cb) + #return d def expunge(self): """ @@ -445,10 +466,7 @@ class IMAPMailbox(object): """ if not self.isWriteable(): raise imap4.ReadOnlyMailbox - d = defer.Deferred() - # FIXME actually broken. - # Iterate through index, and do a expunge. - return d + return self.collection.delete_all_flagged() # FIXME -- get last_uid from mbox_indexer def _bound_seq(self, messages_asked): @@ -465,12 +483,12 @@ class IMAPMailbox(object): except TypeError: # looks like we cannot iterate try: + # XXX fixme, does not exist messages_asked.last = self.last_uid except ValueError: pass return messages_asked - # TODO -- needed? --- we can get the sequence from the indexer. def _filter_msg_seq(self, messages_asked): """ Filter a message sequence returning only the ones that do exist in the @@ -480,10 +498,16 @@ class IMAPMailbox(object): :type messages_asked: MessageSet :rtype: set """ - set_asked = set(messages_asked) - set_exist = set(self.messages.all_uid_iter()) - seq_messg = set_asked.intersection(set_exist) - return seq_messg + # TODO we could pass the asked sequence to the indexer + # all_uid_iter, and bound the sql query instead. + def filter_by_asked(sequence): + set_asked = set(messages_asked) + set_exist = set(sequence) + return set_asked.intersection(set_exist) + + d = self.collection.all_uid_iter() + d.addCallback(filter_by_asked) + return d def fetch(self, messages_asked, uid): """ @@ -509,26 +533,41 @@ class IMAPMailbox(object): sequence = False # sequence = True if uid == 0 else False + getmsg = self.collection.get_message_by_uid messages_asked = self._bound_seq(messages_asked) - seq_messg = self._filter_msg_seq(messages_asked) - getmsg = self.collection.get_msg_by_uid + d_sequence = self._filter_msg_seq(messages_asked) + + def get_imap_messages_for_sequence(sequence): + def _zip_msgid(messages): + return zip( + list(sequence), + map(self.get_imap_message, messages)) + + def _unset_recent(sequence): + reactor.callLater(0, self.unset_recent_flags, sequence) + return sequence + + d_msg = [] + for msgid in sequence: + d_msg.append(getmsg(msgid)) + + d = defer.gatherResults(d_msg) + d.addCallback(_zip_msgid) + return d # for sequence numbers (uid = 0) if sequence: logger.debug("Getting msg by index: INEFFICIENT call!") # TODO --- implement sequences in mailbox indexer raise NotImplementedError + else: - got_msg = ((msgid, getmsg(msgid)) for msgid in seq_messg) - result = ((msgid, msg) for msgid, msg in got_msg - if msg is not None) - reactor.callLater(0, self.unset_recent_flags, seq_messg) + d_sequence.addCallback(get_imap_messages_for_sequence) # TODO -- call signal_to_ui # d.addCallback(self.cb_signal_unread_to_ui) - - return result + return d_sequence def fetch_flags(self, messages_asked, uid): """ @@ -755,6 +794,7 @@ class IMAPMailbox(object): # :raise IllegalQueryError: Raised when query is not valid. # example query: # ['UNDELETED', 'HEADER', 'Message-ID', + # XXX fixme, does not exist # '52D44F11.9060107@dev.bitmask.net'] # TODO hardcoding for now! -- we'll support generic queries later on @@ -821,7 +861,7 @@ class IMAPMailbox(object): Representation string for this mailbox. """ return u"" % ( - self.mbox_name, self.messages.count()) + self.mbox_name, self.collection.count()) _INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 883da35..9b00162 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -18,18 +18,12 @@ IMAPMessage and IMAPMessageCollection. """ import logging -# import StringIO from twisted.mail import imap4 from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type -from leap.common.decorators import memoized_method -from leap.common.mail import get_email_charset - from leap.mail.utils import find_charset -from leap.mail.imap.messageparts import MessagePart -# from leap.mail.imap.messagepargs import MessagePartDoc logger = logging.getLogger(__name__) @@ -116,13 +110,17 @@ class IMAPMessage(object): # IMessagePart # - def getBodyFile(self): + def getBodyFile(self, store=None): """ Retrieve a file object containing only the body of this message. :return: file-like object opened for reading :rtype: StringIO """ + if store is None: + store = self.store + return self.message.get_body_file(store) + # TODO refactor with getBodyFile in MessagePart #body = bdoc_content.get(self.RAW_KEY, "") @@ -141,7 +139,6 @@ class IMAPMessage(object): #finally: #return write_fd(body) - return self.message.get_body_file() def getSize(self): """ diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index cf0ba74..b4f320a 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -Leap IMAP4 Server Implementation. +LEAP IMAP4 Server Implementation. """ from copy import copy @@ -36,9 +36,9 @@ from twisted.mail.imap4 import IllegalClientResponse from twisted.mail.imap4 import LiteralString, LiteralFile -class LeapIMAPServer(imap4.IMAP4Server): +class LEAPIMAPServer(imap4.IMAP4Server): """ - An IMAP4 Server with mailboxes backed by soledad + An IMAP4 Server with a LEAP Storage Backend. """ def __init__(self, *args, **kwargs): # pop extraneous arguments @@ -51,7 +51,6 @@ class LeapIMAPServer(imap4.IMAP4Server): leap_assert(uuid, "need a user in the initialization") self._userid = userid - self.reactor = reactor # initialize imap server! imap4.IMAP4Server.__init__(self, *args, **kwargs) @@ -146,12 +145,15 @@ class LeapIMAPServer(imap4.IMAP4Server): """ Notify new messages to listeners. """ - self.reactor.callFromThread(self.mbox.notify_new) + reactor.callFromThread(self.mbox.notify_new) def _cbSelectWork(self, mbox, cmdName, tag): """ - Callback for selectWork, patched to avoid conformance errors due to - incomplete UIDVALIDITY line. + Callback for selectWork + + * patched to avoid conformance errors due to incomplete UIDVALIDITY + line. + * patched to accept deferreds for messagecount and recent count """ if mbox is None: self.sendNegativeResponse(tag, 'No such mailbox') @@ -161,9 +163,9 @@ class LeapIMAPServer(imap4.IMAP4Server): return flags = mbox.getFlags() + self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS') self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT') - self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) # Patched ------------------------------------------------------- # imaptest was complaining about the incomplete line, we're adding @@ -353,12 +355,9 @@ class LeapIMAPServer(imap4.IMAP4Server): self.sendPositiveResponse(tag, '%s completed' % (cmdName,)) # -------------------- end isSubscribed patch ----------- - # TODO ---- - # subscribe method had also to be changed to accomodate - # deferred - # Revert to regular methods as soon as we implement non-deferred memory - # cache. + # TODO subscribe method had also to be changed to accomodate deferred def do_SUBSCRIBE(self, tag, name): + print "DOING SUBSCRIBE" name = self._parseMbox(name) def _subscribeCb(_): @@ -421,7 +420,8 @@ class LeapIMAPServer(imap4.IMAP4Server): def _renameEb(failure): m = failure.value - print "rename failure!" + print "SERVER rename failure!" + print m if failure.check(TypeError): self.sendBadResponse(tag, 'Invalid command syntax') elif failure.check(imap4.MailboxException): @@ -479,7 +479,7 @@ class LeapIMAPServer(imap4.IMAP4Server): if failure.check(imap4.MailboxException): self.sendNegativeResponse(tag, str(m)) else: - print "other error" + print "SERVER: other error" log.err() self.sendBadResponse( tag, diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 10ba32a..5d88a79 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # imap.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013-2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,15 +15,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -Imap service initialization +IMAP service initialization """ import logging import os -import time -from twisted.internet import defer, threads -from twisted.internet.protocol import ServerFactory +from twisted.internet import reactor from twisted.internet.error import CannotListenError +from twisted.internet.protocol import ServerFactory from twisted.mail import imap4 from twisted.python import log @@ -32,42 +31,14 @@ logger = logging.getLogger(__name__) from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type, leap_check from leap.keymanager import KeyManager -from leap.mail.imap.account import SoledadBackedAccount +from leap.mail.imap.account import IMAPAccount from leap.mail.imap.fetch import LeapIncomingMail -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.server import LeapIMAPServer -from leap.mail.imap.soledadstore import SoledadStore +from leap.mail.imap.server import LEAPIMAPServer from leap.soledad.client import Soledad -# The default port in which imap service will run -IMAP_PORT = 1984 - -# The period between succesive checks of the incoming mail -# queue (in seconds) -INCOMING_CHECK_PERIOD = 60 - from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED from leap.common.events.events_pb2 import IMAP_SERVICE_FAILED_TO_START -###################################################### -# Temporary workaround for RecursionLimit when using -# qt4reactor. Do remove when we move to poll or select -# reactor, which do not show those problems. See #4974 -import resource -import sys - -try: - sys.setrecursionlimit(10**7) -except Exception: - print "Error setting recursion limit" -try: - # Increase max stack size from 8MB to 256MB - resource.setrlimit(resource.RLIMIT_STACK, (2**28, -1)) -except Exception: - print "Error setting stack size" - -###################################################### - DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) if DO_MANHOLE: from leap.mail.imap.service import manhole @@ -81,6 +52,13 @@ if DO_PROFILE: pr = cProfile.Profile() pr.enable() +# The default port in which imap service will run +IMAP_PORT = 1984 + +# The period between succesive checks of the incoming mail +# queue (in seconds) +INCOMING_CHECK_PERIOD = 60 + class IMAPAuthRealm(object): """ @@ -114,12 +92,8 @@ class LeapIMAPFactory(ServerFactory): self._uuid = uuid self._userid = userid self._soledad = soledad - self._memstore = MemoryStore( - permanent_store=SoledadStore(soledad)) - theAccount = SoledadBackedAccount( - uuid, soledad=soledad, - memstore=self._memstore) + theAccount = IMAPAccount(uuid, soledad) self.theAccount = theAccount # XXX how to pass the store along? @@ -131,7 +105,8 @@ class LeapIMAPFactory(ServerFactory): :param addr: remote ip address :type addr: str """ - imapProtocol = LeapIMAPServer( + # XXX addr not used??! + imapProtocol = LEAPIMAPServer( uuid=self._uuid, userid=self._userid, soledad=self._soledad) @@ -139,41 +114,18 @@ class LeapIMAPFactory(ServerFactory): imapProtocol.factory = self return imapProtocol - def doStop(self, cv=None): + def doStop(self): """ Stops imap service (fetcher, factory and port). - - :param cv: A condition variable to which we can signal when imap - indeed stops. - :type cv: threading.Condition - :return: a Deferred that stops and flushes the in memory store data to - disk in another thread. - :rtype: Deferred """ + # TODO should wait for all the pending deferreds, + # the twisted way! if DO_PROFILE: log.msg("Stopping PROFILING") pr.disable() pr.dump_stats(PROFILE_DAT) - ServerFactory.doStop(self) - - if cv is not None: - def _stop_imap_cb(): - logger.debug('Stopping in memory store.') - self._memstore.stop_and_flush() - while not self._memstore.producer.is_queue_empty(): - logger.debug('Waiting for queue to be empty.') - # TODO use a gatherResults over the new/dirty - # deferred list, - # as in memorystore's expunge() method. - time.sleep(1) - # notify that service has stopped - logger.debug('Notifying that service has stopped.') - cv.acquire() - cv.notify() - cv.release() - - return threads.deferToThread(_stop_imap_cb) + return ServerFactory.doStop(self) def run_service(*args, **kwargs): @@ -185,11 +137,6 @@ def run_service(*args, **kwargs): the reactor when starts listening, and the factory for the protocol. """ - from twisted.internet import reactor - # it looks like qtreactor does not honor this, - # but other reactors should. - reactor.suggestThreadPoolSize(20) - leap_assert(len(args) == 2) soledad, keymanager = args leap_assert_type(soledad, Soledad) @@ -201,13 +148,14 @@ def run_service(*args, **kwargs): leap_check(userid is not None, "need an user id") offline = kwargs.get('offline', False) - uuid = soledad._get_uuid() + uuid = soledad.uuid factory = LeapIMAPFactory(uuid, userid, soledad) try: tport = reactor.listenTCP(port, factory, interface="localhost") if not offline: + # FIXME --- update after meskio's work fetcher = LeapIncomingMail( keymanager, soledad, @@ -236,6 +184,8 @@ def run_service(*args, **kwargs): interface="127.0.0.1") logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) leap_events.signal(IMAP_SERVICE_STARTED, str(port)) + + # FIXME -- change service signature return fetcher, tport, factory # not ok, signal error. diff --git a/src/leap/mail/imap/tests/leap_tests_imap.zsh b/src/leap/mail/imap/tests/leap_tests_imap.zsh deleted file mode 100755 index 544faca..0000000 --- a/src/leap/mail/imap/tests/leap_tests_imap.zsh +++ /dev/null @@ -1,178 +0,0 @@ -#!/bin/zsh -# BATCH STRESS TEST FOR IMAP ---------------------- -# http://imgs.xkcd.com/comics/science.jpg -# -# Run imaptest against a LEAP IMAP server -# for a fixed period of time, and collect output. -# -# Author: Kali Kaneko -# Date: 2014 01 26 -# -# To run, you need to have `imaptest` in your path. -# See: -# http://www.imapwiki.org/ImapTest/Installation -# -# For the tests, I'm using a 10MB file sample that -# can be downloaded from: -# http://www.dovecot.org/tmp/dovecot-crlf -# -# Want to contribute to benchmarking? -# -# 1. Create a pristine account in a bitmask provider. -# -# 2. Launch your bitmask client, with different flags -# if you desire. -# -# For example to try the nosync flag in sqlite: -# -# LEAP_SQLITE_NOSYNC=1 bitmask --debug -N --offline -l /tmp/leap.log -# -# 3. Run at several points in time (ie: just after -# launching the bitmask client. one minute after, -# ten minutes after) -# -# mkdir data -# cd data -# ../leap_tests_imap.zsh | tee sqlite_nosync_run2.log -# -# 4. Submit your results to: kali at leap dot se -# together with the logs of the bitmask run. -# -# Please provide also details about your system, and -# the type of hard disk setup you are running against. -# - -# ------------------------------------------------ -# Edit these variables if you are too lazy to pass -# the user and mbox as parameters. Like me. - -USER="test_f14@dev.bitmask.net" -MBOX="~/leap/imaptest/data/dovecot-crlf" - -HOST="localhost" -PORT="1984" - -# in case you have it aliased -GREP="/bin/grep" -IMAPTEST="imaptest" - -# ----------------------------------------------- -# -# These should be kept constant across benchmarking -# runs across different machines, for comparability. - -DURATION=200 -NUM_MSG=200 - - -# TODO add another function, and a cli flag, to be able -# to take several aggretates spaced in time, along a period -# of several minutes. - -imaptest_cmd() { - stdbuf -o0 ${IMAPTEST} user=${USER} pass=1234 host=${HOST} \ - port=${PORT} mbox=${MBOX} clients=1 msgs=${NUM_MSG} \ - no_pipelining 2>/dev/null -} - -stress_imap() { - mkfifo imap_pipe - cat imap_pipe | tee output & - imaptest_cmd >> imap_pipe -} - -wait_and_kill() { - while : - do - sleep $DURATION - pkill -2 imaptest - rm imap_pipe - break - done -} - -print_results() { - sleep 1 - echo - echo - echo "AGGREGATED RESULTS" - echo "----------------------" - echo "\tavg\tstdev" - $GREP "avg" ./output | sed -e 's/^ *//g' -e 's/ *$//g' | \ - gawk ' -function avg(data, count) { - sum=0; - for( x=0; x <= count-1; x++) { - sum += data[x]; - } - return sum/count; -} -function std_dev(data, count) { - sum=0; - for( x=0; x <= count-1; x++) { - sum += data[x]; - } - average = sum/count; - - sumsq=0; - for( x=0; x <= count-1; x++) { - sumsq += (data[x] - average)^2; - } - return sqrt(sumsq/count); -} -BEGIN { - cnt = 0 -} END { - -printf("LOGI:\t%04.2lf\t%04.2f\n", avg(array[1], NR), std_dev(array[1], NR)); -printf("LIST:\t%04.2lf\t%04.2f\n", avg(array[2], NR), std_dev(array[2], NR)); -printf("STAT:\t%04.2lf\t%04.2f\n", avg(array[3], NR), std_dev(array[3], NR)); -printf("SELE:\t%04.2lf\t%04.2f\n", avg(array[4], NR), std_dev(array[4], NR)); -printf("FETC:\t%04.2lf\t%04.2f\n", avg(array[5], NR), std_dev(array[5], NR)); -printf("FET2:\t%04.2lf\t%04.2f\n", avg(array[6], NR), std_dev(array[6], NR)); -printf("STOR:\t%04.2lf\t%04.2f\n", avg(array[7], NR), std_dev(array[7], NR)); -printf("DELE:\t%04.2lf\t%04.2f\n", avg(array[8], NR), std_dev(array[8], NR)); -printf("EXPU:\t%04.2lf\t%04.2f\n", avg(array[9], NR), std_dev(array[9], NR)); -printf("APPE:\t%04.2lf\t%04.2f\n", avg(array[10], NR), std_dev(array[10], NR)); -printf("LOGO:\t%04.2lf\t%04.2f\n", avg(array[11], NR), std_dev(array[11], NR)); - -print "" -print "TOT samples", NR; -} -{ - it = cnt++; - array[1][it] = $1; - array[2][it] = $2; - array[3][it] = $3; - array[4][it] = $4; - array[5][it] = $5; - array[6][it] = $6; - array[7][it] = $7; - array[8][it] = $8; - array[9][it] = $9; - array[10][it] = $10; - array[11][it] = $11; -}' -} - - -{ test $1 = "--help" } && { - echo "Usage: $0 [user@provider] [/path/to/sample.mbox]" - exit 0 -} - -# If the first parameter is passed, take it as the user -{ test $1 } && { - USER=$1 -} - -# If the second parameter is passed, take it as the mbox -{ test $2 } && { - MBOX=$2 -} - -echo "[+] LEAP IMAP TESTS" -echo "[+] Running imaptest for $DURATION seconds with $NUM_MSG messages" -wait_and_kill & -stress_imap -print_results diff --git a/src/leap/mail/imap/tests/stress_tests_imap.zsh b/src/leap/mail/imap/tests/stress_tests_imap.zsh new file mode 100755 index 0000000..544faca --- /dev/null +++ b/src/leap/mail/imap/tests/stress_tests_imap.zsh @@ -0,0 +1,178 @@ +#!/bin/zsh +# BATCH STRESS TEST FOR IMAP ---------------------- +# http://imgs.xkcd.com/comics/science.jpg +# +# Run imaptest against a LEAP IMAP server +# for a fixed period of time, and collect output. +# +# Author: Kali Kaneko +# Date: 2014 01 26 +# +# To run, you need to have `imaptest` in your path. +# See: +# http://www.imapwiki.org/ImapTest/Installation +# +# For the tests, I'm using a 10MB file sample that +# can be downloaded from: +# http://www.dovecot.org/tmp/dovecot-crlf +# +# Want to contribute to benchmarking? +# +# 1. Create a pristine account in a bitmask provider. +# +# 2. Launch your bitmask client, with different flags +# if you desire. +# +# For example to try the nosync flag in sqlite: +# +# LEAP_SQLITE_NOSYNC=1 bitmask --debug -N --offline -l /tmp/leap.log +# +# 3. Run at several points in time (ie: just after +# launching the bitmask client. one minute after, +# ten minutes after) +# +# mkdir data +# cd data +# ../leap_tests_imap.zsh | tee sqlite_nosync_run2.log +# +# 4. Submit your results to: kali at leap dot se +# together with the logs of the bitmask run. +# +# Please provide also details about your system, and +# the type of hard disk setup you are running against. +# + +# ------------------------------------------------ +# Edit these variables if you are too lazy to pass +# the user and mbox as parameters. Like me. + +USER="test_f14@dev.bitmask.net" +MBOX="~/leap/imaptest/data/dovecot-crlf" + +HOST="localhost" +PORT="1984" + +# in case you have it aliased +GREP="/bin/grep" +IMAPTEST="imaptest" + +# ----------------------------------------------- +# +# These should be kept constant across benchmarking +# runs across different machines, for comparability. + +DURATION=200 +NUM_MSG=200 + + +# TODO add another function, and a cli flag, to be able +# to take several aggretates spaced in time, along a period +# of several minutes. + +imaptest_cmd() { + stdbuf -o0 ${IMAPTEST} user=${USER} pass=1234 host=${HOST} \ + port=${PORT} mbox=${MBOX} clients=1 msgs=${NUM_MSG} \ + no_pipelining 2>/dev/null +} + +stress_imap() { + mkfifo imap_pipe + cat imap_pipe | tee output & + imaptest_cmd >> imap_pipe +} + +wait_and_kill() { + while : + do + sleep $DURATION + pkill -2 imaptest + rm imap_pipe + break + done +} + +print_results() { + sleep 1 + echo + echo + echo "AGGREGATED RESULTS" + echo "----------------------" + echo "\tavg\tstdev" + $GREP "avg" ./output | sed -e 's/^ *//g' -e 's/ *$//g' | \ + gawk ' +function avg(data, count) { + sum=0; + for( x=0; x <= count-1; x++) { + sum += data[x]; + } + return sum/count; +} +function std_dev(data, count) { + sum=0; + for( x=0; x <= count-1; x++) { + sum += data[x]; + } + average = sum/count; + + sumsq=0; + for( x=0; x <= count-1; x++) { + sumsq += (data[x] - average)^2; + } + return sqrt(sumsq/count); +} +BEGIN { + cnt = 0 +} END { + +printf("LOGI:\t%04.2lf\t%04.2f\n", avg(array[1], NR), std_dev(array[1], NR)); +printf("LIST:\t%04.2lf\t%04.2f\n", avg(array[2], NR), std_dev(array[2], NR)); +printf("STAT:\t%04.2lf\t%04.2f\n", avg(array[3], NR), std_dev(array[3], NR)); +printf("SELE:\t%04.2lf\t%04.2f\n", avg(array[4], NR), std_dev(array[4], NR)); +printf("FETC:\t%04.2lf\t%04.2f\n", avg(array[5], NR), std_dev(array[5], NR)); +printf("FET2:\t%04.2lf\t%04.2f\n", avg(array[6], NR), std_dev(array[6], NR)); +printf("STOR:\t%04.2lf\t%04.2f\n", avg(array[7], NR), std_dev(array[7], NR)); +printf("DELE:\t%04.2lf\t%04.2f\n", avg(array[8], NR), std_dev(array[8], NR)); +printf("EXPU:\t%04.2lf\t%04.2f\n", avg(array[9], NR), std_dev(array[9], NR)); +printf("APPE:\t%04.2lf\t%04.2f\n", avg(array[10], NR), std_dev(array[10], NR)); +printf("LOGO:\t%04.2lf\t%04.2f\n", avg(array[11], NR), std_dev(array[11], NR)); + +print "" +print "TOT samples", NR; +} +{ + it = cnt++; + array[1][it] = $1; + array[2][it] = $2; + array[3][it] = $3; + array[4][it] = $4; + array[5][it] = $5; + array[6][it] = $6; + array[7][it] = $7; + array[8][it] = $8; + array[9][it] = $9; + array[10][it] = $10; + array[11][it] = $11; +}' +} + + +{ test $1 = "--help" } && { + echo "Usage: $0 [user@provider] [/path/to/sample.mbox]" + exit 0 +} + +# If the first parameter is passed, take it as the user +{ test $1 } && { + USER=$1 +} + +# If the second parameter is passed, take it as the mbox +{ test $2 } && { + MBOX=$2 +} + +echo "[+] LEAP IMAP TESTS" +echo "[+] Running imaptest for $DURATION seconds with $NUM_MSG messages" +wait_and_kill & +stress_imap +print_results diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index dbb823f..d7fcdce 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -17,7 +17,7 @@ """ Test case for leap.email.imap.server TestCases taken from twisted tests and modified to make them work -against SoledadBackedAccount. +against our implementation of the IMAPAccount. @authors: Kali Kaneko, XXX add authors from the original twisted tests. @@ -32,19 +32,13 @@ import types from twisted.mail import imap4 from twisted.internet import defer -from twisted.trial import unittest from twisted.python import util from twisted.python import failure from twisted import cred - -# import u1db - -from leap.mail.imap.mailbox import SoledadMailbox -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.messages import MessageCollection -from leap.mail.imap.server import LeapIMAPServer +from leap.mail.imap.mailbox import IMAPMailbox +from leap.mail.imap.messages import IMAPMessageCollection from leap.mail.imap.tests.utils import IMAP4HelperMixin @@ -81,7 +75,7 @@ class TestRealm: # TestCases # -class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): +class MessageCollectionTestCase(IMAP4HelperMixin): """ Tests for the MessageCollection class """ @@ -95,10 +89,9 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): """ super(MessageCollectionTestCase, self).setUp() - # TODO deprecate memstore - memstore = MemoryStore() - self.messages = MessageCollection("testmbox%s" % (self.count,), - self._soledad, memstore=memstore) + # FIXME --- update initialization + self.messages = IMAPMessageCollection( + "testmbox%s" % (self.count,), self._soledad) MessageCollectionTestCase.count += 1 def tearDown(self): @@ -207,23 +200,18 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase): add_1().addCallback(lambda ignored: self.assertEqual( mc.count(), 3)) - # XXX this has to be redone to fit memstore ------------# - #newmsg = mc._get_empty_doc() - #newmsg['mailbox'] = "mailbox/foo" - #mc._soledad.create_doc(newmsg) - #self.assertEqual(mc.count(), 3) - #self.assertEqual( - #len(mc._soledad.get_from_index(mc.TYPE_IDX, "flags")), 4) + +# DEBUG --- +#from twisted.internet.base import DelayedCall +#DelayedCall.debug = True -class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): - # TODO this currently will use a memory-only store. - # create a different one for testing soledad sync. +class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): """ - Tests for the generic behavior of the LeapIMAP4Server + Tests for the generic behavior of the LEAPIMAP4Server which, right now, it's just implemented in this test file as - LeapIMAPServer. We will move the implementation, together with + LEAPIMAPServer. We will move the implementation, together with authentication bits, to leap.mail.imap.server so it can be instantiated from the tac file. @@ -243,6 +231,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ succeed = ('testbox', 'test/box', 'test/', 'test/box/box', 'foobox') fail = ('testbox', 'test/box') + acc = self.server.theAccount def cb(): self.result.append(1) @@ -267,24 +256,23 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d1 = self.connected.addCallback(strip(login)) d1.addCallback(strip(create)) d2 = self.loopback() - d = defer.gatherResults([d1, d2]) + d = defer.gatherResults([d1, d2], consumeErrors=True) + d.addCallback(lambda _: acc.account.list_all_mailbox_names()) return d.addCallback(self._cbTestCreate, succeed, fail) - def _cbTestCreate(self, ignored, succeed, fail): + def _cbTestCreate(self, mailboxes, succeed, fail): self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail)) - mboxes = LeapIMAPServer.theAccount.mailboxes - answers = ([u'INBOX', u'testbox', u'test/box', u'test', u'test/box/box', 'foobox']) - self.assertEqual(sorted(mboxes), sorted([a for a in answers])) + self.assertEqual(sorted(mailboxes), sorted([a for a in answers])) def testDelete(self): """ Test whether we can delete mailboxes """ - acc = LeapIMAPServer.theAccount - d0 = lambda: acc.addMailbox('test-delete/me') + def add_mailbox(): + return self.server.theAccount.addMailbox('test-delete/me') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -292,15 +280,17 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def delete(): return self.client.delete('test-delete/me') - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) + acc = self.server.theAccount.account + + d1 = self.connected.addCallback(add_mailbox) + d1.addCallback(strip(login)) d1.addCallbacks(strip(delete), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback( - lambda _: self.assertEqual( - LeapIMAPServer.theAccount.mailboxes, ['INBOX'])) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(lambda mboxes: self.assertEqual( + mboxes, ['INBOX'])) return d def testIllegalInboxDelete(self): @@ -359,29 +349,34 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): Try deleting a mailbox with sub-folders, and \NoSelect flag set. An exception is expected. """ - acc = LeapIMAPServer.theAccount - d_del0 = lambda: acc.addMailbox('delete') - d_del1 = lambda: acc.addMailbox('delete/me') - - def set_noselect_flag(): - mbox = acc.getMailbox('delete') - mbox.setFlags((r'\Noselect',)) + acc = self.server.theAccount def login(): return self.client.login(TEST_USER, TEST_PASSWD) - def delete(): + def create_mailboxes(): + d1 = acc.addMailbox('delete') + d2 = acc.addMailbox('delete/me') + d = defer.gatherResults([d1, d2]) + return d + + def get_noselect_mailbox(mboxes): + mbox = mboxes[0] + return mbox.setFlags((r'\Noselect',)) + + def delete_mbox(ignored): return self.client.delete('delete') def deleteFailed(failure): self.failure = failure self.failure = None + d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d_del0)) - d1.addCallback(strip(d_del1)) - d1.addCallback(strip(set_noselect_flag)) - d1.addCallback(strip(delete)).addErrback(deleteFailed) + d1.addCallback(strip(create_mailboxes)) + d1.addCallback(get_noselect_mailbox) + + d1.addCallback(delete_mbox).addErrback(deleteFailed) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) @@ -393,11 +388,15 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.assertEqual(str(self.failure.value), expected)) return d + # FIXME --- this test sometimes FAILS (timing issue). + # Some of the deferreds used in the rename op is not waiting for the + # operations properly def testRename(self): """ Test whether we can rename a mailbox """ - d0 = lambda: LeapIMAPServer.theAccount.addMailbox('oldmbox') + def create_mbox(): + return self.server.theAccount.addMailbox('oldmbox') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -405,16 +404,16 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def rename(): return self.client.rename('oldmbox', 'newname') - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) + d1 = self.connected.addCallback(strip(create_mbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(rename), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) d.addCallback(lambda _: - self.assertEqual( - LeapIMAPServer.theAccount.mailboxes, - ['INBOX', 'newname'])) + self.server.theAccount.account.list_all_mailbox_names()) + d.addCallback(lambda mboxes: + self.assertItemsEqual(mboxes, ['INBOX', 'newname'])) return d def testIllegalInboxRename(self): @@ -448,7 +447,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Try to rename hierarchical mailboxes """ - acc = LeapIMAPServer.theAccount + acc = LEAPIMAPServer.theAccount dc1 = lambda: acc.create('oldmbox/m1') dc2 = lambda: acc.create('oldmbox/m2') @@ -468,7 +467,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestHierarchicalRename) def _cbTestHierarchicalRename(self, ignored): - mboxes = LeapIMAPServer.theAccount.mailboxes + mboxes = LEAPIMAPServer.theAccount.mailboxes expected = ['INBOX', 'newname', 'newname/m1', 'newname/m2'] self.assertEqual(sorted(mboxes), sorted([s for s in expected])) @@ -476,6 +475,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test whether we can mark a mailbox as subscribed to """ + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('this/mbox') + def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -483,9 +487,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.subscribe('this/mbox') def get_subscriptions(ignored): - return LeapIMAPServer.theAccount.getSubscriptions() + return self.server.theAccount.getSubscriptions() - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(subscribe), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() @@ -500,7 +505,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test whether we can unsubscribe from a set of mailboxes """ - acc = LeapIMAPServer.theAccount + acc = self.server.theAccount + + def add_mailboxes(): + return defer.gatherResults([ + acc.addMailbox('this/mbox'), + acc.addMailbox('that/mbox')]) dc1 = lambda: acc.subscribe('this/mbox') dc2 = lambda: acc.subscribe('that/mbox') @@ -512,9 +522,10 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return self.client.unsubscribe('this/mbox') def get_subscriptions(ignored): - return LeapIMAPServer.theAccount.getSubscriptions() + return acc.getSubscriptions() - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailboxes)) + d1.addCallback(strip(login)) d1.addCallback(strip(dc1)) d1.addCallback(strip(dc2)) d1.addCallbacks(strip(unsubscribe), self._ebGeneral) @@ -531,10 +542,14 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Try to select a mailbox """ - acc = self.server.theAccount - d0 = lambda: acc.addMailbox('TESTMAILBOX-SELECT', creation_ts=42) + mbox_name = "TESTMAILBOXSELECT" self.selectedArgs = None + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox(mbox_name, creation_ts=42) + def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -542,30 +557,26 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def selected(args): self.selectedArgs = args self._cbStopClient(None) - d = self.client.select('TESTMAILBOX-SELECT') + d = self.client.select(mbox_name) d.addCallback(selected) return d - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallback(strip(select)) - d1.addErrback(self._ebGeneral) + #d1.addErrback(self._ebGeneral) d2 = self.loopback() - return defer.gatherResults([d1, d2]).addCallback(self._cbTestSelect) + + d = defer.gatherResults([d1, d2]) + d.addCallback(self._cbTestSelect) + return d def _cbTestSelect(self, ignored): - mbox = LeapIMAPServer.theAccount.getMailbox('TESTMAILBOX-SELECT') - self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox) - # XXX UIDVALIDITY should be "42" if the creation_ts is passed along - # to the memory store. However, the current state of the account - # implementation is incomplete and we're writing to soledad store - # directly there. We should handle the UIDVALIDITY timestamping - # mechanism in a separate test suite. + self.assertTrue(self.selectedArgs is not None) self.assertEqual(self.selectedArgs, { - 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 0, - # 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, + 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged', '\\Deleted', '\\Draft', '\\Recent', 'List'), 'READ-WRITE': True @@ -583,13 +594,16 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): caps.update(c) self.server.transport.loseConnection() return self.client.getCapabilities().addCallback(gotCaps) - d1 = self.connected.addCallback( + + d1 = self.connected + d1.addCallback( strip(getCaps)).addErrback(self._ebGeneral) + d = defer.gatherResults([self.loopback(), d1]) expected = {'IMAP4rev1': None, 'NAMESPACE': None, 'LITERAL+': None, 'IDLE': None} - - return d.addCallback(lambda _: self.assertEqual(expected, caps)) + d.addCallback(lambda _: self.assertEqual(expected, caps)) + return d def testCapabilityWithAuth(self): caps = {} @@ -610,7 +624,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): 'IDLE': None, 'LITERAL+': None, 'AUTH': ['CRAM-MD5']} - return d.addCallback(lambda _: self.assertEqual(expCap, caps)) + d.addCallback(lambda _: self.assertEqual(expCap, caps)) + return d # # authentication @@ -658,7 +673,6 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestLogin) def _cbTestLogin(self, ignored): - self.assertEqual(self.server.account, LeapIMAPServer.theAccount) self.assertEqual(self.server.state, 'auth') def testFailedLogin(self): @@ -696,7 +710,6 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestLoginRequiringQuoting) def _cbTestLoginRequiringQuoting(self, ignored): - self.assertEqual(self.server.account, LeapIMAPServer.theAccount) self.assertEqual(self.server.state, 'auth') # @@ -743,11 +756,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): for details. """ # TODO implement the IMAP4ClientExamineTests testcase. - - self.server.theAccount.addMailbox('test-mailbox-e', - creation_ts=42) + mbox_name = "test_mailbox_e" + acc = self.server.theAccount self.examinedArgs = None + def add_mailbox(): + return acc.addMailbox(mbox_name, creation_ts=42) + def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -755,11 +770,12 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def examined(args): self.examinedArgs = args self._cbStopClient(None) - d = self.client.examine('test-mailbox-e') + d = self.client.examine(mbox_name) d.addCallback(examined) return d - d1 = self.connected.addCallback(strip(login)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallback(strip(examine)) d1.addErrback(self._ebGeneral) d2 = self.loopback() @@ -767,27 +783,19 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): return d.addCallback(self._cbTestExamine) def _cbTestExamine(self, ignored): - mbox = self.server.theAccount.getMailbox('test-mailbox-e') - self.assertEqual(self.server.mbox.messages.mbox, mbox.messages.mbox) - - # XXX UIDVALIDITY should be "42" if the creation_ts is passed along - # to the memory store. However, the current state of the account - # implementation is incomplete and we're writing to soledad store - # directly there. We should handle the UIDVALIDITY timestamping - # mechanism in a separate test suite. self.assertEqual(self.examinedArgs, { - 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 0, - # 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, + 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged', '\\Deleted', '\\Draft', '\\Recent', 'List'), 'READ-WRITE': False}) def _listSetup(self, f, f2=None): - acc = LeapIMAPServer.theAccount - dc1 = lambda: acc.addMailbox('root/subthing', creation_ts=42) - dc2 = lambda: acc.addMailbox('root/another-thing', creation_ts=42) - dc3 = lambda: acc.addMailbox('non-root/subthing', creation_ts=42) + acc = self.server.theAccount + + dc1 = lambda: acc.addMailbox('root_subthing', creation_ts=42) + dc2 = lambda: acc.addMailbox('root_another_thing', creation_ts=42) + dc3 = lambda: acc.addMailbox('non_root_subthing', creation_ts=42) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -816,12 +824,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ def list(): return self.client.list('root', '%') + d = self._listSetup(list) d.addCallback(lambda listed: self.assertEqual( sortNest(listed), sortNest([ - (SoledadMailbox.INIT_FLAGS, "/", "root/subthing"), - (SoledadMailbox.INIT_FLAGS, "/", "root/another-thing") + (IMAPMailbox.init_flags, "/", "root_subthing"), + (IMAPMailbox.init_flags, "/", "root_another_thing") ]) )) return d @@ -830,28 +839,28 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): """ Test LSub command """ - acc = LeapIMAPServer.theAccount + acc = self.server.theAccount def subs_mailbox(): # why not client.subscribe instead? - return acc.subscribe('root/subthing') + return acc.subscribe('root_subthing') def lsub(): return self.client.lsub('root', '%') d = self._listSetup(lsub, strip(subs_mailbox)) d.addCallback(self.assertEqual, - [(SoledadMailbox.INIT_FLAGS, "/", "root/subthing")]) + [(IMAPMailbox.init_flags, "/", "root_subthing")]) return d def testStatus(self): """ Test Status command """ - acc = LeapIMAPServer.theAccount + acc = self.server.theAccount def add_mailbox(): - return acc.addMailbox('root/subthings') + return acc.addMailbox('root_subthings') # XXX FIXME ---- should populate this a little bit, # with unseen etc... @@ -861,7 +870,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def status(): return self.client.status( - 'root/subthings', 'MESSAGES', 'UIDNEXT', 'UNSEEN') + 'root_subthings', 'MESSAGES', 'UIDNEXT', 'UNSEEN') def statused(result): self.statused = result @@ -927,7 +936,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): infile = util.sibpath(__file__, 'rfc822.message') message = open(infile) acc = self.server.theAccount - mailbox_name = "root_subthing" + mailbox_name = "root/subthing" def add_mailbox(): return acc.addMailbox(mailbox_name) @@ -948,42 +957,62 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: acc.getMailbox(mailbox_name)) - def print_mb(mb): - print "MB ----", mb - return mb - d.addCallback(print_mb) - d.addCallback(lambda mb: mb.collection.get_message_by_uid(1)) + d.addCallback(lambda _: acc.getMailbox(mailbox_name)) + d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True)) return d.addCallback(self._cbTestFullAppend, infile) - def _cbTestFullAppend(self, msg, infile): - # TODO --- move to deferreds - self.assertEqual( - set(('\\Recent', '\\SEEN', '\\DELETED')), - set(msg.getFlags())) + def _cbTestFullAppend(self, fetched, infile): + self.assertTrue(len(fetched) == 1) + self.assertTrue(len(fetched[0]) == 2) + uid, msg = fetched[0] + parsed = self.parser.parse(open(infile)) + expected_body = parsed.get_payload() + expected_headers = dict(parsed.items()) - self.assertEqual( - 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', - msg.getInternalDate()) + def assert_flags(flags): + self.assertEqual( + set(('\\SEEN', '\\DELETED')), + set(flags)) - parsed = self.parser.parse(open(infile)) - body = parsed.get_payload() - headers = dict(parsed.items()) - self.assertEqual( - body, - msg.getBodyFile().read()) - gotheaders = msg.getHeaders(True) + def assert_date(date): + self.assertEqual( + 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', + date) + + def assert_body(body): + gotbody = body.read() + self.assertEqual(expected_body, gotbody) + + def assert_headers(headers): + self.assertItemsEqual(expected_headers, headers) + + d = defer.maybeDeferred(msg.getFlags) + d.addCallback(assert_flags) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getInternalDate)) + d.addCallback(assert_date) - self.assertItemsEqual( - headers, gotheaders) + d.addCallback( + lambda _: defer.maybeDeferred( + msg.getBodyFile, self._soledad)) + d.addCallback(assert_body) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getHeaders, True)) + d.addCallback(assert_headers) + + return d def testPartialAppend(self): """ Test partially appending a message to the mailbox """ infile = util.sibpath(__file__, 'rfc822.message') - d0 = lambda: LeapIMAPServer.theAccount.addMailbox('PARTIAL/SUBTHING') + + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('PARTIAL/SUBTHING') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -998,34 +1027,46 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): (), self.client._IMAP4Client__cbContinueAppend, message ) ) - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(append), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) + + d.addCallback(lambda _: acc.getMailbox("PARTIAL/SUBTHING")) + d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True)) return d.addCallback( self._cbTestPartialAppend, infile) - def _cbTestPartialAppend(self, ignored, infile): - mb = LeapIMAPServer.theAccount.getMailbox('PARTIAL/SUBTHING') - self.assertEqual(1, len(mb.messages)) - msg = mb.messages.get_msg_by_uid(1) - self.assertEqual( - set(('\\SEEN', '\\Recent')), - set(msg.getFlags()) - ) + def _cbTestPartialAppend(self, fetched, infile): + self.assertTrue(len(fetched) == 1) + self.assertTrue(len(fetched[0]) == 2) + uid, msg = fetched[0] parsed = self.parser.parse(open(infile)) - body = parsed.get_payload() - self.assertEqual( - body, - msg.getBodyFile().read()) + expected_body = parsed.get_payload() + + def assert_flags(flags): + self.assertEqual( + set((['\\SEEN'])), set(flags)) + + def assert_body(body): + gotbody = body.read() + self.assertEqual(expected_body, gotbody) + + d = defer.maybeDeferred(msg.getFlags) + d.addCallback(assert_flags) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getBodyFile)) + d.addCallback(assert_body) + return d def testCheck(self): """ Test check command """ - LeapIMAPServer.theAccount.addMailbox('root/subthing') + def add_mailbox(): + return self.server.theAccount.addMailbox('root/subthing') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -1036,98 +1077,106 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): def check(): return self.client.check() - d = self.connected.addCallback(strip(login)) + d = self.connected.addCallbacks( + strip(add_mailbox), self._ebGeneral) + d.addCallbacks(lambda _: login(), self._ebGeneral) d.addCallbacks(strip(select), self._ebGeneral) d.addCallbacks(strip(check), self._ebGeneral) d.addCallbacks(self._cbStopClient, self._ebGeneral) - return self.loopback() - - # Okay, that was fun - - def testClose(self): - """ - Test closing the mailbox. We expect to get deleted all messages flagged - as such. - """ - acc = self.server.theAccount - name = 'mailbox-close' - - d0 = lambda: acc.addMailbox(name) - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def select(): - return self.client.select(name) - - def get_mailbox(): - self.mailbox = LeapIMAPServer.theAccount.getMailbox(name) - - def add_messages(): - d1 = self.mailbox.messages.add_msg( - 'test 1', subject="Message 1", - flags=('\\Deleted', 'AnotherFlag')) - d2 = self.mailbox.messages.add_msg( - 'test 2', subject="Message 2", - flags=('AnotherFlag',)) - d3 = self.mailbox.messages.add_msg( - 'test 3', subject="Message 3", - flags=('\\Deleted',)) - d = defer.gatherResults([d1, d2, d3]) - return d - - def close(): - return self.client.close() - - d = self.connected.addCallback(strip(login)) - d.addCallback(strip(d0)) - d.addCallbacks(strip(select), self._ebGeneral) - d.addCallback(strip(get_mailbox)) - d.addCallbacks(strip(add_messages), self._ebGeneral) - d.addCallbacks(strip(close), self._ebGeneral) - d.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() - return defer.gatherResults([d, d2]).addCallback(self._cbTestClose) - - def _cbTestClose(self, ignored): - self.assertEqual(len(self.mailbox.messages), 1) - msg = self.mailbox.messages.get_msg_by_uid(2) - self.assertTrue(msg is not None) - - self.assertEqual( - dict(msg.hdoc.content)['subject'], - 'Message 2') - self.failUnless(self.mailbox.closed) + return defer.gatherResults([d, d2]) + + # Okay, that was much fun indeed + + # skipping close test: we just need expunge for now. + #def testClose(self): + #""" + #Test closing the mailbox. We expect to get deleted all messages flagged + #as such. + #""" + #acc = self.server.theAccount + #mailbox_name = 'mailbox-close' +# + #def add_mailbox(): + #return acc.addMailbox(mailbox_name) +# + #def login(): + #return self.client.login(TEST_USER, TEST_PASSWD) +# + #def select(): + #return self.client.select(mailbox_name) +# + #def get_mailbox(): + #def _save_mbox(mailbox): + #self.mailbox = mailbox + #d = self.server.theAccount.getMailbox(mailbox_name) + #d.addCallback(_save_mbox) + #return d +# + #def add_messages(): + #d1 = self.mailbox.addMessage( + #'test 1', flags=('\\Deleted', 'AnotherFlag')) + #d2 = self.mailbox.addMessage( + #'test 2', flags=('AnotherFlag',)) + #d3 = self.mailbox.addMessage( + #'test 3', flags=('\\Deleted',)) + #d = defer.gatherResults([d1, d2, d3]) + #return d +# + #def close(): + #return self.client.close() +# + #d = self.connected.addCallback(strip(add_mailbox)) + #d.addCallback(strip(login)) + #d.addCallbacks(strip(select), self._ebGeneral) + #d.addCallback(strip(get_mailbox)) + #d.addCallbacks(strip(add_messages), self._ebGeneral) + #d.addCallbacks(strip(close), self._ebGeneral) + #d.addCallbacks(self._cbStopClient, self._ebGeneral) + #d2 = self.loopback() + #d1 = defer.gatherResults([d, d2]) + #d1.addCallback(lambda _: self.mailbox.getMessageCount()) + #d1.addCallback(self._cbTestClose) + #return d1 +# + #def _cbTestClose(self, count): + # TODO is this correct? count should not take into account those + # flagged as deleted??? + #self.assertEqual(count, 1) + # TODO --- assert flags are those of the message #2 + #self.failUnless(self.mailbox.closed) def testExpunge(self): """ Test expunge command """ acc = self.server.theAccount - name = 'mailbox-expunge' + mailbox_name = 'mailboxexpunge' - d0 = lambda: acc.addMailbox(name) + def add_mailbox(): + return acc.addMailbox(mailbox_name) def login(): return self.client.login(TEST_USER, TEST_PASSWD) def select(): - return self.client.select('mailbox-expunge') + return self.client.select(mailbox_name) + + def save_mailbox(mailbox): + self.mailbox = mailbox def get_mailbox(): - self.mailbox = LeapIMAPServer.theAccount.getMailbox(name) + d = acc.getMailbox(mailbox_name) + d.addCallback(save_mailbox) + return d def add_messages(): - d1 = self.mailbox.messages.add_msg( - 'test 1', subject="Message 1", - flags=('\\Deleted', 'AnotherFlag')) - d2 = self.mailbox.messages.add_msg( - 'test 2', subject="Message 2", - flags=('AnotherFlag',)) - d3 = self.mailbox.messages.add_msg( - 'test 3', subject="Message 3", - flags=('\\Deleted',)) - d = defer.gatherResults([d1, d2, d3]) + d = self.mailbox.addMessage( + 'test 1', flags=('\\Deleted', 'AnotherFlag')) + d.addCallback(lambda _: self.mailbox.addMessage( + 'test 2', flags=('AnotherFlag',))) + d.addCallback(lambda _: self.mailbox.addMessage( + 'test 3', flags=('\\Deleted',))) return d def expunge(): @@ -1138,49 +1187,53 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase): self.results = results self.results = None - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(d0)) - d1.addCallbacks(strip(select), self._ebGeneral) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) d1.addCallback(strip(get_mailbox)) d1.addCallbacks(strip(add_messages), self._ebGeneral) + d1.addCallbacks(strip(select), self._ebGeneral) d1.addCallbacks(strip(expunge), self._ebGeneral) d1.addCallbacks(expunged, self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: self.mailbox.getMessageCount()) return d.addCallback(self._cbTestExpunge) - def _cbTestExpunge(self, ignored): + def _cbTestExpunge(self, count): # we only left 1 mssage with no deleted flag - self.assertEqual(len(self.mailbox.messages), 1) - msg = self.mailbox.messages.get_msg_by_uid(2) - - msg = list(self.mailbox.messages)[0] - self.assertTrue(msg is not None) - - self.assertEqual( - msg.hdoc.content['subject'], - 'Message 2') - + self.assertEqual(count, 1) # the uids of the deleted messages self.assertItemsEqual(self.results, [1, 3]) -class AccountTestCase(IMAP4HelperMixin, unittest.TestCase): +class AccountTestCase(IMAP4HelperMixin): """ Test the Account. """ def _create_empty_mailbox(self): - LeapIMAPServer.theAccount.addMailbox('') + return self.server.theAccount.addMailbox('') def _create_one_mailbox(self): - LeapIMAPServer.theAccount.addMailbox('one') + return self.server.theAccount.addMailbox('one') def test_illegalMailboxCreate(self): - self.assertRaises(AssertionError, self._create_empty_mailbox) + # FIXME --- account.addMailbox needs to raise a failure, + # not the direct exception. + self.stashed = None + + def stash(result): + self.stashed = result + + d = self._create_empty_mailbox() + d.addBoth(stash) + d.addCallback(lambda _: self.failUnless(isinstance(self.stashed, + failure.Failure))) + return d + #self.assertRaises(AssertionError, self._create_empty_mailbox) -class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase): +class IMAP4ServerSearchTestCase(IMAP4HelperMixin): """ Tests for the behavior of the search_* functions in L{imap5.IMAP4Server}. """ diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index 920eeb0..5708787 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -1,30 +1,44 @@ -import os -import tempfile -import shutil - +# -*- coding: utf-8 -*- +# utils.py +# Copyright (C) 2014, 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Common utilities for testing Soledad IMAP Server. +""" from email import parser from mock import Mock from twisted.mail import imap4 from twisted.internet import defer from twisted.protocols import loopback +from twisted.python import log -from leap.common.testing.basetest import BaseLeapTest -from leap.mail.imap.account import SoledadBackedAccount -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.server import LeapIMAPServer -from leap.soledad.client import Soledad +from leap.mail.adaptors import soledad as soledad_adaptor +from leap.mail.imap.account import IMAPAccount +from leap.mail.imap.server import LEAPIMAPServer +from leap.mail.tests.common import SoledadTestMixin TEST_USER = "testuser@leap.se" TEST_PASSWD = "1234" + # # Simple IMAP4 Client for testing # - class SimpleClient(imap4.IMAP4Client): - """ A Simple IMAP4 Client to test our Soledad-LEAPServer @@ -51,160 +65,57 @@ class SimpleClient(imap4.IMAP4Client): self.transport.loseConnection() -# XXX move to common helper -def initialize_soledad(email, gnupg_home, tempdir): - """ - Initializes soledad by hand - - :param email: ID for the user - :param gnupg_home: path to home used by gnupg - :param tempdir: path to temporal dir - :rtype: Soledad instance - """ - - uuid = "foobar-uuid" - passphrase = u"verysecretpassphrase" - secret_path = os.path.join(tempdir, "secret.gpg") - local_db_path = os.path.join(tempdir, "soledad.u1db") - server_url = "https://provider" - cert_file = "" - - class MockSharedDB(object): - - get_doc = Mock(return_value=None) - put_doc = Mock() - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) - - def __call__(self): - return self - - Soledad._shared_db = MockSharedDB() - - _soledad = Soledad( - uuid, - passphrase, - secret_path, - local_db_path, - server_url, - cert_file, - syncable=False) - - return _soledad - - -# XXX this is not properly a mixin, since helper already inherits from -# uniittest.Testcase -class IMAP4HelperMixin(BaseLeapTest): +class IMAP4HelperMixin(SoledadTestMixin): """ MixIn containing several utilities to be shared across different TestCases """ - serverCTX = None clientCTX = None - # setUpClass cannot be a classmethod in trial, see: - # https://twistedmatrix.com/trac/ticket/1870 - def setUp(self): - """ - Setup method for each test. - - Initializes and run a LEAP IMAP4 Server. - """ - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir - - # Soledad: config info - self.gnupg_home = "%s/gnupg" % self.tempdir - self.email = 'leap@leap.se' - - # initialize soledad by hand so we can control keys - self._soledad = initialize_soledad( - self.email, - self.gnupg_home, - self.tempdir) - UUID = 'deadbeef', - USERID = TEST_USER - memstore = MemoryStore() - ########### + soledad_adaptor.cleanup_deferred_locks() - d_server_ready = defer.Deferred() + UUID = 'deadbeef', + USERID = TEST_USER - self.server = LeapIMAPServer( - uuid=UUID, userid=USERID, - contextFactory=self.serverCTX, - soledad=self._soledad) + def setup_server(account): + self.server = LEAPIMAPServer( + uuid=UUID, userid=USERID, + contextFactory=self.serverCTX, + soledad=self._soledad) + self.server.theAccount = account - self.client = SimpleClient( - d_server_ready, contextFactory=self.clientCTX) + d_server_ready = defer.Deferred() + self.client = SimpleClient( + d_server_ready, contextFactory=self.clientCTX) + self.connected = d_server_ready - theAccount = SoledadBackedAccount( - USERID, - soledad=self._soledad, - memstore=memstore) - d_account_ready = theAccount.callWhenReady(lambda r: None) - LeapIMAPServer.theAccount = theAccount + def setup_account(_): + self.parser = parser.Parser() - self.connected = defer.gatherResults( - [d_server_ready, d_account_ready]) + # XXX this should be fixed in soledad. + # Soledad sync makes trial block forever. The sync it's mocked to + # fix this problem. _mock_soledad_get_from_index can be used from + # the tests to provide documents. + self._soledad.sync = Mock() - # XXX FIXME -------------------------------------------- - # XXX this needs to be done differently, - # have to be hooked on initialization callback instead. - # in case we get something from previous tests... - #for mb in self.server.theAccount.mailboxes: - #self.server.theAccount.delete(mb) + d = defer.Deferred() + self.acc = IMAPAccount(USERID, self._soledad, d=d) + return d - # email parser - self.parser = parser.Parser() + d = super(IMAP4HelperMixin, self).setUp() + d.addCallback(setup_account) + d.addCallback(setup_server) + return d def tearDown(self): - """ - tearDown method called after each test. - """ - try: - self._soledad.close() - except Exception: - print "ERROR WHILE CLOSING SOLEDAD" - finally: - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check - assert 'leap_tests-' in self.tempdir - shutil.rmtree(self.tempdir) - - def populateMessages(self): - """ - Populates soledad instance with several simple messages - """ - # XXX we should encapsulate this thru SoledadBackedAccount - # instead. - - # XXX we also should put this in a mailbox! - - self._soledad.messages.add_msg('', subject="test1") - self._soledad.messages.add_msg('', subject="test2") - self._soledad.messages.add_msg('', subject="test3") - # XXX should change Flags too - self._soledad.messages.add_msg('', subject="test4") - - def delete_all_docs(self): - """ - Deletes all the docs in the testing instance of the - SoledadBackedAccount. - """ - self.server.theAccount.deleteAllMessages( - iknowhatiamdoing=True) + SoledadTestMixin.tearDown(self) + del self._soledad + del self.client + del self.server + del self.connected def _cbStopClient(self, ignore): self.client.transport.loseConnection() @@ -212,11 +123,8 @@ class IMAP4HelperMixin(BaseLeapTest): def _ebGeneral(self, failure): self.client.transport.loseConnection() self.server.transport.loseConnection() - # can we do something similar? - # I guess this was ok with trial, but not in noseland... - # log.err(failure, "Problem with %r" % (self.function,)) - raise failure.value - # failure.trap(Exception) + if hasattr(self, 'function'): + log.err(failure, "Problem with %r" % (self.function,)) def loopback(self): return loopback.loopbackAsync(self.server, self.client) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index b2caa33..8137f97 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -172,7 +172,7 @@ class Message(object): :return: An RFC822-formatted date string. :rtype: str """ - return self._wrapper.fdoc.date + return self._wrapper.hdoc.date # imap.IMessageParts @@ -271,9 +271,10 @@ class MessageCollection(object): store = None messageklass = Message - def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): + def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None, + count=None): """ - Constructore for a MessageCollection. + Constructor for a MessageCollection. """ self.adaptor = adaptor self.store = store @@ -317,7 +318,7 @@ class MessageCollection(object): wrapper = getattr(self, "mbox_wrapper", None) if not wrapper: return None - return wrapper.mbox_uuid + return wrapper.uuid def get_mbox_attr(self, attr): return getattr(self.mbox_wrapper, attr) @@ -364,10 +365,18 @@ class MessageCollection(object): self.messageklass, self.store, doc_id, uid=uid, get_cdocs=get_cdocs) - d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_name, uid) + d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_uuid, uid) d.addCallback(get_msg_from_mdoc_id) return d + # TODO deprecate ??? --- + def _prime_count(self): + def update_count(count): + self._count = count + d = self.mbox_indexer.count(self.mbox_name) + d.addCallback(update_count) + return d + def count(self): """ Count the messages in this collection. @@ -376,7 +385,17 @@ class MessageCollection(object): """ if not self.is_mailbox_collection(): raise NotImplementedError() - return self.mbox_indexer.count(self.mbox_name) + + d = self.mbox_indexer.count(self.mbox_uuid) + return d + + def count_recent(self): + # FIXME HACK + return 0 + + def count_unseen(self): + # FIXME hack + return 0 def get_uid_next(self): """ @@ -385,7 +404,13 @@ class MessageCollection(object): :return: a Deferred that will fire with the integer for the next uid. :rtype: Deferred """ - return self.mbox_indexer.get_next_uid(self.mbox_name) + return self.mbox_indexer.get_next_uid(self.mbox_uuid) + + def all_uid_iter(self): + """ + Iterator through all the uids for this collection. + """ + return self.mbox_indexer.all_uid_iter(self.mbox_uuid) # Manipulate messages @@ -397,6 +422,7 @@ class MessageCollection(object): flags = tuple() if not tags: tags = tuple() + leap_assert_type(flags, tuple) leap_assert_type(date, str) @@ -408,10 +434,10 @@ class MessageCollection(object): else: mbox_id = self.mbox_uuid + wrapper.set_mbox_uuid(mbox_id) wrapper.set_flags(flags) wrapper.set_tags(tags) wrapper.set_date(date) - wrapper.set_mbox_uuid(mbox_id) def insert_mdoc_id(_, wrapper): doc_id = wrapper.mdoc.doc_id @@ -420,6 +446,8 @@ class MessageCollection(object): d = wrapper.create(self.store) d.addCallback(insert_mdoc_id, wrapper) + d.addErrback(lambda f: f.printTraceback()) + return d def copy_msg(self, msg, newmailbox): @@ -453,8 +481,47 @@ class MessageCollection(object): d.addCallback(delete_mdoc_id, wrapper) return d + def delete_all_flagged(self): + """ + Delete all messages flagged as \\Deleted. + Used from IMAPMailbox.expunge() + """ + def get_uid_list(hashes): + d = [] + for h in hashes: + d.append(self.mbox_indexer.get_uid_from_doc_id( + self.mbox_uuid, h)) + return defer.gatherResults(d), hashes + + def delete_uid_entries((uids, hashes)): + d = [] + for h in hashes: + d.append(self.mbox_indexer.delete_doc_by_hash( + self.mbox_uuid, h)) + return defer.gatherResults(d).addCallback( + lambda _: uids) + + mdocs_deleted = self.adaptor.del_all_flagged_messages( + self.store, self.mbox_uuid) + mdocs_deleted.addCallback(get_uid_list) + mdocs_deleted.addCallback(delete_uid_entries) + return mdocs_deleted + # TODO should add a delete-by-uid to collection? + def delete_all_docs(self): + def del_all_uid(uid_list): + deferreds = [] + for uid in uid_list: + d = self.get_message_by_uid(uid) + d.addCallback(lambda msg: msg.delete()) + deferreds.append(d) + return defer.gatherResults(deferreds) + + d = self.all_uid_iter() + d.addCallback(del_all_uid) + return d + def _update_flags_or_tags(self, old, new, mode): if mode == Flagsmode.APPEND: final = list((set(tuple(old) + new))) @@ -509,45 +576,49 @@ class Account(object): adaptor_class = SoledadMailAdaptor store = None - def __init__(self, store): + def __init__(self, store, ready_cb=None): self.store = store self.adaptor = self.adaptor_class() self.mbox_indexer = MailboxIndexer(self.store) + self.deferred_initialization = defer.Deferred() self._initialized = False - self._deferred_initialization = defer.Deferred() + self._ready_cb = ready_cb - self._initialize_storage() + self._init_d = self._initialize_storage() def _initialize_storage(self): def add_mailbox_if_none(mboxes): if not mboxes: - self.add_mailbox(INBOX_NAME) + return self.add_mailbox(INBOX_NAME) def finish_initialization(result): self._initialized = True - self._deferred_initialization.callback(None) + self.deferred_initialization.callback(None) + if self._ready_cb is not None: + self._ready_cb() d = self.adaptor.initialize_store(self.store) d.addCallback(lambda _: self.list_all_mailbox_names()) d.addCallback(add_mailbox_if_none) d.addCallback(finish_initialization) + return d def callWhenReady(self, cb, *args, **kw): - # use adaptor.store_ready instead? if self._initialized: cb(self, *args, **kw) return defer.succeed(None) else: - self._deferred_initialization.addCallback(cb, *args, **kw) - return self._deferred_initialization + self.deferred_initialization.addCallback(cb, *args, **kw) + return self.deferred_initialization # # Public API Starts # def list_all_mailbox_names(self): + def filter_names(mboxes): return [m.mbox for m in mboxes] @@ -563,8 +634,11 @@ class Account(object): def create_uuid(wrapper): if not wrapper.uuid: - wrapper.uuid = uuid.uuid4() - return wrapper.update(self.store) + wrapper.uuid = str(uuid.uuid4()) + d = wrapper.update(self.store) + d.addCallback(lambda _: wrapper) + return d + return wrapper def create_uid_table_cb(wrapper): d = self.mbox_indexer.create_table(wrapper.uuid) @@ -599,7 +673,7 @@ class Account(object): def _rename_mbox(wrapper): wrapper.mbox = newname - return wrapper.update(self.store) + return wrapper, wrapper.update(self.store) d = self.adaptor.get_or_create_mbox(self.store, oldname) d.addCallback(_rename_mbox) @@ -616,8 +690,6 @@ class Account(object): return MessageCollection( self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) - mboxwrapper_klass = self.adaptor.mboxwrapper_klass - #d = mboxwrapper_klass.get_or_create(name) d = self.adaptor.get_or_create_mbox(self.store, name) d.addCallback(get_collection_for_mailbox) return d diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 6155a7a..22e57d4 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -114,6 +114,24 @@ class MailboxIndexer(object): preffix=self.table_preffix, name=sanitize(mailbox_id))) return self._query(sql) + def rename_table(self, oldmailbox, newmailbox): + """ + Delete the UID table for a given mailbox. + :param oldmailbox: the old mailbox name + :type oldmailbox: str + :param newmailbox: the new mailbox name + :type newmailbox: str + :rtype: Deferred + """ + assert oldmailbox + assert newmailbox + assert oldmailbox != newmailbox + sql = ("ALTER TABLE {preffix}{old} " + "RENAME TO {preffix}{new}".format( + preffix=self.table_preffix, + old=sanitize(oldmailbox), new=sanitize(newmailbox))) + return self._query(sql) + def insert_doc(self, mailbox_id, doc_id): """ Insert the doc_id for a MetaMsg in the UID table for a given mailbox. @@ -134,7 +152,7 @@ class MailboxIndexer(object): assert doc_id mailbox_id = mailbox_id.replace('-', '_') - if not re.findall(METAMSGID_RE.format(mbox=mailbox_id), doc_id): + if not re.findall(METAMSGID_RE.format(mbox_uuid=mailbox_id), doc_id): raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") def get_rowid(result): @@ -148,9 +166,11 @@ class MailboxIndexer(object): sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( preffix=self.table_preffix, name=sanitize(mailbox_id)) + d = self._query(sql, values) d.addCallback(lambda _: self._query(sql_last)) d.addCallback(get_rowid) + d.addErrback(lambda f: f.printTraceback()) return d def delete_doc_by_uid(self, mailbox_id, uid): @@ -199,13 +219,14 @@ class MailboxIndexer(object): """ Get the doc_id for a MetaMsg in the UID table for a given mailbox. - :param mailbox: the mailbox uuid + :param mailbox_id: the mailbox uuid :type mailbox: str :param uid: the uid for the MetaMsg for this mailbox :type uid: int :rtype: Deferred """ check_good_uuid(mailbox_id) + mailbox_id = mailbox_id.replace('-', '_') def get_hash(result): return _maybe_first_query_item(result) @@ -218,7 +239,24 @@ class MailboxIndexer(object): d.addCallback(get_hash) return d - def get_doc_ids_from_uids(self, mailbox, uids): + def get_uid_from_doc_id(self, mailbox_id, doc_id): + check_good_uuid(mailbox_id) + mailbox_id = mailbox_id.replace('-', '_') + + def get_uid(result): + return _maybe_first_query_item(result) + + sql = ("SELECT uid from {preffix}{name} " + "WHERE hash=?".format( + preffix=self.table_preffix, name=sanitize(mailbox_id))) + values = (doc_id,) + d = self._query(sql, values) + d.addCallback(get_uid) + return d + + + + def get_doc_ids_from_uids(self, mailbox_id, uids): # For IMAP relative numbering /sequences. # XXX dereference the range (n,*) raise NotImplementedError() @@ -227,8 +265,8 @@ class MailboxIndexer(object): """ Get the number of entries in the UID table for a given mailbox. - :param mailbox: the mailbox name - :type mailbox: str + :param mailbox_id: the mailbox uuid + :type mailbox_id: str :return: a deferred that will fire with an integer returning the count. :rtype: Deferred """ @@ -241,6 +279,7 @@ class MailboxIndexer(object): preffix=self.table_preffix, name=sanitize(mailbox_id))) d = self._query(sql) d.addCallback(get_count) + d.addErrback(lambda _: 0) return d def get_next_uid(self, mailbox_id): @@ -252,7 +291,7 @@ class MailboxIndexer(object): only thing that can be assured is that it will be equal or greater than the value returned. - :param mailbox: the mailbox name + :param mailbox_id: the mailbox uuid :type mailbox: str :return: a deferred that will fire with an integer returning the next uid. @@ -273,3 +312,22 @@ class MailboxIndexer(object): d = self._query(sql) d.addCallback(increment) return d + + def all_uid_iter(self, mailbox_id): + """ + Get a sequence of all the uids in this mailbox. + + :param mailbox_id: the mailbox uuid + :type mailbox_id: str + """ + check_good_uuid(mailbox_id) + + sql = ("SELECT uid from {preffix}{name} ").format( + preffix=self.table_preffix, name=sanitize(mailbox_id)) + + def get_results(result): + return [x[0] for x in result] + + d = self._query(sql) + d.addCallback(get_results) + return d diff --git a/src/leap/mail/tests/common.py b/src/leap/mail/tests/common.py index fefa7ee..a411b2d 100644 --- a/src/leap/mail/tests/common.py +++ b/src/leap/mail/tests/common.py @@ -21,6 +21,9 @@ import os import shutil import tempfile +from twisted.internet import defer +from twisted.trial import unittest + from leap.common.testing.basetest import BaseLeapTest from leap.soledad.client import Soledad @@ -60,7 +63,7 @@ def _initialize_soledad(email, gnupg_home, tempdir): return soledad -class SoledadTestMixin(BaseLeapTest): +class SoledadTestMixin(unittest.TestCase, BaseLeapTest): """ It is **VERY** important that this base is added *AFTER* unittest.TestCase """ @@ -68,15 +71,7 @@ class SoledadTestMixin(BaseLeapTest): def setUp(self): self.results = [] - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir + self.setUpEnv() # Soledad: config info self.gnupg_home = "%s/gnupg" % self.tempdir @@ -88,6 +83,8 @@ class SoledadTestMixin(BaseLeapTest): self.gnupg_home, self.tempdir) + return defer.succeed(True) + def tearDown(self): """ tearDown method called after each test. diff --git a/src/leap/mail/tests/test_mailbox_indexer.py b/src/leap/mail/tests/test_mailbox_indexer.py index 2edf1d8..b82fd2d 100644 --- a/src/leap/mail/tests/test_mailbox_indexer.py +++ b/src/leap/mail/tests/test_mailbox_indexer.py @@ -84,18 +84,6 @@ class MailboxIndexerTestCase(SoledadTestMixin): d.addCallback(assert_table_deleted) return d - #def test_rename_table(self): - #def assert_table_renamed(tables): - #self.assertEqual( - #tables, ["leapmail_uid_foomailbox"]) -# - #m_uid = self.get_mbox_uid() - #d = m_uid.create_table('inbox') - #d.addCallback(lambda _: m_uid.rename_table('inbox', 'foomailbox')) - #d.addCallback(self.list_mail_tables_cb) - #d.addCallback(assert_table_renamed) - #return d - def test_insert_doc(self): m_uid = self.get_mbox_uid() @@ -168,7 +156,6 @@ class MailboxIndexerTestCase(SoledadTestMixin): def test_get_doc_id_from_uid(self): m_uid = self.get_mbox_uid() - #mbox = 'foomailbox' h1 = fmt_hash(mbox_id, hash_test0) @@ -183,7 +170,6 @@ class MailboxIndexerTestCase(SoledadTestMixin): def test_count(self): m_uid = self.get_mbox_uid() - #mbox = 'foomailbox' h1 = fmt_hash(mbox_id, hash_test0) h2 = fmt_hash(mbox_id, hash_test1) @@ -216,7 +202,6 @@ class MailboxIndexerTestCase(SoledadTestMixin): def test_get_next_uid(self): m_uid = self.get_mbox_uid() - #mbox = 'foomailbox' h1 = fmt_hash(mbox_id, hash_test0) h2 = fmt_hash(mbox_id, hash_test1) @@ -237,3 +222,29 @@ class MailboxIndexerTestCase(SoledadTestMixin): d.addCallback(lambda _: m_uid.get_next_uid(mbox_id)) d.addCallback(partial(assert_next_uid, expected=6)) return d + + def test_all_uid_iter(self): + + m_uid = self.get_mbox_uid() + + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) + d.addCallback(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1)) + d.addCallback(lambda _: m_uid.delete_doc_by_uid(mbox_id, 4)) + + def assert_all_uid(result, expected=[2, 3, 5]): + self.assertEquals(result, expected) + + d.addCallback(lambda _: m_uid.all_uid_iter(mbox_id)) + d.addCallback(partial(assert_all_uid)) + return d -- cgit v1.2.3 From cf702f32225e50a8f283d2fdda64c93116c12724 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 14 Jan 2015 01:09:19 -0400 Subject: patch cbSelect to accept deferreds for count* --- src/leap/mail/imap/account.py | 8 +++++--- src/leap/mail/imap/server.py | 14 ++++++++++++-- src/leap/mail/imap/tests/test_imap.py | 20 +++++++++++--------- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index dfc0d62..8a6e87e 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -330,8 +330,7 @@ class IMAPAccount(object): oldname = normalize_mailbox(oldname) newname = normalize_mailbox(newname) - def rename_inferiors(inferiors_result): - inferiors, mailboxes = inferiors_result + def rename_inferiors((inferiors, mailboxes)): rename_deferreds = [] inferiors = [ (o, o.replace(oldname, newname, 1)) for o in inferiors] @@ -347,7 +346,10 @@ class IMAPAccount(object): d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) return d1 - d = self._inferiorNames(oldname) + d1 = self._inferiorNames(oldname) + d2 = self.account.list_all_mailbox_names() + + d = defer.gatherResults([d1, d2]) d.addCallback(rename_inferiors) return d diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index b4f320a..32c921d 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -162,10 +162,20 @@ class LEAPIMAPServer(imap4.IMAP4Server): self.sendNegativeResponse(tag, 'Mailbox cannot be selected') return + d1 = defer.maybeDeferred(mbox.getMessageCount) + d2 = defer.maybeDeferred(mbox.getRecentCount) + return defer.gatherResults([d1, d2]).addCallback( + self.__cbSelectWork, mbox, cmdName, tag) + + def __cbSelectWork(self, ((msg_count, recent_count)), mbox, cmdName, tag): flags = mbox.getFlags() self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) - self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS') - self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT') + + # Patched ------------------------------------------------------- + # accept deferreds for the count + self.sendUntaggedResponse(str(msg_count) + ' EXISTS') + self.sendUntaggedResponse(str(recent_count) + ' RECENT') + # ---------------------------------------------------------------- # Patched ------------------------------------------------------- # imaptest was complaining about the incomplete line, we're adding diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index d7fcdce..6be41cd 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -447,9 +447,12 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): """ Try to rename hierarchical mailboxes """ - acc = LEAPIMAPServer.theAccount - dc1 = lambda: acc.create('oldmbox/m1') - dc2 = lambda: acc.create('oldmbox/m2') + acc = self.server.theAccount + + def add_mailboxes(): + return defer.gatherResults([ + acc.addMailbox('oldmbox/m1'), + acc.addMailbox('oldmbox/m2')]) def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -457,19 +460,18 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): def rename(): return self.client.rename('oldmbox', 'newname') - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(dc1)) - d1.addCallback(strip(dc2)) + d1 = self.connected.addCallback(strip(add_mailboxes)) + d1.addCallback(strip(login)) d1.addCallbacks(strip(rename), self._ebGeneral) d1.addCallbacks(self._cbStopClient, self._ebGeneral) d2 = self.loopback() d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: acc.account.list_all_mailbox_names()) return d.addCallback(self._cbTestHierarchicalRename) - def _cbTestHierarchicalRename(self, ignored): - mboxes = LEAPIMAPServer.theAccount.mailboxes + def _cbTestHierarchicalRename(self, mailboxes): expected = ['INBOX', 'newname', 'newname/m1', 'newname/m2'] - self.assertEqual(sorted(mboxes), sorted([s for s in expected])) + self.assertEqual(sorted(mailboxes), sorted([s for s in expected])) def testSubscribe(self): """ -- cgit v1.2.3 From 338a5d6c1e0d6bae05d2756eb6e4c20b38d444d9 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Wed, 14 Jan 2015 13:50:02 -0600 Subject: Refactor fetch into leap.mail.incoming IService --- src/leap/mail/imap/account.py | 2 +- src/leap/mail/imap/fetch.py | 749 -------------------- src/leap/mail/imap/service/imap.py | 30 +- src/leap/mail/imap/tests/test_incoming_mail.py | 193 ------ src/leap/mail/incoming/__init__.py | 0 src/leap/mail/incoming/service.py | 760 +++++++++++++++++++++ src/leap/mail/incoming/tests/__init__.py | 0 src/leap/mail/incoming/tests/test_incoming_mail.py | 202 ++++++ src/leap/mail/mail.py | 17 +- 9 files changed, 971 insertions(+), 982 deletions(-) delete mode 100644 src/leap/mail/imap/fetch.py delete mode 100644 src/leap/mail/imap/tests/test_incoming_mail.py create mode 100644 src/leap/mail/incoming/__init__.py create mode 100644 src/leap/mail/incoming/service.py create mode 100644 src/leap/mail/incoming/tests/__init__.py create mode 100644 src/leap/mail/incoming/tests/test_incoming_mail.py diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 8a6e87e..146d066 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -63,7 +63,7 @@ class IMAPAccount(object): selected = None closed = False - def __init__(self, user_id, store, d=None): + def __init__(self, user_id, store, d=defer.Deferred()): """ Keeps track of the mailboxes and subscriptions handled by this account. diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py deleted file mode 100644 index dbc726a..0000000 --- a/src/leap/mail/imap/fetch.py +++ /dev/null @@ -1,749 +0,0 @@ -# -*- coding: utf-8 -*- -# fetch.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Incoming mail fetcher. -""" -import copy -import logging -import shlex -import threading -import time -import traceback -import warnings - -from email.parser import Parser -from email.generator import Generator -from email.utils import parseaddr -from StringIO import StringIO -from urlparse import urlparse - -from twisted.python import log -from twisted.internet import defer, reactor -from twisted.internet.task import LoopingCall -from twisted.internet.task import deferLater -from u1db import errors as u1db_errors - -from leap.common import events as leap_events -from leap.common.check import leap_assert, leap_assert_type -from leap.common.events.events_pb2 import IMAP_FETCHED_INCOMING -from leap.common.events.events_pb2 import IMAP_MSG_PROCESSING -from leap.common.events.events_pb2 import IMAP_MSG_DECRYPTED -from leap.common.events.events_pb2 import IMAP_MSG_SAVED_LOCALLY -from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING -from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL -from leap.common.events.events_pb2 import SOLEDAD_INVALID_AUTH_TOKEN -from leap.common.mail import get_email_charset -from leap.keymanager import errors as keymanager_errors -from leap.keymanager.openpgp import OpenPGPKey -from leap.mail.decorators import deferred_to_thread -from leap.mail.imap.fields import fields -from leap.mail.utils import json_loads, empty, first -from leap.soledad.client import Soledad -from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY -from leap.soledad.common.errors import InvalidAuthTokenError - - -logger = logging.getLogger(__name__) - -MULTIPART_ENCRYPTED = "multipart/encrypted" -MULTIPART_SIGNED = "multipart/signed" -PGP_BEGIN = "-----BEGIN PGP MESSAGE-----" -PGP_END = "-----END PGP MESSAGE-----" - - -class MalformedMessage(Exception): - """ - Raised when a given message is not well formed. - """ - pass - - -class LeapIncomingMail(object): - """ - Fetches and process mail from the incoming pool. - - This object has public methods start_loop and stop that will - actually initiate a LoopingCall with check_period recurrency. - The LoopingCall itself will invoke the fetch method each time - that the check_period expires. - - This loop will sync the soledad db with the remote server and - process all the documents found tagged as incoming mail. - """ - - RECENT_FLAG = "\\Recent" - CONTENT_KEY = "content" - - LEAP_SIGNATURE_HEADER = 'X-Leap-Signature' - """ - Header added to messages when they are decrypted by the IMAP fetcher, - which states the validity of an eventual signature that might be included - in the encrypted blob. - """ - LEAP_SIGNATURE_VALID = 'valid' - LEAP_SIGNATURE_INVALID = 'invalid' - LEAP_SIGNATURE_COULD_NOT_VERIFY = 'could not verify' - - fetching_lock = threading.Lock() - - def __init__(self, keymanager, soledad, imap_account, - check_period, userid): - - """ - Initialize LeapIncomingMail.. - - :param keymanager: a keymanager instance - :type keymanager: keymanager.KeyManager - - :param soledad: a soledad instance - :type soledad: Soledad - - :param imap_account: the account to fetch periodically - :type imap_account: SoledadBackedAccount - - :param check_period: the period to fetch new mail, in seconds. - :type check_period: int - """ - - leap_assert(keymanager, "need a keymanager to initialize") - leap_assert_type(soledad, Soledad) - leap_assert(check_period, "need a period to check incoming mail") - leap_assert_type(check_period, int) - leap_assert(userid, "need a userid to initialize") - - self._keymanager = keymanager - self._soledad = soledad - self.imapAccount = imap_account - self._inbox = self.imapAccount.getMailbox('inbox') - self._userid = userid - - self._loop = None - self._check_period = check_period - - # initialize a mail parser only once - self._parser = Parser() - - # - # Public API: fetch, start_loop, stop. - # - - def fetch(self): - """ - Fetch incoming mail, to be called periodically. - - Calls a deferred that will execute the fetch callback - in a separate thread - """ - def syncSoledadCallback(result): - # FIXME this needs a matching change in mx!!! - # --> need to add ERROR_DECRYPTING_KEY = False - # as default. - try: - doclist = self._soledad.get_from_index( - fields.JUST_MAIL_IDX, "*", "0") - except u1db_errors.InvalidGlobbing: - # It looks like we are a dealing with an outdated - # mx. Fallback to the version of the index - warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", - DeprecationWarning) - doclist = self._soledad.get_from_index( - fields.JUST_MAIL_COMPAT_IDX, "*") - return self._process_doclist(doclist) - - logger.debug("fetching mail for: %s %s" % ( - self._soledad.uuid, self._userid)) - if not self.fetching_lock.locked(): - d1 = self._sync_soledad() - d = defer.gatherResults([d1], consumeErrors=True) - d.addCallbacks(syncSoledadCallback, self._errback) - d.addCallbacks(self._signal_fetch_to_ui, self._errback) - return d - else: - logger.debug("Already fetching mail.") - - def start_loop(self): - """ - Starts a loop to fetch mail. - """ - if self._loop is None: - self._loop = LoopingCall(self.fetch) - self._loop.start(self._check_period) - else: - logger.warning("Tried to start an already running fetching loop.") - - def stop(self): - # XXX change the name to stop_loop, for consistency. - """ - Stops the loop that fetches mail. - """ - if self._loop and self._loop.running is True: - self._loop.stop() - self._loop = None - - # - # Private methods. - # - - # synchronize incoming mail - - def _errback(self, failure): - logger.exception(failure.value) - traceback.print_exc() - - @deferred_to_thread - def _sync_soledad(self): - """ - Synchronize with remote soledad. - - :returns: a list of LeapDocuments, or None. - :rtype: iterable or None - """ - with self.fetching_lock: - try: - log.msg('FETCH: syncing soledad...') - self._soledad.sync() - log.msg('FETCH soledad SYNCED.') - except InvalidAuthTokenError: - # if the token is invalid, send an event so the GUI can - # disable mail and show an error message. - leap_events.signal(SOLEDAD_INVALID_AUTH_TOKEN) - - def _signal_fetch_to_ui(self, doclist): - """ - Send leap events to ui. - - :param doclist: iterable with msg documents. - :type doclist: iterable. - :returns: doclist - :rtype: iterable - """ - doclist = first(doclist) # gatherResults pass us a list - if doclist: - fetched_ts = time.mktime(time.gmtime()) - num_mails = len(doclist) if doclist is not None else 0 - if num_mails != 0: - log.msg("there are %s mails" % (num_mails,)) - leap_events.signal( - IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts)) - return doclist - - def _signal_unread_to_ui(self, *args): - """ - Sends unread event to ui. - """ - leap_events.signal( - IMAP_UNREAD_MAIL, str(self._inbox.getUnseenCount())) - - # process incoming mail. - - def _process_doclist(self, doclist): - """ - Iterates through the doclist, checks if each doc - looks like a message, and yields a deferred that will decrypt and - process the message. - - :param doclist: iterable with msg documents. - :type doclist: iterable. - :returns: a list of deferreds for individual messages. - """ - log.msg('processing doclist') - if not doclist: - logger.debug("no docs found") - return - num_mails = len(doclist) - - deferreds = [] - for index, doc in enumerate(doclist): - logger.debug("processing doc %d of %d" % (index + 1, num_mails)) - leap_events.signal( - IMAP_MSG_PROCESSING, str(index), str(num_mails)) - - keys = doc.content.keys() - - # TODO Compatibility check with the index in pre-0.6 mx - # that does not write the ERROR_DECRYPTING_KEY - # This should be removed in 0.7 - - has_errors = doc.content.get(fields.ERROR_DECRYPTING_KEY, None) - if has_errors is None: - warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", - DeprecationWarning) - - if has_errors: - logger.debug("skipping msg with decrypting errors...") - elif self._is_msg(keys): - d = self._decrypt_doc(doc) - d.addCallback(self._extract_keys) - d.addCallbacks(self._add_message_locally, self._errback) - deferreds.append(d) - return defer.gatherResults(deferreds, consumeErrors=True) - - # - # operations on individual messages - # - - @deferred_to_thread - def _decrypt_doc(self, doc): - """ - Decrypt the contents of a document. - - :param doc: A document containing an encrypted message. - :type doc: SoledadDocument - - :return: A Deferred that will be fired with the document and the - decrypted message. - :rtype: SoledadDocument, str - """ - log.msg('decrypting msg') - - def process_decrypted(res): - if isinstance(res, tuple): - decrdata, _ = res - success = True - else: - decrdata = "" - success = False - - leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0") - - data = self._process_decrypted_doc(doc, decrdata) - return doc, data - - d = self._keymanager.decrypt( - doc.content[ENC_JSON_KEY], - self._userid, OpenPGPKey) - d.addErrback(self._errback) - d.addCallback(process_decrypted) - return d - - def _process_decrypted_doc(self, doc, data): - """ - Process a document containing a succesfully decrypted message. - - :param doc: the incoming message - :type doc: SoledadDocument - :param data: the json-encoded, decrypted content of the incoming - message - :type data: str - - :return: the processed data. - :rtype: str - """ - log.msg('processing decrypted doc') - - # XXX turn this into an errBack for each one of - # the deferreds that would process an individual document - try: - msg = json_loads(data) - except UnicodeError as exc: - logger.error("Error while decrypting %s" % (doc.doc_id,)) - logger.exception(exc) - - # we flag the message as "with decrypting errors", - # to avoid further decryption attempts during sync - # cycles until we're prepared to deal with that. - # What is the same, when Ivan deals with it... - # A new decrypting attempt event could be triggered by a - # future a library upgrade, or a cli flag to the client, - # we just `defer` that for now... :) - doc.content[fields.ERROR_DECRYPTING_KEY] = True - deferLater(reactor, 0, self._update_incoming_message, doc) - - # FIXME this is just a dirty hack to delay the proper - # deferred organization here... - # and remember, boys, do not do this at home. - return [] - - if not isinstance(msg, dict): - defer.returnValue(False) - if not msg.get(fields.INCOMING_KEY, False): - defer.returnValue(False) - - # ok, this is an incoming message - rawmsg = msg.get(self.CONTENT_KEY, None) - if not rawmsg: - return "" - return self._maybe_decrypt_msg(rawmsg) - - @deferred_to_thread - def _update_incoming_message(self, doc): - """ - Do a put for a soledad document. This probably has been called only - in the case that we've needed to update the ERROR_DECRYPTING_KEY - flag in an incoming message, to get it out of the decrypting queue. - - :param doc: the SoledadDocument to update - :type doc: SoledadDocument - """ - log.msg("Updating SoledadDoc %s" % (doc.doc_id)) - self._soledad.put_doc(doc) - - @deferred_to_thread - def _delete_incoming_message(self, doc): - """ - Delete document. - - :param doc: the SoledadDocument to delete - :type doc: SoledadDocument - """ - log.msg("Deleting Incoming message: %s" % (doc.doc_id,)) - self._soledad.delete_doc(doc) - - def _maybe_decrypt_msg(self, data): - """ - Tries to decrypt a gpg message if data looks like one. - - :param data: the text to be decrypted. - :type data: str - :return: data, possibly decrypted. - :rtype: str - """ - leap_assert_type(data, str) - log.msg('maybe decrypting doc') - - # parse the original message - encoding = get_email_charset(data) - msg = self._parser.parsestr(data) - - fromHeader = msg.get('from', None) - senderAddress = None - if (fromHeader is not None - and (msg.get_content_type() == MULTIPART_ENCRYPTED - or msg.get_content_type() == MULTIPART_SIGNED)): - senderAddress = parseaddr(fromHeader) - - def add_leap_header(decrmsg, signkey): - if (senderAddress is None or - isinstance(signkey, keymanager_errors.KeyNotFound)): - decrmsg.add_header( - self.LEAP_SIGNATURE_HEADER, - self.LEAP_SIGNATURE_COULD_NOT_VERIFY) - elif isinstance(signkey, keymanager_errors.InvalidSignature): - decrmsg.add_header( - self.LEAP_SIGNATURE_HEADER, - self.LEAP_SIGNATURE_INVALID) - else: - decrmsg.add_header( - self.LEAP_SIGNATURE_HEADER, - self.LEAP_SIGNATURE_VALID, - pubkey=signkey.key_id) - return decrmsg.as_string() - - if msg.get_content_type() == MULTIPART_ENCRYPTED: - d = self._decrypt_multipart_encrypted_msg( - msg, encoding, senderAddress) - else: - d = self._maybe_decrypt_inline_encrypted_msg( - msg, encoding, senderAddress) - d.addCallback(add_leap_header) - return d - - def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderAddress): - """ - Decrypt a message with content-type 'multipart/encrypted'. - - :param msg: The original encrypted message. - :type msg: Message - :param encoding: The encoding of the email message. - :type encoding: str - :param senderAddress: The email address of the sender of the message. - :type senderAddress: str - - :return: A Deferred that will be fired with a tuple containing a - decrypted Message and the signing OpenPGPKey if the signature - is valid or InvalidSignature or KeyNotFound. - :rtype: Deferred - """ - log.msg('decrypting multipart encrypted msg') - msg = copy.deepcopy(msg) - self._msg_multipart_sanity_check(msg) - - # parse message and get encrypted content - pgpencmsg = msg.get_payload()[1] - encdata = pgpencmsg.get_payload() - - # decrypt or fail gracefully - def build_msg(res): - decrdata, signkey = res - - decrmsg = self._parser.parsestr(decrdata) - # remove original message's multipart/encrypted content-type - del(msg['content-type']) - - # replace headers back in original message - for hkey, hval in decrmsg.items(): - try: - # this will raise KeyError if header is not present - msg.replace_header(hkey, hval) - except KeyError: - msg[hkey] = hval - - # all ok, replace payload by unencrypted payload - msg.set_payload(decrmsg.get_payload()) - return (msg, signkey) - - d = self._keymanager.decrypt( - encdata, self._userid, OpenPGPKey, - verify=senderAddress) - d.addCallbacks(build_msg, self._decryption_error, errbackArgs=(msg,)) - return d - - def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding, - senderAddress): - """ - Possibly decrypt an inline OpenPGP encrypted message. - - :param origmsg: The original, possibly encrypted message. - :type origmsg: Message - :param encoding: The encoding of the email message. - :type encoding: str - :param senderAddress: The email address of the sender of the message. - :type senderAddress: str - - :return: A Deferred that will be fired with a tuple containing a - decrypted Message and the signing OpenPGPKey if the signature - is valid or InvalidSignature or KeyNotFound. - :rtype: Deferred - """ - log.msg('maybe decrypting inline encrypted msg') - # serialize the original message - buf = StringIO() - g = Generator(buf) - g.flatten(origmsg) - data = buf.getvalue() - - def decrypted_data(res): - decrdata, signkey = res - return data.replace(pgp_message, decrdata), signkey - - def encode_and_return(res): - data, signkey = res - if isinstance(data, unicode): - data = data.encode(encoding, 'replace') - return (self._parser.parsestr(data), signkey) - - # handle exactly one inline PGP message - if PGP_BEGIN in data: - begin = data.find(PGP_BEGIN) - end = data.find(PGP_END) - pgp_message = data[begin:end + len(PGP_END)] - d = self._keymanager.decrypt( - pgp_message, self._userid, OpenPGPKey, - verify=senderAddress) - d.addCallbacks(decrypted_data, self._decryption_error, - errbackArgs=(data,)) - else: - d = defer.succeed((data, None)) - d.addCallback(encode_and_return) - return d - - def _decryption_error(self, failure, msg): - """ - Check for known decryption errors - """ - if failure.check(keymanager_errors.DecryptError): - logger.warning('Failed to decrypt encrypted message (%s). ' - 'Storing message without modifications.' - % str(failure.value)) - return (msg, None) - elif failure.check(keymanager_errors.KeyNotFound): - logger.error('Failed to find private key for decryption (%s). ' - 'Storing message without modifications.' - % str(failure.value)) - return (msg, None) - else: - return failure - - def _extract_keys(self, msgtuple): - """ - Retrieve attached keys to the mesage and parse message headers for an - *OpenPGP* header as described on the `IETF draft - ` - only urls with https and the same hostname than the email are supported - for security reasons. - - :param msgtuple: a tuple consisting of a SoledadDocument - instance containing the incoming message - and data, the json-encoded, decrypted content of the - incoming message - :type msgtuple: (SoledadDocument, str) - - :return: A Deferred that will be fired with msgtuple when key - extraction finishes - :rtype: Deferred - """ - OpenPGP_HEADER = 'OpenPGP' - doc, data = msgtuple - - # XXX the parsing of the message is done in mailbox.addMessage, maybe - # we should do it in this module so we don't need to parse it again - # here - msg = self._parser.parsestr(data) - _, fromAddress = parseaddr(msg['from']) - - header = msg.get(OpenPGP_HEADER, None) - dh = defer.success() - if header is not None: - dh = self._extract_openpgp_header(header, fromAddress) - - da = defer.success() - if msg.is_multipart(): - da = self._extract_attached_key(msg.get_payload(), fromAddress) - - d = defer.gatherResults([dh, da]) - d.addCallback(lambda _: msgtuple) - return d - - def _extract_openpgp_header(self, header, address): - """ - Import keys from the OpenPGP header - - :param header: OpenPGP header string - :type header: str - :param address: email address in the from header - :type address: str - - :return: A Deferred that will be fired when header extraction is done - :rtype: Deferred - """ - d = defer.success() - fields = dict([f.strip(' ').split('=') for f in header.split(';')]) - if 'url' in fields: - url = shlex.split(fields['url'])[0] # remove quotations - urlparts = urlparse(url) - addressHostname = address.split('@')[1] - if (urlparts.scheme == 'https' - and urlparts.hostname == addressHostname): - def fetch_error(failure): - if failure.check(keymanager_errors.KeyNotFound): - logger.warning("Url from OpenPGP header %s failed" - % (url,)) - elif failure.check(keymanager_errors.KeyAttributesDiffer): - logger.warning("Key from OpenPGP header url %s didn't " - "match the from address %s" - % (url, address)) - else: - return failure - - d = self._keymanager.fetch_key(address, url, OpenPGPKey) - d.addCallback( - lambda _: - logger.info("Imported key from header %s" % (url,))) - d.addErrback(fetch_error) - else: - logger.debug("No valid url on OpenPGP header %s" % (url,)) - else: - logger.debug("There is no url on the OpenPGP header: %s" - % (header,)) - return d - - def _extract_attached_key(self, attachments, address): - """ - Import keys from the attachments - - :param attachments: email attachment list - :type attachments: list(email.Message) - :param address: email address in the from header - :type address: str - - :return: A Deferred that will be fired when all the keys are stored - :rtype: Deferred - """ - MIME_KEY = "application/pgp-keys" - - deferreds = [] - for attachment in attachments: - if MIME_KEY == attachment.get_content_type(): - logger.debug("Add key from attachment") - d = self._keymanager.put_raw_key( - attachment.get_payload(), - OpenPGPKey, - address=address) - deferreds.append(d) - return defer.gatherResults(deferreds) - - def _add_message_locally(self, msgtuple): - """ - Adds a message to local inbox and delete it from the incoming db - in soledad. - - :param msgtuple: a tuple consisting of a SoledadDocument - instance containing the incoming message - and data, the json-encoded, decrypted content of the - incoming message - :type msgtuple: (SoledadDocument, str) - - :return: A Deferred that will be fired when the messages is stored - :rtype: Defferred - """ - doc, data = msgtuple - log.msg('adding message %s to local db' % (doc.doc_id,)) - - if isinstance(data, list): - if empty(data): - return False - data = data[0] - - def msgSavedCallback(result): - if not empty(result): - leap_events.signal(IMAP_MSG_SAVED_LOCALLY) - deferLater(reactor, 0, self._delete_incoming_message, doc) - leap_events.signal(IMAP_MSG_DELETED_INCOMING) - - d = self._inbox.addMessage(data, flags=(self.RECENT_FLAG,), - notify_on_disk=True) - d.addCallbacks(msgSavedCallback, self._errback) - return d - - # - # helpers - # - - def _msg_multipart_sanity_check(self, msg): - """ - Performs a sanity check against a multipart encrypted msg - - :param msg: The original encrypted message. - :type msg: Message - """ - # sanity check - payload = msg.get_payload() - if len(payload) != 2: - raise MalformedMessage( - 'Multipart/encrypted messages should have exactly 2 body ' - 'parts (instead of %d).' % len(payload)) - if payload[0].get_content_type() != 'application/pgp-encrypted': - raise MalformedMessage( - "Multipart/encrypted messages' first body part should " - "have content type equal to 'application/pgp-encrypted' " - "(instead of %s)." % payload[0].get_content_type()) - if payload[1].get_content_type() != 'application/octet-stream': - raise MalformedMessage( - "Multipart/encrypted messages' second body part should " - "have content type equal to 'octet-stream' (instead of " - "%s)." % payload[1].get_content_type()) - - def _is_msg(self, keys): - """ - Checks if the keys of a dictionary match the signature - of the document type we use for messages. - - :param keys: iterable containing the strings to match. - :type keys: iterable of strings. - :rtype: bool - """ - return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 5d88a79..93e4d62 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -30,10 +30,9 @@ logger = logging.getLogger(__name__) from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type, leap_check -from leap.keymanager import KeyManager from leap.mail.imap.account import IMAPAccount -from leap.mail.imap.fetch import LeapIncomingMail from leap.mail.imap.server import LEAPIMAPServer +from leap.mail.incoming import IncomingMail from leap.soledad.client import Soledad from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED @@ -55,10 +54,6 @@ if DO_PROFILE: # The default port in which imap service will run IMAP_PORT = 1984 -# The period between succesive checks of the incoming mail -# queue (in seconds) -INCOMING_CHECK_PERIOD = 60 - class IMAPAuthRealm(object): """ @@ -132,21 +127,16 @@ def run_service(*args, **kwargs): """ Main entry point to run the service from the client. - :returns: the LoopingCall instance that will have to be stoppped - before shutting down the client, the port as returned by - the reactor when starts listening, and the factory for - the protocol. + :returns: the port as returned by the reactor when starts listening, and + the factory for the protocol. """ leap_assert(len(args) == 2) - soledad, keymanager = args + soledad = args leap_assert_type(soledad, Soledad) - leap_assert_type(keymanager, KeyManager) port = kwargs.get('port', IMAP_PORT) - check_period = kwargs.get('check_period', INCOMING_CHECK_PERIOD) userid = kwargs.get('userid', None) leap_check(userid is not None, "need an user id") - offline = kwargs.get('offline', False) uuid = soledad.uuid factory = LeapIMAPFactory(uuid, userid, soledad) @@ -154,16 +144,6 @@ def run_service(*args, **kwargs): try: tport = reactor.listenTCP(port, factory, interface="localhost") - if not offline: - # FIXME --- update after meskio's work - fetcher = LeapIncomingMail( - keymanager, - soledad, - factory.theAccount, - check_period, - userid) - else: - fetcher = None except CannotListenError: logger.error("IMAP Service failed to start: " "cannot listen in port %s" % (port,)) @@ -186,7 +166,7 @@ def run_service(*args, **kwargs): leap_events.signal(IMAP_SERVICE_STARTED, str(port)) # FIXME -- change service signature - return fetcher, tport, factory + return tport, factory # not ok, signal error. leap_events.signal(IMAP_SERVICE_FAILED_TO_START, str(port)) diff --git a/src/leap/mail/imap/tests/test_incoming_mail.py b/src/leap/mail/imap/tests/test_incoming_mail.py deleted file mode 100644 index 03c0164..0000000 --- a/src/leap/mail/imap/tests/test_incoming_mail.py +++ /dev/null @@ -1,193 +0,0 @@ -# -*- coding: utf-8 -*- -# test_imap.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test case for leap.email.imap.fetch - -@authors: Ruben Pollan, - -@license: GPLv3, see included LICENSE file -""" - -import json - -from email.mime.application import MIMEApplication -from email.mime.multipart import MIMEMultipart -from email.parser import Parser -from mock import Mock - -from leap.keymanager.openpgp import OpenPGPKey -from leap.mail.imap.account import SoledadBackedAccount -from leap.mail.imap.fetch import LeapIncomingMail -from leap.mail.imap.fields import fields -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.service.imap import INCOMING_CHECK_PERIOD -from leap.mail.tests import ( - TestCaseWithKeyManager, - ADDRESS, -) -from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.crypto import ( - EncryptionSchemes, - ENC_JSON_KEY, - ENC_SCHEME_KEY, -) - - -class LeapIncomingMailTestCase(TestCaseWithKeyManager): - """ - Tests for the incoming mail parser - """ - NICKSERVER = "http://domain" - FROM_ADDRESS = "test@somedomain.com" - BODY = """ -Governments of the Industrial World, you weary giants of flesh and steel, I -come from Cyberspace, the new home of Mind. On behalf of the future, I ask -you of the past to leave us alone. You are not welcome among us. You have -no sovereignty where we gather. - """ - EMAIL = """from: Test from SomeDomain <%(from)s> -to: %(to)s -subject: independence of cyberspace - -%(body)s - """ % { - "from": FROM_ADDRESS, - "to": ADDRESS, - "body": BODY - } - - def setUp(self): - super(LeapIncomingMailTestCase, self).setUp() - - # Soledad sync makes trial block forever. The sync it's mocked to fix - # this problem. _mock_soledad_get_from_index can be used from the tests - # to provide documents. - self._soledad.sync = Mock() - - memstore = MemoryStore() - theAccount = SoledadBackedAccount( - ADDRESS, - soledad=self._soledad, - memstore=memstore) - self.fetcher = LeapIncomingMail( - self._km, - self._soledad, - theAccount, - INCOMING_CHECK_PERIOD, - ADDRESS) - - def tearDown(self): - del self.fetcher - super(LeapIncomingMailTestCase, self).tearDown() - - def testExtractOpenPGPHeader(self): - """ - Test the OpenPGP header key extraction - """ - KEYURL = "https://somedomain.com/key.txt" - OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) - - message = Parser().parsestr(self.EMAIL) - message.add_header("OpenPGP", OpenPGP) - email = self._create_incoming_email(message.as_string()) - self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]) - self.fetcher._keymanager.fetch_key = Mock() - - def fetch_key_called(ret): - self.fetcher._keymanager.fetch_key.assert_called_once_with( - self.FROM_ADDRESS, KEYURL, OpenPGPKey) - - d = self.fetcher.fetch() - d.addCallback(fetch_key_called) - return d - - def testExtractOpenPGPHeaderInvalidUrl(self): - """ - Test the OpenPGP header key extraction - """ - KEYURL = "https://someotherdomain.com/key.txt" - OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) - - message = Parser().parsestr(self.EMAIL) - message.add_header("OpenPGP", OpenPGP) - email = self._create_incoming_email(message.as_string()) - self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]) - self.fetcher._keymanager.fetch_key = Mock() - - def fetch_key_called(ret): - self.assertFalse(self.fetcher._keymanager.fetch_key.called) - - d = self.fetcher.fetch() - d.addCallback(fetch_key_called) - return d - - def testExtractAttachedKey(self): - """ - Test the OpenPGP header key extraction - """ - KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." - - message = MIMEMultipart() - message.add_header("from", self.FROM_ADDRESS) - key = MIMEApplication("", "pgp-keys") - key.set_payload(KEY) - message.attach(key) - - def put_raw_key_called(ret): - self.fetcher._keymanager.put_raw_key.assert_called_once_with( - KEY, OpenPGPKey, address=self.FROM_ADDRESS) - - d = self.mock_fetch(message.as_string()) - d.addCallback(put_raw_key_called) - return d - - def _mock_fetch(self, message): - self.fetcher._keymanager.fetch_key = Mock() - d = self._create_incoming_email(message) - d.addCallback( - lambda email: - self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) - d.addCallback(lambda _: self.fetcher.fetch()) - return d - - def _create_incoming_email(self, email_str): - email = SoledadDocument() - data = json.dumps( - {"incoming": True, "content": email_str}, - ensure_ascii=False) - - def set_email_content(pubkey): - email.content = { - fields.INCOMING_KEY: True, - fields.ERROR_DECRYPTING_KEY: False, - ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, - ENC_JSON_KEY: str(self._km.encrypt(data, pubkey)) - } - return email - - d = self._km.get_key(ADDRESS, OpenPGPKey) - d.addCallback(set_email_content) - return d - - def _mock_soledad_get_from_index(self, index_name, value): - get_from_index = self._soledad.get_from_index - - def soledad_mock(idx_name, *key_values): - if index_name == idx_name: - return value - return get_from_index(idx_name, *key_values) - self.fetcher._soledad.get_from_index = Mock(side_effect=soledad_mock) diff --git a/src/leap/mail/incoming/__init__.py b/src/leap/mail/incoming/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py new file mode 100644 index 0000000..e52c727 --- /dev/null +++ b/src/leap/mail/incoming/service.py @@ -0,0 +1,760 @@ +# -*- coding: utf-8 -*- +# service.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Incoming mail fetcher. +""" +import copy +import logging +import shlex +import threading +import time +import traceback +import warnings + +from email.parser import Parser +from email.generator import Generator +from email.utils import parseaddr +from StringIO import StringIO +from urlparse import urlparse + +from twisted.application.service import Service +from twisted.python import log +from twisted.internet import defer, reactor +from twisted.internet.task import LoopingCall +from twisted.internet.task import deferLater +from u1db import errors as u1db_errors + +from leap.common import events as leap_events +from leap.common.check import leap_assert, leap_assert_type +from leap.common.events.events_pb2 import IMAP_FETCHED_INCOMING +from leap.common.events.events_pb2 import IMAP_MSG_PROCESSING +from leap.common.events.events_pb2 import IMAP_MSG_DECRYPTED +from leap.common.events.events_pb2 import IMAP_MSG_SAVED_LOCALLY +from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING +from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL +from leap.common.events.events_pb2 import SOLEDAD_INVALID_AUTH_TOKEN +from leap.common.mail import get_email_charset +from leap.keymanager import errors as keymanager_errors +from leap.keymanager.openpgp import OpenPGPKey +from leap.mail.adaptors import soledad_indexes as fields +from leap.mail.decorators import deferred_to_thread +from leap.mail.utils import json_loads, empty, first +from leap.soledad.client import Soledad +from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY +from leap.soledad.common.errors import InvalidAuthTokenError + + +logger = logging.getLogger(__name__) + +MULTIPART_ENCRYPTED = "multipart/encrypted" +MULTIPART_SIGNED = "multipart/signed" +PGP_BEGIN = "-----BEGIN PGP MESSAGE-----" +PGP_END = "-----END PGP MESSAGE-----" + +# The period between succesive checks of the incoming mail +# queue (in seconds) +INCOMING_CHECK_PERIOD = 60 + + +class MalformedMessage(Exception): + """ + Raised when a given message is not well formed. + """ + pass + + +class IncomingMail(Service): + """ + Fetches and process mail from the incoming pool. + + This object implements IService interface, has public methods + startService and stopService that will actually initiate a + LoopingCall with check_period recurrency. + The LoopingCall itself will invoke the fetch method each time + that the check_period expires. + + This loop will sync the soledad db with the remote server and + process all the documents found tagged as incoming mail. + """ + + name = "IncomingMail" + + RECENT_FLAG = "\\Recent" + CONTENT_KEY = "content" + + LEAP_SIGNATURE_HEADER = 'X-Leap-Signature' + """ + Header added to messages when they are decrypted by the IMAP fetcher, + which states the validity of an eventual signature that might be included + in the encrypted blob. + """ + LEAP_SIGNATURE_VALID = 'valid' + LEAP_SIGNATURE_INVALID = 'invalid' + LEAP_SIGNATURE_COULD_NOT_VERIFY = 'could not verify' + + fetching_lock = threading.Lock() + + def __init__(self, keymanager, soledad, inbox, userid, + check_period=INCOMING_CHECK_PERIOD): + + """ + Initialize IncomingMail.. + + :param keymanager: a keymanager instance + :type keymanager: keymanager.KeyManager + + :param soledad: a soledad instance + :type soledad: Soledad + + :param inbox: the inbox where the new emails will be stored + :type inbox: IMAPMailbox + + :param check_period: the period to fetch new mail, in seconds. + :type check_period: int + """ + + leap_assert(keymanager, "need a keymanager to initialize") + leap_assert_type(soledad, Soledad) + leap_assert(check_period, "need a period to check incoming mail") + leap_assert_type(check_period, int) + leap_assert(userid, "need a userid to initialize") + + self._keymanager = keymanager + self._soledad = soledad + self._inbox = inbox + self._userid = userid + + self._loop = None + self._check_period = check_period + + # initialize a mail parser only once + self._parser = Parser() + + # + # Public API: fetch, start_loop, stop. + # + + def fetch(self): + """ + Fetch incoming mail, to be called periodically. + + Calls a deferred that will execute the fetch callback + in a separate thread + """ + def mail_compat(failure): + if failure.check(u1db_errors.InvalidGlobbing): + # It looks like we are a dealing with an outdated + # mx. Fallback to the version of the index + warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", + DeprecationWarning) + return self._soledad.get_from_index( + fields.JUST_MAIL_COMPAT_IDX, "*") + return failure + + def syncSoledadCallback(_): + d = self._soledad.get_from_index( + fields.JUST_MAIL_IDX, "*", "0") + d.addErrback(mail_compat) + d.addCallback(self._process_doclist) + return d + + logger.debug("fetching mail for: %s %s" % ( + self._soledad.uuid, self._userid)) + if not self.fetching_lock.locked(): + d1 = self._sync_soledad() + d = defer.gatherResults([d1], consumeErrors=True) + d.addCallbacks(syncSoledadCallback, self._errback) + d.addCallbacks(self._signal_fetch_to_ui, self._errback) + return d + else: + logger.debug("Already fetching mail.") + + def startService(self): + """ + Starts a loop to fetch mail. + """ + Service.startService(self) + if self._loop is None: + self._loop = LoopingCall(self.fetch) + self._loop.start(self._check_period) + else: + logger.warning("Tried to start an already running fetching loop.") + + def stopService(self): + """ + Stops the loop that fetches mail. + """ + if self._loop and self._loop.running is True: + self._loop.stop() + self._loop = None + Service.stopService(self) + + # + # Private methods. + # + + # synchronize incoming mail + + def _errback(self, failure): + logger.exception(failure.value) + traceback.print_exc() + + @deferred_to_thread + def _sync_soledad(self): + """ + Synchronize with remote soledad. + + :returns: a list of LeapDocuments, or None. + :rtype: iterable or None + """ + with self.fetching_lock: + try: + log.msg('FETCH: syncing soledad...') + self._soledad.sync() + log.msg('FETCH soledad SYNCED.') + except InvalidAuthTokenError: + # if the token is invalid, send an event so the GUI can + # disable mail and show an error message. + leap_events.signal(SOLEDAD_INVALID_AUTH_TOKEN) + + def _signal_fetch_to_ui(self, doclist): + """ + Send leap events to ui. + + :param doclist: iterable with msg documents. + :type doclist: iterable. + :returns: doclist + :rtype: iterable + """ + doclist = first(doclist) # gatherResults pass us a list + if doclist: + fetched_ts = time.mktime(time.gmtime()) + num_mails = len(doclist) if doclist is not None else 0 + if num_mails != 0: + log.msg("there are %s mails" % (num_mails,)) + leap_events.signal( + IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts)) + return doclist + + def _signal_unread_to_ui(self, *args): + """ + Sends unread event to ui. + """ + leap_events.signal( + IMAP_UNREAD_MAIL, str(self._inbox.getUnseenCount())) + + # process incoming mail. + + def _process_doclist(self, doclist): + """ + Iterates through the doclist, checks if each doc + looks like a message, and yields a deferred that will decrypt and + process the message. + + :param doclist: iterable with msg documents. + :type doclist: iterable. + :returns: a list of deferreds for individual messages. + """ + log.msg('processing doclist') + if not doclist: + logger.debug("no docs found") + return + num_mails = len(doclist) + + deferreds = [] + for index, doc in enumerate(doclist): + logger.debug("processing doc %d of %d" % (index + 1, num_mails)) + leap_events.signal( + IMAP_MSG_PROCESSING, str(index), str(num_mails)) + + keys = doc.content.keys() + + # TODO Compatibility check with the index in pre-0.6 mx + # that does not write the ERROR_DECRYPTING_KEY + # This should be removed in 0.7 + + has_errors = doc.content.get(fields.ERROR_DECRYPTING_KEY, None) + if has_errors is None: + warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", + DeprecationWarning) + + if has_errors: + logger.debug("skipping msg with decrypting errors...") + elif self._is_msg(keys): + d = self._decrypt_doc(doc) + d.addCallback(self._extract_keys) + d.addCallbacks(self._add_message_locally, self._errback) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + # + # operations on individual messages + # + + #FIXME: @deferred_to_thread + def _decrypt_doc(self, doc): + """ + Decrypt the contents of a document. + + :param doc: A document containing an encrypted message. + :type doc: SoledadDocument + + :return: A Deferred that will be fired with the document and the + decrypted message. + :rtype: SoledadDocument, str + """ + log.msg('decrypting msg') + + def process_decrypted(res): + if isinstance(res, tuple): + decrdata, _ = res + success = True + else: + decrdata = "" + success = False + + leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0") + return self._process_decrypted_doc(doc, decrdata) + + d = self._keymanager.decrypt( + doc.content[ENC_JSON_KEY], + self._userid, OpenPGPKey) + d.addErrback(self._errback) + d.addCallback(process_decrypted) + d.addCallback(lambda data: (doc, data)) + return d + + def _process_decrypted_doc(self, doc, data): + """ + Process a document containing a succesfully decrypted message. + + :param doc: the incoming message + :type doc: SoledadDocument + :param data: the json-encoded, decrypted content of the incoming + message + :type data: str + + :return: a Deferred that will be fired with an str of the proccessed + data. + :rtype: Deferred + """ + log.msg('processing decrypted doc') + + # XXX turn this into an errBack for each one of + # the deferreds that would process an individual document + try: + msg = json_loads(data) + except UnicodeError as exc: + logger.error("Error while decrypting %s" % (doc.doc_id,)) + logger.exception(exc) + + # we flag the message as "with decrypting errors", + # to avoid further decryption attempts during sync + # cycles until we're prepared to deal with that. + # What is the same, when Ivan deals with it... + # A new decrypting attempt event could be triggered by a + # future a library upgrade, or a cli flag to the client, + # we just `defer` that for now... :) + doc.content[fields.ERROR_DECRYPTING_KEY] = True + deferLater(reactor, 0, self._update_incoming_message, doc) + + # FIXME this is just a dirty hack to delay the proper + # deferred organization here... + # and remember, boys, do not do this at home. + return [] + + if not isinstance(msg, dict): + defer.returnValue(False) + if not msg.get(fields.INCOMING_KEY, False): + defer.returnValue(False) + + # ok, this is an incoming message + rawmsg = msg.get(self.CONTENT_KEY, None) + if not rawmsg: + return "" + return self._maybe_decrypt_msg(rawmsg) + + @deferred_to_thread + def _update_incoming_message(self, doc): + """ + Do a put for a soledad document. This probably has been called only + in the case that we've needed to update the ERROR_DECRYPTING_KEY + flag in an incoming message, to get it out of the decrypting queue. + + :param doc: the SoledadDocument to update + :type doc: SoledadDocument + """ + log.msg("Updating SoledadDoc %s" % (doc.doc_id)) + self._soledad.put_doc(doc) + + @deferred_to_thread + def _delete_incoming_message(self, doc): + """ + Delete document. + + :param doc: the SoledadDocument to delete + :type doc: SoledadDocument + """ + log.msg("Deleting Incoming message: %s" % (doc.doc_id,)) + self._soledad.delete_doc(doc) + + def _maybe_decrypt_msg(self, data): + """ + Tries to decrypt a gpg message if data looks like one. + + :param data: the text to be decrypted. + :type data: str + + :return: a Deferred that will be fired with an str of data, possibly + decrypted. + :rtype: Deferred + """ + leap_assert_type(data, str) + log.msg('maybe decrypting doc') + + # parse the original message + encoding = get_email_charset(data) + msg = self._parser.parsestr(data) + + fromHeader = msg.get('from', None) + senderAddress = None + if (fromHeader is not None + and (msg.get_content_type() == MULTIPART_ENCRYPTED + or msg.get_content_type() == MULTIPART_SIGNED)): + senderAddress = parseaddr(fromHeader) + + def add_leap_header(ret): + decrmsg, signkey = ret + if (senderAddress is None or + isinstance(signkey, keymanager_errors.KeyNotFound)): + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_COULD_NOT_VERIFY) + elif isinstance(signkey, keymanager_errors.InvalidSignature): + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_INVALID) + else: + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_VALID, + pubkey=signkey.key_id) + return decrmsg.as_string() + + if msg.get_content_type() == MULTIPART_ENCRYPTED: + d = self._decrypt_multipart_encrypted_msg( + msg, encoding, senderAddress) + else: + d = self._maybe_decrypt_inline_encrypted_msg( + msg, encoding, senderAddress) + d.addCallback(add_leap_header) + return d + + def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderAddress): + """ + Decrypt a message with content-type 'multipart/encrypted'. + + :param msg: The original encrypted message. + :type msg: Message + :param encoding: The encoding of the email message. + :type encoding: str + :param senderAddress: The email address of the sender of the message. + :type senderAddress: str + + :return: A Deferred that will be fired with a tuple containing a + decrypted Message and the signing OpenPGPKey if the signature + is valid or InvalidSignature or KeyNotFound. + :rtype: Deferred + """ + log.msg('decrypting multipart encrypted msg') + msg = copy.deepcopy(msg) + self._msg_multipart_sanity_check(msg) + + # parse message and get encrypted content + pgpencmsg = msg.get_payload()[1] + encdata = pgpencmsg.get_payload() + + # decrypt or fail gracefully + def build_msg(res): + decrdata, signkey = res + + decrmsg = self._parser.parsestr(decrdata) + # remove original message's multipart/encrypted content-type + del(msg['content-type']) + + # replace headers back in original message + for hkey, hval in decrmsg.items(): + try: + # this will raise KeyError if header is not present + msg.replace_header(hkey, hval) + except KeyError: + msg[hkey] = hval + + # all ok, replace payload by unencrypted payload + msg.set_payload(decrmsg.get_payload()) + return (msg, signkey) + + d = self._keymanager.decrypt( + encdata, self._userid, OpenPGPKey, + verify=senderAddress) + d.addCallbacks(build_msg, self._decryption_error, errbackArgs=(msg,)) + return d + + def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding, + senderAddress): + """ + Possibly decrypt an inline OpenPGP encrypted message. + + :param origmsg: The original, possibly encrypted message. + :type origmsg: Message + :param encoding: The encoding of the email message. + :type encoding: str + :param senderAddress: The email address of the sender of the message. + :type senderAddress: str + + :return: A Deferred that will be fired with a tuple containing a + decrypted Message and the signing OpenPGPKey if the signature + is valid or InvalidSignature or KeyNotFound. + :rtype: Deferred + """ + log.msg('maybe decrypting inline encrypted msg') + # serialize the original message + buf = StringIO() + g = Generator(buf) + g.flatten(origmsg) + data = buf.getvalue() + + def decrypted_data(res): + decrdata, signkey = res + return data.replace(pgp_message, decrdata), signkey + + def encode_and_return(res): + data, signkey = res + if isinstance(data, unicode): + data = data.encode(encoding, 'replace') + return (self._parser.parsestr(data), signkey) + + # handle exactly one inline PGP message + if PGP_BEGIN in data: + begin = data.find(PGP_BEGIN) + end = data.find(PGP_END) + pgp_message = data[begin:end + len(PGP_END)] + d = self._keymanager.decrypt( + pgp_message, self._userid, OpenPGPKey, + verify=senderAddress) + d.addCallbacks(decrypted_data, self._decryption_error, + errbackArgs=(data,)) + else: + d = defer.succeed((data, None)) + d.addCallback(encode_and_return) + return d + + def _decryption_error(self, failure, msg): + """ + Check for known decryption errors + """ + if failure.check(keymanager_errors.DecryptError): + logger.warning('Failed to decrypt encrypted message (%s). ' + 'Storing message without modifications.' + % str(failure.value)) + return (msg, None) + elif failure.check(keymanager_errors.KeyNotFound): + logger.error('Failed to find private key for decryption (%s). ' + 'Storing message without modifications.' + % str(failure.value)) + return (msg, None) + else: + return failure + + def _extract_keys(self, msgtuple): + """ + Retrieve attached keys to the mesage and parse message headers for an + *OpenPGP* header as described on the `IETF draft + ` + only urls with https and the same hostname than the email are supported + for security reasons. + + :param msgtuple: a tuple consisting of a SoledadDocument + instance containing the incoming message + and data, the json-encoded, decrypted content of the + incoming message + :type msgtuple: (SoledadDocument, str) + + :return: A Deferred that will be fired with msgtuple when key + extraction finishes + :rtype: Deferred + """ + OpenPGP_HEADER = 'OpenPGP' + doc, data = msgtuple + + # XXX the parsing of the message is done in mailbox.addMessage, maybe + # we should do it in this module so we don't need to parse it again + # here + msg = self._parser.parsestr(data) + _, fromAddress = parseaddr(msg['from']) + + header = msg.get(OpenPGP_HEADER, None) + dh = defer.succeed(None) + if header is not None: + dh = self._extract_openpgp_header(header, fromAddress) + + da = defer.succeed(None) + if msg.is_multipart(): + da = self._extract_attached_key(msg.get_payload(), fromAddress) + + d = defer.gatherResults([dh, da]) + d.addCallback(lambda _: msgtuple) + return d + + def _extract_openpgp_header(self, header, address): + """ + Import keys from the OpenPGP header + + :param header: OpenPGP header string + :type header: str + :param address: email address in the from header + :type address: str + + :return: A Deferred that will be fired when header extraction is done + :rtype: Deferred + """ + d = defer.succeed(None) + fields = dict([f.strip(' ').split('=') for f in header.split(';')]) + if 'url' in fields: + url = shlex.split(fields['url'])[0] # remove quotations + urlparts = urlparse(url) + addressHostname = address.split('@')[1] + if (urlparts.scheme == 'https' + and urlparts.hostname == addressHostname): + def fetch_error(failure): + if failure.check(keymanager_errors.KeyNotFound): + logger.warning("Url from OpenPGP header %s failed" + % (url,)) + elif failure.check(keymanager_errors.KeyAttributesDiffer): + logger.warning("Key from OpenPGP header url %s didn't " + "match the from address %s" + % (url, address)) + else: + return failure + + d = self._keymanager.fetch_key(address, url, OpenPGPKey) + d.addCallback( + lambda _: + logger.info("Imported key from header %s" % (url,))) + d.addErrback(fetch_error) + else: + logger.debug("No valid url on OpenPGP header %s" % (url,)) + else: + logger.debug("There is no url on the OpenPGP header: %s" + % (header,)) + return d + + def _extract_attached_key(self, attachments, address): + """ + Import keys from the attachments + + :param attachments: email attachment list + :type attachments: list(email.Message) + :param address: email address in the from header + :type address: str + + :return: A Deferred that will be fired when all the keys are stored + :rtype: Deferred + """ + MIME_KEY = "application/pgp-keys" + + deferreds = [] + for attachment in attachments: + if MIME_KEY == attachment.get_content_type(): + logger.debug("Add key from attachment") + d = self._keymanager.put_raw_key( + attachment.get_payload(), + OpenPGPKey, + address=address) + deferreds.append(d) + return defer.gatherResults(deferreds) + + def _add_message_locally(self, msgtuple): + """ + Adds a message to local inbox and delete it from the incoming db + in soledad. + + :param msgtuple: a tuple consisting of a SoledadDocument + instance containing the incoming message + and data, the json-encoded, decrypted content of the + incoming message + :type msgtuple: (SoledadDocument, str) + + :return: A Deferred that will be fired when the messages is stored + :rtype: Defferred + """ + doc, data = msgtuple + log.msg('adding message %s to local db' % (doc.doc_id,)) + + if isinstance(data, list): + if empty(data): + return False + data = data[0] + + def msgSavedCallback(result): + if not empty(result): + leap_events.signal(IMAP_MSG_SAVED_LOCALLY) + deferLater(reactor, 0, self._delete_incoming_message, doc) + leap_events.signal(IMAP_MSG_DELETED_INCOMING) + + d = self._inbox.addMessage(data, (self.RECENT_FLAG,)) + d.addCallbacks(msgSavedCallback, self._errback) + return d + + # + # helpers + # + + def _msg_multipart_sanity_check(self, msg): + """ + Performs a sanity check against a multipart encrypted msg + + :param msg: The original encrypted message. + :type msg: Message + """ + # sanity check + payload = msg.get_payload() + if len(payload) != 2: + raise MalformedMessage( + 'Multipart/encrypted messages should have exactly 2 body ' + 'parts (instead of %d).' % len(payload)) + if payload[0].get_content_type() != 'application/pgp-encrypted': + raise MalformedMessage( + "Multipart/encrypted messages' first body part should " + "have content type equal to 'application/pgp-encrypted' " + "(instead of %s)." % payload[0].get_content_type()) + if payload[1].get_content_type() != 'application/octet-stream': + raise MalformedMessage( + "Multipart/encrypted messages' second body part should " + "have content type equal to 'octet-stream' (instead of " + "%s)." % payload[1].get_content_type()) + + def _is_msg(self, keys): + """ + Checks if the keys of a dictionary match the signature + of the document type we use for messages. + + :param keys: iterable containing the strings to match. + :type keys: iterable of strings. + :rtype: bool + """ + return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys diff --git a/src/leap/mail/incoming/tests/__init__.py b/src/leap/mail/incoming/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/mail/incoming/tests/test_incoming_mail.py b/src/leap/mail/incoming/tests/test_incoming_mail.py new file mode 100644 index 0000000..bf95b1d --- /dev/null +++ b/src/leap/mail/incoming/tests/test_incoming_mail.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- +# test_incoming_mail.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test case for leap.mail.incoming.service + +@authors: Ruben Pollan, + +@license: GPLv3, see included LICENSE file +""" + +import json + +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email.parser import Parser +from mock import Mock +from twisted.internet import defer + +from leap.keymanager.openpgp import OpenPGPKey +from leap.mail.adaptors import soledad_indexes as fields +from leap.mail.constants import INBOX_NAME +from leap.mail.imap.account import IMAPAccount +from leap.mail.incoming.service import IncomingMail +from leap.mail.tests import ( + TestCaseWithKeyManager, + ADDRESS, +) +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.crypto import ( + EncryptionSchemes, + ENC_JSON_KEY, + ENC_SCHEME_KEY, +) + + +class IncomingMailTestCase(TestCaseWithKeyManager): + """ + Tests for the incoming mail parser + """ + NICKSERVER = "http://domain" + FROM_ADDRESS = "test@somedomain.com" + BODY = """ +Governments of the Industrial World, you weary giants of flesh and steel, I +come from Cyberspace, the new home of Mind. On behalf of the future, I ask +you of the past to leave us alone. You are not welcome among us. You have +no sovereignty where we gather. + """ + EMAIL = """from: Test from SomeDomain <%(from)s> +to: %(to)s +subject: independence of cyberspace + +%(body)s + """ % { + "from": FROM_ADDRESS, + "to": ADDRESS, + "body": BODY + } + + def setUp(self): + def getInbox(_): + theAccount = IMAPAccount(ADDRESS, self._soledad) + return theAccount.callWhenReady( + lambda _: theAccount.getMailbox(INBOX_NAME)) + + def setUpFetcher(inbox): + # Soledad sync makes trial block forever. The sync it's mocked to + # fix this problem. _mock_soledad_get_from_index can be used from + # the tests to provide documents. + self._soledad.sync = Mock() + + self.fetcher = IncomingMail( + self._km, + self._soledad, + inbox, + ADDRESS) + + d = super(IncomingMailTestCase, self).setUp() + d.addCallback(getInbox) + d.addCallback(setUpFetcher) + return d + + def tearDown(self): + del self.fetcher + return super(IncomingMailTestCase, self).tearDown() + + def testExtractOpenPGPHeader(self): + """ + Test the OpenPGP header key extraction + """ + KEYURL = "https://somedomain.com/key.txt" + OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) + + message = Parser().parsestr(self.EMAIL) + message.add_header("OpenPGP", OpenPGP) + self.fetcher._keymanager.fetch_key = Mock( + return_value=defer.succeed(None)) + + def fetch_key_called(ret): + self.fetcher._keymanager.fetch_key.assert_called_once_with( + self.FROM_ADDRESS, KEYURL, OpenPGPKey) + + d = self._create_incoming_email(message.as_string()) + d.addCallback( + lambda email: + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) + d.addCallback(lambda _: self.fetcher.fetch()) + d.addCallback(fetch_key_called) + return d + + def testExtractOpenPGPHeaderInvalidUrl(self): + """ + Test the OpenPGP header key extraction + """ + KEYURL = "https://someotherdomain.com/key.txt" + OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) + + message = Parser().parsestr(self.EMAIL) + message.add_header("OpenPGP", OpenPGP) + self.fetcher._keymanager.fetch_key = Mock() + + def fetch_key_called(ret): + self.assertFalse(self.fetcher._keymanager.fetch_key.called) + + d = self._create_incoming_email(message.as_string()) + d.addCallback( + lambda email: + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) + d.addCallback(lambda _: self.fetcher.fetch()) + d.addCallback(fetch_key_called) + return d + + def testExtractAttachedKey(self): + """ + Test the OpenPGP header key extraction + """ + KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." + + message = MIMEMultipart() + message.add_header("from", self.FROM_ADDRESS) + key = MIMEApplication("", "pgp-keys") + key.set_payload(KEY) + message.attach(key) + self.fetcher._keymanager.put_raw_key = Mock( + return_value=defer.succeed(None)) + + def put_raw_key_called(_): + self.fetcher._keymanager.put_raw_key.assert_called_once_with( + KEY, OpenPGPKey, address=self.FROM_ADDRESS) + + d = self._mock_fetch(message.as_string()) + d.addCallback(put_raw_key_called) + return d + + def _mock_fetch(self, message): + self.fetcher._keymanager.fetch_key = Mock() + d = self._create_incoming_email(message) + d.addCallback( + lambda email: + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) + d.addCallback(lambda _: self.fetcher.fetch()) + return d + + def _create_incoming_email(self, email_str): + email = SoledadDocument() + data = json.dumps( + {"incoming": True, "content": email_str}, + ensure_ascii=False) + + def set_email_content(encr_data): + email.content = { + fields.INCOMING_KEY: True, + fields.ERROR_DECRYPTING_KEY: False, + ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, + ENC_JSON_KEY: encr_data + } + return email + d = self._km.encrypt(data, ADDRESS, OpenPGPKey, fetch_remote=False) + d.addCallback(set_email_content) + return d + + def _mock_soledad_get_from_index(self, index_name, value): + get_from_index = self._soledad.get_from_index + + def soledad_mock(idx_name, *key_values): + if index_name == idx_name: + return defer.succeed(value) + return get_from_index(idx_name, *key_values) + self.fetcher._soledad.get_from_index = Mock(side_effect=soledad_mock) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 8137f97..cb37d25 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -414,15 +414,10 @@ class MessageCollection(object): # Manipulate messages - def add_msg(self, raw_msg, flags=None, tags=None, date=None): + def add_msg(self, raw_msg, flags=tuple(), tags=tuple(), date=""): """ Add a message to this collection. """ - if not flags: - flags = tuple() - if not tags: - tags = tuple() - leap_assert_type(flags, tuple) leap_assert_type(date, str) @@ -582,7 +577,6 @@ class Account(object): self.mbox_indexer = MailboxIndexer(self.store) self.deferred_initialization = defer.Deferred() - self._initialized = False self._ready_cb = ready_cb self._init_d = self._initialize_storage() @@ -594,7 +588,6 @@ class Account(object): return self.add_mailbox(INBOX_NAME) def finish_initialization(result): - self._initialized = True self.deferred_initialization.callback(None) if self._ready_cb is not None: self._ready_cb() @@ -606,12 +599,8 @@ class Account(object): return d def callWhenReady(self, cb, *args, **kw): - if self._initialized: - cb(self, *args, **kw) - return defer.succeed(None) - else: - self.deferred_initialization.addCallback(cb, *args, **kw) - return self.deferred_initialization + self.deferred_initialization.addCallback(cb, *args, **kw) + return self.deferred_initialization # # Public API Starts -- cgit v1.2.3 From 26034f854a3b1cc4a39b2e46b57140af27a08ddc Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 15 Jan 2015 15:50:20 -0400 Subject: update mail/imap tests --- src/leap/mail/mail.py | 6 ++---- src/leap/mail/tests/test_mail.py | 18 +++++++++--------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index cb37d25..8629d0e 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -285,6 +285,8 @@ class MessageCollection(object): self.mbox_indexer = mbox_indexer self.mbox_wrapper = mbox_wrapper + # TODO --- review this count shit. I think it's better to patch server + # to accept deferreds. # TODO need to initialize count here because imap server does not # expect a defered for the count. caller should return the deferred for # prime_count (ie, initialize) when returning the collection @@ -295,10 +297,6 @@ class MessageCollection(object): count = 0 self._count = count - #def initialize(self): - #d = self.prime_count() - #return d - def is_mailbox_collection(self): """ Return True if this collection represents a Mailbox. diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py index 2c4b919..9bc553f 100644 --- a/src/leap/mail/tests/test_mail.py +++ b/src/leap/mail/tests/test_mail.py @@ -17,8 +17,10 @@ """ Tests for the mail module. """ -import time import os +import time +import uuid + from functools import partial from email.parser import Parser from email.Utils import formatdate @@ -28,9 +30,6 @@ from leap.mail.mail import MessageCollection, Account from leap.mail.mailbox_indexer import MailboxIndexer from leap.mail.tests.common import SoledadTestMixin -# from twisted.internet import defer -from twisted.trial import unittest - HERE = os.path.split(os.path.abspath(__file__))[0] @@ -67,24 +66,26 @@ class CollectionMixin(object): if mbox_collection: mbox_indexer = MailboxIndexer(store) mbox_name = "TestMbox" + mbox_uuid = str(uuid.uuid4()) else: mbox_indexer = mbox_name = None def get_collection_from_mbox_wrapper(wrapper): + wrapper.uuid = mbox_uuid return MessageCollection( adaptor, store, mbox_indexer=mbox_indexer, mbox_wrapper=wrapper) d = adaptor.initialize_store(store) if mbox_collection: - d.addCallback(lambda _: mbox_indexer.create_table(mbox_name)) + d.addCallback(lambda _: mbox_indexer.create_table(mbox_uuid)) d.addCallback(lambda _: adaptor.get_or_create_mbox(store, mbox_name)) d.addCallback(get_collection_from_mbox_wrapper) return d # TODO profile add_msg. Why are these tests so SLOW??! -class MessageTestCase(unittest.TestCase, SoledadTestMixin, CollectionMixin): +class MessageTestCase(SoledadTestMixin, CollectionMixin): """ Tests for the Message class. """ @@ -204,8 +205,7 @@ class MessageTestCase(unittest.TestCase, SoledadTestMixin, CollectionMixin): self.assertEquals(msg.get_tags(), self.msg_tags) -class MessageCollectionTestCase(unittest.TestCase, - SoledadTestMixin, CollectionMixin): +class MessageCollectionTestCase(SoledadTestMixin, CollectionMixin): """ Tests for the MessageCollection class. """ @@ -294,7 +294,7 @@ class MessageCollectionTestCase(unittest.TestCase, pass -class AccountTestCase(unittest.TestCase, SoledadTestMixin): +class AccountTestCase(SoledadTestMixin): """ Tests for the Account class. """ -- cgit v1.2.3 From 49f29ea0430d75ab99563e85019fd58ea7370f96 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 16 Jan 2015 20:26:37 -0400 Subject: return the deferred from the incoming.startService() call --- src/leap/mail/incoming/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index e52c727..71c356f 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -190,7 +190,7 @@ class IncomingMail(Service): Service.startService(self) if self._loop is None: self._loop = LoopingCall(self.fetch) - self._loop.start(self._check_period) + return self._loop.start(self._check_period) else: logger.warning("Tried to start an already running fetching loop.") -- cgit v1.2.3 From 3e8458b3640167078ad24584fed4a071cae934d9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 16 Jan 2015 20:27:24 -0400 Subject: tests: add link related to trial block with sync --- src/leap/mail/imap/tests/utils.py | 1 + src/leap/mail/incoming/tests/test_incoming_mail.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index 5708787..83c3f29 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -99,6 +99,7 @@ class IMAP4HelperMixin(SoledadTestMixin): # Soledad sync makes trial block forever. The sync it's mocked to # fix this problem. _mock_soledad_get_from_index can be used from # the tests to provide documents. + # TODO see here, possibly related? -- http://www.pythoneye.com/83_20424875/ self._soledad.sync = Mock() d = defer.Deferred() diff --git a/src/leap/mail/incoming/tests/test_incoming_mail.py b/src/leap/mail/incoming/tests/test_incoming_mail.py index bf95b1d..0745ee0 100644 --- a/src/leap/mail/incoming/tests/test_incoming_mail.py +++ b/src/leap/mail/incoming/tests/test_incoming_mail.py @@ -80,6 +80,7 @@ subject: independence of cyberspace # Soledad sync makes trial block forever. The sync it's mocked to # fix this problem. _mock_soledad_get_from_index can be used from # the tests to provide documents. + # TODO ---- see here http://www.pythoneye.com/83_20424875/ self._soledad.sync = Mock() self.fetcher = IncomingMail( -- cgit v1.2.3 From c3f487002e33b4b4cb65a04f9c192c4e59ec4908 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 20 Jan 2015 01:20:42 -0400 Subject: bug: fix empty definition; remove threading use this fixes a bug by which incoming service was not deleting the message from incoming after correclty saving all the message subparts into soledad. --- src/leap/mail/incoming/service.py | 66 +++++++++++++++++++-------------------- src/leap/mail/utils.py | 3 ++ 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index 71c356f..0b2f7c2 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -20,7 +20,6 @@ Incoming mail fetcher. import copy import logging import shlex -import threading import time import traceback import warnings @@ -51,7 +50,6 @@ from leap.common.mail import get_email_charset from leap.keymanager import errors as keymanager_errors from leap.keymanager.openpgp import OpenPGPKey from leap.mail.adaptors import soledad_indexes as fields -from leap.mail.decorators import deferred_to_thread from leap.mail.utils import json_loads, empty, first from leap.soledad.client import Soledad from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY @@ -90,6 +88,7 @@ class IncomingMail(Service): This loop will sync the soledad db with the remote server and process all the documents found tagged as incoming mail. """ + # TODO implements IService? name = "IncomingMail" @@ -106,8 +105,6 @@ class IncomingMail(Service): LEAP_SIGNATURE_INVALID = 'invalid' LEAP_SIGNATURE_COULD_NOT_VERIFY = 'could not verify' - fetching_lock = threading.Lock() - def __init__(self, keymanager, soledad, inbox, userid, check_period=INCOMING_CHECK_PERIOD): @@ -174,14 +171,10 @@ class IncomingMail(Service): logger.debug("fetching mail for: %s %s" % ( self._soledad.uuid, self._userid)) - if not self.fetching_lock.locked(): - d1 = self._sync_soledad() - d = defer.gatherResults([d1], consumeErrors=True) - d.addCallbacks(syncSoledadCallback, self._errback) - d.addCallbacks(self._signal_fetch_to_ui, self._errback) - return d - else: - logger.debug("Already fetching mail.") + d = self._sync_soledad() + d.addCallbacks(syncSoledadCallback, self._errback) + d.addCallbacks(self._signal_fetch_to_ui, self._errback) + return d def startService(self): """ @@ -213,7 +206,6 @@ class IncomingMail(Service): logger.exception(failure.value) traceback.print_exc() - @deferred_to_thread def _sync_soledad(self): """ Synchronize with remote soledad. @@ -221,15 +213,21 @@ class IncomingMail(Service): :returns: a list of LeapDocuments, or None. :rtype: iterable or None """ - with self.fetching_lock: - try: - log.msg('FETCH: syncing soledad...') - self._soledad.sync() - log.msg('FETCH soledad SYNCED.') - except InvalidAuthTokenError: - # if the token is invalid, send an event so the GUI can - # disable mail and show an error message. - leap_events.signal(SOLEDAD_INVALID_AUTH_TOKEN) + def _log_synced(result): + log.msg('FETCH soledad SYNCED.') + print "Result: ", result + return result + try: + log.msg('FETCH: syncing soledad...') + d = self._soledad.sync() + d.addCallback(_log_synced) + return d + # TODO is this still raised? or should we do failure.trap + # instead? + except InvalidAuthTokenError: + # if the token is invalid, send an event so the GUI can + # disable mail and show an error message. + leap_events.signal(SOLEDAD_INVALID_AUTH_TOKEN) def _signal_fetch_to_ui(self, doclist): """ @@ -305,7 +303,6 @@ class IncomingMail(Service): # operations on individual messages # - #FIXME: @deferred_to_thread def _decrypt_doc(self, doc): """ Decrypt the contents of a document. @@ -388,7 +385,6 @@ class IncomingMail(Service): return "" return self._maybe_decrypt_msg(rawmsg) - @deferred_to_thread def _update_incoming_message(self, doc): """ Do a put for a soledad document. This probably has been called only @@ -398,10 +394,9 @@ class IncomingMail(Service): :param doc: the SoledadDocument to update :type doc: SoledadDocument """ - log.msg("Updating SoledadDoc %s" % (doc.doc_id)) - self._soledad.put_doc(doc) + log.msg("Updating Incoming MSG: SoledadDoc %s" % (doc.doc_id)) + return self._soledad.put_doc(doc) - @deferred_to_thread def _delete_incoming_message(self, doc): """ Delete document. @@ -409,8 +404,9 @@ class IncomingMail(Service): :param doc: the SoledadDocument to delete :type doc: SoledadDocument """ + print "DELETING INCOMING MESSAGE" log.msg("Deleting Incoming message: %s" % (doc.doc_id,)) - self._soledad.delete_doc(doc) + return self._soledad.delete_doc(doc) def _maybe_decrypt_msg(self, data): """ @@ -705,16 +701,18 @@ class IncomingMail(Service): doc, data = msgtuple log.msg('adding message %s to local db' % (doc.doc_id,)) - if isinstance(data, list): - if empty(data): - return False - data = data[0] + #if isinstance(data, list): + #if empty(data): + #return False + #data = data[0] def msgSavedCallback(result): if not empty(result): leap_events.signal(IMAP_MSG_SAVED_LOCALLY) - deferLater(reactor, 0, self._delete_incoming_message, doc) - leap_events.signal(IMAP_MSG_DELETED_INCOMING) + print "DEFERRING THE DELETION ----->" + return self._delete_incoming_message(doc) + # TODO add notification as a callback + #leap_events.signal(IMAP_MSG_DELETED_INCOMING) d = self._inbox.addMessage(data, (self.RECENT_FLAG,)) d.addCallbacks(msgSavedCallback, self._errback) diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py index 457097b..8e51024 100644 --- a/src/leap/mail/utils.py +++ b/src/leap/mail/utils.py @@ -45,9 +45,12 @@ def first(things): def empty(thing): """ Return True if a thing is None or its length is zero. + If thing is a number (int, float, long), return False. """ if thing is None: return True + if isinstance(thing, (int, float, long)): + return False if isinstance(thing, SoledadDocument): thing = thing.content try: -- cgit v1.2.3 From 7542de6aeb370d647840a0b78b71f69c6fdacf8e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 20 Jan 2015 13:48:21 -0400 Subject: imap: complete FETCH implementation --- src/leap/mail/adaptors/soledad.py | 38 +++++++++++++--- src/leap/mail/imap/mailbox.py | 95 ++++++++++++++++++++++++++------------- src/leap/mail/imap/messages.py | 23 +++++++++- src/leap/mail/imap/server.py | 3 ++ src/leap/mail/mail.py | 49 ++++++++++++++++---- src/leap/mail/mailbox_indexer.py | 14 +++++- 6 files changed, 171 insertions(+), 51 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index d99f677..46dbe4c 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -364,6 +364,12 @@ class FlagsDocWrapper(SoledadDocumentWrapper): self._future_doc_id = new_id self.mbox_uuid = mbox_uuid + def get_flags(self): + """ + Get the flags for this message (as a tuple of strings, not unicode). + """ + return map(str, self.flags) + class HeaderDocWrapper(SoledadDocumentWrapper): @@ -727,11 +733,6 @@ class SoledadMailAdaptor(SoledadIndexMixin): mboxwrapper_klass = MailboxWrapper - def __init__(self): - SoledadIndexMixin.__init__(self) - - mboxwrapper_klass = MailboxWrapper - def __init__(self): SoledadIndexMixin.__init__(self) @@ -792,7 +793,7 @@ class SoledadMailAdaptor(SoledadIndexMixin): fdoc, hdoc = doc_list[:3] cdocs = dict(enumerate(doc_list[3:], 1)) return self.get_msg_from_docs( - msg_class, mdoc, fdoc, hdoc, cdocs, uid=None) + msg_class, mdoc, fdoc, hdoc, cdocs, uid=uid) def get_msg_from_mdoc_id(self, MessageClass, store, mdoc_id, uid=None, get_cdocs=False): @@ -847,6 +848,30 @@ class SoledadMailAdaptor(SoledadIndexMixin): msg_class=MessageClass, uid=uid)) return d + def get_flags_from_mdoc_id(self, store, mdoc_id): + """ + # XXX stuff here... + """ + mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0] + chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0] + + def _get_fdoc_id_from_mdoc_id(): + return constants.FDOCID.format(mbox_uuid=mbox, chash=chash) + + fdoc_id = _get_fdoc_id_from_mdoc_id() + + def wrap_fdoc(doc): + cls = FlagsDocWrapper + return cls(doc_id=doc.doc_id, **doc.content) + + def get_flags(fdoc_wrapper): + return fdoc_wrapper.get_flags() + + d = store.get_doc(fdoc_id) + d.addCallback(wrap_fdoc) + d.addCallback(get_flags) + return d + def create_msg(self, store, msg): """ :param store: an instance of soledad, or anything that behaves alike @@ -881,7 +906,6 @@ class SoledadMailAdaptor(SoledadIndexMixin): Delete all messages flagged as deleted. """ def err(f): - print "ERROR GETTING FROM INDEX" f.printTraceback() def delete_fdoc_and_mdoc_flagged(fdocs): diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index e1eb6bf..a000133 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -143,9 +143,9 @@ class IMAPMailbox(object): return self._listeners[self.mbox_name] def get_imap_message(self, message): - msg = IMAPMessage(message) - msg.store = self.collection.store - return msg + d = defer.Deferred() + IMAPMessage(message, store=self.collection.store, d=d) + return d # FIXME this grows too crazily when many instances are fired, like # during imaptest stress testing. Should have a queue of limited size @@ -468,7 +468,6 @@ class IMAPMailbox(object): raise imap4.ReadOnlyMailbox return self.collection.delete_all_flagged() - # FIXME -- get last_uid from mbox_indexer def _bound_seq(self, messages_asked): """ Put an upper bound to a messages sequence if this is open. @@ -477,16 +476,18 @@ class IMAPMailbox(object): :type messages_asked: MessageSet :rtype: MessageSet """ + def set_last(last_uid): + messages_asked.last = last_uid + return messages_asked + if not messages_asked.last: try: iter(messages_asked) except TypeError: # looks like we cannot iterate - try: - # XXX fixme, does not exist - messages_asked.last = self.last_uid - except ValueError: - pass + d = self.collection.get_last_uid() + d.addCallback(set_last) + return d return messages_asked def _filter_msg_seq(self, messages_asked): @@ -524,50 +525,64 @@ class IMAPMailbox(object): otherwise. :type uid: bool - :rtype: deferred + :rtype: deferred with a generator that yields... """ # For the moment our UID is sequential, so we # can treat them all the same. # Change this to the flag that twisted expects when we # switch to content-hash based index + local UID table. - sequence = False - # sequence = True if uid == 0 else False + is_sequence = True if uid == 0 else False getmsg = self.collection.get_message_by_uid + getimapmsg = self.get_imap_message - messages_asked = self._bound_seq(messages_asked) - d_sequence = self._filter_msg_seq(messages_asked) + def get_imap_messages_for_sequence(msg_sequence): + + def _get_imap_msg(messages): + d_imapmsg = [] + for msg in messages: + d_imapmsg.append(getimapmsg(msg)) + return defer.gatherResults(d_imapmsg) - def get_imap_messages_for_sequence(sequence): - def _zip_msgid(messages): - return zip( - list(sequence), - map(self.get_imap_message, messages)) + def _zip_msgid(imap_messages): + zipped = zip( + list(msg_sequence), imap_messages) + return (item for item in zipped) def _unset_recent(sequence): reactor.callLater(0, self.unset_recent_flags, sequence) return sequence d_msg = [] - for msgid in sequence: + for msgid in msg_sequence: d_msg.append(getmsg(msgid)) d = defer.gatherResults(d_msg) + d.addCallback(_get_imap_msg) d.addCallback(_zip_msgid) return d # for sequence numbers (uid = 0) - if sequence: + if is_sequence: logger.debug("Getting msg by index: INEFFICIENT call!") # TODO --- implement sequences in mailbox indexer raise NotImplementedError else: - d_sequence.addCallback(get_imap_messages_for_sequence) + d = self._get_sequence_of_messages(messages_asked) + d.addCallback(get_imap_messages_for_sequence) # TODO -- call signal_to_ui # d.addCallback(self.cb_signal_unread_to_ui) - return d_sequence + return d + + def _get_sequence_of_messages(self, messages_asked): + def get_sequence(messages_asked): + return self._filter_msg_seq(messages_asked) + + d = defer.maybeDeferred(self._bound_seq, messages_asked) + d.addCallback(get_sequence) + return d def fetch_flags(self, messages_asked, uid): """ @@ -611,8 +626,8 @@ class IMAPMailbox(object): :param d: deferred whose callback will be called with result. :type d: Deferred - :rtype: A tuple of two-tuples of message sequence numbers and - flagsPart + :rtype: A generator that yields two-tuples of message sequence numbers + and flagsPart """ class flagsPart(object): def __init__(self, uid, flags): @@ -625,14 +640,30 @@ class IMAPMailbox(object): def getFlags(self): return map(str, self.flags) - messages_asked = self._bound_seq(messages_asked) - seq_messg = self._filter_msg_seq(messages_asked) + def pack_flags(result): + #if result is None: + #print "No result" + #return + _uid, _flags = result + return _uid, flagsPart(_uid, _flags) - # FIXME use deferreds here - all_flags = self.collection.get_all_flags(self.mbox_name) - result = ((msgid, flagsPart( - msgid, all_flags.get(msgid, tuple()))) for msgid in seq_messg) - d.callback(result) + def get_flags_for_seq(sequence): + d_all_flags = [] + for msgid in sequence: + d_flags_per_uid = self.collection.get_flags_by_uid(msgid) + d_flags_per_uid.addCallback(pack_flags) + d_all_flags.append(d_flags_per_uid) + gotflags = defer.gatherResults(d_all_flags) + gotflags.addCallback(get_uid_flag_generator) + return gotflags + + def get_uid_flag_generator(result): + generator = (item for item in result) + d.callback(generator) + + d_seq = self._get_sequence_of_messages(messages_asked) + d_seq.addCallback(get_flags_for_seq) + return d_seq def fetch_headers(self, messages_asked, uid): """ diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 9b00162..d4b5d1f 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -19,6 +19,7 @@ IMAPMessage and IMAPMessageCollection. """ import logging from twisted.mail import imap4 +from twisted.internet import defer from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type @@ -40,11 +41,17 @@ class IMAPMessage(object): implements(imap4.IMessage) - def __init__(self, message): + def __init__(self, message, prefetch_body=True, + store=None, d=defer.Deferred()): """ Initializes a LeapMessage. """ self.message = message + self.__body_fd = None + self.store = store + if prefetch_body: + gotbody = self.__prefetch_body_file() + gotbody.addCallback(lambda _: d.callback(self)) # IMessage implementation @@ -109,14 +116,26 @@ class IMAPMessage(object): # # IMessagePart # + def __prefetch_body_file(self): + def assign_body_fd(fd): + self.__body_fd = fd + return fd + d = self.getBodyFile() + d.addCallback(assign_body_fd) + return d def getBodyFile(self, store=None): """ Retrieve a file object containing only the body of this message. :return: file-like object opened for reading - :rtype: StringIO + :rtype: a deferred that will fire with a StringIO object. """ + if self.__body_fd is not None: + fd = self.__body_fd + fd.seek(0) + return fd + if store is None: store = self.store return self.message.get_body_file(store) diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 32c921d..38a3fd4 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -112,6 +112,7 @@ class LEAPIMAPServer(imap4.IMAP4Server): ebFetch = self._IMAP4Server__ebFetch if len(query) == 1 and str(query[0]) == "flags": + print ">>>>>>>>> fetching flags" self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator maybeDeferred( @@ -121,6 +122,7 @@ class LEAPIMAPServer(imap4.IMAP4Server): ).addErrback(ebFetch, tag) elif len(query) == 1 and str(query[0]) == "rfc822.header": + print ">>>>>>>> fetching headers" self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator maybeDeferred( @@ -129,6 +131,7 @@ class LEAPIMAPServer(imap4.IMAP4Server): cbFetch, tag, query, uid ).addErrback(ebFetch, tag) else: + print ">>>>>>> Fetching other" self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator maybeDeferred( diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 8629d0e..976df5a 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -155,7 +155,7 @@ class Message(object): Get flags for this message. :rtype: tuple """ - return tuple(self._wrapper.fdoc.flags) + return self._wrapper.fdoc.get_flags() def get_internal_date(self): """ @@ -184,6 +184,7 @@ class Message(object): def get_body_file(self, store): """ + Get a file descriptor with the body content. """ def write_and_rewind_if_found(cdoc): if not cdoc: @@ -367,14 +368,36 @@ class MessageCollection(object): d.addCallback(get_msg_from_mdoc_id) return d - # TODO deprecate ??? --- - def _prime_count(self): - def update_count(count): - self._count = count - d = self.mbox_indexer.count(self.mbox_name) - d.addCallback(update_count) + def get_flags_by_uid(self, uid, absolute=True): + if not absolute: + raise NotImplementedError("Does not support relative ids yet") + + def get_flags_from_mdoc_id(doc_id): + if doc_id is None: # XXX needed? or bug? + return None + return self.adaptor.get_flags_from_mdoc_id( + self.store, doc_id) + + def wrap_in_tuple(flags): + return (uid, flags) + + d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_uuid, uid) + d.addCallback(get_flags_from_mdoc_id) + d.addCallback(wrap_in_tuple) return d + # TODO ------------------------------ FIXME FIXME FIXME implement this! + def set_flags(self, *args, **kw): + pass + + # TODO deprecate ??? --- + #def _prime_count(self): + #def update_count(count): + #self._count = count + #d = self.mbox_indexer.count(self.mbox_name) + #d.addCallback(update_count) + #return d + def count(self): """ Count the messages in this collection. @@ -389,11 +412,13 @@ class MessageCollection(object): def count_recent(self): # FIXME HACK - return 0 + # TODO ------------------------ implement this + return 3 def count_unseen(self): # FIXME hack - return 0 + # TODO ------------------------ implement this + return 3 def get_uid_next(self): """ @@ -404,6 +429,12 @@ class MessageCollection(object): """ return self.mbox_indexer.get_next_uid(self.mbox_uuid) + def get_last_uid(self): + """ + Get the last UID for this mailbox. + """ + return self.mbox_indexer.get_last_uid(self.mbox_uuid) + def all_uid_iter(self): """ Iterator through all the uids for this collection. diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 22e57d4..43a1f60 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -305,12 +305,24 @@ class MailboxIndexer(object): return 1 return uid + 1 + d = self.get_last_uid(mailbox_id) + d.addCallback(increment) + return d + + def get_last_uid(self, mailbox_id): + """ + Get the highest UID for a given mailbox. + """ + check_good_uuid(mailbox_id) sql = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( preffix=self.table_preffix, name=sanitize(mailbox_id)) + def getit(result): + return _maybe_first_query_item(result) + d = self._query(sql) - d.addCallback(increment) + d.addCallback(getit) return d def all_uid_iter(self, mailbox_id): -- cgit v1.2.3 From ff57477e5ef84f85985f0531d6d745aa45529145 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 21 Jan 2015 00:50:27 -0400 Subject: imap: implement setting of message flags --- src/leap/mail/imap/mailbox.py | 51 +++++++++++++++++++++++------------------- src/leap/mail/imap/messages.py | 23 ------------------- src/leap/mail/imap/server.py | 6 ----- src/leap/mail/mail.py | 38 +++++++++---------------------- 4 files changed, 39 insertions(+), 79 deletions(-) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index a000133..58fc514 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -231,7 +231,7 @@ class IMAPMailbox(object): """ return self.collection.get_mbox_attr("created") - def getUID(self, message): + def getUID(self, message_number): """ Return the UID of a message in the mailbox @@ -239,15 +239,15 @@ class IMAPMailbox(object): but in the future will be useful to get absolute UIDs from message sequence numbers. - :param message: the message uid + :param message: the message sequence number. :type message: int :rtype: int + :return: the UID of the message. """ - # TODO --- return the uid if it has it!!! - d = self.collection.get_msg_by_uid(message) - d.addCallback(lambda m: m.getUID()) - return d + # TODO support relative sequences. The (imap) message should + # receive a sequence number attribute: a deferred is not expected + return message_number def getUIDNext(self): """ @@ -451,15 +451,6 @@ class IMAPMailbox(object): def _close_cb(self, result): self.closed = True - # TODO server already calls expunge for closing - #def close(self): - #""" - #Expunge and mark as closed - #""" - #d = self.expunge() - #d.addCallback(self._close_cb) - #return d - def expunge(self): """ Remove all messages flagged \\Deleted @@ -641,9 +632,6 @@ class IMAPMailbox(object): return map(str, self.flags) def pack_flags(result): - #if result is None: - #print "No result" - #return _uid, _flags = result return _uid, flagsPart(_uid, _flags) @@ -790,14 +778,31 @@ class IMAPMailbox(object): :type observer: deferred """ # XXX implement also sequence (uid = 0) - # XXX we should prevent client from setting Recent flag? + # TODO we should prevent client from setting Recent flag leap_assert(not isinstance(flags, basestring), "flags cannot be a string") flags = tuple(flags) - messages_asked = self._bound_seq(messages_asked) - seq_messg = self._filter_msg_seq(messages_asked) - self.collection.set_flags( - self.mbox_name, seq_messg, flags, mode, observer) + + def set_flags_for_seq(sequence): + + def return_result_dict(list_of_flags): + result = dict(zip(list(sequence), list_of_flags)) + observer.callback(result) + return result + + d_all_set = [] + for msgid in sequence: + d = self.collection.get_message_by_uid(msgid) + d.addCallback(lambda msg: self.collection.update_flags( + msg, flags, mode)) + d_all_set.append(d) + got_flags_setted = defer.gatherResults(d_all_set) + got_flags_setted.addCallback(return_result_dict) + return got_flags_setted + + d_seq = self._get_sequence_of_messages(messages_asked) + d_seq.addCallback(set_flags_for_seq) + return d_seq # ISearchableMailbox diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index d4b5d1f..df50323 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -73,29 +73,6 @@ class IMAPMessage(object): """ return self.message.get_flags() - # setFlags not in the interface spec but we use it with store command. - - # XXX if we can move it to a collection method, we don't need to pass - # collection to the IMAPMessage - - # lookup method? IMAPMailbox? - - #def setFlags(self, flags, mode): - #""" - #Sets the flags for this message -# - #:param flags: the flags to update in the message. - #:type flags: tuple of str - #:param mode: the mode for setting. 1 is append, -1 is remove, 0 set. - #:type mode: int - #""" - #leap_assert(isinstance(flags, tuple), "flags need to be a tuple") - # XXX - # return new flags - # map to str - #self.message.set_flags(flags, mode) - #self.collection.update_flags(self.message, flags, mode) - def getInternalDate(self): """ Retrieve the date internally associated with this message diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 38a3fd4..f294f42 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -112,7 +112,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): ebFetch = self._IMAP4Server__ebFetch if len(query) == 1 and str(query[0]) == "flags": - print ">>>>>>>>> fetching flags" self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator maybeDeferred( @@ -122,7 +121,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): ).addErrback(ebFetch, tag) elif len(query) == 1 and str(query[0]) == "rfc822.header": - print ">>>>>>>> fetching headers" self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator maybeDeferred( @@ -131,7 +129,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): cbFetch, tag, query, uid ).addErrback(ebFetch, tag) else: - print ">>>>>>> Fetching other" self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator maybeDeferred( @@ -370,7 +367,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): # TODO subscribe method had also to be changed to accomodate deferred def do_SUBSCRIBE(self, tag, name): - print "DOING SUBSCRIBE" name = self._parseMbox(name) def _subscribeCb(_): @@ -433,8 +429,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): def _renameEb(failure): m = failure.value - print "SERVER rename failure!" - print m if failure.check(TypeError): self.sendBadResponse(tag, 'Invalid command syntax') elif failure.check(imap4.MailboxException): diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 976df5a..9b7a562 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -272,8 +272,7 @@ class MessageCollection(object): store = None messageklass = Message - def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None, - count=None): + def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): """ Constructor for a MessageCollection. """ @@ -288,15 +287,6 @@ class MessageCollection(object): # TODO --- review this count shit. I think it's better to patch server # to accept deferreds. - # TODO need to initialize count here because imap server does not - # expect a defered for the count. caller should return the deferred for - # prime_count (ie, initialize) when returning the collection - # TODO should increment and decrement when adding/deleting. - # TODO recent count should also be static. - - if not count: - count = 0 - self._count = count def is_mailbox_collection(self): """ @@ -386,18 +376,6 @@ class MessageCollection(object): d.addCallback(wrap_in_tuple) return d - # TODO ------------------------------ FIXME FIXME FIXME implement this! - def set_flags(self, *args, **kw): - pass - - # TODO deprecate ??? --- - #def _prime_count(self): - #def update_count(count): - #self._count = count - #d = self.mbox_indexer.count(self.mbox_name) - #d.addCallback(update_count) - #return d - def count(self): """ Count the messages in this collection. @@ -479,6 +457,7 @@ class MessageCollection(object): Copy the message to another collection. (it only makes sense for mailbox collections) """ + # TODO currently broken ------------------FIXME- if not self.is_mailbox_collection(): raise NotImplementedError() @@ -555,19 +534,21 @@ class MessageCollection(object): final = new return final - def udpate_flags(self, msg, flags, mode): + def update_flags(self, msg, flags, mode): """ Update flags for a given message. """ wrapper = msg.get_wrapper() current = wrapper.fdoc.flags - newflags = self._update_flags_or_tags(current, flags, mode) + newflags = map(str, self._update_flags_or_tags(current, flags, mode)) wrapper.fdoc.flags = newflags wrapper.fdoc.seen = MessageFlags.SEEN_FLAG in newflags wrapper.fdoc.deleted = MessageFlags.DELETED_FLAG in newflags - return self.adaptor.update_msg(self.store, msg) + d = self.adaptor.update_msg(self.store, msg) + d.addCallback(lambda _: newflags) + return d def update_tags(self, msg, tags, mode): """ @@ -576,8 +557,11 @@ class MessageCollection(object): wrapper = msg.get_wrapper() current = wrapper.fdoc.tags newtags = self._update_flags_or_tags(current, tags, mode) + wrapper.fdoc.tags = newtags - return self.adaptor.update_msg(self.store, msg) + d = self.adaptor.update_msg(self.store, msg) + d.addCallback(newtags) + return d class Account(object): -- cgit v1.2.3 From c3da2530d6cc6c202dac056aa569f7703f0a0963 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 21 Jan 2015 11:52:09 -0400 Subject: rename confusing attribute for account --- src/leap/mail/imap/account.py | 11 ++++++++++- src/leap/mail/imap/server.py | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 146d066..0cf583b 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -61,7 +61,7 @@ class IMAPAccount(object): implements(imap4.IAccount, imap4.INamespacePresenter) selected = None - closed = False + session_ended = False def __init__(self, user_id, store, d=defer.Deferred()): """ @@ -92,6 +92,15 @@ class IMAPAccount(object): return None mbox = IMAPMailbox(collection, rw=readwrite) return mbox + def end_session(self): + """ + Used to mark when the session has closed, and we should not allow any + more commands from the client. + + Right now it's called from the client backend. + """ + # TODO move its use to the service shutdown in leap.mail + self.session_ended = True def callWhenReady(self, cb, *args, **kw): d = self.account.callWhenReady(cb, *args, **kw) diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index f294f42..23ddefc 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -67,7 +67,7 @@ class LEAPIMAPServer(imap4.IMAP4Server): :param line: the line from the server, without the line delimiter. :type line: str """ - if self.theAccount.closed is True and self.state != "unauth": + if self.theAccount.session_ended is True and self.state != "unauth": log.msg("Closing the session. State: unauth") self.state = "unauth" -- cgit v1.2.3 From b67997517d7901cf92eda9d9c68440bb8424e439 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 16 Jan 2015 19:20:37 -0400 Subject: lots of little fixes after meskio's review mostly having to do with poor, missing or outdated documentation, naming of confusing things and reordering of code blocks for improved readability. --- src/leap/mail/adaptors/soledad.py | 85 ++++++++++------------ src/leap/mail/imap/account.py | 82 ++++++++++++---------- src/leap/mail/imap/mailbox.py | 43 +++++++----- src/leap/mail/imap/messages.py | 28 +++++++- src/leap/mail/imap/server.py | 10 ++- src/leap/mail/imap/service/imap.py | 20 +++--- src/leap/mail/imap/tests/test_imap.py | 61 +--------------- src/leap/mail/mail.py | 70 ++++++++++++------- src/leap/mail/mailbox_indexer.py | 128 ++++++++++++++-------------------- src/leap/mail/walk.py | 4 +- 10 files changed, 255 insertions(+), 276 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 46dbe4c..9f0bb30 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -19,7 +19,6 @@ Soledadad MailAdaptor module. import re from collections import defaultdict from email import message_from_string -from functools import partial from pycryptopp.hash import sha256 from twisted.internet import defer @@ -72,6 +71,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): deletion. """ # TODO we could also use a _dirty flag (in models) + # TODO add a get_count() method ??? -- that is extended over u1db. # We keep a dictionary with DeferredLocks, that will be # unique to every subclass of SoledadDocumentWrapper. @@ -86,10 +86,9 @@ class SoledadDocumentWrapper(models.DocumentWrapper): """ return cls._k_locks[cls.__name__] - def __init__(self, **kwargs): - doc_id = kwargs.pop('doc_id', None) + def __init__(self, doc_id=None, future_doc_id=None, **kwargs): self._doc_id = doc_id - self._future_doc_id = kwargs.pop('future_doc_id', None) + self._future_doc_id = future_doc_id self._lock = defer.DeferredLock() super(SoledadDocumentWrapper, self).__init__(**kwargs) @@ -123,7 +122,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): def update_doc_id(doc): self._doc_id = doc.doc_id - self._future_doc_id = None + self.set_future_doc_id(None) return doc if self.future_doc_id is None: @@ -201,6 +200,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): @classmethod def _get_or_create(cls, store, index, value): + # TODO shorten this method. assert store is not None assert index is not None assert value is not None @@ -211,6 +211,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): except AttributeError: raise RuntimeError("The model is badly defined") + # TODO separate into another method? def try_to_get_doc_from_index(indexes): values = [] idx_def = dict(indexes)[index] @@ -323,9 +324,6 @@ class SoledadDocumentWrapper(models.DocumentWrapper): d.addCallback(wrap_docs) return d - # TODO - # [ ] get_count() ??? - def __repr__(self): try: idx = getattr(self, self.model.__meta__.index) @@ -442,29 +440,23 @@ class MessageWrapper(object): integers, beginning at one, and the values are dictionaries with the content of the content-docs. """ - if isinstance(mdoc, SoledadDocument): - mdoc_id = mdoc.doc_id - mdoc = mdoc.content - else: - mdoc_id = None - if not mdoc: - mdoc = {} - self.mdoc = MetaMsgDocWrapper(doc_id=mdoc_id, **mdoc) - - if isinstance(fdoc, SoledadDocument): - fdoc_id = fdoc.doc_id - fdoc = fdoc.content - else: - fdoc_id = None - self.fdoc = FlagsDocWrapper(doc_id=fdoc_id, **fdoc) + + def get_doc_wrapper(doc, cls): + if isinstance(doc, SoledadDocument): + doc_id = doc.doc_id + doc = doc.content + else: + doc_id = None + if not doc: + doc = {} + return cls(doc_id=doc_id, **doc) + + self.mdoc = get_doc_wrapper(mdoc, MetaMsgDocWrapper) + + self.fdoc = get_doc_wrapper(fdoc, FlagsDocWrapper) self.fdoc.set_future_doc_id(self.mdoc.fdoc) - if isinstance(hdoc, SoledadDocument): - hdoc_id = hdoc.doc_id - hdoc = hdoc.content - else: - hdoc_id = None - self.hdoc = HeaderDocWrapper(doc_id=hdoc_id, **hdoc) + self.hdoc = get_doc_wrapper(hdoc, HeaderDocWrapper) self.hdoc.set_future_doc_id(self.mdoc.hdoc) if cdocs is None: @@ -489,10 +481,6 @@ class MessageWrapper(object): "Cannot create: fdoc has a doc_id") # TODO check that the doc_ids in the mdoc are coherent - # TODO I think we need to tolerate the no hdoc.doc_id case, for when we - # are doing a copy to another mailbox. - # leap_assert(self.hdoc.doc_id is None, - # "Cannot create: hdoc has a doc_id") d = [] d.append(self.mdoc.create(store)) d.append(self.fdoc.create(store)) @@ -566,8 +554,9 @@ class MessageWrapper(object): def get_subpart_dict(self, index): """ - :param index: index, 1-indexed + :param index: the part to lookup, 1-indexed :type index: int + :rtype: dict """ return self.hdoc.part_map[str(index)] @@ -785,16 +774,6 @@ class SoledadMailAdaptor(SoledadIndexMixin): assert(MessageClass is not None) return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs), uid=uid) - def _get_msg_from_variable_doc_list(self, doc_list, msg_class, uid=None): - if len(doc_list) == 3: - mdoc, fdoc, hdoc = doc_list - cdocs = None - elif len(doc_list) > 3: - fdoc, hdoc = doc_list[:3] - cdocs = dict(enumerate(doc_list[3:], 1)) - return self.get_msg_from_docs( - msg_class, mdoc, fdoc, hdoc, cdocs, uid=uid) - def get_msg_from_mdoc_id(self, MessageClass, store, mdoc_id, uid=None, get_cdocs=False): @@ -844,10 +823,21 @@ class SoledadMailAdaptor(SoledadIndexMixin): else: d = get_parts_doc_from_mdoc_id() - d.addCallback(partial(self._get_msg_from_variable_doc_list, - msg_class=MessageClass, uid=uid)) + d.addCallback(self._get_msg_from_variable_doc_list, + msg_class=MessageClass, uid=uid) return d + def _get_msg_from_variable_doc_list(self, doc_list, msg_class, uid=None): + if len(doc_list) == 3: + mdoc, fdoc, hdoc = doc_list + cdocs = None + elif len(doc_list) > 3: + # XXX is this case used? + mdoc, fdoc, hdoc = doc_list[:3] + cdocs = dict(enumerate(doc_list[3:], 1)) + return self.get_msg_from_docs( + msg_class, mdoc, fdoc, hdoc, cdocs, uid=uid) + def get_flags_from_mdoc_id(self, store, mdoc_id): """ # XXX stuff here... @@ -875,7 +865,6 @@ class SoledadMailAdaptor(SoledadIndexMixin): def create_msg(self, store, msg): """ :param store: an instance of soledad, or anything that behaves alike - :type store: :param msg: a Message object. :return: a Deferred that is fired when all the underlying documents @@ -889,8 +878,6 @@ class SoledadMailAdaptor(SoledadIndexMixin): """ :param msg: a Message object. :param store: an instance of soledad, or anything that behaves alike - :type store: - :param msg: a Message object. :return: a Deferred that is fired when all the underlying documents have been updated (actually, it's only the fdoc that's allowed to update). diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 0cf583b..38df845 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -49,9 +49,6 @@ if PROFILE_CMD: # Soledad IMAP Account ####################################### -# XXX watchout, account needs to be ready... so we should maybe return -# a deferred to the IMAP service when it's initialized - class IMAPAccount(object): """ An implementation of an imap4 Account @@ -67,8 +64,14 @@ class IMAPAccount(object): """ Keeps track of the mailboxes and subscriptions handled by this account. - :param account: The name of the account (user id). - :type account: str + The account is not ready to be used, since the store needs to be + initialized and we also need to do some initialization routines. + You can either pass a deferred to this constructor, or use + `callWhenReady` method. + + :param user_id: The name of the account (user id, in the form + user@provider). + :type user_id: str :param store: a Soledad instance. :type store: Soledad @@ -87,11 +90,6 @@ class IMAPAccount(object): self.user_id = user_id self.account = Account(store, ready_cb=lambda: d.callback(self)) - def _return_mailbox_from_collection(self, collection, readwrite=1): - if collection is None: - return None - mbox = IMAPMailbox(collection, rw=readwrite) - return mbox def end_session(self): """ Used to mark when the session has closed, and we should not allow any @@ -103,6 +101,12 @@ class IMAPAccount(object): self.session_ended = True def callWhenReady(self, cb, *args, **kw): + """ + Execute callback when the account is ready to be used. + XXX note that this callback will be called with a first ignored + parameter. + """ + # TODO ignore the first parameter and change tests accordingly. d = self.account.callWhenReady(cb, *args, **kw) return d @@ -129,6 +133,12 @@ class IMAPAccount(object): d.addCallback(self._return_mailbox_from_collection) return d + def _return_mailbox_from_collection(self, collection, readwrite=1): + if collection is None: + return None + mbox = IMAPMailbox(collection, rw=readwrite) + return mbox + # # IAccount # @@ -146,7 +156,7 @@ class IMAPAccount(object): :type creation_ts: int :returns: a Deferred that will contain the document if successful. - :rtype: bool + :rtype: defer.Deferred """ name = normalize_mailbox(name) @@ -190,25 +200,15 @@ class IMAPAccount(object): :return: A deferred that will fire with a true value if the creation - succeeds. + succeeds. The deferred might fail with a MailboxException + if the mailbox cannot be added. :rtype: Deferred - :raise MailboxException: Raised if this mailbox cannot be added. """ - paths = filter(None, normalize_mailbox(pathspec).split('/')) - subs = [] - sep = '/' - def pass_on_collision(failure): failure.trap(imap4.MailboxCollision) return True - for accum in range(1, len(paths)): - partial_path = sep.join(paths[:accum]) - d = self.addMailbox(partial_path) - d.addErrback(pass_on_collision) - subs.append(d) - def handle_collision(failure): failure.trap(imap4.MailboxCollision) if not pathspec.endswith('/'): @@ -216,19 +216,26 @@ class IMAPAccount(object): else: return defer.succeed(True) + def all_good(result): + return all(result) + + paths = filter(None, normalize_mailbox(pathspec).split('/')) + subs = [] + sep = '/' + + for accum in range(1, len(paths)): + partial_path = sep.join(paths[:accum]) + d = self.addMailbox(partial_path) + d.addErrback(pass_on_collision) + subs.append(d) + df = self.addMailbox(sep.join(paths)) df.addErrback(handle_collision) subs.append(df) - def all_good(result): - return all(result) - - if subs: - d1 = defer.gatherResults(subs) - d1.addCallback(all_good) - return d1 - else: - return defer.succeed(False) + d1 = defer.gatherResults(subs) + d1.addCallback(all_good) + return d1 def select(self, name, readwrite=1): """ @@ -285,8 +292,7 @@ class IMAPAccount(object): global _mboxes _mboxes = mailboxes if name not in mailboxes: - err = imap4.MailboxException("No such mailbox: %r" % name) - return defer.fail(err) + raise imap4.MailboxException("No such mailbox: %r" % name) def get_mailbox(_): return self.getMailbox(name) @@ -303,10 +309,9 @@ class IMAPAccount(object): # as part of their root. for others in _mboxes: if others != name and others.startswith(name): - err = imap4.MailboxException( + raise imap4.MailboxException( "Hierarchically inferior mailboxes " "exist and \\Noselect is set") - return defer.fail(err) return mbox d = self.account.list_all_mailbox_names() @@ -324,8 +329,6 @@ class IMAPAccount(object): # if self._inferiorNames(name) > 1: # self._index.removeMailbox(name) - # TODO use mail.Account.rename_mailbox - # TODO finish conversion to deferreds def rename(self, oldname, newname): """ Renames a mailbox. @@ -460,6 +463,9 @@ class IMAPAccount(object): :type name: str :rtype: Deferred """ + # TODO should raise MailboxException if attempted to unsubscribe + # from a mailbox that is not currently subscribed. + # TODO factor out with subscribe method. name = normalize_mailbox(name) def set_unsubscribed(mbox): diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 58fc514..52f4dd5 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -91,8 +91,9 @@ class IMAPMailbox(object): implements( imap4.IMailbox, imap4.IMailboxInfo, - #imap4.ICloseableMailbox, imap4.ISearchableMailbox, + # XXX I think we do not need to implement CloseableMailbox, do we? + # imap4.ICloseableMailbox imap4.IMessageCopier) init_flags = INIT_FLAGS @@ -108,8 +109,6 @@ class IMAPMailbox(object): def __init__(self, collection, rw=1): """ - SoledadMailbox constructor. - :param collection: instance of IMAPMessageCollection :type collection: IMAPMessageCollection @@ -117,14 +116,10 @@ class IMAPMailbox(object): :type rw: int """ self.rw = rw - self.closed = False self._uidvalidity = None self.collection = collection - if not self.getFlags(): - self.setFlags(self.init_flags) - @property def mbox_name(self): return self.collection.mbox_name @@ -201,6 +196,7 @@ class IMAPMailbox(object): "flags expected to be a tuple") return self.collection.set_mbox_attr("flags", flags) + # TODO - not used? @property def is_closed(self): """ @@ -211,6 +207,7 @@ class IMAPMailbox(object): """ return self.collection.get_mbox_attr("closed") + # TODO - not used? def set_closed(self, closed): """ Set the closed attribute for this mailbox. @@ -448,9 +445,6 @@ class IMAPMailbox(object): d.addCallback(remove_mbox) return d - def _close_cb(self, result): - self.closed = True - def expunge(self): """ Remove all messages flagged \\Deleted @@ -555,24 +549,23 @@ class IMAPMailbox(object): # for sequence numbers (uid = 0) if is_sequence: - logger.debug("Getting msg by index: INEFFICIENT call!") # TODO --- implement sequences in mailbox indexer raise NotImplementedError else: - d = self._get_sequence_of_messages(messages_asked) + d = self._get_messages_range(messages_asked) d.addCallback(get_imap_messages_for_sequence) # TODO -- call signal_to_ui # d.addCallback(self.cb_signal_unread_to_ui) return d - def _get_sequence_of_messages(self, messages_asked): - def get_sequence(messages_asked): + def _get_messages_range(self, messages_asked): + def get_range(messages_asked): return self._filter_msg_seq(messages_asked) d = defer.maybeDeferred(self._bound_seq, messages_asked) - d.addCallback(get_sequence) + d.addCallback(get_range) return d def fetch_flags(self, messages_asked, uid): @@ -599,6 +592,10 @@ class IMAPMailbox(object): MessagePart. :rtype: tuple """ + is_sequence = True if uid == 0 else False + if is_sequence: + raise NotImplementedError + d = defer.Deferred() reactor.callLater(0, self._do_fetch_flags, messages_asked, uid, d) if PROFILE_CMD: @@ -649,7 +646,7 @@ class IMAPMailbox(object): generator = (item for item in result) d.callback(generator) - d_seq = self._get_sequence_of_messages(messages_asked) + d_seq = self._get_messages_range(messages_asked) d_seq.addCallback(get_flags_for_seq) return d_seq @@ -677,7 +674,11 @@ class IMAPMailbox(object): MessagePart. :rtype: tuple """ + # TODO implement sequences # TODO how often is thunderbird doing this? + is_sequence = True if uid == 0 else False + if is_sequence: + raise NotImplementedError class headersPart(object): def __init__(self, uid, headers): @@ -753,6 +754,12 @@ class IMAPMailbox(object): :raise ReadOnlyMailbox: Raised if this mailbox is not open for read-write. """ + # TODO implement sequences + # TODO how often is thunderbird doing this? + is_sequence = True if uid == 0 else False + if is_sequence: + raise NotImplementedError + if not self.isWriteable(): log.msg('read only mailbox!') raise imap4.ReadOnlyMailbox @@ -777,7 +784,7 @@ class IMAPMailbox(object): done. :type observer: deferred """ - # XXX implement also sequence (uid = 0) + # TODO implement also sequence (uid = 0) # TODO we should prevent client from setting Recent flag leap_assert(not isinstance(flags, basestring), "flags cannot be a string") @@ -800,7 +807,7 @@ class IMAPMailbox(object): got_flags_setted.addCallback(return_result_dict) return got_flags_setted - d_seq = self._get_sequence_of_messages(messages_asked) + d_seq = self._get_messages_range(messages_asked) d_seq.addCallback(set_flags_for_seq) return d_seq diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index df50323..8f4c953 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -36,16 +36,38 @@ logger = logging.getLogger(__name__) class IMAPMessage(object): """ - The main representation of a message. + The main representation of a message as seen by the IMAP Server. + This class implements the semantics specific to IMAP specification. """ - implements(imap4.IMessage) def __init__(self, message, prefetch_body=True, store=None, d=defer.Deferred()): """ - Initializes a LeapMessage. + Get an IMAPMessage. A mail.Message is needed, since many of the methods + are proxied to that object. + + + If you do not need to prefetch the body of the message, you can set + `prefetch_body` to False, but the current imap server implementation + expect the getBodyFile method to return inmediately. + + When the prefetch_body option is used, a deferred is also expected as a + parameter, and this will fire when the deferred initialization has + taken place, with this instance of IMAPMessage as a parameter. + + :param message: the abstract message + :type message: mail.Message + :param prefetch_body: Whether to prefetch the content doc for the body. + :type prefetch_body: bool + :param store: an instance of soledad, or anything that behaves like it. + :param d: an optional deferred, that will be fired with the instance of + the IMAPMessage being initialized + :type d: defer.Deferred """ + # TODO substitute the use of the deferred initialization by a factory + # function, maybe. + self.message = message self.__body_fd = None self.store = store diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 23ddefc..027fd7a 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -500,13 +500,18 @@ class LEAPIMAPServer(imap4.IMAP4Server): select_DELETE = auth_DELETE # Need to override the command table after patching - # arg_astring and arg_literal + # arg_astring and arg_literal, except on the methods that we are already + # overriding. + # TODO -------------------------------------------- + # Check if we really need to override these + # methods, or we can monkeypatch. # do_DELETE = imap4.IMAP4Server.do_DELETE # do_CREATE = imap4.IMAP4Server.do_CREATE # do_RENAME = imap4.IMAP4Server.do_RENAME # do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE # do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE + # ------------------------------------------------- do_LOGIN = imap4.IMAP4Server.do_LOGIN do_STATUS = imap4.IMAP4Server.do_STATUS do_APPEND = imap4.IMAP4Server.do_APPEND @@ -530,8 +535,11 @@ class LEAPIMAPServer(imap4.IMAP4Server): auth_EXAMINE = (_selectWork, arg_astring, 0, 'EXAMINE') select_EXAMINE = auth_EXAMINE + # TODO ----------------------------------------------- + # re-add if we stop overriding DELETE # auth_DELETE = (do_DELETE, arg_astring) # select_DELETE = auth_DELETE + # ---------------------------------------------------- auth_RENAME = (do_RENAME, arg_astring, arg_astring) select_RENAME = auth_RENAME diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 93e4d62..cc76e3a 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -17,6 +17,7 @@ """ IMAP service initialization """ +# TODO: leave only an implementor of IService in here import logging import os @@ -29,10 +30,9 @@ from twisted.python import log logger = logging.getLogger(__name__) from leap.common import events as leap_events -from leap.common.check import leap_assert, leap_assert_type, leap_check +from leap.common.check import leap_assert_type, leap_check from leap.mail.imap.account import IMAPAccount from leap.mail.imap.server import LEAPIMAPServer -from leap.mail.incoming import IncomingMail from leap.soledad.client import Soledad from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED @@ -113,6 +113,10 @@ class LeapIMAPFactory(ServerFactory): """ Stops imap service (fetcher, factory and port). """ + # mark account as unusable, so any imap command will fail + # with unauth state. + self.theAccount.end_session() + # TODO should wait for all the pending deferreds, # the twisted way! if DO_PROFILE: @@ -123,23 +127,23 @@ class LeapIMAPFactory(ServerFactory): return ServerFactory.doStop(self) -def run_service(*args, **kwargs): +def run_service(store, **kwargs): """ Main entry point to run the service from the client. + :param store: a soledad instance + :returns: the port as returned by the reactor when starts listening, and the factory for the protocol. """ - leap_assert(len(args) == 2) - soledad = args - leap_assert_type(soledad, Soledad) + leap_assert_type(store, Soledad) port = kwargs.get('port', IMAP_PORT) userid = kwargs.get('userid', None) leap_check(userid is not None, "need an user id") - uuid = soledad.uuid - factory = LeapIMAPFactory(uuid, userid, soledad) + uuid = store.uuid + factory = LeapIMAPFactory(uuid, userid, store) try: tport = reactor.listenTCP(port, factory, diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 6be41cd..67a24cd 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -75,6 +75,7 @@ class TestRealm: # TestCases # +# TODO rename to IMAPMessageCollection class MessageCollectionTestCase(IMAP4HelperMixin): """ Tests for the MessageCollection class @@ -87,6 +88,7 @@ class MessageCollectionTestCase(IMAP4HelperMixin): We override mixin method since we are only testing MessageCollection interface in this particular TestCase """ + # FIXME -- return deferred super(MessageCollectionTestCase, self).setUp() # FIXME --- update initialization @@ -1090,64 +1092,6 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): # Okay, that was much fun indeed - # skipping close test: we just need expunge for now. - #def testClose(self): - #""" - #Test closing the mailbox. We expect to get deleted all messages flagged - #as such. - #""" - #acc = self.server.theAccount - #mailbox_name = 'mailbox-close' -# - #def add_mailbox(): - #return acc.addMailbox(mailbox_name) -# - #def login(): - #return self.client.login(TEST_USER, TEST_PASSWD) -# - #def select(): - #return self.client.select(mailbox_name) -# - #def get_mailbox(): - #def _save_mbox(mailbox): - #self.mailbox = mailbox - #d = self.server.theAccount.getMailbox(mailbox_name) - #d.addCallback(_save_mbox) - #return d -# - #def add_messages(): - #d1 = self.mailbox.addMessage( - #'test 1', flags=('\\Deleted', 'AnotherFlag')) - #d2 = self.mailbox.addMessage( - #'test 2', flags=('AnotherFlag',)) - #d3 = self.mailbox.addMessage( - #'test 3', flags=('\\Deleted',)) - #d = defer.gatherResults([d1, d2, d3]) - #return d -# - #def close(): - #return self.client.close() -# - #d = self.connected.addCallback(strip(add_mailbox)) - #d.addCallback(strip(login)) - #d.addCallbacks(strip(select), self._ebGeneral) - #d.addCallback(strip(get_mailbox)) - #d.addCallbacks(strip(add_messages), self._ebGeneral) - #d.addCallbacks(strip(close), self._ebGeneral) - #d.addCallbacks(self._cbStopClient, self._ebGeneral) - #d2 = self.loopback() - #d1 = defer.gatherResults([d, d2]) - #d1.addCallback(lambda _: self.mailbox.getMessageCount()) - #d1.addCallback(self._cbTestClose) - #return d1 -# - #def _cbTestClose(self, count): - # TODO is this correct? count should not take into account those - # flagged as deleted??? - #self.assertEqual(count, 1) - # TODO --- assert flags are those of the message #2 - #self.failUnless(self.mailbox.closed) - def testExpunge(self): """ Test expunge command @@ -1209,6 +1153,7 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): self.assertItemsEqual(self.results, [1, 3]) +# TODO -------- Fix this testcase class AccountTestCase(IMAP4HelperMixin): """ Test the Account. diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 9b7a562..59fd57c 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -58,9 +58,16 @@ def _write_and_rewind(payload): class MessagePart(object): - # TODO pass cdocs in init - def __init__(self, part_map, cdocs={}): + """ + :param part_map: a dictionary mapping the subparts for + this MessagePart (1-indexed). + :type part_map: dict + :param cdoc: optional, a dict of content documents + """ + # TODO document the expected keys in the part_map dict. + # TODO add abstraction layer between the cdocs and this class. Only + # adaptor should know about the format of the cdocs. self._pmap = part_map self._cdocs = cdocs @@ -266,6 +273,10 @@ class MessageCollection(object): # [ ] To guarantee synchronicity of the documents sent together during a # sync, we could get hold of a deferredLock that inhibits # synchronization while we are updating (think more about this!) + # [ ] review the serveral count_ methods. I think it's better to patch + # server to accept deferreds. + # [ ] Use inheritance for the mailbox-collection instead of handling the + # special cases everywhere? # Account should provide an adaptor instance when creating this collection. adaptor = None @@ -285,9 +296,6 @@ class MessageCollection(object): self.mbox_indexer = mbox_indexer self.mbox_wrapper = mbox_wrapper - # TODO --- review this count shit. I think it's better to patch server - # to accept deferreds. - def is_mailbox_collection(self): """ Return True if this collection represents a Mailbox. @@ -297,22 +305,26 @@ class MessageCollection(object): @property def mbox_name(self): - wrapper = getattr(self, "mbox_wrapper", None) - if not wrapper: + # TODO raise instead? + if self.mbox_wrapper is None: return None - return wrapper.mbox + return self.mbox_wrapper.mbox @property def mbox_uuid(self): - wrapper = getattr(self, "mbox_wrapper", None) - if not wrapper: + # TODO raise instead? + if self.mbox_wrapper is None: return None - return wrapper.uuid + return self.mbox_wrapper.uuid def get_mbox_attr(self, attr): + if self.mbox_wrapper is None: + raise RuntimeError("This is not a mailbox collection") return getattr(self.mbox_wrapper, attr) def set_mbox_attr(self, attr, value): + if self.mbox_wrapper is None: + raise RuntimeError("This is not a mailbox collection") setattr(self.mbox_wrapper, attr, value) return self.mbox_wrapper.update(self.store) @@ -323,10 +335,10 @@ class MessageCollection(object): Retrieve a message by its content hash. :rtype: Deferred """ - if not self.is_mailbox_collection(): - # instead of getting the metamsg by chash, query by (meta) index - # or use the internal collection of pointers-to-docs. + # TODO instead of getting the metamsg by chash, in this case we + # should query by (meta) index or use the internal collection of + # pointers-to-docs. raise NotImplementedError() metamsg_id = _get_mdoc_id(self.mbox_name, chash) @@ -462,6 +474,9 @@ class MessageCollection(object): raise NotImplementedError() def insert_copied_mdoc_id(wrapper): + # TODO this needs to be implemented before the copy + # interface works. + newmailbox_uuid = get_mbox_uuid_from_msg_wrapper(wrapper) return self.mbox_indexer.insert_doc( newmailbox_uuid, wrapper.mdoc.doc_id) @@ -525,15 +540,6 @@ class MessageCollection(object): d.addCallback(del_all_uid) return d - def _update_flags_or_tags(self, old, new, mode): - if mode == Flagsmode.APPEND: - final = list((set(tuple(old) + new))) - elif mode == Flagsmode.REMOVE: - final = list(set(old).difference(set(new))) - elif mode == Flagsmode.SET: - final = new - return final - def update_flags(self, msg, flags, mode): """ Update flags for a given message. @@ -563,6 +569,15 @@ class MessageCollection(object): d.addCallback(newtags) return d + def _update_flags_or_tags(self, old, new, mode): + if mode == Flagsmode.APPEND: + final = list((set(tuple(old) + new))) + elif mode == Flagsmode.REMOVE: + final = list(set(old).difference(set(new))) + elif mode == Flagsmode.SET: + final = new + return final + class Account(object): """ @@ -573,7 +588,7 @@ class Account(object): basic collection handled by traditional MUAs, but it can also handle other types of Collections (tag based, for instance). - leap.mail.imap.SoledadBackedAccount partially proxies methods in this + leap.mail.imap.IMAPAccount partially proxies methods in this class. """ @@ -582,7 +597,6 @@ class Account(object): # the Account class. adaptor_class = SoledadMailAdaptor - store = None def __init__(self, store, ready_cb=None): self.store = store @@ -612,6 +626,12 @@ class Account(object): return d def callWhenReady(self, cb, *args, **kw): + """ + Execute the callback when the initialization of the Account is ready. + Note that the callback will receive a first meaningless parameter. + """ + # TODO this should ignore the first parameter explicitely + # lambda _: cb(*args, **kw) self.deferred_initialization.addCallback(cb, *args, **kw) return self.deferred_initialization diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 43a1f60..4eb0fa8 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -38,23 +38,23 @@ class WrongMetaDocIDError(Exception): pass -def sanitize(mailbox_id): - return mailbox_id.replace("-", "_") +def sanitize(mailbox_uuid): + return mailbox_uuid.replace("-", "_") -def check_good_uuid(mailbox_id): +def check_good_uuid(mailbox_uuid): """ Check that the passed mailbox identifier is a valid UUID. - :param mailbox_id: the uuid to check - :type mailbox_id: str + :param mailbox_uuid: the uuid to check + :type mailbox_uuid: str :return: None :raises: AssertionError if a wrong uuid was passed. """ try: - uuid.UUID(str(mailbox_id)) + uuid.UUID(str(mailbox_uuid)) except (AttributeError, ValueError): raise AssertionError( - "the mbox_id is not a valid uuid: %s" % mailbox_id) + "the mbox_id is not a valid uuid: %s" % mailbox_uuid) class MailboxIndexer(object): @@ -88,51 +88,33 @@ class MailboxIndexer(object): assert self.store is not None return self.store.raw_sqlcipher_query(*args, **kw) - def create_table(self, mailbox_id): + def create_table(self, mailbox_uuid): """ Create the UID table for a given mailbox. :param mailbox: the mailbox identifier. :type mailbox: str :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) sql = ("CREATE TABLE if not exists {preffix}{name}( " "uid INTEGER PRIMARY KEY AUTOINCREMENT, " "hash TEXT UNIQUE NOT NULL)".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) return self._query(sql) - def delete_table(self, mailbox_id): + def delete_table(self, mailbox_uuid): """ Delete the UID table for a given mailbox. :param mailbox: the mailbox name :type mailbox: str :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) sql = ("DROP TABLE if exists {preffix}{name}".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) return self._query(sql) - def rename_table(self, oldmailbox, newmailbox): - """ - Delete the UID table for a given mailbox. - :param oldmailbox: the old mailbox name - :type oldmailbox: str - :param newmailbox: the new mailbox name - :type newmailbox: str - :rtype: Deferred - """ - assert oldmailbox - assert newmailbox - assert oldmailbox != newmailbox - sql = ("ALTER TABLE {preffix}{old} " - "RENAME TO {preffix}{new}".format( - preffix=self.table_preffix, - old=sanitize(oldmailbox), new=sanitize(newmailbox))) - return self._query(sql) - - def insert_doc(self, mailbox_id, doc_id): + def insert_doc(self, mailbox_uuid, doc_id): """ Insert the doc_id for a MetaMsg in the UID table for a given mailbox. @@ -148,11 +130,11 @@ class MailboxIndexer(object): document. :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) assert doc_id - mailbox_id = mailbox_id.replace('-', '_') + mailbox_uuid = mailbox_uuid.replace('-', '_') - if not re.findall(METAMSGID_RE.format(mbox_uuid=mailbox_id), doc_id): + if not re.findall(METAMSGID_RE.format(mbox_uuid=mailbox_uuid), doc_id): raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") def get_rowid(result): @@ -160,12 +142,12 @@ class MailboxIndexer(object): sql = ("INSERT INTO {preffix}{name} VALUES (" "NULL, ?)".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) values = (doc_id,) sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( - preffix=self.table_preffix, name=sanitize(mailbox_id)) + preffix=self.table_preffix, name=sanitize(mailbox_uuid)) d = self._query(sql, values) d.addCallback(lambda _: self._query(sql_last)) @@ -173,25 +155,25 @@ class MailboxIndexer(object): d.addErrback(lambda f: f.printTraceback()) return d - def delete_doc_by_uid(self, mailbox_id, uid): + def delete_doc_by_uid(self, mailbox_uuid, uid): """ Delete the entry for a MetaMsg in the UID table for a given mailbox. - :param mailbox_id: the mailbox uuid + :param mailbox_uuid: the mailbox uuid :type mailbox: str :param uid: the UID of the message. :type uid: int :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) assert uid sql = ("DELETE FROM {preffix}{name} " "WHERE uid=?".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) values = (uid,) return self._query(sql, values) - def delete_doc_by_hash(self, mailbox_id, doc_id): + def delete_doc_by_hash(self, mailbox_uuid, doc_id): """ Delete the entry for a MetaMsg in the UID table for a given mailbox. @@ -199,7 +181,7 @@ class MailboxIndexer(object): M-- - :param mailbox_id: the mailbox uuid + :param mailbox_uuid: the mailbox uuid :type mailbox: str :param doc_id: the doc_id for the MetaMsg :type doc_id: str @@ -207,82 +189,80 @@ class MailboxIndexer(object): document. :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) assert doc_id sql = ("DELETE FROM {preffix}{name} " "WHERE hash=?".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) values = (doc_id,) return self._query(sql, values) - def get_doc_id_from_uid(self, mailbox_id, uid): + def get_doc_id_from_uid(self, mailbox_uuid, uid): """ Get the doc_id for a MetaMsg in the UID table for a given mailbox. - :param mailbox_id: the mailbox uuid + :param mailbox_uuid: the mailbox uuid :type mailbox: str :param uid: the uid for the MetaMsg for this mailbox :type uid: int :rtype: Deferred """ - check_good_uuid(mailbox_id) - mailbox_id = mailbox_id.replace('-', '_') + check_good_uuid(mailbox_uuid) + mailbox_uuid = mailbox_uuid.replace('-', '_') def get_hash(result): return _maybe_first_query_item(result) sql = ("SELECT hash from {preffix}{name} " "WHERE uid=?".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) values = (uid,) d = self._query(sql, values) d.addCallback(get_hash) return d - def get_uid_from_doc_id(self, mailbox_id, doc_id): - check_good_uuid(mailbox_id) - mailbox_id = mailbox_id.replace('-', '_') + def get_uid_from_doc_id(self, mailbox_uuid, doc_id): + check_good_uuid(mailbox_uuid) + mailbox_uuid = mailbox_uuid.replace('-', '_') def get_uid(result): return _maybe_first_query_item(result) sql = ("SELECT uid from {preffix}{name} " "WHERE hash=?".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) values = (doc_id,) d = self._query(sql, values) d.addCallback(get_uid) return d - - - def get_doc_ids_from_uids(self, mailbox_id, uids): + def get_doc_ids_from_uids(self, mailbox_uuid, uids): # For IMAP relative numbering /sequences. # XXX dereference the range (n,*) raise NotImplementedError() - def count(self, mailbox_id): + def count(self, mailbox_uuid): """ Get the number of entries in the UID table for a given mailbox. - :param mailbox_id: the mailbox uuid - :type mailbox_id: str + :param mailbox_uuid: the mailbox uuid + :type mailbox_uuid: str :return: a deferred that will fire with an integer returning the count. :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) def get_count(result): return _maybe_first_query_item(result) sql = ("SELECT Count(*) FROM {preffix}{name};".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) d = self._query(sql) d.addCallback(get_count) d.addErrback(lambda _: 0) return d - def get_next_uid(self, mailbox_id): + def get_next_uid(self, mailbox_uuid): """ Get the next integer beyond the highest UID count for a given mailbox. @@ -291,13 +271,13 @@ class MailboxIndexer(object): only thing that can be assured is that it will be equal or greater than the value returned. - :param mailbox_id: the mailbox uuid + :param mailbox_uuid: the mailbox uuid :type mailbox: str :return: a deferred that will fire with an integer returning the next uid. :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) def increment(result): uid = _maybe_first_query_item(result) @@ -305,18 +285,18 @@ class MailboxIndexer(object): return 1 return uid + 1 - d = self.get_last_uid(mailbox_id) + d = self.get_last_uid(mailbox_uuid) d.addCallback(increment) return d - def get_last_uid(self, mailbox_id): + def get_last_uid(self, mailbox_uuid): """ Get the highest UID for a given mailbox. """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) sql = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( - preffix=self.table_preffix, name=sanitize(mailbox_id)) + preffix=self.table_preffix, name=sanitize(mailbox_uuid)) def getit(result): return _maybe_first_query_item(result) @@ -325,17 +305,17 @@ class MailboxIndexer(object): d.addCallback(getit) return d - def all_uid_iter(self, mailbox_id): + def all_uid_iter(self, mailbox_uuid): """ Get a sequence of all the uids in this mailbox. - :param mailbox_id: the mailbox uuid - :type mailbox_id: str + :param mailbox_uuid: the mailbox uuid + :type mailbox_uuid: str """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) sql = ("SELECT uid from {preffix}{name} ").format( - preffix=self.table_preffix, name=sanitize(mailbox_id)) + preffix=self.table_preffix, name=sanitize(mailbox_uuid)) def get_results(result): return [x[0] for x in result] diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 8653a5f..891abdc 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -122,7 +122,7 @@ def walk_msg_tree(parts, body_phash=None): documents that will be stored in Soledad. It walks down the subparts in the parsed message tree, and collapses - the leaf docuents into a wrapper document until no multipart submessages + the leaf documents into a wrapper document until no multipart submessages are left. To achieve this, it iteratively calculates a wrapper vector of all documents in the sequence that have more than one part and have unitary documents to their right. To collapse a multipart, take as many @@ -171,7 +171,7 @@ def walk_msg_tree(parts, body_phash=None): HEADERS: dict(parts[wind][HEADERS]) } - # remove subparts and substitue wrapper + # remove subparts and substitute wrapper map(lambda i: parts.remove(i), slic) parts[wind] = cwra -- cgit v1.2.3 From f8c07f66bfcd97e3ae7085071c1f1efbd80e0286 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 22 Jan 2015 00:47:42 -0400 Subject: fix typo --- src/leap/mail/smtp/gateway.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index 222ef3f..1a187cf 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -219,7 +219,7 @@ class SMTPDelivery(object): d = self._km.get_key(address, OpenPGPKey) # might raise KeyNotFound d.addCallbacks(found, not_found) - d.addCallbac(lambda _: EncryptedMessage(user, self._outgoing_mail)) + d.addCallback(lambda _: EncryptedMessage(user, self._outgoing_mail)) return d def validateFrom(self, helo, origin): @@ -306,4 +306,6 @@ class EncryptedMessage(object): log.err() signal(proto.SMTP_CONNECTION_LOST, self._user.dest.addrstr) # unexpected loss of connection; don't save + + self._lines = [] -- cgit v1.2.3 From 4c02942472be00486d04d854e987d6eda8c7df12 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 22 Jan 2015 03:16:32 -0400 Subject: re-add support for basic multipart messages --- src/leap/mail/adaptors/soledad.py | 8 ++- src/leap/mail/imap/mailbox.py | 6 +- src/leap/mail/imap/messages.py | 144 ++++++++++++++++++++------------------ src/leap/mail/mail.py | 102 ++++++++++++++++----------- 4 files changed, 147 insertions(+), 113 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 9f0bb30..d21638c 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -463,8 +463,9 @@ class MessageWrapper(object): cdocs = {} cdocs_keys = cdocs.keys() assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1) - self.cdocs = dict([(key, ContentDocWrapper(**doc)) for (key, doc) in - cdocs.items()]) + self.cdocs = dict([ + (key, ContentDocWrapper(**doc.content)) + for (key, doc) in cdocs.items()]) for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()): cdoc.set_future_doc_id(doc_id) @@ -560,6 +561,9 @@ class MessageWrapper(object): """ return self.hdoc.part_map[str(index)] + def get_subpart_indexes(self): + return self.hdoc.part_map.keys() + def get_body(self, store): """ :rtype: deferred diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 52f4dd5..045636e 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -540,7 +540,11 @@ class IMAPMailbox(object): d_msg = [] for msgid in msg_sequence: - d_msg.append(getmsg(msgid)) + # XXX We want cdocs because we "probably" are asked for the + # body. We should be smarted at do_FETCH and pass a parameter + # to this method in order not to prefetch cdocs if they're not + # going to be used. + d_msg.append(getmsg(msgid, get_cdocs=True)) d = defer.gatherResults(d_msg) d.addCallback(_get_imap_msg) diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 8f4c953..b7bb6ee 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -115,13 +115,6 @@ class IMAPMessage(object): # # IMessagePart # - def __prefetch_body_file(self): - def assign_body_fd(fd): - self.__body_fd = fd - return fd - d = self.getBodyFile() - d.addCallback(assign_body_fd) - return d def getBodyFile(self, store=None): """ @@ -139,25 +132,6 @@ class IMAPMessage(object): store = self.store return self.message.get_body_file(store) - # TODO refactor with getBodyFile in MessagePart - - #body = bdoc_content.get(self.RAW_KEY, "") - #content_type = bdoc_content.get('content-type', "") - #charset = find_charset(content_type) - #if charset is None: - #charset = self._get_charset(body) - #try: - #if isinstance(body, unicode): - #body = body.encode(charset) - #except UnicodeError as exc: - #logger.error( - #"Unicode error, using 'replace'. {0!r}".format(exc)) - #logger.debug("Attempted to encode with: %s" % charset) - #body = body.encode(charset, 'replace') - #finally: - #return write_fd(body) - - def getSize(self): """ Return the total size, in octets, of this message. @@ -182,48 +156,8 @@ class IMAPMessage(object): :return: A mapping of header field names to header field values :rtype: dict """ - # TODO split in smaller methods -- format_headers()? - # XXX refactor together with MessagePart method - headers = self.message.get_headers() - - # XXX keep this in the imap imessage implementation, - # because the server impl. expects content-type to be present. - if not headers: - logger.warning("No headers found") - return {str('content-type'): str('')} - - names = map(lambda s: s.upper(), names) - if negate: - cond = lambda key: key.upper() not in names - else: - cond = lambda key: key.upper() in names - - if isinstance(headers, list): - headers = dict(headers) - - # default to most likely standard - charset = find_charset(headers, "utf-8") - headers2 = dict() - for key, value in headers.items(): - # twisted imap server expects *some* headers to be lowercase - # We could use a CaseInsensitiveDict here... - if key.lower() == "content-type": - key = key.lower() - - if not isinstance(key, str): - key = key.encode(charset, 'replace') - if not isinstance(value, str): - value = value.encode(charset, 'replace') - - if value.endswith(";"): - # bastards - value = value[:-1] - - # filter original dict by negate-condition - if cond(key): - headers2[key] = value - return headers2 + return _format_headers(headers, negate, *names) def isMultipart(self): """ @@ -242,7 +176,81 @@ class IMAPMessage(object): :rtype: Any object implementing C{IMessagePart}. :return: The specified sub-part. """ - return self.message.get_subpart(part) + subpart = self.message.get_subpart(part) + return IMAPMessagePart(subpart) + + def __prefetch_body_file(self): + def assign_body_fd(fd): + self.__body_fd = fd + return fd + d = self.getBodyFile() + d.addCallback(assign_body_fd) + return d + + +class IMAPMessagePart(object): + + def __init__(self, message_part): + self.message_part = message_part + + def getBodyFile(self, store=None): + return self.message_part.get_body_file() + + def getSize(self): + return self.message_part.get_size() + + def getHeaders(self, negate, *names): + headers = self.message_part.get_headers() + return _format_headers(headers, negate, *names) + + def isMultipart(self): + return self.message_part.is_multipart() + + def getSubPart(self, part): + subpart = self.message_part.get_subpart(part) + return IMAPMessagePart(subpart) + + +def _format_headers(headers, negate, *names): + # current server impl. expects content-type to be present, so if for + # some reason we do not have headers, we have to return at least that + # one + if not headers: + logger.warning("No headers found") + return {str('content-type'): str('')} + + names = map(lambda s: s.upper(), names) + if negate: + cond = lambda key: key.upper() not in names + else: + cond = lambda key: key.upper() in names + + if isinstance(headers, list): + headers = dict(headers) + + # default to most likely standard + charset = find_charset(headers, "utf-8") + + _headers = dict() + for key, value in headers.items(): + # twisted imap server expects *some* headers to be lowercase + # We could use a CaseInsensitiveDict here... + if key.lower() == "content-type": + key = key.lower() + + if not isinstance(key, str): + key = key.encode(charset, 'replace') + if not isinstance(value, str): + value = value.encode(charset, 'replace') + + if value.endswith(";"): + # bastards + value = value[:-1] + + # filter original dict by negate-condition + if cond(key): + _headers[key] = value + return _headers class IMAPMessageCollection(object): diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 59fd57c..aa499c0 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -22,6 +22,7 @@ import logging import StringIO from twisted.internet import defer +from twisted.python import log from leap.common.check import leap_assert_type from leap.common.mail import get_email_charset @@ -30,7 +31,7 @@ from leap.mail.adaptors.soledad import SoledadMailAdaptor from leap.mail.constants import INBOX_NAME from leap.mail.constants import MessageFlags from leap.mail.mailbox_indexer import MailboxIndexer -from leap.mail.utils import empty, find_charset +from leap.mail.utils import empty # find_charset logger = logging.getLogger(name=__name__) @@ -57,61 +58,57 @@ def _write_and_rewind(payload): class MessagePart(object): + # TODO This class should be better abstracted from the data model. + # TODO support arbitrarily nested multiparts (right now we only support + # the trivial case) - def __init__(self, part_map, cdocs={}): + def __init__(self, part_map, index=1, cdocs={}): """ :param part_map: a dictionary mapping the subparts for this MessagePart (1-indexed). :type part_map: dict - :param cdoc: optional, a dict of content documents + + The format for the part_map is as follows: + + {u'1': {u'ctype': u'text/plain', + u'headers': [[u'Content-Type', u'text/plain; charset="utf-8"'], + [u'Content-Transfer-Encoding', u'8bit']], + u'multi': False, + u'parts': 1, + u'phash': u'02D82B29F6BB0C8612D1C', + u'size': 132}} + + :param index: which index in the content-doc is this subpart + representing. + :param cdocs: optional, a reference to the top-level dict of wrappers + for content-docs (1-indexed). """ - # TODO document the expected keys in the part_map dict. - # TODO add abstraction layer between the cdocs and this class. Only - # adaptor should know about the format of the cdocs. + # TODO: Pass only the cdoc wrapper for this part. self._pmap = part_map + self._index = index self._cdocs = cdocs def get_size(self): return self._pmap['size'] def get_body_file(self): + payload = "" pmap = self._pmap multi = pmap.get('multi') if not multi: - phash = pmap.get("phash") + payload = self._get_payload(self._index) else: - pmap_ = pmap.get('part_map') - first_part = pmap_.get('1', None) - if not empty(first_part): - phash = first_part['phash'] - else: - phash = "" - - payload = self._get_payload(phash) - + # XXX uh, multi also... should recurse" + raise NotImplementedError if payload: - # FIXME - # content_type = self._get_ctype_from_document(phash) - # charset = find_charset(content_type) - charset = None - if charset is None: - charset = get_email_charset(payload) - try: - if isinstance(payload, unicode): - payload = payload.encode(charset) - except UnicodeError as exc: - logger.error( - "Unicode error, using 'replace'. {0!r}".format(exc)) - payload = payload.encode(charset, 'replace') - + payload = self._format_payload(payload) return _write_and_rewind(payload) def get_headers(self): return self._pmap.get("headers", []) def is_multipart(self): - multi = self._pmap.get("multi", False) - return multi + return self._pmap.get("multi", False) def get_subpart(self, part): if not self.is_multipart(): @@ -123,10 +120,30 @@ class MessagePart(object): except KeyError: logger.debug("getSubpart for %s: KeyError" % (part,)) raise IndexError - return MessagePart(self._soledad, part_map) - - def _get_payload(self, phash): - return self._cdocs.get(phash, "") + return MessagePart(part_map, cdocs={1: self._cdocs.get(1, {})}) + + def _get_payload(self, index): + cdoc_wrapper = self._cdocs.get(index, None) + if cdoc_wrapper: + return cdoc_wrapper.raw + return "" + + def _format_payload(self, payload): + # FIXME ----------------------------------------------- + # Test against unicode payloads... + # content_type = self._get_ctype_from_document(phash) + # charset = find_charset(content_type) + charset = None + if charset is None: + charset = get_email_charset(payload) + try: + if isinstance(payload, unicode): + payload = payload.encode(charset) + except UnicodeError as exc: + logger.error( + "Unicode error, using 'replace'. {0!r}".format(exc)) + payload = payload.encode(charset, 'replace') + return payload class Message(object): @@ -224,17 +241,18 @@ class Message(object): raise TypeError part_index = part + 1 try: - subpart_dict = self._wrapper.get_subpart_dict( - part_index) + subpart_dict = self._wrapper.get_subpart_dict(part_index) except KeyError: - raise TypeError - # XXX pass cdocs - return MessagePart(subpart_dict) + raise IndexError + + return MessagePart( + subpart_dict, index=part_index, cdocs=self._wrapper.cdocs) # Custom methods. def get_tags(self): """ + Get the tags for this message. """ return tuple(self._wrapper.fdoc.tags) @@ -290,7 +308,7 @@ class MessageCollection(object): self.adaptor = adaptor self.store = store - # XXX I have to think about what to do when there is no mbox passed to + # XXX think about what to do when there is no mbox passed to # the initialization. We could still get the MetaMsg by index, instead # of by doc_id. See get_message_by_content_hash self.mbox_indexer = mbox_indexer -- cgit v1.2.3 From d586ea252e187e811cd2851522cdfab74428fd75 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 22 Jan 2015 13:02:18 -0400 Subject: add TODO to the adaptor interface --- src/leap/mail/interfaces.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/leap/mail/interfaces.py b/src/leap/mail/interfaces.py index 5838ce9..899400f 100644 --- a/src/leap/mail/interfaces.py +++ b/src/leap/mail/interfaces.py @@ -33,6 +33,9 @@ class IMessageWrapper(Interface): cdocs = Attribute('A dictionary with the content-docs, one-indexed') +# TODO [ ] Catch up with the actual implementation! +# Lot of stuff added there ... + class IMailAdaptor(Interface): """ I know how to store the standard representation for messages and mailboxes, -- cgit v1.2.3 From 66adeda5603aa5a1a29f027bfe6a8536c488ac97 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 22 Jan 2015 14:02:28 -0400 Subject: Fix recent/unseen notifications --- src/leap/mail/adaptors/soledad.py | 40 +++++++++++++++++++++++++++++++ src/leap/mail/adaptors/soledad_indexes.py | 4 ++-- src/leap/mail/imap/mailbox.py | 5 +--- src/leap/mail/mail.py | 23 +++++++++++++----- 4 files changed, 60 insertions(+), 12 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index d21638c..74c34f9 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -930,6 +930,42 @@ class SoledadMailAdaptor(SoledadIndexMixin): d.addCallbacks(delete_fdoc_and_mdoc_flagged, err) return d + # count messages + + def get_count_unseen(self, store, mbox_uuid): + """ + Get the number of unseen messages for a given mailbox. + + :param store: instance of Soledad. + :param mbox_uuid: the uuid for this mailbox. + :rtype: int + """ + type_ = FlagsDocWrapper.model.type_ + uuid = mbox_uuid.replace('-', '_') + + unseen_index = indexes.TYPE_MBOX_SEEN_IDX + + d = store.get_count_from_index(unseen_index, type_, uuid, "0") + d.addErrback(self._errback) + return d + + def get_count_recent(self, store, mbox_uuid): + """ + Get the number of recent messages for a given mailbox. + + :param store: instance of Soledad. + :param mbox_uuid: the uuid for this mailbox. + :rtype: int + """ + type_ = FlagsDocWrapper.model.type_ + uuid = mbox_uuid.replace('-', '_') + + recent_index = indexes.TYPE_MBOX_RECENT_IDX + + d = store.get_count_from_index(recent_index, type_, uuid, "1") + d.addErrback(self._errback) + return d + # Mailbox handling def get_or_create_mbox(self, store, name): @@ -937,6 +973,7 @@ class SoledadMailAdaptor(SoledadIndexMixin): Get the mailbox with the given name, or create one if it does not exist. + :param store: instance of Soledad :param name: the name of the mailbox :type name: str """ @@ -970,6 +1007,9 @@ class SoledadMailAdaptor(SoledadIndexMixin): """ return MailboxWrapper.get_all(store) + def _errback(self, f): + f.printTraceback() + def _split_into_parts(raw): # TODO signal that we can delete the original message!----- diff --git a/src/leap/mail/adaptors/soledad_indexes.py b/src/leap/mail/adaptors/soledad_indexes.py index 856dfb4..d2f8b71 100644 --- a/src/leap/mail/adaptors/soledad_indexes.py +++ b/src/leap/mail/adaptors/soledad_indexes.py @@ -51,7 +51,7 @@ TYPE_MBOX_UUID_IDX = 'by-type-and-mbox-uuid' TYPE_SUBS_IDX = 'by-type-and-subscribed' TYPE_MSGID_IDX = 'by-type-and-message-id' TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' -TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' +TYPE_MBOX_RECENT_IDX = 'by-type-and-mbox-and-recent' TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash' TYPE_C_HASH_IDX = 'by-type-and-contenthash' @@ -97,7 +97,7 @@ MAIL_INDEXES = { # messages TYPE_MBOX_SEEN_IDX: [TYPE, MBOX_UUID, 'bool(seen)'], - TYPE_MBOX_RECT_IDX: [TYPE, MBOX_UUID, 'bool(recent)'], + TYPE_MBOX_RECENT_IDX: [TYPE, MBOX_UUID, 'bool(recent)'], TYPE_MBOX_DEL_IDX: [TYPE, MBOX_UUID, 'bool(deleted)'], # incoming queue diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 045636e..c826e86 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -714,13 +714,10 @@ class IMAPMailbox(object): :param result: ignored """ - d = self._get_unseen_deferred() + d = defer.maybeDeferred(self.getUnseenCount) d.addCallback(self.__cb_signal_unread_to_ui) return result - def _get_unseen_deferred(self): - return defer.maybeDeferred(self.getUnseenCount) - def __cb_signal_unread_to_ui(self, unseen): """ Send the unread signal to UI. diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index aa499c0..d986f59 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -295,6 +295,7 @@ class MessageCollection(object): # server to accept deferreds. # [ ] Use inheritance for the mailbox-collection instead of handling the # special cases everywhere? + # [ ] or maybe a mailbox_only decorator... # Account should provide an adaptor instance when creating this collection. adaptor = None @@ -419,14 +420,24 @@ class MessageCollection(object): return d def count_recent(self): - # FIXME HACK - # TODO ------------------------ implement this - return 3 + """ + Count the recent messages in this collection. + :return: a Deferred that will fire with the integer for the count. + :rtype: Deferred + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + return self.adaptor.get_count_recent(self.store, self.mbox_uuid) def count_unseen(self): - # FIXME hack - # TODO ------------------------ implement this - return 3 + """ + Count the unseen messages in this collection. + :return: a Deferred that will fire with the integer for the count. + :rtype: Deferred + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + return self.adaptor.get_count_unseen(self.store, self.mbox_uuid) def get_uid_next(self): """ -- cgit v1.2.3 From f211669206d30088181f7299aaa0ffb65a89becc Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Thu, 22 Jan 2015 17:51:47 -0600 Subject: Fix SMTP async tests --- src/leap/mail/smtp/gateway.py | 32 +++++++++--------- src/leap/mail/smtp/tests/test_gateway.py | 57 ++++++++++++++++++++++---------- 2 files changed, 57 insertions(+), 32 deletions(-) diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index 1a187cf..9d78474 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -204,22 +204,24 @@ class SMTPDelivery(object): signal(proto.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, user.dest.addrstr) def not_found(failure): - if failure.check(KeyNotFound): - # if key was not found, check config to see if will send anyway - if self._encrypted_only: - signal(proto.SMTP_RECIPIENT_REJECTED, user.dest.addrstr) - raise smtp.SMTPBadRcpt(user.dest.addrstr) - log.msg("Warning: will send an unencrypted message (because " - "encrypted_only' is set to False).") - signal( - proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, - user.dest.addrstr) - else: - return failure - - d = self._km.get_key(address, OpenPGPKey) # might raise KeyNotFound + failure.trap(KeyNotFound) + + # if key was not found, check config to see if will send anyway + if self._encrypted_only: + signal(proto.SMTP_RECIPIENT_REJECTED, user.dest.addrstr) + raise smtp.SMTPBadRcpt(user.dest.addrstr) + log.msg("Warning: will send an unencrypted message (because " + "encrypted_only' is set to False).") + signal( + proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, + user.dest.addrstr) + + def encrypt_func(_): + return lambda: EncryptedMessage(user, self._outgoing_mail) + + d = self._km.get_key(address, OpenPGPKey) d.addCallbacks(found, not_found) - d.addCallback(lambda _: EncryptedMessage(user, self._outgoing_mail)) + d.addCallback(encrypt_func) return d def validateFrom(self, helo, origin): diff --git a/src/leap/mail/smtp/tests/test_gateway.py b/src/leap/mail/smtp/tests/test_gateway.py index 8cbff8f..0b9a364 100644 --- a/src/leap/mail/smtp/tests/test_gateway.py +++ b/src/leap/mail/smtp/tests/test_gateway.py @@ -23,7 +23,8 @@ SMTP gateway tests. import re from datetime import datetime -from twisted.internet.defer import inlineCallbacks, fail +from twisted.internet import reactor +from twisted.internet.defer import inlineCallbacks, fail, succeed, Deferred from twisted.test import proto_helpers from mock import Mock @@ -70,6 +71,7 @@ class TestSmtpGateway(TestCaseWithKeyManager): % (string, pattern)) raise self.failureException(msg) + @inlineCallbacks def test_gateway_accepts_valid_email(self): """ Test if SMTP server responds correctly for valid interaction. @@ -93,11 +95,11 @@ class TestSmtpGateway(TestCaseWithKeyManager): # snip... transport = proto_helpers.StringTransport() proto.makeConnection(transport) + reply = "" for i, line in enumerate(self.EMAIL_DATA): - proto.lineReceived(line + '\r\n') - self.assertMatch(transport.value(), - '\r\n'.join(SMTP_ANSWERS[0:i + 1]), - 'Did not get expected answer from gateway.') + reply += yield self.getReply(line + '\r\n', proto, transport) + self.assertMatch(reply, '\r\n'.join(SMTP_ANSWERS), + 'Did not get expected answer from gateway.') proto.setTimeout(None) @inlineCallbacks @@ -122,15 +124,16 @@ class TestSmtpGateway(TestCaseWithKeyManager): outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) transport = proto_helpers.StringTransport() proto.makeConnection(transport) - proto.lineReceived(self.EMAIL_DATA[0] + '\r\n') - proto.lineReceived(self.EMAIL_DATA[1] + '\r\n') - proto.lineReceived(self.EMAIL_DATA[2] + '\r\n') + yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport) + yield self.getReply(self.EMAIL_DATA[1] + '\r\n', proto, transport) + reply = yield self.getReply(self.EMAIL_DATA[2] + '\r\n', + proto, transport) # ensure the address was rejected - lines = transport.value().rstrip().split('\n') self.assertEqual( - '550 Cannot receive for specified address', - lines[-1], + '550 Cannot receive for specified address\r\n', + reply, 'Address should have been rejecetd with appropriate message.') + proto.setTimeout(None) @inlineCallbacks def test_missing_key_accepts_address(self): @@ -153,12 +156,32 @@ class TestSmtpGateway(TestCaseWithKeyManager): False, outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) transport = proto_helpers.StringTransport() proto.makeConnection(transport) - proto.lineReceived(self.EMAIL_DATA[0] + '\r\n') - proto.lineReceived(self.EMAIL_DATA[1] + '\r\n') - proto.lineReceived(self.EMAIL_DATA[2] + '\r\n') + yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport) + yield self.getReply(self.EMAIL_DATA[1] + '\r\n', proto, transport) + reply = yield self.getReply(self.EMAIL_DATA[2] + '\r\n', + proto, transport) # ensure the address was accepted - lines = transport.value().rstrip().split('\n') self.assertEqual( - '250 Recipient address accepted', - lines[-1], + '250 Recipient address accepted\r\n', + reply, 'Address should have been accepted with appropriate message.') + proto.setTimeout(None) + + def getReply(self, line, proto, transport): + proto.lineReceived(line) + + if line[:4] not in ['HELO', 'MAIL', 'RCPT', 'DATA']: + return succeed("") + + def check_transport(_): + reply = transport.value() + if reply: + transport.clear() + return succeed(reply) + + d = Deferred() + d.addCallback(check_transport) + reactor.callLater(0, lambda: d.callback(None)) + return d + + return check_transport(None) -- cgit v1.2.3 From 459524ffa25eb9dae95a5f761024d15b6bdb25df Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 22 Jan 2015 20:27:08 -0400 Subject: re-add the missing hacking docs; add tb imap logging hint --- docs/conf.py | 2 +- docs/hacking.rst | 163 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 2 + 3 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 docs/hacking.rst diff --git a/docs/conf.py b/docs/conf.py index 8e08f09..51e9d66 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -57,7 +57,7 @@ copyright = u'2014, Kali Kaneko' # built documents. # # The short X.Y version. -version = '0.3.9' +version = '0.4.0alpha1' # The full version, including alpha/beta/rc tags. release = '0.3.9' diff --git a/docs/hacking.rst b/docs/hacking.rst new file mode 100644 index 0000000..daa3762 --- /dev/null +++ b/docs/hacking.rst @@ -0,0 +1,163 @@ +.. _hacking: + +======== +Hacking +======== + +Some hints oriented to `leap.mail` hackers. These notes are mostly related to +the imap server, although they probably will be useful for other pieces too. + +Don't panic! Just manhole into it +================================= + +If you want to inspect the objects living in your application memory, in +realtime, you can manhole into it. + +First of all, check that the modules ``PyCrypto`` and ``pyasn1`` are installed +into your system, they are needed for it to work. + +You just have to pass the ``LEAP_MAIL_MANHOLE=1`` enviroment variable while +launching the client:: + + LEAP_MAIL_MANHOLE=1 bitmask --debug + +And then you can ssh into your application! (password is "leap"):: + + ssh boss@localhost -p 2222 + +Did I mention how *awesome* twisted is?? ``:)`` + + +Profiling +========= +If using ``twistd`` to launch the server, you can use twisted profiling +capabities:: + + LEAP_MAIL_CONF=~/.leapmailrc twistd --profile=/tmp/mail-profiling -n -y imap-server.tac + +``--profiler`` option allows you to select different profilers (default is +"hotshot"). + +You can also do profiling when using the ``bitmask`` client. Enable the +``LEAP_PROFILE_IMAPCMD`` environment flag to get profiling of certain IMAP +commands:: + + LEAP_PROFILE_IMAPCMD=1 bitmask --debug + +Offline mode +============ + +The client has an ``--offline`` flag that will make the Mail services (imap, +currently) not try to sync with remote replicas. Very useful during development, +although you need to login with the remote server at least once before being +able to use it. + +Running the service with twistd +=============================== + +In order to run the mail service (currently, the imap server only), you will +need a config with this info:: + + [leap_mail] + userid = "user@provider" + uuid = "deadbeefdeadabad" + passwd = "foobar" # Optional + +In the ``LEAP_MAIL_CONF`` enviroment variable. If you do not specify a password +parameter, you'll be prompted for it. + +In order to get the user uid (uuid), look into the +``~/.config/leap/leap-backend.conf`` file after you have logged in into your +provider at least once. + +Run the twisted service:: + + LEAP_IMAP_CONFIG=~/.leapmailrc twistd -n -y imap-server.tac + +Now you can telnet into your local IMAP server and read your mail like a real +programmer™:: + + % telnet localhost 1984 + Trying 127.0.0.1... + Connected to localhost. + Escape character is '^]'. + * OK [CAPABILITY IMAP4rev1 LITERAL+ IDLE NAMESPACE] Twisted IMAP4rev1 Ready + tag LOGIN me@myprovider.net mahsikret + tag OK LOGIN succeeded + tag SELECT Inbox + * 2 EXISTS + * 1 RECENT + * FLAGS (\Seen \Answered \Flagged \Deleted \Draft \Recent List) + * OK [UIDVALIDITY 1410453885932] UIDs valid + tag OK [READ-WRITE] SELECT successful + ^] + telnet> Connection closed. + + +Although you probably prefer to use ``offlineimap`` for tests:: + + offlineimap -c LEAPofflineimapRC-tests + + +Minimal offlineimap configuration +--------------------------------- + +You can use this as a sample offlineimap config file:: + + [general] + accounts = leap-local + + [Account leap-local] + localrepository = LocalLeap + remoterepository = RemoteLeap + + [Repository LocalLeap] + type = Maildir + localfolders = ~/LEAPMail/Mail + + [Repository RemoteLeap] + type = IMAP + ssl = no + remotehost = localhost + remoteport = 1984 + remoteuser = user + remotepass = pass + +Testing utilities +----------------- +There are a bunch of utilities to test IMAP delivery in ``imap/tests`` folder. +If looking for a quick way of inspecting mailboxes, have a look at ``getmail``:: + + ./getmail me@testprovider.net mahsikret + 1. Drafts + 2. INBOX + 3. Trash + Which mailbox? [1] 2 + 1 Subject: this is the time of the revolution + 2 Subject: ignore me + + Which message? [1] (Q quits) 1 + 1 X-Leap-Provenance: Thu, 11 Sep 2014 16:52:11 -0000; pubkey="C1F8DE10BD151F99" + Received: from mx1.testprovider.net(mx1.testprovider.net [198.197.196.195]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (Client CN "*.foobar.net", Issuer "Gandi Standard SSL CA" (not verified)) + by blackhole (Postfix) with ESMTPS id DEADBEEF + for ; Thu, 11 Sep 2014 16:52:10 +0000 (UTC) + Delivered-To: 926d4915cfd42b6d96d38660c04613af@testprovider.net + Message-Id: <20140911165205.GB8054@samsara> + From: Kali + + (snip) + + +Debugging IMAP commands +======================= + +Use ``ngrep`` to obtain logs of the commands:: + + sudo ngrep -d lo -W byline port 1984 + +To get verbose output from thunderbird/icedove, set the following environment +variable:: + + NSPR_LOG_MODULES="imap:5" icedove diff --git a/docs/index.rst b/docs/index.rst index d8634ea..8bacc51 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -38,6 +38,8 @@ server as another code entity that uses this lower layer. .. toctree:: :maxdepth: 2 + hacking + .. intro .. tutorial -- cgit v1.2.3 From 59f7005a05e50700f4d8f1fba4b036318fa246e6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 26 Jan 2015 23:41:19 -0400 Subject: fix initialization of cdocs --- src/leap/mail/adaptors/soledad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 74c34f9..542ad94 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -464,7 +464,7 @@ class MessageWrapper(object): cdocs_keys = cdocs.keys() assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1) self.cdocs = dict([ - (key, ContentDocWrapper(**doc.content)) + (key, get_doc_wrapper(doc, ContentDocWrapper)) for (key, doc) in cdocs.items()]) for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()): cdoc.set_future_doc_id(doc_id) -- cgit v1.2.3 From ad113e53d59213a5accc4f71564fd6ad0f73f74f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 26 Jan 2015 23:41:52 -0400 Subject: save drafts: search by msg-id --- src/leap/mail/adaptors/soledad.py | 24 +++++++++++++++++ src/leap/mail/imap/mailbox.py | 13 +++++---- src/leap/mail/imap/server.py | 57 ++++++++++++++++++++++++++++++++++----- src/leap/mail/mail.py | 19 +++++++++++++ src/leap/mail/mailbox_indexer.py | 2 +- 5 files changed, 100 insertions(+), 15 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 542ad94..4dc02a1 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -966,6 +966,30 @@ class SoledadMailAdaptor(SoledadIndexMixin): d.addErrback(self._errback) return d + # search api + + def get_mdoc_id_from_msgid(self, store, mbox_uuid, msgid): + """ + Get the UID for a message with the passed msgid (the one in the headers + msg-id). + This is used by the MUA to retrieve the recently saved draft. + """ + type_ = HeaderDocWrapper.model.type_ + uuid = mbox_uuid.replace('-', '_') + + msgid_index = indexes.TYPE_MSGID_IDX + + def get_mdoc_id(hdoc): + if not hdoc: + return None + hdoc = hdoc[0] + mdoc_id = hdoc.doc_id.replace("H-", "M-%s-" % uuid) + return mdoc_id + + d = store.get_from_index(msgid_index, type_, msgid) + d.addCallback(get_mdoc_id) + return d + # Mailbox handling def get_or_create_mbox(self, store, name): diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index c826e86..be7f70c 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -842,18 +842,17 @@ class IMAPMailbox(object): # '52D44F11.9060107@dev.bitmask.net'] # TODO hardcoding for now! -- we'll support generic queries later on - # but doing a quickfix for avoiding duplicat saves in the draft folder. - # See issue #4209 + # but doing a quickfix for avoiding duplicate saves in the draft + # folder. # See issue #4209 if len(query) > 2: if query[1] == 'HEADER' and query[2].lower() == "message-id": msgid = str(query[3]).strip() logger.debug("Searching for %s" % (msgid,)) - d = self.messages._get_uid_from_msgid(str(msgid)) - # XXX remove gatherResults - d1 = defer.gatherResults([d]) - # we want a list, so return it all the same - return d1 + + d = self.collection.get_uid_from_msgid(str(msgid)) + d.addCallback(lambda result: [result]) + return d # nothing implemented for any other query logger.warning("Cannot process query: %s" % (query,)) diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 027fd7a..abe16be 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -199,8 +199,7 @@ class LEAPIMAPServer(imap4.IMAP4Server): a deferred, the client will only be informed of success (or failure) when the deferred's callback (or errback) is invoked. """ - # TODO return the output of _memstore.is_writing - # XXX and that should return a deferred! + # TODO implement a collection of ongoing deferreds? return None ############################################################# @@ -499,6 +498,50 @@ class LEAPIMAPServer(imap4.IMAP4Server): auth_DELETE = (do_DELETE, arg_astring) select_DELETE = auth_DELETE + # ----------------------------------------------------------------------- + # Patched just to allow __cbAppend to receive a deferred from messageCount + # TODO format and send upstream. + def do_APPEND(self, tag, mailbox, flags, date, message): + mailbox = self._parseMbox(mailbox) + maybeDeferred(self.account.select, mailbox).addCallback( + self._cbAppendGotMailbox, tag, flags, date, message).addErrback( + self._ebAppendGotMailbox, tag) + + def __ebAppend(self, failure, tag): + self.sendBadResponse(tag, 'APPEND failed: ' + str(failure.value)) + + def _cbAppendGotMailbox(self, mbox, tag, flags, date, message): + if not mbox: + self.sendNegativeResponse(tag, '[TRYCREATE] No such mailbox') + return + + d = mbox.addMessage(message, flags, date) + d.addCallback(self.__cbAppend, tag, mbox) + d.addErrback(self.__ebAppend, tag) + + def _ebAppendGotMailbox(self, failure, tag): + self.sendBadResponse( + tag, "Server error encountered while opening mailbox.") + log.err(failure) + + def __cbAppend(self, result, tag, mbox): + + # XXX patched --------------------------------- + def send_response(count): + self.sendUntaggedResponse('%d EXISTS' % count) + self.sendPositiveResponse(tag, 'APPEND complete') + + d = mbox.getMessageCount() + d.addCallback(send_response) + return d + # XXX patched --------------------------------- + + # ----------------------------------------------------------------------- + + auth_APPEND = (do_APPEND, arg_astring, imap4.IMAP4Server.opt_plist, + imap4.IMAP4Server.opt_datetime, arg_literal) + select_APPEND = auth_APPEND + # Need to override the command table after patching # arg_astring and arg_literal, except on the methods that we are already # overriding. @@ -511,10 +554,10 @@ class LEAPIMAPServer(imap4.IMAP4Server): # do_RENAME = imap4.IMAP4Server.do_RENAME # do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE # do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE + # do_APPEND = imap4.IMAP4Server.do_APPEND # ------------------------------------------------- do_LOGIN = imap4.IMAP4Server.do_LOGIN do_STATUS = imap4.IMAP4Server.do_STATUS - do_APPEND = imap4.IMAP4Server.do_APPEND do_COPY = imap4.IMAP4Server.do_COPY _selectWork = imap4.IMAP4Server._selectWork @@ -539,6 +582,10 @@ class LEAPIMAPServer(imap4.IMAP4Server): # re-add if we stop overriding DELETE # auth_DELETE = (do_DELETE, arg_astring) # select_DELETE = auth_DELETE + # auth_APPEND = (do_APPEND, arg_astring, opt_plist, opt_datetime, + # arg_literal) + # select_APPEND = auth_APPEND + # ---------------------------------------------------- auth_RENAME = (do_RENAME, arg_astring, arg_astring) @@ -559,10 +606,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): auth_STATUS = (do_STATUS, arg_astring, arg_plist) select_STATUS = auth_STATUS - auth_APPEND = (do_APPEND, arg_astring, opt_plist, opt_datetime, - arg_literal) - select_APPEND = auth_APPEND - select_COPY = (do_COPY, arg_seqset, arg_astring) ############################################################# diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index d986f59..2766524 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -460,6 +460,25 @@ class MessageCollection(object): """ return self.mbox_indexer.all_uid_iter(self.mbox_uuid) + def get_uid_from_msgid(self, msgid): + """ + Return the UID(s) of the matching msg-ids for this mailbox collection. + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + + def get_uid(mdoc_id): + if not mdoc_id: + return None + d = self.mbox_indexer.get_uid_from_doc_id( + self.mbox_uuid, mdoc_id) + return d + + d = self.adaptor.get_mdoc_id_from_msgid( + self.store, self.mbox_uuid, msgid) + d.addCallback(get_uid) + return d + # Manipulate messages def add_msg(self, raw_msg, flags=tuple(), tags=tuple(), date=""): diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 4eb0fa8..3bec41e 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -120,7 +120,7 @@ class MailboxIndexer(object): The doc_id must be in the format: - M++ + M-- :param mailbox: the mailbox name :type mailbox: str -- cgit v1.2.3 From 3e4c33021250df02cc50868016b0a73d2f069a40 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 27 Jan 2015 11:16:48 -0400 Subject: allow text/html for bodies --- src/leap/mail/walk.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 891abdc..9f5098d 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -62,7 +62,8 @@ def get_body_phash(msg): Find the body payload-hash for this message. """ for part in msg.walk(): - if part.get_content_type() == "text/plain": + # XXX what other ctypes should be considered body? + if part.get_content_type() in ("text/plain", "text/html"): # XXX avoid hashing again return get_hash(part.get_payload()) -- cgit v1.2.3 From a4305a4a57afb9a84fb56b6031947a3d2a717973 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 27 Jan 2015 11:18:04 -0400 Subject: return empty string if we couldn't find body so the server doesn't choke. --- src/leap/mail/mail.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 2766524..4306ec3 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -22,7 +22,6 @@ import logging import StringIO from twisted.internet import defer -from twisted.python import log from leap.common.check import leap_assert_type from leap.common.mail import get_email_charset @@ -31,7 +30,6 @@ from leap.mail.adaptors.soledad import SoledadMailAdaptor from leap.mail.constants import INBOX_NAME from leap.mail.constants import MessageFlags from leap.mail.mailbox_indexer import MailboxIndexer -from leap.mail.utils import empty # find_charset logger = logging.getLogger(name=__name__) @@ -211,9 +209,8 @@ class Message(object): Get a file descriptor with the body content. """ def write_and_rewind_if_found(cdoc): - if not cdoc: - return None - return _write_and_rewind(cdoc.raw) + payload = cdoc.raw if cdoc else "" + return _write_and_rewind(payload) d = defer.maybeDeferred(self._wrapper.get_body, store) d.addCallback(write_and_rewind_if_found) -- cgit v1.2.3 From d8b078504b522add8542ebe3bdbe4da39e4ffaac Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 27 Jan 2015 11:19:14 -0400 Subject: rename lingering , that is ambiguous --- src/leap/mail/imap/mailbox.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index be7f70c..c91f127 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -521,7 +521,7 @@ class IMAPMailbox(object): getmsg = self.collection.get_message_by_uid getimapmsg = self.get_imap_message - def get_imap_messages_for_sequence(msg_sequence): + def get_imap_messages_for_range(msg_range): def _get_imap_msg(messages): d_imapmsg = [] @@ -531,7 +531,7 @@ class IMAPMailbox(object): def _zip_msgid(imap_messages): zipped = zip( - list(msg_sequence), imap_messages) + list(msg_range), imap_messages) return (item for item in zipped) def _unset_recent(sequence): @@ -539,7 +539,7 @@ class IMAPMailbox(object): return sequence d_msg = [] - for msgid in msg_sequence: + for msgid in msg_range: # XXX We want cdocs because we "probably" are asked for the # body. We should be smarted at do_FETCH and pass a parameter # to this method in order not to prefetch cdocs if they're not @@ -558,7 +558,7 @@ class IMAPMailbox(object): else: d = self._get_messages_range(messages_asked) - d.addCallback(get_imap_messages_for_sequence) + d.addCallback(get_imap_messages_for_range) # TODO -- call signal_to_ui # d.addCallback(self.cb_signal_unread_to_ui) -- cgit v1.2.3 From 3e723f7a003f72a33d761874e64cbce0604d37a9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 27 Jan 2015 14:13:55 -0400 Subject: implement copy interface --- src/leap/mail/adaptors/soledad.py | 28 ++++++++++++++++++++++------ src/leap/mail/imap/mailbox.py | 4 ++-- src/leap/mail/mail.py | 18 ++++++++++-------- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 4dc02a1..721f25e 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -16,6 +16,7 @@ """ Soledadad MailAdaptor module. """ +import logging import re from collections import defaultdict from email import message_from_string @@ -512,15 +513,30 @@ class MessageWrapper(object): d.append(self.fdoc.delete(store)) return defer.gatherResults(d) - def copy(self, store, newmailbox): + def copy(self, store, new_mbox_uuid): """ Return a copy of this MessageWrapper in a new mailbox. + + :param store: an instance of Soledad, or anything that behaves alike. + :param new_mbox_uuid: the uuid of the mailbox where we are copying this + message to. + :type new_mbox_uuid: str + :rtype: MessageWrapper """ - # 1. copy the fdoc, mdoc - # 2. remove the doc_id of that fdoc - # 3. create it (with new doc_id) - # 4. return new wrapper (new meta too!) - raise NotImplementedError() + new_mdoc = self.mdoc.serialize() + new_fdoc = self.fdoc.serialize() + + # the future doc_ids is properly set because we modified + # the pointers in mdoc, which has precedence. + new_wrapper = MessageWrapper(new_mdoc, new_fdoc, None, None) + new_wrapper.hdoc = self.hdoc + new_wrapper.cdocs = self.cdocs + new_wrapper.set_mbox_uuid(new_mbox_uuid) + + # XXX could flag so that it only creates mdoc/fdoc... + d = new_wrapper.create(store) + d.addCallback(lambda result: new_wrapper) + return d def set_mbox_uuid(self, mbox_uuid): """ diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index c91f127..1bc530e 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -880,8 +880,8 @@ class IMAPMailbox(object): #deferLater(self.reactor, 0, self._do_copy, message, d) #return d - # FIXME not implemented !!! --- - return self.collection.copy_msg(message, self.mbox_name) + return self.collection.copy_msg(message.message, + self.collection.mbox_uuid) # convenience fun diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 4306ec3..b46d223 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -509,25 +509,27 @@ class MessageCollection(object): return d - def copy_msg(self, msg, newmailbox): + def copy_msg(self, msg, new_mbox_uuid): """ Copy the message to another collection. (it only makes sense for mailbox collections) """ - # TODO currently broken ------------------FIXME- if not self.is_mailbox_collection(): raise NotImplementedError() - def insert_copied_mdoc_id(wrapper): - # TODO this needs to be implemented before the copy - # interface works. - newmailbox_uuid = get_mbox_uuid_from_msg_wrapper(wrapper) + def insert_copied_mdoc_id(wrapper_new_msg): return self.mbox_indexer.insert_doc( - newmailbox_uuid, wrapper.mdoc.doc_id) + new_mbox_uuid, wrapper.mdoc.doc_id) wrapper = msg.get_wrapper() - d = wrapper.copy(self.store, newmailbox) + + def print_result(result): + print "COPY CALLBACK:>>>", result + return result + + d = wrapper.copy(self.store, new_mbox_uuid) d.addCallback(insert_copied_mdoc_id) + d.addCallback(print_result) return d def delete_msg(self, msg): -- cgit v1.2.3 From fdbc6a7a448ca5f329bb452dc4517bd6be8804b4 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 27 Jan 2015 15:20:27 -0400 Subject: ignore revisionconflicts on puts. we surely already have that part. --- src/leap/mail/adaptors/soledad.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 721f25e..470562d 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -18,12 +18,14 @@ Soledadad MailAdaptor module. """ import logging import re + from collections import defaultdict from email import message_from_string from pycryptopp.hash import sha256 from twisted.internet import defer from zope.interface import implements +import u1db from leap.common.check import leap_assert, leap_assert_type @@ -40,6 +42,8 @@ from leap.mail.interfaces import IMailAdaptor, IMessageWrapper from leap.soledad.common.document import SoledadDocument +logger = logging.getLogger(__name__) + # TODO # [ ] Convenience function to create mail specifying subject, date, etc? @@ -151,12 +155,24 @@ class SoledadDocumentWrapper(models.DocumentWrapper): def update_and_put_doc(doc): doc.content.update(self.serialize()) - return store.put_doc(doc) + d = store.put_doc(doc) + d.addErrback(self._catch_revision_conflict, doc.doc_id) + return d d = store.get_doc(self._doc_id) d.addCallback(update_and_put_doc) return d + def _catch_revision_conflict(self, failure, doc_id): + # XXX We can have some RevisionConflicts if we try + # to put the docs that are already there. + # This can happen right now when creating/saving the cdocs + # during a copy. Instead of catching and ignoring this + # error, we should mark them in the copy so there is no attempt to + # create/update them. + failure.trap(u1db.errors.RevisionConflict) + logger.debug("Got conflict while putting %s" % doc_id) + def delete(self, store): """ Delete the documents for this wrapper. -- cgit v1.2.3 From 9d2cf05fa575aa5a01e255837738f3aa2740b19e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 28 Jan 2015 02:03:12 -0400 Subject: append/copy performance improvement --- src/leap/mail/adaptors/soledad.py | 63 +++++++++++++++++++++++++++++++-------- src/leap/mail/imap/mailbox.py | 55 ++++++++++++---------------------- src/leap/mail/mail.py | 26 ++++++++++------ 3 files changed, 86 insertions(+), 58 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 470562d..490e014 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -136,6 +136,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): d = store.create_doc(self.serialize(), doc_id=self.future_doc_id) d.addCallback(update_doc_id) + d.addErrback(self._catch_revision_conflict, self.future_doc_id) return d def update(self, store): @@ -447,7 +448,7 @@ class MessageWrapper(object): implements(IMessageWrapper) - def __init__(self, mdoc, fdoc, hdoc, cdocs=None): + def __init__(self, mdoc, fdoc, hdoc, cdocs=None, is_copy=False): """ Need at least a metamsg-document, a flag-document and a header-document to instantiate a MessageWrapper. Content-documents can be retrieved @@ -456,7 +457,11 @@ class MessageWrapper(object): cdocs, if any, should be a dictionary in which the keys are ascending integers, beginning at one, and the values are dictionaries with the content of the content-docs. + + is_copy, if set to True, will only attempt to create mdoc and fdoc + (because hdoc and cdocs are supposed to exist already) """ + self._is_copy = is_copy def get_doc_wrapper(doc, cls): if isinstance(doc, SoledadDocument): @@ -486,9 +491,33 @@ class MessageWrapper(object): for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()): cdoc.set_future_doc_id(doc_id) - def create(self, store): + def create(self, store, notify_just_mdoc=False): """ Create all the parts for this message in the store. + + :param store: an instance of Soledad + + :param notify_just_mdoc: + if set to True, this method will return *only* the deferred + corresponding to the creation of the meta-message document. + Be warned that in that case there will be no record of failures + when creating the other part-documents. + + Other-wise, this method will return a deferred that will wait for + the creation of all the part documents. + + Setting this flag to True is mostly a convenient workaround for the + fact that massive serial appends will take too much time, and in + most of the cases the MUA will only switch to the mailbox where the + appends have happened after a certain time, which in most of the + times will be enough to have all the queued insert operations + finished. + :type notify_just_mdoc: bool + + :return: a deferred whose callback will be called when either all the + part documents have been written, or just the metamsg-doc, + depending on the value of the notify_just_mdoc flag + :rtype: defer.Deferred """ leap_assert(self.cdocs, "Need non empty cdocs to create the " @@ -500,17 +529,24 @@ class MessageWrapper(object): # TODO check that the doc_ids in the mdoc are coherent d = [] - d.append(self.mdoc.create(store)) + mdoc_created = self.mdoc.create(store) + d.append(mdoc_created) d.append(self.fdoc.create(store)) - if self.hdoc.doc_id is None: - d.append(self.hdoc.create(store)) - for cdoc in self.cdocs.values(): - if cdoc.doc_id is not None: - # we could be just linking to an existing - # content-doc. - continue - d.append(cdoc.create(store)) - return defer.gatherResults(d) + + if not self._is_copy: + if self.hdoc.doc_id is None: + d.append(self.hdoc.create(store)) + for cdoc in self.cdocs.values(): + if cdoc.doc_id is not None: + # we could be just linking to an existing + # content-doc. + continue + d.append(cdoc.create(store)) + + if notify_just_mdoc: + return mdoc_created + else: + return defer.gatherResults(d) def update(self, store): """ @@ -544,7 +580,8 @@ class MessageWrapper(object): # the future doc_ids is properly set because we modified # the pointers in mdoc, which has precedence. - new_wrapper = MessageWrapper(new_mdoc, new_fdoc, None, None) + new_wrapper = MessageWrapper(new_mdoc, new_fdoc, None, None, + is_copy=True) new_wrapper.hdoc = self.hdoc new_wrapper.cdocs = self.cdocs new_wrapper.set_mbox_uuid(new_mbox_uuid) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 1bc530e..9ec6ea8 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -158,7 +158,8 @@ class IMAPMailbox(object): if not NOTIFY_NEW: return - logger.debug('adding mailbox listener: %s' % listener) + logger.debug('adding mailbox listener: %s. Total: %s' % ( + listener, len(self.listeners))) self.listeners.add(listener) def removeListener(self, listener): @@ -196,29 +197,6 @@ class IMAPMailbox(object): "flags expected to be a tuple") return self.collection.set_mbox_attr("flags", flags) - # TODO - not used? - @property - def is_closed(self): - """ - Return the closed attribute for this mailbox. - - :return: True if the mailbox is closed - :rtype: bool - """ - return self.collection.get_mbox_attr("closed") - - # TODO - not used? - def set_closed(self, closed): - """ - Set the closed attribute for this mailbox. - - :param closed: the state to be set - :type closed: bool - - :rtype: Deferred - """ - return self.collection.set_mbox_attr("closed", closed) - def getUIDValidity(self): """ Return the unique validity identifier for this mailbox. @@ -345,8 +323,10 @@ class IMAPMailbox(object): :param date: timestamp :type date: str - :return: a deferred that evals to None + :return: a deferred that will be triggered with the UID of the added + message. """ + # TODO should raise ReadOnlyMailbox if not rw. # TODO have a look at the cases for internal date in the rfc if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)): message = message.getvalue() @@ -362,20 +342,23 @@ class IMAPMailbox(object): if date is None: date = formatdate(time.time()) + # A better place for this would be the COPY/APPEND dispatcher # if PROFILE_CMD: # do_profile_cmd(d, "APPEND") - # XXX should review now that we're not using qtreactor. - # A better place for this would be the COPY/APPEND dispatcher - # in server.py, but qtreactor hangs when I do that, so this seems - # to work fine for now. + # just_mdoc=True: feels HACKY, but improves a *lot* the responsiveness + # of the APPENDS: we just need to be notified when the mdoc + # is saved, and let's hope that the other parts are doing just fine. + # This will not catch any errors when the inserts of the other parts + # fail, but on the other hand allows us to return very quickly, which + # seems a good compromise given that we have to serialize the appends. + # A better solution will probably involve implementing MULTIAPPEND + # or patching imap server to support pipelining. - def notifyCallback(x): - reactor.callLater(0, self.notify_new) - return x + d = self.collection.add_msg(message, flags=flags, date=date, + notify_just_mdoc=True) - d = self.collection.add_msg(message, flags=flags, date=date) - d.addCallback(notifyCallback) + # XXX signal to UI? --- should do it only if INBOX... d.addErrback(lambda f: log.msg(f.getTraceback())) return d @@ -486,9 +469,9 @@ class IMAPMailbox(object): """ # TODO we could pass the asked sequence to the indexer # all_uid_iter, and bound the sql query instead. - def filter_by_asked(sequence): + def filter_by_asked(all_msg_uid): set_asked = set(messages_asked) - set_exist = set(sequence) + set_exist = set(all_msg_uid) return set_asked.intersection(set_exist) d = self.collection.all_uid_iter() diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index b46d223..d74f6b8 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -478,10 +478,24 @@ class MessageCollection(object): # Manipulate messages - def add_msg(self, raw_msg, flags=tuple(), tags=tuple(), date=""): + def add_msg(self, raw_msg, flags=tuple(), tags=tuple(), date="", + notify_just_mdoc=False): """ Add a message to this collection. + + :param notify_just_mdoc: + boolean passed to the wrapper.create method, + to indicate whether we're interested in being notified when only + the mdoc has been written (faster, but potentially unsafe), or we + want to wait untill all the parts have been written. + Used by the imap mailbox implementation to get faster responses. + :type notify_just_mdoc: bool + + :returns: a deferred that will fire with the UID of the inserted + message. + :rtype: deferred """ + # XXX mdoc ref is a leaky abstraction here. generalize. leap_assert_type(flags, tuple) leap_assert_type(date, str) @@ -503,10 +517,9 @@ class MessageCollection(object): return self.mbox_indexer.insert_doc( self.mbox_uuid, doc_id) - d = wrapper.create(self.store) + d = wrapper.create(self.store, notify_just_mdoc=notify_just_mdoc) d.addCallback(insert_mdoc_id, wrapper) d.addErrback(lambda f: f.printTraceback()) - return d def copy_msg(self, msg, new_mbox_uuid): @@ -519,17 +532,12 @@ class MessageCollection(object): def insert_copied_mdoc_id(wrapper_new_msg): return self.mbox_indexer.insert_doc( - new_mbox_uuid, wrapper.mdoc.doc_id) + new_mbox_uuid, wrapper_new_msg.mdoc.doc_id) wrapper = msg.get_wrapper() - def print_result(result): - print "COPY CALLBACK:>>>", result - return result - d = wrapper.copy(self.store, new_mbox_uuid) d.addCallback(insert_copied_mdoc_id) - d.addCallback(print_result) return d def delete_msg(self, msg): -- cgit v1.2.3 From 078970756ef995b73e39cd00ba4cc27d42913750 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Fri, 23 Jan 2015 11:52:42 -0600 Subject: Remove footer --- changes/feature-4692_remove_footer | 1 + src/leap/mail/outgoing/service.py | 13 ------------- src/leap/mail/outgoing/tests/test_outgoing.py | 4 +--- 3 files changed, 2 insertions(+), 16 deletions(-) create mode 100644 changes/feature-4692_remove_footer diff --git a/changes/feature-4692_remove_footer b/changes/feature-4692_remove_footer new file mode 100644 index 0000000..8eca883 --- /dev/null +++ b/changes/feature-4692_remove_footer @@ -0,0 +1 @@ +- Don't add any footer to the emails (Closes: #4692) diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py index b70b3b1..88b8895 100644 --- a/src/leap/mail/outgoing/service.py +++ b/src/leap/mail/outgoing/service.py @@ -67,8 +67,6 @@ class OutgoingMail: A service for handling encrypted outgoing mail. """ - FOOTER_STRING = "I prefer encrypted email" - def __init__(self, from_address, keymanager, cert, key, host, port): """ Initialize the mail service. @@ -220,7 +218,6 @@ class OutgoingMail: :rtype: Deferred """ # pass if the original message's content-type is "multipart/encrypted" - lines = raw.split('\r\n') origmsg = Parser().parsestr(raw) if origmsg.get_content_type() == 'multipart/encrypted': @@ -230,16 +227,6 @@ class OutgoingMail: username, domain = from_address.split('@') to_address = validate_address(recipient.dest.addrstr) - # add a nice footer to the outgoing message - # XXX: footer will eventually optional or be removed - if origmsg.get_content_type() == 'text/plain': - lines.append('--') - lines.append('%s - https://%s/key/%s' % - (self.FOOTER_STRING, domain, username)) - lines.append('') - - origmsg = Parser().parsestr('\r\n'.join(lines)) - def signal_encrypt_sign(newmsg): signal(proto.SMTP_END_ENCRYPT_AND_SIGN, "%s,%s" % (self._from_address, to_address)) diff --git a/src/leap/mail/outgoing/tests/test_outgoing.py b/src/leap/mail/outgoing/tests/test_outgoing.py index fa50c30..d7423b6 100644 --- a/src/leap/mail/outgoing/tests/test_outgoing.py +++ b/src/leap/mail/outgoing/tests/test_outgoing.py @@ -58,9 +58,7 @@ class TestOutgoingMail(TestCaseWithKeyManager): self.lines = [line for line in self.EMAIL_DATA[4:12]] self.lines.append('') # add a trailing newline self.raw = '\r\n'.join(self.lines) - self.expected_body = ('\r\n'.join(self.EMAIL_DATA[9:12]) + - "\r\n\r\n--\r\nI prefer encrypted email - " - "https://leap.se/key/anotheruser\r\n") + self.expected_body = '\r\n'.join(self.EMAIL_DATA[9:12]) + "\r\n" self.fromAddr = ADDRESS_2 def init_outgoing_and_proto(_): -- cgit v1.2.3 From 3cd1e6ace64fad041873cb08b74789f9b78c50c9 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Mon, 26 Jan 2015 13:04:54 -0600 Subject: If not signature don't fail --- src/leap/mail/incoming/service.py | 2 +- src/leap/mail/incoming/tests/test_incoming_mail.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index 0b2f7c2..23c216c 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -435,7 +435,7 @@ class IncomingMail(Service): def add_leap_header(ret): decrmsg, signkey = ret - if (senderAddress is None or + if (senderAddress is None or signkey is None or isinstance(signkey, keymanager_errors.KeyNotFound)): decrmsg.add_header( self.LEAP_SIGNATURE_HEADER, diff --git a/src/leap/mail/incoming/tests/test_incoming_mail.py b/src/leap/mail/incoming/tests/test_incoming_mail.py index 0745ee0..f8652b3 100644 --- a/src/leap/mail/incoming/tests/test_incoming_mail.py +++ b/src/leap/mail/incoming/tests/test_incoming_mail.py @@ -46,6 +46,8 @@ from leap.soledad.common.crypto import ( ENC_SCHEME_KEY, ) +# TODO: add some tests for encrypted, unencrypted, signed and unsgined messages + class IncomingMailTestCase(TestCaseWithKeyManager): """ -- cgit v1.2.3 From 50afe275cdb1e030cf7caa231fa4ffce4b45588c Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Wed, 28 Jan 2015 14:24:32 -0600 Subject: Add public key as attachment --- changes/feature-6617_attach_public_key | 1 + src/leap/mail/outgoing/service.py | 42 +++++++++++++++++++- src/leap/mail/outgoing/tests/test_outgoing.py | 55 +++++++++++++++++++++++++-- src/leap/mail/smtp/gateway.py | 1 - 4 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 changes/feature-6617_attach_public_key diff --git a/changes/feature-6617_attach_public_key b/changes/feature-6617_attach_public_key new file mode 100644 index 0000000..49b444b --- /dev/null +++ b/changes/feature-6617_attach_public_key @@ -0,0 +1 @@ +- add public key as attachment (Closes: #6617) diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py index 88b8895..9777187 100644 --- a/src/leap/mail/outgoing/service.py +++ b/src/leap/mail/outgoing/service.py @@ -18,6 +18,8 @@ import re from StringIO import StringIO from email.parser import Parser from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText from OpenSSL import SSL @@ -250,10 +252,48 @@ class OutgoingMail: % (to_address, from_address)) signal(proto.SMTP_START_ENCRYPT_AND_SIGN, "%s,%s" % (self._from_address, to_address)) - d = self._encrypt_and_sign(origmsg, to_address, from_address) + d = self._maybe_attach_key(origmsg, from_address, to_address) + d.addCallback(self._encrypt_and_sign, to_address, from_address) d.addCallbacks(signal_encrypt_sign, if_key_not_found_send_unencrypted) return d + def _maybe_attach_key(self, origmsg, from_address, to_address): + filename = "%s-email-key.asc" % (from_address,) + + def attach_if_address_hasnt_encrypted(to_key): + # if the sign_used flag is true that means that we got an encrypted + # email from this address, because we conly check signatures on + # encrypted emails. In this case we don't attach. + # XXX: this might not be true some time in the future + if to_key.sign_used: + return origmsg + + d = self._keymanager.get_key(from_address, OpenPGPKey, + fetch_remote=False) + d.addCallback(attach_key) + return d + + def attach_key(from_key): + msg = origmsg + if not origmsg.is_multipart(): + msg = MIMEMultipart() + for h, v in origmsg.items(): + msg.add_header(h, v) + msg.attach(MIMEText(origmsg.get_payload())) + + keymsg = MIMEApplication(from_key.key_data, _subtype='pgp-keys', + _encoder=lambda x: x) + keymsg.add_header('content-disposition', 'attachment', + filename=filename) + msg.attach(keymsg) + return msg + + d = self._keymanager.get_key(to_address, OpenPGPKey, + fetch_remote=False) + d.addCallback(attach_if_address_hasnt_encrypted) + d.addErrback(lambda _: origmsg) + return d + def _encrypt_and_sign(self, origmsg, encrypt_address, sign_address): """ Create an RFC 3156 compliang PGP encrypted and signed message using diff --git a/src/leap/mail/outgoing/tests/test_outgoing.py b/src/leap/mail/outgoing/tests/test_outgoing.py index d7423b6..0eb05c8 100644 --- a/src/leap/mail/outgoing/tests/test_outgoing.py +++ b/src/leap/mail/outgoing/tests/test_outgoing.py @@ -21,6 +21,7 @@ SMTP gateway tests. """ import re +from email.parser import Parser from datetime import datetime from twisted.internet.defer import fail from twisted.mail.smtp import User @@ -33,10 +34,14 @@ from leap.mail.tests import ( TestCaseWithKeyManager, ADDRESS, ADDRESS_2, + PUBLIC_KEY_2, ) from leap.keymanager import openpgp, errors +BEGIN_PUBLIC_KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----" + + class TestOutgoingMail(TestCaseWithKeyManager): EMAIL_DATA = ['HELO gateway.leap.se', 'MAIL FROM: <%s>' % ADDRESS_2, @@ -71,7 +76,7 @@ class TestOutgoingMail(TestCaseWithKeyManager): self._km, self._config['encrypted_only'], self.outgoing_mail).buildProtocol(('127.0.0.1', 0)) - self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS) + self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS_2) d = TestCaseWithKeyManager.setUp(self) d.addCallback(init_outgoing_and_proto) @@ -88,7 +93,10 @@ class TestOutgoingMail(TestCaseWithKeyManager): decrypted, 'Decrypted text differs from plaintext.') - d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + d = self._set_sign_used(ADDRESS) + d.addCallback( + lambda _: + self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest)) d.addCallback(self._assert_encrypted) d.addCallback(lambda message: self._km.decrypt( message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey)) @@ -109,7 +117,10 @@ class TestOutgoingMail(TestCaseWithKeyManager): self.assertTrue(ADDRESS_2 in signkey.address, "Verification failed") - d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + d = self._set_sign_used(ADDRESS) + d.addCallback( + lambda _: + self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest)) d.addCallback(self._assert_encrypted) d.addCallback(lambda message: self._km.decrypt( message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey, @@ -158,7 +169,7 @@ class TestOutgoingMail(TestCaseWithKeyManager): def assert_verify(key): self.assertTrue(ADDRESS_2 in key.address, - 'Signature could not be verified.') + 'Signature could not be verified.') d = self._km.verify( signed_text, ADDRESS_2, openpgp.OpenPGPKey, @@ -171,6 +182,42 @@ class TestOutgoingMail(TestCaseWithKeyManager): d.addCallback(verify) return d + def test_attach_key(self): + def check_headers(message): + msgstr = message.as_string(unixfrom=False) + for header in self.EMAIL_DATA[4:8]: + self.assertTrue(header in msgstr, + "Missing header: %s" % (header,)) + return message + + def check_attachment((decrypted, _)): + msg = Parser().parsestr(decrypted) + for payload in msg.get_payload(): + if 'application/pgp-keys' == payload.get_content_type(): + keylines = PUBLIC_KEY_2.split('\n') + key = BEGIN_PUBLIC_KEY + '\n\n' + '\n'.join(keylines[4:-1]) + self.assertTrue(key in payload.get_payload(), + "Key attachment don't match") + return + self.fail("No public key attachment found") + + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + d.addCallback(self._assert_encrypted) + d.addCallback(check_headers) + d.addCallback(lambda message: self._km.decrypt( + message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey)) + d.addCallback(check_attachment) + return d + + def _set_sign_used(self, address): + def set_sign(key): + key.sign_used = True + return self._km.put_key(key, address) + + d = self._km.get_key(address, openpgp.OpenPGPKey, fetch_remote=False) + d.addCallback(set_sign) + return d + def _assert_encrypted(self, res): message, _ = res self.assertTrue('Content-Type' in message) diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index 9d78474..954a7d0 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -309,5 +309,4 @@ class EncryptedMessage(object): signal(proto.SMTP_CONNECTION_LOST, self._user.dest.addrstr) # unexpected loss of connection; don't save - self._lines = [] -- cgit v1.2.3 From b2c69f5d01c5d31967453a0de6bb55ec50c5b5e2 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Mon, 2 Feb 2015 14:21:31 -0600 Subject: Attach key for addresses without known key This seems to fix the problem with some headers dissapearing (#6692) --- src/leap/mail/outgoing/service.py | 43 ++++++++++------- src/leap/mail/outgoing/tests/test_outgoing.py | 68 ++++++++++++++++++--------- 2 files changed, 72 insertions(+), 39 deletions(-) diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py index 9777187..c881830 100644 --- a/src/leap/mail/outgoing/service.py +++ b/src/leap/mail/outgoing/service.py @@ -16,6 +16,7 @@ # along with this program. If not, see . import re from StringIO import StringIO +from copy import deepcopy from email.parser import Parser from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart @@ -33,7 +34,7 @@ from leap.common.check import leap_assert_type, leap_assert from leap.common.events import proto, signal from leap.keymanager import KeyManager from leap.keymanager.openpgp import OpenPGPKey -from leap.keymanager.errors import KeyNotFound +from leap.keymanager.errors import KeyNotFound, KeyAddressMismatch from leap.mail import __version__ from leap.mail.utils import validate_address from leap.mail.smtp.rfc3156 import MultipartEncrypted @@ -229,32 +230,37 @@ class OutgoingMail: username, domain = from_address.split('@') to_address = validate_address(recipient.dest.addrstr) + def maybe_encrypt_and_sign(message): + d = self._encrypt_and_sign(message, to_address, from_address) + d.addCallbacks(signal_encrypt_sign, + if_key_not_found_send_unencrypted, + errbackArgs=(message,)) + return d + def signal_encrypt_sign(newmsg): signal(proto.SMTP_END_ENCRYPT_AND_SIGN, "%s,%s" % (self._from_address, to_address)) return newmsg, recipient + def if_key_not_found_send_unencrypted(failure, message): + failure.trap(KeyNotFound, KeyAddressMismatch) + + log.msg('Will send unencrypted message to %s.' % to_address) + signal(proto.SMTP_START_SIGN, self._from_address) + d = self._sign(message, from_address) + d.addCallback(signal_sign) + return d + def signal_sign(newmsg): signal(proto.SMTP_END_SIGN, self._from_address) return newmsg, recipient - def if_key_not_found_send_unencrypted(failure): - if failure.check(KeyNotFound): - log.msg('Will send unencrypted message to %s.' % to_address) - signal(proto.SMTP_START_SIGN, self._from_address) - d = self._sign(origmsg, from_address) - d.addCallback(signal_sign) - return d - else: - return failure - log.msg("Will encrypt the message with %s and sign with %s." % (to_address, from_address)) signal(proto.SMTP_START_ENCRYPT_AND_SIGN, "%s,%s" % (self._from_address, to_address)) d = self._maybe_attach_key(origmsg, from_address, to_address) - d.addCallback(self._encrypt_and_sign, to_address, from_address) - d.addCallbacks(signal_encrypt_sign, if_key_not_found_send_unencrypted) + d.addCallback(maybe_encrypt_and_sign) return d def _maybe_attach_key(self, origmsg, from_address, to_address): @@ -267,7 +273,9 @@ class OutgoingMail: # XXX: this might not be true some time in the future if to_key.sign_used: return origmsg + return get_key_and_attach(None) + def get_key_and_attach(_): d = self._keymanager.get_key(from_address, OpenPGPKey, fetch_remote=False) d.addCallback(attach_key) @@ -290,7 +298,7 @@ class OutgoingMail: d = self._keymanager.get_key(to_address, OpenPGPKey, fetch_remote=False) - d.addCallback(attach_if_address_hasnt_encrypted) + d.addCallbacks(attach_if_address_hasnt_encrypted, get_key_and_attach) d.addErrback(lambda _: origmsg) return d @@ -386,7 +394,7 @@ class OutgoingMail: d.addCallback(create_signed_message) return d - def _fix_headers(self, origmsg, newmsg, sign_address): + def _fix_headers(self, msg, newmsg, sign_address): """ Move some headers from C{origmsg} to C{newmsg}, delete unwanted headers from C{origmsg} and add new headers to C{newms}. @@ -416,8 +424,8 @@ class OutgoingMail: - User-Agent - :param origmsg: The original message. - :type origmsg: email.message.Message + :param msg: The original message. + :type msg: email.message.Message :param newmsg: The new message being created. :type newmsg: email.message.Message :param sign_address: The address used to sign C{newmsg} @@ -428,6 +436,7 @@ class OutgoingMail: original Message with headers removed) :rtype: Deferred """ + origmsg = deepcopy(msg) # move headers from origmsg to newmsg headers = origmsg.items() passthrough = [ diff --git a/src/leap/mail/outgoing/tests/test_outgoing.py b/src/leap/mail/outgoing/tests/test_outgoing.py index 0eb05c8..2376da9 100644 --- a/src/leap/mail/outgoing/tests/test_outgoing.py +++ b/src/leap/mail/outgoing/tests/test_outgoing.py @@ -21,6 +21,7 @@ SMTP gateway tests. """ import re +from StringIO import StringIO from email.parser import Parser from datetime import datetime from twisted.internet.defer import fail @@ -29,6 +30,7 @@ from twisted.mail.smtp import User from mock import Mock from leap.mail.smtp.gateway import SMTPFactory +from leap.mail.smtp.rfc3156 import RFC3156CompliantGenerator from leap.mail.outgoing.service import OutgoingMail from leap.mail.tests import ( TestCaseWithKeyManager, @@ -149,8 +151,11 @@ class TestOutgoingMail(TestCaseWithKeyManager): message.get_param('protocol')) self.assertEqual('pgp-sha512', message.get_param('micalg')) # assert content of message + body = (message.get_payload(0) + .get_payload(0) + .get_payload(decode=True)) self.assertEqual(self.expected_body, - message.get_payload(0).get_payload(decode=True)) + body) # assert content of signature self.assertTrue( message.get_payload(1).get_payload().startswith( @@ -164,8 +169,12 @@ class TestOutgoingMail(TestCaseWithKeyManager): def verify(message): # replace EOL before verifying (according to rfc3156) + fp = StringIO() + g = RFC3156CompliantGenerator( + fp, mangle_from_=False, maxheaderlen=76) + g.flatten(message.get_payload(0)) signed_text = re.sub('\r?\n', '\r\n', - message.get_payload(0).as_string()) + fp.getvalue()) def assert_verify(key): self.assertTrue(ADDRESS_2 in key.address, @@ -183,32 +192,47 @@ class TestOutgoingMail(TestCaseWithKeyManager): return d def test_attach_key(self): - def check_headers(message): - msgstr = message.as_string(unixfrom=False) - for header in self.EMAIL_DATA[4:8]: - self.assertTrue(header in msgstr, - "Missing header: %s" % (header,)) - return message - - def check_attachment((decrypted, _)): - msg = Parser().parsestr(decrypted) - for payload in msg.get_payload(): - if 'application/pgp-keys' == payload.get_content_type(): - keylines = PUBLIC_KEY_2.split('\n') - key = BEGIN_PUBLIC_KEY + '\n\n' + '\n'.join(keylines[4:-1]) - self.assertTrue(key in payload.get_payload(), - "Key attachment don't match") - return - self.fail("No public key attachment found") - d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) d.addCallback(self._assert_encrypted) - d.addCallback(check_headers) + d.addCallback(self._check_headers, self.lines[:4]) d.addCallback(lambda message: self._km.decrypt( message.get_payload(1).get_payload(), ADDRESS, openpgp.OpenPGPKey)) - d.addCallback(check_attachment) + d.addCallback(lambda (decrypted, _): + self._check_key_attachment(Parser().parsestr(decrypted))) return d + def test_attach_key_not_known(self): + address = "someunknownaddress@somewhere.com" + lines = self.lines + lines[1] = "To: <%s>" % (address,) + raw = '\r\n'.join(lines) + dest = User(address, 'gateway.leap.se', self.proto, ADDRESS_2) + + d = self.outgoing_mail._maybe_encrypt_and_sign(raw, dest) + d.addCallback(lambda (message, _): + self._check_headers(message, lines[:4])) + d.addCallback(self._check_key_attachment) + return d + + def _check_headers(self, message, headers): + msgstr = message.as_string(unixfrom=False) + for header in headers: + self.assertTrue(header in msgstr, + "Missing header: %s" % (header,)) + return message + + def _check_key_attachment(self, message): + for payload in message.get_payload(): + if payload.is_multipart(): + return self._check_key_attachment(payload) + if 'application/pgp-keys' == payload.get_content_type(): + keylines = PUBLIC_KEY_2.split('\n') + key = BEGIN_PUBLIC_KEY + '\n\n' + '\n'.join(keylines[4:-1]) + self.assertTrue(key in payload.get_payload(decode=True), + "Key attachment don't match") + return + self.fail("No public key attachment found") + def _set_sign_used(self, address): def set_sign(key): key.sign_used = True -- cgit v1.2.3 From d7177ea01c968e5ee221aae90eb6e1f815c09972 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Thu, 5 Feb 2015 09:51:47 -0600 Subject: Fix incoming email decryption problems --- src/leap/mail/incoming/service.py | 5 +- src/leap/mail/incoming/tests/test_incoming_mail.py | 59 ++++++++++++++++++---- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index 23c216c..2902141 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -215,7 +215,6 @@ class IncomingMail(Service): """ def _log_synced(result): log.msg('FETCH soledad SYNCED.') - print "Result: ", result return result try: log.msg('FETCH: syncing soledad...') @@ -404,7 +403,6 @@ class IncomingMail(Service): :param doc: the SoledadDocument to delete :type doc: SoledadDocument """ - print "DELETING INCOMING MESSAGE" log.msg("Deleting Incoming message: %s" % (doc.doc_id,)) return self._soledad.delete_doc(doc) @@ -431,7 +429,7 @@ class IncomingMail(Service): if (fromHeader is not None and (msg.get_content_type() == MULTIPART_ENCRYPTED or msg.get_content_type() == MULTIPART_SIGNED)): - senderAddress = parseaddr(fromHeader) + senderAddress = parseaddr(fromHeader)[1] def add_leap_header(ret): decrmsg, signkey = ret @@ -709,7 +707,6 @@ class IncomingMail(Service): def msgSavedCallback(result): if not empty(result): leap_events.signal(IMAP_MSG_SAVED_LOCALLY) - print "DEFERRING THE DELETION ----->" return self._delete_incoming_message(doc) # TODO add notification as a callback #leap_events.signal(IMAP_MSG_DELETED_INCOMING) diff --git a/src/leap/mail/incoming/tests/test_incoming_mail.py b/src/leap/mail/incoming/tests/test_incoming_mail.py index f8652b3..0b1e696 100644 --- a/src/leap/mail/incoming/tests/test_incoming_mail.py +++ b/src/leap/mail/incoming/tests/test_incoming_mail.py @@ -35,9 +35,11 @@ from leap.mail.adaptors import soledad_indexes as fields from leap.mail.constants import INBOX_NAME from leap.mail.imap.account import IMAPAccount from leap.mail.incoming.service import IncomingMail +from leap.mail.smtp.rfc3156 import MultipartEncrypted, PGPEncrypted from leap.mail.tests import ( TestCaseWithKeyManager, ADDRESS, + ADDRESS_2, ) from leap.soledad.common.document import SoledadDocument from leap.soledad.common.crypto import ( @@ -54,7 +56,6 @@ class IncomingMailTestCase(TestCaseWithKeyManager): Tests for the incoming mail parser """ NICKSERVER = "http://domain" - FROM_ADDRESS = "test@somedomain.com" BODY = """ Governments of the Industrial World, you weary giants of flesh and steel, I come from Cyberspace, the new home of Mind. On behalf of the future, I ask @@ -67,23 +68,25 @@ subject: independence of cyberspace %(body)s """ % { - "from": FROM_ADDRESS, + "from": ADDRESS_2, "to": ADDRESS, "body": BODY } def setUp(self): def getInbox(_): - theAccount = IMAPAccount(ADDRESS, self._soledad) - return theAccount.callWhenReady( + d = defer.Deferred() + theAccount = IMAPAccount(ADDRESS, self._soledad, d=d) + d.addCallback( lambda _: theAccount.getMailbox(INBOX_NAME)) + return d def setUpFetcher(inbox): # Soledad sync makes trial block forever. The sync it's mocked to # fix this problem. _mock_soledad_get_from_index can be used from # the tests to provide documents. # TODO ---- see here http://www.pythoneye.com/83_20424875/ - self._soledad.sync = Mock() + self._soledad.sync = Mock(return_value=defer.succeed(None)) self.fetcher = IncomingMail( self._km, @@ -91,6 +94,9 @@ subject: independence of cyberspace inbox, ADDRESS) + # The messages don't exist on soledad will fail on deletion + self.fetcher._delete_incoming_message = Mock(return_value=None) + d = super(IncomingMailTestCase, self).setUp() d.addCallback(getInbox) d.addCallback(setUpFetcher) @@ -104,7 +110,7 @@ subject: independence of cyberspace """ Test the OpenPGP header key extraction """ - KEYURL = "https://somedomain.com/key.txt" + KEYURL = "https://leap.se/key.txt" OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) message = Parser().parsestr(self.EMAIL) @@ -114,7 +120,7 @@ subject: independence of cyberspace def fetch_key_called(ret): self.fetcher._keymanager.fetch_key.assert_called_once_with( - self.FROM_ADDRESS, KEYURL, OpenPGPKey) + ADDRESS_2, KEYURL, OpenPGPKey) d = self._create_incoming_email(message.as_string()) d.addCallback( @@ -153,7 +159,7 @@ subject: independence of cyberspace KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." message = MIMEMultipart() - message.add_header("from", self.FROM_ADDRESS) + message.add_header("from", ADDRESS_2) key = MIMEApplication("", "pgp-keys") key.set_payload(KEY) message.attach(key) @@ -162,12 +168,47 @@ subject: independence of cyberspace def put_raw_key_called(_): self.fetcher._keymanager.put_raw_key.assert_called_once_with( - KEY, OpenPGPKey, address=self.FROM_ADDRESS) + KEY, OpenPGPKey, address=ADDRESS_2) d = self._mock_fetch(message.as_string()) d.addCallback(put_raw_key_called) return d + def testDecryptEmail(self): + self.fetcher._decryption_error = Mock() + + def create_encrypted_message(encstr): + message = Parser().parsestr(self.EMAIL) + newmsg = MultipartEncrypted('application/pgp-encrypted') + for hkey, hval in message.items(): + newmsg.add_header(hkey, hval) + + encmsg = MIMEApplication( + encstr, _subtype='octet-stream', _encoder=lambda x: x) + encmsg.add_header('content-disposition', 'attachment', + filename='msg.asc') + # create meta message + metamsg = PGPEncrypted() + metamsg.add_header('Content-Disposition', 'attachment') + # attach pgp message parts to new message + newmsg.attach(metamsg) + newmsg.attach(encmsg) + return newmsg + + def decryption_error_not_called(_): + self.assertFalse(self.fetcher._decyption_error.called, + "There was some errors with decryption") + + d = self._km.encrypt( + self.EMAIL, + ADDRESS, OpenPGPKey, sign=ADDRESS_2) + d.addCallback(create_encrypted_message) + d.addCallback( + lambda message: + self._mock_fetch(message.as_string())) + + return d + def _mock_fetch(self, message): self.fetcher._keymanager.fetch_key = Mock() d = self._create_incoming_email(message) -- cgit v1.2.3 From 87c15e3ec5c1ca607890bf72b0d90bb0f114cbb1 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Wed, 18 Feb 2015 10:24:03 -0600 Subject: Add callbacks to inbox insertions --- src/leap/mail/incoming/service.py | 38 ++++++++++++++++------ src/leap/mail/incoming/tests/test_incoming_mail.py | 24 +++++++++++--- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index 2902141..fadfd9f 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -135,12 +135,26 @@ class IncomingMail(Service): self._inbox = inbox self._userid = userid + self._listeners = [] self._loop = None self._check_period = check_period # initialize a mail parser only once self._parser = Parser() + def add_listener(self, listener): + """ + Add a listener to inbox insertions. + + This listener function will be called for each message added to the + inbox with its uid as parameter. This function should not be blocking + or it will block the incoming queue. + + :param listener: the listener function + :type listener: callable + """ + self._listeners.append(listener) + # # Public API: fetch, start_loop, stop. # @@ -699,17 +713,21 @@ class IncomingMail(Service): doc, data = msgtuple log.msg('adding message %s to local db' % (doc.doc_id,)) - #if isinstance(data, list): - #if empty(data): - #return False - #data = data[0] - def msgSavedCallback(result): - if not empty(result): - leap_events.signal(IMAP_MSG_SAVED_LOCALLY) - return self._delete_incoming_message(doc) - # TODO add notification as a callback - #leap_events.signal(IMAP_MSG_DELETED_INCOMING) + if empty(result): + return + + for listener in self._listeners: + listener(result) + + def signal_deleted(doc_id): + leap_events.signal(IMAP_MSG_DELETED_INCOMING) + return doc_id + + leap_events.signal(IMAP_MSG_SAVED_LOCALLY) + d = self._delete_incoming_message(doc) + d.addCallback(signal_deleted) + return d d = self._inbox.addMessage(data, (self.RECENT_FLAG,)) d.addCallbacks(msgSavedCallback, self._errback) diff --git a/src/leap/mail/incoming/tests/test_incoming_mail.py b/src/leap/mail/incoming/tests/test_incoming_mail.py index 0b1e696..a932a95 100644 --- a/src/leap/mail/incoming/tests/test_incoming_mail.py +++ b/src/leap/mail/incoming/tests/test_incoming_mail.py @@ -95,7 +95,8 @@ subject: independence of cyberspace ADDRESS) # The messages don't exist on soledad will fail on deletion - self.fetcher._delete_incoming_message = Mock(return_value=None) + self.fetcher._delete_incoming_message = Mock( + return_value=defer.succeed(None)) d = super(IncomingMailTestCase, self).setUp() d.addCallback(getInbox) @@ -165,12 +166,13 @@ subject: independence of cyberspace message.attach(key) self.fetcher._keymanager.put_raw_key = Mock( return_value=defer.succeed(None)) + self.fetcher._keymanager.fetch_key = Mock() def put_raw_key_called(_): self.fetcher._keymanager.put_raw_key.assert_called_once_with( KEY, OpenPGPKey, address=ADDRESS_2) - d = self._mock_fetch(message.as_string()) + d = self._do_fetch(message.as_string()) d.addCallback(put_raw_key_called) return d @@ -205,12 +207,24 @@ subject: independence of cyberspace d.addCallback(create_encrypted_message) d.addCallback( lambda message: - self._mock_fetch(message.as_string())) + self._do_fetch(message.as_string())) + return d + + def testListener(self): + self.called = False + def listener(uid): + self.called = True + + def listener_called(_): + self.assertTrue(self.called) + + self.fetcher.add_listener(listener) + d = self._do_fetch(self.EMAIL) + d.addCallback(listener_called) return d - def _mock_fetch(self, message): - self.fetcher._keymanager.fetch_key = Mock() + def _do_fetch(self, message): d = self._create_incoming_email(message) d.addCallback( lambda email: -- cgit v1.2.3 From 2af7083d4543be9a1b1476de75fafb17bfe78b95 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Wed, 18 Feb 2015 11:33:42 -0600 Subject: Use MessageCollection instead of IMAPMailbox in IncomingMail --- src/leap/mail/imap/mailbox.py | 42 +---------------- src/leap/mail/incoming/service.py | 8 ++-- src/leap/mail/incoming/tests/test_incoming_mail.py | 2 +- src/leap/mail/mail.py | 53 ++++++++++++++++++++++ 4 files changed, 60 insertions(+), 45 deletions(-) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 9ec6ea8..2f33aa0 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -19,13 +19,9 @@ IMAP Mailbox. """ import re import logging -import StringIO -import cStringIO -import time import os from collections import defaultdict -from email.utils import formatdate from twisted.internet import defer from twisted.internet import reactor @@ -36,7 +32,7 @@ from zope.interface import implements from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL -from leap.common.check import leap_assert, leap_assert_type +from leap.common.check import leap_assert from leap.mail.constants import INBOX_NAME, MessageFlags from leap.mail.imap.messages import IMAPMessage @@ -326,41 +322,7 @@ class IMAPMailbox(object): :return: a deferred that will be triggered with the UID of the added message. """ - # TODO should raise ReadOnlyMailbox if not rw. - # TODO have a look at the cases for internal date in the rfc - if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)): - message = message.getvalue() - - # XXX we could treat the message as an IMessage from here - leap_assert_type(message, basestring) - - if flags is None: - flags = tuple() - else: - flags = tuple(str(flag) for flag in flags) - - if date is None: - date = formatdate(time.time()) - - # A better place for this would be the COPY/APPEND dispatcher - # if PROFILE_CMD: - # do_profile_cmd(d, "APPEND") - - # just_mdoc=True: feels HACKY, but improves a *lot* the responsiveness - # of the APPENDS: we just need to be notified when the mdoc - # is saved, and let's hope that the other parts are doing just fine. - # This will not catch any errors when the inserts of the other parts - # fail, but on the other hand allows us to return very quickly, which - # seems a good compromise given that we have to serialize the appends. - # A better solution will probably involve implementing MULTIAPPEND - # or patching imap server to support pipelining. - - d = self.collection.add_msg(message, flags=flags, date=date, - notify_just_mdoc=True) - - # XXX signal to UI? --- should do it only if INBOX... - d.addErrback(lambda f: log.msg(f.getTraceback())) - return d + return self.collection.add_raw_msg(message, flags, date) def notify_new(self, *args): """ diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index fadfd9f..8b5c371 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -97,7 +97,7 @@ class IncomingMail(Service): LEAP_SIGNATURE_HEADER = 'X-Leap-Signature' """ - Header added to messages when they are decrypted by the IMAP fetcher, + Header added to messages when they are decrypted by the fetcher, which states the validity of an eventual signature that might be included in the encrypted blob. """ @@ -118,7 +118,7 @@ class IncomingMail(Service): :type soledad: Soledad :param inbox: the inbox where the new emails will be stored - :type inbox: IMAPMailbox + :type inbox: MessageCollection :param check_period: the period to fetch new mail, in seconds. :type check_period: int @@ -266,7 +266,7 @@ class IncomingMail(Service): Sends unread event to ui. """ leap_events.signal( - IMAP_UNREAD_MAIL, str(self._inbox.getUnseenCount())) + IMAP_UNREAD_MAIL, str(self._inbox.count_unseen())) # process incoming mail. @@ -729,7 +729,7 @@ class IncomingMail(Service): d.addCallback(signal_deleted) return d - d = self._inbox.addMessage(data, (self.RECENT_FLAG,)) + d = self._inbox.add_raw_message(data, (self.RECENT_FLAG,)) d.addCallbacks(msgSavedCallback, self._errback) return d diff --git a/src/leap/mail/incoming/tests/test_incoming_mail.py b/src/leap/mail/incoming/tests/test_incoming_mail.py index a932a95..f43f746 100644 --- a/src/leap/mail/incoming/tests/test_incoming_mail.py +++ b/src/leap/mail/incoming/tests/test_incoming_mail.py @@ -91,7 +91,7 @@ subject: independence of cyberspace self.fetcher = IncomingMail( self._km, self._soledad, - inbox, + inbox.collection, ADDRESS) # The messages don't exist on soledad will fail on deletion diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index d74f6b8..b7b0981 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -20,7 +20,10 @@ Generic Access to Mail objects: Public LEAP Mail API. import uuid import logging import StringIO +import cStringIO +import time +from email.utils import formatdate from twisted.internet import defer from leap.common.check import leap_assert_type @@ -522,6 +525,56 @@ class MessageCollection(object): d.addErrback(lambda f: f.printTraceback()) return d + def add_raw_message(self, message, flags, date=None): + """ + Adds a message to this collection. + + :param message: the raw message + :type message: str + + :param flags: flag list + :type flags: list of str + + :param date: timestamp + :type date: str + + :return: a deferred that will be triggered with the UID of the added + message. + """ + # TODO should raise ReadOnlyMailbox if not rw. + # TODO have a look at the cases for internal date in the rfc + if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)): + message = message.getvalue() + + # XXX we could treat the message as an IMessage from here + leap_assert_type(message, basestring) + + if flags is None: + flags = tuple() + else: + flags = tuple(str(flag) for flag in flags) + + if date is None: + date = formatdate(time.time()) + + # A better place for this would be the COPY/APPEND dispatcher + # if PROFILE_CMD: + # do_profile_cmd(d, "APPEND") + + # just_mdoc=True: feels HACKY, but improves a *lot* the responsiveness + # of the APPENDS: we just need to be notified when the mdoc + # is saved, and let's hope that the other parts are doing just fine. + # This will not catch any errors when the inserts of the other parts + # fail, but on the other hand allows us to return very quickly, which + # seems a good compromise given that we have to serialize the appends. + # A better solution will probably involve implementing MULTIAPPEND + # or patching imap server to support pipelining. + + d = self.add_msg(message, flags=flags, date=date, + notify_just_mdoc=True) + d.addErrback(lambda f: logger.warning(f.getTraceback())) + return d + def copy_msg(self, msg, new_mbox_uuid): """ Copy the message to another collection. (it only makes sense for -- cgit v1.2.3 From 6fb497a9bdf32ddba5e91ad7f049bda9588eefb6 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Thu, 19 Feb 2015 10:35:56 -0600 Subject: Add missing changes about IncomingMail --- changes/feature-6598_refactor_incoming_mail | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/feature-6598_refactor_incoming_mail diff --git a/changes/feature-6598_refactor_incoming_mail b/changes/feature-6598_refactor_incoming_mail new file mode 100644 index 0000000..1db8c28 --- /dev/null +++ b/changes/feature-6598_refactor_incoming_mail @@ -0,0 +1,2 @@ +- Refactor email fetching outside IMAP to it's own independient IncomingMail class (Closes: #6361) +- Add listener for each email added to inbox in IncomingMail (Closes: #6742) -- cgit v1.2.3 From 68e28e1590fcddf4937afd9de340542aff8c6868 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 13 Feb 2015 12:38:19 -0400 Subject: update release in docs --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 51e9d66..95d919b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -50,7 +50,7 @@ master_doc = 'index' # General information about the project. project = u'leap.mail' -copyright = u'2014, Kali Kaneko' +copyright = u'2014-2015, The LEAP Encryption Access Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -59,7 +59,7 @@ copyright = u'2014, Kali Kaneko' # The short X.Y version. version = '0.4.0alpha1' # The full version, including alpha/beta/rc tags. -release = '0.3.9' +release = '0.4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -- cgit v1.2.3 From 1a0c34b145f0dfc1ea34d3670d93b4a0ce9ae4db Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 19 Feb 2015 00:58:17 -0400 Subject: change environment variable for mail config --- docs/hacking.rst | 6 ++--- src/leap/mail/imap/service/imap-server.tac | 38 ++++++++---------------------- 2 files changed, 13 insertions(+), 31 deletions(-) diff --git a/docs/hacking.rst b/docs/hacking.rst index daa3762..bd9f792 100644 --- a/docs/hacking.rst +++ b/docs/hacking.rst @@ -33,7 +33,7 @@ Profiling If using ``twistd`` to launch the server, you can use twisted profiling capabities:: - LEAP_MAIL_CONF=~/.leapmailrc twistd --profile=/tmp/mail-profiling -n -y imap-server.tac + LEAP_MAIL_CONFIG=~/.leapmailrc twistd --profile=/tmp/mail-profiling -n -y imap-server.tac ``--profiler`` option allows you to select different profilers (default is "hotshot"). @@ -63,7 +63,7 @@ need a config with this info:: uuid = "deadbeefdeadabad" passwd = "foobar" # Optional -In the ``LEAP_MAIL_CONF`` enviroment variable. If you do not specify a password +In the ``LEAP_MAIL_CONFIG`` enviroment variable. If you do not specify a password parameter, you'll be prompted for it. In order to get the user uid (uuid), look into the @@ -72,7 +72,7 @@ provider at least once. Run the twisted service:: - LEAP_IMAP_CONFIG=~/.leapmailrc twistd -n -y imap-server.tac + LEAP_MAIL_CONFIG=~/.leapmailrc twistd -n -y imap-server.tac Now you can telnet into your local IMAP server and read your mail like a real programmer™:: diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac index 651f71b..49a29b1 100644 --- a/src/leap/mail/imap/service/imap-server.tac +++ b/src/leap/mail/imap/service/imap-server.tac @@ -23,9 +23,9 @@ For now, and for debugging/testing purposes, you need to pass a config file with the following structure: [leap_mail] -userid = "user@provider" -uuid = "deadbeefdeadabad" -passwd = "supersecret" # optional, will get prompted if not found. +userid = 'user@provider' +uuid = 'deadbeefdeadabad' +passwd = 'supersecret' # optional, will get prompted if not found. """ import ConfigParser import getpass @@ -53,38 +53,17 @@ def initialize_soledad(uuid, email, passwd, :param tempdir: path to temporal dir :rtype: Soledad instance """ - # XXX TODO unify with an authoritative source of mocks - # for soledad (or partial initializations). - # This is copied from the imap tests. - server_url = "http://provider" cert_file = "" - class Mock(object): - def __init__(self, return_value=None): - self._return = return_value - - def __call__(self, *args, **kwargs): - return self._return - - class MockSharedDB(object): - - get_doc = Mock() - put_doc = Mock() - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) - - def __call__(self): - return self - - Soledad._shared_db = MockSharedDB() soledad = Soledad( uuid, passwd, secrets, localdb, server_url, - cert_file) + cert_file, + syncable=False) return soledad @@ -95,9 +74,9 @@ def initialize_soledad(uuid, email, passwd, print "[+] Running LEAP IMAP Service" -bmconf = os.environ.get("LEAP_MAIL_CONF", "") +bmconf = os.environ.get("LEAP_MAIL_CONFIG", "") if not bmconf: - print ("[-] Please set LEAP_MAIL_CONF environment variable " + print ("[-] Please set LEAP_MAIL_CONFIG environment variable " "pointing to your config.") sys.exit(1) @@ -131,6 +110,7 @@ tempdir = "/tmp/" # Ad-hoc soledad/keymanager initialization. +print "[~] user:", userid soledad = initialize_soledad(uuid, userid, passwd, secrets, localdb, gnupg_home, tempdir) km_args = (userid, "https://localhost", soledad) @@ -144,6 +124,8 @@ km_kwargs = { } keymanager = KeyManager(*km_args, **km_kwargs) +# XXX Do we need to wait until keymanager is properly initialized? + ################################################## # Ok, let's expose the application object for the twistd application -- cgit v1.2.3 From 6eb1f599d49adc860efba3ccc30b15035dcaa6b0 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 19 Feb 2015 00:58:52 -0400 Subject: fix broken multipart attachment rendering --- src/leap/mail/imap/mailbox.py | 7 +++++++ src/leap/mail/imap/service/imap-server.tac | 2 -- src/leap/mail/imap/service/imap.py | 4 +++- src/leap/mail/mail.py | 25 ++++++++++++++++--------- src/leap/mail/outgoing/service.py | 5 ++++- 5 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 2f33aa0..499226c 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -463,6 +463,13 @@ class IMAPMailbox(object): # switch to content-hash based index + local UID table. is_sequence = True if uid == 0 else False + + # XXX DEBUG --- if you attempt to use the `getmail` utility under + # imap/tests, it will choke until we implement sequence numbers. This + # is an easy hack meanwhile. + # is_sequence = False + # ----------------------------------------------------------------- + getmsg = self.collection.get_message_by_uid getimapmsg = self.get_imap_message diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac index 49a29b1..2045757 100644 --- a/src/leap/mail/imap/service/imap-server.tac +++ b/src/leap/mail/imap/service/imap-server.tac @@ -124,8 +124,6 @@ km_kwargs = { } keymanager = KeyManager(*km_args, **km_kwargs) -# XXX Do we need to wait until keymanager is properly initialized? - ################################################## # Ok, let's expose the application object for the twistd application diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index cc76e3a..3a6b7b8 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -136,7 +136,9 @@ def run_service(store, **kwargs): :returns: the port as returned by the reactor when starts listening, and the factory for the protocol. """ - leap_assert_type(store, Soledad) + leap_check(store, "store cannot be None") + # XXX this can also be a ProxiedObject, FIXME + # leap_assert_type(store, Soledad) port = kwargs.get('port', IMAP_PORT) userid = kwargs.get('userid', None) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index b7b0981..47c8cba 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -63,7 +63,7 @@ class MessagePart(object): # TODO support arbitrarily nested multiparts (right now we only support # the trivial case) - def __init__(self, part_map, index=1, cdocs={}): + def __init__(self, part_map, cdocs={}): """ :param part_map: a dictionary mapping the subparts for this MessagePart (1-indexed). @@ -71,24 +71,29 @@ class MessagePart(object): The format for the part_map is as follows: - {u'1': {u'ctype': u'text/plain', + {u'ctype': u'text/plain', u'headers': [[u'Content-Type', u'text/plain; charset="utf-8"'], [u'Content-Transfer-Encoding', u'8bit']], u'multi': False, u'parts': 1, u'phash': u'02D82B29F6BB0C8612D1C', - u'size': 132}} + u'size': 132} - :param index: which index in the content-doc is this subpart - representing. :param cdocs: optional, a reference to the top-level dict of wrappers for content-docs (1-indexed). """ - # TODO: Pass only the cdoc wrapper for this part. self._pmap = part_map - self._index = index self._cdocs = cdocs + index = 1 + phash = part_map.get('phash', None) + if phash: + for i, cdoc_wrapper in self._cdocs.items(): + if cdoc_wrapper.phash == phash: + index = i + break + self._index = index + def get_size(self): return self._pmap['size'] @@ -121,7 +126,7 @@ class MessagePart(object): except KeyError: logger.debug("getSubpart for %s: KeyError" % (part,)) raise IndexError - return MessagePart(part_map, cdocs={1: self._cdocs.get(1, {})}) + return MessagePart(part_map, cdocs={1: self._cdocs.get(part + 1, {})}) def _get_payload(self, index): cdoc_wrapper = self._cdocs.get(index, None) @@ -245,8 +250,10 @@ class Message(object): except KeyError: raise IndexError + # FIXME instead of passing the index, let the MessagePart figure it out + # by getting the phash and iterating through the cdocs return MessagePart( - subpart_dict, index=part_index, cdocs=self._wrapper.cdocs) + subpart_dict, cdocs=self._wrapper.cdocs) # Custom methods. diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py index c881830..f3c2320 100644 --- a/src/leap/mail/outgoing/service.py +++ b/src/leap/mail/outgoing/service.py @@ -91,7 +91,10 @@ class OutgoingMail: # assert params leap_assert_type(from_address, str) leap_assert('@' in from_address) - leap_assert_type(keymanager, KeyManager) + + # XXX it can be a zope.proxy too + # leap_assert_type(keymanager, KeyManager) + leap_assert_type(host, str) leap_assert(host != '') leap_assert_type(port, int) -- cgit v1.2.3 From 209d433d92d867b9fb6df6b444930ce96804b060 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 20 Feb 2015 11:28:35 -0400 Subject: notify MUA on closing connection --- changes/feature_send_bye | 1 + src/leap/mail/imap/server.py | 14 ++++++++++++++ src/leap/mail/imap/service/imap.py | 19 ++++++++++++++++--- 3 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 changes/feature_send_bye diff --git a/changes/feature_send_bye b/changes/feature_send_bye new file mode 100644 index 0000000..5bc3e60 --- /dev/null +++ b/changes/feature_send_bye @@ -0,0 +1 @@ +- Send a BYE command to all open connections, so that the MUA is notified when the server is shutted down. diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index abe16be..3e10171 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -80,6 +80,20 @@ class LEAPIMAPServer(imap4.IMAP4Server): log.msg('rcv (%s): %s' % (self.state, msg)) imap4.IMAP4Server.lineReceived(self, line) + def close_server_connection(self): + """ + Send a BYE command so that the MUA at least knows that we're closing + the connection. + """ + self.sendLine( + '* BYE LEAP IMAP Proxy is shutting down; ' + 'so long and thanks for all the fish') + self.transport.loseConnection() + if self.mbox: + self.mbox.removeListener(self) + self.mbox = None + self.state = 'unauth' + def authenticateLogin(self, username, password): """ Lookup the account with the given parameters, and deny diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 3a6b7b8..b3282d4 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -21,6 +21,8 @@ IMAP service initialization import logging import os +from collections import defaultdict + from twisted.internet import reactor from twisted.internet.error import CannotListenError from twisted.internet.protocol import ServerFactory @@ -70,6 +72,7 @@ class LeapIMAPFactory(ServerFactory): Factory for a IMAP4 server with soledad remote sync and gpg-decryption capabilities. """ + protocol = LEAPIMAPServer def __init__(self, uuid, userid, soledad): """ @@ -91,6 +94,7 @@ class LeapIMAPFactory(ServerFactory): theAccount = IMAPAccount(uuid, soledad) self.theAccount = theAccount + self._connections = defaultdict() # XXX how to pass the store along? def buildProtocol(self, addr): @@ -100,15 +104,25 @@ class LeapIMAPFactory(ServerFactory): :param addr: remote ip address :type addr: str """ - # XXX addr not used??! - imapProtocol = LEAPIMAPServer( + # TODO should reject anything from addr != localhost, + # just in case. + log.msg("Building protocol for connection %s" % addr) + imapProtocol = self.protocol( uuid=self._uuid, userid=self._userid, soledad=self._soledad) imapProtocol.theAccount = self.theAccount imapProtocol.factory = self + + self._connections[addr] = imapProtocol return imapProtocol + def stopFactory(self): + # say bye! + for conn, proto in self._connections.items(): + log.msg("Closing connections for %s" % conn) + proto.close_server_connection() + def doStop(self): """ Stops imap service (fetcher, factory and port). @@ -157,7 +171,6 @@ def run_service(store, **kwargs): logger.error("Error launching IMAP service: %r" % (exc,)) else: # all good. - # (the caller has still to call fetcher.start_loop) if DO_MANHOLE: # TODO get pass from env var.too. -- cgit v1.2.3 From b46fbeda9ea33d1dad01af7f74b9856dd32cf855 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 20 Feb 2015 16:07:49 -0400 Subject: fix typo on method name --- src/leap/mail/imap/mailbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 499226c..c29d572 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -322,7 +322,7 @@ class IMAPMailbox(object): :return: a deferred that will be triggered with the UID of the added message. """ - return self.collection.add_raw_msg(message, flags, date) + return self.collection.add_raw_message(message, flags, date) def notify_new(self, *args): """ -- cgit v1.2.3 From 9a52e73518c0918ff8bb372127e9c6aab02b496d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 20 Feb 2015 16:09:38 -0400 Subject: factor out unicode formatting --- src/leap/mail/mail.py | 62 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 47c8cba..9a32483 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -33,6 +33,7 @@ from leap.mail.adaptors.soledad import SoledadMailAdaptor from leap.mail.constants import INBOX_NAME from leap.mail.constants import MessageFlags from leap.mail.mailbox_indexer import MailboxIndexer +from leap.mail.utils import find_charset logger = logging.getLogger(name=__name__) @@ -58,6 +59,45 @@ def _write_and_rewind(payload): return fd +def _encode_payload(payload, ctype=""): + """ + Properly encode an unicode payload (which can be string or unicode) as a + string. + + :param payload: the payload to encode. currently soledad returns unicode + strings. + :type payload: basestring + :param ctype: optional, the content of the content-type header for this + payload. + :type ctype: str + :rtype: str + """ + # TODO Related, it's proposed that we're able to pass + # the encoding to the soledad documents. Better to store the charset there? + # FIXME ----------------------------------------------- + # this need a dedicated test-suite + charset = find_charset(ctype) + + # XXX get from mail headers if not multipart! + # Beware also that we should pass the proper encoding to + # soledad when it's creating the documents. + # if not charset: + # charset = get_email_charset(payload) + #------------------------------------------------------ + + if not charset: + charset = "utf-8" + + try: + if isinstance(payload, unicode): + payload = payload.encode(charset) + except UnicodeError as exc: + logger.error( + "Unicode error, using 'replace'. {0!r}".format(exc)) + payload = payload.encode(charset, 'replace') + return payload + + class MessagePart(object): # TODO This class should be better abstracted from the data model. # TODO support arbitrarily nested multiparts (right now we only support @@ -107,7 +147,7 @@ class MessagePart(object): # XXX uh, multi also... should recurse" raise NotImplementedError if payload: - payload = self._format_payload(payload) + payload = _encode_payload(payload) return _write_and_rewind(payload) def get_headers(self): @@ -134,23 +174,6 @@ class MessagePart(object): return cdoc_wrapper.raw return "" - def _format_payload(self, payload): - # FIXME ----------------------------------------------- - # Test against unicode payloads... - # content_type = self._get_ctype_from_document(phash) - # charset = find_charset(content_type) - charset = None - if charset is None: - charset = get_email_charset(payload) - try: - if isinstance(payload, unicode): - payload = payload.encode(charset) - except UnicodeError as exc: - logger.error( - "Unicode error, using 'replace'. {0!r}".format(exc)) - payload = payload.encode(charset, 'replace') - return payload - class Message(object): """ @@ -218,6 +241,9 @@ class Message(object): """ def write_and_rewind_if_found(cdoc): payload = cdoc.raw if cdoc else "" + # XXX pass ctype from headers if not multipart? + if payload: + payload = _encode_payload(payload, ctype=cdoc.content_type) return _write_and_rewind(payload) d = defer.maybeDeferred(self._wrapper.get_body, store) -- cgit v1.2.3 From 3570f697335814a768f35bb984498f31d9de7ace Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 24 Feb 2015 23:05:48 -0400 Subject: move notifications cb to mail module --- src/leap/mail/imap/mailbox.py | 27 ++------------------------- src/leap/mail/mail.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index c29d572..3769a3e 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -30,8 +30,6 @@ from twisted.python import log from twisted.mail import imap4 from zope.interface import implements -from leap.common import events as leap_events -from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert from leap.mail.constants import INBOX_NAME, MessageFlags from leap.mail.imap.messages import IMAPMessage @@ -340,7 +338,7 @@ class IMAPMailbox(object): d = self._get_notify_count() d.addCallback(cbNotifyNew) - d.addCallback(self.cb_signal_unread_to_ui) + d.addCallback(self.collection.cb_signal_unread_to_ui) def _get_notify_count(self): """ @@ -512,8 +510,6 @@ class IMAPMailbox(object): d = self._get_messages_range(messages_asked) d.addCallback(get_imap_messages_for_range) - # TODO -- call signal_to_ui - # d.addCallback(self.cb_signal_unread_to_ui) return d def _get_messages_range(self, messages_asked): @@ -659,25 +655,6 @@ class IMAPMailbox(object): for msgid in seq_messg) return result - def cb_signal_unread_to_ui(self, result): - """ - Sends unread event to ui. - Used as a callback in several commands. - - :param result: ignored - """ - d = defer.maybeDeferred(self.getUnseenCount) - d.addCallback(self.__cb_signal_unread_to_ui) - return result - - def __cb_signal_unread_to_ui(self, unseen): - """ - Send the unread signal to UI. - :param unseen: number of unseen messages. - :type unseen: int - """ - leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) - def store(self, messages_asked, flags, mode, uid): """ Sets the flags of one or more messages. @@ -722,7 +699,7 @@ class IMAPMailbox(object): mode, uid, d) if PROFILE_CMD: do_profile_cmd(d, "STORE") - d.addCallback(self.cb_signal_unread_to_ui) + d.addCallback(self.collection.cb_signal_unread_to_ui) d.addErrback(lambda f: log.msg(f.getTraceback())) return d diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 9a32483..3127ef5 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -27,6 +27,8 @@ from email.utils import formatdate from twisted.internet import defer from leap.common.check import leap_assert_type +from leap.common import events as leap_events +from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.mail import get_email_charset from leap.mail.adaptors.soledad import SoledadMailAdaptor @@ -606,8 +608,30 @@ class MessageCollection(object): d = self.add_msg(message, flags=flags, date=date, notify_just_mdoc=True) d.addErrback(lambda f: logger.warning(f.getTraceback())) + d.addCallback(self.cb_signal_unread_to_ui) return d + def cb_signal_unread_to_ui(self, result): + """ + Sends unread event to ui. + Used as a callback in several commands. + + :param result: ignored + """ + if self.mbox_name.lower() == "inbox": + d = defer.maybeDeferred(self.count_unseen) + d.addCallback(self.__cb_signal_unread_to_ui) + return result + + def __cb_signal_unread_to_ui(self, unseen): + """ + Send the unread signal to UI. + :param unseen: number of unseen messages. + :type unseen: int + """ + # TODO change name of the signal, non-imap now. + leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) + def copy_msg(self, msg, new_mbox_uuid): """ Copy the message to another collection. (it only makes sense for -- cgit v1.2.3 From 70a42a8de216468175048bfc517035aa66ce9e95 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 24 Feb 2015 23:54:56 -0400 Subject: undo duplication of add_msg method in mail api * Set the internal date from within the incoming mail service. --- src/leap/mail/imap/mailbox.py | 39 ++++++++++++++++++-- src/leap/mail/incoming/service.py | 14 ++++--- src/leap/mail/mail.py | 77 +++++++++++---------------------------- 3 files changed, 66 insertions(+), 64 deletions(-) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 3769a3e..c501614 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -20,8 +20,12 @@ IMAP Mailbox. import re import logging import os +import cStringIO +import StringIO +import time from collections import defaultdict +from email.utils import formatdate from twisted.internet import defer from twisted.internet import reactor @@ -31,6 +35,7 @@ from twisted.mail import imap4 from zope.interface import implements from leap.common.check import leap_assert +from leap.common.check import leap_assert_type from leap.mail.constants import INBOX_NAME, MessageFlags from leap.mail.imap.messages import IMAPMessage @@ -50,7 +55,6 @@ NOTIFY_NEW = not os.environ.get('LEAP_SKIPNOTIFY', False) PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False) if PROFILE_CMD: - import time def _debugProfiling(result, cmdname, start): took = (time.time() - start) * 1000 @@ -315,12 +319,41 @@ class IMAPMailbox(object): :type flags: list of str :param date: timestamp - :type date: str + :type date: str, or None :return: a deferred that will be triggered with the UID of the added message. """ - return self.collection.add_raw_message(message, flags, date) + # TODO should raise ReadOnlyMailbox if not rw. + # TODO have a look at the cases for internal date in the rfc + # XXX we could treat the message as an IMessage from here + + if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)): + message = message.getvalue() + + leap_assert_type(message, basestring) + + if flags is None: + flags = tuple() + else: + flags = tuple(str(flag) for flag in flags) + + if date is None: + date = formatdate(time.time()) + + # notify_just_mdoc=True: feels HACKY, but improves a *lot* the + # responsiveness of the APPENDS: we just need to be notified when the + # mdoc is saved, and let's hope that the other parts are doing just + # fine. This will not catch any errors when the inserts of the other + # parts fail, but on the other hand allows us to return very quickly, + # which seems a good compromise given that we have to serialize the + # appends. + # A better solution will probably involve implementing MULTIAPPEND + # extension or patching imap server to support pipelining. + + # TODO add notify_new as a callback here... + return self.collection.add_msg(message, flags, date, + notify_just_mdoc=True) def notify_new(self, *args): """ diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index 8b5c371..ea790fe 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -27,6 +27,7 @@ import warnings from email.parser import Parser from email.generator import Generator from email.utils import parseaddr +from email.utils import formatdate from StringIO import StringIO from urlparse import urlparse @@ -117,7 +118,8 @@ class IncomingMail(Service): :param soledad: a soledad instance :type soledad: Soledad - :param inbox: the inbox where the new emails will be stored + :param inbox: the collection for the inbox where the new emails will be + stored :type inbox: MessageCollection :param check_period: the period to fetch new mail, in seconds. @@ -132,7 +134,7 @@ class IncomingMail(Service): self._keymanager = keymanager self._soledad = soledad - self._inbox = inbox + self._inbox_collection = inbox self._userid = userid self._listeners = [] @@ -266,7 +268,7 @@ class IncomingMail(Service): Sends unread event to ui. """ leap_events.signal( - IMAP_UNREAD_MAIL, str(self._inbox.count_unseen())) + IMAP_UNREAD_MAIL, str(self._inbox_collection.count_unseen())) # process incoming mail. @@ -710,7 +712,8 @@ class IncomingMail(Service): :return: A Deferred that will be fired when the messages is stored :rtype: Defferred """ - doc, data = msgtuple + doc, raw_data = msgtuple + insertion_date = formatdate(time.time()) log.msg('adding message %s to local db' % (doc.doc_id,)) def msgSavedCallback(result): @@ -729,7 +732,8 @@ class IncomingMail(Service): d.addCallback(signal_deleted) return d - d = self._inbox.add_raw_message(data, (self.RECENT_FLAG,)) + d = self._inbox_collection.add_msg( + raw_data, (self.RECENT_FLAG,), date=insertion_date) d.addCallbacks(msgSavedCallback, self._errback) return d diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 3127ef5..f9e99f0 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -20,10 +20,6 @@ Generic Access to Mail objects: Public LEAP Mail API. import uuid import logging import StringIO -import cStringIO -import time - -from email.utils import formatdate from twisted.internet import defer from leap.common.check import leap_assert_type @@ -521,6 +517,15 @@ class MessageCollection(object): """ Add a message to this collection. + :param raw_message: the raw message + :param flags: tuple of flags for this message + :param tags: tuple of tags for this message + :param date: + formatted date, it will be used to retrieve the internal + date for this message. According to the spec, this is NOT the date + and time in the RFC-822 header, but rather a date and time that + reflects when the message was received. + :type date: str :param notify_just_mdoc: boolean passed to the wrapper.create method, to indicate whether we're interested in being notified when only @@ -533,7 +538,11 @@ class MessageCollection(object): message. :rtype: deferred """ + # TODO watch out if the use of this method in IMAP COPY/APPEND is + # passing the right date. + # XXX mdoc ref is a leaky abstraction here. generalize. + leap_assert_type(flags, tuple) leap_assert_type(date, str) @@ -558,66 +567,22 @@ class MessageCollection(object): d = wrapper.create(self.store, notify_just_mdoc=notify_just_mdoc) d.addCallback(insert_mdoc_id, wrapper) d.addErrback(lambda f: f.printTraceback()) - return d - - def add_raw_message(self, message, flags, date=None): - """ - Adds a message to this collection. - - :param message: the raw message - :type message: str - - :param flags: flag list - :type flags: list of str - - :param date: timestamp - :type date: str - - :return: a deferred that will be triggered with the UID of the added - message. - """ - # TODO should raise ReadOnlyMailbox if not rw. - # TODO have a look at the cases for internal date in the rfc - if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)): - message = message.getvalue() - - # XXX we could treat the message as an IMessage from here - leap_assert_type(message, basestring) - - if flags is None: - flags = tuple() - else: - flags = tuple(str(flag) for flag in flags) - - if date is None: - date = formatdate(time.time()) - - # A better place for this would be the COPY/APPEND dispatcher - # if PROFILE_CMD: - # do_profile_cmd(d, "APPEND") - - # just_mdoc=True: feels HACKY, but improves a *lot* the responsiveness - # of the APPENDS: we just need to be notified when the mdoc - # is saved, and let's hope that the other parts are doing just fine. - # This will not catch any errors when the inserts of the other parts - # fail, but on the other hand allows us to return very quickly, which - # seems a good compromise given that we have to serialize the appends. - # A better solution will probably involve implementing MULTIAPPEND - # or patching imap server to support pipelining. - - d = self.add_msg(message, flags=flags, date=date, - notify_just_mdoc=True) - d.addErrback(lambda f: logger.warning(f.getTraceback())) d.addCallback(self.cb_signal_unread_to_ui) return d def cb_signal_unread_to_ui(self, result): """ - Sends unread event to ui. + Sends an unread event to ui, passing *only* the number of unread + messages if *this* is the inbox. This event is catched, for instance, + in the Bitmask client that displays a message with the number of unread + mails in the INBOX. + Used as a callback in several commands. :param result: ignored """ + # TODO it might make sense to modify the event so that + # it receives both the mailbox name AND the number of unread messages. if self.mbox_name.lower() == "inbox": d = defer.maybeDeferred(self.count_unseen) d.addCallback(self.__cb_signal_unread_to_ui) @@ -629,7 +594,7 @@ class MessageCollection(object): :param unseen: number of unseen messages. :type unseen: int """ - # TODO change name of the signal, non-imap now. + # TODO change name of the signal, independent from imap now. leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) def copy_msg(self, msg, new_mbox_uuid): -- cgit v1.2.3 From 4a0a785750e8b6a2de374c9859b98c855d807984 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 25 Feb 2015 11:33:12 -0400 Subject: properly catch TypeError exception * fix get_next_uid test * remove duplication of maybe_first_query_item, since get_last_uid also do it now. --- src/leap/mail/mailbox_indexer.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 3bec41e..732a6ee 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -30,7 +30,7 @@ def _maybe_first_query_item(thing): """ try: return thing[0][0] - except IndexError: + except (TypeError, IndexError): return None @@ -280,10 +280,7 @@ class MailboxIndexer(object): check_good_uuid(mailbox_uuid) def increment(result): - uid = _maybe_first_query_item(result) - if uid is None: - return 1 - return uid + 1 + return result + 1 d = self.get_last_uid(mailbox_uuid) d.addCallback(increment) -- cgit v1.2.3 From e8ca689b9eaf4ea39f15ea0ff172dda4dd29a9b2 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 25 Feb 2015 14:08:01 -0400 Subject: fix delete_msg test bug: delete_msg was still passing the mbox_name instead of the mbox_uuid as it should. --- src/leap/mail/mail.py | 2 +- src/leap/mail/tests/test_mail.py | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index f9e99f0..37ab829 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -624,7 +624,7 @@ class MessageCollection(object): def delete_mdoc_id(_, wrapper): doc_id = wrapper.mdoc.doc_id return self.mbox_indexer.delete_doc_by_hash( - self.mbox_name, doc_id) + self.mbox_uuid, doc_id) d = wrapper.delete(self.store) d.addCallback(delete_mdoc_id, wrapper) return d diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py index 9bc553f..7009b2f 100644 --- a/src/leap/mail/tests/test_mail.py +++ b/src/leap/mail/tests/test_mail.py @@ -56,17 +56,19 @@ def _get_msg_time(): class CollectionMixin(object): - def get_collection(self, mbox_collection=True): + def get_collection(self, mbox_collection=True, mbox_name=None, + mbox_uuid=None): """ Get a collection for tests. """ adaptor = SoledadMailAdaptor() store = self._soledad adaptor.store = store + if mbox_collection: mbox_indexer = MailboxIndexer(store) - mbox_name = "TestMbox" - mbox_uuid = str(uuid.uuid4()) + mbox_name = mbox_name or "TestMbox" + mbox_uuid = mbox_uuid or str(uuid.uuid4()) else: mbox_indexer = mbox_name = None @@ -209,6 +211,8 @@ class MessageCollectionTestCase(SoledadTestMixin, CollectionMixin): """ Tests for the MessageCollection class. """ + _mbox_uuid = None + def assert_collection_count(self, _, expected): def _assert_count(count): self.assertEqual(count, expected) @@ -222,8 +226,12 @@ class MessageCollectionTestCase(SoledadTestMixin, CollectionMixin): raw = _get_raw_msg() def add_msg_to_collection(collection): + # We keep the uuid in case we need to instantiate the same + # collection afterwards. + self._mbox_uuid = collection.mbox_uuid d = collection.add_msg(raw, date=_get_msg_time()) return d + d = self.get_collection() d.addCallback(add_msg_to_collection) return d @@ -253,7 +261,7 @@ class MessageCollectionTestCase(SoledadTestMixin, CollectionMixin): def _test_add_and_count_msg_cb(self, _): return partial(self.assert_collection_count, expected=1) - def test_coppy_msg(self): + def test_copy_msg(self): # TODO ---- update when implementing messagecopier # interface self.fail("Not Yet Implemented") @@ -263,13 +271,16 @@ class MessageCollectionTestCase(SoledadTestMixin, CollectionMixin): def del_msg(collection): def _delete_it(msg): + self.assertTrue(msg is not None) return collection.delete_msg(msg) d = collection.get_message_by_uid(1) d.addCallback(_delete_it) return d - d.addCallback(lambda _: self.get_collection()) + # We need to instantiate an mbox collection with the same uuid that + # the one in which we inserted the doc. + d.addCallback(lambda _: self.get_collection(mbox_uuid=self._mbox_uuid)) d.addCallback(del_msg) d.addCallback(self._test_delete_msg_cb) return d -- cgit v1.2.3 From c49b378efdf61b7f06f3596e6ecf4ac0d27128fa Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 26 Feb 2015 12:59:37 -0400 Subject: fix MessageTestCase: keep ref to inserted mbox uuid --- src/leap/mail/tests/test_mail.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py index 7009b2f..24dcc8b 100644 --- a/src/leap/mail/tests/test_mail.py +++ b/src/leap/mail/tests/test_mail.py @@ -101,16 +101,21 @@ class MessageTestCase(SoledadTestMixin, CollectionMixin): """ Inserts and return a regular message, for tests. """ + def insert_message(collection): + self._mbox_uuid = collection.mbox_uuid + return collection.add_msg( + raw, flags=self.msg_flags, tags=self.msg_tags, + date=self.internal_date) + raw = _get_raw_msg(multi=multi) + d = self.get_collection() - d.addCallback(lambda col: col.add_msg( - raw, flags=self.msg_flags, tags=self.msg_tags, - date=self.internal_date)) + d.addCallback(insert_message) return d def get_inserted_msg(self, multi=False): d = self._do_insert_msg(multi=multi) - d.addCallback(lambda _: self.get_collection()) + d.addCallback(lambda _: self.get_collection(mbox_uuid=self._mbox_uuid)) d.addCallback(lambda col: col.get_message_by_uid(1)) return d @@ -121,7 +126,7 @@ class MessageTestCase(SoledadTestMixin, CollectionMixin): def _test_get_flags_cb(self, msg): self.assertTrue(msg is not None) - self.assertEquals(msg.get_flags(), self.msg_flags) + self.assertEquals(tuple(msg.get_flags()), self.msg_flags) def test_get_internal_date(self): d = self.get_inserted_msg() -- cgit v1.2.3 From f6375e6bed217974369d77b7ff7191c3cb9fbb24 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 26 Feb 2015 13:04:30 -0400 Subject: [bug] increment: avoid TypeError when there's no entries in table --- src/leap/mail/mailbox_indexer.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 732a6ee..664d580 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -278,12 +278,8 @@ class MailboxIndexer(object): :rtype: Deferred """ check_good_uuid(mailbox_uuid) - - def increment(result): - return result + 1 - d = self.get_last_uid(mailbox_uuid) - d.addCallback(increment) + d.addCallback(lambda uid: uid + 1) return d def get_last_uid(self, mailbox_uuid): @@ -296,7 +292,10 @@ class MailboxIndexer(object): preffix=self.table_preffix, name=sanitize(mailbox_uuid)) def getit(result): - return _maybe_first_query_item(result) + rowid = _maybe_first_query_item(result) + if not rowid: + rowid = 0 + return rowid d = self._query(sql) d.addCallback(getit) -- cgit v1.2.3 From 63a252fb5dadb8b57212ea6d5e09737ea22d470f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 26 Feb 2015 13:10:18 -0400 Subject: fix rename_mailbox implementation, make test pass --- src/leap/mail/mail.py | 10 +++------- src/leap/mail/tests/test_mail.py | 6 +++--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 37ab829..9906ddf 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -816,16 +816,12 @@ class Account(object): return d def rename_mailbox(self, oldname, newname): - # TODO incomplete/wrong!!! - # Should rename also ALL of the document ids that are pointing - # to the old mailbox!!! - - # TODO part-docs identifiers should have the UID_validity of the - # mailbox embedded, instead of the name! (so they can survive a rename) def _rename_mbox(wrapper): wrapper.mbox = newname - return wrapper, wrapper.update(self.store) + d = wrapper.update(self.store) + d.addCallback(lambda result: wrapper) + return d d = self.adaptor.get_or_create_mbox(self.store, oldname) d.addCallback(_rename_mbox) diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py index 24dcc8b..d326ca8 100644 --- a/src/leap/mail/tests/test_mail.py +++ b/src/leap/mail/tests/test_mail.py @@ -342,9 +342,9 @@ class AccountTestCase(SoledadTestMixin): def test_rename_mailbox(self): acc = self.get_account() - d = acc.callWhenReady(lambda _: acc.add_mailbox("TestMailbox")) - d = acc.callWhenReady(lambda _: acc.rename_mailbox( - "TestMailbox", "RenamedMailbox")) + d = acc.callWhenReady(lambda _: acc.add_mailbox("OriginalMailbox")) + d.addCallback(lambda _: acc.rename_mailbox( + "OriginalMailbox", "RenamedMailbox")) d.addCallback(lambda _: acc.list_all_mailbox_names()) d.addCallback(self._test_rename_mailbox_cb) return d -- cgit v1.2.3 From fac71359f77d5649cf0324f7cd9bfdcf13cb120d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 26 Feb 2015 13:28:45 -0400 Subject: pass date explicitely, fix partial_append test it is interpreted as "tags" otherwise. --- src/leap/mail/imap/mailbox.py | 2 +- src/leap/mail/imap/tests/test_imap.py | 1 + src/leap/mail/mail.py | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index c501614..2653ae4 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -352,7 +352,7 @@ class IMAPMailbox(object): # extension or patching imap server to support pipelining. # TODO add notify_new as a callback here... - return self.collection.add_msg(message, flags, date, + return self.collection.add_msg(message, flags, date=date, notify_just_mdoc=True) def notify_new(self, *args): diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 67a24cd..738a674 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -1044,6 +1044,7 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): self._cbTestPartialAppend, infile) def _cbTestPartialAppend(self, fetched, infile): + fetched = list(fetched) self.assertTrue(len(fetched) == 1) self.assertTrue(len(fetched[0]) == 2) uid, msg = fetched[0] diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 9906ddf..584cc4a 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -540,7 +540,6 @@ class MessageCollection(object): """ # TODO watch out if the use of this method in IMAP COPY/APPEND is # passing the right date. - # XXX mdoc ref is a leaky abstraction here. generalize. leap_assert_type(flags, tuple) -- cgit v1.2.3 From 6b3a4017861df498f99ea62ce9de41ecf7caf12b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 26 Feb 2015 13:52:18 -0400 Subject: cast generator to list; change expected after rename --- src/leap/mail/imap/tests/test_imap.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 738a674..802bc9d 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -472,7 +472,7 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): return d.addCallback(self._cbTestHierarchicalRename) def _cbTestHierarchicalRename(self, mailboxes): - expected = ['INBOX', 'newname', 'newname/m1', 'newname/m2'] + expected = ['INBOX', 'newname/m1', 'newname/m2'] self.assertEqual(sorted(mailboxes), sorted([s for s in expected])) def testSubscribe(self): @@ -967,6 +967,7 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): return d.addCallback(self._cbTestFullAppend, infile) def _cbTestFullAppend(self, fetched, infile): + fetched = list(fetched) self.assertTrue(len(fetched) == 1) self.assertTrue(len(fetched[0]) == 2) uid, msg = fetched[0] -- cgit v1.2.3 From 91115f6bc8430fbef03b5e8e7e3975e5846eb50f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 2 Mar 2015 11:49:35 -0400 Subject: [refactor] remove dead code in IMAP implementation while updating the tests, I found that IMAPMessageCollection was not actually being used: all the work is done in IMAPMailbox, using directly the MessageCollection instance. So, this extra level of abstraction was finally not used. Releases: 0.9.0 --- src/leap/mail/imap/mailbox.py | 11 +- src/leap/mail/imap/messages.py | 250 +--------------------------------- src/leap/mail/imap/tests/test_imap.py | 130 ------------------ 3 files changed, 7 insertions(+), 384 deletions(-) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 2653ae4..91c6549 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -82,9 +82,9 @@ class IMAPMailbox(object): A Soledad-backed IMAP mailbox. Implements the high-level method needed for the Mailbox interfaces. - The low-level database methods are contained in IMAPMessageCollection - class, which we instantiate and make accessible in the `messages` - attribute. + The low-level database methods are contained in the generic + MessageCollection class. We receive an instance of it and it is made + accessible in the `collection` attribute. """ implements( imap4.IMailbox, @@ -107,14 +107,13 @@ class IMAPMailbox(object): def __init__(self, collection, rw=1): """ - :param collection: instance of IMAPMessageCollection - :type collection: IMAPMessageCollection + :param collection: instance of MessageCollection + :type collection: MessageCollection :param rw: read-and-write flag for this mailbox :type rw: int """ self.rw = rw - self._uidvalidity = None self.collection = collection diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index b7bb6ee..02aac2e 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -15,23 +15,20 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -IMAPMessage and IMAPMessageCollection. +IMAPMessage implementation. """ import logging from twisted.mail import imap4 from twisted.internet import defer from zope.interface import implements -from leap.common.check import leap_assert, leap_assert_type from leap.mail.utils import find_charset logger = logging.getLogger(__name__) -# TODO ------------------------------------------------------------ - +# TODO # [ ] Add ref to incoming message during add_msg. -# [ ] Delete incoming mail only after successful write. class IMAPMessage(object): @@ -251,246 +248,3 @@ def _format_headers(headers, negate, *names): if cond(key): _headers[key] = value return _headers - - -class IMAPMessageCollection(object): - """ - A collection of messages, surprisingly. - - It is tied to a selected mailbox name that is passed to its constructor. - Implements a filter query over the messages contained in a soledad - database. - """ - - messageklass = IMAPMessage - - # TODO - # [ ] Add RECENT flags docs to mailbox-doc attributes (list-of-uids) - # [ ] move Query for all the headers documents to Collection - - # TODO this should be able to produce a MessageSet methinks - # TODO --- reimplement, review and prune documentation below. - - FLAGS_DOC = "FLAGS" - HEADERS_DOC = "HEADERS" - CONTENT_DOC = "CONTENT" - """ - RECENT_DOC is a document that stores a list of the UIDs - with the recent flag for this mailbox. It deserves a special treatment - because: - (1) it cannot be set by the user - (2) it's a flag that we set inmediately after a fetch, which is quite - often. - (3) we need to be able to set/unset it in batches without doing a single - write for each element in the sequence. - """ - RECENT_DOC = "RECENT" - """ - HDOCS_SET_DOC is a document that stores a set of the Document-IDs - (the u1db index) for all the headers documents for a given mailbox. - We use it to prefetch massively all the headers for a mailbox. - This is the second massive query, after fetching all the FLAGS, that - a typical IMAP MUA will do in a case where we do not have local disk cache. - """ - HDOCS_SET_DOC = "HDOCS_SET" - - def __init__(self, collection): - """ - Constructor for IMAPMessageCollection. - - :param collection: an instance of a MessageCollection - :type collection: MessageCollection - """ - leap_assert( - collection.is_mailbox_collection(), - "Need a mailbox name to initialize") - mbox_name = collection.mbox_name - leap_assert(mbox_name.strip() != "", "mbox cannot be blank space") - leap_assert(isinstance(mbox_name, (str, unicode)), - "mbox needs to be a string") - self.collection = collection - - # XXX this has to be done in IMAPAccount - # (Where the collection must be instantiated and passed to us) - # self.mbox = normalize_mailbox(mbox) - - @property - def mbox_name(self): - """ - Return the string that identifies this mailbox. - """ - return self.collection.mbox_name - - def add_msg(self, raw, flags=None, date=None): - """ - Creates a new message document. - - :param raw: the raw message - :type raw: str - - :param flags: flags - :type flags: list - - :param date: the received date for the message - :type date: str - - :return: a deferred that will be fired with the message - uid when the adding succeed. - :rtype: deferred - """ - if flags is None: - flags = tuple() - leap_assert_type(flags, tuple) - return self.collection.add_msg(raw, flags, date) - - def get_msg_by_uid(self, uid, absolute=True): - """ - Retrieves a IMAPMessage by UID. - This is used primarity in the Mailbox fetch and store methods. - - :param uid: the message uid to query by - :type uid: int - - :rtype: IMAPMessage - """ - def make_imap_msg(msg): - kls = self.messageklass - # TODO --- remove ref to collection - return kls(msg, self.collection) - - d = self.collection.get_msg_by_uid(uid, absolute=absolute) - d.addCalback(make_imap_msg) - return d - - - # TODO -- move this to collection too - # Used for the Search (Drafts) queries? - def _get_uid_from_msgid(self, msgid): - """ - Return a UID for a given message-id. - - It first gets the headers-doc for that msg-id, and - it found it queries the flags doc for the current mailbox - for the matching content-hash. - - :return: A UID, or None - """ - return self._get_uid_from_msgidCb(msgid) - - # TODO handle deferreds - def set_flags(self, messages, flags, mode): - """ - Set flags for a sequence of messages. - - :param mbox: the mbox this message belongs to - :type mbox: str or unicode - :param messages: the messages to iterate through - :type messages: sequence - :flags: the flags to be set - :type flags: tuple - :param mode: the mode for setting. 1 is append, -1 is remove, 0 set. - :type mode: int - :param observer: a deferred that will be called with the dictionary - mapping UIDs to flags after the operation has been - done. - :type observer: deferred - """ - getmsg = self.get_msg_by_uid - - def set_flags(uid, flags, mode): - msg = getmsg(uid) - if msg is not None: - # XXX IMAPMessage needs access to the collection - # to be able to set flags. Better if we make use - # of collection... here. - return uid, msg.setFlags(flags, mode) - - setted_flags = [set_flags(uid, flags, mode) for uid in messages] - result = dict(filter(None, setted_flags)) - # XXX return gatherResults or something - return result - - def count(self): - """ - Return the count of messages for this mailbox. - - :rtype: int - """ - return self.collection.count() - - # headers query - - def all_headers(self): - """ - Return a dict with all the header documents for this - mailbox. - - :rtype: dict - """ - # Use self.collection.mbox_indexer - # and derive all the doc_ids for the hdocs - raise NotImplementedError() - - # unseen messages - - def unseen_iter(self): - """ - Get an iterator for the message UIDs with no `seen` flag - for this mailbox. - - :return: iterator through unseen message doc UIDs - :rtype: iterable - """ - raise NotImplementedError() - - def count_unseen(self): - """ - Count all messages with the `Unseen` flag. - - :returns: count - :rtype: int - """ - return len(list(self.unseen_iter())) - - def get_unseen(self): - """ - Get all messages with the `Unseen` flag - - :returns: a list of LeapMessages - :rtype: list - """ - raise NotImplementedError() - #return [self.messageklass(self._soledad, doc_id, self.mbox) - #for doc_id in self.unseen_iter()] - - # recent messages - - def count_recent(self): - """ - Count all messages with the `Recent` flag. - It just retrieves the length of the recent_flags set, - which is stored in a specific type of document for - this collection. - - :returns: count - :rtype: int - """ - raise NotImplementedError() - - # magic - - def __len__(self): - """ - Returns the number of messages on this mailbox. - :rtype: int - """ - return self.count() - - def __repr__(self): - """ - Representation string for this object. - """ - return u"" % ( - self.mbox_name, self.count()) - - # TODO implement __iter__ ? diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 802bc9d..fbe02d4 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -38,7 +38,6 @@ from twisted.python import failure from twisted import cred from leap.mail.imap.mailbox import IMAPMailbox -from leap.mail.imap.messages import IMAPMessageCollection from leap.mail.imap.tests.utils import IMAP4HelperMixin @@ -70,139 +69,10 @@ class TestRealm: def requestAvatar(self, avatarId, mind, *interfaces): return imap4.IAccount, self.theAccount, lambda: None - # # TestCases # -# TODO rename to IMAPMessageCollection -class MessageCollectionTestCase(IMAP4HelperMixin): - """ - Tests for the MessageCollection class - """ - count = 0 - - def setUp(self): - """ - setUp method for each test - We override mixin method since we are only testing - MessageCollection interface in this particular TestCase - """ - # FIXME -- return deferred - super(MessageCollectionTestCase, self).setUp() - - # FIXME --- update initialization - self.messages = IMAPMessageCollection( - "testmbox%s" % (self.count,), self._soledad) - MessageCollectionTestCase.count += 1 - - def tearDown(self): - """ - tearDown method for each test - """ - del self.messages - - def testEmptyMessage(self): - """ - Test empty message and collection - """ - em = self.messages._get_empty_doc() - self.assertEqual( - em, - { - "chash": '', - "deleted": False, - "flags": [], - "mbox": "inbox", - "seen": False, - "multi": False, - "size": 0, - "type": "flags", - "uid": 1, - }) - self.assertEqual(self.messages.count(), 0) - - def testMultipleAdd(self): - """ - Add multiple messages - """ - mc = self.messages - self.assertEqual(self.messages.count(), 0) - - def add_first(): - d = defer.gatherResults([ - mc.add_msg('Stuff 1', subject="test1"), - mc.add_msg('Stuff 2', subject="test2"), - mc.add_msg('Stuff 3', subject="test3"), - mc.add_msg('Stuff 4', subject="test4")]) - return d - - def add_second(result): - d = defer.gatherResults([ - mc.add_msg('Stuff 5', subject="test5"), - mc.add_msg('Stuff 6', subject="test6"), - mc.add_msg('Stuff 7', subject="test7")]) - return d - - def check_second(result): - return self.assertEqual(mc.count(), 7) - - d1 = add_first() - d1.addCallback(add_second) - d1.addCallback(check_second) - - def testRecentCount(self): - """ - Test the recent count - """ - mc = self.messages - countrecent = mc.count_recent - eq = self.assertEqual - - self.assertEqual(countrecent(), 0) - - d = mc.add_msg('Stuff', subject="test1") - # For the semantics defined in the RFC, we auto-add the - # recent flag by default. - - def add2(_): - return mc.add_msg('Stuff', subject="test2", - flags=('\\Deleted',)) - - def add3(_): - return mc.add_msg('Stuff', subject="test3", - flags=('\\Recent',)) - - def add4(_): - return mc.add_msg('Stuff', subject="test4", - flags=('\\Deleted', '\\Recent')) - - d.addCallback(lambda r: eq(countrecent(), 1)) - d.addCallback(add2) - d.addCallback(lambda r: eq(countrecent(), 2)) - d.addCallback(add3) - d.addCallback(lambda r: eq(countrecent(), 3)) - d.addCallback(add4) - d.addCallback(lambda r: eq(countrecent(), 4)) - - def testFilterByMailbox(self): - """ - Test that queries filter by selected mailbox - """ - mc = self.messages - self.assertEqual(self.messages.count(), 0) - - def add_1(): - d1 = mc.add_msg('msg 1', subject="test1") - d2 = mc.add_msg('msg 2', subject="test2") - d3 = mc.add_msg('msg 3', subject="test3") - d = defer.gatherResults([d1, d2, d3]) - return d - - add_1().addCallback(lambda ignored: self.assertEqual( - mc.count(), 3)) - - # DEBUG --- #from twisted.internet.base import DelayedCall #DelayedCall.debug = True -- cgit v1.2.3 From 90a7d68039c95824711e32463c27e44870ec165e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 2 Mar 2015 12:25:39 -0400 Subject: [docs] add git commit template. because the commits can be prettier this way :) --- docs/git-commit-message.txt | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 docs/git-commit-message.txt diff --git a/docs/git-commit-message.txt b/docs/git-commit-message.txt new file mode 100644 index 0000000..1b28baf --- /dev/null +++ b/docs/git-commit-message.txt @@ -0,0 +1,8 @@ +[bug|feature|docs|pkg] + +... + +Resolves: #XXX +Related: #XXX +Documentation: #XXX +Releases: XXX -- cgit v1.2.3 From 0f491cc4f94c00cd9c6e2afe7e3163dfaaad3714 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 2 Mar 2015 14:34:40 -0400 Subject: [bug] Fix IllegalMailboxCreate We're raising the exception now, not a Failure. Releases: 0.9.0 --- src/leap/mail/imap/tests/test_imap.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index fbe02d4..a94cea7 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -1025,7 +1025,6 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): self.assertItemsEqual(self.results, [1, 3]) -# TODO -------- Fix this testcase class AccountTestCase(IMAP4HelperMixin): """ Test the Account. @@ -1037,19 +1036,7 @@ class AccountTestCase(IMAP4HelperMixin): return self.server.theAccount.addMailbox('one') def test_illegalMailboxCreate(self): - # FIXME --- account.addMailbox needs to raise a failure, - # not the direct exception. - self.stashed = None - - def stash(result): - self.stashed = result - - d = self._create_empty_mailbox() - d.addBoth(stash) - d.addCallback(lambda _: self.failUnless(isinstance(self.stashed, - failure.Failure))) - return d - #self.assertRaises(AssertionError, self._create_empty_mailbox) + self.assertRaises(AssertionError, self._create_empty_mailbox) class IMAP4ServerSearchTestCase(IMAP4HelperMixin): -- cgit v1.2.3 From bd5f777ff5e9f4fd81afeebb17074326b2671cd1 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 2 Mar 2015 15:00:12 -0400 Subject: [bug] Fix testExpunge tests this test was failing randomly because we were returning the deferred before all the documents were saved into soledad store. changed also the delete_msg deferred chaining for better readability. Releases: 0.9.0 --- src/leap/mail/imap/mailbox.py | 29 +++++++++++++++++------------ src/leap/mail/imap/tests/test_imap.py | 10 +++++++--- src/leap/mail/mail.py | 9 +++++++-- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 91c6549..61baca5 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -307,7 +307,7 @@ class IMAPMailbox(object): d.addCallback(as_a_dict) return d - def addMessage(self, message, flags, date=None): + def addMessage(self, message, flags, date=None, notify_just_mdoc=True): """ Adds a message to this mailbox. @@ -327,6 +327,21 @@ class IMAPMailbox(object): # TODO have a look at the cases for internal date in the rfc # XXX we could treat the message as an IMessage from here + # TODO notify_just_mdoc *sometimes* make the append tests fail. + # have to find a better solution for this. A workaround could probably + # be to have a list of the ongoing deferreds related to append, so that + # we queue for later all the requests having to do with these. + + # notify_just_mdoc=True: feels HACKY, but improves a *lot* the + # responsiveness of the APPENDS: we just need to be notified when the + # mdoc is saved, and let's hope that the other parts are doing just + # fine. This will not catch any errors when the inserts of the other + # parts fail, but on the other hand allows us to return very quickly, + # which seems a good compromise given that we have to serialize the + # appends. + # A better solution will probably involve implementing MULTIAPPEND + # extension or patching imap server to support pipelining. + if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)): message = message.getvalue() @@ -340,19 +355,9 @@ class IMAPMailbox(object): if date is None: date = formatdate(time.time()) - # notify_just_mdoc=True: feels HACKY, but improves a *lot* the - # responsiveness of the APPENDS: we just need to be notified when the - # mdoc is saved, and let's hope that the other parts are doing just - # fine. This will not catch any errors when the inserts of the other - # parts fail, but on the other hand allows us to return very quickly, - # which seems a good compromise given that we have to serialize the - # appends. - # A better solution will probably involve implementing MULTIAPPEND - # extension or patching imap server to support pipelining. - # TODO add notify_new as a callback here... return self.collection.add_msg(message, flags, date=date, - notify_just_mdoc=True) + notify_just_mdoc=notify_just_mdoc) def notify_new(self, *args): """ diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index a94cea7..c4f752b 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -882,6 +882,7 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): """ Test partially appending a message to the mailbox """ + # TODO this test sometimes will fail because of the notify_just_mdoc infile = util.sibpath(__file__, 'rfc822.message') acc = self.server.theAccount @@ -990,11 +991,14 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): def add_messages(): d = self.mailbox.addMessage( - 'test 1', flags=('\\Deleted', 'AnotherFlag')) + 'test 1', flags=('\\Deleted', 'AnotherFlag'), + notify_just_mdoc=False) d.addCallback(lambda _: self.mailbox.addMessage( - 'test 2', flags=('AnotherFlag',))) + 'test 2', flags=('AnotherFlag',), + notify_just_mdoc=False)) d.addCallback(lambda _: self.mailbox.addMessage( - 'test 3', flags=('\\Deleted',))) + 'test 3', flags=('\\Deleted',), + notify_just_mdoc=False)) return d def expunge(): diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 584cc4a..ef9a0d9 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -645,8 +645,13 @@ class MessageCollection(object): for h in hashes: d.append(self.mbox_indexer.delete_doc_by_hash( self.mbox_uuid, h)) - return defer.gatherResults(d).addCallback( - lambda _: uids) + + def return_uids_when_deleted(ignored): + return uids + + all_deleted = defer.gatherResults(d).addCallback( + return_uids_when_deleted) + return all_deleted mdocs_deleted = self.adaptor.del_all_flagged_messages( self.store, self.mbox_uuid) -- cgit v1.2.3 From 8603664204f9d760c1e332e70686e4dd63108091 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 5 Mar 2015 12:18:11 -0400 Subject: [feature] Keep mapping of collections it is a weakref dictionary so that the collections can be garbage collected when out of scope. Releases: 0.4.0 --- src/leap/mail/mail.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index ef9a0d9..57d96ef 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -20,6 +20,8 @@ Generic Access to Mail objects: Public LEAP Mail API. import uuid import logging import StringIO +import weakref + from twisted.internet import defer from leap.common.check import leap_assert_type @@ -732,6 +734,14 @@ class Account(object): adaptor_class = SoledadMailAdaptor + # This is a mapping to collection instances so that we always + # return a reference to them instead of creating new ones. However, being a + # dictionary of weakrefs values, they automagically vanish from the dict + # when no hard refs is left to them (so they can be garbage collected) + # This is important because the different wrappers rely on several + # kinds of deferredLocks that are kept as class or instance variables + _collection_mapping = weakref.WeakValueDictionary() + def __init__(self, store, ready_cb=None): self.store = store self.adaptor = self.adaptor_class() @@ -835,12 +845,19 @@ class Account(object): def get_collection_by_mailbox(self, name): """ - :rtype: MessageCollection + :rtype: deferred + :return: a deferred that will fire with a MessageCollection """ + collection = self._collection_mapping.get(name, None) + if collection: + return defer.succeed(collection) + # imap select will use this, passing the collection to SoledadMailbox def get_collection_for_mailbox(mbox_wrapper): - return MessageCollection( + collection = MessageCollection( self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) + self._collection_mapping[name] = collection + return collection d = self.adaptor.get_or_create_mbox(self.store, name) d.addCallback(get_collection_for_mailbox) -- cgit v1.2.3 From 1568761f4fa7a7f8a9b0caec79ed2552b6c73ba7 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 5 Mar 2015 12:20:45 -0400 Subject: [bug] catch null doc_id error, and log it as such Catch null doc_id so that we don't interrupt server This bug needs further investigation Related: #6769 ? --- src/leap/mail/mail.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 57d96ef..d92ff79 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -562,6 +562,15 @@ class MessageCollection(object): def insert_mdoc_id(_, wrapper): doc_id = wrapper.mdoc.doc_id + if not doc_id: + # --- BUG ----------------------------------------- + # XXX why from time to time mdoc doesn't have doc_id + # here??? + logger.error("BUG: (please report) Null doc_id for " + "document %s" % + (wrapper.mdoc.serialize(),)) + return defer.succeed("mdoc_id not inserted") + # XXX BUG ----------------------------------------- return self.mbox_indexer.insert_doc( self.mbox_uuid, doc_id) -- cgit v1.2.3 From 57ec516fe350787d2b6e99a401c96520de1a43b3 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 23 Mar 2015 12:41:49 -0400 Subject: [bug] add extra CRLF to avoid bad mime parsing in Thunderbird Thunderbird (as of 37.0b1) will display a blank body (with no attachments) if some conditions are met: * disk synchronization is disabled * mime_part_on_demand = true * msg size is bigger than the parts_on_demand threshold (30000 by default). Comparing the logs with a well behaved imap server (dovecot, on this case), it's easy to see that twisted implementation is lacking an extra line separator at the end of each group of headers that is rendered in response to each of the `BODY.PEEK[X.MIME]` command that the mime_parts_on_demand will issue after getting the BODYSTRUCTURE. This change patches the spew_body command on the body server. We still would have to see if this is a bad behaviour in the thunderbird side. The most similar bug I've found is: https://bugzilla.mozilla.org/show_bug.cgi?id=149771 Which apparently was happening with exchange server. We should send the patch to upstream twisted as well. Note that this fix is not enough: the following commit, about fixing the case of the boundary passed in the BODYSTRUCTURE response is also needed to fix the bug (since a bad parsing happens all the same). Resolves: #6773, #5010 Documentation: #6773 Releases: 0.4.0 --- src/leap/mail/imap/server.py | 61 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 3e10171..26d2001 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -60,6 +60,67 @@ class LEAPIMAPServer(imap4.IMAP4Server): # populate the test account properly (and only once # per session) + ############################################################# + # + # Twisted imap4 patch to workaround bad mime rendering in TB. + # See https://leap.se/code/issues/6773 + # and https://bugzilla.mozilla.org/show_bug.cgi?id=149771 + # Still unclear if this is a thunderbird bug. + # TODO send this patch upstream + # + ############################################################# + + def spew_body(self, part, id, msg, _w=None, _f=None): + if _w is None: + _w = self.transport.write + for p in part.part: + if msg.isMultipart(): + msg = msg.getSubPart(p) + elif p > 0: + # Non-multipart messages have an implicit first part but no + # other parts - reject any request for any other part. + raise TypeError("Requested subpart of non-multipart message") + + if part.header: + hdrs = msg.getHeaders(part.header.negate, *part.header.fields) + hdrs = imap4._formatHeaders(hdrs) + _w(str(part) + ' ' + imap4._literal(hdrs)) + elif part.text: + _w(str(part) + ' ') + _f() + return imap4.FileProducer(msg.getBodyFile() + ).beginProducing(self.transport + ) + elif part.mime: + hdrs = imap4._formatHeaders(msg.getHeaders(True)) + + ###### PATCHED ##################################### + _w(str(part) + ' ' + imap4._literal(hdrs + "\r\n")) + ###### END PATCHED ################################# + + elif part.empty: + _w(str(part) + ' ') + _f() + if part.part: + return imap4.FileProducer(msg.getBodyFile() + ).beginProducing(self.transport + ) + else: + mf = imap4.IMessageFile(msg, None) + if mf is not None: + return imap4.FileProducer(mf.open()).beginProducing(self.transport) + return imap4.MessageProducer(msg, None, self._scheduler).beginProducing(self.transport) + + else: + _w('BODY ' + imap4.collapseNestedLists([imap4.getBodyStructure(msg)])) + + ################################################################## + # + # END Twisted imap4 patch to workaround bad mime rendering in TB. + # #6773 + # + ################################################################## + def lineReceived(self, line): """ Attempt to parse a single line from the server. -- cgit v1.2.3 From e139fb997c4c482e70cea3dec10b9a2365165772 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 23 Mar 2015 14:39:39 -0400 Subject: [bug] fix wrong case in the boundary passed in BODYSTRUCTURE By removing this call to lower(), we avoid a bug in which the BODYSTRUCTURE response returns a boundary all in lower case. Should send patch upstream to twisted. Related: #6773 --- src/leap/mail/imap/server.py | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 26d2001..3aeca54 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -36,6 +36,38 @@ from twisted.mail.imap4 import IllegalClientResponse from twisted.mail.imap4 import LiteralString, LiteralFile +def _getContentType(msg): + """ + Return a two-tuple of the main and subtype of the given message. + """ + attrs = None + mm = msg.getHeaders(False, 'content-type').get('content-type', None) + if mm: + mm = ''.join(mm.splitlines()) + mimetype = mm.split(';') + if mimetype: + type = mimetype[0].split('/', 1) + if len(type) == 1: + major = type[0] + minor = None + elif len(type) == 2: + major, minor = type + else: + major = minor = None + # XXX patched --------------------------------------------- + attrs = dict(x.strip().split('=', 1) for x in mimetype[1:]) + # XXX patched --------------------------------------------- + else: + major = minor = None + else: + major = minor = None + return major, minor, attrs + +# Monkey-patch _getContentType to avoid bug that passes lower-case boundary in +# BODYSTRUCTURE response. +imap4._getContentType = _getContentType + + class LEAPIMAPServer(imap4.IMAP4Server): """ An IMAP4 Server with a LEAP Storage Backend. @@ -84,7 +116,9 @@ class LEAPIMAPServer(imap4.IMAP4Server): if part.header: hdrs = msg.getHeaders(part.header.negate, *part.header.fields) hdrs = imap4._formatHeaders(hdrs) - _w(str(part) + ' ' + imap4._literal(hdrs)) + # PATCHED ########################################## + _w(str(part) + ' ' + imap4._literal(hdrs + "\r\n")) + # PATCHED ########################################## elif part.text: _w(str(part) + ' ') _f() @@ -94,9 +128,9 @@ class LEAPIMAPServer(imap4.IMAP4Server): elif part.mime: hdrs = imap4._formatHeaders(msg.getHeaders(True)) - ###### PATCHED ##################################### + # PATCHED ########################################## _w(str(part) + ' ' + imap4._literal(hdrs + "\r\n")) - ###### END PATCHED ################################# + # END PATCHED ###################################### elif part.empty: _w(str(part) + ' ') -- cgit v1.2.3 From efb6028e8d4096d113c565a3953919a5e30d0947 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 23 Mar 2015 12:57:02 -0400 Subject: [bug] report the correct size for mime parts The MIME size is the size of the body, w/o counting the headers. Releases: 0.4.0 --- src/leap/mail/mail.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index d92ff79..fd2f39a 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -135,7 +135,15 @@ class MessagePart(object): self._index = index def get_size(self): - return self._pmap['size'] + """ + Size of the body, in octets. + """ + total = self._pmap['size'] + _h = self.get_headers() + headers = len( + '\n'.join(["%s: %s" % (k, v) for k, v in dict(_h).items()])) + # have to subtract 2 blank lines + return total - headers - 2 def get_body_file(self): payload = "" @@ -148,6 +156,7 @@ class MessagePart(object): raise NotImplementedError if payload: payload = _encode_payload(payload) + return _write_and_rewind(payload) def get_headers(self): @@ -252,9 +261,10 @@ class Message(object): def get_size(self): """ - Size, in octets. + Size of the whole message, in octets (including headers). """ - return self._wrapper.fdoc.size + total = self._wrapper.fdoc.size + return total def is_multipart(self): """ -- cgit v1.2.3 From f97104e25fe504993615f194825a757d4c381a24 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 23 Mar 2015 12:59:12 -0400 Subject: [bug] re-add fix for multiple headers This fix stores as multi-line headers that are repeated, and that were being discarded when storing them in a regular dict. It had been removed during the last refactor. I also store headers now as a case-insensitive dict, which solves other problems with the implementation of the twisted imap. Releases: 0.4.0 --- src/leap/mail/adaptors/soledad.py | 18 ++++++++---------- src/leap/mail/imap/messages.py | 25 +++++++++++++++++++------ src/leap/mail/mail.py | 20 +++++++++++++++++++- src/leap/mail/tests/test_mail.py | 8 +++++--- 4 files changed, 51 insertions(+), 20 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 490e014..7a1a92d 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -1114,6 +1114,7 @@ def _split_into_parts(raw): msg, parts, chash, multi = _parse_msg(raw) size = len(msg.as_string()) + body_phash = walk.get_body_phash(msg) parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) @@ -1161,16 +1162,13 @@ def _build_headers_doc(msg, chash, body_phash, parts_map): It takes into account possibly repeated headers. """ - headers = msg.items() - - # TODO move this manipulation to IMAP - #headers = defaultdict(list) - #for k, v in msg.items(): - #headers[k].append(v) - ## "fix" for repeated headers. - #for k, v in headers.items(): - #newline = "\n%s: " % (k,) - #headers[k] = newline.join(v) + headers = defaultdict(list) + for k, v in msg.items(): + headers[k].append(v) + # "fix" for repeated headers (as in "Received:" + for k, v in headers.items(): + newline = "\n%s: " % (k.lower(),) + headers[k] = newline.join(v) lower_headers = lowerdict(dict(headers)) msgid = first(_MSGID_RE.findall( diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 02aac2e..13943b1 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -208,6 +208,18 @@ class IMAPMessagePart(object): return IMAPMessagePart(subpart) +class CaseInsensitiveDict(dict): + """ + A dictionary subclass that will allow case-insenstive key lookups. + """ + + def __setitem__(self, key, value): + super(CaseInsensitiveDict, self).__setitem__(key.lower(), value) + + def __getitem__(self, key): + return super(CaseInsensitiveDict, self).__getitem__(key.lower()) + + def _format_headers(headers, negate, *names): # current server impl. expects content-type to be present, so if for # some reason we do not have headers, we have to return at least that @@ -228,13 +240,13 @@ def _format_headers(headers, negate, *names): # default to most likely standard charset = find_charset(headers, "utf-8") - _headers = dict() - for key, value in headers.items(): - # twisted imap server expects *some* headers to be lowercase - # We could use a CaseInsensitiveDict here... - if key.lower() == "content-type": - key = key.lower() + # We will return a copy of the headers dictionary that + # will allow case-insensitive lookups. In some parts of the twisted imap + # server code the keys are expected to be in lower case, and in this way + # we avoid having to convert them. + _headers = CaseInsensitiveDict() + for key, value in headers.items(): if not isinstance(key, str): key = key.encode(charset, 'replace') if not isinstance(value, str): @@ -247,4 +259,5 @@ def _format_headers(headers, negate, *names): # filter original dict by negate-condition if cond(key): _headers[key] = value + return _headers diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index fd2f39a..99c3873 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -17,6 +17,7 @@ """ Generic Access to Mail objects: Public LEAP Mail API. """ +import itertools import uuid import logging import StringIO @@ -98,6 +99,23 @@ def _encode_payload(payload, ctype=""): return payload +def _unpack_headers(headers_dict): + """ + Take a "packed" dict containing headers (with repeated keys represented as + line breaks inside each value, preceded by the header key) and return a + list of tuples in which each repeated key has a different tuple. + """ + headers_l = headers_dict.items() + for i, (k, v) in enumerate(headers_l): + splitted = v.split(k.lower() + ": ") + if len(splitted) != 1: + inner = zip( + itertools.cycle([k]), + map(lambda l: l.rstrip('\n'), splitted)) + headers_l = headers_l[:i] + inner + headers_l[i+1:] + return headers_l + + class MessagePart(object): # TODO This class should be better abstracted from the data model. # TODO support arbitrarily nested multiparts (right now we only support @@ -242,7 +260,7 @@ class Message(object): """ Get the raw headers document. """ - return [tuple(item) for item in self._wrapper.hdoc.headers] + return self._wrapper.hdoc.headers def get_body_file(self, store): """ diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py index d326ca8..2c03933 100644 --- a/src/leap/mail/tests/test_mail.py +++ b/src/leap/mail/tests/test_mail.py @@ -26,7 +26,7 @@ from email.parser import Parser from email.Utils import formatdate from leap.mail.adaptors.soledad import SoledadMailAdaptor -from leap.mail.mail import MessageCollection, Account +from leap.mail.mail import MessageCollection, Account, _unpack_headers from leap.mail.mailbox_indexer import MailboxIndexer from leap.mail.tests.common import SoledadTestMixin @@ -144,8 +144,10 @@ class MessageTestCase(SoledadTestMixin, CollectionMixin): def _test_get_headers_cb(self, msg): self.assertTrue(msg is not None) - expected = _get_parsed_msg().items() - self.assertEqual(msg.get_headers(), expected) + expected = [ + (str(key.lower()), str(value)) + for (key, value) in _get_parsed_msg().items()] + self.assertItemsEqual(_unpack_headers(msg.get_headers()), expected) def test_get_body_file(self): d = self.get_inserted_msg(multi=True) -- cgit v1.2.3 From 09979be91930c234975c6b3e621cb5791ec05260 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 23 Mar 2015 13:01:54 -0400 Subject: [bug] temporary workaround to allow display on some muas Until we implement sequences, this avoids breaking with certain MUAs like mutt. Releases: 0.4.0 --- src/leap/mail/imap/mailbox.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 61baca5..5d4e597 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -500,9 +500,9 @@ class IMAPMailbox(object): is_sequence = True if uid == 0 else False # XXX DEBUG --- if you attempt to use the `getmail` utility under - # imap/tests, it will choke until we implement sequence numbers. This - # is an easy hack meanwhile. - # is_sequence = False + # imap/tests, or muas like mutt, it will choke until we implement + # sequence numbers. This is an easy hack meanwhile. + is_sequence = False # ----------------------------------------------------------------- getmsg = self.collection.get_message_by_uid @@ -581,7 +581,14 @@ class IMAPMailbox(object): MessagePart. :rtype: tuple """ - is_sequence = True if uid == 0 else False + # is_sequence = True if uid == 0 else False + + # XXX FIXME ----------------------------------------------------- + # imap/tests, or muas like mutt, it will choke until we implement + # sequence numbers. This is an easy hack meanwhile. + is_sequence = False + # --------------------------------------------------------------- + if is_sequence: raise NotImplementedError -- cgit v1.2.3 From 30ffa594249960da99d299b77f007ebc20072fbc Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 23 Mar 2015 13:03:10 -0400 Subject: [feature] make deferred list error-tolerant just in case Releases: 0.4.0 --- src/leap/mail/imap/mailbox.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 5d4e597..0eff317 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -514,13 +514,14 @@ class IMAPMailbox(object): d_imapmsg = [] for msg in messages: d_imapmsg.append(getimapmsg(msg)) - return defer.gatherResults(d_imapmsg) + return defer.gatherResults(d_imapmsg, consumeErrors=True) def _zip_msgid(imap_messages): zipped = zip( list(msg_range), imap_messages) return (item for item in zipped) + # XXX not called?? def _unset_recent(sequence): reactor.callLater(0, self.unset_recent_flags, sequence) return sequence @@ -528,12 +529,12 @@ class IMAPMailbox(object): d_msg = [] for msgid in msg_range: # XXX We want cdocs because we "probably" are asked for the - # body. We should be smarted at do_FETCH and pass a parameter + # body. We should be smarter at do_FETCH and pass a parameter # to this method in order not to prefetch cdocs if they're not # going to be used. d_msg.append(getmsg(msgid, get_cdocs=True)) - d = defer.gatherResults(d_msg) + d = defer.gatherResults(d_msg, consumeErrors=True) d.addCallback(_get_imap_msg) d.addCallback(_zip_msgid) return d -- cgit v1.2.3 From d11c03d1126f1744789d107b0f9bd04fc8a2f50b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 23 Mar 2015 13:08:24 -0400 Subject: [bug] move creation_ts to mail generic api This also fixes a bug in which INBOX wasn't being given a creation timestamp, and therefore always being identified with the same UIDVALIDITY = 1, which could be confusing MUAs since this value should be unique, and it's relied on to uniquely identifying a given message. Releases: 0.4.0 --- src/leap/mail/imap/account.py | 15 ++------------- src/leap/mail/mail.py | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 38df845..ccb4b75 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -163,28 +163,17 @@ class IMAPAccount(object): # FIXME --- return failure instead of AssertionError # See AccountTestCase... leap_assert(name, "Need a mailbox name to create a mailbox") - if creation_ts is None: - # by default, we pass an int value - # taken from the current time - # we make sure to take enough decimals to get a unique - # mailbox-uidvalidity. - creation_ts = int(time.time() * 10E2) def check_it_does_not_exist(mailboxes): if name in mailboxes: raise imap4.MailboxCollision, repr(name) return mailboxes - def set_mbox_creation_ts(collection): - d = collection.set_mbox_attr("created", creation_ts) - d.addCallback(lambda _: collection) - return d - d = self.account.list_all_mailbox_names() d.addCallback(check_it_does_not_exist) - d.addCallback(lambda _: self.account.add_mailbox(name)) + d.addCallback(lambda _: self.account.add_mailbox( + name, creation_ts=creation_ts)) d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) - d.addCallback(set_mbox_creation_ts) d.addCallback(self._return_mailbox_from_collection) return d diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 99c3873..89f89b0 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -21,6 +21,7 @@ import itertools import uuid import logging import StringIO +import time import weakref from twisted.internet import defer @@ -833,7 +834,20 @@ class Account(object): d = self.adaptor.get_all_mboxes(self.store) return d - def add_mailbox(self, name): + def add_mailbox(self, name, creation_ts=None): + + if creation_ts is None: + # by default, we pass an int value + # taken from the current time + # we make sure to take enough decimals to get a unique + # mailbox-uidvalidity. + creation_ts = int(time.time() * 10E2) + + def set_creation_ts(wrapper): + wrapper.created = creation_ts + d = wrapper.update(self.store) + d.addCallback(lambda _: wrapper) + return d def create_uuid(wrapper): if not wrapper.uuid: @@ -849,6 +863,7 @@ class Account(object): return d d = self.adaptor.get_or_create_mbox(self.store, name) + d.addCallback(set_creation_ts) d.addCallback(create_uuid) d.addCallback(create_uid_table_cb) return d -- cgit v1.2.3 From 8e916eeadfcd76d50b54a2621d789e6a296dcce6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 26 Mar 2015 15:59:33 -0400 Subject: [bug] fix early append notification There's a workaround for "slow" APPENDS to an inbox, and it is that we have a flag to allow returning early when JUST the mdoc (the meta-document) has been written. However, this was givin a problem when doing a FETCH right after an APPEND (with notify_just_mdoc=True) has been done. This commit fixes it by making the FETCH command first check if there's an ongoing pending write, and queueing itself right after the write queue has been completed. This fixes the testFullAppend regression. Releases: 0.4.0 --- src/leap/mail/adaptors/soledad.py | 34 ++++++++++++++++++++++++++-------- src/leap/mail/imap/mailbox.py | 12 ++---------- src/leap/mail/imap/messages.py | 14 +------------- src/leap/mail/imap/tests/test_imap.py | 13 +++++++------ src/leap/mail/mail.py | 32 ++++++++++++++++++++++++++------ src/leap/mail/utils.py | 21 +++++++++++++++++++++ 6 files changed, 83 insertions(+), 43 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 7a1a92d..b8e5fd4 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -491,7 +491,7 @@ class MessageWrapper(object): for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()): cdoc.set_future_doc_id(doc_id) - def create(self, store, notify_just_mdoc=False): + def create(self, store, notify_just_mdoc=False, pending_inserts_dict=None): """ Create all the parts for this message in the store. @@ -503,7 +503,7 @@ class MessageWrapper(object): Be warned that in that case there will be no record of failures when creating the other part-documents. - Other-wise, this method will return a deferred that will wait for + Otherwise, this method will return a deferred that will wait for the creation of all the part documents. Setting this flag to True is mostly a convenient workaround for the @@ -513,6 +513,9 @@ class MessageWrapper(object): times will be enough to have all the queued insert operations finished. :type notify_just_mdoc: bool + :param pending_inserts_dict: + a dictionary with the pending inserts ids. + :type pending_inserts_dict: dict :return: a deferred whose callback will be called when either all the part documents have been written, or just the metamsg-doc, @@ -527,26 +530,41 @@ class MessageWrapper(object): leap_assert(self.fdoc.doc_id is None, "Cannot create: fdoc has a doc_id") + def unblock_pending_insert(result): + msgid = self.hdoc.headers.get('Message-Id', None) + try: + d = pending_inserts_dict[msgid] + d.callback(msgid) + except KeyError: + pass + return result + # TODO check that the doc_ids in the mdoc are coherent - d = [] + self.d = [] + mdoc_created = self.mdoc.create(store) - d.append(mdoc_created) - d.append(self.fdoc.create(store)) + fdoc_created = self.fdoc.create(store) + + self.d.append(mdoc_created) + self.d.append(fdoc_created) if not self._is_copy: if self.hdoc.doc_id is None: - d.append(self.hdoc.create(store)) + self.d.append(self.hdoc.create(store)) for cdoc in self.cdocs.values(): if cdoc.doc_id is not None: # we could be just linking to an existing # content-doc. continue - d.append(cdoc.create(store)) + self.d.append(cdoc.create(store)) + + self.all_inserted_d = defer.gatherResults(self.d) if notify_just_mdoc: + self.all_inserted_d.addCallback(unblock_pending_insert) return mdoc_created else: - return defer.gatherResults(d) + return self.all_inserted_d def update(self, store): """ diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 0eff317..1412344 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -492,13 +492,8 @@ class IMAPMailbox(object): :rtype: deferred with a generator that yields... """ - # For the moment our UID is sequential, so we - # can treat them all the same. - # Change this to the flag that twisted expects when we - # switch to content-hash based index + local UID table. - - is_sequence = True if uid == 0 else False - + # TODO implement sequence + # is_sequence = True if uid == 0 else False # XXX DEBUG --- if you attempt to use the `getmail` utility under # imap/tests, or muas like mutt, it will choke until we implement # sequence numbers. This is an easy hack meanwhile. @@ -583,7 +578,6 @@ class IMAPMailbox(object): :rtype: tuple """ # is_sequence = True if uid == 0 else False - # XXX FIXME ----------------------------------------------------- # imap/tests, or muas like mutt, it will choke until we implement # sequence numbers. This is an easy hack meanwhile. @@ -672,7 +666,6 @@ class IMAPMailbox(object): :rtype: tuple """ # TODO implement sequences - # TODO how often is thunderbird doing this? is_sequence = True if uid == 0 else False if is_sequence: raise NotImplementedError @@ -730,7 +723,6 @@ class IMAPMailbox(object): read-write. """ # TODO implement sequences - # TODO how often is thunderbird doing this? is_sequence = True if uid == 0 else False if is_sequence: raise NotImplementedError diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 13943b1..4c6f10d 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -22,7 +22,7 @@ from twisted.mail import imap4 from twisted.internet import defer from zope.interface import implements -from leap.mail.utils import find_charset +from leap.mail.utils import find_charset, CaseInsensitiveDict logger = logging.getLogger(__name__) @@ -208,18 +208,6 @@ class IMAPMessagePart(object): return IMAPMessagePart(subpart) -class CaseInsensitiveDict(dict): - """ - A dictionary subclass that will allow case-insenstive key lookups. - """ - - def __setitem__(self, key, value): - super(CaseInsensitiveDict, self).__setitem__(key.lower(), value) - - def __getitem__(self, key): - return super(CaseInsensitiveDict, self).__getitem__(key.lower()) - - def _format_headers(headers, negate, *names): # current server impl. expects content-type to be present, so if for # some reason we do not have headers, we have to return at least that diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index c4f752b..af1bd69 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -25,8 +25,8 @@ XXX add authors from the original twisted tests. @license: GPLv3, see included LICENSE file """ # XXX review license of the original tests!!! - import os +import string import types @@ -38,6 +38,7 @@ from twisted.python import failure from twisted import cred from leap.mail.imap.mailbox import IMAPMailbox +from leap.mail.imap.messages import CaseInsensitiveDict from leap.mail.imap.tests.utils import IMAP4HelperMixin @@ -74,8 +75,8 @@ class TestRealm: # # DEBUG --- -#from twisted.internet.base import DelayedCall -#DelayedCall.debug = True +# from twisted.internet.base import DelayedCall +# DelayedCall.debug = True class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): @@ -810,7 +811,7 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): infile = util.sibpath(__file__, 'rfc822.message') message = open(infile) acc = self.server.theAccount - mailbox_name = "root/subthing" + mailbox_name = "appendmbox/subthing" def add_mailbox(): return acc.addMailbox(mailbox_name) @@ -843,7 +844,7 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): uid, msg = fetched[0] parsed = self.parser.parse(open(infile)) expected_body = parsed.get_payload() - expected_headers = dict(parsed.items()) + expected_headers = CaseInsensitiveDict(parsed.items()) def assert_flags(flags): self.assertEqual( @@ -860,7 +861,7 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): self.assertEqual(expected_body, gotbody) def assert_headers(headers): - self.assertItemsEqual(expected_headers, headers) + self.assertItemsEqual(map(string.lower, expected_headers), headers) d = defer.maybeDeferred(msg.getFlags) d.addCallback(assert_flags) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 89f89b0..4fe08a6 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -35,7 +35,7 @@ from leap.mail.adaptors.soledad import SoledadMailAdaptor from leap.mail.constants import INBOX_NAME from leap.mail.constants import MessageFlags from leap.mail.mailbox_indexer import MailboxIndexer -from leap.mail.utils import find_charset +from leap.mail.utils import find_charset, CaseInsensitiveDict logger = logging.getLogger(name=__name__) @@ -179,7 +179,7 @@ class MessagePart(object): return _write_and_rewind(payload) def get_headers(self): - return self._pmap.get("headers", []) + return CaseInsensitiveDict(self._pmap.get("headers", [])) def is_multipart(self): return self._pmap.get("multi", False) @@ -261,7 +261,7 @@ class Message(object): """ Get the raw headers document. """ - return self._wrapper.hdoc.headers + return CaseInsensitiveDict(self._wrapper.hdoc.headers) def get_body_file(self, store): """ @@ -364,6 +364,8 @@ class MessageCollection(object): store = None messageklass = Message + _pending_inserts = dict() + def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): """ Constructor for a MessageCollection. @@ -440,6 +442,8 @@ class MessageCollection(object): if not absolute: raise NotImplementedError("Does not support relative ids yet") + get_doc_fun = self.mbox_indexer.get_doc_id_from_uid + def get_msg_from_mdoc_id(doc_id): if doc_id is None: return None @@ -447,7 +451,16 @@ class MessageCollection(object): self.messageklass, self.store, doc_id, uid=uid, get_cdocs=get_cdocs) - d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_uuid, uid) + def cleanup_and_get_doc_after_pending_insert(result): + for key in result: + self._pending_inserts.pop(key) + return get_doc_fun(self.mbox_uuid, uid) + + if not self._pending_inserts: + d = get_doc_fun(self.mbox_uuid, uid) + else: + d = defer.gatherResults(self._pending_inserts.values()) + d.addCallback(cleanup_and_get_doc_after_pending_insert) d.addCallback(get_msg_from_mdoc_id) return d @@ -572,13 +585,16 @@ class MessageCollection(object): # TODO watch out if the use of this method in IMAP COPY/APPEND is # passing the right date. # XXX mdoc ref is a leaky abstraction here. generalize. - leap_assert_type(flags, tuple) leap_assert_type(date, str) msg = self.adaptor.get_msg_from_string(Message, raw_msg) wrapper = msg.get_wrapper() + if notify_just_mdoc: + msgid = msg.get_headers()['message-id'] + self._pending_inserts[msgid] = defer.Deferred() + if not self.is_mailbox_collection(): raise NotImplementedError() @@ -600,10 +616,14 @@ class MessageCollection(object): (wrapper.mdoc.serialize(),)) return defer.succeed("mdoc_id not inserted") # XXX BUG ----------------------------------------- + return self.mbox_indexer.insert_doc( self.mbox_uuid, doc_id) - d = wrapper.create(self.store, notify_just_mdoc=notify_just_mdoc) + d = wrapper.create( + self.store, + notify_just_mdoc=notify_just_mdoc, + pending_inserts_dict=self._pending_inserts) d.addCallback(insert_mdoc_id, wrapper) d.addErrback(lambda f: f.printTraceback()) d.addCallback(self.cb_signal_unread_to_ui) diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py index 8e51024..029e9f5 100644 --- a/src/leap/mail/utils.py +++ b/src/leap/mail/utils.py @@ -351,3 +351,24 @@ def json_loads(data): obj = json.loads(data, cls=json.JSONDecoder) return obj + + +class CaseInsensitiveDict(dict): + """ + A dictionary subclass that will allow case-insenstive key lookups. + """ + def __init__(self, d=None): + if d is None: + d = [] + if isinstance(d, dict): + for key, value in d.items(): + self[key] = value + else: + for key, value in d: + self[key] = value + + def __setitem__(self, key, value): + super(CaseInsensitiveDict, self).__setitem__(key.lower(), value) + + def __getitem__(self, key): + return super(CaseInsensitiveDict, self).__getitem__(key.lower()) -- cgit v1.2.3 From 154e1aa347bb9ee0a9ed9abd60ed703a81cce017 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 14 Apr 2015 01:13:29 -0400 Subject: [feature] implement substring body fetch Current twisted implementation correctly parses partial fetches using substrings by use of angle brackets (see section 6.4.5 of imap rfc), but no use is made of the requested substring in the spew_body method. this commit minimally implements conformance to the substring request, although further boundary checks should be made (ie, checking whether the starting octet is beyond the end of the text). Resolves: #6841 Releases: 0.4.0 --- src/leap/mail/imap/server.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 3aeca54..45da535 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -17,6 +17,7 @@ """ LEAP IMAP4 Server Implementation. """ +import StringIO from copy import copy from twisted import cred @@ -136,7 +137,23 @@ class LEAPIMAPServer(imap4.IMAP4Server): _w(str(part) + ' ') _f() if part.part: - return imap4.FileProducer(msg.getBodyFile() + # PATCHED ############################################# + # implement partial FETCH + # TODO implement boundary checks + # TODO see if there's a more efficient way, without + # copying the original content into a new buffer. + fd = msg.getBodyFile() + begin = getattr(part, "partialBegin", None) + _len = getattr(part, "partialLength", None) + if begin is not None and _len is not None: + _fd = StringIO.StringIO() + fd.seek(part.partialBegin) + _fd.write(fd.read(part.partialLength)) + _fd.seek(0) + else: + _fd = fd + return imap4.FileProducer(_fd + # END PATCHED #########################3 ).beginProducing(self.transport ) else: -- cgit v1.2.3 From d5d9c3794566b999bfd42398072c9f7e3177d1b9 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Mon, 11 May 2015 18:04:29 -0300 Subject: [feat] adapt to new events api on common - Related: #6359 --- changes/VERSION_COMPAT | 1 + changes/feature_adapt-to-new-events-on-common | 1 + src/leap/mail/imap/server.py | 5 ++--- src/leap/mail/imap/service/imap.py | 8 +++----- src/leap/mail/incoming/service.py | 29 ++++++++++----------------- src/leap/mail/mail.py | 5 ++--- src/leap/mail/outgoing/service.py | 16 +++++++-------- src/leap/mail/smtp/__init__.py | 6 +++--- src/leap/mail/smtp/gateway.py | 12 +++++------ 9 files changed, 37 insertions(+), 46 deletions(-) create mode 100644 changes/feature_adapt-to-new-events-on-common diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT index 12822ac..a5c0caa 100644 --- a/changes/VERSION_COMPAT +++ b/changes/VERSION_COMPAT @@ -10,3 +10,4 @@ # leap.foo.bar>=x.y.z leap.keymanager>=0.4.0 leap.soledad.client>=0.7.0 +leap.common>=0.4 diff --git a/changes/feature_adapt-to-new-events-on-common b/changes/feature_adapt-to-new-events-on-common new file mode 100644 index 0000000..e57e777 --- /dev/null +++ b/changes/feature_adapt-to-new-events-on-common @@ -0,0 +1 @@ +- Adapt to new events api on leap.common. Related to #5359. diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 45da535..2b670c1 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -26,9 +26,8 @@ from twisted.internet.defer import maybeDeferred from twisted.mail import imap4 from twisted.python import log -from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type -from leap.common.events.events_pb2 import IMAP_CLIENT_LOGIN +from leap.common.events import emit, catalog from leap.soledad.client import Soledad # imports for LITERAL+ patch @@ -222,7 +221,7 @@ class LEAPIMAPServer(imap4.IMAP4Server): # bad username, reject. raise cred.error.UnauthorizedLogin() # any dummy password is allowed so far. use realm instead! - leap_events.signal(IMAP_CLIENT_LOGIN, "1") + emit(catalog.IMAP_CLIENT_LOGIN, "1") return imap4.IAccount, self.theAccount, lambda: None def do_FETCH(self, tag, messages, query, uid=0): diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index b3282d4..370c513 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -31,14 +31,12 @@ from twisted.python import log logger = logging.getLogger(__name__) -from leap.common import events as leap_events +from leap.common.events import emit, catalog from leap.common.check import leap_assert_type, leap_check from leap.mail.imap.account import IMAPAccount from leap.mail.imap.server import LEAPIMAPServer from leap.soledad.client import Soledad -from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED -from leap.common.events.events_pb2 import IMAP_SERVICE_FAILED_TO_START DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) if DO_MANHOLE: @@ -182,10 +180,10 @@ def run_service(store, **kwargs): reactor.listenTCP(manhole.MANHOLE_PORT, manhole_factory, interface="127.0.0.1") logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) - leap_events.signal(IMAP_SERVICE_STARTED, str(port)) + emit(catalog.IMAP_SERVICE_STARTED, str(port)) # FIXME -- change service signature return tport, factory # not ok, signal error. - leap_events.signal(IMAP_SERVICE_FAILED_TO_START, str(port)) + emit(catalog.IMAP_SERVICE_FAILED_TO_START, str(port)) diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index ea790fe..be37396 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -38,15 +38,8 @@ from twisted.internet.task import LoopingCall from twisted.internet.task import deferLater from u1db import errors as u1db_errors -from leap.common import events as leap_events +from leap.common.events import emit, catalog from leap.common.check import leap_assert, leap_assert_type -from leap.common.events.events_pb2 import IMAP_FETCHED_INCOMING -from leap.common.events.events_pb2 import IMAP_MSG_PROCESSING -from leap.common.events.events_pb2 import IMAP_MSG_DECRYPTED -from leap.common.events.events_pb2 import IMAP_MSG_SAVED_LOCALLY -from leap.common.events.events_pb2 import IMAP_MSG_DELETED_INCOMING -from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL -from leap.common.events.events_pb2 import SOLEDAD_INVALID_AUTH_TOKEN from leap.common.mail import get_email_charset from leap.keymanager import errors as keymanager_errors from leap.keymanager.openpgp import OpenPGPKey @@ -242,7 +235,7 @@ class IncomingMail(Service): except InvalidAuthTokenError: # if the token is invalid, send an event so the GUI can # disable mail and show an error message. - leap_events.signal(SOLEDAD_INVALID_AUTH_TOKEN) + emit(catalog.SOLEDAD_INVALID_AUTH_TOKEN) def _signal_fetch_to_ui(self, doclist): """ @@ -259,16 +252,16 @@ class IncomingMail(Service): num_mails = len(doclist) if doclist is not None else 0 if num_mails != 0: log.msg("there are %s mails" % (num_mails,)) - leap_events.signal( - IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts)) + emit(catalog.MAIL_FETCHED_INCOMING, + str(num_mails), str(fetched_ts)) return doclist def _signal_unread_to_ui(self, *args): """ Sends unread event to ui. """ - leap_events.signal( - IMAP_UNREAD_MAIL, str(self._inbox_collection.count_unseen())) + emit(catalog.MAIL_UNREAD_MESSAGES, + str(self._inbox_collection.count_unseen())) # process incoming mail. @@ -291,8 +284,8 @@ class IncomingMail(Service): deferreds = [] for index, doc in enumerate(doclist): logger.debug("processing doc %d of %d" % (index + 1, num_mails)) - leap_events.signal( - IMAP_MSG_PROCESSING, str(index), str(num_mails)) + emit(catalog.MAIL_MSG_PROCESSING, + str(index), str(num_mails)) keys = doc.content.keys() @@ -339,7 +332,7 @@ class IncomingMail(Service): decrdata = "" success = False - leap_events.signal(IMAP_MSG_DECRYPTED, "1" if success else "0") + emit(catalog.MAIL_MSG_DECRYPTED, "1" if success else "0") return self._process_decrypted_doc(doc, decrdata) d = self._keymanager.decrypt( @@ -724,10 +717,10 @@ class IncomingMail(Service): listener(result) def signal_deleted(doc_id): - leap_events.signal(IMAP_MSG_DELETED_INCOMING) + emit(catalog.MAIL_MSG_DELETED_INCOMING) return doc_id - leap_events.signal(IMAP_MSG_SAVED_LOCALLY) + emit(catalog.MAIL_MSG_SAVED_LOCALLY) d = self._delete_incoming_message(doc) d.addCallback(signal_deleted) return d diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 4fe08a6..1649d4a 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -27,8 +27,7 @@ import weakref from twisted.internet import defer from leap.common.check import leap_assert_type -from leap.common import events as leap_events -from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL +from leap.common.events import emit, catalog from leap.common.mail import get_email_charset from leap.mail.adaptors.soledad import SoledadMailAdaptor @@ -654,7 +653,7 @@ class MessageCollection(object): :type unseen: int """ # TODO change name of the signal, independent from imap now. - leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) + emit(catalog.MAIL_UNREAD_MESSAGES, str(unseen)) def copy_msg(self, msg, new_mbox_uuid): """ diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py index f3c2320..60ba8f5 100644 --- a/src/leap/mail/outgoing/service.py +++ b/src/leap/mail/outgoing/service.py @@ -31,7 +31,7 @@ from twisted.protocols.amp import ssl from twisted.python import log from leap.common.check import leap_assert_type, leap_assert -from leap.common.events import proto, signal +from leap.common.events import emit, catalog from leap.keymanager import KeyManager from leap.keymanager.openpgp import OpenPGPKey from leap.keymanager.errors import KeyNotFound, KeyAddressMismatch @@ -136,7 +136,7 @@ class OutgoingMail: """ dest_addrstr = smtp_sender_result[1][0][0] log.msg('Message sent to %s' % dest_addrstr) - signal(proto.SMTP_SEND_MESSAGE_SUCCESS, dest_addrstr) + emit(catalog.SMTP_SEND_MESSAGE_SUCCESS, dest_addrstr) def sendError(self, failure): """ @@ -146,7 +146,7 @@ class OutgoingMail: :type e: anything """ # XXX: need to get the address from the exception to send signal - # signal(proto.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr) + # emit(catalog.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr) err = failure.value log.err(err) raise err @@ -179,7 +179,7 @@ class OutgoingMail: requireAuthentication=False, requireTransportSecurity=True) factory.domain = __version__ - signal(proto.SMTP_SEND_MESSAGE_START, recipient.dest.addrstr) + emit(catalog.SMTP_SEND_MESSAGE_START, recipient.dest.addrstr) reactor.connectSSL( self._host, self._port, factory, contextFactory=SSLContextFactory(self._cert, self._key)) @@ -241,7 +241,7 @@ class OutgoingMail: return d def signal_encrypt_sign(newmsg): - signal(proto.SMTP_END_ENCRYPT_AND_SIGN, + emit(catalog.SMTP_END_ENCRYPT_AND_SIGN, "%s,%s" % (self._from_address, to_address)) return newmsg, recipient @@ -249,18 +249,18 @@ class OutgoingMail: failure.trap(KeyNotFound, KeyAddressMismatch) log.msg('Will send unencrypted message to %s.' % to_address) - signal(proto.SMTP_START_SIGN, self._from_address) + emit(catalog.SMTP_START_SIGN, self._from_address) d = self._sign(message, from_address) d.addCallback(signal_sign) return d def signal_sign(newmsg): - signal(proto.SMTP_END_SIGN, self._from_address) + emit(catalog.SMTP_END_SIGN, self._from_address) return newmsg, recipient log.msg("Will encrypt the message with %s and sign with %s." % (to_address, from_address)) - signal(proto.SMTP_START_ENCRYPT_AND_SIGN, + emit(catalog.SMTP_START_ENCRYPT_AND_SIGN, "%s,%s" % (self._from_address, to_address)) d = self._maybe_attach_key(origmsg, from_address, to_address) d.addCallback(maybe_encrypt_and_sign) diff --git a/src/leap/mail/smtp/__init__.py b/src/leap/mail/smtp/__init__.py index 24402b4..3ef016b 100644 --- a/src/leap/mail/smtp/__init__.py +++ b/src/leap/mail/smtp/__init__.py @@ -26,7 +26,7 @@ from leap.mail.outgoing.service import OutgoingMail logger = logging.getLogger(__name__) -from leap.common.events import proto, signal +from leap.common.events import emit, catalog from leap.mail.smtp.gateway import SMTPFactory @@ -65,12 +65,12 @@ def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port, factory = SMTPFactory(userid, keymanager, encrypted_only, outgoing_mail) try: tport = reactor.listenTCP(port, factory, interface="localhost") - signal(proto.SMTP_SERVICE_STARTED, str(port)) + emit(catalog.SMTP_SERVICE_STARTED, str(port)) return factory, tport except CannotListenError: logger.error("STMP Service failed to start: " "cannot listen in port %s" % port) - signal(proto.SMTP_SERVICE_FAILED_TO_START, str(port)) + emit(catalog.SMTP_SERVICE_FAILED_TO_START, str(port)) except Exception as exc: logger.error("Unhandled error while launching smtp gateway service") logger.exception(exc) diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index 954a7d0..dd2c32d 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -39,7 +39,7 @@ from twisted.python import log from email.Header import Header from leap.common.check import leap_assert_type -from leap.common.events import proto, signal +from leap.common.events import emit, catalog from leap.keymanager.openpgp import OpenPGPKey from leap.keymanager.errors import KeyNotFound from leap.mail.utils import validate_address @@ -201,19 +201,19 @@ class SMTPDelivery(object): # verify if recipient key is available in keyring def found(_): log.msg("Accepting mail for %s..." % user.dest.addrstr) - signal(proto.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, user.dest.addrstr) + emit(catalog.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, user.dest.addrstr) def not_found(failure): failure.trap(KeyNotFound) # if key was not found, check config to see if will send anyway if self._encrypted_only: - signal(proto.SMTP_RECIPIENT_REJECTED, user.dest.addrstr) + emit(catalog.SMTP_RECIPIENT_REJECTED, user.dest.addrstr) raise smtp.SMTPBadRcpt(user.dest.addrstr) log.msg("Warning: will send an unencrypted message (because " "encrypted_only' is set to False).") - signal( - proto.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, + emit( + catalog.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, user.dest.addrstr) def encrypt_func(_): @@ -306,7 +306,7 @@ class EncryptedMessage(object): """ log.msg("Connection lost unexpectedly!") log.err() - signal(proto.SMTP_CONNECTION_LOST, self._user.dest.addrstr) + emit(catalog.SMTP_CONNECTION_LOST, self._user.dest.addrstr) # unexpected loss of connection; don't save self._lines = [] -- cgit v1.2.3 From 8915db2e1c1a339cf36ae0e28da627bb1a8e040e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 21 May 2015 12:47:27 -0400 Subject: [feature] post-sync mail processing hooks using the new soledad plugin capablity, mail hooks to the post-sync event by subscribing to the Meta-Doc type of documents. In this way, we can create the uid tables and the uid entries needed to keep local indexes for mail that has been processed in another instance. however, this won't prevent a conflict if a given mail is received and processed in two different instances. that is a problem that we still have to deal with. Resolves: #6996 Releases: 0.4.0 --- .gitignore | 1 + src/leap/mail/imap/service/imap.py | 9 +++ src/leap/mail/mail.py | 2 - src/leap/mail/plugins/__init__.py | 3 + src/leap/mail/plugins/soledad_sync_hooks.py | 19 +++++ src/leap/mail/sync_hooks.py | 121 ++++++++++++++++++++++++++++ 6 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 src/leap/mail/plugins/__init__.py create mode 100644 src/leap/mail/plugins/soledad_sync_hooks.py create mode 100644 src/leap/mail/sync_hooks.py diff --git a/.gitignore b/.gitignore index 7ac8289..aafbdd1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc +dropin.cache build/ dist/ *.egg diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 370c513..e401283 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -35,6 +35,7 @@ from leap.common.events import emit, catalog from leap.common.check import leap_assert_type, leap_check from leap.mail.imap.account import IMAPAccount from leap.mail.imap.server import LEAPIMAPServer +from leap.mail.plugins import soledad_sync_hooks from leap.soledad.client import Soledad @@ -91,10 +92,17 @@ class LeapIMAPFactory(ServerFactory): theAccount = IMAPAccount(uuid, soledad) self.theAccount = theAccount + self._initialize_sync_hooks() self._connections = defaultdict() # XXX how to pass the store along? + def _initialize_sync_hooks(self): + soledad_sync_hooks.post_sync_uid_reindexer.set_account(self.theAccount) + + def _teardown_sync_hooks(self): + soledad_sync_hooks.post_sync_uid_reindexer.set_account(None) + def buildProtocol(self, addr): """ Return a protocol suitable for the job. @@ -128,6 +136,7 @@ class LeapIMAPFactory(ServerFactory): # mark account as unusable, so any imap command will fail # with unauth state. self.theAccount.end_session() + self._teardown_sync_hooks() # TODO should wait for all the pending deferreds, # the twisted way! diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 1649d4a..bab73cb 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -42,8 +42,6 @@ logger = logging.getLogger(name=__name__) # TODO LIST # [ ] Probably change the name of this module to "api" or "account", mail is # too generic (there's also IncomingMail, and OutgoingMail -# [ ] Change the doc_ids scheme for part-docs: use mailbox UID validity -# identifier, instead of name! (renames are broken!) # [ ] Profile add_msg. def _get_mdoc_id(mbox, chash): diff --git a/src/leap/mail/plugins/__init__.py b/src/leap/mail/plugins/__init__.py new file mode 100644 index 0000000..ddb8691 --- /dev/null +++ b/src/leap/mail/plugins/__init__.py @@ -0,0 +1,3 @@ +from twisted.plugin import pluginPackagePaths +__path__.extend(pluginPackagePaths(__name__)) +__all__ = [] diff --git a/src/leap/mail/plugins/soledad_sync_hooks.py b/src/leap/mail/plugins/soledad_sync_hooks.py new file mode 100644 index 0000000..9d48126 --- /dev/null +++ b/src/leap/mail/plugins/soledad_sync_hooks.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# soledad_sync_hooks.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from leap.mail.sync_hooks import MailProcessingPostSyncHook +post_sync_uid_reindexer = MailProcessingPostSyncHook() diff --git a/src/leap/mail/sync_hooks.py b/src/leap/mail/sync_hooks.py new file mode 100644 index 0000000..b5bded5 --- /dev/null +++ b/src/leap/mail/sync_hooks.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +# sync_hooks.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Soledad PostSync Hooks. + +Process every new document of interest after every soledad synchronization, +using the hooks that soledad exposes via plugins. +""" +import logging + +from re import compile as regex_compile + +from zope.interface import implements +from twisted.internet import defer +from twisted.plugin import IPlugin +from twisted.python import log + +from leap.soledad.client.interfaces import ISoledadPostSyncPlugin +from leap.mail import constants + + +logger = logging.getLogger(__name__) + +_get_doc_type_preffix = lambda s: s[:2] + + +class MailProcessingPostSyncHook(object): + implements(IPlugin, ISoledadPostSyncPlugin) + + META_DOC_PREFFIX = _get_doc_type_preffix(constants.METAMSGID) + watched_doc_types = (META_DOC_PREFFIX, ) + + _account = None + _pending_docs = [] + _processing_deferreds = [] + + def process_received_docs(self, doc_id_list): + if self._has_configured_account(): + process_fun = self._make_uid_index + else: + self._processing_deferreds = [] + process_fun = self._queue_doc_id + + for doc_id in doc_id_list: + if _get_doc_type_preffix(doc_id) in self.watched_doc_types: + log.msg("Mail post-sync hook: processing %s" % doc_id) + process_fun(doc_id) + + if self._processing_deferreds: + return defer.gatherResults(self._processing_deferreds) + + def set_account(self, account): + self._account = account + if account: + self._process_queued_docs() + + def _has_configured_account(self): + return self._account is not None + + def _queue_doc_id(self, doc_id): + self._pending_docs.append(doc_id) + + def _make_uid_index(self, mdoc_id): + indexer = self._account.account.mbox_indexer + mbox_uuid = _get_mbox_uuid(mdoc_id) + if mbox_uuid: + chash = _get_chash_from_mdoc(mdoc_id) + logger.debug("Making index table for %s:%s" % (mbox_uuid, chash)) + index_docid = constants.METAMSGID.format( + mbox_uuid=mbox_uuid.replace('-', '_'), + chash=chash) + # XXX could avoid creating table if I track which ones I already + # have seen -- but make sure *it's already created* before + # inserting the index entry!. + d = indexer.create_table(mbox_uuid) + d.addCallback(lambda _: indexer.insert_doc(mbox_uuid, index_docid)) + self._processing_deferreds.append(d) + + def _process_queued_docs(self): + assert(self._has_configured_account()) + pending = self._pending_docs + log.msg("Mail post-sync hook: processing queued docs") + + def remove_pending_docs(res): + self._pending_docs = [] + return res + + d = self.process_received_docs(pending) + if d: + d.addCallback(remove_pending_docs) + return d + + +_mbox_uuid_regex = regex_compile(constants.METAMSGID_MBOX_RE) +_mdoc_chash_regex = regex_compile(constants.METAMSGID_CHASH_RE) + + +def _get_mbox_uuid(doc_id): + matches = _mbox_uuid_regex.findall(doc_id) + if matches: + return matches[0].replace('_', '-') + + +def _get_chash_from_mdoc(doc_id): + matches = _mdoc_chash_regex.findall(doc_id) + if matches: + return matches[0] -- cgit v1.2.3 From 4c12d05d063d42171808c432bee90de67c4042b6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 28 May 2015 19:03:24 -0400 Subject: [refactor] move hooks to account --- src/leap/mail/imap/account.py | 7 +++++-- src/leap/mail/imap/service/imap.py | 13 +------------ src/leap/mail/incoming/service.py | 1 + src/leap/mail/mail.py | 22 ++++++++++++++++++++++ src/leap/mail/sync_hooks.py | 2 +- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index ccb4b75..cc56fff 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -58,7 +58,6 @@ class IMAPAccount(object): implements(imap4.IAccount, imap4.INamespacePresenter) selected = None - session_ended = False def __init__(self, user_id, store, d=defer.Deferred()): """ @@ -98,7 +97,11 @@ class IMAPAccount(object): Right now it's called from the client backend. """ # TODO move its use to the service shutdown in leap.mail - self.session_ended = True + self.account.end_session() + + @property + def session_ended(self): + return self.account.session_ended def callWhenReady(self, cb, *args, **kw): """ diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index e401283..92d05cc 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -32,11 +32,9 @@ from twisted.python import log logger = logging.getLogger(__name__) from leap.common.events import emit, catalog -from leap.common.check import leap_assert_type, leap_check +from leap.common.check import leap_check from leap.mail.imap.account import IMAPAccount from leap.mail.imap.server import LEAPIMAPServer -from leap.mail.plugins import soledad_sync_hooks -from leap.soledad.client import Soledad DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) @@ -92,17 +90,9 @@ class LeapIMAPFactory(ServerFactory): theAccount = IMAPAccount(uuid, soledad) self.theAccount = theAccount - self._initialize_sync_hooks() - self._connections = defaultdict() # XXX how to pass the store along? - def _initialize_sync_hooks(self): - soledad_sync_hooks.post_sync_uid_reindexer.set_account(self.theAccount) - - def _teardown_sync_hooks(self): - soledad_sync_hooks.post_sync_uid_reindexer.set_account(None) - def buildProtocol(self, addr): """ Return a protocol suitable for the job. @@ -136,7 +126,6 @@ class LeapIMAPFactory(ServerFactory): # mark account as unusable, so any imap command will fail # with unauth state. self.theAccount.end_session() - self._teardown_sync_hooks() # TODO should wait for all the pending deferreds, # the twisted way! diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index be37396..23aff3d 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -246,6 +246,7 @@ class IncomingMail(Service): :returns: doclist :rtype: iterable """ + # FIXME WTF len(doclist) is 69? doclist = first(doclist) # gatherResults pass us a list if doclist: fetched_ts = time.mktime(time.gmtime()) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index bab73cb..fe8226e 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -34,6 +34,7 @@ from leap.mail.adaptors.soledad import SoledadMailAdaptor from leap.mail.constants import INBOX_NAME from leap.mail.constants import MessageFlags from leap.mail.mailbox_indexer import MailboxIndexer +from leap.mail.plugins import soledad_sync_hooks from leap.mail.utils import find_charset, CaseInsensitiveDict logger = logging.getLogger(name=__name__) @@ -802,10 +803,17 @@ class Account(object): self.adaptor = self.adaptor_class() self.mbox_indexer = MailboxIndexer(self.store) + # This flag is only used from the imap service for the moment. + # In the future, we should prevent any public method to continue if + # this is set to True. Also, it would be good to plug to the + # authentication layer. + self.session_ended = False + self.deferred_initialization = defer.Deferred() self._ready_cb = ready_cb self._init_d = self._initialize_storage() + self._initialize_sync_hooks() def _initialize_storage(self): @@ -834,6 +842,14 @@ class Account(object): self.deferred_initialization.addCallback(cb, *args, **kw) return self.deferred_initialization + # Sync hooks + + def _initialize_sync_hooks(self): + soledad_sync_hooks.post_sync_uid_reindexer.set_account(self) + + def _teardown_sync_hooks(self): + soledad_sync_hooks.post_sync_uid_reindexer.set_account(None) + # # Public API Starts # @@ -946,3 +962,9 @@ class Account(object): :rtype: MessageCollection """ raise NotImplementedError() + + # Session handling + + def end_session(self): + self._teardown_sync_hooks() + self.session_ended = True diff --git a/src/leap/mail/sync_hooks.py b/src/leap/mail/sync_hooks.py index b5bded5..3cf858b 100644 --- a/src/leap/mail/sync_hooks.py +++ b/src/leap/mail/sync_hooks.py @@ -75,7 +75,7 @@ class MailProcessingPostSyncHook(object): self._pending_docs.append(doc_id) def _make_uid_index(self, mdoc_id): - indexer = self._account.account.mbox_indexer + indexer = self._account.mbox_indexer mbox_uuid = _get_mbox_uuid(mdoc_id) if mbox_uuid: chash = _get_chash_from_mdoc(mdoc_id) -- cgit v1.2.3 From 2cf8aa4a285cde6e8bdcaaf1d374b8762d36714c Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 4 Jun 2015 10:22:34 -0400 Subject: [bug] prevent missing uid table exception --- src/leap/mail/mail.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index fe8226e..8cb0b4a 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -615,8 +615,13 @@ class MessageCollection(object): return defer.succeed("mdoc_id not inserted") # XXX BUG ----------------------------------------- - return self.mbox_indexer.insert_doc( - self.mbox_uuid, doc_id) + # XXX BUG sometimes the table is not yet created, + # so workaround is to make sure we always check for it before + # inserting the doc. I should debug into the real cause. + d = self.mbox_indexer.create_table(self.mbox_uuid) + d.addCallback(lambda _: self.mbox_indexer.insert_doc( + self.mbox_uuid, doc_id)) + return d d = wrapper.create( self.store, -- cgit v1.2.3 From e46b14e6966045414a61103b3819a4277af7d211 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 4 Jun 2015 10:23:07 -0400 Subject: [feature] use operation, doesn't return result --- src/leap/mail/mailbox_indexer.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 664d580..ab0967d 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -88,6 +88,10 @@ class MailboxIndexer(object): assert self.store is not None return self.store.raw_sqlcipher_query(*args, **kw) + def _operation(self, *args, **kw): + assert self.store is not None + return self.store.raw_sqlcipher_operation(*args, **kw) + def create_table(self, mailbox_uuid): """ Create the UID table for a given mailbox. @@ -100,7 +104,8 @@ class MailboxIndexer(object): "uid INTEGER PRIMARY KEY AUTOINCREMENT, " "hash TEXT UNIQUE NOT NULL)".format( preffix=self.table_preffix, name=sanitize(mailbox_uuid))) - return self._query(sql) + print "CREATING TABLE..." + return self._operation(sql) def delete_table(self, mailbox_uuid): """ @@ -112,7 +117,7 @@ class MailboxIndexer(object): check_good_uuid(mailbox_uuid) sql = ("DROP TABLE if exists {preffix}{name}".format( preffix=self.table_preffix, name=sanitize(mailbox_uuid))) - return self._query(sql) + return self._operation(sql) def insert_doc(self, mailbox_uuid, doc_id): """ @@ -149,7 +154,7 @@ class MailboxIndexer(object): "LIMIT 1;").format( preffix=self.table_preffix, name=sanitize(mailbox_uuid)) - d = self._query(sql, values) + d = self._operation(sql, values) d.addCallback(lambda _: self._query(sql_last)) d.addCallback(get_rowid) d.addErrback(lambda f: f.printTraceback()) -- cgit v1.2.3 From 9f31b26909a62d5d76e06b51e1c7204778b1882d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 4 Jun 2015 20:08:46 -0400 Subject: [refactor] deprecate old incoming index --- src/leap/mail/adaptors/soledad_indexes.py | 5 +---- src/leap/mail/incoming/service.py | 16 +++++----------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/leap/mail/adaptors/soledad_indexes.py b/src/leap/mail/adaptors/soledad_indexes.py index d2f8b71..eec7d28 100644 --- a/src/leap/mail/adaptors/soledad_indexes.py +++ b/src/leap/mail/adaptors/soledad_indexes.py @@ -101,9 +101,6 @@ MAIL_INDEXES = { TYPE_MBOX_DEL_IDX: [TYPE, MBOX_UUID, 'bool(deleted)'], # incoming queue - JUST_MAIL_IDX: [INCOMING_KEY, + JUST_MAIL_IDX: ["bool(%s)" % (INCOMING_KEY,), "bool(%s)" % (ERROR_DECRYPTING_KEY,)], - - # the backward-compatible index, will be deprecated at 0.7 - JUST_MAIL_COMPAT_IDX: [INCOMING_KEY], } diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index 23aff3d..71edf08 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -161,21 +161,15 @@ class IncomingMail(Service): Calls a deferred that will execute the fetch callback in a separate thread """ - def mail_compat(failure): - if failure.check(u1db_errors.InvalidGlobbing): - # It looks like we are a dealing with an outdated - # mx. Fallback to the version of the index - warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", - DeprecationWarning) - return self._soledad.get_from_index( - fields.JUST_MAIL_COMPAT_IDX, "*") - return failure + def _sync_errback(failure): + failure.printTraceback() def syncSoledadCallback(_): + # XXX this should be moved to adaptors d = self._soledad.get_from_index( - fields.JUST_MAIL_IDX, "*", "0") - d.addErrback(mail_compat) + fields.JUST_MAIL_IDX, "1", "0") d.addCallback(self._process_doclist) + d.addErrback(_sync_errback) return d logger.debug("fetching mail for: %s %s" % ( -- cgit v1.2.3 From ccc627d7b84b8649c1ec9c49504a1b662af40707 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 5 Jun 2015 16:52:25 -0400 Subject: [pkg] changes file for post-sync hooks feature --- changes/feature_6996-post-sync-hooks | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/feature_6996-post-sync-hooks diff --git a/changes/feature_6996-post-sync-hooks b/changes/feature_6996-post-sync-hooks new file mode 100644 index 0000000..e03c28e --- /dev/null +++ b/changes/feature_6996-post-sync-hooks @@ -0,0 +1 @@ +- Ability to reindex local UIDs after a soledad sync. Closes: #6996 -- cgit v1.2.3 From a94677b94326ffa2900141092c70d9559e01853a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 9 Jun 2015 11:32:33 -0400 Subject: [refactor] log failure properly --- src/leap/mail/incoming/service.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index 71edf08..4738dd4 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -162,7 +162,7 @@ class IncomingMail(Service): in a separate thread """ def _sync_errback(failure): - failure.printTraceback() + log.err(failure) def syncSoledadCallback(_): # XXX this should be moved to adaptors @@ -206,8 +206,7 @@ class IncomingMail(Service): # synchronize incoming mail def _errback(self, failure): - logger.exception(failure.value) - traceback.print_exc() + log.err(failure) def _sync_soledad(self): """ -- cgit v1.2.3 From 03cc09d35c880a5805ac9bdbad4a8b6ebe8e717f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 9 Jun 2015 11:32:41 -0400 Subject: [bug] pass the doclist to the ui signal before, we were taking the length of a string, signalling an incorrect number to the ui. currently this event is not being used, just only logged. in the future the ui could probably might want to make use of this info to keep record of a separate counter (how many mails received in the last sync). --- src/leap/mail/incoming/service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index 4738dd4..3daf86b 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -239,8 +239,6 @@ class IncomingMail(Service): :returns: doclist :rtype: iterable """ - # FIXME WTF len(doclist) is 69? - doclist = first(doclist) # gatherResults pass us a list if doclist: fetched_ts = time.mktime(time.gmtime()) num_mails = len(doclist) if doclist is not None else 0 @@ -299,7 +297,9 @@ class IncomingMail(Service): d.addCallback(self._extract_keys) d.addCallbacks(self._add_message_locally, self._errback) deferreds.append(d) - return defer.gatherResults(deferreds, consumeErrors=True) + d = defer.gatherResults(deferreds, consumeErrors=True) + d.addCallback(lambda _: doclist) + return d # # operations on individual messages -- cgit v1.2.3 From b7d1c1b40b18c8336edc07c76ef9b3047d0bf95b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 10 Jun 2015 10:49:34 -0400 Subject: [refactor] remove unneeded conditional --- src/leap/mail/sync_hooks.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/leap/mail/sync_hooks.py b/src/leap/mail/sync_hooks.py index 3cf858b..a8a69c9 100644 --- a/src/leap/mail/sync_hooks.py +++ b/src/leap/mail/sync_hooks.py @@ -60,8 +60,7 @@ class MailProcessingPostSyncHook(object): log.msg("Mail post-sync hook: processing %s" % doc_id) process_fun(doc_id) - if self._processing_deferreds: - return defer.gatherResults(self._processing_deferreds) + return defer.gatherResults(self._processing_deferreds) def set_account(self, account): self._account = account @@ -100,9 +99,8 @@ class MailProcessingPostSyncHook(object): return res d = self.process_received_docs(pending) - if d: - d.addCallback(remove_pending_docs) - return d + d.addCallback(remove_pending_docs) + return d _mbox_uuid_regex = regex_compile(constants.METAMSGID_MBOX_RE) -- cgit v1.2.3 From 061024899e29f745b3c375930b60247de36e2fd3 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 10 Jun 2015 17:50:09 -0400 Subject: [docs] minimal mutt configuration snippet because we love to test with mutt. --- docs/hacking.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/hacking.rst b/docs/hacking.rst index bd9f792..d5669e1 100644 --- a/docs/hacking.rst +++ b/docs/hacking.rst @@ -52,6 +52,19 @@ currently) not try to sync with remote replicas. Very useful during development, although you need to login with the remote server at least once before being able to use it. +Mutt config +=========== + +You cannot live without mutt? You're lucky! Use the following minimal config +with the imap service:: + + set folder="imap://user@provider@localhost:1984" + set spoolfile="imap://user@provider@localhost:1984/INBOX" + set ssl_starttls = no + set ssl_force_tls = no + set imap_pass=MAHSIKRET + + Running the service with twistd =============================== -- cgit v1.2.3 From f1c1eda351ccb1c8c21c33f0ea83ed93b16dd368 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 18 Jun 2015 12:29:37 -0300 Subject: [doc] update smtp gateway doc on message encryption Closes: #7169. --- changes/bug_7169_update-smtp-gateway-doc | 1 + src/leap/mail/smtp/README.rst | 41 ++++++++++++++++++++++++-------- src/leap/mail/smtp/gateway.py | 32 ++++++++++++------------- 3 files changed, 48 insertions(+), 26 deletions(-) create mode 100644 changes/bug_7169_update-smtp-gateway-doc diff --git a/changes/bug_7169_update-smtp-gateway-doc b/changes/bug_7169_update-smtp-gateway-doc new file mode 100644 index 0000000..5b86140 --- /dev/null +++ b/changes/bug_7169_update-smtp-gateway-doc @@ -0,0 +1 @@ + o Update SMTP gateway docs. Closes #7169. diff --git a/src/leap/mail/smtp/README.rst b/src/leap/mail/smtp/README.rst index f625441..1d3a903 100644 --- a/src/leap/mail/smtp/README.rst +++ b/src/leap/mail/smtp/README.rst @@ -1,18 +1,39 @@ Leap SMTP Gateway ================= +The Bitmask Client runs a thin SMTP gateway on the user's device, which +intends to encrypt and sign outgoing messages to achieve point to point +encryption. + +The gateway is bound to localhost and the user's MUA should be configured to +send messages to it. After doing its thing, the gateway will relay the +messages to the remote SMTP server. + Outgoing mail workflow: - * LEAP client runs a thin SMTP proxy on the user's device, bound to - localhost. - * User's MUA is configured outgoing SMTP to localhost. - * When SMTP proxy receives an email from MUA: - * SMTP proxy queries Key Manager for the user's private key and public - keys of all recipients. - * Message is signed by sender and encrypted to recipients. - * If recipient's key is missing, email goes out in cleartext (unless - user has configured option to send only encrypted email). - * Finally, message is gatewayed to provider's SMTP server. + * SMTP gateway receives a message from the MUA. + + * SMTP gateway queries Key Manager for the user's private key. + + * For each recipient (including addresses in "To", "Cc" anc "Bcc" fields), + the following happens: + + - The recipient's address is validated against RFC2822. + + - An attempt is made to fetch the recipient's public PGP key. + + - If key is not found: + + - If the gateway is configured to only send encrypted messages the + recipient is rejected. + + - Otherwise, the message is signed and sent as plain text. + + - If the key is found, the message is encrypted to the recipient and + signed with the sender's private PGP key. + + * Finally, one message for each recipient is gatewayed to provider's SMTP + server. Running tests diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index dd2c32d..f6182a2 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -21,15 +21,13 @@ The following classes comprise the SMTP gateway service: * SMTPFactory - A twisted.internet.protocol.ServerFactory that provides the SMTPDelivery protocol. + * SMTPDelivery - A twisted.mail.smtp.IMessageDelivery implementation. It knows how to validate sender and receiver of messages and it generates an EncryptedMessage for each recipient. - * SSLContextFactory - Contains the relevant ssl information for the - connection. + * EncryptedMessage - An implementation of twisted.mail.smtp.IMessage that knows how to encrypt/sign itself before sending. - - """ from zope.interface import implements @@ -173,27 +171,29 @@ class SMTPDelivery(object): def validateTo(self, user): """ - Validate the address of C{user}, a recipient of the message. + Validate the address of a recipient of the message, possibly + rejecting it if the recipient key is not available. + + This method is called once for each recipient, i.e. for each SMTP + protocol line beginning with "RCPT TO:", which includes all addresses + in "To", "Cc" and "Bcc" MUA fields. - This method is called once for each recipient and validates the - C{user}'s address against the RFC 2822 definition. If the - configuration option ENCRYPTED_ONLY_KEY is True, it also asserts the - existence of the user's key. + The recipient's address is validated against the RFC 2822 definition. + If self._encrypted_only is True and no key is found for a recipient, + then that recipient is rejected. - In the end, it returns an encrypted message object that is able to - send itself to the C{user}'s address. + The method returns an encrypted message object that is able to send + itself to the user's address. :param user: The user whose address we wish to validate. :type: twisted.mail.smtp.User - @return: A Deferred which becomes, or a callable which takes no - arguments and returns an object implementing IMessage. This will - be called and the returned object used to deliver the message when - it arrives. + @return: A callable which takes no arguments and returns an + encryptedMessage. @rtype: no-argument callable @raise SMTPBadRcpt: Raised if messages to the address are not to be - accepted. + accepted. """ # try to find recipient's public key address = validate_address(user.dest.addrstr) -- cgit v1.2.3 From 1348798e88c6026f57ff938c59a2dce979a9a891 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 18 Jun 2015 12:34:43 -0300 Subject: [style] pep8 and unused imports cleanup --- src/leap/mail/outgoing/service.py | 9 ++++----- src/leap/mail/smtp/gateway.py | 5 ++++- src/leap/mail/smtp/rfc3156.py | 2 -- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py index 60ba8f5..838a908 100644 --- a/src/leap/mail/outgoing/service.py +++ b/src/leap/mail/outgoing/service.py @@ -32,7 +32,6 @@ from twisted.python import log from leap.common.check import leap_assert_type, leap_assert from leap.common.events import emit, catalog -from leap.keymanager import KeyManager from leap.keymanager.openpgp import OpenPGPKey from leap.keymanager.errors import KeyNotFound, KeyAddressMismatch from leap.mail import __version__ @@ -169,8 +168,8 @@ class OutgoingMail: # we don't pass an ssl context factory to the ESMTPSenderFactory # because ssl will be handled by reactor.connectSSL() below. factory = smtp.ESMTPSenderFactory( - "", # username is blank because client auth is done on SSL protocol level - "", # password is blank because client auth is done on SSL protocol level + "", # username is blank, no client auth here + "", # password is blank, no client auth here self._from_address, recipient.dest.addrstr, StringIO(msg), @@ -242,7 +241,7 @@ class OutgoingMail: def signal_encrypt_sign(newmsg): emit(catalog.SMTP_END_ENCRYPT_AND_SIGN, - "%s,%s" % (self._from_address, to_address)) + "%s,%s" % (self._from_address, to_address)) return newmsg, recipient def if_key_not_found_send_unencrypted(failure, message): @@ -261,7 +260,7 @@ class OutgoingMail: log.msg("Will encrypt the message with %s and sign with %s." % (to_address, from_address)) emit(catalog.SMTP_START_ENCRYPT_AND_SIGN, - "%s,%s" % (self._from_address, to_address)) + "%s,%s" % (self._from_address, to_address)) d = self._maybe_attach_key(origmsg, from_address, to_address) d.addCallback(maybe_encrypt_and_sign) return d diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index f6182a2..7dae907 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -111,7 +111,10 @@ class SMTPFactory(ServerFactory): @return: The protocol. @rtype: SMTPDelivery """ - smtpProtocol = SMTPHeloLocalhost(SMTPDelivery(self._userid, self._km, self._encrypted_only, self._outgoing_mail)) + smtpProtocol = SMTPHeloLocalhost( + SMTPDelivery( + self._userid, self._km, self._encrypted_only, + self._outgoing_mail)) smtpProtocol.factory = self return smtpProtocol diff --git a/src/leap/mail/smtp/rfc3156.py b/src/leap/mail/smtp/rfc3156.py index 62a0675..7d7bc0f 100644 --- a/src/leap/mail/smtp/rfc3156.py +++ b/src/leap/mail/smtp/rfc3156.py @@ -19,9 +19,7 @@ Implements RFC 3156: MIME Security with OpenPGP. """ -import re import base64 -from abc import ABCMeta, abstractmethod from StringIO import StringIO from twisted.python import log -- cgit v1.2.3 From c8dfed5b5f4ccb87003119f14e189566219365bb Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 18 Jun 2015 23:20:45 -0400 Subject: [bug] fixes for display attachments and move between folders - Add errback handling to catch properly errors that were not allowing the complete insertion of the parts for a given message. Fixes blank attachments and moving of messages to different folders. - Force overwritting of mdoc when it is a copy. This was avoiding a message to be copied back to a folder from where it already had been copied to another (since the mdoc was already existing there, with the same doc_id, which was forbidding the creation of the new one). This case also needs special care in the indexer, since we have to delete the old hash entry first. Closes: #7178, #7158 --- src/leap/mail/adaptors/soledad.py | 45 ++++++++++++++++++++++++++++------- src/leap/mail/imap/mailbox.py | 5 ++-- src/leap/mail/mail.py | 49 +++++++++++++++++++++++++++++++++++---- src/leap/mail/mailbox_indexer.py | 4 +--- src/leap/mail/sync_hooks.py | 2 +- 5 files changed, 86 insertions(+), 19 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index b8e5fd4..dc0960f 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -24,6 +24,7 @@ from email import message_from_string from pycryptopp.hash import sha256 from twisted.internet import defer +from twisted.python import log from zope.interface import implements import u1db @@ -108,7 +109,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): def set_future_doc_id(self, doc_id): self._future_doc_id = doc_id - def create(self, store): + def create(self, store, is_copy=False): """ Create the documents for this wrapper. Since this method will not check for duplication, the @@ -130,13 +131,28 @@ class SoledadDocumentWrapper(models.DocumentWrapper): self.set_future_doc_id(None) return doc + def update_wrapper(failure): + # In the case of some copies (for instance, from one folder to + # another and back to the original folder), the document that we + # want to insert already exists. In this case, putting it + # and overwriting the document with that doc_id is the right thing + # to do. + failure.trap(u1db.errors.RevisionConflict) + self._doc_id = self.future_doc_id + self._future_doc_id = None + return self.update(store) + if self.future_doc_id is None: d = store.create_doc(self.serialize()) else: d = store.create_doc(self.serialize(), doc_id=self.future_doc_id) d.addCallback(update_doc_id) - d.addErrback(self._catch_revision_conflict, self.future_doc_id) + + if is_copy: + d.addErrback(update_wrapper), + else: + d.addErrback(self._catch_revision_conflict, self.future_doc_id) return d def update(self, store): @@ -542,8 +558,8 @@ class MessageWrapper(object): # TODO check that the doc_ids in the mdoc are coherent self.d = [] - mdoc_created = self.mdoc.create(store) - fdoc_created = self.fdoc.create(store) + mdoc_created = self.mdoc.create(store, is_copy=self._is_copy) + fdoc_created = self.fdoc.create(store, is_copy=self._is_copy) self.d.append(mdoc_created) self.d.append(fdoc_created) @@ -558,7 +574,12 @@ class MessageWrapper(object): continue self.d.append(cdoc.create(store)) - self.all_inserted_d = defer.gatherResults(self.d) + def log_all_inserted(result): + log.msg("All parts inserted for msg!") + return result + + self.all_inserted_d = defer.gatherResults(self.d, consumeErrors=True) + self.all_inserted_d.addCallback(log_all_inserted) if notify_just_mdoc: self.all_inserted_d.addCallback(unblock_pending_insert) @@ -605,8 +626,10 @@ class MessageWrapper(object): new_wrapper.set_mbox_uuid(new_mbox_uuid) # XXX could flag so that it only creates mdoc/fdoc... + d = new_wrapper.create(store) d.addCallback(lambda result: new_wrapper) + d.addErrback(lambda failure: log.err(failure)) return d def set_mbox_uuid(self, mbox_uuid): @@ -942,10 +965,14 @@ class SoledadMailAdaptor(SoledadIndexMixin): fdoc_id = _get_fdoc_id_from_mdoc_id() def wrap_fdoc(doc): + if not doc: + return cls = FlagsDocWrapper return cls(doc_id=doc.doc_id, **doc.content) def get_flags(fdoc_wrapper): + if not fdoc_wrapper: + return [] return fdoc_wrapper.get_flags() d = store.get_doc(fdoc_id) @@ -983,8 +1010,8 @@ class SoledadMailAdaptor(SoledadIndexMixin): """ Delete all messages flagged as deleted. """ - def err(f): - f.printTraceback() + def err(failure): + log.err(failure) def delete_fdoc_and_mdoc_flagged(fdocs): # low level here, not using the wrappers... @@ -1118,8 +1145,8 @@ class SoledadMailAdaptor(SoledadIndexMixin): """ return MailboxWrapper.get_all(store) - def _errback(self, f): - f.printTraceback() + def _errback(self, failure): + log.err(failure) def _split_into_parts(raw): diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 1412344..c4821ff 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -846,8 +846,9 @@ class IMAPMailbox(object): #deferLater(self.reactor, 0, self._do_copy, message, d) #return d - return self.collection.copy_msg(message.message, - self.collection.mbox_uuid) + d = self.collection.copy_msg(message.message, + self.collection.mbox_uuid) + return d # convenience fun diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 8cb0b4a..bf5b34d 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -25,6 +25,7 @@ import time import weakref from twisted.internet import defer +from twisted.python import log from leap.common.check import leap_assert_type from leap.common.events import emit, catalog @@ -559,7 +560,7 @@ class MessageCollection(object): """ Add a message to this collection. - :param raw_message: the raw message + :param raw_msg: the raw message :param flags: tuple of flags for this message :param tags: tuple of tags for this message :param date: @@ -619,7 +620,7 @@ class MessageCollection(object): # so workaround is to make sure we always check for it before # inserting the doc. I should debug into the real cause. d = self.mbox_indexer.create_table(self.mbox_uuid) - d.addCallback(lambda _: self.mbox_indexer.insert_doc( + d.addBoth(lambda _: self.mbox_indexer.insert_doc( self.mbox_uuid, doc_id)) return d @@ -664,12 +665,52 @@ class MessageCollection(object): Copy the message to another collection. (it only makes sense for mailbox collections) """ + # TODO should CHECK first if the mdoc is present in the mailbox + # WITH a Deleted flag... and just simply remove the flag... + # Another option is to delete the previous mdoc if it already exists + # (so we get a new UID) + if not self.is_mailbox_collection(): raise NotImplementedError() + def delete_mdoc_entry_and_insert(failure, mbox_uuid, doc_id): + d = self.mbox_indexer.delete_doc_by_hash(mbox_uuid, doc_id) + d.addCallback(lambda _: self.mbox_indexer.insert_doc( + new_mbox_uuid, doc_id)) + return d + def insert_copied_mdoc_id(wrapper_new_msg): - return self.mbox_indexer.insert_doc( - new_mbox_uuid, wrapper_new_msg.mdoc.doc_id) + # XXX FIXME -- since this is already saved, the future_doc_id + # should be already copied into the doc_id! + # Investigate why we are not receiving the already saved doc_id + doc_id = wrapper_new_msg.mdoc.doc_id + if not doc_id: + doc_id = wrapper_new_msg.mdoc._future_doc_id + + def insert_conditionally(uid, mbox_uuid, doc_id): + indexer = self.mbox_indexer + if uid: + d = indexer.delete_doc_by_hash(mbox_uuid, doc_id) + d.addCallback(lambda _: indexer.insert_doc( + new_mbox_uuid, doc_id)) + return d + else: + d = indexer.insert_doc(mbox_uuid, doc_id) + return d + + def log_result(result): + return result + + def insert_doc(_, mbox_uuid, doc_id): + d = self.mbox_indexer.get_uid_from_doc_id(mbox_uuid, doc_id) + d.addCallback(insert_conditionally, mbox_uuid, doc_id) + d.addErrback(lambda err: log.failure(err)) + d.addCallback(log_result) + return d + + d = self.mbox_indexer.create_table(new_mbox_uuid) + d.addBoth(insert_doc, new_mbox_uuid, doc_id) + return d wrapper = msg.get_wrapper() diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index ab0967d..08e5f10 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -104,7 +104,6 @@ class MailboxIndexer(object): "uid INTEGER PRIMARY KEY AUTOINCREMENT, " "hash TEXT UNIQUE NOT NULL)".format( preffix=self.table_preffix, name=sanitize(mailbox_uuid))) - print "CREATING TABLE..." return self._operation(sql) def delete_table(self, mailbox_uuid): @@ -190,8 +189,7 @@ class MailboxIndexer(object): :type mailbox: str :param doc_id: the doc_id for the MetaMsg :type doc_id: str - :return: a deferred that will fire with the uid of the newly inserted - document. + :return: a deferred that will fire when the deletion has succed. :rtype: Deferred """ check_good_uuid(mailbox_uuid) diff --git a/src/leap/mail/sync_hooks.py b/src/leap/mail/sync_hooks.py index a8a69c9..bd8d88d 100644 --- a/src/leap/mail/sync_hooks.py +++ b/src/leap/mail/sync_hooks.py @@ -86,7 +86,7 @@ class MailProcessingPostSyncHook(object): # have seen -- but make sure *it's already created* before # inserting the index entry!. d = indexer.create_table(mbox_uuid) - d.addCallback(lambda _: indexer.insert_doc(mbox_uuid, index_docid)) + d.addBoth(lambda _: indexer.insert_doc(mbox_uuid, index_docid)) self._processing_deferreds.append(d) def _process_queued_docs(self): -- cgit v1.2.3 From 9db1529e60cadff872cd3066e837e15413216b71 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 25 Jun 2015 09:42:25 -0400 Subject: [bug] saving message to drafts folder hangs the bug consist on a fetch-while-pending-inserts hanging. the pending insert dict was not being cleaned up because the lookup for the Message-Id *is* case-sensitive (in the headers dict). by using a temporary all-keys-lowercase dict the lookup can be performed right, and the fetch returns successfully. at this point there's still a pending bug with Drafts, and it is that the new version is inserted but the MUA (TB) doesn't hide the older version (although a Delete flag is added). Resolves: #7189, #7190 Releases: 0.4.0 --- src/leap/mail/adaptors/soledad.py | 10 ++++++++-- src/leap/mail/imap/mailbox.py | 7 +++++++ src/leap/mail/mail.py | 11 +++++++++-- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index dc0960f..7e41f94 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -547,7 +547,9 @@ class MessageWrapper(object): "Cannot create: fdoc has a doc_id") def unblock_pending_insert(result): - msgid = self.hdoc.headers.get('Message-Id', None) + h = self.hdoc.headers + ci_headers = dict([(k.lower(), v) for (k, v) in h.items()]) + msgid = ci_headers.get('message-id', None) try: d = pending_inserts_dict[msgid] d.callback(msgid) @@ -561,6 +563,9 @@ class MessageWrapper(object): mdoc_created = self.mdoc.create(store, is_copy=self._is_copy) fdoc_created = self.fdoc.create(store, is_copy=self._is_copy) + mdoc_created.addErrback(lambda f: log.err(f)) + fdoc_created.addErrback(lambda f: log.err(f)) + self.d.append(mdoc_created) self.d.append(fdoc_created) @@ -580,9 +585,10 @@ class MessageWrapper(object): self.all_inserted_d = defer.gatherResults(self.d, consumeErrors=True) self.all_inserted_d.addCallback(log_all_inserted) + self.all_inserted_d.addCallback(unblock_pending_insert) + self.all_inserted_d.addErrback(lambda failure: log.err(failure)) if notify_just_mdoc: - self.all_inserted_d.addCallback(unblock_pending_insert) return mdoc_created else: return self.all_inserted_d diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index c4821ff..72f5a43 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -504,8 +504,13 @@ class IMAPMailbox(object): getimapmsg = self.get_imap_message def get_imap_messages_for_range(msg_range): + print + print + print + print "GETTING FOR RANGE", msg_range def _get_imap_msg(messages): + print "GETTING IMAP MSG FOR", messages d_imapmsg = [] for msg in messages: d_imapmsg.append(getimapmsg(msg)) @@ -532,6 +537,7 @@ class IMAPMailbox(object): d = defer.gatherResults(d_msg, consumeErrors=True) d.addCallback(_get_imap_msg) d.addCallback(_zip_msgid) + d.addErrback(lambda failure: log.err(failure)) return d # for sequence numbers (uid = 0) @@ -542,6 +548,7 @@ class IMAPMailbox(object): else: d = self._get_messages_range(messages_asked) d.addCallback(get_imap_messages_for_range) + d.addErrback(lambda failure: log.err(failure)) return d diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index bf5b34d..b4602b3 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -622,6 +622,12 @@ class MessageCollection(object): d = self.mbox_indexer.create_table(self.mbox_uuid) d.addBoth(lambda _: self.mbox_indexer.insert_doc( self.mbox_uuid, doc_id)) + # XXX--------------------------------- + def print_inserted(r): + print "INSERTED", r + return r + d.addCallback(print_inserted) + # XXX--------------------------------- return d d = wrapper.create( @@ -629,8 +635,9 @@ class MessageCollection(object): notify_just_mdoc=notify_just_mdoc, pending_inserts_dict=self._pending_inserts) d.addCallback(insert_mdoc_id, wrapper) - d.addErrback(lambda f: f.printTraceback()) - d.addCallback(self.cb_signal_unread_to_ui) + d.addErrback(lambda failure: log.err(failure)) + #d.addCallback(self.cb_signal_unread_to_ui) + return d def cb_signal_unread_to_ui(self, result): -- cgit v1.2.3 From cc487c6bc77769079fc4cdc23b6ad2bdd7c22394 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 25 Jun 2015 10:45:46 -0400 Subject: [bug] avoid duplication of copies in draft folder Although this draft-saving feature seems to be somehow brittle, since it breaks from time to time, the causes are different and subtle on each case. This time the bug is related to having the notify_just_mdoc (or fast_notifies as I'm tempted to call it in the future for more clarity) set to True by default in the APPENDS. Fast notifies break the "save draft" functionality because what Thunderbird does is an Append of the newest message, followed by deletion of the old message and a SEARCH by Message-ID. For this we need the headers-doc to be already inserted in the store. Since the fast-notify makes a *big* difference in terms of insertion times for serialized appends, I've opted for using a heuristic for detecting if it's the case of a Draft message, using a mozilla-specific header. If this is found, we set the notify_just_mdoc unconditionally to False. We'll deal with other MUAs on its due time. Releases: 0.4.0 --- src/leap/mail/adaptors/soledad.py | 4 ++-- src/leap/mail/imap/mailbox.py | 41 +++++++++++++++++++++++++-------------- src/leap/mail/mail.py | 28 ++++++++++++++------------ 3 files changed, 44 insertions(+), 29 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 7e41f94..0565877 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -547,8 +547,7 @@ class MessageWrapper(object): "Cannot create: fdoc has a doc_id") def unblock_pending_insert(result): - h = self.hdoc.headers - ci_headers = dict([(k.lower(), v) for (k, v) in h.items()]) + ci_headers = lowerdict(self.hdoc.headers) msgid = ci_headers.get('message-id', None) try: d = pending_inserts_dict[msgid] @@ -1101,6 +1100,7 @@ class SoledadMailAdaptor(SoledadIndexMixin): def get_mdoc_id(hdoc): if not hdoc: + log.msg("Could not find a HDOC with MSGID %s" % msgid) return None hdoc = hdoc[0] mdoc_id = hdoc.doc_id.replace("H-", "M-%s-" % uuid) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 72f5a43..139ae66 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -320,6 +320,24 @@ class IMAPMailbox(object): :param date: timestamp :type date: str, or None + :param notify_just_mdoc: + boolean passed to the wrapper.create method, to indicate whether + we're insterested in being notified right after the mdoc has been + written (as it's the first doc to be written, and quite small, this + is faster, though potentially unsafe). + Setting it to True improves a *lot* the responsiveness of the + APPENDS: we just need to be notified when the mdoc is saved, and + let's just expect that the other parts are doing just fine. This + will not catch any errors when the inserts of the other parts + fail, but on the other hand allows us to return very quickly, + which seems a good compromise given that we have to serialize the + appends. + However, some operations like the saving of drafts need to wait for + all the parts to be saved, so if some heuristics are met down in + the call chain a Draft message will unconditionally set this flag + to False, and therefore ignoring the setting of this flag here. + :type notify_just_mdoc: bool + :return: a deferred that will be triggered with the UID of the added message. """ @@ -327,18 +345,14 @@ class IMAPMailbox(object): # TODO have a look at the cases for internal date in the rfc # XXX we could treat the message as an IMessage from here + # TODO change notify_just_mdoc to something more meaningful, like + # fast_insert_notify? + # TODO notify_just_mdoc *sometimes* make the append tests fail. # have to find a better solution for this. A workaround could probably # be to have a list of the ongoing deferreds related to append, so that # we queue for later all the requests having to do with these. - # notify_just_mdoc=True: feels HACKY, but improves a *lot* the - # responsiveness of the APPENDS: we just need to be notified when the - # mdoc is saved, and let's hope that the other parts are doing just - # fine. This will not catch any errors when the inserts of the other - # parts fail, but on the other hand allows us to return very quickly, - # which seems a good compromise given that we have to serialize the - # appends. # A better solution will probably involve implementing MULTIAPPEND # extension or patching imap server to support pipelining. @@ -355,9 +369,11 @@ class IMAPMailbox(object): if date is None: date = formatdate(time.time()) - # TODO add notify_new as a callback here... - return self.collection.add_msg(message, flags, date=date, - notify_just_mdoc=notify_just_mdoc) + d = self.collection.add_msg(message, flags, date=date, + notify_just_mdoc=notify_just_mdoc) + d.addCallback(self.notify_new) + d.addErrback(lambda failure: log.err(failure)) + return d def notify_new(self, *args): """ @@ -504,13 +520,8 @@ class IMAPMailbox(object): getimapmsg = self.get_imap_message def get_imap_messages_for_range(msg_range): - print - print - print - print "GETTING FOR RANGE", msg_range def _get_imap_msg(messages): - print "GETTING IMAP MSG FOR", messages d_imapmsg = [] for msg in messages: d_imapmsg.append(getimapmsg(msg)) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index b4602b3..faaabf6 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -37,6 +37,7 @@ from leap.mail.constants import MessageFlags from leap.mail.mailbox_indexer import MailboxIndexer from leap.mail.plugins import soledad_sync_hooks from leap.mail.utils import find_charset, CaseInsensitiveDict +from leap.mail.utils import lowerdict logger = logging.getLogger(name=__name__) @@ -570,11 +571,14 @@ class MessageCollection(object): reflects when the message was received. :type date: str :param notify_just_mdoc: - boolean passed to the wrapper.create method, - to indicate whether we're interested in being notified when only - the mdoc has been written (faster, but potentially unsafe), or we - want to wait untill all the parts have been written. + boolean passed to the wrapper.create method, to indicate whether + we're insterested in being notified right after the mdoc has been + written (as it's the first doc to be written, and quite small, this + is faster, though potentially unsafe), or on the contrary we want + to wait untill all the parts have been written. Used by the imap mailbox implementation to get faster responses. + This will be ignored (and set to False) if a heuristic for a Draft + message is met, which currently is a specific mozilla header. :type notify_just_mdoc: bool :returns: a deferred that will fire with the UID of the inserted @@ -590,8 +594,14 @@ class MessageCollection(object): msg = self.adaptor.get_msg_from_string(Message, raw_msg) wrapper = msg.get_wrapper() + headers = lowerdict(msg.get_headers()) + moz_draft_hdr = "X-Mozilla-Draft-Info" + if moz_draft_hdr.lower() in headers: + log.msg("Setting fast notify to False, Draft detected") + notify_just_mdoc = False + if notify_just_mdoc: - msgid = msg.get_headers()['message-id'] + msgid = headers['message-id'] self._pending_inserts[msgid] = defer.Deferred() if not self.is_mailbox_collection(): @@ -622,12 +632,6 @@ class MessageCollection(object): d = self.mbox_indexer.create_table(self.mbox_uuid) d.addBoth(lambda _: self.mbox_indexer.insert_doc( self.mbox_uuid, doc_id)) - # XXX--------------------------------- - def print_inserted(r): - print "INSERTED", r - return r - d.addCallback(print_inserted) - # XXX--------------------------------- return d d = wrapper.create( @@ -636,7 +640,7 @@ class MessageCollection(object): pending_inserts_dict=self._pending_inserts) d.addCallback(insert_mdoc_id, wrapper) d.addErrback(lambda failure: log.err(failure)) - #d.addCallback(self.cb_signal_unread_to_ui) + d.addCallback(self.cb_signal_unread_to_ui) return d -- cgit v1.2.3 From c0f3a6afa81f93f8d1b078a62e4411b2321ba9f0 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 29 Jun 2015 09:53:36 -0400 Subject: [bug] avoid KeyError on pending_insert_docs lookup in a previous commit, there was a bug inserted in which a key lookup was being made unconditionally, even when the pending_insert_docs was None. Releases: 0.4.0 --- src/leap/mail/adaptors/soledad.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 0565877..4020bd0 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -150,7 +150,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): d.addCallback(update_doc_id) if is_copy: - d.addErrback(update_wrapper), + d.addErrback(update_wrapper) else: d.addErrback(self._catch_revision_conflict, self.future_doc_id) return d @@ -507,7 +507,7 @@ class MessageWrapper(object): for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()): cdoc.set_future_doc_id(doc_id) - def create(self, store, notify_just_mdoc=False, pending_inserts_dict=None): + def create(self, store, notify_just_mdoc=False, pending_inserts_dict={}): """ Create all the parts for this message in the store. @@ -547,13 +547,14 @@ class MessageWrapper(object): "Cannot create: fdoc has a doc_id") def unblock_pending_insert(result): - ci_headers = lowerdict(self.hdoc.headers) - msgid = ci_headers.get('message-id', None) - try: - d = pending_inserts_dict[msgid] - d.callback(msgid) - except KeyError: - pass + if pending_inserts_dict: + ci_headers = lowerdict(self.hdoc.headers) + msgid = ci_headers.get('message-id', None) + try: + d = pending_inserts_dict[msgid] + d.callback(msgid) + except KeyError: + pass return result # TODO check that the doc_ids in the mdoc are coherent -- cgit v1.2.3 From 0dbf2be49db228e43fe4b196199f82ea281886bf Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 29 Jun 2015 11:57:20 -0400 Subject: [bug] allow mailbox to be notified of collection changes in a previous refactor, we decoupled the incoming mail service from the IMAP layer. However, this left the IMAPMailbox unable to react to changes in the underlying collection when a new message is inserted. in this commit, we add a Listener mechanism to the collection itself, so that IMAPMailbox (and any other object that uses it) can subscribe to changes on the number of messages of the collection. Resolves: #7191 Releases: 0.4.0 --- src/leap/mail/adaptors/soledad.py | 2 ++ src/leap/mail/imap/mailbox.py | 14 +++++++++++--- src/leap/mail/mail.py | 16 +++++++++++++++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 4020bd0..2b1d2ff 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -505,6 +505,8 @@ class MessageWrapper(object): (key, get_doc_wrapper(doc, ContentDocWrapper)) for (key, doc) in cdocs.items()]) for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()): + if cdoc.raw == "": + log.msg("Empty raw field in cdoc %s" % doc_id) cdoc.set_future_doc_id(doc_id) def create(self, store, notify_just_mdoc=False, pending_inserts_dict={}): diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 139ae66..0de4b40 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -91,6 +91,8 @@ class IMAPMailbox(object): imap4.IMailboxInfo, imap4.ISearchableMailbox, # XXX I think we do not need to implement CloseableMailbox, do we? + # We could remove ourselves from the collectionListener, although I + # think it simply will be garbage collected. # imap4.ICloseableMailbox imap4.IMessageCopier) @@ -116,6 +118,7 @@ class IMAPMailbox(object): self.rw = rw self._uidvalidity = None self.collection = collection + self.collection.addListener(self) @property def mbox_name(self): @@ -155,9 +158,10 @@ class IMAPMailbox(object): if not NOTIFY_NEW: return + listeners = self.listeners logger.debug('adding mailbox listener: %s. Total: %s' % ( - listener, len(self.listeners))) - self.listeners.add(listener) + listener, len(listeners))) + listeners.add(listener) def removeListener(self, listener): """ @@ -371,13 +375,16 @@ class IMAPMailbox(object): d = self.collection.add_msg(message, flags, date=date, notify_just_mdoc=notify_just_mdoc) - d.addCallback(self.notify_new) d.addErrback(lambda failure: log.err(failure)) return d def notify_new(self, *args): """ Notify of new messages to all the listeners. + This will be called indirectly by the underlying collection, that will + notify this IMAPMailbox whenever there are changes in the number of + messages in the collection, since we have added ourselves to the + collection listeners. :param args: ignored. """ @@ -392,6 +399,7 @@ class IMAPMailbox(object): d = self._get_notify_count() d.addCallback(cbNotifyNew) d.addCallback(self.collection.cb_signal_unread_to_ui) + d.addErrback(lambda failure: log.err(failure)) def _get_notify_count(self): """ diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index faaabf6..4a73186 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -378,6 +378,7 @@ class MessageCollection(object): # of by doc_id. See get_message_by_content_hash self.mbox_indexer = mbox_indexer self.mbox_wrapper = mbox_wrapper + self._listeners = set([]) def is_mailbox_collection(self): """ @@ -639,11 +640,24 @@ class MessageCollection(object): notify_just_mdoc=notify_just_mdoc, pending_inserts_dict=self._pending_inserts) d.addCallback(insert_mdoc_id, wrapper) - d.addErrback(lambda failure: log.err(failure)) d.addCallback(self.cb_signal_unread_to_ui) + d.addCallback(self.notify_new_to_listeners) + d.addErrback(lambda failure: log.err(failure)) return d + # Listeners + + def addListener(self, listener): + self._listeners.add(listener) + + def removeListener(self, listener): + self._listeners.remove(listener) + + def notify_new_to_listeners(self, *args): + for listener in self._listeners: + listener.notify_new() + def cb_signal_unread_to_ui(self, result): """ Sends an unread event to ui, passing *only* the number of unread -- cgit v1.2.3 From 844c9633b7cf24c5cf5785ed7b6a176142b75447 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Thu, 2 Jul 2015 18:39:25 -0400 Subject: [bug] tear down of leap.common.events properly in tests The tests where writting their own implementation of env tear down instead of using leap.common's one. Using it fixes many tests. --- src/leap/mail/tests/common.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/leap/mail/tests/common.py b/src/leap/mail/tests/common.py index a411b2d..6ef5d17 100644 --- a/src/leap/mail/tests/common.py +++ b/src/leap/mail/tests/common.py @@ -92,12 +92,8 @@ class SoledadTestMixin(unittest.TestCase, BaseLeapTest): self.results = [] try: self._soledad.close() - except Exception as exc: + except Exception: print "ERROR WHILE CLOSING SOLEDAD" # logging.exception(exc) finally: - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check - assert 'leap_tests-' in self.tempdir - shutil.rmtree(self.tempdir) + self.tearDownEnv() -- cgit v1.2.3 From 4c63caf644429f8ebc4d9e971bd7afb715dd185d Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Thu, 2 Jul 2015 23:37:37 -0400 Subject: [bug] return the message uid after add it to the collection MessageCollection.add_msg was not returning the uid of the message added making the incoming service not deleting the emails from the incoming queue. * Related: #7158 --- src/leap/mail/mail.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 4a73186..1a60c6d 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -654,9 +654,10 @@ class MessageCollection(object): def removeListener(self, listener): self._listeners.remove(listener) - def notify_new_to_listeners(self, *args): + def notify_new_to_listeners(self, result): for listener in self._listeners: listener.notify_new() + return result def cb_signal_unread_to_ui(self, result): """ -- cgit v1.2.3 From 853f4de55705ecba39c552e851057c2b0c66e810 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Fri, 3 Jul 2015 17:10:05 -0400 Subject: [bug] notify copied emails When copying emails there was not notification produced, that makes thunderbird to see messages moved between folders until it gets restarted. Now the MaillCollection copy does notify the listeners. * Resolves: #7158 --- src/leap/mail/mail.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 1a60c6d..2f190d4 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -619,8 +619,8 @@ class MessageCollection(object): doc_id = wrapper.mdoc.doc_id if not doc_id: # --- BUG ----------------------------------------- - # XXX why from time to time mdoc doesn't have doc_id - # here??? + # XXX watch out, sometimes mdoc doesn't have doc_id + # but it has future_id. Should be solved already. logger.error("BUG: (please report) Null doc_id for " "document %s" % (wrapper.mdoc.serialize(),)) @@ -742,6 +742,7 @@ class MessageCollection(object): d = wrapper.copy(self.store, new_mbox_uuid) d.addCallback(insert_copied_mdoc_id) + d.addCallback(self.notify_new_to_listeners) return d def delete_msg(self, msg): -- cgit v1.2.3 From 3e4c030dbcd261028810fec223127d364c1491b3 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Fri, 10 Jul 2015 17:09:46 -0300 Subject: [pkg] fold in changes --- CHANGELOG | 18 ++++++++++++++++++ changes/bug-6601_port_enum34 | 1 - changes/bug_7169_update-smtp-gateway-doc | 1 - changes/feature-3879_openpgp_header | 1 - changes/feature-4692_remove_footer | 1 - changes/feature-5937_key_attachment | 1 - ...ctor_encryptio_and_sending_out_of_encrypted_message | 1 - changes/feature-6598_refactor_incoming_mail | 2 -- changes/feature-6617_attach_public_key | 1 - changes/feature_6996-post-sync-hooks | 1 - changes/feature_adapt-to-new-events-on-common | 1 - changes/feature_send_bye | 1 - 12 files changed, 18 insertions(+), 12 deletions(-) delete mode 100644 changes/bug-6601_port_enum34 delete mode 100644 changes/bug_7169_update-smtp-gateway-doc delete mode 100644 changes/feature-3879_openpgp_header delete mode 100644 changes/feature-4692_remove_footer delete mode 100644 changes/feature-5937_key_attachment delete mode 100644 changes/feature-6357_factor_encryptio_and_sending_out_of_encrypted_message delete mode 100644 changes/feature-6598_refactor_incoming_mail delete mode 100644 changes/feature-6617_attach_public_key delete mode 100644 changes/feature_6996-post-sync-hooks delete mode 100644 changes/feature_adapt-to-new-events-on-common delete mode 100644 changes/feature_send_bye diff --git a/CHANGELOG b/CHANGELOG index 4c3da7b..885871f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,21 @@ +0.4.0rc1 Jul 10, 2015: + o Parse OpenPGP header and import keys from it. Closes: #3879. + o Don't add any footer to the emails. Closes: #4692. + o Adapt to new events api on leap.common. Related to #5359. + o Discover public keys via attachment. Closes: #5937. + o Creates a OutgoingMail class that has the logic for encrypting, signing and + sending messages. Factors that logic out of EncryptedMessage so it can be + used by other clients. Closes: #6357. + o Refactor email fetching outside IMAP to it's own independient IncomingMail + class. Closes: #6361. + o Port `enum` to `enum34`. Closes #6601. + o Add public key as attachment. Closes: #6617. + o Add listener for each email added to inbox in IncomingMail. Closes: #6742. + o Ability to reindex local UIDs after a soledad sync. Closes: #6996. + o Update SMTP gateway docs. Closes #7169. + o Send a BYE command to all open connections, so that the MUA is notified + when the server is shutted down. + 0.3.10 Sept 26, 2014: o MessageCollection iterator now creates the LeapMessage with the collection reference, so setFlags will work properly. diff --git a/changes/bug-6601_port_enum34 b/changes/bug-6601_port_enum34 deleted file mode 100644 index 2ca551d..0000000 --- a/changes/bug-6601_port_enum34 +++ /dev/null @@ -1 +0,0 @@ -- Port `enum` to `enum34` (Closes #6601) diff --git a/changes/bug_7169_update-smtp-gateway-doc b/changes/bug_7169_update-smtp-gateway-doc deleted file mode 100644 index 5b86140..0000000 --- a/changes/bug_7169_update-smtp-gateway-doc +++ /dev/null @@ -1 +0,0 @@ - o Update SMTP gateway docs. Closes #7169. diff --git a/changes/feature-3879_openpgp_header b/changes/feature-3879_openpgp_header deleted file mode 100644 index e04c925..0000000 --- a/changes/feature-3879_openpgp_header +++ /dev/null @@ -1 +0,0 @@ -- Parse OpenPGP header and import keys from it (Closes: #3879) diff --git a/changes/feature-4692_remove_footer b/changes/feature-4692_remove_footer deleted file mode 100644 index 8eca883..0000000 --- a/changes/feature-4692_remove_footer +++ /dev/null @@ -1 +0,0 @@ -- Don't add any footer to the emails (Closes: #4692) diff --git a/changes/feature-5937_key_attachment b/changes/feature-5937_key_attachment deleted file mode 100644 index 08c37e0..0000000 --- a/changes/feature-5937_key_attachment +++ /dev/null @@ -1 +0,0 @@ -- Discover public keys via attachment (Closes: #5937) diff --git a/changes/feature-6357_factor_encryptio_and_sending_out_of_encrypted_message b/changes/feature-6357_factor_encryptio_and_sending_out_of_encrypted_message deleted file mode 100644 index 6b95c6a..0000000 --- a/changes/feature-6357_factor_encryptio_and_sending_out_of_encrypted_message +++ /dev/null @@ -1 +0,0 @@ -- Creates a OutgoingMail class that has the logic for encrypting, signing and sending messages. Factors that logic out of EncryptedMessage so it can be used by other clients (Closes: #6357) diff --git a/changes/feature-6598_refactor_incoming_mail b/changes/feature-6598_refactor_incoming_mail deleted file mode 100644 index 1db8c28..0000000 --- a/changes/feature-6598_refactor_incoming_mail +++ /dev/null @@ -1,2 +0,0 @@ -- Refactor email fetching outside IMAP to it's own independient IncomingMail class (Closes: #6361) -- Add listener for each email added to inbox in IncomingMail (Closes: #6742) diff --git a/changes/feature-6617_attach_public_key b/changes/feature-6617_attach_public_key deleted file mode 100644 index 49b444b..0000000 --- a/changes/feature-6617_attach_public_key +++ /dev/null @@ -1 +0,0 @@ -- add public key as attachment (Closes: #6617) diff --git a/changes/feature_6996-post-sync-hooks b/changes/feature_6996-post-sync-hooks deleted file mode 100644 index e03c28e..0000000 --- a/changes/feature_6996-post-sync-hooks +++ /dev/null @@ -1 +0,0 @@ -- Ability to reindex local UIDs after a soledad sync. Closes: #6996 diff --git a/changes/feature_adapt-to-new-events-on-common b/changes/feature_adapt-to-new-events-on-common deleted file mode 100644 index e57e777..0000000 --- a/changes/feature_adapt-to-new-events-on-common +++ /dev/null @@ -1 +0,0 @@ -- Adapt to new events api on leap.common. Related to #5359. diff --git a/changes/feature_send_bye b/changes/feature_send_bye deleted file mode 100644 index 5bc3e60..0000000 --- a/changes/feature_send_bye +++ /dev/null @@ -1 +0,0 @@ -- Send a BYE command to all open connections, so that the MUA is notified when the server is shutted down. -- cgit v1.2.3 From c4a7bc9b2d19e70565877c4526ba2ff31acbdca1 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Fri, 10 Jul 2015 17:11:58 -0300 Subject: [pkg] bump dependencies --- changes/VERSION_COMPAT | 3 --- pkg/requirements.pip | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT index a5c0caa..cc00ecf 100644 --- a/changes/VERSION_COMPAT +++ b/changes/VERSION_COMPAT @@ -8,6 +8,3 @@ # # BEGIN DEPENDENCY LIST ------------------------- # leap.foo.bar>=x.y.z -leap.keymanager>=0.4.0 -leap.soledad.client>=0.7.0 -leap.common>=0.4 diff --git a/pkg/requirements.pip b/pkg/requirements.pip index 20f93a6..d77059a 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -1,7 +1,7 @@ zope.interface -leap.soledad.client>=0.4.5 -leap.common>=0.3.7 -leap.keymanager>=0.3.8 +leap.soledad.client>=0.7.0 +leap.common>=0.4.0 +leap.keymanager>=0.4.0 twisted # >= 12.0.3 ?? zope.proxy service-identity -- cgit v1.2.3 From 221a168e07a5d8ff32573c80ab2cad54d03d3038 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 8 Jul 2015 12:47:04 -0400 Subject: [bug] fix the rendering of nested multipart This commit fix a very simplistic and until now broken handling of nested multipart that went undetected due to the structure of the mails used in tests until now. Incidentally, the way that Mail.app structures attachments made this bug noticeable. There was also an off-by-one indexing error when retrieving the subpart message for a given subpart. Be aware that the current implementation will only handle correctly 2 levels of multipart nesting. Extending beyond in a more generic way will need further work. Closes: #7244 --- changes/bug_7244_fix_nested_multipart | 1 + src/leap/mail/mail.py | 48 +++++++++++++++++++++++++++-------- 2 files changed, 38 insertions(+), 11 deletions(-) create mode 100644 changes/bug_7244_fix_nested_multipart diff --git a/changes/bug_7244_fix_nested_multipart b/changes/bug_7244_fix_nested_multipart new file mode 100644 index 0000000..2d9cd8d --- /dev/null +++ b/changes/bug_7244_fix_nested_multipart @@ -0,0 +1 @@ +- Fix nested multipart rendering. Closes: #7244 diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 2f190d4..0aede6b 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -117,6 +117,33 @@ def _unpack_headers(headers_dict): return headers_l +def _get_index_for_cdoc(part_map, cdocs_dict): + """ + Get, if possible, the index for a given content-document matching the phash + of the passed part_map. + + This is used when we are initializing a MessagePart, because we just pass a + reference to the parent message cdocs container and we need to iterate + through the cdocs to figure out which content-doc matches the phash of the + part we're currently rendering. + + It is also used when recursing through a nested multipart message, because + in the initialization of the child MessagePart we pass a dictionary only + for the referenced cdoc. + + :param part_map: a dict describing the mapping of the parts for the current + message-part. + :param cdocs: a dict of content-documents, 0-indexed. + :rtype: int + """ + phash = part_map.get('phash', None) + if phash: + for i, cdoc_wrapper in cdocs_dict.items(): + if cdoc_wrapper.phash == phash: + return i + return None + + class MessagePart(object): # TODO This class should be better abstracted from the data model. # TODO support arbitrarily nested multiparts (right now we only support @@ -144,13 +171,7 @@ class MessagePart(object): self._pmap = part_map self._cdocs = cdocs - index = 1 - phash = part_map.get('phash', None) - if phash: - for i, cdoc_wrapper in self._cdocs.items(): - if cdoc_wrapper.phash == phash: - index = i - break + index = _get_index_for_cdoc(part_map, self._cdocs) or 1 self._index = index def get_size(self): @@ -171,7 +192,8 @@ class MessagePart(object): if not multi: payload = self._get_payload(self._index) else: - # XXX uh, multi also... should recurse" + # XXX uh, multi also... should recurse. + # This needs to be implemented in a more general and elegant way. raise NotImplementedError if payload: payload = _encode_payload(payload) @@ -190,11 +212,15 @@ class MessagePart(object): sub_pmap = self._pmap.get("part_map", {}) try: - part_map = sub_pmap[str(part + 1)] + part_map = sub_pmap[str(part)] except KeyError: - logger.debug("getSubpart for %s: KeyError" % (part,)) + log.msg("getSubpart for %s: KeyError" % (part,)) raise IndexError - return MessagePart(part_map, cdocs={1: self._cdocs.get(part + 1, {})}) + + cdoc_index = _get_index_for_cdoc(part_map, self._cdocs) + cdoc = self._cdocs.get(cdoc_index, {}) + + return MessagePart(part_map, cdocs={1: cdoc}) def _get_payload(self, index): cdoc_wrapper = self._cdocs.get(index, None) -- cgit v1.2.3 From 4ab5660b5c3a2fc29f2d08d06e1958fc91ee5af7 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 10 Jul 2015 12:44:16 -0400 Subject: [bug] workaround for off-by-one error on nested multipart For some reason that I haven't discovered yet, nested multipart is hitting an off-by-one error (that had been wrongly ammended in a previous commit, breaking many other cases but fixing the particular Mail.app sample I was working with). This is just a temporary hack to make all the current regression tests happy, but further investigation is needed to discover the cause of the off-by-one part retrieval solved and correctly documented. --- src/leap/mail/mail.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 0aede6b..772b6db 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -211,6 +211,16 @@ class MessagePart(object): raise TypeError sub_pmap = self._pmap.get("part_map", {}) + + # XXX BUG --- workaround. Subparts with more than 1 subparts + # need to get the requested index for the subpart decremented. + # Off-by-one error, should investigate which is the real reason and + # fix it, this is only a quick workaround. + num_parts = self._pmap.get("parts", 0) + if num_parts > 1: + part = part - 1 + # ------------------------------------------------------------- + try: part_map = sub_pmap[str(part)] except KeyError: -- cgit v1.2.3 From 1b04096571cfa9002f9bd1cb75d99cdbbb7d7936 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 10 Jul 2015 12:47:21 -0400 Subject: [bug] fix keyerror when inserting msg on pending_inserts dict The decission to index the pending_inserts dict by message-id is a bit brittle. I assume any well-formed message in the RealWorld (tm) will have this header, but many samples used in the tests will break this assumption. --- src/leap/mail/mail.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 772b6db..feed11b 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -638,8 +638,9 @@ class MessageCollection(object): notify_just_mdoc = False if notify_just_mdoc: - msgid = headers['message-id'] - self._pending_inserts[msgid] = defer.Deferred() + msgid = headers.get('message-id') + if msgid: + self._pending_inserts[msgid] = defer.Deferred() if not self.is_mailbox_collection(): raise NotImplementedError() -- cgit v1.2.3 From 040a5ae5c8ef07251fce19143439b5ff9714b4de Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 9 Jul 2015 15:41:23 -0400 Subject: [bug] do not raise if the pending insert cannot be removed --- src/leap/mail/mail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index feed11b..f6936dd 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -490,7 +490,7 @@ class MessageCollection(object): def cleanup_and_get_doc_after_pending_insert(result): for key in result: - self._pending_inserts.pop(key) + self._pending_inserts.pop(key, None) return get_doc_fun(self.mbox_uuid, uid) if not self._pending_inserts: -- cgit v1.2.3 From 7a8e2ad2e6f848098977a78b6cfd8c597df93691 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 10 Jul 2015 12:01:40 -0400 Subject: [bug] Return the first cdoc if no body found In the case of a message with just one non-text attachment, something has to be returned for the body payload. (For instance, a message with only one image). --- src/leap/mail/adaptors/soledad.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 2b1d2ff..d114707 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -688,7 +688,8 @@ class MessageWrapper(object): """ body_phash = self.hdoc.body if not body_phash: - return None + if self.cdocs: + return self.cdocs[1] d = store.get_doc('C-' + body_phash) d.addCallback(lambda doc: ContentDocWrapper(**doc.content)) return d -- cgit v1.2.3 From 1810e1b704835257a8e4cba5328df29229e221ee Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 10 Jul 2015 12:41:14 -0400 Subject: [feature] add very basic support for message sequence numbers this is just the bare minimum implementation of MSN (message sequence numbers). It is not enough feature-wise, but I'm doing it now just to support testing with the default imap client that we're using with the regression tests. --- src/leap/mail/imap/mailbox.py | 100 ++++++++++++++++++++++------------------- src/leap/mail/imap/messages.py | 4 +- src/leap/mail/mail.py | 31 ++++++++++--- 3 files changed, 81 insertions(+), 54 deletions(-) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 0de4b40..4339bd2 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -457,7 +457,33 @@ class IMAPMailbox(object): raise imap4.ReadOnlyMailbox return self.collection.delete_all_flagged() - def _bound_seq(self, messages_asked): + def _get_message_fun(self, uid): + """ + Return the proper method to get a message for this mailbox, depending + on the passed uid flag. + + :param uid: If true, the IDs specified in the query are UIDs; + otherwise they are message sequence IDs. + :type uid: bool + :rtype: callable + """ + get_message_fun = [ + self.collection.get_message_by_sequence_number, + self.collection.get_message_by_uid][uid] + return get_message_fun + + def _get_messages_range(self, messages_asked, uid=True): + + def get_range(messages_asked): + return self._filter_msg_seq(messages_asked) + + d = defer.maybeDeferred(self._bound_seq, messages_asked, uid) + if uid: + d.addCallback(get_range) + d.addErrback(lambda f: log.err(f)) + return d + + def _bound_seq(self, messages_asked, uid): """ Put an upper bound to a messages sequence if this is open. @@ -465,17 +491,26 @@ class IMAPMailbox(object): :type messages_asked: MessageSet :rtype: MessageSet """ - def set_last(last_uid): + + def set_last_uid(last_uid): messages_asked.last = last_uid return messages_asked + def set_last_seq(all_uid): + messages_asked.last = len(all_uid) + return messages_asked + if not messages_asked.last: try: iter(messages_asked) except TypeError: # looks like we cannot iterate - d = self.collection.get_last_uid() - d.addCallback(set_last) + if uid: + d = self.collection.get_last_uid() + d.addCallback(set_last_uid) + else: + d = self.collection.all_uid_iter() + d.addCallback(set_last_seq) return d return messages_asked @@ -516,15 +551,7 @@ class IMAPMailbox(object): :rtype: deferred with a generator that yields... """ - # TODO implement sequence - # is_sequence = True if uid == 0 else False - # XXX DEBUG --- if you attempt to use the `getmail` utility under - # imap/tests, or muas like mutt, it will choke until we implement - # sequence numbers. This is an easy hack meanwhile. - is_sequence = False - # ----------------------------------------------------------------- - - getmsg = self.collection.get_message_by_uid + get_msg_fun = self._get_message_fun(uid) getimapmsg = self.get_imap_message def get_imap_messages_for_range(msg_range): @@ -551,7 +578,7 @@ class IMAPMailbox(object): # body. We should be smarter at do_FETCH and pass a parameter # to this method in order not to prefetch cdocs if they're not # going to be used. - d_msg.append(getmsg(msgid, get_cdocs=True)) + d_msg.append(get_msg_fun(msgid, get_cdocs=True)) d = defer.gatherResults(d_msg, consumeErrors=True) d.addCallback(_get_imap_msg) @@ -559,24 +586,9 @@ class IMAPMailbox(object): d.addErrback(lambda failure: log.err(failure)) return d - # for sequence numbers (uid = 0) - if is_sequence: - # TODO --- implement sequences in mailbox indexer - raise NotImplementedError - - else: - d = self._get_messages_range(messages_asked) - d.addCallback(get_imap_messages_for_range) - d.addErrback(lambda failure: log.err(failure)) - - return d - - def _get_messages_range(self, messages_asked): - def get_range(messages_asked): - return self._filter_msg_seq(messages_asked) - - d = defer.maybeDeferred(self._bound_seq, messages_asked) - d.addCallback(get_range) + d = self._get_messages_range(messages_asked, uid) + d.addCallback(get_imap_messages_for_range) + d.addErrback(lambda failure: log.err(failure)) return d def fetch_flags(self, messages_asked, uid): @@ -611,7 +623,8 @@ class IMAPMailbox(object): # --------------------------------------------------------------- if is_sequence: - raise NotImplementedError + raise NotImplementedError( + "FETCH FLAGS NOT IMPLEMENTED FOR MESSAGE SEQUENCE NUMBERS YET") d = defer.Deferred() reactor.callLater(0, self._do_fetch_flags, messages_asked, uid, d) @@ -652,6 +665,7 @@ class IMAPMailbox(object): def get_flags_for_seq(sequence): d_all_flags = [] for msgid in sequence: + # TODO implement sequence numbers here too d_flags_per_uid = self.collection.get_flags_by_uid(msgid) d_flags_per_uid.addCallback(pack_flags) d_all_flags.append(d_flags_per_uid) @@ -663,7 +677,7 @@ class IMAPMailbox(object): generator = (item for item in result) d.callback(generator) - d_seq = self._get_messages_range(messages_asked) + d_seq = self._get_messages_range(messages_asked, uid) d_seq.addCallback(get_flags_for_seq) return d_seq @@ -694,7 +708,8 @@ class IMAPMailbox(object): # TODO implement sequences is_sequence = True if uid == 0 else False if is_sequence: - raise NotImplementedError + raise NotImplementedError( + "FETCH HEADERS NOT IMPLEMENTED FOR SEQUENCE NUMBER YET") class headersPart(object): def __init__(self, uid, headers): @@ -748,11 +763,6 @@ class IMAPMailbox(object): :raise ReadOnlyMailbox: Raised if this mailbox is not open for read-write. """ - # TODO implement sequences - is_sequence = True if uid == 0 else False - if is_sequence: - raise NotImplementedError - if not self.isWriteable(): log.msg('read only mailbox!') raise imap4.ReadOnlyMailbox @@ -762,8 +772,9 @@ class IMAPMailbox(object): mode, uid, d) if PROFILE_CMD: do_profile_cmd(d, "STORE") + d.addCallback(self.collection.cb_signal_unread_to_ui) - d.addErrback(lambda f: log.msg(f.getTraceback())) + d.addErrback(lambda f: log.err(f)) return d def _do_store(self, messages_asked, flags, mode, uid, observer): @@ -777,14 +788,13 @@ class IMAPMailbox(object): done. :type observer: deferred """ - # TODO implement also sequence (uid = 0) # TODO we should prevent client from setting Recent flag + get_msg_fun = self._get_message_fun(uid) leap_assert(not isinstance(flags, basestring), "flags cannot be a string") flags = tuple(flags) def set_flags_for_seq(sequence): - def return_result_dict(list_of_flags): result = dict(zip(list(sequence), list_of_flags)) observer.callback(result) @@ -792,7 +802,7 @@ class IMAPMailbox(object): d_all_set = [] for msgid in sequence: - d = self.collection.get_message_by_uid(msgid) + d = get_msg_fun(msgid) d.addCallback(lambda msg: self.collection.update_flags( msg, flags, mode)) d_all_set.append(d) @@ -800,7 +810,7 @@ class IMAPMailbox(object): got_flags_setted.addCallback(return_result_dict) return got_flags_setted - d_seq = self._get_messages_range(messages_asked) + d_seq = self._get_messages_range(messages_asked, uid) d_seq.addCallback(set_flags_for_seq) return d_seq diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 4c6f10d..9af4c99 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -173,7 +173,7 @@ class IMAPMessage(object): :rtype: Any object implementing C{IMessagePart}. :return: The specified sub-part. """ - subpart = self.message.get_subpart(part) + subpart = self.message.get_subpart(part + 1) return IMAPMessagePart(subpart) def __prefetch_body_file(self): @@ -204,7 +204,7 @@ class IMAPMessagePart(object): return self.message_part.is_multipart() def getSubPart(self, part): - subpart = self.message_part.get_subpart(part) + subpart = self.message_part.get_subpart(part + 1) return IMAPMessagePart(subpart) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index f6936dd..6a7c558 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -149,7 +149,7 @@ class MessagePart(object): # TODO support arbitrarily nested multiparts (right now we only support # the trivial case) - def __init__(self, part_map, cdocs={}): + def __init__(self, part_map, cdocs={}, nested=False): """ :param part_map: a dictionary mapping the subparts for this MessagePart (1-indexed). @@ -170,6 +170,7 @@ class MessagePart(object): """ self._pmap = part_map self._cdocs = cdocs + self._nested = nested index = _get_index_for_cdoc(part_map, self._cdocs) or 1 self._index = index @@ -230,7 +231,7 @@ class MessagePart(object): cdoc_index = _get_index_for_cdoc(part_map, self._cdocs) cdoc = self._cdocs.get(cdoc_index, {}) - return MessagePart(part_map, cdocs={1: cdoc}) + return MessagePart(part_map, cdocs={1: cdoc}, nested=True) def _get_payload(self, index): cdoc_wrapper = self._cdocs.get(index, None) @@ -329,20 +330,17 @@ class Message(object): def get_subpart(self, part): """ - :param part: The number of the part to retrieve, indexed from 0. + :param part: The number of the part to retrieve, indexed from 1. :type part: int :rtype: MessagePart """ if not self.is_multipart(): raise TypeError - part_index = part + 1 try: - subpart_dict = self._wrapper.get_subpart_dict(part_index) + subpart_dict = self._wrapper.get_subpart_dict(part) except KeyError: raise IndexError - # FIXME instead of passing the index, let the MessagePart figure it out - # by getting the phash and iterating through the cdocs return MessagePart( subpart_dict, cdocs=self._wrapper.cdocs) @@ -467,6 +465,21 @@ class MessageCollection(object): self.messageklass, self.store, metamsg_id, get_cdocs=get_cdocs) + def get_message_by_sequence_number(self, msn, get_cdocs=False): + """ + Retrieve a message by its Message Sequence Number. + :rtype: Deferred + """ + def get_uid_for_msn(all_uid): + return all_uid[msn - 1] + d = self.all_uid_iter() + d.addCallback(get_uid_for_msn) + d.addCallback( + lambda uid: self.get_message_by_uid( + uid, get_cdocs=get_cdocs)) + d.addErrback(lambda f: log.err(f)) + return d + def get_message_by_uid(self, uid, absolute=True, get_cdocs=False): """ Retrieve a message by its Unique Identifier. @@ -476,6 +489,8 @@ class MessageCollection(object): flag. For now, only absolute identifiers are supported. :rtype: Deferred """ + # TODO deprecate absolute flag, it doesn't make sense UID and + # !absolute. use _by_sequence_number instead. if not absolute: raise NotImplementedError("Does not support relative ids yet") @@ -502,6 +517,7 @@ class MessageCollection(object): return d def get_flags_by_uid(self, uid, absolute=True): + # TODO use sequence numbers if not absolute: raise NotImplementedError("Does not support relative ids yet") @@ -825,6 +841,7 @@ class MessageCollection(object): self.store, self.mbox_uuid) mdocs_deleted.addCallback(get_uid_list) mdocs_deleted.addCallback(delete_uid_entries) + mdocs_deleted.addErrback(lambda f: log.err(f)) return mdocs_deleted # TODO should add a delete-by-uid to collection? -- cgit v1.2.3 From e9cae5495b589ac6f6c347db156beac76a5413c6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 9 Jul 2015 12:01:53 -0400 Subject: [debug] add some more debug info to the regressions script --- src/leap/mail/imap/tests/regressions | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/leap/mail/imap/tests/regressions b/src/leap/mail/imap/tests/regressions index efe3f46..475c893 100755 --- a/src/leap/mail/imap/tests/regressions +++ b/src/leap/mail/imap/tests/regressions @@ -263,7 +263,7 @@ def cbSelectMbox(result, proto): if result["EXISTS"] != 0: # Flag as deleted, expunge, and do an examine again. - #print "There is mail here, will delete..." + print "There is mail here, will delete..." return cbDeleteAndExpungeTestFolder(proto) else: @@ -276,11 +276,15 @@ def ebSelectMbox(failure, proto, folder): Creates the folder. """ - print failure.getTraceback() + log.err(failure) log.msg("Folder %r does not exist. Creating..." % (folder,)) return proto.create(folder).addCallback(cbAuthentication, proto) +def ebExpunge(failure): + log.err(failure) + + def cbDeleteAndExpungeTestFolder(proto): """ Callback invoked fom cbExamineMbox when the number of messages in the @@ -292,7 +296,9 @@ def cbDeleteAndExpungeTestFolder(proto): ).addCallback( lambda r: proto.expunge() ).addCallback( - cbExpunge, proto) + cbExpunge, proto + ).addErrback( + ebExpunge) def cbExpunge(result, proto): -- cgit v1.2.3 From 2737ab3147ddb08ae024b7fd49c69153e01609c4 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 9 Jul 2015 12:03:22 -0400 Subject: [style] slightly more meaningful test module name --- src/leap/mail/imap/tests/regressions | 459 ----------------------- src/leap/mail/imap/tests/regressions_mime_struct | 459 +++++++++++++++++++++++ 2 files changed, 459 insertions(+), 459 deletions(-) delete mode 100755 src/leap/mail/imap/tests/regressions create mode 100755 src/leap/mail/imap/tests/regressions_mime_struct diff --git a/src/leap/mail/imap/tests/regressions b/src/leap/mail/imap/tests/regressions deleted file mode 100755 index 475c893..0000000 --- a/src/leap/mail/imap/tests/regressions +++ /dev/null @@ -1,459 +0,0 @@ -#!/usr/bin/env python - -# -*- coding: utf-8 -*- -# regressions -# Copyright (C) 2014 LEAP -# Copyright (c) Twisted Matrix Laboratories. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Simple Regression Tests using IMAP4 client. - -Iterates trough all mails under a given folder and tries to APPEND them to -the server being tested. After FETCHING the pushed message, it compares -the received version with the one that was saved, and exits with an error -code if they do not match. -""" -import os -import StringIO -import sys - -from email.parser import Parser - -from twisted.internet import protocol -from twisted.internet import ssl -from twisted.internet import defer -from twisted.internet import stdio -from twisted.mail import imap4 -from twisted.protocols import basic -from twisted.python import log - - -REGRESSIONS_FOLDER = "regressions_test" - -parser = Parser() - - -def get_msg_parts(raw): - """ - Return a representation of the parts of a message suitable for - comparison. - - :param raw: string for the message - :type raw: str - """ - m = parser.parsestr(raw) - return [dict(part.items()) - if part.is_multipart() - else part.get_payload() - for part in m.walk()] - - -def compare_msg_parts(a, b): - """ - Compare two sequences of parts of messages. - - :param a: part sequence for message a - :param b: part sequence for message b - - :return: True if both message sequences are equivalent. - :rtype: bool - """ - # XXX This could be smarter and show the differences in the - # different parts when/where they differ. - #import pprint; pprint.pprint(a[0]) - #import pprint; pprint.pprint(b[0]) - - def lowerkey(d): - return dict((k.lower(), v.replace('\r', '')) - for k, v in d.iteritems()) - - def eq(x, y): - # For dicts, we compare a variation with their keys - # in lowercase, and \r removed from their values - if all(map(lambda i: isinstance(i, dict), (x, y))): - x, y = map(lowerkey, (x, y)) - return x == y - - compare_vector = map(lambda tup: eq(tup[0], tup[1]), zip(a, b)) - all_match = all(compare_vector) - - if not all_match: - print "PARTS MISMATCH!" - print "vector: ", compare_vector - index = compare_vector.index(False) - from pprint import pprint - print "Expected:" - pprint(a[index]) - print ("***") - print "Found:" - pprint(b[index]) - print - - return all_match - - -def get_fd(string): - """ - Return a file descriptor with the passed string - as content. - """ - fd = StringIO.StringIO() - fd.write(string) - fd.seek(0) - return fd - - -class TrivialPrompter(basic.LineReceiver): - promptDeferred = None - - def prompt(self, msg): - assert self.promptDeferred is None - self.display(msg) - self.promptDeferred = defer.Deferred() - return self.promptDeferred - - def display(self, msg): - self.transport.write(msg) - - def lineReceived(self, line): - if self.promptDeferred is None: - return - d, self.promptDeferred = self.promptDeferred, None - d.callback(line) - - -class SimpleIMAP4Client(imap4.IMAP4Client): - """ - A client with callbacks for greeting messages from an IMAP server. - """ - greetDeferred = None - - def serverGreeting(self, caps): - self.serverCapabilities = caps - if self.greetDeferred is not None: - d, self.greetDeferred = self.greetDeferred, None - d.callback(self) - - -class SimpleIMAP4ClientFactory(protocol.ClientFactory): - usedUp = False - protocol = SimpleIMAP4Client - - def __init__(self, username, onConn): - self.ctx = ssl.ClientContextFactory() - - self.username = username - self.onConn = onConn - - def buildProtocol(self, addr): - """ - Initiate the protocol instance. Since we are building a simple IMAP - client, we don't bother checking what capabilities the server has. We - just add all the authenticators twisted.mail has. Note: Gmail no - longer uses any of the methods below, it's been using XOAUTH since - 2010. - """ - assert not self.usedUp - self.usedUp = True - - p = self.protocol(self.ctx) - p.factory = self - p.greetDeferred = self.onConn - - p.registerAuthenticator(imap4.PLAINAuthenticator(self.username)) - p.registerAuthenticator(imap4.LOGINAuthenticator(self.username)) - p.registerAuthenticator( - imap4.CramMD5ClientAuthenticator(self.username)) - - return p - - def clientConnectionFailed(self, connector, reason): - d, self.onConn = self.onConn, None - d.errback(reason) - - -def cbServerGreeting(proto, username, password): - """ - Initial callback - invoked after the server sends us its greet message. - """ - # Hook up stdio - tp = TrivialPrompter() - stdio.StandardIO(tp) - - # And make it easily accessible - proto.prompt = tp.prompt - proto.display = tp.display - - # Try to authenticate securely - return proto.authenticate( - password).addCallback( - cbAuthentication, - proto).addErrback( - ebAuthentication, proto, username, password - ) - - -def ebConnection(reason): - """ - Fallback error-handler. If anything goes wrong, log it and quit. - """ - log.startLogging(sys.stdout) - log.err(reason) - return reason - - -def cbAuthentication(result, proto): - """ - Callback after authentication has succeeded. - - Lists a bunch of mailboxes. - """ - return proto.select( - REGRESSIONS_FOLDER - ).addCallback( - cbSelectMbox, proto - ).addErrback( - ebSelectMbox, proto, REGRESSIONS_FOLDER) - - -def ebAuthentication(failure, proto, username, password): - """ - Errback invoked when authentication fails. - - If it failed because no SASL mechanisms match, offer the user the choice - of logging in insecurely. - - If you are trying to connect to your Gmail account, you will be here! - """ - failure.trap(imap4.NoSupportedAuthentication) - return InsecureLogin(proto, username, password) - - -def InsecureLogin(proto, username, password): - """ - Raise insecure-login error. - """ - return proto.login( - username, password - ).addCallback( - cbAuthentication, proto) - - -def cbSelectMbox(result, proto): - """ - Callback invoked when select command finishes successfully. - - If any message is in the test folder, it will flag them as deleted and - expunge. - If no messages found, it will start with the APPEND tests. - """ - print "SELECT: %s EXISTS " % result.get("EXISTS", "??") - - if result["EXISTS"] != 0: - # Flag as deleted, expunge, and do an examine again. - print "There is mail here, will delete..." - return cbDeleteAndExpungeTestFolder(proto) - - else: - return cbAppendNextMessage(proto) - - -def ebSelectMbox(failure, proto, folder): - """ - Errback invoked when the examine command fails. - - Creates the folder. - """ - log.err(failure) - log.msg("Folder %r does not exist. Creating..." % (folder,)) - return proto.create(folder).addCallback(cbAuthentication, proto) - - -def ebExpunge(failure): - log.err(failure) - - -def cbDeleteAndExpungeTestFolder(proto): - """ - Callback invoked fom cbExamineMbox when the number of messages in the - mailbox is not zero. It flags all messages as deleted and expunge the - mailbox. - """ - return proto.setFlags( - "1:*", ("\\Deleted",) - ).addCallback( - lambda r: proto.expunge() - ).addCallback( - cbExpunge, proto - ).addErrback( - ebExpunge) - - -def cbExpunge(result, proto): - return proto.select( - REGRESSIONS_FOLDER - ).addCallback( - cbSelectMbox, proto - ).addErrback(ebSettingDeleted, proto) - - -def ebSettingDeleted(failure, proto): - """ - Report errors during deletion of messages in the mailbox. - """ - print failure.getTraceback() - - -def cbAppendNextMessage(proto): - """ - Appends the next message in the global queue to the test folder. - """ - # 1. Get the next test message from global tuple. - try: - next_sample = SAMPLES.pop() - except IndexError: - # we're done! - return proto.logout() - - print "\nAPPEND %s" % (next_sample,) - raw = open(next_sample).read() - msg = get_fd(raw) - return proto.append( - REGRESSIONS_FOLDER, msg - ).addCallback( - lambda r: proto.select(REGRESSIONS_FOLDER) - ).addCallback( - cbAppend, proto, raw - ).addErrback( - ebAppend, proto, raw) - - -def cbAppend(result, proto, orig_msg): - """ - Fetches the message right after an append. - """ - # XXX keep account of highest UID - uid = "1:*" - - return proto.fetchSpecific( - '%s' % uid, - headerType='', - headerArgs=['BODY.PEEK[]'], - ).addCallback( - cbCompareMessage, proto, orig_msg - ).addErrback(ebAppend, proto, orig_msg) - - -def ebAppend(failure, proto, raw): - """ - Errorback for the append operation - """ - print "ERROR WHILE APPENDING!" - print failure.getTraceback() - - -def cbPickMessage(result, proto): - """ - Pick a message. - """ - return proto.fetchSpecific( - '%s' % result, - headerType='', - headerArgs=['BODY.PEEK[]'], - ).addCallback(cbCompareMessage, proto) - - -def cbCompareMessage(result, proto, raw): - """ - Display message and compare it with the original one. - """ - parts_orig = get_msg_parts(raw) - - if result: - keys = result.keys() - keys.sort() - else: - print "[-] GOT NO RESULT" - return proto.logout() - - latest = max(keys) - - fetched_msg = result[latest][0][2] - parts_fetched = get_msg_parts(fetched_msg) - - equal = compare_msg_parts( - parts_orig, - parts_fetched) - - if equal: - print "[+] MESSAGES MATCH" - return cbAppendNextMessage(proto) - else: - print "[-] ERROR: MESSAGES DO NOT MATCH !!!" - print " ABORTING COMPARISON..." - # FIXME logout and print the subject ... - return proto.logout() - - -def cbClose(result): - """ - Close the connection when we finish everything. - """ - from twisted.internet import reactor - reactor.stop() - - -def main(): - import glob - import sys - - if len(sys.argv) != 4: - print "Usage: regressions " - sys.exit() - - hostname = "localhost" - port = "1984" - username = sys.argv[1] - password = sys.argv[2] - - samplesdir = sys.argv[3] - - if not os.path.isdir(samplesdir): - print ("Could not find samples folder! " - "Make sure of copying mail_breaker contents there.") - sys.exit() - - samples = glob.glob(samplesdir + '/*') - - global SAMPLES - SAMPLES = [] - SAMPLES += samples - - onConn = defer.Deferred( - ).addCallback( - cbServerGreeting, username, password - ).addErrback( - ebConnection - ).addBoth(cbClose) - - factory = SimpleIMAP4ClientFactory(username, onConn) - - from twisted.internet import reactor - reactor.connectTCP(hostname, int(port), factory) - reactor.run() - - -if __name__ == '__main__': - main() diff --git a/src/leap/mail/imap/tests/regressions_mime_struct b/src/leap/mail/imap/tests/regressions_mime_struct new file mode 100755 index 0000000..1e0e870 --- /dev/null +++ b/src/leap/mail/imap/tests/regressions_mime_struct @@ -0,0 +1,459 @@ +#!/usr/bin/env python + +# -*- coding: utf-8 -*- +# regression_mime_struct +# Copyright (C) 2014 LEAP +# Copyright (c) Twisted Matrix Laboratories. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Simple Regression Tests for checking MIME struct handling using IMAP4 client. + +Iterates trough all mails under a given folder and tries to APPEND them to +the server being tested. After FETCHING the pushed message, it compares +the received version with the one that was saved, and exits with an error +code if they do not match. +""" +import os +import StringIO +import sys + +from email.parser import Parser + +from twisted.internet import protocol +from twisted.internet import ssl +from twisted.internet import defer +from twisted.internet import stdio +from twisted.mail import imap4 +from twisted.protocols import basic +from twisted.python import log + + +REGRESSIONS_FOLDER = "regressions_test" + +parser = Parser() + + +def get_msg_parts(raw): + """ + Return a representation of the parts of a message suitable for + comparison. + + :param raw: string for the message + :type raw: str + """ + m = parser.parsestr(raw) + return [dict(part.items()) + if part.is_multipart() + else part.get_payload() + for part in m.walk()] + + +def compare_msg_parts(a, b): + """ + Compare two sequences of parts of messages. + + :param a: part sequence for message a + :param b: part sequence for message b + + :return: True if both message sequences are equivalent. + :rtype: bool + """ + # XXX This could be smarter and show the differences in the + # different parts when/where they differ. + #import pprint; pprint.pprint(a[0]) + #import pprint; pprint.pprint(b[0]) + + def lowerkey(d): + return dict((k.lower(), v.replace('\r', '')) + for k, v in d.iteritems()) + + def eq(x, y): + # For dicts, we compare a variation with their keys + # in lowercase, and \r removed from their values + if all(map(lambda i: isinstance(i, dict), (x, y))): + x, y = map(lowerkey, (x, y)) + return x == y + + compare_vector = map(lambda tup: eq(tup[0], tup[1]), zip(a, b)) + all_match = all(compare_vector) + + if not all_match: + print "PARTS MISMATCH!" + print "vector: ", compare_vector + index = compare_vector.index(False) + from pprint import pprint + print "Expected:" + pprint(a[index]) + print ("***") + print "Found:" + pprint(b[index]) + print + + return all_match + + +def get_fd(string): + """ + Return a file descriptor with the passed string + as content. + """ + fd = StringIO.StringIO() + fd.write(string) + fd.seek(0) + return fd + + +class TrivialPrompter(basic.LineReceiver): + promptDeferred = None + + def prompt(self, msg): + assert self.promptDeferred is None + self.display(msg) + self.promptDeferred = defer.Deferred() + return self.promptDeferred + + def display(self, msg): + self.transport.write(msg) + + def lineReceived(self, line): + if self.promptDeferred is None: + return + d, self.promptDeferred = self.promptDeferred, None + d.callback(line) + + +class SimpleIMAP4Client(imap4.IMAP4Client): + """ + A client with callbacks for greeting messages from an IMAP server. + """ + greetDeferred = None + + def serverGreeting(self, caps): + self.serverCapabilities = caps + if self.greetDeferred is not None: + d, self.greetDeferred = self.greetDeferred, None + d.callback(self) + + +class SimpleIMAP4ClientFactory(protocol.ClientFactory): + usedUp = False + protocol = SimpleIMAP4Client + + def __init__(self, username, onConn): + self.ctx = ssl.ClientContextFactory() + + self.username = username + self.onConn = onConn + + def buildProtocol(self, addr): + """ + Initiate the protocol instance. Since we are building a simple IMAP + client, we don't bother checking what capabilities the server has. We + just add all the authenticators twisted.mail has. Note: Gmail no + longer uses any of the methods below, it's been using XOAUTH since + 2010. + """ + assert not self.usedUp + self.usedUp = True + + p = self.protocol(self.ctx) + p.factory = self + p.greetDeferred = self.onConn + + p.registerAuthenticator(imap4.PLAINAuthenticator(self.username)) + p.registerAuthenticator(imap4.LOGINAuthenticator(self.username)) + p.registerAuthenticator( + imap4.CramMD5ClientAuthenticator(self.username)) + + return p + + def clientConnectionFailed(self, connector, reason): + d, self.onConn = self.onConn, None + d.errback(reason) + + +def cbServerGreeting(proto, username, password): + """ + Initial callback - invoked after the server sends us its greet message. + """ + # Hook up stdio + tp = TrivialPrompter() + stdio.StandardIO(tp) + + # And make it easily accessible + proto.prompt = tp.prompt + proto.display = tp.display + + # Try to authenticate securely + return proto.authenticate( + password).addCallback( + cbAuthentication, + proto).addErrback( + ebAuthentication, proto, username, password + ) + + +def ebConnection(reason): + """ + Fallback error-handler. If anything goes wrong, log it and quit. + """ + log.startLogging(sys.stdout) + log.err(reason) + return reason + + +def cbAuthentication(result, proto): + """ + Callback after authentication has succeeded. + + Lists a bunch of mailboxes. + """ + return proto.select( + REGRESSIONS_FOLDER + ).addCallback( + cbSelectMbox, proto + ).addErrback( + ebSelectMbox, proto, REGRESSIONS_FOLDER) + + +def ebAuthentication(failure, proto, username, password): + """ + Errback invoked when authentication fails. + + If it failed because no SASL mechanisms match, offer the user the choice + of logging in insecurely. + + If you are trying to connect to your Gmail account, you will be here! + """ + failure.trap(imap4.NoSupportedAuthentication) + return InsecureLogin(proto, username, password) + + +def InsecureLogin(proto, username, password): + """ + Raise insecure-login error. + """ + return proto.login( + username, password + ).addCallback( + cbAuthentication, proto) + + +def cbSelectMbox(result, proto): + """ + Callback invoked when select command finishes successfully. + + If any message is in the test folder, it will flag them as deleted and + expunge. + If no messages found, it will start with the APPEND tests. + """ + print "SELECT: %s EXISTS " % result.get("EXISTS", "??") + + if result["EXISTS"] != 0: + # Flag as deleted, expunge, and do an examine again. + print "There is mail here, will delete..." + return cbDeleteAndExpungeTestFolder(proto) + + else: + return cbAppendNextMessage(proto) + + +def ebSelectMbox(failure, proto, folder): + """ + Errback invoked when the examine command fails. + + Creates the folder. + """ + log.err(failure) + log.msg("Folder %r does not exist. Creating..." % (folder,)) + return proto.create(folder).addCallback(cbAuthentication, proto) + + +def ebExpunge(failure): + log.err(failure) + + +def cbDeleteAndExpungeTestFolder(proto): + """ + Callback invoked fom cbExamineMbox when the number of messages in the + mailbox is not zero. It flags all messages as deleted and expunge the + mailbox. + """ + return proto.setFlags( + "1:*", ("\\Deleted",) + ).addCallback( + lambda r: proto.expunge() + ).addCallback( + cbExpunge, proto + ).addErrback( + ebExpunge) + + +def cbExpunge(result, proto): + return proto.select( + REGRESSIONS_FOLDER + ).addCallback( + cbSelectMbox, proto + ).addErrback(ebSettingDeleted, proto) + + +def ebSettingDeleted(failure, proto): + """ + Report errors during deletion of messages in the mailbox. + """ + print failure.getTraceback() + + +def cbAppendNextMessage(proto): + """ + Appends the next message in the global queue to the test folder. + """ + # 1. Get the next test message from global tuple. + try: + next_sample = SAMPLES.pop() + except IndexError: + # we're done! + return proto.logout() + + print "\nAPPEND %s" % (next_sample,) + raw = open(next_sample).read() + msg = get_fd(raw) + return proto.append( + REGRESSIONS_FOLDER, msg + ).addCallback( + lambda r: proto.select(REGRESSIONS_FOLDER) + ).addCallback( + cbAppend, proto, raw + ).addErrback( + ebAppend, proto, raw) + + +def cbAppend(result, proto, orig_msg): + """ + Fetches the message right after an append. + """ + # XXX keep account of highest UID + uid = "1:*" + + return proto.fetchSpecific( + '%s' % uid, + headerType='', + headerArgs=['BODY.PEEK[]'], + ).addCallback( + cbCompareMessage, proto, orig_msg + ).addErrback(ebAppend, proto, orig_msg) + + +def ebAppend(failure, proto, raw): + """ + Errorback for the append operation + """ + print "ERROR WHILE APPENDING!" + print failure.getTraceback() + + +def cbPickMessage(result, proto): + """ + Pick a message. + """ + return proto.fetchSpecific( + '%s' % result, + headerType='', + headerArgs=['BODY.PEEK[]'], + ).addCallback(cbCompareMessage, proto) + + +def cbCompareMessage(result, proto, raw): + """ + Display message and compare it with the original one. + """ + parts_orig = get_msg_parts(raw) + + if result: + keys = result.keys() + keys.sort() + else: + print "[-] GOT NO RESULT" + return proto.logout() + + latest = max(keys) + + fetched_msg = result[latest][0][2] + parts_fetched = get_msg_parts(fetched_msg) + + equal = compare_msg_parts( + parts_orig, + parts_fetched) + + if equal: + print "[+] MESSAGES MATCH" + return cbAppendNextMessage(proto) + else: + print "[-] ERROR: MESSAGES DO NOT MATCH !!!" + print " ABORTING COMPARISON..." + # FIXME logout and print the subject ... + return proto.logout() + + +def cbClose(result): + """ + Close the connection when we finish everything. + """ + from twisted.internet import reactor + reactor.stop() + + +def main(): + import glob + import sys + + if len(sys.argv) != 4: + print "Usage: regressions " + sys.exit() + + hostname = "localhost" + port = "1984" + username = sys.argv[1] + password = sys.argv[2] + + samplesdir = sys.argv[3] + + if not os.path.isdir(samplesdir): + print ("Could not find samples folder! " + "Make sure of copying mail_breaker contents there.") + sys.exit() + + samples = glob.glob(samplesdir + '/*') + + global SAMPLES + SAMPLES = [] + SAMPLES += samples + + onConn = defer.Deferred( + ).addCallback( + cbServerGreeting, username, password + ).addErrback( + ebConnection + ).addBoth(cbClose) + + factory = SimpleIMAP4ClientFactory(username, onConn) + + from twisted.internet import reactor + reactor.connectTCP(hostname, int(port), factory) + reactor.run() + + +if __name__ == '__main__': + main() -- cgit v1.2.3 From 91269c0305b7f7d562f27fd93c94a8d601219ea1 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 10 Jul 2015 11:57:48 -0400 Subject: [tests] new sample mail added For #7244, this is needed for regression tests --- src/leap/mail/tests/rfc822.multi-nested.message | 619 ++++++++++++++++++++++++ 1 file changed, 619 insertions(+) create mode 100644 src/leap/mail/tests/rfc822.multi-nested.message diff --git a/src/leap/mail/tests/rfc822.multi-nested.message b/src/leap/mail/tests/rfc822.multi-nested.message new file mode 100644 index 0000000..694bef5 --- /dev/null +++ b/src/leap/mail/tests/rfc822.multi-nested.message @@ -0,0 +1,619 @@ +From: TEST +Content-Type: multipart/alternative; + boundary="Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8" +X-Smtp-Server: smtp.example.com:test.bitmask +Subject: test simple attachment +X-Universally-Unique-Identifier: 0ea1b4b2-cdb8-43c3-b54c-dc88a19c6e0a +Date: Wed, 8 Jul 2015 04:25:56 +0900 +Message-Id: <47278179-628A-43F5-95C9-BC7E1753C521@example.com> +To: test_alpha14_001@dev.bitmask.net +Mime-Version: 1.0 (Apple Message framework v1251.1) + + +--Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8 +Content-Transfer-Encoding: 7bit +Content-Type: text/plain; + charset=us-ascii + +this is a simple attachment +--Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8 +Content-Type: multipart/related; + type="text/html"; + boundary="Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B" + + +--Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B +Content-Transfer-Encoding: 7bit +Content-Type: text/html; + charset=us-ascii + +this is a simple attachment +--Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B +Content-Transfer-Encoding: base64 +Content-Disposition: inline; + filename="saing_ergol.jpg" +Content-Type: image/jpg; + x-mac-hide-extension=yes; + x-unix-mode=0600; + name="saint_ergol.jpg" +Content-Id: <163B7957-4342-485F-8FD6-D46A4A53A2C1> + +/9j/4AAQSkZJRgABAQEAYABgAAD/4QCURXhpZgAASUkqAAgAAAACADEBAgALAAAAJgAAAGmHBAAB +AAAAMgAAAAAAAABQaWNhc2EgMy4wAAAEAAKgBAABAAAALAEAAAOgBAABAAAAHgEAAACQBwAEAAAA +MDIxMAWgBAABAAAAaAAAAAAAAAACAAEAAgAFAAAAhgAAAAIABwAEAAAAMDEwMAAAAAAgICAgAAD/ +7QAcUGhvdG9zaG9wIDMuMAA4QklNBAQAAAAAAAD//gAmRmlsZSB3cml0dGVuIGJ5IEFkb2JlIFBo +b3Rvc2hvcKggNS4y/9sAQwAFAwQEBAMFBAQEBQUFBgcMCAcHBwcPCwsJDBEPEhIRDxERExYcFxMU +GhURERghGBodHR8fHxMXIiQiHiQcHh8e/9sAQwEFBQUHBgcOCAgOHhQRFB4eHh4eHh4eHh4eHh4e +Hh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4e/8AAEQgBHgEsAwEiAAIRAQMRAf/E +AB0AAAEFAQEBAQAAAAAAAAAAAAUCAwQGBwgAAQn/xABQEAABAwIEAwUFBQUGBAQFAgcCAwQFARIA +BhETFCEiBzEyQVEVI0JSYSQzYnGBCBZDcqElNIKRkqJTscHRRGOy0jVUc4PhF2SkwsPT4vDy/8QA +FAEBAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/ANW2 +3RMhJzJOiUK4up+4Lv8A4JUt/rgjBZfmN1JRBsaDchEklXfSaQ15GkdfFX6YteR4tNtDiuuCC7gh +6iHpt89O+vLFg+ARv8WAQ0bizSSaD1CkNo4fuL58J2/BdhXSJ+PAex64sL+PCDTEjK47vw4D1qnz +49b1+Pw4bMrf+IOFWj/NgPdV5Wn/AIceSUuMus/5sKMRw2Y+Lo6i8OAV4kiETtL5sMGpaACofUXh +/F+WPN1vvei0R/1YV7tRLpC4fxYBvcT2hJMztIenxXD/AP76YVvCICQnd+G/COpMNwvu/iMvh/Om +Bkgs63QXYtk+IIv4h9FuAJm4+0XCdpdN4+Lp+v8A3xGBZYXBJ3mVt1nzW/T1w6qmperb0koNoeG8 +f+44Emo6QBJN9YoSnVv+FJUvr5p4CVxBKXEmsdxdNvwCPpT5S+mIvGSBN1d4zbOkytuLqEgH1p/1 +phDtZwg1VIt+0LbwEKXjr6j/ABKfWmGGjhQmSvE/CVqVp3Xaf8Mq+Gv0rgLEyUW4JJVT4vx3D+mv +w4dN04IVyRv6Oq6y4fr3YQ0IXMeLkQMRtutIOr/LAqY4VNJqMgYJo2/iG3/TgGJDMyyCS5JuQEh6 +UhctlQH9a24EuJbjnFzGShyISuV3HiqRidPpUfBhSpEoBptjfKIp+7AUXgFdp8WhjiM3FYd1y5fr +kj/+7igI0h/FZXATmSz7dIkW0omSfUYIyQGCo18x6sNHMSjFVVZ28mEESK1tcwFURL5a2692BgLM +RBJMloRe4iIbQWQJL8Q99448kzjyVVJNtFqXCJf/ABJUAX+nOnTgGpiWeO0l0F5K5NTqVFRFZsqg +VO46Wj4cRop8pxC7FN4+EhSEr05gDScj+C8qFStPyxMWEuldBaUUWTIhDZeJKkkVP4daENLh9MMG +Kj5IWOy6ZFaSi7ZaNAyIfmoQlTAS28lME0/vkpvUuFuuTluYpDX/AIlL+/DDSQmkFVV1JibJqmRE +ZKMwIkD866hWuqeGHCjdNp7SF4xJO0UUC9lGIL+VqlKH5YdSKNjY8XJLNdkfERM1ht15W1pd4cA0 +bjMSkO1TXkl7R98u54ZUCS6unwjzGuGnr6U2lVBklEBUEb7XLojSLyU0t8OBTh8oSSCaaLVNqur0 +ELlZK3q0tKlRLpri6/um4FoRIGgIiQ2JEe6QpU/g+XTrgKe79rPrU1JWR4xMbiJs8dW20LkuOo86 +euLXkyPnEGntKSknwrOfAgm5JUEh7q8lNO/lXBiJg4tsDYkGa+8gO4Jl/wCmnV3YK+8USVLZ6vh6 +PX9cA3/aCaVvtJdRa3wiil1f7sKbuH3FluP1yH+QLS8vmx401ib7lnw22eHCVk1t0RFY+rqO0BL/ +AK+eAc4p0nu778y+XoDp/rhYLOCMS4ldNNPqUuttLUfzxBj94jESbGSY9VpIj1f7sPpKOBcEPiR8 +I22l3eeAfCSJNv8AfXeK0yWG78PnhJksropxT1O6mtorDp/zwhuKyiRD4VBL8FuHiJ0VdRqadPl6 +cBToqUKPcIFwx2q9JG5tAiL5SKnQWL5GPEXzIXYgpaXzYzA0U2J7DZYLrSIjK9iRad415bZV/wAW +Lbld4oLtBMUdtuokV1yJCV/0INRL/PAWsxHCTG48KuIvD1Y8ZF04BIF1+D/Vh3wn4MNH81/+rHkl +FLyuDAKtuAui7DG5aYiQdP8A6cP9VmEqjcA3dJCXTbgHbv8Adho07rh/04i++Td9R/Z/iH4h/LEq +4f5R8WAauIlbbOr5sNJIrClcXVb8v/TEhZMd3cI/F0/LiPcKYffGPwl8PV8v0r9cB5VToLr/AAl/ +3w2bckw93Z7zxB4bvy+WuEGPujITu6rb7P8AbXDpimolaQWl8v4cBFSWTLpUW3xTK0yILSSLy19M +J4cve9ZkXSN/iK38VPOmJjtPwiSxipb0mPUQ/wDfDTdmSYbdgW9V3i/1U/PADDTWEFSQC7cK4Osv +Ll7s/g/LFbnSeEqKaaNyYn9qVIPCXounT/1ji7pR6nUQncRdRJWdKunnWnrhjg7URVH3dtxXl4kv +yKvfT6VwDGWnlzUWy5nb4UNzquGnfYfxYmSo7bQiT8SfUIpmIjZX+bWlcDJXejW5Wsw2SIulO4ki +9Knp1BX60w5BOJJTpK9dmRWmC3jQ/wDudyg4AYlH+9XJyz30yt6lGHr8VydKcsNfYU2/EpmDZQSF +Mi49VLp9DpUsWl6mok4FZAA92Nto3D0V/Lz9MQ9kSbiShuhu6SEljLqr9K0wFb2RJXiWjl7t/Akm +5SPq86p1KlS/TDSqJCkkx3nztFUSWICYIkJD8vIeRUxMerJppCg5O1MS6QWATEvxDU9LcDHCm84J +sotHEiRD0u0UhEiqPK6oFUqVwC3DFumqlILhHDtESaS7mNJIh8tK1DTDjRSPdpW3tUFhuG4X5pEk +XzUpUudK4S9jfY2X9xcDQapl9pVbPFRJIfmr088KbpuvZiqyDx0u3QtUQJN4KpD+KnPUh591cAwH +Ctm5Nr0C3SuMBmCucl8wal34YVki4dqgUk6+0+AikgK4e+xTX4sSjGYUZIERrkiJEoR3pWkdS5VD +UuRU+XEVVGSkHpFs76atyZbkakQql5jqBVtL64B9wimQCuKxjtD1A5lfuh8k66d419cXBJwptIKc +GaC20IkIgJj4eY618WBOX4V42PcI1+DBINseGAC8VemulcWAy6/dhaJXdI2Db/XAI6VQLYDbLwkJ +AIkWvnzwwkTwXApkshaVvQNtwjTyxISIdolFPDbdaVn/AErjLc5zXs3tryVEoOTLjUFxVSTBK0Q6 +a3F1YDVne8QCQrAh8XgHqH64jKl96oRgIj1FcAlb+euHHAjYkmR3DddulZ4qfrjAP2jc/SBOEsoZ +QACcO1RRVGwS4nXltAdK8vxYAjCdpEl2g9qbWCyS5XbQMWRLSL4Ww3LlTltalTQRrz6sbU3RtutC +34hGwBIv9uM77Gsh/uLk9nHqAHHEJOHRpoo+Ov47uqlMXreWvJRS8fitLa9/08q63eeAmfcJWqBc +RfDYHh/yw8KgFTVR0FS8/By/pga7HfAVCcroDd0j7rw/L4sRyXSGtqbghEeVKJrJW/8AqwAqHb9D +wRBBNFUusGjwXIJLXaXCmoOtLq9/PEHedRbhrw14iToU3IIgQKiXz7ZXUrSv81MTNtuILryiICok +vtkT1htEvr8eoUpX/Fgrl8dj3fGLrt0uoy4wHNw1/PUsBZEpBO9dN2sCbhLqMRO60fLyxNPwfNis +N3DUnBKNt9QkyK4rFi6q/StcSmT5RNJK5suSZdV4omQ3fStfLAGrbvEGEqjaZW9JW3Ya4gbBu3BI +urqAsKbum6hlaYEVvhE7iup5aUwDqRDtdXi+LCjttx4BEgust+a3Dd2302GWAcMRsu+L/wBOGNxM +XBCXT8mJXwXYQqnuWlZ1D4S+IcBD3hTPpDcu6enqtH6YdNNEjJT4bbbvFcP5YYVuG5AQuIuoPhu9 +dPr9MeScEKQWmChEPSRBbd6/kXrgPJKfZ/uTEfiEerp/7Y8BCQDaBl8tvi/w4TasQEW9bcQiN3hH +/tXD7RNTiOo9serpEOkvzwDrdEhD357iheIvw4UZWmPXaNuFH8OB0hxCjhL7xAfDfvANuAlGp8Im +Al8JYGPVhbRi4qLIcQp8qxCJfrpX/liY7uF2P2k0xttHrH/rgE4UFRJUk3LohTIRsTchd/lgCPHN +xSVInIbJJCVhH0W07x8OJzQkRbjsBanaJdJ9PPu0xXjUUct0k0wXXJUesSRAwVG3zqI9OFRjxxGp +L3AuLceoS4AhtKnLnpTSuAKO+OT30yNclFLRSuciJF+tB6cQnpPBb2+/Hbt97xI3fkXThYSm4xIn +exxCdqgincHTXnd1/wBcQwkmq6tzaYBe64TMTStH/wAutNP64CO4cOC3RHilRTESEBcpF0111tuH +nXEF66TFUSdn94NoGswEwIbddCqBU0KmIOY2/wDYSqbYFyRQIVC4ZEF10Du8VtB1KnPvwwkxWsJy +ViZKiKaqAmq2FyFP4yfVpgCDRwjcP2xqoip4hK9Akh/lOh0KnpiU4FYj8Cgo23CAtgXD86VAg/yx +GbxrjhLWz90797uAuRi5EUqFrs1Guv6Urh9uiixVEk2xoN1BJR4AomgSRU7ipTy78AObrM7+GUbM +dsi3PdrEBEVC5EmBjpQ6fniL7NeNJAlF5VdMiPbQ3rhSXHv+8Ctaa0/lwb2SFwkmJm7akO8oCljw +efcVKVpUqaYRw6KIDcFpeIQbLKobpV53CmdbaF9MBKhJoU24i5WXXT+Ix97bp6+Gv+3BoH0W52iY +uQUEhu6ekuX0rirTCKZHuF7sRG5VdRtZ/wDxCendiGqo3kFUl00eGFoQ3P8AeErht15Kj02187sB +fEU00EiTU6iH8HixgvbLJIse1iFmbwIotdoN42ltCtuUK6nTrroPnjaoeWTctBtPc6RHqWSEhL8v ++WOY/wBo1qnJZgmpCPA3b5SRZxjVIQEw3QuKpnpy+LAbJ2wdozHKGWnjRMFymFUi4YSRG0ip6dVe +eMr/AGX8llO5gddpE62kRTTVL2ULvrAhLxndypr+mM37SE3Ehn1DK+aJv3iSqZTrps2E0m2gD9zQ +afJprSvcXPHQX/6pdnOR4GHyvl8150U0uGQZsAWclcI8x8VeeA15X3jgSTR27vFuIjaWn+L+mBzg +nC73qZqKIh4fcjbb6eLA7JMo+norjpbKr2H1+6auESuKlfi/pixNExELtlBsiXhSURtL/wDGAU4J +QgEU23u+nw22jiNSig1KgpbdNfD0f+3D52iGwIAJfONvhr9MOJN0doOIVQUU05lshz/pgKvk8lFz +VUFmCagpEo2BQ1hEdf4ZXjTSmJwC+GPXJjv9VvUpduj1cxpy50wJdSDx63SU3uJJUhECUDqJGnMh +U0KminLBSPTeNGQoILe5V6QcE2L3WpcwL3uAfVFwrH+Bds8EiEDELhtr8NfUa4fjGJF9rXRQFS0h +JIWxDcXxFrTCQZkTRW6xsI3dSiJ9I0/I+6uByKg2L75tbum0RWVSIS+bnrgH3AppgK6CKaZJ+7Ax +WVQO30PpxKSdPOLSUQDdTtLdEdpX8+dCuL/LEe5w2cWpuXyjhRK4myblJcSD56ajTnhi4SbjuGgT +dQvdEo2JIhKnKo3AXTWn9cBa4yQRcpGRGA+ERHqD+h0piZ8H8uK9GLcc3VUFYE9vwCR7/VTz0rQS +wdaERNxu6VLbiEfDgFbnXb4RHqLouwoyHpK/xYYcbZJXXn09XThO4NgkXhU8VuAfcJpqBtl1CXVg +c4TtcESnURl/qKnd+VfriVdtq2j4bekvl/8A8cIaKIuTVtv3E+kx/wCX+H64BIbJN+pbcISJMiIL +bvw1/wC+GnBOlHbXYM00RLqIfEJW/HTzH64mXeL8PUY/DhaRJkrb0XD8vit9MA4Y9YXdXy/zeuIb +jb4hLrtHq6Cs/wAu/E8y6Pw4iqkmVwqI7lxeGzAQTFwKvUZkn/wyAOn/AHYrr1PfkEF1D21Ej3Ac +popWkXop1f0xaXu8JpJoLbZXeKwekfm78AnCaKDhXrAhckN4rGAi5DuqQ/iwEG14mqSbYNxQUh4p +mTYRBXXvIK0r54UDdmpHtXKCIL7dyYlsmNpV+A7aV7sLcTBNIx8pe64dLpEhRG9IaFpb4ueKVlLO +0LmLMBRbaSapzyAkso2EDteI15Xjofl50wF1cOkWjRIWPFfMYC5VAhGneVNR8OuIAOnTlVBBiG4S +hWqr3pGC4V7+8qdWBj3bF17QjX/92LZJVS4dg689sqVu7/LD/tR0xkHTYTNNuyS60hWASI+/3fuu +dNO/ALVUYk0JymsCaKDrhh3GYXCXmFbCr04cBTgXDoUwPbQEVthNYwND0LQhp04Y4ckHaBDYo82F +FhDZExV10r16EPOmEx7MY+PSUXcroKLqk4VeEBCSQV52c7u/y9MAo3SjaQFpH3uZJ21EnI+6MHI9 +1fi13KUx5u+dNgERcgTe4UwdKXttoqeJI7ht/rgK92ZSP33Ky/Bqq7zoLBI0BEukkzGwvFjzdq+9 +sJJrtgFq594qwJyQk5DyWoJUPAGHr4V27qZURNOPRIiEyRFU0iEtPEFa1tuxFip5u+BUXZuouQbf +AS3uhKo6X0opbdTzxYjhWq6RIIHc12OHFIvAqN1dddK06sAnDUk5Uk2KwXDcmVywmIhQvuq0IfP+ +bATm7N8pGKrw7BBR4ncLpSOW2CVV7+jToLX88VtkNsgKd5xcgr4mrkCZkRVHUxu+7V/K7ErMUoSG +TYpPLrnhm6zpcXBX7H2ig60Cteq3XyxnPY/2mDmjKUxF9qMkbtvHKkK/EIjutgqdaCpRYdNa0r0+ +H4cBfXs5H5Fy4q9/u0ekVtt5ILqq17kaWUIVDrXlTSuMjnc0PMkuFc3yDMHeYph4LlJqp9+x1HSq +NCDWu5pt8jtwf7WG62TnCGYM2vAkW7RcigOH6mz4qhXkuPz2/wAQe6uAfYpl192n57LtEzeCCka0 +K1mwcrEHWOltdaD1269/ngCGTOynOHaQD7NGf5J1Cx8ouKhxgomKq4jyEj0Hl3Y3jKnZ/k3LcU1a +QmXmrZG7cMyRI1SK3kWteeDEYs3UcERGC/iESTvLp9MSDWETSQFsdviG5Eur6eLAIdtWqloi29yn +aXUF135YlJC3La2D6bbQ9zb03eHDUg3UXO0um23q2S6S/wBWFJDa48B3CN1+zaN3n8WAYVaqL+Hq +IS8JMwL/AA8ywpOrlMKDukNfOgtktNf9WPo7yO+oXSSnh9z1D/uwtFOhJ0qoNCL1FHl/zwFO9k7j +Rdjw32pdUiExRRISGnKleQ+PTExu1TbGQuUTURFIRO6NMBL/AEfH+WJziFRcu1UyYIdXUJcMBjdT +vLmV2EybNmRigIG2IhG4RBUbtC+YRrTAQ0nEgs0SYNHMcSbsis23JpKiPkHVXvpg+CaijcUxcuvl +L3wL2/QvXXENgK3tAVOMkRuK1UU3IKj/AJV6ta1w6qXRwyjZdQSVtAl2Y2l5W9+v64CL7LT6uJZg +O3aKSpMLbTu8N4UpqGPJM2qavwJ7he9Fs5VSC6heLSpYdVcC2aJJp8K2RHpvT3Q7tenSg1078Smj +wrBUI/fWimKAvBu8XfoemAkt2ai0vxithDZaIltKXfioVPDgoBEoFyVlvV1YQyIiAeu64viMSIfp +ywsFveql4huEREeq71wDlvw9H83iwwbVQTAkztG7q/CWF2rCJEO2RfD0fDhpJwN6qafwlcfuS/7Y +BXD7Ye7C3DAD767wlaQl8wj5a/hxOBQSDx4S4RIvuz27vDb8vy4BhJQiNXcC23wl4ht+anqP0wwk +jaZEhfb/AD+H8q+mJwC3TtH/AIf4+kdcJuTEBEQtTLp6Qu/5YBzq2hEvF8VuEqkXUJeH/EX/ACwO +OQtVFAjQtUO26+20qfniUagqAuh19Nt1wWjbXz1wA6QTH3q5AHu/CWzcVv5V78cO/tF5mkn3aLIx +t5sGrQxRsTAgAituvT07u/yx3ce2SVrYNzb8I+L+uMy7YOzGBzxDnxKJoPkklCQFFYrhOvxDbSv6 +0rgOaA7ZMwNsqOstyyISSyrNNuMin0iQcrhP1L8eAHYllOSzb2kRLaJM40RLiFV0zMSQAS1v1pXz +7sCc8ZHlss5lGJcrA9HwpKiZCJaDqVNC0rTBPsizxIZCzGM3HxpySl3CGkodo7NedRGlO8qaYDtF +2PAzo8MiabpISUVQUNUUnw28iHnbWtMQZNRvMmkPEgW6uK1yh2mx0LXqofw64g5S7RIHOzRV9ls9 +xqyt3+g+JYlUdbtKU8OJztZRo9Vkhc2oqJCNxbtr4bfS3kWAYVUTkGk0SgbbpRcW6pJgkREAc6KJ +20uw5Jk4kIol48zUtS2WwiCw8SlQeevPSheuILJaQTj1Xz4EE1I+5RmgIB039wqXFStcDknDpCPQ +UFtt2tVFJHoIeDMir1UqF3Td3YCHNvHSdqjFYF3X90YCpYXeXUioFerlXBaCJYdpdofDJqK7Krlb +dI0CHxDSuultcQ527LcU1cvgaqEggTl0k5edSqp9G8G5b82AEnIZils1rwmVI105FNJNpvpgCSQg +Y31PfCtev1wF3kM8ZbYvUkJZ5HNm5CooIEsiJidDLQuXPAKMzZE5tMUI+YQHiVxbGxTvG0R67yvr +pSug4qmcOyOaQdrzcyftJwo1TRSjE1ituoWlDVLTrGmObZhHMmX5NWQ3jjnC6pJhwhiJX09RGtef +0wHZMK+Yy3tDJahoC+kEifxxJgFhKp1tr4KaXW4wrKLHJ8tHv3c+/Xi3kI+X4tspq3SeJEqVqJqj +omZUPXvxXYftokGhtXCDBBo+ZvE3aZprHtlaNqgCnSmlx0xdf2pURm8nwXaDle9zluYK1dIQtJo4 +Itba0p6nry+bAEOzDN2X88LSPZZma9eHWIih3ghyaHXwhUx8yH0xdVe0px2ayC+Uhio4W8Paja/m +EkDVRt8YBWt35YwXKSOXYl6xEWexMKrp2MFrVV17kipRbcqVBS6vEHiHG3ZwJrm3ITybUh8uzWds +otxbvOJuVbJJFzopS2lbyHTADFf2hnhNFZlTIy4sbiEFxniSBUaFp0jUuf1wYaftCErk8syM8mOj +jWx7bhdaVtJL1rTq1PT6Y5omHHD5cXbOcvQ7aWc+8J4udV3TwVCvpVNGn3I2lrz8sPqsYtOPZsYJ +zl17JCQtjBPdJVyZBWhFQ1BEQGn/AKsB1UH7RWU49ukhmCEn4VZVIVrVmxq9Fe4/5cSm/b92XruN +v95F7i6R/s1a0tO7TXGB5oymcW+y3HlA5eczCqVq8EymCNddGo8ycOK0tAR05DrimRXGRsxPxooy +jBNcdkmcY5ScpEN1dBWXu0Cn1wHbDTPGVZtuxdxs3HKIyglwt3SatvfpSv8AXBU3xt6AndbS2laU +2fLHBEfGtyBiIg1zHFtECWkQjjJsbYSLXQlDtuL8sEY6NynJJG6mO0CQg16qVoDAk1XBIp/CNVB5 +FXTzwHfLgkWjvcXNBMi94S5ImIEXy60LH33K6pKILXCPVamsQld81taYHAoiVqHGAmoQ7hEm8MRI +bvQywg953aKDyR2WxkIriiC5CVPhry10rrgJircrxUUsLxdZIiYjr9QqOE3Ji3LrNdr0ioCZkYjp +3cq1oVP88BxcOFAJBNsxSTH3hoCCzUyG7xJlXQcE2ixEsQrhvolbwypbS+6NvrTrp+WAD5gtTNJB +8bHqVJS29Ud0KDyGvf7z0w/Dvickk5E91MhEWwkYGqOg8xr7umLAYpkySacS6QvK3pMw+vnzw+yj +yJ6q7XM/utkTvuEgu+SvxU+bADuIkuNSQbAZIiO4vaA+6O3y00xMZJuCtc8MumpaX3iJDb/u88Fg +FuhbaiFw/wCrCjcCNokZ/wCgsBW5XjEw3E0UFOq0bgMSEfTShYU3U2zFSwLbeohckI8h76UrSuLE +bpMQIusbvwFiCq8TUZELkwHc8XwiOndzr64Ac3dPHIAKJmV5WiqVq4/49KDg+BEKQ7h+8+IhC0eQ +/rgY3dMyblw2wmsXjJMwEh/PElJ8iqAknYSalw2isFpFTv0/LAJSj1hcKqqLApu23iSJF/W7DUgJ +IBxIo9Q9NxLEI/0w+ahDcuJmQlb4QEsOt3gk4Ftfa4EbiEukrfXTABVpJqmaSDkHS6aolaQgBj/W +lK64aSISMUxNBQh6htvSIR/FrUqYLTEaMk32uhD8RIiZf1pgA7FRNVJB3Zw4+7ESvQ3fw6F0lXAH +G6xEyEhO0RPrAguL+bkXdj0rJM4mHXdvjBNMEiUIhC278sVlw8GEaOncksxaIppXWFYNofmGla4w +ftI7YGso1eSEei6QZsEPdEsaoiurUtEqbZV08XxYDMO0gX072tlGtHi7twuuLQFyMeo1CtU8uXfi +29tfYf8Auy4/eDI58SzSFNNdgiZqqpHQaX6d3u6+eGP2QstupntT/eBf7SnG3EuQheIrGPLn3c+/ +HXOaBTFwTlN+u2WttIk3Ipd/cVR88Bz7+y/2fuo/L6+ZnYBFupJ1cglsqkIpCX3ZUup01r640aYb +uIsCtA1FFnQqGxTckIthp3Kp1IS11xfmjxFCPtF4gXT1qk8G0jp5VpSvLFWNw8KbfO1DBRNNJNMQ +RkrDH8SZGXd64BjbUdtHRXoLqSFok5JyBJKhTnbzGnOnngBl9jISUfLLsWdzVR4mmBuQNIeFEaCp +Sp39Y8vTAztleKZd7MppQTTXFJmVwEaSlxqFyup81PWmMo7D83ZwhJOAybNrIO4GSSEUHKhkQpDX +4aV88A7+0ApJZdmIwpts1dkquo4SapuSVQ2e8RKtaa/pjqPscnonNeQY6Zi0UGyKo2mgj0ikdOVR +rypzw1nXIcPmnK/AkCBN10veKiFxEGt1LP8AFTXAWHZw/Zrld409sAgN3ELmodypDXl+dacsAW7T +WakzCSMM2kgbFaKjxfwkklXlqNaefpjjjtDheEy5ttg2BTeJqM2ajkbyCpW7ltRvIirXW6qn6Yum +YO1bicvyKDGedEirIkntCwVAHI1PSiZrH4Nfp4cV6dzQ6YuDXm8vQ79FdcU3C8iZqAkr82zSvvCp +T1wGPzYqcaF3uyEtnYTRt2redvfX19caP2F9o0XFoyOQ88Gf7nTvTVVPxR61e5UK110/6Yoz2aUX +lXT5tY2RUXEngCijYJ9VKbIUHSg24BSCwru1VEkQTTU8IfKNOXOn1wGvZ17OEsmZ9QUmlgex7tcS +iiYPNpWSRrTQVRUMSpuXVG/F3/Z4lEYvtVQj1DfPWsw1KHkzWAOHFal9QBNQdL6UpXTw4pnY92kw +q8Gl2adosX7VywfUwclaS8atQdakNa/DjXshZByu2ze17SkM+Qj/ACwxVTcoCmAJEJAJDQbB00Pn +gMwzaz9jTs1lJMPZKjZ0omIRkUTlyulQtRGq6p17vSnw4qTduK/Zk8gpCbmxeRZE4OKbQgECVvxE +4pW6nV3/AFxrL2Jku0bND6ZUWQfw71cmDAE5Uw4NZQ613KgBeQ1540fIWT4WJhyaZLeLuU5Z+LAT +dnckg1bnQlST/CVuAweT7Jc9LysO0fxuVoseA41BVBZUeJCtLy3CqV1a8vw+LFn7MuyEnmeGK+b4 +eOUhZ2HN+q2ZOTQQZhTW3purUtdO+tca6xcOs2vppKQZoMlJmTGFaq9RAqxQG6ulfmrrin5tfOGs +rcxm2rkZaWHLisWwAiXYx6HKtEbeq+lOoq4AZlfsty64exzR8xdTjqSQcroA593wzIPBubdt1alb +1Y+9k+R8szGQo54tm4YdX3iZs0yREUqioVNNFBIu6lO+uCrJ44TcTXsebeu2+ZHiUPliTICEWyQc +1EK+VtLfF8WM17Ycms8ydoEh7IynGsvZdsc7obsU91wnSlTUpQOWlb6c68+WA6bRfOmwqsf7bQIx +EjQcMBck0qXylSvUOHYq7qTj0Ytyool71dRFVrv6fCVLeWKeqs6THbTMF1F3hNxXcrOGptv/ACi6 +i6dfPGgx4vBjxXXNdk4JAhVaovANJUacrqEY19cApwTxNuPArGKdvUkm8EiSKny6+VcCnEgiLjfc +tnW2V3VwAqiqXdcFUyraWBLuQlhbuicg+bR6CRJ8U/ZgqIl+aZDy0x5kUW5cPHaxxaCLZAViFNZZ +C3T4/FUcBeoov7PSTEwIfhtMgMf8J0pzwRVU21RJQzES8IiZdReflilQT58o3auRcunYqD0E2eIr +3fWuoULzwa450SpIyRmKIW2kszIBE/xFQ68/0wDUxnyFi8wDCSRmm66Ss2VS3Rr4Oqg201+tcFHc +oJASiBh0/ESN4/y10xmmeGL6NnWeeMttuPUYCXtEOPvBVGg89AIfHTvxYsnrZfncuK5kykYOWrtX +cVSUM7R08dnV3YC0gXFt99dHY6rvd33Fp5jW3COOInFwrIWq9NxLGQ+mmhDTA5u6auWQin/cfF1X +gaQ/NTqrityEw6d2pwjlBR9cRAYvBFJUKFzHqTr7yuAtjhZZFVVS8BTSStEiMCLn5U58+7EdJ069 +n7gguTciFQSEEiG36UvxBh3Cz4NwQfcOkXUKjMCJA6fDrrzHnhTh43TVdXbnvEulsTAhFUafxB0L +ASqqLKEIkTJdYri3SbKpDbQta30pSumPJSTxA9tA7vmBFyKto9/TQ7f6YAKuBT4VpxgCsqraDlNY +wt0/grUK7nicZEo3ErHTZNoqQ3+6MxV/QaahgLTCSRPmiBR5oLpkBEBiBiH+LpwnMbgULXK6xijt +FYArWkRelOWmAkYTxNwKhM0EFPESXUhvj849RDg/GCpIe8Jb7OmIiKBAJdX1rp5YAAlDpzKTOSkr +yRH7pJRG01R8r/yxzN+0b2d5mj5hVzEw67mJekSxigAiKStPSl2vdjtAExEOoLrR6R8Q/pgBnMm4 +tEt899S0iSQ2b7it9KVp/wA8ByX2BZ4cdnMezYzsI6Qg5R4XErleKu7UtKFSlR6gpT0x1bMPk0AF +2ma5AraILtgAejyqQHWmuOUs6tXGf+0OMy/lmKek6YPEyVMXO77q64zKlelIraaUpTHU1roWl3vy +RQHbHbZ9XL1oZVwCm6inDoJke24ckRWkYbRDTnzqNa86jirTcojFgRPeOTZgqo7XSJZI7RpzpVJS +71+HFkBR8lH742JkRdR8MApfnXn008sZF+0BnDKsW7jsuyjldRRJVNZVsmAEKQ99C6fFTz0wGV5t +lJbtYzB/Z8lKxuX36pCqCzYEiV+QaUu67dCrX0xecqdk8S0ewqEXNryjGJSInK5GRJb1eeqdKd/P +/LFGzX+7ck9SnSmDUg2F1qVhWrq9OvMCGzv/AFxfMpdsWUVAVhFwNo3SakRk2RtSSSoOpDTn0cu/ +xYC7uMxPISPKSQzICEKhamSFhHb1adWlOWvlijdrbeazXkyRmcszzGShU0hLYK1QV7br+fwlTXGI +NO1DM0FNSycO5aqRMo6JTg3CO5tbnOhCNa89MDgzpPSkYWX2zlrGxqxC2EUQ4YCGpaqFWvO3u5+m +Ahw6cauaRKImg3JISSIXIiVwDfUqHWvRWpVwSz5KrLxJi9Bom6cEmk5QTtqu5NOnPW2taAkJfXqr +gN9jbOjbQz81yXXFDg+GEwVSE61oW4X/AD0prgY72U4z3bb7Urt2rl1EI0HU7K00+L1wEN6s4ILn +bnfWuuG4yIh+tNKaYYVFGwRTO4h8Zjdb+mEGI2EQmA+HovuLnh1u3cOd3hkbhRSJQ7flp8WAmZcj +XktNNYuLZqPZBc+hvYJbpU56aVrTWnLHRvYR2Px9PZ55ohJh0+cy224aqGKTBAgG60wqVx9/y4uP +7PuScsw+So6WGNXQlm1sobpw2LfVK21MNdem6teQ4uj2cLLpyDZ2ALy0XGKSjm463KOnJWpiH5UH +AO5XynFxMPc0ZsY1x7RkH5N0UbRIRIgDWvcGlBxFhk28fmjJrRg/BdbL0Au5etRutVSLpvp611rg +J+/yJRAkig6e5fb5d2J5cQ0SbOz16OfVUr/Fz5YrQR/7vvc0sZDM5/vZGwiDbL6qP3pN1g5IV+E9 +DtHXAWPJiR25IqJIL5eeuXz1q5QO3gVfx6+nriN2eos5jMTB9Hnt5uj2cgslxN1r65wqNSv0tPp8 +6d2Bzh1IQTKamYTLfARLCJbR00xerW2rLFosSI87CprStfmw48FpFu5WJeLOXsfHtm0Pld41C0UH +CydK1Cp0+MrvFgG27zJ8S4YxCiK7iHlIBzahfVUo18Neszr/AA9a9xYsuRs8ZZylkuHh84ZffNpm +jWii+raq9VKVKtp1UGlaFWo0pz1wEcNXjYcxSy8UyZOsrw4wr6OI7ReAuH3255/TpxSs3yfbBD0h +o9nmJoTdOKR2qptrumtSrSla611rSmlPLu7sBvbt8+cuECEHRFdtoLtJVJUFQt8KiZlUe/Dhk8JI +k3ca+TbkVrls5ihMufxBtjz7sDAUWlgJyTYyUEi2gUYAok5C3ysOtL8SmTpSNbk2QBD3hEiYuWyo +GzuHWzpErqcu/AOgUeobVsoiDS4x2leDcJA5G3S2tKcrsDpNNiWTJaPmcyNYlEhuamKytyHVqGoK +V5h64hw75mu7dMXbbhk0FxUPZfqggudvIk7g78DO2BqtLZXVzAxlTFZpc4bKjJFYI0GtaCsnb318 +sBRcuZuyeLtqnm0GsTNOUuIbzsUAqtXmhW9QV1prXTGzwTN49iUH0bmaONS4C3VDMe8uXh8q4xMH +2RZmCVhs2g+y88FJO102MTYuSMdR7vXSvfpg/lLsxybGuI6Wgu1ddMVCTWXbk5EgeCP3gn1eHywG +pTWR1loCWDMkws9QfhYItEbditfQ/FX9cRf2Ymfs/Ji7NS/cbPlUwuAQ6KaacqcrsaQ9WJONFZsC +bkSEbB+EhqXi/LzxWcnt1ISdXaKAvsu7VErrdsjqVddPxYAFNsXmWc1ul2xmqzfiSiW4ssZpKkVd +wk9K+HTyxBBFqMZ7PXW3ESElHKQuQsXGni27+odMXDPEgm5BWJFyug4X+ztl0ekxPzK7T4fPHKXb +xmDPmUJ5rCN8yLv492gL1AXaIkZABeKulfiwHQ0Y6byTIXaDMxjR6W7rgw9+VOVito8tMOpN9twL +tzY2WuEgbE5cIKpH3ahQi58vLHJeR+2iciZVV7NxsdmGHdl9sakiQFb8JUrTkOnljoLJXadl/MWw +5aPzTdbu2k248hNmVvLVMw5j+VeWA0GScPlNpBj9pkF+o9t+BJLjTxaUUr4qUx6VYltJLoLL7KA2 +++jRMf5S0p1054CK5wYtJAWhTDFzIF7x0kmsiaRacq2VMxtKnfpgjl99EzqROWhoOxUIrz4MhK6n +rQa1pWuAlMkxFLcFHY2BJQGye8kQ+pJ0+WvlgrCPm8fxUlIPNtqQ9a6jkiAdPz+L1xmHaB2jR8I0 +XGC2J2SESFBg2MhVZlXoutrS4hrWnOmKKGX8zdoOWkpbP8qom1IRFqzvIWbnq5ipUepuf5jgNjnu +37s1i+JTGVXeqIEQ/ZESMVSpbyvp0/FjIe2Dt2azbjhMshIpkfu26qjbbAXHwa106ufr04Mfui3h +FUigoEPbEb769yF75AKjz5U1Byly9dcYZnXPk1JNF3cgzjmUW9ebiDFozAAfLInaRLalcn3c+WA2 +f9kKHY/uo6za+eIO5JdUkxBNYjVEaFpW6lOd1fTF67UO17KOR/7NlpV0TxVDcBq2bGSpfz7nhpXH +Ks3mqcyFmh0hlSSdRryQa/2ogQBtidedBCytR6acqYpntK5w6cyRunssSorA/JzcSRfNpXkWA1yT +z52idqvthpDOYHLkftCmux4xJInNLuVOqvj/AJeWMsk4V4KpLiEiusIiiLn70COnKo0UT5VpTFp2 +ZCZOJzFLZk9pLO91RfeZkqDFEen3tQ5ipWlOXLBaCy/ltSVdS2zl1eLuFFmKjxaMNIg5VqI2V6/M +q1wGWu3jxoyViRld9mv7xVIQtuOnffSvPliG0UIkiTIAVRFJQhSUOwRGo+Kn5Y3HOGR4UocY8Tn4 +VwLVNZq1ethkGdlSOu6LhMq1oPfrdTFNmMhtUuMXbSUc7GPFInIRRmuNhANd7W3/AP5wFbh414Nz +xSNB2KAisYCAmJXDoI1rT8+dPLD6pNU4f7YCiaaaBWJJ3g2XVqXhPX4x89Pw4GMotQnrxEXOwLIr +lTLUkr/DQLw1pTXXTDTtq3QBVcVkLSG5AbyIyKvLly0rpgFhMELjdFEy6RtVvK4Sp3fQvpriZmCU +ZKSRC2hWsOqKRJqVTvExr53p16afp04rpiJK2+IvDh9JqSgJbBm5WXK1JBMCI1dO/XAJSTUduEEU +kffEIpjthbd389Mat2FZRy7mTtaistzbAHLUUFBXJot0kdOoiMq+n4MGOx/9n/MmavZUtOouo6De +qkiJDaS6Zc9CINaVGmOlezzJbHsnyk6ThIcHLrj9gVbxVUckQ0ChVrXTb+o4CzTcKtINI6ICSuj3 +MiLlxsrEJi2R60xCvfpcI0xW+0KPWfNYxy9Uatvt6s0u4W16EkBtBHT0rUsW2RB86kpMmtlVl004 +9sSev2bUdVO/TFGmG6a6XCRLw5ZrJFwBLisQBHx6ZarK93jvwAyCasZDK8i3m2a6DMUlJybjhvHd +MyIkqDr/AA7benFYzXKFsZNicxN4tpLOf7TQnWhiRJNkAqYifxeVnpgwjF5oLMDObjX+1JPZEhbe ++/v0S15UQ/mK27FSexKcsD7NCiKjaJzvIkJ8Mjc5jGrfrM9NenWzq0wBNlNZmkHpOXbM085ZgXT9 +pxdg8OvFJalRdPX0pT/dh/8AeaDcxr6LYsJVCBzfOpezJMj2towEAU7uoLSHzxGSdSGa3sYxkIqR +TzJmMTRh5RsskkaESjpStdLuV/xD54hqrM4admFN6HdumSow8dBJgXDJAQCBuhGlLjL5vTAOx7VH +MGeJoUZVf2lJZiQQYKuLwGTatOag1qPir04NS6mR835kmXzp08iF2j0mSjZJZK0STEedPpzwOzA3 +9l5ShYqAePZBjDvCSy/NsrRVKQMap0Rt+USLmVfhw8lm7KGXboOeyvVWdZ125VROy1RzpSplzrr5 +0wF93G+0kmobFoR3CbMkVkFROvKpp2lbSuFw6nCSqQsTi3LghJvxSMqqBl3aCompfzxYYyBFKMSE +pV8wFAyWbf2qKto+mt3ViUrEpqPVXKDlBRwukIqmQNy3dPOtMBXQUWdyYx6iMoTdNcidCLlurtad +xDTb18WDBIqOyXeoIqJoubhtIEVUFdS+MaDStP8APHm+XxXMehiKyZWmqLNvfb3+VMTEoFu0t3Gy +GyXSYjGjYJU8+kfPAYfmPs/GNz7wmXXMim4ciT1WMJmBtbrdOYkXxelMBIpx2hRMgxhk8kwD9R28 +EbhZ2mkHmRj8AjjXs4ZNg52P2FzQTkBVuYP2TZUV0NC8NCGndTASVynnZKPXaKdp0qmiqI/afZR8 +SOheEdBwFszB2qZTyo9Z5Zm8yRzKSFr78bCsEre7kX+WKjm3t4yfHxSrmLmAeqJEO+QtiP6Uspd3 +fXBPLnZTktSMtfM46aklPeOZNbeJUir8+vg/lxXnH7PvZm5kN9BHhritJDiXCSRF52lWuAfjO1SN +mYwVCB1tkKYkkQCJFryBKnmF5eIsYZ+0tLN5bPqUPlt+g76dl4qif8YyoO3rXwpjrpTSuBna7leW +7O84Euxm+LR3SJqSLm5XaEdLSrrqWg8sVbs5j3WYsykXsEJFmQkifRtpCZgVA+l2tfPAezQKKb2O +TEweikgmjJs4zwCYEVKdfdzuwhxl2ej99ZaBXbLXEInxNpoBQdyvPypVMhxenr5EjSj3MlHQTNyS +aJtmACRiksJDXSvdVShtx10+fEyQRZzcqzhG0VIuyJBJy+VcrEkIk36V6VofxVSEBwGTTcMoi0Xe +iKI7a4p1ETIlSM6X86/Fbpprg12aZyzNBSDptDTxsE3KRe9WvMUir00KmhUsrz01xq+aHzFskWX3 +LZi9JOHGJJVkYkZGoO4ianyaENA5/NjHcrxu7mhncBshUMbEBDirQp86ddTNO5Pn6YDcOzRNjDO3 +xJooKOhJJRVq/WudLkI0uVZOaeIty/UK40Z7m4W0Yk+bLG/uVJuq+FsIgqVDsseofBS6um5TGTSq +y0M9YxLZm1i/aCqb8mbtYBSfXnrvtFf/AA31HpxufZ+xh42Q9koLA/lhako8NZEQkVUT53KDXk4p ++LqwFbeymfvaboU8sAnHtvdpNifikbYrdaG1cW9YFr4SxzvnXL+ZI126zM59lcUkqIvEiMLxSUHT +3iPOlda95jXG+dsuSZ7NbJ0ojmpjCw6g/wDzJhHLkPcCievu1PzxzhP5fy+0Vaw0PmF1IzYobNjo +w4MdeeqKlOXKvngKjsuLCT3gUtEVCK/15aa151LBXK+XXkzvu7DQh2FvtN+IXi2AuVK1p54smX+y +uWdsvaksDpNmLVJ+SrSxURaVKmpfnTGjZHy3FyjKMF2TLbRVOOkWTJYox+Il1N11R6dz9cBJ7H8k +wPBCmq2ay3GkqwIhWVbLktdUm6hBdb1jUNNaYub/ACyyWn2LsmLpszIVHLKPk2YEgnICFRVQUUrT +uUpdgnGR/wDZls3x0pIKEhGOR6UHzZUC0buK1pp8NupYNSTpxKNVY1u8jicSlrd7Ey5+8QkAG4Kh +9K24DJ89rReRTdT2yvBOJJLiIl02WNdiuPKizJVK7QfpjOZicarsmMfHsGMOiW6/XVYLbAuWVRvJ +Dc5jUrqlTmOLX2kSxLzSUAhJOsqx7ISJzHSLO9ik+AeoAOtNOuleWMHcCK6S8goANk1CUJqkmBEF +1S1qI1+HT64B+VdKIGTRia7SPJIREbLCco18BKUp01LT6YgmoS6pKEdpfD8o6fTyx5K5D3lgESvu +QBQLum3S4a+tMWfsvyS+z1mVrl2JWQTeK9RG4uEREe8dfWtMBEyFk+Yz1m1jleEBEnTu4h3j0AQp +3kX5Y7M7GuyNnkuRy8+JmnGzqrddJ6isjRcOXdtH8PPqw32UdmMHkn2dmRFgg7e0kybqul7gNsld +bSgU/mxrPGf3xBis6TUiVd9QVEamSiNR1tCnf+WAQxW/shjKe4lnTZUhVVZdA866EWnxF4cNyDNw +LSTJP3CzR1xIiytM1xt8VRr8WI6ScW9YrwSPu0ZZInbBJFEkLRoVKkN3z3Yr2Zc3IoO2eZLHQoxZ +cPOt2R3Ls9O5Q6U5kl64AxlREVz2FJUHMogahDcZCSorc7lA8unAfPDyJj5VjDMXhsnUoJQ7VBEA +2hAupVxX8tNNcVHtrzY6guz95M5dkoQnQvCYEq5RsV2lPB1U7ra89ccoXN1pOVLMWZJFzKMmYptV +WixOTcuCLUhEi15UH5eWA6NzpmrIeW8iTSbSadFwy4xOX0kD3HKAB94oPnpU7tcNuM4QMzl11nRs +8h2QzTwYdVqssQWsg++UEKF0GQ0xzetGx7kJCbgeEg04ZqgtsOn9CdOVa0pzRqNddfit+HDrLJst +JBKyTZG5jCtUnr5eVAhItwh6qJ+Ihrry+mA6BdosSyFO5oGeapov0k2WXWvEikUQ1qVaVWurS+6u +3gV2VZ6iZSS9sNm7V3N5SjnKEYigjti+bWaGssZa2mXixgjd4pPCvJLM3eYZZBdN65cXltC2DkQE +Ff8ABg2bNnKRM7MNHy45sdr7iELCI3ICy8St9Q5VGg4DpKEgY8v3Zy+pKoKwbRgvmtd8hcKSCxDX +poQlTpuK+mtPhxKyoz7O3eWmEp2hMWimY5BMnLpRzXQ1KVMqAX5VARxyzJZmzMmq8KWmHTIphJsg +5jhbVTJWP0upbpTQE6DTw0w52lTMzmzNKkzlaKzA3hTRSSZhUSPoAaDrr+dK4D9ApCWJiyLaYOnq +hdSQJswC4fUNS0rpiuhmBRR0cWg/fETAvtS/sfrQ6dbS1rS6ldfhwFcIpuZB45TCHdsUepVqUqtc +JULkaVdeQ/TFdZTDOJZcSQLisuvw7V40myJQdda2qApStMBdWWbCd/aX1jISVJFsKkOZJPPTQh1I +f8sSm8wzT+1oItd63qFQFmyo6d9eQldSlcU521kBSHL/ALKn2zh2qLm1OVAkHOo8ttTTo/TuwWXW +nRcKtiRznwrUwHifcmq0ER500s1MK4C6qyDdy0QIuolREhtcpEJF5UG4hLX9MPgoo7aAgu2Xeoiu +PSVwqj9dNNP64zSkhHrv15Jw+uV2hG+RhCSQIOeg1qNelX8sfcvyDcW68g2coKOLiEUE3izZygNC +58lCOh0p/wAsBa8xqFDuFU09gXRKiQkW7db+KohWluH1U1CbquU1uGLxEk2kiEVfOpDcHf8ATDTR +wT5uKC7ldRZyIqCScqBDbT01D/OmDG2m7DYvejcXSSeyuIlTurXppgMK/a6h2st2coTblF9vMVd5 +i6TciokQmPvEz7rK4yPs3i4Fj2eJTLk5h64epL77NtcIbyJXgOunSW2JV1xrn7UBFIJZdyTIOV4l +u5kUydOiAUklUSOy7l08tNe7GV51ecXxOWxeTD9wgWyDZojsAubfpqXLuubbutcBSMvqAUw2YkjH +QqTtW0V1A3XHWV4FStNddCT0/wAWLRHqSzTtIdZiXigcyyAlIuuPeCkKoh/eAER18VbqaYp+Wk48 +p15KO3nCNU1U0RQ+/dJDXWoKBbpQbTAf9WFuH3t2VkXyhxyDwRUckq5c3kSwD7wenT7wufy4A/Hv +Nr2jHtph1JRMgJNBZsg9+KtfeJjQq/CKop0rprgJ2SIqFnJdBjxzZ8kNzVdMBJdJahiFp6102yuO +hYH5aWJpxX9pOkEbVEySaBcrbQrxKlfl3KBrUbcK7N3gx+ZUnLnikyXVUTMiuFBfUhrVI606uf5+ +K3AbBmBYWPaEu8QRdRqkA63HTUmwqoRBVAQuOl1dxuoVPLw4tv7OrhTM2aprMzaN2ExXJRy2HxNt +OSazMu+o28iHFEzFmx9GlxUWunuHus2b9yFoKpU6jYOhrrU1OfKuNA/Z49m5bhH0zvPmEeO0JKrg +QlFLEOpAQ+betcBG/aVzYo2cEhGrINBXSJQ3hdTCVAdKVTrSn8al+KB2aN27Ro6Fszjm0kyQF7w0 +rdYuSB1WBIFCGmlwVHp88Fs+PNJ/jXzYHca7XEn8ZZ0lvXgLhkPwhWtBxZM0Q7okozJqEkg2cEIk +8GcRvJJ23SHh6XjbbQxoA4CoquHDRw1YyEIu2br7sm6dZfWuAopyNtR56fdEY91MXfs4bt5dornS +YRa5pRjx9mLk3bbTwUaF9ncc/FpjM4SQKC35RMJSHlFXigsSaBxjYmv/AIlqVK8h07+WNpaM3CLS +MaOVvaaKTUl20xCmIE8ji+83EqfEkWAkPXQiAuZtZeWRXL2TLNlLUpBsdVSq2W1u59NRxTMwZsWY +wjopQ/a0optxzmOeo8LIC48Ld6Ned5YBSeYo+Seus3yxxeZYWHEYU3LTdSeKjX+7uqhd7whrXTux +nmaFn0obGSlJv94czSCXALtnNwPmKqJe75fXAO9pco6bQ6GW1ZibXkl7VJ2MmkRJVs6HWgkB+hCX +PFFNwomqJCz2G4qipwal2wqQ9J6curu54Ovvbk3uqLzwSUo7AnEik7MRXSNDptur9K8vXEFJvJKR +6DGQW2BTtKOZufEqKneSf4fPvwEGQInyqTZMzQUVX22bMjHYSAy+f0u79cdr9kmXW/Zdk9dRys64 +qNeJrvBRWFcVxMRpdTlcKfP/AG45O7KomPlM4MfaUachDpltyZKARglr4CrQK07yx2Tm6Sh04JnP +Ciu7y3LsxinhRAEa/LUU+XoPPAEc2yEXCCUI9mzGPn19yMdJ9RtnNSvp/h53UxVp7tBl2GRoLNAt +XUhLtnCLBddvaSbsqlQVRVTpXWnTqeAvahFtXfZ6/aSje5jlkmhQuYm95GkFo6b4UrQqVDzLAU8v +i+zxNRb7NQNnDtdKYi5ODAuFuT6bFh50Cta1wF0zlmRGLOai5J++lYtyz41g4ikfexi1fhM6V6aa +088AJWJmJdWMc5sc8W4ncuiizmIY6bguEhqr9oSpXQtRIe6pYlzEg0lX2Xczt1nuUnk4t7HeOxQB +WOkkqULmr8I6lTliM4y2xyFlxBo5eKQE5HyhLDNtDM2LYFz01KhVrZ0aUtwEbMCeYMwZMSzt/Zyj +p/lhRo8S4Yb1VrqAIop61tP1+XGJw8C+Ty+8mYsGLB5lUuLfSah7rpUlx29nQ7R93p/vxsTLNGYB +ZDFwkOcopkiT2ym2RgQKokXLdCtPFXvrjPc1SUPlXtNGJ7Tma7tEnjmYdNmH3BGuAcONRpzOmoc+ +eAo80WRYn2EtEwq889YoEU05WWIW6rg/uu6lcMrs4mUCFfPsyS87NS6SqktHNek0ACmoCVSrSnLT +E6cdZjdrA9zB7Ljsv5hoMsuzRAU/ctjqFltKUK/o8OEPVE5ZaThcrZGWi/az8ZFs+vO9CPoNaV59 +9vO6tcBEn2843gIzMDSUQbp5qQUYKRsZpv7KZU0Egp6+eD2SW2didx9MloxeXRzKkWXyoJ9R7I+9 +VurTp566liI3DKGVo6dpAS9ZPNNJFJpCl4bUbfen+HW7TAIo9mWT3j9bMjptKRcmLCLiU1qmaomV +yxCdNPMsBbcrzUSxkJ+GY5Y9pSU1Gex2JktxXDOKdBERlbSgkPUOnhwf7LU89SOTmwU7Vo/L6DEi +Zt2pElStQTrpd1VpWutalT9MZ+oxdJuJvNGR33sGHi3gNEkH61AWBRYbS6dK92terHyQkMt5Wl30 +MyZ5WnWyS1yb18moZqaiNS0rSult12mA7DNN5HtEsuj7bXTQuWXFaHFddtqWoEnURqJjSvpgPmBm +3lJVBd2jCKN0iTUGRe5eVDfGnKqahCGlOde/BBW5o99qWIOUzuJs5aT1y7ET9aENOnCTmJCLV4Rd +nMP5J2kVzZvMImk5C74btOfPAAHZQqcg8kpuEgItq2XHhWqm8kSpAFK7iJ/T5aYkwjNP2Oq9YzDX +7SqNkxHTxJLoalcKawKFTXniwcGsukqJHnMmLa29JZZFVdidPi7/AE7sfFW71eTWkicSiZCIi0X9 +ipCguJDySOgnW7TzrgIgN5InfsYQmBcOUBXcoNJ5IgXG7QySKpa3euJKQvnzhBym5zEMfH7ggus2 +SeGloOm2ppqZDy8WAcemom3XjVNgnjkRJditCGkqzCpFqaZUKtCHl3a4NOPZ6IthjVoQniZdbwQc +IJK6d4LDbWndgHeMkkDVc8HuLLq9LUoEiSc6fxk6iOtC/PFhb8PxYkojHDtDcuqpFKpXF/NSnI8V +ZwnEuXvu20cm1SH+6+0lkhJXvGqNahp+uuJDJ0QsiUiXi/FOfdgqUr1JFT/jCYUr+umAxDtozEjJ +dpEnIJv0EGrRrwy4E2NURSIdFNsa06NB6q/NjK2TrMk+9VHem3LwldlquTmwCWt5VrrW2nuNylLc +aD2sSikhnV9BSG+yJVUbVeJCwnVtEwUqpTuDp0trTnjNcryTFtmiRTlDAWLv7ETp2ZEbYiL77p77 +aYATGL1bTSbdoxRXBYT3UB0I7CLQx3S7q0s8WJSojHZ0S99FIikKayRJt6rp6UGhpjSg0rW6vdcO +JKrVEnHtBi/aqMRdWr3e6bX07+impbZ0AefrgZm4kyV4hoZizVMVG6QtiSFIi6zS1r5BrywHm6hI +OF10DkXqKqRCBCj0Kj3q9/OlKV54GR6zy+1s5feLcEEzLxVIaUrWtPX1xYnyLcR4lRm63thJFwo5 +eD0u1OtRYaU52EFD8sV+0mL0hQcmg4Iegkwt7++mtfywGsnAySm++Js6fouUFU3gSqxIARgiPQI1 +57g/Cf642Psqdey4p+io8azDN2km2YvnVojIDZpwLsK/dq0+b4sc3qziLmPXkBB9LS3UJryPS3sF +uFKGNNdKKhp3fFTGrZKzV+7qpMc5Ngko2WYIO3XAWkC6VdKcYGuhbwVr5DgLRIMY9lmhq8LLAPYt +sRsF2r1baeR5kI6CN3eI1rqGBWcPbhOUsly0q19sTqqbB+3mg90SVOTVwmqPiIvPFuSnC9poCJx2 +bCbMFVHL7wjJxVSGlNSry309e/GaZ9nEcr8Y2Xee2m67UWiDCaO41WKhVNFVJYdbKpVPTATGUG6T +zQ1IY11khNgqJbROQOOKVQHw6a9AKjQufnhrMEx7GhUF20bw00/JWRy2/wAvgRilcVjhsaXy+fpi +hOMzoRPDskMxyUS8oahzKVnFIKukepuv1acjrQaVwGLNUtLO5HNrlm+bzSio+y3EZ0oIOP4nR+Mf +TAJza42XCEWo8ayLWLX3OPadLwt3romXrtlX9K4qzh06dqqvnyxvXSokQqEt1pFTuxMbuikm6SCs +Ugu8VdKkR+A1yW5ac+XIu6uGFkXDSNFpwaI75itvuQtVuDpMB58g19cARkBg+I/styuqzMUCQ40N +pVU6iV4606dKV88QpBRQjHccm5ERtDr3REaF4Rr8uLY3kvZKAIRbUBZlHJXk6bA5tCp13FKlSvRT +ny88V0eFbTcj7Jcg/R3VU25KI2gqlXuKnp9MBpX7N6hITGZE05hdg4SYC9bG2WEhVNEqHQSGvjHl +jeLJRGCmBWkv3cTlzTmI542MQbLq23KN6VpzDWzz/FjmHsceN4ftCh5CQeMWylyZMzco7qCtxUEk +ldK9PKv+rG/59mE4KTfISBuk0Y+fQdoZbegRpPEVAqFKNFadPVUi6a4A1O5kWzQ4gnbZFTKo5mYq +tnKoo8S1eOiDRMDMdR0qXLXxYEQgyWX5PIexsZRlnoqxM1J7KRtnJgNSpSpV+OpiPixAcC4HImZk +xZh+7sPmJJwOXn5iLpijS2qg6ULw693PDqrhQssZ1yc0bIv2MS+SlIyJkUSSdEjUqKKbR879K8qY +CBmuPfQ3ZnP5eeNnXtKDmE13oogarGZGp3cw/g9+Ck6pmb94HMIKzXKMHmyEJZIXv25qurbzAj51 +R0HFhZTyjTtA/d/Ksr7DFeCJ4ULNh7pVxXTp3C5Dp81LsDoThUHGSJIkTym8FVdF+T1EVY5yag21 +06rdCLuPXAAIx06j8ykxUikMoxecWqTYXzY0nMe5cIh7ylacxompSleunVgZnvslmJvMTV3lvY3H +48RIvCMV0A2PuxbnX1592Dcgj+6mZZFynMBDx6UiJTCorA8jBEuVABCuhBXnz0xVu2ftOfZOWLL+ +T34Nm7Yk1EnEc/FdmSRcxGg94FzU19MBTc0Ry6E4zRztFryjuj1N4s4aHurizMddqiQdA3V/33YI +TD6ed5Ris7DmAIKJeoFltmyaGRuSYpczI/P4fLniY3F1m+elY3LL+Oyr7QgN6VQTWE2zkadd4HWu +t3Pn8pYocetAxuWpbjo2RZZiUJsUOQnupII/xSqfdrXAWTPcejKGxzBknJgQuW5BgMcz9yO+5JMr +lnFNPBWlKeIsR0pDJLGZnmuQ8vTWYTfQ4NIp4sFSUFyQ6qr2fMOvl54EKzTeSaRkNJP5TM4x5KIx +0S0MgSED51PcpS4i9enGqZSedpTRxlYWmQzjW8ag5btn1gG5JqQ8ysu8XPxYCi5Fjch/vBIscySr +rMyyrBMWrBgiQC5kFBtoHT4qhr48VrMyT5F2gxcFGRq7JCjdVAUTcHQhIvvCpStKHpWlK08tMaOB +M8s/u9JZL9lu57LMiqnJuXrbbN46cFYnTbrWlS8XfTw4nNx7JYWUmG3bQo6d5xVkFF3SzQ67ZCYj +UdLa6cudP0wGuKqChNryintHeepE3bLrZeAmznpuHWzTQtKaYVKy0a0AicsIddRTbI2ZRSqTlnr0 +kQaF5fTEFWPdOXqUbBAaiLYSKTim2aritoNdCT1PlWpYfaFMCDWQcts4D0kjFPhnkVdoq/w1dD0L +n5lgGnsxEoSrWEYv4FZqVpFKbzpK4qcqAp161/zxKCeatlXke0Wjk49ICJdqnNmW6NedNute7XHy +MTzZDuV2BLZi42UAVXDNyi3XQVqPItsufV8VuPswM9Ht9tocw/jUVRJUnOXgI2fT4k+nw09MBLSz +ADY2uYCRevUySFNur7bAjZiWtLVKWeHWmFqyE4g7SiSZzacw7SJwaCazdVBcPirrZyLTuwFcOhey +ass59iEi2ajtG9y2qCDnUtLS6PFr6YFm4btAdSC+W8tMnDkdvg1GbgVxO7TcR1HX/LAXBxCyD7g4 +ti2zEUS2IiXFQG97Y6lr0UIK6j64nyrgSMpbhnwuCLgmrrg0bS/CtSo8qcsZzIJsWyQxZOYTe3QT +VmE+ItEhLWgLcvp59+CMgMWM3aoGT2BIKpkK5IuCZvLh8VSoNl1f88BVO2XsXzBMoFmiFZqLTF32 +xi/NIAIBLWlUraUvprz54wfMEwjJH7Pkoq2aL7O5VTtARVDpT0HT079fFjrmKeRqkZIoKMMrSQq3 +DwDdsqqrf57dbeY6YfZQMfKSaCacCxU4JL3qrSBEDEq+FMtwacsBwsKaiZrjvdKfjHwj4vT9MWmM +i56ZS4ZSBkZLc8KiIGZIFUtaFprbppjrbL/ZjklebSevssQ6klvq8UKiwqhcXKmqSetpaeXdjbMu +Q8XCMhj4Zgg0appDdw1oiRUHTupgPzwkMn5mUh0F2mT3TZ0kuqJqkiRXCXLnUq1w7lTshzxmRVqK +cDIpt1PdkuQFakXrrXH6ImLMVeqwiIbSut8PrzwFdvBbKkV/DJ/GIgj1D5FgPz87QMt5iydmBLLu +YpUEiFcXAfZvddIUDd5/y24C5XfQLFwTl88kRkGzW6OVQ6gQdAXKlde9M8dS/tYKZTd5KJOWcoFM +L/3EhBIlyOncPRztr544/epuG32ZREB96XWPVu6eh/FpgLFIZ0mJaH9kqbDZvvquQBqsSQiR2XjX +8FbfDiLOzDPi0PZvFLs21nDIPwuFKlQpen81vlTnit/BcOJyRLPnou11kFFE+ohWP4RHkPVy+mmA +eScLGyXUbImInci/dKdfiOlac64fVlH14rtlgYNytUBJsZWoaDZXQfIipiKrIJ7Qi0RNp0+8AViI +FSEumtRr6YigoRKkSl5EoJdXxXYAjKyTqQcM0yWNZuxSFBsSiIgQpUK6mttKeuHQFaQcWoXuXjlV +NMWwgV6pl3cyrXTA5kLh2rsJ76rhQSTSAQvIit8OLXlqHFSMeO5JHaTZLtnLqTJYd1BIh+6Ee+4v +pgBSRNWaRfaXrZbhyFdLZ92q4A6e50+WnmXdh+6UzA3eSSbACFV0ncCIW9Zcuke7EVuIyQLlGooE +4SarkrvWiWzQx0PWvTfzwYbtYeNy+845hNoShPGiiSpBYLZvaNaqV0+f4cAFJ83hHsY+i0VxfNhL +juJREhJW7mOleVeWOqnqjfMTeTbNv7UazWVknqCEi5sEVUb/AO6lXuqNfixyS4TubpOxeAv94RId +RbAVLTXn82N67GnEpmjskKClIr2s3glSdteJbFdwVeSopqVppWlO/TAFuzpixksuvGwt32cBzJl0 +Xj5LwPkHCI2VoB/xPB4a9+CeXJaNzJnDJ6c68DNyhZbUbjwVyD9odCGhjUKF1HSnljzSNFePkSyL +MLuYPLLpB7HKkCoyKDUwAlE076aKDXXSuuD8bkWPXjJqZerLryEbKJSscbREGb4gr4xPXTTv5lTA +AuzprOSErCy04f72Novi4VzDPURSfM0SrcJqXa1ryHywWzbnDsvjcqHE+23T+BQfisvlh3aJilQu +Yp6jdaJfDriozXadNPms6xhMsLy0jv8AEnJulhB4KNO/bpTn+WmMwtZ5kz3HfY805jTlECHbfvwS +dEdvIQOlfDb83iwE/tDksvvpqRyyhnBCRy6RcawkWsbU1xVLvTUoNaXUEddSrgBk94WW2SuYIRtx +Lxirw66i8UK7HaLXqLXwF3Yiw7GacxMq+bMFxdQlopq8eKSrEKd47d1x0t86YPum76GyvGJMniDR +bMKW88eJzBKtnwU1rtLBr7uuAhw8HMSVkWTORaSUy+J22YiwERVRMfeKJqU500+X0xbofsb7VM1p +IZfs4CFZJFsG/tAUgvpqNdOZFrjeuwfIMDl/KrHOM62XXkE0AdM1yuV2hMeQI0r+HlSmLa4zMzUA +UYmEdPZK3ebwSYWigZeFRwfhDXvrrgAHZ12R9n/ZvDoZqYm6UkBSFMnjm0jVKvIhANOm4saek3ap +gMo5A2zzhR+MbmKVvhDlypjOWWallXq6Aj+9WZWyopvHg+7i4oq940KvLpp3+uCj3MzqfdIM8uma +yKCoDIyyaIiTmtC+5R9da95YCH2gdmMHnhJdyoi1bSnGEXEpo3CIiWtDPTQqVqGONu1ns+Xgc8vm +yKEo9ZrVos0Xq1JSppV5Urrr60L9NMfoI3XEW679K9Nsklc+e7Ni65o/h+LliUlFNn1KvFxRMV9F +ERJtzBOtOQ9351/XAc4KvGKEUggo/hFuLfbiqS0Csg6Yifg8OvL6YNRieX3b5JoLbJNW6DYhVETc +bTvSvK2tR92VOeuJTdOYFIZltmSY2XLoRjnnEt3JFdztPVPX9a4RBNZxoy9ksUZsX3FEtJoOY1uY +9WupI9HdXANmOWXbRV65hMrOWaH3V0qsKrYu4j8OttPpgg3j30Tlwtixyo7K0XyeYbiEO8KKVqGv +08OJUhHvHMm2T490KLZIrn37tpCCRDzqKnT50xOVUkrFXYo2qCltpAnAgKDz0KuvxeWArIfvI0ky +hkwP2SkhvOmaOYQVVV1+JO4KD/XA5xIE+SdTMstnMYOPIeFdFJN+JbFQdK0qFNbqa4LN2cgLImy4 +bjp+JEbN3l4RXQC6nhIK0rbTHnELJC4axK4MXe0qKhOW0CYqCdC1DcGp6FSlMBHis1QLFw+kpLMM +3x0jtJtSIESQc/L0UrprX64ek3m1lr2WutmVRR2vuJMHINSP/wC2d+mnpiTKotUHCssu24Zumqm2 +SQTYNwbKn4daiYkQ1wYgOzdZ26Xkm0ko33xK1s7Z3igRd9lfl/LAAIwVE3DVsL/MUkiySIiSUNFq +5E7uQ7lCrfph2PGWeZt4ko18om/Q2Wy8m8V3UtPhrYnbbjSIrJsShFey5BsDsU7bjURtK6n4/FSn +pixshUTZC3HpTQLpAjIitwETLUe4Yx+w5Bim6ErjVZdN3prrTXBVuoV/u/eD8REf/wCMeNNbiPdg +FvxdHw4C59zdA5Jy/wC1Jl+1ZCRbaQqHbcfpywBN3am3VUc2J9f8Y7REfm/7Ywft77dMt5UinkFl +8wfzSokmBtDEhQP8Rf8AbGNdrfbpmbPCLlWCWOJjQZi2ftVFh96VVR0IK1pz1+mMphGbds3XknzA +1xds1eBBPq97Qg6i86efVgDUw3fOeGls1LBLPljbEKDsyBVdIiurRLXw059RYgoFFCs8vP2SzVB8 +idzbfa601NNFLnrdrTv9cWY0XE2ynXb6eZSyiUSzJAVkS3VxHuRR8wGnd+LA9pl+YUzR7ETZrxb5 +R4V4uzE2LNJYCpXWtaeLngIGc4FjHhwblgEXLHwzlO1YlEEm5BXWqqlKeKpUp5YoppkPUQeLqEv8 +WmNVPJok0SUdrOmzM4xTbXJapjLum6vhEfIba4gTuVXQtBdiwXF4kgD8WyZiSAtFOZWU8Vw93fgK +GbNZOMGQJseyuqSaCvTaWneNfQqYYBMt3bs6h+H5S+uDRpyGW3aqCgfZVCFbach0rh5KDTEOQ4N2 +quszPgk9q4m6nivqXhCtO/ASsqPvZc2lLKIggmJKJgQhdadvlz5fzYdm5BZ8bVGbBASFq2TA23SC +QXU6lPmU078QFfajaQJfxOEEvEICQiNAp5d3Lzw1vfZ9hC9sQkmRJEF9x0Hxa+VPpgLVGuIeHNmU +lCBmGPTSckyTTebZKGVvvlNBrWg0qPIcCfbkwpHrrqTy6lzxJbaU6iVOgjShV5fBgOCniU+7U6iE +hDqIq/DyxaOyrIcpnrMCUXG/ZkSuFd4QXiloPdpy54B3svyXIZ6zGQqAunHkW4+eJtrtrq1rp9cd +rNI2DhMvw4sWz1BNgQpgkiz2isqNtSPnXWldcTsg5Dj8r5cSho+NBNNMbVSEFR3T8y8Xng29TTE/ +cHcNvSF5kQjXl1dXdgMj7bZLL+VJOMkk4tkLiLFRsmXH7S6iRDckfSNaaUKuMbb5+zJn2Pmoly2a +yxKx1zM1HgoOUjEqU8ddKH/LjSO3rs0eyT2MkYiHZKVaIW+8BUkypdeSapVPwVuxhKrMouP2pRaO +FGJV4lJiszK1cy5ElRelaVLTXlgI2SVGse6Z5km4prMRLBXhnzEnmw6VMrhpTlXqHXDsJDqzaM62 +iYqIYrR6nugUf7D5OlSqVoqV5KaeHy6cSwJvCNEkF4Fk/LNDW09yNVA4y4unZO7qLl54Hyb7jp2H +uABbwtqLp8jFWroJUOoe+GuoqcvpgLHOw8lG5fytmhRbKzlvbsk+bPLzVEipW50jTqP5K4v/AGP9 +n/HTbXPso5jkItgqSjX2UwMhkyPlUNs/CI6emKbkLIcfnTOEwSh7EHHvE1imGzYWYiZF/wAM6F5d +w46Imm5Rskll85J6yjbLQbNgEpGXoI/wRGlNoK69ReeAUWYM0T+bXgxsWCbOFMUWJJrDwLEqhS41 +y7jUp5CPSPrhcwxfPY8WSaMiMKo6tLhFhSfTzmveVS+BCmGcsCDU0GkkzNZ6venTLrJaxlGpVKtb +3KnmVvfdj7K5iTkJ1WSQcGhGtEuCKWEOi6v/AIdgn/Er6n5YATIZZnHMOllltEx1rZUVCjG7kgjG +ZXV63a3iWOnyYvWS4OLy62XzNMZgB8T8dk3BaoJICHKiLVH6lit8Uqmm1QUy86KTL3kTlNNb3hF5 +Onp/154cjmLpzmBjISrxjPvodUl5KRdnawjTr/CQTpoJGNPPAXqBRmpucOdlHBobqRDEw5XACaWv +NZTlzMv6YsbOOQvcGUu/MjWqRUFwIiFdKUtpT05YqXtQZlkC3tVeJy+ouHDPCP7U+K7XopXwp17u +7FoZLSJUV32LFjopWgJ1UEiqOlNCKvrXAY81hyeSrqUdsItYRLbBq4gSFquN3QVwjh2dFq0ZE5Jm +xJ4oW2ZDGuAXZiXnrQLraaU0xbmmT5BtFJM1odld1EXCPzQAi8iIKiWnP64aOBzQ2VSubSKjdsO5 +sJzAkV1O4R1T6+/zwFKjLVEmKe9DruFCIVVUXjhAlypypuh82nzYOyDxi2aIRbT92kEdpNZRs5kj +FIhoVKdOnT3+uDUfH5gUk0lXLbMSYgO8ZXo+KvwkPnh1JrKIKkmUVNrtV7uhZsir3/W6lmArLJrF +rSZPiZwim1btF7buXEq/8Ot3hwVBu3eq8W7YRyjpPcTQBY1lXP16qUrdyxKDLc4+ZcIoEi0R/FtJ +GI/Qqa4Iw+VyX2l5s3xOGCv2beciqRB566UHvwEDK+V49y3SUUbAmIqkQkLDaMht0tKhUofd54vk +ZFsY1kLZszaotx8ACFo3fNhYEJOBuMC2/hs8P9cSXCyKDcl3KwIopjcRKdIiP64BsGqI22+7EfDb +04CZwzhlfKrdJXMkq1ZCors3KdQ3/Wvl+uMP7bf2gCi3CENkB+1XcLlauvZeaWpWXJ07j7vXHOeb +fbE7Ie0p2YMnkhxKxGpcW6QiVBCiX8MqeHAbr2n/ALSBOUfZ+Q2N9FW5bij3pMeu2m2P8TX6YwqT +zVOPpvjp+SueJCuoJubrBKgWW0T8QFyx5WLko8yTFyDSUTdNNpmXU8G0CrQkuWnnzw1KyCbF2l7Q +bNXMhIEqLxUjuVLUq6mY91K4CrsmrVeSSduXMdw5Ok0yS6hEurxfgH1wfgWbVs8QbvtlNFzxjUhY +LbpuR6SEbKfw/rg7GZJzZmtJ5+6jNr7PX94e5YBkKfeI869NK4fylBvhmGcpEgBKKLk0YbgAIKnQ +feCfV0/T5sAdaTUe5jF49oaDuWd5fSZXqAIixMfgS+ZSvzYGLTzt8/22MPwUcLhBw3YqBfVy+Qro +e8VfWl3PBplkFu2cbjQAbJyjwmvtXZtOKc+SNKXcta4s2X+zNGQSJzKNuGar7kU6atnJBwL2pa0X +rqOtplTAQ4x9JL5rGZTZxyEwqkT+FZtFr2ot6jY5Tsryv88QTZtWjRr7lrJCwLi4VsSI3yqJnWpp +qUDyArun0xO/cPgpj7cC6cpvpsnD5Y7eDdpiRUUpp8JjbTXEbKgxvtNrLQz/AG5Rd4rtIEHTGSNC +qNe+v3a1KcqeHAVHtgyaOXYJi7UeLu30gkLtjt3ihwneTbq77O//AA4yllBvnaW8j8TVR2l0F1BQ +qUqNMbj2zuCmYVrleJWUXYOTN2wF17so94A+/QEvTv6cUd7CxKcYl7LeLprPYziYzbPoScCWi6P+ +L0wFGjykHYKimG41ZDvKpdNgjbpdZXl5YgKrEoBXABLKdRFf4vp6UwYZOIlSFdIqImMom6FZILOg +kf4iZUwHIRILrwtUIrh+L9aeWA1LsJ7I1s9SyTmQ308vp/emmsKSpfldWmOx4Ls/g8utEm0TDhti +kmIri2b320HlWpeeAH7N7MlOw+AklLxJVAi6Tuu0LTutxp+yRJCShmoXhH4em3lgBMeooQKioYEp +4blASErfXDCTVuglamG2REVpfZ+r/diUyRZiqruHt7YkJeL/ANuFAioNxJublPh6y8/8GAGO2qKa +SqCh9KgkJCoCJCQenf3YBPct5ZdtxbO42LdtytIgUbN7breVdK4siorJ3Fx7obRtG4yL/wDp4fbv +FkwFteBCI9RqGV3h/wDp4DDO1XsxlJSHVGGzCC8agO41jHLkEmzMR8k6J186+WMuyZ2eyUpMR0yM +r7YRkFfZ0mLB+YvGPwkoY16yHHWM7KLR7QXMe2XfqFamkzZHcapV/nClMZTKs5RDMq8s7W/dxw7X +QEkhfibldKpaWLogGlOX4sBcm7f3oxOXZVDjGyWzNZtdoiRFp3UD4VFfrTw4nTEe3jWDh+pmF1E+ +0rWwSNm/KPq/8NOlfAPpbgTCzAk+blMIoZjzImkS0XARCJA3Zh3UJQq0pS6vzFiaxlHrycSJq2az +mbDoXGPCPWOgk6d9KFWmhFT6YAItDrNpZNqUPxpF/cctIreI7uTqQV7v01w7QZAsxKsvaUc9lIdC +59IpgKUZlwa+KidK+NbT/wDOLTl5ipJUk0YSXdA1clbJz3/iXytOVjf0GndriBG5Ti0zGCXhduBa +WrNYfeI3L5xXvXdf/nAU9FwLZBzKISL5CHfq7DP45bMqtfkrXrTRxNeJOXqjPKqjBjIzTfbXVjiO +nszLqPfQ3H/FV+nfi0N2b5jJlEi/QTneFNxKTBJ0MIxvTkCSNO4a6d2IMSxGG9mNIlguUK5dXJC4 +6nU47IruIWLv2x1uwEl0+i4J6rmTMDx09sVSQZyaiP36tf8Aw7VtT17rtMfMy9oDJtKmMtnqDyg5 +KlD9mORE1gGvcSnylXzp5aYmzb5rmJ4q1i6pruI8th5NlaLWP1p7wUa15bn1piBl/KkO+jRdQOSc +vSTAyLbfTRVq5d1pXSqta176VrrpX6YDXwEbLS+XxEeEq2l1D1KeEcO3dFw//wA2Gj3PFYHi/FgE +AJCqQkd13gIrbhH/ACwkFFB3RK8tzq8Y4dVJbduTC78N/ThwfD1B4huL+bANgO4A3I2/zW4UaY+G +z+Yit/7Y9cVnTjztwLRkq5UPpSSJQrvoOAizchGwUUvLyCiCCKAXEfTdbT4afnjj/tY7UMydoO+c +a/OHy+oquixFYOle0R1oWnxVu8OC3aln5x2oybZoPHQ+X2zxsnvj1GkqZa0qQ08X0xT27OUQyuxi +WiKAtXa7695s3hICJB0080y5d+AqPCx8M9SaLsF01kFWZElfc6Z6JAdTTpT4da64am5xRtGcCpwI +iuKqgPy6l1xu1tU+QvTBLMqkXDZoEXMa+iU1CbOQVW63iWgD90pX4fTngc4h2Mw9JVOeais53CJy +7uA0FS50FXTzL8WAcAph89Zk0Dgk3rpJxGLufeurwHnRNWml3f4cWmPa5dTzw6cyFijctwvaKkad +yUhXuQWCtbaUr+WJmR41wnl9BWSilFY1svsNUhf+8inVPCuA1r93XXvxdY/Lco7jJYXz90TpchRz +OzIAO5Kpe6dpV/64Co5HiWbmMllF544laQdJovB2SD2eY/dq8u5Mq4ubjLcWo0XdqMGrIk9uOlTT +/wDDOqc0nQ0p30LzLFyy4izjY9djIPPanBCKD5WwbnzQ/u1/xKD54aOLdNJsmy7YHLdJAWD4iMbX +LI+aS9KfgrgIcOzbpwjxSZRNgL0uAndsyIUFqfcvE9ddNaW1rXFgaLSTmPMpY2q6g2x06SICJX0/ +u7sdfTENoxR4dWPmQXUKNH2TLGJ3AqxMdUXP6a484fC0VSaO1gtu9kzu4HXtGOjdz+etvPAQ8xrM +ZBwqpOgumJF7JlgLq2lbfszmmnrz54ocnDzCfHbiLGJREW0C/LhupdWhe6c0L4aW1HqwYm5R02zG ++bSxtSWSIYN+YnYJBUa1QdemtO7ATOE5INGW5KOd/wCx+xZYSO4L6fcrjX0+uAxjPDhOUzGSD43X +tbqTctm11qr6hWgVfS7D+bszNV4J+LbLy8W1XVTUY/CTOQSGgrf6vp+uEvk55zJ8FLP2UWm9dcI6 +kekPfD1pl6iNbcCc4IqJxQu5A13pOVRJR4oY3cSF1DEtPmGod+ATldrLIO2slHooPbhVeiFlxKkA ++8E9fyuwCklouxsMWC+4Ilvmp/F1LlXT1xZmmXXybRBzEzCDYlIdV6QLdKgiJFRZL866d2KbduJJ +IJtrrbi6QuIvzwHf37NXDl2KZbubOl0xQIkzTAwHxV/F/wAsaM4TFc/44ppj4CA7i05992M1/ZtE +WnYrllMg3CTQ8Sm0J21IuWla3UxfnCJO3Y7DY7R6bx2THn69WAUrxgpWkBlb4rWxl8XL4v8APEF2 +T5NkXCNjXIS6REFfF9OrE5JqimoTUQREfAoRbOhV8vPHm7cW1qCbbcT8IkIIj1V9OrAAOOmuHLiU +V7i6rU0Vv/fgnxT7h0hFgvcVolcBj3fWpYHvWMb1DwwJl1e9JFv0+XLUsUrtdz84yhkfcI0BcKCK +LYljbkQiRcyENdTpQeelMAMzXOReZMxlH/urMOU0FdkZVOSVj0t2pa7Y9XUVbcXlkms5fICKOWoA +nKRNyal9seKlbpT3tPTGLdh80nmIlynQi5SS4pQhKefkG6rSz3gt+9Pp+mN/yoo1afZoRyyeorqq +LPHTQBJBmVB16Cry78AJyu1a5SW9jqNuARcpWqmosSr+QK3S0R8VBp64hybOSXSVjVofeRXG5llh +qtRsCAUr986Wpzur8uBaVUWs8+kMpyXFunStkjmd/XiFbA6qos0ud3d4R6cSHD54WWnSctFO2jV8 +vti2Jzuy0qVfD3fdAX+2mAJtX0+vGkjl32VIyLRUkwep+4jIilRoJCH/ABa0wHZS0fCZYeKx8w+Q +Ypq2yeaX4Fvyat2m22pXx615Ut6Rw5KQryPjmLaWigfrIICMblSIMkmaF3eTlSug10/F+mBqKcxO +5iZu2z+HIou4eJRbVKOgQoNfubh0WXr9e7ASlt5y4atHeXl4mDVtcDE8SRScu6/h73O4E6aa1uw5 +MOHjl5JySz+2XaIbLldH+4wLb+IIF3EqQ4XBQ8e2buZBT2w5GSVJFWWX6X0qsXcknp90l04OSbXL +cW1jovMjsItND7WhCMlqlugFNffW+Pn54CgS66KjWIqpFGqyUSJPLGUbDSVXr/8AOOufSHxVu8se +aZeknqW+5j5rOrita0VkWL7hWol5ooAI6bYd1K+ddcWdim4nMyqi5jVGj2SSFebfkdvBtO5JkHnQ +i9MWFyhKt1OFr2iMcqgjTbTjWzZJWiYU8NSIud9ad+A0S7o6f92PGJCHT4vhwsOrqLHumy7ANXKX +9QBh0LvF1/y4909JeL8OIsrJRsM04uUfoNG9wp7qx2jfXuHAPqrN2jddd2tsopBcqqXTaP1xyd28 +dpUhmuaKPgni/sVk6UT2mi1irnQBPeEudCHn4cQO3DtWzFnGUk8ops30XBpAuK7ZoFzxWqdn31PJ +Hq15YpeT/wB3Y2PZsXKzV6m0dXLoIn7q1RELTFbv5VwEuM9sJpKyCCyHGNl4/hpEfumwW8hcBX4K +V7ywluxnotk6Xj3JucxNl11pMr72pN1NOpGmJ7eDnJ1psSDZAnTlqm2iXLBawSNDuSXoWtD105XY +tsJAymWYLLPDHvsydKEPEgIL8R8bRTX+GXlgK2yiZJ20jkHkU6kpJsKai43iYLtO8FU/xjixR+X1 +pR2qWyu/WU95MJEzFMXKNC+9T/8ANDzpg7lSBFtIIPmKK8a1dvCWauiMrox3TxM1PLbKvKmLarvL +yCD4nPsclXRJpJEdvBvh7hrp/CUwAVWDZsTXknwb7xghbZ8ErH106qUp8YYkg6WJug7iVkFFmCW4 +1VLxScZX7xOvzEHP8sRjavH1qaiJoOifEo1uMvsb2njQ5fw1R7sCXrV05kEJCEZrobW64Ztr+lm4 +Aq8S109DpgDsS4KNkEnLFmo7YtPtbE7x+0x6w+8RL+StcO5iUaxMggu7M3Ps9Ir/AISXjl//AO2W +AWYpD2EkzGL3yaikMtGAPgVRPku3H8NNcGDnE/Zi6ikbxIxoj0F1krHr6a86/KWAlQ7cRaf2g5BR +u0ujpW7+OxU5orV/LXA7OdwskmLta0Ykhjp0B61V2Rfdq0L6d+HTT9lhdIOTKNQSJg8EQuEminNB +YvPp1wEe5mFNk1kE4037xJJSDnUr7Ssr0pql6/8AtwFIzatGzOVxfKIrk+hyKOmBUMhNVqoXuF66 +U6qU08WKlmJ019scDNyQRaIoJxMml4iVtG5F1X0pzxa3ZFDO9vMD9BAd32LLeEt1oQ3pkNKd9Brj +KMyyCJSAtBhEHakfuMF3ThYrXIGdaIKV9NBt0wDEISMlNi0bLHLOH6oo77joFBa+m2pTv1qWJXaL +Le1M0JFGw4JyAkIrtkw6Rdj0qDSndXURphrL7N5JSr4ZQ+CZiPBKumwWg2dAOqJV09a0w12XrOCz +AkxXRBckiVepHeQqbolS8qV118NC/PAEcwQaL4J9SEeAm3YIJvT3ukiKoU32wV9Br5UxRW7wmwKq +JommRNdki3v0qXd3YueblmMpDz60XvoRaboHMWneJFtLlot+XPXFAMiFIrunpK7owH6F9ggvG3Y/ +lklzBRRRn4yW+G6v4MXlkRNktxy52/is3iMbv9FMVbs0ZjE9lWW2xAuntsExIRMrefPnp+eLYkQp +pCK5r2l1ERXXf564BSq3UShIoEPwnvEJFr/gww7cLJnsCFpEXwrFcI/6MRXr5HdVJB509IhcZdJe +fnhbclF26qm8Cl3hK9W39a0LuwENInBXFfcJDcFrkrrfyqljC/2k5hqpJsYld41TcJtTUFV6iSrY +RqFeWlBoVFKeWN7abiCVxH74riG01vDXy8WOeP2hXzdt2ipLprXOkWe4OyiRkzGvKp1E61osPPQw +8sBmGTM0I5fVffukaDZm5QEXLmTbCb4deSotT1tP6a2+LG65dTcZhyoxi4394l0XbUU2sY90bhaJ +dSq6g08Gn+rHO7Ru1ThxdxcJxqLR0oL6Vd//AA/r+Krf+F5Y6c7AlE22V44bZV28XIRbILPN3fSp +zKz0RGv9MBdDkIlhJpREAim9ogCiiT0QAWcMNB0rqp5lX6YGKz2V4dAW8XIKXL+/eTJBU1V/LRHl +1FX6eHE+byOnKNI6LFxttWC6qx8MG0xsv02lBHxFQcGsj5TiY9v7SFb2kS9ynHOQEdoe6gop06QH +8sAHcJvJ8UOJbLtotIRIIdM/tj7XkNXBfAniZDwck9Fq+mWbVJREi4CHa/3ZidvKq1afeH+mLO4T +Yxrfi0TQZboihxxdRkVS0Eda864oWcpKU4h4CbNeFy+SpdTc7X004qPgQD4KfiwBGWnFI9RVPLyL +aazOqkImO9QWbOo69VS0rQKU154pq3s32euUlnY2CJXDIyzQPtMg5IetJvqNS2h+lMQMvwLpiqMM +5hDkZd2qLn9245ztM4xLnYTtenUodfOhV6sX7LUGaL5WZfBHyuYU1RaJJNAHh4YflH09a4DNQWYx +7eJkI2NmxFddRbL8E5O59Juy8Ltf0SHy1xbzn5ZgXCQ/Zw6zTt/3ySMtdxzXmoNK6c6DWumv0xKk +6lAZxUXYM/3gnHSFs2/UPrQC3QG6FPgIq/DTy6sD3uXidEkpMzWZYtxt0oLGDoVW7UNa1onWuvUf +PWtfrTAbKAiPhDHjtEPBh0cJ54BCRDZ8v+3HIn7X+eJJ9m9vHwj/AG42NQVRXVbnRcCcW3+AdbCH +TlWuOoe0CY/d/IsxLCdqiDVTbH5jqPTSlPPH55cGIq8c7cuiuJDiVWx2GruJFeFKV1offpdgFhNL +RJiS5rpPFCUHaRW+03LAHXv/ABjy501xfsiZX4FoSko23EyY2yO2Hul0ajaRp/8AmI6eKmKj2Pps +SzR7ScogvHtEhTcoKASpIJEZUorr5EJbeNGOQcZZnXjknJi4UdEmLYgtSSdW6VoeuugLBbX/ABYC +5wWWYdQxfSD/AH3iAppya49IkNf7q/T07q911aYuQRayce+kJ1mu9UVIUZZIfB/5b5Gnd8uumK63 +mm8f7H9nw7XhVWpC1QUP79uX94Yl+MPEP8mLFFPkRkGrSPeLuSZNVHUYfVZIR9fvGta1+NOtMAAz +Q4UbOHSEzKmmmQi2mCT6Rsr/AHd8nr6fHiBmCQUIFXMle9UbCnHTZtguEh/gPaUph3tAkEyaFLII +oO24sxJIVg/vkeZe9RrT50ueA/taNy+qzj2J7jNNgLeTVL7pePXKgp6V56qBd+mAsTTNBNGSskoz +MXSiqbKRJQ7iSMeaLmn83dXFPk86EvmNedbb7Rwp0qpdIglIJ+L/AAqDhjNCgwj1JjJb66jYSaPC +E/vWp9SC3Lx26YxrNeYiJukuu/4lwJbLwU/wF7lan5jb1f0wGmZ1zNIcQKmX2G3FtBJ+ld1ESS3Q +sKf0pf8Ay4Hg6zBlJwqNhvWLBnauPzNHPhLX4rS8sCXc08Uyo6fQiJiMIqRBudRKslx0P/TWuGof +MWZlAdLyFijVkw9mOTI7t1FYtUlO74dPF/TAbPl8XBZMZtJkAXcdMLLFvWmLVQbm6nPnpS7vxVJV +ZZSPVUlpLbRSX9iyzMeki2x90ty/FQerzxKyEsop7MY5k4UhmGqkK+citdaYf3danp5YVmt0oSTU +iZsWjp+gpCzB32kg6H7tauvzad+AyTNuaFk5OMF2w2pJkko0kFVgtuKv3alafNQfXFRVcM5J2zUk +FnTl5ukm5JG7qSHkJU/Fj02soU2q5mX/ALSkt9Ru5SHp8OggV2PQTV8nxiHBgSyoi0uWO0kDIuRU +wBuCJOLyEqTZzxac6KjV01K33TgCuSP+mFdnMaQpSc2Jmo6gkk3CDZQ7CVC73g8++nP8sO9ocgmh +FRgkHDSRCSb8bB6XCHSBUp5aj/niz5S3I3syatpBFihIEqMixfEdxLslCqCwV9a0rpgKVnNq1aC6 +9pGbSUcr8RtJ2kgk1UAVUxoNO4+vFbj2sgvbGsWy6ijshEEk/ER+WlMWHtAGFjZD932hnJCwFQUn +wncTkSG5On+DXEFks8dyApxZoNk+P4hql1AQqiOtREtPDgP0PyOm+QyVGIPjuecGnviRq/CNKacx +xOcKLIbRCB/CJW7vn+mIMOs+KHY76IIqEgmoqJIjddZTWv3mH5AR4clLLiHwkmHi/wB+A89WWFUh +URX2x+IuI8X6Dhpo6dIOCHZ8NtvvlhHn60IaYSyTRdgP2YBISttUAh/xfe1xM4URMkyR2PiExDw/ +n11wDr1ZYUkiEF1BIrSFO8/r6Yxz9p3LLybyuxzEga6AxKvEFts1SXFKheVbf89cbKkis23U11kC +HpISJEh/y6sCcxwrObhHkavYui5SK3oIhIajz/iUwHJLtwhOspydJJbMq9jZyMwij9lZgNw2vGvn +Wmvi0xrv7IScehDzDSNmGr94Dol1XDICJszRr1WDfSnfjnzMrdbLshLQSDORJG3h0AFyKBiND199 +WmoKUGvdSunjxrH7Ms9MFnV004N6uioSRSL5EAbEkNB5I1R+Lq+KneOA6rboorAguPS13dxqCNwd +487vXHgWcJ1QTd7ZSRCptCjdtW0+avliSipaqQie6oRCQoeHYCuAc81nFEiicuufZokW44k1g3bQ +r3inTl1f8sBHzFmJvDAlx6KktMKpgSUSyDdLdpzoX4efxFgFlaJzNNSCWZMyOWSEskqSfAo+8CMb +lSvRSvmtXzLFnh4FrHtVUYu8U10ivkSW+0qHd3c6d2IuYJRuhHruXL4IWJtEvaN9qqytK06aBWnn +TAMQuVYeGZPo6CNRkzXIl3stxNy+7QqV0I689MB5iaWcpKqZbftcvQZFacmLa9d8t3e5D4/5sU9X +NTrPk8xi2gcBCgqSoQSlyDiQAf4y5/w0fw18WA/aHna12upk1YJaatJgxVbo3JCr/wAFmn4en41M +A7mDNEbEGuXttCAcJiQpM1LyXvU6auXNaU5KWVKwPXuwmAkO015FIuGGf4TIUeVPscZIMAJ0ol5L +q386Edda6emmEdmXZqUb/a2dFo6Smo10Lt+g7P3DMqjeSypV++UpTu+EcFs3Z8yCi8bKTEPGzyq7 +aiyT1+6FJVRKpFQenypyrp9MBvvx+PCvFj4Ajf3Y+15a6eXdgOef21c1DE5XjMvqMzXbyhksRitt +EFnPpr82OSMzPh3UOEeLuUWQ8MAEY2pWleOnzjpTHQ37Y7mQLPgtmpN1ExiVK2OQuFIfnT5clPrj +niPTkpmBbMGz6gIryQAILDr11Gtda179K178BrHZkKcRldisq5ZOU3KSj10aNu4qxUKgLJ10+JMq +AXLCpVmom7k4l3MOnLe1NGVc2CuRMq9TR0H8o1pTEuDi3ySqpRps29VGwv0E6JWp0EfdronSneJ0 +pStC76YrmYiFmo4BZRfaibQUogdRq6jVOfDnX5g1roXPXzwB/J8g8sdRcoZthTVFTitkruKoXuXQ +6+GivhP+fFuj5SWKCeC2cgyJdUlIwyusZvh+8b//AE1B54xxw+UXhXHBSkkiq1dIxyaihUKtY5au +qadfUxrSnfyp5YvPZ1OSUoJxrrbMFqVaODLqqLlKlyTkOXKunIqf1wBSVzA8kG8SMbGmLfj+JEiO +4Gbug/aGqn/ln8OKHnucYje0y2AezySUWSbEdxIJGNaKtS1+IddaY88zlKZlaydFUko9OQUFNxwZ +VCou0g04gPSpUpTWnLXz1xWmsWwWjk0UKrIqOo83Jr/GDgB5lSmulRLzp/TAWOQJ1Fx4zruVCWeQ +opCaSJiO+xUGtKcq/LjPOIasZtBRyw9z7wjKwT3UTLpLTw8sG7G45faS3CpaRZJgonSmm+gdNSCt +fLn5c6Yqjt0g+lG7ZFExZm6Mm4qncVAIuQl66YDR27daJj3S7ZY3rFgW2qF5CRMl/ARUr36VrTAV +V1IIN3jZewk0Ek2D74QVSMtUVOXkOuLhInXJCLriCJ8ghRSMdDTkSqRjWo6etaVpTWtef1xnjxN8 +3kF2ajlMkkwSarjZdQklK6hSmvnSvr+mAs2Qmb5tJycfKSX2hRUY4x6fcEI02luXw/XF57VWbeWj +Gs2+bLuXDlLgpNsn4kpABtTUqP1xnmX8ssOMRZLLOarlMnErGBaCoppqB189KV8+/F6gZprIQ1ZJ +03VApAVI2QBMqVEl0qag4CleV2vnXngMMlUVGxl9j4Ikvsy4F4t2nOpYNQ8SUkySlnkkajcnQsnl +txEgdvuir6jgVmhJYaJyLxyaq7xRSqtfFrUK6U7/AKYumSE2yOUgUqlU208ktHvEa+RhzSVGvrSv +f3fngKzm6SWlpMnLttwyxCKLnc6iVWEbalWlPDrX1xa80Ok2zTK0XHtvbUD1P2LYfvRG2lFkSLxV +0t10xRYtwqEok4TbtlXJGK4KLUuqKqZUMq93dXu07vWmLmbqSWziM3Bk3bK0aDKKJLhqFCK4STHS +ngrTy0wAKbavF5D3fAsmuwLlnVQxDfRqdSTEa+RU8PLvxFhE5Sbm2DHe6XbwVitttEqnShF0+HDe +amjhGdfbrmqttU6lry0pULqCP0p5YtHYvEm/zxEhHPFUttRBZxRbnQq7g00HT6euA7n2xQh2aZLI +ESaSfSRpCPIdPMce4xum3SQJsxIVCuG1yl/1HywaogKBkSKCQKiVhaEVtPoP0xJSZvCVvMkan8VL +yt/TlgBDckb7d5raPyrI+L5uQ+eHTdEmqNphvEVxANl1tPXpxPJo/TQNusadEKd20udC8X5YivnH +AK1SJV2e5TSlarkVtP1wCjeEuZKEBqD8IkiJdXp3YfBMRuTFHqHwe5G23/viIaTUgEbVKlTuqVcR +hdtA02xWpeNbeVOnXv8APAczftC5VTY9oTqQbNkH60ta7bRg3CkSoDWil4dxdNdenFB7NZ5qnmZJ +9LM15tZBAukZIkgbABVITTMa0qoIj01Avlx1J2y5ZaZ5yO9hUXKzV8nQKtHJDTRM7uWunPT1xyqG +XvbMs0bOHCQzbt3wIUogPB1KgiJGY9+te+6lNcB3BkWW9t5fYzKjYGzqQapuJECuE0BqHSP+WCqo +qKKhILouk02R2t0kTuFca6UuKmOUexfPS2Ss8o5RclJvSqqSEuuo8qsDkxPQSAD0oNKUx0a+zgqW +X46Rat9l5JOeDaUrXVNGl2lxU8+XlgDrgniDhmgKPFut9Qb07tpsFfn/AE7sZznXNULxaSe9FT2Y +I5JTiXqxiTOPIfGVAryJT8NOeLJPrPqSq8G2XpHgtRKi7lr0rKLHTS7XlbSlPTHOud8tqJyuVhlp +g0Ip46dxyKUYzTTWbGJXblD5VIiKlK1KvPAXTPce1aZCnXzGSN63XFstNkK21KvBIuYqKV+5S08v +lxj7+YmMoTb7NeX5iIQ9hPmzZiggtvpNmiojoIadFtteo9NSrimzWa4R07NzKUnnaziKqksqbqh7 +6gqHQLwrXSyg0py9cQ6ZkZSUll1lBZajIWWZtiaruRHdTdFdX3pDWnIvrzr9cAfzrmyYlM2lDz2f ++JbkZIqP2QUJrw6/OtKhpcVvpX8sQYma7I6sE0815azFNSiWqRvkX1oLCNdBIRrzGmmnLywh5x/a +T2oUbxMbE5bfE1rRbhiOiKhohdVTSlNbq9/59+KPmSUSeTTlZywboL3UFQWw2p1KlKUrWlPLXTX8 +64D/2Q== + +--Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B-- + +--Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8-- -- cgit v1.2.3 From aae0daf7a6348c3977e9c16f3499731f77edfb7f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 10 Jul 2015 12:03:55 -0400 Subject: [tests] add symlink to the nested sample file --- src/leap/mail/imap/tests/rfc822.multi-nested.message | 1 + 1 file changed, 1 insertion(+) create mode 120000 src/leap/mail/imap/tests/rfc822.multi-nested.message diff --git a/src/leap/mail/imap/tests/rfc822.multi-nested.message b/src/leap/mail/imap/tests/rfc822.multi-nested.message new file mode 120000 index 0000000..306d0de --- /dev/null +++ b/src/leap/mail/imap/tests/rfc822.multi-nested.message @@ -0,0 +1 @@ +../../tests/rfc822.multi-nested.message \ No newline at end of file -- cgit v1.2.3 From 5cbb56ab995c620595ab6b840ebabd535fb31244 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 10 Jul 2015 12:13:37 -0400 Subject: [tests] get REGRESSIONS_FOLDER from env var in order to start with a fresh folder each time. current test script has some troubles with dirty state. --- src/leap/mail/imap/tests/regressions_mime_struct | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/leap/mail/imap/tests/regressions_mime_struct b/src/leap/mail/imap/tests/regressions_mime_struct index 1e0e870..0332664 100755 --- a/src/leap/mail/imap/tests/regressions_mime_struct +++ b/src/leap/mail/imap/tests/regressions_mime_struct @@ -40,7 +40,9 @@ from twisted.protocols import basic from twisted.python import log -REGRESSIONS_FOLDER = "regressions_test" +REGRESSIONS_FOLDER = os.environ.get( + "REGRESSIONS_FOLDER", "regressions_test") +print "[+] Using regressions folder:", REGRESSIONS_FOLDER parser = Parser() -- cgit v1.2.3 From a36a3bd419bdf33d66ed77277089db537e8b7b36 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 10 Jul 2015 12:57:44 -0400 Subject: [docs] add some documentation about imap regression tests This is quite manual for the moment being, and it's not integrated into the unittests. But it is useful to have it documented, with some luck we can automate the process even more and add it to the CI quite soon. --- README.rst | 11 +++++++++++ docs/hacking.rst | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/README.rst b/README.rst index 317389a..81b4cec 100644 --- a/README.rst +++ b/README.rst @@ -24,3 +24,14 @@ fails:: trial -u leap.mail.imap Read the *trial* manpage for more options . + +imap regressions +---------------- + +For testing the IMAP server implementation, there are a couple of utilities. +From the ``leap.mail.imap.tests`` folder, and with an already initialized server +running:: + + ./regressions_mime_struct user@provider pass path_to_samples/ + +You can find several message samples in the ``leap/mail/tests`` folder. diff --git a/docs/hacking.rst b/docs/hacking.rst index d5669e1..6c49c21 100644 --- a/docs/hacking.rst +++ b/docs/hacking.rst @@ -161,6 +161,25 @@ If looking for a quick way of inspecting mailboxes, have a look at ``getmail``:: From: Kali (snip) + +IMAP Message Rendering Regressions +---------------------------------- + +For testing the IMAP server implementation, there is a litte regressions script +that needs some manual work from your side. + +First of all, you need an already initialized account. Which for now basically +means you have created a new account with a provider that offers the Encrypted +Mail Service, using the Bitmask Client wizard. Then you need to log in with that +account, and let it generate the secrets and sync with the remote for a first +time. After this you can run the twistd server locally and offline. + +From the ``leap.mail.imap.tests`` folder, and with an already initialized server +running:: + + ./regressions_mime_struct user@provider pass path_to_samples/ + +You can find several message samples in the ``leap/mail/tests`` folder. Debugging IMAP commands -- cgit v1.2.3 From 6ab67b4c69be4d3f3fb34dd3f76cd36822e7e1ca Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Tue, 21 Jul 2015 19:16:53 -0300 Subject: Fixed all the pep8 warnings in the code --- src/leap/mail/imap/mailbox.py | 6 +++--- src/leap/mail/imap/server.py | 16 ++++++++-------- src/leap/mail/imap/tests/__init__.py | 6 +++--- src/leap/mail/imap/tests/test_imap.py | 2 +- src/leap/mail/imap/tests/test_imap_store_fetch.py | 12 ++++++++---- src/leap/mail/imap/tests/walktree.py | 2 +- src/leap/mail/mail.py | 4 ++-- src/leap/mail/utils.py | 7 ++++--- src/leap/mail/walk.py | 2 +- 9 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 4339bd2..c52a2e3 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -878,9 +878,9 @@ class IMAPMailbox(object): # A better place for this would be the COPY/APPEND dispatcher # in server.py, but qtreactor hangs when I do that, so this seems # to work fine for now. - #d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) - #deferLater(self.reactor, 0, self._do_copy, message, d) - #return d + # d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) + # deferLater(self.reactor, 0, self._do_copy, message, d) + # return d d = self.collection.copy_msg(message.message, self.collection.mbox_uuid) diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 2b670c1..050521a 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -122,9 +122,9 @@ class LEAPIMAPServer(imap4.IMAP4Server): elif part.text: _w(str(part) + ' ') _f() - return imap4.FileProducer(msg.getBodyFile() - ).beginProducing(self.transport - ) + return imap4.FileProducer( + msg.getBodyFile() + ).beginProducing(self.transport) elif part.mime: hdrs = imap4._formatHeaders(msg.getHeaders(True)) @@ -151,10 +151,10 @@ class LEAPIMAPServer(imap4.IMAP4Server): _fd.seek(0) else: _fd = fd - return imap4.FileProducer(_fd - # END PATCHED #########################3 - ).beginProducing(self.transport - ) + return imap4.FileProducer( + _fd + # END PATCHED #########################3 + ).beginProducing(self.transport) else: mf = imap4.IMessageFile(msg, None) if mf is not None: @@ -459,7 +459,7 @@ class LEAPIMAPServer(imap4.IMAP4Server): mailboxes.addCallback(self._cbSubscribed) mailboxes.addCallback( self._cbListWork, tag, sub, cmdName, - ).addErrback(self._ebListWork, tag) + ).addErrback(self._ebListWork, tag) def _cbSubscribed(self, mailboxes): subscribed = [ diff --git a/src/leap/mail/imap/tests/__init__.py b/src/leap/mail/imap/tests/__init__.py index f3d5ca6..32dacee 100644 --- a/src/leap/mail/imap/tests/__init__.py +++ b/src/leap/mail/imap/tests/__init__.py @@ -1,4 +1,4 @@ -#-*- encoding: utf-8 -*- +# -*- encoding: utf-8 -*- """ leap/email/imap/tests/__init__.py ---------------------------------- @@ -26,10 +26,10 @@ from leap.soledad.client import Soledad from leap.soledad.common.document import SoledadDocument -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Some tests inherit from BaseSoledadTest in order to have a working Soledad # instance in each test. -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- class BaseSoledadIMAPTest(BaseLeapTest): """ diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index af1bd69..ffe59c3 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -439,7 +439,7 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): d1 = self.connected.addCallback(strip(add_mailbox)) d1.addCallback(strip(login)) d1.addCallback(strip(select)) - #d1.addErrback(self._ebGeneral) + # d1.addErrback(self._ebGeneral) d2 = self.loopback() diff --git a/src/leap/mail/imap/tests/test_imap_store_fetch.py b/src/leap/mail/imap/tests/test_imap_store_fetch.py index 6da8581..81f88fe 100644 --- a/src/leap/mail/imap/tests/test_imap_store_fetch.py +++ b/src/leap/mail/imap/tests/test_imap_store_fetch.py @@ -43,10 +43,14 @@ class StoreAndFetchTestCase(IMAP4HelperMixin): self.connected.addCallback( self._addSignedMessage).addCallback( - lambda uid: self.function( - uids, uid=uid) # do NOT use seq numbers! - ).addCallback(result).addCallback( - self._cbStopClient).addErrback(self._ebGeneral) + lambda uid: self.function( + uids, uid=uid + ) # do NOT use seq numbers! + ).addCallback( + result + ).addCallback( + self._cbStopClient + ).addErrback(self._ebGeneral) d = loopback.loopbackTCP(self.server, self.client, noisy=False) d.addCallback(lambda x: self.assertEqual(self.result, self.expected)) diff --git a/src/leap/mail/imap/tests/walktree.py b/src/leap/mail/imap/tests/walktree.py index 695f487..4544856 100644 --- a/src/leap/mail/imap/tests/walktree.py +++ b/src/leap/mail/imap/tests/walktree.py @@ -1,4 +1,4 @@ -#t -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- # walktree.py # Copyright (C) 2013 LEAP # diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index 6a7c558..540a493 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -85,7 +85,7 @@ def _encode_payload(payload, ctype=""): # soledad when it's creating the documents. # if not charset: # charset = get_email_charset(payload) - #------------------------------------------------------ + # ----------------------------------------------------- if not charset: charset = "utf-8" @@ -113,7 +113,7 @@ def _unpack_headers(headers_dict): inner = zip( itertools.cycle([k]), map(lambda l: l.rstrip('\n'), splitted)) - headers_l = headers_l[:i] + inner + headers_l[i+1:] + headers_l = headers_l[:i] + inner + headers_l[i + 1:] return headers_l diff --git a/src/leap/mail/utils.py b/src/leap/mail/utils.py index 029e9f5..64fca98 100644 --- a/src/leap/mail/utils.py +++ b/src/leap/mail/utils.py @@ -254,6 +254,7 @@ def validate_address(address): # String manipulation # + class CustomJsonScanner(object): """ This class is a context manager definition used to monkey patch the default @@ -299,13 +300,13 @@ class CustomJsonScanner(object): end = s.find("\"", idx) while not found: try: - if s[end-1] != "\\": + if s[end - 1] != "\\": found = True else: - end = s.find("\"", end+1) + end = s.find("\"", end + 1) except Exception: found = True - return s[idx:end].decode("string-escape"), end+1 + return s[idx:end].decode("string-escape"), end + 1 def __enter__(self): """ diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 9f5098d..6d79b83 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -187,7 +187,7 @@ def walk_msg_tree(parts, body_phash=None): last_part = max(main_pmap.keys()) main_pmap[last_part][PART_MAP] = {} for partind in range(len(pv) - 1): - print partind+1, len(parts) + print partind + 1, len(parts) main_pmap[last_part][PART_MAP][partind] = parts[partind + 1] outer = parts[0] -- cgit v1.2.3 From bb132dcc1b558cfdfd0bf438779fe2ed07827fd5 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Tue, 21 Jul 2015 19:30:33 -0300 Subject: Updated pep8 and fixed import and line break warnings --- src/leap/mail/_version.py | 13 ++++--------- src/leap/mail/imap/service/imap.py | 6 +++--- src/leap/mail/imap/tests/__init__.py | 14 +++++++------- src/leap/mail/imap/tests/walktree.py | 2 +- src/leap/mail/incoming/service.py | 12 +++++++----- src/leap/mail/smtp/__init__.py | 4 ++-- 6 files changed, 24 insertions(+), 27 deletions(-) diff --git a/src/leap/mail/_version.py b/src/leap/mail/_version.py index d80ec47..b77d552 100644 --- a/src/leap/mail/_version.py +++ b/src/leap/mail/_version.py @@ -1,3 +1,7 @@ +import subprocess +import sys +import re +import os.path IN_LONG_VERSION_PY = True # This file helps to compute a version number in source trees obtained from @@ -14,10 +18,6 @@ git_refnames = "$Format:%d$" git_full = "$Format:%H$" -import subprocess -import sys - - def run_command(args, cwd=None, verbose=False): try: # remember shell=False, so use git.cmd on windows, not just git @@ -38,11 +38,6 @@ def run_command(args, cwd=None, verbose=False): return stdout -import sys -import re -import os.path - - def get_expanded_variables(versionfile_source): # the code embedded in _version.py can just fetch the value of these # variables. When used from setup.py, we don't want to import diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 92d05cc..c3ae59a 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -17,7 +17,6 @@ """ IMAP service initialization """ -# TODO: leave only an implementor of IService in here import logging import os @@ -29,13 +28,14 @@ from twisted.internet.protocol import ServerFactory from twisted.mail import imap4 from twisted.python import log -logger = logging.getLogger(__name__) - from leap.common.events import emit, catalog from leap.common.check import leap_check from leap.mail.imap.account import IMAPAccount from leap.mail.imap.server import LEAPIMAPServer +# TODO: leave only an implementor of IService in here + +logger = logging.getLogger(__name__) DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) if DO_MANHOLE: diff --git a/src/leap/mail/imap/tests/__init__.py b/src/leap/mail/imap/tests/__init__.py index 32dacee..5cf60ed 100644 --- a/src/leap/mail/imap/tests/__init__.py +++ b/src/leap/mail/imap/tests/__init__.py @@ -10,13 +10,6 @@ code, using twisted.trial, for testing leap_mx. @copyright: © 2013 Kali Kaneko, see COPYLEFT file """ -__all__ = ['test_imap'] - - -def run(): - """xxx fill me in""" - pass - import os import u1db @@ -25,12 +18,19 @@ from leap.common.testing.basetest import BaseLeapTest from leap.soledad.client import Soledad from leap.soledad.common.document import SoledadDocument +__all__ = ['test_imap'] + + +def run(): + """xxx fill me in""" + pass # ----------------------------------------------------------------------------- # Some tests inherit from BaseSoledadTest in order to have a working Soledad # instance in each test. # ----------------------------------------------------------------------------- + class BaseSoledadIMAPTest(BaseLeapTest): """ Instantiates GPG and Soledad for usage in LeapIMAPServer tests. diff --git a/src/leap/mail/imap/tests/walktree.py b/src/leap/mail/imap/tests/walktree.py index 4544856..f259a55 100644 --- a/src/leap/mail/imap/tests/walktree.py +++ b/src/leap/mail/imap/tests/walktree.py @@ -19,6 +19,7 @@ Tests for the walktree module. """ import os import sys +import pprint from email import parser from leap.mail import walk as W @@ -118,7 +119,6 @@ if DEBUG and DO_CHECK: print "Structure: OK" -import pprint print print "RAW DOCS" pprint.pprint(raw_docs) diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index 3daf86b..2bc6751 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -429,9 +429,9 @@ class IncomingMail(Service): fromHeader = msg.get('from', None) senderAddress = None - if (fromHeader is not None - and (msg.get_content_type() == MULTIPART_ENCRYPTED - or msg.get_content_type() == MULTIPART_SIGNED)): + if (fromHeader is not None and + (msg.get_content_type() == MULTIPART_ENCRYPTED or + msg.get_content_type() == MULTIPART_SIGNED)): senderAddress = parseaddr(fromHeader)[1] def add_leap_header(ret): @@ -635,8 +635,10 @@ class IncomingMail(Service): url = shlex.split(fields['url'])[0] # remove quotations urlparts = urlparse(url) addressHostname = address.split('@')[1] - if (urlparts.scheme == 'https' - and urlparts.hostname == addressHostname): + if ( + urlparts.scheme == 'https' and + urlparts.hostname == addressHostname + ): def fetch_error(failure): if failure.check(keymanager_errors.KeyNotFound): logger.warning("Url from OpenPGP header %s failed" diff --git a/src/leap/mail/smtp/__init__.py b/src/leap/mail/smtp/__init__.py index 3ef016b..2ff14d7 100644 --- a/src/leap/mail/smtp/__init__.py +++ b/src/leap/mail/smtp/__init__.py @@ -24,11 +24,11 @@ from twisted.internet import reactor from twisted.internet.error import CannotListenError from leap.mail.outgoing.service import OutgoingMail -logger = logging.getLogger(__name__) - from leap.common.events import emit, catalog from leap.mail.smtp.gateway import SMTPFactory +logger = logging.getLogger(__name__) + def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port, smtp_cert, smtp_key, encrypted_only): -- cgit v1.2.3 From 63e643466882996f32eba1c0f79ebdd3ceb5f3b3 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Tue, 21 Jul 2015 19:44:47 -0300 Subject: Pep8 warns about lambdas being assigned to a variable, changed walk and sync_hooks to lambdas to real functions --- src/leap/mail/sync_hooks.py | 5 +-- src/leap/mail/walk.py | 84 ++++++++++++++++++++++++++++----------------- 2 files changed, 56 insertions(+), 33 deletions(-) diff --git a/src/leap/mail/sync_hooks.py b/src/leap/mail/sync_hooks.py index bd8d88d..8efbb7c 100644 --- a/src/leap/mail/sync_hooks.py +++ b/src/leap/mail/sync_hooks.py @@ -32,10 +32,11 @@ from twisted.python import log from leap.soledad.client.interfaces import ISoledadPostSyncPlugin from leap.mail import constants - logger = logging.getLogger(__name__) -_get_doc_type_preffix = lambda s: s[:2] + +def _get_doc_type_preffix(s): + return s[:2] class MailProcessingPostSyncHook(object): diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 6d79b83..f613309 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -26,35 +26,49 @@ from leap.mail.utils import first DEBUG = os.environ.get("BITMASK_MAIL_DEBUG") if DEBUG: - get_hash = lambda s: sha256.SHA256(s).hexdigest()[:10] + def get_hash(s): + return sha256.SHA256(s).hexdigest()[:10] else: - get_hash = lambda s: sha256.SHA256(s).hexdigest() + def get_hash(s): + return sha256.SHA256(s).hexdigest() """ Get interesting message parts """ -get_parts = lambda msg: [ - {'multi': part.is_multipart(), - 'ctype': part.get_content_type(), - 'size': len(part.as_string()), - 'parts': len(part.get_payload()) - if isinstance(part.get_payload(), list) - else 1, - 'headers': part.items(), - 'phash': get_hash(part.get_payload()) - if not part.is_multipart() else None} - for part in msg.walk()] + + +def get_parts(msg): + return [ + { + 'multi': part.is_multipart(), + 'ctype': part.get_content_type(), + 'size': len(part.as_string()), + 'parts': + len(part.get_payload()) + if isinstance(part.get_payload(), list) + else 1, + 'headers': part.items(), + 'phash': + get_hash(part.get_payload()) + if not part.is_multipart() + else None + } for part in msg.walk()] """ Utility lambda functions for getting the parts vector and the payloads from the original message. """ -get_parts_vector = lambda parts: (x.get('parts', 1) for x in parts) -get_payloads = lambda msg: ((x.get_payload(), - dict(((str.lower(k), v) for k, v in (x.items())))) - for x in msg.walk()) + +def get_parts_vector(parts): + return (x.get('parts', 1) for x in parts) + + +def get_payloads(msg): + return ((x.get_payload(), + dict(((str.lower(k), v) for k, v in (x.items())))) + for x in msg.walk()) def get_body_phash(msg): @@ -73,18 +87,22 @@ index the content. Here we remove any mutable part, as the the filename in the content disposition. """ -get_raw_docs = lambda msg, parts: ( - {"type": "cnt", # type content they'll be - "raw": payload if not DEBUG else payload[:100], - "phash": get_hash(payload), - "content-disposition": first(headers.get( - 'content-disposition', '').split(';')), - "content-type": headers.get( - 'content-type', ''), - "content-transfer-encoding": headers.get( - 'content-transfer-type', '')} - for payload, headers in get_payloads(msg) - if not isinstance(payload, list)) + +def get_raw_docs(msg, parts): + return ( + { + "type": "cnt", # type content they'll be + "raw": payload if not DEBUG else payload[:100], + "phash": get_hash(payload), + "content-disposition": first(headers.get( + 'content-disposition', '').split(';')), + "content-type": headers.get( + 'content-type', ''), + "content-transfer-encoding": headers.get( + 'content-transfer-type', '') + } for payload, headers in get_payloads(msg) + if not isinstance(payload, list)) + """ Groucho Marx: Now pay particular attention to this first clause, because it's @@ -155,8 +173,12 @@ def walk_msg_tree(parts, body_phash=None): print # wrappers vector - getwv = lambda pv: [True if pv[i] != 1 and pv[i + 1] == 1 else False - for i in range(len(pv) - 1)] + def getwv(pv): + return [ + True if pv[i] != 1 and pv[i + 1] == 1 + else False + for i in range(len(pv) - 1) + ] wv = getwv(pv) # do until no wrapper document is left -- cgit v1.2.3 From 11d4373226c8ab32c31fa92beed8aedb962dd756 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Tue, 21 Jul 2015 19:56:43 -0300 Subject: Transformed assigned lambdas to functions in models and test_models because of pep8 --- src/leap/mail/adaptors/models.py | 6 ++---- src/leap/mail/adaptors/tests/test_models.py | 7 +++++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/leap/mail/adaptors/models.py b/src/leap/mail/adaptors/models.py index 88e0e4e..c5b838a 100644 --- a/src/leap/mail/adaptors/models.py +++ b/src/leap/mail/adaptors/models.py @@ -115,10 +115,8 @@ class DocumentWrapper(object): def _normalize_dict(_dict): items = _dict.items() - not_callable = lambda (k, v): not callable(v) - not_private = lambda(k, v): not k.startswith('_') - for cond in not_callable, not_private: - items = filter(cond, items) + items = filter(lambda k, v: not callable(v), items) + items = filter(lambda k, v: not k.startswith('_')) items = [(k, v) if not k.endswith('_') else (k[:-1], v) for (k, v) in items] items = [(k.replace('-', '_'), v) for (k, v) in items] diff --git a/src/leap/mail/adaptors/tests/test_models.py b/src/leap/mail/adaptors/tests/test_models.py index efe0bf2..b82cfad 100644 --- a/src/leap/mail/adaptors/tests/test_models.py +++ b/src/leap/mail/adaptors/tests/test_models.py @@ -39,7 +39,8 @@ class SerializableModelsTestCase(unittest.TestCase): class IgnoreMe(object): pass - killmeplease = lambda x: x + def killmeplease(x): + return x serialized = M.serialize() expected = {'foo': 42, 'bar': 33, 'baaz': None} @@ -88,7 +89,9 @@ class DocumentWrapperTestCase(unittest.TestCase): class Wrapper(models.DocumentWrapper): class model(models.SerializableModel): foo = 42 - getwrapper = lambda: Wrapper(bar=1) + + def getwrapper(): + return Wrapper(bar=1) self.assertRaises(RuntimeError, getwrapper) def test_no_model_wrapper(self): -- cgit v1.2.3 From 3f171207e73790b512cbfa187f8c01e0b5f076a9 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Tue, 21 Jul 2015 20:11:29 -0300 Subject: Changed imap tests and messages assigned lambdas to functions because of pep8 --- src/leap/mail/imap/messages.py | 7 +++++-- src/leap/mail/imap/tests/test_imap.py | 18 +++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py index 9af4c99..d1c7b93 100644 --- a/src/leap/mail/imap/messages.py +++ b/src/leap/mail/imap/messages.py @@ -217,10 +217,13 @@ def _format_headers(headers, negate, *names): return {str('content-type'): str('')} names = map(lambda s: s.upper(), names) + if negate: - cond = lambda key: key.upper() not in names + def cond(key): + return key.upper() not in names else: - cond = lambda key: key.upper() in names + def cond(key): + return key.upper() in names if isinstance(headers, list): headers = dict(headers) diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index ffe59c3..62c3c41 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -387,8 +387,11 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): acc.addMailbox('this/mbox'), acc.addMailbox('that/mbox')]) - dc1 = lambda: acc.subscribe('this/mbox') - dc2 = lambda: acc.subscribe('that/mbox') + def dc1(): + return acc.subscribe('this/mbox') + + def dc2(): + return acc.subscribe('that/mbox') def login(): return self.client.login(TEST_USER, TEST_PASSWD) @@ -668,9 +671,14 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): acc = self.server.theAccount - dc1 = lambda: acc.addMailbox('root_subthing', creation_ts=42) - dc2 = lambda: acc.addMailbox('root_another_thing', creation_ts=42) - dc3 = lambda: acc.addMailbox('non_root_subthing', creation_ts=42) + def dc1(): + return acc.addMailbox('root_subthing', creation_ts=42) + + def dc2(): + return acc.addMailbox('root_another_thing', creation_ts=42) + + def dc3(): + return acc.addMailbox('non_root_subthing', creation_ts=42) def login(): return self.client.login(TEST_USER, TEST_PASSWD) -- cgit v1.2.3 From 23a1077401b987a64c22efe6a0b9751c8549e865 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 22 Jul 2015 12:23:18 -0400 Subject: [pkg] separate leap requirements this is part of a process to make the setup of the development mode less troublesome. from now on, setting up a virtualenv in pure development mode will be as easy as telling pip to just install the external dependencies:: pip install -r pkg/requirements.pip and traversing all the leap repos for the needed leap dependencies doing:: python setup.py develop - Related: #7288 --- pkg/requirements-leap.pip | 3 +++ pkg/requirements.pip | 3 --- pkg/utils.py | 29 +++++++++++++++++++++++------ setup.py | 18 +++++++++++++++++- 4 files changed, 43 insertions(+), 10 deletions(-) create mode 100644 pkg/requirements-leap.pip diff --git a/pkg/requirements-leap.pip b/pkg/requirements-leap.pip new file mode 100644 index 0000000..f50487e --- /dev/null +++ b/pkg/requirements-leap.pip @@ -0,0 +1,3 @@ +leap.common>=0.4.0 +leap.soledad.client>=0.7.0 +leap.keymanager>=0.4.0 diff --git a/pkg/requirements.pip b/pkg/requirements.pip index d77059a..0caa66b 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -1,7 +1,4 @@ zope.interface -leap.soledad.client>=0.7.0 -leap.common>=0.4.0 -leap.keymanager>=0.4.0 twisted # >= 12.0.3 ?? zope.proxy service-identity diff --git a/pkg/utils.py b/pkg/utils.py index deace14..d168010 100644 --- a/pkg/utils.py +++ b/pkg/utils.py @@ -14,20 +14,34 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - """ Utils to help in the setup process """ - import os import re import sys +def is_develop_mode(): + """ + Returns True if we're calling the setup script using the argument for + setuptools development mode. + + This avoids messing up with dependency pinning and order, the + responsibility of installing the leap dependencies is left to the + developer. + """ + args = sys.argv + devflags = "setup.py", "develop" + if (args[0], args[1]) == devflags: + return True + return False + + def get_reqs_from_files(reqfiles): """ Returns the contents of the top requirement file listed as a - string list with the lines + string list with the lines. @param reqfiles: requirement files to parse @type reqfiles: list of str @@ -43,6 +57,9 @@ def parse_requirements(reqfiles=['requirements.txt', """ Parses the requirement files provided. + The passed reqfiles list is a list of possible locations to try, the + function will return the contents of the first path found. + Checks the value of LEAP_VENV_SKIP_PYSIDE to see if it should return PySide as a dep or not. Don't set, or set to 0 if you want to install it through pip. @@ -58,9 +75,9 @@ def parse_requirements(reqfiles=['requirements.txt', if re.match(r'\s*-e\s+', line): pass # do not try to do anything with externals on vcs - #requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', - #line)) - # http://foo.bar/baz/foobar/zipball/master#egg=foobar + # requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', + # line)) + # http://foo.bar/baz/foobar/zipball/master#egg=foobar elif re.match(r'\s*https?:', line): requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', line)) diff --git a/setup.py b/setup.py index 499a9ee..ead8982 100644 --- a/setup.py +++ b/setup.py @@ -111,6 +111,22 @@ cmdclass["freeze_debianver"] = freeze_debianver # XXX add ref to docs +requirements = utils.parse_requirements() + +if utils.is_develop_mode(): + print + print ("[WARNING] Skipping leap-specific dependencies " + "because development mode is detected.") + print ("[WARNING] You can install " + "the latest published versions with " + "'pip install -r pkg/requirements-leap.pip'") + print ("[WARNING] Or you can instead do 'python setup.py develop' " + "from the parent folder of each one of them.") + print +else: + requirements += utils.parse_requirements( + reqfiles=["pkg/requirements-leap.pip"]) + setup( name='leap.mail', version=VERSION, @@ -130,7 +146,7 @@ setup( package_dir={'': 'src'}, packages=find_packages('src'), test_suite='leap.mail.load_tests.load_tests', - install_requires=utils.parse_requirements(), + install_requires=requirements, tests_require=utils.parse_requirements( reqfiles=['pkg/requirements-testing.pip']), ) -- cgit v1.2.3 From 35cd9bd76676dea872dc02f4a854e4bcbac30648 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 27 Jul 2015 22:12:01 -0400 Subject: [pkg] add AUTHORS file + one-liner to generate it --- AUTHORS | 7 +++++++ pkg/tools/get_authors.sh | 2 ++ 2 files changed, 9 insertions(+) create mode 100644 AUTHORS create mode 100755 pkg/tools/get_authors.sh diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..de3e0e3 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,7 @@ +Kali Kaneko +Tomás Touceda +drebs +Ivan Alejandro +Ruben Pollan +Bruno Wagner Goncalves +Duda Dornelles diff --git a/pkg/tools/get_authors.sh b/pkg/tools/get_authors.sh new file mode 100755 index 0000000..0169bb1 --- /dev/null +++ b/pkg/tools/get_authors.sh @@ -0,0 +1,2 @@ +#!/bin/sh +git log --format='%aN <%aE>' | awk '{arr[$0]++} END{for (i in arr){print arr[i], i;}}' | sort -rn | cut -d' ' -f2- -- cgit v1.2.3 From f27ad9a3fdb2a6746ddb13fe4c117614c4b59364 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 27 Jul 2015 22:15:23 -0400 Subject: [pkg] add script to install base requirements - update pip - install base reqs --- pkg/pip_install_requirements.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100755 pkg/pip_install_requirements.sh diff --git a/pkg/pip_install_requirements.sh b/pkg/pip_install_requirements.sh new file mode 100755 index 0000000..29f03f3 --- /dev/null +++ b/pkg/pip_install_requirements.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# Update pip and install LEAP base requirements. +pip install -U pip +pip install -r pkg/requirements.pip -- cgit v1.2.3 From 5027dd2dd3c7679a7eea025d838a7d472c355623 Mon Sep 17 00:00:00 2001 From: Folker Bernitt Date: Wed, 29 Jul 2015 16:11:33 +0200 Subject: [bug] fixed syntax error in models.py The lambdas take two args, so it needs to be a tuple. Furthermore filter needs a collection. --- src/leap/mail/adaptors/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/leap/mail/adaptors/models.py b/src/leap/mail/adaptors/models.py index c5b838a..2bf9e60 100644 --- a/src/leap/mail/adaptors/models.py +++ b/src/leap/mail/adaptors/models.py @@ -115,8 +115,8 @@ class DocumentWrapper(object): def _normalize_dict(_dict): items = _dict.items() - items = filter(lambda k, v: not callable(v), items) - items = filter(lambda k, v: not k.startswith('_')) + items = filter(lambda (k, v): not callable(v), items) + items = filter(lambda (k, v): not k.startswith('_'), items) items = [(k, v) if not k.endswith('_') else (k[:-1], v) for (k, v) in items] items = [(k.replace('-', '_'), v) for (k, v) in items] -- cgit v1.2.3 From 34f8a5999ef024a3c1004a5bc9ff62201b4c5faa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Parm=C3=A9nides=20GV?= Date: Fri, 31 Jul 2015 09:02:45 +0200 Subject: [feat] use wheels to install dependencies generate_wheels uses $WHEELHOUSE to generate and store the wheels for requirements.pip and requirements-testing.pip (if it exists). pip_install_requirements.sh installs requirements.pip from them if possible (if not, then it fetches them from pypi) or, if passed the --testing flag, it installs requirements-testing.pip. Related: #7327 --- pkg/generate_wheels.sh | 14 ++++++++ pkg/pip_install_requirements.sh | 71 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 3 deletions(-) create mode 100755 pkg/generate_wheels.sh diff --git a/pkg/generate_wheels.sh b/pkg/generate_wheels.sh new file mode 100755 index 0000000..6679d1d --- /dev/null +++ b/pkg/generate_wheels.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Generate wheels for dependencies +# For convenience, u1db and dirspec are allowed with insecure flags enabled. +# Use at your own risk. + +if [ "$WHEELHOUSE" = "" ]; then + WHEELHOUSE=$HOME/wheelhouse +fi + +pip wheel --wheel-dir $WHEELHOUSE pip +pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements.pip +if [ -f pkg/requirements-testing.pip ]; then + pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip +fi diff --git a/pkg/pip_install_requirements.sh b/pkg/pip_install_requirements.sh index 29f03f3..bd44457 100755 --- a/pkg/pip_install_requirements.sh +++ b/pkg/pip_install_requirements.sh @@ -1,4 +1,69 @@ #!/bin/sh -# Update pip and install LEAP base requirements. -pip install -U pip -pip install -r pkg/requirements.pip +# Update pip and install LEAP base/testing requirements. +# For convenience, $insecure_packages are allowed with insecure flags enabled. +# Use at your own risk. +# See $usage for help + +insecure_packages="" + +return_wheelhouse() { + if [ "$WHEELHOUSE" = "" ]; then + WHEELHOUSE=$HOME/wheelhouse + fi + + if [ ! -d "$WHEELHOUSE" ]; then + mkdir $WHEELHOUSE + fi + + echo "$WHEELHOUSE" +} + +show_help() { + usage="Usage: $0 [--testing]\n --testing\tInstall dependencies from requirements-testing.pip\n +\t\tOtherwise, it will install requirements.pip" + echo $usage + + exit 1 +} + +process_arguments() { + testing=false + while [ "$#" -gt 0 ]; do + # From http://stackoverflow.com/a/31443098 + case "$1" in + --help) show_help;; + --testing) testing=true; shift 1;; + + -h) show_help;; + -*) echo "unknown option: $1" >&2; exit 1;; + esac + done +} + +return_insecure_flags() { + for insecure_package in $insecure_packages; do + flags="$flags --allow-external $insecure_package --allow-unverified $insecure_package" + done + + echo $flags +} + +return_packages() { + if $testing ; then + packages="-r pkg/requirements-testing.pip" + else + packages="-r pkg/requirements.pip" + fi + + echo $packages +} + +process_arguments $@ +wheelhouse=`return_wheelhouse` +install_options="-U --find-links=$wheelhouse" +insecure_flags=`return_insecure_flags` +packages=`return_packages` + +pip install -U wheel +pip install $install_options pip +pip install $install_options $insecure_flags $packages -- cgit v1.2.3 From 16a7f2987c54ff38a08d39a7f2eaf1662a154115 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Mon, 3 Aug 2015 17:40:04 -0300 Subject: [tests] Added requirements-latest help HEAD development Added requirements-latest to make sure we always use the latest develop of all dependencies in an automated way --- pkg/requirements-latest.pip | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 pkg/requirements-latest.pip diff --git a/pkg/requirements-latest.pip b/pkg/requirements-latest.pip new file mode 100644 index 0000000..846a319 --- /dev/null +++ b/pkg/requirements-latest.pip @@ -0,0 +1,9 @@ +--index-url https://pypi.python.org/simple/ + +--allow-external u1db --allow-unverified u1db +--allow-external dirspec --allow-unverified dirspec +-e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common' +-e 'git+https://github.com/pixelated-project/soledad.git@develop#egg=leap.soledad.common&subdirectory=common/' +-e 'git+https://github.com/pixelated-project/soledad.git@develop#egg=leap.soledad.client&subdirectory=client/' +-e 'git+https://github.com/pixelated-project/keymanager.git@develop#egg=leap.keymanager' +-e . -- cgit v1.2.3 From 4777ad0d3ec5378de8b0c7ffbcecf2e19ad9c180 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Mon, 3 Aug 2015 19:18:31 -0300 Subject: [tests] Skipped tests that were not implemented Functionality that is not implemented should not fail, instead it should be skipped until it is implemented. Added a return to test_imap_store_fetch setup to make sure the deferred is being handled --- src/leap/mail/adaptors/tests/test_soledad_adaptor.py | 2 +- src/leap/mail/imap/tests/test_imap_store_fetch.py | 2 +- src/leap/mail/tests/test_mail.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py index 3dc79fe..bdc2c48 100644 --- a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -364,7 +364,7 @@ class SoledadMailAdaptorTestCase(SoledadTestMixin): def test_get_msg_from_metamsg_doc_id(self): # TODO complete-me! - self.fail() + self.skipTest("Not yet implemented") def test_create_msg(self): adaptor = self.get_adaptor() diff --git a/src/leap/mail/imap/tests/test_imap_store_fetch.py b/src/leap/mail/imap/tests/test_imap_store_fetch.py index 81f88fe..a71ff45 100644 --- a/src/leap/mail/imap/tests/test_imap_store_fetch.py +++ b/src/leap/mail/imap/tests/test_imap_store_fetch.py @@ -14,9 +14,9 @@ class StoreAndFetchTestCase(IMAP4HelperMixin): """ def setUp(self): - IMAP4HelperMixin.setUp(self) self.received_messages = self.received_uid = None self.result = None + return IMAP4HelperMixin.setUp(self) def addListener(self, x): pass diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py index 2c03933..2872f15 100644 --- a/src/leap/mail/tests/test_mail.py +++ b/src/leap/mail/tests/test_mail.py @@ -271,7 +271,7 @@ class MessageCollectionTestCase(SoledadTestMixin, CollectionMixin): def test_copy_msg(self): # TODO ---- update when implementing messagecopier # interface - self.fail("Not Yet Implemented") + self.skipTest("Not yet implemented") def test_delete_msg(self): d = self.add_msg_to_collection() @@ -390,7 +390,7 @@ class AccountTestCase(SoledadTestMixin): # XXX not yet implemented def test_get_collection_by_docs(self): - self.fail("Not Yet Implemented") + self.skipTest("Not Yet Implemented") def test_get_collection_by_tag(self): - self.fail("Not Yet Implemented") + self.skipTest("Not Yet Implemented") -- cgit v1.2.3 From 570bf35249b1271a9453d5970059e080f2947d17 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Tue, 4 Aug 2015 12:48:56 -0300 Subject: [tests] Removed outdated test This test was not updated for a while and it doesn't make sense in this context, when we create acceptance tests we can make sure we cover this feature The issue for acceptance tests is: https://leap.se/code/issues/7341 --- src/leap/mail/imap/tests/test_imap_store_fetch.py | 75 ----------------------- 1 file changed, 75 deletions(-) delete mode 100644 src/leap/mail/imap/tests/test_imap_store_fetch.py diff --git a/src/leap/mail/imap/tests/test_imap_store_fetch.py b/src/leap/mail/imap/tests/test_imap_store_fetch.py deleted file mode 100644 index a71ff45..0000000 --- a/src/leap/mail/imap/tests/test_imap_store_fetch.py +++ /dev/null @@ -1,75 +0,0 @@ -from twisted.protocols import loopback -from twisted.python import util - -from leap.mail.imap.tests.utils import IMAP4HelperMixin - -TEST_USER = "testuser@leap.se" -TEST_PASSWD = "1234" - - -class StoreAndFetchTestCase(IMAP4HelperMixin): - """ - Several tests to check that the internal storage representation - is able to render the message structures as we expect them. - """ - - def setUp(self): - self.received_messages = self.received_uid = None - self.result = None - return IMAP4HelperMixin.setUp(self) - - def addListener(self, x): - pass - - def removeListener(self, x): - pass - - def _addSignedMessage(self, _): - self.server.state = 'select' - infile = util.sibpath(__file__, 'rfc822.multi-signed.message') - raw = open(infile).read() - MBOX_NAME = "multipart/SIGNED" - - self.server.theAccount.addMailbox(MBOX_NAME) - mbox = self.server.theAccount.getMailbox(MBOX_NAME) - self.server.mbox = mbox - # return a deferred that will fire with UID - return self.server.mbox.messages.add_msg(raw) - - def _fetchWork(self, uids): - - def result(R): - self.result = R - - self.connected.addCallback( - self._addSignedMessage).addCallback( - lambda uid: self.function( - uids, uid=uid - ) # do NOT use seq numbers! - ).addCallback( - result - ).addCallback( - self._cbStopClient - ).addErrback(self._ebGeneral) - - d = loopback.loopbackTCP(self.server, self.client, noisy=False) - d.addCallback(lambda x: self.assertEqual(self.result, self.expected)) - return d - - def testMultiBody(self): - """ - Test that a multipart signed message is retrieved the same - as we stored it. - """ - self.function = self.client.fetchBody - messages = '1' - - # XXX review. This probably should give everything? - - self.expected = {1: { - 'RFC822.TEXT': 'This is an example of a signed message,\n' - 'with attachments.\n\n\n--=20\n' - 'Nihil sine chao! =E2=88=B4\n', - 'UID': '1'}} - # print "test multi: fetch uid", messages - return self._fetchWork(messages) -- cgit v1.2.3 From 5a97000032cf9cdb8dee5310126f6de11b09d942 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 4 Aug 2015 12:18:50 -0400 Subject: [style] pep8 fixes + pep8 script --- pep8 | 2 ++ src/leap/mail/imap/server.py | 9 ++++++--- src/leap/mail/imap/tests/utils.py | 3 ++- src/leap/mail/tests/__init__.py | 3 ++- 4 files changed, 12 insertions(+), 5 deletions(-) create mode 100755 pep8 diff --git a/pep8 b/pep8 new file mode 100755 index 0000000..d0da40e --- /dev/null +++ b/pep8 @@ -0,0 +1,2 @@ +#!/bin/sh +pep8 src/leap/mail diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 050521a..39f483f 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -158,11 +158,14 @@ class LEAPIMAPServer(imap4.IMAP4Server): else: mf = imap4.IMessageFile(msg, None) if mf is not None: - return imap4.FileProducer(mf.open()).beginProducing(self.transport) - return imap4.MessageProducer(msg, None, self._scheduler).beginProducing(self.transport) + return imap4.FileProducer( + mf.open()).beginProducing(self.transport) + return imap4.MessageProducer( + msg, None, self._scheduler).beginProducing(self.transport) else: - _w('BODY ' + imap4.collapseNestedLists([imap4.getBodyStructure(msg)])) + _w('BODY ' + + imap4.collapseNestedLists([imap4.getBodyStructure(msg)])) ################################################################## # diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index 83c3f29..a34538b 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -99,7 +99,8 @@ class IMAP4HelperMixin(SoledadTestMixin): # Soledad sync makes trial block forever. The sync it's mocked to # fix this problem. _mock_soledad_get_from_index can be used from # the tests to provide documents. - # TODO see here, possibly related? -- http://www.pythoneye.com/83_20424875/ + # TODO see here, possibly related? + # -- http://www.pythoneye.com/83_20424875/ self._soledad.sync = Mock() d = defer.Deferred() diff --git a/src/leap/mail/tests/__init__.py b/src/leap/mail/tests/__init__.py index b35107d..de0088f 100644 --- a/src/leap/mail/tests/__init__.py +++ b/src/leap/mail/tests/__init__.py @@ -34,7 +34,8 @@ from leap.common.testing.basetest import BaseLeapTest def _find_gpg(): gpg_path = distutils.spawn.find_executable('gpg') - return os.path.realpath(gpg_path) if gpg_path is not None else "/usr/bin/gpg" + return (os.path.realpath(gpg_path) + if gpg_path is not None else "/usr/bin/gpg") class TestCaseWithKeyManager(unittest.TestCase, BaseLeapTest): -- cgit v1.2.3 From 521b0621563aad45d2354ff86af698d3c3261fb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Parm=C3=A9nides=20GV?= Date: Thu, 6 Aug 2015 08:55:50 +0200 Subject: [feat] WHEELHOUSE can be a url + --use-leap-wheels --use-leap-wheels sets --trusted-host (remove it when we have a proper cert) and WHEELHOUSE to https://ftp.lizard.leap.se Until we get ftp.lizard cname, use lizard as the wheels server. - Related: #7339 --- pkg/pip_install_requirements.sh | 53 +++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/pkg/pip_install_requirements.sh b/pkg/pip_install_requirements.sh index bd44457..57732e2 100755 --- a/pkg/pip_install_requirements.sh +++ b/pkg/pip_install_requirements.sh @@ -1,38 +1,31 @@ -#!/bin/sh +#!/bin/bash # Update pip and install LEAP base/testing requirements. # For convenience, $insecure_packages are allowed with insecure flags enabled. # Use at your own risk. # See $usage for help insecure_packages="" - -return_wheelhouse() { - if [ "$WHEELHOUSE" = "" ]; then - WHEELHOUSE=$HOME/wheelhouse - fi - - if [ ! -d "$WHEELHOUSE" ]; then - mkdir $WHEELHOUSE - fi - - echo "$WHEELHOUSE" -} +leap_wheelhouse=https://lizard.leap.se/wheels show_help() { - usage="Usage: $0 [--testing]\n --testing\tInstall dependencies from requirements-testing.pip\n -\t\tOtherwise, it will install requirements.pip" - echo $usage + usage="Usage: $0 [--testing] [--use-leap-wheels]\n --testing\t\tInstall dependencies from requirements-testing.pip\n +\t\t\tOtherwise, it will install requirements.pip\n +--use-leap-wheels\tUse wheels from leap.se" + echo -e $usage exit 1 } process_arguments() { testing=false + use_leap_wheels=false + while [ "$#" -gt 0 ]; do # From http://stackoverflow.com/a/31443098 case "$1" in --help) show_help;; --testing) testing=true; shift 1;; + --use-leap-wheels) use_leap_wheels=true; shift 1;; -h) show_help;; -*) echo "unknown option: $1" >&2; exit 1;; @@ -40,6 +33,31 @@ process_arguments() { done } +return_wheelhouse() { + if $use_leap_wheels ; then + WHEELHOUSE=$leap_wheelhouse + elif [ "$WHEELHOUSE" = "" ]; then + WHEELHOUSE=$HOME/wheelhouse + fi + + # Tested with bash and zsh + if [[ $WHEELHOUSE != http* && ! -d "$WHEELHOUSE" ]]; then + mkdir $WHEELHOUSE + fi + + echo "$WHEELHOUSE" +} + +return_install_options() { + wheelhouse=`return_wheelhouse` + install_options="-U --find-links=$wheelhouse" + if $use_leap_wheels ; then + install_options="$install_options --trusted-host lizard.leap.se" + fi + + echo $install_options +} + return_insecure_flags() { for insecure_package in $insecure_packages; do flags="$flags --allow-external $insecure_package --allow-unverified $insecure_package" @@ -59,8 +77,7 @@ return_packages() { } process_arguments $@ -wheelhouse=`return_wheelhouse` -install_options="-U --find-links=$wheelhouse" +install_options=`return_install_options` insecure_flags=`return_insecure_flags` packages=`return_packages` -- cgit v1.2.3 From 68fdc62d4c2792fb7fdea90f1117cf6c6f1c5ffc Mon Sep 17 00:00:00 2001 From: Folker Bernitt Date: Tue, 11 Aug 2015 13:38:56 +0200 Subject: [bug] Fix typo in content-transfer-encoding in walk.py. The get_raw_docs method accesses header field content-transfer-encoding using the string 'content-transfer-type' so the raw doc dict always ends up with that value set to empty string. --- src/leap/mail/adaptors/tests/test_soledad_adaptor.py | 14 ++++++++++++++ src/leap/mail/walk.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py index bdc2c48..0ddea30 100644 --- a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -28,6 +28,9 @@ from leap.mail.adaptors.soledad import SoledadIndexMixin from leap.mail.adaptors.soledad import SoledadMailAdaptor from leap.mail.tests.common import SoledadTestMixin +from email.MIMEMultipart import MIMEMultipart +from email.mime.text import MIMEText + # DEBUG # import logging # logging.basicConfig(level=logging.DEBUG) @@ -330,6 +333,17 @@ class SoledadMailAdaptorTestCase(SoledadTestMixin): self.assertEqual(msg.wrapper.hdoc.subject, subject) self.assertEqual(msg.wrapper.cdocs[1].phash, phash) + def test_get_msg_from_string_multipart(self): + msg = MIMEMultipart() + msg['Subject'] = 'Test multipart mail' + msg.attach(MIMEText(u'a utf8 message', _charset='utf-8')) + adaptor = self.get_adaptor() + + msg = adaptor.get_msg_from_string(TestMessageClass, msg.as_string()) + + self.assertEqual('base64', msg.wrapper.cdocs[1].content_transfer_encoding) + self.assertEqual('YSB1dGY4IG1lc3NhZ2U=\n', msg.wrapper.cdocs[1].raw) + def test_get_msg_from_docs(self): adaptor = self.get_adaptor() mdoc = dict( diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index f613309..1c74366 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -99,7 +99,7 @@ def get_raw_docs(msg, parts): "content-type": headers.get( 'content-type', ''), "content-transfer-encoding": headers.get( - 'content-transfer-type', '') + 'content-transfer-encoding', '') } for payload, headers in get_payloads(msg) if not isinstance(payload, list)) -- cgit v1.2.3 From 490effb589ad94f2f6f520b385399864b08def91 Mon Sep 17 00:00:00 2001 From: Folker Bernitt Date: Tue, 11 Aug 2015 13:42:24 +0200 Subject: [bug] Fix missing _normailize_dict in DocumentWrapper constructor. In the constructor values already is normalized (i.e. with underscores), while kwargs contains items that are not normalized (i.e. with dashes). Joining the dicts resulted in two entries that only differed by dash or underscores. The setattr then set the value that occurred later in items, thereby sometimes overriding the correct value with the default one. --- src/leap/mail/adaptors/models.py | 2 +- src/leap/mail/adaptors/tests/test_soledad_adaptor.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/leap/mail/adaptors/models.py b/src/leap/mail/adaptors/models.py index 2bf9e60..49460f7 100644 --- a/src/leap/mail/adaptors/models.py +++ b/src/leap/mail/adaptors/models.py @@ -76,7 +76,7 @@ class DocumentWrapper(object): if kwargs: values = copy.deepcopy(defaults) - values.update(kwargs) + values.update(_normalize_dict(kwargs)) else: values = defaults diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py index 0ddea30..499c2b1 100644 --- a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -342,6 +342,7 @@ class SoledadMailAdaptorTestCase(SoledadTestMixin): msg = adaptor.get_msg_from_string(TestMessageClass, msg.as_string()) self.assertEqual('base64', msg.wrapper.cdocs[1].content_transfer_encoding) + self.assertEqual('text/plain; charset="utf-8"', msg.wrapper.cdocs[1].content_type) self.assertEqual('YSB1dGY4IG1lc3NhZ2U=\n', msg.wrapper.cdocs[1].raw) def test_get_msg_from_docs(self): -- cgit v1.2.3 From ccc885df223c3e66dc8d560c8054752a30e3f08c Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 11 Aug 2015 22:58:34 -0400 Subject: [docs] add folker to authors + pep8 --- AUTHORS | 1 + src/leap/mail/adaptors/tests/test_soledad_adaptor.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index de3e0e3..db070f4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -5,3 +5,4 @@ Ivan Alejandro Ruben Pollan Bruno Wagner Goncalves Duda Dornelles +Folker Bernitt diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py index 499c2b1..b67e738 100644 --- a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -341,9 +341,12 @@ class SoledadMailAdaptorTestCase(SoledadTestMixin): msg = adaptor.get_msg_from_string(TestMessageClass, msg.as_string()) - self.assertEqual('base64', msg.wrapper.cdocs[1].content_transfer_encoding) - self.assertEqual('text/plain; charset="utf-8"', msg.wrapper.cdocs[1].content_type) - self.assertEqual('YSB1dGY4IG1lc3NhZ2U=\n', msg.wrapper.cdocs[1].raw) + self.assertEqual( + 'base64', msg.wrapper.cdocs[1].content_transfer_encoding) + self.assertEqual( + 'text/plain; charset="utf-8"', msg.wrapper.cdocs[1].content_type) + self.assertEqual( + 'YSB1dGY4IG1lc3NhZ2U=\n', msg.wrapper.cdocs[1].raw) def test_get_msg_from_docs(self): adaptor = self.get_adaptor() -- cgit v1.2.3 From c279e6587f9240d7eb9806996bc84d09cc5d8c97 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 18 Aug 2015 11:29:57 -0400 Subject: [style] pep8 excludes + fixes --- pep8 | 2 -- setup.cfg | 10 ++++++++++ setup.py | 8 ++++---- 3 files changed, 14 insertions(+), 6 deletions(-) delete mode 100755 pep8 create mode 100644 setup.cfg diff --git a/pep8 b/pep8 deleted file mode 100755 index d0da40e..0000000 --- a/pep8 +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -pep8 src/leap/mail diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..51070c6 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,10 @@ +[aliases] +test = trial + +[pep8] +exclude = versioneer.py,_version.py,*.egg,build,docs +ignore = E731 + +[flake8] +exclude = versioneer.py,_version.py,*.egg,build,docs +ignore = E731 diff --git a/setup.py b/setup.py index ead8982..575a6ec 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,10 @@ Setup file for leap.mail import re from setuptools import setup from setuptools import find_packages +from setuptools import Command + +from pkg import utils + import versioneer versioneer.versionfile_source = 'src/leap/mail/_version.py' @@ -27,7 +31,6 @@ versioneer.versionfile_build = 'leap/mail/_version.py' versioneer.tag_prefix = '' # tags are like 1.2.0 versioneer.parentdir_prefix = 'leap.mail-' -from pkg import utils trove_classifiers = [ 'Development Status :: 4 - Beta', @@ -63,9 +66,6 @@ if len(_version_short) > 0: cmdclass = versioneer.get_cmdclass() -from setuptools import Command - - class freeze_debianver(Command): """ Freezes the version in a debian branch. -- cgit v1.2.3 From da03f8933a59bde6b9c928416ca333630f04819a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 20 Aug 2015 23:45:19 -0400 Subject: [tests] properly skip not implemented tests, do not raise --- src/leap/mail/adaptors/tests/test_soledad_adaptor.py | 4 +++- src/leap/mail/tests/test_mail.py | 13 ++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py index b67e738..61e387c 100644 --- a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -382,7 +382,9 @@ class SoledadMailAdaptorTestCase(SoledadTestMixin): def test_get_msg_from_metamsg_doc_id(self): # TODO complete-me! - self.skipTest("Not yet implemented") + pass + + test_get_msg_from_metamsg_doc_id.skip = "Not yet implemented" def test_create_msg(self): adaptor = self.get_adaptor() diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py index 2872f15..9f40ffb 100644 --- a/src/leap/mail/tests/test_mail.py +++ b/src/leap/mail/tests/test_mail.py @@ -271,7 +271,8 @@ class MessageCollectionTestCase(SoledadTestMixin, CollectionMixin): def test_copy_msg(self): # TODO ---- update when implementing messagecopier # interface - self.skipTest("Not yet implemented") + pass + test_copy_msg.skip = "Not yet implemented" def test_delete_msg(self): d = self.add_msg_to_collection() @@ -387,10 +388,12 @@ class AccountTestCase(SoledadTestMixin): d.addCallback(assert_uid_next_empty_collection) return d - # XXX not yet implemented - def test_get_collection_by_docs(self): - self.skipTest("Not Yet Implemented") + pass + + test_get_collection_by_docs.skip = "Not yet implemented" def test_get_collection_by_tag(self): - self.skipTest("Not Yet Implemented") + pass + + test_get_collection_by_tag.skip = "Not yet implemented" -- cgit v1.2.3 From 43c920f38d1c114e9044d2da15d4a05d5faab79d Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Wed, 26 Aug 2015 18:09:22 -0300 Subject: [pkg] fold in changes --- CHANGELOG | 8 ++++++++ changes/bug_7244_fix_nested_multipart | 1 - 2 files changed, 8 insertions(+), 1 deletion(-) delete mode 100644 changes/bug_7244_fix_nested_multipart diff --git a/CHANGELOG b/CHANGELOG index 885871f..c114f09 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,11 @@ +0.4.0rc2 Aug 26, 2015: + o Fix nested multipart rendering. Closes: #7244 + o Bugfix: fix keyerror when inserting msg on pending_inserts dict. + o Bugfix: Return the first cdoc if no body found + o Feature: add very basic support for message sequence numbers. + o Lots of style fixes and tests updates. + o Bugfix: fixed syntax error in models.py. + 0.4.0rc1 Jul 10, 2015: o Parse OpenPGP header and import keys from it. Closes: #3879. o Don't add any footer to the emails. Closes: #4692. diff --git a/changes/bug_7244_fix_nested_multipart b/changes/bug_7244_fix_nested_multipart deleted file mode 100644 index 2d9cd8d..0000000 --- a/changes/bug_7244_fix_nested_multipart +++ /dev/null @@ -1 +0,0 @@ -- Fix nested multipart rendering. Closes: #7244 -- cgit v1.2.3