diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/leap/mail/adaptors/soledad.py | 173 | ||||
| l---------[-rw-r--r--] | src/leap/mail/adaptors/tests/rfc822.message | 87 | ||||
| -rw-r--r-- | src/leap/mail/adaptors/tests/test_soledad_adaptor.py | 110 | ||||
| -rw-r--r-- | src/leap/mail/constants.py | 17 | ||||
| -rw-r--r-- | src/leap/mail/mail.py | 296 | ||||
| -rw-r--r-- | src/leap/mail/mailbox_indexer.py | 254 | ||||
| -rw-r--r-- | src/leap/mail/tests/common.py | 106 | ||||
| -rw-r--r-- | src/leap/mail/tests/rfc822.message | 86 | ||||
| -rw-r--r-- | src/leap/mail/tests/test_mail.py | 95 | ||||
| -rw-r--r-- | src/leap/mail/tests/test_mailbox_indexer.py | 241 |
10 files changed, 1204 insertions, 261 deletions
diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 2e25f04..0b97869 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # soledad.py # Copyright (C) 2014 LEAP # @@ -20,6 +19,7 @@ Soledadad MailAdaptor module. import re from collections import defaultdict from email import message_from_string +from functools import partial from pycryptopp.hash import sha256 from twisted.internet import defer @@ -27,6 +27,7 @@ from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type +from leap.mail import constants from leap.mail import walk from leap.mail.adaptors import soledad_indexes as indexes from leap.mail.constants import INBOX_NAME @@ -60,7 +61,6 @@ class SoledadDocumentWrapper(models.DocumentWrapper): It ensures atomicity of the document operations on creation, update and deletion. """ - # TODO we could also use a _dirty flag (in models) # We keep a dictionary with DeferredLocks, that will be @@ -79,6 +79,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): def __init__(self, **kwargs): doc_id = kwargs.pop('doc_id', None) self._doc_id = doc_id + self._future_doc_id = kwargs.pop('future_doc_id', None) self._lock = defer.DeferredLock() super(SoledadDocumentWrapper, self).__init__(**kwargs) @@ -86,6 +87,13 @@ class SoledadDocumentWrapper(models.DocumentWrapper): def doc_id(self): return self._doc_id + @property + def future_doc_id(self): + return self._future_doc_id + + def set_future_doc_id(self, doc_id): + self._future_doc_id = doc_id + def create(self, store): """ Create the documents for this wrapper. @@ -105,8 +113,14 @@ class SoledadDocumentWrapper(models.DocumentWrapper): def update_doc_id(doc): self._doc_id = doc.doc_id + self._future_doc_id = None return doc - d = store.create_doc(self.serialize()) + + if self.future_doc_id is None: + d = store.create_doc(self.serialize()) + else: + d = store.create_doc(self.serialize(), + doc_id=self.future_doc_id) d.addCallback(update_doc_id) return d @@ -333,6 +347,12 @@ class FlagsDocWrapper(SoledadDocumentWrapper): class __meta__(object): index = "mbox" + def set_mbox(self, mbox): + # XXX raise error if already created, should use copy instead + new_id = constants.FDOCID.format(mbox=mbox, chash=self.chash) + self._future_doc_id = new_id + self.mbox = mbox + class HeaderDocWrapper(SoledadDocumentWrapper): @@ -370,6 +390,23 @@ class ContentDocWrapper(SoledadDocumentWrapper): index = "phash" +class MetaMsgDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "meta" + fdoc = "" + hdoc = "" + cdocs = [] + + def set_mbox(self, mbox): + # XXX raise error if already created, should use copy instead + chash = re.findall(constants.FDOCID_CHASH_RE, self.fdoc)[0] + new_id = constants.METAMSGID.format(mbox=mbox, chash=chash) + new_fdoc_id = constants.FDOCID.format(mbox=mbox, chash=chash) + self._future_doc_id = new_id + self.fdoc = new_fdoc_id + + class MessageWrapper(object): # TODO generalize wrapper composition? @@ -378,23 +415,32 @@ class MessageWrapper(object): implements(IMessageWrapper) - def __init__(self, fdoc, hdoc, cdocs=None): + def __init__(self, mdoc, fdoc, hdoc, cdocs=None): """ - Need at least a flag-document and a header-document to instantiate a - MessageWrapper. Content-documents can be retrieved lazily. + Need at least a metamsg-document, a flag-document and a header-document + to instantiate a MessageWrapper. Content-documents can be retrieved + lazily. cdocs, if any, should be a dictionary in which the keys are ascending integers, beginning at one, and the values are dictionaries with the content of the content-docs. """ + self.mdoc = MetaMsgDocWrapper(**mdoc) + self.fdoc = FlagsDocWrapper(**fdoc) + self.fdoc.set_future_doc_id(self.mdoc.fdoc) + self.hdoc = HeaderDocWrapper(**hdoc) + self.hdoc.set_future_doc_id(self.mdoc.hdoc) + if cdocs is None: cdocs = {} cdocs_keys = cdocs.keys() assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1) self.cdocs = dict([(key, ContentDocWrapper(**doc)) for (key, doc) in cdocs.items()]) + for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()): + cdoc.set_future_doc_id(doc_id) def create(self, store): """ @@ -403,16 +449,21 @@ class MessageWrapper(object): leap_assert(self.cdocs, "Need non empty cdocs to create the " "MessageWrapper documents") + leap_assert(self.mdoc.doc_id is None, + "Cannot create: mdoc has a doc_id") leap_assert(self.fdoc.doc_id is None, "Cannot create: fdoc has a doc_id") + # TODO check that the doc_ids in the mdoc are coherent # TODO I think we need to tolerate the no hdoc.doc_id case, for when we # are doing a copy to another mailbox. - leap_assert(self.hdoc.doc_id is None, - "Cannot create: hdoc has a doc_id") + # leap_assert(self.hdoc.doc_id is None, + # "Cannot create: hdoc has a doc_id") d = [] + d.append(self.mdoc.create(store)) d.append(self.fdoc.create(store)) - d.append(self.hdoc.create(store)) + if self.hdoc.doc_id is None: + d.append(self.hdoc.create(store)) for cdoc in self.cdocs.values(): if cdoc.doc_id is not None: # we could be just linking to an existing @@ -432,6 +483,25 @@ class MessageWrapper(object): # garbage collector. At least the fdoc can be unlinked. raise NotImplementedError() + def copy(self, store, newmailbox): + """ + Return a copy of this MessageWrapper in a new mailbox. + """ + # 1. copy the fdoc, mdoc + # 2. remove the doc_id of that fdoc + # 3. create it (with new doc_id) + # 4. return new wrapper (new meta too!) + raise NotImplementedError() + + def set_mbox(self, mbox): + """ + Set the mailbox for this wrapper. + This method should only be used before the Documents for the + MessageWrapper have been created, will raise otherwise. + """ + self.mdoc.set_mbox(mbox) + self.fdoc.set_mbox(mbox) + # # Mailboxes # @@ -535,6 +605,7 @@ class SoledadMailAdaptor(SoledadIndexMixin): store = None indexes = indexes.MAIL_INDEXES + mboxwrapper_klass = MailboxWrapper # Message handling @@ -552,11 +623,11 @@ class SoledadMailAdaptor(SoledadIndexMixin): :rtype: MessageClass instance. """ assert(MessageClass is not None) - fdoc, hdoc, cdocs = _split_into_parts(raw_msg) + mdoc, fdoc, hdoc, cdocs = _split_into_parts(raw_msg) return self.get_msg_from_docs( - MessageClass, fdoc, hdoc, cdocs) + MessageClass, mdoc, fdoc, hdoc, cdocs) - def get_msg_from_docs(self, MessageClass, fdoc, hdoc, cdocs=None): + def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None): """ Get an instance of a MessageClass initialized with a MessageWrapper that contains the passed part documents. @@ -582,7 +653,62 @@ class SoledadMailAdaptor(SoledadIndexMixin): :rtype: MessageClass instance. """ assert(MessageClass is not None) - return MessageClass(MessageWrapper(fdoc, hdoc, cdocs)) + return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs)) + + def _get_msg_from_variable_doc_list(self, doc_list, msg_class): + if len(doc_list) == 2: + fdoc, hdoc = doc_list + cdocs = None + elif len(doc_list) > 2: + fdoc, hdoc = doc_list[:2] + cdocs = dict(enumerate(doc_list[2:], 1)) + return self.get_msg_from_docs(msg_class, fdoc, hdoc, cdocs) + + def get_msg_from_mdoc_id(self, MessageClass, store, doc_id, + get_cdocs=False): + metamsg_id = doc_id + + def wrap_meta_doc(doc): + cls = MetaMsgDocWrapper + return cls(doc_id=doc.doc_id, **doc.content) + + def get_part_docs_from_mdoc_wrapper(wrapper): + d_docs = [] + d_docs.append(store.get_doc(wrapper.fdoc)) + d_docs.append(store.get_doc(wrapper.hdoc)) + for cdoc in wrapper.cdocs: + d_docs.append(store.get_doc(cdoc)) + d = defer.gatherResults(d_docs) + return d + + def get_parts_doc_from_mdoc_id(): + mbox = re.findall(constants.METAMSGID_MBOX_RE, doc_id)[0] + chash = re.findall(constants.METAMSGID_CHASH_RE, doc_id)[0] + + def _get_fdoc_id_from_mdoc_id(): + return constants.FDOCID.format(mbox=mbox, chash=chash) + + def _get_hdoc_id_from_mdoc_id(): + return constants.FDOCID.format(mbox=mbox, chash=chash) + + d_docs = [] + fdoc_id = _get_fdoc_id_from_mdoc_id(doc_id) + hdoc_id = _get_hdoc_id_from_mdoc_id(doc_id) + d_docs.append(store.get_doc(fdoc_id)) + d_docs.append(store.get_doc(hdoc_id)) + d = defer.gatherResults(d_docs) + return d + + if get_cdocs: + d = store.get_doc(metamsg_id) + d.addCallback(wrap_meta_doc) + d.addCallback(get_part_docs_from_mdoc_wrapper) + else: + d = get_parts_doc_from_mdoc_id() + + d.addCallback(partial(self._get_msg_from_variable_doc_list, + msg_class=MessageClass)) + return d def create_msg(self, store, msg): """ @@ -615,7 +741,7 @@ class SoledadMailAdaptor(SoledadIndexMixin): def get_or_create_mbox(self, store, name): """ - Get the mailbox with the given name, or creatre one if it does not + Get the mailbox with the given name, or create one if it does not exist. :param name: the name of the mailbox @@ -636,6 +762,9 @@ class SoledadMailAdaptor(SoledadIndexMixin): """ return mbox_wrapper.update(store) + def delete_mbox(self, store, mbox_wrapper): + return mbox_wrapper.delete(store) + def get_all_mboxes(self, store): """ Retrieve a list with wrappers for all the mailboxes. @@ -660,15 +789,17 @@ def _split_into_parts(raw): walk.get_body_phash_multi][int(multi)] body_phash = body_phash_fun(walk.get_payloads(msg)) parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) + cdocs_list = list(walk.get_raw_docs(msg, parts)) + cdocs_phashes = [c['phash'] for c in cdocs_list] + mdoc = _build_meta_doc(chash, cdocs_phashes) fdoc = _build_flags_doc(chash, size, multi) hdoc = _build_headers_doc(msg, chash, parts_map) # The MessageWrapper expects a dict, one-indexed - cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1)) + cdocs = dict(enumerate(cdocs_list, 1)) - # XXX convert each to_dicts... - return fdoc, hdoc, cdocs + return mdoc, fdoc, hdoc, cdocs def _parse_msg(raw): @@ -680,6 +811,14 @@ def _parse_msg(raw): return msg, parts, chash, size, multi +def _build_meta_doc(chash, cdocs_phashes): + _mdoc = MetaMsgDocWrapper() + _mdoc.fdoc = constants.FDOCID.format(mbox=INBOX_NAME, chash=chash) + _mdoc.hdoc = constants.HDOCID.format(chash=chash) + _mdoc.cdocs = [constants.CDOCID.format(phash=p) for p in cdocs_phashes] + return _mdoc.serialize() + + def _build_flags_doc(chash, size, multi): _fdoc = FlagsDocWrapper(chash=chash, size=size, multi=multi) return _fdoc.serialize() diff --git a/src/leap/mail/adaptors/tests/rfc822.message b/src/leap/mail/adaptors/tests/rfc822.message index ee97ab9..b19cc28 100644..120000 --- a/src/leap/mail/adaptors/tests/rfc822.message +++ b/src/leap/mail/adaptors/tests/rfc822.message @@ -1,86 +1 @@ -Return-Path: <twisted-commits-admin@twistedmatrix.com> -Delivered-To: exarkun@meson.dyndns.org -Received: from localhost [127.0.0.1] - by localhost with POP3 (fetchmail-6.2.1) - for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) -Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) - by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 - for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST) -Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) - by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) - id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 -Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) - id 18w63j-0007VK-00 - for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600 -To: twisted-commits@twistedmatrix.com -From: etrepum CVS <etrepum@twistedmatrix.com> -Reply-To: twisted-python@twistedmatrix.com -X-Mailer: CVSToys -Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com> -Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. -Sender: twisted-commits-admin@twistedmatrix.com -Errors-To: twisted-commits-admin@twistedmatrix.com -X-BeenThere: twisted-commits@twistedmatrix.com -X-Mailman-Version: 2.0.11 -Precedence: bulk -List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help> -List-Post: <mailto:twisted-commits@twistedmatrix.com> -List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>, - <mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe> -List-Id: <twisted-commits.twistedmatrix.com> -List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>, - <mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe> -List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/> -Date: Thu, 20 Mar 2003 13:50:39 -0600 - -Modified files: -Twisted/twisted/python/rebuild.py 1.19 1.20 - -Log message: -rebuild now works on python versions from 2.2.0 and up. - - -ViewCVS links: -http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted - -Index: Twisted/twisted/python/rebuild.py -diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 ---- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 -+++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 -@@ -206,15 +206,27 @@ - clazz.__dict__.clear() - clazz.__getattr__ = __getattr__ - clazz.__module__ = module.__name__ -+ if newclasses: -+ import gc -+ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): -+ hasBrokenRebuild = 1 -+ gc_objects = gc.get_objects() -+ else: -+ hasBrokenRebuild = 0 - for nclass in newclasses: - ga = getattr(module, nclass.__name__) - if ga is nclass: - log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) - else: -- import gc -- for r in gc.get_referrers(nclass): -- if isinstance(r, nclass): -+ if hasBrokenRebuild: -+ for r in gc_objects: -+ if not getattr(r, '__class__', None) is nclass: -+ continue - r.__class__ = ga -+ else: -+ for r in gc.get_referrers(nclass): -+ if getattr(r, '__class__', None) is nclass: -+ r.__class__ = ga - if doLog: - log.msg('') - log.msg(' (fixing %s): ' % str(module.__name__)) - - -_______________________________________________ -Twisted-commits mailing list -Twisted-commits@twistedmatrix.com -http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits +../../tests/rfc822.message
\ No newline at end of file diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py index 657a602..0cca5ef 100644 --- a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -18,106 +18,22 @@ Tests for the Soledad Adaptor module - leap.mail.adaptors.soledad """ import os -import shutil -import tempfile - from functools import partial from twisted.internet import defer from twisted.trial import unittest -from leap.common.testing.basetest import BaseLeapTest from leap.mail.adaptors import models from leap.mail.adaptors.soledad import SoledadDocumentWrapper from leap.mail.adaptors.soledad import SoledadIndexMixin from leap.mail.adaptors.soledad import SoledadMailAdaptor -from leap.soledad.client import Soledad - -TEST_USER = "testuser@leap.se" -TEST_PASSWD = "1234" +from leap.mail.tests.common import SoledadTestMixin # DEBUG # import logging # logging.basicConfig(level=logging.DEBUG) -def initialize_soledad(email, gnupg_home, tempdir): - """ - Initializes soledad by hand - - :param email: ID for the user - :param gnupg_home: path to home used by gnupg - :param tempdir: path to temporal dir - :rtype: Soledad instance - """ - - uuid = "foobar-uuid" - passphrase = u"verysecretpassphrase" - secret_path = os.path.join(tempdir, "secret.gpg") - local_db_path = os.path.join(tempdir, "soledad.u1db") - server_url = "https://provider" - cert_file = "" - - soledad = Soledad( - uuid, - passphrase, - secret_path, - local_db_path, - server_url, - cert_file, - syncable=False) - - return soledad - - -# TODO move to common module -# XXX remove duplication -class SoledadTestMixin(BaseLeapTest): - """ - It is **VERY** important that this base is added *AFTER* unittest.TestCase - """ - - def setUp(self): - self.results = [] - - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir - - # Soledad: config info - self.gnupg_home = "%s/gnupg" % self.tempdir - self.email = 'leap@leap.se' - - # initialize soledad by hand so we can control keys - self._soledad = initialize_soledad( - self.email, - self.gnupg_home, - self.tempdir) - - def tearDown(self): - """ - tearDown method called after each test. - """ - self.results = [] - try: - self._soledad.close() - except Exception as exc: - print "ERROR WHILE CLOSING SOLEDAD" - # logging.exception(exc) - finally: - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check - assert 'leap_tests-' in self.tempdir - shutil.rmtree(self.tempdir) - - class CounterWrapper(SoledadDocumentWrapper): class model(models.SerializableModel): counter = 0 @@ -357,7 +273,7 @@ class SoledadDocWrapperTestCase(unittest.TestCase, SoledadTestMixin): d.addCallback(assert_actor_list_is_expected) return d -here = os.path.split(os.path.abspath(__file__))[0] +HERE = os.path.split(os.path.abspath(__file__))[0] class TestMessageClass(object): @@ -391,7 +307,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): def test_get_msg_from_string(self): adaptor = self.get_adaptor() - with open(os.path.join(here, "rfc822.message")) as f: + with open(os.path.join(HERE, "rfc822.message")) as f: raw = f.read() msg = adaptor.get_msg_from_string(TestMessageClass, raw) @@ -416,6 +332,10 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): def test_get_msg_from_docs(self): adaptor = self.get_adaptor() + mdoc = dict( + fdoc="F-Foobox-deadbeef", + hdoc="H-deadbeef", + cdocs=["C-deadabad"]) fdoc = dict( mbox="Foobox", flags=('\Seen', '\Nice'), @@ -423,13 +343,14 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): seen=False, deleted=False, recent=False, multi=False) hdoc = dict( + chash="deadbeef", subject="Test Msg") cdocs = { 1: dict( raw='This is a test message')} msg = adaptor.get_msg_from_docs( - TestMessageClass, fdoc, hdoc, cdocs=cdocs) + TestMessageClass, mdoc, fdoc, hdoc, cdocs=cdocs) self.assertEqual(msg.wrapper.fdoc.flags, ('\Seen', '\Nice')) self.assertEqual(msg.wrapper.fdoc.tags, @@ -441,15 +362,20 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): self.assertEqual(msg.wrapper.cdocs[1].raw, "This is a test message") + def test_get_msg_from_metamsg_doc_id(self): + # XXX complete-me! + self.fail() + def test_create_msg(self): adaptor = self.get_adaptor() - with open(os.path.join(here, "rfc822.message")) as f: + with open(os.path.join(HERE, "rfc822.message")) as f: raw = f.read() msg = adaptor.get_msg_from_string(TestMessageClass, raw) def check_create_result(created): - self.assertEqual(len(created), 3) + # that's one mdoc, one hdoc, one fdoc, one cdoc + self.assertEqual(len(created), 4) for doc in created: self.assertTrue( doc.__class__.__name__, @@ -461,7 +387,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): def test_update_msg(self): adaptor = self.get_adaptor() - with open(os.path.join(here, "rfc822.message")) as f: + with open(os.path.join(HERE, "rfc822.message")) as f: raw = f.read() def assert_msg_has_doc_id(ignored, msg): @@ -493,7 +419,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): msg = adaptor.get_msg_from_string(TestMessageClass, raw) d = adaptor.create_msg(adaptor.store, msg) d.addCallback(lambda _: adaptor.store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 3)) + d.addCallback(partial(self.assert_num_docs, 4)) d.addCallback(assert_msg_has_doc_id, msg) d.addCallback(assert_msg_has_no_flags, msg) diff --git a/src/leap/mail/constants.py b/src/leap/mail/constants.py index 55bf1da..bf1db7f 100644 --- a/src/leap/mail/constants.py +++ b/src/leap/mail/constants.py @@ -19,3 +19,20 @@ Constants for leap.mail. """ INBOX_NAME = "INBOX" + +# Regular expressions for the identifiers to be used in the Message Data Layer. + +METAMSGID = "M-{mbox}-{chash}" +METAMSGID_RE = "M\-{mbox}\-[0-9a-fA-F]+" +METAMSGID_CHASH_RE = "M\-\w+\-([0-9a-fA-F]+)" +METAMSGID_MBOX_RE = "M\-(\w+)\-[0-9a-fA-F]+" + +FDOCID = "F-{mbox}-{chash}" +FDOCID_RE = "F\-{mbox}\-[0-9a-fA-F]+" +FDOCID_CHASH_RE = "F\-\w+\-([0-9a-fA-F]+)" + +HDOCID = "H-{chash}" +HDOCID_RE = "H\-[0-9a-fA-F]+" + +CDOCID = "C-{phash}" +CDOCID_RE = "C\-[0-9a-fA-F]+" diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index ea9c95e..ca07f67 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -20,6 +20,7 @@ Generic Access to Mail objects: Public LEAP Mail API. from twisted.internet import defer from leap.mail.constants import INBOX_NAME +from leap.mail.mailbox_indexer import MailboxIndexer from leap.mail.adaptors.soledad import SoledadMailAdaptor @@ -27,8 +28,17 @@ from leap.mail.adaptors.soledad import SoledadMailAdaptor # [ ] Probably change the name of this module to "api" or "account", mail is # too generic (there's also IncomingMail, and OutgoingMail +def _get_mdoc_id(mbox, chash): + """ + Get the doc_id for the metamsg document. + """ + return "M+{mbox}+{chash}".format(mbox=mbox, chash=chash) + class Message(object): + """ + Represents a single message, and gives access to all its attributes. + """ def __init__(self, wrapper): """ @@ -37,45 +47,56 @@ class Message(object): self._wrapper = wrapper def get_wrapper(self): + """ + Get the wrapper for this message. + """ return self._wrapper # imap.IMessage methods - def get_flags(): + def get_flags(self): """ """ + return tuple(self._wrapper.fdoc.flags) - def get_internal_date(): + def get_internal_date(self): """ """ + return self._wrapper.fdoc.date # imap.IMessageParts - def get_headers(): + def get_headers(self): """ """ + # XXX process here? from imap.messages + return self._wrapper.hdoc.headers - def get_body_file(): + def get_body_file(self): """ """ - def get_size(): + def get_size(self): """ """ + return self._wrapper.fdoc.size - def is_multipart(): + def is_multipart(self): """ """ + return self._wrapper.fdoc.multi - def get_subpart(part): + def get_subpart(self, part): """ """ + # XXX ??? return MessagePart? # Custom methods. - def get_tags(): + def get_tags(self): """ """ + return tuple(self._wrapper.fdoc.tags) class MessageCollection(object): @@ -85,43 +106,174 @@ class MessageCollection(object): master documents. Since LEAP Mail is primarily oriented to store mail in Soledad, the default - (and, so far, only) implementation of the store is contained in this - Soledad Mail Adaptor. If you need to use a different adaptor, change the + (and, so far, only) implementation of the store is contained in the + Soledad Mail Adaptor, which is passed to every collection on creation by + the root Account object. If you need to use a different adaptor, change the adaptor class attribute in your Account object. Store is a reference to a particular instance of the message store (soledad instance or proxy, for instance). """ - # TODO look at IMessageSet methods + # TODO + # [ ] look at IMessageSet methods + # [ ] make constructor with a per-instance deferredLock to use on + # creation/deletion? + # [ ] instead of a mailbox, we could pass an arbitrary container with + # pointers to different doc_ids (type: foo) + # [ ] To guarantee synchronicity of the documents sent together during a + # sync, we could get hold of a deferredLock that inhibits + # synchronization while we are updating (think more about this!) # Account should provide an adaptor instance when creating this collection. adaptor = None store = None + messageklass = Message - def get_message_by_doc_id(self, doc_id): - # ... get from soledad etc - # ... but that should be part of adaptor/store too... :/ - fdoc, hdoc = None - return self.adaptor.from_docs(Message, fdoc=fdoc, hdoc=hdoc) + def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): + """ + """ + self.adaptor = adaptor + self.store = store - # TODO review if this is the best place for: + # TODO I have to think about what to do when there is no mbox passed to + # the initialization. We could still get the MetaMsg by index, instead + # of by doc_id. See get_message_by_content_hash + self.mbox_indexer = mbox_indexer + self.mbox_wrapper = mbox_wrapper - def create_docs(): - pass + def is_mailbox_collection(self): + """ + Return True if this collection represents a Mailbox. + :rtype: bool + """ + return bool(self.mbox_wrapper) + + # Get messages + + def get_message_by_content_hash(self, chash, get_cdocs=False): + """ + Retrieve a message by its content hash. + :rtype: Deferred + """ + + if not self.is_mailbox_collection(): + # instead of getting the metamsg by chash, query by (meta) index + # or use the internal collection of pointers-to-docs. + raise NotImplementedError() + + metamsg_id = _get_mdoc_id(self.mbox_wrapper.mbox, chash) + + return self.adaptor.get_msg_from_mdoc_id( + self.messageklass, self.store, + metamsg_id, get_cdocs=get_cdocs) + + def get_message_by_uid(self, uid, absolute=True, get_cdocs=False): + """ + Retrieve a message by its Unique Identifier. + + If this is a Mailbox collection, that is the message UID, unique for a + given mailbox, or a relative sequence number depending on the absolute + flag. For now, only absolute identifiers are supported. + :rtype: Deferred + """ + if not absolute: + raise NotImplementedError("Does not support relative ids yet") + + def get_msg_from_mdoc_id(doc_id): + return self.adaptor.get_msg_from_mdoc_id( + self.messageklass, self.store, + doc_id, get_cdocs=get_cdocs) + + d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_wrapper.mbox, uid) + d.addCallback(get_msg_from_mdoc_id) + return d + + def count(self): + """ + Count the messages in this collection. + :rtype: int + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + return self.mbox_indexer.count(self.mbox_wrapper.mbox) + + # Manipulate messages + + def add_msg(self, raw_msg): + """ + Add a message to this collection. + """ + msg = self.adaptor.get_msg_from_string(Message, raw_msg) + wrapper = msg.get_wrapper() + + if self.is_mailbox_collection(): + mbox = self.mbox_wrapper.mbox + wrapper.set_mbox(mbox) + + def insert_mdoc_id(_): + # XXX does this work? + doc_id = wrapper.mdoc.doc_id + return self.mbox_indexer.insert_doc( + self.mbox_wrapper.mbox, doc_id) + + d = wrapper.create(self.store) + d.addCallback(insert_mdoc_id) + return d + + def copy_msg(self, msg, newmailbox): + """ + Copy the message to another collection. (it only makes sense for + mailbox collections) + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + + def insert_copied_mdoc_id(wrapper): + return self.mbox_indexer.insert_doc( + newmailbox, wrapper.mdoc.doc_id) - def udpate_flags(): + wrapper = msg.get_wrapper() + d = wrapper.copy(self.store, newmailbox) + d.addCallback(insert_copied_mdoc_id) + return d + + def delete_msg(self, msg): + """ + Delete this message. + """ + wrapper = msg.get_wrapper() + + def delete_mdoc_id(_): + # XXX does this work? + doc_id = wrapper.mdoc.doc_id + return self.mbox_indexer.delete_doc_by_hash( + self.mbox_wrapper.mbox, doc_id) + d = wrapper.delete(self.store) + d.addCallback(delete_mdoc_id) + return d + + # TODO should add a delete-by-uid to collection? + + def udpate_flags(self, msg, flags, mode): + """ + Update flags for a given message. + """ + wrapper = msg.get_wrapper() # 1. update the flags in the message wrapper --- stored where??? - # 2. call adaptor.update_msg(store) + # 2. update the special flags in the wrapper (seen, etc) + # 3. call adaptor.update_msg(store) pass - def update_tags(): + def update_tags(self, msg, tags, mode): + """ + Update tags for a given message. + """ + wrapper = msg.get_wrapper() # 1. update the tags in the message wrapper --- stored where??? # 2. call adaptor.update_msg(store) pass - # TODO add delete methods here? - class Account(object): """ @@ -147,8 +299,8 @@ class Account(object): def __init__(self, store): self.store = store self.adaptor = self.adaptor_class() + self.mbox_indexer = MailboxIndexer(self.store) - self.__mailboxes = set([]) self._initialized = False self._deferred_initialization = defer.Deferred() @@ -156,23 +308,16 @@ class Account(object): def _initialize_storage(self): - def add_mailbox_if_none(result): - # every user should have the right to an inbox folder - # at least, so let's make one! - if not self.mailboxes: + def add_mailbox_if_none(mboxes): + if not mboxes: self.add_mailbox(INBOX_NAME) def finish_initialization(result): self._initialized = True self._deferred_initialization.callback(None) - def load_mbox_cache(result): - d = self._load_mailboxes() - d.addCallback(lambda _: result) - return d - d = self.adaptor.initialize_store(self.store) - d.addCallback(load_mbox_cache) + d.addCallback(self.list_all_mailbox_names) d.addCallback(add_mailbox_if_none) d.addCallback(finish_initialization) @@ -185,64 +330,83 @@ class Account(object): self._deferred_initialization.addCallback(cb) return self._deferred_initialization - @property - def mailboxes(self): - """ - A list of the current mailboxes for this account. - :rtype: set - """ - return sorted(self.__mailboxes) + # + # Public API Starts + # - def _load_mailboxes(self): + def list_all_mailbox_names(self): + def filter_names(mboxes): + return [m.name for m in mboxes] - def update_mailboxes(mbox_names): - self.__mailboxes.update(mbox_names) + d = self.get_all_mailboxes() + d.addCallback(filter_names) + return d + def get_all_mailboxes(self): d = self.adaptor.get_all_mboxes(self.store) - d.addCallback(update_mailboxes) return d - # - # Public API Starts - # + def add_mailbox(self, name): - # XXX params for IMAP only??? - def list_mailboxes(self, ref, wildcard): - self.adaptor.get_all_mboxes(self.store) - - def add_mailbox(self, name, mbox=None): - pass + def create_uid_table_cb(res): + d = self.mbox_uid.create_table(name) + d.addCallback(lambda _: res) + return d - def create_mailbox(self, pathspec): - pass + d = self.adaptor.__class__.get_or_create(name) + d.addCallback(create_uid_table_cb) + return d def delete_mailbox(self, name): - pass + def delete_uid_table_cb(res): + d = self.mbox_uid.delete_table(name) + d.addCallback(lambda _: res) + return d + + d = self.adaptor.delete_mbox(self.store) + d.addCallback(delete_uid_table_cb) + return d def rename_mailbox(self, oldname, newname): - pass + def _rename_mbox(wrapper): + wrapper.mbox = newname + return wrapper.update() - # FIXME yet to be decided if it belongs here... + def rename_uid_table_cb(res): + d = self.mbox_uid.rename_table(oldname, newname) + d.addCallback(lambda _: res) + return d + + d = self.adaptor.__class__.get_or_create(oldname) + d.addCallback(_rename_mbox) + d.addCallback(rename_uid_table_cb) + return d def get_collection_by_mailbox(self, name): """ :rtype: MessageCollection """ # imap select will use this, passing the collection to SoledadMailbox - # XXX pass adaptor to MessageCollection - pass + def get_collection_for_mailbox(mbox_wrapper): + return MessageCollection( + self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) + + mboxwrapper_klass = self.adaptor.mboxwrapper_klass + d = mboxwrapper_klass.get_or_create(name) + d.addCallback(get_collection_for_mailbox) + return d def get_collection_by_docs(self, docs): """ :rtype: MessageCollection """ # get a collection of docs by a list of doc_id - # XXX pass adaptor to MessageCollection - pass + # get.docs(...) --> it should be a generator. does it behave in the + # threadpool? + raise NotImplementedError() def get_collection_by_tag(self, tag): """ :rtype: MessageCollection """ - # is this a good idea? - pass + raise NotImplementedError() diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py new file mode 100644 index 0000000..bc298ea --- /dev/null +++ b/src/leap/mail/mailbox_indexer.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 -*- +# mailbox_indexer.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Local tables to store the message Unique Identifiers for a given mailbox. +""" +import re + +from leap.mail.constants import METAMSGID_RE + + +class WrongMetaDocIDError(Exception): + pass + + +class MailboxIndexer(object): + """ + This class contains the commands needed to create, modify and alter the + local-only UID tables for a given mailbox. + + Its purpouse is to keep a local-only index with the messages in each + mailbox, mainly to satisfy the demands of the IMAP specification, but + useful too for any effective listing of the messages in a mailbox. + + Since the incoming mail can be processed at any time in any replica, it's + preferred not to attempt to maintain a global chronological global index. + + These indexes are Message Attributes needed for the IMAP specification (rfc + 3501), although they can be useful for other non-imap store + implementations. + """ + # The uids are expected to be 32-bits values, but the ROWIDs in sqlite + # are 64-bit values. I *don't* think it really matters for any + # practical use, but it's good to remmeber we've got that difference going + # on. + + store = None + table_preffix = "leapmail_uid_" + + def __init__(self, store): + self.store = store + + def _query(self, *args, **kw): + assert self.store is not None + return self.store.raw_sqlcipher_query(*args, **kw) + + def create_table(self, mailbox): + """ + Create the UID table for a given mailbox. + :param mailbox: the mailbox name + :type mailbox: str + :rtype: Deferred + """ + assert mailbox + sql = ("CREATE TABLE if not exists {preffix}{name}( " + "uid INTEGER PRIMARY KEY AUTOINCREMENT, " + "hash TEXT UNIQUE NOT NULL)".format( + preffix=self.table_preffix, name=mailbox)) + return self._query(sql) + + def delete_table(self, mailbox): + """ + Delete the UID table for a given mailbox. + :param mailbox: the mailbox name + :type mailbox: str + :rtype: Deferred + """ + assert mailbox + sql = ("DROP TABLE if exists {preffix}{name}".format( + preffix=self.table_preffix, name=mailbox)) + return self._query(sql) + + def rename_table(self, oldmailbox, newmailbox): + """ + Delete the UID table for a given mailbox. + :param oldmailbox: the old mailbox name + :type oldmailbox: str + :param newmailbox: the new mailbox name + :type newmailbox: str + :rtype: Deferred + """ + assert oldmailbox + assert newmailbox + assert oldmailbox != newmailbox + sql = ("ALTER TABLE {preffix}{old} " + "RENAME TO {preffix}{new}".format( + preffix=self.table_preffix, + old=oldmailbox, new=newmailbox)) + return self._query(sql) + + def insert_doc(self, mailbox, doc_id): + """ + Insert the doc_id for a MetaMsg in the UID table for a given mailbox. + + The doc_id must be in the format: + + M+<mailbox>+<content-hash-of-the-message> + + :param mailbox: the mailbox name + :type mailbox: str + :param doc_id: the doc_id for the MetaMsg + :type doc_id: str + :return: a deferred that will fire with the uid of the newly inserted + document. + :rtype: Deferred + """ + assert mailbox + assert doc_id + + if not re.findall(METAMSGID_RE.format(mbox=mailbox), doc_id): + raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") + + def get_rowid(result): + return result[0][0] + + sql = ("INSERT INTO {preffix}{name} VALUES (" + "NULL, ?)".format( + preffix=self.table_preffix, name=mailbox)) + values = (doc_id,) + + sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " + "LIMIT 1;").format( + preffix=self.table_preffix, name=mailbox) + d = self._query(sql, values) + d.addCallback(lambda _: self._query(sql_last)) + d.addCallback(get_rowid) + return d + + def delete_doc_by_uid(self, mailbox, uid): + """ + Delete the entry for a MetaMsg in the UID table for a given mailbox. + + :param mailbox: the mailbox name + :type mailbox: str + :param uid: the UID of the message. + :type uid: int + :rtype: Deferred + """ + assert mailbox + assert uid + sql = ("DELETE FROM {preffix}{name} " + "WHERE uid=?".format( + preffix=self.table_preffix, name=mailbox)) + values = (uid,) + return self._query(sql, values) + + def delete_doc_by_hash(self, mailbox, doc_id): + """ + Delete the entry for a MetaMsg in the UID table for a given mailbox. + + The doc_id must be in the format: + + M+<mailbox>+<content-hash-of-the-message> + + :param mailbox: the mailbox name + :type mailbox: str + :param doc_id: the doc_id for the MetaMsg + :type doc_id: str + :return: a deferred that will fire with the uid of the newly inserted + document. + :rtype: Deferred + """ + assert mailbox + assert doc_id + sql = ("DELETE FROM {preffix}{name} " + "WHERE hash=?".format( + preffix=self.table_preffix, name=mailbox)) + values = (doc_id,) + return self._query(sql, values) + + def get_doc_id_from_uid(self, mailbox, uid): + """ + Get the doc_id for a MetaMsg in the UID table for a given mailbox. + + :param mailbox: the mailbox name + :type mailbox: str + :param uid: the uid for the MetaMsg for this mailbox + :type uid: int + :rtype: Deferred + """ + def get_hash(result): + return result[0][0] + + sql = ("SELECT hash from {preffix}{name} " + "WHERE uid=?".format( + preffix=self.table_preffix, name=mailbox)) + values = (uid,) + d = self._query(sql, values) + d.addCallback(get_hash) + return d + + def get_doc_ids_from_uids(self, mailbox, uids): + # For IMAP relative numbering /sequences. + # XXX dereference the range (n,*) + raise NotImplementedError() + + def count(self, mailbox): + """ + Get the number of entries in the UID table for a given mailbox. + + :param mailbox: the mailbox name + :type mailbox: str + :return: a deferred that will fire with an integer returning the count. + :rtype: Deferred + """ + def get_count(result): + return result[0][0] + + sql = ("SELECT Count(*) FROM {preffix}{name};".format( + preffix=self.table_preffix, name=mailbox)) + d = self._query(sql) + d.addCallback(get_count) + return d + + def get_next_uid(self, mailbox): + """ + Get the next integer beyond the highest UID count for a given mailbox. + + This is expected by the IMAP implementation. There are no guarantees + that a document to be inserted in the future gets the returned UID: the + only thing that can be assured is that it will be equal or greater than + the value returned. + + :param mailbox: the mailbox name + :type mailbox: str + :return: a deferred that will fire with an integer returning the next + uid. + :rtype: Deferred + """ + assert mailbox + + def increment(result): + return result[0][0] + 1 + + sql = ("SELECT MAX(rowid) FROM {preffix}{name} " + "LIMIT 1;").format( + preffix=self.table_preffix, name=mailbox) + + d = self._query(sql) + d.addCallback(increment) + return d diff --git a/src/leap/mail/tests/common.py b/src/leap/mail/tests/common.py new file mode 100644 index 0000000..fefa7ee --- /dev/null +++ b/src/leap/mail/tests/common.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +# common.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Common utilities for testing Soledad. +""" +import os +import shutil +import tempfile + +from leap.common.testing.basetest import BaseLeapTest +from leap.soledad.client import Soledad + +# TODO move to common module, or Soledad itself +# XXX remove duplication + +TEST_USER = "testuser@leap.se" +TEST_PASSWD = "1234" + + +def _initialize_soledad(email, gnupg_home, tempdir): + """ + Initializes soledad by hand + + :param email: ID for the user + :param gnupg_home: path to home used by gnupg + :param tempdir: path to temporal dir + :rtype: Soledad instance + """ + + uuid = "foobar-uuid" + passphrase = u"verysecretpassphrase" + secret_path = os.path.join(tempdir, "secret.gpg") + local_db_path = os.path.join(tempdir, "soledad.u1db") + server_url = "https://provider" + cert_file = "" + + soledad = Soledad( + uuid, + passphrase, + secret_path, + local_db_path, + server_url, + cert_file, + syncable=False) + + return soledad + + +class SoledadTestMixin(BaseLeapTest): + """ + It is **VERY** important that this base is added *AFTER* unittest.TestCase + """ + + def setUp(self): + self.results = [] + + self.old_path = os.environ['PATH'] + self.old_home = os.environ['HOME'] + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.home = self.tempdir + bin_tdir = os.path.join( + self.tempdir, + 'bin') + os.environ["PATH"] = bin_tdir + os.environ["HOME"] = self.tempdir + + # Soledad: config info + self.gnupg_home = "%s/gnupg" % self.tempdir + self.email = 'leap@leap.se' + + # initialize soledad by hand so we can control keys + self._soledad = _initialize_soledad( + self.email, + self.gnupg_home, + self.tempdir) + + def tearDown(self): + """ + tearDown method called after each test. + """ + self.results = [] + try: + self._soledad.close() + except Exception as exc: + print "ERROR WHILE CLOSING SOLEDAD" + # logging.exception(exc) + finally: + os.environ["PATH"] = self.old_path + os.environ["HOME"] = self.old_home + # safety check + assert 'leap_tests-' in self.tempdir + shutil.rmtree(self.tempdir) diff --git a/src/leap/mail/tests/rfc822.message b/src/leap/mail/tests/rfc822.message new file mode 100644 index 0000000..ee97ab9 --- /dev/null +++ b/src/leap/mail/tests/rfc822.message @@ -0,0 +1,86 @@ +Return-Path: <twisted-commits-admin@twistedmatrix.com> +Delivered-To: exarkun@meson.dyndns.org +Received: from localhost [127.0.0.1] + by localhost with POP3 (fetchmail-6.2.1) + for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) +Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) + by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 + for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST) +Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) + by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) + id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 +Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) + id 18w63j-0007VK-00 + for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600 +To: twisted-commits@twistedmatrix.com +From: etrepum CVS <etrepum@twistedmatrix.com> +Reply-To: twisted-python@twistedmatrix.com +X-Mailer: CVSToys +Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com> +Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. +Sender: twisted-commits-admin@twistedmatrix.com +Errors-To: twisted-commits-admin@twistedmatrix.com +X-BeenThere: twisted-commits@twistedmatrix.com +X-Mailman-Version: 2.0.11 +Precedence: bulk +List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help> +List-Post: <mailto:twisted-commits@twistedmatrix.com> +List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>, + <mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe> +List-Id: <twisted-commits.twistedmatrix.com> +List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>, + <mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe> +List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/> +Date: Thu, 20 Mar 2003 13:50:39 -0600 + +Modified files: +Twisted/twisted/python/rebuild.py 1.19 1.20 + +Log message: +rebuild now works on python versions from 2.2.0 and up. + + +ViewCVS links: +http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted + +Index: Twisted/twisted/python/rebuild.py +diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 +--- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 ++++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 +@@ -206,15 +206,27 @@ + clazz.__dict__.clear() + clazz.__getattr__ = __getattr__ + clazz.__module__ = module.__name__ ++ if newclasses: ++ import gc ++ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): ++ hasBrokenRebuild = 1 ++ gc_objects = gc.get_objects() ++ else: ++ hasBrokenRebuild = 0 + for nclass in newclasses: + ga = getattr(module, nclass.__name__) + if ga is nclass: + log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) + else: +- import gc +- for r in gc.get_referrers(nclass): +- if isinstance(r, nclass): ++ if hasBrokenRebuild: ++ for r in gc_objects: ++ if not getattr(r, '__class__', None) is nclass: ++ continue + r.__class__ = ga ++ else: ++ for r in gc.get_referrers(nclass): ++ if getattr(r, '__class__', None) is nclass: ++ r.__class__ = ga + if doLog: + log.msg('') + log.msg(' (fixing %s): ' % str(module.__name__)) + + +_______________________________________________ +Twisted-commits mailing list +Twisted-commits@twistedmatrix.com +http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py new file mode 100644 index 0000000..ce2366c --- /dev/null +++ b/src/leap/mail/tests/test_mail.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +# test_mail.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Tests for the mail module. +""" +import os +from functools import partial + +from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.mail.mail import MessageCollection +from leap.mail.mailbox_indexer import MailboxIndexer +from leap.mail.tests.common import SoledadTestMixin + +from twisted.internet import defer +from twisted.trial import unittest + +HERE = os.path.split(os.path.abspath(__file__))[0] + + +class MessageCollectionTestCase(unittest.TestCase, SoledadTestMixin): + """ + Tests for the SoledadDocumentWrapper. + """ + + def get_collection(self, mbox_collection=True): + """ + Get a collection for tests. + """ + adaptor = SoledadMailAdaptor() + store = self._soledad + adaptor.store = store + if mbox_collection: + mbox_indexer = MailboxIndexer(store) + mbox_name = "TestMbox" + else: + mbox_indexer = mbox_name = None + + def get_collection_from_mbox_wrapper(wrapper): + return MessageCollection( + adaptor, store, + mbox_indexer=mbox_indexer, mbox_wrapper=wrapper) + + d = adaptor.initialize_store(store) + if mbox_collection: + d.addCallback(lambda _: mbox_indexer.create_table(mbox_name)) + d.addCallback(lambda _: adaptor.get_or_create_mbox(store, mbox_name)) + d.addCallback(get_collection_from_mbox_wrapper) + return d + + def test_is_mailbox_collection(self): + + def assert_is_mbox_collection(collection): + self.assertTrue(collection.is_mailbox_collection()) + + d = self.get_collection() + d.addCallback(assert_is_mbox_collection) + return d + + def assert_collection_count(self, _, expected, collection): + + def _assert_count(count): + self.assertEqual(count, expected) + d = collection.count() + d.addCallback(_assert_count) + return d + + def test_add_msg(self): + + with open(os.path.join(HERE, "rfc822.message")) as f: + raw = f.read() + + def add_msg_to_collection_and_assert_count(collection): + d = collection.add_msg(raw) + d.addCallback(partial( + self.assert_collection_count, + expected=1, collection=collection)) + return d + + d = self.get_collection() + d.addCallback(add_msg_to_collection_and_assert_count) + return d diff --git a/src/leap/mail/tests/test_mailbox_indexer.py b/src/leap/mail/tests/test_mailbox_indexer.py new file mode 100644 index 0000000..47a3bdc --- /dev/null +++ b/src/leap/mail/tests/test_mailbox_indexer.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- +# test_mailbox_indexer.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Tests for the mailbox_indexer module. +""" +from functools import partial + +from twisted.trial import unittest + +from leap.mail import mailbox_indexer as mi +from leap.mail.tests.common import SoledadTestMixin + +hash_test0 = '590c9f8430c7435807df8ba9a476e3f1295d46ef210f6efae2043a4c085a569e' +hash_test1 = '1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014' +hash_test2 = '60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752' +hash_test3 = 'fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13' +hash_test4 = 'a4e624d686e03ed2767c0abd85c14426b0b1157d2ce81d27bb4fe4f6f01d688a' + + +def fmt_hash(mailbox, hash): + return "M-" + mailbox + "-" + hash + + +class MailboxIndexerTestCase(unittest.TestCase, SoledadTestMixin): + """ + Tests for the MailboxUID class. + """ + def get_mbox_uid(self): + m_uid = mi.MailboxIndexer(self._soledad) + return m_uid + + def list_mail_tables_cb(self, ignored): + def filter_mailuid_tables(tables): + filtered = [ + table[0] for table in tables if + table[0].startswith(mi.MailboxIndexer.table_preffix)] + return filtered + + sql = "SELECT name FROM sqlite_master WHERE type='table';" + d = self._soledad.raw_sqlcipher_query(sql) + d.addCallback(filter_mailuid_tables) + return d + + def select_uid_rows(self, mailbox): + sql = "SELECT * FROM %s%s;" % ( + mi.MailboxIndexer.table_preffix, mailbox) + d = self._soledad.raw_sqlcipher_query(sql) + return d + + def test_create_table(self): + def assert_table_created(tables): + self.assertEqual( + tables, ["leapmail_uid_inbox"]) + + m_uid = self.get_mbox_uid() + d = m_uid.create_table('inbox') + d.addCallback(self.list_mail_tables_cb) + d.addCallback(assert_table_created) + return d + + def test_create_and_delete_table(self): + def assert_table_deleted(tables): + self.assertEqual(tables, []) + + m_uid = self.get_mbox_uid() + d = m_uid.create_table('inbox') + d.addCallback(lambda _: m_uid.delete_table('inbox')) + d.addCallback(self.list_mail_tables_cb) + d.addCallback(assert_table_deleted) + return d + + def test_rename_table(self): + def assert_table_renamed(tables): + self.assertEqual( + tables, ["leapmail_uid_foomailbox"]) + + m_uid = self.get_mbox_uid() + d = m_uid.create_table('inbox') + d.addCallback(lambda _: m_uid.rename_table('inbox', 'foomailbox')) + d.addCallback(self.list_mail_tables_cb) + d.addCallback(assert_table_renamed) + return d + + def test_insert_doc(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + h1 = fmt_hash(mbox, hash_test0) + h2 = fmt_hash(mbox, hash_test1) + h3 = fmt_hash(mbox, hash_test2) + h4 = fmt_hash(mbox, hash_test3) + h5 = fmt_hash(mbox, hash_test4) + + def assert_uid_rows(rows): + expected = [(1, h1), (2, h2), (3, h3), (4, h4), (5, h5)] + self.assertEquals(rows, expected) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + d.addCallback(lambda _: self.select_uid_rows(mbox)) + d.addCallback(assert_uid_rows) + return d + + def test_insert_doc_return(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + def assert_rowid(rowid, expected=None): + self.assertEqual(rowid, expected) + + h1 = fmt_hash(mbox, hash_test0) + h2 = fmt_hash(mbox, hash_test1) + h3 = fmt_hash(mbox, hash_test2) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(partial(assert_rowid, expected=1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(partial(assert_rowid, expected=2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(partial(assert_rowid, expected=3)) + return d + + def test_delete_doc(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + h1 = fmt_hash(mbox, hash_test0) + h2 = fmt_hash(mbox, hash_test1) + h3 = fmt_hash(mbox, hash_test2) + h4 = fmt_hash(mbox, hash_test3) + h5 = fmt_hash(mbox, hash_test4) + + def assert_uid_rows(rows): + expected = [(4, h4), (5, h5)] + self.assertEquals(rows, expected) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 1)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 2)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_hash(mbox, h3)) + + d.addCallback(lambda _: self.select_uid_rows(mbox)) + d.addCallback(assert_uid_rows) + return d + + def test_get_doc_id_from_uid(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + h1 = fmt_hash(mbox, hash_test0) + + def assert_doc_hash(res): + self.assertEqual(res, h1) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(lambda _: m_uid.get_doc_id_from_uid(mbox, 1)) + d.addCallback(assert_doc_hash) + return d + + def test_count(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + h1 = fmt_hash(mbox, hash_test0) + h2 = fmt_hash(mbox, hash_test1) + h3 = fmt_hash(mbox, hash_test2) + h4 = fmt_hash(mbox, hash_test3) + h5 = fmt_hash(mbox, hash_test4) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + + def assert_count_after_inserts(count): + self.assertEquals(count, 5) + + d.addCallback(lambda _: m_uid.count(mbox)) + d.addCallback(assert_count_after_inserts) + + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 1)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 2)) + + def assert_count_after_deletions(count): + self.assertEquals(count, 3) + + d.addCallback(lambda _: m_uid.count(mbox)) + d.addCallback(assert_count_after_deletions) + return d + + def test_get_next_uid(self): + m_uid = self.get_mbox_uid() + mbox = 'foomailbox' + + h1 = fmt_hash(mbox, hash_test0) + h2 = fmt_hash(mbox, hash_test1) + h3 = fmt_hash(mbox, hash_test2) + h4 = fmt_hash(mbox, hash_test3) + h5 = fmt_hash(mbox, hash_test4) + + d = m_uid.create_table(mbox) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox, h5)) + + def assert_next_uid(result, expected=1): + self.assertEquals(result, expected) + + d.addCallback(lambda _: m_uid.get_next_uid(mbox)) + d.addCallback(partial(assert_next_uid, expected=6)) + return d |