diff options
Diffstat (limited to 'src/leap/mail/adaptors')
| -rw-r--r-- | src/leap/mail/adaptors/soledad.py | 173 | ||||
| l---------[-rw-r--r--] | src/leap/mail/adaptors/tests/rfc822.message | 87 | ||||
| -rw-r--r-- | src/leap/mail/adaptors/tests/test_soledad_adaptor.py | 110 |
3 files changed, 175 insertions, 195 deletions
diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 2e25f04..0b97869 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # soledad.py # Copyright (C) 2014 LEAP # @@ -20,6 +19,7 @@ Soledadad MailAdaptor module. import re from collections import defaultdict from email import message_from_string +from functools import partial from pycryptopp.hash import sha256 from twisted.internet import defer @@ -27,6 +27,7 @@ from zope.interface import implements from leap.common.check import leap_assert, leap_assert_type +from leap.mail import constants from leap.mail import walk from leap.mail.adaptors import soledad_indexes as indexes from leap.mail.constants import INBOX_NAME @@ -60,7 +61,6 @@ class SoledadDocumentWrapper(models.DocumentWrapper): It ensures atomicity of the document operations on creation, update and deletion. """ - # TODO we could also use a _dirty flag (in models) # We keep a dictionary with DeferredLocks, that will be @@ -79,6 +79,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper): def __init__(self, **kwargs): doc_id = kwargs.pop('doc_id', None) self._doc_id = doc_id + self._future_doc_id = kwargs.pop('future_doc_id', None) self._lock = defer.DeferredLock() super(SoledadDocumentWrapper, self).__init__(**kwargs) @@ -86,6 +87,13 @@ class SoledadDocumentWrapper(models.DocumentWrapper): def doc_id(self): return self._doc_id + @property + def future_doc_id(self): + return self._future_doc_id + + def set_future_doc_id(self, doc_id): + self._future_doc_id = doc_id + def create(self, store): """ Create the documents for this wrapper. @@ -105,8 +113,14 @@ class SoledadDocumentWrapper(models.DocumentWrapper): def update_doc_id(doc): self._doc_id = doc.doc_id + self._future_doc_id = None return doc - d = store.create_doc(self.serialize()) + + if self.future_doc_id is None: + d = store.create_doc(self.serialize()) + else: + d = store.create_doc(self.serialize(), + doc_id=self.future_doc_id) d.addCallback(update_doc_id) return d @@ -333,6 +347,12 @@ class FlagsDocWrapper(SoledadDocumentWrapper): class __meta__(object): index = "mbox" + def set_mbox(self, mbox): + # XXX raise error if already created, should use copy instead + new_id = constants.FDOCID.format(mbox=mbox, chash=self.chash) + self._future_doc_id = new_id + self.mbox = mbox + class HeaderDocWrapper(SoledadDocumentWrapper): @@ -370,6 +390,23 @@ class ContentDocWrapper(SoledadDocumentWrapper): index = "phash" +class MetaMsgDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "meta" + fdoc = "" + hdoc = "" + cdocs = [] + + def set_mbox(self, mbox): + # XXX raise error if already created, should use copy instead + chash = re.findall(constants.FDOCID_CHASH_RE, self.fdoc)[0] + new_id = constants.METAMSGID.format(mbox=mbox, chash=chash) + new_fdoc_id = constants.FDOCID.format(mbox=mbox, chash=chash) + self._future_doc_id = new_id + self.fdoc = new_fdoc_id + + class MessageWrapper(object): # TODO generalize wrapper composition? @@ -378,23 +415,32 @@ class MessageWrapper(object): implements(IMessageWrapper) - def __init__(self, fdoc, hdoc, cdocs=None): + def __init__(self, mdoc, fdoc, hdoc, cdocs=None): """ - Need at least a flag-document and a header-document to instantiate a - MessageWrapper. Content-documents can be retrieved lazily. + Need at least a metamsg-document, a flag-document and a header-document + to instantiate a MessageWrapper. Content-documents can be retrieved + lazily. cdocs, if any, should be a dictionary in which the keys are ascending integers, beginning at one, and the values are dictionaries with the content of the content-docs. """ + self.mdoc = MetaMsgDocWrapper(**mdoc) + self.fdoc = FlagsDocWrapper(**fdoc) + self.fdoc.set_future_doc_id(self.mdoc.fdoc) + self.hdoc = HeaderDocWrapper(**hdoc) + self.hdoc.set_future_doc_id(self.mdoc.hdoc) + if cdocs is None: cdocs = {} cdocs_keys = cdocs.keys() assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1) self.cdocs = dict([(key, ContentDocWrapper(**doc)) for (key, doc) in cdocs.items()]) + for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()): + cdoc.set_future_doc_id(doc_id) def create(self, store): """ @@ -403,16 +449,21 @@ class MessageWrapper(object): leap_assert(self.cdocs, "Need non empty cdocs to create the " "MessageWrapper documents") + leap_assert(self.mdoc.doc_id is None, + "Cannot create: mdoc has a doc_id") leap_assert(self.fdoc.doc_id is None, "Cannot create: fdoc has a doc_id") + # TODO check that the doc_ids in the mdoc are coherent # TODO I think we need to tolerate the no hdoc.doc_id case, for when we # are doing a copy to another mailbox. - leap_assert(self.hdoc.doc_id is None, - "Cannot create: hdoc has a doc_id") + # leap_assert(self.hdoc.doc_id is None, + # "Cannot create: hdoc has a doc_id") d = [] + d.append(self.mdoc.create(store)) d.append(self.fdoc.create(store)) - d.append(self.hdoc.create(store)) + if self.hdoc.doc_id is None: + d.append(self.hdoc.create(store)) for cdoc in self.cdocs.values(): if cdoc.doc_id is not None: # we could be just linking to an existing @@ -432,6 +483,25 @@ class MessageWrapper(object): # garbage collector. At least the fdoc can be unlinked. raise NotImplementedError() + def copy(self, store, newmailbox): + """ + Return a copy of this MessageWrapper in a new mailbox. + """ + # 1. copy the fdoc, mdoc + # 2. remove the doc_id of that fdoc + # 3. create it (with new doc_id) + # 4. return new wrapper (new meta too!) + raise NotImplementedError() + + def set_mbox(self, mbox): + """ + Set the mailbox for this wrapper. + This method should only be used before the Documents for the + MessageWrapper have been created, will raise otherwise. + """ + self.mdoc.set_mbox(mbox) + self.fdoc.set_mbox(mbox) + # # Mailboxes # @@ -535,6 +605,7 @@ class SoledadMailAdaptor(SoledadIndexMixin): store = None indexes = indexes.MAIL_INDEXES + mboxwrapper_klass = MailboxWrapper # Message handling @@ -552,11 +623,11 @@ class SoledadMailAdaptor(SoledadIndexMixin): :rtype: MessageClass instance. """ assert(MessageClass is not None) - fdoc, hdoc, cdocs = _split_into_parts(raw_msg) + mdoc, fdoc, hdoc, cdocs = _split_into_parts(raw_msg) return self.get_msg_from_docs( - MessageClass, fdoc, hdoc, cdocs) + MessageClass, mdoc, fdoc, hdoc, cdocs) - def get_msg_from_docs(self, MessageClass, fdoc, hdoc, cdocs=None): + def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None): """ Get an instance of a MessageClass initialized with a MessageWrapper that contains the passed part documents. @@ -582,7 +653,62 @@ class SoledadMailAdaptor(SoledadIndexMixin): :rtype: MessageClass instance. """ assert(MessageClass is not None) - return MessageClass(MessageWrapper(fdoc, hdoc, cdocs)) + return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs)) + + def _get_msg_from_variable_doc_list(self, doc_list, msg_class): + if len(doc_list) == 2: + fdoc, hdoc = doc_list + cdocs = None + elif len(doc_list) > 2: + fdoc, hdoc = doc_list[:2] + cdocs = dict(enumerate(doc_list[2:], 1)) + return self.get_msg_from_docs(msg_class, fdoc, hdoc, cdocs) + + def get_msg_from_mdoc_id(self, MessageClass, store, doc_id, + get_cdocs=False): + metamsg_id = doc_id + + def wrap_meta_doc(doc): + cls = MetaMsgDocWrapper + return cls(doc_id=doc.doc_id, **doc.content) + + def get_part_docs_from_mdoc_wrapper(wrapper): + d_docs = [] + d_docs.append(store.get_doc(wrapper.fdoc)) + d_docs.append(store.get_doc(wrapper.hdoc)) + for cdoc in wrapper.cdocs: + d_docs.append(store.get_doc(cdoc)) + d = defer.gatherResults(d_docs) + return d + + def get_parts_doc_from_mdoc_id(): + mbox = re.findall(constants.METAMSGID_MBOX_RE, doc_id)[0] + chash = re.findall(constants.METAMSGID_CHASH_RE, doc_id)[0] + + def _get_fdoc_id_from_mdoc_id(): + return constants.FDOCID.format(mbox=mbox, chash=chash) + + def _get_hdoc_id_from_mdoc_id(): + return constants.FDOCID.format(mbox=mbox, chash=chash) + + d_docs = [] + fdoc_id = _get_fdoc_id_from_mdoc_id(doc_id) + hdoc_id = _get_hdoc_id_from_mdoc_id(doc_id) + d_docs.append(store.get_doc(fdoc_id)) + d_docs.append(store.get_doc(hdoc_id)) + d = defer.gatherResults(d_docs) + return d + + if get_cdocs: + d = store.get_doc(metamsg_id) + d.addCallback(wrap_meta_doc) + d.addCallback(get_part_docs_from_mdoc_wrapper) + else: + d = get_parts_doc_from_mdoc_id() + + d.addCallback(partial(self._get_msg_from_variable_doc_list, + msg_class=MessageClass)) + return d def create_msg(self, store, msg): """ @@ -615,7 +741,7 @@ class SoledadMailAdaptor(SoledadIndexMixin): def get_or_create_mbox(self, store, name): """ - Get the mailbox with the given name, or creatre one if it does not + Get the mailbox with the given name, or create one if it does not exist. :param name: the name of the mailbox @@ -636,6 +762,9 @@ class SoledadMailAdaptor(SoledadIndexMixin): """ return mbox_wrapper.update(store) + def delete_mbox(self, store, mbox_wrapper): + return mbox_wrapper.delete(store) + def get_all_mboxes(self, store): """ Retrieve a list with wrappers for all the mailboxes. @@ -660,15 +789,17 @@ def _split_into_parts(raw): walk.get_body_phash_multi][int(multi)] body_phash = body_phash_fun(walk.get_payloads(msg)) parts_map = walk.walk_msg_tree(parts, body_phash=body_phash) + cdocs_list = list(walk.get_raw_docs(msg, parts)) + cdocs_phashes = [c['phash'] for c in cdocs_list] + mdoc = _build_meta_doc(chash, cdocs_phashes) fdoc = _build_flags_doc(chash, size, multi) hdoc = _build_headers_doc(msg, chash, parts_map) # The MessageWrapper expects a dict, one-indexed - cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1)) + cdocs = dict(enumerate(cdocs_list, 1)) - # XXX convert each to_dicts... - return fdoc, hdoc, cdocs + return mdoc, fdoc, hdoc, cdocs def _parse_msg(raw): @@ -680,6 +811,14 @@ def _parse_msg(raw): return msg, parts, chash, size, multi +def _build_meta_doc(chash, cdocs_phashes): + _mdoc = MetaMsgDocWrapper() + _mdoc.fdoc = constants.FDOCID.format(mbox=INBOX_NAME, chash=chash) + _mdoc.hdoc = constants.HDOCID.format(chash=chash) + _mdoc.cdocs = [constants.CDOCID.format(phash=p) for p in cdocs_phashes] + return _mdoc.serialize() + + def _build_flags_doc(chash, size, multi): _fdoc = FlagsDocWrapper(chash=chash, size=size, multi=multi) return _fdoc.serialize() diff --git a/src/leap/mail/adaptors/tests/rfc822.message b/src/leap/mail/adaptors/tests/rfc822.message index ee97ab9..b19cc28 100644..120000 --- a/src/leap/mail/adaptors/tests/rfc822.message +++ b/src/leap/mail/adaptors/tests/rfc822.message @@ -1,86 +1 @@ -Return-Path: <twisted-commits-admin@twistedmatrix.com> -Delivered-To: exarkun@meson.dyndns.org -Received: from localhost [127.0.0.1] - by localhost with POP3 (fetchmail-6.2.1) - for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) -Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) - by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 - for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST) -Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) - by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) - id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 -Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) - id 18w63j-0007VK-00 - for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600 -To: twisted-commits@twistedmatrix.com -From: etrepum CVS <etrepum@twistedmatrix.com> -Reply-To: twisted-python@twistedmatrix.com -X-Mailer: CVSToys -Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com> -Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. -Sender: twisted-commits-admin@twistedmatrix.com -Errors-To: twisted-commits-admin@twistedmatrix.com -X-BeenThere: twisted-commits@twistedmatrix.com -X-Mailman-Version: 2.0.11 -Precedence: bulk -List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help> -List-Post: <mailto:twisted-commits@twistedmatrix.com> -List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>, - <mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe> -List-Id: <twisted-commits.twistedmatrix.com> -List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>, - <mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe> -List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/> -Date: Thu, 20 Mar 2003 13:50:39 -0600 - -Modified files: -Twisted/twisted/python/rebuild.py 1.19 1.20 - -Log message: -rebuild now works on python versions from 2.2.0 and up. - - -ViewCVS links: -http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted - -Index: Twisted/twisted/python/rebuild.py -diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 ---- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 -+++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 -@@ -206,15 +206,27 @@ - clazz.__dict__.clear() - clazz.__getattr__ = __getattr__ - clazz.__module__ = module.__name__ -+ if newclasses: -+ import gc -+ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): -+ hasBrokenRebuild = 1 -+ gc_objects = gc.get_objects() -+ else: -+ hasBrokenRebuild = 0 - for nclass in newclasses: - ga = getattr(module, nclass.__name__) - if ga is nclass: - log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) - else: -- import gc -- for r in gc.get_referrers(nclass): -- if isinstance(r, nclass): -+ if hasBrokenRebuild: -+ for r in gc_objects: -+ if not getattr(r, '__class__', None) is nclass: -+ continue - r.__class__ = ga -+ else: -+ for r in gc.get_referrers(nclass): -+ if getattr(r, '__class__', None) is nclass: -+ r.__class__ = ga - if doLog: - log.msg('') - log.msg(' (fixing %s): ' % str(module.__name__)) - - -_______________________________________________ -Twisted-commits mailing list -Twisted-commits@twistedmatrix.com -http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits +../../tests/rfc822.message
\ No newline at end of file diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py index 657a602..0cca5ef 100644 --- a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py +++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -18,106 +18,22 @@ Tests for the Soledad Adaptor module - leap.mail.adaptors.soledad """ import os -import shutil -import tempfile - from functools import partial from twisted.internet import defer from twisted.trial import unittest -from leap.common.testing.basetest import BaseLeapTest from leap.mail.adaptors import models from leap.mail.adaptors.soledad import SoledadDocumentWrapper from leap.mail.adaptors.soledad import SoledadIndexMixin from leap.mail.adaptors.soledad import SoledadMailAdaptor -from leap.soledad.client import Soledad - -TEST_USER = "testuser@leap.se" -TEST_PASSWD = "1234" +from leap.mail.tests.common import SoledadTestMixin # DEBUG # import logging # logging.basicConfig(level=logging.DEBUG) -def initialize_soledad(email, gnupg_home, tempdir): - """ - Initializes soledad by hand - - :param email: ID for the user - :param gnupg_home: path to home used by gnupg - :param tempdir: path to temporal dir - :rtype: Soledad instance - """ - - uuid = "foobar-uuid" - passphrase = u"verysecretpassphrase" - secret_path = os.path.join(tempdir, "secret.gpg") - local_db_path = os.path.join(tempdir, "soledad.u1db") - server_url = "https://provider" - cert_file = "" - - soledad = Soledad( - uuid, - passphrase, - secret_path, - local_db_path, - server_url, - cert_file, - syncable=False) - - return soledad - - -# TODO move to common module -# XXX remove duplication -class SoledadTestMixin(BaseLeapTest): - """ - It is **VERY** important that this base is added *AFTER* unittest.TestCase - """ - - def setUp(self): - self.results = [] - - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir - - # Soledad: config info - self.gnupg_home = "%s/gnupg" % self.tempdir - self.email = 'leap@leap.se' - - # initialize soledad by hand so we can control keys - self._soledad = initialize_soledad( - self.email, - self.gnupg_home, - self.tempdir) - - def tearDown(self): - """ - tearDown method called after each test. - """ - self.results = [] - try: - self._soledad.close() - except Exception as exc: - print "ERROR WHILE CLOSING SOLEDAD" - # logging.exception(exc) - finally: - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check - assert 'leap_tests-' in self.tempdir - shutil.rmtree(self.tempdir) - - class CounterWrapper(SoledadDocumentWrapper): class model(models.SerializableModel): counter = 0 @@ -357,7 +273,7 @@ class SoledadDocWrapperTestCase(unittest.TestCase, SoledadTestMixin): d.addCallback(assert_actor_list_is_expected) return d -here = os.path.split(os.path.abspath(__file__))[0] +HERE = os.path.split(os.path.abspath(__file__))[0] class TestMessageClass(object): @@ -391,7 +307,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): def test_get_msg_from_string(self): adaptor = self.get_adaptor() - with open(os.path.join(here, "rfc822.message")) as f: + with open(os.path.join(HERE, "rfc822.message")) as f: raw = f.read() msg = adaptor.get_msg_from_string(TestMessageClass, raw) @@ -416,6 +332,10 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): def test_get_msg_from_docs(self): adaptor = self.get_adaptor() + mdoc = dict( + fdoc="F-Foobox-deadbeef", + hdoc="H-deadbeef", + cdocs=["C-deadabad"]) fdoc = dict( mbox="Foobox", flags=('\Seen', '\Nice'), @@ -423,13 +343,14 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): seen=False, deleted=False, recent=False, multi=False) hdoc = dict( + chash="deadbeef", subject="Test Msg") cdocs = { 1: dict( raw='This is a test message')} msg = adaptor.get_msg_from_docs( - TestMessageClass, fdoc, hdoc, cdocs=cdocs) + TestMessageClass, mdoc, fdoc, hdoc, cdocs=cdocs) self.assertEqual(msg.wrapper.fdoc.flags, ('\Seen', '\Nice')) self.assertEqual(msg.wrapper.fdoc.tags, @@ -441,15 +362,20 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): self.assertEqual(msg.wrapper.cdocs[1].raw, "This is a test message") + def test_get_msg_from_metamsg_doc_id(self): + # XXX complete-me! + self.fail() + def test_create_msg(self): adaptor = self.get_adaptor() - with open(os.path.join(here, "rfc822.message")) as f: + with open(os.path.join(HERE, "rfc822.message")) as f: raw = f.read() msg = adaptor.get_msg_from_string(TestMessageClass, raw) def check_create_result(created): - self.assertEqual(len(created), 3) + # that's one mdoc, one hdoc, one fdoc, one cdoc + self.assertEqual(len(created), 4) for doc in created: self.assertTrue( doc.__class__.__name__, @@ -461,7 +387,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): def test_update_msg(self): adaptor = self.get_adaptor() - with open(os.path.join(here, "rfc822.message")) as f: + with open(os.path.join(HERE, "rfc822.message")) as f: raw = f.read() def assert_msg_has_doc_id(ignored, msg): @@ -493,7 +419,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin): msg = adaptor.get_msg_from_string(TestMessageClass, raw) d = adaptor.create_msg(adaptor.store, msg) d.addCallback(lambda _: adaptor.store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 3)) + d.addCallback(partial(self.assert_num_docs, 4)) d.addCallback(assert_msg_has_doc_id, msg) d.addCallback(assert_msg_has_no_flags, msg) |