summaryrefslogtreecommitdiff
path: root/src/leap/mail
diff options
context:
space:
mode:
Diffstat (limited to 'src/leap/mail')
-rw-r--r--src/leap/mail/adaptors/soledad.py173
l---------[-rw-r--r--]src/leap/mail/adaptors/tests/rfc822.message87
-rw-r--r--src/leap/mail/adaptors/tests/test_soledad_adaptor.py110
-rw-r--r--src/leap/mail/constants.py17
-rw-r--r--src/leap/mail/mail.py296
-rw-r--r--src/leap/mail/mailbox_indexer.py254
-rw-r--r--src/leap/mail/tests/common.py106
-rw-r--r--src/leap/mail/tests/rfc822.message86
-rw-r--r--src/leap/mail/tests/test_mail.py95
-rw-r--r--src/leap/mail/tests/test_mailbox_indexer.py241
10 files changed, 1204 insertions, 261 deletions
diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py
index 2e25f04..0b97869 100644
--- a/src/leap/mail/adaptors/soledad.py
+++ b/src/leap/mail/adaptors/soledad.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# soledad.py
# Copyright (C) 2014 LEAP
#
@@ -20,6 +19,7 @@ Soledadad MailAdaptor module.
import re
from collections import defaultdict
from email import message_from_string
+from functools import partial
from pycryptopp.hash import sha256
from twisted.internet import defer
@@ -27,6 +27,7 @@ from zope.interface import implements
from leap.common.check import leap_assert, leap_assert_type
+from leap.mail import constants
from leap.mail import walk
from leap.mail.adaptors import soledad_indexes as indexes
from leap.mail.constants import INBOX_NAME
@@ -60,7 +61,6 @@ class SoledadDocumentWrapper(models.DocumentWrapper):
It ensures atomicity of the document operations on creation, update and
deletion.
"""
-
# TODO we could also use a _dirty flag (in models)
# We keep a dictionary with DeferredLocks, that will be
@@ -79,6 +79,7 @@ class SoledadDocumentWrapper(models.DocumentWrapper):
def __init__(self, **kwargs):
doc_id = kwargs.pop('doc_id', None)
self._doc_id = doc_id
+ self._future_doc_id = kwargs.pop('future_doc_id', None)
self._lock = defer.DeferredLock()
super(SoledadDocumentWrapper, self).__init__(**kwargs)
@@ -86,6 +87,13 @@ class SoledadDocumentWrapper(models.DocumentWrapper):
def doc_id(self):
return self._doc_id
+ @property
+ def future_doc_id(self):
+ return self._future_doc_id
+
+ def set_future_doc_id(self, doc_id):
+ self._future_doc_id = doc_id
+
def create(self, store):
"""
Create the documents for this wrapper.
@@ -105,8 +113,14 @@ class SoledadDocumentWrapper(models.DocumentWrapper):
def update_doc_id(doc):
self._doc_id = doc.doc_id
+ self._future_doc_id = None
return doc
- d = store.create_doc(self.serialize())
+
+ if self.future_doc_id is None:
+ d = store.create_doc(self.serialize())
+ else:
+ d = store.create_doc(self.serialize(),
+ doc_id=self.future_doc_id)
d.addCallback(update_doc_id)
return d
@@ -333,6 +347,12 @@ class FlagsDocWrapper(SoledadDocumentWrapper):
class __meta__(object):
index = "mbox"
+ def set_mbox(self, mbox):
+ # XXX raise error if already created, should use copy instead
+ new_id = constants.FDOCID.format(mbox=mbox, chash=self.chash)
+ self._future_doc_id = new_id
+ self.mbox = mbox
+
class HeaderDocWrapper(SoledadDocumentWrapper):
@@ -370,6 +390,23 @@ class ContentDocWrapper(SoledadDocumentWrapper):
index = "phash"
+class MetaMsgDocWrapper(SoledadDocumentWrapper):
+
+ class model(models.SerializableModel):
+ type_ = "meta"
+ fdoc = ""
+ hdoc = ""
+ cdocs = []
+
+ def set_mbox(self, mbox):
+ # XXX raise error if already created, should use copy instead
+ chash = re.findall(constants.FDOCID_CHASH_RE, self.fdoc)[0]
+ new_id = constants.METAMSGID.format(mbox=mbox, chash=chash)
+ new_fdoc_id = constants.FDOCID.format(mbox=mbox, chash=chash)
+ self._future_doc_id = new_id
+ self.fdoc = new_fdoc_id
+
+
class MessageWrapper(object):
# TODO generalize wrapper composition?
@@ -378,23 +415,32 @@ class MessageWrapper(object):
implements(IMessageWrapper)
- def __init__(self, fdoc, hdoc, cdocs=None):
+ def __init__(self, mdoc, fdoc, hdoc, cdocs=None):
"""
- Need at least a flag-document and a header-document to instantiate a
- MessageWrapper. Content-documents can be retrieved lazily.
+ Need at least a metamsg-document, a flag-document and a header-document
+ to instantiate a MessageWrapper. Content-documents can be retrieved
+ lazily.
cdocs, if any, should be a dictionary in which the keys are ascending
integers, beginning at one, and the values are dictionaries with the
content of the content-docs.
"""
+ self.mdoc = MetaMsgDocWrapper(**mdoc)
+
self.fdoc = FlagsDocWrapper(**fdoc)
+ self.fdoc.set_future_doc_id(self.mdoc.fdoc)
+
self.hdoc = HeaderDocWrapper(**hdoc)
+ self.hdoc.set_future_doc_id(self.mdoc.hdoc)
+
if cdocs is None:
cdocs = {}
cdocs_keys = cdocs.keys()
assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1)
self.cdocs = dict([(key, ContentDocWrapper(**doc)) for (key, doc) in
cdocs.items()])
+ for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()):
+ cdoc.set_future_doc_id(doc_id)
def create(self, store):
"""
@@ -403,16 +449,21 @@ class MessageWrapper(object):
leap_assert(self.cdocs,
"Need non empty cdocs to create the "
"MessageWrapper documents")
+ leap_assert(self.mdoc.doc_id is None,
+ "Cannot create: mdoc has a doc_id")
leap_assert(self.fdoc.doc_id is None,
"Cannot create: fdoc has a doc_id")
+ # TODO check that the doc_ids in the mdoc are coherent
# TODO I think we need to tolerate the no hdoc.doc_id case, for when we
# are doing a copy to another mailbox.
- leap_assert(self.hdoc.doc_id is None,
- "Cannot create: hdoc has a doc_id")
+ # leap_assert(self.hdoc.doc_id is None,
+ # "Cannot create: hdoc has a doc_id")
d = []
+ d.append(self.mdoc.create(store))
d.append(self.fdoc.create(store))
- d.append(self.hdoc.create(store))
+ if self.hdoc.doc_id is None:
+ d.append(self.hdoc.create(store))
for cdoc in self.cdocs.values():
if cdoc.doc_id is not None:
# we could be just linking to an existing
@@ -432,6 +483,25 @@ class MessageWrapper(object):
# garbage collector. At least the fdoc can be unlinked.
raise NotImplementedError()
+ def copy(self, store, newmailbox):
+ """
+ Return a copy of this MessageWrapper in a new mailbox.
+ """
+ # 1. copy the fdoc, mdoc
+ # 2. remove the doc_id of that fdoc
+ # 3. create it (with new doc_id)
+ # 4. return new wrapper (new meta too!)
+ raise NotImplementedError()
+
+ def set_mbox(self, mbox):
+ """
+ Set the mailbox for this wrapper.
+ This method should only be used before the Documents for the
+ MessageWrapper have been created, will raise otherwise.
+ """
+ self.mdoc.set_mbox(mbox)
+ self.fdoc.set_mbox(mbox)
+
#
# Mailboxes
#
@@ -535,6 +605,7 @@ class SoledadMailAdaptor(SoledadIndexMixin):
store = None
indexes = indexes.MAIL_INDEXES
+ mboxwrapper_klass = MailboxWrapper
# Message handling
@@ -552,11 +623,11 @@ class SoledadMailAdaptor(SoledadIndexMixin):
:rtype: MessageClass instance.
"""
assert(MessageClass is not None)
- fdoc, hdoc, cdocs = _split_into_parts(raw_msg)
+ mdoc, fdoc, hdoc, cdocs = _split_into_parts(raw_msg)
return self.get_msg_from_docs(
- MessageClass, fdoc, hdoc, cdocs)
+ MessageClass, mdoc, fdoc, hdoc, cdocs)
- def get_msg_from_docs(self, MessageClass, fdoc, hdoc, cdocs=None):
+ def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None):
"""
Get an instance of a MessageClass initialized with a MessageWrapper
that contains the passed part documents.
@@ -582,7 +653,62 @@ class SoledadMailAdaptor(SoledadIndexMixin):
:rtype: MessageClass instance.
"""
assert(MessageClass is not None)
- return MessageClass(MessageWrapper(fdoc, hdoc, cdocs))
+ return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs))
+
+ def _get_msg_from_variable_doc_list(self, doc_list, msg_class):
+ if len(doc_list) == 2:
+ fdoc, hdoc = doc_list
+ cdocs = None
+ elif len(doc_list) > 2:
+ fdoc, hdoc = doc_list[:2]
+ cdocs = dict(enumerate(doc_list[2:], 1))
+ return self.get_msg_from_docs(msg_class, fdoc, hdoc, cdocs)
+
+ def get_msg_from_mdoc_id(self, MessageClass, store, doc_id,
+ get_cdocs=False):
+ metamsg_id = doc_id
+
+ def wrap_meta_doc(doc):
+ cls = MetaMsgDocWrapper
+ return cls(doc_id=doc.doc_id, **doc.content)
+
+ def get_part_docs_from_mdoc_wrapper(wrapper):
+ d_docs = []
+ d_docs.append(store.get_doc(wrapper.fdoc))
+ d_docs.append(store.get_doc(wrapper.hdoc))
+ for cdoc in wrapper.cdocs:
+ d_docs.append(store.get_doc(cdoc))
+ d = defer.gatherResults(d_docs)
+ return d
+
+ def get_parts_doc_from_mdoc_id():
+ mbox = re.findall(constants.METAMSGID_MBOX_RE, doc_id)[0]
+ chash = re.findall(constants.METAMSGID_CHASH_RE, doc_id)[0]
+
+ def _get_fdoc_id_from_mdoc_id():
+ return constants.FDOCID.format(mbox=mbox, chash=chash)
+
+ def _get_hdoc_id_from_mdoc_id():
+ return constants.FDOCID.format(mbox=mbox, chash=chash)
+
+ d_docs = []
+ fdoc_id = _get_fdoc_id_from_mdoc_id(doc_id)
+ hdoc_id = _get_hdoc_id_from_mdoc_id(doc_id)
+ d_docs.append(store.get_doc(fdoc_id))
+ d_docs.append(store.get_doc(hdoc_id))
+ d = defer.gatherResults(d_docs)
+ return d
+
+ if get_cdocs:
+ d = store.get_doc(metamsg_id)
+ d.addCallback(wrap_meta_doc)
+ d.addCallback(get_part_docs_from_mdoc_wrapper)
+ else:
+ d = get_parts_doc_from_mdoc_id()
+
+ d.addCallback(partial(self._get_msg_from_variable_doc_list,
+ msg_class=MessageClass))
+ return d
def create_msg(self, store, msg):
"""
@@ -615,7 +741,7 @@ class SoledadMailAdaptor(SoledadIndexMixin):
def get_or_create_mbox(self, store, name):
"""
- Get the mailbox with the given name, or creatre one if it does not
+ Get the mailbox with the given name, or create one if it does not
exist.
:param name: the name of the mailbox
@@ -636,6 +762,9 @@ class SoledadMailAdaptor(SoledadIndexMixin):
"""
return mbox_wrapper.update(store)
+ def delete_mbox(self, store, mbox_wrapper):
+ return mbox_wrapper.delete(store)
+
def get_all_mboxes(self, store):
"""
Retrieve a list with wrappers for all the mailboxes.
@@ -660,15 +789,17 @@ def _split_into_parts(raw):
walk.get_body_phash_multi][int(multi)]
body_phash = body_phash_fun(walk.get_payloads(msg))
parts_map = walk.walk_msg_tree(parts, body_phash=body_phash)
+ cdocs_list = list(walk.get_raw_docs(msg, parts))
+ cdocs_phashes = [c['phash'] for c in cdocs_list]
+ mdoc = _build_meta_doc(chash, cdocs_phashes)
fdoc = _build_flags_doc(chash, size, multi)
hdoc = _build_headers_doc(msg, chash, parts_map)
# The MessageWrapper expects a dict, one-indexed
- cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1))
+ cdocs = dict(enumerate(cdocs_list, 1))
- # XXX convert each to_dicts...
- return fdoc, hdoc, cdocs
+ return mdoc, fdoc, hdoc, cdocs
def _parse_msg(raw):
@@ -680,6 +811,14 @@ def _parse_msg(raw):
return msg, parts, chash, size, multi
+def _build_meta_doc(chash, cdocs_phashes):
+ _mdoc = MetaMsgDocWrapper()
+ _mdoc.fdoc = constants.FDOCID.format(mbox=INBOX_NAME, chash=chash)
+ _mdoc.hdoc = constants.HDOCID.format(chash=chash)
+ _mdoc.cdocs = [constants.CDOCID.format(phash=p) for p in cdocs_phashes]
+ return _mdoc.serialize()
+
+
def _build_flags_doc(chash, size, multi):
_fdoc = FlagsDocWrapper(chash=chash, size=size, multi=multi)
return _fdoc.serialize()
diff --git a/src/leap/mail/adaptors/tests/rfc822.message b/src/leap/mail/adaptors/tests/rfc822.message
index ee97ab9..b19cc28 100644..120000
--- a/src/leap/mail/adaptors/tests/rfc822.message
+++ b/src/leap/mail/adaptors/tests/rfc822.message
@@ -1,86 +1 @@
-Return-Path: <twisted-commits-admin@twistedmatrix.com>
-Delivered-To: exarkun@meson.dyndns.org
-Received: from localhost [127.0.0.1]
- by localhost with POP3 (fetchmail-6.2.1)
- for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST)
-Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105])
- by intarweb.us (Postfix) with ESMTP id 4A4A513EA4
- for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST)
-Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com)
- by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian))
- id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600
-Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian))
- id 18w63j-0007VK-00
- for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600
-To: twisted-commits@twistedmatrix.com
-From: etrepum CVS <etrepum@twistedmatrix.com>
-Reply-To: twisted-python@twistedmatrix.com
-X-Mailer: CVSToys
-Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com>
-Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up.
-Sender: twisted-commits-admin@twistedmatrix.com
-Errors-To: twisted-commits-admin@twistedmatrix.com
-X-BeenThere: twisted-commits@twistedmatrix.com
-X-Mailman-Version: 2.0.11
-Precedence: bulk
-List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help>
-List-Post: <mailto:twisted-commits@twistedmatrix.com>
-List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
- <mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe>
-List-Id: <twisted-commits.twistedmatrix.com>
-List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
- <mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe>
-List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/>
-Date: Thu, 20 Mar 2003 13:50:39 -0600
-
-Modified files:
-Twisted/twisted/python/rebuild.py 1.19 1.20
-
-Log message:
-rebuild now works on python versions from 2.2.0 and up.
-
-
-ViewCVS links:
-http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted
-
-Index: Twisted/twisted/python/rebuild.py
-diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20
---- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003
-+++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003
-@@ -206,15 +206,27 @@
- clazz.__dict__.clear()
- clazz.__getattr__ = __getattr__
- clazz.__module__ = module.__name__
-+ if newclasses:
-+ import gc
-+ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2):
-+ hasBrokenRebuild = 1
-+ gc_objects = gc.get_objects()
-+ else:
-+ hasBrokenRebuild = 0
- for nclass in newclasses:
- ga = getattr(module, nclass.__name__)
- if ga is nclass:
- log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass))
- else:
-- import gc
-- for r in gc.get_referrers(nclass):
-- if isinstance(r, nclass):
-+ if hasBrokenRebuild:
-+ for r in gc_objects:
-+ if not getattr(r, '__class__', None) is nclass:
-+ continue
- r.__class__ = ga
-+ else:
-+ for r in gc.get_referrers(nclass):
-+ if getattr(r, '__class__', None) is nclass:
-+ r.__class__ = ga
- if doLog:
- log.msg('')
- log.msg(' (fixing %s): ' % str(module.__name__))
-
-
-_______________________________________________
-Twisted-commits mailing list
-Twisted-commits@twistedmatrix.com
-http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits
+../../tests/rfc822.message \ No newline at end of file
diff --git a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py
index 657a602..0cca5ef 100644
--- a/src/leap/mail/adaptors/tests/test_soledad_adaptor.py
+++ b/src/leap/mail/adaptors/tests/test_soledad_adaptor.py
@@ -18,106 +18,22 @@
Tests for the Soledad Adaptor module - leap.mail.adaptors.soledad
"""
import os
-import shutil
-import tempfile
-
from functools import partial
from twisted.internet import defer
from twisted.trial import unittest
-from leap.common.testing.basetest import BaseLeapTest
from leap.mail.adaptors import models
from leap.mail.adaptors.soledad import SoledadDocumentWrapper
from leap.mail.adaptors.soledad import SoledadIndexMixin
from leap.mail.adaptors.soledad import SoledadMailAdaptor
-from leap.soledad.client import Soledad
-
-TEST_USER = "testuser@leap.se"
-TEST_PASSWD = "1234"
+from leap.mail.tests.common import SoledadTestMixin
# DEBUG
# import logging
# logging.basicConfig(level=logging.DEBUG)
-def initialize_soledad(email, gnupg_home, tempdir):
- """
- Initializes soledad by hand
-
- :param email: ID for the user
- :param gnupg_home: path to home used by gnupg
- :param tempdir: path to temporal dir
- :rtype: Soledad instance
- """
-
- uuid = "foobar-uuid"
- passphrase = u"verysecretpassphrase"
- secret_path = os.path.join(tempdir, "secret.gpg")
- local_db_path = os.path.join(tempdir, "soledad.u1db")
- server_url = "https://provider"
- cert_file = ""
-
- soledad = Soledad(
- uuid,
- passphrase,
- secret_path,
- local_db_path,
- server_url,
- cert_file,
- syncable=False)
-
- return soledad
-
-
-# TODO move to common module
-# XXX remove duplication
-class SoledadTestMixin(BaseLeapTest):
- """
- It is **VERY** important that this base is added *AFTER* unittest.TestCase
- """
-
- def setUp(self):
- self.results = []
-
- self.old_path = os.environ['PATH']
- self.old_home = os.environ['HOME']
- self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
- self.home = self.tempdir
- bin_tdir = os.path.join(
- self.tempdir,
- 'bin')
- os.environ["PATH"] = bin_tdir
- os.environ["HOME"] = self.tempdir
-
- # Soledad: config info
- self.gnupg_home = "%s/gnupg" % self.tempdir
- self.email = 'leap@leap.se'
-
- # initialize soledad by hand so we can control keys
- self._soledad = initialize_soledad(
- self.email,
- self.gnupg_home,
- self.tempdir)
-
- def tearDown(self):
- """
- tearDown method called after each test.
- """
- self.results = []
- try:
- self._soledad.close()
- except Exception as exc:
- print "ERROR WHILE CLOSING SOLEDAD"
- # logging.exception(exc)
- finally:
- os.environ["PATH"] = self.old_path
- os.environ["HOME"] = self.old_home
- # safety check
- assert 'leap_tests-' in self.tempdir
- shutil.rmtree(self.tempdir)
-
-
class CounterWrapper(SoledadDocumentWrapper):
class model(models.SerializableModel):
counter = 0
@@ -357,7 +273,7 @@ class SoledadDocWrapperTestCase(unittest.TestCase, SoledadTestMixin):
d.addCallback(assert_actor_list_is_expected)
return d
-here = os.path.split(os.path.abspath(__file__))[0]
+HERE = os.path.split(os.path.abspath(__file__))[0]
class TestMessageClass(object):
@@ -391,7 +307,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin):
def test_get_msg_from_string(self):
adaptor = self.get_adaptor()
- with open(os.path.join(here, "rfc822.message")) as f:
+ with open(os.path.join(HERE, "rfc822.message")) as f:
raw = f.read()
msg = adaptor.get_msg_from_string(TestMessageClass, raw)
@@ -416,6 +332,10 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin):
def test_get_msg_from_docs(self):
adaptor = self.get_adaptor()
+ mdoc = dict(
+ fdoc="F-Foobox-deadbeef",
+ hdoc="H-deadbeef",
+ cdocs=["C-deadabad"])
fdoc = dict(
mbox="Foobox",
flags=('\Seen', '\Nice'),
@@ -423,13 +343,14 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin):
seen=False, deleted=False,
recent=False, multi=False)
hdoc = dict(
+ chash="deadbeef",
subject="Test Msg")
cdocs = {
1: dict(
raw='This is a test message')}
msg = adaptor.get_msg_from_docs(
- TestMessageClass, fdoc, hdoc, cdocs=cdocs)
+ TestMessageClass, mdoc, fdoc, hdoc, cdocs=cdocs)
self.assertEqual(msg.wrapper.fdoc.flags,
('\Seen', '\Nice'))
self.assertEqual(msg.wrapper.fdoc.tags,
@@ -441,15 +362,20 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin):
self.assertEqual(msg.wrapper.cdocs[1].raw,
"This is a test message")
+ def test_get_msg_from_metamsg_doc_id(self):
+ # XXX complete-me!
+ self.fail()
+
def test_create_msg(self):
adaptor = self.get_adaptor()
- with open(os.path.join(here, "rfc822.message")) as f:
+ with open(os.path.join(HERE, "rfc822.message")) as f:
raw = f.read()
msg = adaptor.get_msg_from_string(TestMessageClass, raw)
def check_create_result(created):
- self.assertEqual(len(created), 3)
+ # that's one mdoc, one hdoc, one fdoc, one cdoc
+ self.assertEqual(len(created), 4)
for doc in created:
self.assertTrue(
doc.__class__.__name__,
@@ -461,7 +387,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin):
def test_update_msg(self):
adaptor = self.get_adaptor()
- with open(os.path.join(here, "rfc822.message")) as f:
+ with open(os.path.join(HERE, "rfc822.message")) as f:
raw = f.read()
def assert_msg_has_doc_id(ignored, msg):
@@ -493,7 +419,7 @@ class SoledadMailAdaptorTestCase(unittest.TestCase, SoledadTestMixin):
msg = adaptor.get_msg_from_string(TestMessageClass, raw)
d = adaptor.create_msg(adaptor.store, msg)
d.addCallback(lambda _: adaptor.store.get_all_docs())
- d.addCallback(partial(self.assert_num_docs, 3))
+ d.addCallback(partial(self.assert_num_docs, 4))
d.addCallback(assert_msg_has_doc_id, msg)
d.addCallback(assert_msg_has_no_flags, msg)
diff --git a/src/leap/mail/constants.py b/src/leap/mail/constants.py
index 55bf1da..bf1db7f 100644
--- a/src/leap/mail/constants.py
+++ b/src/leap/mail/constants.py
@@ -19,3 +19,20 @@ Constants for leap.mail.
"""
INBOX_NAME = "INBOX"
+
+# Regular expressions for the identifiers to be used in the Message Data Layer.
+
+METAMSGID = "M-{mbox}-{chash}"
+METAMSGID_RE = "M\-{mbox}\-[0-9a-fA-F]+"
+METAMSGID_CHASH_RE = "M\-\w+\-([0-9a-fA-F]+)"
+METAMSGID_MBOX_RE = "M\-(\w+)\-[0-9a-fA-F]+"
+
+FDOCID = "F-{mbox}-{chash}"
+FDOCID_RE = "F\-{mbox}\-[0-9a-fA-F]+"
+FDOCID_CHASH_RE = "F\-\w+\-([0-9a-fA-F]+)"
+
+HDOCID = "H-{chash}"
+HDOCID_RE = "H\-[0-9a-fA-F]+"
+
+CDOCID = "C-{phash}"
+CDOCID_RE = "C\-[0-9a-fA-F]+"
diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py
index ea9c95e..ca07f67 100644
--- a/src/leap/mail/mail.py
+++ b/src/leap/mail/mail.py
@@ -20,6 +20,7 @@ Generic Access to Mail objects: Public LEAP Mail API.
from twisted.internet import defer
from leap.mail.constants import INBOX_NAME
+from leap.mail.mailbox_indexer import MailboxIndexer
from leap.mail.adaptors.soledad import SoledadMailAdaptor
@@ -27,8 +28,17 @@ from leap.mail.adaptors.soledad import SoledadMailAdaptor
# [ ] Probably change the name of this module to "api" or "account", mail is
# too generic (there's also IncomingMail, and OutgoingMail
+def _get_mdoc_id(mbox, chash):
+ """
+ Get the doc_id for the metamsg document.
+ """
+ return "M+{mbox}+{chash}".format(mbox=mbox, chash=chash)
+
class Message(object):
+ """
+ Represents a single message, and gives access to all its attributes.
+ """
def __init__(self, wrapper):
"""
@@ -37,45 +47,56 @@ class Message(object):
self._wrapper = wrapper
def get_wrapper(self):
+ """
+ Get the wrapper for this message.
+ """
return self._wrapper
# imap.IMessage methods
- def get_flags():
+ def get_flags(self):
"""
"""
+ return tuple(self._wrapper.fdoc.flags)
- def get_internal_date():
+ def get_internal_date(self):
"""
"""
+ return self._wrapper.fdoc.date
# imap.IMessageParts
- def get_headers():
+ def get_headers(self):
"""
"""
+ # XXX process here? from imap.messages
+ return self._wrapper.hdoc.headers
- def get_body_file():
+ def get_body_file(self):
"""
"""
- def get_size():
+ def get_size(self):
"""
"""
+ return self._wrapper.fdoc.size
- def is_multipart():
+ def is_multipart(self):
"""
"""
+ return self._wrapper.fdoc.multi
- def get_subpart(part):
+ def get_subpart(self, part):
"""
"""
+ # XXX ??? return MessagePart?
# Custom methods.
- def get_tags():
+ def get_tags(self):
"""
"""
+ return tuple(self._wrapper.fdoc.tags)
class MessageCollection(object):
@@ -85,43 +106,174 @@ class MessageCollection(object):
master documents.
Since LEAP Mail is primarily oriented to store mail in Soledad, the default
- (and, so far, only) implementation of the store is contained in this
- Soledad Mail Adaptor. If you need to use a different adaptor, change the
+ (and, so far, only) implementation of the store is contained in the
+ Soledad Mail Adaptor, which is passed to every collection on creation by
+ the root Account object. If you need to use a different adaptor, change the
adaptor class attribute in your Account object.
Store is a reference to a particular instance of the message store (soledad
instance or proxy, for instance).
"""
- # TODO look at IMessageSet methods
+ # TODO
+ # [ ] look at IMessageSet methods
+ # [ ] make constructor with a per-instance deferredLock to use on
+ # creation/deletion?
+ # [ ] instead of a mailbox, we could pass an arbitrary container with
+ # pointers to different doc_ids (type: foo)
+ # [ ] To guarantee synchronicity of the documents sent together during a
+ # sync, we could get hold of a deferredLock that inhibits
+ # synchronization while we are updating (think more about this!)
# Account should provide an adaptor instance when creating this collection.
adaptor = None
store = None
+ messageklass = Message
- def get_message_by_doc_id(self, doc_id):
- # ... get from soledad etc
- # ... but that should be part of adaptor/store too... :/
- fdoc, hdoc = None
- return self.adaptor.from_docs(Message, fdoc=fdoc, hdoc=hdoc)
+ def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None):
+ """
+ """
+ self.adaptor = adaptor
+ self.store = store
- # TODO review if this is the best place for:
+ # TODO I have to think about what to do when there is no mbox passed to
+ # the initialization. We could still get the MetaMsg by index, instead
+ # of by doc_id. See get_message_by_content_hash
+ self.mbox_indexer = mbox_indexer
+ self.mbox_wrapper = mbox_wrapper
- def create_docs():
- pass
+ def is_mailbox_collection(self):
+ """
+ Return True if this collection represents a Mailbox.
+ :rtype: bool
+ """
+ return bool(self.mbox_wrapper)
+
+ # Get messages
+
+ def get_message_by_content_hash(self, chash, get_cdocs=False):
+ """
+ Retrieve a message by its content hash.
+ :rtype: Deferred
+ """
+
+ if not self.is_mailbox_collection():
+ # instead of getting the metamsg by chash, query by (meta) index
+ # or use the internal collection of pointers-to-docs.
+ raise NotImplementedError()
+
+ metamsg_id = _get_mdoc_id(self.mbox_wrapper.mbox, chash)
+
+ return self.adaptor.get_msg_from_mdoc_id(
+ self.messageklass, self.store,
+ metamsg_id, get_cdocs=get_cdocs)
+
+ def get_message_by_uid(self, uid, absolute=True, get_cdocs=False):
+ """
+ Retrieve a message by its Unique Identifier.
+
+ If this is a Mailbox collection, that is the message UID, unique for a
+ given mailbox, or a relative sequence number depending on the absolute
+ flag. For now, only absolute identifiers are supported.
+ :rtype: Deferred
+ """
+ if not absolute:
+ raise NotImplementedError("Does not support relative ids yet")
+
+ def get_msg_from_mdoc_id(doc_id):
+ return self.adaptor.get_msg_from_mdoc_id(
+ self.messageklass, self.store,
+ doc_id, get_cdocs=get_cdocs)
+
+ d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_wrapper.mbox, uid)
+ d.addCallback(get_msg_from_mdoc_id)
+ return d
+
+ def count(self):
+ """
+ Count the messages in this collection.
+ :rtype: int
+ """
+ if not self.is_mailbox_collection():
+ raise NotImplementedError()
+ return self.mbox_indexer.count(self.mbox_wrapper.mbox)
+
+ # Manipulate messages
+
+ def add_msg(self, raw_msg):
+ """
+ Add a message to this collection.
+ """
+ msg = self.adaptor.get_msg_from_string(Message, raw_msg)
+ wrapper = msg.get_wrapper()
+
+ if self.is_mailbox_collection():
+ mbox = self.mbox_wrapper.mbox
+ wrapper.set_mbox(mbox)
+
+ def insert_mdoc_id(_):
+ # XXX does this work?
+ doc_id = wrapper.mdoc.doc_id
+ return self.mbox_indexer.insert_doc(
+ self.mbox_wrapper.mbox, doc_id)
+
+ d = wrapper.create(self.store)
+ d.addCallback(insert_mdoc_id)
+ return d
+
+ def copy_msg(self, msg, newmailbox):
+ """
+ Copy the message to another collection. (it only makes sense for
+ mailbox collections)
+ """
+ if not self.is_mailbox_collection():
+ raise NotImplementedError()
+
+ def insert_copied_mdoc_id(wrapper):
+ return self.mbox_indexer.insert_doc(
+ newmailbox, wrapper.mdoc.doc_id)
- def udpate_flags():
+ wrapper = msg.get_wrapper()
+ d = wrapper.copy(self.store, newmailbox)
+ d.addCallback(insert_copied_mdoc_id)
+ return d
+
+ def delete_msg(self, msg):
+ """
+ Delete this message.
+ """
+ wrapper = msg.get_wrapper()
+
+ def delete_mdoc_id(_):
+ # XXX does this work?
+ doc_id = wrapper.mdoc.doc_id
+ return self.mbox_indexer.delete_doc_by_hash(
+ self.mbox_wrapper.mbox, doc_id)
+ d = wrapper.delete(self.store)
+ d.addCallback(delete_mdoc_id)
+ return d
+
+ # TODO should add a delete-by-uid to collection?
+
+ def udpate_flags(self, msg, flags, mode):
+ """
+ Update flags for a given message.
+ """
+ wrapper = msg.get_wrapper()
# 1. update the flags in the message wrapper --- stored where???
- # 2. call adaptor.update_msg(store)
+ # 2. update the special flags in the wrapper (seen, etc)
+ # 3. call adaptor.update_msg(store)
pass
- def update_tags():
+ def update_tags(self, msg, tags, mode):
+ """
+ Update tags for a given message.
+ """
+ wrapper = msg.get_wrapper()
# 1. update the tags in the message wrapper --- stored where???
# 2. call adaptor.update_msg(store)
pass
- # TODO add delete methods here?
-
class Account(object):
"""
@@ -147,8 +299,8 @@ class Account(object):
def __init__(self, store):
self.store = store
self.adaptor = self.adaptor_class()
+ self.mbox_indexer = MailboxIndexer(self.store)
- self.__mailboxes = set([])
self._initialized = False
self._deferred_initialization = defer.Deferred()
@@ -156,23 +308,16 @@ class Account(object):
def _initialize_storage(self):
- def add_mailbox_if_none(result):
- # every user should have the right to an inbox folder
- # at least, so let's make one!
- if not self.mailboxes:
+ def add_mailbox_if_none(mboxes):
+ if not mboxes:
self.add_mailbox(INBOX_NAME)
def finish_initialization(result):
self._initialized = True
self._deferred_initialization.callback(None)
- def load_mbox_cache(result):
- d = self._load_mailboxes()
- d.addCallback(lambda _: result)
- return d
-
d = self.adaptor.initialize_store(self.store)
- d.addCallback(load_mbox_cache)
+ d.addCallback(self.list_all_mailbox_names)
d.addCallback(add_mailbox_if_none)
d.addCallback(finish_initialization)
@@ -185,64 +330,83 @@ class Account(object):
self._deferred_initialization.addCallback(cb)
return self._deferred_initialization
- @property
- def mailboxes(self):
- """
- A list of the current mailboxes for this account.
- :rtype: set
- """
- return sorted(self.__mailboxes)
+ #
+ # Public API Starts
+ #
- def _load_mailboxes(self):
+ def list_all_mailbox_names(self):
+ def filter_names(mboxes):
+ return [m.name for m in mboxes]
- def update_mailboxes(mbox_names):
- self.__mailboxes.update(mbox_names)
+ d = self.get_all_mailboxes()
+ d.addCallback(filter_names)
+ return d
+ def get_all_mailboxes(self):
d = self.adaptor.get_all_mboxes(self.store)
- d.addCallback(update_mailboxes)
return d
- #
- # Public API Starts
- #
+ def add_mailbox(self, name):
- # XXX params for IMAP only???
- def list_mailboxes(self, ref, wildcard):
- self.adaptor.get_all_mboxes(self.store)
-
- def add_mailbox(self, name, mbox=None):
- pass
+ def create_uid_table_cb(res):
+ d = self.mbox_uid.create_table(name)
+ d.addCallback(lambda _: res)
+ return d
- def create_mailbox(self, pathspec):
- pass
+ d = self.adaptor.__class__.get_or_create(name)
+ d.addCallback(create_uid_table_cb)
+ return d
def delete_mailbox(self, name):
- pass
+ def delete_uid_table_cb(res):
+ d = self.mbox_uid.delete_table(name)
+ d.addCallback(lambda _: res)
+ return d
+
+ d = self.adaptor.delete_mbox(self.store)
+ d.addCallback(delete_uid_table_cb)
+ return d
def rename_mailbox(self, oldname, newname):
- pass
+ def _rename_mbox(wrapper):
+ wrapper.mbox = newname
+ return wrapper.update()
- # FIXME yet to be decided if it belongs here...
+ def rename_uid_table_cb(res):
+ d = self.mbox_uid.rename_table(oldname, newname)
+ d.addCallback(lambda _: res)
+ return d
+
+ d = self.adaptor.__class__.get_or_create(oldname)
+ d.addCallback(_rename_mbox)
+ d.addCallback(rename_uid_table_cb)
+ return d
def get_collection_by_mailbox(self, name):
"""
:rtype: MessageCollection
"""
# imap select will use this, passing the collection to SoledadMailbox
- # XXX pass adaptor to MessageCollection
- pass
+ def get_collection_for_mailbox(mbox_wrapper):
+ return MessageCollection(
+ self.adaptor, self.store, self.mbox_indexer, mbox_wrapper)
+
+ mboxwrapper_klass = self.adaptor.mboxwrapper_klass
+ d = mboxwrapper_klass.get_or_create(name)
+ d.addCallback(get_collection_for_mailbox)
+ return d
def get_collection_by_docs(self, docs):
"""
:rtype: MessageCollection
"""
# get a collection of docs by a list of doc_id
- # XXX pass adaptor to MessageCollection
- pass
+ # get.docs(...) --> it should be a generator. does it behave in the
+ # threadpool?
+ raise NotImplementedError()
def get_collection_by_tag(self, tag):
"""
:rtype: MessageCollection
"""
- # is this a good idea?
- pass
+ raise NotImplementedError()
diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py
new file mode 100644
index 0000000..bc298ea
--- /dev/null
+++ b/src/leap/mail/mailbox_indexer.py
@@ -0,0 +1,254 @@
+# -*- coding: utf-8 -*-
+# mailbox_indexer.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Local tables to store the message Unique Identifiers for a given mailbox.
+"""
+import re
+
+from leap.mail.constants import METAMSGID_RE
+
+
+class WrongMetaDocIDError(Exception):
+ pass
+
+
+class MailboxIndexer(object):
+ """
+ This class contains the commands needed to create, modify and alter the
+ local-only UID tables for a given mailbox.
+
+ Its purpouse is to keep a local-only index with the messages in each
+ mailbox, mainly to satisfy the demands of the IMAP specification, but
+ useful too for any effective listing of the messages in a mailbox.
+
+ Since the incoming mail can be processed at any time in any replica, it's
+ preferred not to attempt to maintain a global chronological global index.
+
+ These indexes are Message Attributes needed for the IMAP specification (rfc
+ 3501), although they can be useful for other non-imap store
+ implementations.
+ """
+ # The uids are expected to be 32-bits values, but the ROWIDs in sqlite
+ # are 64-bit values. I *don't* think it really matters for any
+ # practical use, but it's good to remmeber we've got that difference going
+ # on.
+
+ store = None
+ table_preffix = "leapmail_uid_"
+
+ def __init__(self, store):
+ self.store = store
+
+ def _query(self, *args, **kw):
+ assert self.store is not None
+ return self.store.raw_sqlcipher_query(*args, **kw)
+
+ def create_table(self, mailbox):
+ """
+ Create the UID table for a given mailbox.
+ :param mailbox: the mailbox name
+ :type mailbox: str
+ :rtype: Deferred
+ """
+ assert mailbox
+ sql = ("CREATE TABLE if not exists {preffix}{name}( "
+ "uid INTEGER PRIMARY KEY AUTOINCREMENT, "
+ "hash TEXT UNIQUE NOT NULL)".format(
+ preffix=self.table_preffix, name=mailbox))
+ return self._query(sql)
+
+ def delete_table(self, mailbox):
+ """
+ Delete the UID table for a given mailbox.
+ :param mailbox: the mailbox name
+ :type mailbox: str
+ :rtype: Deferred
+ """
+ assert mailbox
+ sql = ("DROP TABLE if exists {preffix}{name}".format(
+ preffix=self.table_preffix, name=mailbox))
+ return self._query(sql)
+
+ def rename_table(self, oldmailbox, newmailbox):
+ """
+ Delete the UID table for a given mailbox.
+ :param oldmailbox: the old mailbox name
+ :type oldmailbox: str
+ :param newmailbox: the new mailbox name
+ :type newmailbox: str
+ :rtype: Deferred
+ """
+ assert oldmailbox
+ assert newmailbox
+ assert oldmailbox != newmailbox
+ sql = ("ALTER TABLE {preffix}{old} "
+ "RENAME TO {preffix}{new}".format(
+ preffix=self.table_preffix,
+ old=oldmailbox, new=newmailbox))
+ return self._query(sql)
+
+ def insert_doc(self, mailbox, doc_id):
+ """
+ Insert the doc_id for a MetaMsg in the UID table for a given mailbox.
+
+ The doc_id must be in the format:
+
+ M+<mailbox>+<content-hash-of-the-message>
+
+ :param mailbox: the mailbox name
+ :type mailbox: str
+ :param doc_id: the doc_id for the MetaMsg
+ :type doc_id: str
+ :return: a deferred that will fire with the uid of the newly inserted
+ document.
+ :rtype: Deferred
+ """
+ assert mailbox
+ assert doc_id
+
+ if not re.findall(METAMSGID_RE.format(mbox=mailbox), doc_id):
+ raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id")
+
+ def get_rowid(result):
+ return result[0][0]
+
+ sql = ("INSERT INTO {preffix}{name} VALUES ("
+ "NULL, ?)".format(
+ preffix=self.table_preffix, name=mailbox))
+ values = (doc_id,)
+
+ sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} "
+ "LIMIT 1;").format(
+ preffix=self.table_preffix, name=mailbox)
+ d = self._query(sql, values)
+ d.addCallback(lambda _: self._query(sql_last))
+ d.addCallback(get_rowid)
+ return d
+
+ def delete_doc_by_uid(self, mailbox, uid):
+ """
+ Delete the entry for a MetaMsg in the UID table for a given mailbox.
+
+ :param mailbox: the mailbox name
+ :type mailbox: str
+ :param uid: the UID of the message.
+ :type uid: int
+ :rtype: Deferred
+ """
+ assert mailbox
+ assert uid
+ sql = ("DELETE FROM {preffix}{name} "
+ "WHERE uid=?".format(
+ preffix=self.table_preffix, name=mailbox))
+ values = (uid,)
+ return self._query(sql, values)
+
+ def delete_doc_by_hash(self, mailbox, doc_id):
+ """
+ Delete the entry for a MetaMsg in the UID table for a given mailbox.
+
+ The doc_id must be in the format:
+
+ M+<mailbox>+<content-hash-of-the-message>
+
+ :param mailbox: the mailbox name
+ :type mailbox: str
+ :param doc_id: the doc_id for the MetaMsg
+ :type doc_id: str
+ :return: a deferred that will fire with the uid of the newly inserted
+ document.
+ :rtype: Deferred
+ """
+ assert mailbox
+ assert doc_id
+ sql = ("DELETE FROM {preffix}{name} "
+ "WHERE hash=?".format(
+ preffix=self.table_preffix, name=mailbox))
+ values = (doc_id,)
+ return self._query(sql, values)
+
+ def get_doc_id_from_uid(self, mailbox, uid):
+ """
+ Get the doc_id for a MetaMsg in the UID table for a given mailbox.
+
+ :param mailbox: the mailbox name
+ :type mailbox: str
+ :param uid: the uid for the MetaMsg for this mailbox
+ :type uid: int
+ :rtype: Deferred
+ """
+ def get_hash(result):
+ return result[0][0]
+
+ sql = ("SELECT hash from {preffix}{name} "
+ "WHERE uid=?".format(
+ preffix=self.table_preffix, name=mailbox))
+ values = (uid,)
+ d = self._query(sql, values)
+ d.addCallback(get_hash)
+ return d
+
+ def get_doc_ids_from_uids(self, mailbox, uids):
+ # For IMAP relative numbering /sequences.
+ # XXX dereference the range (n,*)
+ raise NotImplementedError()
+
+ def count(self, mailbox):
+ """
+ Get the number of entries in the UID table for a given mailbox.
+
+ :param mailbox: the mailbox name
+ :type mailbox: str
+ :return: a deferred that will fire with an integer returning the count.
+ :rtype: Deferred
+ """
+ def get_count(result):
+ return result[0][0]
+
+ sql = ("SELECT Count(*) FROM {preffix}{name};".format(
+ preffix=self.table_preffix, name=mailbox))
+ d = self._query(sql)
+ d.addCallback(get_count)
+ return d
+
+ def get_next_uid(self, mailbox):
+ """
+ Get the next integer beyond the highest UID count for a given mailbox.
+
+ This is expected by the IMAP implementation. There are no guarantees
+ that a document to be inserted in the future gets the returned UID: the
+ only thing that can be assured is that it will be equal or greater than
+ the value returned.
+
+ :param mailbox: the mailbox name
+ :type mailbox: str
+ :return: a deferred that will fire with an integer returning the next
+ uid.
+ :rtype: Deferred
+ """
+ assert mailbox
+
+ def increment(result):
+ return result[0][0] + 1
+
+ sql = ("SELECT MAX(rowid) FROM {preffix}{name} "
+ "LIMIT 1;").format(
+ preffix=self.table_preffix, name=mailbox)
+
+ d = self._query(sql)
+ d.addCallback(increment)
+ return d
diff --git a/src/leap/mail/tests/common.py b/src/leap/mail/tests/common.py
new file mode 100644
index 0000000..fefa7ee
--- /dev/null
+++ b/src/leap/mail/tests/common.py
@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+# common.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Common utilities for testing Soledad.
+"""
+import os
+import shutil
+import tempfile
+
+from leap.common.testing.basetest import BaseLeapTest
+from leap.soledad.client import Soledad
+
+# TODO move to common module, or Soledad itself
+# XXX remove duplication
+
+TEST_USER = "testuser@leap.se"
+TEST_PASSWD = "1234"
+
+
+def _initialize_soledad(email, gnupg_home, tempdir):
+ """
+ Initializes soledad by hand
+
+ :param email: ID for the user
+ :param gnupg_home: path to home used by gnupg
+ :param tempdir: path to temporal dir
+ :rtype: Soledad instance
+ """
+
+ uuid = "foobar-uuid"
+ passphrase = u"verysecretpassphrase"
+ secret_path = os.path.join(tempdir, "secret.gpg")
+ local_db_path = os.path.join(tempdir, "soledad.u1db")
+ server_url = "https://provider"
+ cert_file = ""
+
+ soledad = Soledad(
+ uuid,
+ passphrase,
+ secret_path,
+ local_db_path,
+ server_url,
+ cert_file,
+ syncable=False)
+
+ return soledad
+
+
+class SoledadTestMixin(BaseLeapTest):
+ """
+ It is **VERY** important that this base is added *AFTER* unittest.TestCase
+ """
+
+ def setUp(self):
+ self.results = []
+
+ self.old_path = os.environ['PATH']
+ self.old_home = os.environ['HOME']
+ self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
+ self.home = self.tempdir
+ bin_tdir = os.path.join(
+ self.tempdir,
+ 'bin')
+ os.environ["PATH"] = bin_tdir
+ os.environ["HOME"] = self.tempdir
+
+ # Soledad: config info
+ self.gnupg_home = "%s/gnupg" % self.tempdir
+ self.email = 'leap@leap.se'
+
+ # initialize soledad by hand so we can control keys
+ self._soledad = _initialize_soledad(
+ self.email,
+ self.gnupg_home,
+ self.tempdir)
+
+ def tearDown(self):
+ """
+ tearDown method called after each test.
+ """
+ self.results = []
+ try:
+ self._soledad.close()
+ except Exception as exc:
+ print "ERROR WHILE CLOSING SOLEDAD"
+ # logging.exception(exc)
+ finally:
+ os.environ["PATH"] = self.old_path
+ os.environ["HOME"] = self.old_home
+ # safety check
+ assert 'leap_tests-' in self.tempdir
+ shutil.rmtree(self.tempdir)
diff --git a/src/leap/mail/tests/rfc822.message b/src/leap/mail/tests/rfc822.message
new file mode 100644
index 0000000..ee97ab9
--- /dev/null
+++ b/src/leap/mail/tests/rfc822.message
@@ -0,0 +1,86 @@
+Return-Path: <twisted-commits-admin@twistedmatrix.com>
+Delivered-To: exarkun@meson.dyndns.org
+Received: from localhost [127.0.0.1]
+ by localhost with POP3 (fetchmail-6.2.1)
+ for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST)
+Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105])
+ by intarweb.us (Postfix) with ESMTP id 4A4A513EA4
+ for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST)
+Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com)
+ by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian))
+ id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600
+Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian))
+ id 18w63j-0007VK-00
+ for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600
+To: twisted-commits@twistedmatrix.com
+From: etrepum CVS <etrepum@twistedmatrix.com>
+Reply-To: twisted-python@twistedmatrix.com
+X-Mailer: CVSToys
+Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com>
+Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up.
+Sender: twisted-commits-admin@twistedmatrix.com
+Errors-To: twisted-commits-admin@twistedmatrix.com
+X-BeenThere: twisted-commits@twistedmatrix.com
+X-Mailman-Version: 2.0.11
+Precedence: bulk
+List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help>
+List-Post: <mailto:twisted-commits@twistedmatrix.com>
+List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
+ <mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe>
+List-Id: <twisted-commits.twistedmatrix.com>
+List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
+ <mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe>
+List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/>
+Date: Thu, 20 Mar 2003 13:50:39 -0600
+
+Modified files:
+Twisted/twisted/python/rebuild.py 1.19 1.20
+
+Log message:
+rebuild now works on python versions from 2.2.0 and up.
+
+
+ViewCVS links:
+http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted
+
+Index: Twisted/twisted/python/rebuild.py
+diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20
+--- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003
++++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003
+@@ -206,15 +206,27 @@
+ clazz.__dict__.clear()
+ clazz.__getattr__ = __getattr__
+ clazz.__module__ = module.__name__
++ if newclasses:
++ import gc
++ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2):
++ hasBrokenRebuild = 1
++ gc_objects = gc.get_objects()
++ else:
++ hasBrokenRebuild = 0
+ for nclass in newclasses:
+ ga = getattr(module, nclass.__name__)
+ if ga is nclass:
+ log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass))
+ else:
+- import gc
+- for r in gc.get_referrers(nclass):
+- if isinstance(r, nclass):
++ if hasBrokenRebuild:
++ for r in gc_objects:
++ if not getattr(r, '__class__', None) is nclass:
++ continue
+ r.__class__ = ga
++ else:
++ for r in gc.get_referrers(nclass):
++ if getattr(r, '__class__', None) is nclass:
++ r.__class__ = ga
+ if doLog:
+ log.msg('')
+ log.msg(' (fixing %s): ' % str(module.__name__))
+
+
+_______________________________________________
+Twisted-commits mailing list
+Twisted-commits@twistedmatrix.com
+http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits
diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py
new file mode 100644
index 0000000..ce2366c
--- /dev/null
+++ b/src/leap/mail/tests/test_mail.py
@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+# test_mail.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the mail module.
+"""
+import os
+from functools import partial
+
+from leap.mail.adaptors.soledad import SoledadMailAdaptor
+from leap.mail.mail import MessageCollection
+from leap.mail.mailbox_indexer import MailboxIndexer
+from leap.mail.tests.common import SoledadTestMixin
+
+from twisted.internet import defer
+from twisted.trial import unittest
+
+HERE = os.path.split(os.path.abspath(__file__))[0]
+
+
+class MessageCollectionTestCase(unittest.TestCase, SoledadTestMixin):
+ """
+ Tests for the SoledadDocumentWrapper.
+ """
+
+ def get_collection(self, mbox_collection=True):
+ """
+ Get a collection for tests.
+ """
+ adaptor = SoledadMailAdaptor()
+ store = self._soledad
+ adaptor.store = store
+ if mbox_collection:
+ mbox_indexer = MailboxIndexer(store)
+ mbox_name = "TestMbox"
+ else:
+ mbox_indexer = mbox_name = None
+
+ def get_collection_from_mbox_wrapper(wrapper):
+ return MessageCollection(
+ adaptor, store,
+ mbox_indexer=mbox_indexer, mbox_wrapper=wrapper)
+
+ d = adaptor.initialize_store(store)
+ if mbox_collection:
+ d.addCallback(lambda _: mbox_indexer.create_table(mbox_name))
+ d.addCallback(lambda _: adaptor.get_or_create_mbox(store, mbox_name))
+ d.addCallback(get_collection_from_mbox_wrapper)
+ return d
+
+ def test_is_mailbox_collection(self):
+
+ def assert_is_mbox_collection(collection):
+ self.assertTrue(collection.is_mailbox_collection())
+
+ d = self.get_collection()
+ d.addCallback(assert_is_mbox_collection)
+ return d
+
+ def assert_collection_count(self, _, expected, collection):
+
+ def _assert_count(count):
+ self.assertEqual(count, expected)
+ d = collection.count()
+ d.addCallback(_assert_count)
+ return d
+
+ def test_add_msg(self):
+
+ with open(os.path.join(HERE, "rfc822.message")) as f:
+ raw = f.read()
+
+ def add_msg_to_collection_and_assert_count(collection):
+ d = collection.add_msg(raw)
+ d.addCallback(partial(
+ self.assert_collection_count,
+ expected=1, collection=collection))
+ return d
+
+ d = self.get_collection()
+ d.addCallback(add_msg_to_collection_and_assert_count)
+ return d
diff --git a/src/leap/mail/tests/test_mailbox_indexer.py b/src/leap/mail/tests/test_mailbox_indexer.py
new file mode 100644
index 0000000..47a3bdc
--- /dev/null
+++ b/src/leap/mail/tests/test_mailbox_indexer.py
@@ -0,0 +1,241 @@
+# -*- coding: utf-8 -*-
+# test_mailbox_indexer.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the mailbox_indexer module.
+"""
+from functools import partial
+
+from twisted.trial import unittest
+
+from leap.mail import mailbox_indexer as mi
+from leap.mail.tests.common import SoledadTestMixin
+
+hash_test0 = '590c9f8430c7435807df8ba9a476e3f1295d46ef210f6efae2043a4c085a569e'
+hash_test1 = '1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014'
+hash_test2 = '60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752'
+hash_test3 = 'fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13'
+hash_test4 = 'a4e624d686e03ed2767c0abd85c14426b0b1157d2ce81d27bb4fe4f6f01d688a'
+
+
+def fmt_hash(mailbox, hash):
+ return "M-" + mailbox + "-" + hash
+
+
+class MailboxIndexerTestCase(unittest.TestCase, SoledadTestMixin):
+ """
+ Tests for the MailboxUID class.
+ """
+ def get_mbox_uid(self):
+ m_uid = mi.MailboxIndexer(self._soledad)
+ return m_uid
+
+ def list_mail_tables_cb(self, ignored):
+ def filter_mailuid_tables(tables):
+ filtered = [
+ table[0] for table in tables if
+ table[0].startswith(mi.MailboxIndexer.table_preffix)]
+ return filtered
+
+ sql = "SELECT name FROM sqlite_master WHERE type='table';"
+ d = self._soledad.raw_sqlcipher_query(sql)
+ d.addCallback(filter_mailuid_tables)
+ return d
+
+ def select_uid_rows(self, mailbox):
+ sql = "SELECT * FROM %s%s;" % (
+ mi.MailboxIndexer.table_preffix, mailbox)
+ d = self._soledad.raw_sqlcipher_query(sql)
+ return d
+
+ def test_create_table(self):
+ def assert_table_created(tables):
+ self.assertEqual(
+ tables, ["leapmail_uid_inbox"])
+
+ m_uid = self.get_mbox_uid()
+ d = m_uid.create_table('inbox')
+ d.addCallback(self.list_mail_tables_cb)
+ d.addCallback(assert_table_created)
+ return d
+
+ def test_create_and_delete_table(self):
+ def assert_table_deleted(tables):
+ self.assertEqual(tables, [])
+
+ m_uid = self.get_mbox_uid()
+ d = m_uid.create_table('inbox')
+ d.addCallback(lambda _: m_uid.delete_table('inbox'))
+ d.addCallback(self.list_mail_tables_cb)
+ d.addCallback(assert_table_deleted)
+ return d
+
+ def test_rename_table(self):
+ def assert_table_renamed(tables):
+ self.assertEqual(
+ tables, ["leapmail_uid_foomailbox"])
+
+ m_uid = self.get_mbox_uid()
+ d = m_uid.create_table('inbox')
+ d.addCallback(lambda _: m_uid.rename_table('inbox', 'foomailbox'))
+ d.addCallback(self.list_mail_tables_cb)
+ d.addCallback(assert_table_renamed)
+ return d
+
+ def test_insert_doc(self):
+ m_uid = self.get_mbox_uid()
+ mbox = 'foomailbox'
+
+ h1 = fmt_hash(mbox, hash_test0)
+ h2 = fmt_hash(mbox, hash_test1)
+ h3 = fmt_hash(mbox, hash_test2)
+ h4 = fmt_hash(mbox, hash_test3)
+ h5 = fmt_hash(mbox, hash_test4)
+
+ def assert_uid_rows(rows):
+ expected = [(1, h1), (2, h2), (3, h3), (4, h4), (5, h5)]
+ self.assertEquals(rows, expected)
+
+ d = m_uid.create_table(mbox)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h3))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h4))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h5))
+ d.addCallback(lambda _: self.select_uid_rows(mbox))
+ d.addCallback(assert_uid_rows)
+ return d
+
+ def test_insert_doc_return(self):
+ m_uid = self.get_mbox_uid()
+ mbox = 'foomailbox'
+
+ def assert_rowid(rowid, expected=None):
+ self.assertEqual(rowid, expected)
+
+ h1 = fmt_hash(mbox, hash_test0)
+ h2 = fmt_hash(mbox, hash_test1)
+ h3 = fmt_hash(mbox, hash_test2)
+
+ d = m_uid.create_table(mbox)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h1))
+ d.addCallback(partial(assert_rowid, expected=1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h2))
+ d.addCallback(partial(assert_rowid, expected=2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h3))
+ d.addCallback(partial(assert_rowid, expected=3))
+ return d
+
+ def test_delete_doc(self):
+ m_uid = self.get_mbox_uid()
+ mbox = 'foomailbox'
+
+ h1 = fmt_hash(mbox, hash_test0)
+ h2 = fmt_hash(mbox, hash_test1)
+ h3 = fmt_hash(mbox, hash_test2)
+ h4 = fmt_hash(mbox, hash_test3)
+ h5 = fmt_hash(mbox, hash_test4)
+
+ def assert_uid_rows(rows):
+ expected = [(4, h4), (5, h5)]
+ self.assertEquals(rows, expected)
+
+ d = m_uid.create_table(mbox)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h3))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h4))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h5))
+
+ d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 1))
+ d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 2))
+ d.addCallbacks(lambda _: m_uid.delete_doc_by_hash(mbox, h3))
+
+ d.addCallback(lambda _: self.select_uid_rows(mbox))
+ d.addCallback(assert_uid_rows)
+ return d
+
+ def test_get_doc_id_from_uid(self):
+ m_uid = self.get_mbox_uid()
+ mbox = 'foomailbox'
+
+ h1 = fmt_hash(mbox, hash_test0)
+
+ def assert_doc_hash(res):
+ self.assertEqual(res, h1)
+
+ d = m_uid.create_table(mbox)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h1))
+ d.addCallback(lambda _: m_uid.get_doc_id_from_uid(mbox, 1))
+ d.addCallback(assert_doc_hash)
+ return d
+
+ def test_count(self):
+ m_uid = self.get_mbox_uid()
+ mbox = 'foomailbox'
+
+ h1 = fmt_hash(mbox, hash_test0)
+ h2 = fmt_hash(mbox, hash_test1)
+ h3 = fmt_hash(mbox, hash_test2)
+ h4 = fmt_hash(mbox, hash_test3)
+ h5 = fmt_hash(mbox, hash_test4)
+
+ d = m_uid.create_table(mbox)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h3))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h4))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h5))
+
+ def assert_count_after_inserts(count):
+ self.assertEquals(count, 5)
+
+ d.addCallback(lambda _: m_uid.count(mbox))
+ d.addCallback(assert_count_after_inserts)
+
+ d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 1))
+ d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox, 2))
+
+ def assert_count_after_deletions(count):
+ self.assertEquals(count, 3)
+
+ d.addCallback(lambda _: m_uid.count(mbox))
+ d.addCallback(assert_count_after_deletions)
+ return d
+
+ def test_get_next_uid(self):
+ m_uid = self.get_mbox_uid()
+ mbox = 'foomailbox'
+
+ h1 = fmt_hash(mbox, hash_test0)
+ h2 = fmt_hash(mbox, hash_test1)
+ h3 = fmt_hash(mbox, hash_test2)
+ h4 = fmt_hash(mbox, hash_test3)
+ h5 = fmt_hash(mbox, hash_test4)
+
+ d = m_uid.create_table(mbox)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h3))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h4))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox, h5))
+
+ def assert_next_uid(result, expected=1):
+ self.assertEquals(result, expected)
+
+ d.addCallback(lambda _: m_uid.get_next_uid(mbox))
+ d.addCallback(partial(assert_next_uid, expected=6))
+ return d
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-28 16:36:05 +0000