summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorKali Kaneko (leap communications) <kali@leap.se>2016-08-29 23:10:17 -0400
committerKali Kaneko (leap communications) <kali@leap.se>2016-08-29 23:11:41 -0400
commit5a3a2012bb8982ad0884ed659e61e969345e6fde (patch)
treefc2310d8d3244987bf5a1d2632cab99a60ba93f1 /src
parent43df4205af42fce5d097f70bb0345b69e9d16f1c (diff)
[pkg] move mail source to leap.bitmask.mail
Diffstat (limited to 'src')
-rw-r--r--src/leap/bitmask/mail/__init__.py26
-rw-r--r--src/leap/bitmask/mail/_version.py484
-rw-r--r--src/leap/bitmask/mail/adaptors/__init__.py0
-rw-r--r--src/leap/bitmask/mail/adaptors/models.py123
-rw-r--r--src/leap/bitmask/mail/adaptors/soledad.py1268
-rw-r--r--src/leap/bitmask/mail/adaptors/soledad_indexes.py106
l---------src/leap/bitmask/mail/adaptors/tests/rfc822.message1
-rw-r--r--src/leap/bitmask/mail/adaptors/tests/test_models.py106
-rw-r--r--src/leap/bitmask/mail/adaptors/tests/test_soledad_adaptor.py529
-rw-r--r--src/leap/bitmask/mail/constants.py52
-rw-r--r--src/leap/bitmask/mail/cred.py80
-rw-r--r--src/leap/bitmask/mail/decorators.py149
-rw-r--r--src/leap/bitmask/mail/errors.py27
-rw-r--r--src/leap/bitmask/mail/generator.py23
-rw-r--r--src/leap/bitmask/mail/imap/__init__.py0
-rw-r--r--src/leap/bitmask/mail/imap/account.py498
-rw-r--r--src/leap/bitmask/mail/imap/mailbox.py970
-rw-r--r--src/leap/bitmask/mail/imap/messages.py254
-rw-r--r--src/leap/bitmask/mail/imap/server.py693
-rw-r--r--src/leap/bitmask/mail/imap/service/README.rst39
-rw-r--r--src/leap/bitmask/mail/imap/service/__init__.py0
-rw-r--r--src/leap/bitmask/mail/imap/service/imap-server.tac145
-rw-r--r--src/leap/bitmask/mail/imap/service/imap.py208
-rw-r--r--src/leap/bitmask/mail/imap/service/manhole.py130
-rw-r--r--src/leap/bitmask/mail/imap/service/notes.txt81
-rw-r--r--src/leap/bitmask/mail/imap/service/rfc822.message86
-rw-r--r--src/leap/bitmask/mail/imap/tests/.gitignore1
-rwxr-xr-xsrc/leap/bitmask/mail/imap/tests/getmail344
-rwxr-xr-xsrc/leap/bitmask/mail/imap/tests/imapclient.py207
-rwxr-xr-xsrc/leap/bitmask/mail/imap/tests/regressions_mime_struct461
l---------src/leap/bitmask/mail/imap/tests/rfc822.message1
l---------src/leap/bitmask/mail/imap/tests/rfc822.multi-minimal.message1
l---------src/leap/bitmask/mail/imap/tests/rfc822.multi-nested.message1
l---------src/leap/bitmask/mail/imap/tests/rfc822.multi-signed.message1
l---------src/leap/bitmask/mail/imap/tests/rfc822.multi.message1
l---------src/leap/bitmask/mail/imap/tests/rfc822.plain.message1
-rwxr-xr-xsrc/leap/bitmask/mail/imap/tests/stress_tests_imap.zsh178
-rw-r--r--src/leap/bitmask/mail/imap/tests/test_imap.py1060
-rw-r--r--src/leap/bitmask/mail/imap/tests/walktree.py127
-rw-r--r--src/leap/bitmask/mail/incoming/__init__.py0
-rw-r--r--src/leap/bitmask/mail/incoming/service.py844
-rw-r--r--src/leap/bitmask/mail/incoming/tests/rfc822.multi-encrypt-signed.message61
-rw-r--r--src/leap/bitmask/mail/incoming/tests/test_incoming_mail.py391
-rw-r--r--src/leap/bitmask/mail/interfaces.py215
-rw-r--r--src/leap/bitmask/mail/load_tests.py29
-rw-r--r--src/leap/bitmask/mail/mail.py1070
-rw-r--r--src/leap/bitmask/mail/mailbox_indexer.py327
-rw-r--r--src/leap/bitmask/mail/outgoing/__init__.py0
-rw-r--r--src/leap/bitmask/mail/outgoing/service.py518
-rw-r--r--src/leap/bitmask/mail/outgoing/tests/test_outgoing.py263
-rw-r--r--src/leap/bitmask/mail/plugins/__init__.py3
-rw-r--r--src/leap/bitmask/mail/plugins/soledad_sync_hooks.py19
-rw-r--r--src/leap/bitmask/mail/rfc3156.py390
-rw-r--r--src/leap/bitmask/mail/size.py57
-rw-r--r--src/leap/bitmask/mail/smtp/README.rst44
-rw-r--r--src/leap/bitmask/mail/smtp/__init__.py73
-rw-r--r--src/leap/bitmask/mail/smtp/bounces.py90
-rw-r--r--src/leap/bitmask/mail/smtp/gateway.py413
-rw-r--r--src/leap/bitmask/mail/smtp/tests/185CA770.key79
-rw-r--r--src/leap/bitmask/mail/smtp/tests/185CA770.pub52
-rw-r--r--src/leap/bitmask/mail/smtp/tests/cert/server.crt29
-rw-r--r--src/leap/bitmask/mail/smtp/tests/cert/server.key51
-rw-r--r--src/leap/bitmask/mail/smtp/tests/mail.txt10
-rw-r--r--src/leap/bitmask/mail/smtp/tests/test_gateway.py181
-rw-r--r--src/leap/bitmask/mail/sync_hooks.py120
-rw-r--r--src/leap/bitmask/mail/testing/__init__.py353
-rw-r--r--src/leap/bitmask/mail/testing/__init__.py~358
-rw-r--r--src/leap/bitmask/mail/testing/common.py109
-rw-r--r--src/leap/bitmask/mail/testing/imap.py186
-rw-r--r--src/leap/bitmask/mail/testing/smtp.py51
-rw-r--r--src/leap/bitmask/mail/tests/rfc822.bounce.message152
-rw-r--r--src/leap/bitmask/mail/tests/rfc822.message86
-rw-r--r--src/leap/bitmask/mail/tests/rfc822.multi-minimal.message16
-rw-r--r--src/leap/bitmask/mail/tests/rfc822.multi-nested.message619
-rw-r--r--src/leap/bitmask/mail/tests/rfc822.multi-signed.message238
-rw-r--r--src/leap/bitmask/mail/tests/rfc822.multi.message96
-rw-r--r--src/leap/bitmask/mail/tests/rfc822.plain.message66
-rw-r--r--src/leap/bitmask/mail/tests/test_mail.py399
-rw-r--r--src/leap/bitmask/mail/tests/test_mailbox_indexer.py250
-rw-r--r--src/leap/bitmask/mail/tests/test_walk.py81
-rw-r--r--src/leap/bitmask/mail/utils.py375
-rw-r--r--src/leap/bitmask/mail/walk.py107
82 files changed, 17333 insertions, 0 deletions
diff --git a/src/leap/bitmask/mail/__init__.py b/src/leap/bitmask/mail/__init__.py
new file mode 100644
index 00000000..4b25fe62
--- /dev/null
+++ b/src/leap/bitmask/mail/__init__.py
@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# __init__.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+Client mail bits.
+"""
+
+
+from ._version import get_versions
+__version__ = get_versions()['version']
+del get_versions
diff --git a/src/leap/bitmask/mail/_version.py b/src/leap/bitmask/mail/_version.py
new file mode 100644
index 00000000..954f4882
--- /dev/null
+++ b/src/leap/bitmask/mail/_version.py
@@ -0,0 +1,484 @@
+
+# This file helps to compute a version number in source trees obtained from
+# git-archive tarball (such as those provided by githubs download-from-tag
+# feature). Distribution tarballs (built by setup.py sdist) and build
+# directories (produced by setup.py build) will contain a much shorter file
+# that just contains the computed version number.
+
+# This file is released into the public domain. Generated by
+# versioneer-0.16 (https://github.com/warner/python-versioneer)
+
+"""Git implementation of _version.py."""
+
+import errno
+import os
+import re
+import subprocess
+import sys
+
+
+def get_keywords():
+ """Get the keywords needed to look up the version information."""
+ # these strings will be replaced by git during git-archive.
+ # setup.py/versioneer.py will grep for the variable names, so they must
+ # each be defined on a line of their own. _version.py will just call
+ # get_keywords().
+ git_refnames = "$Format:%d$"
+ git_full = "$Format:%H$"
+ keywords = {"refnames": git_refnames, "full": git_full}
+ return keywords
+
+
+class VersioneerConfig:
+ """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+ """Create, populate and return the VersioneerConfig() object."""
+ # these strings are filled in when 'setup.py versioneer' creates
+ # _version.py
+ cfg = VersioneerConfig()
+ cfg.VCS = "git"
+ cfg.style = "pep440"
+ cfg.tag_prefix = ""
+ cfg.parentdir_prefix = "None"
+ cfg.versionfile_source = "src/leap/mail/_version.py"
+ cfg.verbose = False
+ return cfg
+
+
+class NotThisMethod(Exception):
+ """Exception raised if a method is not valid for the current scenario."""
+
+
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method): # decorator
+ """Decorator to mark a method as the handler for a particular VCS."""
+ def decorate(f):
+ """Store f in HANDLERS[vcs][method]."""
+ if vcs not in HANDLERS:
+ HANDLERS[vcs] = {}
+ HANDLERS[vcs][method] = f
+ return f
+ return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+ """Call the given command(s)."""
+ assert isinstance(commands, list)
+ p = None
+ for c in commands:
+ try:
+ dispcmd = str([c] + args)
+ # remember shell=False, so use git.cmd on windows, not just git
+ p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+ stderr=(subprocess.PIPE if hide_stderr
+ else None))
+ break
+ except EnvironmentError:
+ e = sys.exc_info()[1]
+ if e.errno == errno.ENOENT:
+ continue
+ if verbose:
+ print("unable to run %s" % dispcmd)
+ print(e)
+ return None
+ else:
+ if verbose:
+ print("unable to find command, tried %s" % (commands,))
+ return None
+ stdout = p.communicate()[0].strip()
+ if sys.version_info[0] >= 3:
+ stdout = stdout.decode()
+ if p.returncode != 0:
+ if verbose:
+ print("unable to run %s (error)" % dispcmd)
+ return None
+ return stdout
+
+
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+ """Try to determine the version from the parent directory name.
+
+ Source tarballs conventionally unpack into a directory that includes
+ both the project name and a version string.
+ """
+ dirname = os.path.basename(root)
+ if not dirname.startswith(parentdir_prefix):
+ if verbose:
+ print("guessing rootdir is '%s', but '%s' doesn't start with "
+ "prefix '%s'" % (root, dirname, parentdir_prefix))
+ raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+ return {"version": dirname[len(parentdir_prefix):],
+ "full-revisionid": None,
+ "dirty": False, "error": None}
+
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+ """Extract version information from the given file."""
+ # the code embedded in _version.py can just fetch the value of these
+ # keywords. When used from setup.py, we don't want to import _version.py,
+ # so we do it with a regexp instead. This function is not used from
+ # _version.py.
+ keywords = {}
+ try:
+ f = open(versionfile_abs, "r")
+ for line in f.readlines():
+ if line.strip().startswith("git_refnames ="):
+ mo = re.search(r'=\s*"(.*)"', line)
+ if mo:
+ keywords["refnames"] = mo.group(1)
+ if line.strip().startswith("git_full ="):
+ mo = re.search(r'=\s*"(.*)"', line)
+ if mo:
+ keywords["full"] = mo.group(1)
+ f.close()
+ except EnvironmentError:
+ pass
+ return keywords
+
+
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+ """Get version information from git keywords."""
+ if not keywords:
+ raise NotThisMethod("no keywords at all, weird")
+ refnames = keywords["refnames"].strip()
+ if refnames.startswith("$Format"):
+ if verbose:
+ print("keywords are unexpanded, not using")
+ raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
+ refs = set([r.strip() for r in refnames.strip("()").split(",")])
+ # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
+ # just "foo-1.0". If we see a "tag: " prefix, prefer those.
+ TAG = "tag: "
+ tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
+ if not tags:
+ # Either we're using git < 1.8.3, or there really are no tags. We use
+ # a heuristic: assume all version tags have a digit. The old git %d
+ # expansion behaves like git log --decorate=short and strips out the
+ # refs/heads/ and refs/tags/ prefixes that would let us distinguish
+ # between branches and tags. By ignoring refnames without digits, we
+ # filter out many common branch names like "release" and
+ # "stabilization", as well as "HEAD" and "master".
+ tags = set([r for r in refs if re.search(r'\d', r)])
+ if verbose:
+ print("discarding '%s', no digits" % ",".join(refs-tags))
+ if verbose:
+ print("likely tags: %s" % ",".join(sorted(tags)))
+ for ref in sorted(tags):
+ # sorting will prefer e.g. "2.0" over "2.0rc1"
+ if ref.startswith(tag_prefix):
+ r = ref[len(tag_prefix):]
+ if verbose:
+ print("picking %s" % r)
+ return {"version": r,
+ "full-revisionid": keywords["full"].strip(),
+ "dirty": False, "error": None
+ }
+ # no suitable tags, so version is "0+unknown", but full hex is still there
+ if verbose:
+ print("no suitable tags, using unknown + full revision id")
+ return {"version": "0+unknown",
+ "full-revisionid": keywords["full"].strip(),
+ "dirty": False, "error": "no suitable tags"}
+
+
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+ """Get version from 'git describe' in the root of the source tree.
+
+ This only gets called if the git-archive 'subst' keywords were *not*
+ expanded, and _version.py hasn't already been rewritten with a short
+ version string, meaning we're inside a checked out source tree.
+ """
+ if not os.path.exists(os.path.join(root, ".git")):
+ if verbose:
+ print("no .git in %s" % root)
+ raise NotThisMethod("no .git directory")
+
+ GITS = ["git"]
+ if sys.platform == "win32":
+ GITS = ["git.cmd", "git.exe"]
+ # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+ # if there isn't one, this yields HEX[-dirty] (no NUM)
+ describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
+ "--always", "--long",
+ "--match", "%s*" % tag_prefix],
+ cwd=root)
+ # --long was added in git-1.5.5
+ if describe_out is None:
+ raise NotThisMethod("'git describe' failed")
+ describe_out = describe_out.strip()
+ full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+ if full_out is None:
+ raise NotThisMethod("'git rev-parse' failed")
+ full_out = full_out.strip()
+
+ pieces = {}
+ pieces["long"] = full_out
+ pieces["short"] = full_out[:7] # maybe improved later
+ pieces["error"] = None
+
+ # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+ # TAG might have hyphens.
+ git_describe = describe_out
+
+ # look for -dirty suffix
+ dirty = git_describe.endswith("-dirty")
+ pieces["dirty"] = dirty
+ if dirty:
+ git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+ # now we have TAG-NUM-gHEX or HEX
+
+ if "-" in git_describe:
+ # TAG-NUM-gHEX
+ mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+ if not mo:
+ # unparseable. Maybe git-describe is misbehaving?
+ pieces["error"] = ("unable to parse git-describe output: '%s'"
+ % describe_out)
+ return pieces
+
+ # tag
+ full_tag = mo.group(1)
+ if not full_tag.startswith(tag_prefix):
+ if verbose:
+ fmt = "tag '%s' doesn't start with prefix '%s'"
+ print(fmt % (full_tag, tag_prefix))
+ pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+ % (full_tag, tag_prefix))
+ return pieces
+ pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+ # distance: number of commits since tag
+ pieces["distance"] = int(mo.group(2))
+
+ # commit: short hex revision ID
+ pieces["short"] = mo.group(3)
+
+ else:
+ # HEX: no tags
+ pieces["closest-tag"] = None
+ count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
+ cwd=root)
+ pieces["distance"] = int(count_out) # total number of commits
+
+ return pieces
+
+
+def plus_or_dot(pieces):
+ """Return a + if we don't already have one, else return a ."""
+ if "+" in pieces.get("closest-tag", ""):
+ return "."
+ return "+"
+
+
+def render_pep440(pieces):
+ """Build up version string, with post-release "local version identifier".
+
+ Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+ get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+ Exceptions:
+ 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"] or pieces["dirty"]:
+ rendered += plus_or_dot(pieces)
+ rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+ if pieces["dirty"]:
+ rendered += ".dirty"
+ else:
+ # exception #1
+ rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+ pieces["short"])
+ if pieces["dirty"]:
+ rendered += ".dirty"
+ return rendered
+
+
+def render_pep440_pre(pieces):
+ """TAG[.post.devDISTANCE] -- No -dirty.
+
+ Exceptions:
+ 1: no tags. 0.post.devDISTANCE
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"]:
+ rendered += ".post.dev%d" % pieces["distance"]
+ else:
+ # exception #1
+ rendered = "0.post.dev%d" % pieces["distance"]
+ return rendered
+
+
+def render_pep440_post(pieces):
+ """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+ The ".dev0" means dirty. Note that .dev0 sorts backwards
+ (a dirty tree will appear "older" than the corresponding clean one),
+ but you shouldn't be releasing software with -dirty anyways.
+
+ Exceptions:
+ 1: no tags. 0.postDISTANCE[.dev0]
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"] or pieces["dirty"]:
+ rendered += ".post%d" % pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ rendered += plus_or_dot(pieces)
+ rendered += "g%s" % pieces["short"]
+ else:
+ # exception #1
+ rendered = "0.post%d" % pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ rendered += "+g%s" % pieces["short"]
+ return rendered
+
+
+def render_pep440_old(pieces):
+ """TAG[.postDISTANCE[.dev0]] .
+
+ The ".dev0" means dirty.
+
+ Eexceptions:
+ 1: no tags. 0.postDISTANCE[.dev0]
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"] or pieces["dirty"]:
+ rendered += ".post%d" % pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ else:
+ # exception #1
+ rendered = "0.post%d" % pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ return rendered
+
+
+def render_git_describe(pieces):
+ """TAG[-DISTANCE-gHEX][-dirty].
+
+ Like 'git describe --tags --dirty --always'.
+
+ Exceptions:
+ 1: no tags. HEX[-dirty] (note: no 'g' prefix)
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"]:
+ rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+ else:
+ # exception #1
+ rendered = pieces["short"]
+ if pieces["dirty"]:
+ rendered += "-dirty"
+ return rendered
+
+
+def render_git_describe_long(pieces):
+ """TAG-DISTANCE-gHEX[-dirty].
+
+ Like 'git describe --tags --dirty --always -long'.
+ The distance/hash is unconditional.
+
+ Exceptions:
+ 1: no tags. HEX[-dirty] (note: no 'g' prefix)
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+ else:
+ # exception #1
+ rendered = pieces["short"]
+ if pieces["dirty"]:
+ rendered += "-dirty"
+ return rendered
+
+
+def render(pieces, style):
+ """Render the given version pieces into the requested style."""
+ if pieces["error"]:
+ return {"version": "unknown",
+ "full-revisionid": pieces.get("long"),
+ "dirty": None,
+ "error": pieces["error"]}
+
+ if not style or style == "default":
+ style = "pep440" # the default
+
+ if style == "pep440":
+ rendered = render_pep440(pieces)
+ elif style == "pep440-pre":
+ rendered = render_pep440_pre(pieces)
+ elif style == "pep440-post":
+ rendered = render_pep440_post(pieces)
+ elif style == "pep440-old":
+ rendered = render_pep440_old(pieces)
+ elif style == "git-describe":
+ rendered = render_git_describe(pieces)
+ elif style == "git-describe-long":
+ rendered = render_git_describe_long(pieces)
+ else:
+ raise ValueError("unknown style '%s'" % style)
+
+ return {"version": rendered, "full-revisionid": pieces["long"],
+ "dirty": pieces["dirty"], "error": None}
+
+
+def get_versions():
+ """Get version information or return default if unable to do so."""
+ # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
+ # __file__, we can work backwards from there to the root. Some
+ # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
+ # case we can only use expanded keywords.
+
+ cfg = get_config()
+ verbose = cfg.verbose
+
+ try:
+ return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
+ verbose)
+ except NotThisMethod:
+ pass
+
+ try:
+ root = os.path.realpath(__file__)
+ # versionfile_source is the relative path from the top of the source
+ # tree (where the .git directory might live) to this file. Invert
+ # this to find the root from __file__.
+ for i in cfg.versionfile_source.split('/'):
+ root = os.path.dirname(root)
+ except NameError:
+ return {"version": "0+unknown", "full-revisionid": None,
+ "dirty": None,
+ "error": "unable to find root of source tree"}
+
+ try:
+ pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+ return render(pieces, cfg.style)
+ except NotThisMethod:
+ pass
+
+ try:
+ if cfg.parentdir_prefix:
+ return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+ except NotThisMethod:
+ pass
+
+ return {"version": "0+unknown", "full-revisionid": None,
+ "dirty": None,
+ "error": "unable to compute version"}
diff --git a/src/leap/bitmask/mail/adaptors/__init__.py b/src/leap/bitmask/mail/adaptors/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/src/leap/bitmask/mail/adaptors/__init__.py
diff --git a/src/leap/bitmask/mail/adaptors/models.py b/src/leap/bitmask/mail/adaptors/models.py
new file mode 100644
index 00000000..49460f74
--- /dev/null
+++ b/src/leap/bitmask/mail/adaptors/models.py
@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+# models.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Generic Models to be used by the Document Adaptors.
+"""
+import copy
+
+
+class SerializableModel(object):
+ """
+ A Generic document model, that can be serialized into a dictionary.
+
+ Subclasses of this `SerializableModel` are meant to be added as class
+ attributes of classes inheriting from DocumentWrapper.
+
+ A subclass __meta__ of this SerializableModel might exist, and contain info
+ relative to particularities of this model.
+
+ For instance, the use of `__meta__.index` marks the existence of a primary
+ index in the model, which will be used to do unique queries (in which case
+ all the other indexed fields in the underlying document will be filled with
+ the default info contained in the model definition).
+ """
+
+ @classmethod
+ def serialize(klass):
+ """
+ Get a dictionary representation of the public attributes in the model
+ class. To avoid collisions with builtin functions, any occurrence of an
+ attribute ended in '_' (like 'type_') will be normalized by removing
+ the trailing underscore.
+
+ This classmethod is used from within the serialized method of a
+ DocumentWrapper instance: it provides defaults for the
+ empty document.
+ """
+ assert isinstance(klass, type)
+ return _normalize_dict(klass.__dict__)
+
+
+class DocumentWrapper(object):
+ """
+ A Wrapper object that can be manipulated, passed around, and serialized in
+ a format that the store understands.
+ It is related to a SerializableModel, which must be specified as the
+ ``model`` class attribute. The instance of this DocumentWrapper will not
+ allow any other *public* attributes than those defined in the corresponding
+ model.
+ """
+ # TODO we could do some very basic type checking here
+ # TODO set a dirty flag (on __setattr__, whenever the value is != from
+ # before)
+ # TODO we could enforce the existence of a correct "model" attribute
+ # in some other way (other than in the initializer)
+
+ def __init__(self, **kwargs):
+ if not getattr(self, 'model', None):
+ raise RuntimeError(
+ 'DocumentWrapper class needs a model attribute')
+
+ defaults = self.model.serialize()
+
+ if kwargs:
+ values = copy.deepcopy(defaults)
+ values.update(_normalize_dict(kwargs))
+ else:
+ values = defaults
+
+ for k, v in values.items():
+ k = k.replace('-', '_')
+ setattr(self, k, v)
+
+ def __setattr__(self, attr, value):
+ normalized = _normalize_dict(self.model.__dict__)
+ if not attr.startswith('_') and attr not in normalized:
+ raise RuntimeError(
+ "Cannot set attribute because it's not defined "
+ "in the model %s: %s" % (self.__class__, attr))
+ object.__setattr__(self, attr, value)
+
+ def serialize(self):
+ return _normalize_dict(self.__dict__)
+
+ def create(self):
+ raise NotImplementedError()
+
+ def update(self):
+ raise NotImplementedError()
+
+ def delete(self):
+ raise NotImplementedError()
+
+ @classmethod
+ def get_or_create(self):
+ raise NotImplementedError()
+
+ @classmethod
+ def get_all(self):
+ raise NotImplementedError()
+
+
+def _normalize_dict(_dict):
+ items = _dict.items()
+ items = filter(lambda (k, v): not callable(v), items)
+ items = filter(lambda (k, v): not k.startswith('_'), items)
+ items = [(k, v) if not k.endswith('_') else (k[:-1], v)
+ for (k, v) in items]
+ items = [(k.replace('-', '_'), v) for (k, v) in items]
+ return dict(items)
diff --git a/src/leap/bitmask/mail/adaptors/soledad.py b/src/leap/bitmask/mail/adaptors/soledad.py
new file mode 100644
index 00000000..ca8f741d
--- /dev/null
+++ b/src/leap/bitmask/mail/adaptors/soledad.py
@@ -0,0 +1,1268 @@
+# soledad.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledadad MailAdaptor module.
+"""
+import logging
+import re
+
+from collections import defaultdict
+from email import message_from_string
+
+from twisted.internet import defer
+from twisted.python import log
+from zope.interface import implements
+from leap.soledad.common import l2db
+
+from leap.common.check import leap_assert, leap_assert_type
+
+from leap.mail import constants
+from leap.mail import walk
+from leap.mail.adaptors import soledad_indexes as indexes
+from leap.mail.constants import INBOX_NAME
+from leap.mail.adaptors import models
+from leap.mail.imap.mailbox import normalize_mailbox
+from leap.mail.utils import lowerdict, first
+from leap.mail.utils import stringify_parts_map
+from leap.mail.interfaces import IMailAdaptor, IMessageWrapper
+
+from leap.soledad.common.document import SoledadDocument
+
+
+logger = logging.getLogger(__name__)
+
+# TODO
+# [ ] Convenience function to create mail specifying subject, date, etc?
+
+
+_MSGID_PATTERN = r"""<([\w@.]+)>"""
+_MSGID_RE = re.compile(_MSGID_PATTERN)
+
+
+class DuplicatedDocumentError(Exception):
+ """
+ Raised when a duplicated document is detected.
+ """
+ pass
+
+
+def cleanup_deferred_locks():
+ """
+ Need to use this from within trial to cleanup the reactor before
+ each run.
+ """
+ SoledadDocumentWrapper._k_locks = defaultdict(defer.DeferredLock)
+
+
+class SoledadDocumentWrapper(models.DocumentWrapper):
+ """
+ A Wrapper object that can be manipulated, passed around, and serialized in
+ a format that the Soledad Store understands.
+
+ It ensures atomicity of the document operations on creation, update and
+ deletion.
+ """
+ # TODO we could also use a _dirty flag (in models)
+ # TODO add a get_count() method ??? -- that is extended over l2db.
+
+ # We keep a dictionary with DeferredLocks, that will be
+ # unique to every subclass of SoledadDocumentWrapper.
+ _k_locks = defaultdict(defer.DeferredLock)
+
+ @classmethod
+ def _get_klass_lock(cls):
+ """
+ Get a DeferredLock that is unique for this subclass name.
+ Used to lock the access to indexes in the `get_or_create` call
+ for a particular DocumentWrapper.
+ """
+ return cls._k_locks[cls.__name__]
+
+ def __init__(self, doc_id=None, future_doc_id=None, **kwargs):
+ self._doc_id = doc_id
+ self._future_doc_id = future_doc_id
+ self._lock = defer.DeferredLock()
+ super(SoledadDocumentWrapper, self).__init__(**kwargs)
+
+ @property
+ def doc_id(self):
+ return self._doc_id
+
+ @property
+ def future_doc_id(self):
+ return self._future_doc_id
+
+ def set_future_doc_id(self, doc_id):
+ self._future_doc_id = doc_id
+
+ def create(self, store, is_copy=False):
+ """
+ Create the documents for this wrapper.
+ Since this method will not check for duplication, the
+ responsibility of avoiding duplicates is left to the caller.
+
+ You might be interested in using `get_or_create` classmethod
+ instead (that's the preferred way of creating documents from
+ the wrapper object).
+
+ :return: a deferred that will fire when the underlying
+ Soledad document has been created.
+ :rtype: Deferred
+ """
+ leap_assert(self._doc_id is None,
+ "This document already has a doc_id!")
+
+ def update_doc_id(doc):
+ self._doc_id = doc.doc_id
+ self.set_future_doc_id(None)
+ return doc
+
+ def update_wrapper(failure):
+ # In the case of some copies (for instance, from one folder to
+ # another and back to the original folder), the document that we
+ # want to insert already exists. In this case, putting it
+ # and overwriting the document with that doc_id is the right thing
+ # to do.
+ failure.trap(l2db.errors.RevisionConflict)
+ self._doc_id = self.future_doc_id
+ self._future_doc_id = None
+ return self.update(store)
+
+ if self.future_doc_id is None:
+ d = store.create_doc(self.serialize())
+ else:
+ d = store.create_doc(self.serialize(),
+ doc_id=self.future_doc_id)
+ d.addCallback(update_doc_id)
+
+ if is_copy:
+ d.addErrback(update_wrapper)
+ else:
+ d.addErrback(self._catch_revision_conflict, self.future_doc_id)
+ return d
+
+ def update(self, store):
+ """
+ Update the documents for this wrapper.
+
+ :return: a deferred that will fire when the underlying
+ Soledad document has been updated.
+ :rtype: Deferred
+ """
+ # the deferred lock guards against revision conflicts
+ return self._lock.run(self._update, store)
+
+ def _update(self, store):
+ leap_assert(self._doc_id is not None,
+ "Need to create doc before updating")
+
+ def update_and_put_doc(doc):
+ doc.content.update(self.serialize())
+ d = store.put_doc(doc)
+ d.addErrback(self._catch_revision_conflict, doc.doc_id)
+ return d
+
+ d = store.get_doc(self._doc_id)
+ d.addCallback(update_and_put_doc)
+ return d
+
+ def _catch_revision_conflict(self, failure, doc_id):
+ # XXX We can have some RevisionConflicts if we try
+ # to put the docs that are already there.
+ # This can happen right now when creating/saving the cdocs
+ # during a copy. Instead of catching and ignoring this
+ # error, we should mark them in the copy so there is no attempt to
+ # create/update them.
+ failure.trap(l2db.errors.RevisionConflict)
+ logger.debug("Got conflict while putting %s" % doc_id)
+
+ def delete(self, store):
+ """
+ Delete the documents for this wrapper.
+
+ :return: a deferred that will fire when the underlying
+ Soledad document has been deleted.
+ :rtype: Deferred
+ """
+ # the deferred lock guards against conflicts while updating
+ return self._lock.run(self._delete, store)
+
+ def _delete(self, store):
+ leap_assert(self._doc_id is not None,
+ "Need to create doc before deleting")
+ # XXX might want to flag this DocumentWrapper to avoid
+ # updating it by mistake. This could go in models.DocumentWrapper
+
+ def delete_doc(doc):
+ return store.delete_doc(doc)
+
+ d = store.get_doc(self._doc_id)
+ d.addCallback(delete_doc)
+ return d
+
+ @classmethod
+ def get_or_create(cls, store, index, value):
+ """
+ Get a unique DocumentWrapper by index, or create a new one if the
+ matching query does not exist.
+
+ :param index: the primary index for the model.
+ :type index: str
+ :param value: the value to query the primary index.
+ :type value: str
+
+ :return: a deferred that will be fired with the SoledadDocumentWrapper
+ matching the index query, either existing or just created.
+ :rtype: Deferred
+ """
+ return cls._get_klass_lock().run(
+ cls._get_or_create, store, index, value)
+
+ @classmethod
+ def _get_or_create(cls, store, index, value):
+ # TODO shorten this method.
+ assert store is not None
+ assert index is not None
+ assert value is not None
+
+ def get_main_index():
+ try:
+ return cls.model.__meta__.index
+ except AttributeError:
+ raise RuntimeError("The model is badly defined")
+
+ # TODO separate into another method?
+ def try_to_get_doc_from_index(indexes):
+ values = []
+ idx_def = dict(indexes)[index]
+ if len(idx_def) == 1:
+ values = [value]
+ else:
+ main_index = get_main_index()
+ fields = cls.model.serialize()
+ for field in idx_def:
+ if field == main_index:
+ values.append(value)
+ else:
+ values.append(fields[field])
+ d = store.get_from_index(index, *values)
+ return d
+
+ def get_first_doc_if_any(docs):
+ if not docs:
+ return None
+ if len(docs) > 1:
+ raise DuplicatedDocumentError
+ return docs[0]
+
+ def wrap_existing_or_create_new(doc):
+ if doc:
+ return cls(doc_id=doc.doc_id, **doc.content)
+ else:
+ return create_and_wrap_new_doc()
+
+ def create_and_wrap_new_doc():
+ # XXX use closure to store indexes instead of
+ # querying for them again.
+ d = store.list_indexes()
+ d.addCallback(get_wrapper_instance_from_index)
+ d.addCallback(return_wrapper_when_created)
+ return d
+
+ def get_wrapper_instance_from_index(indexes):
+ init_values = {}
+ idx_def = dict(indexes)[index]
+ if len(idx_def) == 1:
+ init_value = {idx_def[0]: value}
+ return cls(**init_value)
+ main_index = get_main_index()
+ fields = cls.model.serialize()
+ for field in idx_def:
+ if field == main_index:
+ init_values[field] = value
+ else:
+ init_values[field] = fields[field]
+ return cls(**init_values)
+
+ def return_wrapper_when_created(wrapper):
+ d = wrapper.create(store)
+ d.addCallback(lambda doc: wrapper)
+ return d
+
+ d = store.list_indexes()
+ d.addCallback(try_to_get_doc_from_index)
+ d.addCallback(get_first_doc_if_any)
+ d.addCallback(wrap_existing_or_create_new)
+ return d
+
+ @classmethod
+ def get_all(cls, store):
+ """
+ Get a collection of wrappers around all the documents belonging
+ to this kind.
+
+ For this to work, the model.__meta__ needs to include a tuple with
+ the index to be used for listing purposes, and which is the field to be
+ used to query the index.
+
+ Note that this method only supports indexes of a single field at the
+ moment. It also might be too expensive to return all the documents
+ matching the query, so handle with care.
+
+ class __meta__(object):
+ index = "name"
+ list_index = ("by-type", "type_")
+
+ :return: a deferred that will be fired with an iterable containing
+ as many SoledadDocumentWrapper are matching the index defined
+ in the model as the `list_index`.
+ :rtype: Deferred
+ """
+ # TODO LIST (get_all)
+ # [ ] extend support to indexes with n-ples
+ # [ ] benchmark the cost of querying and returning indexes in a big
+ # database. This might badly need pagination before being put to
+ # serious use.
+ return cls._get_klass_lock().run(cls._get_all, store)
+
+ @classmethod
+ def _get_all(cls, store):
+ try:
+ list_index, list_attr = cls.model.__meta__.list_index
+ except AttributeError:
+ raise RuntimeError("The model is badly defined: no list_index")
+ try:
+ index_value = getattr(cls.model, list_attr)
+ except AttributeError:
+ raise RuntimeError("The model is badly defined: "
+ "no attribute matching list_index")
+
+ def wrap_docs(docs):
+ return (cls(doc_id=doc.doc_id, **doc.content) for doc in docs)
+
+ d = store.get_from_index(list_index, index_value)
+ d.addCallback(wrap_docs)
+ return d
+
+ def __repr__(self):
+ try:
+ idx = getattr(self, self.model.__meta__.index)
+ except AttributeError:
+ idx = ""
+ return "<%s: %s (%s)>" % (self.__class__.__name__,
+ idx, self._doc_id)
+
+
+#
+# Message documents
+#
+
+class FlagsDocWrapper(SoledadDocumentWrapper):
+
+ class model(models.SerializableModel):
+ type_ = "flags"
+ chash = ""
+
+ mbox_uuid = ""
+ seen = False
+ deleted = False
+ recent = False
+ flags = []
+ tags = []
+ size = 0
+ multi = False
+
+ class __meta__(object):
+ index = "mbox"
+
+ def set_mbox_uuid(self, mbox_uuid):
+ # XXX raise error if already created, should use copy instead
+ mbox_uuid = mbox_uuid.replace('-', '_')
+ new_id = constants.FDOCID.format(mbox_uuid=mbox_uuid, chash=self.chash)
+ self._future_doc_id = new_id
+ self.mbox_uuid = mbox_uuid
+
+ def get_flags(self):
+ """
+ Get the flags for this message (as a tuple of strings, not unicode).
+ """
+ return map(str, self.flags)
+
+
+class HeaderDocWrapper(SoledadDocumentWrapper):
+
+ class model(models.SerializableModel):
+ type_ = "head"
+ chash = ""
+
+ date = ""
+ subject = ""
+ headers = {}
+ part_map = {}
+ body = "" # link to phash of body
+ msgid = ""
+ multi = False
+
+ class __meta__(object):
+ index = "chash"
+
+
+class ContentDocWrapper(SoledadDocumentWrapper):
+
+ class model(models.SerializableModel):
+ type_ = "cnt"
+ phash = ""
+
+ ctype = "" # XXX index by ctype too?
+ lkf = [] # XXX not implemented yet!
+ raw = ""
+
+ content_disposition = ""
+ content_transfer_encoding = ""
+ content_type = ""
+
+ class __meta__(object):
+ index = "phash"
+
+
+class MetaMsgDocWrapper(SoledadDocumentWrapper):
+
+ class model(models.SerializableModel):
+ type_ = "meta"
+ fdoc = ""
+ hdoc = ""
+ cdocs = []
+
+ def set_mbox_uuid(self, mbox_uuid):
+ # XXX raise error if already created, should use copy instead
+ mbox_uuid = mbox_uuid.replace('-', '_')
+ chash = re.findall(constants.FDOCID_CHASH_RE, self.fdoc)[0]
+ new_id = constants.METAMSGID.format(mbox_uuid=mbox_uuid, chash=chash)
+ new_fdoc_id = constants.FDOCID.format(mbox_uuid=mbox_uuid, chash=chash)
+ self._future_doc_id = new_id
+ self.fdoc = new_fdoc_id
+
+
+class MessageWrapper(object):
+
+ # This could benefit of a DeferredLock to create/update all the
+ # documents at the same time maybe, and defend against concurrent updates?
+
+ implements(IMessageWrapper)
+
+ def __init__(self, mdoc, fdoc, hdoc, cdocs=None, is_copy=False):
+ """
+ Need at least a metamsg-document, a flag-document and a header-document
+ to instantiate a MessageWrapper. Content-documents can be retrieved
+ lazily.
+
+ cdocs, if any, should be a dictionary in which the keys are ascending
+ integers, beginning at one, and the values are dictionaries with the
+ content of the content-docs.
+
+ is_copy, if set to True, will only attempt to create mdoc and fdoc
+ (because hdoc and cdocs are supposed to exist already)
+ """
+ self._is_copy = is_copy
+
+ def get_doc_wrapper(doc, cls):
+ if isinstance(doc, SoledadDocument):
+ doc_id = doc.doc_id
+ doc = doc.content
+ else:
+ doc_id = None
+ if not doc:
+ doc = {}
+ return cls(doc_id=doc_id, **doc)
+
+ self.mdoc = get_doc_wrapper(mdoc, MetaMsgDocWrapper)
+
+ self.fdoc = get_doc_wrapper(fdoc, FlagsDocWrapper)
+ self.fdoc.set_future_doc_id(self.mdoc.fdoc)
+
+ self.hdoc = get_doc_wrapper(hdoc, HeaderDocWrapper)
+ self.hdoc.set_future_doc_id(self.mdoc.hdoc)
+
+ if cdocs is None:
+ cdocs = {}
+ cdocs_keys = cdocs.keys()
+ assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1)
+ self.cdocs = dict([
+ (key, get_doc_wrapper(doc, ContentDocWrapper))
+ for (key, doc) in cdocs.items()])
+ for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()):
+ if cdoc.raw == "":
+ log.msg("Empty raw field in cdoc %s" % doc_id)
+ cdoc.set_future_doc_id(doc_id)
+
+ def create(self, store, notify_just_mdoc=False, pending_inserts_dict=None):
+ """
+ Create all the parts for this message in the store.
+
+ :param store: an instance of Soledad
+
+ :param notify_just_mdoc:
+ if set to True, this method will return *only* the deferred
+ corresponding to the creation of the meta-message document.
+ Be warned that in that case there will be no record of failures
+ when creating the other part-documents.
+
+ Otherwise, this method will return a deferred that will wait for
+ the creation of all the part documents.
+
+ Setting this flag to True is mostly a convenient workaround for the
+ fact that massive serial appends will take too much time, and in
+ most of the cases the MUA will only switch to the mailbox where the
+ appends have happened after a certain time, which in most of the
+ times will be enough to have all the queued insert operations
+ finished.
+ :type notify_just_mdoc: bool
+ :param pending_inserts_dict:
+ a dictionary with the pending inserts ids.
+ :type pending_inserts_dict: dict
+
+ :return: a deferred whose callback will be called when either all the
+ part documents have been written, or just the metamsg-doc,
+ depending on the value of the notify_just_mdoc flag
+ :rtype: defer.Deferred
+ """
+ if pending_inserts_dict is None:
+ pending_inserts_dict = {}
+
+ leap_assert(self.cdocs,
+ "Need non empty cdocs to create the "
+ "MessageWrapper documents")
+ leap_assert(self.mdoc.doc_id is None,
+ "Cannot create: mdoc has a doc_id")
+ leap_assert(self.fdoc.doc_id is None,
+ "Cannot create: fdoc has a doc_id")
+
+ def unblock_pending_insert(result):
+ if pending_inserts_dict:
+ ci_headers = lowerdict(self.hdoc.headers)
+ msgid = ci_headers.get('message-id', None)
+ try:
+ d = pending_inserts_dict[msgid]
+ d.callback(msgid)
+ except KeyError:
+ pass
+ return result
+
+ # TODO check that the doc_ids in the mdoc are coherent
+ self.d = []
+
+ mdoc_created = self.mdoc.create(store, is_copy=self._is_copy)
+ fdoc_created = self.fdoc.create(store, is_copy=self._is_copy)
+
+ mdoc_created.addErrback(lambda f: log.err(f))
+ fdoc_created.addErrback(lambda f: log.err(f))
+
+ self.d.append(mdoc_created)
+ self.d.append(fdoc_created)
+
+ if not self._is_copy:
+ if self.hdoc.doc_id is None:
+ self.d.append(self.hdoc.create(store))
+ for cdoc in self.cdocs.values():
+ if cdoc.doc_id is not None:
+ # we could be just linking to an existing
+ # content-doc.
+ continue
+ self.d.append(cdoc.create(store))
+
+ def log_all_inserted(result):
+ log.msg("All parts inserted for msg!")
+ return result
+
+ self.all_inserted_d = defer.gatherResults(self.d, consumeErrors=True)
+ self.all_inserted_d.addCallback(log_all_inserted)
+ self.all_inserted_d.addCallback(unblock_pending_insert)
+ self.all_inserted_d.addErrback(lambda failure: log.err(failure))
+
+ if notify_just_mdoc:
+ return mdoc_created
+ else:
+ return self.all_inserted_d
+
+ def update(self, store):
+ """
+ Update the only mutable parts, which are within the flags document.
+ """
+ return self.fdoc.update(store)
+
+ def delete(self, store):
+ # TODO
+ # Eventually this would have to do the duplicate search or send for the
+ # garbage collector. At least mdoc and t the mdoc and fdoc can be
+ # unlinked.
+ d = []
+ if self.mdoc.doc_id:
+ d.append(self.mdoc.delete(store))
+ d.append(self.fdoc.delete(store))
+ return defer.gatherResults(d)
+
+ def copy(self, store, new_mbox_uuid):
+ """
+ Return a copy of this MessageWrapper in a new mailbox.
+
+ :param store: an instance of Soledad, or anything that behaves alike.
+ :param new_mbox_uuid: the uuid of the mailbox where we are copying this
+ message to.
+ :type new_mbox_uuid: str
+ :rtype: MessageWrapper
+ """
+ new_mdoc = self.mdoc.serialize()
+ new_fdoc = self.fdoc.serialize()
+
+ # the future doc_ids is properly set because we modified
+ # the pointers in mdoc, which has precedence.
+ new_wrapper = MessageWrapper(new_mdoc, new_fdoc, None, None,
+ is_copy=True)
+ new_wrapper.hdoc = self.hdoc
+ new_wrapper.cdocs = self.cdocs
+ new_wrapper.set_mbox_uuid(new_mbox_uuid)
+
+ # XXX could flag so that it only creates mdoc/fdoc...
+
+ d = new_wrapper.create(store)
+ d.addCallback(lambda result: new_wrapper)
+ d.addErrback(lambda failure: log.err(failure))
+ return d
+
+ def set_mbox_uuid(self, mbox_uuid):
+ """
+ Set the mailbox for this wrapper.
+ This method should only be used before the Documents for the
+ MessageWrapper have been created, will raise otherwise.
+ """
+ mbox_uuid = mbox_uuid.replace('-', '_')
+ self.mdoc.set_mbox_uuid(mbox_uuid)
+ self.fdoc.set_mbox_uuid(mbox_uuid)
+
+ def set_flags(self, flags):
+ # TODO serialize the get + update
+ if flags is None:
+ flags = tuple()
+ leap_assert_type(flags, tuple)
+ self.fdoc.flags = list(flags)
+ self.fdoc.deleted = "\\Deleted" in flags
+ self.fdoc.seen = "\\Seen" in flags
+ self.fdoc.recent = "\\Recent" in flags
+
+ def set_tags(self, tags):
+ # TODO serialize the get + update
+ if tags is None:
+ tags = tuple()
+ leap_assert_type(tags, tuple)
+ self.fdoc.tags = list(tags)
+
+ def set_date(self, date):
+ # XXX assert valid date format
+ self.hdoc.date = date
+
+ def get_subpart_dict(self, index):
+ """
+ :param index: the part to lookup, 1-indexed
+ :type index: int
+ :rtype: dict
+ """
+ return self.hdoc.part_map[str(index)]
+
+ def get_subpart_indexes(self):
+ return self.hdoc.part_map.keys()
+
+ def get_body(self, store):
+ """
+ :rtype: deferred
+ """
+ body_phash = self.hdoc.body
+ if body_phash:
+ d = store.get_doc('C-' + body_phash)
+ d.addCallback(lambda doc: ContentDocWrapper(**doc.content))
+ return d
+ elif self.cdocs:
+ return self.cdocs[1]
+ else:
+ return ''
+
+#
+# Mailboxes
+#
+
+
+class MailboxWrapper(SoledadDocumentWrapper):
+
+ class model(models.SerializableModel):
+ type_ = "mbox"
+ mbox = INBOX_NAME
+ uuid = None
+ flags = []
+ recent = []
+ created = 1
+ closed = False
+ subscribed = False
+
+ class __meta__(object):
+ index = "mbox"
+ list_index = (indexes.TYPE_IDX, 'type_')
+
+
+#
+# Soledad Adaptor
+#
+
+class SoledadIndexMixin(object):
+ """
+ This will need a class attribute `indexes`, that is a dictionary containing
+ the index definitions for the underlying l2db store underlying soledad.
+
+ It needs to be in the following format:
+ {'index-name': ['field1', 'field2']}
+
+ You can also add a class attribute `wait_for_indexes` to any class
+ inheriting from this Mixin, that should be a list of strings representing
+ the methods that need to wait until the indexes have been initialized
+ before being able to work properly.
+ """
+ # TODO move this mixin to soledad itself
+ # so that each application can pass a set of indexes for their data model.
+
+ # TODO could have a wrapper class for indexes, supporting introspection
+ # and __getattr__
+
+ # TODO make this an interface?
+
+ indexes = {}
+ wait_for_indexes = []
+ store_ready = False
+
+ def initialize_store(self, store):
+ """
+ Initialize the indexes in the database.
+
+ :param store: store
+ :returns: a Deferred that will fire when the store is correctly
+ initialized.
+ :rtype: deferred
+ """
+ # TODO I think we *should* get another deferredLock in here, but
+ # global to the soledad namespace, to protect from several points
+ # initializing soledad indexes at the same time.
+ self._wait_for_indexes()
+
+ d = self._init_indexes(store)
+ d.addCallback(self._restore_waiting_methods)
+ return d
+
+ def _init_indexes(self, store):
+ """
+ Initialize the database indexes.
+ """
+ leap_assert(store, "Cannot init indexes with null soledad")
+ leap_assert_type(self.indexes, dict)
+
+ def _create_index(name, expression):
+ return store.create_index(name, *expression)
+
+ def init_idexes(indexes):
+ deferreds = []
+ db_indexes = dict(indexes)
+ # Loop through the indexes we expect to find.
+ for name, expression in self.indexes.items():
+ if name not in db_indexes:
+ # The index does not yet exist.
+ d = _create_index(name, expression)
+ deferreds.append(d)
+ elif expression != db_indexes[name]:
+ # The index exists but the definition is not what expected,
+ # so we delete it and add the proper index expression.
+ d = store.delete_index(name)
+ d.addCallback(
+ lambda _: _create_index(name, *expression))
+ deferreds.append(d)
+ return defer.gatherResults(deferreds, consumeErrors=True)
+
+ def store_ready(whatever):
+ self.store_ready = True
+ return whatever
+
+ self.deferred_indexes = store.list_indexes()
+ self.deferred_indexes.addCallback(init_idexes)
+ self.deferred_indexes.addCallback(store_ready)
+ return self.deferred_indexes
+
+ def _wait_for_indexes(self):
+ """
+ Make the marked methods to wait for the indexes to be ready.
+ Heavily based on
+ http://blogs.fluidinfo.com/terry/2009/05/11/a-mixin-class-allowing-python-__init__-methods-to-work-with-twisted-deferreds/
+
+ :param methods: methods that need to wait for the indexes to be ready
+ :type methods: tuple(str)
+ """
+ leap_assert_type(self.wait_for_indexes, list)
+ methods = self.wait_for_indexes
+
+ self.waiting = []
+ self.stored = {}
+
+ def makeWrapper(method):
+ def wrapper(*args, **kw):
+ d = defer.Deferred()
+ d.addCallback(lambda _: self.stored[method](*args, **kw))
+ self.waiting.append(d)
+ return d
+ return wrapper
+
+ for method in methods:
+ self.stored[method] = getattr(self, method)
+ setattr(self, method, makeWrapper(method))
+
+ def _restore_waiting_methods(self, _):
+ for method in self.stored:
+ setattr(self, method, self.stored[method])
+ for d in self.waiting:
+ d.callback(None)
+
+
+class SoledadMailAdaptor(SoledadIndexMixin):
+
+ implements(IMailAdaptor)
+ store = None
+
+ indexes = indexes.MAIL_INDEXES
+ wait_for_indexes = ['get_or_create_mbox', 'update_mbox', 'get_all_mboxes']
+
+ mboxwrapper_klass = MailboxWrapper
+
+ def __init__(self):
+ SoledadIndexMixin.__init__(self)
+
+ # Message handling
+
+ def get_msg_from_string(self, MessageClass, raw_msg):
+ """
+ Get an instance of a MessageClass initialized with a MessageWrapper
+ that contains all the parts obtained from parsing the raw string for
+ the message.
+
+ :param MessageClass: any Message class that can be initialized passing
+ an instance of an IMessageWrapper implementor.
+ :type MessageClass: type
+ :param raw_msg: a string containing the raw email message.
+ :type raw_msg: str
+ :rtype: MessageClass instance.
+ """
+ assert(MessageClass is not None)
+ mdoc, fdoc, hdoc, cdocs = _split_into_parts(raw_msg)
+ return self.get_msg_from_docs(
+ MessageClass, mdoc, fdoc, hdoc, cdocs)
+
+ def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None,
+ uid=None):
+ """
+ Get an instance of a MessageClass initialized with a MessageWrapper
+ that contains the passed part documents.
+
+ This is not the recommended way of obtaining a message, unless you know
+ how to take care of ensuring the internal consistency between the part
+ documents, or unless you are glueing together the part documents that
+ have been previously generated by `get_msg_from_string`.
+
+ :param MessageClass: any Message class that can be initialized passing
+ an instance of an IMessageWrapper implementor.
+ :type MessageClass: type
+ :param fdoc: a dictionary containing values from which a
+ FlagsDocWrapper can be initialized
+ :type fdoc: dict
+ :param hdoc: a dictionary containing values from which a
+ HeaderDocWrapper can be initialized
+ :type hdoc: dict
+ :param cdocs: None, or a dictionary mapping integers (1-indexed) to
+ dicts from where a ContentDocWrapper can be initialized.
+ :type cdocs: dict, or None
+
+ :rtype: MessageClass instance.
+ """
+ assert(MessageClass is not None)
+ return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs), uid=uid)
+
+ def get_msg_from_mdoc_id(self, MessageClass, store, mdoc_id,
+ uid=None, get_cdocs=False):
+
+ def wrap_meta_doc(doc):
+ cls = MetaMsgDocWrapper
+ return cls(doc_id=doc.doc_id, **doc.content)
+
+ def get_part_docs_from_mdoc_wrapper(wrapper):
+ d_docs = []
+ d_docs.append(store.get_doc(wrapper.fdoc))
+ d_docs.append(store.get_doc(wrapper.hdoc))
+ for cdoc in wrapper.cdocs:
+ d_docs.append(store.get_doc(cdoc))
+
+ def add_mdoc(doc_list):
+ return [wrapper.serialize()] + doc_list
+
+ d = defer.gatherResults(d_docs)
+ d.addCallback(add_mdoc)
+ return d
+
+ def get_parts_doc_from_mdoc_id():
+ mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0]
+ chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0]
+
+ def _get_fdoc_id_from_mdoc_id():
+ return constants.FDOCID.format(mbox_uuid=mbox, chash=chash)
+
+ def _get_hdoc_id_from_mdoc_id():
+ return constants.HDOCID.format(mbox_uuid=mbox, chash=chash)
+
+ d_docs = []
+ fdoc_id = _get_fdoc_id_from_mdoc_id()
+ hdoc_id = _get_hdoc_id_from_mdoc_id()
+
+ d_docs.append(store.get_doc(mdoc_id))
+ d_docs.append(store.get_doc(fdoc_id))
+ d_docs.append(store.get_doc(hdoc_id))
+
+ d = defer.gatherResults(d_docs)
+ return d
+
+ def _err_log_failure_part_docs(failure):
+ # See https://leap.se/code/issues/7495.
+ # This avoids blocks, but the real cause still needs to be
+ # isolated (0.9.0rc3) -- kali
+ log.msg("BUG ---------------------------------------------------")
+ log.msg("BUG: Error while retrieving part docs for mdoc id %s" %
+ mdoc_id)
+ log.err(failure)
+ log.msg("BUG (please report above info) ------------------------")
+ return []
+
+ def _err_log_cannot_find_msg(failure):
+ log.msg("BUG: Error while getting msg (uid=%s)" % uid)
+ return None
+
+ if get_cdocs:
+ d = store.get_doc(mdoc_id)
+ d.addCallback(wrap_meta_doc)
+ d.addCallback(get_part_docs_from_mdoc_wrapper)
+ d.addErrback(_err_log_failure_part_docs)
+
+ else:
+ d = get_parts_doc_from_mdoc_id()
+
+ d.addCallback(self._get_msg_from_variable_doc_list,
+ msg_class=MessageClass, uid=uid)
+ d.addErrback(_err_log_cannot_find_msg)
+ return d
+
+ def _get_msg_from_variable_doc_list(self, doc_list, msg_class, uid=None):
+ if len(doc_list) == 3:
+ mdoc, fdoc, hdoc = doc_list
+ cdocs = None
+ elif len(doc_list) > 3:
+ # XXX is this case used?
+ mdoc, fdoc, hdoc = doc_list[:3]
+ cdocs = dict(enumerate(doc_list[3:], 1))
+ return self.get_msg_from_docs(
+ msg_class, mdoc, fdoc, hdoc, cdocs, uid=uid)
+
+ def get_flags_from_mdoc_id(self, store, mdoc_id):
+ """
+ # XXX stuff here...
+ """
+ mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0]
+ chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0]
+
+ def _get_fdoc_id_from_mdoc_id():
+ return constants.FDOCID.format(mbox_uuid=mbox, chash=chash)
+
+ fdoc_id = _get_fdoc_id_from_mdoc_id()
+
+ def wrap_fdoc(doc):
+ if not doc:
+ return
+ cls = FlagsDocWrapper
+ return cls(doc_id=doc.doc_id, **doc.content)
+
+ def get_flags(fdoc_wrapper):
+ if not fdoc_wrapper:
+ return []
+ return fdoc_wrapper.get_flags()
+
+ d = store.get_doc(fdoc_id)
+ d.addCallback(wrap_fdoc)
+ d.addCallback(get_flags)
+ return d
+
+ def create_msg(self, store, msg):
+ """
+ :param store: an instance of soledad, or anything that behaves alike
+ :param msg: a Message object.
+
+ :return: a Deferred that is fired when all the underlying documents
+ have been created.
+ :rtype: defer.Deferred
+ """
+ wrapper = msg.get_wrapper()
+ return wrapper.create(store)
+
+ def update_msg(self, store, msg):
+ """
+ :param msg: a Message object.
+ :param store: an instance of soledad, or anything that behaves alike
+ :return: a Deferred that is fired when all the underlying documents
+ have been updated (actually, it's only the fdoc that's allowed
+ to update).
+ :rtype: defer.Deferred
+ """
+ wrapper = msg.get_wrapper()
+ return wrapper.update(store)
+
+ # batch deletion
+
+ def del_all_flagged_messages(self, store, mbox_uuid):
+ """
+ Delete all messages flagged as deleted.
+ """
+ def err(failure):
+ log.err(failure)
+
+ def delete_fdoc_and_mdoc_flagged(fdocs):
+ # low level here, not using the wrappers...
+ # get meta doc ids from the flag doc ids
+ fdoc_ids = [doc.doc_id for doc in fdocs]
+ mdoc_ids = map(lambda s: "M" + s[1:], fdoc_ids)
+
+ def delete_all_docs(mdocs, fdocs):
+ mdocs = list(mdocs)
+ doc_ids = [m.doc_id for m in mdocs]
+ _d = []
+ docs = mdocs + fdocs
+ for doc in docs:
+ _d.append(store.delete_doc(doc))
+ d = defer.gatherResults(_d)
+ # return the mdocs ids only
+ d.addCallback(lambda _: doc_ids)
+ return d
+
+ d = store.get_docs(mdoc_ids)
+ d.addCallback(delete_all_docs, fdocs)
+ d.addErrback(err)
+ return d
+
+ type_ = FlagsDocWrapper.model.type_
+ uuid = mbox_uuid.replace('-', '_')
+ deleted_index = indexes.TYPE_MBOX_DEL_IDX
+
+ d = store.get_from_index(deleted_index, type_, uuid, "1")
+ d.addCallbacks(delete_fdoc_and_mdoc_flagged, err)
+ return d
+
+ # count messages
+
+ def get_count_unseen(self, store, mbox_uuid):
+ """
+ Get the number of unseen messages for a given mailbox.
+
+ :param store: instance of Soledad.
+ :param mbox_uuid: the uuid for this mailbox.
+ :rtype: int
+ """
+ type_ = FlagsDocWrapper.model.type_
+ uuid = mbox_uuid.replace('-', '_')
+
+ unseen_index = indexes.TYPE_MBOX_SEEN_IDX
+
+ d = store.get_count_from_index(unseen_index, type_, uuid, "0")
+ d.addErrback(self._errback)
+ return d
+
+ def get_count_recent(self, store, mbox_uuid):
+ """
+ Get the number of recent messages for a given mailbox.
+
+ :param store: instance of Soledad.
+ :param mbox_uuid: the uuid for this mailbox.
+ :rtype: int
+ """
+ type_ = FlagsDocWrapper.model.type_
+ uuid = mbox_uuid.replace('-', '_')
+
+ recent_index = indexes.TYPE_MBOX_RECENT_IDX
+
+ d = store.get_count_from_index(recent_index, type_, uuid, "1")
+ d.addErrback(self._errback)
+ return d
+
+ # search api
+
+ def get_mdoc_id_from_msgid(self, store, mbox_uuid, msgid):
+ """
+ Get the UID for a message with the passed msgid (the one in the headers
+ msg-id).
+ This is used by the MUA to retrieve the recently saved draft.
+ """
+ type_ = HeaderDocWrapper.model.type_
+ uuid = mbox_uuid.replace('-', '_')
+
+ msgid_index = indexes.TYPE_MSGID_IDX
+
+ def get_mdoc_id(hdoc):
+ if not hdoc:
+ log.msg("Could not find a HDOC with MSGID %s" % msgid)
+ return None
+ hdoc = hdoc[0]
+ mdoc_id = hdoc.doc_id.replace("H-", "M-%s-" % uuid)
+ return mdoc_id
+
+ d = store.get_from_index(msgid_index, type_, msgid)
+ d.addCallback(get_mdoc_id)
+ return d
+
+ # Mailbox handling
+
+ def get_or_create_mbox(self, store, name):
+ """
+ Get the mailbox with the given name, or create one if it does not
+ exist.
+
+ :param store: instance of Soledad
+ :param name: the name of the mailbox
+ :type name: str
+ """
+ index = indexes.TYPE_MBOX_IDX
+ mbox = normalize_mailbox(name)
+ return MailboxWrapper.get_or_create(store, index, mbox)
+
+ def update_mbox(self, store, mbox_wrapper):
+ """
+ Update the documents for a given mailbox.
+ :param mbox_wrapper: MailboxWrapper instance
+ :type mbox_wrapper: MailboxWrapper
+ :return: a Deferred that will be fired when the mailbox documents
+ have been updated.
+ :rtype: defer.Deferred
+ """
+ leap_assert_type(mbox_wrapper, SoledadDocumentWrapper)
+ return mbox_wrapper.update(store)
+
+ def delete_mbox(self, store, mbox_wrapper):
+ leap_assert_type(mbox_wrapper, SoledadDocumentWrapper)
+ return mbox_wrapper.delete(store)
+
+ def get_all_mboxes(self, store):
+ """
+ Retrieve a list with wrappers for all the mailboxes.
+
+ :return: a deferred that will be fired with a list of all the
+ MailboxWrappers found.
+ :rtype: defer.Deferred
+ """
+ return MailboxWrapper.get_all(store)
+
+ def _errback(self, failure):
+ log.err(failure)
+
+
+def _split_into_parts(raw):
+ # TODO signal that we can delete the original message!-----
+ # when all the processing is done.
+ # TODO add the linked-from info !
+ # TODO add reference to the original message?
+ # TODO populate Default FLAGS/TAGS (unseen?)
+ # TODO seed propely the content_docs with defaults??
+
+ msg, chash, multi = _parse_msg(raw)
+ size = len(msg.as_string())
+
+ parts_map = walk.get_tree(msg)
+ cdocs_list = list(walk.get_raw_docs(msg))
+ cdocs_phashes = [c['phash'] for c in cdocs_list]
+ body_phash = walk.get_body_phash(msg)
+
+ mdoc = _build_meta_doc(chash, cdocs_phashes)
+ fdoc = _build_flags_doc(chash, size, multi)
+ hdoc = _build_headers_doc(msg, chash, body_phash, parts_map)
+
+ # The MessageWrapper expects a dict, one-indexed
+ cdocs = dict(enumerate(cdocs_list, 1))
+
+ return mdoc, fdoc, hdoc, cdocs
+
+
+def _parse_msg(raw):
+ msg = message_from_string(raw)
+ chash = walk.get_hash(raw)
+ multi = msg.is_multipart()
+ return msg, chash, multi
+
+
+def _build_meta_doc(chash, cdocs_phashes):
+ _mdoc = MetaMsgDocWrapper()
+ # FIXME passing the inbox name because we don't have the uuid at this
+ # point.
+
+ _mdoc.fdoc = constants.FDOCID.format(mbox_uuid=INBOX_NAME, chash=chash)
+ _mdoc.hdoc = constants.HDOCID.format(chash=chash)
+ _mdoc.cdocs = [constants.CDOCID.format(phash=p) for p in cdocs_phashes]
+
+ return _mdoc.serialize()
+
+
+def _build_flags_doc(chash, size, multi):
+ _fdoc = FlagsDocWrapper(chash=chash, size=size, multi=multi)
+ return _fdoc.serialize()
+
+
+def _build_headers_doc(msg, chash, body_phash, parts_map):
+ """
+ Assemble a headers document from the original parsed message, the
+ content-hash, and the parts map.
+
+ It takes into account possibly repeated headers.
+ """
+ headers = defaultdict(list)
+ for k, v in msg.items():
+ headers[k].append(v)
+ # "fix" for repeated headers (as in "Received:"
+ for k, v in headers.items():
+ newline = "\n%s: " % (k.lower(),)
+ headers[k] = newline.join(v)
+
+ lower_headers = lowerdict(dict(headers))
+ msgid = first(_MSGID_RE.findall(
+ lower_headers.get('message-id', '')))
+
+ _hdoc = HeaderDocWrapper(
+ chash=chash, headers=headers, body=body_phash,
+ msgid=msgid)
+
+ def copy_attr(headers, key, doc):
+ if key in headers:
+ setattr(doc, key, headers[key])
+
+ copy_attr(lower_headers, "subject", _hdoc)
+ copy_attr(lower_headers, "date", _hdoc)
+
+ hdoc = _hdoc.serialize()
+ # add some of the attr from the parts map to header doc
+ for key in parts_map:
+ if key in ('body', 'multi', 'part_map'):
+ hdoc[key] = parts_map[key]
+ return stringify_parts_map(hdoc)
diff --git a/src/leap/bitmask/mail/adaptors/soledad_indexes.py b/src/leap/bitmask/mail/adaptors/soledad_indexes.py
new file mode 100644
index 00000000..eec7d286
--- /dev/null
+++ b/src/leap/bitmask/mail/adaptors/soledad_indexes.py
@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+# soledad_indexes.py
+# Copyright (C) 2013, 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledad Indexes for Mail Documents.
+"""
+
+# TODO
+# [ ] hide most of the constants here
+
+# Document Type, for indexing
+
+TYPE = "type"
+MBOX = "mbox"
+MBOX_UUID = "mbox_uuid"
+FLAGS = "flags"
+HEADERS = "head"
+CONTENT = "cnt"
+RECENT = "rct"
+HDOCS_SET = "hdocset"
+
+INCOMING_KEY = "incoming"
+ERROR_DECRYPTING_KEY = "errdecr"
+
+# indexing keys
+CONTENT_HASH = "chash"
+PAYLOAD_HASH = "phash"
+MSGID = "msgid"
+UID = "uid"
+
+
+# Index types
+# --------------
+
+TYPE_IDX = 'by-type'
+TYPE_MBOX_IDX = 'by-type-and-mbox'
+TYPE_MBOX_UUID_IDX = 'by-type-and-mbox-uuid'
+TYPE_SUBS_IDX = 'by-type-and-subscribed'
+TYPE_MSGID_IDX = 'by-type-and-message-id'
+TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
+TYPE_MBOX_RECENT_IDX = 'by-type-and-mbox-and-recent'
+TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted'
+TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash'
+TYPE_C_HASH_IDX = 'by-type-and-contenthash'
+TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber'
+TYPE_P_HASH_IDX = 'by-type-and-payloadhash'
+
+# Soledad index for incoming mail, without decrypting errors.
+# and the backward-compatible index, will be deprecated at 0.7
+JUST_MAIL_IDX = "just-mail"
+JUST_MAIL_COMPAT_IDX = "just-mail-compat"
+
+
+# TODO
+# it would be nice to measure the cost of indexing
+# by many fields.
+
+# TODO
+# make the indexes dict more readable!
+
+MAIL_INDEXES = {
+ # generic
+ TYPE_IDX: [TYPE],
+ TYPE_MBOX_IDX: [TYPE, MBOX],
+ TYPE_MBOX_UUID_IDX: [TYPE, MBOX_UUID],
+
+ # XXX deprecate 0.4.0
+ # TYPE_MBOX_UID_IDX: [TYPE, MBOX, UID],
+
+ # mailboxes
+ TYPE_SUBS_IDX: [TYPE, 'bool(subscribed)'],
+
+ # fdocs uniqueness
+ TYPE_MBOX_C_HASH_IDX: [TYPE, MBOX, CONTENT_HASH],
+
+ # headers doc - search by msgid.
+ TYPE_MSGID_IDX: [TYPE, MSGID],
+
+ # content, headers doc
+ TYPE_C_HASH_IDX: [TYPE, CONTENT_HASH],
+
+ # attachment payload dedup
+ TYPE_P_HASH_IDX: [TYPE, PAYLOAD_HASH],
+
+ # messages
+ TYPE_MBOX_SEEN_IDX: [TYPE, MBOX_UUID, 'bool(seen)'],
+ TYPE_MBOX_RECENT_IDX: [TYPE, MBOX_UUID, 'bool(recent)'],
+ TYPE_MBOX_DEL_IDX: [TYPE, MBOX_UUID, 'bool(deleted)'],
+
+ # incoming queue
+ JUST_MAIL_IDX: ["bool(%s)" % (INCOMING_KEY,),
+ "bool(%s)" % (ERROR_DECRYPTING_KEY,)],
+}
diff --git a/src/leap/bitmask/mail/adaptors/tests/rfc822.message b/src/leap/bitmask/mail/adaptors/tests/rfc822.message
new file mode 120000
index 00000000..b19cc280
--- /dev/null
+++ b/src/leap/bitmask/mail/adaptors/tests/rfc822.message
@@ -0,0 +1 @@
+../../tests/rfc822.message \ No newline at end of file
diff --git a/src/leap/bitmask/mail/adaptors/tests/test_models.py b/src/leap/bitmask/mail/adaptors/tests/test_models.py
new file mode 100644
index 00000000..b82cfad0
--- /dev/null
+++ b/src/leap/bitmask/mail/adaptors/tests/test_models.py
@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+# test_models.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the leap.mail.adaptors.models module.
+"""
+from twisted.trial import unittest
+
+from leap.mail.adaptors import models
+
+
+class SerializableModelsTestCase(unittest.TestCase):
+
+ def test_good_serialized_model(self):
+
+ class M(models.SerializableModel):
+ foo = 42
+ bar = 33
+ baaz_ = None
+ _nope = 0
+ __nope = 0
+
+ def not_today(self):
+ pass
+
+ class IgnoreMe(object):
+ pass
+
+ def killmeplease(x):
+ return x
+
+ serialized = M.serialize()
+ expected = {'foo': 42, 'bar': 33, 'baaz': None}
+ self.assertEqual(serialized, expected)
+
+
+class DocumentWrapperTestCase(unittest.TestCase):
+
+ def test_wrapper_defaults(self):
+
+ class Wrapper(models.DocumentWrapper):
+ class model(models.SerializableModel):
+ foo = 42
+ bar = 11
+
+ wrapper = Wrapper()
+ wrapper._ignored = True
+ serialized = wrapper.serialize()
+ expected = {'foo': 42, 'bar': 11}
+ self.assertEqual(serialized, expected)
+
+ def test_initialized_wrapper(self):
+
+ class Wrapper(models.DocumentWrapper):
+ class model(models.SerializableModel):
+ foo = 42
+ bar_ = 11
+
+ wrapper = Wrapper(foo=0, bar=-1)
+ serialized = wrapper.serialize()
+ expected = {'foo': 0, 'bar': -1}
+ self.assertEqual(serialized, expected)
+
+ wrapper.foo = 23
+ serialized = wrapper.serialize()
+ expected = {'foo': 23, 'bar': -1}
+ self.assertEqual(serialized, expected)
+
+ wrapper = Wrapper(foo=0)
+ serialized = wrapper.serialize()
+ expected = {'foo': 0, 'bar': 11}
+ self.assertEqual(serialized, expected)
+
+ def test_invalid_initialized_wrapper(self):
+
+ class Wrapper(models.DocumentWrapper):
+ class model(models.SerializableModel):
+ foo = 42
+
+ def getwrapper():
+ return Wrapper(bar=1)
+ self.assertRaises(RuntimeError, getwrapper)
+
+ def test_no_model_wrapper(self):
+
+ class Wrapper(models.DocumentWrapper):
+ pass
+
+ def getwrapper():
+ w = Wrapper()
+ w.foo = None
+
+ self.assertRaises(RuntimeError, getwrapper)
diff --git a/src/leap/bitmask/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/bitmask/mail/adaptors/tests/test_soledad_adaptor.py
new file mode 100644
index 00000000..73eaf164
--- /dev/null
+++ b/src/leap/bitmask/mail/adaptors/tests/test_soledad_adaptor.py
@@ -0,0 +1,529 @@
+# -*- coding: utf-8 -*-
+# test_soledad_adaptor.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the Soledad Adaptor module - leap.mail.adaptors.soledad
+"""
+import os
+from functools import partial
+
+from twisted.internet import defer
+
+from leap.mail.adaptors import models
+from leap.mail.adaptors.soledad import SoledadDocumentWrapper
+from leap.mail.adaptors.soledad import SoledadIndexMixin
+from leap.mail.adaptors.soledad import SoledadMailAdaptor
+from leap.mail.testing.common import SoledadTestMixin
+
+from email.MIMEMultipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+# DEBUG
+# import logging
+# logging.basicConfig(level=logging.DEBUG)
+
+
+class CounterWrapper(SoledadDocumentWrapper):
+ class model(models.SerializableModel):
+ counter = 0
+ flag = None
+
+
+class CharacterWrapper(SoledadDocumentWrapper):
+ class model(models.SerializableModel):
+ name = ""
+ age = 20
+
+
+class ActorWrapper(SoledadDocumentWrapper):
+ class model(models.SerializableModel):
+ type_ = "actor"
+ name = None
+
+ class __meta__(object):
+ index = "name"
+ list_index = ("by-type", "type_")
+
+
+class TestAdaptor(SoledadIndexMixin):
+ indexes = {'by-name': ['name'],
+ 'by-type-and-name': ['type', 'name'],
+ 'by-type': ['type']}
+
+
+class SoledadDocWrapperTestCase(SoledadTestMixin):
+ """
+ Tests for the SoledadDocumentWrapper.
+ """
+ def assert_num_docs(self, num, docs):
+ self.assertEqual(len(docs[1]), num)
+
+ def test_create_single(self):
+
+ store = self._soledad
+ wrapper = CounterWrapper()
+
+ def assert_one_doc(docs):
+ self.assertEqual(docs[0], 1)
+
+ d = wrapper.create(store)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(assert_one_doc)
+ return d
+
+ def test_create_many(self):
+
+ store = self._soledad
+ w1 = CounterWrapper()
+ w2 = CounterWrapper(counter=1)
+ w3 = CounterWrapper(counter=2)
+ w4 = CounterWrapper(counter=3)
+ w5 = CounterWrapper(counter=4)
+
+ d1 = [w1.create(store),
+ w2.create(store),
+ w3.create(store),
+ w4.create(store),
+ w5.create(store)]
+
+ d = defer.gatherResults(d1)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 5))
+ return d
+
+ def test_multiple_updates(self):
+
+ store = self._soledad
+ wrapper = CounterWrapper(counter=1)
+ MAX = 100
+
+ def assert_doc_id(doc):
+ self.assertTrue(wrapper._doc_id is not None)
+ return doc
+
+ def assert_counter_initial_ok(doc):
+ self.assertEqual(wrapper.counter, 1)
+
+ def increment_counter(ignored):
+ d1 = []
+
+ def record_revision(revision):
+ rev = int(revision.split(':')[1])
+ self.results.append(rev)
+
+ for i in list(range(MAX)):
+ wrapper.counter += 1
+ wrapper.flag = i % 2 == 0
+ d = wrapper.update(store)
+ d.addCallback(record_revision)
+ d1.append(d)
+
+ return defer.gatherResults(d1)
+
+ def assert_counter_final_ok(doc):
+ self.assertEqual(doc.content['counter'], MAX + 1)
+ self.assertEqual(doc.content['flag'], False)
+
+ def assert_results_ordered_list(ignored):
+ self.assertEqual(self.results, sorted(range(2, MAX + 2)))
+
+ d = wrapper.create(store)
+ d.addCallback(assert_doc_id)
+ d.addCallback(assert_counter_initial_ok)
+ d.addCallback(increment_counter)
+ d.addCallback(lambda _: store.get_doc(wrapper._doc_id))
+ d.addCallback(assert_counter_final_ok)
+ d.addCallback(assert_results_ordered_list)
+ return d
+
+ def test_delete(self):
+ adaptor = TestAdaptor()
+ store = self._soledad
+
+ wrapper_list = []
+
+ def get_or_create_bob(ignored):
+ def add_to_list(wrapper):
+ wrapper_list.append(wrapper)
+ return wrapper
+ wrapper = CharacterWrapper.get_or_create(
+ store, 'by-name', 'bob')
+ wrapper.addCallback(add_to_list)
+ return wrapper
+
+ def delete_bob(ignored):
+ wrapper = wrapper_list[0]
+ return wrapper.delete(store)
+
+ d = adaptor.initialize_store(store)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 0))
+
+ # this should create bob document
+ d.addCallback(get_or_create_bob)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+
+ d.addCallback(delete_bob)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 0))
+ return d
+
+ def test_get_or_create(self):
+ adaptor = TestAdaptor()
+ store = self._soledad
+
+ def get_or_create_bob(ignored):
+ wrapper = CharacterWrapper.get_or_create(
+ store, 'by-name', 'bob')
+ return wrapper
+
+ d = adaptor.initialize_store(store)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 0))
+
+ # this should create bob document
+ d.addCallback(get_or_create_bob)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+
+ # this should get us bob document
+ d.addCallback(get_or_create_bob)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+ return d
+
+ def test_get_or_create_multi_index(self):
+ adaptor = TestAdaptor()
+ store = self._soledad
+
+ def get_or_create_actor_harry(ignored):
+ wrapper = ActorWrapper.get_or_create(
+ store, 'by-type-and-name', 'harrison')
+ return wrapper
+
+ def create_director_harry(ignored):
+ wrapper = ActorWrapper(name="harrison", type="director")
+ return wrapper.create(store)
+
+ d = adaptor.initialize_store(store)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 0))
+
+ # this should create harrison document
+ d.addCallback(get_or_create_actor_harry)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+
+ # this should get us harrison document
+ d.addCallback(get_or_create_actor_harry)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+
+ # create director harry, should create new doc
+ d.addCallback(create_director_harry)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 2))
+
+ # this should get us harrison document, still 2 docs
+ d.addCallback(get_or_create_actor_harry)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 2))
+ return d
+
+ def test_get_all(self):
+ adaptor = TestAdaptor()
+ store = self._soledad
+ actor_names = ["harry", "carrie", "mark", "david"]
+
+ def create_some_actors(ignored):
+ deferreds = []
+ for name in actor_names:
+ dw = ActorWrapper.get_or_create(
+ store, 'by-type-and-name', name)
+ deferreds.append(dw)
+ return defer.gatherResults(deferreds)
+
+ d = adaptor.initialize_store(store)
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 0))
+
+ d.addCallback(create_some_actors)
+
+ d.addCallback(lambda _: store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 4))
+
+ def assert_actor_list_is_expected(res):
+ got = set([actor.name for actor in res])
+ expected = set(actor_names)
+ self.assertEqual(got, expected)
+
+ d.addCallback(lambda _: ActorWrapper.get_all(store))
+ d.addCallback(assert_actor_list_is_expected)
+ return d
+
+HERE = os.path.split(os.path.abspath(__file__))[0]
+
+
+class MessageClass(object):
+ def __init__(self, wrapper, uid):
+ self.wrapper = wrapper
+ self.uid = uid
+
+ def get_wrapper(self):
+ return self.wrapper
+
+
+class SoledadMailAdaptorTestCase(SoledadTestMixin):
+ """
+ Tests for the SoledadMailAdaptor.
+ """
+
+ def get_adaptor(self):
+ adaptor = SoledadMailAdaptor()
+ adaptor.store = self._soledad
+ return adaptor
+
+ def assert_num_docs(self, num, docs):
+ self.assertEqual(len(docs[1]), num)
+
+ def test_mail_adaptor_init(self):
+ adaptor = self.get_adaptor()
+ self.assertTrue(isinstance(adaptor.indexes, dict))
+ self.assertTrue(len(adaptor.indexes) != 0)
+
+ # Messages
+
+ def test_get_msg_from_string(self):
+ adaptor = self.get_adaptor()
+
+ with open(os.path.join(HERE, "rfc822.message")) as f:
+ raw = f.read()
+
+ msg = adaptor.get_msg_from_string(MessageClass, raw)
+
+ chash = ("D27B2771C0DCCDCB468EE65A4540438"
+ "09DBD11588E87E951545BE0CBC321C308")
+ phash = ("64934534C1C80E0D4FA04BE1CCBA104"
+ "F07BCA5F469C86E2C0ABE1D41310B7299")
+ subject = ("[Twisted-commits] rebuild now works on "
+ "python versions from 2.2.0 and up.")
+ self.assertTrue(msg.wrapper.fdoc is not None)
+ self.assertTrue(msg.wrapper.hdoc is not None)
+ self.assertTrue(msg.wrapper.cdocs is not None)
+ self.assertEquals(len(msg.wrapper.cdocs), 1)
+ self.assertEquals(msg.wrapper.fdoc.chash, chash)
+ self.assertEquals(msg.wrapper.fdoc.size, 3837)
+ self.assertEquals(msg.wrapper.hdoc.chash, chash)
+ self.assertEqual(dict(msg.wrapper.hdoc.headers)['Subject'],
+ subject)
+ self.assertEqual(msg.wrapper.hdoc.subject, subject)
+ self.assertEqual(msg.wrapper.cdocs[1].phash, phash)
+
+ def test_get_msg_from_string_multipart(self):
+ msg = MIMEMultipart()
+ msg['Subject'] = 'Test multipart mail'
+ msg.attach(MIMEText(u'a utf8 message', _charset='utf-8'))
+ adaptor = self.get_adaptor()
+
+ msg = adaptor.get_msg_from_string(MessageClass, msg.as_string())
+
+ self.assertEqual(
+ 'base64', msg.wrapper.cdocs[1].content_transfer_encoding)
+ self.assertEqual(
+ 'text/plain', msg.wrapper.cdocs[1].content_type)
+ self.assertEqual(
+ 'YSB1dGY4IG1lc3NhZ2U=\n', msg.wrapper.cdocs[1].raw)
+
+ def test_get_msg_from_docs(self):
+ adaptor = self.get_adaptor()
+ mdoc = dict(
+ fdoc="F-Foobox-deadbeef",
+ hdoc="H-deadbeef",
+ cdocs=["C-deadabad"])
+ fdoc = dict(
+ mbox_uuid="Foobox",
+ flags=('\Seen', '\Nice'),
+ tags=('Personal', 'TODO'),
+ seen=False, deleted=False,
+ recent=False, multi=False)
+ hdoc = dict(
+ chash="deadbeef",
+ subject="Test Msg")
+ cdocs = {
+ 1: dict(
+ raw='This is a test message')}
+
+ msg = adaptor.get_msg_from_docs(
+ MessageClass, mdoc, fdoc, hdoc, cdocs=cdocs)
+ self.assertEqual(msg.wrapper.fdoc.flags,
+ ('\Seen', '\Nice'))
+ self.assertEqual(msg.wrapper.fdoc.tags,
+ ('Personal', 'TODO'))
+ self.assertEqual(msg.wrapper.fdoc.mbox_uuid, "Foobox")
+ self.assertEqual(msg.wrapper.hdoc.multi, False)
+ self.assertEqual(msg.wrapper.hdoc.subject,
+ "Test Msg")
+ self.assertEqual(msg.wrapper.cdocs[1].raw,
+ "This is a test message")
+
+ def test_get_msg_from_metamsg_doc_id(self):
+ # TODO complete-me!
+ pass
+
+ test_get_msg_from_metamsg_doc_id.skip = "Not yet implemented"
+
+ def test_create_msg(self):
+ adaptor = self.get_adaptor()
+
+ with open(os.path.join(HERE, "rfc822.message")) as f:
+ raw = f.read()
+ msg = adaptor.get_msg_from_string(MessageClass, raw)
+
+ def check_create_result(created):
+ # that's one mdoc, one hdoc, one fdoc, one cdoc
+ self.assertEqual(len(created), 4)
+ for doc in created:
+ self.assertTrue(
+ doc.__class__.__name__,
+ "SoledadDocument")
+
+ d = adaptor.create_msg(adaptor.store, msg)
+ d.addCallback(check_create_result)
+ return d
+
+ def test_update_msg(self):
+ adaptor = self.get_adaptor()
+ with open(os.path.join(HERE, "rfc822.message")) as f:
+ raw = f.read()
+
+ def assert_msg_has_doc_id(ignored, msg):
+ wrapper = msg.get_wrapper()
+ self.assertTrue(wrapper.fdoc.doc_id is not None)
+
+ def assert_msg_has_no_flags(ignored, msg):
+ wrapper = msg.get_wrapper()
+ self.assertEqual(wrapper.fdoc.flags, [])
+
+ def update_msg_flags(ignored, msg):
+ wrapper = msg.get_wrapper()
+ wrapper.fdoc.flags = ["This", "That"]
+ return wrapper.update(adaptor.store)
+
+ def assert_msg_has_flags(ignored, msg):
+ wrapper = msg.get_wrapper()
+ self.assertEqual(wrapper.fdoc.flags, ["This", "That"])
+
+ def get_fdoc_and_check_flags(ignored):
+ def assert_doc_has_flags(doc):
+ self.assertEqual(doc.content['flags'],
+ ['This', 'That'])
+ wrapper = msg.get_wrapper()
+ d = adaptor.store.get_doc(wrapper.fdoc.doc_id)
+ d.addCallback(assert_doc_has_flags)
+ return d
+
+ msg = adaptor.get_msg_from_string(MessageClass, raw)
+ d = adaptor.create_msg(adaptor.store, msg)
+ d.addCallback(lambda _: adaptor.store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 4))
+ d.addCallback(assert_msg_has_doc_id, msg)
+ d.addCallback(assert_msg_has_no_flags, msg)
+
+ # update it!
+ d.addCallback(update_msg_flags, msg)
+ d.addCallback(assert_msg_has_flags, msg)
+ d.addCallback(get_fdoc_and_check_flags)
+ return d
+
+ # Mailboxes
+
+ def test_get_or_create_mbox(self):
+ adaptor = self.get_adaptor()
+
+ def get_or_create_mbox(ignored):
+ d = adaptor.get_or_create_mbox(adaptor.store, "Trash")
+ return d
+
+ def assert_good_doc(mbox_wrapper):
+ self.assertTrue(mbox_wrapper.doc_id is not None)
+ self.assertEqual(mbox_wrapper.mbox, "Trash")
+ self.assertEqual(mbox_wrapper.type, "mbox")
+ self.assertEqual(mbox_wrapper.closed, False)
+ self.assertEqual(mbox_wrapper.subscribed, False)
+
+ d = adaptor.initialize_store(adaptor.store)
+ d.addCallback(get_or_create_mbox)
+ d.addCallback(assert_good_doc)
+ d.addCallback(lambda _: adaptor.store.get_all_docs())
+ d.addCallback(partial(self.assert_num_docs, 1))
+ return d
+
+ def test_update_mbox(self):
+ adaptor = self.get_adaptor()
+
+ wrapper_ref = []
+
+ def get_or_create_mbox(ignored):
+ d = adaptor.get_or_create_mbox(adaptor.store, "Trash")
+ return d
+
+ def update_wrapper(wrapper, wrapper_ref):
+ wrapper_ref.append(wrapper)
+ wrapper.subscribed = True
+ wrapper.closed = True
+ d = adaptor.update_mbox(adaptor.store, wrapper)
+ return d
+
+ def get_mbox_doc_and_check_flags(res, wrapper_ref):
+ wrapper = wrapper_ref[0]
+
+ def assert_doc_has_flags(doc):
+ self.assertEqual(doc.content['subscribed'], True)
+ self.assertEqual(doc.content['closed'], True)
+ d = adaptor.store.get_doc(wrapper.doc_id)
+ d.addCallback(assert_doc_has_flags)
+ return d
+
+ d = adaptor.initialize_store(adaptor.store)
+ d.addCallback(get_or_create_mbox)
+ d.addCallback(update_wrapper, wrapper_ref)
+ d.addCallback(get_mbox_doc_and_check_flags, wrapper_ref)
+ return d
+
+ def test_get_all_mboxes(self):
+ adaptor = self.get_adaptor()
+ mboxes = ("Sent", "Trash", "Personal", "ListFoo")
+
+ def get_or_create_mboxes(ignored):
+ d = []
+ for mbox in mboxes:
+ d.append(adaptor.get_or_create_mbox(
+ adaptor.store, mbox))
+ return defer.gatherResults(d)
+
+ def get_all_mboxes(ignored):
+ return adaptor.get_all_mboxes(adaptor.store)
+
+ def assert_mboxes_match_expected(wrappers):
+ names = [m.mbox for m in wrappers]
+ self.assertEqual(set(names), set(mboxes))
+
+ d = adaptor.initialize_store(adaptor.store)
+ d.addCallback(get_or_create_mboxes)
+ d.addCallback(get_all_mboxes)
+ d.addCallback(assert_mboxes_match_expected)
+ return d
diff --git a/src/leap/bitmask/mail/constants.py b/src/leap/bitmask/mail/constants.py
new file mode 100644
index 00000000..4ef42cbb
--- /dev/null
+++ b/src/leap/bitmask/mail/constants.py
@@ -0,0 +1,52 @@
+# *- coding: utf-8 -*-
+# constants.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Constants for leap.mail.
+"""
+
+INBOX_NAME = "INBOX"
+
+# Regular expressions for the identifiers to be used in the Message Data Layer.
+
+METAMSGID = "M-{mbox_uuid}-{chash}"
+METAMSGID_RE = "M\-{mbox_uuid}\-[0-9a-fA-F]+"
+METAMSGID_CHASH_RE = "M\-\w+\-([0-9a-fA-F]+)"
+METAMSGID_MBOX_RE = "M\-(\w+)\-[0-9a-fA-F]+"
+
+FDOCID = "F-{mbox_uuid}-{chash}"
+FDOCID_RE = "F\-{mbox_uuid}\-[0-9a-fA-F]+"
+FDOCID_CHASH_RE = "F\-\w+\-([0-9a-fA-F]+)"
+
+HDOCID = "H-{chash}"
+HDOCID_RE = "H\-[0-9a-fA-F]+"
+
+CDOCID = "C-{phash}"
+CDOCID_RE = "C\-[0-9a-fA-F]+"
+
+
+class MessageFlags(object):
+ """
+ Flags used in Message and Mailbox.
+ """
+ SEEN_FLAG = "\\Seen"
+ RECENT_FLAG = "\\Recent"
+ ANSWERED_FLAG = "\\Answered"
+ FLAGGED_FLAG = "\\Flagged" # yo dawg
+ DELETED_FLAG = "\\Deleted"
+ DRAFT_FLAG = "\\Draft"
+ NOSELECT_FLAG = "\\Noselect"
+ LIST_FLAG = "List" # is this OK? (no \. ie, no system flag)
diff --git a/src/leap/bitmask/mail/cred.py b/src/leap/bitmask/mail/cred.py
new file mode 100644
index 00000000..7eab1f04
--- /dev/null
+++ b/src/leap/bitmask/mail/cred.py
@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+# cred.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Credentials handling.
+"""
+
+from zope.interface import implementer
+from twisted.cred.checkers import ICredentialsChecker
+from twisted.cred.credentials import IUsernamePassword
+from twisted.cred.error import UnauthorizedLogin
+from twisted.internet import defer
+
+
+@implementer(ICredentialsChecker)
+class LocalSoledadTokenChecker(object):
+
+ """
+ A Credentials Checker for a LocalSoledad store.
+
+ It checks that:
+
+ 1) The Local SoledadStorage has been correctly unlocked for the given
+ user. This currently means that the right passphrase has been passed
+ to the Local SoledadStorage.
+
+ 2) The password passed in the credentials matches whatever token has
+ been stored in the local encrypted SoledadStorage, associated to the
+ Protocol that is requesting the authentication.
+ """
+
+ credentialInterfaces = (IUsernamePassword,)
+ service = None
+
+ def __init__(self, soledad_sessions):
+ """
+ :param soledad_sessions: a dict-like object, containing instances
+ of a Store (soledad instances), indexed by
+ userid.
+ """
+ self._soledad_sessions = soledad_sessions
+
+ def requestAvatarId(self, credentials):
+ if self.service is None:
+ raise NotImplementedError(
+ "this checker has not defined its service name")
+ username, password = credentials.username, credentials.password
+ d = self.checkSoledadToken(username, password, self.service)
+ d.addErrback(lambda f: defer.fail(UnauthorizedLogin()))
+ return d
+
+ def checkSoledadToken(self, username, password, service):
+ soledad = self._soledad_sessions.get(username)
+ if not soledad:
+ return defer.fail(Exception("No soledad"))
+
+ def match_token(token):
+ if token is None:
+ raise RuntimeError('no token')
+ if token == password:
+ return username
+ else:
+ raise RuntimeError('bad token')
+
+ d = soledad.get_or_create_service_token(service)
+ d.addCallback(match_token)
+ return d
diff --git a/src/leap/bitmask/mail/decorators.py b/src/leap/bitmask/mail/decorators.py
new file mode 100644
index 00000000..5105de92
--- /dev/null
+++ b/src/leap/bitmask/mail/decorators.py
@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+# decorators.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Useful decorators for mail package.
+"""
+import logging
+import os
+
+from functools import wraps
+
+from twisted.internet.threads import deferToThread
+
+
+logger = logging.getLogger(__name__)
+
+
+# TODO
+# Should write a helper to be able to pass a timeout argument.
+# See this answer: http://stackoverflow.com/a/19019648/1157664
+# And the notes by glyph and jpcalderone
+
+def deferred_to_thread(f):
+ """
+ Decorator, for deferring methods to Threads.
+
+ It will do a deferToThread of the decorated method
+ unless the environment variable LEAPMAIL_DEBUG is set.
+
+ It uses a descriptor to delay the definition of the
+ method wrapper.
+ """
+ class descript(object):
+ """
+ The class to be used as decorator.
+
+ It takes any method as the passed object.
+ """
+
+ def __init__(self, f):
+ """
+ Initializes the decorator object.
+
+ :param f: the decorated function
+ :type f: callable
+ """
+ self.f = f
+
+ def __get__(self, instance, klass):
+ """
+ Descriptor implementation.
+
+ At creation time, the decorated `method` is unbound.
+
+ It will dispatch the make_unbound method if we still do not
+ have an instance available, and the make_bound method when the
+ method has already been bound to the instance.
+
+ :param instance: the instance of the class, or None if not exist.
+ :type instance: instantiated class or None.
+ """
+ if instance is None:
+ # Class method was requested
+ return self.make_unbound(klass)
+ return self.make_bound(instance)
+
+ def _errback(self, failure):
+ """
+ Errorback that logs the exception catched.
+
+ :param failure: a twisted failure
+ :type failure: Failure
+ """
+ logger.warning('Error in method: %s' % (self.f.__name__))
+ logger.exception(failure.getTraceback())
+
+ def make_unbound(self, klass):
+ """
+ Return a wrapped function with the unbound call, during the
+ early access to the decortad method. This gets passed
+ only the class (not the instance since it does not yet exist).
+
+ :param klass: the class to which the still unbound method belongs
+ :type klass: type
+ """
+
+ @wraps(self.f)
+ def wrapper(*args, **kwargs):
+ """
+ We're temporarily wrapping the decorated method, but this
+ should not be called, since our application should use
+ the bound-wrapped method after this decorator class has been
+ used.
+
+ This documentation will vanish at runtime.
+ """
+ raise TypeError(
+ 'unbound method {}() must be called with {} instance '
+ 'as first argument (got nothing instead)'.format(
+ self.f.__name__,
+ klass.__name__)
+ )
+ return wrapper
+
+ def make_bound(self, instance):
+ """
+ Return a function that wraps the bound method call,
+ after we are able to access the instance object.
+
+ :param instance: an instance of the class the decorated method,
+ now bound, belongs to.
+ :type instance: object
+ """
+
+ @wraps(self.f)
+ def wrapper(*args, **kwargs):
+ """
+ Do a proper function wrapper that defers the decorated method
+ call to a separated thread if the LEAPMAIL_DEBUG
+ environment variable is set.
+
+ This documentation will vanish at runtime.
+ """
+ if not os.environ.get('LEAPMAIL_DEBUG'):
+ d = deferToThread(self.f, instance, *args, **kwargs)
+ d.addErrback(self._errback)
+ return d
+ else:
+ return self.f(instance, *args, **kwargs)
+
+ # This instance does not need the descriptor anymore,
+ # let it find the wrapper directly next time:
+ setattr(instance, self.f.__name__, wrapper)
+ return wrapper
+
+ return descript(f)
diff --git a/src/leap/bitmask/mail/errors.py b/src/leap/bitmask/mail/errors.py
new file mode 100644
index 00000000..2f18e87f
--- /dev/null
+++ b/src/leap/bitmask/mail/errors.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# errors.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Exceptions for leap.mail
+"""
+
+
+class AuthenticationError(Exception):
+ pass
+
+
+class ConfigurationError(Exception):
+ pass
diff --git a/src/leap/bitmask/mail/generator.py b/src/leap/bitmask/mail/generator.py
new file mode 100644
index 00000000..bb3f26e2
--- /dev/null
+++ b/src/leap/bitmask/mail/generator.py
@@ -0,0 +1,23 @@
+from email.generator import Generator as EmailGenerator
+
+
+class Generator(EmailGenerator):
+ """
+ Generates output from a Message object tree, keeping signatures.
+
+ This code was extracted from Mailman.Generator.Generator, version 2.1.4:
+
+ Most other Generator will be created not setting the foldheader flag,
+ as we do not overwrite clone(). The original clone() does not
+ set foldheaders.
+
+ So you need to set foldheaders if you want the toplevel to fold headers
+
+ TODO: Python 3.3 is patched against this problems. See issue 1590744 on
+ python bug tracker.
+ """
+ def _write_headers(self, msg):
+ for h, v in msg.items():
+ print >> self._fp, '%s:' % h,
+ print >> self._fp, v
+ print >> self._fp
diff --git a/src/leap/bitmask/mail/imap/__init__.py b/src/leap/bitmask/mail/imap/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/__init__.py
diff --git a/src/leap/bitmask/mail/imap/account.py b/src/leap/bitmask/mail/imap/account.py
new file mode 100644
index 00000000..e795c1b9
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/account.py
@@ -0,0 +1,498 @@
+# -*- coding: utf-8 -*-
+# account.py
+# Copyright (C) 2013-2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledad Backed IMAP Account.
+"""
+import logging
+import os
+import time
+from functools import partial
+
+from twisted.internet import defer
+from twisted.mail import imap4
+from twisted.python import log
+from zope.interface import implements
+
+from leap.common.check import leap_assert, leap_assert_type
+
+from leap.mail.constants import MessageFlags
+from leap.mail.mail import Account
+from leap.mail.imap.mailbox import IMAPMailbox, normalize_mailbox
+from leap.soledad.client import Soledad
+
+logger = logging.getLogger(__name__)
+
+PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False)
+
+if PROFILE_CMD:
+ def _debugProfiling(result, cmdname, start):
+ took = (time.time() - start) * 1000
+ log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec")
+ return result
+
+
+#######################################
+# Soledad IMAP Account
+#######################################
+
+
+class IMAPAccount(object):
+ """
+ An implementation of an imap4 Account
+ that is backed by Soledad Encrypted Documents.
+ """
+
+ implements(imap4.IAccount, imap4.INamespacePresenter)
+
+ selected = None
+
+ def __init__(self, store, user_id, d=defer.Deferred()):
+ """
+ Keeps track of the mailboxes and subscriptions handled by this account.
+
+ The account is not ready to be used, since the store needs to be
+ initialized and we also need to do some initialization routines.
+ You can either pass a deferred to this constructor, or use
+ `callWhenReady` method.
+
+ :param store: a Soledad instance.
+ :type store: Soledad
+
+ :param user_id: The identifier of the user this account belongs to
+ (user id, in the form user@provider).
+ :type user_id: str
+
+
+ :param d: a deferred that will be fired with this IMAPAccount instance
+ when the account is ready to be used.
+ :type d: defer.Deferred
+ """
+ leap_assert(store, "Need a store instance to initialize")
+ leap_assert_type(store, Soledad)
+
+ # TODO assert too that the name matches the user/uuid with which
+ # soledad has been initialized. Although afaik soledad doesn't know
+ # about user_id, only the client backend.
+
+ self.user_id = user_id
+ self.account = Account(
+ store, user_id, ready_cb=lambda: d.callback(self))
+
+ def end_session(self):
+ """
+ Used to mark when the session has closed, and we should not allow any
+ more commands from the client.
+
+ Right now it's called from the client backend.
+ """
+ # TODO move its use to the service shutdown in leap.mail
+ self.account.end_session()
+
+ @property
+ def session_ended(self):
+ return self.account.session_ended
+
+ def callWhenReady(self, cb, *args, **kw):
+ """
+ Execute callback when the account is ready to be used.
+ XXX note that this callback will be called with a first ignored
+ parameter.
+ """
+ # TODO ignore the first parameter and change tests accordingly.
+ d = self.account.callWhenReady(cb, *args, **kw)
+ return d
+
+ def getMailbox(self, name):
+ """
+ Return a Mailbox with that name, without selecting it.
+
+ :param name: name of the mailbox
+ :type name: str
+
+ :returns: an IMAPMailbox instance
+ :rtype: IMAPMailbox
+ """
+ name = normalize_mailbox(name)
+
+ def check_it_exists(mailboxes):
+ if name not in mailboxes:
+ raise imap4.MailboxException("No such mailbox: %r" % name)
+ return True
+
+ d = self.account.list_all_mailbox_names()
+ d.addCallback(check_it_exists)
+ d.addCallback(lambda _: self.account.get_collection_by_mailbox(name))
+ d.addCallback(self._return_mailbox_from_collection)
+ return d
+
+ def _return_mailbox_from_collection(self, collection, readwrite=1):
+ if collection is None:
+ return None
+ mbox = IMAPMailbox(collection, rw=readwrite)
+ return mbox
+
+ #
+ # IAccount
+ #
+
+ def addMailbox(self, name, creation_ts=None):
+ """
+ Add a mailbox to the account.
+
+ :param name: the name of the mailbox
+ :type name: str
+
+ :param creation_ts: an optional creation timestamp to be used as
+ mailbox id. A timestamp will be used if no
+ one is provided.
+ :type creation_ts: int
+
+ :returns: a Deferred that will contain the document if successful.
+ :rtype: defer.Deferred
+ """
+ name = normalize_mailbox(name)
+
+ # FIXME --- return failure instead of AssertionError
+ # See AccountTestCase...
+ leap_assert(name, "Need a mailbox name to create a mailbox")
+
+ def check_it_does_not_exist(mailboxes):
+ if name in mailboxes:
+ raise imap4.MailboxCollision, repr(name)
+ return mailboxes
+
+ d = self.account.list_all_mailbox_names()
+ d.addCallback(check_it_does_not_exist)
+ d.addCallback(lambda _: self.account.add_mailbox(
+ name, creation_ts=creation_ts))
+ d.addCallback(lambda _: self.account.get_collection_by_mailbox(name))
+ d.addCallback(self._return_mailbox_from_collection)
+ return d
+
+ def create(self, pathspec):
+ """
+ Create a new mailbox from the given hierarchical name.
+
+ :param pathspec:
+ The full hierarchical name of a new mailbox to create.
+ If any of the inferior hierarchical names to this one
+ do not exist, they are created as well.
+ :type pathspec: str
+
+ :return:
+ A deferred that will fire with a true value if the creation
+ succeeds. The deferred might fail with a MailboxException
+ if the mailbox cannot be added.
+ :rtype: Deferred
+
+ """
+ def pass_on_collision(failure):
+ failure.trap(imap4.MailboxCollision)
+ return True
+
+ def handle_collision(failure):
+ failure.trap(imap4.MailboxCollision)
+ if not pathspec.endswith('/'):
+ return defer.succeed(False)
+ else:
+ return defer.succeed(True)
+
+ def all_good(result):
+ return all(result)
+
+ paths = filter(None, normalize_mailbox(pathspec).split('/'))
+ subs = []
+ sep = '/'
+
+ for accum in range(1, len(paths)):
+ partial_path = sep.join(paths[:accum])
+ d = self.addMailbox(partial_path)
+ d.addErrback(pass_on_collision)
+ subs.append(d)
+
+ df = self.addMailbox(sep.join(paths))
+ df.addErrback(handle_collision)
+ subs.append(df)
+
+ d1 = defer.gatherResults(subs)
+ d1.addCallback(all_good)
+ return d1
+
+ def select(self, name, readwrite=1):
+ """
+ Selects a mailbox.
+
+ :param name: the mailbox to select
+ :type name: str
+
+ :param readwrite: 1 for readwrite permissions.
+ :type readwrite: int
+
+ :rtype: IMAPMailbox
+ """
+ name = normalize_mailbox(name)
+
+ def check_it_exists(mailboxes):
+ if name not in mailboxes:
+ logger.warning("SELECT: No such mailbox!")
+ return None
+ return name
+
+ def set_selected(_):
+ self.selected = name
+
+ def get_collection(name):
+ if name is None:
+ return None
+ return self.account.get_collection_by_mailbox(name)
+
+ d = self.account.list_all_mailbox_names()
+ d.addCallback(check_it_exists)
+ d.addCallback(get_collection)
+ d.addCallback(partial(
+ self._return_mailbox_from_collection, readwrite=readwrite))
+ return d
+
+ def delete(self, name, force=False):
+ """
+ Deletes a mailbox.
+
+ :param name: the mailbox to be deleted
+ :type name: str
+
+ :param force:
+ if True, it will not check for noselect flag or inferior
+ names. use with care.
+ :type force: bool
+ :rtype: Deferred
+ """
+ name = normalize_mailbox(name)
+ _mboxes = None
+
+ def check_it_exists(mailboxes):
+ global _mboxes
+ _mboxes = mailboxes
+ if name not in mailboxes:
+ raise imap4.MailboxException("No such mailbox: %r" % name)
+
+ def get_mailbox(_):
+ return self.getMailbox(name)
+
+ def destroy_mailbox(mbox):
+ return mbox.destroy()
+
+ def check_can_be_deleted(mbox):
+ global _mboxes
+ # See if this box is flagged \Noselect
+ mbox_flags = mbox.getFlags()
+ if MessageFlags.NOSELECT_FLAG in mbox_flags:
+ # Check for hierarchically inferior mailboxes with this one
+ # as part of their root.
+ for others in _mboxes:
+ if others != name and others.startswith(name):
+ raise imap4.MailboxException(
+ "Hierarchically inferior mailboxes "
+ "exist and \\Noselect is set")
+ return mbox
+
+ d = self.account.list_all_mailbox_names()
+ d.addCallback(check_it_exists)
+ d.addCallback(get_mailbox)
+ if not force:
+ d.addCallback(check_can_be_deleted)
+ d.addCallback(destroy_mailbox)
+ return d
+
+ # FIXME --- not honoring the inferior names...
+ # if there are no hierarchically inferior names, we will
+ # delete it from our ken.
+ # XXX is this right?
+ # if self._inferiorNames(name) > 1:
+ # self._index.removeMailbox(name)
+
+ def rename(self, oldname, newname):
+ """
+ Renames a mailbox.
+
+ :param oldname: old name of the mailbox
+ :type oldname: str
+
+ :param newname: new name of the mailbox
+ :type newname: str
+ """
+ oldname = normalize_mailbox(oldname)
+ newname = normalize_mailbox(newname)
+
+ def rename_inferiors((inferiors, mailboxes)):
+ rename_deferreds = []
+ inferiors = [
+ (o, o.replace(oldname, newname, 1)) for o in inferiors]
+
+ for (old, new) in inferiors:
+ if new in mailboxes:
+ raise imap4.MailboxCollision(repr(new))
+
+ for (old, new) in inferiors:
+ d = self.account.rename_mailbox(old, new)
+ rename_deferreds.append(d)
+
+ d1 = defer.gatherResults(rename_deferreds, consumeErrors=True)
+ return d1
+
+ d1 = self._inferiorNames(oldname)
+ d2 = self.account.list_all_mailbox_names()
+
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(rename_inferiors)
+ return d
+
+ def _inferiorNames(self, name):
+ """
+ Return hierarchically inferior mailboxes.
+
+ :param name: name of the mailbox
+ :rtype: list
+ """
+ # XXX use wildcard query instead
+ def filter_inferiors(mailboxes):
+ inferiors = []
+ for infname in mailboxes:
+ if infname.startswith(name):
+ inferiors.append(infname)
+ return inferiors
+
+ d = self.account.list_all_mailbox_names()
+ d.addCallback(filter_inferiors)
+ return d
+
+ def listMailboxes(self, ref, wildcard):
+ """
+ List the mailboxes.
+
+ from rfc 3501:
+ returns a subset of names from the complete set
+ of all names available to the client. Zero or more untagged LIST
+ replies are returned, containing the name attributes, hierarchy
+ delimiter, and name.
+
+ :param ref: reference name
+ :type ref: str
+
+ :param wildcard: mailbox name with possible wildcards
+ :type wildcard: str
+ """
+ wildcard = imap4.wildcardToRegexp(wildcard, '/')
+
+ def get_list(mboxes, mboxes_names):
+ return zip(mboxes_names, mboxes)
+
+ def filter_inferiors(ref):
+ mboxes = [mbox for mbox in ref if wildcard.match(mbox)]
+ mbox_d = defer.gatherResults([self.getMailbox(m) for m in mboxes])
+
+ mbox_d.addCallback(get_list, mboxes)
+ return mbox_d
+
+ d = self._inferiorNames(normalize_mailbox(ref))
+ d.addCallback(filter_inferiors)
+ return d
+
+ #
+ # The rest of the methods are specific for leap.mail.imap.account.Account
+ #
+
+ def isSubscribed(self, name):
+ """
+ Returns True if user is subscribed to this mailbox.
+
+ :param name: the mailbox to be checked.
+ :type name: str
+
+ :rtype: Deferred (will fire with bool)
+ """
+ name = normalize_mailbox(name)
+
+ def get_subscribed(mbox):
+ return mbox.collection.get_mbox_attr("subscribed")
+
+ d = self.getMailbox(name)
+ d.addCallback(get_subscribed)
+ return d
+
+ def subscribe(self, name):
+ """
+ Subscribe to this mailbox if not already subscribed.
+
+ :param name: name of the mailbox
+ :type name: str
+ :rtype: Deferred
+ """
+ name = normalize_mailbox(name)
+
+ def set_subscribed(mbox):
+ return mbox.collection.set_mbox_attr("subscribed", True)
+
+ d = self.getMailbox(name)
+ d.addCallback(set_subscribed)
+ return d
+
+ def unsubscribe(self, name):
+ """
+ Unsubscribe from this mailbox
+
+ :param name: name of the mailbox
+ :type name: str
+ :rtype: Deferred
+ """
+ # TODO should raise MailboxException if attempted to unsubscribe
+ # from a mailbox that is not currently subscribed.
+ # TODO factor out with subscribe method.
+ name = normalize_mailbox(name)
+
+ def set_unsubscribed(mbox):
+ return mbox.collection.set_mbox_attr("subscribed", False)
+
+ d = self.getMailbox(name)
+ d.addCallback(set_unsubscribed)
+ return d
+
+ def getSubscriptions(self):
+ def get_subscribed(mailboxes):
+ return [x.mbox for x in mailboxes if x.subscribed]
+
+ d = self.account.get_all_mailboxes()
+ d.addCallback(get_subscribed)
+ return d
+
+ #
+ # INamespacePresenter
+ #
+
+ def getPersonalNamespaces(self):
+ return [["", "/"]]
+
+ def getSharedNamespaces(self):
+ return None
+
+ def getOtherNamespaces(self):
+ return None
+
+ def __repr__(self):
+ """
+ Representation string for this object.
+ """
+ return "<IMAPAccount (%s)>" % self.user_id
diff --git a/src/leap/bitmask/mail/imap/mailbox.py b/src/leap/bitmask/mail/imap/mailbox.py
new file mode 100644
index 00000000..e70a1d80
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/mailbox.py
@@ -0,0 +1,970 @@
+# *- coding: utf-8 -*-
+# mailbox.py
+# Copyright (C) 2013-2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+IMAP Mailbox.
+"""
+import re
+import logging
+import os
+import cStringIO
+import StringIO
+import time
+
+from collections import defaultdict
+from email.utils import formatdate
+
+from twisted.internet import defer
+from twisted.internet import reactor
+from twisted.python import log
+
+from twisted.mail import imap4
+from zope.interface import implements
+
+from leap.common.check import leap_assert
+from leap.common.check import leap_assert_type
+from leap.mail.constants import INBOX_NAME, MessageFlags
+from leap.mail.imap.messages import IMAPMessage
+
+logger = logging.getLogger(__name__)
+
+# TODO LIST
+# [ ] Restore profile_cmd instrumentation
+# [ ] finish the implementation of IMailboxListener
+# [ ] implement the rest of ISearchableMailbox
+
+
+"""
+If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid
+notifying clients of new messages. Use during stress tests.
+"""
+NOTIFY_NEW = not os.environ.get('LEAP_SKIPNOTIFY', False)
+PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False)
+
+if PROFILE_CMD:
+
+ def _debugProfiling(result, cmdname, start):
+ took = (time.time() - start) * 1000
+ log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec")
+ return result
+
+ def do_profile_cmd(d, name):
+ """
+ Add the profiling debug to the passed callback.
+ :param d: deferred
+ :param name: name of the command
+ :type name: str
+ """
+ d.addCallback(_debugProfiling, name, time.time())
+ d.addErrback(lambda f: log.msg(f.getTraceback()))
+
+INIT_FLAGS = (MessageFlags.SEEN_FLAG, MessageFlags.ANSWERED_FLAG,
+ MessageFlags.FLAGGED_FLAG, MessageFlags.DELETED_FLAG,
+ MessageFlags.DRAFT_FLAG, MessageFlags.RECENT_FLAG,
+ MessageFlags.LIST_FLAG)
+
+
+def make_collection_listener(mailbox):
+ """
+ Wrap a mailbox in a class that can be hashed according to the mailbox name.
+
+ This means that dicts or sets will use this new equality rule, so we won't
+ collect multiple instances of the same mailbox in collections like the
+ MessageCollection set where we keep track of listeners.
+ """
+
+ class HashableMailbox(object):
+
+ def __init__(self, mbox):
+ self.mbox = mbox
+
+ # See #8083, pixelated adaptor seems to be misusing this class.
+ self.mailbox_name = self.mbox.mbox_name
+
+ def __hash__(self):
+ return hash(self.mbox.mbox_name)
+
+ def __eq__(self, other):
+ return self.mbox.mbox_name == other.mbox.mbox_name
+
+ def notify_new(self):
+ self.mbox.notify_new()
+
+ return HashableMailbox(mailbox)
+
+
+class IMAPMailbox(object):
+ """
+ A Soledad-backed IMAP mailbox.
+
+ Implements the high-level method needed for the Mailbox interfaces.
+ The low-level database methods are contained in the generic
+ MessageCollection class. We receive an instance of it and it is made
+ accessible in the `collection` attribute.
+ """
+ implements(
+ imap4.IMailbox,
+ imap4.IMailboxInfo,
+ imap4.ISearchableMailbox,
+ # XXX I think we do not need to implement CloseableMailbox, do we?
+ # We could remove ourselves from the collectionListener, although I
+ # think it simply will be garbage collected.
+ # imap4.ICloseableMailbox
+ imap4.IMessageCopier)
+
+ init_flags = INIT_FLAGS
+
+ CMD_MSG = "MESSAGES"
+ CMD_RECENT = "RECENT"
+ CMD_UIDNEXT = "UIDNEXT"
+ CMD_UIDVALIDITY = "UIDVALIDITY"
+ CMD_UNSEEN = "UNSEEN"
+
+ # TODO we should turn this into a datastructure with limited capacity
+ _listeners = defaultdict(set)
+
+ def __init__(self, collection, rw=1):
+ """
+ :param collection: instance of MessageCollection
+ :type collection: MessageCollection
+
+ :param rw: read-and-write flag for this mailbox
+ :type rw: int
+ """
+ self.rw = rw
+ self._uidvalidity = None
+ self.collection = collection
+ self.collection.addListener(make_collection_listener(self))
+
+ @property
+ def mbox_name(self):
+ return self.collection.mbox_name
+
+ @property
+ def listeners(self):
+ """
+ Returns listeners for this mbox.
+
+ The server itself is a listener to the mailbox.
+ so we can notify it (and should!) after changes in flags
+ and number of messages.
+
+ :rtype: set
+ """
+ return self._listeners[self.mbox_name]
+
+ def get_imap_message(self, message):
+ d = defer.Deferred()
+ IMAPMessage(message, store=self.collection.store, d=d)
+ return d
+
+ # FIXME this grows too crazily when many instances are fired, like
+ # during imaptest stress testing. Should have a queue of limited size
+ # instead.
+
+ def addListener(self, listener):
+ """
+ Add a listener to the listeners queue.
+ The server adds itself as a listener when there is a SELECT,
+ so it can send EXIST commands.
+
+ :param listener: listener to add
+ :type listener: an object that implements IMailboxListener
+ """
+ if not NOTIFY_NEW:
+ return
+
+ listeners = self.listeners
+ logger.debug('adding mailbox listener: %s. Total: %s' % (
+ listener, len(listeners)))
+ listeners.add(listener)
+
+ def removeListener(self, listener):
+ """
+ Remove a listener from the listeners queue.
+
+ :param listener: listener to remove
+ :type listener: an object that implements IMailboxListener
+ """
+ self.listeners.remove(listener)
+
+ def getFlags(self):
+ """
+ Returns the flags defined for this mailbox.
+
+ :returns: tuple of flags for this mailbox
+ :rtype: tuple of str
+ """
+ flags = self.collection.mbox_wrapper.flags
+ if not flags:
+ flags = self.init_flags
+ flags_str = map(str, flags)
+ return flags_str
+
+ def setFlags(self, flags):
+ """
+ Sets flags for this mailbox.
+
+ :param flags: a tuple with the flags
+ :type flags: tuple of str
+ """
+ # XXX this is setting (overriding) old flags.
+ # Better pass a mode flag
+ leap_assert(isinstance(flags, tuple),
+ "flags expected to be a tuple")
+ return self.collection.set_mbox_attr("flags", flags)
+
+ def getUIDValidity(self):
+ """
+ Return the unique validity identifier for this mailbox.
+
+ :return: unique validity identifier
+ :rtype: int
+ """
+ return self.collection.get_mbox_attr("created")
+
+ def getUID(self, message_number):
+ """
+ Return the UID of a message in the mailbox
+
+ .. note:: this implementation does not make much sense RIGHT NOW,
+ but in the future will be useful to get absolute UIDs from
+ message sequence numbers.
+
+
+ :param message: the message sequence number.
+ :type message: int
+
+ :rtype: int
+ :return: the UID of the message.
+
+ """
+ # TODO support relative sequences. The (imap) message should
+ # receive a sequence number attribute: a deferred is not expected
+ return message_number
+
+ def getUIDNext(self):
+ """
+ Return the likely UID for the next message added to this
+ mailbox. Currently it returns the higher UID incremented by
+ one.
+
+ :return: deferred with int
+ :rtype: Deferred
+ """
+ d = self.collection.get_uid_next()
+ return d
+
+ def getMessageCount(self):
+ """
+ Returns the total count of messages in this mailbox.
+
+ :return: deferred with int
+ :rtype: Deferred
+ """
+ return self.collection.count()
+
+ def getUnseenCount(self):
+ """
+ Returns the number of messages with the 'Unseen' flag.
+
+ :return: count of messages flagged `unseen`
+ :rtype: int
+ """
+ return self.collection.count_unseen()
+
+ def getRecentCount(self):
+ """
+ Returns the number of messages with the 'Recent' flag.
+
+ :return: count of messages flagged `recent`
+ :rtype: int
+ """
+ return self.collection.count_recent()
+
+ def isWriteable(self):
+ """
+ Get the read/write status of the mailbox.
+
+ :return: 1 if mailbox is read-writeable, 0 otherwise.
+ :rtype: int
+ """
+ # XXX We don't need to store it in the mbox doc, do we?
+ # return int(self.collection.get_mbox_attr('rw'))
+ return self.rw
+
+ def getHierarchicalDelimiter(self):
+ """
+ Returns the character used to delimite hierarchies in mailboxes.
+
+ :rtype: str
+ """
+ return '/'
+
+ def requestStatus(self, names):
+ """
+ Handles a status request by gathering the output of the different
+ status commands.
+
+ :param names: a list of strings containing the status commands
+ :type names: iter
+ """
+ r = {}
+ maybe = defer.maybeDeferred
+ if self.CMD_MSG in names:
+ r[self.CMD_MSG] = maybe(self.getMessageCount)
+ if self.CMD_RECENT in names:
+ r[self.CMD_RECENT] = maybe(self.getRecentCount)
+ if self.CMD_UIDNEXT in names:
+ r[self.CMD_UIDNEXT] = maybe(self.getUIDNext)
+ if self.CMD_UIDVALIDITY in names:
+ r[self.CMD_UIDVALIDITY] = maybe(self.getUIDValidity)
+ if self.CMD_UNSEEN in names:
+ r[self.CMD_UNSEEN] = maybe(self.getUnseenCount)
+
+ def as_a_dict(values):
+ return dict(zip(r.keys(), values))
+
+ d = defer.gatherResults(r.values())
+ d.addCallback(as_a_dict)
+ return d
+
+ def addMessage(self, message, flags, date=None, notify_just_mdoc=True):
+ """
+ Adds a message to this mailbox.
+
+ :param message: the raw message
+ :type message: str
+
+ :param flags: flag list
+ :type flags: list of str
+
+ :param date: timestamp
+ :type date: str, or None
+
+ :param notify_just_mdoc:
+ boolean passed to the wrapper.create method, to indicate whether
+ we're insterested in being notified right after the mdoc has been
+ written (as it's the first doc to be written, and quite small, this
+ is faster, though potentially unsafe).
+ Setting it to True improves a *lot* the responsiveness of the
+ APPENDS: we just need to be notified when the mdoc is saved, and
+ let's just expect that the other parts are doing just fine. This
+ will not catch any errors when the inserts of the other parts
+ fail, but on the other hand allows us to return very quickly,
+ which seems a good compromise given that we have to serialize the
+ appends.
+ However, some operations like the saving of drafts need to wait for
+ all the parts to be saved, so if some heuristics are met down in
+ the call chain a Draft message will unconditionally set this flag
+ to False, and therefore ignoring the setting of this flag here.
+ :type notify_just_mdoc: bool
+
+ :return: a deferred that will be triggered with the UID of the added
+ message.
+ """
+ # TODO should raise ReadOnlyMailbox if not rw.
+ # TODO have a look at the cases for internal date in the rfc
+ # XXX we could treat the message as an IMessage from here
+
+ # TODO change notify_just_mdoc to something more meaningful, like
+ # fast_insert_notify?
+
+ # TODO notify_just_mdoc *sometimes* make the append tests fail.
+ # have to find a better solution for this. A workaround could probably
+ # be to have a list of the ongoing deferreds related to append, so that
+ # we queue for later all the requests having to do with these.
+
+ # A better solution will probably involve implementing MULTIAPPEND
+ # extension or patching imap server to support pipelining.
+
+ if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)):
+ message = message.getvalue()
+
+ leap_assert_type(message, basestring)
+
+ if flags is None:
+ flags = tuple()
+ else:
+ flags = tuple(str(flag) for flag in flags)
+
+ if date is None:
+ date = formatdate(time.time())
+
+ d = self.collection.add_msg(message, flags, date=date,
+ notify_just_mdoc=notify_just_mdoc)
+ d.addErrback(lambda failure: log.err(failure))
+ return d
+
+ def notify_new(self, *args):
+ """
+ Notify of new messages to all the listeners.
+
+ This will be called indirectly by the underlying collection, that will
+ notify this IMAPMailbox whenever there are changes in the number of
+ messages in the collection, since we have added ourselves to the
+ collection listeners.
+
+ :param args: ignored.
+ """
+ if not NOTIFY_NEW:
+ return
+
+ def cbNotifyNew(result):
+ exists, recent = result
+ for listener in self.listeners:
+ listener.newMessages(exists, recent)
+
+ d = self._get_notify_count()
+ d.addCallback(cbNotifyNew)
+ d.addCallback(self.collection.cb_signal_unread_to_ui)
+ d.addErrback(lambda failure: log.err(failure))
+
+ def _get_notify_count(self):
+ """
+ Get message count and recent count for this mailbox.
+
+ :return: a deferred that will fire with a tuple, with number of
+ messages and number of recent messages.
+ :rtype: Deferred
+ """
+ # XXX this is way too expensive in cases like multiple APPENDS.
+ # We should have a way of keep a cache or do a self-increment for that
+ # kind of calls.
+ d_exists = defer.maybeDeferred(self.getMessageCount)
+ d_recent = defer.maybeDeferred(self.getRecentCount)
+ d_list = [d_exists, d_recent]
+
+ def log_num_msg(result):
+ exists, recent = tuple(result)
+ logger.debug("NOTIFY (%r): there are %s messages, %s recent" % (
+ self.mbox_name, exists, recent))
+ return result
+
+ d = defer.gatherResults(d_list)
+ d.addCallback(log_num_msg)
+ return d
+
+ # commands, do not rename methods
+
+ def destroy(self):
+ """
+ Called before this mailbox is permanently deleted.
+
+ Should cleanup resources, and set the \\Noselect flag
+ on the mailbox.
+
+ """
+ # XXX this will overwrite all the existing flags
+ # should better simply addFlag
+ self.setFlags((MessageFlags.NOSELECT_FLAG,))
+
+ def remove_mbox(_):
+ uuid = self.collection.mbox_uuid
+ d = self.collection.mbox_wrapper.delete(self.collection.store)
+ d.addCallback(
+ lambda _: self.collection.mbox_indexer.delete_table(uuid))
+ return d
+
+ d = self.deleteAllDocs()
+ d.addCallback(remove_mbox)
+ return d
+
+ def expunge(self):
+ """
+ Remove all messages flagged \\Deleted
+ """
+ if not self.isWriteable():
+ raise imap4.ReadOnlyMailbox
+ return self.collection.delete_all_flagged()
+
+ def _get_message_fun(self, uid):
+ """
+ Return the proper method to get a message for this mailbox, depending
+ on the passed uid flag.
+
+ :param uid: If true, the IDs specified in the query are UIDs;
+ otherwise they are message sequence IDs.
+ :type uid: bool
+ :rtype: callable
+ """
+ get_message_fun = [
+ self.collection.get_message_by_sequence_number,
+ self.collection.get_message_by_uid][uid]
+ return get_message_fun
+
+ def _get_messages_range(self, messages_asked, uid=True):
+
+ def get_range(messages_asked):
+ return self._filter_msg_seq(messages_asked)
+
+ d = self._bound_seq(messages_asked, uid)
+ if uid:
+ d.addCallback(get_range)
+ d.addErrback(lambda f: log.err(f))
+ return d
+
+ def _bound_seq(self, messages_asked, uid):
+ """
+ Put an upper bound to a messages sequence if this is open.
+
+ :param messages_asked: IDs of the messages.
+ :type messages_asked: MessageSet
+ :return: a Deferred that will fire with a MessageSet
+ """
+
+ def set_last_uid(last_uid):
+ messages_asked.last = last_uid
+ return messages_asked
+
+ def set_last_seq(all_uid):
+ messages_asked.last = len(all_uid)
+ return messages_asked
+
+ if not messages_asked.last:
+ try:
+ iter(messages_asked)
+ except TypeError:
+ # looks like we cannot iterate
+ if uid:
+ d = self.collection.get_last_uid()
+ d.addCallback(set_last_uid)
+ else:
+ d = self.collection.all_uid_iter()
+ d.addCallback(set_last_seq)
+ return d
+ return defer.succeed(messages_asked)
+
+ def _filter_msg_seq(self, messages_asked):
+ """
+ Filter a message sequence returning only the ones that do exist in the
+ collection.
+
+ :param messages_asked: IDs of the messages.
+ :type messages_asked: MessageSet
+ :rtype: set
+ """
+ # TODO we could pass the asked sequence to the indexer
+ # all_uid_iter, and bound the sql query instead.
+ def filter_by_asked(all_msg_uid):
+ set_asked = set(messages_asked)
+ set_exist = set(all_msg_uid)
+ return set_asked.intersection(set_exist)
+
+ d = self.collection.all_uid_iter()
+ d.addCallback(filter_by_asked)
+ return d
+
+ def fetch(self, messages_asked, uid):
+ """
+ Retrieve one or more messages in this mailbox.
+
+ from rfc 3501: The data items to be fetched can be either a single atom
+ or a parenthesized list.
+
+ :param messages_asked: IDs of the messages to retrieve information
+ about
+ :type messages_asked: MessageSet
+
+ :param uid: If true, the IDs are UIDs. They are message sequence IDs
+ otherwise.
+ :type uid: bool
+
+ :rtype: deferred with a generator that yields...
+ """
+ get_msg_fun = self._get_message_fun(uid)
+ getimapmsg = self.get_imap_message
+
+ def get_imap_messages_for_range(msg_range):
+
+ def _get_imap_msg(messages):
+ d_imapmsg = []
+ # just in case we got bad data in here
+ for msg in filter(None, messages):
+ d_imapmsg.append(getimapmsg(msg))
+ return defer.gatherResults(d_imapmsg, consumeErrors=True)
+
+ def _zip_msgid(imap_messages):
+ zipped = zip(
+ list(msg_range), imap_messages)
+ return (item for item in zipped)
+
+ # XXX not called??
+ def _unset_recent(sequence):
+ reactor.callLater(0, self.unset_recent_flags, sequence)
+ return sequence
+
+ d_msg = []
+ for msgid in msg_range:
+ # XXX We want cdocs because we "probably" are asked for the
+ # body. We should be smarter at do_FETCH and pass a parameter
+ # to this method in order not to prefetch cdocs if they're not
+ # going to be used.
+ d_msg.append(get_msg_fun(msgid, get_cdocs=True))
+
+ d = defer.gatherResults(d_msg, consumeErrors=True)
+ d.addCallback(_get_imap_msg)
+ d.addCallback(_zip_msgid)
+ d.addErrback(lambda failure: log.err(failure))
+ return d
+
+ d = self._get_messages_range(messages_asked, uid)
+ d.addCallback(get_imap_messages_for_range)
+ d.addErrback(lambda failure: log.err(failure))
+ return d
+
+ def fetch_flags(self, messages_asked, uid):
+ """
+ A fast method to fetch all flags, tricking just the
+ needed subset of the MIME interface that's needed to satisfy
+ a generic FLAGS query.
+
+ Given how LEAP Mail is supposed to work without local cache,
+ this query is going to be quite common, and also we expect
+ it to be in the form 1:* at the beginning of a session, so
+ it's not bad to fetch all the FLAGS docs at once.
+
+ :param messages_asked: IDs of the messages to retrieve information
+ about
+ :type messages_asked: MessageSet
+
+ :param uid: If 1, the IDs are UIDs. They are message sequence IDs
+ otherwise.
+ :type uid: int
+
+ :return: A tuple of two-tuples of message sequence numbers and
+ flagsPart, which is a only a partial implementation of
+ MessagePart.
+ :rtype: tuple
+ """
+ # is_sequence = True if uid == 0 else False
+ # XXX FIXME -----------------------------------------------------
+ # imap/tests, or muas like mutt, it will choke until we implement
+ # sequence numbers. This is an easy hack meanwhile.
+ is_sequence = False
+ # ---------------------------------------------------------------
+
+ if is_sequence:
+ raise NotImplementedError(
+ "FETCH FLAGS NOT IMPLEMENTED FOR MESSAGE SEQUENCE NUMBERS YET")
+
+ d = defer.Deferred()
+ reactor.callLater(0, self._do_fetch_flags, messages_asked, uid, d)
+ if PROFILE_CMD:
+ do_profile_cmd(d, "FETCH-ALL-FLAGS")
+ return d
+
+ def _do_fetch_flags(self, messages_asked, uid, d):
+ """
+ :param messages_asked: IDs of the messages to retrieve information
+ about
+ :type messages_asked: MessageSet
+
+ :param uid: If 1, the IDs are UIDs. They are message sequence IDs
+ otherwise.
+ :type uid: int
+ :param d: deferred whose callback will be called with result.
+ :type d: Deferred
+
+ :rtype: A generator that yields two-tuples of message sequence numbers
+ and flagsPart
+ """
+ class flagsPart(object):
+ def __init__(self, uid, flags):
+ self.uid = uid
+ self.flags = flags
+
+ def getUID(self):
+ return self.uid
+
+ def getFlags(self):
+ return map(str, self.flags)
+
+ def pack_flags(result):
+ _uid, _flags = result
+ return _uid, flagsPart(_uid, _flags)
+
+ def get_flags_for_seq(sequence):
+ d_all_flags = []
+ for msgid in sequence:
+ # TODO implement sequence numbers here too
+ d_flags_per_uid = self.collection.get_flags_by_uid(msgid)
+ d_flags_per_uid.addCallback(pack_flags)
+ d_all_flags.append(d_flags_per_uid)
+ gotflags = defer.gatherResults(d_all_flags)
+ gotflags.addCallback(get_uid_flag_generator)
+ return gotflags
+
+ def get_uid_flag_generator(result):
+ generator = (item for item in result)
+ d.callback(generator)
+
+ d_seq = self._get_messages_range(messages_asked, uid)
+ d_seq.addCallback(get_flags_for_seq)
+ return d_seq
+
+ @defer.inlineCallbacks
+ def fetch_headers(self, messages_asked, uid):
+ """
+ A fast method to fetch all headers, tricking just the
+ needed subset of the MIME interface that's needed to satisfy
+ a generic HEADERS query.
+
+ Given how LEAP Mail is supposed to work without local cache,
+ this query is going to be quite common, and also we expect
+ it to be in the form 1:* at the beginning of a session, so
+ **MAYBE** it's not too bad to fetch all the HEADERS docs at once.
+
+ :param messages_asked: IDs of the messages to retrieve information
+ about
+ :type messages_asked: MessageSet
+
+ :param uid: If true, the IDs are UIDs. They are message sequence IDs
+ otherwise.
+ :type uid: bool
+
+ :return: A tuple of two-tuples of message sequence numbers and
+ headersPart, which is a only a partial implementation of
+ MessagePart.
+ :rtype: tuple
+ """
+ # TODO implement sequences
+ is_sequence = True if uid == 0 else False
+ if is_sequence:
+ raise NotImplementedError(
+ "FETCH HEADERS NOT IMPLEMENTED FOR SEQUENCE NUMBER YET")
+
+ class headersPart(object):
+ def __init__(self, uid, headers):
+ self.uid = uid
+ self.headers = headers
+
+ def getUID(self):
+ return self.uid
+
+ def getHeaders(self, _):
+ return dict(
+ (str(key), str(value))
+ for key, value in
+ self.headers.items())
+
+ messages_asked = yield self._bound_seq(messages_asked, uid)
+ seq_messg = yield self._filter_msg_seq(messages_asked)
+
+ result = []
+ for msgid in seq_messg:
+ msg = yield self.collection.get_message_by_uid(msgid)
+ headers = headersPart(msgid, msg.get_headers())
+ result.append((msgid, headers))
+ defer.returnValue(iter(result))
+
+ def store(self, messages_asked, flags, mode, uid):
+ """
+ Sets the flags of one or more messages.
+
+ :param messages: The identifiers of the messages to set the flags
+ :type messages: A MessageSet object with the list of messages requested
+
+ :param flags: The flags to set, unset, or add.
+ :type flags: sequence of str
+
+ :param mode: If mode is -1, these flags should be removed from the
+ specified messages. If mode is 1, these flags should be
+ added to the specified messages. If mode is 0, all
+ existing flags should be cleared and these flags should be
+ added.
+ :type mode: -1, 0, or 1
+
+ :param uid: If true, the IDs specified in the query are UIDs;
+ otherwise they are message sequence IDs.
+ :type uid: bool
+
+ :return: A deferred, that will be called with a dict mapping message
+ sequence numbers to sequences of str representing the flags
+ set on the message after this operation has been performed.
+ :rtype: deferred
+
+ :raise ReadOnlyMailbox: Raised if this mailbox is not open for
+ read-write.
+ """
+ if not self.isWriteable():
+ log.msg('read only mailbox!')
+ raise imap4.ReadOnlyMailbox
+
+ d = defer.Deferred()
+ reactor.callLater(0, self._do_store, messages_asked, flags,
+ mode, uid, d)
+ if PROFILE_CMD:
+ do_profile_cmd(d, "STORE")
+
+ d.addCallback(self.collection.cb_signal_unread_to_ui)
+ d.addErrback(lambda f: log.err(f))
+ return d
+
+ def _do_store(self, messages_asked, flags, mode, uid, observer):
+ """
+ Helper method, invoke set_flags method in the IMAPMessageCollection.
+
+ See the documentation for the `store` method for the parameters.
+
+ :param observer: a deferred that will be called with the dictionary
+ mapping UIDs to flags after the operation has been
+ done.
+ :type observer: deferred
+ """
+ # TODO we should prevent client from setting Recent flag
+ get_msg_fun = self._get_message_fun(uid)
+ leap_assert(not isinstance(flags, basestring),
+ "flags cannot be a string")
+ flags = tuple(flags)
+
+ def set_flags_for_seq(sequence):
+ def return_result_dict(list_of_flags):
+ result = dict(zip(list(sequence), list_of_flags))
+ observer.callback(result)
+ return result
+
+ d_all_set = []
+ for msgid in sequence:
+ d = get_msg_fun(msgid)
+ d.addCallback(lambda msg: self.collection.update_flags(
+ msg, flags, mode))
+ d_all_set.append(d)
+ got_flags_setted = defer.gatherResults(d_all_set)
+ got_flags_setted.addCallback(return_result_dict)
+ return got_flags_setted
+
+ d_seq = self._get_messages_range(messages_asked, uid)
+ d_seq.addCallback(set_flags_for_seq)
+ return d_seq
+
+ # ISearchableMailbox
+
+ def search(self, query, uid):
+ """
+ Search for messages that meet the given query criteria.
+
+ Warning: this is half-baked, and it might give problems since
+ it offers the SearchableInterface.
+ We'll be implementing it asap.
+
+ :param query: The search criteria
+ :type query: list
+
+ :param uid: If true, the IDs specified in the query are UIDs;
+ otherwise they are message sequence IDs.
+ :type uid: bool
+
+ :return: A list of message sequence numbers or message UIDs which
+ match the search criteria or a C{Deferred} whose callback
+ will be invoked with such a list.
+ :rtype: C{list} or C{Deferred}
+ """
+ # TODO see if we can raise w/o interrupting flow
+ # :raise IllegalQueryError: Raised when query is not valid.
+ # example query:
+ # ['UNDELETED', 'HEADER', 'Message-ID',
+ # XXX fixme, does not exist
+ # '52D44F11.9060107@dev.bitmask.net']
+
+ # TODO hardcoding for now! -- we'll support generic queries later on
+ # but doing a quickfix for avoiding duplicate saves in the draft
+ # folder. # See issue #4209
+
+ if len(query) > 2:
+ if query[1] == 'HEADER' and query[2].lower() == "message-id":
+ msgid = str(query[3]).strip()
+ logger.debug("Searching for %s" % (msgid,))
+
+ d = self.collection.get_uid_from_msgid(str(msgid))
+ d.addCallback(lambda result: [result])
+ return d
+
+ # nothing implemented for any other query
+ logger.warning("Cannot process query: %s" % (query,))
+ return []
+
+ # IMessageCopier
+
+ def copy(self, message):
+ """
+ Copy the given message object into this mailbox.
+
+ :param message: an IMessage implementor
+ :type message: LeapMessage
+ :return: a deferred that will be fired with the message
+ uid when the copy succeed.
+ :rtype: Deferred
+ """
+ # if PROFILE_CMD:
+ # do_profile_cmd(d, "COPY")
+
+ # A better place for this would be the COPY/APPEND dispatcher
+ # in server.py, but qtreactor hangs when I do that, so this seems
+ # to work fine for now.
+ # d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new))
+ # deferLater(self.reactor, 0, self._do_copy, message, d)
+ # return d
+
+ d = self.collection.copy_msg(message.message,
+ self.collection.mbox_uuid)
+ return d
+
+ # convenience fun
+
+ def deleteAllDocs(self):
+ """
+ Delete all docs in this mailbox
+ """
+ # FIXME not implemented
+ return self.collection.delete_all_docs()
+
+ def unset_recent_flags(self, uid_seq):
+ """
+ Unset Recent flag for a sequence of UIDs.
+ """
+ # FIXME not implemented
+ return self.collection.unset_recent_flags(uid_seq)
+
+ def __repr__(self):
+ """
+ Representation string for this mailbox.
+ """
+ return u"<IMAPMailbox: mbox '%s' (%s)>" % (
+ self.mbox_name, self.collection.count())
+
+
+_INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
+
+
+def normalize_mailbox(name):
+ """
+ Return a normalized representation of the mailbox ``name``.
+
+ This method ensures that an eventual initial 'inbox' part of a
+ mailbox name is made uppercase.
+
+ :param name: the name of the mailbox
+ :type name: unicode
+
+ :rtype: unicode
+ """
+ # XXX maybe it would make sense to normalize common folders too:
+ # trash, sent, drafts, etc...
+ if _INBOX_RE.match(name):
+ # ensure inital INBOX is uppercase
+ return INBOX_NAME + name[len(INBOX_NAME):]
+ return name
diff --git a/src/leap/bitmask/mail/imap/messages.py b/src/leap/bitmask/mail/imap/messages.py
new file mode 100644
index 00000000..d1c7b93c
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/messages.py
@@ -0,0 +1,254 @@
+# -*- coding: utf-8 -*-
+# imap/messages.py
+# Copyright (C) 2013-2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+IMAPMessage implementation.
+"""
+import logging
+from twisted.mail import imap4
+from twisted.internet import defer
+from zope.interface import implements
+
+from leap.mail.utils import find_charset, CaseInsensitiveDict
+
+
+logger = logging.getLogger(__name__)
+
+# TODO
+# [ ] Add ref to incoming message during add_msg.
+
+
+class IMAPMessage(object):
+ """
+ The main representation of a message as seen by the IMAP Server.
+ This class implements the semantics specific to IMAP specification.
+ """
+ implements(imap4.IMessage)
+
+ def __init__(self, message, prefetch_body=True,
+ store=None, d=defer.Deferred()):
+ """
+ Get an IMAPMessage. A mail.Message is needed, since many of the methods
+ are proxied to that object.
+
+
+ If you do not need to prefetch the body of the message, you can set
+ `prefetch_body` to False, but the current imap server implementation
+ expect the getBodyFile method to return inmediately.
+
+ When the prefetch_body option is used, a deferred is also expected as a
+ parameter, and this will fire when the deferred initialization has
+ taken place, with this instance of IMAPMessage as a parameter.
+
+ :param message: the abstract message
+ :type message: mail.Message
+ :param prefetch_body: Whether to prefetch the content doc for the body.
+ :type prefetch_body: bool
+ :param store: an instance of soledad, or anything that behaves like it.
+ :param d: an optional deferred, that will be fired with the instance of
+ the IMAPMessage being initialized
+ :type d: defer.Deferred
+ """
+ # TODO substitute the use of the deferred initialization by a factory
+ # function, maybe.
+
+ self.message = message
+ self.__body_fd = None
+ self.store = store
+ if prefetch_body:
+ gotbody = self.__prefetch_body_file()
+ gotbody.addCallback(lambda _: d.callback(self))
+
+ # IMessage implementation
+
+ def getUID(self):
+ """
+ Retrieve the unique identifier associated with this Message.
+
+ :return: uid for this message
+ :rtype: int
+ """
+ return self.message.get_uid()
+
+ def getFlags(self):
+ """
+ Retrieve the flags associated with this Message.
+
+ :return: The flags, represented as strings
+ :rtype: tuple
+ """
+ return self.message.get_flags()
+
+ def getInternalDate(self):
+ """
+ Retrieve the date internally associated with this message
+
+ According to the spec, this is NOT the date and time in the
+ RFC-822 header, but rather a date and time that reflects when the
+ message was received.
+
+ * In SMTP, date and time of final delivery.
+ * In COPY, internal date/time of the source message.
+ * In APPEND, date/time specified.
+
+ :return: An RFC822-formatted date string.
+ :rtype: str
+ """
+ return self.message.get_internal_date()
+
+ #
+ # IMessagePart
+ #
+
+ def getBodyFile(self, store=None):
+ """
+ Retrieve a file object containing only the body of this message.
+
+ :return: file-like object opened for reading
+ :rtype: a deferred that will fire with a StringIO object.
+ """
+ if self.__body_fd is not None:
+ fd = self.__body_fd
+ fd.seek(0)
+ return fd
+
+ if store is None:
+ store = self.store
+ return self.message.get_body_file(store)
+
+ def getSize(self):
+ """
+ Return the total size, in octets, of this message.
+
+ :return: size of the message, in octets
+ :rtype: int
+ """
+ return self.message.get_size()
+
+ def getHeaders(self, negate, *names):
+ """
+ Retrieve a group of message headers.
+
+ :param names: The names of the headers to retrieve or omit.
+ :type names: tuple of str
+
+ :param negate: If True, indicates that the headers listed in names
+ should be omitted from the return value, rather
+ than included.
+ :type negate: bool
+
+ :return: A mapping of header field names to header field values
+ :rtype: dict
+ """
+ headers = self.message.get_headers()
+ return _format_headers(headers, negate, *names)
+
+ def isMultipart(self):
+ """
+ Return True if this message is multipart.
+ """
+ return self.message.is_multipart()
+
+ def getSubPart(self, part):
+ """
+ Retrieve a MIME submessage
+
+ :type part: C{int}
+ :param part: The number of the part to retrieve, indexed from 0.
+ :raise IndexError: Raised if the specified part does not exist.
+ :raise TypeError: Raised if this message is not multipart.
+ :rtype: Any object implementing C{IMessagePart}.
+ :return: The specified sub-part.
+ """
+ subpart = self.message.get_subpart(part + 1)
+ return IMAPMessagePart(subpart)
+
+ def __prefetch_body_file(self):
+ def assign_body_fd(fd):
+ self.__body_fd = fd
+ return fd
+ d = self.getBodyFile()
+ d.addCallback(assign_body_fd)
+ return d
+
+
+class IMAPMessagePart(object):
+
+ def __init__(self, message_part):
+ self.message_part = message_part
+
+ def getBodyFile(self, store=None):
+ return self.message_part.get_body_file()
+
+ def getSize(self):
+ return self.message_part.get_size()
+
+ def getHeaders(self, negate, *names):
+ headers = self.message_part.get_headers()
+ return _format_headers(headers, negate, *names)
+
+ def isMultipart(self):
+ return self.message_part.is_multipart()
+
+ def getSubPart(self, part):
+ subpart = self.message_part.get_subpart(part + 1)
+ return IMAPMessagePart(subpart)
+
+
+def _format_headers(headers, negate, *names):
+ # current server impl. expects content-type to be present, so if for
+ # some reason we do not have headers, we have to return at least that
+ # one
+ if not headers:
+ logger.warning("No headers found")
+ return {str('content-type'): str('')}
+
+ names = map(lambda s: s.upper(), names)
+
+ if negate:
+ def cond(key):
+ return key.upper() not in names
+ else:
+ def cond(key):
+ return key.upper() in names
+
+ if isinstance(headers, list):
+ headers = dict(headers)
+
+ # default to most likely standard
+ charset = find_charset(headers, "utf-8")
+
+ # We will return a copy of the headers dictionary that
+ # will allow case-insensitive lookups. In some parts of the twisted imap
+ # server code the keys are expected to be in lower case, and in this way
+ # we avoid having to convert them.
+
+ _headers = CaseInsensitiveDict()
+ for key, value in headers.items():
+ if not isinstance(key, str):
+ key = key.encode(charset, 'replace')
+ if not isinstance(value, str):
+ value = value.encode(charset, 'replace')
+
+ if value.endswith(";"):
+ # bastards
+ value = value[:-1]
+
+ # filter original dict by negate-condition
+ if cond(key):
+ _headers[key] = value
+
+ return _headers
diff --git a/src/leap/bitmask/mail/imap/server.py b/src/leap/bitmask/mail/imap/server.py
new file mode 100644
index 00000000..5a63af01
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/server.py
@@ -0,0 +1,693 @@
+# -*- coding: utf-8 -*-
+# server.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+LEAP IMAP4 Server Implementation.
+"""
+import StringIO
+from copy import copy
+
+from twisted.internet.defer import maybeDeferred
+from twisted.mail import imap4
+from twisted.python import log
+
+# imports for LITERAL+ patch
+from twisted.internet import defer, interfaces
+from twisted.mail.imap4 import IllegalClientResponse
+from twisted.mail.imap4 import LiteralString, LiteralFile
+
+from leap.common.events import emit_async, catalog
+
+
+def _getContentType(msg):
+ """
+ Return a two-tuple of the main and subtype of the given message.
+ """
+ attrs = None
+ mm = msg.getHeaders(False, 'content-type').get('content-type', None)
+ if mm:
+ mm = ''.join(mm.splitlines())
+ mimetype = mm.split(';')
+ if mimetype:
+ type = mimetype[0].split('/', 1)
+ if len(type) == 1:
+ major = type[0]
+ minor = None
+ elif len(type) == 2:
+ major, minor = type
+ else:
+ major = minor = None
+ # XXX patched ---------------------------------------------
+ attrs = dict(x.strip().split('=', 1) for x in mimetype[1:])
+ # XXX patched ---------------------------------------------
+ else:
+ major = minor = None
+ else:
+ major = minor = None
+ return major, minor, attrs
+
+# Monkey-patch _getContentType to avoid bug that passes lower-case boundary in
+# BODYSTRUCTURE response.
+imap4._getContentType = _getContentType
+
+
+class LEAPIMAPServer(imap4.IMAP4Server):
+ """
+ An IMAP4 Server with a LEAP Storage Backend.
+ """
+
+ #############################################################
+ #
+ # Twisted imap4 patch to workaround bad mime rendering in TB.
+ # See https://leap.se/code/issues/6773
+ # and https://bugzilla.mozilla.org/show_bug.cgi?id=149771
+ # Still unclear if this is a thunderbird bug.
+ # TODO send this patch upstream
+ #
+ #############################################################
+
+ def spew_body(self, part, id, msg, _w=None, _f=None):
+ if _w is None:
+ _w = self.transport.write
+ for p in part.part:
+ if msg.isMultipart():
+ msg = msg.getSubPart(p)
+ elif p > 0:
+ # Non-multipart messages have an implicit first part but no
+ # other parts - reject any request for any other part.
+ raise TypeError("Requested subpart of non-multipart message")
+
+ if part.header:
+ hdrs = msg.getHeaders(part.header.negate, *part.header.fields)
+ hdrs = imap4._formatHeaders(hdrs)
+ # PATCHED ##########################################
+ _w(str(part) + ' ' + imap4._literal(hdrs + "\r\n"))
+ # PATCHED ##########################################
+ elif part.text:
+ _w(str(part) + ' ')
+ _f()
+ return imap4.FileProducer(
+ msg.getBodyFile()
+ ).beginProducing(self.transport)
+ elif part.mime:
+ hdrs = imap4._formatHeaders(msg.getHeaders(True))
+
+ # PATCHED ##########################################
+ _w(str(part) + ' ' + imap4._literal(hdrs + "\r\n"))
+ # END PATCHED ######################################
+
+ elif part.empty:
+ _w(str(part) + ' ')
+ _f()
+ if part.part:
+ # PATCHED #############################################
+ # implement partial FETCH
+ # TODO implement boundary checks
+ # TODO see if there's a more efficient way, without
+ # copying the original content into a new buffer.
+ fd = msg.getBodyFile()
+ begin = getattr(part, "partialBegin", None)
+ _len = getattr(part, "partialLength", None)
+ if begin is not None and _len is not None:
+ _fd = StringIO.StringIO()
+ fd.seek(part.partialBegin)
+ _fd.write(fd.read(part.partialLength))
+ _fd.seek(0)
+ else:
+ _fd = fd
+ return imap4.FileProducer(
+ _fd
+ # END PATCHED #########################3
+ ).beginProducing(self.transport)
+ else:
+ mf = imap4.IMessageFile(msg, None)
+ if mf is not None:
+ return imap4.FileProducer(
+ mf.open()).beginProducing(self.transport)
+ return imap4.MessageProducer(
+ msg, None, self._scheduler).beginProducing(self.transport)
+
+ else:
+ _w('BODY ' +
+ imap4.collapseNestedLists([imap4.getBodyStructure(msg)]))
+
+ ##################################################################
+ #
+ # END Twisted imap4 patch to workaround bad mime rendering in TB.
+ # #6773
+ #
+ ##################################################################
+
+ def lineReceived(self, line):
+ """
+ Attempt to parse a single line from the server.
+
+ :param line: the line from the server, without the line delimiter.
+ :type line: str
+ """
+ if "login" in line.lower():
+ # avoid to log the pass, even though we are using a dummy auth
+ # by now.
+ msg = line[:7] + " [...]"
+ else:
+ msg = copy(line)
+ log.msg('rcv (%s): %s' % (self.state, msg))
+ imap4.IMAP4Server.lineReceived(self, line)
+
+ def close_server_connection(self):
+ """
+ Send a BYE command so that the MUA at least knows that we're closing
+ the connection.
+ """
+ self.sendLine(
+ '* BYE LEAP IMAP Proxy is shutting down; '
+ 'so long and thanks for all the fish')
+ self.transport.loseConnection()
+ if self.mbox:
+ self.mbox.removeListener(self)
+ self.mbox = None
+ self.state = 'unauth'
+
+ def do_FETCH(self, tag, messages, query, uid=0):
+ """
+ Overwritten fetch dispatcher to use the fast fetch_flags
+ method
+ """
+ if not query:
+ self.sendPositiveResponse(tag, 'FETCH complete')
+ return
+
+ cbFetch = self._IMAP4Server__cbFetch
+ ebFetch = self._IMAP4Server__ebFetch
+
+ if len(query) == 1 and str(query[0]) == "flags":
+ self._oldTimeout = self.setTimeout(None)
+ # no need to call iter, we get a generator
+ maybeDeferred(
+ self.mbox.fetch_flags, messages, uid=uid
+ ).addCallback(
+ cbFetch, tag, query, uid
+ ).addErrback(ebFetch, tag)
+
+ elif len(query) == 1 and str(query[0]) == "rfc822.header":
+ self._oldTimeout = self.setTimeout(None)
+ # no need to call iter, we get a generator
+ maybeDeferred(
+ self.mbox.fetch_headers, messages, uid=uid
+ ).addCallback(
+ cbFetch, tag, query, uid
+ ).addErrback(ebFetch, tag)
+ else:
+ self._oldTimeout = self.setTimeout(None)
+ # no need to call iter, we get a generator
+ maybeDeferred(
+ self.mbox.fetch, messages, uid=uid
+ ).addCallback(
+ cbFetch, tag, query, uid
+ ).addErrback(
+ ebFetch, tag)
+
+ select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset,
+ imap4.IMAP4Server.arg_fetchatt)
+
+ def _cbSelectWork(self, mbox, cmdName, tag):
+ """
+ Callback for selectWork
+
+ * patched to avoid conformance errors due to incomplete UIDVALIDITY
+ line.
+ * patched to accept deferreds for messagecount and recent count
+ """
+ if mbox is None:
+ self.sendNegativeResponse(tag, 'No such mailbox')
+ return
+ if '\\noselect' in [s.lower() for s in mbox.getFlags()]:
+ self.sendNegativeResponse(tag, 'Mailbox cannot be selected')
+ return
+
+ d1 = defer.maybeDeferred(mbox.getMessageCount)
+ d2 = defer.maybeDeferred(mbox.getRecentCount)
+ return defer.gatherResults([d1, d2]).addCallback(
+ self.__cbSelectWork, mbox, cmdName, tag)
+
+ def __cbSelectWork(self, ((msg_count, recent_count)), mbox, cmdName, tag):
+ flags = mbox.getFlags()
+ self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags))
+
+ # Patched -------------------------------------------------------
+ # accept deferreds for the count
+ self.sendUntaggedResponse(str(msg_count) + ' EXISTS')
+ self.sendUntaggedResponse(str(recent_count) + ' RECENT')
+ # ----------------------------------------------------------------
+
+ # Patched -------------------------------------------------------
+ # imaptest was complaining about the incomplete line, we're adding
+ # "UIDs valid" here.
+ self.sendPositiveResponse(
+ None, '[UIDVALIDITY %d] UIDs valid' % mbox.getUIDValidity())
+ # ----------------------------------------------------------------
+
+ s = mbox.isWriteable() and 'READ-WRITE' or 'READ-ONLY'
+ mbox.addListener(self)
+ self.sendPositiveResponse(tag, '[%s] %s successful' % (s, cmdName))
+ self.state = 'select'
+ self.mbox = mbox
+
+ def checkpoint(self):
+ """
+ Called when the client issues a CHECK command.
+
+ This should perform any checkpoint operations required by the server.
+ It may be a long running operation, but may not block. If it returns
+ a deferred, the client will only be informed of success (or failure)
+ when the deferred's callback (or errback) is invoked.
+ """
+ # TODO implement a collection of ongoing deferreds?
+ return None
+
+ #############################################################
+ #
+ # Twisted imap4 patch to support LITERAL+ extension
+ # TODO send this patch upstream asap!
+ #
+ #############################################################
+
+ def capabilities(self):
+ cap = {'AUTH': self.challengers.keys()}
+ if self.ctx and self.canStartTLS:
+ t = self.transport
+ ti = interfaces.ISSLTransport
+ if not self.startedTLS and ti(t, None) is None:
+ cap['LOGINDISABLED'] = None
+ cap['STARTTLS'] = None
+ cap['NAMESPACE'] = None
+ cap['IDLE'] = None
+ # patched ############
+ cap['LITERAL+'] = None
+ ######################
+ return cap
+
+ def _stringLiteral(self, size, literal_plus=False):
+ if size > self._literalStringLimit:
+ raise IllegalClientResponse(
+ "Literal too long! I accept at most %d octets" %
+ (self._literalStringLimit,))
+ d = defer.Deferred()
+ self.parseState = 'pending'
+ self._pendingLiteral = LiteralString(size, d)
+ # Patched ###########################################################
+ if not literal_plus:
+ self.sendContinuationRequest('Ready for %d octets of text' % size)
+ #####################################################################
+ self.setRawMode()
+ return d
+
+ def _fileLiteral(self, size, literal_plus=False):
+ d = defer.Deferred()
+ self.parseState = 'pending'
+ self._pendingLiteral = LiteralFile(size, d)
+ if not literal_plus:
+ self.sendContinuationRequest('Ready for %d octets of data' % size)
+ self.setRawMode()
+ return d
+
+ def arg_astring(self, line):
+ """
+ Parse an astring from the line, return (arg, rest), possibly
+ via a deferred (to handle literals)
+ """
+ line = line.strip()
+ if not line:
+ raise IllegalClientResponse("Missing argument")
+ d = None
+ arg, rest = None, None
+ if line[0] == '"':
+ try:
+ spam, arg, rest = line.split('"', 2)
+ rest = rest[1:] # Strip space
+ except ValueError:
+ raise IllegalClientResponse("Unmatched quotes")
+ elif line[0] == '{':
+ # literal
+ if line[-1] != '}':
+ raise IllegalClientResponse("Malformed literal")
+
+ # Patched ################
+ if line[-2] == "+":
+ literalPlus = True
+ size_end = -2
+ else:
+ literalPlus = False
+ size_end = -1
+
+ try:
+ size = int(line[1:size_end])
+ except ValueError:
+ raise IllegalClientResponse(
+ "Bad literal size: " + line[1:size_end])
+ d = self._stringLiteral(size, literalPlus)
+ ##########################
+ else:
+ arg = line.split(' ', 1)
+ if len(arg) == 1:
+ arg.append('')
+ arg, rest = arg
+ return d or (arg, rest)
+
+ def arg_literal(self, line):
+ """
+ Parse a literal from the line
+ """
+ if not line:
+ raise IllegalClientResponse("Missing argument")
+
+ if line[0] != '{':
+ raise IllegalClientResponse("Missing literal")
+
+ if line[-1] != '}':
+ raise IllegalClientResponse("Malformed literal")
+
+ # Patched ##################
+ if line[-2] == "+":
+ literalPlus = True
+ size_end = -2
+ else:
+ literalPlus = False
+ size_end = -1
+
+ try:
+ size = int(line[1:size_end])
+ except ValueError:
+ raise IllegalClientResponse(
+ "Bad literal size: " + line[1:size_end])
+
+ return self._fileLiteral(size, literalPlus)
+ #############################
+
+ # --------------------------------- isSubscribed patch
+ # TODO -- send patch upstream.
+ # There is a bug in twisted implementation:
+ # in cbListWork, it's assumed that account.isSubscribed IS a callable,
+ # although in the interface documentation it's stated that it can be
+ # a deferred.
+
+ def _listWork(self, tag, ref, mbox, sub, cmdName):
+ mbox = self._parseMbox(mbox)
+ mailboxes = maybeDeferred(self.account.listMailboxes, ref, mbox)
+ mailboxes.addCallback(self._cbSubscribed)
+ mailboxes.addCallback(
+ self._cbListWork, tag, sub, cmdName,
+ ).addErrback(self._ebListWork, tag)
+
+ def _cbSubscribed(self, mailboxes):
+ subscribed = [
+ maybeDeferred(self.account.isSubscribed, name)
+ for (name, box) in mailboxes]
+
+ def get_mailboxes_and_subs(result):
+ subscribed = [i[0] for i, yes in zip(mailboxes, result) if yes]
+ return mailboxes, subscribed
+
+ d = defer.gatherResults(subscribed)
+ d.addCallback(get_mailboxes_and_subs)
+ return d
+
+ def _cbListWork(self, mailboxes_subscribed, tag, sub, cmdName):
+ mailboxes, subscribed = mailboxes_subscribed
+
+ for (name, box) in mailboxes:
+ if not sub or name in subscribed:
+ flags = box.getFlags()
+ delim = box.getHierarchicalDelimiter()
+ resp = (imap4.DontQuoteMe(cmdName),
+ map(imap4.DontQuoteMe, flags),
+ delim, name.encode('imap4-utf-7'))
+ self.sendUntaggedResponse(
+ imap4.collapseNestedLists(resp))
+ self.sendPositiveResponse(tag, '%s completed' % (cmdName,))
+ # -------------------- end isSubscribed patch -----------
+
+ # TODO subscribe method had also to be changed to accomodate deferred
+ def do_SUBSCRIBE(self, tag, name):
+ name = self._parseMbox(name)
+
+ def _subscribeCb(_):
+ self.sendPositiveResponse(tag, 'Subscribed')
+
+ def _subscribeEb(failure):
+ m = failure.value
+ log.err()
+ if failure.check(imap4.MailboxException):
+ self.sendNegativeResponse(tag, str(m))
+ else:
+ self.sendBadResponse(
+ tag,
+ "Server error encountered while subscribing to mailbox")
+
+ d = self.account.subscribe(name)
+ d.addCallbacks(_subscribeCb, _subscribeEb)
+ return d
+
+ auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring)
+ select_SUBSCRIBE = auth_SUBSCRIBE
+
+ def do_UNSUBSCRIBE(self, tag, name):
+ # unsubscribe method had also to be changed to accomodate
+ # deferred
+ name = self._parseMbox(name)
+
+ def _unsubscribeCb(_):
+ self.sendPositiveResponse(tag, 'Unsubscribed')
+
+ def _unsubscribeEb(failure):
+ m = failure.value
+ log.err()
+ if failure.check(imap4.MailboxException):
+ self.sendNegativeResponse(tag, str(m))
+ else:
+ self.sendBadResponse(
+ tag,
+ "Server error encountered while unsubscribing "
+ "from mailbox")
+
+ d = self.account.unsubscribe(name)
+ d.addCallbacks(_unsubscribeCb, _unsubscribeEb)
+ return d
+
+ auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring)
+ select_UNSUBSCRIBE = auth_UNSUBSCRIBE
+
+ def do_RENAME(self, tag, oldname, newname):
+ oldname, newname = [self._parseMbox(n) for n in oldname, newname]
+ if oldname.lower() == 'inbox' or newname.lower() == 'inbox':
+ self.sendNegativeResponse(
+ tag,
+ 'You cannot rename the inbox, or '
+ 'rename another mailbox to inbox.')
+ return
+
+ def _renameCb(_):
+ self.sendPositiveResponse(tag, 'Mailbox renamed')
+
+ def _renameEb(failure):
+ m = failure.value
+ if failure.check(TypeError):
+ self.sendBadResponse(tag, 'Invalid command syntax')
+ elif failure.check(imap4.MailboxException):
+ self.sendNegativeResponse(tag, str(m))
+ else:
+ log.err()
+ self.sendBadResponse(
+ tag,
+ "Server error encountered while "
+ "renaming mailbox")
+
+ d = self.account.rename(oldname, newname)
+ d.addCallbacks(_renameCb, _renameEb)
+ return d
+
+ auth_RENAME = (do_RENAME, arg_astring, arg_astring)
+ select_RENAME = auth_RENAME
+
+ def do_CREATE(self, tag, name):
+ name = self._parseMbox(name)
+
+ def _createCb(result):
+ if result:
+ self.sendPositiveResponse(tag, 'Mailbox created')
+ else:
+ self.sendNegativeResponse(tag, 'Mailbox not created')
+
+ def _createEb(failure):
+ c = failure.value
+ if failure.check(imap4.MailboxException):
+ self.sendNegativeResponse(tag, str(c))
+ else:
+ log.err()
+ self.sendBadResponse(
+ tag, "Server error encountered while creating mailbox")
+
+ d = self.account.create(name)
+ d.addCallbacks(_createCb, _createEb)
+ return d
+
+ auth_CREATE = (do_CREATE, arg_astring)
+ select_CREATE = auth_CREATE
+
+ def do_DELETE(self, tag, name):
+ name = self._parseMbox(name)
+ if name.lower() == 'inbox':
+ self.sendNegativeResponse(tag, 'You cannot delete the inbox')
+ return
+
+ def _deleteCb(result):
+ self.sendPositiveResponse(tag, 'Mailbox deleted')
+
+ def _deleteEb(failure):
+ m = failure.value
+ if failure.check(imap4.MailboxException):
+ self.sendNegativeResponse(tag, str(m))
+ else:
+ print "SERVER: other error"
+ log.err()
+ self.sendBadResponse(
+ tag,
+ "Server error encountered while deleting mailbox")
+
+ d = self.account.delete(name)
+ d.addCallbacks(_deleteCb, _deleteEb)
+ return d
+
+ auth_DELETE = (do_DELETE, arg_astring)
+ select_DELETE = auth_DELETE
+
+ # -----------------------------------------------------------------------
+ # Patched just to allow __cbAppend to receive a deferred from messageCount
+ # TODO format and send upstream.
+ def do_APPEND(self, tag, mailbox, flags, date, message):
+ mailbox = self._parseMbox(mailbox)
+ maybeDeferred(self.account.select, mailbox).addCallback(
+ self._cbAppendGotMailbox, tag, flags, date, message).addErrback(
+ self._ebAppendGotMailbox, tag)
+
+ def __ebAppend(self, failure, tag):
+ self.sendBadResponse(tag, 'APPEND failed: ' + str(failure.value))
+
+ def _cbAppendGotMailbox(self, mbox, tag, flags, date, message):
+ if not mbox:
+ self.sendNegativeResponse(tag, '[TRYCREATE] No such mailbox')
+ return
+
+ d = mbox.addMessage(message, flags, date)
+ d.addCallback(self.__cbAppend, tag, mbox)
+ d.addErrback(self.__ebAppend, tag)
+
+ def _ebAppendGotMailbox(self, failure, tag):
+ self.sendBadResponse(
+ tag, "Server error encountered while opening mailbox.")
+ log.err(failure)
+
+ def __cbAppend(self, result, tag, mbox):
+
+ # XXX patched ---------------------------------
+ def send_response(count):
+ self.sendUntaggedResponse('%d EXISTS' % count)
+ self.sendPositiveResponse(tag, 'APPEND complete')
+
+ d = mbox.getMessageCount()
+ d.addCallback(send_response)
+ return d
+ # XXX patched ---------------------------------
+ # -----------------------------------------------------------------------
+
+ auth_APPEND = (do_APPEND, arg_astring, imap4.IMAP4Server.opt_plist,
+ imap4.IMAP4Server.opt_datetime, arg_literal)
+ select_APPEND = auth_APPEND
+
+ # Need to override the command table after patching
+ # arg_astring and arg_literal, except on the methods that we are already
+ # overriding.
+
+ # TODO --------------------------------------------
+ # Check if we really need to override these
+ # methods, or we can monkeypatch.
+ # do_DELETE = imap4.IMAP4Server.do_DELETE
+ # do_CREATE = imap4.IMAP4Server.do_CREATE
+ # do_RENAME = imap4.IMAP4Server.do_RENAME
+ # do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE
+ # do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE
+ # do_APPEND = imap4.IMAP4Server.do_APPEND
+ # -------------------------------------------------
+ do_LOGIN = imap4.IMAP4Server.do_LOGIN
+ do_STATUS = imap4.IMAP4Server.do_STATUS
+ do_COPY = imap4.IMAP4Server.do_COPY
+
+ _selectWork = imap4.IMAP4Server._selectWork
+
+ arg_plist = imap4.IMAP4Server.arg_plist
+ arg_seqset = imap4.IMAP4Server.arg_seqset
+ opt_plist = imap4.IMAP4Server.opt_plist
+ opt_datetime = imap4.IMAP4Server.opt_datetime
+
+ unauth_LOGIN = (do_LOGIN, arg_astring, arg_astring)
+
+ auth_SELECT = (_selectWork, arg_astring, 1, 'SELECT')
+ select_SELECT = auth_SELECT
+
+ auth_CREATE = (do_CREATE, arg_astring)
+ select_CREATE = auth_CREATE
+
+ auth_EXAMINE = (_selectWork, arg_astring, 0, 'EXAMINE')
+ select_EXAMINE = auth_EXAMINE
+
+ # TODO -----------------------------------------------
+ # re-add if we stop overriding DELETE
+ # auth_DELETE = (do_DELETE, arg_astring)
+ # select_DELETE = auth_DELETE
+ # auth_APPEND = (do_APPEND, arg_astring, opt_plist, opt_datetime,
+ # arg_literal)
+ # select_APPEND = auth_APPEND
+
+ # ----------------------------------------------------
+
+ auth_RENAME = (do_RENAME, arg_astring, arg_astring)
+ select_RENAME = auth_RENAME
+
+ auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring)
+ select_SUBSCRIBE = auth_SUBSCRIBE
+
+ auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring)
+ select_UNSUBSCRIBE = auth_UNSUBSCRIBE
+
+ auth_LIST = (_listWork, arg_astring, arg_astring, 0, 'LIST')
+ select_LIST = auth_LIST
+
+ auth_LSUB = (_listWork, arg_astring, arg_astring, 1, 'LSUB')
+ select_LSUB = auth_LSUB
+
+ auth_STATUS = (do_STATUS, arg_astring, arg_plist)
+ select_STATUS = auth_STATUS
+
+ select_COPY = (do_COPY, arg_seqset, arg_astring)
+
+ #############################################################
+ # END of Twisted imap4 patch to support LITERAL+ extension
+ #############################################################
+
+ def authenticateLogin(self, user, passwd):
+ result = imap4.IMAP4Server.authenticateLogin(self, user, passwd)
+ emit_async(catalog.IMAP_CLIENT_LOGIN, str(user))
+ return result
diff --git a/src/leap/bitmask/mail/imap/service/README.rst b/src/leap/bitmask/mail/imap/service/README.rst
new file mode 100644
index 00000000..2cca9b38
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/service/README.rst
@@ -0,0 +1,39 @@
+testing the service
+===================
+
+Run the twisted service::
+
+ twistd -n -y imap-server.tac
+
+And use offlineimap for tests::
+
+ offlineimap -c LEAPofflineimapRC-tests
+
+minimal offlineimap configuration
+---------------------------------
+
+[general]
+accounts = leap-local
+
+[Account leap-local]
+localrepository = LocalLeap
+remoterepository = RemoteLeap
+
+[Repository LocalLeap]
+type = Maildir
+localfolders = ~/LEAPMail/Mail
+
+[Repository RemoteLeap]
+type = IMAP
+ssl = no
+remotehost = localhost
+remoteport = 9930
+remoteuser = user
+remotepass = pass
+
+debugging
+---------
+
+Use ngrep to obtain logs of the sequences::
+
+ sudo ngrep -d lo -W byline port 9930
diff --git a/src/leap/bitmask/mail/imap/service/__init__.py b/src/leap/bitmask/mail/imap/service/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/service/__init__.py
diff --git a/src/leap/bitmask/mail/imap/service/imap-server.tac b/src/leap/bitmask/mail/imap/service/imap-server.tac
new file mode 100644
index 00000000..c4d602db
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/service/imap-server.tac
@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# imap-server.tac
+# Copyright (C) 2013,2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+TAC file for initialization of the imap service using twistd.
+
+Use this for debugging and testing the imap server using a native reactor.
+
+For now, and for debugging/testing purposes, you need
+to pass a config file with the following structure:
+
+[leap_mail]
+userid = 'user@provider'
+uuid = 'deadbeefdeadabad'
+passwd = 'supersecret' # optional, will get prompted if not found.
+"""
+
+# TODO -- this .tac file should be deprecated in favor of bitmask.core.bitmaskd
+
+import ConfigParser
+import getpass
+import os
+import sys
+
+from leap.keymanager import KeyManager
+from leap.mail.imap.service import imap
+from leap.soledad.client import Soledad
+
+from twisted.application import service, internet
+
+
+# TODO should get this initializers from some authoritative mocked source
+# We might want to put them the soledad itself.
+
+def initialize_soledad(uuid, email, passwd,
+ secrets, localdb,
+ gnupg_home, tempdir):
+ """
+ Initializes soledad by hand
+
+ :param email: ID for the user
+ :param gnupg_home: path to home used by gnupg
+ :param tempdir: path to temporal dir
+ :rtype: Soledad instance
+ """
+ server_url = "http://provider"
+ cert_file = ""
+
+ soledad = Soledad(
+ uuid,
+ passwd,
+ secrets,
+ localdb,
+ server_url,
+ cert_file,
+ syncable=False)
+
+ return soledad
+
+######################################################################
+# Remember to set your config files, see module documentation above!
+######################################################################
+
+print "[+] Running LEAP IMAP Service"
+
+
+bmconf = os.environ.get("LEAP_MAIL_CONFIG", "")
+if not bmconf:
+ print ("[-] Please set LEAP_MAIL_CONFIG environment variable "
+ "pointing to your config.")
+ sys.exit(1)
+
+SECTION = "leap_mail"
+cp = ConfigParser.ConfigParser()
+cp.read(bmconf)
+
+userid = cp.get(SECTION, "userid")
+uuid = cp.get(SECTION, "uuid")
+passwd = unicode(cp.get(SECTION, "passwd"))
+
+# XXX get this right from the environment variable !!!
+port = 1984
+
+if not userid or not uuid:
+ print "[-] Config file missing userid or uuid field"
+ sys.exit(1)
+
+if not passwd:
+ passwd = unicode(getpass.getpass("Soledad passphrase: "))
+
+
+secrets = os.path.expanduser("~/.config/leap/soledad/%s.secret" % (uuid,))
+localdb = os.path.expanduser("~/.config/leap/soledad/%s.db" % (uuid,))
+
+# XXX Is this really used? Should point it to user var dirs defined in xdg?
+gnupg_home = "/tmp/"
+tempdir = "/tmp/"
+
+###################################################
+
+# Ad-hoc soledad/keymanager initialization.
+
+print "[~] user:", userid
+soledad = initialize_soledad(uuid, userid, passwd, secrets,
+ localdb, gnupg_home, tempdir, userid=userid)
+km_args = (userid, "https://localhost", soledad)
+km_kwargs = {
+ "token": "",
+ "ca_cert_path": "",
+ "api_uri": "",
+ "api_version": "",
+ "uid": uuid,
+ "gpgbinary": "/usr/bin/gpg"
+}
+keymanager = KeyManager(*km_args, **km_kwargs)
+
+##################################################
+
+# Ok, let's expose the application object for the twistd application
+# framework to pick up from here...
+
+
+def getIMAPService():
+ soledad_sessions = {userid: soledad}
+ factory = imap.LeapIMAPFactory(soledad_sessions)
+ return internet.TCPServer(port, factory, interface="localhost")
+
+
+application = service.Application("LEAP IMAP Application")
+service = getIMAPService()
+service.setServiceParent(application)
diff --git a/src/leap/bitmask/mail/imap/service/imap.py b/src/leap/bitmask/mail/imap/service/imap.py
new file mode 100644
index 00000000..4663854a
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/service/imap.py
@@ -0,0 +1,208 @@
+# -*- coding: utf-8 -*-
+# imap.py
+# Copyright (C) 2013-2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+IMAP Service Initialization.
+"""
+import logging
+import os
+
+from collections import defaultdict
+
+from twisted.cred.portal import Portal, IRealm
+from twisted.mail.imap4 import IAccount
+from twisted.internet import defer
+from twisted.internet import reactor
+from twisted.internet.error import CannotListenError
+from twisted.internet.protocol import ServerFactory
+from twisted.python import log
+from zope.interface import implementer
+
+from leap.common.events import emit_async, catalog
+from leap.mail.cred import LocalSoledadTokenChecker
+from leap.mail.imap.account import IMAPAccount
+from leap.mail.imap.server import LEAPIMAPServer
+
+# TODO: leave only an implementor of IService in here
+
+logger = logging.getLogger(__name__)
+
+DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None)
+if DO_MANHOLE:
+ from leap.mail.imap.service import manhole
+
+# The default port in which imap service will run
+
+IMAP_PORT = 1984
+
+#
+# Credentials Handling
+#
+
+
+@implementer(IRealm)
+class LocalSoledadIMAPRealm(object):
+
+ _encoding = 'utf-8'
+
+ def __init__(self, soledad_sessions):
+ """
+ :param soledad_sessions: a dict-like object, containing instances
+ of a Store (soledad instances), indexed by
+ userid.
+ """
+ self._soledad_sessions = soledad_sessions
+
+ def requestAvatar(self, avatarId, mind, *interfaces):
+ if isinstance(avatarId, str):
+ avatarId = avatarId.decode(self._encoding)
+
+ def gotSoledad(soledad):
+ for iface in interfaces:
+ if iface is IAccount:
+ avatar = IMAPAccount(soledad, avatarId)
+ return (IAccount, avatar,
+ getattr(avatar, 'logout', lambda: None))
+ raise NotImplementedError(self, interfaces)
+
+ return self.lookupSoledadInstance(avatarId).addCallback(gotSoledad)
+
+ def lookupSoledadInstance(self, userid):
+ soledad = self._soledad_sessions[userid]
+ # XXX this should return the instance after whenReady callback
+ return defer.succeed(soledad)
+
+
+class IMAPTokenChecker(LocalSoledadTokenChecker):
+ """A credentials checker that will lookup a token for the IMAP service.
+ For now it will be using the same identifier than SMTPTokenChecker"""
+
+ service = 'mail_auth'
+
+
+class LocalSoledadIMAPServer(LEAPIMAPServer):
+
+ """
+ An IMAP Server that authenticates against a LocalSoledad store.
+ """
+
+ def __init__(self, soledad_sessions, *args, **kw):
+
+ LEAPIMAPServer.__init__(self, *args, **kw)
+
+ realm = LocalSoledadIMAPRealm(soledad_sessions)
+ portal = Portal(realm)
+ checker = IMAPTokenChecker(soledad_sessions)
+ self.checker = checker
+ self.portal = portal
+ portal.registerChecker(checker)
+
+
+class LeapIMAPFactory(ServerFactory):
+
+ """
+ Factory for a IMAP4 server with soledad remote sync and gpg-decryption
+ capabilities.
+ """
+
+ protocol = LocalSoledadIMAPServer
+
+ def __init__(self, soledad_sessions):
+ """
+ Initializes the server factory.
+
+ :param soledad_sessions: a dict-like object, containing instances
+ of a Store (soledad instances), indexed by
+ userid.
+ """
+ self._soledad_sessions = soledad_sessions
+ self._connections = defaultdict()
+
+ def buildProtocol(self, addr):
+ """
+ Return a protocol suitable for the job.
+
+ :param addr: remote ip address
+ :type addr: str
+ """
+ # TODO should reject anything from addr != localhost,
+ # just in case.
+ log.msg("Building protocol for connection %s" % addr)
+ imapProtocol = self.protocol(self._soledad_sessions)
+ self._connections[addr] = imapProtocol
+ return imapProtocol
+
+ def stopFactory(self):
+ # say bye!
+ for conn, proto in self._connections.items():
+ log.msg("Closing connections for %s" % conn)
+ proto.close_server_connection()
+
+ def doStop(self):
+ """
+ Stops imap service (fetcher, factory and port).
+ """
+ return ServerFactory.doStop(self)
+
+
+def run_service(soledad_sessions, port=IMAP_PORT):
+ """
+ Main entry point to run the service from the client.
+
+ :param soledad_sessions: a dict-like object, containing instances
+ of a Store (soledad instances), indexed by userid.
+
+ :returns: the port as returned by the reactor when starts listening, and
+ the factory for the protocol.
+ :rtype: tuple
+ """
+ factory = LeapIMAPFactory(soledad_sessions)
+
+ try:
+ interface = "localhost"
+ # don't bind just to localhost if we are running on docker since we
+ # won't be able to access imap from the host
+ if os.environ.get("LEAP_DOCKERIZED"):
+ interface = ''
+
+ # TODO use Endpoints !!!
+ tport = reactor.listenTCP(port, factory,
+ interface=interface)
+ except CannotListenError:
+ logger.error("IMAP Service failed to start: "
+ "cannot listen in port %s" % (port,))
+ except Exception as exc:
+ logger.error("Error launching IMAP service: %r" % (exc,))
+ else:
+ # all good.
+
+ if DO_MANHOLE:
+ # TODO get pass from env var.too.
+ manhole_factory = manhole.getManholeFactory(
+ {'f': factory,
+ 'gm': factory.theAccount.getMailbox},
+ "boss", "leap")
+ # TODO use Endpoints !!!
+ reactor.listenTCP(manhole.MANHOLE_PORT, manhole_factory,
+ interface="127.0.0.1")
+ logger.debug("IMAP4 Server is RUNNING in port %s" % (port,))
+ emit_async(catalog.IMAP_SERVICE_STARTED, str(port))
+
+ # FIXME -- change service signature
+ return tport, factory
+
+ # not ok, signal error.
+ emit_async(catalog.IMAP_SERVICE_FAILED_TO_START, str(port))
diff --git a/src/leap/bitmask/mail/imap/service/manhole.py b/src/leap/bitmask/mail/imap/service/manhole.py
new file mode 100644
index 00000000..c83ae899
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/service/manhole.py
@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+# manhole.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Utilities for enabling the manhole administrative interface into the
+LEAP Mail application.
+"""
+MANHOLE_PORT = 2222
+
+
+def getManholeFactory(namespace, user, secret):
+ """
+ Get an administrative manhole into the application.
+
+ :param namespace: the namespace to show in the manhole
+ :type namespace: dict
+ :param user: the user to authenticate into the administrative shell.
+ :type user: str
+ :param secret: pass for this manhole
+ :type secret: str
+ """
+ import string
+
+ from twisted.cred.portal import Portal
+ from twisted.conch import manhole, manhole_ssh
+ from twisted.conch.insults import insults
+ from twisted.cred.checkers import (
+ InMemoryUsernamePasswordDatabaseDontUse as MemoryDB)
+
+ from rlcompleter import Completer
+
+ class EnhancedColoredManhole(manhole.ColoredManhole):
+ """
+ A Manhole with some primitive autocomplete support.
+ """
+ # TODO use introspection to make life easier
+
+ def find_common(self, l):
+ """
+ find common parts in thelist items
+ ex: 'ab' for ['abcd','abce','abf']
+ requires an ordered list
+ """
+ if len(l) == 1:
+ return l[0]
+
+ init = l[0]
+ for item in l[1:]:
+ for i, (x, y) in enumerate(zip(init, item)):
+ if x != y:
+ init = "".join(init[:i])
+ break
+
+ if not init:
+ return None
+ return init
+
+ def handle_TAB(self):
+ """
+ Trap the TAB keystroke.
+ """
+ necessarypart = "".join(self.lineBuffer).split(' ')[-1]
+ completer = Completer(globals())
+ if completer.complete(necessarypart, 0):
+ matches = list(set(completer.matches)) # has multiples
+
+ if len(matches) == 1:
+ length = len(necessarypart)
+ self.lineBuffer = self.lineBuffer[:-length]
+ self.lineBuffer.extend(matches[0])
+ self.lineBufferIndex = len(self.lineBuffer)
+ else:
+ matches.sort()
+ commons = self.find_common(matches)
+ if commons:
+ length = len(necessarypart)
+ self.lineBuffer = self.lineBuffer[:-length]
+ self.lineBuffer.extend(commons)
+ self.lineBufferIndex = len(self.lineBuffer)
+
+ self.terminal.nextLine()
+ while matches:
+ matches, part = matches[4:], matches[:4]
+ for item in part:
+ self.terminal.write('%s' % item.ljust(30))
+ self.terminal.write('\n')
+ self.terminal.nextLine()
+
+ self.terminal.eraseLine()
+ self.terminal.cursorBackward(self.lineBufferIndex + 5)
+ self.terminal.write("%s %s" % (
+ self.ps[self.pn], "".join(self.lineBuffer)))
+
+ def keystrokeReceived(self, keyID, modifier):
+ """
+ Act upon any keystroke received.
+ """
+ self.keyHandlers.update({'\b': self.handle_BACKSPACE})
+ m = self.keyHandlers.get(keyID)
+ if m is not None:
+ m()
+ elif keyID in string.printable:
+ self.characterReceived(keyID, False)
+
+ sshRealm = manhole_ssh.TerminalRealm()
+
+ def chainedProtocolFactory():
+ return insults.ServerProtocol(EnhancedColoredManhole, namespace)
+
+ sshRealm = manhole_ssh.TerminalRealm()
+ sshRealm.chainedProtocolFactory = chainedProtocolFactory
+
+ portal = Portal(
+ sshRealm, [MemoryDB(**{user: secret})])
+
+ f = manhole_ssh.ConchFactory(portal)
+ return f
diff --git a/src/leap/bitmask/mail/imap/service/notes.txt b/src/leap/bitmask/mail/imap/service/notes.txt
new file mode 100644
index 00000000..623e1224
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/service/notes.txt
@@ -0,0 +1,81 @@
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+* OK [CAPABILITY IMAP4rev1 IDLE NAMESPACE] Twisted IMAP4rev1 Ready.
+
+##
+T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP]
+NCLJ1 CAPABILITY.
+
+##
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+* CAPABILITY IMAP4rev1 IDLE NAMESPACE.
+NCLJ1 OK CAPABILITY completed.
+
+##
+T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP]
+NCLJ2 LOGIN user "pass".
+
+#
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+NCLJ2 OK LOGIN succeeded.
+
+##
+T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP]
+NCLJ3 CAPABILITY.
+
+#
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+* CAPABILITY IMAP4rev1 IDLE NAMESPACE.
+NCLJ3 OK CAPABILITY completed.
+
+#
+T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP]
+NCLJ4 LIST "" "".
+
+##
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+* LIST (\Seen \Answered \Flagged \Deleted \Draft \Recent List) "/" "INBOX".
+NCLJ4 OK LIST completed.
+
+#
+T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP]
+NCLJ5 LIST "" "*".
+
+##
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+* LIST (\Seen \Answered \Flagged \Deleted \Draft \Recent List) "/" "INBOX".
+NCLJ5 OK LIST completed.
+
+#
+T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP]
+NCLJ6 SELECT INBOX.
+
+#
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+* 0 EXISTS.
+* 3 RECENT.
+* FLAGS (\Seen \Answered \Flagged \Deleted \Draft \Recent List).
+* OK [UIDVALIDITY 42].
+NCLJ6 OK [READ-WRITE] SELECT successful.
+
+#
+T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP]
+NCLJ7 EXAMINE INBOX.
+
+##
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+* 0 EXISTS.
+* 3 RECENT.
+* FLAGS (\Seen \Answered \Flagged \Deleted \Draft \Recent List).
+* OK [UIDVALIDITY 42].
+NCLJ7 OK [READ-ONLY] EXAMINE successful.
+
+#
+T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP]
+NCLJ8 LOGOUT.
+
+##
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+* BYE Nice talking to you.
+NCLJ8 OK LOGOUT successful.
+
+
diff --git a/src/leap/bitmask/mail/imap/service/rfc822.message b/src/leap/bitmask/mail/imap/service/rfc822.message
new file mode 100644
index 00000000..ee97ab92
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/service/rfc822.message
@@ -0,0 +1,86 @@
+Return-Path: <twisted-commits-admin@twistedmatrix.com>
+Delivered-To: exarkun@meson.dyndns.org
+Received: from localhost [127.0.0.1]
+ by localhost with POP3 (fetchmail-6.2.1)
+ for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST)
+Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105])
+ by intarweb.us (Postfix) with ESMTP id 4A4A513EA4
+ for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST)
+Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com)
+ by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian))
+ id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600
+Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian))
+ id 18w63j-0007VK-00
+ for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600
+To: twisted-commits@twistedmatrix.com
+From: etrepum CVS <etrepum@twistedmatrix.com>
+Reply-To: twisted-python@twistedmatrix.com
+X-Mailer: CVSToys
+Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com>
+Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up.
+Sender: twisted-commits-admin@twistedmatrix.com
+Errors-To: twisted-commits-admin@twistedmatrix.com
+X-BeenThere: twisted-commits@twistedmatrix.com
+X-Mailman-Version: 2.0.11
+Precedence: bulk
+List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help>
+List-Post: <mailto:twisted-commits@twistedmatrix.com>
+List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
+ <mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe>
+List-Id: <twisted-commits.twistedmatrix.com>
+List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
+ <mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe>
+List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/>
+Date: Thu, 20 Mar 2003 13:50:39 -0600
+
+Modified files:
+Twisted/twisted/python/rebuild.py 1.19 1.20
+
+Log message:
+rebuild now works on python versions from 2.2.0 and up.
+
+
+ViewCVS links:
+http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted
+
+Index: Twisted/twisted/python/rebuild.py
+diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20
+--- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003
++++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003
+@@ -206,15 +206,27 @@
+ clazz.__dict__.clear()
+ clazz.__getattr__ = __getattr__
+ clazz.__module__ = module.__name__
++ if newclasses:
++ import gc
++ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2):
++ hasBrokenRebuild = 1
++ gc_objects = gc.get_objects()
++ else:
++ hasBrokenRebuild = 0
+ for nclass in newclasses:
+ ga = getattr(module, nclass.__name__)
+ if ga is nclass:
+ log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass))
+ else:
+- import gc
+- for r in gc.get_referrers(nclass):
+- if isinstance(r, nclass):
++ if hasBrokenRebuild:
++ for r in gc_objects:
++ if not getattr(r, '__class__', None) is nclass:
++ continue
+ r.__class__ = ga
++ else:
++ for r in gc.get_referrers(nclass):
++ if getattr(r, '__class__', None) is nclass:
++ r.__class__ = ga
+ if doLog:
+ log.msg('')
+ log.msg(' (fixing %s): ' % str(module.__name__))
+
+
+_______________________________________________
+Twisted-commits mailing list
+Twisted-commits@twistedmatrix.com
+http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits
diff --git a/src/leap/bitmask/mail/imap/tests/.gitignore b/src/leap/bitmask/mail/imap/tests/.gitignore
new file mode 100644
index 00000000..60baa9cb
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/tests/.gitignore
@@ -0,0 +1 @@
+data/*
diff --git a/src/leap/bitmask/mail/imap/tests/getmail b/src/leap/bitmask/mail/imap/tests/getmail
new file mode 100755
index 00000000..dd3fa0bb
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/tests/getmail
@@ -0,0 +1,344 @@
+#!/usr/bin/env python
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE in twisted for details.
+
+# Modifications by LEAP Developers 2014 to fit
+# Bitmask configuration settings.
+"""
+Simple IMAP4 client which displays the subjects of all messages in a
+particular mailbox.
+"""
+
+import os
+import sys
+
+from twisted.internet import protocol
+from twisted.internet import ssl
+from twisted.internet import defer
+from twisted.internet import stdio
+from twisted.mail import imap4
+from twisted.protocols import basic
+from twisted.python import log
+
+# Global options stored here from main
+_opts = {}
+
+
+class TrivialPrompter(basic.LineReceiver):
+ from os import linesep as delimiter
+
+ promptDeferred = None
+
+ def prompt(self, msg):
+ assert self.promptDeferred is None
+ self.display(msg)
+ self.promptDeferred = defer.Deferred()
+ return self.promptDeferred
+
+ def display(self, msg):
+ self.transport.write(msg)
+
+ def lineReceived(self, line):
+ if self.promptDeferred is None:
+ return
+ d, self.promptDeferred = self.promptDeferred, None
+ d.callback(line)
+
+
+class SimpleIMAP4Client(imap4.IMAP4Client):
+ """
+ A client with callbacks for greeting messages from an IMAP server.
+ """
+ greetDeferred = None
+
+ def serverGreeting(self, caps):
+ self.serverCapabilities = caps
+ if self.greetDeferred is not None:
+ d, self.greetDeferred = self.greetDeferred, None
+ d.callback(self)
+
+
+class SimpleIMAP4ClientFactory(protocol.ClientFactory):
+ usedUp = False
+
+ protocol = SimpleIMAP4Client
+
+ def __init__(self, username, onConn):
+ self.ctx = ssl.ClientContextFactory()
+
+ self.username = username
+ self.onConn = onConn
+
+ def buildProtocol(self, addr):
+ """
+ Initiate the protocol instance. Since we are building a simple IMAP
+ client, we don't bother checking what capabilities the server has. We
+ just add all the authenticators twisted.mail has.
+ """
+ assert not self.usedUp
+ self.usedUp = True
+
+ p = self.protocol(self.ctx)
+ p.factory = self
+ p.greetDeferred = self.onConn
+
+ p.registerAuthenticator(imap4.PLAINAuthenticator(self.username))
+ p.registerAuthenticator(imap4.LOGINAuthenticator(self.username))
+ p.registerAuthenticator(
+ imap4.CramMD5ClientAuthenticator(self.username))
+
+ return p
+
+ def clientConnectionFailed(self, connector, reason):
+ d, self.onConn = self.onConn, None
+ d.errback(reason)
+
+
+def cbServerGreeting(proto, username, password):
+ """
+ Initial callback - invoked after the server sends us its greet message.
+ """
+ # Hook up stdio
+ tp = TrivialPrompter()
+ stdio.StandardIO(tp)
+
+ # And make it easily accessible
+ proto.prompt = tp.prompt
+ proto.display = tp.display
+
+ # Try to authenticate securely
+ return proto.authenticate(
+ password).addCallback(
+ cbAuthentication,
+ proto).addErrback(
+ ebAuthentication, proto, username, password
+ )
+
+
+def ebConnection(reason):
+ """
+ Fallback error-handler. If anything goes wrong, log it and quit.
+ """
+ log.startLogging(sys.stdout)
+ log.err(reason)
+ return reason
+
+
+def cbAuthentication(result, proto):
+ """
+ Callback after authentication has succeeded.
+
+ Lists a bunch of mailboxes.
+ """
+ return proto.list("", "*"
+ ).addCallback(cbMailboxList, proto
+ )
+
+
+def ebAuthentication(failure, proto, username, password):
+ """
+ Errback invoked when authentication fails.
+
+ If it failed because no SASL mechanisms match, offer the user the choice
+ of logging in insecurely.
+
+ If you are trying to connect to your Gmail account, you will be here!
+ """
+ failure.trap(imap4.NoSupportedAuthentication)
+ return InsecureLogin(proto, username, password)
+
+
+def InsecureLogin(proto, username, password):
+ """
+ insecure-login.
+ """
+ return proto.login(username, password
+ ).addCallback(cbAuthentication, proto
+ )
+
+
+def cbMailboxList(result, proto):
+ """
+ Callback invoked when a list of mailboxes has been retrieved.
+ If we have a selected mailbox in the global options, we directly pick it.
+ Otherwise, we offer a prompt to let user choose one.
+ """
+ all_mbox_list = [e[2] for e in result]
+ s = '\n'.join(['%d. %s' % (n + 1, m) for (n, m) in zip(range(len(all_mbox_list)), all_mbox_list)])
+ if not s:
+ return defer.fail(Exception("No mailboxes exist on server!"))
+
+ selected_mailbox = _opts.get('mailbox')
+
+ if not selected_mailbox:
+ return proto.prompt(s + "\nWhich mailbox? [1] "
+ ).addCallback(cbPickMailbox, proto, all_mbox_list
+ )
+ else:
+ mboxes_lower = map(lambda s: s.lower(), all_mbox_list)
+ index = mboxes_lower.index(selected_mailbox.lower()) + 1
+ return cbPickMailbox(index, proto, all_mbox_list)
+
+
+def cbPickMailbox(result, proto, mboxes):
+ """
+ When the user selects a mailbox, "examine" it.
+ """
+ mbox = mboxes[int(result or '1') - 1]
+ return proto.examine(mbox
+ ).addCallback(cbExamineMbox, proto
+ )
+
+
+def cbExamineMbox(result, proto):
+ """
+ Callback invoked when examine command completes.
+
+ Retrieve the subject header of every message in the mailbox.
+ """
+ return proto.fetchSpecific('1:*',
+ headerType='HEADER.FIELDS',
+ headerArgs=['SUBJECT'],
+ ).addCallback(cbFetch, proto,
+ )
+
+
+def cbFetch(result, proto):
+ """
+ Display a listing of the messages in the mailbox, based on the collected
+ headers.
+ """
+ selected_subject = _opts.get('subject', None)
+ index = None
+
+ if result:
+ keys = result.keys()
+ keys.sort()
+
+ if selected_subject:
+ for k in keys:
+ # remove 'Subject: ' preffix plus eol
+ subject = result[k][0][2][9:].rstrip('\r\n')
+ if subject.lower() == selected_subject.lower():
+ index = k
+ break
+ else:
+ for k in keys:
+ proto.display('%s %s' % (k, result[k][0][2]))
+ else:
+ print "Hey, an empty mailbox!"
+
+ if not index:
+ return proto.prompt("\nWhich message? [1] (Q quits) "
+ ).addCallback(cbPickMessage, proto)
+ else:
+ return cbPickMessage(index, proto)
+
+
+def cbPickMessage(result, proto):
+ """
+ Pick a message.
+ """
+ if result == "Q":
+ print "Bye!"
+ return proto.logout()
+
+ return proto.fetchSpecific(
+ '%s' % result,
+ headerType='',
+ headerArgs=['BODY.PEEK[]'],
+ ).addCallback(cbShowmessage, proto)
+
+
+def cbShowmessage(result, proto):
+ """
+ Display message.
+ """
+ if result:
+ keys = result.keys()
+ keys.sort()
+ for k in keys:
+ proto.display('%s %s' % (k, result[k][0][2]))
+ else:
+ print "Hey, an empty message!"
+
+ return proto.logout()
+
+
+def cbClose(result):
+ """
+ Close the connection when we finish everything.
+ """
+ from twisted.internet import reactor
+ reactor.stop()
+
+
+def main():
+ import argparse
+ import ConfigParser
+ import sys
+ from twisted.internet import reactor
+
+ description = (
+ 'Get messages from a LEAP IMAP Proxy.\nThis is a '
+ 'debugging tool, do not use this to retrieve any sensitive '
+ 'information, or we will send ninjas to your house!')
+ epilog = (
+ 'In case you want to automate the usage of this utility '
+ 'you can place your credentials in a file pointed by '
+ 'BITMASK_CREDENTIALS. You need to have a [Credentials] '
+ 'section, with username=<user@provider> and password fields')
+
+ parser = argparse.ArgumentParser(description=description, epilog=epilog)
+ credentials = os.environ.get('BITMASK_CREDENTIALS')
+
+ if credentials:
+ try:
+ config = ConfigParser.ConfigParser()
+ config.read(credentials)
+ username = config.get('Credentials', 'username')
+ password = config.get('Credentials', 'password')
+ except Exception, e:
+ print "Error reading credentials file: {0}".format(e)
+ sys.exit()
+ else:
+ parser.add_argument('username', type=str)
+ parser.add_argument('password', type=str)
+
+ parser.add_argument('--mailbox', dest='mailbox', default=None,
+ help='Which mailbox to retrieve. Empty for interactive prompt.')
+ parser.add_argument('--subject', dest='subject', default=None,
+ help='A subject for retrieve a mail that matches. Empty for interactive prompt.')
+
+ ns = parser.parse_args()
+
+ if not credentials:
+ username = ns.username
+ password = ns.password
+
+ _opts['mailbox'] = ns.mailbox
+ _opts['subject'] = ns.subject
+
+ hostname = "localhost"
+ port = "1984"
+
+ onConn = defer.Deferred(
+ ).addCallback(cbServerGreeting, username, password
+ ).addErrback(ebConnection
+ ).addBoth(cbClose)
+
+ factory = SimpleIMAP4ClientFactory(username, onConn)
+
+ if port == '993':
+ reactor.connectSSL(
+ hostname, int(port), factory, ssl.ClientContextFactory())
+ else:
+ if not port:
+ port = 143
+ reactor.connectTCP(hostname, int(port), factory)
+ reactor.run()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/src/leap/bitmask/mail/imap/tests/imapclient.py b/src/leap/bitmask/mail/imap/tests/imapclient.py
new file mode 100755
index 00000000..c353ceed
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/tests/imapclient.py
@@ -0,0 +1,207 @@
+#!/usr/bin/env python
+
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Simple IMAP4 client which connects to our custome
+IMAP4 server: imapserver.py.
+"""
+
+import sys
+
+from twisted.internet import protocol
+from twisted.internet import defer
+from twisted.internet import stdio
+from twisted.mail import imap4
+from twisted.protocols import basic
+from twisted.python import util
+from twisted.python import log
+
+
+class TrivialPrompter(basic.LineReceiver):
+ # from os import linesep as delimiter
+
+ promptDeferred = None
+
+ def prompt(self, msg):
+ assert self.promptDeferred is None
+ self.display(msg)
+ self.promptDeferred = defer.Deferred()
+ return self.promptDeferred
+
+ def display(self, msg):
+ self.transport.write(msg)
+
+ def lineReceived(self, line):
+ if self.promptDeferred is None:
+ return
+ d, self.promptDeferred = self.promptDeferred, None
+ d.callback(line)
+
+
+class SimpleIMAP4Client(imap4.IMAP4Client):
+
+ """
+ Add callbacks when the client receives greeting messages from
+ an IMAP server.
+ """
+ greetDeferred = None
+
+ def serverGreeting(self, caps):
+ self.serverCapabilities = caps
+ if self.greetDeferred is not None:
+ d, self.greetDeferred = self.greetDeferred, None
+ d.callback(self)
+
+
+class SimpleIMAP4ClientFactory(protocol.ClientFactory):
+ usedUp = False
+ protocol = SimpleIMAP4Client
+
+ def __init__(self, username, onConn):
+ self.username = username
+ self.onConn = onConn
+
+ def buildProtocol(self, addr):
+ assert not self.usedUp
+ self.usedUp = True
+
+ p = self.protocol()
+ p.factory = self
+ p.greetDeferred = self.onConn
+
+ p.registerAuthenticator(imap4.PLAINAuthenticator(self.username))
+ p.registerAuthenticator(imap4.LOGINAuthenticator(self.username))
+ p.registerAuthenticator(
+ imap4.CramMD5ClientAuthenticator(self.username))
+
+ return p
+
+ def clientConnectionFailed(self, connector, reason):
+ d, self.onConn = self.onConn, None
+ d.errback(reason)
+
+
+def cbServerGreeting(proto, username, password):
+ """
+ Initial callback - invoked after the server sends us its greet message.
+ """
+ # Hook up stdio
+ tp = TrivialPrompter()
+ stdio.StandardIO(tp)
+
+ # And make it easily accessible
+ proto.prompt = tp.prompt
+ proto.display = tp.display
+
+ # Try to authenticate securely
+ return proto.authenticate(
+ password).addCallback(
+ cbAuthentication, proto).addErrback(
+ ebAuthentication, proto, username, password)
+
+
+def ebConnection(reason):
+ """
+ Fallback error-handler. If anything goes wrong, log it and quit.
+ """
+ log.startLogging(sys.stdout)
+ log.err(reason)
+ return reason
+
+
+def cbAuthentication(result, proto):
+ """
+ Callback after authentication has succeeded.
+ List a bunch of mailboxes.
+ """
+ return proto.list("", "*"
+ ).addCallback(cbMailboxList, proto
+ )
+
+
+def ebAuthentication(failure, proto, username, password):
+ """
+ Errback invoked when authentication fails.
+ If it failed because no SASL mechanisms match, offer the user the choice
+ of logging in insecurely.
+ If you are trying to connect to your Gmail account, you will be here!
+ """
+ failure.trap(imap4.NoSupportedAuthentication)
+ return proto.prompt(
+ "No secure authentication available. Login insecurely? (y/N) "
+ ).addCallback(cbInsecureLogin, proto, username, password
+ )
+
+
+def cbInsecureLogin(result, proto, username, password):
+ """
+ Callback for "insecure-login" prompt.
+ """
+ if result.lower() == "y":
+ # If they said yes, do it.
+ return proto.login(username, password
+ ).addCallback(cbAuthentication, proto
+ )
+ return defer.fail(Exception("Login failed for security reasons."))
+
+
+def cbMailboxList(result, proto):
+ """
+ Callback invoked when a list of mailboxes has been retrieved.
+ """
+ result = [e[2] for e in result]
+ s = '\n'.join(
+ ['%d. %s' % (n + 1, m) for (n, m) in zip(range(len(result)), result)])
+ if not s:
+ return defer.fail(Exception("No mailboxes exist on server!"))
+ return proto.prompt(s + "\nWhich mailbox? [1] "
+ ).addCallback(cbPickMailbox, proto, result
+ )
+
+
+def cbPickMailbox(result, proto, mboxes):
+ """
+ When the user selects a mailbox, "examine" it.
+ """
+ mbox = mboxes[int(result or '1') - 1]
+ return proto.status(mbox, 'MESSAGES', 'UNSEEN'
+ ).addCallback(cbMboxStatus, proto)
+
+
+def cbMboxStatus(result, proto):
+ print "You have %s messages (%s unseen)!" % (
+ result['MESSAGES'], result['UNSEEN'])
+ return proto.logout()
+
+
+def cbClose(result):
+ """
+ Close the connection when we finish everything.
+ """
+ from twisted.internet import reactor
+ reactor.stop()
+
+
+def main():
+ hostname = raw_input('IMAP4 Server Hostname: ')
+ port = raw_input('IMAP4 Server Port (the default is 143): ')
+ username = raw_input('IMAP4 Username: ')
+ password = util.getPassword('IMAP4 Password: ')
+
+ onConn = defer.Deferred(
+ ).addCallback(cbServerGreeting, username, password
+ ).addErrback(ebConnection
+ ).addBoth(cbClose)
+
+ factory = SimpleIMAP4ClientFactory(username, onConn)
+
+ from twisted.internet import reactor
+ conn = reactor.connectTCP(hostname, int(port), factory)
+ reactor.run()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/src/leap/bitmask/mail/imap/tests/regressions_mime_struct b/src/leap/bitmask/mail/imap/tests/regressions_mime_struct
new file mode 100755
index 00000000..03326646
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/tests/regressions_mime_struct
@@ -0,0 +1,461 @@
+#!/usr/bin/env python
+
+# -*- coding: utf-8 -*-
+# regression_mime_struct
+# Copyright (C) 2014 LEAP
+# Copyright (c) Twisted Matrix Laboratories.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Simple Regression Tests for checking MIME struct handling using IMAP4 client.
+
+Iterates trough all mails under a given folder and tries to APPEND them to
+the server being tested. After FETCHING the pushed message, it compares
+the received version with the one that was saved, and exits with an error
+code if they do not match.
+"""
+import os
+import StringIO
+import sys
+
+from email.parser import Parser
+
+from twisted.internet import protocol
+from twisted.internet import ssl
+from twisted.internet import defer
+from twisted.internet import stdio
+from twisted.mail import imap4
+from twisted.protocols import basic
+from twisted.python import log
+
+
+REGRESSIONS_FOLDER = os.environ.get(
+ "REGRESSIONS_FOLDER", "regressions_test")
+print "[+] Using regressions folder:", REGRESSIONS_FOLDER
+
+parser = Parser()
+
+
+def get_msg_parts(raw):
+ """
+ Return a representation of the parts of a message suitable for
+ comparison.
+
+ :param raw: string for the message
+ :type raw: str
+ """
+ m = parser.parsestr(raw)
+ return [dict(part.items())
+ if part.is_multipart()
+ else part.get_payload()
+ for part in m.walk()]
+
+
+def compare_msg_parts(a, b):
+ """
+ Compare two sequences of parts of messages.
+
+ :param a: part sequence for message a
+ :param b: part sequence for message b
+
+ :return: True if both message sequences are equivalent.
+ :rtype: bool
+ """
+ # XXX This could be smarter and show the differences in the
+ # different parts when/where they differ.
+ #import pprint; pprint.pprint(a[0])
+ #import pprint; pprint.pprint(b[0])
+
+ def lowerkey(d):
+ return dict((k.lower(), v.replace('\r', ''))
+ for k, v in d.iteritems())
+
+ def eq(x, y):
+ # For dicts, we compare a variation with their keys
+ # in lowercase, and \r removed from their values
+ if all(map(lambda i: isinstance(i, dict), (x, y))):
+ x, y = map(lowerkey, (x, y))
+ return x == y
+
+ compare_vector = map(lambda tup: eq(tup[0], tup[1]), zip(a, b))
+ all_match = all(compare_vector)
+
+ if not all_match:
+ print "PARTS MISMATCH!"
+ print "vector: ", compare_vector
+ index = compare_vector.index(False)
+ from pprint import pprint
+ print "Expected:"
+ pprint(a[index])
+ print ("***")
+ print "Found:"
+ pprint(b[index])
+ print
+
+ return all_match
+
+
+def get_fd(string):
+ """
+ Return a file descriptor with the passed string
+ as content.
+ """
+ fd = StringIO.StringIO()
+ fd.write(string)
+ fd.seek(0)
+ return fd
+
+
+class TrivialPrompter(basic.LineReceiver):
+ promptDeferred = None
+
+ def prompt(self, msg):
+ assert self.promptDeferred is None
+ self.display(msg)
+ self.promptDeferred = defer.Deferred()
+ return self.promptDeferred
+
+ def display(self, msg):
+ self.transport.write(msg)
+
+ def lineReceived(self, line):
+ if self.promptDeferred is None:
+ return
+ d, self.promptDeferred = self.promptDeferred, None
+ d.callback(line)
+
+
+class SimpleIMAP4Client(imap4.IMAP4Client):
+ """
+ A client with callbacks for greeting messages from an IMAP server.
+ """
+ greetDeferred = None
+
+ def serverGreeting(self, caps):
+ self.serverCapabilities = caps
+ if self.greetDeferred is not None:
+ d, self.greetDeferred = self.greetDeferred, None
+ d.callback(self)
+
+
+class SimpleIMAP4ClientFactory(protocol.ClientFactory):
+ usedUp = False
+ protocol = SimpleIMAP4Client
+
+ def __init__(self, username, onConn):
+ self.ctx = ssl.ClientContextFactory()
+
+ self.username = username
+ self.onConn = onConn
+
+ def buildProtocol(self, addr):
+ """
+ Initiate the protocol instance. Since we are building a simple IMAP
+ client, we don't bother checking what capabilities the server has. We
+ just add all the authenticators twisted.mail has. Note: Gmail no
+ longer uses any of the methods below, it's been using XOAUTH since
+ 2010.
+ """
+ assert not self.usedUp
+ self.usedUp = True
+
+ p = self.protocol(self.ctx)
+ p.factory = self
+ p.greetDeferred = self.onConn
+
+ p.registerAuthenticator(imap4.PLAINAuthenticator(self.username))
+ p.registerAuthenticator(imap4.LOGINAuthenticator(self.username))
+ p.registerAuthenticator(
+ imap4.CramMD5ClientAuthenticator(self.username))
+
+ return p
+
+ def clientConnectionFailed(self, connector, reason):
+ d, self.onConn = self.onConn, None
+ d.errback(reason)
+
+
+def cbServerGreeting(proto, username, password):
+ """
+ Initial callback - invoked after the server sends us its greet message.
+ """
+ # Hook up stdio
+ tp = TrivialPrompter()
+ stdio.StandardIO(tp)
+
+ # And make it easily accessible
+ proto.prompt = tp.prompt
+ proto.display = tp.display
+
+ # Try to authenticate securely
+ return proto.authenticate(
+ password).addCallback(
+ cbAuthentication,
+ proto).addErrback(
+ ebAuthentication, proto, username, password
+ )
+
+
+def ebConnection(reason):
+ """
+ Fallback error-handler. If anything goes wrong, log it and quit.
+ """
+ log.startLogging(sys.stdout)
+ log.err(reason)
+ return reason
+
+
+def cbAuthentication(result, proto):
+ """
+ Callback after authentication has succeeded.
+
+ Lists a bunch of mailboxes.
+ """
+ return proto.select(
+ REGRESSIONS_FOLDER
+ ).addCallback(
+ cbSelectMbox, proto
+ ).addErrback(
+ ebSelectMbox, proto, REGRESSIONS_FOLDER)
+
+
+def ebAuthentication(failure, proto, username, password):
+ """
+ Errback invoked when authentication fails.
+
+ If it failed because no SASL mechanisms match, offer the user the choice
+ of logging in insecurely.
+
+ If you are trying to connect to your Gmail account, you will be here!
+ """
+ failure.trap(imap4.NoSupportedAuthentication)
+ return InsecureLogin(proto, username, password)
+
+
+def InsecureLogin(proto, username, password):
+ """
+ Raise insecure-login error.
+ """
+ return proto.login(
+ username, password
+ ).addCallback(
+ cbAuthentication, proto)
+
+
+def cbSelectMbox(result, proto):
+ """
+ Callback invoked when select command finishes successfully.
+
+ If any message is in the test folder, it will flag them as deleted and
+ expunge.
+ If no messages found, it will start with the APPEND tests.
+ """
+ print "SELECT: %s EXISTS " % result.get("EXISTS", "??")
+
+ if result["EXISTS"] != 0:
+ # Flag as deleted, expunge, and do an examine again.
+ print "There is mail here, will delete..."
+ return cbDeleteAndExpungeTestFolder(proto)
+
+ else:
+ return cbAppendNextMessage(proto)
+
+
+def ebSelectMbox(failure, proto, folder):
+ """
+ Errback invoked when the examine command fails.
+
+ Creates the folder.
+ """
+ log.err(failure)
+ log.msg("Folder %r does not exist. Creating..." % (folder,))
+ return proto.create(folder).addCallback(cbAuthentication, proto)
+
+
+def ebExpunge(failure):
+ log.err(failure)
+
+
+def cbDeleteAndExpungeTestFolder(proto):
+ """
+ Callback invoked fom cbExamineMbox when the number of messages in the
+ mailbox is not zero. It flags all messages as deleted and expunge the
+ mailbox.
+ """
+ return proto.setFlags(
+ "1:*", ("\\Deleted",)
+ ).addCallback(
+ lambda r: proto.expunge()
+ ).addCallback(
+ cbExpunge, proto
+ ).addErrback(
+ ebExpunge)
+
+
+def cbExpunge(result, proto):
+ return proto.select(
+ REGRESSIONS_FOLDER
+ ).addCallback(
+ cbSelectMbox, proto
+ ).addErrback(ebSettingDeleted, proto)
+
+
+def ebSettingDeleted(failure, proto):
+ """
+ Report errors during deletion of messages in the mailbox.
+ """
+ print failure.getTraceback()
+
+
+def cbAppendNextMessage(proto):
+ """
+ Appends the next message in the global queue to the test folder.
+ """
+ # 1. Get the next test message from global tuple.
+ try:
+ next_sample = SAMPLES.pop()
+ except IndexError:
+ # we're done!
+ return proto.logout()
+
+ print "\nAPPEND %s" % (next_sample,)
+ raw = open(next_sample).read()
+ msg = get_fd(raw)
+ return proto.append(
+ REGRESSIONS_FOLDER, msg
+ ).addCallback(
+ lambda r: proto.select(REGRESSIONS_FOLDER)
+ ).addCallback(
+ cbAppend, proto, raw
+ ).addErrback(
+ ebAppend, proto, raw)
+
+
+def cbAppend(result, proto, orig_msg):
+ """
+ Fetches the message right after an append.
+ """
+ # XXX keep account of highest UID
+ uid = "1:*"
+
+ return proto.fetchSpecific(
+ '%s' % uid,
+ headerType='',
+ headerArgs=['BODY.PEEK[]'],
+ ).addCallback(
+ cbCompareMessage, proto, orig_msg
+ ).addErrback(ebAppend, proto, orig_msg)
+
+
+def ebAppend(failure, proto, raw):
+ """
+ Errorback for the append operation
+ """
+ print "ERROR WHILE APPENDING!"
+ print failure.getTraceback()
+
+
+def cbPickMessage(result, proto):
+ """
+ Pick a message.
+ """
+ return proto.fetchSpecific(
+ '%s' % result,
+ headerType='',
+ headerArgs=['BODY.PEEK[]'],
+ ).addCallback(cbCompareMessage, proto)
+
+
+def cbCompareMessage(result, proto, raw):
+ """
+ Display message and compare it with the original one.
+ """
+ parts_orig = get_msg_parts(raw)
+
+ if result:
+ keys = result.keys()
+ keys.sort()
+ else:
+ print "[-] GOT NO RESULT"
+ return proto.logout()
+
+ latest = max(keys)
+
+ fetched_msg = result[latest][0][2]
+ parts_fetched = get_msg_parts(fetched_msg)
+
+ equal = compare_msg_parts(
+ parts_orig,
+ parts_fetched)
+
+ if equal:
+ print "[+] MESSAGES MATCH"
+ return cbAppendNextMessage(proto)
+ else:
+ print "[-] ERROR: MESSAGES DO NOT MATCH !!!"
+ print " ABORTING COMPARISON..."
+ # FIXME logout and print the subject ...
+ return proto.logout()
+
+
+def cbClose(result):
+ """
+ Close the connection when we finish everything.
+ """
+ from twisted.internet import reactor
+ reactor.stop()
+
+
+def main():
+ import glob
+ import sys
+
+ if len(sys.argv) != 4:
+ print "Usage: regressions <user> <pass> <samples-folder>"
+ sys.exit()
+
+ hostname = "localhost"
+ port = "1984"
+ username = sys.argv[1]
+ password = sys.argv[2]
+
+ samplesdir = sys.argv[3]
+
+ if not os.path.isdir(samplesdir):
+ print ("Could not find samples folder! "
+ "Make sure of copying mail_breaker contents there.")
+ sys.exit()
+
+ samples = glob.glob(samplesdir + '/*')
+
+ global SAMPLES
+ SAMPLES = []
+ SAMPLES += samples
+
+ onConn = defer.Deferred(
+ ).addCallback(
+ cbServerGreeting, username, password
+ ).addErrback(
+ ebConnection
+ ).addBoth(cbClose)
+
+ factory = SimpleIMAP4ClientFactory(username, onConn)
+
+ from twisted.internet import reactor
+ reactor.connectTCP(hostname, int(port), factory)
+ reactor.run()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/src/leap/bitmask/mail/imap/tests/rfc822.message b/src/leap/bitmask/mail/imap/tests/rfc822.message
new file mode 120000
index 00000000..b19cc280
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/tests/rfc822.message
@@ -0,0 +1 @@
+../../tests/rfc822.message \ No newline at end of file
diff --git a/src/leap/bitmask/mail/imap/tests/rfc822.multi-minimal.message b/src/leap/bitmask/mail/imap/tests/rfc822.multi-minimal.message
new file mode 120000
index 00000000..e0aa678b
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/tests/rfc822.multi-minimal.message
@@ -0,0 +1 @@
+../../tests/rfc822.multi-minimal.message \ No newline at end of file
diff --git a/src/leap/bitmask/mail/imap/tests/rfc822.multi-nested.message b/src/leap/bitmask/mail/imap/tests/rfc822.multi-nested.message
new file mode 120000
index 00000000..306d0dec
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/tests/rfc822.multi-nested.message
@@ -0,0 +1 @@
+../../tests/rfc822.multi-nested.message \ No newline at end of file
diff --git a/src/leap/bitmask/mail/imap/tests/rfc822.multi-signed.message b/src/leap/bitmask/mail/imap/tests/rfc822.multi-signed.message
new file mode 120000
index 00000000..4172244e
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/tests/rfc822.multi-signed.message
@@ -0,0 +1 @@
+../../tests/rfc822.multi-signed.message \ No newline at end of file
diff --git a/src/leap/bitmask/mail/imap/tests/rfc822.multi.message b/src/leap/bitmask/mail/imap/tests/rfc822.multi.message
new file mode 120000
index 00000000..62057d20
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/tests/rfc822.multi.message
@@ -0,0 +1 @@
+../../tests/rfc822.multi.message \ No newline at end of file
diff --git a/src/leap/bitmask/mail/imap/tests/rfc822.plain.message b/src/leap/bitmask/mail/imap/tests/rfc822.plain.message
new file mode 120000
index 00000000..5bab0e8d
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/tests/rfc822.plain.message
@@ -0,0 +1 @@
+../../tests/rfc822.plain.message \ No newline at end of file
diff --git a/src/leap/bitmask/mail/imap/tests/stress_tests_imap.zsh b/src/leap/bitmask/mail/imap/tests/stress_tests_imap.zsh
new file mode 100755
index 00000000..544facaa
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/tests/stress_tests_imap.zsh
@@ -0,0 +1,178 @@
+#!/bin/zsh
+# BATCH STRESS TEST FOR IMAP ----------------------
+# http://imgs.xkcd.com/comics/science.jpg
+#
+# Run imaptest against a LEAP IMAP server
+# for a fixed period of time, and collect output.
+#
+# Author: Kali Kaneko
+# Date: 2014 01 26
+#
+# To run, you need to have `imaptest` in your path.
+# See:
+# http://www.imapwiki.org/ImapTest/Installation
+#
+# For the tests, I'm using a 10MB file sample that
+# can be downloaded from:
+# http://www.dovecot.org/tmp/dovecot-crlf
+#
+# Want to contribute to benchmarking?
+#
+# 1. Create a pristine account in a bitmask provider.
+#
+# 2. Launch your bitmask client, with different flags
+# if you desire.
+#
+# For example to try the nosync flag in sqlite:
+#
+# LEAP_SQLITE_NOSYNC=1 bitmask --debug -N --offline -l /tmp/leap.log
+#
+# 3. Run at several points in time (ie: just after
+# launching the bitmask client. one minute after,
+# ten minutes after)
+#
+# mkdir data
+# cd data
+# ../leap_tests_imap.zsh | tee sqlite_nosync_run2.log
+#
+# 4. Submit your results to: kali at leap dot se
+# together with the logs of the bitmask run.
+#
+# Please provide also details about your system, and
+# the type of hard disk setup you are running against.
+#
+
+# ------------------------------------------------
+# Edit these variables if you are too lazy to pass
+# the user and mbox as parameters. Like me.
+
+USER="test_f14@dev.bitmask.net"
+MBOX="~/leap/imaptest/data/dovecot-crlf"
+
+HOST="localhost"
+PORT="1984"
+
+# in case you have it aliased
+GREP="/bin/grep"
+IMAPTEST="imaptest"
+
+# -----------------------------------------------
+#
+# These should be kept constant across benchmarking
+# runs across different machines, for comparability.
+
+DURATION=200
+NUM_MSG=200
+
+
+# TODO add another function, and a cli flag, to be able
+# to take several aggretates spaced in time, along a period
+# of several minutes.
+
+imaptest_cmd() {
+ stdbuf -o0 ${IMAPTEST} user=${USER} pass=1234 host=${HOST} \
+ port=${PORT} mbox=${MBOX} clients=1 msgs=${NUM_MSG} \
+ no_pipelining 2>/dev/null
+}
+
+stress_imap() {
+ mkfifo imap_pipe
+ cat imap_pipe | tee output &
+ imaptest_cmd >> imap_pipe
+}
+
+wait_and_kill() {
+ while :
+ do
+ sleep $DURATION
+ pkill -2 imaptest
+ rm imap_pipe
+ break
+ done
+}
+
+print_results() {
+ sleep 1
+ echo
+ echo
+ echo "AGGREGATED RESULTS"
+ echo "----------------------"
+ echo "\tavg\tstdev"
+ $GREP "avg" ./output | sed -e 's/^ *//g' -e 's/ *$//g' | \
+ gawk '
+function avg(data, count) {
+ sum=0;
+ for( x=0; x <= count-1; x++) {
+ sum += data[x];
+ }
+ return sum/count;
+}
+function std_dev(data, count) {
+ sum=0;
+ for( x=0; x <= count-1; x++) {
+ sum += data[x];
+ }
+ average = sum/count;
+
+ sumsq=0;
+ for( x=0; x <= count-1; x++) {
+ sumsq += (data[x] - average)^2;
+ }
+ return sqrt(sumsq/count);
+}
+BEGIN {
+ cnt = 0
+} END {
+
+printf("LOGI:\t%04.2lf\t%04.2f\n", avg(array[1], NR), std_dev(array[1], NR));
+printf("LIST:\t%04.2lf\t%04.2f\n", avg(array[2], NR), std_dev(array[2], NR));
+printf("STAT:\t%04.2lf\t%04.2f\n", avg(array[3], NR), std_dev(array[3], NR));
+printf("SELE:\t%04.2lf\t%04.2f\n", avg(array[4], NR), std_dev(array[4], NR));
+printf("FETC:\t%04.2lf\t%04.2f\n", avg(array[5], NR), std_dev(array[5], NR));
+printf("FET2:\t%04.2lf\t%04.2f\n", avg(array[6], NR), std_dev(array[6], NR));
+printf("STOR:\t%04.2lf\t%04.2f\n", avg(array[7], NR), std_dev(array[7], NR));
+printf("DELE:\t%04.2lf\t%04.2f\n", avg(array[8], NR), std_dev(array[8], NR));
+printf("EXPU:\t%04.2lf\t%04.2f\n", avg(array[9], NR), std_dev(array[9], NR));
+printf("APPE:\t%04.2lf\t%04.2f\n", avg(array[10], NR), std_dev(array[10], NR));
+printf("LOGO:\t%04.2lf\t%04.2f\n", avg(array[11], NR), std_dev(array[11], NR));
+
+print ""
+print "TOT samples", NR;
+}
+{
+ it = cnt++;
+ array[1][it] = $1;
+ array[2][it] = $2;
+ array[3][it] = $3;
+ array[4][it] = $4;
+ array[5][it] = $5;
+ array[6][it] = $6;
+ array[7][it] = $7;
+ array[8][it] = $8;
+ array[9][it] = $9;
+ array[10][it] = $10;
+ array[11][it] = $11;
+}'
+}
+
+
+{ test $1 = "--help" } && {
+ echo "Usage: $0 [user@provider] [/path/to/sample.mbox]"
+ exit 0
+}
+
+# If the first parameter is passed, take it as the user
+{ test $1 } && {
+ USER=$1
+}
+
+# If the second parameter is passed, take it as the mbox
+{ test $2 } && {
+ MBOX=$2
+}
+
+echo "[+] LEAP IMAP TESTS"
+echo "[+] Running imaptest for $DURATION seconds with $NUM_MSG messages"
+wait_and_kill &
+stress_imap
+print_results
diff --git a/src/leap/bitmask/mail/imap/tests/test_imap.py b/src/leap/bitmask/mail/imap/tests/test_imap.py
new file mode 100644
index 00000000..9cca17ff
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/tests/test_imap.py
@@ -0,0 +1,1060 @@
+# -*- coding: utf-8 -*-
+# test_imap.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Test case for leap.email.imap.server
+TestCases taken from twisted tests and modified to make them work
+against our implementation of the IMAPAccount.
+
+@authors: Kali Kaneko, <kali@leap.se>
+XXX add authors from the original twisted tests.
+
+@license: GPLv3, see included LICENSE file
+"""
+# XXX review license of the original tests!!!
+import os
+import string
+import types
+
+
+from twisted.mail import imap4
+from twisted.internet import defer
+from twisted.python import util
+from twisted.python import failure
+
+from twisted import cred
+
+from leap.mail.imap.mailbox import IMAPMailbox
+from leap.mail.imap.messages import CaseInsensitiveDict
+from leap.mail.testing.imap import IMAP4HelperMixin
+
+
+TEST_USER = "testuser@leap.se"
+TEST_PASSWD = "1234"
+
+
+def strip(f):
+ return lambda result, f=f: f()
+
+
+def sortNest(l):
+ l = l[:]
+ l.sort()
+ for i in range(len(l)):
+ if isinstance(l[i], types.ListType):
+ l[i] = sortNest(l[i])
+ elif isinstance(l[i], types.TupleType):
+ l[i] = tuple(sortNest(list(l[i])))
+ return l
+
+
+class TestRealm:
+ """
+ A minimal auth realm for testing purposes only
+ """
+ theAccount = None
+
+ def requestAvatar(self, avatarId, mind, *interfaces):
+ return imap4.IAccount, self.theAccount, lambda: None
+
+#
+# TestCases
+#
+
+# DEBUG ---
+# from twisted.internet.base import DelayedCall
+# DelayedCall.debug = True
+
+
+class LEAPIMAP4ServerTestCase(IMAP4HelperMixin):
+
+ """
+ Tests for the generic behavior of the LEAPIMAP4Server
+ which, right now, it's just implemented in this test file as
+ LEAPIMAPServer. We will move the implementation, together with
+ authentication bits, to leap.mail.imap.server so it can be instantiated
+ from the tac file.
+
+ Right now this TestCase tries to mimmick as close as possible the
+ organization from the twisted.mail.imap tests so we can achieve
+ a complete implementation. The order in which they appear reflect
+ the intended order of implementation.
+ """
+
+ #
+ # mailboxes operations
+ #
+
+ def testCreate(self):
+ """
+ Test whether we can create mailboxes
+ """
+ succeed = ('testbox', 'test/box', 'test/', 'test/box/box', 'foobox')
+ fail = ('testbox', 'test/box')
+ acc = self.server.theAccount
+
+ def cb():
+ self.result.append(1)
+
+ def eb(failure):
+ self.result.append(0)
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def create():
+ create_deferreds = []
+ for name in succeed + fail:
+ d = self.client.create(name)
+ d.addCallback(strip(cb)).addErrback(eb)
+ create_deferreds.append(d)
+ dd = defer.gatherResults(create_deferreds)
+ dd.addCallbacks(self._cbStopClient, self._ebGeneral)
+ return dd
+
+ self.result = []
+ d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(create))
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2], consumeErrors=True)
+ d.addCallback(lambda _: acc.account.list_all_mailbox_names())
+ return d.addCallback(self._cbTestCreate, succeed, fail)
+
+ def _cbTestCreate(self, mailboxes, succeed, fail):
+ self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail))
+
+ answers = ([u'INBOX', u'testbox', u'test/box', u'test',
+ u'test/box/box', 'foobox'])
+ self.assertEqual(sorted(mailboxes), sorted([a for a in answers]))
+
+ def testDelete(self):
+ """
+ Test whether we can delete mailboxes
+ """
+ def add_mailbox():
+ return self.server.theAccount.addMailbox('test-delete/me')
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def delete():
+ return self.client.delete('test-delete/me')
+
+ acc = self.server.theAccount.account
+
+ d1 = self.connected.addCallback(add_mailbox)
+ d1.addCallback(strip(login))
+ d1.addCallbacks(strip(delete), self._ebGeneral)
+ d1.addCallbacks(self._cbStopClient, self._ebGeneral)
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(lambda _: acc.list_all_mailbox_names())
+ d.addCallback(lambda mboxes: self.assertEqual(
+ mboxes, ['INBOX']))
+ return d
+
+ def testIllegalInboxDelete(self):
+ """
+ Test what happens if we try to delete the user Inbox.
+ We expect that operation to fail.
+ """
+ self.stashed = None
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def delete():
+ return self.client.delete('inbox')
+
+ def stash(result):
+ self.stashed = result
+
+ d1 = self.connected.addCallback(strip(login))
+ d1.addCallbacks(strip(delete), self._ebGeneral)
+ d1.addBoth(stash)
+ d1.addCallbacks(self._cbStopClient, self._ebGeneral)
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(lambda _: self.failUnless(isinstance(self.stashed,
+ failure.Failure)))
+ return d
+
+ def testNonExistentDelete(self):
+ """
+ Test what happens if we try to delete a non-existent mailbox.
+ We expect an error raised stating 'No such mailbox'
+ """
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def delete():
+ return self.client.delete('delete/me')
+ self.failure = failure
+
+ def deleteFailed(failure):
+ self.failure = failure
+
+ self.failure = None
+ d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(delete)).addErrback(deleteFailed)
+ d1.addCallbacks(self._cbStopClient, self._ebGeneral)
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(lambda _: self.assertTrue(
+ str(self.failure.value).startswith('No such mailbox')))
+ return d
+
+ def testIllegalDelete(self):
+ """
+ Try deleting a mailbox with sub-folders, and \NoSelect flag set.
+ An exception is expected.
+ """
+ acc = self.server.theAccount
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def create_mailboxes():
+ d1 = acc.addMailbox('delete')
+ d2 = acc.addMailbox('delete/me')
+ d = defer.gatherResults([d1, d2])
+ return d
+
+ def get_noselect_mailbox(mboxes):
+ mbox = mboxes[0]
+ return mbox.setFlags((r'\Noselect',))
+
+ def delete_mbox(ignored):
+ return self.client.delete('delete')
+
+ def deleteFailed(failure):
+ self.failure = failure
+
+ self.failure = None
+
+ d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(create_mailboxes))
+ d1.addCallback(get_noselect_mailbox)
+
+ d1.addCallback(delete_mbox).addErrback(deleteFailed)
+ d1.addCallbacks(self._cbStopClient, self._ebGeneral)
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2])
+ expected = ("Hierarchically inferior mailboxes exist "
+ "and \\Noselect is set")
+ d.addCallback(lambda _:
+ self.assertTrue(self.failure is not None))
+ d.addCallback(lambda _:
+ self.assertEqual(str(self.failure.value), expected))
+ return d
+
+ # FIXME --- this test sometimes FAILS (timing issue).
+ # Some of the deferreds used in the rename op is not waiting for the
+ # operations properly
+ def testRename(self):
+ """
+ Test whether we can rename a mailbox
+ """
+ def create_mbox():
+ return self.server.theAccount.addMailbox('oldmbox')
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def rename():
+ return self.client.rename('oldmbox', 'newname')
+
+ d1 = self.connected.addCallback(strip(create_mbox))
+ d1.addCallback(strip(login))
+ d1.addCallbacks(strip(rename), self._ebGeneral)
+ d1.addCallbacks(self._cbStopClient, self._ebGeneral)
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(lambda _:
+ self.server.theAccount.account.list_all_mailbox_names())
+ d.addCallback(lambda mboxes:
+ self.assertItemsEqual(mboxes, ['INBOX', 'newname']))
+ return d
+
+ def testIllegalInboxRename(self):
+ """
+ Try to rename inbox. We expect it to fail. Then it would be not
+ an inbox anymore, would it?
+ """
+ self.stashed = None
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def rename():
+ return self.client.rename('inbox', 'frotz')
+
+ def stash(stuff):
+ self.stashed = stuff
+
+ d1 = self.connected.addCallback(strip(login))
+ d1.addCallbacks(strip(rename), self._ebGeneral)
+ d1.addBoth(stash)
+ d1.addCallbacks(self._cbStopClient, self._ebGeneral)
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(lambda _:
+ self.failUnless(isinstance(
+ self.stashed, failure.Failure)))
+ return d
+
+ def testHierarchicalRename(self):
+ """
+ Try to rename hierarchical mailboxes
+ """
+ acc = self.server.theAccount
+
+ def add_mailboxes():
+ return defer.gatherResults([
+ acc.addMailbox('oldmbox/m1'),
+ acc.addMailbox('oldmbox/m2')])
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def rename():
+ return self.client.rename('oldmbox', 'newname')
+
+ d1 = self.connected.addCallback(strip(add_mailboxes))
+ d1.addCallback(strip(login))
+ d1.addCallbacks(strip(rename), self._ebGeneral)
+ d1.addCallbacks(self._cbStopClient, self._ebGeneral)
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(lambda _: acc.account.list_all_mailbox_names())
+ return d.addCallback(self._cbTestHierarchicalRename)
+
+ def _cbTestHierarchicalRename(self, mailboxes):
+ expected = ['INBOX', 'newname/m1', 'newname/m2']
+ self.assertEqual(sorted(mailboxes), sorted([s for s in expected]))
+
+ def testSubscribe(self):
+ """
+ Test whether we can mark a mailbox as subscribed to
+ """
+ acc = self.server.theAccount
+
+ def add_mailbox():
+ return acc.addMailbox('this/mbox')
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def subscribe():
+ return self.client.subscribe('this/mbox')
+
+ def get_subscriptions(ignored):
+ return self.server.theAccount.getSubscriptions()
+
+ d1 = self.connected.addCallback(strip(add_mailbox))
+ d1.addCallback(strip(login))
+ d1.addCallbacks(strip(subscribe), self._ebGeneral)
+ d1.addCallbacks(self._cbStopClient, self._ebGeneral)
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(get_subscriptions)
+ d.addCallback(lambda subscriptions:
+ self.assertEqual(subscriptions,
+ ['this/mbox']))
+ return d
+
+ def testUnsubscribe(self):
+ """
+ Test whether we can unsubscribe from a set of mailboxes
+ """
+ acc = self.server.theAccount
+
+ def add_mailboxes():
+ return defer.gatherResults([
+ acc.addMailbox('this/mbox'),
+ acc.addMailbox('that/mbox')])
+
+ def dc1():
+ return acc.subscribe('this/mbox')
+
+ def dc2():
+ return acc.subscribe('that/mbox')
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def unsubscribe():
+ return self.client.unsubscribe('this/mbox')
+
+ def get_subscriptions(ignored):
+ return acc.getSubscriptions()
+
+ d1 = self.connected.addCallback(strip(add_mailboxes))
+ d1.addCallback(strip(login))
+ d1.addCallback(strip(dc1))
+ d1.addCallback(strip(dc2))
+ d1.addCallbacks(strip(unsubscribe), self._ebGeneral)
+ d1.addCallbacks(self._cbStopClient, self._ebGeneral)
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(get_subscriptions)
+ d.addCallback(lambda subscriptions:
+ self.assertEqual(subscriptions,
+ ['that/mbox']))
+ return d
+
+ def testSelect(self):
+ """
+ Try to select a mailbox
+ """
+ mbox_name = "TESTMAILBOXSELECT"
+ self.selectedArgs = None
+
+ acc = self.server.theAccount
+
+ def add_mailbox():
+ return acc.addMailbox(mbox_name, creation_ts=42)
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def select():
+ def selected(args):
+ self.selectedArgs = args
+ self._cbStopClient(None)
+ d = self.client.select(mbox_name)
+ d.addCallback(selected)
+ return d
+
+ d1 = self.connected.addCallback(strip(add_mailbox))
+ d1.addCallback(strip(login))
+ d1.addCallback(strip(select))
+ # d1.addErrback(self._ebGeneral)
+
+ d2 = self.loopback()
+
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(self._cbTestSelect)
+ return d
+
+ def _cbTestSelect(self, ignored):
+ self.assertTrue(self.selectedArgs is not None)
+
+ self.assertEqual(self.selectedArgs, {
+ 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42,
+ 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged',
+ '\\Deleted', '\\Draft', '\\Recent', 'List'),
+ 'READ-WRITE': True
+ })
+
+ #
+ # capabilities
+ #
+
+ def testCapability(self):
+ caps = {}
+
+ def getCaps():
+ def gotCaps(c):
+ caps.update(c)
+ self.server.transport.loseConnection()
+ return self.client.getCapabilities().addCallback(gotCaps)
+
+ d1 = self.connected
+ d1.addCallback(
+ strip(getCaps)).addErrback(self._ebGeneral)
+
+ d = defer.gatherResults([self.loopback(), d1])
+ expected = {'IMAP4rev1': None, 'NAMESPACE': None, 'LITERAL+': None,
+ 'IDLE': None}
+ d.addCallback(lambda _: self.assertEqual(expected, caps))
+ return d
+
+ def testCapabilityWithAuth(self):
+ caps = {}
+ self.server.challengers[
+ 'CRAM-MD5'] = cred.credentials.CramMD5Credentials
+
+ def getCaps():
+ def gotCaps(c):
+ caps.update(c)
+ self.server.transport.loseConnection()
+ return self.client.getCapabilities().addCallback(gotCaps)
+ d1 = self.connected.addCallback(
+ strip(getCaps)).addErrback(self._ebGeneral)
+
+ d = defer.gatherResults([self.loopback(), d1])
+
+ expCap = {'IMAP4rev1': None, 'NAMESPACE': None,
+ 'IDLE': None, 'LITERAL+': None,
+ 'AUTH': ['CRAM-MD5']}
+
+ d.addCallback(lambda _: self.assertEqual(expCap, caps))
+ return d
+
+ #
+ # authentication
+ #
+
+ def testLogout(self):
+ """
+ Test log out
+ """
+ self.loggedOut = 0
+
+ def logout():
+ def setLoggedOut():
+ self.loggedOut = 1
+ self.client.logout().addCallback(strip(setLoggedOut))
+ self.connected.addCallback(strip(logout)).addErrback(self._ebGeneral)
+ d = self.loopback()
+ return d.addCallback(lambda _: self.assertEqual(self.loggedOut, 1))
+
+ def testNoop(self):
+ """
+ Test noop command
+ """
+ self.responses = None
+
+ def noop():
+ def setResponses(responses):
+ self.responses = responses
+ self.server.transport.loseConnection()
+ self.client.noop().addCallback(setResponses)
+ self.connected.addCallback(strip(noop)).addErrback(self._ebGeneral)
+ d = self.loopback()
+ return d.addCallback(lambda _: self.assertEqual(self.responses, []))
+
+ def testLogin(self):
+ """
+ Test login
+ """
+ def login():
+ d = self.client.login(TEST_USER, TEST_PASSWD)
+ d.addCallback(self._cbStopClient)
+ d1 = self.connected.addCallback(
+ strip(login)).addErrback(self._ebGeneral)
+ d = defer.gatherResults([d1, self.loopback()])
+ return d.addCallback(self._cbTestLogin)
+
+ def _cbTestLogin(self, ignored):
+ self.assertEqual(self.server.state, 'auth')
+
+ def testFailedLogin(self):
+ """
+ Test bad login
+ """
+ def login():
+ d = self.client.login("bad_user@leap.se", TEST_PASSWD)
+ d.addBoth(self._cbStopClient)
+
+ d1 = self.connected.addCallback(
+ strip(login)).addErrback(self._ebGeneral)
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2])
+ return d.addCallback(self._cbTestFailedLogin)
+
+ def _cbTestFailedLogin(self, ignored):
+ self.assertEqual(self.server.state, 'unauth')
+ self.assertEqual(self.server.account, None)
+
+ def testLoginRequiringQuoting(self):
+ """
+ Test login requiring quoting
+ """
+ self.server.checker.userid = '{test}user@leap.se'
+ self.server.checker.password = '{test}password'
+
+ def login():
+ d = self.client.login('{test}user@leap.se', '{test}password')
+ d.addBoth(self._cbStopClient)
+
+ d1 = self.connected.addCallback(
+ strip(login)).addErrback(self._ebGeneral)
+ d = defer.gatherResults([self.loopback(), d1])
+ return d.addCallback(self._cbTestLoginRequiringQuoting)
+
+ def _cbTestLoginRequiringQuoting(self, ignored):
+ self.assertEqual(self.server.state, 'auth')
+
+ #
+ # Inspection
+ #
+
+ def testNamespace(self):
+ """
+ Test retrieving namespace
+ """
+ self.namespaceArgs = None
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def namespace():
+ def gotNamespace(args):
+ self.namespaceArgs = args
+ self._cbStopClient(None)
+ return self.client.namespace().addCallback(gotNamespace)
+
+ d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(namespace))
+ d1.addErrback(self._ebGeneral)
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(lambda _: self.assertEqual(self.namespaceArgs,
+ [[['', '/']], [], []]))
+ return d
+
+ def testExamine(self):
+ """
+ L{IMAP4Client.examine} issues an I{EXAMINE} command to the server and
+ returns a L{Deferred} which fires with a C{dict} with as many of the
+ following keys as the server includes in its response: C{'FLAGS'},
+ C{'EXISTS'}, C{'RECENT'}, C{'UNSEEN'}, C{'READ-WRITE'}, C{'READ-ONLY'},
+ C{'UIDVALIDITY'}, and C{'PERMANENTFLAGS'}.
+
+ Unfortunately the server doesn't generate all of these so it's hard to
+ test the client's handling of them here. See
+ L{IMAP4ClientExamineTests} below.
+
+ See U{RFC 3501<http://www.faqs.org/rfcs/rfc3501.html>}, section 6.3.2,
+ for details.
+ """
+ # TODO implement the IMAP4ClientExamineTests testcase.
+ mbox_name = "test_mailbox_e"
+ acc = self.server.theAccount
+ self.examinedArgs = None
+
+ def add_mailbox():
+ return acc.addMailbox(mbox_name, creation_ts=42)
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def examine():
+ def examined(args):
+ self.examinedArgs = args
+ self._cbStopClient(None)
+ d = self.client.examine(mbox_name)
+ d.addCallback(examined)
+ return d
+
+ d1 = self.connected.addCallback(strip(add_mailbox))
+ d1.addCallback(strip(login))
+ d1.addCallback(strip(examine))
+ d1.addErrback(self._ebGeneral)
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2])
+ return d.addCallback(self._cbTestExamine)
+
+ def _cbTestExamine(self, ignored):
+ self.assertEqual(self.examinedArgs, {
+ 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42,
+ 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged',
+ '\\Deleted', '\\Draft', '\\Recent', 'List'),
+ 'READ-WRITE': False})
+
+ def _listSetup(self, f, f2=None):
+
+ acc = self.server.theAccount
+
+ def dc1():
+ return acc.addMailbox('root_subthing', creation_ts=42)
+
+ def dc2():
+ return acc.addMailbox('root_another_thing', creation_ts=42)
+
+ def dc3():
+ return acc.addMailbox('non_root_subthing', creation_ts=42)
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def listed(answers):
+ self.listed = answers
+
+ self.listed = None
+ d1 = self.connected.addCallback(strip(login))
+ d1.addCallback(strip(dc1))
+ d1.addCallback(strip(dc2))
+ d1.addCallback(strip(dc3))
+
+ if f2 is not None:
+ d1.addCallback(f2)
+
+ d1.addCallbacks(strip(f), self._ebGeneral)
+ d1.addCallbacks(listed, self._ebGeneral)
+ d1.addCallbacks(self._cbStopClient, self._ebGeneral)
+ d2 = self.loopback()
+ return defer.gatherResults([d1, d2]).addCallback(lambda _: self.listed)
+
+ def testList(self):
+ """
+ Test List command
+ """
+ def list():
+ return self.client.list('root', '%')
+
+ d = self._listSetup(list)
+ d.addCallback(lambda listed: self.assertEqual(
+ sortNest(listed),
+ sortNest([
+ (IMAPMailbox.init_flags, "/", "root_subthing"),
+ (IMAPMailbox.init_flags, "/", "root_another_thing")
+ ])
+ ))
+ return d
+
+ def testLSub(self):
+ """
+ Test LSub command
+ """
+ acc = self.server.theAccount
+
+ def subs_mailbox():
+ # why not client.subscribe instead?
+ return acc.subscribe('root_subthing')
+
+ def lsub():
+ return self.client.lsub('root', '%')
+
+ d = self._listSetup(lsub, strip(subs_mailbox))
+ d.addCallback(self.assertEqual,
+ [(IMAPMailbox.init_flags, "/", "root_subthing")])
+ return d
+
+ def testStatus(self):
+ """
+ Test Status command
+ """
+ acc = self.server.theAccount
+
+ def add_mailbox():
+ return acc.addMailbox('root_subthings')
+
+ # XXX FIXME ---- should populate this a little bit,
+ # with unseen etc...
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def status():
+ return self.client.status(
+ 'root_subthings', 'MESSAGES', 'UIDNEXT', 'UNSEEN')
+
+ def statused(result):
+ self.statused = result
+
+ self.statused = None
+
+ d1 = self.connected.addCallback(strip(add_mailbox))
+ d1.addCallback(strip(login))
+ d1.addCallbacks(strip(status), self._ebGeneral)
+ d1.addCallbacks(statused, self._ebGeneral)
+ d1.addCallbacks(self._cbStopClient, self._ebGeneral)
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(lambda _: self.assertEqual(
+ self.statused,
+ {'MESSAGES': 0, 'UIDNEXT': '1', 'UNSEEN': 0}
+ ))
+ return d
+
+ def testFailedStatus(self):
+ """
+ Test failed status command with a non-existent mailbox
+ """
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def status():
+ return self.client.status(
+ 'root/nonexistent', 'MESSAGES', 'UIDNEXT', 'UNSEEN')
+
+ def statused(result):
+ self.statused = result
+
+ def failed(failure):
+ self.failure = failure
+
+ self.statused = self.failure = None
+ d1 = self.connected.addCallback(strip(login))
+ d1.addCallbacks(strip(status), self._ebGeneral)
+ d1.addCallbacks(statused, failed)
+ d1.addCallbacks(self._cbStopClient, self._ebGeneral)
+ d2 = self.loopback()
+ return defer.gatherResults([d1, d2]).addCallback(
+ self._cbTestFailedStatus)
+
+ def _cbTestFailedStatus(self, ignored):
+ self.assertEqual(
+ self.statused, None
+ )
+ self.assertEqual(
+ self.failure.value.args,
+ ('Could not open mailbox',)
+ )
+
+ #
+ # messages
+ #
+
+ def testFullAppend(self):
+ """
+ Test appending a full message to the mailbox
+ """
+ infile = util.sibpath(__file__, 'rfc822.message')
+ message = open(infile)
+ acc = self.server.theAccount
+ mailbox_name = "appendmbox/subthing"
+
+ def add_mailbox():
+ return acc.addMailbox(mailbox_name)
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def append():
+ return self.client.append(
+ mailbox_name, message,
+ ('\\SEEN', '\\DELETED'),
+ 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)',
+ )
+
+ d1 = self.connected.addCallback(strip(add_mailbox))
+ d1.addCallback(strip(login))
+ d1.addCallbacks(strip(append), self._ebGeneral)
+ d1.addCallbacks(self._cbStopClient, self._ebGeneral)
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2])
+
+ d.addCallback(lambda _: acc.getMailbox(mailbox_name))
+ d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True))
+ return d.addCallback(self._cbTestFullAppend, infile)
+
+ def _cbTestFullAppend(self, fetched, infile):
+ fetched = list(fetched)
+ self.assertTrue(len(fetched) == 1)
+ self.assertTrue(len(fetched[0]) == 2)
+ uid, msg = fetched[0]
+ parsed = self.parser.parse(open(infile))
+ expected_body = parsed.get_payload()
+ expected_headers = CaseInsensitiveDict(parsed.items())
+
+ def assert_flags(flags):
+ self.assertEqual(
+ set(('\\SEEN', '\\DELETED')),
+ set(flags))
+
+ def assert_date(date):
+ self.assertEqual(
+ 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)',
+ date)
+
+ def assert_body(body):
+ gotbody = body.read()
+ self.assertEqual(expected_body, gotbody)
+
+ def assert_headers(headers):
+ self.assertItemsEqual(map(string.lower, expected_headers), headers)
+
+ d = defer.maybeDeferred(msg.getFlags)
+ d.addCallback(assert_flags)
+
+ d.addCallback(lambda _: defer.maybeDeferred(msg.getInternalDate))
+ d.addCallback(assert_date)
+
+ d.addCallback(
+ lambda _: defer.maybeDeferred(
+ msg.getBodyFile, self._soledad))
+ d.addCallback(assert_body)
+
+ d.addCallback(lambda _: defer.maybeDeferred(msg.getHeaders, True))
+ d.addCallback(assert_headers)
+
+ return d
+
+ def testPartialAppend(self):
+ """
+ Test partially appending a message to the mailbox
+ """
+ # TODO this test sometimes will fail because of the notify_just_mdoc
+ infile = util.sibpath(__file__, 'rfc822.message')
+
+ acc = self.server.theAccount
+
+ def add_mailbox():
+ return acc.addMailbox('PARTIAL/SUBTHING')
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def append():
+ message = file(infile)
+ return self.client.sendCommand(
+ imap4.Command(
+ 'APPEND',
+ 'PARTIAL/SUBTHING (\\SEEN) "Right now" '
+ '{%d}' % os.path.getsize(infile),
+ (), self.client._IMAP4Client__cbContinueAppend, message
+ )
+ )
+ d1 = self.connected.addCallback(strip(add_mailbox))
+ d1.addCallback(strip(login))
+ d1.addCallbacks(strip(append), self._ebGeneral)
+ d1.addCallbacks(self._cbStopClient, self._ebGeneral)
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2])
+
+ d.addCallback(lambda _: acc.getMailbox("PARTIAL/SUBTHING"))
+ d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True))
+ return d.addCallback(
+ self._cbTestPartialAppend, infile)
+
+ def _cbTestPartialAppend(self, fetched, infile):
+ fetched = list(fetched)
+ self.assertTrue(len(fetched) == 1)
+ self.assertTrue(len(fetched[0]) == 2)
+ uid, msg = fetched[0]
+ parsed = self.parser.parse(open(infile))
+ expected_body = parsed.get_payload()
+
+ def assert_flags(flags):
+ self.assertEqual(
+ set((['\\SEEN'])), set(flags))
+
+ def assert_body(body):
+ gotbody = body.read()
+ self.assertEqual(expected_body, gotbody)
+
+ d = defer.maybeDeferred(msg.getFlags)
+ d.addCallback(assert_flags)
+
+ d.addCallback(lambda _: defer.maybeDeferred(msg.getBodyFile))
+ d.addCallback(assert_body)
+ return d
+
+ def testCheck(self):
+ """
+ Test check command
+ """
+ def add_mailbox():
+ return self.server.theAccount.addMailbox('root/subthing')
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def select():
+ return self.client.select('root/subthing')
+
+ def check():
+ return self.client.check()
+
+ d = self.connected.addCallbacks(
+ strip(add_mailbox), self._ebGeneral)
+ d.addCallbacks(lambda _: login(), self._ebGeneral)
+ d.addCallbacks(strip(select), self._ebGeneral)
+ d.addCallbacks(strip(check), self._ebGeneral)
+ d.addCallbacks(self._cbStopClient, self._ebGeneral)
+ d2 = self.loopback()
+ return defer.gatherResults([d, d2])
+
+ # Okay, that was much fun indeed
+
+ def testExpunge(self):
+ """
+ Test expunge command
+ """
+ acc = self.server.theAccount
+ mailbox_name = 'mailboxexpunge'
+
+ def add_mailbox():
+ return acc.addMailbox(mailbox_name)
+
+ def login():
+ return self.client.login(TEST_USER, TEST_PASSWD)
+
+ def select():
+ return self.client.select(mailbox_name)
+
+ def save_mailbox(mailbox):
+ self.mailbox = mailbox
+
+ def get_mailbox():
+ d = acc.getMailbox(mailbox_name)
+ d.addCallback(save_mailbox)
+ return d
+
+ def add_messages():
+ d = self.mailbox.addMessage(
+ 'test 1', flags=('\\Deleted', 'AnotherFlag'),
+ notify_just_mdoc=False)
+ d.addCallback(lambda _: self.mailbox.addMessage(
+ 'test 2', flags=('AnotherFlag',),
+ notify_just_mdoc=False))
+ d.addCallback(lambda _: self.mailbox.addMessage(
+ 'test 3', flags=('\\Deleted',),
+ notify_just_mdoc=False))
+ return d
+
+ def expunge():
+ return self.client.expunge()
+
+ def expunged(results):
+ self.failIf(self.server.mbox is None)
+ self.results = results
+
+ self.results = None
+ d1 = self.connected.addCallback(strip(add_mailbox))
+ d1.addCallback(strip(login))
+ d1.addCallback(strip(get_mailbox))
+ d1.addCallbacks(strip(add_messages), self._ebGeneral)
+ d1.addCallbacks(strip(select), self._ebGeneral)
+ d1.addCallbacks(strip(expunge), self._ebGeneral)
+ d1.addCallbacks(expunged, self._ebGeneral)
+ d1.addCallbacks(self._cbStopClient, self._ebGeneral)
+ d2 = self.loopback()
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(lambda _: self.mailbox.getMessageCount())
+ return d.addCallback(self._cbTestExpunge)
+
+ def _cbTestExpunge(self, count):
+ # we only left 1 mssage with no deleted flag
+ self.assertEqual(count, 1)
+ # the uids of the deleted messages
+ self.assertItemsEqual(self.results, [1, 3])
+
+
+class AccountTestCase(IMAP4HelperMixin):
+ """
+ Test the Account.
+ """
+ def _create_empty_mailbox(self):
+ return self.server.theAccount.addMailbox('')
+
+ def _create_one_mailbox(self):
+ return self.server.theAccount.addMailbox('one')
+
+ def test_illegalMailboxCreate(self):
+ self.assertRaises(AssertionError, self._create_empty_mailbox)
+
+
+class IMAP4ServerSearchTestCase(IMAP4HelperMixin):
+ """
+ Tests for the behavior of the search_* functions in L{imap5.IMAP4Server}.
+ """
+ # XXX coming soon to your screens!
+ pass
diff --git a/src/leap/bitmask/mail/imap/tests/walktree.py b/src/leap/bitmask/mail/imap/tests/walktree.py
new file mode 100644
index 00000000..f259a556
--- /dev/null
+++ b/src/leap/bitmask/mail/imap/tests/walktree.py
@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+# walktree.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the walktree module.
+"""
+import os
+import sys
+import pprint
+from email import parser
+
+from leap.mail import walk as W
+
+DEBUG = os.environ.get("BITMASK_MAIL_DEBUG")
+
+
+p = parser.Parser()
+
+# TODO pass an argument of the type of message
+
+##################################################
+# Input from hell
+
+if len(sys.argv) > 1:
+ FILENAME = sys.argv[1]
+else:
+ FILENAME = "rfc822.multi-signed.message"
+
+"""
+FILENAME = "rfc822.plain.message"
+FILENAME = "rfc822.multi-minimal.message"
+"""
+
+msg = p.parse(open(FILENAME))
+DO_CHECK = False
+#################################################
+
+parts = W.get_parts(msg)
+
+if DEBUG:
+ def trim(item):
+ item = item[:10]
+ [trim(part["phash"]) for part in parts if part.get('phash', None)]
+
+raw_docs = list(W.get_raw_docs(msg, parts))
+
+body_phash_fun = [W.get_body_phash_simple,
+ W.get_body_phash_multi][int(msg.is_multipart())]
+body_phash = body_phash_fun(W.get_payloads(msg))
+parts_map = W.walk_msg_tree(parts, body_phash=body_phash)
+
+
+# TODO add missing headers!
+expected = {
+ 'body': '1ddfa80485',
+ 'multi': True,
+ 'part_map': {
+ 1: {
+ 'headers': {'Content-Disposition': 'inline',
+ 'Content-Type': 'multipart/mixed; '
+ 'boundary="z0eOaCaDLjvTGF2l"'},
+ 'multi': True,
+ 'part_map': {1: {'ctype': 'text/plain',
+ 'headers': [
+ ('Content-Type',
+ 'text/plain; charset=utf-8'),
+ ('Content-Disposition',
+ 'inline'),
+ ('Content-Transfer-Encoding',
+ 'quoted-printable')],
+ 'multi': False,
+ 'parts': 1,
+ 'phash': '1ddfa80485',
+ 'size': 206},
+ 2: {'ctype': 'text/plain',
+ 'headers': [('Content-Type',
+ 'text/plain; charset=us-ascii'),
+ ('Content-Disposition',
+ 'attachment; '
+ 'filename="attach.txt"')],
+ 'multi': False,
+ 'parts': 1,
+ 'phash': '7a94e4d769',
+ 'size': 133},
+ 3: {'ctype': 'application/octet-stream',
+ 'headers': [('Content-Type',
+ 'application/octet-stream'),
+ ('Content-Disposition',
+ 'attachment; filename="hack.ico"'),
+ ('Content-Transfer-Encoding',
+ 'base64')],
+ 'multi': False,
+ 'parts': 1,
+ 'phash': 'c42cccebbd',
+ 'size': 12736}}},
+ 2: {'ctype': 'application/pgp-signature',
+ 'headers': [('Content-Type', 'application/pgp-signature')],
+ 'multi': False,
+ 'parts': 1,
+ 'phash': '8f49fbf749',
+ 'size': 877}}}
+
+if DEBUG and DO_CHECK:
+ # TODO turn this into a proper unittest
+ assert(parts_map == expected)
+ print "Structure: OK"
+
+
+print
+print "RAW DOCS"
+pprint.pprint(raw_docs)
+print
+print "PARTS MAP"
+pprint.pprint(parts_map)
diff --git a/src/leap/bitmask/mail/incoming/__init__.py b/src/leap/bitmask/mail/incoming/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/src/leap/bitmask/mail/incoming/__init__.py
diff --git a/src/leap/bitmask/mail/incoming/service.py b/src/leap/bitmask/mail/incoming/service.py
new file mode 100644
index 00000000..1e20862b
--- /dev/null
+++ b/src/leap/bitmask/mail/incoming/service.py
@@ -0,0 +1,844 @@
+# -*- coding: utf-8 -*-
+# service.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Incoming mail fetcher.
+"""
+import copy
+import logging
+import shlex
+import time
+import warnings
+
+from email.parser import Parser
+from email.utils import parseaddr
+from email.utils import formatdate
+from StringIO import StringIO
+from urlparse import urlparse
+
+from twisted.application.service import Service
+from twisted.python import log
+from twisted.python.failure import Failure
+from twisted.internet import defer, reactor
+from twisted.internet.task import LoopingCall
+from twisted.internet.task import deferLater
+
+from leap.common.events import emit_async, catalog
+from leap.common.check import leap_assert, leap_assert_type
+from leap.common.mail import get_email_charset
+from leap.keymanager import errors as keymanager_errors
+from leap.mail.adaptors import soledad_indexes as fields
+from leap.mail.generator import Generator
+from leap.mail.utils import json_loads, empty
+from leap.soledad.client import Soledad
+from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
+from leap.soledad.common.errors import InvalidAuthTokenError
+
+
+logger = logging.getLogger(__name__)
+
+MULTIPART_ENCRYPTED = "multipart/encrypted"
+MULTIPART_SIGNED = "multipart/signed"
+PGP_BEGIN = "-----BEGIN PGP MESSAGE-----"
+PGP_END = "-----END PGP MESSAGE-----"
+
+# The period between succesive checks of the incoming mail
+# queue (in seconds)
+INCOMING_CHECK_PERIOD = 60
+
+
+class MalformedMessage(Exception):
+ """
+ Raised when a given message is not well formed.
+ """
+ pass
+
+
+class IncomingMail(Service):
+ """
+ Fetches and process mail from the incoming pool.
+
+ This object implements IService interface, has public methods
+ startService and stopService that will actually initiate a
+ LoopingCall with check_period recurrency.
+ The LoopingCall itself will invoke the fetch method each time
+ that the check_period expires.
+
+ This loop will sync the soledad db with the remote server and
+ process all the documents found tagged as incoming mail.
+ """
+ # TODO implements IService?
+
+ name = "IncomingMail"
+
+ RECENT_FLAG = "\\Recent"
+ CONTENT_KEY = "content"
+
+ LEAP_SIGNATURE_HEADER = 'X-Leap-Signature'
+ LEAP_ENCRYPTION_HEADER = 'X-Leap-Encryption'
+ """
+ Header added to messages when they are decrypted by the fetcher,
+ which states the validity of an eventual signature that might be included
+ in the encrypted blob.
+ """
+ LEAP_SIGNATURE_VALID = 'valid'
+ LEAP_SIGNATURE_INVALID = 'invalid'
+ LEAP_SIGNATURE_COULD_NOT_VERIFY = 'could not verify'
+
+ LEAP_ENCRYPTION_DECRYPTED = 'decrypted'
+
+ def __init__(self, keymanager, soledad, inbox, userid,
+ check_period=INCOMING_CHECK_PERIOD):
+
+ """
+ Initialize IncomingMail..
+
+ :param keymanager: a keymanager instance
+ :type keymanager: keymanager.KeyManager
+
+ :param soledad: a soledad instance
+ :type soledad: Soledad
+
+ :param inbox: the collection for the inbox where the new emails will be
+ stored
+ :type inbox: MessageCollection
+
+ :param check_period: the period to fetch new mail, in seconds.
+ :type check_period: int
+ """
+ leap_assert(keymanager, "need a keymanager to initialize")
+ leap_assert_type(soledad, Soledad)
+ leap_assert(check_period, "need a period to check incoming mail")
+ leap_assert_type(check_period, int)
+ leap_assert(userid, "need a userid to initialize")
+
+ self._keymanager = keymanager
+ self._soledad = soledad
+ self._inbox_collection = inbox
+ self._userid = userid
+
+ self._listeners = []
+ self._loop = None
+ self._check_period = check_period
+
+ # initialize a mail parser only once
+ self._parser = Parser()
+
+ def add_listener(self, listener):
+ """
+ Add a listener to inbox insertions.
+
+ This listener function will be called for each message added to the
+ inbox with its uid as parameter. This function should not be blocking
+ or it will block the incoming queue.
+
+ :param listener: the listener function
+ :type listener: callable
+ """
+ self._listeners.append(listener)
+
+ #
+ # Public API: fetch, start_loop, stop.
+ #
+
+ def fetch(self):
+ """
+ Fetch incoming mail, to be called periodically.
+
+ Calls a deferred that will execute the fetch callback.
+ """
+ def _sync_errback(failure):
+ log.err(failure)
+
+ def syncSoledadCallback(_):
+ # XXX this should be moved to adaptors
+ d = self._soledad.get_from_index(
+ fields.JUST_MAIL_IDX, "1", "0")
+ d.addCallback(self._process_doclist)
+ d.addErrback(_sync_errback)
+ return d
+
+ logger.debug("fetching mail for: %s %s" % (
+ self._soledad.uuid, self._userid))
+ d = self._sync_soledad()
+ d.addCallbacks(syncSoledadCallback, self._errback)
+ d.addCallbacks(self._signal_fetch_to_ui, self._errback)
+ return d
+
+ def startService(self):
+ """
+ Starts a loop to fetch mail.
+
+ :returns: A Deferred whose callback will be invoked with
+ the LoopingCall instance when loop.stop is called, or
+ whose errback will be invoked when the function raises an
+ exception or returned a deferred that has its errback
+ invoked.
+ """
+ Service.startService(self)
+ if self._loop is None:
+ self._loop = LoopingCall(self.fetch)
+ stop_deferred = self._loop.start(self._check_period)
+ return stop_deferred
+ else:
+ logger.warning("Tried to start an already running fetching loop.")
+ return defer.fail(Failure('Already running loop.'))
+
+ def stopService(self):
+ """
+ Stops the loop that fetches mail.
+ """
+ if self._loop and self._loop.running is True:
+ self._loop.stop()
+ self._loop = None
+ Service.stopService(self)
+
+ #
+ # Private methods.
+ #
+
+ # synchronize incoming mail
+
+ def _errback(self, failure):
+ log.err(failure)
+
+ def _sync_soledad(self):
+ """
+ Synchronize with remote soledad.
+
+ :returns: a list of LeapDocuments, or None.
+ :rtype: iterable or None
+ """
+ def _log_synced(result):
+ log.msg('FETCH soledad SYNCED.')
+ return result
+
+ def _signal_invalid_auth(failure):
+ failure.trap(InvalidAuthTokenError)
+ # if the token is invalid, send an event so the GUI can
+ # disable mail and show an error message.
+ emit_async(catalog.SOLEDAD_INVALID_AUTH_TOKEN, self._userid)
+
+ log.msg('FETCH: syncing soledad...')
+ d = self._soledad.sync()
+ d.addCallbacks(_log_synced, _signal_invalid_auth)
+ return d
+
+ def _signal_fetch_to_ui(self, doclist):
+ """
+ Send leap events to ui.
+
+ :param doclist: iterable with msg documents.
+ :type doclist: iterable.
+ :returns: doclist
+ :rtype: iterable
+ """
+ if doclist:
+ fetched_ts = time.mktime(time.gmtime())
+ num_mails = len(doclist) if doclist is not None else 0
+ if num_mails != 0:
+ log.msg("there are %s mails" % (num_mails,))
+ emit_async(catalog.MAIL_FETCHED_INCOMING, self._userid,
+ str(num_mails), str(fetched_ts))
+ return doclist
+
+ def _signal_unread_to_ui(self, *args):
+ """
+ Sends unread event to ui.
+ """
+ emit_async(catalog.MAIL_UNREAD_MESSAGES, self._userid,
+ str(self._inbox_collection.count_unseen()))
+
+ # process incoming mail.
+
+ def _process_doclist(self, doclist):
+ """
+ Iterates through the doclist, checks if each doc
+ looks like a message, and yields a deferred that will decrypt and
+ process the message.
+
+ :param doclist: iterable with msg documents.
+ :type doclist: iterable.
+ :returns: a list of deferreds for individual messages.
+ """
+ log.msg('processing doclist')
+ if not doclist:
+ logger.debug("no docs found")
+ return
+ num_mails = len(doclist)
+
+ deferreds = []
+ for index, doc in enumerate(doclist):
+ logger.debug("processing doc %d of %d" % (index + 1, num_mails))
+ emit_async(catalog.MAIL_MSG_PROCESSING, self._userid,
+ str(index), str(num_mails))
+
+ keys = doc.content.keys()
+
+ # TODO Compatibility check with the index in pre-0.6 mx
+ # that does not write the ERROR_DECRYPTING_KEY
+ # This should be removed in 0.7
+
+ has_errors = doc.content.get(fields.ERROR_DECRYPTING_KEY, None)
+
+ if has_errors is None:
+ warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!",
+ DeprecationWarning)
+
+ if has_errors:
+ logger.debug("skipping msg with decrypting errors...")
+ elif self._is_msg(keys):
+ # TODO this pipeline is a bit obscure!
+ d = self._decrypt_doc(doc)
+ d.addCallback(self._maybe_extract_keys)
+ d.addCallbacks(self._add_message_locally, self._errback)
+ deferreds.append(d)
+
+ d = defer.gatherResults(deferreds, consumeErrors=True)
+ d.addCallback(lambda _: doclist)
+ return d
+
+ #
+ # operations on individual messages
+ #
+
+ def _decrypt_doc(self, doc):
+ """
+ Decrypt the contents of a document.
+
+ :param doc: A document containing an encrypted message.
+ :type doc: SoledadDocument
+
+ :return: A Deferred that will be fired with the document and the
+ decrypted message.
+ :rtype: SoledadDocument, str
+ """
+ log.msg('decrypting msg')
+
+ def process_decrypted(res):
+ if isinstance(res, tuple):
+ decrdata, _ = res
+ success = True
+ else:
+ decrdata = ""
+ success = False
+
+ emit_async(catalog.MAIL_MSG_DECRYPTED, self._userid,
+ "1" if success else "0")
+ return self._process_decrypted_doc(doc, decrdata)
+
+ d = self._keymanager.decrypt(doc.content[ENC_JSON_KEY], self._userid)
+ d.addErrback(self._errback)
+ d.addCallback(process_decrypted)
+ d.addCallback(lambda data: (doc, data))
+ return d
+
+ def _process_decrypted_doc(self, doc, data):
+ """
+ Process a document containing a succesfully decrypted message.
+
+ :param doc: the incoming message
+ :type doc: SoledadDocument
+ :param data: the json-encoded, decrypted content of the incoming
+ message
+ :type data: str
+
+ :return: a Deferred that will be fired with an str of the proccessed
+ data.
+ :rtype: Deferred
+ """
+ log.msg('processing decrypted doc')
+
+ # XXX turn this into an errBack for each one of
+ # the deferreds that would process an individual document
+ try:
+ msg = json_loads(data)
+ except UnicodeError as exc:
+ logger.error("Error while decrypting %s" % (doc.doc_id,))
+ logger.exception(exc)
+
+ # we flag the message as "with decrypting errors",
+ # to avoid further decryption attempts during sync
+ # cycles until we're prepared to deal with that.
+ # What is the same, when Ivan deals with it...
+ # A new decrypting attempt event could be triggered by a
+ # future a library upgrade, or a cli flag to the client,
+ # we just `defer` that for now... :)
+ doc.content[fields.ERROR_DECRYPTING_KEY] = True
+ deferLater(reactor, 0, self._update_incoming_message, doc)
+
+ # FIXME this is just a dirty hack to delay the proper
+ # deferred organization here...
+ # and remember, boys, do not do this at home.
+ return []
+
+ if not isinstance(msg, dict):
+ defer.returnValue(False)
+ if not msg.get(fields.INCOMING_KEY, False):
+ defer.returnValue(False)
+
+ # ok, this is an incoming message
+ rawmsg = msg.get(self.CONTENT_KEY, None)
+ if rawmsg is None:
+ return ""
+ return self._maybe_decrypt_msg(rawmsg)
+
+ def _update_incoming_message(self, doc):
+ """
+ Do a put for a soledad document. This probably has been called only
+ in the case that we've needed to update the ERROR_DECRYPTING_KEY
+ flag in an incoming message, to get it out of the decrypting queue.
+
+ :param doc: the SoledadDocument to update
+ :type doc: SoledadDocument
+ """
+ log.msg("Updating Incoming MSG: SoledadDoc %s" % (doc.doc_id))
+ return self._soledad.put_doc(doc)
+
+ def _delete_incoming_message(self, doc):
+ """
+ Delete document.
+
+ :param doc: the SoledadDocument to delete
+ :type doc: SoledadDocument
+ """
+ log.msg("Deleting Incoming message: %s" % (doc.doc_id,))
+ return self._soledad.delete_doc(doc)
+
+ def _maybe_decrypt_msg(self, data):
+ """
+ Tries to decrypt a gpg message if data looks like one.
+
+ :param data: the text to be decrypted.
+ :type data: str
+
+ :return: a Deferred that will be fired with an str of data, possibly
+ decrypted.
+ :rtype: Deferred
+ """
+ leap_assert_type(data, str)
+ log.msg('maybe decrypting doc')
+
+ # parse the original message
+ encoding = get_email_charset(data)
+ msg = self._parser.parsestr(data)
+
+ fromHeader = msg.get('from', None)
+ senderAddress = None
+
+ if (fromHeader is not None and
+ (msg.get_content_type() == MULTIPART_ENCRYPTED or
+ msg.get_content_type() == MULTIPART_SIGNED)):
+ senderAddress = parseaddr(fromHeader)[1]
+
+ def add_leap_header(ret):
+ decrmsg, signkey = ret
+ if (senderAddress is None or signkey is None or
+ isinstance(signkey, keymanager_errors.KeyNotFound)):
+ decrmsg.add_header(
+ self.LEAP_SIGNATURE_HEADER,
+ self.LEAP_SIGNATURE_COULD_NOT_VERIFY)
+ elif isinstance(signkey, keymanager_errors.InvalidSignature):
+ decrmsg.add_header(
+ self.LEAP_SIGNATURE_HEADER,
+ self.LEAP_SIGNATURE_INVALID)
+ else:
+ self._add_verified_signature_header(decrmsg,
+ signkey.fingerprint)
+ return decrmsg.as_string()
+
+ if msg.get_content_type() == MULTIPART_ENCRYPTED:
+ d = self._decrypt_multipart_encrypted_msg(
+ msg, encoding, senderAddress)
+ elif msg.get_content_type() == MULTIPART_SIGNED:
+ d = self._verify_signature_not_encrypted_msg(msg, senderAddress)
+ else:
+ d = self._maybe_decrypt_inline_encrypted_msg(
+ msg, encoding, senderAddress)
+ d.addCallback(add_leap_header)
+ return d
+
+ def _add_verified_signature_header(self, decrmsg, fingerprint):
+ decrmsg.add_header(
+ self.LEAP_SIGNATURE_HEADER,
+ self.LEAP_SIGNATURE_VALID,
+ pubkey=fingerprint)
+
+ def _add_decrypted_header(self, msg):
+ msg.add_header(self.LEAP_ENCRYPTION_HEADER,
+ self.LEAP_ENCRYPTION_DECRYPTED)
+
+ def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderAddress):
+ """
+ Decrypt a message with content-type 'multipart/encrypted'.
+
+ :param msg: The original encrypted message.
+ :type msg: Message
+ :param encoding: The encoding of the email message.
+ :type encoding: str
+ :param senderAddress: The email address of the sender of the message.
+ :type senderAddress: str
+
+ :return: A Deferred that will be fired with a tuple containing a
+ decrypted Message and the signing OpenPGPKey if the signature
+ is valid or InvalidSignature or KeyNotFound.
+ :rtype: Deferred
+ """
+ log.msg('decrypting multipart encrypted msg')
+ msg = copy.deepcopy(msg)
+ self._msg_multipart_sanity_check(msg)
+
+ # parse message and get encrypted content
+ pgpencmsg = msg.get_payload()[1]
+ encdata = pgpencmsg.get_payload()
+
+ # decrypt or fail gracefully
+ def build_msg(res):
+ decrdata, signkey = res
+
+ decrmsg = self._parser.parsestr(decrdata)
+ # remove original message's multipart/encrypted content-type
+ del(msg['content-type'])
+
+ # replace headers back in original message
+ for hkey, hval in decrmsg.items():
+ try:
+ # this will raise KeyError if header is not present
+ msg.replace_header(hkey, hval)
+ except KeyError:
+ msg[hkey] = hval
+
+ # all ok, replace payload by unencrypted payload
+ msg.set_payload(decrmsg.get_payload())
+ self._add_decrypted_header(msg)
+ return (msg, signkey)
+
+ def verify_signature_after_decrypt_an_email(res):
+ decrdata, signkey = res
+ if decrdata.get_content_type() == MULTIPART_SIGNED:
+ res = self._verify_signature_not_encrypted_msg(decrdata,
+ senderAddress)
+ return res
+
+ d = self._keymanager.decrypt(
+ encdata, self._userid, verify=senderAddress)
+ d.addCallbacks(build_msg, self._decryption_error, errbackArgs=(msg,))
+ d.addCallbacks(verify_signature_after_decrypt_an_email)
+ return d
+
+ def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding,
+ senderAddress):
+ """
+ Possibly decrypt an inline OpenPGP encrypted message.
+
+ :param origmsg: The original, possibly encrypted message.
+ :type origmsg: Message
+ :param encoding: The encoding of the email message.
+ :type encoding: str
+ :param senderAddress: The email address of the sender of the message.
+ :type senderAddress: str
+
+ :return: A Deferred that will be fired with a tuple containing a
+ decrypted Message and the signing OpenPGPKey if the signature
+ is valid or InvalidSignature or KeyNotFound.
+ :rtype: Deferred
+ """
+ log.msg('maybe decrypting inline encrypted msg')
+
+ data = self._serialize_msg(origmsg)
+
+ def decrypted_data(res):
+ decrdata, signkey = res
+ replaced_data = data.replace(pgp_message, decrdata)
+ self._add_decrypted_header(origmsg)
+ return replaced_data, signkey
+
+ def encode_and_return(res):
+ data, signkey = res
+ if isinstance(data, unicode):
+ data = data.encode(encoding, 'replace')
+ return (self._parser.parsestr(data), signkey)
+
+ # handle exactly one inline PGP message
+ if PGP_BEGIN in data:
+ begin = data.find(PGP_BEGIN)
+ end = data.find(PGP_END)
+ pgp_message = data[begin:end + len(PGP_END)]
+ d = self._keymanager.decrypt(
+ pgp_message, self._userid, verify=senderAddress)
+ d.addCallbacks(decrypted_data, self._decryption_error,
+ errbackArgs=(data,))
+ else:
+ d = defer.succeed((data, None))
+ d.addCallback(encode_and_return)
+ return d
+
+ def _verify_signature_not_encrypted_msg(self, origmsg, sender_address):
+ """
+ Possibly decrypt an inline OpenPGP encrypted message.
+
+ :param origmsg: The original, possibly encrypted message.
+ :type origmsg: Message
+ :param sender_address: The email address of the sender of the message.
+ :type sender_address: str
+
+ :return: A Deferred that will be fired with a tuple containing a
+ signed Message and the signing OpenPGPKey if the signature
+ is valid or InvalidSignature.
+ :rtype: Deferred
+ """
+ msg = copy.deepcopy(origmsg)
+ data = self._serialize_msg(msg.get_payload(0))
+ detached_sig = self._extract_signature(msg)
+ d = self._keymanager.verify(data, sender_address, detached_sig)
+
+ d.addCallback(lambda sign_key: (msg, sign_key))
+ d.addErrback(lambda _: (msg, keymanager_errors.InvalidSignature()))
+ return d
+
+ def _serialize_msg(self, origmsg):
+ buf = StringIO()
+ g = Generator(buf)
+ g.flatten(origmsg)
+ return buf.getvalue()
+
+ def _extract_signature(self, msg):
+ body = msg.get_payload(0).get_payload()
+
+ if isinstance(body, str):
+ body = msg.get_payload(0)
+
+ detached_sig = msg.get_payload(1).get_payload()
+ msg.set_payload(body)
+ return detached_sig
+
+ def _decryption_error(self, failure, msg):
+ """
+ Check for known decryption errors
+ """
+ if failure.check(keymanager_errors.DecryptError):
+ logger.warning('Failed to decrypt encrypted message (%s). '
+ 'Storing message without modifications.'
+ % str(failure.value))
+ return (msg, None)
+ elif failure.check(keymanager_errors.KeyNotFound):
+ logger.error('Failed to find private key for decryption (%s). '
+ 'Storing message without modifications.'
+ % str(failure.value))
+ return (msg, None)
+ else:
+ return failure
+
+ @defer.inlineCallbacks
+ def _maybe_extract_keys(self, msgtuple):
+ """
+ Retrieve attached keys to the mesage and parse message headers for an
+ *OpenPGP* header as described on the `IETF draft
+ <http://tools.ietf.org/html/draft-josefsson-openpgp-mailnews-header-06>`
+ only urls with https and the same hostname than the email are supported
+ for security reasons.
+
+ :param msgtuple: a tuple consisting of a SoledadDocument
+ instance containing the incoming message
+ and data, the json-encoded, decrypted content of the
+ incoming message
+ :type msgtuple: (SoledadDocument, str)
+
+ :return: A Deferred that will be fired with msgtuple when key
+ extraction finishes
+ :rtype: Deferred
+ """
+ OpenPGP_HEADER = 'OpenPGP'
+ doc, data = msgtuple
+
+ # XXX the parsing of the message is done in mailbox.addMessage, maybe
+ # we should do it in this module so we don't need to parse it again
+ # here
+ msg = self._parser.parsestr(data)
+ _, fromAddress = parseaddr(msg['from'])
+
+ valid_attachment = False
+ if msg.is_multipart():
+ valid_attachment = yield self._maybe_extract_attached_key(
+ msg.get_payload(), fromAddress)
+
+ if not valid_attachment:
+ header = msg.get(OpenPGP_HEADER, None)
+ if header is not None:
+ yield self._maybe_extract_openpgp_header(header, fromAddress)
+
+ defer.returnValue(msgtuple)
+
+ def _maybe_extract_openpgp_header(self, header, address):
+ """
+ Import keys from the OpenPGP header
+
+ :param header: OpenPGP header string
+ :type header: str
+ :param address: email address in the from header
+ :type address: str
+
+ :return: A Deferred that will be fired when header extraction is done
+ :rtype: Deferred
+ """
+ d = defer.succeed(None)
+ fields = dict([f.strip(' ').split('=') for f in header.split(';')])
+ if 'url' in fields:
+ url = shlex.split(fields['url'])[0] # remove quotations
+ urlparts = urlparse(url)
+ addressHostname = address.split('@')[1]
+ if (
+ urlparts.scheme == 'https' and
+ urlparts.hostname == addressHostname
+ ):
+ def fetch_error(failure):
+ if failure.check(keymanager_errors.KeyNotFound):
+ logger.warning("Url from OpenPGP header %s failed"
+ % (url,))
+ elif failure.check(keymanager_errors.KeyAttributesDiffer):
+ logger.warning("Key from OpenPGP header url %s didn't "
+ "match the from address %s"
+ % (url, address))
+ else:
+ return failure
+
+ d = self._keymanager.fetch_key(address, url)
+ d.addCallback(
+ lambda _:
+ logger.info("Imported key from header %s" % (url,)))
+ d.addErrback(fetch_error)
+ else:
+ logger.debug("No valid url on OpenPGP header %s" % (url,))
+ else:
+ logger.debug("There is no url on the OpenPGP header: %s"
+ % (header,))
+ return d
+
+ def _maybe_extract_attached_key(self, attachments, address):
+ """
+ Import keys from the attachments
+
+ :param attachments: email attachment list
+ :type attachments: list(email.Message)
+ :param address: email address in the from header
+ :type address: str
+
+ :return: A Deferred that will be fired when all the keys are stored
+ with a boolean: True if there was a valid key attached, or
+ False otherwise.
+ :rtype: Deferred
+ """
+ MIME_KEY = "application/pgp-keys"
+
+ def log_key_added(ignored):
+ logger.debug('Added key found in attachment for %s' % address)
+ return True
+
+ def failed_put_key(failure):
+ logger.info("An error has ocurred adding attached key for %s: %s"
+ % (address, failure.getErrorMessage()))
+ return False
+
+ deferreds = []
+ for attachment in attachments:
+ if MIME_KEY == attachment.get_content_type():
+ d = self._keymanager.put_raw_key(
+ attachment.get_payload(decode=True), address=address)
+ d.addCallbacks(log_key_added, failed_put_key)
+ deferreds.append(d)
+ d = defer.gatherResults(deferreds)
+ d.addCallback(lambda result: any(result))
+ return d
+
+ def _add_message_locally(self, msgtuple):
+ """
+ Adds a message to local inbox and delete it from the incoming db
+ in soledad.
+
+ :param msgtuple: a tuple consisting of a SoledadDocument
+ instance containing the incoming message
+ and data, the json-encoded, decrypted content of the
+ incoming message
+ :type msgtuple: (SoledadDocument, str)
+
+ :return: A Deferred that will be fired when the messages is stored
+ :rtype: Defferred
+ """
+ doc, raw_data = msgtuple
+ insertion_date = formatdate(time.time())
+ log.msg('adding message %s to local db' % (doc.doc_id,))
+
+ def msgSavedCallback(result):
+ if empty(result):
+ return
+
+ for listener in self._listeners:
+ listener(result)
+
+ def signal_deleted(doc_id):
+ emit_async(catalog.MAIL_MSG_DELETED_INCOMING,
+ self._userid)
+ return doc_id
+
+ emit_async(catalog.MAIL_MSG_SAVED_LOCALLY, self._userid)
+ d = self._delete_incoming_message(doc)
+ d.addCallback(signal_deleted)
+ return d
+
+ d = self._inbox_collection.add_msg(
+ raw_data, (self.RECENT_FLAG,), date=insertion_date,
+ notify_just_mdoc=True)
+ d.addCallbacks(msgSavedCallback, self._errback)
+ return d
+
+ #
+ # helpers
+ #
+
+ def _msg_multipart_sanity_check(self, msg):
+ """
+ Performs a sanity check against a multipart encrypted msg
+
+ :param msg: The original encrypted message.
+ :type msg: Message
+ """
+ # sanity check
+ payload = msg.get_payload()
+ if len(payload) != 2:
+ raise MalformedMessage(
+ 'Multipart/encrypted messages should have exactly 2 body '
+ 'parts (instead of %d).' % len(payload))
+ if payload[0].get_content_type() != 'application/pgp-encrypted':
+ raise MalformedMessage(
+ "Multipart/encrypted messages' first body part should "
+ "have content type equal to 'application/pgp-encrypted' "
+ "(instead of %s)." % payload[0].get_content_type())
+ if payload[1].get_content_type() != 'application/octet-stream':
+ raise MalformedMessage(
+ "Multipart/encrypted messages' second body part should "
+ "have content type equal to 'octet-stream' (instead of "
+ "%s)." % payload[1].get_content_type())
+
+ def _is_msg(self, keys):
+ """
+ Checks if the keys of a dictionary match the signature
+ of the document type we use for messages.
+
+ :param keys: iterable containing the strings to match.
+ :type keys: iterable of strings.
+ :rtype: bool
+ """
+ return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys
diff --git a/src/leap/bitmask/mail/incoming/tests/rfc822.multi-encrypt-signed.message b/src/leap/bitmask/mail/incoming/tests/rfc822.multi-encrypt-signed.message
new file mode 100644
index 00000000..98304f24
--- /dev/null
+++ b/src/leap/bitmask/mail/incoming/tests/rfc822.multi-encrypt-signed.message
@@ -0,0 +1,61 @@
+Content-Type: multipart/encrypted;
+ boundary="Apple-Mail=_C01A1464-6C43-43BF-8F62-157335B7E25B";
+ protocol="application/pgp-encrypted";
+Subject: Enc signed
+Mime-Version: 1.0 (Mac OS X Mail 9.3 \(3124\))
+From: Leap Test Key <leap@leap.se>
+Date: Tue, 24 May 2016 11:47:24 -0300
+Content-Description: OpenPGP encrypted message
+To: leap@leap.se
+
+This is an OpenPGP/MIME encrypted message (RFC 2440 and 3156)
+--Apple-Mail=_C01A1464-6C43-43BF-8F62-157335B7E25B
+Content-Type: application/pgp-encrypted
+Content-Description: PGP/MIME Versions Identification
+
+--Apple-Mail=_C01A1464-6C43-43BF-8F62-157335B7E25B
+Content-Disposition: inline;
+ filename=encrypted.asc
+Content-Type: application/octet-stream;
+ name=encrypted.asc
+Content-Description: OpenPGP encrypted message
+
+-----BEGIN PGP MESSAGE-----
+Version: GnuPG v2
+
+hQIMAyj9aG/xtZOwAQ/9Gft0KmOpgzL6z4wmVlLm2aeAvHolXmxWb7N/ByL/dZ4n
+YZd/GPRj42X3BwUrDEL5aO3Mcp+rqq8ACh9hsZXiau0Q9cs1K7Gr55Y06qLrIjom
+2fLqwLFBxCL2sAX1dvClgStyfsRFk9Y/+5tX+IjWaD8dAoRdxCO8IbUDuYGnaKld
+bB9h0NMfKVddCAvuQvX1Zc1Nx0Yb3Hd+ocDD7i9BVgX1BBiGu4/ElS3d32TAVCFs
+Na3tjitWB2G472CYu1O6exY7h1F5V4FHfXH6iMRJSYnvV2Jr+oPZENzNdEEA5H/H
+fUbpWrpKzPafjho9S5rJBBM/tqtmBQFBIdgFVcBVb+bXO6DJ8SMTLiiGcVUvvm1b
+9N2VQIhsxtZ8DpcHHSqFVgT2Gt4UkSrEleSoReg36TzS1s8Uw0oU068PwTe3K0Gx
+2pLMdT9NA6X/t7movpXP6tih1l6P5z62dxFl6W12J9OcegISCt0Q7gex1gk/a8zM
+rzBJC3mVxRiFlvHPBgD6oUKarnTJPQx5f5dFXg8DXBWR1Eh/aFjPQIzhZBYpmOi8
+HqgjcAA+WhMQ7v5c0enJoJJS+8Xfai/MK2vTUGsfAT6HqHLw1HSIn6XQGEf4sQ/U
+NfLeFHHbe9rTk8QhyjrSl2vvek2H4EBQVLF08/FUrAfPELUttOFtysQfC3+M0+PS
+6QGyeIlUjKpBJG7HBd4ibuKMQ5vnA+ACsg/TySYeCO6P85xsN+Lmqlr8cAICn/hR
+ezFSzlibaIelRgfDEDJdjVyCsa7qBMjhRCvGYBdkyTzIRq53qwD9pkhrQ6nwWQrv
+bBzyLrl+NVR8CTEOwbeFLI6qf68kblojk3lwo3Qi3psmeMJdiaV9uevsHrgmEFTH
+lZ3rFECPWzmrkMSfVjWu5d8jJqMcqa4lnGzFQKaB76I8BzGhCWrnuvHPB9c9SVhI
+AnAwNw3gY5xgsbXMxZhnPgYeBSViPkQkgRCWl8Jz41eiAJ3Gtj8QSSFWGHpX+MgP
+ohBaPHz6Fnkhz7Lok97e2AcuRZrDVKV6i28r8mizI3B2Mah6ZV0Yuv0EYNtzBv/v
+yV3nu4DWuOOU0301CXBayxJGX0h07z1Ycv7jWD6LNiBXa1vahtbU4WSYNkF0OJaz
+nf8O3CZy5twMq5kQYoPacdNNLregAmWquvE1nxqWbtHFMjtXitP7czxzUTU/DE+C
+jr+irDoYEregEKg9xov91UCRPZgxL+TML71+tSYOMO3JG6lbGw77PQ8s2So7xore
+8+FeDFPaaJqh6uhF5LETRSx8x/haZiXLd+WtO7wF8S3+Vz7AJIFIe8MUadZrYwnH
+wfMAktQKbep3iHCeZ5jHYA461AOhnCca2y+GoyHZUDDFwS1pC1RN4lMkafSE1AgH
+cmEcjLYsw1gqT0+DfqrvjbXmMjGgkgnkMybJH7df5TKu36Q0Nqvcbc2XLFkalr5V
+Vk0SScqKYnKL+cJjabqA8rKkeAh22E2FBCpKPqxSS3te2bRb3XBX26bP0LshkJuy
+GPu6LKvwmUn0obPKCnLJvb9ImIGZToXu6Fb/Cd2c3DG1IK5PptQz4f7ZRW98huPO
+2w59Bswwt5q4lQqsMEzVRnIDH45MmnhEUeS4NaxqLTO7eJpMpb4VxT2u/Ac3XWKp
+o2RE6CbqTyJ+n8tY9OwBRMKzdVd9RFAMqMHTzWTAuU4BgW2vT2sHYZdAsX8sktBr
+5mo9P3MqvgdPNpg8+AOB03JlIv0dzrAFWCZxxLLGIIIz0eXsjghHzQ9QjGfr0xFH
+Z79AKDjsoRisWyWCnadS2oM9fdAg4T/h1STnfxc44o7N1+ym7u58ODICFi+Kg8IR
+JBHIp3CK02JLTLd/WFhUVyWgc6l8gn+oBK+r7Dw+FTWhqX2/ZHCO8qKK1ZK3NIMn
+MBcSVvHSnTPtppb+oND5nk38xazVVHnwxNHaIh7g3NxDB4hl5rBhrWsgTNuqDDRU
+w7ufvMYr1AOV+8e92cHCEKPM19nFKEgaBFECEptEObesGI3QZPAESlojzQ3cDeBa
+=tEyc
+-----END PGP MESSAGE-----
+
+--Apple-Mail=_C01A1464-6C43-43BF-8F62-157335B7E25B-- \ No newline at end of file
diff --git a/src/leap/bitmask/mail/incoming/tests/test_incoming_mail.py b/src/leap/bitmask/mail/incoming/tests/test_incoming_mail.py
new file mode 100644
index 00000000..29422ecc
--- /dev/null
+++ b/src/leap/bitmask/mail/incoming/tests/test_incoming_mail.py
@@ -0,0 +1,391 @@
+# -*- coding: utf-8 -*-
+# test_incoming_mail.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Test case for leap.mail.incoming.service
+
+@authors: Ruben Pollan, <meskio@sindominio.net>
+
+@license: GPLv3, see included LICENSE file
+"""
+
+import json
+import os
+import tempfile
+import uuid
+
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from email.parser import Parser
+from mock import Mock
+
+from twisted.internet import defer
+from twisted.python import log
+
+from leap.keymanager.errors import KeyAddressMismatch
+from leap.mail.adaptors import soledad_indexes as fields
+from leap.mail.adaptors.soledad import cleanup_deferred_locks
+from leap.mail.adaptors.soledad import SoledadMailAdaptor
+from leap.mail.mail import MessageCollection
+from leap.mail.mailbox_indexer import MailboxIndexer
+
+from leap.mail.incoming.service import IncomingMail
+from leap.mail.rfc3156 import MultipartEncrypted, PGPEncrypted
+from leap.mail.testing import KeyManagerWithSoledadTestCase
+from leap.mail.testing import ADDRESS, ADDRESS_2
+from leap.soledad.common.document import SoledadDocument
+from leap.soledad.common.crypto import (
+ EncryptionSchemes,
+ ENC_JSON_KEY,
+ ENC_SCHEME_KEY,
+)
+
+HERE = os.path.split(os.path.abspath(__file__))[0]
+
+# TODO: add some tests for encrypted, unencrypted, signed and unsgined messages
+
+
+class IncomingMailTestCase(KeyManagerWithSoledadTestCase):
+ """
+ Tests for the incoming mail parser
+ """
+ NICKSERVER = "http://domain"
+ BODY = """
+Governments of the Industrial World, you weary giants of flesh and steel, I
+come from Cyberspace, the new home of Mind. On behalf of the future, I ask
+you of the past to leave us alone. You are not welcome among us. You have
+no sovereignty where we gather.
+ """
+ EMAIL = """from: Test from SomeDomain <%(from)s>
+to: %(to)s
+subject: independence of cyberspace
+
+%(body)s
+ """ % {
+ "from": ADDRESS_2,
+ "to": ADDRESS,
+ "body": BODY
+ }
+
+ def setUp(self):
+ cleanup_deferred_locks()
+ try:
+ del self._soledad
+ del self.km
+ except AttributeError:
+ pass
+
+ # pytest handles correctly the setupEnv for the class,
+ # but trial ignores it.
+ if not getattr(self, 'tempdir', None):
+ self.tempdir = tempfile.mkdtemp()
+
+ def getCollection(_):
+ adaptor = SoledadMailAdaptor()
+ store = self._soledad
+ adaptor.store = store
+ mbox_indexer = MailboxIndexer(store)
+ mbox_name = "INBOX"
+ mbox_uuid = str(uuid.uuid4())
+
+ def get_collection_from_mbox_wrapper(wrapper):
+ wrapper.uuid = mbox_uuid
+ return MessageCollection(
+ adaptor, store,
+ mbox_indexer=mbox_indexer, mbox_wrapper=wrapper)
+
+ d = adaptor.initialize_store(store)
+ d.addCallback(lambda _: mbox_indexer.create_table(mbox_uuid))
+ d.addCallback(
+ lambda _: adaptor.get_or_create_mbox(store, mbox_name))
+ d.addCallback(get_collection_from_mbox_wrapper)
+ return d
+
+ def setUpFetcher(inbox_collection):
+ self.fetcher = IncomingMail(
+ self.km,
+ self._soledad,
+ inbox_collection,
+ ADDRESS)
+
+ # The messages don't exist on soledad will fail on deletion
+ self.fetcher._delete_incoming_message = Mock(
+ return_value=defer.succeed(None))
+
+ d = KeyManagerWithSoledadTestCase.setUp(self)
+ d.addCallback(getCollection)
+ d.addCallback(setUpFetcher)
+ d.addErrback(log.err)
+ return d
+
+ def tearDown(self):
+ d = KeyManagerWithSoledadTestCase.tearDown(self)
+ return d
+
+ def testExtractOpenPGPHeader(self):
+ """
+ Test the OpenPGP header key extraction
+ """
+ KEYURL = "https://leap.se/key.txt"
+ OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,)
+
+ message = Parser().parsestr(self.EMAIL)
+ message.add_header("OpenPGP", OpenPGP)
+ self.fetcher._keymanager.fetch_key = Mock(
+ return_value=defer.succeed(None))
+
+ def fetch_key_called(ret):
+ self.fetcher._keymanager.fetch_key.assert_called_once_with(
+ ADDRESS_2, KEYURL)
+
+ d = self._create_incoming_email(message.as_string())
+ d.addCallback(
+ lambda email:
+ self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]))
+ d.addCallback(lambda _: self.fetcher.fetch())
+ d.addCallback(fetch_key_called)
+ return d
+
+ def testExtractOpenPGPHeaderInvalidUrl(self):
+ """
+ Test the OpenPGP header key extraction
+ """
+ KEYURL = "https://someotherdomain.com/key.txt"
+ OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,)
+
+ message = Parser().parsestr(self.EMAIL)
+ message.add_header("OpenPGP", OpenPGP)
+ self.fetcher._keymanager.fetch_key = Mock()
+
+ def fetch_key_called(ret):
+ self.assertFalse(self.fetcher._keymanager.fetch_key.called)
+
+ d = self._create_incoming_email(message.as_string())
+ d.addCallback(
+ lambda email:
+ self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]))
+ d.addCallback(lambda _: self.fetcher.fetch())
+ d.addCallback(fetch_key_called)
+ return d
+
+ def testExtractAttachedKey(self):
+ KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..."
+
+ message = MIMEMultipart()
+ message.add_header("from", ADDRESS_2)
+ key = MIMEApplication("", "pgp-keys")
+ key.set_payload(KEY)
+ message.attach(key)
+ self.fetcher._keymanager.put_raw_key = Mock(
+ return_value=defer.succeed(None))
+
+ def put_raw_key_called(_):
+ self.fetcher._keymanager.put_raw_key.assert_called_once_with(
+ KEY, address=ADDRESS_2)
+
+ d = self._do_fetch(message.as_string())
+ d.addCallback(put_raw_key_called)
+ return d
+
+ def testExtractInvalidAttachedKey(self):
+ KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..."
+
+ message = MIMEMultipart()
+ message.add_header("from", ADDRESS_2)
+ key = MIMEApplication("", "pgp-keys")
+ key.set_payload(KEY)
+ message.attach(key)
+ self.fetcher._keymanager.put_raw_key = Mock(
+ return_value=defer.fail(KeyAddressMismatch()))
+
+ def put_raw_key_called(_):
+ self.fetcher._keymanager.put_raw_key.assert_called_once_with(
+ KEY, address=ADDRESS_2)
+
+ d = self._do_fetch(message.as_string())
+ d.addCallback(put_raw_key_called)
+ d.addErrback(log.err)
+ return d
+
+ def testExtractAttachedKeyAndNotOpenPGPHeader(self):
+ KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..."
+ KEYURL = "https://leap.se/key.txt"
+ OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,)
+
+ message = MIMEMultipart()
+ message.add_header("from", ADDRESS_2)
+ message.add_header("OpenPGP", OpenPGP)
+ key = MIMEApplication("", "pgp-keys")
+ key.set_payload(KEY)
+ message.attach(key)
+
+ self.fetcher._keymanager.put_raw_key = Mock(
+ return_value=defer.succeed(None))
+ self.fetcher._keymanager.fetch_key = Mock()
+
+ def put_raw_key_called(_):
+ self.fetcher._keymanager.put_raw_key.assert_called_once_with(
+ KEY, address=ADDRESS_2)
+ self.assertFalse(self.fetcher._keymanager.fetch_key.called)
+
+ d = self._do_fetch(message.as_string())
+ d.addCallback(put_raw_key_called)
+ return d
+
+ def testExtractOpenPGPHeaderIfInvalidAttachedKey(self):
+ KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..."
+ KEYURL = "https://leap.se/key.txt"
+ OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,)
+
+ message = MIMEMultipart()
+ message.add_header("from", ADDRESS_2)
+ message.add_header("OpenPGP", OpenPGP)
+ key = MIMEApplication("", "pgp-keys")
+ key.set_payload(KEY)
+ message.attach(key)
+
+ self.fetcher._keymanager.put_raw_key = Mock(
+ return_value=defer.fail(KeyAddressMismatch()))
+ self.fetcher._keymanager.fetch_key = Mock()
+
+ def put_raw_key_called(_):
+ self.fetcher._keymanager.put_raw_key.assert_called_once_with(
+ KEY, address=ADDRESS_2)
+ self.fetcher._keymanager.fetch_key.assert_called_once_with(
+ ADDRESS_2, KEYURL)
+
+ d = self._do_fetch(message.as_string())
+ d.addCallback(put_raw_key_called)
+ return d
+
+ def testAddDecryptedHeader(self):
+ class DummyMsg():
+
+ def __init__(self):
+ self.headers = {}
+
+ def add_header(self, k, v):
+ self.headers[k] = v
+
+ msg = DummyMsg()
+ self.fetcher._add_decrypted_header(msg)
+
+ self.assertEquals(msg.headers['X-Leap-Encryption'], 'decrypted')
+
+ def testDecryptEmail(self):
+
+ self.fetcher._decryption_error = Mock()
+ self.fetcher._add_decrypted_header = Mock()
+
+ def create_encrypted_message(encstr):
+ message = Parser().parsestr(self.EMAIL)
+ newmsg = MultipartEncrypted('application/pgp-encrypted')
+ for hkey, hval in message.items():
+ newmsg.add_header(hkey, hval)
+
+ encmsg = MIMEApplication(
+ encstr, _subtype='octet-stream', _encoder=lambda x: x)
+ encmsg.add_header('content-disposition', 'attachment',
+ filename='msg.asc')
+ # create meta message
+ metamsg = PGPEncrypted()
+ metamsg.add_header('Content-Disposition', 'attachment')
+ # attach pgp message parts to new message
+ newmsg.attach(metamsg)
+ newmsg.attach(encmsg)
+ return newmsg
+
+ def decryption_error_not_called(_):
+ self.assertFalse(self.fetcher._decryption_error.called,
+ "There was some errors with decryption")
+
+ def add_decrypted_header_called(_):
+ self.assertTrue(self.fetcher._add_decrypted_header.called,
+ "There was some errors with decryption")
+
+ d = self.km.encrypt(self.EMAIL, ADDRESS, sign=ADDRESS_2)
+ d.addCallback(create_encrypted_message)
+ d.addCallback(
+ lambda message:
+ self._do_fetch(message.as_string()))
+ d.addCallback(decryption_error_not_called)
+ d.addCallback(add_decrypted_header_called)
+ return d
+
+ def testValidateSignatureFromEncryptedEmailFromAppleMail(self):
+ enc_signed_file = os.path.join(
+ HERE, 'rfc822.multi-encrypt-signed.message')
+ self.fetcher._add_verified_signature_header = Mock()
+
+ def add_verified_signature_header_called(_):
+ self.assertTrue(self.fetcher._add_verified_signature_header.called,
+ "There was some errors verifying signature")
+
+ with open(enc_signed_file) as f:
+ enc_signed_raw = f.read()
+
+ d = self._do_fetch(enc_signed_raw)
+ d.addCallback(add_verified_signature_header_called)
+ return d
+
+ def testListener(self):
+ self.called = False
+
+ def listener(uid):
+ self.called = True
+
+ def listener_called(_):
+ self.assertTrue(self.called)
+
+ self.fetcher.add_listener(listener)
+ d = self._do_fetch(self.EMAIL)
+ d.addCallback(listener_called)
+ return d
+
+ def _do_fetch(self, message):
+ d = self._create_incoming_email(message)
+ d.addCallback(
+ lambda email:
+ self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email]))
+ d.addCallback(lambda _: self.fetcher.fetch())
+ return d
+
+ def _create_incoming_email(self, email_str):
+ email = SoledadDocument()
+ data = json.dumps(
+ {"incoming": True, "content": email_str},
+ ensure_ascii=False)
+
+ def set_email_content(encr_data):
+ email.content = {
+ fields.INCOMING_KEY: True,
+ fields.ERROR_DECRYPTING_KEY: False,
+ ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY,
+ ENC_JSON_KEY: encr_data
+ }
+ return email
+ d = self.km.encrypt(data, ADDRESS, fetch_remote=False)
+ d.addCallback(set_email_content)
+ return d
+
+ def _mock_soledad_get_from_index(self, index_name, value):
+ get_from_index = self._soledad.get_from_index
+
+ def soledad_mock(idx_name, *key_values):
+ if index_name == idx_name:
+ return defer.succeed(value)
+ return get_from_index(idx_name, *key_values)
+ self.fetcher._soledad.get_from_index = Mock(side_effect=soledad_mock)
diff --git a/src/leap/bitmask/mail/interfaces.py b/src/leap/bitmask/mail/interfaces.py
new file mode 100644
index 00000000..10f51237
--- /dev/null
+++ b/src/leap/bitmask/mail/interfaces.py
@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+# interfaces.py
+# Copyright (C) 2014,2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Interfaces for the leap.mail module.
+"""
+from zope.interface import Interface, Attribute
+
+
+class IMessageWrapper(Interface):
+ """
+ I know how to access the different parts into which a given message is
+ splitted into.
+
+ :ivar fdoc: dict with flag document.
+ :ivar hdoc: dict with flag document.
+ :ivar cdocs: dict with content-documents, one-indexed.
+ """
+
+ fdoc = Attribute('A dictionaly-like containing the flags document '
+ '(mutable)')
+ hdoc = Attribute('A dictionary-like containing the headers document '
+ '(immutable)')
+ cdocs = Attribute('A dictionary with the content-docs, one-indexed')
+
+ def create(self, store, notify_just_mdoc=False, pending_inserts_dict={}):
+ """
+ Create the underlying wrapper.
+ """
+
+ def update(self, store):
+ """
+ Update the only mutable parts, which are within the flags document.
+ """
+
+ def delete(self, store):
+ """
+ Delete the parts for this wrapper that are not referenced from anywhere
+ else.
+ """
+
+ def copy(self, store, new_mbox_uuid):
+ """
+ Return a copy of this IMessageWrapper in a new mailbox.
+ """
+
+ def set_mbox_uuid(self, mbox_uuid):
+ """
+ Set the mailbox for this wrapper.
+ """
+
+ def set_flags(self, flags):
+ """
+ """
+
+ def set_tags(self, tags):
+ """
+ """
+
+ def set_date(self, date):
+ """
+ """
+
+ def get_subpart_dict(self, index):
+ """
+ :param index: the part to lookup, 1-indexed
+ """
+
+ def get_subpart_indexes(self):
+ """
+ """
+
+ def get_body(self, store):
+ """
+ """
+
+
+# TODO -- split into smaller interfaces? separate mailbox interface at least?
+
+class IMailAdaptor(Interface):
+ """
+ I know how to store the standard representation for messages and mailboxes,
+ and how to update the relevant mutable parts when needed.
+ """
+
+ def initialize_store(self, store):
+ """
+ Performs whatever initialization is needed before the store can be
+ used (creating indexes, sanity checks, etc).
+
+ :param store: store
+ :returns: a Deferred that will fire when the store is correctly
+ initialized.
+ :rtype: deferred
+ """
+
+ def get_msg_from_string(self, MessageClass, raw_msg):
+ """
+ Get an instance of a MessageClass initialized with a MessageWrapper
+ that contains all the parts obtained from parsing the raw string for
+ the message.
+
+ :param MessageClass: an implementor of IMessage
+ :type raw_msg: str
+ :rtype: implementor of leap.mail.IMessage
+ """
+
+ def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None,
+ uid=None):
+ """
+ Get an instance of a MessageClass initialized with a MessageWrapper
+ that contains the passed part documents.
+
+ This is not the recommended way of obtaining a message, unless you know
+ how to take care of ensuring the internal consistency between the part
+ documents, or unless you are glueing together the part documents that
+ have been previously generated by `get_msg_from_string`.
+ """
+
+ def get_flags_from_mdoc_id(self, store, mdoc_id):
+ """
+ """
+
+ def create_msg(self, store, msg):
+ """
+ :param store: an instance of soledad, or anything that behaves alike
+ :param msg: a Message object.
+
+ :return: a Deferred that is fired when all the underlying documents
+ have been created.
+ :rtype: defer.Deferred
+ """
+
+ def update_msg(self, store, msg):
+ """
+ :param msg: a Message object.
+ :param store: an instance of soledad, or anything that behaves alike
+ :return: a Deferred that is fired when all the underlying documents
+ have been updated (actually, it's only the fdoc that's allowed
+ to update).
+ :rtype: defer.Deferred
+ """
+
+ def get_count_unseen(self, store, mbox_uuid):
+ """
+ Get the number of unseen messages for a given mailbox.
+
+ :param store: instance of Soledad.
+ :param mbox_uuid: the uuid for this mailbox.
+ :rtype: int
+ """
+
+ def get_count_recent(self, store, mbox_uuid):
+ """
+ Get the number of recent messages for a given mailbox.
+
+ :param store: instance of Soledad.
+ :param mbox_uuid: the uuid for this mailbox.
+ :rtype: int
+ """
+
+ def get_mdoc_id_from_msgid(self, store, mbox_uuid, msgid):
+ """
+ Get the UID for a message with the passed msgid (the one in the headers
+ msg-id).
+ This is used by the MUA to retrieve the recently saved draft.
+ """
+
+ # mbox handling
+
+ def get_or_create_mbox(self, store, name):
+ """
+ Get the mailbox with the given name, or create one if it does not
+ exist.
+
+ :param store: instance of Soledad
+ :param name: the name of the mailbox
+ :type name: str
+ """
+
+ def update_mbox(self, store, mbox_wrapper):
+ """
+ Update the documents for a given mailbox.
+ :param mbox_wrapper: MailboxWrapper instance
+ :type mbox_wrapper: MailboxWrapper
+ :return: a Deferred that will be fired when the mailbox documents
+ have been updated.
+ :rtype: defer.Deferred
+ """
+
+ def delete_mbox(self, store, mbox_wrapper):
+ """
+ """
+
+ def get_all_mboxes(self, store):
+ """
+ Retrieve a list with wrappers for all the mailboxes.
+
+ :return: a deferred that will be fired with a list of all the
+ MailboxWrappers found.
+ :rtype: defer.Deferred
+ """
diff --git a/src/leap/bitmask/mail/load_tests.py b/src/leap/bitmask/mail/load_tests.py
new file mode 100644
index 00000000..be65b8d9
--- /dev/null
+++ b/src/leap/bitmask/mail/load_tests.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# tests.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Provide a function for loading tests.
+"""
+import unittest
+
+
+def load_tests():
+ suite = unittest.TestSuite()
+ for test in unittest.defaultTestLoader.discover(
+ './src/leap/mail/',
+ top_level_dir='./src/'):
+ suite.addTest(test)
+ return suite
diff --git a/src/leap/bitmask/mail/mail.py b/src/leap/bitmask/mail/mail.py
new file mode 100644
index 00000000..2fde3a1b
--- /dev/null
+++ b/src/leap/bitmask/mail/mail.py
@@ -0,0 +1,1070 @@
+# -*- coding: utf-8 -*-
+# mail.py
+# Copyright (C) 2014,2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Generic Access to Mail objects.
+
+This module holds the public LEAP Mail API, which should be viewed as the main
+entry point for message and account manipulation, in a protocol-agnostic way.
+
+In the future, pluggable transports will expose this generic API.
+"""
+import itertools
+import uuid
+import logging
+import StringIO
+import time
+import weakref
+
+from collections import defaultdict
+
+from twisted.internet import defer
+from twisted.python import log
+
+from leap.common.check import leap_assert_type
+from leap.common.events import emit_async, catalog
+
+from leap.mail.adaptors.soledad import SoledadMailAdaptor
+from leap.mail.constants import INBOX_NAME
+from leap.mail.constants import MessageFlags
+from leap.mail.mailbox_indexer import MailboxIndexer
+from leap.mail.plugins import soledad_sync_hooks
+from leap.mail.utils import find_charset, CaseInsensitiveDict
+from leap.mail.utils import lowerdict
+
+logger = logging.getLogger(name=__name__)
+
+
+# TODO LIST
+# [ ] Probably change the name of this module to "api" or "account", mail is
+# too generic (there's also IncomingMail, and OutgoingMail
+# [ ] Profile add_msg.
+
+def _get_mdoc_id(mbox, chash):
+ """
+ Get the doc_id for the metamsg document.
+ """
+ return "M+{mbox}+{chash}".format(mbox=mbox, chash=chash)
+
+
+def _write_and_rewind(payload):
+ fd = StringIO.StringIO()
+ fd.write(payload)
+ fd.seek(0)
+ return fd
+
+
+def _encode_payload(payload, ctype=""):
+ """
+ Properly encode an unicode payload (which can be string or unicode) as a
+ string.
+
+ :param payload: the payload to encode. currently soledad returns unicode
+ strings.
+ :type payload: basestring
+ :param ctype: optional, the content of the content-type header for this
+ payload.
+ :type ctype: str
+ :rtype: str
+ """
+ # TODO Related, it's proposed that we're able to pass
+ # the encoding to the soledad documents. Better to store the charset there?
+ # FIXME -----------------------------------------------
+ # this need a dedicated test-suite
+ charset = find_charset(ctype)
+
+ # XXX get from mail headers if not multipart!
+ # Beware also that we should pass the proper encoding to
+ # soledad when it's creating the documents.
+ # if not charset:
+ # charset = get_email_charset(payload)
+ # -----------------------------------------------------
+
+ if not charset:
+ charset = "utf-8"
+
+ try:
+ if isinstance(payload, unicode):
+ payload = payload.encode(charset)
+ except UnicodeError as exc:
+ logger.error(
+ "Unicode error, using 'replace'. {0!r}".format(exc))
+ payload = payload.encode(charset, 'replace')
+ return payload
+
+
+def _unpack_headers(headers_dict):
+ """
+ Take a "packed" dict containing headers (with repeated keys represented as
+ line breaks inside each value, preceded by the header key) and return a
+ list of tuples in which each repeated key has a different tuple.
+ """
+ headers_l = headers_dict.items()
+ for i, (k, v) in enumerate(headers_l):
+ splitted = v.split(k.lower() + ": ")
+ if len(splitted) != 1:
+ inner = zip(
+ itertools.cycle([k]),
+ map(lambda l: l.rstrip('\n'), splitted))
+ headers_l = headers_l[:i] + inner + headers_l[i + 1:]
+ return headers_l
+
+
+class MessagePart(object):
+ # TODO This class should be better abstracted from the data model.
+ # TODO support arbitrarily nested multiparts (right now we only support
+ # the trivial case)
+ """
+ Represents a part of a multipart MIME Message.
+ """
+
+ def __init__(self, part_map, cdocs=None, nested=False):
+ """
+ :param part_map: a dictionary mapping the subparts for
+ this MessagePart (1-indexed).
+ :type part_map: dict
+
+ The format for the part_map is as follows:
+
+ {u'ctype': u'text/plain',
+ u'headers': [[u'Content-Type', u'text/plain; charset="utf-8"'],
+ [u'Content-Transfer-Encoding', u'8bit']],
+ u'multi': False,
+ u'parts': 1,
+ u'phash': u'02D82B29F6BB0C8612D1C',
+ u'size': 132}
+
+ :param cdocs: optional, a reference to the top-level dict of wrappers
+ for content-docs (1-indexed).
+ """
+ if cdocs is None:
+ cdocs = {}
+ self._pmap = part_map
+ self._cdocs = cdocs
+ self._nested = nested
+
+ def get_size(self):
+ """
+ Size of the body, in octets.
+ """
+ total = self._pmap['size']
+ _h = self.get_headers()
+ headers = len(
+ '\n'.join(["%s: %s" % (k, v) for k, v in dict(_h).items()]))
+ # have to subtract 2 blank lines
+ return total - headers - 2
+
+ def get_body_file(self):
+ payload = ""
+ pmap = self._pmap
+
+ multi = pmap.get('multi')
+ if not multi:
+ payload = self._get_payload(pmap.get('phash'))
+ if payload:
+ payload = _encode_payload(payload)
+
+ return _write_and_rewind(payload)
+
+ def get_headers(self):
+ return CaseInsensitiveDict(self._pmap.get("headers", []))
+
+ def is_multipart(self):
+ return self._pmap.get("multi", False)
+
+ def get_subpart(self, part):
+ if not self.is_multipart():
+ raise TypeError
+ sub_pmap = self._pmap.get("part_map", {})
+
+ try:
+ part_map = sub_pmap[str(part)]
+ except KeyError:
+ log.msg("getSubpart for %s: KeyError" % (part,))
+ raise IndexError
+ return MessagePart(part_map, cdocs=self._cdocs, nested=True)
+
+ def _get_payload(self, phash):
+ for cdocw in self._cdocs.values():
+ if cdocw.phash == phash:
+ return cdocw.raw
+ return ""
+
+
+class Message(object):
+ """
+ Represents a single message, and gives access to all its attributes.
+ """
+
+ def __init__(self, wrapper, uid=None):
+ """
+ :param wrapper: an instance of an implementor of IMessageWrapper
+ :param uid:
+ :type uid: int
+ """
+ self._wrapper = wrapper
+ self._uid = uid
+
+ def get_wrapper(self):
+ """
+ Get the wrapper for this message.
+ """
+ return self._wrapper
+
+ def get_uid(self):
+ """
+ Get the (optional) UID.
+ """
+ return self._uid
+
+ # imap.IMessage methods
+
+ def get_flags(self):
+ """
+ Get flags for this message.
+ :rtype: tuple
+ """
+ return self._wrapper.fdoc.get_flags()
+
+ def get_internal_date(self):
+ """
+ Retrieve the date internally associated with this message
+
+ According to the spec, this is NOT the date and time in the
+ RFC-822 header, but rather a date and time that reflects when the
+ message was received.
+
+ * In SMTP, date and time of final delivery.
+ * In COPY, internal date/time of the source message.
+ * In APPEND, date/time specified.
+
+ :return: An RFC822-formatted date string.
+ :rtype: str
+ """
+ return self._wrapper.hdoc.date
+
+ # imap.IMessageParts
+
+ def get_headers(self):
+ """
+ Get the raw headers document.
+ """
+ return CaseInsensitiveDict(self._wrapper.hdoc.headers)
+
+ def get_body_file(self, store):
+ """
+ Get a file descriptor with the body content.
+ """
+ def write_and_rewind_if_found(cdoc):
+ payload = cdoc.raw if cdoc else ""
+ # XXX pass ctype from headers if not multipart?
+ if payload:
+ payload = _encode_payload(payload, ctype=cdoc.content_type)
+ return _write_and_rewind(payload)
+
+ d = defer.maybeDeferred(self._wrapper.get_body, store)
+ d.addCallback(write_and_rewind_if_found)
+ return d
+
+ def get_size(self):
+ """
+ Size of the whole message, in octets (including headers).
+ """
+ total = self._wrapper.fdoc.size
+ return total
+
+ def is_multipart(self):
+ """
+ Return True if this message is multipart.
+ """
+ return self._wrapper.fdoc.multi
+
+ def get_subpart(self, part):
+ """
+ :param part: The number of the part to retrieve, indexed from 1.
+ :type part: int
+ :rtype: MessagePart
+ """
+ if not self.is_multipart():
+ raise TypeError
+ try:
+ subpart_dict = self._wrapper.get_subpart_dict(part)
+ except KeyError:
+ raise IndexError
+
+ return MessagePart(
+ subpart_dict, cdocs=self._wrapper.cdocs)
+
+ # Custom methods.
+
+ def get_tags(self):
+ """
+ Get the tags for this message.
+ """
+ return tuple(self._wrapper.fdoc.tags)
+
+
+class Flagsmode(object):
+ """
+ Modes for setting the flags/tags.
+ """
+ APPEND = 1
+ REMOVE = -1
+ SET = 0
+
+
+class MessageCollection(object):
+ """
+ A generic collection of messages. It can be messages sharing the same
+ mailbox, tag, the result of a given query, or just a bunch of ids for
+ master documents.
+
+ Since LEAP Mail is primarily oriented to store mail in Soledad, the default
+ (and, so far, only) implementation of the store is contained in the
+ Soledad Mail Adaptor, which is passed to every collection on creation by
+ the root Account object. If you need to use a different adaptor, change the
+ adaptor class attribute in your Account object.
+
+ Store is a reference to a particular instance of the message store (soledad
+ instance or proxy, for instance).
+ """
+
+ # TODO LIST
+ # [ ] look at IMessageSet methods
+ # [ ] make constructor with a per-instance deferredLock to use on
+ # creation/deletion?
+ # [ ] instead of a mailbox, we could pass an arbitrary container with
+ # pointers to different doc_ids (type: foo)
+ # [ ] To guarantee synchronicity of the documents sent together during a
+ # sync, we could get hold of a deferredLock that inhibits
+ # synchronization while we are updating (think more about this!)
+ # [ ] review the serveral count_ methods. I think it's better to patch
+ # server to accept deferreds.
+ # [ ] Use inheritance for the mailbox-collection instead of handling the
+ # special cases everywhere?
+ # [ ] or maybe a mailbox_only decorator...
+
+ # Account should provide an adaptor instance when creating this collection.
+ adaptor = None
+ store = None
+ messageklass = Message
+
+ _pending_inserts = dict()
+
+ def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None):
+ """
+ Constructor for a MessageCollection.
+ """
+ self.adaptor = adaptor
+ self.store = store
+
+ # XXX think about what to do when there is no mbox passed to
+ # the initialization. We could still get the MetaMsg by index, instead
+ # of by doc_id. See get_message_by_content_hash
+ self.mbox_indexer = mbox_indexer
+ self.mbox_wrapper = mbox_wrapper
+ self._listeners = set([])
+
+ def is_mailbox_collection(self):
+ """
+ Return True if this collection represents a Mailbox.
+ :rtype: bool
+ """
+ return bool(self.mbox_wrapper)
+
+ @property
+ def mbox_name(self):
+ # TODO raise instead?
+ if self.mbox_wrapper is None:
+ return None
+ return self.mbox_wrapper.mbox
+
+ @property
+ def mbox_uuid(self):
+ # TODO raise instead?
+ if self.mbox_wrapper is None:
+ return None
+ return self.mbox_wrapper.uuid
+
+ def get_mbox_attr(self, attr):
+ if self.mbox_wrapper is None:
+ raise RuntimeError("This is not a mailbox collection")
+ return getattr(self.mbox_wrapper, attr)
+
+ def set_mbox_attr(self, attr, value):
+ if self.mbox_wrapper is None:
+ raise RuntimeError("This is not a mailbox collection")
+ setattr(self.mbox_wrapper, attr, value)
+ return self.mbox_wrapper.update(self.store)
+
+ # Get messages
+
+ def get_message_by_content_hash(self, chash, get_cdocs=False):
+ """
+ Retrieve a message by its content hash.
+ :rtype: Deferred
+ """
+ if not self.is_mailbox_collection():
+ # TODO instead of getting the metamsg by chash, in this case we
+ # should query by (meta) index or use the internal collection of
+ # pointers-to-docs.
+ raise NotImplementedError()
+
+ metamsg_id = _get_mdoc_id(self.mbox_name, chash)
+
+ return self.adaptor.get_msg_from_mdoc_id(
+ self.messageklass, self.store,
+ metamsg_id, get_cdocs=get_cdocs)
+
+ def get_message_by_sequence_number(self, msn, get_cdocs=False):
+ """
+ Retrieve a message by its Message Sequence Number.
+ :rtype: Deferred
+ """
+ def get_uid_for_msn(all_uid):
+ return all_uid[msn - 1]
+ d = self.all_uid_iter()
+ d.addCallback(get_uid_for_msn)
+ d.addCallback(
+ lambda uid: self.get_message_by_uid(
+ uid, get_cdocs=get_cdocs))
+ d.addErrback(lambda f: log.err(f))
+ return d
+
+ def get_message_by_uid(self, uid, absolute=True, get_cdocs=False):
+ """
+ Retrieve a message by its Unique Identifier.
+
+ If this is a Mailbox collection, that is the message UID, unique for a
+ given mailbox, or a relative sequence number depending on the absolute
+ flag. For now, only absolute identifiers are supported.
+ :rtype: Deferred
+ """
+ # TODO deprecate absolute flag, it doesn't make sense UID and
+ # !absolute. use _by_sequence_number instead.
+ if not absolute:
+ raise NotImplementedError("Does not support relative ids yet")
+
+ get_doc_fun = self.mbox_indexer.get_doc_id_from_uid
+
+ def get_msg_from_mdoc_id(doc_id):
+ if doc_id is None:
+ return None
+ return self.adaptor.get_msg_from_mdoc_id(
+ self.messageklass, self.store,
+ doc_id, uid=uid, get_cdocs=get_cdocs)
+
+ def cleanup_and_get_doc_after_pending_insert(result):
+ for key in result:
+ self._pending_inserts.pop(key, None)
+ return get_doc_fun(self.mbox_uuid, uid)
+
+ if not self._pending_inserts:
+ d = get_doc_fun(self.mbox_uuid, uid)
+ else:
+ d = defer.gatherResults(self._pending_inserts.values())
+ d.addCallback(cleanup_and_get_doc_after_pending_insert)
+ d.addCallback(get_msg_from_mdoc_id)
+ return d
+
+ def get_flags_by_uid(self, uid, absolute=True):
+ # TODO use sequence numbers
+ if not absolute:
+ raise NotImplementedError("Does not support relative ids yet")
+
+ def get_flags_from_mdoc_id(doc_id):
+ if doc_id is None: # XXX needed? or bug?
+ return None
+ return self.adaptor.get_flags_from_mdoc_id(
+ self.store, doc_id)
+
+ def wrap_in_tuple(flags):
+ return (uid, flags)
+
+ d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_uuid, uid)
+ d.addCallback(get_flags_from_mdoc_id)
+ d.addCallback(wrap_in_tuple)
+ return d
+
+ def count(self):
+ """
+ Count the messages in this collection.
+ :return: a Deferred that will fire with the integer for the count.
+ :rtype: Deferred
+ """
+ if not self.is_mailbox_collection():
+ raise NotImplementedError()
+
+ d = self.mbox_indexer.count(self.mbox_uuid)
+ return d
+
+ def count_recent(self):
+ """
+ Count the recent messages in this collection.
+ :return: a Deferred that will fire with the integer for the count.
+ :rtype: Deferred
+ """
+ if not self.is_mailbox_collection():
+ raise NotImplementedError()
+ return self.adaptor.get_count_recent(self.store, self.mbox_uuid)
+
+ def count_unseen(self):
+ """
+ Count the unseen messages in this collection.
+ :return: a Deferred that will fire with the integer for the count.
+ :rtype: Deferred
+ """
+ if not self.is_mailbox_collection():
+ raise NotImplementedError()
+ return self.adaptor.get_count_unseen(self.store, self.mbox_uuid)
+
+ def get_uid_next(self):
+ """
+ Get the next integer beyond the highest UID count for this mailbox.
+
+ :return: a Deferred that will fire with the integer for the next uid.
+ :rtype: Deferred
+ """
+ return self.mbox_indexer.get_next_uid(self.mbox_uuid)
+
+ def get_last_uid(self):
+ """
+ Get the last UID for this mailbox.
+ """
+ return self.mbox_indexer.get_last_uid(self.mbox_uuid)
+
+ def all_uid_iter(self):
+ """
+ Iterator through all the uids for this collection.
+ """
+ return self.mbox_indexer.all_uid_iter(self.mbox_uuid)
+
+ def get_uid_from_msgid(self, msgid):
+ """
+ Return the UID(s) of the matching msg-ids for this mailbox collection.
+ """
+ if not self.is_mailbox_collection():
+ raise NotImplementedError()
+
+ def get_uid(mdoc_id):
+ if not mdoc_id:
+ return None
+ d = self.mbox_indexer.get_uid_from_doc_id(
+ self.mbox_uuid, mdoc_id)
+ return d
+
+ d = self.adaptor.get_mdoc_id_from_msgid(
+ self.store, self.mbox_uuid, msgid)
+ d.addCallback(get_uid)
+ return d
+
+ # Manipulate messages
+
+ def add_msg(self, raw_msg, flags=tuple(), tags=tuple(), date="",
+ notify_just_mdoc=False):
+ """
+ Add a message to this collection.
+
+ :param raw_msg: the raw message
+ :param flags: tuple of flags for this message
+ :param tags: tuple of tags for this message
+ :param date:
+ formatted date, it will be used to retrieve the internal
+ date for this message. According to the spec, this is NOT the date
+ and time in the RFC-822 header, but rather a date and time that
+ reflects when the message was received.
+ :type date: str
+ :param notify_just_mdoc:
+ boolean passed to the wrapper.create method, to indicate whether
+ we're insterested in being notified right after the mdoc has been
+ written (as it's the first doc to be written, and quite small, this
+ is faster, though potentially unsafe), or on the contrary we want
+ to wait untill all the parts have been written.
+ Used by the imap mailbox implementation to get faster responses.
+ This will be ignored (and set to False) if a heuristic for a Draft
+ message is met, which currently is a specific mozilla header.
+ :type notify_just_mdoc: bool
+
+ :returns: a deferred that will fire with the UID of the inserted
+ message.
+ :rtype: deferred
+ """
+ # TODO watch out if the use of this method in IMAP COPY/APPEND is
+ # passing the right date.
+ # XXX mdoc ref is a leaky abstraction here. generalize.
+ leap_assert_type(flags, tuple)
+ leap_assert_type(date, str)
+
+ msg = self.adaptor.get_msg_from_string(Message, raw_msg)
+ wrapper = msg.get_wrapper()
+
+ headers = lowerdict(msg.get_headers())
+ moz_draft_hdr = "X-Mozilla-Draft-Info"
+ if moz_draft_hdr.lower() in headers:
+ log.msg("Setting fast notify to False, Draft detected")
+ notify_just_mdoc = False
+
+ if notify_just_mdoc:
+ msgid = headers.get('message-id')
+ if msgid:
+ self._pending_inserts[msgid] = defer.Deferred()
+
+ if not self.is_mailbox_collection():
+ raise NotImplementedError()
+
+ else:
+ mbox_id = self.mbox_uuid
+ wrapper.set_mbox_uuid(mbox_id)
+ wrapper.set_flags(flags)
+ wrapper.set_tags(tags)
+ wrapper.set_date(date)
+
+ def insert_mdoc_id(_, wrapper):
+ doc_id = wrapper.mdoc.doc_id
+ if not doc_id:
+ # --- BUG -----------------------------------------
+ # XXX watch out, sometimes mdoc doesn't have doc_id
+ # but it has future_id. Should be solved already.
+ logger.error("BUG: (please report) Null doc_id for "
+ "document %s" %
+ (wrapper.mdoc.serialize(),))
+ return defer.succeed("mdoc_id not inserted")
+ # XXX BUG -----------------------------------------
+
+ # XXX BUG sometimes the table is not yet created,
+ # so workaround is to make sure we always check for it before
+ # inserting the doc. I should debug into the real cause.
+ d = self.mbox_indexer.create_table(self.mbox_uuid)
+ d.addBoth(lambda _: self.mbox_indexer.insert_doc(
+ self.mbox_uuid, doc_id))
+ return d
+
+ d = wrapper.create(
+ self.store,
+ notify_just_mdoc=notify_just_mdoc,
+ pending_inserts_dict=self._pending_inserts)
+ d.addCallback(insert_mdoc_id, wrapper)
+ d.addCallback(self.cb_signal_unread_to_ui)
+ d.addCallback(self.notify_new_to_listeners)
+ d.addErrback(lambda failure: log.err(failure))
+
+ return d
+
+ # Listeners
+
+ def addListener(self, listener):
+ self._listeners.add(listener)
+
+ def removeListener(self, listener):
+ self._listeners.remove(listener)
+
+ def notify_new_to_listeners(self, result):
+ for listener in self._listeners:
+ listener.notify_new()
+ return result
+
+ def cb_signal_unread_to_ui(self, result):
+ """
+ Sends an unread event to ui, passing *only* the number of unread
+ messages if *this* is the inbox. This event is catched, for instance,
+ in the Bitmask client that displays a message with the number of unread
+ mails in the INBOX.
+
+ Used as a callback in several commands.
+
+ :param result: ignored
+ """
+ # TODO it might make sense to modify the event so that
+ # it receives both the mailbox name AND the number of unread messages.
+ if self.mbox_name.lower() == "inbox":
+ d = defer.maybeDeferred(self.count_unseen)
+ d.addCallback(self.__cb_signal_unread_to_ui)
+ return result
+
+ def __cb_signal_unread_to_ui(self, unseen):
+ """
+ Send the unread signal to UI.
+ :param unseen: number of unseen messages.
+ :type unseen: int
+ """
+ emit_async(catalog.MAIL_UNREAD_MESSAGES, self.store.uuid, str(unseen))
+
+ def copy_msg(self, msg, new_mbox_uuid):
+ """
+ Copy the message to another collection. (it only makes sense for
+ mailbox collections)
+ """
+ # TODO should CHECK first if the mdoc is present in the mailbox
+ # WITH a Deleted flag... and just simply remove the flag...
+ # Another option is to delete the previous mdoc if it already exists
+ # (so we get a new UID)
+
+ if not self.is_mailbox_collection():
+ raise NotImplementedError()
+
+ def delete_mdoc_entry_and_insert(failure, mbox_uuid, doc_id):
+ d = self.mbox_indexer.delete_doc_by_hash(mbox_uuid, doc_id)
+ d.addCallback(lambda _: self.mbox_indexer.insert_doc(
+ new_mbox_uuid, doc_id))
+ return d
+
+ def insert_copied_mdoc_id(wrapper_new_msg):
+ # XXX FIXME -- since this is already saved, the future_doc_id
+ # should be already copied into the doc_id!
+ # Investigate why we are not receiving the already saved doc_id
+ doc_id = wrapper_new_msg.mdoc.doc_id
+ if not doc_id:
+ doc_id = wrapper_new_msg.mdoc._future_doc_id
+
+ def insert_conditionally(uid, mbox_uuid, doc_id):
+ indexer = self.mbox_indexer
+ if uid:
+ d = indexer.delete_doc_by_hash(mbox_uuid, doc_id)
+ d.addCallback(lambda _: indexer.insert_doc(
+ new_mbox_uuid, doc_id))
+ return d
+ else:
+ d = indexer.insert_doc(mbox_uuid, doc_id)
+ return d
+
+ def log_result(result):
+ return result
+
+ def insert_doc(_, mbox_uuid, doc_id):
+ d = self.mbox_indexer.get_uid_from_doc_id(mbox_uuid, doc_id)
+ d.addCallback(insert_conditionally, mbox_uuid, doc_id)
+ d.addErrback(lambda err: log.failure(err))
+ d.addCallback(log_result)
+ return d
+
+ d = self.mbox_indexer.create_table(new_mbox_uuid)
+ d.addBoth(insert_doc, new_mbox_uuid, doc_id)
+ return d
+
+ wrapper = msg.get_wrapper()
+
+ d = wrapper.copy(self.store, new_mbox_uuid)
+ d.addCallback(insert_copied_mdoc_id)
+ d.addCallback(self.notify_new_to_listeners)
+ return d
+
+ def delete_msg(self, msg):
+ """
+ Delete this message.
+ """
+ wrapper = msg.get_wrapper()
+
+ def delete_mdoc_id(_, wrapper):
+ doc_id = wrapper.mdoc.doc_id
+ return self.mbox_indexer.delete_doc_by_hash(
+ self.mbox_uuid, doc_id)
+ d = wrapper.delete(self.store)
+ d.addCallback(delete_mdoc_id, wrapper)
+ return d
+
+ def delete_all_flagged(self):
+ """
+ Delete all messages flagged as \\Deleted.
+ Used from IMAPMailbox.expunge()
+ """
+ def get_uid_list(hashes):
+ d = []
+ for h in hashes:
+ d.append(self.mbox_indexer.get_uid_from_doc_id(
+ self.mbox_uuid, h))
+ return defer.gatherResults(d), hashes
+
+ def delete_uid_entries((uids, hashes)):
+ d = []
+ for h in hashes:
+ d.append(self.mbox_indexer.delete_doc_by_hash(
+ self.mbox_uuid, h))
+
+ def return_uids_when_deleted(ignored):
+ return uids
+
+ all_deleted = defer.gatherResults(d).addCallback(
+ return_uids_when_deleted)
+ return all_deleted
+
+ mdocs_deleted = self.adaptor.del_all_flagged_messages(
+ self.store, self.mbox_uuid)
+ mdocs_deleted.addCallback(get_uid_list)
+ mdocs_deleted.addCallback(delete_uid_entries)
+ mdocs_deleted.addErrback(lambda f: log.err(f))
+ return mdocs_deleted
+
+ # TODO should add a delete-by-uid to collection?
+
+ def delete_all_docs(self):
+ def del_all_uid(uid_list):
+ deferreds = []
+ for uid in uid_list:
+ d = self.get_message_by_uid(uid)
+ d.addCallback(lambda msg: msg.delete())
+ deferreds.append(d)
+ return defer.gatherResults(deferreds)
+
+ d = self.all_uid_iter()
+ d.addCallback(del_all_uid)
+ return d
+
+ def update_flags(self, msg, flags, mode):
+ """
+ Update flags for a given message.
+ """
+ wrapper = msg.get_wrapper()
+ current = wrapper.fdoc.flags
+ newflags = map(str, self._update_flags_or_tags(current, flags, mode))
+ wrapper.fdoc.flags = newflags
+
+ wrapper.fdoc.seen = MessageFlags.SEEN_FLAG in newflags
+ wrapper.fdoc.deleted = MessageFlags.DELETED_FLAG in newflags
+
+ d = self.adaptor.update_msg(self.store, msg)
+ d.addCallback(lambda _: newflags)
+ return d
+
+ def update_tags(self, msg, tags, mode):
+ """
+ Update tags for a given message.
+ """
+ wrapper = msg.get_wrapper()
+ current = wrapper.fdoc.tags
+ newtags = self._update_flags_or_tags(current, tags, mode)
+
+ wrapper.fdoc.tags = newtags
+ d = self.adaptor.update_msg(self.store, msg)
+ d.addCallback(newtags)
+ return d
+
+ def _update_flags_or_tags(self, old, new, mode):
+ if mode == Flagsmode.APPEND:
+ final = list((set(tuple(old) + new)))
+ elif mode == Flagsmode.REMOVE:
+ final = list(set(old).difference(set(new)))
+ elif mode == Flagsmode.SET:
+ final = new
+ return final
+
+
+class Account(object):
+ """
+ Account is the top level abstraction to access collections of messages
+ associated with a LEAP Mail Account.
+
+ It primarily handles creation and access of Mailboxes, which will be the
+ basic collection handled by traditional MUAs, but it can also handle other
+ types of Collections (tag based, for instance).
+
+ leap.mail.imap.IMAPAccount partially proxies methods in this
+ class.
+ """
+
+ # Adaptor is passed to the returned MessageCollections, so if you want to
+ # use a different adaptor this is the place to change it, by subclassing
+ # the Account class.
+
+ adaptor_class = SoledadMailAdaptor
+
+ # this is a defaultdict, indexed by userid, that returns a
+ # WeakValueDictionary mapping to collection instances so that we always
+ # return a reference to them instead of creating new ones. however,
+ # being a dictionary of weakrefs values, they automagically vanish
+ # from the dict when no hard refs is left to them (so they can be
+ # garbage collected) this is important because the different wrappers
+ # rely on several kinds of deferredlocks that are kept as class or
+ # instance variables.
+
+ # We need it to be a class property because we create more than one Account
+ # object in the current usage pattern (ie, one in the mail service, and
+ # another one in the IncomingMailService). When we move to a proper service
+ # tree we can let it be an instance attribute.
+ _collection_mapping = defaultdict(weakref.WeakValueDictionary)
+
+ def __init__(self, store, user_id, ready_cb=None):
+ self.store = store
+ self.user_id = user_id
+ self.adaptor = self.adaptor_class()
+
+ self.mbox_indexer = MailboxIndexer(self.store)
+
+ # This flag is only used from the imap service for the moment.
+ # In the future, we should prevent any public method to continue if
+ # this is set to True. Also, it would be good to plug to the
+ # authentication layer.
+ self.session_ended = False
+
+ self.deferred_initialization = defer.Deferred()
+ self._ready_cb = ready_cb
+
+ self._init_d = self._initialize_storage()
+ self._initialize_sync_hooks()
+
+ def _initialize_storage(self):
+
+ def add_mailbox_if_none(mboxes):
+ if not mboxes:
+ return self.add_mailbox(INBOX_NAME)
+
+ def finish_initialization(result):
+ self.deferred_initialization.callback(None)
+ if self._ready_cb is not None:
+ self._ready_cb()
+
+ d = self.adaptor.initialize_store(self.store)
+ d.addCallback(lambda _: self.list_all_mailbox_names())
+ d.addCallback(add_mailbox_if_none)
+ d.addCallback(finish_initialization)
+ return d
+
+ def callWhenReady(self, cb, *args, **kw):
+ """
+ Execute the callback when the initialization of the Account is ready.
+ Note that the callback will receive a first meaningless parameter.
+ """
+ # TODO this should ignore the first parameter explicitely
+ # lambda _: cb(*args, **kw)
+ self.deferred_initialization.addCallback(cb, *args, **kw)
+ return self.deferred_initialization
+
+ # Sync hooks
+
+ def _initialize_sync_hooks(self):
+ soledad_sync_hooks.post_sync_uid_reindexer.set_account(self)
+
+ def _teardown_sync_hooks(self):
+ soledad_sync_hooks.post_sync_uid_reindexer.set_account(None)
+
+ #
+ # Public API Starts
+ #
+
+ def list_all_mailbox_names(self):
+
+ def filter_names(mboxes):
+ return [m.mbox for m in mboxes]
+
+ d = self.get_all_mailboxes()
+ d.addCallback(filter_names)
+ return d
+
+ def get_all_mailboxes(self):
+ d = self.adaptor.get_all_mboxes(self.store)
+ return d
+
+ def add_mailbox(self, name, creation_ts=None):
+
+ if creation_ts is None:
+ # by default, we pass an int value
+ # taken from the current time
+ # we make sure to take enough decimals to get a unique
+ # mailbox-uidvalidity.
+ creation_ts = int(time.time() * 10E2)
+
+ def set_creation_ts(wrapper):
+ wrapper.created = creation_ts
+ d = wrapper.update(self.store)
+ d.addCallback(lambda _: wrapper)
+ return d
+
+ def create_uuid(wrapper):
+ if not wrapper.uuid:
+ wrapper.uuid = str(uuid.uuid4())
+ d = wrapper.update(self.store)
+ d.addCallback(lambda _: wrapper)
+ return d
+ return wrapper
+
+ def create_uid_table_cb(wrapper):
+ d = self.mbox_indexer.create_table(wrapper.uuid)
+ d.addCallback(lambda _: wrapper)
+ return d
+
+ d = self.adaptor.get_or_create_mbox(self.store, name)
+ d.addCallback(set_creation_ts)
+ d.addCallback(create_uuid)
+ d.addCallback(create_uid_table_cb)
+ return d
+
+ def delete_mailbox(self, name):
+
+ def delete_uid_table_cb(wrapper):
+ d = self.mbox_indexer.delete_table(wrapper.uuid)
+ d.addCallback(lambda _: wrapper)
+ return d
+
+ d = self.adaptor.get_or_create_mbox(self.store, name)
+ d.addCallback(delete_uid_table_cb)
+ d.addCallback(
+ lambda wrapper: self.adaptor.delete_mbox(self.store, wrapper))
+ return d
+
+ def rename_mailbox(self, oldname, newname):
+
+ def _rename_mbox(wrapper):
+ wrapper.mbox = newname
+ d = wrapper.update(self.store)
+ d.addCallback(lambda result: wrapper)
+ return d
+
+ d = self.adaptor.get_or_create_mbox(self.store, oldname)
+ d.addCallback(_rename_mbox)
+ return d
+
+ # Get Collections
+
+ def get_collection_by_mailbox(self, name):
+ """
+ :rtype: deferred
+ :return: a deferred that will fire with a MessageCollection
+ """
+ collection = self._collection_mapping[self.user_id].get(
+ name, None)
+ if collection:
+ return defer.succeed(collection)
+
+ # imap select will use this, passing the collection to SoledadMailbox
+ def get_collection_for_mailbox(mbox_wrapper):
+ collection = MessageCollection(
+ self.adaptor, self.store, self.mbox_indexer, mbox_wrapper)
+ self._collection_mapping[self.user_id][name] = collection
+ return collection
+
+ d = self.adaptor.get_or_create_mbox(self.store, name)
+ d.addCallback(get_collection_for_mailbox)
+ return d
+
+ def get_collection_by_docs(self, docs):
+ """
+ :rtype: MessageCollection
+ """
+ # get a collection of docs by a list of doc_id
+ # get.docs(...) --> it should be a generator. does it behave in the
+ # threadpool?
+ raise NotImplementedError()
+
+ def get_collection_by_tag(self, tag):
+ """
+ :rtype: MessageCollection
+ """
+ raise NotImplementedError()
+
+ # Session handling
+
+ def end_session(self):
+ self._teardown_sync_hooks()
+ self.session_ended = True
diff --git a/src/leap/bitmask/mail/mailbox_indexer.py b/src/leap/bitmask/mail/mailbox_indexer.py
new file mode 100644
index 00000000..c49f808b
--- /dev/null
+++ b/src/leap/bitmask/mail/mailbox_indexer.py
@@ -0,0 +1,327 @@
+# -*- coding: utf-8 -*-
+# mailbox_indexer.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+.. :py:module::mailbox_indexer
+
+Local tables to store the message Unique Identifiers for a given mailbox.
+"""
+import re
+import uuid
+
+from leap.mail.constants import METAMSGID_RE
+
+
+def _maybe_first_query_item(thing):
+ """
+ Return the first item the returned query result, or None
+ if empty.
+ """
+ try:
+ return thing[0][0]
+ except (TypeError, IndexError):
+ return None
+
+
+class WrongMetaDocIDError(Exception):
+ pass
+
+
+def sanitize(mailbox_uuid):
+ return mailbox_uuid.replace("-", "_")
+
+
+def check_good_uuid(mailbox_uuid):
+ """
+ Check that the passed mailbox identifier is a valid UUID.
+ :param mailbox_uuid: the uuid to check
+ :type mailbox_uuid: str
+ :return: None
+ :raises: AssertionError if a wrong uuid was passed.
+ """
+ try:
+ uuid.UUID(str(mailbox_uuid))
+ except (AttributeError, ValueError):
+ raise AssertionError(
+ "the mbox_id is not a valid uuid: %s" % mailbox_uuid)
+
+
+class MailboxIndexer(object):
+ """
+ This class contains the commands needed to create, modify and alter the
+ local-only UID tables for a given mailbox.
+
+ Its purpouse is to keep a local-only index with the messages in each
+ mailbox, mainly to satisfy the demands of the IMAP specification, but
+ useful too for any effective listing of the messages in a mailbox.
+
+ Since the incoming mail can be processed at any time in any replica, it's
+ preferred not to attempt to maintain a global chronological global index.
+
+ These indexes are Message Attributes needed for the IMAP specification (rfc
+ 3501), although they can be useful for other non-imap store
+ implementations.
+
+ """
+ # The uids are expected to be 32-bits values, but the ROWIDs in sqlite
+ # are 64-bit values. I *don't* think it really matters for any
+ # practical use, but it's good to remember we've got that difference going
+ # on.
+
+ store = None
+ table_preffix = "leapmail_uid_"
+
+ def __init__(self, store):
+ self.store = store
+
+ def _query(self, *args, **kw):
+ assert self.store is not None
+ return self.store.raw_sqlcipher_query(*args, **kw)
+
+ def _operation(self, *args, **kw):
+ assert self.store is not None
+ return self.store.raw_sqlcipher_operation(*args, **kw)
+
+ def create_table(self, mailbox_uuid):
+ """
+ Create the UID table for a given mailbox.
+ :param mailbox: the mailbox identifier.
+ :type mailbox: str
+ :rtype: Deferred
+ """
+ check_good_uuid(mailbox_uuid)
+ sql = ("CREATE TABLE if not exists {preffix}{name}( "
+ "uid INTEGER PRIMARY KEY AUTOINCREMENT, "
+ "hash TEXT UNIQUE NOT NULL)".format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid)))
+ return self._operation(sql)
+
+ def delete_table(self, mailbox_uuid):
+ """
+ Delete the UID table for a given mailbox.
+ :param mailbox: the mailbox name
+ :type mailbox: str
+ :rtype: Deferred
+ """
+ check_good_uuid(mailbox_uuid)
+ sql = ("DROP TABLE if exists {preffix}{name}".format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid)))
+ return self._operation(sql)
+
+ def insert_doc(self, mailbox_uuid, doc_id):
+ """
+ Insert the doc_id for a MetaMsg in the UID table for a given mailbox.
+
+ The doc_id must be in the format:
+
+ M-<mailbox>-<content-hash-of-the-message>
+
+ :param mailbox: the mailbox name
+ :type mailbox: str
+ :param doc_id: the doc_id for the MetaMsg
+ :type doc_id: str
+ :return: a deferred that will fire with the uid of the newly inserted
+ document.
+ :rtype: Deferred
+ """
+ check_good_uuid(mailbox_uuid)
+ assert doc_id
+ mailbox_uuid = mailbox_uuid.replace('-', '_')
+
+ if not re.findall(METAMSGID_RE.format(mbox_uuid=mailbox_uuid), doc_id):
+ raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id")
+
+ def get_rowid(result):
+ return _maybe_first_query_item(result)
+
+ sql = ("INSERT INTO {preffix}{name} VALUES ("
+ "NULL, ?)".format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid)))
+ values = (doc_id,)
+
+ sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} "
+ "LIMIT 1;").format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid))
+
+ d = self._operation(sql, values)
+ d.addCallback(lambda _: self._query(sql_last))
+ d.addCallback(get_rowid)
+ d.addErrback(lambda f: f.printTraceback())
+ return d
+
+ def delete_doc_by_uid(self, mailbox_uuid, uid):
+ """
+ Delete the entry for a MetaMsg in the UID table for a given mailbox.
+
+ :param mailbox_uuid: the mailbox uuid
+ :type mailbox: str
+ :param uid: the UID of the message.
+ :type uid: int
+ :rtype: Deferred
+ """
+ check_good_uuid(mailbox_uuid)
+ assert uid
+ sql = ("DELETE FROM {preffix}{name} "
+ "WHERE uid=?".format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid)))
+ values = (uid,)
+ return self._query(sql, values)
+
+ def delete_doc_by_hash(self, mailbox_uuid, doc_id):
+ """
+ Delete the entry for a MetaMsg in the UID table for a given mailbox.
+
+ The doc_id must be in the format:
+
+ M-<mailbox_uuid>-<content-hash-of-the-message>
+
+ :param mailbox_uuid: the mailbox uuid
+ :type mailbox: str
+ :param doc_id: the doc_id for the MetaMsg
+ :type doc_id: str
+ :return: a deferred that will fire when the deletion has succed.
+ :rtype: Deferred
+ """
+ check_good_uuid(mailbox_uuid)
+ assert doc_id
+ sql = ("DELETE FROM {preffix}{name} "
+ "WHERE hash=?".format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid)))
+ values = (doc_id,)
+ return self._query(sql, values)
+
+ def get_doc_id_from_uid(self, mailbox_uuid, uid):
+ """
+ Get the doc_id for a MetaMsg in the UID table for a given mailbox.
+
+ :param mailbox_uuid: the mailbox uuid
+ :type mailbox: str
+ :param uid: the uid for the MetaMsg for this mailbox
+ :type uid: int
+ :rtype: Deferred
+ """
+ check_good_uuid(mailbox_uuid)
+ mailbox_uuid = mailbox_uuid.replace('-', '_')
+
+ def get_hash(result):
+ return _maybe_first_query_item(result)
+
+ sql = ("SELECT hash from {preffix}{name} "
+ "WHERE uid=?".format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid)))
+ values = (uid,)
+ d = self._query(sql, values)
+ d.addCallback(get_hash)
+ return d
+
+ def get_uid_from_doc_id(self, mailbox_uuid, doc_id):
+ check_good_uuid(mailbox_uuid)
+ mailbox_uuid = mailbox_uuid.replace('-', '_')
+
+ def get_uid(result):
+ return _maybe_first_query_item(result)
+
+ sql = ("SELECT uid from {preffix}{name} "
+ "WHERE hash=?".format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid)))
+ values = (doc_id,)
+ d = self._query(sql, values)
+ d.addCallback(get_uid)
+ return d
+
+ def get_doc_ids_from_uids(self, mailbox_uuid, uids):
+ # For IMAP relative numbering /sequences.
+ # XXX dereference the range (n,*)
+ raise NotImplementedError()
+
+ def count(self, mailbox_uuid):
+ """
+ Get the number of entries in the UID table for a given mailbox.
+
+ :param mailbox_uuid: the mailbox uuid
+ :type mailbox_uuid: str
+ :return: a deferred that will fire with an integer returning the count.
+ :rtype: Deferred
+ """
+ check_good_uuid(mailbox_uuid)
+
+ def get_count(result):
+ return _maybe_first_query_item(result)
+
+ sql = ("SELECT Count(*) FROM {preffix}{name};".format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid)))
+ d = self._query(sql)
+ d.addCallback(get_count)
+ d.addErrback(lambda _: 0)
+ return d
+
+ def get_next_uid(self, mailbox_uuid):
+ """
+ Get the next integer beyond the highest UID count for a given mailbox.
+
+ This is expected by the IMAP implementation. There are no guarantees
+ that a document to be inserted in the future gets the returned UID: the
+ only thing that can be assured is that it will be equal or greater than
+ the value returned.
+
+ :param mailbox_uuid: the mailbox uuid
+ :type mailbox: str
+ :return: a deferred that will fire with an integer returning the next
+ uid.
+ :rtype: Deferred
+ """
+ check_good_uuid(mailbox_uuid)
+ d = self.get_last_uid(mailbox_uuid)
+ d.addCallback(lambda uid: uid + 1)
+ return d
+
+ def get_last_uid(self, mailbox_uuid):
+ """
+ Get the highest UID for a given mailbox.
+ """
+ check_good_uuid(mailbox_uuid)
+ sql = ("SELECT MAX(rowid) FROM {preffix}{name} "
+ "LIMIT 1;").format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid))
+
+ def getit(result):
+ rowid = _maybe_first_query_item(result)
+ if not rowid:
+ rowid = 0
+ return rowid
+
+ d = self._query(sql)
+ d.addCallback(getit)
+ return d
+
+ def all_uid_iter(self, mailbox_uuid):
+ """
+ Get a sequence of all the uids in this mailbox.
+
+ :param mailbox_uuid: the mailbox uuid
+ :type mailbox_uuid: str
+ """
+ check_good_uuid(mailbox_uuid)
+
+ sql = ("SELECT uid from {preffix}{name} ").format(
+ preffix=self.table_preffix, name=sanitize(mailbox_uuid))
+
+ def get_results(result):
+ return [x[0] for x in result]
+
+ d = self._query(sql)
+ d.addCallback(get_results)
+ return d
diff --git a/src/leap/bitmask/mail/outgoing/__init__.py b/src/leap/bitmask/mail/outgoing/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/src/leap/bitmask/mail/outgoing/__init__.py
diff --git a/src/leap/bitmask/mail/outgoing/service.py b/src/leap/bitmask/mail/outgoing/service.py
new file mode 100644
index 00000000..8b02f2e4
--- /dev/null
+++ b/src/leap/bitmask/mail/outgoing/service.py
@@ -0,0 +1,518 @@
+# -*- coding: utf-8 -*-
+# outgoing/service.py
+# Copyright (C) 2013-2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+OutgoingMail module.
+
+The OutgoingMail class allows to send mail, and encrypts/signs it if needed.
+"""
+
+import os.path
+import re
+from StringIO import StringIO
+from copy import deepcopy
+from email.parser import Parser
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+from OpenSSL import SSL
+
+from twisted.mail import smtp
+from twisted.internet import reactor
+from twisted.internet import defer
+from twisted.protocols.amp import ssl
+from twisted.python import log
+
+from leap.common.check import leap_assert_type, leap_assert
+from leap.common.events import emit_async, catalog
+from leap.keymanager.errors import KeyNotFound, KeyAddressMismatch
+from leap.mail import __version__
+from leap.mail import errors
+from leap.mail.utils import validate_address
+from leap.mail.rfc3156 import MultipartEncrypted
+from leap.mail.rfc3156 import MultipartSigned
+from leap.mail.rfc3156 import encode_base64_rec
+from leap.mail.rfc3156 import RFC3156CompliantGenerator
+from leap.mail.rfc3156 import PGPSignature
+from leap.mail.rfc3156 import PGPEncrypted
+
+# TODO
+# [ ] rename this module to something else, service should be the implementor
+# of IService
+
+
+class SSLContextFactory(ssl.ClientContextFactory):
+ def __init__(self, cert, key):
+ self.cert = cert
+ self.key = key
+
+ def getContext(self):
+ # FIXME -- we should use sslv23 to allow for tlsv1.2
+ # and, if possible, explicitely disable sslv3 clientside.
+ # Servers should avoid sslv3
+ self.method = SSL.TLSv1_METHOD # SSLv23_METHOD
+ ctx = ssl.ClientContextFactory.getContext(self)
+ ctx.use_certificate_file(self.cert)
+ ctx.use_privatekey_file(self.key)
+ return ctx
+
+
+def outgoingFactory(userid, keymanager, opts, check_cert=True, bouncer=None):
+
+ cert = unicode(opts.cert)
+ key = unicode(opts.key)
+ hostname = str(opts.hostname)
+ port = opts.port
+
+ if check_cert:
+ if not os.path.isfile(cert):
+ raise errors.ConfigurationError(
+ 'No valid SMTP certificate could be found for %s!' % userid)
+
+ return OutgoingMail(
+ str(userid), keymanager, cert, key, hostname, port,
+ bouncer)
+
+
+class OutgoingMail(object):
+ """
+ Sends Outgoing Mail, encrypting and signing if needed.
+ """
+
+ def __init__(self, from_address, keymanager, cert, key, host, port,
+ bouncer=None):
+ """
+ Initialize the outgoing mail service.
+
+ :param from_address: The sender address.
+ :type from_address: str
+ :param keymanager: A KeyManager for retrieving recipient's keys.
+ :type keymanager: leap.common.keymanager.KeyManager
+ :param cert: The client certificate for SSL authentication.
+ :type cert: str
+ :param key: The client private key for SSL authentication.
+ :type key: str
+ :param host: The hostname of the remote SMTP server.
+ :type host: str
+ :param port: The port of the remote SMTP server.
+ :type port: int
+ """
+
+ # assert params
+ leap_assert_type(from_address, str)
+ leap_assert('@' in from_address)
+
+ # XXX it can be a zope.proxy too
+ # leap_assert_type(keymanager, KeyManager)
+
+ leap_assert_type(host, str)
+ leap_assert(host != '')
+ leap_assert_type(port, int)
+ leap_assert(port is not 0)
+ leap_assert_type(cert, unicode)
+ leap_assert(cert != '')
+ leap_assert_type(key, unicode)
+ leap_assert(key != '')
+
+ self._port = port
+ self._host = host
+ self._key = key
+ self._cert = cert
+ self._from_address = from_address
+ self._keymanager = keymanager
+ self._bouncer = bouncer
+
+ def send_message(self, raw, recipient):
+ """
+ Sends a message to a recipient. Maybe encrypts and signs.
+
+ :param raw: The raw message
+ :type raw: str
+ :param recipient: The recipient for the message
+ :type recipient: smtp.User
+ :return: a deferred which delivers the message when fired
+ """
+ d = self._maybe_encrypt_and_sign(raw, recipient)
+ d.addCallback(self._route_msg, raw)
+ d.addErrback(self.sendError, raw)
+ return d
+
+ def sendSuccess(self, smtp_sender_result):
+ """
+ Callback for a successful send.
+
+ :param smtp_sender_result: The result from the ESMTPSender from
+ _route_msg
+ :type smtp_sender_result: tuple(int, list(tuple))
+ """
+ dest_addrstr = smtp_sender_result[1][0][0]
+ fromaddr = self._from_address
+ log.msg('Message sent from %s to %s' % (fromaddr, dest_addrstr))
+ emit_async(catalog.SMTP_SEND_MESSAGE_SUCCESS,
+ fromaddr, dest_addrstr)
+
+ def sendError(self, failure, origmsg):
+ """
+ Callback for an unsuccessfull send.
+
+ :param failure: The result from the last errback.
+ :type failure: anything
+ :param origmsg: the original, unencrypted, raw message, to be passed to
+ the bouncer.
+ :type origmsg: str
+ """
+ # XXX: need to get the address from the original message to send signal
+ # emit_async(catalog.SMTP_SEND_MESSAGE_ERROR, self._from_address,
+ # self._user.dest.addrstr)
+
+ # TODO when we implement outgoing queues/long-term-retries, we could
+ # examine the error *here* and delay the notification if it's just a
+ # temporal error. We might want to notify the permanent errors
+ # differently.
+
+ err = failure.value
+ log.err(err)
+
+ if self._bouncer:
+ self._bouncer.bounce_message(
+ err.message, to=self._from_address,
+ orig=origmsg)
+ else:
+ raise err
+
+ def _route_msg(self, encrypt_and_sign_result, raw):
+ """
+ Sends the msg using the ESMTPSenderFactory.
+
+ :param encrypt_and_sign_result: A tuple containing the 'maybe'
+ encrypted message and the recipient
+ :type encrypt_and_sign_result: tuple
+ """
+ message, recipient = encrypt_and_sign_result
+ log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port))
+ msg = message.as_string(False)
+
+ # we construct a defer to pass to the ESMTPSenderFactory
+ d = defer.Deferred()
+ d.addCallback(self.sendSuccess)
+ d.addErrback(self.sendError, raw)
+ # we don't pass an ssl context factory to the ESMTPSenderFactory
+ # because ssl will be handled by reactor.connectSSL() below.
+ factory = smtp.ESMTPSenderFactory(
+ "", # username is blank, no client auth here
+ "", # password is blank, no client auth here
+ self._from_address,
+ recipient.dest.addrstr,
+ StringIO(msg),
+ d,
+ heloFallback=True,
+ requireAuthentication=False,
+ requireTransportSecurity=True)
+ factory.domain = bytes('leap.mail-' + __version__)
+ emit_async(catalog.SMTP_SEND_MESSAGE_START,
+ self._from_address, recipient.dest.addrstr)
+ reactor.connectSSL(
+ self._host, self._port, factory,
+ contextFactory=SSLContextFactory(self._cert, self._key))
+
+ def _maybe_encrypt_and_sign(self, raw, recipient, fetch_remote=True):
+ """
+ Attempt to encrypt and sign the outgoing message.
+
+ The behaviour of this method depends on:
+
+ 1. the original message's content-type, and
+ 2. the availability of the recipient's public key.
+
+ If the original message's content-type is "multipart/encrypted", then
+ the original message is not altered. For any other content-type, the
+ method attempts to fetch the recipient's public key. If the
+ recipient's public key is available, the message is encrypted and
+ signed; otherwise it is only signed.
+
+ Note that, if the C{encrypted_only} configuration is set to True and
+ the recipient's public key is not available, then the recipient
+ address would have been rejected in SMTPDelivery.validateTo().
+
+ The following table summarizes the overall behaviour of the gateway:
+
+ +---------------------------------------------------+----------------+
+ | content-type | rcpt pubkey | enforce encr. | action |
+ +---------------------+-------------+---------------+----------------+
+ | multipart/encrypted | any | any | pass |
+ | other | available | any | encrypt + sign |
+ | other | unavailable | yes | reject |
+ | other | unavailable | no | sign |
+ +---------------------+-------------+---------------+----------------+
+
+ :param raw: The raw message
+ :type raw: str
+ :param recipient: The recipient for the message
+ :type: recipient: smtp.User
+
+ :return: A Deferred that will be fired with a MIMEMultipart message
+ and the original recipient Message
+ :rtype: Deferred
+ """
+ # pass if the original message's content-type is "multipart/encrypted"
+ origmsg = Parser().parsestr(raw)
+
+ if origmsg.get_content_type() == 'multipart/encrypted':
+ return defer.succeed((origmsg, recipient))
+
+ from_address = validate_address(self._from_address)
+ username, domain = from_address.split('@')
+ to_address = validate_address(recipient.dest.addrstr)
+
+ def maybe_encrypt_and_sign(message):
+ d = self._encrypt_and_sign(
+ message, to_address, from_address,
+ fetch_remote=fetch_remote)
+ d.addCallbacks(signal_encrypt_sign,
+ if_key_not_found_send_unencrypted,
+ errbackArgs=(message,))
+ return d
+
+ def signal_encrypt_sign(newmsg):
+ emit_async(catalog.SMTP_END_ENCRYPT_AND_SIGN,
+ self._from_address,
+ "%s,%s" % (self._from_address, to_address))
+ return newmsg, recipient
+
+ def if_key_not_found_send_unencrypted(failure, message):
+ failure.trap(KeyNotFound, KeyAddressMismatch)
+
+ log.msg('Will send unencrypted message to %s.' % to_address)
+ emit_async(catalog.SMTP_START_SIGN, self._from_address, to_address)
+ d = self._sign(message, from_address)
+ d.addCallback(signal_sign)
+ return d
+
+ def signal_sign(newmsg):
+ emit_async(catalog.SMTP_END_SIGN, self._from_address)
+ return newmsg, recipient
+
+ log.msg("Will encrypt the message with %s and sign with %s."
+ % (to_address, from_address))
+ emit_async(catalog.SMTP_START_ENCRYPT_AND_SIGN,
+ self._from_address,
+ "%s,%s" % (self._from_address, to_address))
+ d = self._maybe_attach_key(origmsg, from_address, to_address)
+ d.addCallback(maybe_encrypt_and_sign)
+ return d
+
+ def _maybe_attach_key(self, origmsg, from_address, to_address):
+ filename = "%s-email-key.asc" % (from_address,)
+
+ def attach_if_address_hasnt_encrypted(to_key):
+ # if the sign_used flag is true that means that we got an encrypted
+ # email from this address, because we conly check signatures on
+ # encrypted emails. In this case we don't attach.
+ # XXX: this might not be true some time in the future
+ if to_key.sign_used:
+ return origmsg
+ return get_key_and_attach(None)
+
+ def get_key_and_attach(_):
+ d = self._keymanager.get_key(from_address, fetch_remote=False)
+ d.addCallback(attach_key)
+ return d
+
+ def attach_key(from_key):
+ msg = origmsg
+ if not origmsg.is_multipart():
+ msg = MIMEMultipart()
+ for h, v in origmsg.items():
+ msg.add_header(h, v)
+ msg.attach(MIMEText(origmsg.get_payload()))
+
+ keymsg = MIMEApplication(from_key.key_data, _subtype='pgp-keys',
+ _encoder=lambda x: x)
+ keymsg.add_header('content-disposition', 'attachment',
+ filename=filename)
+ msg.attach(keymsg)
+ return msg
+
+ d = self._keymanager.get_key(to_address, fetch_remote=False)
+ d.addCallbacks(attach_if_address_hasnt_encrypted, get_key_and_attach)
+ d.addErrback(lambda _: origmsg)
+ return d
+
+ def _encrypt_and_sign(self, origmsg, encrypt_address, sign_address,
+ fetch_remote=True):
+ """
+ Create an RFC 3156 compliang PGP encrypted and signed message using
+ C{encrypt_address} to encrypt and C{sign_address} to sign.
+
+ :param origmsg: The original message
+ :type origmsg: email.message.Message
+ :param encrypt_address: The address used to encrypt the message.
+ :type encrypt_address: str
+ :param sign_address: The address used to sign the message.
+ :type sign_address: str
+
+ :return: A Deferred with the MultipartEncrypted message
+ :rtype: Deferred
+ """
+ # create new multipart/encrypted message with 'pgp-encrypted' protocol
+
+ def encrypt(res):
+ newmsg, origmsg = res
+ d = self._keymanager.encrypt(
+ origmsg.as_string(unixfrom=False),
+ encrypt_address, sign=sign_address,
+ fetch_remote=fetch_remote)
+ d.addCallback(lambda encstr: (newmsg, encstr))
+ return d
+
+ def create_encrypted_message(res):
+ newmsg, encstr = res
+ encmsg = MIMEApplication(
+ encstr, _subtype='octet-stream', _encoder=lambda x: x)
+ encmsg.add_header('content-disposition', 'attachment',
+ filename='msg.asc')
+ # create meta message
+ metamsg = PGPEncrypted()
+ metamsg.add_header('Content-Disposition', 'attachment')
+ # attach pgp message parts to new message
+ newmsg.attach(metamsg)
+ newmsg.attach(encmsg)
+ return newmsg
+
+ d = self._fix_headers(
+ origmsg,
+ MultipartEncrypted('application/pgp-encrypted'),
+ sign_address)
+ d.addCallback(encrypt)
+ d.addCallback(create_encrypted_message)
+ return d
+
+ def _sign(self, origmsg, sign_address):
+ """
+ Create an RFC 3156 compliant PGP signed MIME message using
+ C{sign_address}.
+
+ :param origmsg: The original message
+ :type origmsg: email.message.Message
+ :param sign_address: The address used to sign the message.
+ :type sign_address: str
+
+ :return: A Deferred with the MultipartSigned message.
+ :rtype: Deferred
+ """
+ # apply base64 content-transfer-encoding
+ encode_base64_rec(origmsg)
+ # get message text with headers and replace \n for \r\n
+ fp = StringIO()
+ g = RFC3156CompliantGenerator(
+ fp, mangle_from_=False, maxheaderlen=76)
+ g.flatten(origmsg)
+ msgtext = re.sub('\r?\n', '\r\n', fp.getvalue())
+ # make sure signed message ends with \r\n as per OpenPGP stantard.
+ if origmsg.is_multipart():
+ if not msgtext.endswith("\r\n"):
+ msgtext += "\r\n"
+
+ def create_signed_message(res):
+ (msg, _), signature = res
+ sigmsg = PGPSignature(signature)
+ # attach original message and signature to new message
+ msg.attach(origmsg)
+ msg.attach(sigmsg)
+ return msg
+
+ dh = self._fix_headers(
+ origmsg,
+ MultipartSigned('application/pgp-signature', 'pgp-sha512'),
+ sign_address)
+ ds = self._keymanager.sign(
+ msgtext, sign_address, digest_algo='SHA512',
+ clearsign=False, detach=True, binary=False)
+ d = defer.gatherResults([dh, ds])
+ d.addCallback(create_signed_message)
+ return d
+
+ def _fix_headers(self, msg, newmsg, sign_address):
+ """
+ Move some headers from C{origmsg} to C{newmsg}, delete unwanted
+ headers from C{origmsg} and add new headers to C{newms}.
+
+ Outgoing messages are either encrypted and signed or just signed
+ before being sent. Because of that, they are packed inside new
+ messages and some manipulation has to be made on their headers.
+
+ Allowed headers for passing through:
+
+ - From
+ - Date
+ - To
+ - Subject
+ - Reply-To
+ - References
+ - In-Reply-To
+ - Cc
+
+ Headers to be added:
+
+ - Message-ID (i.e. should not use origmsg's Message-Id)
+ - Received (this is added automatically by twisted smtp API)
+ - OpenPGP (see #4447)
+
+ Headers to be deleted:
+
+ - User-Agent
+
+ :param msg: The original message.
+ :type msg: email.message.Message
+ :param newmsg: The new message being created.
+ :type newmsg: email.message.Message
+ :param sign_address: The address used to sign C{newmsg}
+ :type sign_address: str
+
+ :return: A Deferred with a touple:
+ (new Message with the unencrypted headers,
+ original Message with headers removed)
+ :rtype: Deferred
+ """
+ origmsg = deepcopy(msg)
+ # move headers from origmsg to newmsg
+ headers = origmsg.items()
+ passthrough = [
+ 'from', 'date', 'to', 'subject', 'reply-to', 'references',
+ 'in-reply-to', 'cc'
+ ]
+ headers = filter(lambda x: x[0].lower() in passthrough, headers)
+ for hkey, hval in headers:
+ newmsg.add_header(hkey, hval)
+ del (origmsg[hkey])
+ # add a new message-id to newmsg
+ newmsg.add_header('Message-Id', smtp.messageid())
+ # delete user-agent from origmsg
+ del (origmsg['user-agent'])
+
+ def add_openpgp_header(signkey):
+ username, domain = sign_address.split('@')
+ newmsg.add_header(
+ 'OpenPGP', 'id=%s' % signkey.fingerprint,
+ url='https://%s/key/%s' % (domain, username),
+ preference='signencrypt')
+ return newmsg, origmsg
+
+ d = self._keymanager.get_key(sign_address, private=True)
+ d.addCallback(add_openpgp_header)
+ return d
diff --git a/src/leap/bitmask/mail/outgoing/tests/test_outgoing.py b/src/leap/bitmask/mail/outgoing/tests/test_outgoing.py
new file mode 100644
index 00000000..dd053c15
--- /dev/null
+++ b/src/leap/bitmask/mail/outgoing/tests/test_outgoing.py
@@ -0,0 +1,263 @@
+# -*- coding: utf-8 -*-
+# test_gateway.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+SMTP gateway tests.
+"""
+import re
+from copy import deepcopy
+from StringIO import StringIO
+from email.parser import Parser
+from datetime import datetime
+from twisted.internet.defer import fail
+from twisted.mail.smtp import User
+from twisted.python import log
+
+from mock import Mock
+
+from leap.mail.rfc3156 import RFC3156CompliantGenerator
+from leap.mail.outgoing.service import OutgoingMail
+from leap.mail.testing import ADDRESS, ADDRESS_2, PUBLIC_KEY_2
+from leap.mail.testing import KeyManagerWithSoledadTestCase
+from leap.mail.testing.smtp import getSMTPFactory
+from leap.keymanager import errors
+
+
+BEGIN_PUBLIC_KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----"
+
+TEST_USER = u'anotheruser@leap.se'
+
+
+class TestOutgoingMail(KeyManagerWithSoledadTestCase):
+ EMAIL_DATA = ['HELO gateway.leap.se',
+ 'MAIL FROM: <%s>' % ADDRESS_2,
+ 'RCPT TO: <%s>' % ADDRESS,
+ 'DATA',
+ 'From: User <%s>' % ADDRESS_2,
+ 'To: Leap <%s>' % ADDRESS,
+ 'Date: ' + datetime.now().strftime('%c'),
+ 'Subject: test message',
+ '',
+ 'This is a secret message.',
+ 'Yours,',
+ 'A.',
+ '',
+ '.',
+ 'QUIT']
+
+ def setUp(self):
+ self.lines = [line for line in self.EMAIL_DATA[4:12]]
+ self.lines.append('') # add a trailing newline
+ self.raw = '\r\n'.join(self.lines)
+ self.expected_body = '\r\n'.join(self.EMAIL_DATA[9:12]) + "\r\n"
+ self.fromAddr = ADDRESS_2
+
+ class opts:
+ cert = u'/tmp/cert'
+ key = u'/tmp/cert'
+ hostname = 'remote'
+ port = 666
+ self.opts = opts
+
+ def init_outgoing_and_proto(_):
+ self.outgoing_mail = OutgoingMail(
+ self.fromAddr, self.km, opts.cert,
+ opts.key, opts.hostname, opts.port)
+
+ user = TEST_USER
+
+ # TODO -- this shouldn't need SMTP to be tested!? or does it?
+ self.proto = getSMTPFactory(
+ {user: None}, {user: self.km}, {user: None})
+ self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS_2)
+
+ d = KeyManagerWithSoledadTestCase.setUp(self)
+ d.addCallback(init_outgoing_and_proto)
+ return d
+
+ def test_message_encrypt(self):
+ """
+ Test if message gets encrypted to destination email.
+ """
+ def check_decryption(res):
+ decrypted, _ = res
+ self.assertEqual(
+ '\n' + self.expected_body,
+ decrypted,
+ 'Decrypted text differs from plaintext.')
+
+ d = self._set_sign_used(ADDRESS)
+ d.addCallback(
+ lambda _:
+ self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest))
+ d.addCallback(self._assert_encrypted)
+ d.addCallback(lambda message: self.km.decrypt(
+ message.get_payload(1).get_payload(), ADDRESS))
+ d.addCallback(check_decryption)
+ return d
+
+ def test_message_encrypt_sign(self):
+ """
+ Test if message gets encrypted to destination email and signed with
+ sender key.
+ '"""
+ def check_decryption_and_verify(res):
+ decrypted, signkey = res
+ self.assertEqual(
+ '\n' + self.expected_body,
+ decrypted,
+ 'Decrypted text differs from plaintext.')
+ self.assertTrue(ADDRESS_2 in signkey.address,
+ "Verification failed")
+
+ d = self._set_sign_used(ADDRESS)
+ d.addCallback(
+ lambda _:
+ self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest))
+ d.addCallback(self._assert_encrypted)
+ d.addCallback(lambda message: self.km.decrypt(
+ message.get_payload(1).get_payload(), ADDRESS, verify=ADDRESS_2))
+ d.addCallback(check_decryption_and_verify)
+ return d
+
+ def test_message_sign(self):
+ """
+ Test if message is signed with sender key.
+ """
+ # mock the key fetching
+ self.km._fetch_keys_from_server = Mock(
+ return_value=fail(errors.KeyNotFound()))
+ recipient = User('ihavenopubkey@nonleap.se',
+ 'gateway.leap.se', self.proto, ADDRESS)
+ self.outgoing_mail = OutgoingMail(
+ self.fromAddr, self.km, self.opts.cert, self.opts.key,
+ self.opts.hostname, self.opts.port)
+
+ def check_signed(res):
+ message, _ = res
+ self.assertTrue('Content-Type' in message)
+ self.assertEqual('multipart/signed', message.get_content_type())
+ self.assertEqual('application/pgp-signature',
+ message.get_param('protocol'))
+ self.assertEqual('pgp-sha512', message.get_param('micalg'))
+ # assert content of message
+ body = (message.get_payload(0)
+ .get_payload(0)
+ .get_payload(decode=True))
+ self.assertEqual(self.expected_body,
+ body)
+ # assert content of signature
+ self.assertTrue(
+ message.get_payload(1).get_payload().startswith(
+ '-----BEGIN PGP SIGNATURE-----\n'),
+ 'Message does not start with signature header.')
+ self.assertTrue(
+ message.get_payload(1).get_payload().endswith(
+ '-----END PGP SIGNATURE-----\n'),
+ 'Message does not end with signature footer.')
+ return message
+
+ def verify(message):
+ # replace EOL before verifying (according to rfc3156)
+ fp = StringIO()
+ g = RFC3156CompliantGenerator(
+ fp, mangle_from_=False, maxheaderlen=76)
+ g.flatten(message.get_payload(0))
+ signed_text = re.sub('\r?\n', '\r\n',
+ fp.getvalue())
+
+ def assert_verify(key):
+ self.assertTrue(ADDRESS_2 in key.address,
+ 'Signature could not be verified.')
+
+ d = self.km.verify(
+ signed_text, ADDRESS_2,
+ detached_sig=message.get_payload(1).get_payload())
+ d.addCallback(assert_verify)
+ return d
+
+ d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, recipient)
+ d.addCallback(check_signed)
+ d.addCallback(verify)
+ return d
+
+ def test_attach_key(self):
+ d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest)
+ d.addCallback(self._assert_encrypted)
+ d.addCallback(self._check_headers, self.lines[:4])
+ d.addCallback(lambda message: self.km.decrypt(
+ message.get_payload(1).get_payload(), ADDRESS))
+ d.addCallback(lambda (decrypted, _):
+ self._check_key_attachment(Parser().parsestr(decrypted)))
+ return d
+
+ def test_attach_key_not_known(self):
+ unknown_address = "someunknownaddress@somewhere.com"
+ lines = deepcopy(self.lines)
+ lines[1] = "To: <%s>" % (unknown_address,)
+ raw = '\r\n'.join(lines)
+ dest = User(unknown_address, 'gateway.leap.se', self.proto, ADDRESS_2)
+
+ d = self.outgoing_mail._maybe_encrypt_and_sign(
+ raw, dest, fetch_remote=False)
+ d.addCallback(lambda (message, _):
+ self._check_headers(message, lines[:4]))
+ d.addCallback(self._check_key_attachment)
+ d.addErrback(log.err)
+ return d
+
+ def _check_headers(self, message, headers):
+ msgstr = message.as_string(unixfrom=False)
+ for header in headers:
+ self.assertTrue(header in msgstr,
+ "Missing header: %s" % (header,))
+ return message
+
+ def _check_key_attachment(self, message):
+ for payload in message.get_payload():
+ if payload.is_multipart():
+ return self._check_key_attachment(payload)
+ if 'application/pgp-keys' == payload.get_content_type():
+ keylines = PUBLIC_KEY_2.split('\n')
+ key = BEGIN_PUBLIC_KEY + '\n\n' + '\n'.join(keylines[4:-1])
+ self.assertTrue(key in payload.get_payload(decode=True),
+ "Key attachment don't match")
+ return
+ self.fail("No public key attachment found")
+
+ def _set_sign_used(self, address):
+ def set_sign(key):
+ key.sign_used = True
+ return self.km.put_key(key)
+
+ d = self.km.get_key(address, fetch_remote=False)
+ d.addCallback(set_sign)
+ return d
+
+ def _assert_encrypted(self, res):
+ message, _ = res
+ self.assertTrue('Content-Type' in message)
+ self.assertEqual('multipart/encrypted', message.get_content_type())
+ self.assertEqual('application/pgp-encrypted',
+ message.get_param('protocol'))
+ self.assertEqual(2, len(message.get_payload()))
+ self.assertEqual('application/pgp-encrypted',
+ message.get_payload(0).get_content_type())
+ self.assertEqual('application/octet-stream',
+ message.get_payload(1).get_content_type())
+ return message
diff --git a/src/leap/bitmask/mail/plugins/__init__.py b/src/leap/bitmask/mail/plugins/__init__.py
new file mode 100644
index 00000000..ddb86917
--- /dev/null
+++ b/src/leap/bitmask/mail/plugins/__init__.py
@@ -0,0 +1,3 @@
+from twisted.plugin import pluginPackagePaths
+__path__.extend(pluginPackagePaths(__name__))
+__all__ = []
diff --git a/src/leap/bitmask/mail/plugins/soledad_sync_hooks.py b/src/leap/bitmask/mail/plugins/soledad_sync_hooks.py
new file mode 100644
index 00000000..9d48126e
--- /dev/null
+++ b/src/leap/bitmask/mail/plugins/soledad_sync_hooks.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# soledad_sync_hooks.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from leap.mail.sync_hooks import MailProcessingPostSyncHook
+post_sync_uid_reindexer = MailProcessingPostSyncHook()
diff --git a/src/leap/bitmask/mail/rfc3156.py b/src/leap/bitmask/mail/rfc3156.py
new file mode 100644
index 00000000..7d7bc0f0
--- /dev/null
+++ b/src/leap/bitmask/mail/rfc3156.py
@@ -0,0 +1,390 @@
+# -*- coding: utf-8 -*-
+# rfc3156.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Implements RFC 3156: MIME Security with OpenPGP.
+"""
+
+import base64
+from StringIO import StringIO
+
+from twisted.python import log
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from email import errors
+from email.generator import (
+ Generator,
+ fcre,
+ NL,
+ _make_boundary,
+)
+
+
+#
+# A generator that solves http://bugs.python.org/issue14983
+#
+
+class RFC3156CompliantGenerator(Generator):
+ """
+ An email generator that addresses Python's issue #14983 for multipart
+ messages.
+
+ This is just a copy of email.generator.Generator which fixes the following
+ bug: http://bugs.python.org/issue14983
+ """
+
+ def _handle_multipart(self, msg):
+ """
+ A multipart handling implementation that addresses issue #14983.
+
+ This is just a copy of the parent's method which fixes the following
+ bug: http://bugs.python.org/issue14983 (see the line marked with
+ "(***)").
+
+ :param msg: The multipart message to be handled.
+ :type msg: email.message.Message
+ """
+ # The trick here is to write out each part separately, merge them all
+ # together, and then make sure that the boundary we've chosen isn't
+ # present in the payload.
+ msgtexts = []
+ subparts = msg.get_payload()
+ if subparts is None:
+ subparts = []
+ elif isinstance(subparts, basestring):
+ # e.g. a non-strict parse of a message with no starting boundary.
+ self._fp.write(subparts)
+ return
+ elif not isinstance(subparts, list):
+ # Scalar payload
+ subparts = [subparts]
+ for part in subparts:
+ s = StringIO()
+ g = self.clone(s)
+ g.flatten(part, unixfrom=False)
+ msgtexts.append(s.getvalue())
+ # BAW: What about boundaries that are wrapped in double-quotes?
+ boundary = msg.get_boundary()
+ if not boundary:
+ # Create a boundary that doesn't appear in any of the
+ # message texts.
+ alltext = NL.join(msgtexts)
+ boundary = _make_boundary(alltext)
+ msg.set_boundary(boundary)
+ # If there's a preamble, write it out, with a trailing CRLF
+ if msg.preamble is not None:
+ preamble = msg.preamble
+ if self._mangle_from_:
+ preamble = fcre.sub('>From ', msg.preamble)
+ self._fp.write(preamble + '\n')
+ # dash-boundary transport-padding CRLF
+ self._fp.write('--' + boundary + '\n')
+ # body-part
+ if msgtexts:
+ self._fp.write(msgtexts.pop(0))
+ # *encapsulation
+ # --> delimiter transport-padding
+ # --> CRLF body-part
+ for body_part in msgtexts:
+ # delimiter transport-padding CRLF
+ self._fp.write('\n--' + boundary + '\n')
+ # body-part
+ self._fp.write(body_part)
+ # close-delimiter transport-padding
+ self._fp.write('\n--' + boundary + '--' + '\n') # (***) Solve #14983
+ if msg.epilogue is not None:
+ self._fp.write('\n')
+ epilogue = msg.epilogue
+ if self._mangle_from_:
+ epilogue = fcre.sub('>From ', msg.epilogue)
+ self._fp.write(epilogue)
+
+
+#
+# Base64 encoding: these are almost the same as python's email.encoder
+# solution, but a bit modified.
+#
+
+def _bencode(s):
+ """
+ Encode C{s} in base64.
+
+ :param s: The string to be encoded.
+ :type s: str
+ """
+ # We can't quite use base64.encodestring() since it tacks on a "courtesy
+ # newline". Blech!
+ if not s:
+ return s
+ value = base64.encodestring(s)
+ return value[:-1]
+
+
+def encode_base64(msg):
+ """
+ Encode a non-multipart message's payload in Base64 (in place).
+
+ This method modifies the message contents in place and adds or replaces an
+ appropriate Content-Transfer-Encoding header.
+
+ :param msg: The non-multipart message to be encoded.
+ :type msg: email.message.Message
+ """
+ encoding = msg.get('Content-Transfer-Encoding', None)
+ if encoding is not None:
+ encoding = encoding.lower()
+ # XXX Python's email module can only decode quoted-printable, base64 and
+ # uuencoded data, so we might have to implement other decoding schemes in
+ # order to support RFC 3156 properly and correctly calculate signatures
+ # for multipart attachments (eg. 7bit or 8bit encoded attachments). For
+ # now, if content is already encoded as base64 or if it is encoded with
+ # some unknown encoding, we just pass.
+ if encoding in [None, 'quoted-printable', 'x-uuencode', 'uue', 'x-uue']:
+ orig = msg.get_payload(decode=True)
+ encdata = _bencode(orig)
+ msg.set_payload(encdata)
+ # replace or set the Content-Transfer-Encoding header.
+ try:
+ msg.replace_header('Content-Transfer-Encoding', 'base64')
+ except KeyError:
+ msg['Content-Transfer-Encoding'] = 'base64'
+ elif encoding is not 'base64':
+ log.err('Unknown content-transfer-encoding: %s' % encoding)
+
+
+def encode_base64_rec(msg):
+ """
+ Encode (possibly multipart) messages in base64 (in place).
+
+ This method modifies the message contents in place.
+
+ :param msg: The non-multipart message to be encoded.
+ :type msg: email.message.Message
+ """
+ if not msg.is_multipart():
+ encode_base64(msg)
+ else:
+ for sub in msg.get_payload():
+ encode_base64_rec(sub)
+
+
+#
+# RFC 1847: multipart/signed and multipart/encrypted
+#
+
+class MultipartSigned(MIMEMultipart):
+ """
+ Multipart/Signed MIME message according to RFC 1847.
+
+ 2.1. Definition of Multipart/Signed
+
+ (1) MIME type name: multipart
+ (2) MIME subtype name: signed
+ (3) Required parameters: boundary, protocol, and micalg
+ (4) Optional parameters: none
+ (5) Security considerations: Must be treated as opaque while in
+ transit
+
+ The multipart/signed content type contains exactly two body parts.
+ The first body part is the body part over which the digital signature
+ was created, including its MIME headers. The second body part
+ contains the control information necessary to verify the digital
+ signature. The first body part may contain any valid MIME content
+ type, labeled accordingly. The second body part is labeled according
+ to the value of the protocol parameter.
+
+ When the OpenPGP digital signature is generated:
+
+ (1) The data to be signed MUST first be converted to its content-
+ type specific canonical form. For text/plain, this means
+ conversion to an appropriate character set and conversion of
+ line endings to the canonical <CR><LF> sequence.
+
+ (2) An appropriate Content-Transfer-Encoding is then applied; see
+ section 3. In particular, line endings in the encoded data
+ MUST use the canonical <CR><LF> sequence where appropriate
+ (note that the canonical line ending may or may not be present
+ on the last line of encoded data and MUST NOT be included in
+ the signature if absent).
+
+ (3) MIME content headers are then added to the body, each ending
+ with the canonical <CR><LF> sequence.
+
+ (4) As described in section 3 of this document, any trailing
+ whitespace MUST then be removed from the signed material.
+
+ (5) As described in [2], the digital signature MUST be calculated
+ over both the data to be signed and its set of content headers.
+
+ (6) The signature MUST be generated detached from the signed data
+ so that the process does not alter the signed data in any way.
+ """
+
+ def __init__(self, protocol, micalg, boundary=None, _subparts=None):
+ """
+ Initialize the multipart/signed message.
+
+ :param boundary: the multipart boundary string. By default it is
+ calculated as needed.
+ :type boundary: str
+ :param _subparts: a sequence of initial subparts for the payload. It
+ must be an iterable object, such as a list. You can always
+ attach new subparts to the message by using the attach() method.
+ :type _subparts: iterable
+ """
+ MIMEMultipart.__init__(
+ self, _subtype='signed', boundary=boundary,
+ _subparts=_subparts)
+ self.set_param('protocol', protocol)
+ self.set_param('micalg', micalg)
+
+ def attach(self, payload):
+ """
+ Add the C{payload} to the current payload list.
+
+ Also prevent from adding payloads with wrong Content-Type and from
+ exceeding a maximum of 2 payloads.
+
+ :param payload: The payload to be attached.
+ :type payload: email.message.Message
+ """
+ # second payload's content type must be equal to the protocol
+ # parameter given on object creation
+ if len(self.get_payload()) == 1:
+ if payload.get_content_type() != self.get_param('protocol'):
+ raise errors.MultipartConversionError(
+ 'Wrong content type %s.' % payload.get_content_type)
+ # prevent from adding more payloads
+ if len(self._payload) == 2:
+ raise errors.MultipartConversionError(
+ 'Cannot have more than two subparts.')
+ MIMEMultipart.attach(self, payload)
+
+
+class MultipartEncrypted(MIMEMultipart):
+ """
+ Multipart/encrypted MIME message according to RFC 1847.
+
+ 2.2. Definition of Multipart/Encrypted
+
+ (1) MIME type name: multipart
+ (2) MIME subtype name: encrypted
+ (3) Required parameters: boundary, protocol
+ (4) Optional parameters: none
+ (5) Security considerations: none
+
+ The multipart/encrypted content type contains exactly two body parts.
+ The first body part contains the control information necessary to
+ decrypt the data in the second body part and is labeled according to
+ the value of the protocol parameter. The second body part contains
+ the data which was encrypted and is always labeled
+ application/octet-stream.
+ """
+
+ def __init__(self, protocol, boundary=None, _subparts=None):
+ """
+ :param protocol: The encryption protocol to be added as a parameter to
+ the Content-Type header.
+ :type protocol: str
+ :param boundary: the multipart boundary string. By default it is
+ calculated as needed.
+ :type boundary: str
+ :param _subparts: a sequence of initial subparts for the payload. It
+ must be an iterable object, such as a list. You can always
+ attach new subparts to the message by using the attach() method.
+ :type _subparts: iterable
+ """
+ MIMEMultipart.__init__(
+ self, _subtype='encrypted', boundary=boundary,
+ _subparts=_subparts)
+ self.set_param('protocol', protocol)
+
+ def attach(self, payload):
+ """
+ Add the C{payload} to the current payload list.
+
+ Also prevent from adding payloads with wrong Content-Type and from
+ exceeding a maximum of 2 payloads.
+
+ :param payload: The payload to be attached.
+ :type payload: email.message.Message
+ """
+ # first payload's content type must be equal to the protocol parameter
+ # given on object creation
+ if len(self._payload) == 0:
+ if payload.get_content_type() != self.get_param('protocol'):
+ raise errors.MultipartConversionError(
+ 'Wrong content type.')
+ # second payload is always application/octet-stream
+ if len(self._payload) == 1:
+ if payload.get_content_type() != 'application/octet-stream':
+ raise errors.MultipartConversionError(
+ 'Wrong content type %s.' % payload.get_content_type)
+ # prevent from adding more payloads
+ if len(self._payload) == 2:
+ raise errors.MultipartConversionError(
+ 'Cannot have more than two subparts.')
+ MIMEMultipart.attach(self, payload)
+
+
+#
+# RFC 3156: application/pgp-encrypted, application/pgp-signed and
+# application-pgp-signature.
+#
+
+class PGPEncrypted(MIMEApplication):
+ """
+ Application/pgp-encrypted MIME media type according to RFC 3156.
+
+ * MIME media type name: application
+ * MIME subtype name: pgp-encrypted
+ * Required parameters: none
+ * Optional parameters: none
+ """
+
+ def __init__(self, version=1):
+ data = "Version: %d" % version
+ MIMEApplication.__init__(self, data, 'pgp-encrypted')
+
+
+class PGPSignature(MIMEApplication):
+ """
+ Application/pgp-signature MIME media type according to RFC 3156.
+
+ * MIME media type name: application
+ * MIME subtype name: pgp-signature
+ * Required parameters: none
+ * Optional parameters: none
+ """
+ def __init__(self, _data, name='signature.asc'):
+ MIMEApplication.__init__(self, _data, 'pgp-signature',
+ _encoder=lambda x: x, name=name)
+ self.add_header('Content-Description', 'OpenPGP Digital Signature')
+
+
+class PGPKeys(MIMEApplication):
+ """
+ Application/pgp-keys MIME media type according to RFC 3156.
+
+ * MIME media type name: application
+ * MIME subtype name: pgp-keys
+ * Required parameters: none
+ * Optional parameters: none
+ """
+
+ def __init__(self, _data):
+ MIMEApplication.__init__(self, _data, 'pgp-keys')
diff --git a/src/leap/bitmask/mail/size.py b/src/leap/bitmask/mail/size.py
new file mode 100644
index 00000000..c9eaabd3
--- /dev/null
+++ b/src/leap/bitmask/mail/size.py
@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+# size.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Recursively get size of objects.
+"""
+from gc import collect
+from itertools import chain
+from sys import getsizeof
+
+
+def _get_size(item, seen):
+ known_types = {dict: lambda d: chain.from_iterable(d.items())}
+ default_size = getsizeof(0)
+
+ def size_walk(item):
+ if id(item) in seen:
+ return 0
+ seen.add(id(item))
+ s = getsizeof(item, default_size)
+ for _type, fun in known_types.iteritems():
+ if isinstance(item, _type):
+ s += sum(map(size_walk, fun(item)))
+ break
+ return s
+
+ return size_walk(item)
+
+
+def get_size(item):
+ """
+ Return the cumulative size of a given object.
+
+ Currently it supports only dictionaries, and seemingly leaks
+ some memory, so use with care.
+
+ :param item: the item which size wants to be computed
+ :rtype: int
+ """
+ seen = set()
+ size = _get_size(item, seen)
+ del seen
+ collect()
+ return size
diff --git a/src/leap/bitmask/mail/smtp/README.rst b/src/leap/bitmask/mail/smtp/README.rst
new file mode 100644
index 00000000..1d3a9038
--- /dev/null
+++ b/src/leap/bitmask/mail/smtp/README.rst
@@ -0,0 +1,44 @@
+Leap SMTP Gateway
+=================
+
+The Bitmask Client runs a thin SMTP gateway on the user's device, which
+intends to encrypt and sign outgoing messages to achieve point to point
+encryption.
+
+The gateway is bound to localhost and the user's MUA should be configured to
+send messages to it. After doing its thing, the gateway will relay the
+messages to the remote SMTP server.
+
+Outgoing mail workflow:
+
+ * SMTP gateway receives a message from the MUA.
+
+ * SMTP gateway queries Key Manager for the user's private key.
+
+ * For each recipient (including addresses in "To", "Cc" anc "Bcc" fields),
+ the following happens:
+
+ - The recipient's address is validated against RFC2822.
+
+ - An attempt is made to fetch the recipient's public PGP key.
+
+ - If key is not found:
+
+ - If the gateway is configured to only send encrypted messages the
+ recipient is rejected.
+
+ - Otherwise, the message is signed and sent as plain text.
+
+ - If the key is found, the message is encrypted to the recipient and
+ signed with the sender's private PGP key.
+
+ * Finally, one message for each recipient is gatewayed to provider's SMTP
+ server.
+
+
+Running tests
+-------------
+
+Tests are run using Twisted's Trial API, like this::
+
+ python setup.py test -s leap.mail.gateway.tests
diff --git a/src/leap/bitmask/mail/smtp/__init__.py b/src/leap/bitmask/mail/smtp/__init__.py
new file mode 100644
index 00000000..9fab70a7
--- /dev/null
+++ b/src/leap/bitmask/mail/smtp/__init__.py
@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+# __init__.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+SMTP gateway helper function.
+"""
+import logging
+import os
+
+from twisted.internet import reactor
+from twisted.internet.error import CannotListenError
+
+from leap.common.events import emit_async, catalog
+
+from leap.mail.smtp.gateway import SMTPFactory
+
+logger = logging.getLogger(__name__)
+
+
+SMTP_PORT = 2013
+
+
+def run_service(soledad_sessions, keymanager_sessions, sendmail_opts,
+ port=SMTP_PORT):
+ """
+ Main entry point to run the service from the client.
+
+ :param soledad_sessions: a dict-like object, containing instances
+ of a Store (soledad instances), indexed by userid.
+ :param keymanager_sessions: a dict-like object, containing instances
+ of Keymanager, indexed by userid.
+ :param sendmail_opts: a dict-like object of sendmailOptions.
+
+ :returns: the port as returned by the reactor when starts listening, and
+ the factory for the protocol.
+ :rtype: tuple
+ """
+ factory = SMTPFactory(soledad_sessions, keymanager_sessions,
+ sendmail_opts)
+
+ try:
+ interface = "localhost"
+ # don't bind just to localhost if we are running on docker since we
+ # won't be able to access smtp from the host
+ if os.environ.get("LEAP_DOCKERIZED"):
+ interface = ''
+
+ # TODO Use Endpoints instead --------------------------------
+ tport = reactor.listenTCP(port, factory, interface=interface)
+ emit_async(catalog.SMTP_SERVICE_STARTED, str(port))
+
+ return factory, tport
+ except CannotListenError:
+ logger.error("STMP Service failed to start: "
+ "cannot listen in port %s" % port)
+ emit_async(catalog.SMTP_SERVICE_FAILED_TO_START, str(port))
+ except Exception as exc:
+ logger.error("Unhandled error while launching smtp gateway service")
+ logger.exception(exc)
diff --git a/src/leap/bitmask/mail/smtp/bounces.py b/src/leap/bitmask/mail/smtp/bounces.py
new file mode 100644
index 00000000..7a4674b9
--- /dev/null
+++ b/src/leap/bitmask/mail/smtp/bounces.py
@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+# bounces.py
+# Copyright (C) 2016 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Deliver bounces to the user Inbox.
+"""
+import time
+from email.message import Message
+from email.utils import formatdate
+
+from leap.mail.constants import INBOX_NAME
+from leap.mail.mail import Account
+
+
+# TODO implement localization for this template.
+
+BOUNCE_TEMPLATE = """This is your local Bitmask Mail Agent running at localhost.
+
+I'm sorry to have to inform you that your message could not be delivered to one
+or more recipients.
+
+The reasons I got for the error are:
+
+{raw_error}
+
+If the problem persists and it's not a network connectivity issue, you might
+want to contact your provider ({provider}) with this information (remove any
+sensitive data before).
+
+--- Original message (*before* it was encrypted by bitmask) below ----:
+
+{orig}"""
+
+
+class Bouncer(object):
+ """
+ Implements a mechanism to deliver bounces to user inbox.
+ """
+ # TODO this should follow RFC 6522, and compose a correct multipart
+ # attaching the report and the original message. Leaving this for a future
+ # iteration.
+
+ def __init__(self, inbox_collection):
+ self._inbox_collection = inbox_collection
+
+ def bounce_message(self, error_data, to, date=None, orig=''):
+ if not date:
+ date = formatdate(time.time())
+
+ raw_data = self._format_msg(error_data, to, date, orig)
+ d = self._inbox_collection.add_msg(
+ raw_data, ('\\Recent',), date=date)
+ return d
+
+ def _format_msg(self, error_data, to, date, orig):
+ provider = to.split('@')[1]
+
+ msg = Message()
+ msg.add_header(
+ 'From', 'bitmask-bouncer@localhost (Bitmask Local Agent)')
+ msg.add_header('To', to)
+ msg.add_header('Subject', 'Undelivered Message')
+ msg.add_header('Date', date)
+ msg.set_payload(BOUNCE_TEMPLATE.format(
+ raw_error=error_data,
+ provider=provider,
+ orig=orig))
+
+ return msg.as_string()
+
+
+def bouncerFactory(soledad):
+ user_id = soledad.uuid
+ acc = Account(soledad, user_id)
+ d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox(INBOX_NAME))
+ d.addCallback(lambda inbox: Bouncer(inbox))
+ return d
diff --git a/src/leap/bitmask/mail/smtp/gateway.py b/src/leap/bitmask/mail/smtp/gateway.py
new file mode 100644
index 00000000..e49bbe82
--- /dev/null
+++ b/src/leap/bitmask/mail/smtp/gateway.py
@@ -0,0 +1,413 @@
+# -*- coding: utf-8 -*-
+# gateway.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+LEAP SMTP encrypted gateway.
+
+The following classes comprise the SMTP gateway service:
+
+ * SMTPFactory - A twisted.internet.protocol.ServerFactory that provides
+ the SMTPDelivery protocol.
+
+ * SMTPDelivery - A twisted.mail.smtp.IMessageDelivery implementation. It
+ knows how to validate sender and receiver of messages and it generates
+ an EncryptedMessage for each recipient.
+
+ * EncryptedMessage - An implementation of twisted.mail.smtp.IMessage that
+ knows how to encrypt/sign itself before sending.
+"""
+from email.Header import Header
+
+from zope.interface import implements
+from zope.interface import implementer
+
+from twisted.cred.portal import Portal, IRealm
+from twisted.mail import smtp
+from twisted.mail.imap4 import LOGINCredentials, PLAINCredentials
+from twisted.internet import defer, protocol
+from twisted.python import log
+
+from leap.common.check import leap_assert_type
+from leap.common.events import emit_async, catalog
+from leap.mail import errors
+from leap.mail.cred import LocalSoledadTokenChecker
+from leap.mail.utils import validate_address
+from leap.mail.rfc3156 import RFC3156CompliantGenerator
+from leap.mail.outgoing.service import outgoingFactory
+from leap.mail.smtp.bounces import bouncerFactory
+from leap.keymanager.errors import KeyNotFound
+
+# replace email generator with a RFC 3156 compliant one.
+from email import generator
+
+generator.Generator = RFC3156CompliantGenerator
+
+
+LOCAL_FQDN = "bitmask.local"
+
+
+@implementer(IRealm)
+class LocalSMTPRealm(object):
+
+ _encoding = 'utf-8'
+
+ def __init__(self, keymanager_sessions, soledad_sessions, sendmail_opts,
+ encrypted_only=False):
+ """
+ :param keymanager_sessions: a dict-like object, containing instances
+ of a Keymanager objects, indexed by
+ userid.
+ """
+ self._keymanager_sessions = keymanager_sessions
+ self._soledad_sessions = soledad_sessions
+ self._sendmail_opts = sendmail_opts
+ self.encrypted_only = encrypted_only
+
+ def requestAvatar(self, avatarId, mind, *interfaces):
+
+ if isinstance(avatarId, str):
+ avatarId = avatarId.decode(self._encoding)
+
+ def gotKeymanagerAndSoledad(result):
+ keymanager, soledad = result
+ d = bouncerFactory(soledad)
+ d.addCallback(lambda bouncer: (keymanager, soledad, bouncer))
+ return d
+
+ def getMessageDelivery(result):
+ keymanager, soledad, bouncer = result
+ # TODO use IMessageDeliveryFactory instead ?
+ # it could reuse the connections.
+ if smtp.IMessageDelivery in interfaces:
+ userid = avatarId
+ opts = self.getSendingOpts(userid)
+
+ outgoing = outgoingFactory(
+ userid, keymanager, opts, bouncer=bouncer)
+ avatar = SMTPDelivery(userid, keymanager, self.encrypted_only,
+ outgoing)
+
+ return (smtp.IMessageDelivery, avatar,
+ getattr(avatar, 'logout', lambda: None))
+
+ raise NotImplementedError(self, interfaces)
+
+ d1 = self.lookupKeymanagerInstance(avatarId)
+ d2 = self.lookupSoledadInstance(avatarId)
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(gotKeymanagerAndSoledad)
+ d.addCallback(getMessageDelivery)
+ return d
+
+ def lookupKeymanagerInstance(self, userid):
+ try:
+ keymanager = self._keymanager_sessions[userid]
+ except:
+ raise errors.AuthenticationError(
+ 'No keymanager session found for user %s. Is it authenticated?'
+ % userid)
+ # XXX this should return the instance after whenReady callback
+ return defer.succeed(keymanager)
+
+ def lookupSoledadInstance(self, userid):
+ try:
+ soledad = self._soledad_sessions[userid]
+ except:
+ raise errors.AuthenticationError(
+ 'No soledad session found for user %s. Is it authenticated?'
+ % userid)
+ # XXX this should return the instance after whenReady callback
+ return defer.succeed(soledad)
+
+ def getSendingOpts(self, userid):
+ try:
+ opts = self._sendmail_opts[userid]
+ except KeyError:
+ raise errors.ConfigurationError(
+ 'No sendingMail options found for user %s' % userid)
+ return opts
+
+
+class SMTPTokenChecker(LocalSoledadTokenChecker):
+ """A credentials checker that will lookup a token for the SMTP service.
+ For now it will be using the same identifier than IMAPTokenChecker"""
+
+ service = 'mail_auth'
+
+ # TODO besides checking for token credential,
+ # we could also verify the certificate here.
+
+
+class LEAPInitMixin(object):
+
+ """
+ A Mixin that takes care of initialization of all the data needed to access
+ LEAP sessions.
+ """
+ def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts,
+ encrypted_only=False):
+ realm = LocalSMTPRealm(
+ keymanager_sessions, soledad_sessions, sendmail_opts,
+ encrypted_only)
+ portal = Portal(realm)
+
+ checker = SMTPTokenChecker(soledad_sessions)
+ self.checker = checker
+ self.portal = portal
+ portal.registerChecker(checker)
+
+
+class LocalSMTPServer(smtp.ESMTP, LEAPInitMixin):
+ """
+ The Production ESMTP Server: Authentication Needed.
+ Authenticates against SMTP Token stored in Local Soledad instance.
+ The Realm will produce a Delivery Object that handles encryption/signing.
+ """
+
+ # TODO: implement Queue using twisted.mail.mail.MailService
+
+ def __init__(self, soledads, keyms, sendmailopts, *args, **kw):
+ encrypted_only = kw.pop('encrypted_only', False)
+
+ LEAPInitMixin.__init__(self, soledads, keyms, sendmailopts,
+ encrypted_only)
+ smtp.ESMTP.__init__(self, *args, **kw)
+
+
+# TODO implement retries -- see smtp.SenderMixin
+class SMTPFactory(protocol.ServerFactory):
+ """
+ Factory for an SMTP server with encrypted gatewaying capabilities.
+ """
+
+ protocol = LocalSMTPServer
+ domain = LOCAL_FQDN
+ timeout = 600
+ encrypted_only = False
+
+ def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts,
+ deferred=None, retries=3):
+
+ self._soledad_sessions = soledad_sessions
+ self._keymanager_sessions = keymanager_sessions
+ self._sendmail_opts = sendmail_opts
+
+ def buildProtocol(self, addr):
+ p = self.protocol(
+ self._soledad_sessions, self._keymanager_sessions,
+ self._sendmail_opts, encrypted_only=self.encrypted_only)
+ p.factory = self
+ p.host = LOCAL_FQDN
+ p.challengers = {"LOGIN": LOGINCredentials, "PLAIN": PLAINCredentials}
+ return p
+
+
+#
+# SMTPDelivery
+#
+
+@implementer(smtp.IMessageDelivery)
+class SMTPDelivery(object):
+ """
+ Validate email addresses and handle message delivery.
+ """
+
+ def __init__(self, userid, keymanager, encrypted_only, outgoing_mail):
+ """
+ Initialize the SMTP delivery object.
+
+ :param userid: The user currently logged in
+ :type userid: unicode
+ :param keymanager: A Key Manager from where to get recipients' public
+ keys.
+ :param encrypted_only: Whether the SMTP gateway should send unencrypted
+ mail or not.
+ :type encrypted_only: bool
+ :param outgoing_mail: The outgoing mail to send the message
+ :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail
+ """
+ self._userid = userid
+ self._outgoing_mail = outgoing_mail
+ self._km = keymanager
+ self._encrypted_only = encrypted_only
+ self._origin = None
+
+ def receivedHeader(self, helo, origin, recipients):
+ """
+ Generate the 'Received:' header for a message.
+
+ :param helo: The argument to the HELO command and the client's IP
+ address.
+ :type helo: (str, str)
+ :param origin: The address the message is from.
+ :type origin: twisted.mail.smtp.Address
+ :param recipients: A list of the addresses for which this message is
+ bound.
+ :type: list of twisted.mail.smtp.User
+
+ @return: The full "Received" header string.
+ :type: str
+ """
+ myHostname, clientIP = helo
+ headerValue = "by bitmask.local from %s with ESMTP ; %s" % (
+ clientIP, smtp.rfc822date())
+ # email.Header.Header used for automatic wrapping of long lines
+ return "Received: %s" % Header(s=headerValue, header_name='Received')
+
+ def validateTo(self, user):
+ """
+ Validate the address of a recipient of the message, possibly
+ rejecting it if the recipient key is not available.
+
+ This method is called once for each recipient, i.e. for each SMTP
+ protocol line beginning with "RCPT TO:", which includes all addresses
+ in "To", "Cc" and "Bcc" MUA fields.
+
+ The recipient's address is validated against the RFC 2822 definition.
+ If self._encrypted_only is True and no key is found for a recipient,
+ then that recipient is rejected.
+
+ The method returns an encrypted message object that is able to send
+ itself to the user's address.
+
+ :param user: The user whose address we wish to validate.
+ :type: twisted.mail.smtp.User
+
+ @return: A callable which takes no arguments and returns an
+ encryptedMessage.
+ @rtype: no-argument callable
+
+ @raise SMTPBadRcpt: Raised if messages to the address are not to be
+ accepted.
+ """
+ # try to find recipient's public key
+ address = validate_address(user.dest.addrstr)
+
+ # verify if recipient key is available in keyring
+ def found(_):
+ log.msg("Accepting mail for %s..." % user.dest.addrstr)
+ emit_async(catalog.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED,
+ self._userid, user.dest.addrstr)
+
+ def not_found(failure):
+ failure.trap(KeyNotFound)
+
+ # if key was not found, check config to see if will send anyway
+ if self._encrypted_only:
+ emit_async(catalog.SMTP_RECIPIENT_REJECTED, self._userid,
+ user.dest.addrstr)
+ raise smtp.SMTPBadRcpt(user.dest.addrstr)
+ log.msg("Warning: will send an unencrypted message (because "
+ "encrypted_only' is set to False).")
+ emit_async(
+ catalog.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED,
+ self._userid, user.dest.addrstr)
+
+ def encrypt_func(_):
+ return lambda: EncryptedMessage(user, self._outgoing_mail)
+
+ d = self._km.get_key(address)
+ d.addCallbacks(found, not_found)
+ d.addCallback(encrypt_func)
+ return d
+
+ def validateFrom(self, helo, origin):
+ """
+ Validate the address from which the message originates.
+
+ :param helo: The argument to the HELO command and the client's IP
+ address.
+ :type: (str, str)
+ :param origin: The address the message is from.
+ :type origin: twisted.mail.smtp.Address
+
+ @return: origin or a Deferred whose callback will be passed origin.
+ @rtype: Deferred or Address
+
+ @raise twisted.mail.smtp.SMTPBadSender: Raised if messages from this
+ address are not to be accepted.
+ """
+ # accept mail from anywhere. To reject an address, raise
+ # smtp.SMTPBadSender here.
+ if str(origin) != str(self._userid):
+ log.msg("Rejecting sender {0}, expected {1}".format(origin,
+ self._userid))
+ raise smtp.SMTPBadSender(origin)
+ self._origin = origin
+ return origin
+
+
+#
+# EncryptedMessage
+#
+
+class EncryptedMessage(object):
+ """
+ Receive plaintext from client, encrypt it and send message to a
+ recipient.
+ """
+ implements(smtp.IMessage)
+
+ def __init__(self, user, outgoing_mail):
+ """
+ Initialize the encrypted message.
+
+ :param user: The recipient of this message.
+ :type user: twisted.mail.smtp.User
+ :param outgoing_mail: The outgoing mail to send the message
+ :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail
+ """
+ # assert params
+ leap_assert_type(user, smtp.User)
+
+ self._user = user
+ self._lines = []
+ self._outgoing_mail = outgoing_mail
+
+ def lineReceived(self, line):
+ """
+ Handle another line.
+
+ :param line: The received line.
+ :type line: str
+ """
+ self._lines.append(line)
+
+ def eomReceived(self):
+ """
+ Handle end of message.
+
+ This method will encrypt and send the message.
+
+ :returns: a deferred
+ """
+ log.msg("Message data complete.")
+ self._lines.append('') # add a trailing newline
+ raw_mail = '\r\n'.join(self._lines)
+
+ return self._outgoing_mail.send_message(raw_mail, self._user)
+
+ def connectionLost(self):
+ """
+ Log an error when the connection is lost.
+ """
+ log.msg("Connection lost unexpectedly!")
+ log.err()
+ emit_async(catalog.SMTP_CONNECTION_LOST, self._userid,
+ self._user.dest.addrstr)
+ # unexpected loss of connection; don't save
+
+ self._lines = []
diff --git a/src/leap/bitmask/mail/smtp/tests/185CA770.key b/src/leap/bitmask/mail/smtp/tests/185CA770.key
new file mode 100644
index 00000000..587b4164
--- /dev/null
+++ b/src/leap/bitmask/mail/smtp/tests/185CA770.key
@@ -0,0 +1,79 @@
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: GnuPG v1.4.10 (GNU/Linux)
+
+lQIVBFCJNL4BEADFsI1TCD4yq7ZqL7VhdVviTuX6JUps8/mVEhRVOZhojLcTYaqQ
+gs6T6WabRxcK7ymOnf4K8NhYdz6HFoJN46BT87etokx7J/Sl2OhpiqBQEY+jW8Rp
++3MSGrGmvFw0s1lGrz/cXzM7UNgWSTOnYZ5nJS1veMhy0jseZOUK7ekp2oEDjGZh
+pzgd3zICCR2SvlpLIXB2Nr/CUcuRWTcc5LlKmbjMybu0E/uuY14st3JL+7qI6QX0
+atFm0VhFVpagOl0vWKxakUx4hC7j1wH2ADlCvSZPG0StSLUyHkJx3UPsmYxOZFao
+ATED3Okjwga6E7PJEbzyqAkvzw/M973kaZCUSH75ZV0cQnpdgXV3DK1gSa3d3gug
+W1lE0V7pwnN2NTOYfBMi+WloCs/bp4iZSr4QP1duZ3IqKraeBDCk7MoFo4A9Wk07
+kvqPwF9IBgatu62WVEZIzwyViN+asFUGfgp+8D7gtnlWAw0V6y/lSTzyl+dnLP98
+Hfr2eLBylFs+Kl3Pivpg2uHw09LLCrjeLEN3dj9SfBbA9jDIo9Zhs1voiIK/7Shx
+E0BRJaBgG3C4QaytYEu7RFFOKuvBai9w2Y5OfsKFo8rA7v4dxFFDvzKGujCtNnwf
+oyaGlZmMBU5MUmHUNiG8ON21COZBtK5oMScuY1VC9CQonj3OClg3IbU9SQARAQAB
+/gNlAkdOVQG0JGRyZWJzIChncGcgdGVzdCBrZXkpIDxkcmVic0BsZWFwLnNlPokC
+OAQTAQIAIgUCUIk0vgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQty9e
+xhhcp3Bdhw//bdPUNbp6rgIjRRuwYvGJ6IuiFuFWJQ0m3iAuuAoZo5GHAPqZAuGk
+dMVYu0dtCtZ68MJ/QpjBCT9RRL+mgIgfLfUSj2ZknP4nb6baiG5u28l0KId/e5IC
+iQKBnIsjxKxhLBVHSzRaS1P+vZeF2C2R9XyNy0eCnAwyCMcD0R8TVROGQ7i4ZQsM
+bMj1LPpOwhV/EGp23nD+upWOVbn/wQHOYV2kMiA/8fizmWRIWsV4/68uMA+WDP4L
+40AnJ0fcs04f9deM9P6pjlm00VD7qklYEGw6Mpr2g/M73kGh1nlAv+ImQBGlLMle
+RXyzHY3WAhzmRKWO4koFuKeR9Q0EMzk2R4/kuagdWEpM+bhwE4xPV1tPZhn9qFTz
+pQD4p/VT4qNQKOD0+aTFWre65Rt2cFFMLI7UmEHNLi0NB9JCIAi4+l+b9WQNlmaO
+C8EhOGwRzmehUyHmXM3BNW28MnyKFJ7bBFMd7uJz+vAPOrr6OzuNvVCv2I2ICkTs
+ihIj/zw5GXxkPO7YbMu9rKG0nKF1N3JB1gUJ78DHmhbjeaGSvHw85sPD0/1dPZK4
+8Gig8i62aCxf8OlJPlt8ZhBBolzs6ITUNa75Rw9fJsj3UWuv2VFaIuR57bFWmY3s
+A9KPgdf7jVQlAZKlVyli7IkyaZmxDZNFQoTdIC9uo0aggIDP8zKv0n2dBz4EUIk0
+vgEQAOO8BAR7sBdqj2RRMRNeWSA4S9GuHfV3YQARnqYsbITs1jRgAo7jx9Z5C80c
+ZOxOUVK7CJjtTqU0JB9QP/zwV9hk5i6y6aQTysclQyTNN10aXu/3zJla5Duhz+Cs
++5UcVAmNJX9FgTMVvhKDEIY/LNmb9MoBLMut1CkDx+WPCV45WOIBCDdj2HpIjie4
+phs0/65SWjPiVg3WsFZljVxpJCGXP48Eet2bf8afYH1lx3sQMcNbyJACIPtz+YKz
+c7jIKwKSWzg1VyYikbk9eWCxcz6VKNJKi94YH9c7U8X3TdZ8G0kGYUldjYDvesyl
+nuQlcGCtSGKOAhrN/Bu2R0gpFgYl247u79CmjotefMdv8BGUDW6u9/Sep9xN3dW8
+S87h6M/tvs0ChlkDDpJedzCd7ThdikGvFRJfW/8sT/+qoTKskySQaDIeNJnxZuyK
+wELLMBvCZGpamwmnkEGhvuZWq0h/DwyTs4QAE8OVHXJSM3UN7hM4lJIUh+sRKJ1F
+AXXTdSY4cUNaS+OKtj2LJ85zFqhfAZ4pFwLCgYbJtU5hej2LnMJNbYcSkjxbk+c5
+IjkoZRF+ExjZlc0VLYNT57ZriwZ/pX42ofjOyMR/dkHQuFik/4K7v1ZemfaTdm07
+SEMBknR6OZsy/5+viEtXiih3ptTMaT9row+g+cFoxdXkisKvABEBAAH+AwMCIlVK
+Xs3x0Slgwx03cTNIoWXmishkPCJlEEdcjldz2VyQF9hjdp1VIe+npI26chKwCZqm
+U8yYbJh4UBrugUUzKKd4EfnmKfu+/BsJciFRVKwBtiolIiUImzcHPWktYLwo9yzX
+W42teShXXVgWmsJN1/6FqJdsLg8dxWesXMKoaNF4n1P7zx6vKBmDHTRz7PToaI/d
+5/nKrjED7ZT1h+qR5i9UUgbvF0ySp8mlqk/KNqHUSLDB9kf/JDg4XVtPHGGd9Ik/
+60UJ7aDfohi4Z0VgwWmfLBwcQ3It+ENtnPFufH3WHW8c1UA4wVku9tOTqyrRG6tP
+TZGiRfuwsv7Hq3pWT6rntbDkTiVgESM4C1fiZblc98iWUKGXSHqm+te1TwXOUCci
+J/gryXcjQFM8A0rwA/m+EvsoWuzoqIl3x++p3/3/mGux6UD4O7OhJNRVRz+8Mhq1
+ksrR9XkQzpq3Yv3ulTHz7l+WCRRXxw5+XWAkRHHF47Vf/na38NJQHcsCBbRIuLYR
+wBzS48cYzYkF6VejKThdQmdYJ0/fUrlUBCAJWgrfqCihFLDa1s4jJ16/fqi8a97Y
+4raVy2hrF2vFc/wet13hsaddVn4rPRAMDEGdgEmJX7MmU1emT/yaIG9lvjMpI2c5
+ADXGF2yYYa7H8zPIFyHU1RSavlT0S/K9yzIZvv+jA5KbNeGp+WWFT8MLZs0IhoCZ
+d1EgLUYAt7LPUSm2lBy1w/IL+VtYuyn/UVFo2xWiHd1ABiNWl1ji3X9Ki5613QqH
+bvn4z46voCzdZ02rYkAwrdqDr92fiBR8ctwA0AudaG6nf2ztmFKtM3E/RPMkPgKF
+8NHYc7QxS2jruJxXBtjRBMtoIaZ0+AXUO6WuEJrDLDHWaM08WKByQMm808xNCbRr
+CpiK8qyR3SwkfaOMCp22mqViirQ2KfuVvBpBT2pBYlgDKs50nE+stDjUMv+FDKAo
+5NtiyPfNtaBOYnXAEQb/hjjW5bKq7JxHSxIWAYKbNKIWgftJ3ACZAsBMHfaOCFNH
++XLojAoxOI+0zbN6FtjN+YMU1XrLd6K49v7GEiJQZVQSfLCecVDhDU9paNROA/Xq
+/3nDCTKhd3stTPnc8ymLAwhTP0bSoFh/KtU96D9ZMC2cu9XZ+UcSQYES/ncZWcLw
+wTKrt+VwBG1z3DbV2O0ruUiXTLcZMsrwbUSDx1RVhmKZ0i42AttMdauFQ9JaX2CS
+2ddqFBS1b4X6+VCy44KkpdXsmp0NWMgm/PM3PTisCxrha7bI5/LqfXG0b+GuIFb4
+h/lEA0Ae0gMgkzm3ePAPPVlRj7kFl5Osjxm3YVRW23WWGDRF5ywIROlBjbdozA0a
+MyMgXlG9hhJseIpFveoiwqenNE5Wxg0yQbnhMUTKeCQ0xskG82P+c9bvDsevAQUR
+uv1JAGGxDd1/4nk0M5m9/Gf4Bn0uLAz29LdMg0FFUvAm2ol3U3uChm7OISU8dqFy
+JdCFACKBMzAREiXfgH2TrTxAhpy5uVcUSQV8x5J8qJ/mUoTF1WE3meXEm9CIvIAF
+Mz49KKebLS3zGFixMcKLAOKA+s/tUWO7ZZoJyQjvQVerLyDo6UixVb11LQUJQOXb
+ZIuSKV7deCgBDQ26C42SpF3rHfEQa7XH7j7tl1IIW/9DfYJYVQHaz1NTq6zcjWS2
+e+cUexBPhxbadGn0zelXr6DLJqQT7kaVeYOHlkYUHkZXdHE4CWoHqOboeB02uM/A
+e7nge1rDi57ySrsF4AVl59QJYBPR43AOVbCJAh8EGAECAAkFAlCJNL4CGwwACgkQ
+ty9exhhcp3DetA/8D/IscSBlWY3TjCD2P7t3+X34USK8EFD3QJse9dnCWOLcskFQ
+IoIfhRM752evFu2W9owEvxSQdG+otQAOqL72k1EH2g7LsADuV8I4LOYOnLyeIE9I
+b+CFPBkmzTEzrdYp6ITUU7qqgkhcgnltKGHoektIjxE8gtxCKEdyxkzazum6nCQQ
+kSBZOXVU3ezm+A2QHHP6XT1GEbdKbJ0tIuJR8ADu08pBx2c/LDBBreVStrrt1Dbz
+uR+U8MJsfLVcYX/Rw3V+KA24oLRzg91y3cfi3sNU/kmd5Cw42Tj00B+FXQny51Mq
+s4KyqHobj62II68eL5HRB2pcGsoaedQyxu2cYSeVyarBOiUPNYkoGDJoKdDyZRIB
+NNK0W+ASTf0zeHhrY/okt1ybTVtvbt6wkTEbKVePUaYmNmhre1cAj4uNwFzYjkzJ
+cm+8XWftD+TV8cE5DyVdnF00SPDuPzodRAPXaGpQUMLkE4RPr1TAwcuoPH9aFHZ/
+se6rw6TQHLd0vMk0U/DocikXpSJ1N6caE3lRwI/+nGfXNiCr8MIdofgkBeO86+G7
+k0UXS4v5FKk1nwTyt4PkFJDvAJX6rZPxIZ9NmtA5ao5vyu1DT5IhoXgDzwurAe8+
+R+y6gtA324hXIweFNt7SzYPfI4SAjunlmm8PIBf3owBrk3j+w6EQoaCreK4=
+=6HcJ
+-----END PGP PRIVATE KEY BLOCK-----
diff --git a/src/leap/bitmask/mail/smtp/tests/185CA770.pub b/src/leap/bitmask/mail/smtp/tests/185CA770.pub
new file mode 100644
index 00000000..38af19f8
--- /dev/null
+++ b/src/leap/bitmask/mail/smtp/tests/185CA770.pub
@@ -0,0 +1,52 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.10 (GNU/Linux)
+
+mQINBFCJNL4BEADFsI1TCD4yq7ZqL7VhdVviTuX6JUps8/mVEhRVOZhojLcTYaqQ
+gs6T6WabRxcK7ymOnf4K8NhYdz6HFoJN46BT87etokx7J/Sl2OhpiqBQEY+jW8Rp
++3MSGrGmvFw0s1lGrz/cXzM7UNgWSTOnYZ5nJS1veMhy0jseZOUK7ekp2oEDjGZh
+pzgd3zICCR2SvlpLIXB2Nr/CUcuRWTcc5LlKmbjMybu0E/uuY14st3JL+7qI6QX0
+atFm0VhFVpagOl0vWKxakUx4hC7j1wH2ADlCvSZPG0StSLUyHkJx3UPsmYxOZFao
+ATED3Okjwga6E7PJEbzyqAkvzw/M973kaZCUSH75ZV0cQnpdgXV3DK1gSa3d3gug
+W1lE0V7pwnN2NTOYfBMi+WloCs/bp4iZSr4QP1duZ3IqKraeBDCk7MoFo4A9Wk07
+kvqPwF9IBgatu62WVEZIzwyViN+asFUGfgp+8D7gtnlWAw0V6y/lSTzyl+dnLP98
+Hfr2eLBylFs+Kl3Pivpg2uHw09LLCrjeLEN3dj9SfBbA9jDIo9Zhs1voiIK/7Shx
+E0BRJaBgG3C4QaytYEu7RFFOKuvBai9w2Y5OfsKFo8rA7v4dxFFDvzKGujCtNnwf
+oyaGlZmMBU5MUmHUNiG8ON21COZBtK5oMScuY1VC9CQonj3OClg3IbU9SQARAQAB
+tCRkcmVicyAoZ3BnIHRlc3Qga2V5KSA8ZHJlYnNAbGVhcC5zZT6JAjgEEwECACIF
+AlCJNL4CGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELcvXsYYXKdwXYcP
+/23T1DW6eq4CI0UbsGLxieiLohbhViUNJt4gLrgKGaORhwD6mQLhpHTFWLtHbQrW
+evDCf0KYwQk/UUS/poCIHy31Eo9mZJz+J2+m2ohubtvJdCiHf3uSAokCgZyLI8Ss
+YSwVR0s0WktT/r2XhdgtkfV8jctHgpwMMgjHA9EfE1UThkO4uGULDGzI9Sz6TsIV
+fxBqdt5w/rqVjlW5/8EBzmFdpDIgP/H4s5lkSFrFeP+vLjAPlgz+C+NAJydH3LNO
+H/XXjPT+qY5ZtNFQ+6pJWBBsOjKa9oPzO95BodZ5QL/iJkARpSzJXkV8sx2N1gIc
+5kSljuJKBbinkfUNBDM5NkeP5LmoHVhKTPm4cBOMT1dbT2YZ/ahU86UA+Kf1U+Kj
+UCjg9PmkxVq3uuUbdnBRTCyO1JhBzS4tDQfSQiAIuPpfm/VkDZZmjgvBIThsEc5n
+oVMh5lzNwTVtvDJ8ihSe2wRTHe7ic/rwDzq6+js7jb1Qr9iNiApE7IoSI/88ORl8
+ZDzu2GzLvayhtJyhdTdyQdYFCe/Ax5oW43mhkrx8PObDw9P9XT2SuPBooPIutmgs
+X/DpST5bfGYQQaJc7OiE1DWu+UcPXybI91Frr9lRWiLkee2xVpmN7APSj4HX+41U
+JQGSpVcpYuyJMmmZsQ2TRUKE3SAvbqNGoICAz/Myr9J9uQINBFCJNL4BEADjvAQE
+e7AXao9kUTETXlkgOEvRrh31d2EAEZ6mLGyE7NY0YAKO48fWeQvNHGTsTlFSuwiY
+7U6lNCQfUD/88FfYZOYusumkE8rHJUMkzTddGl7v98yZWuQ7oc/grPuVHFQJjSV/
+RYEzFb4SgxCGPyzZm/TKASzLrdQpA8fljwleOVjiAQg3Y9h6SI4nuKYbNP+uUloz
+4lYN1rBWZY1caSQhlz+PBHrdm3/Gn2B9Zcd7EDHDW8iQAiD7c/mCs3O4yCsCkls4
+NVcmIpG5PXlgsXM+lSjSSoveGB/XO1PF903WfBtJBmFJXY2A73rMpZ7kJXBgrUhi
+jgIazfwbtkdIKRYGJduO7u/Qpo6LXnzHb/ARlA1urvf0nqfcTd3VvEvO4ejP7b7N
+AoZZAw6SXncwne04XYpBrxUSX1v/LE//qqEyrJMkkGgyHjSZ8WbsisBCyzAbwmRq
+WpsJp5BBob7mVqtIfw8Mk7OEABPDlR1yUjN1De4TOJSSFIfrESidRQF103UmOHFD
+WkvjirY9iyfOcxaoXwGeKRcCwoGGybVOYXo9i5zCTW2HEpI8W5PnOSI5KGURfhMY
+2ZXNFS2DU+e2a4sGf6V+NqH4zsjEf3ZB0LhYpP+Cu79WXpn2k3ZtO0hDAZJ0ejmb
+Mv+fr4hLV4ood6bUzGk/a6MPoPnBaMXV5IrCrwARAQABiQIfBBgBAgAJBQJQiTS+
+AhsMAAoJELcvXsYYXKdw3rQP/A/yLHEgZVmN04wg9j+7d/l9+FEivBBQ90CbHvXZ
+wlji3LJBUCKCH4UTO+dnrxbtlvaMBL8UkHRvqLUADqi+9pNRB9oOy7AA7lfCOCzm
+Dpy8niBPSG/ghTwZJs0xM63WKeiE1FO6qoJIXIJ5bShh6HpLSI8RPILcQihHcsZM
+2s7pupwkEJEgWTl1VN3s5vgNkBxz+l09RhG3SmydLSLiUfAA7tPKQcdnPywwQa3l
+Ura67dQ287kflPDCbHy1XGF/0cN1figNuKC0c4Pdct3H4t7DVP5JneQsONk49NAf
+hV0J8udTKrOCsqh6G4+tiCOvHi+R0QdqXBrKGnnUMsbtnGEnlcmqwTolDzWJKBgy
+aCnQ8mUSATTStFvgEk39M3h4a2P6JLdcm01bb27esJExGylXj1GmJjZoa3tXAI+L
+jcBc2I5MyXJvvF1n7Q/k1fHBOQ8lXZxdNEjw7j86HUQD12hqUFDC5BOET69UwMHL
+qDx/WhR2f7Huq8Ok0By3dLzJNFPw6HIpF6UidTenGhN5UcCP/pxn1zYgq/DCHaH4
+JAXjvOvhu5NFF0uL+RSpNZ8E8reD5BSQ7wCV+q2T8SGfTZrQOWqOb8rtQ0+SIaF4
+A88LqwHvPkfsuoLQN9uIVyMHhTbe0s2D3yOEgI7p5ZpvDyAX96MAa5N4/sOhEKGg
+q3iu
+=RChS
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/src/leap/bitmask/mail/smtp/tests/cert/server.crt b/src/leap/bitmask/mail/smtp/tests/cert/server.crt
new file mode 100644
index 00000000..a27391c2
--- /dev/null
+++ b/src/leap/bitmask/mail/smtp/tests/cert/server.crt
@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIFBjCCAu4CCQCWn3oMoQrDJTANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJV
+UzETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0
+cyBQdHkgTHRkMB4XDTEzMTAyMzE0NDUwNFoXDTE2MDcxOTE0NDUwNFowRTELMAkG
+A1UEBhMCVVMxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0
+IFdpZGdpdHMgUHR5IEx0ZDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB
+APexTvEvG7cSmZdAERHt9TB11cSor54Y/F7NmYMdSOJNi4Y0kwkSslpdfipi+mt/
+NFg/uGKi1mcgvuXdVbVPZ9rCgVpIzMncO8RAP7a5+I2zKUzqMCCbLH16sYpo/rDk
+VQ5V15TwLsTzOFGG8Cgp68TR8zHuZ4Edf2zMGC1IaiJ6W38LTnJgsowYOCFDAF3z
+L36kxMO5gNGEUYV6tjltx+rAcXka3po+xiAgvW6q65UUgDHcIdEGG2dc9bkxxPl7
+RkprF2RwwADNzYS7Tn+Hpmjy06pfYZHNME+Iw515bCRF3GQFUU4BpGnY7EO+h4P9
+Kb1h948gUT9/oswXG+q2Kwk8AoggMJkUOWDFiCa5UjW1GBoxxb7VtZ+QTJXxlFWc
+M2VzT7M/HX+P4b05vY4MXJjxPAFKrAGS7J8DKW8WJNUnXa9XSDBHg5qijDzZ/zGm
+HTdG6iADnJLmOHBQgFQ12a/n9mYV2GPVC6FlgDzG9f0/SUPBUCafyWYz1LwKY4VM
+2NLx/iwYMQsNIMSZQfNmufNDBr70+BShe3ZpbmKB/J33d87AuJd2HjnsThTEAAr+
+6CejyYmwFutoDUCF8IaKGJEp7OGP2//ub4nt5WwW8DYLRi8EqtzEnxPo5ZiayHMY
+GHR1jpX1O5JVJFUE79bZCFFHKmtJc4kVZS4m4rTLsk83AgMBAAEwDQYJKoZIhvcN
+AQEFBQADggIBAEt4PIRqVuALQSdgZ+GiZYuvEVjxoDVtMSc/ym93Gi8R7DDivFH9
+4suQc5QUiuEF8lpEtkmh+PZ+oFdQkjhBH80h7p4BUSyBy5Yi6dy7ATTlBAqwzCYZ
+4wzHeJzu1SI6FinZLksoULbcw04n410aGHkLa6I9O3vCC4kXSnBlwU1sUsJphxM2
+3pkHBpvv79XYf5kFqZPzF16aO7rxFuVvqgXLyzwuyP9kH5zMA21Kioxs/pNyg1lm
+5h0VinpHLPse+4tYih1L1WLMpEZiSwZgFhoRtlcdIVXokZPaX4G2EkdrMmSQruWg
+Uz8Av6LEYHmRfbYwYM2kEX/+AF8thpTQDbvxjqYk5oyGX4wpKGpih1ac/jYu3O8B
+VLhbxZlBYcLxCqqNsGJrWaiHj2Jf4GhUB0O9hXfaZDMqEGXT9GzOz0yF6b3pDQVy
+H0lKIBb+kQzB/jhZKu4vrTAowXtt/av5d7D+rpAU1SxfUhBOPNSRoJUI5NSBbokp
+a7u4azdB2IQETX3d2rhDk09EbG1XmMi5Vg1oa8nxfMOWXZnDMusJoZClKjrthmwd
+rtR5et44XYhX6p217RBkYMDOVFT7aZpu4SaFeqZIuarVYodSmgXToOFXPsrLppRQ
+adOT0FpU64RPNrQz5NF1bSIjqrHSaRVacf8yr7qqxNnpMsrtkDJzsMBz
+-----END CERTIFICATE-----
diff --git a/src/leap/bitmask/mail/smtp/tests/cert/server.key b/src/leap/bitmask/mail/smtp/tests/cert/server.key
new file mode 100644
index 00000000..197a4496
--- /dev/null
+++ b/src/leap/bitmask/mail/smtp/tests/cert/server.key
@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKQIBAAKCAgEA97FO8S8btxKZl0AREe31MHXVxKivnhj8Xs2Zgx1I4k2LhjST
+CRKyWl1+KmL6a380WD+4YqLWZyC+5d1VtU9n2sKBWkjMydw7xEA/trn4jbMpTOow
+IJssfXqximj+sORVDlXXlPAuxPM4UYbwKCnrxNHzMe5ngR1/bMwYLUhqInpbfwtO
+cmCyjBg4IUMAXfMvfqTEw7mA0YRRhXq2OW3H6sBxeRremj7GICC9bqrrlRSAMdwh
+0QYbZ1z1uTHE+XtGSmsXZHDAAM3NhLtOf4emaPLTql9hkc0wT4jDnXlsJEXcZAVR
+TgGkadjsQ76Hg/0pvWH3jyBRP3+izBcb6rYrCTwCiCAwmRQ5YMWIJrlSNbUYGjHF
+vtW1n5BMlfGUVZwzZXNPsz8df4/hvTm9jgxcmPE8AUqsAZLsnwMpbxYk1Sddr1dI
+MEeDmqKMPNn/MaYdN0bqIAOckuY4cFCAVDXZr+f2ZhXYY9ULoWWAPMb1/T9JQ8FQ
+Jp/JZjPUvApjhUzY0vH+LBgxCw0gxJlB82a580MGvvT4FKF7dmluYoH8nfd3zsC4
+l3YeOexOFMQACv7oJ6PJibAW62gNQIXwhooYkSns4Y/b/+5vie3lbBbwNgtGLwSq
+3MSfE+jlmJrIcxgYdHWOlfU7klUkVQTv1tkIUUcqa0lziRVlLibitMuyTzcCAwEA
+AQKCAgAFQdcqGVTeQt/NrQdvuPw+RhH+dZIcqe0ZWgXLGaEFZJ30gEMqqyHr9xYJ
+ckZcZ7vFr7yLI2enkrNaj6MVERVkOEKkluz5z9nY5YA0safL4iSbRFE3L/P2ydbg
+2C+ns4D2p+3GdH6ZoYvtdw6723/skoQ16Bh8ThL5TS+qLmJKTwyIGsZUeSbxAEaY
+tiJY3COC7Z5bhSFt0QAl9B/QAjt/CQyfhGl7Hp/36Jn8slYDuQariD+TfyyvufJh
+NuQ2Y15vj+xULmx01+lnys30uP1YNuc1M4cPoCpJVd7JBd28u1rdKJu8Kx7BPGBv
+Y6jerU3ofh7SA96VmXDsIgVuquUo51Oklspe6a9VaDmzLvjYqJsBKQ7BH3J2f07x
+NiOob56CGXykX51Ig3WBK1wKn+pA69FL62DbkEa6SykGCqdZPdgBF/kiMc0TESsl
+867Em63Yx/2hq+mG3Dknnq8jWXf+Es/zZSSak6N4154IxPOD3m1hzuUq73PP7Ptt
+KFe6NfU0DmAuTJL3FqNli8F8lFfvJfuwmW2qk5iTMfwPxybSd8FPbGxi7aRgoZdh
+7fIbTFJ0X2f83/SO+9rCzV+B091+d7TM8AaOJ4dEoS74rlRZg53EgmAU0phVnE+l
+taMNKGHy2kpJrv9IHX3w5Gm6CjNJj5t4ccS0J18NFFJ+j077eQKCAQEA/RJNRUBS
+mI5l0eirl78Q9uDPh1usChZpQiLsvscIJITWQ1vtXSRCvP0hVQRRv8+4CtrZr2rX
+v0afkzg/3HNFaNsjYT6aHjgnombFqfpyS/NZN/p3gOzi2h+1Sujzz5fBUGhNLVgZ
+F2GLnJbiIHnM1BmKA6597pHpXcRMh1E3DSjDMQAEEsBgF6MyS+MT9WfNwHvJukii
+k028tNzR4wRq3Xo3WTfvXZRjbX54Ew9Zy3+TFiu19j2FmuOoqyj+ZvMic4EYmTaY
+BWm7viDff4dW34dR9sYCuTWWehLtMJGroA38e7lTLfNOHNDGaUZWkfxs4uJCsxvP
+0fPp3xlbU3NUGwKCAQEA+o8SeHwEN+VN2dZvC3wFvbnRvWLc1aLnNcndRE9QLVcC
+B4LMRuQMpxaNYRiSQPppoPTNq6zWbo6FEjUO5Md7R8I8dbg1vHo4PzuHOu2wXNcm
+DEicocCpSKShSS27NCK6uoSsTqTIlG4u+1x9/R2gJEjlTqjeIkOQkPv7PbWhrUyt
+XqvzPy4bewOz9Brmd6ryi8ZLtNbUSNwMyd64s9b1V4A6JRlYZrMDOQ6kXEZo+mbL
+ynet0vuj7lYxsAZvxoPIq+Gi5i0CrDYtze6JCg+kGahjMX0zXRjXrYh/YID8NWYT
+0GXr2+a0V5pXg86YCDp/jpr3lq75HJJ+vIvm2VHLFQKCAQATEm0GWgmfe6PKxPkh
+j4GsyVZ6gfseK4A1PsKOwhsn/WbUXrotuczZx03axV+P0AyzrLiZErk9rgnao3OU
+no9Njq5E5t3ghyTdhVdCLyCr/qPrpxGYgsG55IfaJGIzc+FauPGQCEKj03MdEvXp
+sqQwG9id3GmbMB3hNij6TbGTaU4EhFbKPvs+7Mqek3dumCsWZX3Xbx/pcANXsgiT
+TkLrfAltzNxaNhOkLdLIxPBkeLHSCutEqnBGMwAEHivGAG7JO6Jp8YZVahl/A6U0
+TDPM1rrjmRqdcJ9thb2gWmoPvt4XSOku3lY1r7o0NtvRVq+yDZEvRFpOHU6zxIpw
+aJGfAoIBAQDiTvvF62379pc8nJwr6VdeKEozHuqL49mmEbBTFLg8W4wvsIpFtZFg
+EdSc0I65NfTWNobV+wSrUvsKmPXc2fiVtfDZ+wo+NL49Ds10Al/7WzC4g5VF3DiK
+rngnGrEtw/iYo2Dmn5uzxVmWG9KIHowYeeb0Bz6sAA7BhXdGI5nmZ41oJzNL659S
+muOdJfboO3Vbnj2fFzMio+7BHvQBK7Tp1Z2vCJd6G1Jb5Me7uLT1BognVbWhDTzh
+9uRmM0oeKcXEycZS1HDHjyAMEtmgRsRXkGoXtxf/jIKx8MnsJlSm/o4C+yvvsQ9O
+2M8W9DEJrZys93eNmHjUv9TNBCf8Pg6JAoIBAQDDItnQPLntCUgd7dy0dDjQYBGN
+4wVRJNINpgjqwJj0hVjB/dmvrcxkXcOG4VAH+iNH8A25qLU+RTDcNipuL3uEFKbF
+O4DSjFih3qL1Y8otTXSrPeqZOMvYpY8dXS5uyI7DSWQQZyZ9bMpeWbxgx4LHqPPH
+rdcVJy9Egw1ZIOA7JBFM02uGn9TVwFzNUJk0G/3xwVHzDxYNbJ98vDfflc2vD4CH
+OAN6un0pOuol2h200F6zFgc5mbETWHCPIom+ZMXIX3bq7g341c/cgqIELPTk8DLS
+s+AgrZ4qYmskrFaD0PHakWsQNHGC8yOh80lgE3Gl4nxSGAvkcR7dkSmsIQFL
+-----END RSA PRIVATE KEY-----
diff --git a/src/leap/bitmask/mail/smtp/tests/mail.txt b/src/leap/bitmask/mail/smtp/tests/mail.txt
new file mode 100644
index 00000000..95420470
--- /dev/null
+++ b/src/leap/bitmask/mail/smtp/tests/mail.txt
@@ -0,0 +1,10 @@
+HELO drebs@riseup.net
+MAIL FROM: drebs@riseup.net
+RCPT TO: drebs@riseup.net
+RCPT TO: drebs@leap.se
+DATA
+Subject: leap test
+
+Hello world!
+.
+QUIT
diff --git a/src/leap/bitmask/mail/smtp/tests/test_gateway.py b/src/leap/bitmask/mail/smtp/tests/test_gateway.py
new file mode 100644
index 00000000..9d88afb8
--- /dev/null
+++ b/src/leap/bitmask/mail/smtp/tests/test_gateway.py
@@ -0,0 +1,181 @@
+# -*- coding: utf-8 -*-
+# test_gateway.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+SMTP gateway tests.
+"""
+import re
+import tempfile
+from datetime import datetime
+
+from twisted.internet import reactor
+from twisted.internet.defer import inlineCallbacks, fail, succeed, Deferred
+from twisted.test import proto_helpers
+
+from mock import Mock
+
+from leap.keymanager import openpgp, errors
+from leap.mail.testing import KeyManagerWithSoledadTestCase
+from leap.mail.testing import ADDRESS, ADDRESS_2
+from leap.mail.testing.smtp import getSMTPFactory, TEST_USER
+
+
+# some regexps
+IP_REGEX = "(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}" + \
+ "([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])"
+HOSTNAME_REGEX = "(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*" + \
+ "([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])"
+IP_OR_HOST_REGEX = '(' + IP_REGEX + '|' + HOSTNAME_REGEX + ')'
+
+
+class TestSmtpGateway(KeyManagerWithSoledadTestCase):
+
+ EMAIL_DATA = ['HELO gateway.leap.se',
+ 'MAIL FROM: <%s>' % ADDRESS_2,
+ 'RCPT TO: <%s>' % ADDRESS,
+ 'DATA',
+ 'From: User <%s>' % ADDRESS_2,
+ 'To: Leap <%s>' % ADDRESS,
+ 'Date: ' + datetime.now().strftime('%c'),
+ 'Subject: test message',
+ '',
+ 'This is a secret message.',
+ 'Yours,',
+ 'A.',
+ '',
+ '.',
+ 'QUIT']
+
+ def setUp(self):
+ # pytest handles correctly the setupEnv for the class,
+ # but trial ignores it.
+ if not getattr(self, 'tempdir', None):
+ self.tempdir = tempfile.mkdtemp()
+ return KeyManagerWithSoledadTestCase.setUp(self)
+
+ def tearDown(self):
+ return KeyManagerWithSoledadTestCase.tearDown(self)
+
+ def assertMatch(self, string, pattern, msg=None):
+ if not re.match(pattern, string):
+ msg = self._formatMessage(msg, '"%s" does not match pattern "%s".'
+ % (string, pattern))
+ raise self.failureException(msg)
+
+ @inlineCallbacks
+ def test_gateway_accepts_valid_email(self):
+ """
+ Test if SMTP server responds correctly for valid interaction.
+ """
+
+ SMTP_ANSWERS = ['220 ' + IP_OR_HOST_REGEX +
+ ' NO UCE NO UBE NO RELAY PROBES',
+ '250 ' + IP_OR_HOST_REGEX + ' Hello ' +
+ IP_OR_HOST_REGEX + ', nice to meet you',
+ '250 Sender address accepted',
+ '250 Recipient address accepted',
+ '354 Continue']
+
+ user = TEST_USER
+ proto = getSMTPFactory({user: None}, {user: self.km}, {user: None})
+ transport = proto_helpers.StringTransport()
+ proto.makeConnection(transport)
+ reply = ""
+ for i, line in enumerate(self.EMAIL_DATA):
+ reply += yield self.getReply(line + '\r\n', proto, transport)
+ self.assertMatch(reply, '\r\n'.join(SMTP_ANSWERS),
+ 'Did not get expected answer from gateway.')
+ proto.setTimeout(None)
+
+ @inlineCallbacks
+ def test_missing_key_rejects_address(self):
+ """
+ Test if server rejects to send unencrypted when 'encrypted_only' is
+ True.
+ """
+ # remove key from key manager
+ pubkey = yield self.km.get_key(ADDRESS)
+ pgp = openpgp.OpenPGPScheme(
+ self._soledad, gpgbinary=self.gpg_binary_path)
+ yield pgp.delete_key(pubkey)
+ # mock the key fetching
+ self.km._fetch_keys_from_server = Mock(
+ return_value=fail(errors.KeyNotFound()))
+ user = TEST_USER
+ proto = getSMTPFactory(
+ {user: None}, {user: self.km}, {user: None},
+ encrypted_only=True)
+ transport = proto_helpers.StringTransport()
+ proto.makeConnection(transport)
+ yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport)
+ yield self.getReply(self.EMAIL_DATA[1] + '\r\n', proto, transport)
+ reply = yield self.getReply(self.EMAIL_DATA[2] + '\r\n',
+ proto, transport)
+ # ensure the address was rejected
+ self.assertEqual(
+ '550 Cannot receive for specified address\r\n',
+ reply,
+ 'Address should have been rejected with appropriate message.')
+ proto.setTimeout(None)
+
+ @inlineCallbacks
+ def test_missing_key_accepts_address(self):
+ """
+ Test if server accepts to send unencrypted when 'encrypted_only' is
+ False.
+ """
+ # remove key from key manager
+ pubkey = yield self.km.get_key(ADDRESS)
+ pgp = openpgp.OpenPGPScheme(
+ self._soledad, gpgbinary=self.gpg_binary_path)
+ yield pgp.delete_key(pubkey)
+ # mock the key fetching
+ self.km._fetch_keys_from_server = Mock(
+ return_value=fail(errors.KeyNotFound()))
+ user = TEST_USER
+ proto = getSMTPFactory({user: None}, {user: self.km}, {user: None})
+ transport = proto_helpers.StringTransport()
+ proto.makeConnection(transport)
+ yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport)
+ yield self.getReply(self.EMAIL_DATA[1] + '\r\n', proto, transport)
+ reply = yield self.getReply(self.EMAIL_DATA[2] + '\r\n',
+ proto, transport)
+ # ensure the address was accepted
+ self.assertEqual(
+ '250 Recipient address accepted\r\n',
+ reply,
+ 'Address should have been accepted with appropriate message.')
+ proto.setTimeout(None)
+
+ def getReply(self, line, proto, transport):
+ proto.lineReceived(line)
+
+ if line[:4] not in ['HELO', 'MAIL', 'RCPT', 'DATA']:
+ return succeed("")
+
+ def check_transport(_):
+ reply = transport.value()
+ if reply:
+ transport.clear()
+ return succeed(reply)
+
+ d = Deferred()
+ d.addCallback(check_transport)
+ reactor.callLater(0, lambda: d.callback(None))
+ return d
+
+ return check_transport(None)
diff --git a/src/leap/bitmask/mail/sync_hooks.py b/src/leap/bitmask/mail/sync_hooks.py
new file mode 100644
index 00000000..8efbb7ce
--- /dev/null
+++ b/src/leap/bitmask/mail/sync_hooks.py
@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+# sync_hooks.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledad PostSync Hooks.
+
+Process every new document of interest after every soledad synchronization,
+using the hooks that soledad exposes via plugins.
+"""
+import logging
+
+from re import compile as regex_compile
+
+from zope.interface import implements
+from twisted.internet import defer
+from twisted.plugin import IPlugin
+from twisted.python import log
+
+from leap.soledad.client.interfaces import ISoledadPostSyncPlugin
+from leap.mail import constants
+
+logger = logging.getLogger(__name__)
+
+
+def _get_doc_type_preffix(s):
+ return s[:2]
+
+
+class MailProcessingPostSyncHook(object):
+ implements(IPlugin, ISoledadPostSyncPlugin)
+
+ META_DOC_PREFFIX = _get_doc_type_preffix(constants.METAMSGID)
+ watched_doc_types = (META_DOC_PREFFIX, )
+
+ _account = None
+ _pending_docs = []
+ _processing_deferreds = []
+
+ def process_received_docs(self, doc_id_list):
+ if self._has_configured_account():
+ process_fun = self._make_uid_index
+ else:
+ self._processing_deferreds = []
+ process_fun = self._queue_doc_id
+
+ for doc_id in doc_id_list:
+ if _get_doc_type_preffix(doc_id) in self.watched_doc_types:
+ log.msg("Mail post-sync hook: processing %s" % doc_id)
+ process_fun(doc_id)
+
+ return defer.gatherResults(self._processing_deferreds)
+
+ def set_account(self, account):
+ self._account = account
+ if account:
+ self._process_queued_docs()
+
+ def _has_configured_account(self):
+ return self._account is not None
+
+ def _queue_doc_id(self, doc_id):
+ self._pending_docs.append(doc_id)
+
+ def _make_uid_index(self, mdoc_id):
+ indexer = self._account.mbox_indexer
+ mbox_uuid = _get_mbox_uuid(mdoc_id)
+ if mbox_uuid:
+ chash = _get_chash_from_mdoc(mdoc_id)
+ logger.debug("Making index table for %s:%s" % (mbox_uuid, chash))
+ index_docid = constants.METAMSGID.format(
+ mbox_uuid=mbox_uuid.replace('-', '_'),
+ chash=chash)
+ # XXX could avoid creating table if I track which ones I already
+ # have seen -- but make sure *it's already created* before
+ # inserting the index entry!.
+ d = indexer.create_table(mbox_uuid)
+ d.addBoth(lambda _: indexer.insert_doc(mbox_uuid, index_docid))
+ self._processing_deferreds.append(d)
+
+ def _process_queued_docs(self):
+ assert(self._has_configured_account())
+ pending = self._pending_docs
+ log.msg("Mail post-sync hook: processing queued docs")
+
+ def remove_pending_docs(res):
+ self._pending_docs = []
+ return res
+
+ d = self.process_received_docs(pending)
+ d.addCallback(remove_pending_docs)
+ return d
+
+
+_mbox_uuid_regex = regex_compile(constants.METAMSGID_MBOX_RE)
+_mdoc_chash_regex = regex_compile(constants.METAMSGID_CHASH_RE)
+
+
+def _get_mbox_uuid(doc_id):
+ matches = _mbox_uuid_regex.findall(doc_id)
+ if matches:
+ return matches[0].replace('_', '-')
+
+
+def _get_chash_from_mdoc(doc_id):
+ matches = _mdoc_chash_regex.findall(doc_id)
+ if matches:
+ return matches[0]
diff --git a/src/leap/bitmask/mail/testing/__init__.py b/src/leap/bitmask/mail/testing/__init__.py
new file mode 100644
index 00000000..8822a5c7
--- /dev/null
+++ b/src/leap/bitmask/mail/testing/__init__.py
@@ -0,0 +1,353 @@
+# -*- coding: utf-8 -*-
+# __init__.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Base classes and keys for leap.mail tests.
+"""
+import os
+import distutils.spawn
+from mock import Mock
+from twisted.internet.defer import gatherResults
+from twisted.trial import unittest
+from twisted.internet import defer
+
+
+from leap.soledad.client import Soledad
+from leap.keymanager import KeyManager
+
+
+from leap.common.testing.basetest import BaseLeapTest
+
+ADDRESS = 'leap@leap.se'
+ADDRESS_2 = 'anotheruser@leap.se'
+
+
+class defaultMockSharedDB(object):
+ get_doc = Mock(return_value=None)
+ put_doc = Mock(side_effect=None)
+ open = Mock(return_value=None)
+ close = Mock(return_value=None)
+ syncable = True
+
+ def __call__(self):
+ return self
+
+
+class KeyManagerWithSoledadTestCase(unittest.TestCase, BaseLeapTest):
+
+ def setUp(self):
+ self.gpg_binary_path = self._find_gpg()
+
+ self._soledad = Soledad(
+ u"leap@leap.se",
+ u"123456",
+ secrets_path=self.tempdir + "/secret.gpg",
+ local_db_path=self.tempdir + "/soledad.u1db",
+ server_url='',
+ cert_file=None,
+ auth_token=None,
+ shared_db=defaultMockSharedDB(),
+ syncable=False)
+
+ self.km = self._key_manager()
+
+ class Response(object):
+ code = 200
+ phrase = ''
+
+ def deliverBody(self, x):
+ return ''
+
+ self.km._async_client_pinned.request = Mock(
+ return_value=defer.succeed(Response()))
+
+ d1 = self.km.put_raw_key(PRIVATE_KEY, ADDRESS)
+ d2 = self.km.put_raw_key(PRIVATE_KEY_2, ADDRESS_2)
+ return gatherResults([d1, d2])
+
+ def tearDown(self):
+ km = self._key_manager()
+ # wait for the indexes to be ready for the tear down
+ d = km._openpgp.deferred_init
+ d.addCallback(lambda _: self.delete_all_keys(km))
+ d.addCallback(lambda _: self._soledad.close())
+ return d
+
+ def delete_all_keys(self, km):
+ def delete_keys(keys):
+ deferreds = []
+ for key in keys:
+ d = km._openpgp.delete_key(key)
+ deferreds.append(d)
+ return gatherResults(deferreds)
+
+ def check_deleted(_, private):
+ d = km.get_all_keys(private=private)
+ d.addCallback(lambda keys: self.assertEqual(keys, []))
+ return d
+
+ deferreds = []
+ for private in [True, False]:
+ d = km.get_all_keys(private=private)
+ d.addCallback(delete_keys)
+ d.addCallback(check_deleted, private)
+ deferreds.append(d)
+ return gatherResults(deferreds)
+
+ def _key_manager(self, user=ADDRESS, url='', token=None,
+ ca_cert_path=None):
+ return KeyManager(user, url, self._soledad, token=token,
+ gpgbinary=self.gpg_binary_path,
+ ca_cert_path=ca_cert_path)
+
+ def _find_gpg(self):
+ gpg_path = distutils.spawn.find_executable('gpg')
+ if gpg_path is not None:
+ return os.path.realpath(gpg_path)
+ else:
+ return "/usr/bin/gpg"
+
+ def get_public_binary_key(self):
+ with open(PATH + '/fixtures/public_key.bin', 'r') as binary_public_key:
+ return binary_public_key.read()
+
+ def get_private_binary_key(self):
+ with open(
+ PATH + '/fixtures/private_key.bin', 'r') as binary_private_key:
+ return binary_private_key.read()
+
+
+# key 24D18DDF: public key "Leap Test Key <leap@leap.se>"
+KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF"
+PUBLIC_KEY = """
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.10 (GNU/Linux)
+
+mQINBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz
+iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO
+zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx
+irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT
+huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs
+d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g
+wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb
+hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv
+U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H
+T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i
+Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB
+tBxMZWFwIFRlc3QgS2V5IDxsZWFwQGxlYXAuc2U+iQI3BBMBCAAhBQJQvfnZAhsD
+BQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEC9FXigk0Y3fT7EQAKH3IuRniOpb
+T/DDIgwwjz3oxB/W0DDMyPXowlhSOuM0rgGfntBpBb3boezEXwL86NPQxNGGruF5
+hkmecSiuPSvOmQlqlS95NGQp6hNG0YaKColh+Q5NTspFXCAkFch9oqUje0LdxfSP
+QfV9UpeEvGyPmk1I9EJV/YDmZ4+Djge1d7qhVZInz4Rx1NrSyF/Tc2EC0VpjQFsU
+Y9Kb2YBBR7ivG6DBc8ty0jJXi7B4WjkFcUEJviQpMF2dCLdonCehYs1PqsN1N7j+
+eFjQd+hqVMJgYuSGKjvuAEfClM6MQw7+FmFwMyLgK/Ew/DttHEDCri77SPSkOGSI
+txCzhTg6798f6mJr7WcXmHX1w1Vcib5FfZ8vTDFVhz/XgAgArdhPo9V6/1dgSSiB
+KPQ/spsco6u5imdOhckERE0lnAYvVT6KE81TKuhF/b23u7x+Wdew6kK0EQhYA7wy
+7LmlaNXc7rMBQJ9Z60CJ4JDtatBWZ0kNrt2VfdDHVdqBTOpl0CraNUjWE5YMDasr
+K2dF5IX8D3uuYtpZnxqg0KzyLg0tzL0tvOL1C2iudgZUISZNPKbS0z0v+afuAAnx
+2pTC3uezbh2Jt8SWTLhll4i0P4Ps5kZ6HQUO56O+/Z1cWovX+mQekYFmERySDR9n
+3k1uAwLilJmRmepGmvYbB8HloV8HqwgguQINBFC9+dkBEAC0I/xn1uborMgDvBtf
+H0sEhwnXBC849/32zic6udB6/3Efk9nzbSpL3FSOuXITZsZgCHPkKarnoQ2ztMcS
+sh1ke1C5gQGms75UVmM/nS+2YI4vY8OX/GC/on2vUyncqdH+bR6xH5hx4NbWpfTs
+iQHmz5C6zzS/kuabGdZyKRaZHt23WQ7JX/4zpjqbC99DjHcP9BSk7tJ8wI4bkMYD
+uFVQdT9O6HwyKGYwUU4sAQRAj7XCTGvVbT0dpgJwH4RmrEtJoHAx4Whg8mJ710E0
+GCmzf2jqkNuOw76ivgk27Kge+Hw00jmJjQhHY0yVbiaoJwcRrPKzaSjEVNgrpgP3
+lXPRGQArgESsIOTeVVHQ8fhK2YtTeCY9rIiO+L0OX2xo9HK7hfHZZWL6rqymXdyS
+fhzh/f6IPyHFWnvj7Brl7DR8heMikygcJqv+ed2yx7iLyCUJ10g12I48+aEj1aLe
+dP7lna32iY8/Z0SHQLNH6PXO9SlPcq2aFUgKqE75A/0FMk7CunzU1OWr2ZtTLNO1
+WT/13LfOhhuEq9jTyTosn0WxBjJKq18lnhzCXlaw6EAtbA7CUwsD3CTPR56aAXFK
+3I7KXOVAqggrvMe5Tpdg5drfYpI8hZovL5aAgb+7Y5ta10TcJdUhS5K3kFAWe/td
+U0cmWUMDP1UMSQ5Jg6JIQVWhSwARAQABiQIfBBgBCAAJBQJQvfnZAhsMAAoJEC9F
+Xigk0Y3fRwsP/i0ElYCyxeLpWJTwo1iCLkMKz2yX1lFVa9nT1BVTPOQwr/IAc5OX
+NdtbJ14fUsKL5pWgW8OmrXtwZm1y4euI1RPWWubG01ouzwnGzv26UcuHeqC5orZj
+cOnKtL40y8VGMm8LoicVkRJH8blPORCnaLjdOtmA3rx/v2EXrJpSa3AhOy0ZSRXk
+ZSrK68AVNwamHRoBSYyo0AtaXnkPX4+tmO8X8BPfj125IljubvwZPIW9VWR9UqCE
+VPfDR1XKegVb6VStIywF7kmrknM1C5qUY28rdZYWgKorw01hBGV4jTW0cqde3N51
+XT1jnIAa+NoXUM9uQoGYMiwrL7vNsLlyyiW5ayDyV92H/rIuiqhFgbJsHTlsm7I8
+oGheR784BagAA1NIKD1qEO9T6Kz9lzlDaeWS5AUKeXrb7ZJLI1TTCIZx5/DxjLqM
+Tt/RFBpVo9geZQrvLUqLAMwdaUvDXC2c6DaCPXTh65oCZj/hqzlJHH+RoTWWzKI+
+BjXxgUWF9EmZUBrg68DSmI+9wuDFsjZ51BcqvJwxyfxtTaWhdoYqH/UQS+D1FP3/
+diZHHlzwVwPICzM9ooNTgbrcDzyxRkIVqsVwBq7EtzcvgYUyX53yG25Giy6YQaQ2
+ZtQ/VymwFL3XdUWV6B/hU4PVAFvO3qlOtdJ6TpE+nEWgcWjCv5g7RjXX
+=MuOY
+-----END PGP PUBLIC KEY BLOCK-----
+"""
+PRIVATE_KEY = """
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: GnuPG v1.4.10 (GNU/Linux)
+
+lQcYBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz
+iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO
+zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx
+irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT
+huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs
+d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g
+wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb
+hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv
+U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H
+T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i
+Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB
+AA/+JHtlL39G1wsH9R6UEfUQJGXR9MiIiwZoKcnRB2o8+DS+OLjg0JOh8XehtuCs
+E/8oGQKtQqa5bEIstX7IZoYmYFiUQi9LOzIblmp2vxOm+HKkxa4JszWci2/ZmC3t
+KtaA4adl9XVnshoQ7pijuCMUKB3naBEOAxd8s9d/JeReGIYkJErdrnVfNk5N71Ds
+FmH5Ll3XtEDvgBUQP3nkA6QFjpsaB94FHjL3gDwum/cxzj6pCglcvHOzEhfY0Ddb
+J967FozQTaf2JW3O+w3LOqtcKWpq87B7+O61tVidQPSSuzPjCtFF0D2LC9R/Hpky
+KTMQ6CaKja4MPhjwywd4QPcHGYSqjMpflvJqi+kYIt8psUK/YswWjnr3r4fbuqVY
+VhtiHvnBHQjz135lUqWvEz4hM3Xpnxydx7aRlv5NlevK8+YIO5oFbWbGNTWsPZI5
+jpoFBpSsnR1Q5tnvtNHauvoWV+XN2qAOBTG+/nEbDYH6Ak3aaE9jrpTdYh0CotYF
+q7csANsDy3JvkAzeU6WnYpsHHaAjqOGyiZGsLej1UcXPFMosE/aUo4WQhiS8Zx2c
+zOVKOi/X5vQ2GdNT9Qolz8AriwzsvFR+bxPzyd8V6ALwDsoXvwEYinYBKK8j0OPv
+OOihSR6HVsuP9NUZNU9ewiGzte/+/r6pNXHvR7wTQ8EWLcEIAN6Zyrb0bHZTIlxt
+VWur/Ht2mIZrBaO50qmM5RD3T5oXzWXi/pjLrIpBMfeZR9DWfwQwjYzwqi7pxtYx
+nJvbMuY505rfnMoYxb4J+cpRXV8MS7Dr1vjjLVUC9KiwSbM3gg6emfd2yuA93ihv
+Pe3mffzLIiQa4mRE3wtGcioC43nWuV2K2e1KjxeFg07JhrezA/1Cak505ab/tmvP
+4YmjR5c44+yL/YcQ3HdFgs4mV+nVbptRXvRcPpolJsgxPccGNdvHhsoR4gwXMS3F
+RRPD2z6x8xeN73Q4KH3bm01swQdwFBZbWVfmUGLxvN7leCdfs9+iFJyqHiCIB6Iv
+mQfp8F0IAOwSo8JhWN+V1dwML4EkIrM8wUb4yecNLkyR6TpPH/qXx4PxVMC+vy6x
+sCtjeHIwKE+9vqnlhd5zOYh7qYXEJtYwdeDDmDbL8oks1LFfd+FyAuZXY33DLwn0
+cRYsr2OEZmaajqUB3NVmj3H4uJBN9+paFHyFSXrH68K1Fk2o3n+RSf2EiX+eICwI
+L6rqoF5sSVUghBWdNegV7qfy4anwTQwrIMGjgU5S6PKW0Dr/3iO5z3qQpGPAj5OW
+ATqPWkDICLbObPxD5cJlyyNE2wCA9VVc6/1d6w4EVwSq9h3/WTpATEreXXxTGptd
+LNiTA1nmakBYNO2Iyo3djhaqBdWjk+EIAKtVEnJH9FAVwWOvaj1RoZMA5DnDMo7e
+SnhrCXl8AL7Z1WInEaybasTJXn1uQ8xY52Ua4b8cbuEKRKzw/70NesFRoMLYoHTO
+dyeszvhoDHberpGRTciVmpMu7Hyi33rM31K9epA4ib6QbbCHnxkWOZB+Bhgj1hJ8
+xb4RBYWiWpAYcg0+DAC3w9gfxQhtUlZPIbmbrBmrVkO2GVGUj8kH6k4UV6kUHEGY
+HQWQR0HcbKcXW81ZXCCD0l7ROuEWQtTe5Jw7dJ4/QFuqZnPutXVRNOZqpl6eRShw
+7X2/a29VXBpmHA95a88rSQsL+qm7Fb3prqRmuMCtrUZgFz7HLSTuUMR867QcTGVh
+cCBUZXN0IEtleSA8bGVhcEBsZWFwLnNlPokCNwQTAQgAIQUCUL352QIbAwULCQgH
+AwUVCgkICwUWAgMBAAIeAQIXgAAKCRAvRV4oJNGN30+xEACh9yLkZ4jqW0/wwyIM
+MI896MQf1tAwzMj16MJYUjrjNK4Bn57QaQW926HsxF8C/OjT0MTRhq7heYZJnnEo
+rj0rzpkJapUveTRkKeoTRtGGigqJYfkOTU7KRVwgJBXIfaKlI3tC3cX0j0H1fVKX
+hLxsj5pNSPRCVf2A5mePg44HtXe6oVWSJ8+EcdTa0shf03NhAtFaY0BbFGPSm9mA
+QUe4rxugwXPLctIyV4uweFo5BXFBCb4kKTBdnQi3aJwnoWLNT6rDdTe4/nhY0Hfo
+alTCYGLkhio77gBHwpTOjEMO/hZhcDMi4CvxMPw7bRxAwq4u+0j0pDhkiLcQs4U4
+Ou/fH+pia+1nF5h19cNVXIm+RX2fL0wxVYc/14AIAK3YT6PVev9XYEkogSj0P7Kb
+HKOruYpnToXJBERNJZwGL1U+ihPNUyroRf29t7u8flnXsOpCtBEIWAO8Muy5pWjV
+3O6zAUCfWetAieCQ7WrQVmdJDa7dlX3Qx1XagUzqZdAq2jVI1hOWDA2rKytnReSF
+/A97rmLaWZ8aoNCs8i4NLcy9Lbzi9QtornYGVCEmTTym0tM9L/mn7gAJ8dqUwt7n
+s24dibfElky4ZZeItD+D7OZGeh0FDuejvv2dXFqL1/pkHpGBZhEckg0fZ95NbgMC
+4pSZkZnqRpr2GwfB5aFfB6sIIJ0HGARQvfnZARAAtCP8Z9bm6KzIA7wbXx9LBIcJ
+1wQvOPf99s4nOrnQev9xH5PZ820qS9xUjrlyE2bGYAhz5Cmq56ENs7THErIdZHtQ
+uYEBprO+VFZjP50vtmCOL2PDl/xgv6J9r1Mp3KnR/m0esR+YceDW1qX07IkB5s+Q
+us80v5LmmxnWcikWmR7dt1kOyV/+M6Y6mwvfQ4x3D/QUpO7SfMCOG5DGA7hVUHU/
+Tuh8MihmMFFOLAEEQI+1wkxr1W09HaYCcB+EZqxLSaBwMeFoYPJie9dBNBgps39o
+6pDbjsO+or4JNuyoHvh8NNI5iY0IR2NMlW4mqCcHEazys2koxFTYK6YD95Vz0RkA
+K4BErCDk3lVR0PH4StmLU3gmPayIjvi9Dl9saPRyu4Xx2WVi+q6spl3ckn4c4f3+
+iD8hxVp74+wa5ew0fIXjIpMoHCar/nndsse4i8glCddINdiOPPmhI9Wi3nT+5Z2t
+9omPP2dEh0CzR+j1zvUpT3KtmhVICqhO+QP9BTJOwrp81NTlq9mbUyzTtVk/9dy3
+zoYbhKvY08k6LJ9FsQYySqtfJZ4cwl5WsOhALWwOwlMLA9wkz0eemgFxStyOylzl
+QKoIK7zHuU6XYOXa32KSPIWaLy+WgIG/u2ObWtdE3CXVIUuSt5BQFnv7XVNHJllD
+Az9VDEkOSYOiSEFVoUsAEQEAAQAP/1AagnZQZyzHDEgw4QELAspYHCWLXE5aZInX
+wTUJhK31IgIXNn9bJ0hFiSpQR2xeMs9oYtRuPOu0P8oOFMn4/z374fkjZy8QVY3e
+PlL+3EUeqYtkMwlGNmVw5a/NbNuNfm5Darb7pEfbYd1gPcni4MAYw7R2SG/57GbC
+9gucvspHIfOSfBNLBthDzmK8xEKe1yD2eimfc2T7IRYb6hmkYfeds5GsqvGI6mwI
+85h4uUHWRc5JOlhVM6yX8hSWx0L60Z3DZLChmc8maWnFXd7C8eQ6P1azJJbW71Ih
+7CoK0XW4LE82vlQurSRFgTwfl7wFYszW2bOzCuhHDDtYnwH86Nsu0DC78ZVRnvxn
+E8Ke/AJgrdhIOo4UAyR+aZD2+2mKd7/waOUTUrUtTzc7i8N3YXGi/EIaNReBXaq+
+ZNOp24BlFzRp+FCF/pptDW9HjPdiV09x0DgICmeZS4Gq/4vFFIahWctg52NGebT0
+Idxngjj+xDtLaZlLQoOz0n5ByjO/Wi0ANmMv1sMKCHhGvdaSws2/PbMR2r4caj8m
+KXpIgdinM/wUzHJ5pZyF2U/qejsRj8Kw8KH/tfX4JCLhiaP/mgeTuWGDHeZQERAT
+xPmRFHaLP9/ZhvGNh6okIYtrKjWTLGoXvKLHcrKNisBLSq+P2WeFrlme1vjvJMo/
+jPwLT5o9CADQmcbKZ+QQ1ZM9v99iDZol7SAMZX43JC019sx6GK0u6xouJBcLfeB4
+OXacTgmSYdTa9RM9fbfVpti01tJ84LV2SyL/VJq/enJF4XQPSynT/tFTn1PAor6o
+tEAAd8fjKdJ6LnD5wb92SPHfQfXqI84rFEO8rUNIE/1ErT6DYifDzVCbfD2KZdoF
+cOSp7TpD77sY1bs74ocBX5ejKtd+aH99D78bJSMM4pSDZsIEwnomkBHTziubPwJb
+OwnATy0LmSMAWOw5rKbsh5nfwCiUTM20xp0t5JeXd+wPVWbpWqI2EnkCEN+RJr9i
+7dp/ymDQ+Yt5wrsN3NwoyiexPOG91WQVCADdErHsnglVZZq9Z8Wx7KwecGCUurJ2
+H6lKudv5YOxPnAzqZS5HbpZd/nRTMZh2rdXCr5m2YOuewyYjvM757AkmUpM09zJX
+MQ1S67/UX2y8/74TcRF97Ncx9HeELs92innBRXoFitnNguvcO6Esx4BTe1OdU6qR
+ER3zAmVf22Le9ciXbu24DN4mleOH+OmBx7X2PqJSYW9GAMTsRB081R6EWKH7romQ
+waxFrZ4DJzZ9ltyosEJn5F32StyLrFxpcrdLUoEaclZCv2qka7sZvi0EvovDVEBU
+e10jOx9AOwf8Gj2ufhquQ6qgVYCzbP+YrodtkFrXRS3IsljIchj1M2ffB/0bfoUs
+rtER9pLvYzCjBPg8IfGLw0o754Qbhh/ReplCRTusP/fQMybvCvfxreS3oyEriu/G
+GufRomjewZ8EMHDIgUsLcYo2UHZsfF7tcazgxMGmMvazp4r8vpgrvW/8fIN/6Adu
+tF+WjWDTvJLFJCe6O+BFJOWrssNrrra1zGtLC1s8s+Wfpe+bGPL5zpHeebGTwH1U
+22eqgJArlEKxrfarz7W5+uHZJHSjF/K9ZvunLGD0n9GOPMpji3UO3zeM8IYoWn7E
+/EWK1XbjnssNemeeTZ+sDh+qrD7BOi+vCX1IyBxbfqnQfJZvmcPWpruy1UsO+aIC
+0GY8Jr3OL69dDQ21jueJAh8EGAEIAAkFAlC9+dkCGwwACgkQL0VeKCTRjd9HCw/+
+LQSVgLLF4ulYlPCjWIIuQwrPbJfWUVVr2dPUFVM85DCv8gBzk5c121snXh9Swovm
+laBbw6ate3BmbXLh64jVE9Za5sbTWi7PCcbO/bpRy4d6oLmitmNw6cq0vjTLxUYy
+bwuiJxWREkfxuU85EKdouN062YDevH+/YResmlJrcCE7LRlJFeRlKsrrwBU3BqYd
+GgFJjKjQC1peeQ9fj62Y7xfwE9+PXbkiWO5u/Bk8hb1VZH1SoIRU98NHVcp6BVvp
+VK0jLAXuSauSczULmpRjbyt1lhaAqivDTWEEZXiNNbRyp17c3nVdPWOcgBr42hdQ
+z25CgZgyLCsvu82wuXLKJblrIPJX3Yf+si6KqEWBsmwdOWybsjygaF5HvzgFqAAD
+U0goPWoQ71PorP2XOUNp5ZLkBQp5etvtkksjVNMIhnHn8PGMuoxO39EUGlWj2B5l
+Cu8tSosAzB1pS8NcLZzoNoI9dOHrmgJmP+GrOUkcf5GhNZbMoj4GNfGBRYX0SZlQ
+GuDrwNKYj73C4MWyNnnUFyq8nDHJ/G1NpaF2hiof9RBL4PUU/f92JkceXPBXA8gL
+Mz2ig1OButwPPLFGQhWqxXAGrsS3Ny+BhTJfnfIbbkaLLphBpDZm1D9XKbAUvdd1
+RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc=
+=JTFu
+-----END PGP PRIVATE KEY BLOCK-----
+"""
+
+# key 7FEE575A: public key "anotheruser <anotheruser@leap.se>"
+PUBLIC_KEY_2 = """
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.10 (GNU/Linux)
+
+mI0EUYwJXgEEAMbTKHuPJ5/Gk34l9Z06f+0WCXTDXdte1UBoDtZ1erAbudgC4MOR
+gquKqoj3Hhw0/ILqJ88GcOJmKK/bEoIAuKaqlzDF7UAYpOsPZZYmtRfPC2pTCnXq
+Z1vdeqLwTbUspqXflkCkFtfhGKMq5rH8GV5a3tXZkRWZhdNwhVXZagC3ABEBAAG0
+IWFub3RoZXJ1c2VyIDxhbm90aGVydXNlckBsZWFwLnNlPoi4BBMBAgAiBQJRjAle
+AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRB/nfpof+5XWotuA/4tLN4E
+gUr7IfLy2HkHAxzw7A4rqfMN92DIM9mZrDGaWRrOn3aVF7VU1UG7MDkHfPvp/cFw
+ezoCw4s4IoHVc/pVlOkcHSyt4/Rfh248tYEJmFCJXGHpkK83VIKYJAithNccJ6Q4
+JE/o06Mtf4uh/cA1HUL4a4ceqUhtpLJULLeKo7iNBFGMCV4BBADsyQI7GR0wSAxz
+VayLjuPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQt
+Z/hwcLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63
+yuRe94WenT1RJd6xU1aaUff4rKizuQARAQABiJ8EGAECAAkFAlGMCV4CGwwACgkQ
+f536aH/uV1rPZQQAqCzRysOlu8ez7PuiBD4SebgRqWlxa1TF1ujzfLmuPivROZ2X
+Kw5aQstxgGSjoB7tac49s0huh4X8XK+BtJBfU84JS8Jc2satlfwoyZ35LH6sDZck
+I+RS/3we6zpMfHs3vvp9xgca6ZupQxivGtxlJs294TpJorx+mFFqbV17AzQ=
+=Thdu
+-----END PGP PUBLIC KEY BLOCK-----
+"""
+
+PRIVATE_KEY_2 = """
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: GnuPG v1.4.10 (GNU/Linux)
+
+lQHYBFGMCV4BBADG0yh7jyefxpN+JfWdOn/tFgl0w13bXtVAaA7WdXqwG7nYAuDD
+kYKriqqI9x4cNPyC6ifPBnDiZiiv2xKCALimqpcwxe1AGKTrD2WWJrUXzwtqUwp1
+6mdb3Xqi8E21LKal35ZApBbX4RijKuax/BleWt7V2ZEVmYXTcIVV2WoAtwARAQAB
+AAP7BLuSAx7tOohnimEs74ks8l/L6dOcsFQZj2bqs4AoY3jFe7bV0tHr4llypb/8
+H3/DYvpf6DWnCjyUS1tTnXSW8JXtx01BUKaAufSmMNg9blKV6GGHlT/Whe9uVyks
+7XHk/+9mebVMNJ/kNlqq2k+uWqJohzC8WWLRK+d1tBeqDsECANZmzltPaqUsGV5X
+C3zszE3tUBgptV/mKnBtopKi+VH+t7K6fudGcG+bAcZDUoH/QVde52mIIjjIdLje
+uajJuHUCAO1mqh+vPoGv4eBLV7iBo3XrunyGXiys4a39eomhxTy3YktQanjjx+ty
+GltAGCs5PbWGO6/IRjjvd46wh53kzvsCAO0J97gsWhzLuFnkxFAJSPk7RRlyl7lI
+1XS/x0Og6j9XHCyY1OYkfBm0to3UlCfkgirzCYlTYObCofzdKFIPDmSqHbQhYW5v
+dGhlcnVzZXIgPGFub3RoZXJ1c2VyQGxlYXAuc2U+iLgEEwECACIFAlGMCV4CGwMG
+CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEH+d+mh/7ldai24D/i0s3gSBSvsh
+8vLYeQcDHPDsDiup8w33YMgz2ZmsMZpZGs6fdpUXtVTVQbswOQd8++n9wXB7OgLD
+izgigdVz+lWU6RwdLK3j9F+Hbjy1gQmYUIlcYemQrzdUgpgkCK2E1xwnpDgkT+jT
+oy1/i6H9wDUdQvhrhx6pSG2kslQst4qjnQHYBFGMCV4BBADsyQI7GR0wSAxzVayL
+juPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQtZ/hw
+cLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63yuRe
+94WenT1RJd6xU1aaUff4rKizuQARAQABAAP9EyElqJ3dq3EErXwwT4mMnbd1SrVC
+rUJrNWQZL59mm5oigS00uIyR0SvusOr+UzTtd8ysRuwHy5d/LAZsbjQStaOMBILx
+77TJveOel0a1QK0YSMF2ywZMCKvquvjli4hAtWYz/EwfuzQN3t23jc5ny+GqmqD2
+3FUxLJosFUfLNmECAO9KhVmJi+L9dswIs+2Dkjd1eiRQzNOEVffvYkGYZyKxNiXF
+UA5kvyZcB4iAN9sWCybE4WHZ9jd4myGB0MPDGxkCAP1RsXJbbuD6zS7BXe5gwunO
+2q4q7ptdSl/sJYQuTe1KNP5d/uGsvlcFfsYjpsopasPjFBIncc/2QThMKlhoEaEB
+/0mVAxpT6SrEvUbJ18z7kna24SgMPr3OnPMxPGfvNLJY/Xv/A17YfoqjmByCvsKE
+JCDjopXtmbcrZyoEZbEht9mko4ifBBgBAgAJBQJRjAleAhsMAAoJEH+d+mh/7lda
+z2UEAKgs0crDpbvHs+z7ogQ+Enm4EalpcWtUxdbo83y5rj4r0TmdlysOWkLLcYBk
+o6Ae7WnOPbNIboeF/FyvgbSQX1POCUvCXNrGrZX8KMmd+Sx+rA2XJCPkUv98Hus6
+THx7N776fcYHGumbqUMYrxrcZSbNveE6SaK8fphRam1dewM0
+=a5gs
+-----END PGP PRIVATE KEY BLOCK-----
+"""
diff --git a/src/leap/bitmask/mail/testing/__init__.py~ b/src/leap/bitmask/mail/testing/__init__.py~
new file mode 100644
index 00000000..ffaadd88
--- /dev/null
+++ b/src/leap/bitmask/mail/testing/__init__.py~
@@ -0,0 +1,358 @@
+# -*- coding: utf-8 -*-
+# __init__.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Base classes and keys for leap.mail tests.
+"""
+import os
+import distutils.spawn
+from mock import Mock
+from twisted.internet.defer import gatherResults, succeed
+from twisted.trial import unittest
+from twisted.web.client import Response
+from twisted.internet import defer
+from twisted.python import log
+
+
+from leap.soledad.client import Soledad
+from leap.keymanager import KeyManager
+
+
+from leap.common.testing.basetest import BaseLeapTest
+
+ADDRESS = 'leap@leap.se'
+ADDRESS_2 = 'anotheruser@leap.se'
+
+class defaultMockSharedDB(object):
+ get_doc = Mock(return_value=None)
+ put_doc = Mock(side_effect=None)
+ open = Mock(return_value=None)
+ close = Mock(return_value=None)
+ syncable = True
+
+ def __call__(self):
+ return self
+
+
+class KeyManagerWithSoledadTestCase(unittest.TestCase, BaseLeapTest):
+
+ def setUp(self):
+ self.gpg_binary_path = self._find_gpg()
+
+ self._soledad = Soledad(
+ u"leap@leap.se",
+ u"123456",
+ secrets_path=self.tempdir + "/secret.gpg",
+ local_db_path=self.tempdir + "/soledad.u1db",
+ server_url='',
+ cert_file=None,
+ auth_token=None,
+ shared_db=defaultMockSharedDB(),
+ syncable=False)
+
+ self.km = self._key_manager()
+
+ class Response(object):
+ code = 200
+ phrase = ''
+ def deliverBody(self, x):
+ return
+
+ # XXX why the fuck is this needed? ------------------------
+ self.km._async_client_pinned.request = Mock(
+ return_value=defer.succeed(Response()))
+ #self.km._async_client.request = Mock(return_value='')
+ #self.km._async_client_pinned.request = Mock(
+ #return_value='')
+ # -------------------------------------------------------
+
+ d1 = self.km.put_raw_key(PRIVATE_KEY, ADDRESS)
+ d2 = self.km.put_raw_key(PRIVATE_KEY_2, ADDRESS_2)
+ return gatherResults([d1, d2])
+
+ def tearDown(self):
+ km = self._key_manager()
+ # wait for the indexes to be ready for the tear down
+ d = km._openpgp.deferred_init
+ d.addCallback(lambda _: self.delete_all_keys(km))
+ d.addCallback(lambda _: self._soledad.close())
+ return d
+
+ def delete_all_keys(self, km):
+ def delete_keys(keys):
+ deferreds = []
+ for key in keys:
+ d = km._openpgp.delete_key(key)
+ deferreds.append(d)
+ return gatherResults(deferreds)
+
+ def check_deleted(_, private):
+ d = km.get_all_keys(private=private)
+ d.addCallback(lambda keys: self.assertEqual(keys, []))
+ return d
+
+ deferreds = []
+ for private in [True, False]:
+ d = km.get_all_keys(private=private)
+ d.addCallback(delete_keys)
+ d.addCallback(check_deleted, private)
+ deferreds.append(d)
+ return gatherResults(deferreds)
+
+ def _key_manager(self, user=ADDRESS, url='', token=None,
+ ca_cert_path=None):
+ return KeyManager(user, url, self._soledad, token=token,
+ gpgbinary=self.gpg_binary_path,
+ ca_cert_path=ca_cert_path)
+
+ def _find_gpg(self):
+ gpg_path = distutils.spawn.find_executable('gpg')
+ if gpg_path is not None:
+ return os.path.realpath(gpg_path)
+ else:
+ return "/usr/bin/gpg"
+
+ def get_public_binary_key(self):
+ with open(PATH + '/fixtures/public_key.bin', 'r') as binary_public_key:
+ return binary_public_key.read()
+
+ def get_private_binary_key(self):
+ with open(
+ PATH + '/fixtures/private_key.bin', 'r') as binary_private_key:
+ return binary_private_key.read()
+
+
+# key 24D18DDF: public key "Leap Test Key <leap@leap.se>"
+KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF"
+PUBLIC_KEY = """
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.10 (GNU/Linux)
+
+mQINBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz
+iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO
+zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx
+irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT
+huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs
+d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g
+wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb
+hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv
+U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H
+T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i
+Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB
+tBxMZWFwIFRlc3QgS2V5IDxsZWFwQGxlYXAuc2U+iQI3BBMBCAAhBQJQvfnZAhsD
+BQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEC9FXigk0Y3fT7EQAKH3IuRniOpb
+T/DDIgwwjz3oxB/W0DDMyPXowlhSOuM0rgGfntBpBb3boezEXwL86NPQxNGGruF5
+hkmecSiuPSvOmQlqlS95NGQp6hNG0YaKColh+Q5NTspFXCAkFch9oqUje0LdxfSP
+QfV9UpeEvGyPmk1I9EJV/YDmZ4+Djge1d7qhVZInz4Rx1NrSyF/Tc2EC0VpjQFsU
+Y9Kb2YBBR7ivG6DBc8ty0jJXi7B4WjkFcUEJviQpMF2dCLdonCehYs1PqsN1N7j+
+eFjQd+hqVMJgYuSGKjvuAEfClM6MQw7+FmFwMyLgK/Ew/DttHEDCri77SPSkOGSI
+txCzhTg6798f6mJr7WcXmHX1w1Vcib5FfZ8vTDFVhz/XgAgArdhPo9V6/1dgSSiB
+KPQ/spsco6u5imdOhckERE0lnAYvVT6KE81TKuhF/b23u7x+Wdew6kK0EQhYA7wy
+7LmlaNXc7rMBQJ9Z60CJ4JDtatBWZ0kNrt2VfdDHVdqBTOpl0CraNUjWE5YMDasr
+K2dF5IX8D3uuYtpZnxqg0KzyLg0tzL0tvOL1C2iudgZUISZNPKbS0z0v+afuAAnx
+2pTC3uezbh2Jt8SWTLhll4i0P4Ps5kZ6HQUO56O+/Z1cWovX+mQekYFmERySDR9n
+3k1uAwLilJmRmepGmvYbB8HloV8HqwgguQINBFC9+dkBEAC0I/xn1uborMgDvBtf
+H0sEhwnXBC849/32zic6udB6/3Efk9nzbSpL3FSOuXITZsZgCHPkKarnoQ2ztMcS
+sh1ke1C5gQGms75UVmM/nS+2YI4vY8OX/GC/on2vUyncqdH+bR6xH5hx4NbWpfTs
+iQHmz5C6zzS/kuabGdZyKRaZHt23WQ7JX/4zpjqbC99DjHcP9BSk7tJ8wI4bkMYD
+uFVQdT9O6HwyKGYwUU4sAQRAj7XCTGvVbT0dpgJwH4RmrEtJoHAx4Whg8mJ710E0
+GCmzf2jqkNuOw76ivgk27Kge+Hw00jmJjQhHY0yVbiaoJwcRrPKzaSjEVNgrpgP3
+lXPRGQArgESsIOTeVVHQ8fhK2YtTeCY9rIiO+L0OX2xo9HK7hfHZZWL6rqymXdyS
+fhzh/f6IPyHFWnvj7Brl7DR8heMikygcJqv+ed2yx7iLyCUJ10g12I48+aEj1aLe
+dP7lna32iY8/Z0SHQLNH6PXO9SlPcq2aFUgKqE75A/0FMk7CunzU1OWr2ZtTLNO1
+WT/13LfOhhuEq9jTyTosn0WxBjJKq18lnhzCXlaw6EAtbA7CUwsD3CTPR56aAXFK
+3I7KXOVAqggrvMe5Tpdg5drfYpI8hZovL5aAgb+7Y5ta10TcJdUhS5K3kFAWe/td
+U0cmWUMDP1UMSQ5Jg6JIQVWhSwARAQABiQIfBBgBCAAJBQJQvfnZAhsMAAoJEC9F
+Xigk0Y3fRwsP/i0ElYCyxeLpWJTwo1iCLkMKz2yX1lFVa9nT1BVTPOQwr/IAc5OX
+NdtbJ14fUsKL5pWgW8OmrXtwZm1y4euI1RPWWubG01ouzwnGzv26UcuHeqC5orZj
+cOnKtL40y8VGMm8LoicVkRJH8blPORCnaLjdOtmA3rx/v2EXrJpSa3AhOy0ZSRXk
+ZSrK68AVNwamHRoBSYyo0AtaXnkPX4+tmO8X8BPfj125IljubvwZPIW9VWR9UqCE
+VPfDR1XKegVb6VStIywF7kmrknM1C5qUY28rdZYWgKorw01hBGV4jTW0cqde3N51
+XT1jnIAa+NoXUM9uQoGYMiwrL7vNsLlyyiW5ayDyV92H/rIuiqhFgbJsHTlsm7I8
+oGheR784BagAA1NIKD1qEO9T6Kz9lzlDaeWS5AUKeXrb7ZJLI1TTCIZx5/DxjLqM
+Tt/RFBpVo9geZQrvLUqLAMwdaUvDXC2c6DaCPXTh65oCZj/hqzlJHH+RoTWWzKI+
+BjXxgUWF9EmZUBrg68DSmI+9wuDFsjZ51BcqvJwxyfxtTaWhdoYqH/UQS+D1FP3/
+diZHHlzwVwPICzM9ooNTgbrcDzyxRkIVqsVwBq7EtzcvgYUyX53yG25Giy6YQaQ2
+ZtQ/VymwFL3XdUWV6B/hU4PVAFvO3qlOtdJ6TpE+nEWgcWjCv5g7RjXX
+=MuOY
+-----END PGP PUBLIC KEY BLOCK-----
+"""
+PRIVATE_KEY = """
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: GnuPG v1.4.10 (GNU/Linux)
+
+lQcYBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz
+iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO
+zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx
+irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT
+huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs
+d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g
+wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb
+hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv
+U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H
+T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i
+Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB
+AA/+JHtlL39G1wsH9R6UEfUQJGXR9MiIiwZoKcnRB2o8+DS+OLjg0JOh8XehtuCs
+E/8oGQKtQqa5bEIstX7IZoYmYFiUQi9LOzIblmp2vxOm+HKkxa4JszWci2/ZmC3t
+KtaA4adl9XVnshoQ7pijuCMUKB3naBEOAxd8s9d/JeReGIYkJErdrnVfNk5N71Ds
+FmH5Ll3XtEDvgBUQP3nkA6QFjpsaB94FHjL3gDwum/cxzj6pCglcvHOzEhfY0Ddb
+J967FozQTaf2JW3O+w3LOqtcKWpq87B7+O61tVidQPSSuzPjCtFF0D2LC9R/Hpky
+KTMQ6CaKja4MPhjwywd4QPcHGYSqjMpflvJqi+kYIt8psUK/YswWjnr3r4fbuqVY
+VhtiHvnBHQjz135lUqWvEz4hM3Xpnxydx7aRlv5NlevK8+YIO5oFbWbGNTWsPZI5
+jpoFBpSsnR1Q5tnvtNHauvoWV+XN2qAOBTG+/nEbDYH6Ak3aaE9jrpTdYh0CotYF
+q7csANsDy3JvkAzeU6WnYpsHHaAjqOGyiZGsLej1UcXPFMosE/aUo4WQhiS8Zx2c
+zOVKOi/X5vQ2GdNT9Qolz8AriwzsvFR+bxPzyd8V6ALwDsoXvwEYinYBKK8j0OPv
+OOihSR6HVsuP9NUZNU9ewiGzte/+/r6pNXHvR7wTQ8EWLcEIAN6Zyrb0bHZTIlxt
+VWur/Ht2mIZrBaO50qmM5RD3T5oXzWXi/pjLrIpBMfeZR9DWfwQwjYzwqi7pxtYx
+nJvbMuY505rfnMoYxb4J+cpRXV8MS7Dr1vjjLVUC9KiwSbM3gg6emfd2yuA93ihv
+Pe3mffzLIiQa4mRE3wtGcioC43nWuV2K2e1KjxeFg07JhrezA/1Cak505ab/tmvP
+4YmjR5c44+yL/YcQ3HdFgs4mV+nVbptRXvRcPpolJsgxPccGNdvHhsoR4gwXMS3F
+RRPD2z6x8xeN73Q4KH3bm01swQdwFBZbWVfmUGLxvN7leCdfs9+iFJyqHiCIB6Iv
+mQfp8F0IAOwSo8JhWN+V1dwML4EkIrM8wUb4yecNLkyR6TpPH/qXx4PxVMC+vy6x
+sCtjeHIwKE+9vqnlhd5zOYh7qYXEJtYwdeDDmDbL8oks1LFfd+FyAuZXY33DLwn0
+cRYsr2OEZmaajqUB3NVmj3H4uJBN9+paFHyFSXrH68K1Fk2o3n+RSf2EiX+eICwI
+L6rqoF5sSVUghBWdNegV7qfy4anwTQwrIMGjgU5S6PKW0Dr/3iO5z3qQpGPAj5OW
+ATqPWkDICLbObPxD5cJlyyNE2wCA9VVc6/1d6w4EVwSq9h3/WTpATEreXXxTGptd
+LNiTA1nmakBYNO2Iyo3djhaqBdWjk+EIAKtVEnJH9FAVwWOvaj1RoZMA5DnDMo7e
+SnhrCXl8AL7Z1WInEaybasTJXn1uQ8xY52Ua4b8cbuEKRKzw/70NesFRoMLYoHTO
+dyeszvhoDHberpGRTciVmpMu7Hyi33rM31K9epA4ib6QbbCHnxkWOZB+Bhgj1hJ8
+xb4RBYWiWpAYcg0+DAC3w9gfxQhtUlZPIbmbrBmrVkO2GVGUj8kH6k4UV6kUHEGY
+HQWQR0HcbKcXW81ZXCCD0l7ROuEWQtTe5Jw7dJ4/QFuqZnPutXVRNOZqpl6eRShw
+7X2/a29VXBpmHA95a88rSQsL+qm7Fb3prqRmuMCtrUZgFz7HLSTuUMR867QcTGVh
+cCBUZXN0IEtleSA8bGVhcEBsZWFwLnNlPokCNwQTAQgAIQUCUL352QIbAwULCQgH
+AwUVCgkICwUWAgMBAAIeAQIXgAAKCRAvRV4oJNGN30+xEACh9yLkZ4jqW0/wwyIM
+MI896MQf1tAwzMj16MJYUjrjNK4Bn57QaQW926HsxF8C/OjT0MTRhq7heYZJnnEo
+rj0rzpkJapUveTRkKeoTRtGGigqJYfkOTU7KRVwgJBXIfaKlI3tC3cX0j0H1fVKX
+hLxsj5pNSPRCVf2A5mePg44HtXe6oVWSJ8+EcdTa0shf03NhAtFaY0BbFGPSm9mA
+QUe4rxugwXPLctIyV4uweFo5BXFBCb4kKTBdnQi3aJwnoWLNT6rDdTe4/nhY0Hfo
+alTCYGLkhio77gBHwpTOjEMO/hZhcDMi4CvxMPw7bRxAwq4u+0j0pDhkiLcQs4U4
+Ou/fH+pia+1nF5h19cNVXIm+RX2fL0wxVYc/14AIAK3YT6PVev9XYEkogSj0P7Kb
+HKOruYpnToXJBERNJZwGL1U+ihPNUyroRf29t7u8flnXsOpCtBEIWAO8Muy5pWjV
+3O6zAUCfWetAieCQ7WrQVmdJDa7dlX3Qx1XagUzqZdAq2jVI1hOWDA2rKytnReSF
+/A97rmLaWZ8aoNCs8i4NLcy9Lbzi9QtornYGVCEmTTym0tM9L/mn7gAJ8dqUwt7n
+s24dibfElky4ZZeItD+D7OZGeh0FDuejvv2dXFqL1/pkHpGBZhEckg0fZ95NbgMC
+4pSZkZnqRpr2GwfB5aFfB6sIIJ0HGARQvfnZARAAtCP8Z9bm6KzIA7wbXx9LBIcJ
+1wQvOPf99s4nOrnQev9xH5PZ820qS9xUjrlyE2bGYAhz5Cmq56ENs7THErIdZHtQ
+uYEBprO+VFZjP50vtmCOL2PDl/xgv6J9r1Mp3KnR/m0esR+YceDW1qX07IkB5s+Q
+us80v5LmmxnWcikWmR7dt1kOyV/+M6Y6mwvfQ4x3D/QUpO7SfMCOG5DGA7hVUHU/
+Tuh8MihmMFFOLAEEQI+1wkxr1W09HaYCcB+EZqxLSaBwMeFoYPJie9dBNBgps39o
+6pDbjsO+or4JNuyoHvh8NNI5iY0IR2NMlW4mqCcHEazys2koxFTYK6YD95Vz0RkA
+K4BErCDk3lVR0PH4StmLU3gmPayIjvi9Dl9saPRyu4Xx2WVi+q6spl3ckn4c4f3+
+iD8hxVp74+wa5ew0fIXjIpMoHCar/nndsse4i8glCddINdiOPPmhI9Wi3nT+5Z2t
+9omPP2dEh0CzR+j1zvUpT3KtmhVICqhO+QP9BTJOwrp81NTlq9mbUyzTtVk/9dy3
+zoYbhKvY08k6LJ9FsQYySqtfJZ4cwl5WsOhALWwOwlMLA9wkz0eemgFxStyOylzl
+QKoIK7zHuU6XYOXa32KSPIWaLy+WgIG/u2ObWtdE3CXVIUuSt5BQFnv7XVNHJllD
+Az9VDEkOSYOiSEFVoUsAEQEAAQAP/1AagnZQZyzHDEgw4QELAspYHCWLXE5aZInX
+wTUJhK31IgIXNn9bJ0hFiSpQR2xeMs9oYtRuPOu0P8oOFMn4/z374fkjZy8QVY3e
+PlL+3EUeqYtkMwlGNmVw5a/NbNuNfm5Darb7pEfbYd1gPcni4MAYw7R2SG/57GbC
+9gucvspHIfOSfBNLBthDzmK8xEKe1yD2eimfc2T7IRYb6hmkYfeds5GsqvGI6mwI
+85h4uUHWRc5JOlhVM6yX8hSWx0L60Z3DZLChmc8maWnFXd7C8eQ6P1azJJbW71Ih
+7CoK0XW4LE82vlQurSRFgTwfl7wFYszW2bOzCuhHDDtYnwH86Nsu0DC78ZVRnvxn
+E8Ke/AJgrdhIOo4UAyR+aZD2+2mKd7/waOUTUrUtTzc7i8N3YXGi/EIaNReBXaq+
+ZNOp24BlFzRp+FCF/pptDW9HjPdiV09x0DgICmeZS4Gq/4vFFIahWctg52NGebT0
+Idxngjj+xDtLaZlLQoOz0n5ByjO/Wi0ANmMv1sMKCHhGvdaSws2/PbMR2r4caj8m
+KXpIgdinM/wUzHJ5pZyF2U/qejsRj8Kw8KH/tfX4JCLhiaP/mgeTuWGDHeZQERAT
+xPmRFHaLP9/ZhvGNh6okIYtrKjWTLGoXvKLHcrKNisBLSq+P2WeFrlme1vjvJMo/
+jPwLT5o9CADQmcbKZ+QQ1ZM9v99iDZol7SAMZX43JC019sx6GK0u6xouJBcLfeB4
+OXacTgmSYdTa9RM9fbfVpti01tJ84LV2SyL/VJq/enJF4XQPSynT/tFTn1PAor6o
+tEAAd8fjKdJ6LnD5wb92SPHfQfXqI84rFEO8rUNIE/1ErT6DYifDzVCbfD2KZdoF
+cOSp7TpD77sY1bs74ocBX5ejKtd+aH99D78bJSMM4pSDZsIEwnomkBHTziubPwJb
+OwnATy0LmSMAWOw5rKbsh5nfwCiUTM20xp0t5JeXd+wPVWbpWqI2EnkCEN+RJr9i
+7dp/ymDQ+Yt5wrsN3NwoyiexPOG91WQVCADdErHsnglVZZq9Z8Wx7KwecGCUurJ2
+H6lKudv5YOxPnAzqZS5HbpZd/nRTMZh2rdXCr5m2YOuewyYjvM757AkmUpM09zJX
+MQ1S67/UX2y8/74TcRF97Ncx9HeELs92innBRXoFitnNguvcO6Esx4BTe1OdU6qR
+ER3zAmVf22Le9ciXbu24DN4mleOH+OmBx7X2PqJSYW9GAMTsRB081R6EWKH7romQ
+waxFrZ4DJzZ9ltyosEJn5F32StyLrFxpcrdLUoEaclZCv2qka7sZvi0EvovDVEBU
+e10jOx9AOwf8Gj2ufhquQ6qgVYCzbP+YrodtkFrXRS3IsljIchj1M2ffB/0bfoUs
+rtER9pLvYzCjBPg8IfGLw0o754Qbhh/ReplCRTusP/fQMybvCvfxreS3oyEriu/G
+GufRomjewZ8EMHDIgUsLcYo2UHZsfF7tcazgxMGmMvazp4r8vpgrvW/8fIN/6Adu
+tF+WjWDTvJLFJCe6O+BFJOWrssNrrra1zGtLC1s8s+Wfpe+bGPL5zpHeebGTwH1U
+22eqgJArlEKxrfarz7W5+uHZJHSjF/K9ZvunLGD0n9GOPMpji3UO3zeM8IYoWn7E
+/EWK1XbjnssNemeeTZ+sDh+qrD7BOi+vCX1IyBxbfqnQfJZvmcPWpruy1UsO+aIC
+0GY8Jr3OL69dDQ21jueJAh8EGAEIAAkFAlC9+dkCGwwACgkQL0VeKCTRjd9HCw/+
+LQSVgLLF4ulYlPCjWIIuQwrPbJfWUVVr2dPUFVM85DCv8gBzk5c121snXh9Swovm
+laBbw6ate3BmbXLh64jVE9Za5sbTWi7PCcbO/bpRy4d6oLmitmNw6cq0vjTLxUYy
+bwuiJxWREkfxuU85EKdouN062YDevH+/YResmlJrcCE7LRlJFeRlKsrrwBU3BqYd
+GgFJjKjQC1peeQ9fj62Y7xfwE9+PXbkiWO5u/Bk8hb1VZH1SoIRU98NHVcp6BVvp
+VK0jLAXuSauSczULmpRjbyt1lhaAqivDTWEEZXiNNbRyp17c3nVdPWOcgBr42hdQ
+z25CgZgyLCsvu82wuXLKJblrIPJX3Yf+si6KqEWBsmwdOWybsjygaF5HvzgFqAAD
+U0goPWoQ71PorP2XOUNp5ZLkBQp5etvtkksjVNMIhnHn8PGMuoxO39EUGlWj2B5l
+Cu8tSosAzB1pS8NcLZzoNoI9dOHrmgJmP+GrOUkcf5GhNZbMoj4GNfGBRYX0SZlQ
+GuDrwNKYj73C4MWyNnnUFyq8nDHJ/G1NpaF2hiof9RBL4PUU/f92JkceXPBXA8gL
+Mz2ig1OButwPPLFGQhWqxXAGrsS3Ny+BhTJfnfIbbkaLLphBpDZm1D9XKbAUvdd1
+RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc=
+=JTFu
+-----END PGP PRIVATE KEY BLOCK-----
+"""
+
+# key 7FEE575A: public key "anotheruser <anotheruser@leap.se>"
+PUBLIC_KEY_2 = """
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.10 (GNU/Linux)
+
+mI0EUYwJXgEEAMbTKHuPJ5/Gk34l9Z06f+0WCXTDXdte1UBoDtZ1erAbudgC4MOR
+gquKqoj3Hhw0/ILqJ88GcOJmKK/bEoIAuKaqlzDF7UAYpOsPZZYmtRfPC2pTCnXq
+Z1vdeqLwTbUspqXflkCkFtfhGKMq5rH8GV5a3tXZkRWZhdNwhVXZagC3ABEBAAG0
+IWFub3RoZXJ1c2VyIDxhbm90aGVydXNlckBsZWFwLnNlPoi4BBMBAgAiBQJRjAle
+AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRB/nfpof+5XWotuA/4tLN4E
+gUr7IfLy2HkHAxzw7A4rqfMN92DIM9mZrDGaWRrOn3aVF7VU1UG7MDkHfPvp/cFw
+ezoCw4s4IoHVc/pVlOkcHSyt4/Rfh248tYEJmFCJXGHpkK83VIKYJAithNccJ6Q4
+JE/o06Mtf4uh/cA1HUL4a4ceqUhtpLJULLeKo7iNBFGMCV4BBADsyQI7GR0wSAxz
+VayLjuPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQt
+Z/hwcLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63
+yuRe94WenT1RJd6xU1aaUff4rKizuQARAQABiJ8EGAECAAkFAlGMCV4CGwwACgkQ
+f536aH/uV1rPZQQAqCzRysOlu8ez7PuiBD4SebgRqWlxa1TF1ujzfLmuPivROZ2X
+Kw5aQstxgGSjoB7tac49s0huh4X8XK+BtJBfU84JS8Jc2satlfwoyZ35LH6sDZck
+I+RS/3we6zpMfHs3vvp9xgca6ZupQxivGtxlJs294TpJorx+mFFqbV17AzQ=
+=Thdu
+-----END PGP PUBLIC KEY BLOCK-----
+"""
+
+PRIVATE_KEY_2 = """
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: GnuPG v1.4.10 (GNU/Linux)
+
+lQHYBFGMCV4BBADG0yh7jyefxpN+JfWdOn/tFgl0w13bXtVAaA7WdXqwG7nYAuDD
+kYKriqqI9x4cNPyC6ifPBnDiZiiv2xKCALimqpcwxe1AGKTrD2WWJrUXzwtqUwp1
+6mdb3Xqi8E21LKal35ZApBbX4RijKuax/BleWt7V2ZEVmYXTcIVV2WoAtwARAQAB
+AAP7BLuSAx7tOohnimEs74ks8l/L6dOcsFQZj2bqs4AoY3jFe7bV0tHr4llypb/8
+H3/DYvpf6DWnCjyUS1tTnXSW8JXtx01BUKaAufSmMNg9blKV6GGHlT/Whe9uVyks
+7XHk/+9mebVMNJ/kNlqq2k+uWqJohzC8WWLRK+d1tBeqDsECANZmzltPaqUsGV5X
+C3zszE3tUBgptV/mKnBtopKi+VH+t7K6fudGcG+bAcZDUoH/QVde52mIIjjIdLje
+uajJuHUCAO1mqh+vPoGv4eBLV7iBo3XrunyGXiys4a39eomhxTy3YktQanjjx+ty
+GltAGCs5PbWGO6/IRjjvd46wh53kzvsCAO0J97gsWhzLuFnkxFAJSPk7RRlyl7lI
+1XS/x0Og6j9XHCyY1OYkfBm0to3UlCfkgirzCYlTYObCofzdKFIPDmSqHbQhYW5v
+dGhlcnVzZXIgPGFub3RoZXJ1c2VyQGxlYXAuc2U+iLgEEwECACIFAlGMCV4CGwMG
+CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEH+d+mh/7ldai24D/i0s3gSBSvsh
+8vLYeQcDHPDsDiup8w33YMgz2ZmsMZpZGs6fdpUXtVTVQbswOQd8++n9wXB7OgLD
+izgigdVz+lWU6RwdLK3j9F+Hbjy1gQmYUIlcYemQrzdUgpgkCK2E1xwnpDgkT+jT
+oy1/i6H9wDUdQvhrhx6pSG2kslQst4qjnQHYBFGMCV4BBADsyQI7GR0wSAxzVayL
+juPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQtZ/hw
+cLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63yuRe
+94WenT1RJd6xU1aaUff4rKizuQARAQABAAP9EyElqJ3dq3EErXwwT4mMnbd1SrVC
+rUJrNWQZL59mm5oigS00uIyR0SvusOr+UzTtd8ysRuwHy5d/LAZsbjQStaOMBILx
+77TJveOel0a1QK0YSMF2ywZMCKvquvjli4hAtWYz/EwfuzQN3t23jc5ny+GqmqD2
+3FUxLJosFUfLNmECAO9KhVmJi+L9dswIs+2Dkjd1eiRQzNOEVffvYkGYZyKxNiXF
+UA5kvyZcB4iAN9sWCybE4WHZ9jd4myGB0MPDGxkCAP1RsXJbbuD6zS7BXe5gwunO
+2q4q7ptdSl/sJYQuTe1KNP5d/uGsvlcFfsYjpsopasPjFBIncc/2QThMKlhoEaEB
+/0mVAxpT6SrEvUbJ18z7kna24SgMPr3OnPMxPGfvNLJY/Xv/A17YfoqjmByCvsKE
+JCDjopXtmbcrZyoEZbEht9mko4ifBBgBAgAJBQJRjAleAhsMAAoJEH+d+mh/7lda
+z2UEAKgs0crDpbvHs+z7ogQ+Enm4EalpcWtUxdbo83y5rj4r0TmdlysOWkLLcYBk
+o6Ae7WnOPbNIboeF/FyvgbSQX1POCUvCXNrGrZX8KMmd+Sx+rA2XJCPkUv98Hus6
+THx7N776fcYHGumbqUMYrxrcZSbNveE6SaK8fphRam1dewM0
+=a5gs
+-----END PGP PRIVATE KEY BLOCK-----
+"""
diff --git a/src/leap/bitmask/mail/testing/common.py b/src/leap/bitmask/mail/testing/common.py
new file mode 100644
index 00000000..1bf1de22
--- /dev/null
+++ b/src/leap/bitmask/mail/testing/common.py
@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+# common.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Common utilities for testing Soledad.
+"""
+import os
+import tempfile
+
+from twisted.internet import defer
+from twisted.trial import unittest
+
+from leap.common.testing.basetest import BaseLeapTest
+from leap.soledad.client import Soledad
+
+# TODO move to common module, or Soledad itself
+# XXX remove duplication
+
+TEST_USER = "testuser@leap.se"
+TEST_PASSWD = "1234"
+
+
+def _initialize_soledad(email, gnupg_home, tempdir):
+ """
+ Initializes soledad by hand
+
+ :param email: ID for the user
+ :param gnupg_home: path to home used by gnupg
+ :param tempdir: path to temporal dir
+ :rtype: Soledad instance
+ """
+
+ uuid = "foobar-uuid"
+ passphrase = u"verysecretpassphrase"
+ secret_path = os.path.join(tempdir, "secret.gpg")
+ local_db_path = os.path.join(tempdir, "soledad.u1db")
+ server_url = "https://provider"
+ cert_file = ""
+
+ soledad = Soledad(
+ uuid,
+ passphrase,
+ secret_path,
+ local_db_path,
+ server_url,
+ cert_file,
+ syncable=False)
+
+ return soledad
+
+
+class SoledadTestMixin(unittest.TestCase, BaseLeapTest):
+ """
+ It is **VERY** important that this base is added *AFTER* unittest.TestCase
+ """
+
+ def setUp(self):
+ self.results = []
+ self.setUpEnv()
+
+ # pytest handles correctly the setupEnv for the class,
+ # but trial ignores it.
+ if not getattr(self, 'tempdir', None):
+ self.tempdir = tempfile.gettempdir()
+
+ # Soledad: config info
+ self.gnupg_home = "%s/gnupg" % self.tempdir
+ self.email = 'leap@leap.se'
+
+ # initialize soledad by hand so we can control keys
+ self._soledad = _initialize_soledad(
+ self.email,
+ self.gnupg_home,
+ self.tempdir)
+
+ return defer.succeed(True)
+
+ def tearDown(self):
+ """
+ tearDown method called after each test.
+ """
+ self.results = []
+ try:
+ self._soledad.close()
+ except Exception:
+ print "ERROR WHILE CLOSING SOLEDAD"
+ # logging.exception(exc)
+ self.tearDownEnv()
+
+ @classmethod
+ def setUpClass(self):
+ pass
+
+ @classmethod
+ def tearDownClass(self):
+ pass
diff --git a/src/leap/bitmask/mail/testing/imap.py b/src/leap/bitmask/mail/testing/imap.py
new file mode 100644
index 00000000..72acbf2e
--- /dev/null
+++ b/src/leap/bitmask/mail/testing/imap.py
@@ -0,0 +1,186 @@
+# -*- coding: utf-8 -*-
+# utils.py
+# Copyright (C) 2014, 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Common utilities for testing Soledad IMAP Server.
+"""
+from email import parser
+
+from mock import Mock
+from twisted.cred.checkers import ICredentialsChecker
+from twisted.cred.credentials import IUsernamePassword
+from twisted.cred.error import UnauthorizedLogin
+from twisted.cred.portal import Portal, IRealm
+from twisted.mail import imap4
+from twisted.internet import defer
+from twisted.protocols import loopback
+from twisted.python import log
+from zope.interface import implementer
+
+from leap.mail.adaptors import soledad as soledad_adaptor
+from leap.mail.imap.account import IMAPAccount
+from leap.mail.imap.server import LEAPIMAPServer
+from leap.mail.testing.common import SoledadTestMixin
+
+TEST_USER = "testuser@leap.se"
+TEST_PASSWD = "1234"
+
+
+#
+# Simple IMAP4 Client for testing
+#
+
+class SimpleClient(imap4.IMAP4Client):
+ """
+ A Simple IMAP4 Client to test our
+ Soledad-LEAPServer
+ """
+
+ def __init__(self, deferred, contextFactory=None):
+ imap4.IMAP4Client.__init__(self, contextFactory)
+ self.deferred = deferred
+ self.events = []
+
+ def serverGreeting(self, caps):
+ self.deferred.callback(None)
+
+ def modeChanged(self, writeable):
+ self.events.append(['modeChanged', writeable])
+ self.transport.loseConnection()
+
+ def flagsChanged(self, newFlags):
+ self.events.append(['flagsChanged', newFlags])
+ self.transport.loseConnection()
+
+ def newMessages(self, exists, recent):
+ self.events.append(['newMessages', exists, recent])
+ self.transport.loseConnection()
+
+#
+# Dummy credentials for tests
+#
+
+
+@implementer(IRealm)
+class TestRealm(object):
+
+ def __init__(self, account):
+ self._account = account
+
+ def requestAvatar(self, avatarId, mind, *interfaces):
+ avatar = self._account
+ return (imap4.IAccount, avatar,
+ getattr(avatar, 'logout', lambda: None))
+
+
+@implementer(ICredentialsChecker)
+class TestCredentialsChecker(object):
+
+ credentialInterfaces = (IUsernamePassword,)
+
+ userid = TEST_USER
+ password = TEST_PASSWD
+
+ def requestAvatarId(self, credentials):
+ username, password = credentials.username, credentials.password
+ d = self.checkTestCredentials(username, password)
+ d.addErrback(lambda f: defer.fail(UnauthorizedLogin()))
+ return d
+
+ def checkTestCredentials(self, username, password):
+ if username == self.userid and password == self.password:
+ return defer.succeed(username)
+ else:
+ return defer.fail(Exception("Wrong credentials"))
+
+
+class TestSoledadIMAPServer(LEAPIMAPServer):
+
+ def __init__(self, account, *args, **kw):
+
+ LEAPIMAPServer.__init__(self, *args, **kw)
+
+ realm = TestRealm(account)
+ portal = Portal(realm)
+ checker = TestCredentialsChecker()
+ self.checker = checker
+ self.portal = portal
+ portal.registerChecker(checker)
+
+
+class IMAP4HelperMixin(SoledadTestMixin):
+ """
+ MixIn containing several utilities to be shared across
+ different TestCases
+ """
+ serverCTX = None
+ clientCTX = None
+
+ def setUp(self):
+
+ soledad_adaptor.cleanup_deferred_locks()
+
+ USERID = TEST_USER
+
+ def setup_server(account):
+ self.server = TestSoledadIMAPServer(
+ account=account,
+ contextFactory=self.serverCTX)
+ self.server.theAccount = account
+
+ d_server_ready = defer.Deferred()
+ self.client = SimpleClient(
+ d_server_ready, contextFactory=self.clientCTX)
+ self.connected = d_server_ready
+
+ def setup_account(_):
+ self.parser = parser.Parser()
+
+ # XXX this should be fixed in soledad.
+ # Soledad sync makes trial block forever. The sync it's mocked to
+ # fix this problem. _mock_soledad_get_from_index can be used from
+ # the tests to provide documents.
+ # TODO see here, possibly related?
+ # -- http://www.pythoneye.com/83_20424875/
+ self._soledad.sync = Mock()
+
+ d = defer.Deferred()
+ self.acc = IMAPAccount(self._soledad, USERID, d=d)
+ return d
+
+ d = super(IMAP4HelperMixin, self).setUp()
+ d.addCallback(setup_account)
+ d.addCallback(setup_server)
+ return d
+
+ def tearDown(self):
+ SoledadTestMixin.tearDown(self)
+ del self._soledad
+ del self.client
+ del self.server
+ del self.connected
+
+ def _cbStopClient(self, ignore):
+ self.client.transport.loseConnection()
+
+ def _ebGeneral(self, failure):
+ self.client.transport.loseConnection()
+ self.server.transport.loseConnection()
+ if hasattr(self, 'function'):
+ log.err(failure, "Problem with %r" % (self.function,))
+
+ def loopback(self):
+ return loopback.loopbackAsync(self.server, self.client)
diff --git a/src/leap/bitmask/mail/testing/smtp.py b/src/leap/bitmask/mail/testing/smtp.py
new file mode 100644
index 00000000..d8690f12
--- /dev/null
+++ b/src/leap/bitmask/mail/testing/smtp.py
@@ -0,0 +1,51 @@
+from twisted.mail import smtp
+
+from leap.mail.smtp.gateway import SMTPFactory, LOCAL_FQDN
+from leap.mail.smtp.gateway import SMTPDelivery
+from leap.mail.outgoing.service import outgoingFactory
+
+TEST_USER = u'anotheruser@leap.se'
+
+
+class UnauthenticatedSMTPServer(smtp.SMTP):
+
+ encrypted_only = False
+
+ def __init__(self, soledads, keyms, opts, encrypted_only=False):
+ smtp.SMTP.__init__(self)
+
+ userid = TEST_USER
+ keym = keyms[userid]
+
+ class Opts:
+ cert = '/tmp/cert'
+ key = '/tmp/cert'
+ hostname = 'remote'
+ port = 666
+
+ outgoing = outgoingFactory(
+ userid, keym, Opts, check_cert=False)
+ avatar = SMTPDelivery(userid, keym, encrypted_only, outgoing)
+ self.delivery = avatar
+
+ def validateFrom(self, helo, origin):
+ return origin
+
+
+class UnauthenticatedSMTPFactory(SMTPFactory):
+ """
+ A Factory that produces a SMTP server that does not authenticate user.
+ Only for tests!
+ """
+ protocol = UnauthenticatedSMTPServer
+ domain = LOCAL_FQDN
+ encrypted_only = False
+
+
+def getSMTPFactory(soledad_s, keymanager_s, sendmail_opts,
+ encrypted_only=False):
+ factory = UnauthenticatedSMTPFactory
+ factory.encrypted_only = encrypted_only
+ proto = factory(
+ soledad_s, keymanager_s, sendmail_opts).buildProtocol(('127.0.0.1', 0))
+ return proto
diff --git a/src/leap/bitmask/mail/tests/rfc822.bounce.message b/src/leap/bitmask/mail/tests/rfc822.bounce.message
new file mode 100644
index 00000000..7a51ac04
--- /dev/null
+++ b/src/leap/bitmask/mail/tests/rfc822.bounce.message
@@ -0,0 +1,152 @@
+Return-Path: <>
+X-Original-To: yoyo@dev.pixelated-project.org
+Delivered-To: a6973ec1af0a6d1e2a1e4db4ff85f6c2@deliver.local
+Received: by dev1.dev.pixelated-project.org (Postfix)
+ id 92CEA83164; Thu, 16 Jun 2016 14:53:34 +0200 (CEST)
+Date: Thu, 16 Jun 2016 14:53:34 +0200 (CEST)
+From: MAILER-DAEMON@dev1.dev.pixelated-project.org (Mail Delivery System)
+Subject: Undelivered Mail Returned to Sender
+To: yoyo@dev.pixelated-project.org
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+ boundary="8F60183010.1466081614/dev1.dev.pixelated-project.org"
+Message-Id: <20160616125334.92CEA83164@dev1.dev.pixelated-project.org>
+
+This is a MIME-encapsulated message.
+
+--8F60183010.1466081614/dev1.dev.pixelated-project.org
+Content-Description: Notification
+Content-Type: text/plain; charset=us-ascii
+
+This is the mail system at host dev1.dev.pixelated-project.org.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please send mail to postmaster.
+
+If you do so, please include this problem report. You can
+delete your own text from the attached returned message.
+
+ The mail system
+
+<nobody@leap.se>: host caribou.leap.se[176.53.69.122] said: 550 5.1.1
+ <nobody@leap.se>: Recipient address rejected: User unknown in virtual alias
+ table (in reply to RCPT TO command)
+
+--8F60183010.1466081614/dev1.dev.pixelated-project.org
+Content-Description: Delivery report
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; dev1.dev.pixelated-project.org
+X-Postfix-Queue-ID: 8F60183010
+X-Postfix-Sender: rfc822; yoyo@dev.pixelated-project.org
+Arrival-Date: Thu, 16 Jun 2016 14:53:33 +0200 (CEST)
+
+Final-Recipient: rfc822; nobody@leap.se
+Original-Recipient: rfc822;nobody@leap.se
+Action: failed
+Status: 5.1.1
+Remote-MTA: dns; caribou.leap.se
+Diagnostic-Code: smtp; 550 5.1.1 <nobody@leap.se>: Recipient address rejected:
+ User unknown in virtual alias table
+
+--8F60183010.1466081614/dev1.dev.pixelated-project.org
+Content-Description: Undelivered Message
+Content-Type: message/rfc822
+
+Return-Path: <yoyo@dev.pixelated-project.org>
+Received: from leap.mail-0.4.0rc1+111.g736ea86 (localhost [127.0.0.1])
+ (using TLSv1 with cipher ECDHE-RSA-AES128-SHA (128/128 bits))
+ (Client CN "yoyo@dev.pixelated-project.org", Issuer "Pixelated Project Root CA (client certificates only!)" (verified OK))
+ by dev1.dev.pixelated-project.org (Postfix) with ESMTPS id 8F60183010
+ for <nobody@leap.se>; Thu, 16 Jun 2016 14:53:33 +0200 (CEST)
+MIME-Version: 1.0
+Content-Type: multipart/signed; protocol="application/pgp-signature";
+ micalg="pgp-sha512"; boundary="===============7598747164910592838=="
+To: nobody@leap.se
+Subject: vrgg
+From: yoyo@dev.pixelated-project.org
+Date: Thu, 16 Jun 2016 13:53:32 -0000
+Message-Id: <20160616125332.16961.677041909.5@dev1.dev.pixelated-project.org>
+OpenPGP: id=CB546109E857BC34DFF2BCB3288870B39C400C24;
+ url="https://dev.pixelated-project.org/key/yoyo"; preference="signencrypt"
+
+--===============7598747164910592838==
+Content-Type: multipart/mixed; boundary="===============3737055506052708210=="
+MIME-Version: 1.0
+To: nobody@leap.se
+Subject: vrgg
+From: yoyo@dev.pixelated-project.org
+Date: Thu, 16 Jun 2016 13:53:32 -0000
+
+--===============3737055506052708210==
+Content-Type: text/plain; charset="utf-8"
+MIME-Version: 1.0
+Content-Transfer-Encoding: base64
+
+
+--===============3737055506052708210==
+Content-Type: application/pgp-keys
+MIME-Version: 1.0
+content-disposition: attachment; filename="yoyo@dev.pixelated-project.org-email-key.asc"
+Content-Transfer-Encoding: base64
+
+LS0tLS1CRUdJTiBQR1AgUFVCTElDIEtFWSBCTE9DSy0tLS0tCgptUUlOQkZkZ01BZ0JFQURIWWpU
+T20wcTdOT0lYVUpoTmlHVXg2S05OZ1M0Q0I2VlMvbGtab2UvYjZuRjdCSENmCkFnRVkxeFlxMkIv
+MzA3YzBtNTZWMEZvOWt2ZmZCUWhQckU5WG9rckI5blRlN1RsSDZUNTdiV09LSWMyMHhNSy8KSlVU
+djZ3UEpybjdLN0VyNEdxbzdrUmpWcFVBcWlBbGFxMkhVYllGd2NEMnBIb0VENmU2L01CZDBVUTFX
+b2s4QQpPNURDc2ZmeWhBZ0NFU1poK2w2VHlsVEJXYTJDTmJvUTl0SWtPZ0ZWTk9kTW9uWkxoTk1N
+Y0tIeU54dmF5bUdCCjhjQlRISVE2UWhGRThvR2JDRTdvczdZWWhyTmNmcUsyMzJJQllzTHNXN3Vk
+QmdwRTA0YkpwQWlvbW1zTHBCYmwKV0pCSjdqeEhwWmhJR3JGL1ltejNsSXpkbm9Mb3BSSWJyS0pC
+MmxaVDhIUHBlTVVJdVE2eHErd3RhQXFJVzlPTgo5U29uZWYyVU5BL3VseW1LeDRkOFhxbEwxY3hE
+aDFQU1E5YVlPcVg0RDlrMklmOXZmR2hET0xVMzR2Y2VFOC8vCnM1WGdTY2ZFbHg2SWlEVWZHdGx2
+aE5zQUM4TmhhUU1sOHJjUXVoRDA2RFdvSUowMVhkeFJVM2JSVVZkc0I1NWMKcXRWSHJMbVBVb256
+NU13MGFURzlTZzZudUlQcU1QOVNKRlBzbVpzR3ZYVnZWbCtSNzl1SFBlc25yWkoyTjZqOQpNaUth
+S045NFBhL1dJUnRoYWdzVnpHeHNtd2orTVZCRkZKRmh0TUtnNlFzYUsvbzRLNGJFR1ZLdWNXQk1i
+MnNxCldmd0o0SndTcHcrOHgyS3p6aXhWTllTZXhRdm9oMkc3RDRmRXdISDJzazNST3k3dTlldjhs
+bEVqUFFBUkFRQUIKdEQ5NWIzbHZRR1JsZGk1d2FYaGxiR0YwWldRdGNISnZhbVZqZEM1dmNtY2dQ
+SGx2ZVc5QVpHVjJMbkJwZUdWcwpZWFJsWkMxd2NtOXFaV04wTG05eVp6NkpBajRFRXdFQ0FDZ0ZB
+bGRnTUFnQ0d5OEZDUUhnTUZnR0N3a0lCd01DCkJoVUlBZ2tLQ3dRV0FnTUJBaDRCQWhlQUFBb0pF
+Q2lJY0xPY1FBd2s4djBQL2o2MmNyNjRUMlZPMVNKdHp1RlEKWjVpeVJsVFVHSGN2NW5hQjlUSDdI
+VVB3cTVwekZiTkg5SnhNRjVFRWtvZjdvV0hWeldWVTFBM1NDdzVNZ2FFbwppWTk5ZFBGNzdHazJ4
+ZEczNXZlWmIwWkg2WkVLdks1S042VXBucG5IeStxaVZVc1FLcE9DdUZKNkF0UlVEOTRJClJ2YnUv
+S1hsMHdORDlzVXFlYkJZN1BBSlRNY1RjLzVEdWpIT1Erd3VlSkFtaFZZbEozVnpZK1lBS2t5U05B
+QVoKZ3VVenNyUm5xQWU5SmU5TGgrcERpcVpHT2tEK1Z3b2kvRlVPQXJwbWFnNzZONTVjR3hiK2VG
+QUlzRHYrM1NNOQpjUDFyQkFON2lEaGgvdkdJeHgzMFlrYUlpMmpmcXg3VXUydnNwSXh6K0NsWWdi
+dm1wZm1CWmFqVzYzR0FsK3YvCngrby92eFZmVTMraTZ3alFjRS8vRTBTR2pvY3lQdUw0ZTZLNERy
+S3k2SHQycjBQckdHVFZ0dUZPaWU2dnVzbVcKL09sdVB1dGszU3o1S1BmRDFpRXBobmpPQ0pNRkZx
+Z2xRM1pPa3MweG00WGdwWW1ycnpQcXc1WWlzK1NEVjhobwp6anlrSzRWUlcrcC9IcUVzU29GQm5a
+MG5XSmg2Q1pZOExIeVNiMVJwaFlMRFpWd21JRXd1OW12Vm1ISVIyWUZVCllNZEx4UExiOFZNei9t
+QWpMb2Q0OGNSSzdSTzBSZ1RoMTUyK0VieXRGR3k5Y2tiS3VzRmJzVTFCQjN2MFJyUlUKenozTTcx
+T3hjcFhVQ0tpWlI0MEVYZnErSnVtZVFudm1wSWdZdUNaQkh5MzJwQUJuOHNDdUlrMStyQnp4bXdt
+bgp0WGh0K0RvNlExYXYyVjZYR00xV2xoKzEKPU8zaHEKLS0tLS1FTkQgUEdQIFBVQkxJQyBLRVkg
+QkxPQ0stLS0tLQo=
+--===============3737055506052708210==--
+
+--===============7598747164910592838==
+Content-Type: application/pgp-signature; name="signature.asc"
+MIME-Version: 1.0
+Content-Description: OpenPGP Digital Signature
+
+-----BEGIN PGP SIGNATURE-----
+
+iQIcBAABCgAGBQJXYqFNAAoJECiIcLOcQAwkDEIQAL67/XJXDv+lusoy18jr7Ony
+WQEP0pIRLp4GywGpH3dAITFAkuamO4VX3QEdVGjOHNoaT8VkSVWf9mnsYLl+Mh2v
+1OIwMv0u8WyVtrcxyXijIznnJv8X1RgyCzpUJcmOh04VZcDyxKbnFHWSDMfJ4Jtq
+qnXDONcfEeT8pwrGjP5qzTgcF/irG3w5svyQjEtj6kycddYtqUc9Hx3cMaRIzsHg
+kuUzznSzU/6P0Z345q/kXyYvU9rlcsP9vogrsqL2ueLwYSipxUJQUrRWG82FYoCo
+PAKNdGIt0xl2gEW+xWZkJqFarPiUFCx//+bVBelKrqj6rjwbj+E7mHJW318JYVHQ
+en3Smv7pEWlT4hZHXnoe8ng6TAvKzQjf7/bUxq2JpKSycp2hDO3Qz3Tv+kc+jC/r
+5UDWe/flR+syq8lAQTRSn6057g3BgDG2RtAwsjedg1aTFSrljSxbKlK4vsj5Muek
+Olq9+MUdMFSE3Jj/JC2COcS3rlt/Qt+JLDYXKahU3CodaSgF2dobikDe1bW0/QNS
+7O4Ng2PK0pA416RCFRUgPXerUnMGiWAiq7BoRHeym9y7fkHYhIYGpPVKXJ6t67y5
+JjvuzwfwG8SZTp4Wy2pg1Mr6znm6uVBxUDxTHyP3BjciI1zpEigOIg9UwJ9nCDxL
+uUGz4VqipNKbkpRkjLLW
+=3IaF
+-----END PGP SIGNATURE-----
+
+--===============7598747164910592838==--
+
+--8F60183010.1466081614/dev1.dev.pixelated-project.org--
diff --git a/src/leap/bitmask/mail/tests/rfc822.message b/src/leap/bitmask/mail/tests/rfc822.message
new file mode 100644
index 00000000..ee97ab92
--- /dev/null
+++ b/src/leap/bitmask/mail/tests/rfc822.message
@@ -0,0 +1,86 @@
+Return-Path: <twisted-commits-admin@twistedmatrix.com>
+Delivered-To: exarkun@meson.dyndns.org
+Received: from localhost [127.0.0.1]
+ by localhost with POP3 (fetchmail-6.2.1)
+ for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST)
+Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105])
+ by intarweb.us (Postfix) with ESMTP id 4A4A513EA4
+ for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST)
+Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com)
+ by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian))
+ id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600
+Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian))
+ id 18w63j-0007VK-00
+ for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600
+To: twisted-commits@twistedmatrix.com
+From: etrepum CVS <etrepum@twistedmatrix.com>
+Reply-To: twisted-python@twistedmatrix.com
+X-Mailer: CVSToys
+Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com>
+Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up.
+Sender: twisted-commits-admin@twistedmatrix.com
+Errors-To: twisted-commits-admin@twistedmatrix.com
+X-BeenThere: twisted-commits@twistedmatrix.com
+X-Mailman-Version: 2.0.11
+Precedence: bulk
+List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help>
+List-Post: <mailto:twisted-commits@twistedmatrix.com>
+List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
+ <mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe>
+List-Id: <twisted-commits.twistedmatrix.com>
+List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
+ <mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe>
+List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/>
+Date: Thu, 20 Mar 2003 13:50:39 -0600
+
+Modified files:
+Twisted/twisted/python/rebuild.py 1.19 1.20
+
+Log message:
+rebuild now works on python versions from 2.2.0 and up.
+
+
+ViewCVS links:
+http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted
+
+Index: Twisted/twisted/python/rebuild.py
+diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20
+--- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003
++++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003
+@@ -206,15 +206,27 @@
+ clazz.__dict__.clear()
+ clazz.__getattr__ = __getattr__
+ clazz.__module__ = module.__name__
++ if newclasses:
++ import gc
++ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2):
++ hasBrokenRebuild = 1
++ gc_objects = gc.get_objects()
++ else:
++ hasBrokenRebuild = 0
+ for nclass in newclasses:
+ ga = getattr(module, nclass.__name__)
+ if ga is nclass:
+ log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass))
+ else:
+- import gc
+- for r in gc.get_referrers(nclass):
+- if isinstance(r, nclass):
++ if hasBrokenRebuild:
++ for r in gc_objects:
++ if not getattr(r, '__class__', None) is nclass:
++ continue
+ r.__class__ = ga
++ else:
++ for r in gc.get_referrers(nclass):
++ if getattr(r, '__class__', None) is nclass:
++ r.__class__ = ga
+ if doLog:
+ log.msg('')
+ log.msg(' (fixing %s): ' % str(module.__name__))
+
+
+_______________________________________________
+Twisted-commits mailing list
+Twisted-commits@twistedmatrix.com
+http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits
diff --git a/src/leap/bitmask/mail/tests/rfc822.multi-minimal.message b/src/leap/bitmask/mail/tests/rfc822.multi-minimal.message
new file mode 100644
index 00000000..582297c6
--- /dev/null
+++ b/src/leap/bitmask/mail/tests/rfc822.multi-minimal.message
@@ -0,0 +1,16 @@
+Content-Type: multipart/mixed; boundary="===============6203542367371144092=="
+MIME-Version: 1.0
+Subject: [TEST] 010 - Inceptos cum lorem risus congue
+From: testmailbitmaskspam@gmail.com
+To: test_c5@dev.bitmask.net
+
+--===============6203542367371144092==
+Content-Type: text/plain; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+
+Howdy from python!
+The subject: [TEST] 010 - Inceptos cum lorem risus congue
+Current date & time: Wed Jan 8 16:36:21 2014
+Trying to attach: []
+--===============6203542367371144092==--
diff --git a/src/leap/bitmask/mail/tests/rfc822.multi-nested.message b/src/leap/bitmask/mail/tests/rfc822.multi-nested.message
new file mode 100644
index 00000000..694bef59
--- /dev/null
+++ b/src/leap/bitmask/mail/tests/rfc822.multi-nested.message
@@ -0,0 +1,619 @@
+From: TEST <test.bitmask@example.com>
+Content-Type: multipart/alternative;
+ boundary="Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8"
+X-Smtp-Server: smtp.example.com:test.bitmask
+Subject: test simple attachment
+X-Universally-Unique-Identifier: 0ea1b4b2-cdb8-43c3-b54c-dc88a19c6e0a
+Date: Wed, 8 Jul 2015 04:25:56 +0900
+Message-Id: <47278179-628A-43F5-95C9-BC7E1753C521@example.com>
+To: test_alpha14_001@dev.bitmask.net
+Mime-Version: 1.0 (Apple Message framework v1251.1)
+
+
+--Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8
+Content-Transfer-Encoding: 7bit
+Content-Type: text/plain;
+ charset=us-ascii
+
+this is a simple attachment
+--Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8
+Content-Type: multipart/related;
+ type="text/html";
+ boundary="Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B"
+
+
+--Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B
+Content-Transfer-Encoding: 7bit
+Content-Type: text/html;
+ charset=us-ascii
+
+<html><head></head><body style="word-wrap: break-word; -webkit-nbsp-mode: space; -webkit-line-break: after-white-space; ">this is a simple attachment<img height="286" width="300" apple-width="yes" apple-height="yes" id="fd3d0c89-709d-419f-b293-a6827f75c8d4" src="cid:163B7957-4342-485F-8FD6-D46A4A53A2C1"></body></html>
+--Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B
+Content-Transfer-Encoding: base64
+Content-Disposition: inline;
+ filename="saing_ergol.jpg"
+Content-Type: image/jpg;
+ x-mac-hide-extension=yes;
+ x-unix-mode=0600;
+ name="saint_ergol.jpg"
+Content-Id: <163B7957-4342-485F-8FD6-D46A4A53A2C1>
+
+/9j/4AAQSkZJRgABAQEAYABgAAD/4QCURXhpZgAASUkqAAgAAAACADEBAgALAAAAJgAAAGmHBAAB
+AAAAMgAAAAAAAABQaWNhc2EgMy4wAAAEAAKgBAABAAAALAEAAAOgBAABAAAAHgEAAACQBwAEAAAA
+MDIxMAWgBAABAAAAaAAAAAAAAAACAAEAAgAFAAAAhgAAAAIABwAEAAAAMDEwMAAAAAAgICAgAAD/
+7QAcUGhvdG9zaG9wIDMuMAA4QklNBAQAAAAAAAD//gAmRmlsZSB3cml0dGVuIGJ5IEFkb2JlIFBo
+b3Rvc2hvcKggNS4y/9sAQwAFAwQEBAMFBAQEBQUFBgcMCAcHBwcPCwsJDBEPEhIRDxERExYcFxMU
+GhURERghGBodHR8fHxMXIiQiHiQcHh8e/9sAQwEFBQUHBgcOCAgOHhQRFB4eHh4eHh4eHh4eHh4e
+Hh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4e/8AAEQgBHgEsAwEiAAIRAQMRAf/E
+AB0AAAEFAQEBAQAAAAAAAAAAAAUCAwQGBwgAAQn/xABQEAABAwIEAwUFBQUGBAQFAgcCAwQFARIA
+BhETFCEiBzEyQVEVI0JSYSQzYnGBCBZDcqElNIKRkqJTscHRRGOy0jVUc4PhF2SkwsPT4vDy/8QA
+FAEBAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/ANW2
+3RMhJzJOiUK4up+4Lv8A4JUt/rgjBZfmN1JRBsaDchEklXfSaQ15GkdfFX6YteR4tNtDiuuCC7gh
+6iHpt89O+vLFg+ARv8WAQ0bizSSaD1CkNo4fuL58J2/BdhXSJ+PAex64sL+PCDTEjK47vw4D1qnz
+49b1+Pw4bMrf+IOFWj/NgPdV5Wn/AIceSUuMus/5sKMRw2Y+Lo6i8OAV4kiETtL5sMGpaACofUXh
+/F+WPN1vvei0R/1YV7tRLpC4fxYBvcT2hJMztIenxXD/AP76YVvCICQnd+G/COpMNwvu/iMvh/Om
+Bkgs63QXYtk+IIv4h9FuAJm4+0XCdpdN4+Lp+v8A3xGBZYXBJ3mVt1nzW/T1w6qmperb0koNoeG8
+f+44Emo6QBJN9YoSnVv+FJUvr5p4CVxBKXEmsdxdNvwCPpT5S+mIvGSBN1d4zbOkytuLqEgH1p/1
+phDtZwg1VIt+0LbwEKXjr6j/ABKfWmGGjhQmSvE/CVqVp3Xaf8Mq+Gv0rgLEyUW4JJVT4vx3D+mv
+w4dN04IVyRv6Oq6y4fr3YQ0IXMeLkQMRtutIOr/LAqY4VNJqMgYJo2/iG3/TgGJDMyyCS5JuQEh6
+UhctlQH9a24EuJbjnFzGShyISuV3HiqRidPpUfBhSpEoBptjfKIp+7AUXgFdp8WhjiM3FYd1y5fr
+kj/+7igI0h/FZXATmSz7dIkW0omSfUYIyQGCo18x6sNHMSjFVVZ28mEESK1tcwFURL5a2692BgLM
+RBJMloRe4iIbQWQJL8Q99448kzjyVVJNtFqXCJf/ABJUAX+nOnTgGpiWeO0l0F5K5NTqVFRFZsqg
+VO46Wj4cRop8pxC7FN4+EhSEr05gDScj+C8qFStPyxMWEuldBaUUWTIhDZeJKkkVP4daENLh9MMG
+Kj5IWOy6ZFaSi7ZaNAyIfmoQlTAS28lME0/vkpvUuFuuTluYpDX/AIlL+/DDSQmkFVV1JibJqmRE
+ZKMwIkD866hWuqeGHCjdNp7SF4xJO0UUC9lGIL+VqlKH5YdSKNjY8XJLNdkfERM1ht15W1pd4cA0
+bjMSkO1TXkl7R98u54ZUCS6unwjzGuGnr6U2lVBklEBUEb7XLojSLyU0t8OBTh8oSSCaaLVNqur0
+ELlZK3q0tKlRLpri6/um4FoRIGgIiQ2JEe6QpU/g+XTrgKe79rPrU1JWR4xMbiJs8dW20LkuOo86
+euLXkyPnEGntKSknwrOfAgm5JUEh7q8lNO/lXBiJg4tsDYkGa+8gO4Jl/wCmnV3YK+8USVLZ6vh6
+PX9cA3/aCaVvtJdRa3wiil1f7sKbuH3FluP1yH+QLS8vmx401ib7lnw22eHCVk1t0RFY+rqO0BL/
+AK+eAc4p0nu778y+XoDp/rhYLOCMS4ldNNPqUuttLUfzxBj94jESbGSY9VpIj1f7sPpKOBcEPiR8
+I22l3eeAfCSJNv8AfXeK0yWG78PnhJksropxT1O6mtorDp/zwhuKyiRD4VBL8FuHiJ0VdRqadPl6
+cBToqUKPcIFwx2q9JG5tAiL5SKnQWL5GPEXzIXYgpaXzYzA0U2J7DZYLrSIjK9iRad415bZV/wAW
+Lbld4oLtBMUdtuokV1yJCV/0INRL/PAWsxHCTG48KuIvD1Y8ZF04BIF1+D/Vh3wn4MNH81/+rHkl
+FLyuDAKtuAui7DG5aYiQdP8A6cP9VmEqjcA3dJCXTbgHbv8Adho07rh/04i++Td9R/Z/iH4h/LEq
+4f5R8WAauIlbbOr5sNJIrClcXVb8v/TEhZMd3cI/F0/LiPcKYffGPwl8PV8v0r9cB5VToLr/AAl/
+3w2bckw93Z7zxB4bvy+WuEGPujITu6rb7P8AbXDpimolaQWl8v4cBFSWTLpUW3xTK0yILSSLy19M
+J4cve9ZkXSN/iK38VPOmJjtPwiSxipb0mPUQ/wDfDTdmSYbdgW9V3i/1U/PADDTWEFSQC7cK4Osv
+Ll7s/g/LFbnSeEqKaaNyYn9qVIPCXounT/1ji7pR6nUQncRdRJWdKunnWnrhjg7URVH3dtxXl4kv
+yKvfT6VwDGWnlzUWy5nb4UNzquGnfYfxYmSo7bQiT8SfUIpmIjZX+bWlcDJXejW5Wsw2SIulO4ki
+9Knp1BX60w5BOJJTpK9dmRWmC3jQ/wDudyg4AYlH+9XJyz30yt6lGHr8VydKcsNfYU2/EpmDZQSF
+Mi49VLp9DpUsWl6mok4FZAA92Nto3D0V/Lz9MQ9kSbiShuhu6SEljLqr9K0wFb2RJXiWjl7t/Akm
+5SPq86p1KlS/TDSqJCkkx3nztFUSWICYIkJD8vIeRUxMerJppCg5O1MS6QWATEvxDU9LcDHCm84J
+sotHEiRD0u0UhEiqPK6oFUqVwC3DFumqlILhHDtESaS7mNJIh8tK1DTDjRSPdpW3tUFhuG4X5pEk
+XzUpUudK4S9jfY2X9xcDQapl9pVbPFRJIfmr088KbpuvZiqyDx0u3QtUQJN4KpD+KnPUh591cAwH
+Ctm5Nr0C3SuMBmCucl8wal34YVki4dqgUk6+0+AikgK4e+xTX4sSjGYUZIERrkiJEoR3pWkdS5VD
+UuRU+XEVVGSkHpFs76atyZbkakQql5jqBVtL64B9wimQCuKxjtD1A5lfuh8k66d419cXBJwptIKc
+GaC20IkIgJj4eY618WBOX4V42PcI1+DBINseGAC8VemulcWAy6/dhaJXdI2Db/XAI6VQLYDbLwkJ
+AIkWvnzwwkTwXApkshaVvQNtwjTyxISIdolFPDbdaVn/AErjLc5zXs3tryVEoOTLjUFxVSTBK0Q6
+a3F1YDVne8QCQrAh8XgHqH64jKl96oRgIj1FcAlb+euHHAjYkmR3DddulZ4qfrjAP2jc/SBOEsoZ
+QACcO1RRVGwS4nXltAdK8vxYAjCdpEl2g9qbWCyS5XbQMWRLSL4Ww3LlTltalTQRrz6sbU3RtutC
+34hGwBIv9uM77Gsh/uLk9nHqAHHEJOHRpoo+Ov47uqlMXreWvJRS8fitLa9/08q63eeAmfcJWqBc
+RfDYHh/yw8KgFTVR0FS8/By/pga7HfAVCcroDd0j7rw/L4sRyXSGtqbghEeVKJrJW/8AqwAqHb9D
+wRBBNFUusGjwXIJLXaXCmoOtLq9/PEHedRbhrw14iToU3IIgQKiXz7ZXUrSv81MTNtuILryiICok
+vtkT1htEvr8eoUpX/Fgrl8dj3fGLrt0uoy4wHNw1/PUsBZEpBO9dN2sCbhLqMRO60fLyxNPwfNis
+N3DUnBKNt9QkyK4rFi6q/StcSmT5RNJK5suSZdV4omQ3fStfLAGrbvEGEqjaZW9JW3Ya4gbBu3BI
+urqAsKbum6hlaYEVvhE7iup5aUwDqRDtdXi+LCjttx4BEgust+a3Dd2302GWAcMRsu+L/wBOGNxM
+XBCXT8mJXwXYQqnuWlZ1D4S+IcBD3hTPpDcu6enqtH6YdNNEjJT4bbbvFcP5YYVuG5AQuIuoPhu9
+dPr9MeScEKQWmChEPSRBbd6/kXrgPJKfZ/uTEfiEerp/7Y8BCQDaBl8tvi/w4TasQEW9bcQiN3hH
+/tXD7RNTiOo9serpEOkvzwDrdEhD357iheIvw4UZWmPXaNuFH8OB0hxCjhL7xAfDfvANuAlGp8Im
+Al8JYGPVhbRi4qLIcQp8qxCJfrpX/liY7uF2P2k0xttHrH/rgE4UFRJUk3LohTIRsTchd/lgCPHN
+xSVInIbJJCVhH0W07x8OJzQkRbjsBanaJdJ9PPu0xXjUUct0k0wXXJUesSRAwVG3zqI9OFRjxxGp
+L3AuLceoS4AhtKnLnpTSuAKO+OT30yNclFLRSuciJF+tB6cQnpPBb2+/Hbt97xI3fkXThYSm4xIn
+exxCdqgincHTXnd1/wBcQwkmq6tzaYBe64TMTStH/wAutNP64CO4cOC3RHilRTESEBcpF0111tuH
+nXEF66TFUSdn94NoGswEwIbddCqBU0KmIOY2/wDYSqbYFyRQIVC4ZEF10Du8VtB1KnPvwwkxWsJy
+ViZKiKaqAmq2FyFP4yfVpgCDRwjcP2xqoip4hK9Akh/lOh0KnpiU4FYj8Cgo23CAtgXD86VAg/yx
+GbxrjhLWz90797uAuRi5EUqFrs1Guv6Urh9uiixVEk2xoN1BJR4AomgSRU7ipTy78AObrM7+GUbM
+dsi3PdrEBEVC5EmBjpQ6fniL7NeNJAlF5VdMiPbQ3rhSXHv+8Ctaa0/lwb2SFwkmJm7akO8oCljw
+efcVKVpUqaYRw6KIDcFpeIQbLKobpV53CmdbaF9MBKhJoU24i5WXXT+Ix97bp6+Gv+3BoH0W52iY
+uQUEhu6ekuX0rirTCKZHuF7sRG5VdRtZ/wDxCendiGqo3kFUl00eGFoQ3P8AeErht15Kj02187sB
+fEU00EiTU6iH8HixgvbLJIse1iFmbwIotdoN42ltCtuUK6nTrroPnjaoeWTctBtPc6RHqWSEhL8v
++WOY/wBo1qnJZgmpCPA3b5SRZxjVIQEw3QuKpnpy+LAbJ2wdozHKGWnjRMFymFUi4YSRG0ip6dVe
+eMr/AGX8llO5gddpE62kRTTVL2ULvrAhLxndypr+mM37SE3Ehn1DK+aJv3iSqZTrps2E0m2gD9zQ
+afJprSvcXPHQX/6pdnOR4GHyvl8150U0uGQZsAWclcI8x8VeeA15X3jgSTR27vFuIjaWn+L+mBzg
+nC73qZqKIh4fcjbb6eLA7JMo+norjpbKr2H1+6auESuKlfi/pixNExELtlBsiXhSURtL/wDGAU4J
+QgEU23u+nw22jiNSig1KgpbdNfD0f+3D52iGwIAJfONvhr9MOJN0doOIVQUU05lshz/pgKvk8lFz
+VUFmCagpEo2BQ1hEdf4ZXjTSmJwC+GPXJjv9VvUpduj1cxpy50wJdSDx63SU3uJJUhECUDqJGnMh
+U0KminLBSPTeNGQoILe5V6QcE2L3WpcwL3uAfVFwrH+Bds8EiEDELhtr8NfUa4fjGJF9rXRQFS0h
+JIWxDcXxFrTCQZkTRW6xsI3dSiJ9I0/I+6uByKg2L75tbum0RWVSIS+bnrgH3AppgK6CKaZJ+7Ax
+WVQO30PpxKSdPOLSUQDdTtLdEdpX8+dCuL/LEe5w2cWpuXyjhRK4myblJcSD56ajTnhi4SbjuGgT
+dQvdEo2JIhKnKo3AXTWn9cBa4yQRcpGRGA+ERHqD+h0piZ8H8uK9GLcc3VUFYE9vwCR7/VTz0rQS
+wdaERNxu6VLbiEfDgFbnXb4RHqLouwoyHpK/xYYcbZJXXn09XThO4NgkXhU8VuAfcJpqBtl1CXVg
+c4TtcESnURl/qKnd+VfriVdtq2j4bekvl/8A8cIaKIuTVtv3E+kx/wCX+H64BIbJN+pbcISJMiIL
+bvw1/wC+GnBOlHbXYM00RLqIfEJW/HTzH64mXeL8PUY/DhaRJkrb0XD8vit9MA4Y9YXdXy/zeuIb
+jb4hLrtHq6Cs/wAu/E8y6Pw4iqkmVwqI7lxeGzAQTFwKvUZkn/wyAOn/AHYrr1PfkEF1D21Ej3Ac
+popWkXop1f0xaXu8JpJoLbZXeKwekfm78AnCaKDhXrAhckN4rGAi5DuqQ/iwEG14mqSbYNxQUh4p
+mTYRBXXvIK0r54UDdmpHtXKCIL7dyYlsmNpV+A7aV7sLcTBNIx8pe64dLpEhRG9IaFpb4ueKVlLO
+0LmLMBRbaSapzyAkso2EDteI15Xjofl50wF1cOkWjRIWPFfMYC5VAhGneVNR8OuIAOnTlVBBiG4S
+hWqr3pGC4V7+8qdWBj3bF17QjX/92LZJVS4dg689sqVu7/LD/tR0xkHTYTNNuyS60hWASI+/3fuu
+dNO/ALVUYk0JymsCaKDrhh3GYXCXmFbCr04cBTgXDoUwPbQEVthNYwND0LQhp04Y4ckHaBDYo82F
+FhDZExV10r16EPOmEx7MY+PSUXcroKLqk4VeEBCSQV52c7u/y9MAo3SjaQFpH3uZJ21EnI+6MHI9
+1fi13KUx5u+dNgERcgTe4UwdKXttoqeJI7ht/rgK92ZSP33Ky/Bqq7zoLBI0BEukkzGwvFjzdq+9
+sJJrtgFq594qwJyQk5DyWoJUPAGHr4V27qZURNOPRIiEyRFU0iEtPEFa1tuxFip5u+BUXZuouQbf
+AS3uhKo6X0opbdTzxYjhWq6RIIHc12OHFIvAqN1dddK06sAnDUk5Uk2KwXDcmVywmIhQvuq0IfP+
+bATm7N8pGKrw7BBR4ncLpSOW2CVV7+jToLX88VtkNsgKd5xcgr4mrkCZkRVHUxu+7V/K7ErMUoSG
+TYpPLrnhm6zpcXBX7H2ig60Cteq3XyxnPY/2mDmjKUxF9qMkbtvHKkK/EIjutgqdaCpRYdNa0r0+
+H4cBfXs5H5Fy4q9/u0ekVtt5ILqq17kaWUIVDrXlTSuMjnc0PMkuFc3yDMHeYph4LlJqp9+x1HSq
+NCDWu5pt8jtwf7WG62TnCGYM2vAkW7RcigOH6mz4qhXkuPz2/wAQe6uAfYpl192n57LtEzeCCka0
+K1mwcrEHWOltdaD1269/ngCGTOynOHaQD7NGf5J1Cx8ouKhxgomKq4jyEj0Hl3Y3jKnZ/k3LcU1a
+QmXmrZG7cMyRI1SK3kWteeDEYs3UcERGC/iESTvLp9MSDWETSQFsdviG5Eur6eLAIdtWqloi29yn
+aXUF135YlJC3La2D6bbQ9zb03eHDUg3UXO0um23q2S6S/wBWFJDa48B3CN1+zaN3n8WAYVaqL+Hq
+IS8JMwL/AA8ywpOrlMKDukNfOgtktNf9WPo7yO+oXSSnh9z1D/uwtFOhJ0qoNCL1FHl/zwFO9k7j
+Rdjw32pdUiExRRISGnKleQ+PTExu1TbGQuUTURFIRO6NMBL/AEfH+WJziFRcu1UyYIdXUJcMBjdT
+vLmV2EybNmRigIG2IhG4RBUbtC+YRrTAQ0nEgs0SYNHMcSbsis23JpKiPkHVXvpg+CaijcUxcuvl
+L3wL2/QvXXENgK3tAVOMkRuK1UU3IKj/AJV6ta1w6qXRwyjZdQSVtAl2Y2l5W9+v64CL7LT6uJZg
+O3aKSpMLbTu8N4UpqGPJM2qavwJ7he9Fs5VSC6heLSpYdVcC2aJJp8K2RHpvT3Q7tenSg1078Smj
+wrBUI/fWimKAvBu8XfoemAkt2ai0vxithDZaIltKXfioVPDgoBEoFyVlvV1YQyIiAeu64viMSIfp
+ywsFveql4huEREeq71wDlvw9H83iwwbVQTAkztG7q/CWF2rCJEO2RfD0fDhpJwN6qafwlcfuS/7Y
+BXD7Ye7C3DAD767wlaQl8wj5a/hxOBQSDx4S4RIvuz27vDb8vy4BhJQiNXcC23wl4ht+anqP0wwk
+jaZEhfb/AD+H8q+mJwC3TtH/AIf4+kdcJuTEBEQtTLp6Qu/5YBzq2hEvF8VuEqkXUJeH/EX/ACwO
+OQtVFAjQtUO26+20qfniUagqAuh19Nt1wWjbXz1wA6QTH3q5AHu/CWzcVv5V78cO/tF5mkn3aLIx
+t5sGrQxRsTAgAituvT07u/yx3ce2SVrYNzb8I+L+uMy7YOzGBzxDnxKJoPkklCQFFYrhOvxDbSv6
+0rgOaA7ZMwNsqOstyyISSyrNNuMin0iQcrhP1L8eAHYllOSzb2kRLaJM40RLiFV0zMSQAS1v1pXz
+7sCc8ZHlss5lGJcrA9HwpKiZCJaDqVNC0rTBPsizxIZCzGM3HxpySl3CGkodo7NedRGlO8qaYDtF
+2PAzo8MiabpISUVQUNUUnw28iHnbWtMQZNRvMmkPEgW6uK1yh2mx0LXqofw64g5S7RIHOzRV9ls9
+xqyt3+g+JYlUdbtKU8OJztZRo9Vkhc2oqJCNxbtr4bfS3kWAYVUTkGk0SgbbpRcW6pJgkREAc6KJ
+20uw5Jk4kIol48zUtS2WwiCw8SlQeevPSheuILJaQTj1Xz4EE1I+5RmgIB039wqXFStcDknDpCPQ
+UFtt2tVFJHoIeDMir1UqF3Td3YCHNvHSdqjFYF3X90YCpYXeXUioFerlXBaCJYdpdofDJqK7Krlb
+dI0CHxDSuultcQ527LcU1cvgaqEggTl0k5edSqp9G8G5b82AEnIZils1rwmVI105FNJNpvpgCSQg
+Y31PfCtev1wF3kM8ZbYvUkJZ5HNm5CooIEsiJidDLQuXPAKMzZE5tMUI+YQHiVxbGxTvG0R67yvr
+pSug4qmcOyOaQdrzcyftJwo1TRSjE1ituoWlDVLTrGmObZhHMmX5NWQ3jjnC6pJhwhiJX09RGtef
+0wHZMK+Yy3tDJahoC+kEifxxJgFhKp1tr4KaXW4wrKLHJ8tHv3c+/Xi3kI+X4tspq3SeJEqVqJqj
+omZUPXvxXYftokGhtXCDBBo+ZvE3aZprHtlaNqgCnSmlx0xdf2pURm8nwXaDle9zluYK1dIQtJo4
+Itba0p6nry+bAEOzDN2X88LSPZZma9eHWIih3ghyaHXwhUx8yH0xdVe0px2ayC+Uhio4W8Paja/m
+EkDVRt8YBWt35YwXKSOXYl6xEWexMKrp2MFrVV17kipRbcqVBS6vEHiHG3ZwJrm3ITybUh8uzWds
+otxbvOJuVbJJFzopS2lbyHTADFf2hnhNFZlTIy4sbiEFxniSBUaFp0jUuf1wYaftCErk8syM8mOj
+jWx7bhdaVtJL1rTq1PT6Y5omHHD5cXbOcvQ7aWc+8J4udV3TwVCvpVNGn3I2lrz8sPqsYtOPZsYJ
+zl17JCQtjBPdJVyZBWhFQ1BEQGn/AKsB1UH7RWU49ukhmCEn4VZVIVrVmxq9Fe4/5cSm/b92XruN
+v95F7i6R/s1a0tO7TXGB5oymcW+y3HlA5eczCqVq8EymCNddGo8ycOK0tAR05DrimRXGRsxPxooy
+jBNcdkmcY5ScpEN1dBWXu0Cn1wHbDTPGVZtuxdxs3HKIyglwt3SatvfpSv8AXBU3xt6AndbS2laU
+2fLHBEfGtyBiIg1zHFtECWkQjjJsbYSLXQlDtuL8sEY6NynJJG6mO0CQg16qVoDAk1XBIp/CNVB5
+FXTzwHfLgkWjvcXNBMi94S5ImIEXy60LH33K6pKILXCPVamsQld81taYHAoiVqHGAmoQ7hEm8MRI
+bvQywg953aKDyR2WxkIriiC5CVPhry10rrgJircrxUUsLxdZIiYjr9QqOE3Ji3LrNdr0ioCZkYjp
+3cq1oVP88BxcOFAJBNsxSTH3hoCCzUyG7xJlXQcE2ixEsQrhvolbwypbS+6NvrTrp+WAD5gtTNJB
+8bHqVJS29Ud0KDyGvf7z0w/Dvickk5E91MhEWwkYGqOg8xr7umLAYpkySacS6QvK3pMw+vnzw+yj
+yJ6q7XM/utkTvuEgu+SvxU+bADuIkuNSQbAZIiO4vaA+6O3y00xMZJuCtc8MumpaX3iJDb/u88Fg
+FuhbaiFw/wCrCjcCNokZ/wCgsBW5XjEw3E0UFOq0bgMSEfTShYU3U2zFSwLbeohckI8h76UrSuLE
+bpMQIusbvwFiCq8TUZELkwHc8XwiOndzr64Ac3dPHIAKJmV5WiqVq4/49KDg+BEKQ7h+8+IhC0eQ
+/rgY3dMyblw2wmsXjJMwEh/PElJ8iqAknYSalw2isFpFTv0/LAJSj1hcKqqLApu23iSJF/W7DUgJ
+IBxIo9Q9NxLEI/0w+ahDcuJmQlb4QEsOt3gk4Ftfa4EbiEukrfXTABVpJqmaSDkHS6aolaQgBj/W
+lK64aSISMUxNBQh6htvSIR/FrUqYLTEaMk32uhD8RIiZf1pgA7FRNVJB3Zw4+7ESvQ3fw6F0lXAH
+G6xEyEhO0RPrAguL+bkXdj0rJM4mHXdvjBNMEiUIhC278sVlw8GEaOncksxaIppXWFYNofmGla4w
+ftI7YGso1eSEei6QZsEPdEsaoiurUtEqbZV08XxYDMO0gX072tlGtHi7twuuLQFyMeo1CtU8uXfi
+29tfYf8Auy4/eDI58SzSFNNdgiZqqpHQaX6d3u6+eGP2QstupntT/eBf7SnG3EuQheIrGPLn3c+/
+HXOaBTFwTlN+u2WttIk3Ipd/cVR88Bz7+y/2fuo/L6+ZnYBFupJ1cglsqkIpCX3ZUup01r640aYb
+uIsCtA1FFnQqGxTckIthp3Kp1IS11xfmjxFCPtF4gXT1qk8G0jp5VpSvLFWNw8KbfO1DBRNNJNMQ
+RkrDH8SZGXd64BjbUdtHRXoLqSFok5JyBJKhTnbzGnOnngBl9jISUfLLsWdzVR4mmBuQNIeFEaCp
+Sp39Y8vTAztleKZd7MppQTTXFJmVwEaSlxqFyup81PWmMo7D83ZwhJOAybNrIO4GSSEUHKhkQpDX
+4aV88A7+0ApJZdmIwpts1dkquo4SapuSVQ2e8RKtaa/pjqPscnonNeQY6Zi0UGyKo2mgj0ikdOVR
+rypzw1nXIcPmnK/AkCBN10veKiFxEGt1LP8AFTXAWHZw/Zrld409sAgN3ELmodypDXl+dacsAW7T
+WakzCSMM2kgbFaKjxfwkklXlqNaefpjjjtDheEy5ttg2BTeJqM2ajkbyCpW7ltRvIirXW6qn6Yum
+YO1bicvyKDGedEirIkntCwVAHI1PSiZrH4Nfp4cV6dzQ6YuDXm8vQ79FdcU3C8iZqAkr82zSvvCp
+T1wGPzYqcaF3uyEtnYTRt2redvfX19caP2F9o0XFoyOQ88Gf7nTvTVVPxR61e5UK110/6Yoz2aUX
+lXT5tY2RUXEngCijYJ9VKbIUHSg24BSCwru1VEkQTTU8IfKNOXOn1wGvZ17OEsmZ9QUmlgex7tcS
+iiYPNpWSRrTQVRUMSpuXVG/F3/Z4lEYvtVQj1DfPWsw1KHkzWAOHFal9QBNQdL6UpXTw4pnY92kw
+q8Gl2adosX7VywfUwclaS8atQdakNa/DjXshZByu2ze17SkM+Qj/ACwxVTcoCmAJEJAJDQbB00Pn
+gMwzaz9jTs1lJMPZKjZ0omIRkUTlyulQtRGq6p17vSnw4qTduK/Zk8gpCbmxeRZE4OKbQgECVvxE
+4pW6nV3/AFxrL2Jku0bND6ZUWQfw71cmDAE5Uw4NZQ613KgBeQ1540fIWT4WJhyaZLeLuU5Z+LAT
+dnckg1bnQlST/CVuAweT7Jc9LysO0fxuVoseA41BVBZUeJCtLy3CqV1a8vw+LFn7MuyEnmeGK+b4
+eOUhZ2HN+q2ZOTQQZhTW3purUtdO+tca6xcOs2vppKQZoMlJmTGFaq9RAqxQG6ulfmrrin5tfOGs
+rcxm2rkZaWHLisWwAiXYx6HKtEbeq+lOoq4AZlfsty64exzR8xdTjqSQcroA593wzIPBubdt1alb
+1Y+9k+R8szGQo54tm4YdX3iZs0yREUqioVNNFBIu6lO+uCrJ44TcTXsebeu2+ZHiUPliTICEWyQc
+1EK+VtLfF8WM17Ycms8ydoEh7IynGsvZdsc7obsU91wnSlTUpQOWlb6c68+WA6bRfOmwqsf7bQIx
+EjQcMBck0qXylSvUOHYq7qTj0Ytyool71dRFVrv6fCVLeWKeqs6THbTMF1F3hNxXcrOGptv/ACi6
+i6dfPGgx4vBjxXXNdk4JAhVaovANJUacrqEY19cApwTxNuPArGKdvUkm8EiSKny6+VcCnEgiLjfc
+tnW2V3VwAqiqXdcFUyraWBLuQlhbuicg+bR6CRJ8U/ZgqIl+aZDy0x5kUW5cPHaxxaCLZAViFNZZ
+C3T4/FUcBeoov7PSTEwIfhtMgMf8J0pzwRVU21RJQzES8IiZdReflilQT58o3auRcunYqD0E2eIr
+3fWuoULzwa450SpIyRmKIW2kszIBE/xFQ68/0wDUxnyFi8wDCSRmm66Ss2VS3Rr4Oqg201+tcFHc
+oJASiBh0/ESN4/y10xmmeGL6NnWeeMttuPUYCXtEOPvBVGg89AIfHTvxYsnrZfncuK5kykYOWrtX
+cVSUM7R08dnV3YC0gXFt99dHY6rvd33Fp5jW3COOInFwrIWq9NxLGQ+mmhDTA5u6auWQin/cfF1X
+gaQ/NTqrityEw6d2pwjlBR9cRAYvBFJUKFzHqTr7yuAtjhZZFVVS8BTSStEiMCLn5U58+7EdJ069
+n7gguTciFQSEEiG36UvxBh3Cz4NwQfcOkXUKjMCJA6fDrrzHnhTh43TVdXbnvEulsTAhFUafxB0L
+ASqqLKEIkTJdYri3SbKpDbQta30pSumPJSTxA9tA7vmBFyKto9/TQ7f6YAKuBT4VpxgCsqraDlNY
+wt0/grUK7nicZEo3ErHTZNoqQ3+6MxV/QaahgLTCSRPmiBR5oLpkBEBiBiH+LpwnMbgULXK6xijt
+FYArWkRelOWmAkYTxNwKhM0EFPESXUhvj849RDg/GCpIe8Jb7OmIiKBAJdX1rp5YAAlDpzKTOSkr
+yRH7pJRG01R8r/yxzN+0b2d5mj5hVzEw67mJekSxigAiKStPSl2vdjtAExEOoLrR6R8Q/pgBnMm4
+tEt899S0iSQ2b7it9KVp/wA8ByX2BZ4cdnMezYzsI6Qg5R4XErleKu7UtKFSlR6gpT0x1bMPk0AF
+2ma5AraILtgAejyqQHWmuOUs6tXGf+0OMy/lmKek6YPEyVMXO77q64zKlelIraaUpTHU1roWl3vy
+RQHbHbZ9XL1oZVwCm6inDoJke24ckRWkYbRDTnzqNa86jirTcojFgRPeOTZgqo7XSJZI7RpzpVJS
+71+HFkBR8lH742JkRdR8MApfnXn008sZF+0BnDKsW7jsuyjldRRJVNZVsmAEKQ99C6fFTz0wGV5t
+lJbtYzB/Z8lKxuX36pCqCzYEiV+QaUu67dCrX0xecqdk8S0ewqEXNryjGJSInK5GRJb1eeqdKd/P
+/LFGzX+7ck9SnSmDUg2F1qVhWrq9OvMCGzv/AFxfMpdsWUVAVhFwNo3SakRk2RtSSSoOpDTn0cu/
+xYC7uMxPISPKSQzICEKhamSFhHb1adWlOWvlijdrbeazXkyRmcszzGShU0hLYK1QV7br+fwlTXGI
+NO1DM0FNSycO5aqRMo6JTg3CO5tbnOhCNa89MDgzpPSkYWX2zlrGxqxC2EUQ4YCGpaqFWvO3u5+m
+Ahw6cauaRKImg3JISSIXIiVwDfUqHWvRWpVwSz5KrLxJi9Bom6cEmk5QTtqu5NOnPW2taAkJfXqr
+gN9jbOjbQz81yXXFDg+GEwVSE61oW4X/AD0prgY72U4z3bb7Urt2rl1EI0HU7K00+L1wEN6s4ILn
+bnfWuuG4yIh+tNKaYYVFGwRTO4h8Zjdb+mEGI2EQmA+HovuLnh1u3cOd3hkbhRSJQ7flp8WAmZcj
+XktNNYuLZqPZBc+hvYJbpU56aVrTWnLHRvYR2Px9PZ55ohJh0+cy224aqGKTBAgG60wqVx9/y4uP
+7PuScsw+So6WGNXQlm1sobpw2LfVK21MNdem6teQ4uj2cLLpyDZ2ALy0XGKSjm463KOnJWpiH5UH
+AO5XynFxMPc0ZsY1x7RkH5N0UbRIRIgDWvcGlBxFhk28fmjJrRg/BdbL0Au5etRutVSLpvp611rg
+J+/yJRAkig6e5fb5d2J5cQ0SbOz16OfVUr/Fz5YrQR/7vvc0sZDM5/vZGwiDbL6qP3pN1g5IV+E9
+DtHXAWPJiR25IqJIL5eeuXz1q5QO3gVfx6+nriN2eos5jMTB9Hnt5uj2cgslxN1r65wqNSv0tPp8
+6d2Bzh1IQTKamYTLfARLCJbR00xerW2rLFosSI87CprStfmw48FpFu5WJeLOXsfHtm0Pld41C0UH
+CydK1Cp0+MrvFgG27zJ8S4YxCiK7iHlIBzahfVUo18Neszr/AA9a9xYsuRs8ZZylkuHh84ZffNpm
+jWii+raq9VKVKtp1UGlaFWo0pz1wEcNXjYcxSy8UyZOsrw4wr6OI7ReAuH3255/TpxSs3yfbBD0h
+o9nmJoTdOKR2qptrumtSrSla611rSmlPLu7sBvbt8+cuECEHRFdtoLtJVJUFQt8KiZlUe/Dhk8JI
+k3ca+TbkVrls5ihMufxBtjz7sDAUWlgJyTYyUEi2gUYAok5C3ysOtL8SmTpSNbk2QBD3hEiYuWyo
+GzuHWzpErqcu/AOgUeobVsoiDS4x2leDcJA5G3S2tKcrsDpNNiWTJaPmcyNYlEhuamKytyHVqGoK
+V5h64hw75mu7dMXbbhk0FxUPZfqggudvIk7g78DO2BqtLZXVzAxlTFZpc4bKjJFYI0GtaCsnb318
+sBRcuZuyeLtqnm0GsTNOUuIbzsUAqtXmhW9QV1prXTGzwTN49iUH0bmaONS4C3VDMe8uXh8q4xMH
+2RZmCVhs2g+y88FJO102MTYuSMdR7vXSvfpg/lLsxybGuI6Wgu1ddMVCTWXbk5EgeCP3gn1eHywG
+pTWR1loCWDMkws9QfhYItEbditfQ/FX9cRf2Ymfs/Ji7NS/cbPlUwuAQ6KaacqcrsaQ9WJONFZsC
+bkSEbB+EhqXi/LzxWcnt1ISdXaKAvsu7VErrdsjqVddPxYAFNsXmWc1ul2xmqzfiSiW4ssZpKkVd
+wk9K+HTyxBBFqMZ7PXW3ESElHKQuQsXGni27+odMXDPEgm5BWJFyug4X+ztl0ekxPzK7T4fPHKXb
+xmDPmUJ5rCN8yLv492gL1AXaIkZABeKulfiwHQ0Y6byTIXaDMxjR6W7rgw9+VOVito8tMOpN9twL
+tzY2WuEgbE5cIKpH3ahQi58vLHJeR+2iciZVV7NxsdmGHdl9sakiQFb8JUrTkOnljoLJXadl/MWw
+5aPzTdbu2k248hNmVvLVMw5j+VeWA0GScPlNpBj9pkF+o9t+BJLjTxaUUr4qUx6VYltJLoLL7KA2
+++jRMf5S0p1054CK5wYtJAWhTDFzIF7x0kmsiaRacq2VMxtKnfpgjl99EzqROWhoOxUIrz4MhK6n
+rQa1pWuAlMkxFLcFHY2BJQGye8kQ+pJ0+WvlgrCPm8fxUlIPNtqQ9a6jkiAdPz+L1xmHaB2jR8I0
+XGC2J2SESFBg2MhVZlXoutrS4hrWnOmKKGX8zdoOWkpbP8qom1IRFqzvIWbnq5ipUepuf5jgNjnu
+37s1i+JTGVXeqIEQ/ZESMVSpbyvp0/FjIe2Dt2azbjhMshIpkfu26qjbbAXHwa106ufr04Mfui3h
+FUigoEPbEb769yF75AKjz5U1Byly9dcYZnXPk1JNF3cgzjmUW9ebiDFozAAfLInaRLalcn3c+WA2
+f9kKHY/uo6za+eIO5JdUkxBNYjVEaFpW6lOd1fTF67UO17KOR/7NlpV0TxVDcBq2bGSpfz7nhpXH
+Ks3mqcyFmh0hlSSdRryQa/2ogQBtidedBCytR6acqYpntK5w6cyRunssSorA/JzcSRfNpXkWA1yT
+z52idqvthpDOYHLkftCmux4xJInNLuVOqvj/AJeWMsk4V4KpLiEiusIiiLn70COnKo0UT5VpTFp2
+ZCZOJzFLZk9pLO91RfeZkqDFEen3tQ5ipWlOXLBaCy/ltSVdS2zl1eLuFFmKjxaMNIg5VqI2V6/M
+q1wGWu3jxoyViRld9mv7xVIQtuOnffSvPliG0UIkiTIAVRFJQhSUOwRGo+Kn5Y3HOGR4UocY8Tn4
+VwLVNZq1ethkGdlSOu6LhMq1oPfrdTFNmMhtUuMXbSUc7GPFInIRRmuNhANd7W3/AP5wFbh414Nz
+xSNB2KAisYCAmJXDoI1rT8+dPLD6pNU4f7YCiaaaBWJJ3g2XVqXhPX4x89Pw4GMotQnrxEXOwLIr
+lTLUkr/DQLw1pTXXTDTtq3QBVcVkLSG5AbyIyKvLly0rpgFhMELjdFEy6RtVvK4Sp3fQvpriZmCU
+ZKSRC2hWsOqKRJqVTvExr53p16afp04rpiJK2+IvDh9JqSgJbBm5WXK1JBMCI1dO/XAJSTUduEEU
+kffEIpjthbd389Mat2FZRy7mTtaistzbAHLUUFBXJot0kdOoiMq+n4MGOx/9n/MmavZUtOouo6De
+qkiJDaS6Zc9CINaVGmOlezzJbHsnyk6ThIcHLrj9gVbxVUckQ0ChVrXTb+o4CzTcKtINI6ICSuj3
+MiLlxsrEJi2R60xCvfpcI0xW+0KPWfNYxy9Uatvt6s0u4W16EkBtBHT0rUsW2RB86kpMmtlVl004
+9sSev2bUdVO/TFGmG6a6XCRLw5ZrJFwBLisQBHx6ZarK93jvwAyCasZDK8i3m2a6DMUlJybjhvHd
+MyIkqDr/AA7benFYzXKFsZNicxN4tpLOf7TQnWhiRJNkAqYifxeVnpgwjF5oLMDObjX+1JPZEhbe
++/v0S15UQ/mK27FSexKcsD7NCiKjaJzvIkJ8Mjc5jGrfrM9NenWzq0wBNlNZmkHpOXbM085ZgXT9
+pxdg8OvFJalRdPX0pT/dh/8AeaDcxr6LYsJVCBzfOpezJMj2towEAU7uoLSHzxGSdSGa3sYxkIqR
+TzJmMTRh5RsskkaESjpStdLuV/xD54hqrM4admFN6HdumSow8dBJgXDJAQCBuhGlLjL5vTAOx7VH
+MGeJoUZVf2lJZiQQYKuLwGTatOag1qPir04NS6mR835kmXzp08iF2j0mSjZJZK0STEedPpzwOzA3
+9l5ShYqAePZBjDvCSy/NsrRVKQMap0Rt+USLmVfhw8lm7KGXboOeyvVWdZ125VROy1RzpSplzrr5
+0wF93G+0kmobFoR3CbMkVkFROvKpp2lbSuFw6nCSqQsTi3LghJvxSMqqBl3aCompfzxYYyBFKMSE
+pV8wFAyWbf2qKto+mt3ViUrEpqPVXKDlBRwukIqmQNy3dPOtMBXQUWdyYx6iMoTdNcidCLlurtad
+xDTb18WDBIqOyXeoIqJoubhtIEVUFdS+MaDStP8APHm+XxXMehiKyZWmqLNvfb3+VMTEoFu0t3Gy
+GyXSYjGjYJU8+kfPAYfmPs/GNz7wmXXMim4ciT1WMJmBtbrdOYkXxelMBIpx2hRMgxhk8kwD9R28
+EbhZ2mkHmRj8AjjXs4ZNg52P2FzQTkBVuYP2TZUV0NC8NCGndTASVynnZKPXaKdp0qmiqI/afZR8
+SOheEdBwFszB2qZTyo9Z5Zm8yRzKSFr78bCsEre7kX+WKjm3t4yfHxSrmLmAeqJEO+QtiP6Uspd3
+fXBPLnZTktSMtfM46aklPeOZNbeJUir8+vg/lxXnH7PvZm5kN9BHhritJDiXCSRF52lWuAfjO1SN
+mYwVCB1tkKYkkQCJFryBKnmF5eIsYZ+0tLN5bPqUPlt+g76dl4qif8YyoO3rXwpjrpTSuBna7leW
+7O84Euxm+LR3SJqSLm5XaEdLSrrqWg8sVbs5j3WYsykXsEJFmQkifRtpCZgVA+l2tfPAezQKKb2O
+TEweikgmjJs4zwCYEVKdfdzuwhxl2ej99ZaBXbLXEInxNpoBQdyvPypVMhxenr5EjSj3MlHQTNyS
+aJtmACRiksJDXSvdVShtx10+fEyQRZzcqzhG0VIuyJBJy+VcrEkIk36V6VofxVSEBwGTTcMoi0Xe
+iKI7a4p1ETIlSM6X86/Fbpprg12aZyzNBSDptDTxsE3KRe9WvMUir00KmhUsrz01xq+aHzFskWX3
+LZi9JOHGJJVkYkZGoO4ianyaENA5/NjHcrxu7mhncBshUMbEBDirQp86ddTNO5Pn6YDcOzRNjDO3
+xJooKOhJJRVq/WudLkI0uVZOaeIty/UK40Z7m4W0Yk+bLG/uVJuq+FsIgqVDsseofBS6um5TGTSq
+y0M9YxLZm1i/aCqb8mbtYBSfXnrvtFf/AA31HpxufZ+xh42Q9koLA/lhako8NZEQkVUT53KDXk4p
++LqwFbeymfvaboU8sAnHtvdpNifikbYrdaG1cW9YFr4SxzvnXL+ZI126zM59lcUkqIvEiMLxSUHT
+3iPOlda95jXG+dsuSZ7NbJ0ojmpjCw6g/wDzJhHLkPcCievu1PzxzhP5fy+0Vaw0PmF1IzYobNjo
+w4MdeeqKlOXKvngKjsuLCT3gUtEVCK/15aa151LBXK+XXkzvu7DQh2FvtN+IXi2AuVK1p54smX+y
+uWdsvaksDpNmLVJ+SrSxURaVKmpfnTGjZHy3FyjKMF2TLbRVOOkWTJYox+Il1N11R6dz9cBJ7H8k
+wPBCmq2ay3GkqwIhWVbLktdUm6hBdb1jUNNaYub/ACyyWn2LsmLpszIVHLKPk2YEgnICFRVQUUrT
+uUpdgnGR/wDZls3x0pIKEhGOR6UHzZUC0buK1pp8NupYNSTpxKNVY1u8jicSlrd7Ey5+8QkAG4Kh
+9K24DJ89rReRTdT2yvBOJJLiIl02WNdiuPKizJVK7QfpjOZicarsmMfHsGMOiW6/XVYLbAuWVRvJ
+Dc5jUrqlTmOLX2kSxLzSUAhJOsqx7ISJzHSLO9ik+AeoAOtNOuleWMHcCK6S8goANk1CUJqkmBEF
+1S1qI1+HT64B+VdKIGTRia7SPJIREbLCco18BKUp01LT6YgmoS6pKEdpfD8o6fTyx5K5D3lgESvu
+QBQLum3S4a+tMWfsvyS+z1mVrl2JWQTeK9RG4uEREe8dfWtMBEyFk+Yz1m1jleEBEnTu4h3j0AQp
+3kX5Y7M7GuyNnkuRy8+JmnGzqrddJ6isjRcOXdtH8PPqw32UdmMHkn2dmRFgg7e0kybqul7gNsld
+bSgU/mxrPGf3xBis6TUiVd9QVEamSiNR1tCnf+WAQxW/shjKe4lnTZUhVVZdA866EWnxF4cNyDNw
+LSTJP3CzR1xIiytM1xt8VRr8WI6ScW9YrwSPu0ZZInbBJFEkLRoVKkN3z3Yr2Zc3IoO2eZLHQoxZ
+cPOt2R3Ls9O5Q6U5kl64AxlREVz2FJUHMogahDcZCSorc7lA8unAfPDyJj5VjDMXhsnUoJQ7VBEA
+2hAupVxX8tNNcVHtrzY6guz95M5dkoQnQvCYEq5RsV2lPB1U7ra89ccoXN1pOVLMWZJFzKMmYptV
+WixOTcuCLUhEi15UH5eWA6NzpmrIeW8iTSbSadFwy4xOX0kD3HKAB94oPnpU7tcNuM4QMzl11nRs
+8h2QzTwYdVqssQWsg++UEKF0GQ0xzetGx7kJCbgeEg04ZqgtsOn9CdOVa0pzRqNddfit+HDrLJst
+JBKyTZG5jCtUnr5eVAhItwh6qJ+Ihrry+mA6BdosSyFO5oGeapov0k2WXWvEikUQ1qVaVWurS+6u
+3gV2VZ6iZSS9sNm7V3N5SjnKEYigjti+bWaGssZa2mXixgjd4pPCvJLM3eYZZBdN65cXltC2DkQE
+Ff8ABg2bNnKRM7MNHy45sdr7iELCI3ICy8St9Q5VGg4DpKEgY8v3Zy+pKoKwbRgvmtd8hcKSCxDX
+poQlTpuK+mtPhxKyoz7O3eWmEp2hMWimY5BMnLpRzXQ1KVMqAX5VARxyzJZmzMmq8KWmHTIphJsg
+5jhbVTJWP0upbpTQE6DTw0w52lTMzmzNKkzlaKzA3hTRSSZhUSPoAaDrr+dK4D9ApCWJiyLaYOnq
+hdSQJswC4fUNS0rpiuhmBRR0cWg/fETAvtS/sfrQ6dbS1rS6ldfhwFcIpuZB45TCHdsUepVqUqtc
+JULkaVdeQ/TFdZTDOJZcSQLisuvw7V40myJQdda2qApStMBdWWbCd/aX1jISVJFsKkOZJPPTQh1I
+f8sSm8wzT+1oItd63qFQFmyo6d9eQldSlcU521kBSHL/ALKn2zh2qLm1OVAkHOo8ttTTo/TuwWXW
+nRcKtiRznwrUwHifcmq0ER500s1MK4C6qyDdy0QIuolREhtcpEJF5UG4hLX9MPgoo7aAgu2Xeoiu
+PSVwqj9dNNP64zSkhHrv15Jw+uV2hG+RhCSQIOeg1qNelX8sfcvyDcW68g2coKOLiEUE3izZygNC
+58lCOh0p/wAsBa8xqFDuFU09gXRKiQkW7db+KohWluH1U1CbquU1uGLxEk2kiEVfOpDcHf8ATDTR
+wT5uKC7ldRZyIqCScqBDbT01D/OmDG2m7DYvejcXSSeyuIlTurXppgMK/a6h2st2coTblF9vMVd5
+i6TciokQmPvEz7rK4yPs3i4Fj2eJTLk5h64epL77NtcIbyJXgOunSW2JV1xrn7UBFIJZdyTIOV4l
+u5kUydOiAUklUSOy7l08tNe7GV51ecXxOWxeTD9wgWyDZojsAubfpqXLuubbutcBSMvqAUw2YkjH
+QqTtW0V1A3XHWV4FStNddCT0/wAWLRHqSzTtIdZiXigcyyAlIuuPeCkKoh/eAER18VbqaYp+Wk48
+p15KO3nCNU1U0RQ+/dJDXWoKBbpQbTAf9WFuH3t2VkXyhxyDwRUckq5c3kSwD7wenT7wufy4A/Hv
+Nr2jHtph1JRMgJNBZsg9+KtfeJjQq/CKop0rprgJ2SIqFnJdBjxzZ8kNzVdMBJdJahiFp6102yuO
+hYH5aWJpxX9pOkEbVEySaBcrbQrxKlfl3KBrUbcK7N3gx+ZUnLnikyXVUTMiuFBfUhrVI606uf5+
+K3AbBmBYWPaEu8QRdRqkA63HTUmwqoRBVAQuOl1dxuoVPLw4tv7OrhTM2aprMzaN2ExXJRy2HxNt
+OSazMu+o28iHFEzFmx9GlxUWunuHus2b9yFoKpU6jYOhrrU1OfKuNA/Z49m5bhH0zvPmEeO0JKrg
+QlFLEOpAQ+betcBG/aVzYo2cEhGrINBXSJQ3hdTCVAdKVTrSn8al+KB2aN27Ro6Fszjm0kyQF7w0
+rdYuSB1WBIFCGmlwVHp88Fs+PNJ/jXzYHca7XEn8ZZ0lvXgLhkPwhWtBxZM0Q7okozJqEkg2cEIk
+8GcRvJJ23SHh6XjbbQxoA4CoquHDRw1YyEIu2br7sm6dZfWuAopyNtR56fdEY91MXfs4bt5dornS
+YRa5pRjx9mLk3bbTwUaF9ncc/FpjM4SQKC35RMJSHlFXigsSaBxjYmv/AIlqVK8h07+WNpaM3CLS
+MaOVvaaKTUl20xCmIE8ji+83EqfEkWAkPXQiAuZtZeWRXL2TLNlLUpBsdVSq2W1u59NRxTMwZsWY
+wjopQ/a0optxzmOeo8LIC48Ld6Ned5YBSeYo+Seus3yxxeZYWHEYU3LTdSeKjX+7uqhd7whrXTux
+nmaFn0obGSlJv94czSCXALtnNwPmKqJe75fXAO9pco6bQ6GW1ZibXkl7VJ2MmkRJVs6HWgkB+hCX
+PFFNwomqJCz2G4qipwal2wqQ9J6curu54Ovvbk3uqLzwSUo7AnEik7MRXSNDptur9K8vXEFJvJKR
+6DGQW2BTtKOZufEqKneSf4fPvwEGQInyqTZMzQUVX22bMjHYSAy+f0u79cdr9kmXW/Zdk9dRys64
+qNeJrvBRWFcVxMRpdTlcKfP/AG45O7KomPlM4MfaUachDpltyZKARglr4CrQK07yx2Tm6Sh04JnP
+Ciu7y3LsxinhRAEa/LUU+XoPPAEc2yEXCCUI9mzGPn19yMdJ9RtnNSvp/h53UxVp7tBl2GRoLNAt
+XUhLtnCLBddvaSbsqlQVRVTpXWnTqeAvahFtXfZ6/aSje5jlkmhQuYm95GkFo6b4UrQqVDzLAU8v
+i+zxNRb7NQNnDtdKYi5ODAuFuT6bFh50Cta1wF0zlmRGLOai5J++lYtyz41g4ikfexi1fhM6V6aa
+088AJWJmJdWMc5sc8W4ncuiizmIY6bguEhqr9oSpXQtRIe6pYlzEg0lX2Xczt1nuUnk4t7HeOxQB
+WOkkqULmr8I6lTliM4y2xyFlxBo5eKQE5HyhLDNtDM2LYFz01KhVrZ0aUtwEbMCeYMwZMSzt/Zyj
+p/lhRo8S4Yb1VrqAIop61tP1+XGJw8C+Ty+8mYsGLB5lUuLfSah7rpUlx29nQ7R93p/vxsTLNGYB
+ZDFwkOcopkiT2ym2RgQKokXLdCtPFXvrjPc1SUPlXtNGJ7Tma7tEnjmYdNmH3BGuAcONRpzOmoc+
+eAo80WRYn2EtEwq889YoEU05WWIW6rg/uu6lcMrs4mUCFfPsyS87NS6SqktHNek0ACmoCVSrSnLT
+E6cdZjdrA9zB7Ljsv5hoMsuzRAU/ctjqFltKUK/o8OEPVE5ZaThcrZGWi/az8ZFs+vO9CPoNaV59
+9vO6tcBEn2843gIzMDSUQbp5qQUYKRsZpv7KZU0Egp6+eD2SW2didx9MloxeXRzKkWXyoJ9R7I+9
+VurTp566liI3DKGVo6dpAS9ZPNNJFJpCl4bUbfen+HW7TAIo9mWT3j9bMjptKRcmLCLiU1qmaomV
+yxCdNPMsBbcrzUSxkJ+GY5Y9pSU1Gex2JktxXDOKdBERlbSgkPUOnhwf7LU89SOTmwU7Vo/L6DEi
+Zt2pElStQTrpd1VpWutalT9MZ+oxdJuJvNGR33sGHi3gNEkH61AWBRYbS6dK92terHyQkMt5Wl30
+MyZ5WnWyS1yb18moZqaiNS0rSult12mA7DNN5HtEsuj7bXTQuWXFaHFddtqWoEnURqJjSvpgPmBm
+3lJVBd2jCKN0iTUGRe5eVDfGnKqahCGlOde/BBW5o99qWIOUzuJs5aT1y7ET9aENOnCTmJCLV4Rd
+nMP5J2kVzZvMImk5C74btOfPAAHZQqcg8kpuEgItq2XHhWqm8kSpAFK7iJ/T5aYkwjNP2Oq9YzDX
+7SqNkxHTxJLoalcKawKFTXniwcGsukqJHnMmLa29JZZFVdidPi7/AE7sfFW71eTWkicSiZCIi0X9
+ipCguJDySOgnW7TzrgIgN5InfsYQmBcOUBXcoNJ5IgXG7QySKpa3euJKQvnzhBym5zEMfH7ggus2
+SeGloOm2ppqZDy8WAcemom3XjVNgnjkRJditCGkqzCpFqaZUKtCHl3a4NOPZ6IthjVoQniZdbwQc
+IJK6d4LDbWndgHeMkkDVc8HuLLq9LUoEiSc6fxk6iOtC/PFhb8PxYkojHDtDcuqpFKpXF/NSnI8V
+ZwnEuXvu20cm1SH+6+0lkhJXvGqNahp+uuJDJ0QsiUiXi/FOfdgqUr1JFT/jCYUr+umAxDtozEjJ
+dpEnIJv0EGrRrwy4E2NURSIdFNsa06NB6q/NjK2TrMk+9VHem3LwldlquTmwCWt5VrrW2nuNylLc
+aD2sSikhnV9BSG+yJVUbVeJCwnVtEwUqpTuDp0trTnjNcryTFtmiRTlDAWLv7ETp2ZEbYiL77p77
+aYATGL1bTSbdoxRXBYT3UB0I7CLQx3S7q0s8WJSojHZ0S99FIikKayRJt6rp6UGhpjSg0rW6vdcO
+JKrVEnHtBi/aqMRdWr3e6bX07+impbZ0AefrgZm4kyV4hoZizVMVG6QtiSFIi6zS1r5BrywHm6hI
+OF10DkXqKqRCBCj0Kj3q9/OlKV54GR6zy+1s5feLcEEzLxVIaUrWtPX1xYnyLcR4lRm63thJFwo5
+eD0u1OtRYaU52EFD8sV+0mL0hQcmg4Iegkwt7++mtfywGsnAySm++Js6fouUFU3gSqxIARgiPQI1
+57g/Cf642Psqdey4p+io8azDN2km2YvnVojIDZpwLsK/dq0+b4sc3qziLmPXkBB9LS3UJryPS3sF
+uFKGNNdKKhp3fFTGrZKzV+7qpMc5Ngko2WYIO3XAWkC6VdKcYGuhbwVr5DgLRIMY9lmhq8LLAPYt
+sRsF2r1baeR5kI6CN3eI1rqGBWcPbhOUsly0q19sTqqbB+3mg90SVOTVwmqPiIvPFuSnC9poCJx2
+bCbMFVHL7wjJxVSGlNSry309e/GaZ9nEcr8Y2Xee2m67UWiDCaO41WKhVNFVJYdbKpVPTATGUG6T
+zQ1IY11khNgqJbROQOOKVQHw6a9AKjQufnhrMEx7GhUF20bw00/JWRy2/wAvgRilcVjhsaXy+fpi
+hOMzoRPDskMxyUS8oahzKVnFIKukepuv1acjrQaVwGLNUtLO5HNrlm+bzSio+y3EZ0oIOP4nR+Mf
+TAJza42XCEWo8ayLWLX3OPadLwt3romXrtlX9K4qzh06dqqvnyxvXSokQqEt1pFTuxMbuikm6SCs
+Ugu8VdKkR+A1yW5ac+XIu6uGFkXDSNFpwaI75itvuQtVuDpMB58g19cARkBg+I/styuqzMUCQ40N
+pVU6iV4606dKV88QpBRQjHccm5ERtDr3REaF4Rr8uLY3kvZKAIRbUBZlHJXk6bA5tCp13FKlSvRT
+ny88V0eFbTcj7Jcg/R3VU25KI2gqlXuKnp9MBpX7N6hITGZE05hdg4SYC9bG2WEhVNEqHQSGvjHl
+jeLJRGCmBWkv3cTlzTmI542MQbLq23KN6VpzDWzz/FjmHsceN4ftCh5CQeMWylyZMzco7qCtxUEk
+ldK9PKv+rG/59mE4KTfISBuk0Y+fQdoZbegRpPEVAqFKNFadPVUi6a4A1O5kWzQ4gnbZFTKo5mYq
+tnKoo8S1eOiDRMDMdR0qXLXxYEQgyWX5PIexsZRlnoqxM1J7KRtnJgNSpSpV+OpiPixAcC4HImZk
+xZh+7sPmJJwOXn5iLpijS2qg6ULw693PDqrhQssZ1yc0bIv2MS+SlIyJkUSSdEjUqKKbR879K8qY
+CBmuPfQ3ZnP5eeNnXtKDmE13oogarGZGp3cw/g9+Ck6pmb94HMIKzXKMHmyEJZIXv25qurbzAj51
+R0HFhZTyjTtA/d/Ksr7DFeCJ4ULNh7pVxXTp3C5Dp81LsDoThUHGSJIkTym8FVdF+T1EVY5yag21
+06rdCLuPXAAIx06j8ykxUikMoxecWqTYXzY0nMe5cIh7ylacxompSleunVgZnvslmJvMTV3lvY3H
+48RIvCMV0A2PuxbnX1592Dcgj+6mZZFynMBDx6UiJTCorA8jBEuVABCuhBXnz0xVu2ftOfZOWLL+
+T34Nm7Yk1EnEc/FdmSRcxGg94FzU19MBTc0Ry6E4zRztFryjuj1N4s4aHurizMddqiQdA3V/33YI
+TD6ed5Ris7DmAIKJeoFltmyaGRuSYpczI/P4fLniY3F1m+elY3LL+Oyr7QgN6VQTWE2zkadd4HWu
+t3Pn8pYocetAxuWpbjo2RZZiUJsUOQnupII/xSqfdrXAWTPcejKGxzBknJgQuW5BgMcz9yO+5JMr
+lnFNPBWlKeIsR0pDJLGZnmuQ8vTWYTfQ4NIp4sFSUFyQ6qr2fMOvl54EKzTeSaRkNJP5TM4x5KIx
+0S0MgSED51PcpS4i9enGqZSedpTRxlYWmQzjW8ag5btn1gG5JqQ8ysu8XPxYCi5Fjch/vBIscySr
+rMyyrBMWrBgiQC5kFBtoHT4qhr48VrMyT5F2gxcFGRq7JCjdVAUTcHQhIvvCpStKHpWlK08tMaOB
+M8s/u9JZL9lu57LMiqnJuXrbbN46cFYnTbrWlS8XfTw4nNx7JYWUmG3bQo6d5xVkFF3SzQ67ZCYj
+UdLa6cudP0wGuKqChNryintHeepE3bLrZeAmznpuHWzTQtKaYVKy0a0AicsIddRTbI2ZRSqTlnr0
+kQaF5fTEFWPdOXqUbBAaiLYSKTim2aritoNdCT1PlWpYfaFMCDWQcts4D0kjFPhnkVdoq/w1dD0L
+n5lgGnsxEoSrWEYv4FZqVpFKbzpK4qcqAp161/zxKCeatlXke0Wjk49ICJdqnNmW6NedNute7XHy
+MTzZDuV2BLZi42UAVXDNyi3XQVqPItsufV8VuPswM9Ht9tocw/jUVRJUnOXgI2fT4k+nw09MBLSz
+ADY2uYCRevUySFNur7bAjZiWtLVKWeHWmFqyE4g7SiSZzacw7SJwaCazdVBcPirrZyLTuwFcOhey
+ass59iEi2ajtG9y2qCDnUtLS6PFr6YFm4btAdSC+W8tMnDkdvg1GbgVxO7TcR1HX/LAXBxCyD7g4
+ti2zEUS2IiXFQG97Y6lr0UIK6j64nyrgSMpbhnwuCLgmrrg0bS/CtSo8qcsZzIJsWyQxZOYTe3QT
+VmE+ItEhLWgLcvp59+CMgMWM3aoGT2BIKpkK5IuCZvLh8VSoNl1f88BVO2XsXzBMoFmiFZqLTF32
+xi/NIAIBLWlUraUvprz54wfMEwjJH7Pkoq2aL7O5VTtARVDpT0HT079fFjrmKeRqkZIoKMMrSQq3
+DwDdsqqrf57dbeY6YfZQMfKSaCacCxU4JL3qrSBEDEq+FMtwacsBwsKaiZrjvdKfjHwj4vT9MWmM
+i56ZS4ZSBkZLc8KiIGZIFUtaFprbppjrbL/ZjklebSevssQ6klvq8UKiwqhcXKmqSetpaeXdjbMu
+Q8XCMhj4Zgg0appDdw1oiRUHTupgPzwkMn5mUh0F2mT3TZ0kuqJqkiRXCXLnUq1w7lTshzxmRVqK
+cDIpt1PdkuQFakXrrXH6ImLMVeqwiIbSut8PrzwFdvBbKkV/DJ/GIgj1D5FgPz87QMt5iydmBLLu
+YpUEiFcXAfZvddIUDd5/y24C5XfQLFwTl88kRkGzW6OVQ6gQdAXKlde9M8dS/tYKZTd5KJOWcoFM
+L/3EhBIlyOncPRztr544/epuG32ZREB96XWPVu6eh/FpgLFIZ0mJaH9kqbDZvvquQBqsSQiR2XjX
+8FbfDiLOzDPi0PZvFLs21nDIPwuFKlQpen81vlTnit/BcOJyRLPnou11kFFE+ohWP4RHkPVy+mmA
+eScLGyXUbImInci/dKdfiOlac64fVlH14rtlgYNytUBJsZWoaDZXQfIipiKrIJ7Qi0RNp0+8AViI
+FSEumtRr6YigoRKkSl5EoJdXxXYAjKyTqQcM0yWNZuxSFBsSiIgQpUK6mttKeuHQFaQcWoXuXjlV
+NMWwgV6pl3cyrXTA5kLh2rsJ76rhQSTSAQvIit8OLXlqHFSMeO5JHaTZLtnLqTJYd1BIh+6Ee+4v
+pgBSRNWaRfaXrZbhyFdLZ92q4A6e50+WnmXdh+6UzA3eSSbACFV0ncCIW9Zcuke7EVuIyQLlGooE
+4SarkrvWiWzQx0PWvTfzwYbtYeNy+845hNoShPGiiSpBYLZvaNaqV0+f4cAFJ83hHsY+i0VxfNhL
+juJREhJW7mOleVeWOqnqjfMTeTbNv7UazWVknqCEi5sEVUb/AO6lXuqNfixyS4TubpOxeAv94RId
+RbAVLTXn82N67GnEpmjskKClIr2s3glSdteJbFdwVeSopqVppWlO/TAFuzpixksuvGwt32cBzJl0
+Xj5LwPkHCI2VoB/xPB4a9+CeXJaNzJnDJ6c68DNyhZbUbjwVyD9odCGhjUKF1HSnljzSNFePkSyL
+MLuYPLLpB7HKkCoyKDUwAlE076aKDXXSuuD8bkWPXjJqZerLryEbKJSscbREGb4gr4xPXTTv5lTA
+AuzprOSErCy04f72Novi4VzDPURSfM0SrcJqXa1ryHywWzbnDsvjcqHE+23T+BQfisvlh3aJilQu
+Yp6jdaJfDriozXadNPms6xhMsLy0jv8AEnJulhB4KNO/bpTn+WmMwtZ5kz3HfY805jTlECHbfvwS
+dEdvIQOlfDb83iwE/tDksvvpqRyyhnBCRy6RcawkWsbU1xVLvTUoNaXUEddSrgBk94WW2SuYIRtx
+Lxirw66i8UK7HaLXqLXwF3Yiw7GacxMq+bMFxdQlopq8eKSrEKd47d1x0t86YPum76GyvGJMniDR
+bMKW88eJzBKtnwU1rtLBr7uuAhw8HMSVkWTORaSUy+J22YiwERVRMfeKJqU500+X0xbofsb7VM1p
+IZfs4CFZJFsG/tAUgvpqNdOZFrjeuwfIMDl/KrHOM62XXkE0AdM1yuV2hMeQI0r+HlSmLa4zMzUA
+UYmEdPZK3ebwSYWigZeFRwfhDXvrrgAHZ12R9n/ZvDoZqYm6UkBSFMnjm0jVKvIhANOm4saek3ap
+gMo5A2zzhR+MbmKVvhDlypjOWWallXq6Aj+9WZWyopvHg+7i4oq940KvLpp3+uCj3MzqfdIM8uma
+yKCoDIyyaIiTmtC+5R9da95YCH2gdmMHnhJdyoi1bSnGEXEpo3CIiWtDPTQqVqGONu1ns+Xgc8vm
+yKEo9ZrVos0Xq1JSppV5Urrr60L9NMfoI3XEW679K9Nsklc+e7Ni65o/h+LliUlFNn1KvFxRMV9F
+ERJtzBOtOQ9351/XAc4KvGKEUggo/hFuLfbiqS0Csg6Yifg8OvL6YNRieX3b5JoLbJNW6DYhVETc
+bTvSvK2tR92VOeuJTdOYFIZltmSY2XLoRjnnEt3JFdztPVPX9a4RBNZxoy9ksUZsX3FEtJoOY1uY
+9WupI9HdXANmOWXbRV65hMrOWaH3V0qsKrYu4j8OttPpgg3j30Tlwtixyo7K0XyeYbiEO8KKVqGv
+08OJUhHvHMm2T490KLZIrn37tpCCRDzqKnT50xOVUkrFXYo2qCltpAnAgKDz0KuvxeWArIfvI0ky
+hkwP2SkhvOmaOYQVVV1+JO4KD/XA5xIE+SdTMstnMYOPIeFdFJN+JbFQdK0qFNbqa4LN2cgLImy4
+bjp+JEbN3l4RXQC6nhIK0rbTHnELJC4axK4MXe0qKhOW0CYqCdC1DcGp6FSlMBHis1QLFw+kpLMM
+3x0jtJtSIESQc/L0UrprX64ek3m1lr2WutmVRR2vuJMHINSP/wC2d+mnpiTKotUHCssu24Zumqm2
+SQTYNwbKn4daiYkQ1wYgOzdZ26Xkm0ko33xK1s7Z3igRd9lfl/LAAIwVE3DVsL/MUkiySIiSUNFq
+5E7uQ7lCrfph2PGWeZt4ko18om/Q2Wy8m8V3UtPhrYnbbjSIrJsShFey5BsDsU7bjURtK6n4/FSn
+pixshUTZC3HpTQLpAjIitwETLUe4Yx+w5Bim6ErjVZdN3prrTXBVuoV/u/eD8REf/wCMeNNbiPdg
+FvxdHw4C59zdA5Jy/wC1Jl+1ZCRbaQqHbcfpywBN3am3VUc2J9f8Y7REfm/7Ywft77dMt5UinkFl
+8wfzSokmBtDEhQP8Rf8AbGNdrfbpmbPCLlWCWOJjQZi2ftVFh96VVR0IK1pz1+mMphGbds3XknzA
+1xds1eBBPq97Qg6i86efVgDUw3fOeGls1LBLPljbEKDsyBVdIiurRLXw059RYgoFFCs8vP2SzVB8
+idzbfa601NNFLnrdrTv9cWY0XE2ynXb6eZSyiUSzJAVkS3VxHuRR8wGnd+LA9pl+YUzR7ETZrxb5
+R4V4uzE2LNJYCpXWtaeLngIGc4FjHhwblgEXLHwzlO1YlEEm5BXWqqlKeKpUp5YoppkPUQeLqEv8
+WmNVPJok0SUdrOmzM4xTbXJapjLum6vhEfIba4gTuVXQtBdiwXF4kgD8WyZiSAtFOZWU8Vw93fgK
+GbNZOMGQJseyuqSaCvTaWneNfQqYYBMt3bs6h+H5S+uDRpyGW3aqCgfZVCFbach0rh5KDTEOQ4N2
+quszPgk9q4m6nivqXhCtO/ASsqPvZc2lLKIggmJKJgQhdadvlz5fzYdm5BZ8bVGbBASFq2TA23SC
+QXU6lPmU078QFfajaQJfxOEEvEICQiNAp5d3Lzw1vfZ9hC9sQkmRJEF9x0Hxa+VPpgLVGuIeHNmU
+lCBmGPTSckyTTebZKGVvvlNBrWg0qPIcCfbkwpHrrqTy6lzxJbaU6iVOgjShV5fBgOCniU+7U6iE
+hDqIq/DyxaOyrIcpnrMCUXG/ZkSuFd4QXiloPdpy54B3svyXIZ6zGQqAunHkW4+eJtrtrq1rp9cd
+rNI2DhMvw4sWz1BNgQpgkiz2isqNtSPnXWldcTsg5Dj8r5cSho+NBNNMbVSEFR3T8y8Xng29TTE/
+cHcNvSF5kQjXl1dXdgMj7bZLL+VJOMkk4tkLiLFRsmXH7S6iRDckfSNaaUKuMbb5+zJn2Pmoly2a
+yxKx1zM1HgoOUjEqU8ddKH/LjSO3rs0eyT2MkYiHZKVaIW+8BUkypdeSapVPwVuxhKrMouP2pRaO
+FGJV4lJiszK1cy5ElRelaVLTXlgI2SVGse6Z5km4prMRLBXhnzEnmw6VMrhpTlXqHXDsJDqzaM62
+iYqIYrR6nugUf7D5OlSqVoqV5KaeHy6cSwJvCNEkF4Fk/LNDW09yNVA4y4unZO7qLl54Hyb7jp2H
+uABbwtqLp8jFWroJUOoe+GuoqcvpgLHOw8lG5fytmhRbKzlvbsk+bPLzVEipW50jTqP5K4v/AGP9
+n/HTbXPso5jkItgqSjX2UwMhkyPlUNs/CI6emKbkLIcfnTOEwSh7EHHvE1imGzYWYiZF/wAM6F5d
+w46Imm5Rskll85J6yjbLQbNgEpGXoI/wRGlNoK69ReeAUWYM0T+bXgxsWCbOFMUWJJrDwLEqhS41
+y7jUp5CPSPrhcwxfPY8WSaMiMKo6tLhFhSfTzmveVS+BCmGcsCDU0GkkzNZ6venTLrJaxlGpVKtb
+3KnmVvfdj7K5iTkJ1WSQcGhGtEuCKWEOi6v/AIdgn/Er6n5YATIZZnHMOllltEx1rZUVCjG7kgjG
+ZXV63a3iWOnyYvWS4OLy62XzNMZgB8T8dk3BaoJICHKiLVH6lit8Uqmm1QUy86KTL3kTlNNb3hF5
+Onp/154cjmLpzmBjISrxjPvodUl5KRdnawjTr/CQTpoJGNPPAXqBRmpucOdlHBobqRDEw5XACaWv
+NZTlzMv6YsbOOQvcGUu/MjWqRUFwIiFdKUtpT05YqXtQZlkC3tVeJy+ouHDPCP7U+K7XopXwp17u
+7FoZLSJUV32LFjopWgJ1UEiqOlNCKvrXAY81hyeSrqUdsItYRLbBq4gSFquN3QVwjh2dFq0ZE5Jm
+xJ4oW2ZDGuAXZiXnrQLraaU0xbmmT5BtFJM1odld1EXCPzQAi8iIKiWnP64aOBzQ2VSubSKjdsO5
+sJzAkV1O4R1T6+/zwFKjLVEmKe9DruFCIVVUXjhAlypypuh82nzYOyDxi2aIRbT92kEdpNZRs5kj
+FIhoVKdOnT3+uDUfH5gUk0lXLbMSYgO8ZXo+KvwkPnh1JrKIKkmUVNrtV7uhZsir3/W6lmArLJrF
+rSZPiZwim1btF7buXEq/8Ot3hwVBu3eq8W7YRyjpPcTQBY1lXP16qUrdyxKDLc4+ZcIoEi0R/FtJ
+GI/Qqa4Iw+VyX2l5s3xOGCv2beciqRB566UHvwEDK+V49y3SUUbAmIqkQkLDaMht0tKhUofd54vk
+ZFsY1kLZszaotx8ACFo3fNhYEJOBuMC2/hs8P9cSXCyKDcl3KwIopjcRKdIiP64BsGqI22+7EfDb
+04CZwzhlfKrdJXMkq1ZCors3KdQ3/Wvl+uMP7bf2gCi3CENkB+1XcLlauvZeaWpWXJ07j7vXHOeb
+fbE7Ie0p2YMnkhxKxGpcW6QiVBCiX8MqeHAbr2n/ALSBOUfZ+Q2N9FW5bij3pMeu2m2P8TX6YwqT
+zVOPpvjp+SueJCuoJubrBKgWW0T8QFyx5WLko8yTFyDSUTdNNpmXU8G0CrQkuWnnzw1KyCbF2l7Q
+bNXMhIEqLxUjuVLUq6mY91K4CrsmrVeSSduXMdw5Ok0yS6hEurxfgH1wfgWbVs8QbvtlNFzxjUhY
+LbpuR6SEbKfw/rg7GZJzZmtJ5+6jNr7PX94e5YBkKfeI869NK4fylBvhmGcpEgBKKLk0YbgAIKnQ
+feCfV0/T5sAdaTUe5jF49oaDuWd5fSZXqAIixMfgS+ZSvzYGLTzt8/22MPwUcLhBw3YqBfVy+Qro
+e8VfWl3PBplkFu2cbjQAbJyjwmvtXZtOKc+SNKXcta4s2X+zNGQSJzKNuGar7kU6atnJBwL2pa0X
+rqOtplTAQ4x9JL5rGZTZxyEwqkT+FZtFr2ot6jY5Tsryv88QTZtWjRr7lrJCwLi4VsSI3yqJnWpp
+qUDyArun0xO/cPgpj7cC6cpvpsnD5Y7eDdpiRUUpp8JjbTXEbKgxvtNrLQz/AG5Rd4rtIEHTGSNC
+qNe+v3a1KcqeHAVHtgyaOXYJi7UeLu30gkLtjt3ihwneTbq77O//AA4yllBvnaW8j8TVR2l0F1BQ
+qUqNMbj2zuCmYVrleJWUXYOTN2wF17so94A+/QEvTv6cUd7CxKcYl7LeLprPYziYzbPoScCWi6P+
+L0wFGjykHYKimG41ZDvKpdNgjbpdZXl5YgKrEoBXABLKdRFf4vp6UwYZOIlSFdIqImMom6FZILOg
+kf4iZUwHIRILrwtUIrh+L9aeWA1LsJ7I1s9SyTmQ308vp/emmsKSpfldWmOx4Ls/g8utEm0TDhti
+kmIri2b320HlWpeeAH7N7MlOw+AklLxJVAi6Tuu0LTutxp+yRJCShmoXhH4em3lgBMeooQKioYEp
+4blASErfXDCTVuglamG2REVpfZ+r/diUyRZiqruHt7YkJeL/ANuFAioNxJublPh6y8/8GAGO2qKa
+SqCh9KgkJCoCJCQenf3YBPct5ZdtxbO42LdtytIgUbN7breVdK4siorJ3Fx7obRtG4yL/wDp4fbv
+FkwFteBCI9RqGV3h/wDp4DDO1XsxlJSHVGGzCC8agO41jHLkEmzMR8k6J186+WMuyZ2eyUpMR0yM
+r7YRkFfZ0mLB+YvGPwkoY16yHHWM7KLR7QXMe2XfqFamkzZHcapV/nClMZTKs5RDMq8s7W/dxw7X
+QEkhfibldKpaWLogGlOX4sBcm7f3oxOXZVDjGyWzNZtdoiRFp3UD4VFfrTw4nTEe3jWDh+pmF1E+
+0rWwSNm/KPq/8NOlfAPpbgTCzAk+blMIoZjzImkS0XARCJA3Zh3UJQq0pS6vzFiaxlHrycSJq2az
+mbDoXGPCPWOgk6d9KFWmhFT6YAItDrNpZNqUPxpF/cctIreI7uTqQV7v01w7QZAsxKsvaUc9lIdC
+59IpgKUZlwa+KidK+NbT/wDOLTl5ipJUk0YSXdA1clbJz3/iXytOVjf0GndriBG5Ti0zGCXhduBa
+WrNYfeI3L5xXvXdf/nAU9FwLZBzKISL5CHfq7DP45bMqtfkrXrTRxNeJOXqjPKqjBjIzTfbXVjiO
+nszLqPfQ3H/FV+nfi0N2b5jJlEi/QTneFNxKTBJ0MIxvTkCSNO4a6d2IMSxGG9mNIlguUK5dXJC4
+6nU47IruIWLv2x1uwEl0+i4J6rmTMDx09sVSQZyaiP36tf8Aw7VtT17rtMfMy9oDJtKmMtnqDyg5
+KlD9mORE1gGvcSnylXzp5aYmzb5rmJ4q1i6pruI8th5NlaLWP1p7wUa15bn1piBl/KkO+jRdQOSc
+vSTAyLbfTRVq5d1pXSqta176VrrpX6YDXwEbLS+XxEeEq2l1D1KeEcO3dFw//wA2Gj3PFYHi/FgE
+AJCqQkd13gIrbhH/ACwkFFB3RK8tzq8Y4dVJbduTC78N/ThwfD1B4huL+bANgO4A3I2/zW4UaY+G
+z+Yit/7Y9cVnTjztwLRkq5UPpSSJQrvoOAizchGwUUvLyCiCCKAXEfTdbT4afnjj/tY7UMydoO+c
+a/OHy+oquixFYOle0R1oWnxVu8OC3aln5x2oybZoPHQ+X2zxsnvj1GkqZa0qQ08X0xT27OUQyuxi
+WiKAtXa7695s3hICJB0080y5d+AqPCx8M9SaLsF01kFWZElfc6Z6JAdTTpT4da64am5xRtGcCpwI
+iuKqgPy6l1xu1tU+QvTBLMqkXDZoEXMa+iU1CbOQVW63iWgD90pX4fTngc4h2Mw9JVOeais53CJy
+7uA0FS50FXTzL8WAcAph89Zk0Dgk3rpJxGLufeurwHnRNWml3f4cWmPa5dTzw6cyFijctwvaKkad
+yUhXuQWCtbaUr+WJmR41wnl9BWSilFY1svsNUhf+8inVPCuA1r93XXvxdY/Lco7jJYXz90TpchRz
+OzIAO5Kpe6dpV/64Co5HiWbmMllF544laQdJovB2SD2eY/dq8u5Mq4ubjLcWo0XdqMGrIk9uOlTT
+/wDDOqc0nQ0p30LzLFyy4izjY9djIPPanBCKD5WwbnzQ/u1/xKD54aOLdNJsmy7YHLdJAWD4iMbX
+LI+aS9KfgrgIcOzbpwjxSZRNgL0uAndsyIUFqfcvE9ddNaW1rXFgaLSTmPMpY2q6g2x06SICJX0/
+u7sdfTENoxR4dWPmQXUKNH2TLGJ3AqxMdUXP6a484fC0VSaO1gtu9kzu4HXtGOjdz+etvPAQ8xrM
+ZBwqpOgumJF7JlgLq2lbfszmmnrz54ocnDzCfHbiLGJREW0C/LhupdWhe6c0L4aW1HqwYm5R02zG
++bSxtSWSIYN+YnYJBUa1QdemtO7ATOE5INGW5KOd/wCx+xZYSO4L6fcrjX0+uAxjPDhOUzGSD43X
+tbqTctm11qr6hWgVfS7D+bszNV4J+LbLy8W1XVTUY/CTOQSGgrf6vp+uEvk55zJ8FLP2UWm9dcI6
+kekPfD1pl6iNbcCc4IqJxQu5A13pOVRJR4oY3cSF1DEtPmGod+ATldrLIO2slHooPbhVeiFlxKkA
++8E9fyuwCklouxsMWC+4Ilvmp/F1LlXT1xZmmXXybRBzEzCDYlIdV6QLdKgiJFRZL866d2KbduJJ
+IJtrrbi6QuIvzwHf37NXDl2KZbubOl0xQIkzTAwHxV/F/wAsaM4TFc/44ppj4CA7i05992M1/ZtE
+WnYrllMg3CTQ8Sm0J21IuWla3UxfnCJO3Y7DY7R6bx2THn69WAUrxgpWkBlb4rWxl8XL4v8APEF2
+T5NkXCNjXIS6REFfF9OrE5JqimoTUQREfAoRbOhV8vPHm7cW1qCbbcT8IkIIj1V9OrAAOOmuHLiU
+V7i6rU0Vv/fgnxT7h0hFgvcVolcBj3fWpYHvWMb1DwwJl1e9JFv0+XLUsUrtdz84yhkfcI0BcKCK
+LYljbkQiRcyENdTpQeelMAMzXOReZMxlH/urMOU0FdkZVOSVj0t2pa7Y9XUVbcXlkms5fICKOWoA
+nKRNyal9seKlbpT3tPTGLdh80nmIlynQi5SS4pQhKefkG6rSz3gt+9Pp+mN/yoo1afZoRyyeorqq
+LPHTQBJBmVB16Cry78AJyu1a5SW9jqNuARcpWqmosSr+QK3S0R8VBp64hybOSXSVjVofeRXG5llh
+qtRsCAUr986Wpzur8uBaVUWs8+kMpyXFunStkjmd/XiFbA6qos0ud3d4R6cSHD54WWnSctFO2jV8
+vti2Jzuy0qVfD3fdAX+2mAJtX0+vGkjl32VIyLRUkwep+4jIilRoJCH/ABa0wHZS0fCZYeKx8w+Q
+Ypq2yeaX4Fvyat2m22pXx615Ut6Rw5KQryPjmLaWigfrIICMblSIMkmaF3eTlSug10/F+mBqKcxO
+5iZu2z+HIou4eJRbVKOgQoNfubh0WXr9e7ASlt5y4atHeXl4mDVtcDE8SRScu6/h73O4E6aa1uw5
+MOHjl5JySz+2XaIbLldH+4wLb+IIF3EqQ4XBQ8e2buZBT2w5GSVJFWWX6X0qsXcknp90l04OSbXL
+cW1jovMjsItND7WhCMlqlugFNffW+Pn54CgS66KjWIqpFGqyUSJPLGUbDSVXr/8AOOufSHxVu8se
+aZeknqW+5j5rOrita0VkWL7hWol5ooAI6bYd1K+ddcWdim4nMyqi5jVGj2SSFebfkdvBtO5JkHnQ
+i9MWFyhKt1OFr2iMcqgjTbTjWzZJWiYU8NSIud9ad+A0S7o6f92PGJCHT4vhwsOrqLHumy7ANXKX
+9QBh0LvF1/y4909JeL8OIsrJRsM04uUfoNG9wp7qx2jfXuHAPqrN2jddd2tsopBcqqXTaP1xyd28
+dpUhmuaKPgni/sVk6UT2mi1irnQBPeEudCHn4cQO3DtWzFnGUk8ops30XBpAuK7ZoFzxWqdn31PJ
+Hq15YpeT/wB3Y2PZsXKzV6m0dXLoIn7q1RELTFbv5VwEuM9sJpKyCCyHGNl4/hpEfumwW8hcBX4K
+V7ywluxnotk6Xj3JucxNl11pMr72pN1NOpGmJ7eDnJ1psSDZAnTlqm2iXLBawSNDuSXoWtD105XY
+tsJAymWYLLPDHvsydKEPEgIL8R8bRTX+GXlgK2yiZJ20jkHkU6kpJsKai43iYLtO8FU/xjixR+X1
+pR2qWyu/WU95MJEzFMXKNC+9T/8ANDzpg7lSBFtIIPmKK8a1dvCWauiMrox3TxM1PLbKvKmLarvL
+yCD4nPsclXRJpJEdvBvh7hrp/CUwAVWDZsTXknwb7xghbZ8ErH106qUp8YYkg6WJug7iVkFFmCW4
+1VLxScZX7xOvzEHP8sRjavH1qaiJoOifEo1uMvsb2njQ5fw1R7sCXrV05kEJCEZrobW64Ztr+lm4
+Aq8S109DpgDsS4KNkEnLFmo7YtPtbE7x+0x6w+8RL+StcO5iUaxMggu7M3Ps9Ir/AISXjl//AO2W
+AWYpD2EkzGL3yaikMtGAPgVRPku3H8NNcGDnE/Zi6ikbxIxoj0F1krHr6a86/KWAlQ7cRaf2g5BR
+u0ujpW7+OxU5orV/LXA7OdwskmLta0Ykhjp0B61V2Rfdq0L6d+HTT9lhdIOTKNQSJg8EQuEminNB
+YvPp1wEe5mFNk1kE4037xJJSDnUr7Ssr0pql6/8AtwFIzatGzOVxfKIrk+hyKOmBUMhNVqoXuF66
+U6qU08WKlmJ019scDNyQRaIoJxMml4iVtG5F1X0pzxa3ZFDO9vMD9BAd32LLeEt1oQ3pkNKd9Brj
+KMyyCJSAtBhEHakfuMF3ThYrXIGdaIKV9NBt0wDEISMlNi0bLHLOH6oo77joFBa+m2pTv1qWJXaL
+Le1M0JFGw4JyAkIrtkw6Rdj0qDSndXURphrL7N5JSr4ZQ+CZiPBKumwWg2dAOqJV09a0w12XrOCz
+AkxXRBckiVepHeQqbolS8qV118NC/PAEcwQaL4J9SEeAm3YIJvT3ukiKoU32wV9Br5UxRW7wmwKq
+JommRNdki3v0qXd3YueblmMpDz60XvoRaboHMWneJFtLlot+XPXFAMiFIrunpK7owH6F9ggvG3Y/
+lklzBRRRn4yW+G6v4MXlkRNktxy52/is3iMbv9FMVbs0ZjE9lWW2xAuntsExIRMrefPnp+eLYkQp
+pCK5r2l1ERXXf564BSq3UShIoEPwnvEJFr/gww7cLJnsCFpEXwrFcI/6MRXr5HdVJB509IhcZdJe
+fnhbclF26qm8Cl3hK9W39a0LuwENInBXFfcJDcFrkrrfyqljC/2k5hqpJsYld41TcJtTUFV6iSrY
+RqFeWlBoVFKeWN7abiCVxH74riG01vDXy8WOeP2hXzdt2ipLprXOkWe4OyiRkzGvKp1E61osPPQw
+8sBmGTM0I5fVffukaDZm5QEXLmTbCb4deSotT1tP6a2+LG65dTcZhyoxi4394l0XbUU2sY90bhaJ
+dSq6g08Gn+rHO7Ru1ThxdxcJxqLR0oL6Vd//AA/r+Krf+F5Y6c7AlE22V44bZV28XIRbILPN3fSp
+zKz0RGv9MBdDkIlhJpREAim9ogCiiT0QAWcMNB0rqp5lX6YGKz2V4dAW8XIKXL+/eTJBU1V/LRHl
+1FX6eHE+byOnKNI6LFxttWC6qx8MG0xsv02lBHxFQcGsj5TiY9v7SFb2kS9ynHOQEdoe6gop06QH
+8sAHcJvJ8UOJbLtotIRIIdM/tj7XkNXBfAniZDwck9Fq+mWbVJREi4CHa/3ZidvKq1afeH+mLO4T
+Yxrfi0TQZboihxxdRkVS0Eda864oWcpKU4h4CbNeFy+SpdTc7X004qPgQD4KfiwBGWnFI9RVPLyL
+aazOqkImO9QWbOo69VS0rQKU154pq3s32euUlnY2CJXDIyzQPtMg5IetJvqNS2h+lMQMvwLpiqMM
+5hDkZd2qLn9245ztM4xLnYTtenUodfOhV6sX7LUGaL5WZfBHyuYU1RaJJNAHh4YflH09a4DNQWYx
+7eJkI2NmxFddRbL8E5O59Juy8Ltf0SHy1xbzn5ZgXCQ/Zw6zTt/3ySMtdxzXmoNK6c6DWumv0xKk
+6lAZxUXYM/3gnHSFs2/UPrQC3QG6FPgIq/DTy6sD3uXidEkpMzWZYtxt0oLGDoVW7UNa1onWuvUf
+PWtfrTAbKAiPhDHjtEPBh0cJ54BCRDZ8v+3HIn7X+eJJ9m9vHwj/AG42NQVRXVbnRcCcW3+AdbCH
+TlWuOoe0CY/d/IsxLCdqiDVTbH5jqPTSlPPH55cGIq8c7cuiuJDiVWx2GruJFeFKV1offpdgFhNL
+RJiS5rpPFCUHaRW+03LAHXv/ABjy501xfsiZX4FoSko23EyY2yO2Hul0ajaRp/8AmI6eKmKj2Pps
+SzR7ScogvHtEhTcoKASpIJEZUorr5EJbeNGOQcZZnXjknJi4UdEmLYgtSSdW6VoeuugLBbX/ABYC
+5wWWYdQxfSD/AH3iAppya49IkNf7q/T07q911aYuQRayce+kJ1mu9UVIUZZIfB/5b5Gnd8uumK63
+mm8f7H9nw7XhVWpC1QUP79uX94Yl+MPEP8mLFFPkRkGrSPeLuSZNVHUYfVZIR9fvGta1+NOtMAAz
+Q4UbOHSEzKmmmQi2mCT6Rsr/AHd8nr6fHiBmCQUIFXMle9UbCnHTZtguEh/gPaUph3tAkEyaFLII
+oO24sxJIVg/vkeZe9RrT50ueA/taNy+qzj2J7jNNgLeTVL7pePXKgp6V56qBd+mAsTTNBNGSskoz
+MXSiqbKRJQ7iSMeaLmn83dXFPk86EvmNedbb7Rwp0qpdIglIJ+L/AAqDhjNCgwj1JjJb66jYSaPC
+E/vWp9SC3Lx26YxrNeYiJukuu/4lwJbLwU/wF7lan5jb1f0wGmZ1zNIcQKmX2G3FtBJ+ld1ESS3Q
+sKf0pf8Ay4Hg6zBlJwqNhvWLBnauPzNHPhLX4rS8sCXc08Uyo6fQiJiMIqRBudRKslx0P/TWuGof
+MWZlAdLyFijVkw9mOTI7t1FYtUlO74dPF/TAbPl8XBZMZtJkAXcdMLLFvWmLVQbm6nPnpS7vxVJV
+ZZSPVUlpLbRSX9iyzMeki2x90ty/FQerzxKyEsop7MY5k4UhmGqkK+citdaYf3danp5YVmt0oSTU
+iZsWjp+gpCzB32kg6H7tauvzad+AyTNuaFk5OMF2w2pJkko0kFVgtuKv3alafNQfXFRVcM5J2zUk
+FnTl5ukm5JG7qSHkJU/Fj02soU2q5mX/ALSkt9Ru5SHp8OggV2PQTV8nxiHBgSyoi0uWO0kDIuRU
+wBuCJOLyEqTZzxac6KjV01K33TgCuSP+mFdnMaQpSc2Jmo6gkk3CDZQ7CVC73g8++nP8sO9ocgmh
+FRgkHDSRCSb8bB6XCHSBUp5aj/niz5S3I3syatpBFihIEqMixfEdxLslCqCwV9a0rpgKVnNq1aC6
+9pGbSUcr8RtJ2kgk1UAVUxoNO4+vFbj2sgvbGsWy6ijshEEk/ER+WlMWHtAGFjZD932hnJCwFQUn
+wncTkSG5On+DXEFks8dyApxZoNk+P4hql1AQqiOtREtPDgP0PyOm+QyVGIPjuecGnviRq/CNKacx
+xOcKLIbRCB/CJW7vn+mIMOs+KHY76IIqEgmoqJIjddZTWv3mH5AR4clLLiHwkmHi/wB+A89WWFUh
+URX2x+IuI8X6Dhpo6dIOCHZ8NtvvlhHn60IaYSyTRdgP2YBISttUAh/xfe1xM4URMkyR2PiExDw/
+n11wDr1ZYUkiEF1BIrSFO8/r6Yxz9p3LLybyuxzEga6AxKvEFts1SXFKheVbf89cbKkis23U11kC
+HpISJEh/y6sCcxwrObhHkavYui5SK3oIhIajz/iUwHJLtwhOspydJJbMq9jZyMwij9lZgNw2vGvn
+Wmvi0xrv7IScehDzDSNmGr94Dol1XDICJszRr1WDfSnfjnzMrdbLshLQSDORJG3h0AFyKBiND199
+WmoKUGvdSunjxrH7Ms9MFnV004N6uioSRSL5EAbEkNB5I1R+Lq+KneOA6rboorAguPS13dxqCNwd
+487vXHgWcJ1QTd7ZSRCptCjdtW0+avliSipaqQie6oRCQoeHYCuAc81nFEiicuufZokW44k1g3bQ
+r3inTl1f8sBHzFmJvDAlx6KktMKpgSUSyDdLdpzoX4efxFgFlaJzNNSCWZMyOWSEskqSfAo+8CMb
+lSvRSvmtXzLFnh4FrHtVUYu8U10ivkSW+0qHd3c6d2IuYJRuhHruXL4IWJtEvaN9qqytK06aBWnn
+TAMQuVYeGZPo6CNRkzXIl3stxNy+7QqV0I689MB5iaWcpKqZbftcvQZFacmLa9d8t3e5D4/5sU9X
+NTrPk8xi2gcBCgqSoQSlyDiQAf4y5/w0fw18WA/aHna12upk1YJaatJgxVbo3JCr/wAFmn4en41M
+A7mDNEbEGuXttCAcJiQpM1LyXvU6auXNaU5KWVKwPXuwmAkO015FIuGGf4TIUeVPscZIMAJ0ol5L
+q386Edda6emmEdmXZqUb/a2dFo6Smo10Lt+g7P3DMqjeSypV++UpTu+EcFs3Z8yCi8bKTEPGzyq7
+aiyT1+6FJVRKpFQenypyrp9MBvvx+PCvFj4Ajf3Y+15a6eXdgOef21c1DE5XjMvqMzXbyhksRitt
+EFnPpr82OSMzPh3UOEeLuUWQ8MAEY2pWleOnzjpTHQ37Y7mQLPgtmpN1ExiVK2OQuFIfnT5clPrj
+niPTkpmBbMGz6gIryQAILDr11Gtda179K178BrHZkKcRldisq5ZOU3KSj10aNu4qxUKgLJ10+JMq
+AXLCpVmom7k4l3MOnLe1NGVc2CuRMq9TR0H8o1pTEuDi3ySqpRps29VGwv0E6JWp0EfdronSneJ0
+pStC76YrmYiFmo4BZRfaibQUogdRq6jVOfDnX5g1roXPXzwB/J8g8sdRcoZthTVFTitkruKoXuXQ
+6+GivhP+fFuj5SWKCeC2cgyJdUlIwyusZvh+8b//AE1B54xxw+UXhXHBSkkiq1dIxyaihUKtY5au
+qadfUxrSnfyp5YvPZ1OSUoJxrrbMFqVaODLqqLlKlyTkOXKunIqf1wBSVzA8kG8SMbGmLfj+JEiO
+4Gbug/aGqn/ln8OKHnucYje0y2AezySUWSbEdxIJGNaKtS1+IddaY88zlKZlaydFUko9OQUFNxwZ
+VCou0g04gPSpUpTWnLXz1xWmsWwWjk0UKrIqOo83Jr/GDgB5lSmulRLzp/TAWOQJ1Fx4zruVCWeQ
+opCaSJiO+xUGtKcq/LjPOIasZtBRyw9z7wjKwT3UTLpLTw8sG7G45faS3CpaRZJgonSmm+gdNSCt
+fLn5c6Yqjt0g+lG7ZFExZm6Mm4qncVAIuQl66YDR27daJj3S7ZY3rFgW2qF5CRMl/ARUr36VrTAV
+V1IIN3jZewk0Ek2D74QVSMtUVOXkOuLhInXJCLriCJ8ghRSMdDTkSqRjWo6etaVpTWtef1xnjxN8
+3kF2ajlMkkwSarjZdQklK6hSmvnSvr+mAs2Qmb5tJycfKSX2hRUY4x6fcEI02luXw/XF57VWbeWj
+Gs2+bLuXDlLgpNsn4kpABtTUqP1xnmX8ssOMRZLLOarlMnErGBaCoppqB189KV8+/F6gZprIQ1ZJ
+03VApAVI2QBMqVEl0qag4CleV2vnXngMMlUVGxl9j4Ikvsy4F4t2nOpYNQ8SUkySlnkkajcnQsnl
+txEgdvuir6jgVmhJYaJyLxyaq7xRSqtfFrUK6U7/AKYumSE2yOUgUqlU208ktHvEa+RhzSVGvrSv
+f3fngKzm6SWlpMnLttwyxCKLnc6iVWEbalWlPDrX1xa80Ok2zTK0XHtvbUD1P2LYfvRG2lFkSLxV
+0t10xRYtwqEok4TbtlXJGK4KLUuqKqZUMq93dXu07vWmLmbqSWziM3Bk3bK0aDKKJLhqFCK4STHS
+ngrTy0wAKbavF5D3fAsmuwLlnVQxDfRqdSTEa+RU8PLvxFhE5Sbm2DHe6XbwVitttEqnShF0+HDe
+amjhGdfbrmqttU6lry0pULqCP0p5YtHYvEm/zxEhHPFUttRBZxRbnQq7g00HT6euA7n2xQh2aZLI
+ESaSfSRpCPIdPMce4xum3SQJsxIVCuG1yl/1HywaogKBkSKCQKiVhaEVtPoP0xJSZvCVvMkan8VL
+yt/TlgBDckb7d5raPyrI+L5uQ+eHTdEmqNphvEVxANl1tPXpxPJo/TQNusadEKd20udC8X5YivnH
+AK1SJV2e5TSlarkVtP1wCjeEuZKEBqD8IkiJdXp3YfBMRuTFHqHwe5G23/viIaTUgEbVKlTuqVcR
+hdtA02xWpeNbeVOnXv8APAczftC5VTY9oTqQbNkH60ta7bRg3CkSoDWil4dxdNdenFB7NZ5qnmZJ
+9LM15tZBAukZIkgbABVITTMa0qoIj01Avlx1J2y5ZaZ5yO9hUXKzV8nQKtHJDTRM7uWunPT1xyqG
+XvbMs0bOHCQzbt3wIUogPB1KgiJGY9+te+6lNcB3BkWW9t5fYzKjYGzqQapuJECuE0BqHSP+WCqo
+qKKhILouk02R2t0kTuFca6UuKmOUexfPS2Ss8o5RclJvSqqSEuuo8qsDkxPQSAD0oNKUx0a+zgqW
+X46Rat9l5JOeDaUrXVNGl2lxU8+XlgDrgniDhmgKPFut9Qb07tpsFfn/AE7sZznXNULxaSe9FT2Y
+I5JTiXqxiTOPIfGVAryJT8NOeLJPrPqSq8G2XpHgtRKi7lr0rKLHTS7XlbSlPTHOud8tqJyuVhlp
+g0Ip46dxyKUYzTTWbGJXblD5VIiKlK1KvPAXTPce1aZCnXzGSN63XFstNkK21KvBIuYqKV+5S08v
+lxj7+YmMoTb7NeX5iIQ9hPmzZiggtvpNmiojoIadFtteo9NSrimzWa4R07NzKUnnaziKqksqbqh7
+6gqHQLwrXSyg0py9cQ6ZkZSUll1lBZajIWWZtiaruRHdTdFdX3pDWnIvrzr9cAfzrmyYlM2lDz2f
++JbkZIqP2QUJrw6/OtKhpcVvpX8sQYma7I6sE0815azFNSiWqRvkX1oLCNdBIRrzGmmnLywh5x/a
+T2oUbxMbE5bfE1rRbhiOiKhohdVTSlNbq9/59+KPmSUSeTTlZywboL3UFQWw2p1KlKUrWlPLXTX8
+64D/2Q==
+
+--Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B--
+
+--Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8--
diff --git a/src/leap/bitmask/mail/tests/rfc822.multi-signed.message b/src/leap/bitmask/mail/tests/rfc822.multi-signed.message
new file mode 100644
index 00000000..9907c2de
--- /dev/null
+++ b/src/leap/bitmask/mail/tests/rfc822.multi-signed.message
@@ -0,0 +1,238 @@
+Date: Mon, 6 Jan 2014 04:40:47 -0400
+From: Kali Kaneko <kali@leap.se>
+To: penguin@example.com
+Subject: signed message
+Message-ID: <20140106084047.GA21317@samsara.lan>
+MIME-Version: 1.0
+Content-Type: multipart/signed; micalg=pgp-sha1;
+ protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy"
+Content-Disposition: inline
+User-Agent: Mutt/1.5.21 (2012-12-30)
+
+
+--z9ECzHErBrwFF8sy
+Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l"
+Content-Disposition: inline
+
+
+--z0eOaCaDLjvTGF2l
+Content-Type: text/plain; charset=utf-8
+Content-Disposition: inline
+Content-Transfer-Encoding: quoted-printable
+
+This is an example of a signed message,
+with attachments.
+
+
+--=20
+Nihil sine chao! =E2=88=B4
+
+--z0eOaCaDLjvTGF2l
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: attachment; filename="attach.txt"
+
+this is attachment in plain text.
+
+--z0eOaCaDLjvTGF2l
+Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="hack.ico"
+Content-Transfer-Encoding: base64
+
+AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA
+KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG
+RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA
+PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl
+5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA
+/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ
+yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A
+Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK
+ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK
+LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP
+QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy
+AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs
+AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA
+AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA
+gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d
+HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA
+x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7
++wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA
+AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5
++QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA
+OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK
+igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA
+JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra
+2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA
+xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj
+owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB
+AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA
+AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d
+XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d
+XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA
+AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB
+AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm
+X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC
+AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B
+bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ
+S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu
+J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y
+AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N
+KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB
+XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A
+AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA
+AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d
+XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA
+AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr
+RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA
+AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A
+Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI
+yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA
+CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys
+rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA
+vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d
+HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA
+urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx
+cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA
+CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo
+6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA
+2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7
+OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA
+UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp
+qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA
+lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa
+WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB
+AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB
+AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB
+AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA
+ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA
+AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB
+AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB
+AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB
+AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB
+AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA
+tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA
+AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF
+wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB
+AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2
+RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB
+AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB
+AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA
+AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd
+AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB
+AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB
+AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB
+AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB
+AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB
+AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8
+ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2
+NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF
+RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB
+lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA
+AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa
+WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA
+AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX
+AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB
+AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB
+AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA
+AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA
+AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA
+AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA
+AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB
+AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB
+AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA
+ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA
+AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7
+LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA
+AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF
+NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB
+AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2
+RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA
+ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4
+RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi
+JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2
+NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK
+T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB
+AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB
+AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB
+AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN
+UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA
+AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA
+W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA
+AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB
+l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB
+AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+
+WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA
+AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv
+RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA
+AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj
+AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB
+AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB
+AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA
+AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA
+AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA
+dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A
+AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB
+AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB
+AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB
+AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW
+pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+
+--z0eOaCaDLjvTGF2l--
+
+--z9ECzHErBrwFF8sy
+Content-Type: application/pgp-signature
+
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.15 (GNU/Linux)
+
+iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv
+kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl
+vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK
+PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC
+w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw
+sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr
+BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN
+QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt
+mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ
+jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8
+gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X
+sSdfcAhT7Tno7PB/Acoh
+=+okv
+-----END PGP SIGNATURE-----
+
+--z9ECzHErBrwFF8sy--
diff --git a/src/leap/bitmask/mail/tests/rfc822.multi.message b/src/leap/bitmask/mail/tests/rfc822.multi.message
new file mode 100644
index 00000000..30f74e52
--- /dev/null
+++ b/src/leap/bitmask/mail/tests/rfc822.multi.message
@@ -0,0 +1,96 @@
+Date: Fri, 19 May 2000 09:55:48 -0400 (EDT)
+From: Doug Sauder <doug@penguin.example.com>
+To: Joe Blow <blow@example.com>
+Subject: Test message from PINE
+Message-ID: <Pine.LNX.4.21.0005190951410.8452-102000@penguin.example.com>
+MIME-Version: 1.0
+Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452"
+
+ This message is in MIME format. The first part should be readable text,
+ while the remaining parts are likely unreadable without MIME-aware tools.
+ Send mail to mime@docserver.cac.washington.edu for more info.
+
+---1463757054-952513540-958744548=:8452
+Content-Type: TEXT/PLAIN; charset=US-ASCII
+
+This is a test message from PINE MUA.
+
+
+---1463757054-952513540-958744548=:8452
+Content-Type: APPLICATION/octet-stream; name="redball.png"
+Content-Transfer-Encoding: BASE64
+Content-ID: <Pine.LNX.4.21.0005190955480.8452@penguin.example.com>
+Content-Description: A PNG graphic file
+Content-Disposition: attachment; filename="redball.png"
+
+iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A
+AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj
+AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv
+AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0
+GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf
+hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl
+rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL
+ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS
+AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP
+AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP
+AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI
+AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR
+AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6
+AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO
+AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8
+AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8
+LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu
+MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF
+BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp
+6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow
+8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G
+ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn
+OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1
+a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T
+VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8
+Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f
+lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/
+joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms
+1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA
+JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7
+vAAAAABJRU5ErkJggg==
+---1463757054-952513540-958744548=:8452
+Content-Type: APPLICATION/octet-stream; name="blueball.png"
+Content-Transfer-Encoding: BASE64
+Content-ID: <Pine.LNX.4.21.0005190955481.8452@penguin.example.com>
+Content-Description: A PNG graphic file
+Content-Disposition: attachment; filename="blueball.png"
+
+iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A
+AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI
+IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y
+Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S
+hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM
+vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB
+Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu
+MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b
+fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo
+Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp
+LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX
+P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M
+jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8
++VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE
+1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42
+YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY
+mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp
+Z3VldDZzO7wAAAAASUVORK5CYII=
+---1463757054-952513540-958744548=:8452--
diff --git a/src/leap/bitmask/mail/tests/rfc822.plain.message b/src/leap/bitmask/mail/tests/rfc822.plain.message
new file mode 100644
index 00000000..fc627c3a
--- /dev/null
+++ b/src/leap/bitmask/mail/tests/rfc822.plain.message
@@ -0,0 +1,66 @@
+From pyar-bounces@python.org.ar Wed Jan 8 14:46:02 2014
+Return-Path: <pyar-bounces@python.org.ar>
+X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net
+X-Spam-Level: **
+X-Spam-Pyzor: Reported 0 times.
+X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE,
+ CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP,
+ NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled
+ version=3.3.2
+Delivered-To: kali@leap.se
+Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33])
+ (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits))
+ (Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified))
+ by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F
+ for <kali@leap.se>; Wed, 8 Jan 2014 18:46:02 +0000 (UTC)
+Received: from pyar.usla.org.ar (unknown [190.228.30.157])
+ by mx1.riseup.net (Postfix) with ESMTP id F244C533F4
+ for <kali@leap.se>; Wed, 8 Jan 2014 10:46:01 -0800 (PST)
+Received: from [127.0.0.1] (localhost [127.0.0.1])
+ by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F
+ for <kali@leap.se>; Wed, 8 Jan 2014 15:46:00 -0300 (ART)
+MIME-Version: 1.0
+Content-Type: text/plain; charset="iso-8859-1"
+Content-Transfer-Encoding: quoted-printable
+From: pyar-request@python.org.ar
+To: kali@leap.se
+Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2
+Reply-To: pyar-request@python.org.ar
+Auto-Submitted: auto-replied
+Message-ID: <mailman.245.1389206759.1579.pyar@python.org.ar>
+Date: Wed, 08 Jan 2014 15:45:59 -0300
+Precedence: bulk
+X-BeenThere: pyar@python.org.ar
+X-Mailman-Version: 2.1.15
+List-Id: Python Argentina <pyar.python.org.ar>
+X-List-Administrivia: yes
+Errors-To: pyar-bounces@python.org.ar
+Sender: "pyar" <pyar-bounces@python.org.ar>
+X-Virus-Scanned: clamav-milter 0.97.8 at mx1
+X-Virus-Status: Clean
+
+Mailing list subscription confirmation notice for mailing list pyar
+
+We have received a request de kaliyuga@riseup.net for subscription of
+your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar
+mailing list. To confirm that you want to be added to this mailing
+list, simply reply to this message, keeping the Subject: header
+intact. Or visit this web page:
+
+ http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b=
+3377148ac2
+
+
+Or include the following line -- and only the following line -- in a
+message to pyar-request@python.org.ar:
+
+ confirm 0e47e4342e4d42508e8c283175b05b3377148ac2
+
+Note that simply sending a `reply' to this message should work from
+most mail readers, since that usually leaves the Subject: line in the
+right form (additional "Re:" text in the Subject: is okay).
+
+If you do not wish to be subscribed to this list, please simply
+disregard this message. If you think you are being maliciously
+subscribed to the list, or have any other questions, send them to
+pyar-owner@python.org.ar.
diff --git a/src/leap/bitmask/mail/tests/test_mail.py b/src/leap/bitmask/mail/tests/test_mail.py
new file mode 100644
index 00000000..f9cded29
--- /dev/null
+++ b/src/leap/bitmask/mail/tests/test_mail.py
@@ -0,0 +1,399 @@
+# -*- coding: utf-8 -*-
+# test_mail.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the mail module.
+"""
+import os
+import time
+import uuid
+
+from functools import partial
+from email.parser import Parser
+from email.Utils import formatdate
+
+from leap.mail.adaptors.soledad import SoledadMailAdaptor
+from leap.mail.mail import MessageCollection, Account, _unpack_headers
+from leap.mail.mailbox_indexer import MailboxIndexer
+from leap.mail.testing.common import SoledadTestMixin
+
+HERE = os.path.split(os.path.abspath(__file__))[0]
+
+
+def _get_raw_msg(multi=False):
+ if multi:
+ sample = "rfc822.multi.message"
+ else:
+ sample = "rfc822.message"
+ with open(os.path.join(HERE, sample)) as f:
+ raw = f.read()
+ return raw
+
+
+def _get_parsed_msg(multi=False):
+ mail_parser = Parser()
+ raw = _get_raw_msg(multi=multi)
+ return mail_parser.parsestr(raw)
+
+
+def _get_msg_time():
+ timestamp = time.mktime((2010, 12, 12, 1, 1, 1, 1, 1, 1))
+ return formatdate(timestamp)
+
+
+class CollectionMixin(object):
+
+ def get_collection(self, mbox_collection=True, mbox_name=None,
+ mbox_uuid=None):
+ """
+ Get a collection for tests.
+ """
+ adaptor = SoledadMailAdaptor()
+ store = self._soledad
+ adaptor.store = store
+
+ if mbox_collection:
+ mbox_indexer = MailboxIndexer(store)
+ mbox_name = mbox_name or "TestMbox"
+ mbox_uuid = mbox_uuid or str(uuid.uuid4())
+ else:
+ mbox_indexer = mbox_name = None
+
+ def get_collection_from_mbox_wrapper(wrapper):
+ wrapper.uuid = mbox_uuid
+ return MessageCollection(
+ adaptor, store,
+ mbox_indexer=mbox_indexer, mbox_wrapper=wrapper)
+
+ d = adaptor.initialize_store(store)
+ if mbox_collection:
+ d.addCallback(lambda _: mbox_indexer.create_table(mbox_uuid))
+ d.addCallback(lambda _: adaptor.get_or_create_mbox(store, mbox_name))
+ d.addCallback(get_collection_from_mbox_wrapper)
+ return d
+
+
+# TODO profile add_msg. Why are these tests so SLOW??!
+class MessageTestCase(SoledadTestMixin, CollectionMixin):
+ """
+ Tests for the Message class.
+ """
+ msg_flags = ('\Recent', '\Unseen', '\TestFlag')
+ msg_tags = ('important', 'todo', 'wonderful')
+ internal_date = "19-Mar-2015 19:22:21 -0500"
+
+ maxDiff = None
+
+ def _do_insert_msg(self, multi=False):
+ """
+ Inserts and return a regular message, for tests.
+ """
+ def insert_message(collection):
+ self._mbox_uuid = collection.mbox_uuid
+ return collection.add_msg(
+ raw, flags=self.msg_flags, tags=self.msg_tags,
+ date=self.internal_date)
+
+ raw = _get_raw_msg(multi=multi)
+
+ d = self.get_collection()
+ d.addCallback(insert_message)
+ return d
+
+ def get_inserted_msg(self, multi=False):
+ d = self._do_insert_msg(multi=multi)
+ d.addCallback(lambda _: self.get_collection(mbox_uuid=self._mbox_uuid))
+ d.addCallback(lambda col: col.get_message_by_uid(1))
+ return d
+
+ def test_get_flags(self):
+ d = self.get_inserted_msg()
+ d.addCallback(self._test_get_flags_cb)
+ return d
+
+ def _test_get_flags_cb(self, msg):
+ self.assertTrue(msg is not None)
+ self.assertEquals(tuple(msg.get_flags()), self.msg_flags)
+
+ def test_get_internal_date(self):
+ d = self.get_inserted_msg()
+ d.addCallback(self._test_get_internal_date_cb)
+
+ def _test_get_internal_date_cb(self, msg):
+ self.assertTrue(msg is not None)
+ self.assertDictEqual(msg.get_internal_date(),
+ self.internal_date)
+
+ def test_get_headers(self):
+ d = self.get_inserted_msg()
+ d.addCallback(self._test_get_headers_cb)
+ return d
+
+ def _test_get_headers_cb(self, msg):
+ self.assertTrue(msg is not None)
+ expected = [
+ (str(key.lower()), str(value))
+ for (key, value) in _get_parsed_msg().items()]
+ self.assertItemsEqual(_unpack_headers(msg.get_headers()), expected)
+
+ def test_get_body_file(self):
+ d = self.get_inserted_msg(multi=True)
+ d.addCallback(self._test_get_body_file_cb)
+ return d
+
+ def _test_get_body_file_cb(self, msg):
+ self.assertTrue(msg is not None)
+ orig = _get_parsed_msg(multi=True)
+ expected = orig.get_payload()[0].get_payload()
+ d = msg.get_body_file(self._soledad)
+
+ def assert_body(fd):
+ self.assertTrue(fd is not None)
+ self.assertEqual(fd.read(), expected)
+ d.addCallback(assert_body)
+ return d
+
+ def test_get_size(self):
+ d = self.get_inserted_msg()
+ d.addCallback(self._test_get_size_cb)
+ return d
+
+ def _test_get_size_cb(self, msg):
+ self.assertTrue(msg is not None)
+ expected = len(_get_parsed_msg().as_string())
+ self.assertEqual(msg.get_size(), expected)
+
+ def test_is_multipart_no(self):
+ d = self.get_inserted_msg()
+ d.addCallback(self._test_is_multipart_no_cb)
+ return d
+
+ def _test_is_multipart_no_cb(self, msg):
+ self.assertTrue(msg is not None)
+ expected = _get_parsed_msg().is_multipart()
+ self.assertEqual(msg.is_multipart(), expected)
+
+ def test_is_multipart_yes(self):
+ d = self.get_inserted_msg(multi=True)
+ d.addCallback(self._test_is_multipart_yes_cb)
+ return d
+
+ def _test_is_multipart_yes_cb(self, msg):
+ self.assertTrue(msg is not None)
+ expected = _get_parsed_msg(multi=True).is_multipart()
+ self.assertEqual(msg.is_multipart(), expected)
+
+ def test_get_subpart(self):
+ d = self.get_inserted_msg(multi=True)
+ d.addCallback(self._test_get_subpart_cb)
+ return d
+
+ def _test_get_subpart_cb(self, msg):
+ self.assertTrue(msg is not None)
+
+ def test_get_tags(self):
+ d = self.get_inserted_msg()
+ d.addCallback(self._test_get_tags_cb)
+ return d
+
+ def _test_get_tags_cb(self, msg):
+ self.assertTrue(msg is not None)
+ self.assertEquals(msg.get_tags(), self.msg_tags)
+
+
+class MessageCollectionTestCase(SoledadTestMixin, CollectionMixin):
+ """
+ Tests for the MessageCollection class.
+ """
+ _mbox_uuid = None
+
+ def assert_collection_count(self, _, expected):
+ def _assert_count(count):
+ self.assertEqual(count, expected)
+
+ d = self.get_collection()
+ d.addCallback(lambda col: col.count())
+ d.addCallback(_assert_count)
+ return d
+
+ def add_msg_to_collection(self):
+ raw = _get_raw_msg()
+
+ def add_msg_to_collection(collection):
+ # We keep the uuid in case we need to instantiate the same
+ # collection afterwards.
+ self._mbox_uuid = collection.mbox_uuid
+ d = collection.add_msg(raw, date=_get_msg_time())
+ return d
+
+ d = self.get_collection()
+ d.addCallback(add_msg_to_collection)
+ return d
+
+ def test_is_mailbox_collection(self):
+ d = self.get_collection()
+ d.addCallback(self._test_is_mailbox_collection_cb)
+ return d
+
+ def _test_is_mailbox_collection_cb(self, collection):
+ self.assertTrue(collection.is_mailbox_collection())
+
+ def test_get_uid_next(self):
+ d = self.add_msg_to_collection()
+ d.addCallback(lambda _: self.get_collection())
+ d.addCallback(lambda col: col.get_uid_next())
+ d.addCallback(self._test_get_uid_next_cb)
+
+ def _test_get_uid_next_cb(self, next_uid):
+ self.assertEqual(next_uid, 2)
+
+ def test_add_and_count_msg(self):
+ d = self.add_msg_to_collection()
+ d.addCallback(self._test_add_and_count_msg_cb)
+ return d
+
+ def _test_add_and_count_msg_cb(self, _):
+ return partial(self.assert_collection_count, expected=1)
+
+ def test_copy_msg(self):
+ # TODO ---- update when implementing messagecopier
+ # interface
+ pass
+ test_copy_msg.skip = "Not yet implemented"
+
+ def test_delete_msg(self):
+ d = self.add_msg_to_collection()
+
+ def del_msg(collection):
+ def _delete_it(msg):
+ self.assertTrue(msg is not None)
+ return collection.delete_msg(msg)
+
+ d = collection.get_message_by_uid(1)
+ d.addCallback(_delete_it)
+ return d
+
+ # We need to instantiate an mbox collection with the same uuid that
+ # the one in which we inserted the doc.
+ d.addCallback(lambda _: self.get_collection(mbox_uuid=self._mbox_uuid))
+ d.addCallback(del_msg)
+ d.addCallback(self._test_delete_msg_cb)
+ return d
+
+ def _test_delete_msg_cb(self, _):
+ return partial(self.assert_collection_count, expected=0)
+
+ def test_update_flags(self):
+ d = self.add_msg_to_collection()
+ d.addCallback(self._test_update_flags_cb)
+ return d
+
+ def _test_update_flags_cb(self, msg):
+ pass
+
+ def test_update_tags(self):
+ d = self.add_msg_to_collection()
+ d.addCallback(self._test_update_tags_cb)
+ return d
+
+ def _test_update_tags_cb(self, msg):
+ pass
+
+
+class AccountTestCase(SoledadTestMixin):
+ """
+ Tests for the Account class.
+ """
+ def get_account(self, user_id):
+ store = self._soledad
+ return Account(store, user_id)
+
+ def test_add_mailbox(self):
+ acc = self.get_account('some_user_id')
+ d = acc.callWhenReady(lambda _: acc.add_mailbox("TestMailbox"))
+ d.addCallback(lambda _: acc.list_all_mailbox_names())
+ d.addCallback(self._test_add_mailbox_cb)
+ return d
+
+ def _test_add_mailbox_cb(self, mboxes):
+ expected = ['INBOX', 'TestMailbox']
+ self.assertItemsEqual(mboxes, expected)
+
+ def test_delete_mailbox(self):
+ acc = self.get_account('some_user_id')
+ d = acc.callWhenReady(lambda _: acc.delete_mailbox("Inbox"))
+ d.addCallback(lambda _: acc.list_all_mailbox_names())
+ d.addCallback(self._test_delete_mailbox_cb)
+ return d
+
+ def _test_delete_mailbox_cb(self, mboxes):
+ expected = []
+ self.assertItemsEqual(mboxes, expected)
+
+ def test_rename_mailbox(self):
+ acc = self.get_account('some_user_id')
+ d = acc.callWhenReady(lambda _: acc.add_mailbox("OriginalMailbox"))
+ d.addCallback(lambda _: acc.rename_mailbox(
+ "OriginalMailbox", "RenamedMailbox"))
+ d.addCallback(lambda _: acc.list_all_mailbox_names())
+ d.addCallback(self._test_rename_mailbox_cb)
+ return d
+
+ def _test_rename_mailbox_cb(self, mboxes):
+ expected = ['INBOX', 'RenamedMailbox']
+ self.assertItemsEqual(mboxes, expected)
+
+ def test_get_all_mailboxes(self):
+ acc = self.get_account('some_user_id')
+ d = acc.callWhenReady(lambda _: acc.add_mailbox("OneMailbox"))
+ d.addCallback(lambda _: acc.add_mailbox("TwoMailbox"))
+ d.addCallback(lambda _: acc.add_mailbox("ThreeMailbox"))
+ d.addCallback(lambda _: acc.add_mailbox("anotherthing"))
+ d.addCallback(lambda _: acc.add_mailbox("anotherthing2"))
+ d.addCallback(lambda _: acc.get_all_mailboxes())
+ d.addCallback(self._test_get_all_mailboxes_cb)
+ return d
+
+ def _test_get_all_mailboxes_cb(self, mailboxes):
+ expected = ["INBOX", "OneMailbox", "TwoMailbox", "ThreeMailbox",
+ "anotherthing", "anotherthing2"]
+ names = [m.mbox for m in mailboxes]
+ self.assertItemsEqual(names, expected)
+
+ def test_get_collection_by_mailbox(self):
+ acc = self.get_account('some_user_id')
+ d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox("INBOX"))
+ d.addCallback(self._test_get_collection_by_mailbox_cb)
+ return d
+
+ def _test_get_collection_by_mailbox_cb(self, collection):
+ self.assertTrue(collection.is_mailbox_collection())
+
+ def assert_uid_next_empty_collection(uid):
+ self.assertEqual(uid, 1)
+ d = collection.get_uid_next()
+ d.addCallback(assert_uid_next_empty_collection)
+ return d
+
+ def test_get_collection_by_docs(self):
+ pass
+
+ test_get_collection_by_docs.skip = "Not yet implemented"
+
+ def test_get_collection_by_tag(self):
+ pass
+
+ test_get_collection_by_tag.skip = "Not yet implemented"
diff --git a/src/leap/bitmask/mail/tests/test_mailbox_indexer.py b/src/leap/bitmask/mail/tests/test_mailbox_indexer.py
new file mode 100644
index 00000000..5c1891d5
--- /dev/null
+++ b/src/leap/bitmask/mail/tests/test_mailbox_indexer.py
@@ -0,0 +1,250 @@
+# -*- coding: utf-8 -*-
+# test_mailbox_indexer.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for the mailbox_indexer module.
+"""
+import uuid
+from functools import partial
+
+from leap.mail import mailbox_indexer as mi
+from leap.mail.testing.common import SoledadTestMixin
+
+hash_test0 = '590c9f8430c7435807df8ba9a476e3f1295d46ef210f6efae2043a4c085a569e'
+hash_test1 = '1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014'
+hash_test2 = '60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752'
+hash_test3 = 'fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13'
+hash_test4 = 'a4e624d686e03ed2767c0abd85c14426b0b1157d2ce81d27bb4fe4f6f01d688a'
+
+
+def fmt_hash(mailbox_uuid, hash):
+ return "M-" + mailbox_uuid.replace('-', '_') + "-" + hash
+
+mbox_id = str(uuid.uuid4())
+
+
+class MailboxIndexerTestCase(SoledadTestMixin):
+ """
+ Tests for the MailboxUID class.
+ """
+ def get_mbox_uid(self):
+ m_uid = mi.MailboxIndexer(self._soledad)
+ return m_uid
+
+ def list_mail_tables_cb(self, ignored):
+ def filter_mailuid_tables(tables):
+ filtered = [
+ table[0] for table in tables if
+ table[0].startswith(mi.MailboxIndexer.table_preffix)]
+ return filtered
+
+ sql = "SELECT name FROM sqlite_master WHERE type='table';"
+ d = self._soledad.raw_sqlcipher_query(sql)
+ d.addCallback(filter_mailuid_tables)
+ return d
+
+ def select_uid_rows(self, mailbox):
+ sql = "SELECT * FROM %s%s;" % (
+ mi.MailboxIndexer.table_preffix, mailbox.replace('-', '_'))
+ d = self._soledad.raw_sqlcipher_query(sql)
+ return d
+
+ def test_create_table(self):
+ def assert_table_created(tables):
+ self.assertEqual(
+ tables, ["leapmail_uid_" + mbox_id.replace('-', '_')])
+
+ m_uid = self.get_mbox_uid()
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(self.list_mail_tables_cb)
+ d.addCallback(assert_table_created)
+ return d
+
+ def test_create_and_delete_table(self):
+ def assert_table_deleted(tables):
+ self.assertEqual(tables, [])
+
+ m_uid = self.get_mbox_uid()
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(lambda _: m_uid.delete_table(mbox_id))
+ d.addCallback(self.list_mail_tables_cb)
+ d.addCallback(assert_table_deleted)
+ return d
+
+ def test_insert_doc(self):
+ m_uid = self.get_mbox_uid()
+
+ h1 = fmt_hash(mbox_id, hash_test0)
+ h2 = fmt_hash(mbox_id, hash_test1)
+ h3 = fmt_hash(mbox_id, hash_test2)
+ h4 = fmt_hash(mbox_id, hash_test3)
+ h5 = fmt_hash(mbox_id, hash_test4)
+
+ def assert_uid_rows(rows):
+ expected = [(1, h1), (2, h2), (3, h3), (4, h4), (5, h5)]
+ self.assertEquals(rows, expected)
+
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5))
+ d.addCallback(lambda _: self.select_uid_rows(mbox_id))
+ d.addCallback(assert_uid_rows)
+ return d
+
+ def test_insert_doc_return(self):
+ m_uid = self.get_mbox_uid()
+
+ def assert_rowid(rowid, expected=None):
+ self.assertEqual(rowid, expected)
+
+ h1 = fmt_hash(mbox_id, hash_test0)
+ h2 = fmt_hash(mbox_id, hash_test1)
+ h3 = fmt_hash(mbox_id, hash_test2)
+
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1))
+ d.addCallback(partial(assert_rowid, expected=1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2))
+ d.addCallback(partial(assert_rowid, expected=2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3))
+ d.addCallback(partial(assert_rowid, expected=3))
+ return d
+
+ def test_delete_doc(self):
+ m_uid = self.get_mbox_uid()
+
+ h1 = fmt_hash(mbox_id, hash_test0)
+ h2 = fmt_hash(mbox_id, hash_test1)
+ h3 = fmt_hash(mbox_id, hash_test2)
+ h4 = fmt_hash(mbox_id, hash_test3)
+ h5 = fmt_hash(mbox_id, hash_test4)
+
+ def assert_uid_rows(rows):
+ expected = [(4, h4), (5, h5)]
+ self.assertEquals(rows, expected)
+
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5))
+
+ d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1))
+ d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 2))
+ d.addCallbacks(lambda _: m_uid.delete_doc_by_hash(mbox_id, h3))
+
+ d.addCallback(lambda _: self.select_uid_rows(mbox_id))
+ d.addCallback(assert_uid_rows)
+ return d
+
+ def test_get_doc_id_from_uid(self):
+ m_uid = self.get_mbox_uid()
+
+ h1 = fmt_hash(mbox_id, hash_test0)
+
+ def assert_doc_hash(res):
+ self.assertEqual(res, h1)
+
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1))
+ d.addCallback(lambda _: m_uid.get_doc_id_from_uid(mbox_id, 1))
+ d.addCallback(assert_doc_hash)
+ return d
+
+ def test_count(self):
+ m_uid = self.get_mbox_uid()
+
+ h1 = fmt_hash(mbox_id, hash_test0)
+ h2 = fmt_hash(mbox_id, hash_test1)
+ h3 = fmt_hash(mbox_id, hash_test2)
+ h4 = fmt_hash(mbox_id, hash_test3)
+ h5 = fmt_hash(mbox_id, hash_test4)
+
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5))
+
+ def assert_count_after_inserts(count):
+ self.assertEquals(count, 5)
+
+ d.addCallback(lambda _: m_uid.count(mbox_id))
+ d.addCallback(assert_count_after_inserts)
+
+ d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1))
+ d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 2))
+
+ def assert_count_after_deletions(count):
+ self.assertEquals(count, 3)
+
+ d.addCallback(lambda _: m_uid.count(mbox_id))
+ d.addCallback(assert_count_after_deletions)
+ return d
+
+ def test_get_next_uid(self):
+ m_uid = self.get_mbox_uid()
+
+ h1 = fmt_hash(mbox_id, hash_test0)
+ h2 = fmt_hash(mbox_id, hash_test1)
+ h3 = fmt_hash(mbox_id, hash_test2)
+ h4 = fmt_hash(mbox_id, hash_test3)
+ h5 = fmt_hash(mbox_id, hash_test4)
+
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5))
+
+ def assert_next_uid(result, expected=1):
+ self.assertEquals(result, expected)
+
+ d.addCallback(lambda _: m_uid.get_next_uid(mbox_id))
+ d.addCallback(partial(assert_next_uid, expected=6))
+ return d
+
+ def test_all_uid_iter(self):
+
+ m_uid = self.get_mbox_uid()
+
+ h1 = fmt_hash(mbox_id, hash_test0)
+ h2 = fmt_hash(mbox_id, hash_test1)
+ h3 = fmt_hash(mbox_id, hash_test2)
+ h4 = fmt_hash(mbox_id, hash_test3)
+ h5 = fmt_hash(mbox_id, hash_test4)
+
+ d = m_uid.create_table(mbox_id)
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4))
+ d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5))
+ d.addCallback(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1))
+ d.addCallback(lambda _: m_uid.delete_doc_by_uid(mbox_id, 4))
+
+ def assert_all_uid(result, expected=[2, 3, 5]):
+ self.assertEquals(result, expected)
+
+ d.addCallback(lambda _: m_uid.all_uid_iter(mbox_id))
+ d.addCallback(partial(assert_all_uid))
+ return d
diff --git a/src/leap/bitmask/mail/tests/test_walk.py b/src/leap/bitmask/mail/tests/test_walk.py
new file mode 100644
index 00000000..826ec10c
--- /dev/null
+++ b/src/leap/bitmask/mail/tests/test_walk.py
@@ -0,0 +1,81 @@
+"""
+Tests for leap.mail.walk module
+"""
+import os.path
+from email.parser import Parser
+
+from leap.mail import walk
+
+CORPUS = {
+ 'simple': 'rfc822.message',
+ 'multimin': 'rfc822.multi-minimal.message',
+ 'multisigned': 'rfc822.multi-signed.message',
+ 'bounced': 'rfc822.bounce.message',
+}
+
+_here = os.path.dirname(__file__)
+_parser = Parser()
+
+
+# tests
+
+
+def test_simple_mail():
+ msg = _parse('simple')
+ tree = walk.get_tree(msg)
+ assert len(tree['part_map']) == 0
+ assert tree['ctype'] == 'text/plain'
+ assert tree['multi'] is False
+
+
+def test_multipart_minimal():
+ msg = _parse('multimin')
+ tree = walk.get_tree(msg)
+
+ assert tree['multi'] is True
+ assert len(tree['part_map']) == 1
+ first = tree['part_map'][1]
+ assert first['multi'] is False
+ assert first['ctype'] == 'text/plain'
+
+
+def test_multi_signed():
+ msg = _parse('multisigned')
+ tree = walk.get_tree(msg)
+ assert tree['multi'] is True
+ assert len(tree['part_map']) == 2
+
+ _first = tree['part_map'][1]
+ _second = tree['part_map'][2]
+ assert len(_first['part_map']) == 3
+ assert(_second['multi'] is False)
+
+
+def test_bounce_mime():
+ msg = _parse('bounced')
+ tree = walk.get_tree(msg)
+
+ ctypes = [tree['part_map'][index]['ctype']
+ for index in sorted(tree['part_map'].keys())]
+ third = tree['part_map'][3]
+ three_one_ctype = third['part_map'][1]['ctype']
+ assert three_one_ctype == 'multipart/signed'
+
+ assert ctypes == [
+ 'text/plain',
+ 'message/delivery-status',
+ 'message/rfc822']
+
+
+# utils
+
+def _parse(name):
+ _str = _get_string_for_message(name)
+ return _parser.parsestr(_str)
+
+
+def _get_string_for_message(name):
+ filename = os.path.join(_here, CORPUS[name])
+ with open(filename) as f:
+ msgstr = f.read()
+ return msgstr
diff --git a/src/leap/bitmask/mail/utils.py b/src/leap/bitmask/mail/utils.py
new file mode 100644
index 00000000..64fca981
--- /dev/null
+++ b/src/leap/bitmask/mail/utils.py
@@ -0,0 +1,375 @@
+# -*- coding: utf-8 -*-
+# utils.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Mail utilities.
+"""
+from email.utils import parseaddr
+import json
+import re
+import traceback
+import Queue
+
+from leap.soledad.common.document import SoledadDocument
+from leap.common.check import leap_assert_type
+from twisted.mail import smtp
+
+
+CHARSET_PATTERN = r"""charset=([\w-]+)"""
+CHARSET_RE = re.compile(CHARSET_PATTERN, re.IGNORECASE)
+
+
+def first(things):
+ """
+ Return the head of a collection.
+ """
+ try:
+ return things[0]
+ except (IndexError, TypeError):
+ return None
+
+
+def empty(thing):
+ """
+ Return True if a thing is None or its length is zero.
+ If thing is a number (int, float, long), return False.
+ """
+ if thing is None:
+ return True
+ if isinstance(thing, (int, float, long)):
+ return False
+ if isinstance(thing, SoledadDocument):
+ thing = thing.content
+ try:
+ return len(thing) == 0
+ except (ReferenceError, TypeError):
+ return True
+
+
+def maybe_call(thing):
+ """
+ Return the same thing, or the result of its invocation if it is a
+ callable.
+ """
+ return thing() if callable(thing) else thing
+
+
+def find_charset(thing, default=None):
+ """
+ Looks into the object 'thing' for a charset specification.
+ It searchs into the object's `repr`.
+
+ :param thing: the object to look into.
+ :type thing: object
+ :param default: the dafault charset to return if no charset is found.
+ :type default: str
+
+ :return: the charset or 'default'
+ :rtype: str or None
+ """
+ charset = first(CHARSET_RE.findall(repr(thing)))
+ if charset is None:
+ charset = default
+ return charset
+
+
+def lowerdict(_dict):
+ """
+ Return a dict with the keys in lowercase.
+
+ :param _dict: the dict to convert
+ :rtype: dict
+ """
+ # TODO should properly implement a CaseInsensitive dict.
+ # Look into requests code.
+ return dict((key.lower(), value)
+ for key, value in _dict.items())
+
+
+PART_MAP = "part_map"
+PHASH = "phash"
+
+
+def _str_dict(d, k):
+ """
+ Convert the dictionary key to string if it was a string.
+
+ :param d: the dict
+ :type d: dict
+ :param k: the key
+ :type k: object
+ """
+ if isinstance(k, int):
+ val = d[k]
+ d[str(k)] = val
+ del(d[k])
+
+
+def stringify_parts_map(d):
+ """
+ Modify a dictionary making all the nested dicts under "part_map" keys
+ having strings as keys.
+
+ :param d: the dictionary to modify
+ :type d: dictionary
+ :rtype: dictionary
+ """
+ for k in d:
+ if k == PART_MAP:
+ pmap = d[k]
+ for kk in pmap.keys():
+ _str_dict(d[k], kk)
+ for kk in pmap.keys():
+ stringify_parts_map(d[k][str(kk)])
+ return d
+
+
+def phash_iter(d):
+ """
+ A recursive generator that extracts all the payload-hashes
+ from an arbitrary nested parts-map dictionary.
+
+ :param d: the dictionary to walk
+ :type d: dictionary
+ :return: a list of all the phashes found
+ :rtype: list
+ """
+ if PHASH in d:
+ yield d[PHASH]
+ if PART_MAP in d:
+ for key in d[PART_MAP]:
+ for phash in phash_iter(d[PART_MAP][key]):
+ yield phash
+
+
+def accumulator(fun, lim):
+ """
+ A simple accumulator that uses a closure and a mutable
+ object to collect items.
+ When the count of items is greater than `lim`, the
+ collection is flushed after invoking a map of the function `fun`
+ over it.
+
+ The returned accumulator can also be flushed at any moment
+ by passing a boolean as a second parameter.
+
+ :param fun: the function to call over the collection
+ when its size is greater than `lim`
+ :type fun: callable
+ :param lim: the turning point for the collection
+ :type lim: int
+ :rtype: function
+
+ >>> from pprint import pprint
+ >>> acc = accumulator(pprint, 2)
+ >>> acc(1)
+ >>> acc(2)
+ [1, 2]
+ >>> acc(3)
+ >>> acc(4)
+ [3, 4]
+ >>> acc = accumulator(pprint, 5)
+ >>> acc(1)
+ >>> acc(2)
+ >>> acc(3)
+ >>> acc(None, flush=True)
+ [1,2,3]
+ """
+ KEY = "items"
+ _o = {KEY: []}
+
+ def _accumulator(item, flush=False):
+ collection = _o[KEY]
+ collection.append(item)
+ if len(collection) >= lim or flush:
+ map(fun, filter(None, collection))
+ _o[KEY] = []
+
+ return _accumulator
+
+
+def accumulator_queue(fun, lim):
+ """
+ A version of the accumulator that uses a queue.
+
+ When the count of items is greater than `lim`, the
+ queue is flushed after invoking the function `fun`
+ over its items.
+
+ The returned accumulator can also be flushed at any moment
+ by passing a boolean as a second parameter.
+
+ :param fun: the function to call over the collection
+ when its size is greater than `lim`
+ :type fun: callable
+ :param lim: the turning point for the collection
+ :type lim: int
+ :rtype: function
+ """
+ _q = Queue.Queue()
+
+ def _accumulator(item, flush=False):
+ _q.put(item)
+ if _q.qsize() >= lim or flush:
+ collection = [_q.get() for i in range(_q.qsize())]
+ map(fun, filter(None, collection))
+
+ return _accumulator
+
+
+def validate_address(address):
+ """
+ Validate C{address} as defined in RFC 2822.
+
+ :param address: The address to be validated.
+ :type address: str
+
+ @return: A valid address.
+ @rtype: str
+
+ @raise smtp.SMTPBadRcpt: Raised if C{address} is invalid.
+ """
+ leap_assert_type(address, str)
+ # in the following, the address is parsed as described in RFC 2822 and
+ # ('', '') is returned if the parse fails.
+ _, address = parseaddr(address)
+ if address == '':
+ raise smtp.SMTPBadRcpt(address)
+ return address
+
+#
+# String manipulation
+#
+
+
+class CustomJsonScanner(object):
+ """
+ This class is a context manager definition used to monkey patch the default
+ json string parsing behavior.
+ The emails can have more than one encoding, so the `str` objects have more
+ than one encoding and json does not support direct work with `str`
+ (only `unicode`).
+ """
+
+ def _parse_string_str(self, s, idx, *args, **kwargs):
+ """
+ Parses the string "s" starting at the point idx and returns an `str`
+ object. Which basically means it works exactly the same as the regular
+ JSON string parsing, except that it doesn't try to decode utf8.
+ We need this because mail raw strings might have bytes in multiple
+ encodings.
+
+ :param s: the string we want to parse
+ :type s: str
+ :param idx: the starting point for parsing
+ :type idx: int
+
+ :returns: the parsed string and the index where the
+ string ends.
+ :rtype: tuple (str, int)
+ """
+ # NOTE: we just want to use this monkey patched version if we are
+ # calling the loads from our custom method. Otherwise, we use the
+ # json's default parser.
+ monkey_patched = False
+ for i in traceback.extract_stack():
+ # look for json_loads method in the call stack
+ if i[2] == json_loads.__name__:
+ monkey_patched = True
+ break
+
+ if not monkey_patched:
+ return self._orig_scanstring(s, idx, *args, **kwargs)
+
+ # TODO profile to see if a compiled regex can get us some
+ # benefit here.
+ found = False
+ end = s.find("\"", idx)
+ while not found:
+ try:
+ if s[end - 1] != "\\":
+ found = True
+ else:
+ end = s.find("\"", end + 1)
+ except Exception:
+ found = True
+ return s[idx:end].decode("string-escape"), end + 1
+
+ def __enter__(self):
+ """
+ Replace the json methods with the needed ones.
+ Also make a backup to restore them later.
+ """
+ # backup original values
+ self._orig_make_scanner = json.scanner.make_scanner
+ self._orig_scanstring = json.decoder.scanstring
+
+ # We need the make_scanner function to be the python one so we can
+ # monkey_patch the json string parsing
+ json.scanner.make_scanner = json.scanner.py_make_scanner
+
+ # And now we monkey patch the money method
+ json.decoder.scanstring = self._parse_string_str
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ """
+ Restores the backuped methods.
+ """
+ # restore original values
+ json.scanner.make_scanner = self._orig_make_scanner
+ json.decoder.scanstring = self._orig_scanstring
+
+
+def json_loads(data):
+ """
+ It works as json.loads but supporting multiple encodings in the same
+ string and accepting an `str` parameter that won't be converted to unicode.
+
+ :param data: the string to load the objects from
+ :type data: str
+
+ :returns: the corresponding python object result of parsing 'data', this
+ behaves similarly as json.loads, with the exception of that
+ returns always `str` instead of `unicode`.
+ """
+ obj = None
+ with CustomJsonScanner():
+ # We need to use the cls parameter in order to trigger the code
+ # that will let us control the string parsing method.
+ obj = json.loads(data, cls=json.JSONDecoder)
+
+ return obj
+
+
+class CaseInsensitiveDict(dict):
+ """
+ A dictionary subclass that will allow case-insenstive key lookups.
+ """
+ def __init__(self, d=None):
+ if d is None:
+ d = []
+ if isinstance(d, dict):
+ for key, value in d.items():
+ self[key] = value
+ else:
+ for key, value in d:
+ self[key] = value
+
+ def __setitem__(self, key, value):
+ super(CaseInsensitiveDict, self).__setitem__(key.lower(), value)
+
+ def __getitem__(self, key):
+ return super(CaseInsensitiveDict, self).__getitem__(key.lower())
diff --git a/src/leap/bitmask/mail/walk.py b/src/leap/bitmask/mail/walk.py
new file mode 100644
index 00000000..d143d61e
--- /dev/null
+++ b/src/leap/bitmask/mail/walk.py
@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+# walk.py
+# Copyright (C) 2013-2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Walk a message tree and generate documents that can be inserted in the backend
+store.
+"""
+from email.parser import Parser
+
+from cryptography.hazmat.backends.multibackend import MultiBackend
+from cryptography.hazmat.backends.openssl.backend import (
+ Backend as OpenSSLBackend)
+from cryptography.hazmat.primitives import hashes
+
+from leap.mail.utils import first
+
+crypto_backend = MultiBackend([OpenSSLBackend()])
+
+_parser = Parser()
+
+
+def get_tree(msg):
+ p = {}
+ p['ctype'] = msg.get_content_type()
+ p['headers'] = msg.items()
+
+ payload = msg.get_payload()
+ is_multi = msg.is_multipart()
+ if is_multi:
+ p['part_map'] = dict(
+ [(idx, get_tree(part)) for idx, part in enumerate(payload, 1)])
+ p['parts'] = len(payload)
+ p['phash'] = None
+ else:
+ p['parts'] = 0
+ p['size'] = len(payload)
+ p['phash'] = get_hash(payload)
+ p['part_map'] = {}
+ p['multi'] = is_multi
+ return p
+
+
+def get_tree_from_string(messagestr):
+ return get_tree(_parser.parsestr(messagestr))
+
+
+def get_body_phash(msg):
+ """
+ Find the body payload-hash for this message.
+ """
+ for part in msg.walk():
+ # XXX what other ctypes should be considered body?
+ if part.get_content_type() in ("text/plain", "text/html"):
+ # XXX avoid hashing again
+ return get_hash(part.get_payload())
+
+
+def get_raw_docs(msg):
+ """
+ We get also some of the headers to be able to
+ index the content. Here we remove any mutable part, as the the filename
+ in the content disposition.
+ """
+ return (
+ {'type': 'cnt',
+ 'raw': part.get_payload(),
+ 'phash': get_hash(part.get_payload()),
+ 'content-type': part.get_content_type(),
+ 'content-disposition': first(part.get(
+ 'content-disposition', '').split(';')),
+ 'content-transfer-encoding': part.get(
+ 'content-transfer-encoding', '')
+ } for part in msg.walk() if not isinstance(part.get_payload(), list))
+
+
+def get_hash(s):
+ digest = hashes.Hash(hashes.SHA256(), crypto_backend)
+ digest.update(s)
+ return digest.finalize().encode("hex").upper()
+
+
+"""
+Groucho Marx: Now pay particular attention to this first clause, because it's
+ most important. There's the party of the first part shall be
+ known in this contract as the party of the first part. How do you
+ like that, that's pretty neat eh?
+
+Chico Marx: No, that's no good.
+Groucho Marx: What's the matter with it?
+
+Chico Marx: I don't know, let's hear it again.
+Groucho Marx: So the party of the first part shall be known in this contract as
+ the party of the first part.
+"""