summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorKali Kaneko <kali@leap.se>2016-04-18 11:53:18 -0400
committerKali Kaneko <kali@leap.se>2016-04-18 11:53:18 -0400
commit6db937b20828bc39ea13836e8a70c056affa593e (patch)
treefea16d104b10dfc993079b763e7bbc64da37ed81 /src
parent9ba85bcc7724f1d9abc3ae200326e5f0a8597374 (diff)
parentd6f260f85f8464c6db6b9e158ecc85cfc02761ac (diff)
Merge tag '0.4.1'
Tag version 0.4.1
Diffstat (limited to 'src')
-rw-r--r--src/leap/mail/_version.py549
-rw-r--r--src/leap/mail/adaptors/soledad.py18
-rw-r--r--src/leap/mail/cred.py80
-rw-r--r--src/leap/mail/errors.py27
-rw-r--r--src/leap/mail/generator.py23
-rw-r--r--src/leap/mail/imap/account.py14
-rw-r--r--src/leap/mail/imap/mailbox.py22
-rw-r--r--src/leap/mail/imap/server.py56
-rw-r--r--src/leap/mail/imap/service/imap-server.tac9
-rw-r--r--src/leap/mail/imap/service/imap.py145
-rw-r--r--src/leap/mail/imap/tests/test_imap.py4
-rw-r--r--src/leap/mail/imap/tests/utils.py66
-rw-r--r--src/leap/mail/incoming/service.py74
-rw-r--r--src/leap/mail/incoming/tests/test_incoming_mail.py2
-rw-r--r--src/leap/mail/mail.py36
-rw-r--r--src/leap/mail/outgoing/service.py90
-rw-r--r--src/leap/mail/outgoing/tests/test_outgoing.py26
-rw-r--r--src/leap/mail/smtp/__init__.py50
-rw-r--r--src/leap/mail/smtp/bounces.py90
-rw-r--r--src/leap/mail/smtp/gateway.py218
-rw-r--r--src/leap/mail/smtp/tests/test_gateway.py91
-rw-r--r--src/leap/mail/tests/__init__.py2
-rw-r--r--src/leap/mail/tests/test_mail.py14
-rw-r--r--src/leap/mail/walk.py26
24 files changed, 1245 insertions, 487 deletions
diff --git a/src/leap/mail/_version.py b/src/leap/mail/_version.py
index b77d552..954f488 100644
--- a/src/leap/mail/_version.py
+++ b/src/leap/mail/_version.py
@@ -1,72 +1,157 @@
-import subprocess
-import sys
-import re
-import os.path
-IN_LONG_VERSION_PY = True
# This file helps to compute a version number in source trees obtained from
# git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (build by setup.py sdist) and build
+# feature). Distribution tarballs (built by setup.py sdist) and build
# directories (produced by setup.py build) will contain a much shorter file
# that just contains the computed version number.
# This file is released into the public domain. Generated by
-# versioneer-0.7+ (https://github.com/warner/python-versioneer)
+# versioneer-0.16 (https://github.com/warner/python-versioneer)
-# these strings will be replaced by git during git-archive
-git_refnames = "$Format:%d$"
-git_full = "$Format:%H$"
+"""Git implementation of _version.py."""
+import errno
+import os
+import re
+import subprocess
+import sys
-def run_command(args, cwd=None, verbose=False):
- try:
- # remember shell=False, so use git.cmd on windows, not just git
- p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
- except EnvironmentError:
- e = sys.exc_info()[1]
+
+def get_keywords():
+ """Get the keywords needed to look up the version information."""
+ # these strings will be replaced by git during git-archive.
+ # setup.py/versioneer.py will grep for the variable names, so they must
+ # each be defined on a line of their own. _version.py will just call
+ # get_keywords().
+ git_refnames = "$Format:%d$"
+ git_full = "$Format:%H$"
+ keywords = {"refnames": git_refnames, "full": git_full}
+ return keywords
+
+
+class VersioneerConfig:
+ """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+ """Create, populate and return the VersioneerConfig() object."""
+ # these strings are filled in when 'setup.py versioneer' creates
+ # _version.py
+ cfg = VersioneerConfig()
+ cfg.VCS = "git"
+ cfg.style = "pep440"
+ cfg.tag_prefix = ""
+ cfg.parentdir_prefix = "None"
+ cfg.versionfile_source = "src/leap/mail/_version.py"
+ cfg.verbose = False
+ return cfg
+
+
+class NotThisMethod(Exception):
+ """Exception raised if a method is not valid for the current scenario."""
+
+
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method): # decorator
+ """Decorator to mark a method as the handler for a particular VCS."""
+ def decorate(f):
+ """Store f in HANDLERS[vcs][method]."""
+ if vcs not in HANDLERS:
+ HANDLERS[vcs] = {}
+ HANDLERS[vcs][method] = f
+ return f
+ return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+ """Call the given command(s)."""
+ assert isinstance(commands, list)
+ p = None
+ for c in commands:
+ try:
+ dispcmd = str([c] + args)
+ # remember shell=False, so use git.cmd on windows, not just git
+ p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+ stderr=(subprocess.PIPE if hide_stderr
+ else None))
+ break
+ except EnvironmentError:
+ e = sys.exc_info()[1]
+ if e.errno == errno.ENOENT:
+ continue
+ if verbose:
+ print("unable to run %s" % dispcmd)
+ print(e)
+ return None
+ else:
if verbose:
- print("unable to run %s" % args[0])
- print(e)
+ print("unable to find command, tried %s" % (commands,))
return None
stdout = p.communicate()[0].strip()
- if sys.version >= '3':
+ if sys.version_info[0] >= 3:
stdout = stdout.decode()
if p.returncode != 0:
if verbose:
- print("unable to run %s (error)" % args[0])
+ print("unable to run %s (error)" % dispcmd)
return None
return stdout
-def get_expanded_variables(versionfile_source):
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+ """Try to determine the version from the parent directory name.
+
+ Source tarballs conventionally unpack into a directory that includes
+ both the project name and a version string.
+ """
+ dirname = os.path.basename(root)
+ if not dirname.startswith(parentdir_prefix):
+ if verbose:
+ print("guessing rootdir is '%s', but '%s' doesn't start with "
+ "prefix '%s'" % (root, dirname, parentdir_prefix))
+ raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+ return {"version": dirname[len(parentdir_prefix):],
+ "full-revisionid": None,
+ "dirty": False, "error": None}
+
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+ """Extract version information from the given file."""
# the code embedded in _version.py can just fetch the value of these
- # variables. When used from setup.py, we don't want to import
- # _version.py, so we do it with a regexp instead. This function is not
- # used from _version.py.
- variables = {}
+ # keywords. When used from setup.py, we don't want to import _version.py,
+ # so we do it with a regexp instead. This function is not used from
+ # _version.py.
+ keywords = {}
try:
- f = open(versionfile_source, "r")
+ f = open(versionfile_abs, "r")
for line in f.readlines():
if line.strip().startswith("git_refnames ="):
mo = re.search(r'=\s*"(.*)"', line)
if mo:
- variables["refnames"] = mo.group(1)
+ keywords["refnames"] = mo.group(1)
if line.strip().startswith("git_full ="):
mo = re.search(r'=\s*"(.*)"', line)
if mo:
- variables["full"] = mo.group(1)
+ keywords["full"] = mo.group(1)
f.close()
except EnvironmentError:
pass
- return variables
+ return keywords
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
- refnames = variables["refnames"].strip()
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+ """Get version information from git keywords."""
+ if not keywords:
+ raise NotThisMethod("no keywords at all, weird")
+ refnames = keywords["refnames"].strip()
if refnames.startswith("$Format"):
if verbose:
- print("variables are unexpanded, not using")
- return {} # unexpanded, so not in an unpacked git-archive tarball
+ print("keywords are unexpanded, not using")
+ raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
refs = set([r.strip() for r in refnames.strip("()").split(",")])
# starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
# just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -82,7 +167,7 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
# "stabilization", as well as "HEAD" and "master".
tags = set([r for r in refs if re.search(r'\d', r)])
if verbose:
- print("discarding '%s', no digits" % ",".join(refs - tags))
+ print("discarding '%s', no digits" % ",".join(refs-tags))
if verbose:
print("likely tags: %s" % ",".join(sorted(tags)))
for ref in sorted(tags):
@@ -92,114 +177,308 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
if verbose:
print("picking %s" % r)
return {"version": r,
- "full": variables["full"].strip()}
- # no suitable tags, so we use the full revision id
+ "full-revisionid": keywords["full"].strip(),
+ "dirty": False, "error": None
+ }
+ # no suitable tags, so version is "0+unknown", but full hex is still there
if verbose:
- print("no suitable tags, using full revision id")
- return {"version": variables["full"].strip(),
- "full": variables["full"].strip()}
-
-
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
- # this runs 'git' from the root of the source tree. That either means
- # someone ran a setup.py command (and this code is in versioneer.py, so
- # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
- # the source tree), or someone ran a project-specific entry point (and
- # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
- # containing directory is somewhere deeper in the source tree). This only
- # gets called if the git-archive 'subst' variables were *not* expanded,
- # and _version.py hasn't already been rewritten with a short version
- # string, meaning we're inside a checked out source tree.
+ print("no suitable tags, using unknown + full revision id")
+ return {"version": "0+unknown",
+ "full-revisionid": keywords["full"].strip(),
+ "dirty": False, "error": "no suitable tags"}
- try:
- here = os.path.abspath(__file__)
- except NameError:
- # some py2exe/bbfreeze/non-CPython implementations don't do __file__
- return {} # not always correct
-
- # versionfile_source is the relative path from the top of the source tree
- # (where the .git directory might live) to this file. Invert this to find
- # the root from __file__.
- root = here
- if IN_LONG_VERSION_PY:
- for i in range(len(versionfile_source.split("/"))):
- root = os.path.dirname(root)
- else:
- root = os.path.dirname(here)
+
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+ """Get version from 'git describe' in the root of the source tree.
+
+ This only gets called if the git-archive 'subst' keywords were *not*
+ expanded, and _version.py hasn't already been rewritten with a short
+ version string, meaning we're inside a checked out source tree.
+ """
if not os.path.exists(os.path.join(root, ".git")):
if verbose:
print("no .git in %s" % root)
- return {}
+ raise NotThisMethod("no .git directory")
- GIT = "git"
+ GITS = ["git"]
if sys.platform == "win32":
- GIT = "git.cmd"
- stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
- cwd=root)
- if stdout is None:
- return {}
- if not stdout.startswith(tag_prefix):
- if verbose:
- print("tag '%s' doesn't start with prefix '%s'" %
- (stdout, tag_prefix))
- return {}
- tag = stdout[len(tag_prefix):]
- stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
- if stdout is None:
- return {}
- full = stdout.strip()
- if tag.endswith("-dirty"):
- full += "-dirty"
- return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source,
- verbose=False):
- if IN_LONG_VERSION_PY:
- # We're running from _version.py. If it's from a source tree
- # (execute-in-place), we can work upwards to find the root of the
- # tree, and then check the parent directory for a version string. If
- # it's in an installed application, there's no hope.
- try:
- here = os.path.abspath(__file__)
- except NameError:
- # py2exe/bbfreeze/non-CPython don't have __file__
- return {} # without __file__, we have no hope
+ GITS = ["git.cmd", "git.exe"]
+ # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+ # if there isn't one, this yields HEX[-dirty] (no NUM)
+ describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
+ "--always", "--long",
+ "--match", "%s*" % tag_prefix],
+ cwd=root)
+ # --long was added in git-1.5.5
+ if describe_out is None:
+ raise NotThisMethod("'git describe' failed")
+ describe_out = describe_out.strip()
+ full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+ if full_out is None:
+ raise NotThisMethod("'git rev-parse' failed")
+ full_out = full_out.strip()
+
+ pieces = {}
+ pieces["long"] = full_out
+ pieces["short"] = full_out[:7] # maybe improved later
+ pieces["error"] = None
+
+ # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+ # TAG might have hyphens.
+ git_describe = describe_out
+
+ # look for -dirty suffix
+ dirty = git_describe.endswith("-dirty")
+ pieces["dirty"] = dirty
+ if dirty:
+ git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+ # now we have TAG-NUM-gHEX or HEX
+
+ if "-" in git_describe:
+ # TAG-NUM-gHEX
+ mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+ if not mo:
+ # unparseable. Maybe git-describe is misbehaving?
+ pieces["error"] = ("unable to parse git-describe output: '%s'"
+ % describe_out)
+ return pieces
+
+ # tag
+ full_tag = mo.group(1)
+ if not full_tag.startswith(tag_prefix):
+ if verbose:
+ fmt = "tag '%s' doesn't start with prefix '%s'"
+ print(fmt % (full_tag, tag_prefix))
+ pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+ % (full_tag, tag_prefix))
+ return pieces
+ pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+ # distance: number of commits since tag
+ pieces["distance"] = int(mo.group(2))
+
+ # commit: short hex revision ID
+ pieces["short"] = mo.group(3)
+
+ else:
+ # HEX: no tags
+ pieces["closest-tag"] = None
+ count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
+ cwd=root)
+ pieces["distance"] = int(count_out) # total number of commits
+
+ return pieces
+
+
+def plus_or_dot(pieces):
+ """Return a + if we don't already have one, else return a ."""
+ if "+" in pieces.get("closest-tag", ""):
+ return "."
+ return "+"
+
+
+def render_pep440(pieces):
+ """Build up version string, with post-release "local version identifier".
+
+ Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+ get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+ Exceptions:
+ 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"] or pieces["dirty"]:
+ rendered += plus_or_dot(pieces)
+ rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+ if pieces["dirty"]:
+ rendered += ".dirty"
+ else:
+ # exception #1
+ rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+ pieces["short"])
+ if pieces["dirty"]:
+ rendered += ".dirty"
+ return rendered
+
+
+def render_pep440_pre(pieces):
+ """TAG[.post.devDISTANCE] -- No -dirty.
+
+ Exceptions:
+ 1: no tags. 0.post.devDISTANCE
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"]:
+ rendered += ".post.dev%d" % pieces["distance"]
+ else:
+ # exception #1
+ rendered = "0.post.dev%d" % pieces["distance"]
+ return rendered
+
+
+def render_pep440_post(pieces):
+ """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+ The ".dev0" means dirty. Note that .dev0 sorts backwards
+ (a dirty tree will appear "older" than the corresponding clean one),
+ but you shouldn't be releasing software with -dirty anyways.
+
+ Exceptions:
+ 1: no tags. 0.postDISTANCE[.dev0]
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"] or pieces["dirty"]:
+ rendered += ".post%d" % pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ rendered += plus_or_dot(pieces)
+ rendered += "g%s" % pieces["short"]
+ else:
+ # exception #1
+ rendered = "0.post%d" % pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ rendered += "+g%s" % pieces["short"]
+ return rendered
+
+
+def render_pep440_old(pieces):
+ """TAG[.postDISTANCE[.dev0]] .
+
+ The ".dev0" means dirty.
+
+ Eexceptions:
+ 1: no tags. 0.postDISTANCE[.dev0]
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"] or pieces["dirty"]:
+ rendered += ".post%d" % pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ else:
+ # exception #1
+ rendered = "0.post%d" % pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ return rendered
+
+
+def render_git_describe(pieces):
+ """TAG[-DISTANCE-gHEX][-dirty].
+
+ Like 'git describe --tags --dirty --always'.
+
+ Exceptions:
+ 1: no tags. HEX[-dirty] (note: no 'g' prefix)
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"]:
+ rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+ else:
+ # exception #1
+ rendered = pieces["short"]
+ if pieces["dirty"]:
+ rendered += "-dirty"
+ return rendered
+
+
+def render_git_describe_long(pieces):
+ """TAG-DISTANCE-gHEX[-dirty].
+
+ Like 'git describe --tags --dirty --always -long'.
+ The distance/hash is unconditional.
+
+ Exceptions:
+ 1: no tags. HEX[-dirty] (note: no 'g' prefix)
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+ else:
+ # exception #1
+ rendered = pieces["short"]
+ if pieces["dirty"]:
+ rendered += "-dirty"
+ return rendered
+
+
+def render(pieces, style):
+ """Render the given version pieces into the requested style."""
+ if pieces["error"]:
+ return {"version": "unknown",
+ "full-revisionid": pieces.get("long"),
+ "dirty": None,
+ "error": pieces["error"]}
+
+ if not style or style == "default":
+ style = "pep440" # the default
+
+ if style == "pep440":
+ rendered = render_pep440(pieces)
+ elif style == "pep440-pre":
+ rendered = render_pep440_pre(pieces)
+ elif style == "pep440-post":
+ rendered = render_pep440_post(pieces)
+ elif style == "pep440-old":
+ rendered = render_pep440_old(pieces)
+ elif style == "git-describe":
+ rendered = render_git_describe(pieces)
+ elif style == "git-describe-long":
+ rendered = render_git_describe_long(pieces)
+ else:
+ raise ValueError("unknown style '%s'" % style)
+
+ return {"version": rendered, "full-revisionid": pieces["long"],
+ "dirty": pieces["dirty"], "error": None}
+
+
+def get_versions():
+ """Get version information or return default if unable to do so."""
+ # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
+ # __file__, we can work backwards from there to the root. Some
+ # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
+ # case we can only use expanded keywords.
+
+ cfg = get_config()
+ verbose = cfg.verbose
+
+ try:
+ return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
+ verbose)
+ except NotThisMethod:
+ pass
+
+ try:
+ root = os.path.realpath(__file__)
# versionfile_source is the relative path from the top of the source
- # tree to _version.py. Invert this to find the root from __file__.
- root = here
- for i in range(len(versionfile_source.split("/"))):
+ # tree (where the .git directory might live) to this file. Invert
+ # this to find the root from __file__.
+ for i in cfg.versionfile_source.split('/'):
root = os.path.dirname(root)
- else:
- # we're running from versioneer.py, which means we're running from
- # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
- here = os.path.abspath(sys.argv[0])
- root = os.path.dirname(here)
+ except NameError:
+ return {"version": "0+unknown", "full-revisionid": None,
+ "dirty": None,
+ "error": "unable to find root of source tree"}
- # Source tarballs conventionally unpack into a directory that includes
- # both the project name and a version string.
- dirname = os.path.basename(root)
- if not dirname.startswith(parentdir_prefix):
- if verbose:
- print("guessing rootdir is '%s', but '%s' doesn't "
- "start with prefix '%s'" %
- (root, dirname, parentdir_prefix))
- return None
- return {"version": dirname[len(parentdir_prefix):], "full": ""}
-
-tag_prefix = ""
-parentdir_prefix = "leap-mail"
-versionfile_source = "src/leap/mail/_version.py"
-
-
-def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
- variables = {"refnames": git_refnames, "full": git_full}
- ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
- if not ver:
- ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
- if not ver:
- ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
- verbose)
- if not ver:
- ver = default
- return ver
+ try:
+ pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+ return render(pieces, cfg.style)
+ except NotThisMethod:
+ pass
+
+ try:
+ if cfg.parentdir_prefix:
+ return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+ except NotThisMethod:
+ pass
+
+ return {"version": "0+unknown", "full-revisionid": None,
+ "dirty": None,
+ "error": "unable to compute version"}
diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py
index 8de83f7..f4af020 100644
--- a/src/leap/mail/adaptors/soledad.py
+++ b/src/leap/mail/adaptors/soledad.py
@@ -22,7 +22,6 @@ import re
from collections import defaultdict
from email import message_from_string
-from pycryptopp.hash import sha256
from twisted.internet import defer
from twisted.python import log
from zope.interface import implements
@@ -687,13 +686,14 @@ class MessageWrapper(object):
:rtype: deferred
"""
body_phash = self.hdoc.body
- if not body_phash:
- if self.cdocs:
- return self.cdocs[1]
- d = store.get_doc('C-' + body_phash)
- d.addCallback(lambda doc: ContentDocWrapper(**doc.content))
- return d
-
+ if body_phash:
+ d = store.get_doc('C-' + body_phash)
+ d.addCallback(lambda doc: ContentDocWrapper(**doc.content))
+ return d
+ elif self.cdocs:
+ return self.cdocs[1]
+ else:
+ return ''
#
# Mailboxes
@@ -1207,7 +1207,7 @@ def _split_into_parts(raw):
def _parse_msg(raw):
msg = message_from_string(raw)
parts = walk.get_parts(msg)
- chash = sha256.SHA256(raw).hexdigest()
+ chash = walk.get_hash(raw)
multi = msg.is_multipart()
return msg, parts, chash, multi
diff --git a/src/leap/mail/cred.py b/src/leap/mail/cred.py
new file mode 100644
index 0000000..7eab1f0
--- /dev/null
+++ b/src/leap/mail/cred.py
@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+# cred.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Credentials handling.
+"""
+
+from zope.interface import implementer
+from twisted.cred.checkers import ICredentialsChecker
+from twisted.cred.credentials import IUsernamePassword
+from twisted.cred.error import UnauthorizedLogin
+from twisted.internet import defer
+
+
+@implementer(ICredentialsChecker)
+class LocalSoledadTokenChecker(object):
+
+ """
+ A Credentials Checker for a LocalSoledad store.
+
+ It checks that:
+
+ 1) The Local SoledadStorage has been correctly unlocked for the given
+ user. This currently means that the right passphrase has been passed
+ to the Local SoledadStorage.
+
+ 2) The password passed in the credentials matches whatever token has
+ been stored in the local encrypted SoledadStorage, associated to the
+ Protocol that is requesting the authentication.
+ """
+
+ credentialInterfaces = (IUsernamePassword,)
+ service = None
+
+ def __init__(self, soledad_sessions):
+ """
+ :param soledad_sessions: a dict-like object, containing instances
+ of a Store (soledad instances), indexed by
+ userid.
+ """
+ self._soledad_sessions = soledad_sessions
+
+ def requestAvatarId(self, credentials):
+ if self.service is None:
+ raise NotImplementedError(
+ "this checker has not defined its service name")
+ username, password = credentials.username, credentials.password
+ d = self.checkSoledadToken(username, password, self.service)
+ d.addErrback(lambda f: defer.fail(UnauthorizedLogin()))
+ return d
+
+ def checkSoledadToken(self, username, password, service):
+ soledad = self._soledad_sessions.get(username)
+ if not soledad:
+ return defer.fail(Exception("No soledad"))
+
+ def match_token(token):
+ if token is None:
+ raise RuntimeError('no token')
+ if token == password:
+ return username
+ else:
+ raise RuntimeError('bad token')
+
+ d = soledad.get_or_create_service_token(service)
+ d.addCallback(match_token)
+ return d
diff --git a/src/leap/mail/errors.py b/src/leap/mail/errors.py
new file mode 100644
index 0000000..2f18e87
--- /dev/null
+++ b/src/leap/mail/errors.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# errors.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Exceptions for leap.mail
+"""
+
+
+class AuthenticationError(Exception):
+ pass
+
+
+class ConfigurationError(Exception):
+ pass
diff --git a/src/leap/mail/generator.py b/src/leap/mail/generator.py
new file mode 100644
index 0000000..bb3f26e
--- /dev/null
+++ b/src/leap/mail/generator.py
@@ -0,0 +1,23 @@
+from email.generator import Generator as EmailGenerator
+
+
+class Generator(EmailGenerator):
+ """
+ Generates output from a Message object tree, keeping signatures.
+
+ This code was extracted from Mailman.Generator.Generator, version 2.1.4:
+
+ Most other Generator will be created not setting the foldheader flag,
+ as we do not overwrite clone(). The original clone() does not
+ set foldheaders.
+
+ So you need to set foldheaders if you want the toplevel to fold headers
+
+ TODO: Python 3.3 is patched against this problems. See issue 1590744 on
+ python bug tracker.
+ """
+ def _write_headers(self, msg):
+ for h, v in msg.items():
+ print >> self._fp, '%s:' % h,
+ print >> self._fp, v
+ print >> self._fp
diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index cc56fff..459b0ba 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -49,6 +49,7 @@ if PROFILE_CMD:
# Soledad IMAP Account
#######################################
+
class IMAPAccount(object):
"""
An implementation of an imap4 Account
@@ -59,7 +60,7 @@ class IMAPAccount(object):
selected = None
- def __init__(self, user_id, store, d=defer.Deferred()):
+ def __init__(self, store, user_id, d=defer.Deferred()):
"""
Keeps track of the mailboxes and subscriptions handled by this account.
@@ -68,13 +69,14 @@ class IMAPAccount(object):
You can either pass a deferred to this constructor, or use
`callWhenReady` method.
- :param user_id: The name of the account (user id, in the form
- user@provider).
- :type user_id: str
-
:param store: a Soledad instance.
:type store: Soledad
+ :param user_id: The identifier of the user this account belongs to
+ (user id, in the form user@provider).
+ :type user_id: str
+
+
:param d: a deferred that will be fired with this IMAPAccount instance
when the account is ready to be used.
:type d: defer.Deferred
@@ -87,7 +89,7 @@ class IMAPAccount(object):
# about user_id, only the client backend.
self.user_id = user_id
- self.account = Account(store, ready_cb=lambda: d.callback(self))
+ self.account = Account(store, user_id, ready_cb=lambda: d.callback(self))
def end_session(self):
"""
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index bfc0bfc..d545c00 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -508,7 +508,7 @@ class IMAPMailbox(object):
def get_range(messages_asked):
return self._filter_msg_seq(messages_asked)
- d = defer.maybeDeferred(self._bound_seq, messages_asked, uid)
+ d = self._bound_seq(messages_asked, uid)
if uid:
d.addCallback(get_range)
d.addErrback(lambda f: log.err(f))
@@ -520,7 +520,7 @@ class IMAPMailbox(object):
:param messages_asked: IDs of the messages.
:type messages_asked: MessageSet
- :rtype: MessageSet
+ :return: a Deferred that will fire with a MessageSet
"""
def set_last_uid(last_uid):
@@ -543,7 +543,7 @@ class IMAPMailbox(object):
d = self.collection.all_uid_iter()
d.addCallback(set_last_seq)
return d
- return messages_asked
+ return defer.succeed(messages_asked)
def _filter_msg_seq(self, messages_asked):
"""
@@ -713,6 +713,7 @@ class IMAPMailbox(object):
d_seq.addCallback(get_flags_for_seq)
return d_seq
+ @defer.inlineCallbacks
def fetch_headers(self, messages_asked, uid):
"""
A fast method to fetch all headers, tricking just the
@@ -757,14 +758,15 @@ class IMAPMailbox(object):
for key, value in
self.headers.items())
- messages_asked = self._bound_seq(messages_asked)
- seq_messg = self._filter_msg_seq(messages_asked)
+ messages_asked = yield self._bound_seq(messages_asked, uid)
+ seq_messg = yield self._filter_msg_seq(messages_asked)
- all_headers = self.messages.all_headers()
- result = ((msgid, headersPart(
- msgid, all_headers.get(msgid, {})))
- for msgid in seq_messg)
- return result
+ result = []
+ for msgid in seq_messg:
+ msg = yield self.collection.get_message_by_uid(msgid)
+ headers = headersPart(msgid, msg.get_headers())
+ result.append((msgid, headers))
+ defer.returnValue(iter(result))
def store(self, messages_asked, flags, mode, uid):
"""
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 99e7174..5a63af0 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -20,21 +20,17 @@ LEAP IMAP4 Server Implementation.
import StringIO
from copy import copy
-from twisted import cred
-from twisted.internet import reactor
from twisted.internet.defer import maybeDeferred
from twisted.mail import imap4
from twisted.python import log
-from leap.common.check import leap_assert, leap_assert_type
-from leap.common.events import emit_async, catalog
-from leap.soledad.client import Soledad
-
# imports for LITERAL+ patch
from twisted.internet import defer, interfaces
from twisted.mail.imap4 import IllegalClientResponse
from twisted.mail.imap4 import LiteralString, LiteralFile
+from leap.common.events import emit_async, catalog
+
def _getContentType(msg):
"""
@@ -72,25 +68,6 @@ class LEAPIMAPServer(imap4.IMAP4Server):
"""
An IMAP4 Server with a LEAP Storage Backend.
"""
- def __init__(self, *args, **kwargs):
- # pop extraneous arguments
- soledad = kwargs.pop('soledad', None)
- uuid = kwargs.pop('uuid', None)
- userid = kwargs.pop('userid', None)
-
- leap_assert(soledad, "need a soledad instance")
- leap_assert_type(soledad, Soledad)
- leap_assert(uuid, "need a user in the initialization")
-
- self._userid = userid
-
- # initialize imap server!
- imap4.IMAP4Server.__init__(self, *args, **kwargs)
-
- # we should initialize the account here,
- # but we move it to the factory so we can
- # populate the test account properly (and only once
- # per session)
#############################################################
#
@@ -181,10 +158,6 @@ class LEAPIMAPServer(imap4.IMAP4Server):
:param line: the line from the server, without the line delimiter.
:type line: str
"""
- if self.theAccount.session_ended is True and self.state != "unauth":
- log.msg("Closing the session. State: unauth")
- self.state = "unauth"
-
if "login" in line.lower():
# avoid to log the pass, even though we are using a dummy auth
# by now.
@@ -208,25 +181,6 @@ class LEAPIMAPServer(imap4.IMAP4Server):
self.mbox = None
self.state = 'unauth'
- def authenticateLogin(self, username, password):
- """
- Lookup the account with the given parameters, and deny
- the improper combinations.
-
- :param username: the username that is attempting authentication.
- :type username: str
- :param password: the password to authenticate with.
- :type password: str
- """
- # XXX this should use portal:
- # return portal.login(cred.credentials.UsernamePassword(user, pass)
- if username != self._userid:
- # bad username, reject.
- raise cred.error.UnauthorizedLogin()
- # any dummy password is allowed so far. use realm instead!
- emit_async(catalog.IMAP_CLIENT_LOGIN, "1")
- return imap4.IAccount, self.theAccount, lambda: None
-
def do_FETCH(self, tag, messages, query, uid=0):
"""
Overwritten fetch dispatcher to use the fast fetch_flags
@@ -657,7 +611,6 @@ class LEAPIMAPServer(imap4.IMAP4Server):
d.addCallback(send_response)
return d
# XXX patched ---------------------------------
-
# -----------------------------------------------------------------------
auth_APPEND = (do_APPEND, arg_astring, imap4.IMAP4Server.opt_plist,
@@ -733,3 +686,8 @@ class LEAPIMAPServer(imap4.IMAP4Server):
#############################################################
# END of Twisted imap4 patch to support LITERAL+ extension
#############################################################
+
+ def authenticateLogin(self, user, passwd):
+ result = imap4.IMAP4Server.authenticateLogin(self, user, passwd)
+ emit_async(catalog.IMAP_CLIENT_LOGIN, str(user))
+ return result
diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac
index 2045757..c4d602d 100644
--- a/src/leap/mail/imap/service/imap-server.tac
+++ b/src/leap/mail/imap/service/imap-server.tac
@@ -1,3 +1,4 @@
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
# imap-server.tac
# Copyright (C) 2013,2014 LEAP
@@ -27,6 +28,9 @@ userid = 'user@provider'
uuid = 'deadbeefdeadabad'
passwd = 'supersecret' # optional, will get prompted if not found.
"""
+
+# TODO -- this .tac file should be deprecated in favor of bitmask.core.bitmaskd
+
import ConfigParser
import getpass
import os
@@ -112,7 +116,7 @@ tempdir = "/tmp/"
print "[~] user:", userid
soledad = initialize_soledad(uuid, userid, passwd, secrets,
- localdb, gnupg_home, tempdir)
+ localdb, gnupg_home, tempdir, userid=userid)
km_args = (userid, "https://localhost", soledad)
km_kwargs = {
"token": "",
@@ -131,7 +135,8 @@ keymanager = KeyManager(*km_args, **km_kwargs)
def getIMAPService():
- factory = imap.LeapIMAPFactory(uuid, userid, soledad)
+ soledad_sessions = {userid: soledad}
+ factory = imap.LeapIMAPFactory(soledad_sessions)
return internet.TCPServer(port, factory, interface="localhost")
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index a50611b..4663854 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -15,21 +15,24 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
-IMAP service initialization
+IMAP Service Initialization.
"""
import logging
import os
from collections import defaultdict
+from twisted.cred.portal import Portal, IRealm
+from twisted.mail.imap4 import IAccount
+from twisted.internet import defer
from twisted.internet import reactor
from twisted.internet.error import CannotListenError
from twisted.internet.protocol import ServerFactory
-from twisted.mail import imap4
from twisted.python import log
+from zope.interface import implementer
from leap.common.events import emit_async, catalog
-from leap.common.check import leap_check
+from leap.mail.cred import LocalSoledadTokenChecker
from leap.mail.imap.account import IMAPAccount
from leap.mail.imap.server import LEAPIMAPServer
@@ -41,57 +44,92 @@ DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None)
if DO_MANHOLE:
from leap.mail.imap.service import manhole
-DO_PROFILE = os.environ.get("LEAP_PROFILE", None)
-if DO_PROFILE:
- import cProfile
- log.msg("Starting PROFILING...")
-
- PROFILE_DAT = "/tmp/leap_mail_profile.pstats"
- pr = cProfile.Profile()
- pr.enable()
-
# The default port in which imap service will run
+
IMAP_PORT = 1984
+#
+# Credentials Handling
+#
+
+
+@implementer(IRealm)
+class LocalSoledadIMAPRealm(object):
+
+ _encoding = 'utf-8'
+
+ def __init__(self, soledad_sessions):
+ """
+ :param soledad_sessions: a dict-like object, containing instances
+ of a Store (soledad instances), indexed by
+ userid.
+ """
+ self._soledad_sessions = soledad_sessions
+
+ def requestAvatar(self, avatarId, mind, *interfaces):
+ if isinstance(avatarId, str):
+ avatarId = avatarId.decode(self._encoding)
+
+ def gotSoledad(soledad):
+ for iface in interfaces:
+ if iface is IAccount:
+ avatar = IMAPAccount(soledad, avatarId)
+ return (IAccount, avatar,
+ getattr(avatar, 'logout', lambda: None))
+ raise NotImplementedError(self, interfaces)
+
+ return self.lookupSoledadInstance(avatarId).addCallback(gotSoledad)
+
+ def lookupSoledadInstance(self, userid):
+ soledad = self._soledad_sessions[userid]
+ # XXX this should return the instance after whenReady callback
+ return defer.succeed(soledad)
+
+
+class IMAPTokenChecker(LocalSoledadTokenChecker):
+ """A credentials checker that will lookup a token for the IMAP service.
+ For now it will be using the same identifier than SMTPTokenChecker"""
+
+ service = 'mail_auth'
+
+
+class LocalSoledadIMAPServer(LEAPIMAPServer):
-class IMAPAuthRealm(object):
"""
- Dummy authentication realm. Do not use in production!
+ An IMAP Server that authenticates against a LocalSoledad store.
"""
- theAccount = None
- def requestAvatar(self, avatarId, mind, *interfaces):
- return imap4.IAccount, self.theAccount, lambda: None
+ def __init__(self, soledad_sessions, *args, **kw):
+
+ LEAPIMAPServer.__init__(self, *args, **kw)
+
+ realm = LocalSoledadIMAPRealm(soledad_sessions)
+ portal = Portal(realm)
+ checker = IMAPTokenChecker(soledad_sessions)
+ self.checker = checker
+ self.portal = portal
+ portal.registerChecker(checker)
class LeapIMAPFactory(ServerFactory):
+
"""
Factory for a IMAP4 server with soledad remote sync and gpg-decryption
capabilities.
"""
- protocol = LEAPIMAPServer
- def __init__(self, uuid, userid, soledad):
+ protocol = LocalSoledadIMAPServer
+
+ def __init__(self, soledad_sessions):
"""
Initializes the server factory.
- :param uuid: user uuid
- :type uuid: str
-
- :param userid: user id (user@provider.org)
- :type userid: str
-
- :param soledad: soledad instance
- :type soledad: Soledad
+ :param soledad_sessions: a dict-like object, containing instances
+ of a Store (soledad instances), indexed by
+ userid.
"""
- self._uuid = uuid
- self._userid = userid
- self._soledad = soledad
-
- theAccount = IMAPAccount(uuid, soledad)
- self.theAccount = theAccount
+ self._soledad_sessions = soledad_sessions
self._connections = defaultdict()
- # XXX how to pass the store along?
def buildProtocol(self, addr):
"""
@@ -103,13 +141,7 @@ class LeapIMAPFactory(ServerFactory):
# TODO should reject anything from addr != localhost,
# just in case.
log.msg("Building protocol for connection %s" % addr)
- imapProtocol = self.protocol(
- uuid=self._uuid,
- userid=self._userid,
- soledad=self._soledad)
- imapProtocol.theAccount = self.theAccount
- imapProtocol.factory = self
-
+ imapProtocol = self.protocol(self._soledad_sessions)
self._connections[addr] = imapProtocol
return imapProtocol
@@ -123,39 +155,21 @@ class LeapIMAPFactory(ServerFactory):
"""
Stops imap service (fetcher, factory and port).
"""
- # mark account as unusable, so any imap command will fail
- # with unauth state.
- self.theAccount.end_session()
-
- # TODO should wait for all the pending deferreds,
- # the twisted way!
- if DO_PROFILE:
- log.msg("Stopping PROFILING")
- pr.disable()
- pr.dump_stats(PROFILE_DAT)
-
return ServerFactory.doStop(self)
-def run_service(store, **kwargs):
+def run_service(soledad_sessions, port=IMAP_PORT):
"""
Main entry point to run the service from the client.
- :param store: a soledad instance
+ :param soledad_sessions: a dict-like object, containing instances
+ of a Store (soledad instances), indexed by userid.
:returns: the port as returned by the reactor when starts listening, and
the factory for the protocol.
+ :rtype: tuple
"""
- leap_check(store, "store cannot be None")
- # XXX this can also be a ProxiedObject, FIXME
- # leap_assert_type(store, Soledad)
-
- port = kwargs.get('port', IMAP_PORT)
- userid = kwargs.get('userid', None)
- leap_check(userid is not None, "need an user id")
-
- uuid = store.uuid
- factory = LeapIMAPFactory(uuid, userid, store)
+ factory = LeapIMAPFactory(soledad_sessions)
try:
interface = "localhost"
@@ -164,6 +178,7 @@ def run_service(store, **kwargs):
if os.environ.get("LEAP_DOCKERIZED"):
interface = ''
+ # TODO use Endpoints !!!
tport = reactor.listenTCP(port, factory,
interface=interface)
except CannotListenError:
@@ -178,9 +193,9 @@ def run_service(store, **kwargs):
# TODO get pass from env var.too.
manhole_factory = manhole.getManholeFactory(
{'f': factory,
- 'a': factory.theAccount,
'gm': factory.theAccount.getMailbox},
"boss", "leap")
+ # TODO use Endpoints !!!
reactor.listenTCP(manhole.MANHOLE_PORT, manhole_factory,
interface="127.0.0.1")
logger.debug("IMAP4 Server is RUNNING in port %s" % (port,))
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index 62c3c41..ccce285 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -575,8 +575,8 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin):
"""
Test login requiring quoting
"""
- self.server._userid = '{test}user@leap.se'
- self.server._password = '{test}password'
+ self.server.checker.userid = '{test}user@leap.se'
+ self.server.checker.password = '{test}password'
def login():
d = self.client.login('{test}user@leap.se', '{test}password')
diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py
index a34538b..64a0326 100644
--- a/src/leap/mail/imap/tests/utils.py
+++ b/src/leap/mail/imap/tests/utils.py
@@ -20,10 +20,15 @@ Common utilities for testing Soledad IMAP Server.
from email import parser
from mock import Mock
+from twisted.cred.checkers import ICredentialsChecker
+from twisted.cred.credentials import IUsernamePassword
+from twisted.cred.error import UnauthorizedLogin
+from twisted.cred.portal import Portal, IRealm
from twisted.mail import imap4
from twisted.internet import defer
from twisted.protocols import loopback
from twisted.python import log
+from zope.interface import implementer
from leap.mail.adaptors import soledad as soledad_adaptor
from leap.mail.imap.account import IMAPAccount
@@ -64,6 +69,57 @@ class SimpleClient(imap4.IMAP4Client):
self.events.append(['newMessages', exists, recent])
self.transport.loseConnection()
+#
+# Dummy credentials for tests
+#
+
+
+@implementer(IRealm)
+class TestRealm(object):
+
+ def __init__(self, account):
+ self._account = account
+
+ def requestAvatar(self, avatarId, mind, *interfaces):
+ avatar = self._account
+ return (imap4.IAccount, avatar,
+ getattr(avatar, 'logout', lambda: None))
+
+
+@implementer(ICredentialsChecker)
+class TestCredentialsChecker(object):
+
+ credentialInterfaces = (IUsernamePassword,)
+
+ userid = TEST_USER
+ password = TEST_PASSWD
+
+ def requestAvatarId(self, credentials):
+ username, password = credentials.username, credentials.password
+ d = self.checkTestCredentials(username, password)
+ d.addErrback(lambda f: defer.fail(UnauthorizedLogin()))
+ return d
+
+ def checkTestCredentials(self, username, password):
+ if username == self.userid and password == self.password:
+ return defer.succeed(username)
+ else:
+ return defer.fail(Exception("Wrong credentials"))
+
+
+class TestSoledadIMAPServer(LEAPIMAPServer):
+
+ def __init__(self, account, *args, **kw):
+
+ LEAPIMAPServer.__init__(self, *args, **kw)
+
+ realm = TestRealm(account)
+ portal = Portal(realm)
+ checker = TestCredentialsChecker()
+ self.checker = checker
+ self.portal = portal
+ portal.registerChecker(checker)
+
class IMAP4HelperMixin(SoledadTestMixin):
"""
@@ -77,14 +133,12 @@ class IMAP4HelperMixin(SoledadTestMixin):
soledad_adaptor.cleanup_deferred_locks()
- UUID = 'deadbeef',
USERID = TEST_USER
def setup_server(account):
- self.server = LEAPIMAPServer(
- uuid=UUID, userid=USERID,
- contextFactory=self.serverCTX,
- soledad=self._soledad)
+ self.server = TestSoledadIMAPServer(
+ account=account,
+ contextFactory=self.serverCTX)
self.server.theAccount = account
d_server_ready = defer.Deferred()
@@ -104,7 +158,7 @@ class IMAP4HelperMixin(SoledadTestMixin):
self._soledad.sync = Mock()
d = defer.Deferred()
- self.acc = IMAPAccount(USERID, self._soledad, d=d)
+ self.acc = IMAPAccount(self._soledad, USERID, d=d)
return d
d = super(IMAP4HelperMixin, self).setUp()
diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py
index d8b91ba..c7d194d 100644
--- a/src/leap/mail/incoming/service.py
+++ b/src/leap/mail/incoming/service.py
@@ -24,7 +24,6 @@ import time
import warnings
from email.parser import Parser
-from email.generator import Generator
from email.utils import parseaddr
from email.utils import formatdate
from StringIO import StringIO
@@ -43,6 +42,7 @@ from leap.common.mail import get_email_charset
from leap.keymanager import errors as keymanager_errors
from leap.keymanager.openpgp import OpenPGPKey
from leap.mail.adaptors import soledad_indexes as fields
+from leap.mail.generator import Generator
from leap.mail.utils import json_loads, empty
from leap.soledad.client import Soledad
from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY
@@ -233,7 +233,7 @@ class IncomingMail(Service):
failure.trap(InvalidAuthTokenError)
# if the token is invalid, send an event so the GUI can
# disable mail and show an error message.
- emit_async(catalog.SOLEDAD_INVALID_AUTH_TOKEN)
+ emit_async(catalog.SOLEDAD_INVALID_AUTH_TOKEN, self._userid)
log.msg('FETCH: syncing soledad...')
d = self._soledad.sync()
@@ -254,7 +254,7 @@ class IncomingMail(Service):
num_mails = len(doclist) if doclist is not None else 0
if num_mails != 0:
log.msg("there are %s mails" % (num_mails,))
- emit_async(catalog.MAIL_FETCHED_INCOMING,
+ emit_async(catalog.MAIL_FETCHED_INCOMING, self._userid,
str(num_mails), str(fetched_ts))
return doclist
@@ -262,7 +262,7 @@ class IncomingMail(Service):
"""
Sends unread event to ui.
"""
- emit_async(catalog.MAIL_UNREAD_MESSAGES,
+ emit_async(catalog.MAIL_UNREAD_MESSAGES, self._userid,
str(self._inbox_collection.count_unseen()))
# process incoming mail.
@@ -286,7 +286,7 @@ class IncomingMail(Service):
deferreds = []
for index, doc in enumerate(doclist):
logger.debug("processing doc %d of %d" % (index + 1, num_mails))
- emit_async(catalog.MAIL_MSG_PROCESSING,
+ emit_async(catalog.MAIL_MSG_PROCESSING, self._userid,
str(index), str(num_mails))
keys = doc.content.keys()
@@ -336,7 +336,8 @@ class IncomingMail(Service):
decrdata = ""
success = False
- emit_async(catalog.MAIL_MSG_DECRYPTED, "1" if success else "0")
+ emit_async(catalog.MAIL_MSG_DECRYPTED, self._userid,
+ "1" if success else "0")
return self._process_decrypted_doc(doc, decrdata)
d = self._keymanager.decrypt(
@@ -393,7 +394,7 @@ class IncomingMail(Service):
# ok, this is an incoming message
rawmsg = msg.get(self.CONTENT_KEY, None)
- if not rawmsg:
+ if rawmsg is None:
return ""
return self._maybe_decrypt_msg(rawmsg)
@@ -439,6 +440,7 @@ class IncomingMail(Service):
fromHeader = msg.get('from', None)
senderAddress = None
+
if (fromHeader is not None and
(msg.get_content_type() == MULTIPART_ENCRYPTED or
msg.get_content_type() == MULTIPART_SIGNED)):
@@ -459,12 +461,14 @@ class IncomingMail(Service):
decrmsg.add_header(
self.LEAP_SIGNATURE_HEADER,
self.LEAP_SIGNATURE_VALID,
- pubkey=signkey.key_id)
+ pubkey=signkey.fingerprint)
return decrmsg.as_string()
if msg.get_content_type() == MULTIPART_ENCRYPTED:
d = self._decrypt_multipart_encrypted_msg(
msg, encoding, senderAddress)
+ elif msg.get_content_type() == MULTIPART_SIGNED:
+ d = self._verify_signature_not_encrypted_msg(msg, senderAddress)
else:
d = self._maybe_decrypt_inline_encrypted_msg(
msg, encoding, senderAddress)
@@ -544,11 +548,8 @@ class IncomingMail(Service):
:rtype: Deferred
"""
log.msg('maybe decrypting inline encrypted msg')
- # serialize the original message
- buf = StringIO()
- g = Generator(buf)
- g.flatten(origmsg)
- data = buf.getvalue()
+
+ data = self._serialize_msg(origmsg)
def decrypted_data(res):
decrdata, signkey = res
@@ -577,6 +578,46 @@ class IncomingMail(Service):
d.addCallback(encode_and_return)
return d
+ def _verify_signature_not_encrypted_msg(self, origmsg, sender_address):
+ """
+ Possibly decrypt an inline OpenPGP encrypted message.
+
+ :param origmsg: The original, possibly encrypted message.
+ :type origmsg: Message
+ :param sender_address: The email address of the sender of the message.
+ :type sender_address: str
+
+ :return: A Deferred that will be fired with a tuple containing a
+ signed Message and the signing OpenPGPKey if the signature
+ is valid or InvalidSignature.
+ :rtype: Deferred
+ """
+ msg = copy.deepcopy(origmsg)
+ data = self._serialize_msg(msg.get_payload(0))
+ detached_sig = self._extract_signature(msg)
+ d = self._keymanager.verify(data, sender_address, OpenPGPKey,
+ detached_sig)
+
+ d.addCallback(lambda sign_key: (msg, sign_key))
+ d.addErrback(lambda _: (msg, keymanager_errors.InvalidSignature()))
+ return d
+
+ def _serialize_msg(self, origmsg):
+ buf = StringIO()
+ g = Generator(buf)
+ g.flatten(origmsg)
+ return buf.getvalue()
+
+ def _extract_signature(self, msg):
+ body = msg.get_payload(0).get_payload()
+
+ if isinstance(body, str):
+ body = msg.get_payload(0)
+
+ detached_sig = msg.get_payload(1).get_payload()
+ msg.set_payload(body)
+ return detached_sig
+
def _decryption_error(self, failure, msg):
"""
Check for known decryption errors
@@ -708,7 +749,7 @@ class IncomingMail(Service):
for attachment in attachments:
if MIME_KEY == attachment.get_content_type():
d = self._keymanager.put_raw_key(
- attachment.get_payload(),
+ attachment.get_payload(decode=True),
OpenPGPKey,
address=address)
d.addCallbacks(log_key_added, failed_put_key)
@@ -743,10 +784,11 @@ class IncomingMail(Service):
listener(result)
def signal_deleted(doc_id):
- emit_async(catalog.MAIL_MSG_DELETED_INCOMING)
+ emit_async(catalog.MAIL_MSG_DELETED_INCOMING,
+ self._userid)
return doc_id
- emit_async(catalog.MAIL_MSG_SAVED_LOCALLY)
+ emit_async(catalog.MAIL_MSG_SAVED_LOCALLY, self._userid)
d = self._delete_incoming_message(doc)
d.addCallback(signal_deleted)
return d
diff --git a/src/leap/mail/incoming/tests/test_incoming_mail.py b/src/leap/mail/incoming/tests/test_incoming_mail.py
index 6880496..754df9f 100644
--- a/src/leap/mail/incoming/tests/test_incoming_mail.py
+++ b/src/leap/mail/incoming/tests/test_incoming_mail.py
@@ -77,7 +77,7 @@ subject: independence of cyberspace
def setUp(self):
def getInbox(_):
d = defer.Deferred()
- theAccount = IMAPAccount(ADDRESS, self._soledad, d=d)
+ theAccount = IMAPAccount(self._soledad, ADDRESS, d=d)
d.addCallback(
lambda _: theAccount.getMailbox(INBOX_NAME))
return d
diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py
index c0e16a6..d3659de 100644
--- a/src/leap/mail/mail.py
+++ b/src/leap/mail/mail.py
@@ -29,6 +29,8 @@ import StringIO
import time
import weakref
+from collections import defaultdict
+
from twisted.internet import defer
from twisted.python import log
@@ -744,7 +746,7 @@ class MessageCollection(object):
:param unseen: number of unseen messages.
:type unseen: int
"""
- emit_async(catalog.MAIL_UNREAD_MESSAGES, str(unseen))
+ emit_async(catalog.MAIL_UNREAD_MESSAGES, self.store.uuid, str(unseen))
def copy_msg(self, msg, new_mbox_uuid):
"""
@@ -924,19 +926,26 @@ class Account(object):
adaptor_class = SoledadMailAdaptor
- def __init__(self, store, ready_cb=None):
+ # this is a defaultdict, indexed by userid, that returns a
+ # WeakValueDictionary mapping to collection instances so that we always
+ # return a reference to them instead of creating new ones. however,
+ # being a dictionary of weakrefs values, they automagically vanish
+ # from the dict when no hard refs is left to them (so they can be
+ # garbage collected) this is important because the different wrappers
+ # rely on several kinds of deferredlocks that are kept as class or
+ # instance variables.
+
+ # We need it to be a class property because we create more than one Account
+ # object in the current usage pattern (ie, one in the mail service, and
+ # another one in the IncomingMailService). When we move to a proper service
+ # tree we can let it be an instance attribute.
+ _collection_mapping = defaultdict(weakref.WeakValueDictionary)
+
+ def __init__(self, store, user_id, ready_cb=None):
self.store = store
+ self.user_id = user_id
self.adaptor = self.adaptor_class()
- # this is a mapping to collection instances so that we always
- # return a reference to them instead of creating new ones. however,
- # being a dictionary of weakrefs values, they automagically vanish
- # from the dict when no hard refs is left to them (so they can be
- # garbage collected) this is important because the different wrappers
- # rely on several kinds of deferredlocks that are kept as class or
- # instance variables
- self._collection_mapping = weakref.WeakValueDictionary()
-
self.mbox_indexer = MailboxIndexer(self.store)
# This flag is only used from the imap service for the moment.
@@ -1069,7 +1078,8 @@ class Account(object):
:rtype: deferred
:return: a deferred that will fire with a MessageCollection
"""
- collection = self._collection_mapping.get(name, None)
+ collection = self._collection_mapping[self.user_id].get(
+ name, None)
if collection:
return defer.succeed(collection)
@@ -1077,7 +1087,7 @@ class Account(object):
def get_collection_for_mailbox(mbox_wrapper):
collection = MessageCollection(
self.adaptor, self.store, self.mbox_indexer, mbox_wrapper)
- self._collection_mapping[name] = collection
+ self._collection_mapping[self.user_id][name] = collection
return collection
d = self.adaptor.get_or_create_mbox(self.store, name)
diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py
index 7cc5a24..335cae4 100644
--- a/src/leap/mail/outgoing/service.py
+++ b/src/leap/mail/outgoing/service.py
@@ -14,6 +14,14 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+OutgoingMail module.
+
+The OutgoingMail class allows to send mail, and encrypts/signs it if needed.
+"""
+
+import os.path
import re
from StringIO import StringIO
from copy import deepcopy
@@ -35,6 +43,7 @@ from leap.common.events import emit_async, catalog
from leap.keymanager.openpgp import OpenPGPKey
from leap.keymanager.errors import KeyNotFound, KeyAddressMismatch
from leap.mail import __version__
+from leap.mail import errors
from leap.mail.utils import validate_address
from leap.mail.rfc3156 import MultipartEncrypted
from leap.mail.rfc3156 import MultipartSigned
@@ -64,14 +73,32 @@ class SSLContextFactory(ssl.ClientContextFactory):
return ctx
-class OutgoingMail:
+def outgoingFactory(userid, keymanager, opts, check_cert=True, bouncer=None):
+
+ cert = unicode(opts.cert)
+ key = unicode(opts.key)
+ hostname = str(opts.hostname)
+ port = opts.port
+
+ if check_cert:
+ if not os.path.isfile(cert):
+ raise errors.ConfigurationError(
+ 'No valid SMTP certificate could be found for %s!' % userid)
+
+ return OutgoingMail(
+ str(userid), keymanager, cert, key, hostname, port,
+ bouncer)
+
+
+class OutgoingMail(object):
"""
- A service for handling encrypted outgoing mail.
+ Sends Outgoing Mail, encrypting and signing if needed.
"""
- def __init__(self, from_address, keymanager, cert, key, host, port):
+ def __init__(self, from_address, keymanager, cert, key, host, port,
+ bouncer=None):
"""
- Initialize the mail service.
+ Initialize the outgoing mail service.
:param from_address: The sender address.
:type from_address: str
@@ -109,6 +136,7 @@ class OutgoingMail:
self._cert = cert
self._from_address = from_address
self._keymanager = keymanager
+ self._bouncer = bouncer
def send_message(self, raw, recipient):
"""
@@ -121,8 +149,8 @@ class OutgoingMail:
:return: a deferred which delivers the message when fired
"""
d = self._maybe_encrypt_and_sign(raw, recipient)
- d.addCallback(self._route_msg)
- d.addErrback(self.sendError)
+ d.addCallback(self._route_msg, raw)
+ d.addErrback(self.sendError, raw)
return d
def sendSuccess(self, smtp_sender_result):
@@ -134,23 +162,41 @@ class OutgoingMail:
:type smtp_sender_result: tuple(int, list(tuple))
"""
dest_addrstr = smtp_sender_result[1][0][0]
- log.msg('Message sent to %s' % dest_addrstr)
- emit_async(catalog.SMTP_SEND_MESSAGE_SUCCESS, dest_addrstr)
+ fromaddr = self._from_address
+ log.msg('Message sent from %s to %s' % (fromaddr, dest_addrstr))
+ emit_async(catalog.SMTP_SEND_MESSAGE_SUCCESS,
+ fromaddr, dest_addrstr)
- def sendError(self, failure):
+ def sendError(self, failure, origmsg):
"""
Callback for an unsuccessfull send.
- :param e: The result from the last errback.
- :type e: anything
+ :param failure: The result from the last errback.
+ :type failure: anything
+ :param origmsg: the original, unencrypted, raw message, to be passed to
+ the bouncer.
+ :type origmsg: str
"""
- # XXX: need to get the address from the exception to send signal
- # emit_async(catalog.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr)
+ # XXX: need to get the address from the original message to send signal
+ # emit_async(catalog.SMTP_SEND_MESSAGE_ERROR, self._from_address,
+ # self._user.dest.addrstr)
+
+ # TODO when we implement outgoing queues/long-term-retries, we could
+ # examine the error *here* and delay the notification if it's just a
+ # temporal error. We might want to notify the permanent errors
+ # differently.
+
err = failure.value
log.err(err)
- raise err
- def _route_msg(self, encrypt_and_sign_result):
+ if self._bouncer:
+ self._bouncer.bounce_message(
+ err.message, to=self._from_address,
+ orig=origmsg)
+ else:
+ raise err
+
+ def _route_msg(self, encrypt_and_sign_result, raw):
"""
Sends the msg using the ESMTPSenderFactory.
@@ -164,7 +210,8 @@ class OutgoingMail:
# we construct a defer to pass to the ESMTPSenderFactory
d = defer.Deferred()
- d.addCallbacks(self.sendSuccess, self.sendError)
+ d.addCallback(self.sendSuccess)
+ d.addErrback(self.sendError, raw)
# we don't pass an ssl context factory to the ESMTPSenderFactory
# because ssl will be handled by reactor.connectSSL() below.
factory = smtp.ESMTPSenderFactory(
@@ -178,7 +225,8 @@ class OutgoingMail:
requireAuthentication=False,
requireTransportSecurity=True)
factory.domain = __version__
- emit_async(catalog.SMTP_SEND_MESSAGE_START, recipient.dest.addrstr)
+ emit_async(catalog.SMTP_SEND_MESSAGE_START,
+ self._from_address, recipient.dest.addrstr)
reactor.connectSSL(
self._host, self._port, factory,
contextFactory=SSLContextFactory(self._cert, self._key))
@@ -226,7 +274,7 @@ class OutgoingMail:
origmsg = Parser().parsestr(raw)
if origmsg.get_content_type() == 'multipart/encrypted':
- return defer.success((origmsg, recipient))
+ return defer.succeed((origmsg, recipient))
from_address = validate_address(self._from_address)
username, domain = from_address.split('@')
@@ -241,6 +289,7 @@ class OutgoingMail:
def signal_encrypt_sign(newmsg):
emit_async(catalog.SMTP_END_ENCRYPT_AND_SIGN,
+ self._from_address,
"%s,%s" % (self._from_address, to_address))
return newmsg, recipient
@@ -248,7 +297,7 @@ class OutgoingMail:
failure.trap(KeyNotFound, KeyAddressMismatch)
log.msg('Will send unencrypted message to %s.' % to_address)
- emit_async(catalog.SMTP_START_SIGN, self._from_address)
+ emit_async(catalog.SMTP_START_SIGN, self._from_address, to_address)
d = self._sign(message, from_address)
d.addCallback(signal_sign)
return d
@@ -260,6 +309,7 @@ class OutgoingMail:
log.msg("Will encrypt the message with %s and sign with %s."
% (to_address, from_address))
emit_async(catalog.SMTP_START_ENCRYPT_AND_SIGN,
+ self._from_address,
"%s,%s" % (self._from_address, to_address))
d = self._maybe_attach_key(origmsg, from_address, to_address)
d.addCallback(maybe_encrypt_and_sign)
@@ -457,7 +507,7 @@ class OutgoingMail:
def add_openpgp_header(signkey):
username, domain = sign_address.split('@')
newmsg.add_header(
- 'OpenPGP', 'id=%s' % signkey.key_id,
+ 'OpenPGP', 'id=%s' % signkey.fingerprint,
url='https://%s/key/%s' % (domain, username),
preference='signencrypt')
return newmsg, origmsg
diff --git a/src/leap/mail/outgoing/tests/test_outgoing.py b/src/leap/mail/outgoing/tests/test_outgoing.py
index 5518b33..ad7803d 100644
--- a/src/leap/mail/outgoing/tests/test_outgoing.py
+++ b/src/leap/mail/outgoing/tests/test_outgoing.py
@@ -29,20 +29,19 @@ from twisted.mail.smtp import User
from mock import Mock
-from leap.mail.smtp.gateway import SMTPFactory
from leap.mail.rfc3156 import RFC3156CompliantGenerator
from leap.mail.outgoing.service import OutgoingMail
-from leap.mail.tests import (
- TestCaseWithKeyManager,
- ADDRESS,
- ADDRESS_2,
- PUBLIC_KEY_2,
-)
+from leap.mail.tests import TestCaseWithKeyManager
+from leap.mail.tests import ADDRESS, ADDRESS_2, PUBLIC_KEY_2
+from leap.mail.smtp.tests.test_gateway import getSMTPFactory
+
from leap.keymanager import openpgp, errors
BEGIN_PUBLIC_KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----"
+TEST_USER = u'anotheruser@leap.se'
+
class TestOutgoingMail(TestCaseWithKeyManager):
EMAIL_DATA = ['HELO gateway.leap.se',
@@ -73,11 +72,12 @@ class TestOutgoingMail(TestCaseWithKeyManager):
self.fromAddr, self._km, self._config['cert'],
self._config['key'], self._config['host'],
self._config['port'])
- self.proto = SMTPFactory(
- u'anotheruser@leap.se',
- self._km,
- self._config['encrypted_only'],
- self.outgoing_mail).buildProtocol(('127.0.0.1', 0))
+
+ user = TEST_USER
+
+ # TODO -- this shouldn't need SMTP to be tested!? or does it?
+ self.proto = getSMTPFactory(
+ {user: None}, {user: self._km}, {user: None})
self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS_2)
d = TestCaseWithKeyManager.setUp(self)
@@ -236,7 +236,7 @@ class TestOutgoingMail(TestCaseWithKeyManager):
def _set_sign_used(self, address):
def set_sign(key):
key.sign_used = True
- return self._km.put_key(key, address)
+ return self._km.put_key(key)
d = self._km.get_key(address, openpgp.OpenPGPKey, fetch_remote=False)
d.addCallback(set_sign)
diff --git a/src/leap/mail/smtp/__init__.py b/src/leap/mail/smtp/__init__.py
index 7b62808..9fab70a 100644
--- a/src/leap/mail/smtp/__init__.py
+++ b/src/leap/mail/smtp/__init__.py
@@ -23,47 +23,35 @@ import os
from twisted.internet import reactor
from twisted.internet.error import CannotListenError
-from leap.mail.outgoing.service import OutgoingMail
from leap.common.events import emit_async, catalog
+
from leap.mail.smtp.gateway import SMTPFactory
logger = logging.getLogger(__name__)
-def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port,
- smtp_cert, smtp_key, encrypted_only):
- """
- Setup SMTP gateway to run with Twisted.
+SMTP_PORT = 2013
+
- This function sets up the SMTP gateway configuration and the Twisted
- reactor.
+def run_service(soledad_sessions, keymanager_sessions, sendmail_opts,
+ port=SMTP_PORT):
+ """
+ Main entry point to run the service from the client.
- :param port: The port in which to run the server.
- :type port: int
- :param userid: The user currently logged in
- :type userid: str
- :param keymanager: A Key Manager from where to get recipients' public
- keys.
- :type keymanager: leap.common.keymanager.KeyManager
- :param smtp_host: The hostname of the remote SMTP server.
- :type smtp_host: str
- :param smtp_port: The port of the remote SMTP server.
- :type smtp_port: int
- :param smtp_cert: The client certificate for authentication.
- :type smtp_cert: str
- :param smtp_key: The client key for authentication.
- :type smtp_key: str
- :param encrypted_only: Whether the SMTP gateway should send unencrypted
- mail or not.
- :type encrypted_only: bool
+ :param soledad_sessions: a dict-like object, containing instances
+ of a Store (soledad instances), indexed by userid.
+ :param keymanager_sessions: a dict-like object, containing instances
+ of Keymanager, indexed by userid.
+ :param sendmail_opts: a dict-like object of sendmailOptions.
- :returns: tuple of SMTPFactory, twisted.internet.tcp.Port
+ :returns: the port as returned by the reactor when starts listening, and
+ the factory for the protocol.
+ :rtype: tuple
"""
- # configure the use of this service with twistd
- outgoing_mail = OutgoingMail(
- userid, keymanager, smtp_cert, smtp_key, smtp_host, smtp_port)
- factory = SMTPFactory(userid, keymanager, encrypted_only, outgoing_mail)
+ factory = SMTPFactory(soledad_sessions, keymanager_sessions,
+ sendmail_opts)
+
try:
interface = "localhost"
# don't bind just to localhost if we are running on docker since we
@@ -71,8 +59,10 @@ def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port,
if os.environ.get("LEAP_DOCKERIZED"):
interface = ''
+ # TODO Use Endpoints instead --------------------------------
tport = reactor.listenTCP(port, factory, interface=interface)
emit_async(catalog.SMTP_SERVICE_STARTED, str(port))
+
return factory, tport
except CannotListenError:
logger.error("STMP Service failed to start: "
diff --git a/src/leap/mail/smtp/bounces.py b/src/leap/mail/smtp/bounces.py
new file mode 100644
index 0000000..7a4674b
--- /dev/null
+++ b/src/leap/mail/smtp/bounces.py
@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+# bounces.py
+# Copyright (C) 2016 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Deliver bounces to the user Inbox.
+"""
+import time
+from email.message import Message
+from email.utils import formatdate
+
+from leap.mail.constants import INBOX_NAME
+from leap.mail.mail import Account
+
+
+# TODO implement localization for this template.
+
+BOUNCE_TEMPLATE = """This is your local Bitmask Mail Agent running at localhost.
+
+I'm sorry to have to inform you that your message could not be delivered to one
+or more recipients.
+
+The reasons I got for the error are:
+
+{raw_error}
+
+If the problem persists and it's not a network connectivity issue, you might
+want to contact your provider ({provider}) with this information (remove any
+sensitive data before).
+
+--- Original message (*before* it was encrypted by bitmask) below ----:
+
+{orig}"""
+
+
+class Bouncer(object):
+ """
+ Implements a mechanism to deliver bounces to user inbox.
+ """
+ # TODO this should follow RFC 6522, and compose a correct multipart
+ # attaching the report and the original message. Leaving this for a future
+ # iteration.
+
+ def __init__(self, inbox_collection):
+ self._inbox_collection = inbox_collection
+
+ def bounce_message(self, error_data, to, date=None, orig=''):
+ if not date:
+ date = formatdate(time.time())
+
+ raw_data = self._format_msg(error_data, to, date, orig)
+ d = self._inbox_collection.add_msg(
+ raw_data, ('\\Recent',), date=date)
+ return d
+
+ def _format_msg(self, error_data, to, date, orig):
+ provider = to.split('@')[1]
+
+ msg = Message()
+ msg.add_header(
+ 'From', 'bitmask-bouncer@localhost (Bitmask Local Agent)')
+ msg.add_header('To', to)
+ msg.add_header('Subject', 'Undelivered Message')
+ msg.add_header('Date', date)
+ msg.set_payload(BOUNCE_TEMPLATE.format(
+ raw_error=error_data,
+ provider=provider,
+ orig=orig))
+
+ return msg.as_string()
+
+
+def bouncerFactory(soledad):
+ user_id = soledad.uuid
+ acc = Account(soledad, user_id)
+ d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox(INBOX_NAME))
+ d.addCallback(lambda inbox: Bouncer(inbox))
+ return d
diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py
index 45560bf..bd0be6f 100644
--- a/src/leap/mail/smtp/gateway.py
+++ b/src/leap/mail/smtp/gateway.py
@@ -29,19 +29,27 @@ The following classes comprise the SMTP gateway service:
* EncryptedMessage - An implementation of twisted.mail.smtp.IMessage that
knows how to encrypt/sign itself before sending.
"""
+from email.Header import Header
from zope.interface import implements
+from zope.interface import implementer
+
+from twisted.cred.portal import Portal, IRealm
from twisted.mail import smtp
-from twisted.internet.protocol import ServerFactory
+from twisted.mail.imap4 import LOGINCredentials, PLAINCredentials
+from twisted.internet import defer, protocol
from twisted.python import log
-from email.Header import Header
from leap.common.check import leap_assert_type
from leap.common.events import emit_async, catalog
-from leap.keymanager.openpgp import OpenPGPKey
-from leap.keymanager.errors import KeyNotFound
+from leap.mail import errors
+from leap.mail.cred import LocalSoledadTokenChecker
from leap.mail.utils import validate_address
from leap.mail.rfc3156 import RFC3156CompliantGenerator
+from leap.mail.outgoing.service import outgoingFactory
+from leap.mail.smtp.bounces import bouncerFactory
+from leap.keymanager.openpgp import OpenPGPKey
+from leap.keymanager.errors import KeyNotFound
# replace email generator with a RFC 3156 compliant one.
from email import generator
@@ -49,84 +57,176 @@ from email import generator
generator.Generator = RFC3156CompliantGenerator
-#
-# Helper utilities
-#
-
LOCAL_FQDN = "bitmask.local"
-class SMTPHeloLocalhost(smtp.SMTP):
+@implementer(IRealm)
+class LocalSMTPRealm(object):
+
+ _encoding = 'utf-8'
+
+ def __init__(self, keymanager_sessions, soledad_sessions, sendmail_opts,
+ encrypted_only=False):
+ """
+ :param keymanager_sessions: a dict-like object, containing instances
+ of a Keymanager objects, indexed by
+ userid.
+ """
+ self._keymanager_sessions = keymanager_sessions
+ self._soledad_sessions = soledad_sessions
+ self._sendmail_opts = sendmail_opts
+ self.encrypted_only = encrypted_only
+
+ def requestAvatar(self, avatarId, mind, *interfaces):
+
+ if isinstance(avatarId, str):
+ avatarId = avatarId.decode(self._encoding)
+
+ def gotKeymanagerAndSoledad(result):
+ keymanager, soledad = result
+ d = bouncerFactory(soledad)
+ d.addCallback(lambda bouncer: (keymanager, soledad, bouncer))
+ return d
+
+ def getMessageDelivery(result):
+ keymanager, soledad, bouncer = result
+ # TODO use IMessageDeliveryFactory instead ?
+ # it could reuse the connections.
+ if smtp.IMessageDelivery in interfaces:
+ userid = avatarId
+ opts = self.getSendingOpts(userid)
+
+ outgoing = outgoingFactory(
+ userid, keymanager, opts, bouncer=bouncer)
+ avatar = SMTPDelivery(userid, keymanager, self.encrypted_only,
+ outgoing)
+
+ return (smtp.IMessageDelivery, avatar,
+ getattr(avatar, 'logout', lambda: None))
+
+ raise NotImplementedError(self, interfaces)
+
+ d1 = self.lookupKeymanagerInstance(avatarId)
+ d2 = self.lookupSoledadInstance(avatarId)
+ d = defer.gatherResults([d1, d2])
+ d.addCallback(gotKeymanagerAndSoledad)
+ d.addCallback(getMessageDelivery)
+ return d
+
+ def lookupKeymanagerInstance(self, userid):
+ print 'getting KM INSTNACE>>>'
+ try:
+ keymanager = self._keymanager_sessions[userid]
+ except:
+ raise errors.AuthenticationError(
+ 'No keymanager session found for user %s. Is it authenticated?'
+ % userid)
+ # XXX this should return the instance after whenReady callback
+ return defer.succeed(keymanager)
+
+ def lookupSoledadInstance(self, userid):
+ try:
+ soledad = self._soledad_sessions[userid]
+ except:
+ raise errors.AuthenticationError(
+ 'No soledad session found for user %s. Is it authenticated?'
+ % userid)
+ # XXX this should return the instance after whenReady callback
+ return defer.succeed(soledad)
+
+ def getSendingOpts(self, userid):
+ try:
+ opts = self._sendmail_opts[userid]
+ except KeyError:
+ raise errors.ConfigurationError(
+ 'No sendingMail options found for user %s' % userid)
+ return opts
+
+
+class SMTPTokenChecker(LocalSoledadTokenChecker):
+ """A credentials checker that will lookup a token for the SMTP service.
+ For now it will be using the same identifier than IMAPTokenChecker"""
+
+ service = 'mail_auth'
+
+ # TODO besides checking for token credential,
+ # we could also verify the certificate here.
+
+
+class LEAPInitMixin(object):
+
+ """
+ A Mixin that takes care of initialization of all the data needed to access
+ LEAP sessions.
"""
- An SMTP class that ensures a proper FQDN
- for localhost.
+ def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts,
+ encrypted_only=False):
+ realm = LocalSMTPRealm(
+ keymanager_sessions, soledad_sessions, sendmail_opts,
+ encrypted_only)
+ portal = Portal(realm)
+
+ checker = SMTPTokenChecker(soledad_sessions)
+ self.checker = checker
+ self.portal = portal
+ portal.registerChecker(checker)
- This avoids a problem in which unproperly configured providers
- would complain about the helo not being a fqdn.
+
+class LocalSMTPServer(smtp.ESMTP, LEAPInitMixin):
+ """
+ The Production ESMTP Server: Authentication Needed.
+ Authenticates against SMTP Token stored in Local Soledad instance.
+ The Realm will produce a Delivery Object that handles encryption/signing.
"""
- def __init__(self, *args):
- smtp.SMTP.__init__(self, *args)
- self.host = LOCAL_FQDN
+ # TODO: implement Queue using twisted.mail.mail.MailService
+
+ def __init__(self, soledads, keyms, sendmailopts, *args, **kw):
+ encrypted_only = kw.pop('encrypted_only', False)
+
+ LEAPInitMixin.__init__(self, soledads, keyms, sendmailopts,
+ encrypted_only)
+ smtp.ESMTP.__init__(self, *args, **kw)
-class SMTPFactory(ServerFactory):
+# TODO implement retries -- see smtp.SenderMixin
+class SMTPFactory(protocol.ServerFactory):
"""
Factory for an SMTP server with encrypted gatewaying capabilities.
"""
+
+ protocol = LocalSMTPServer
domain = LOCAL_FQDN
+ timeout = 600
+ encrypted_only = False
- def __init__(self, userid, keymanager, encrypted_only, outgoing_mail):
- """
- Initialize the SMTP factory.
+ def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts,
+ deferred=None, retries=3):
- :param userid: The user currently logged in
- :type userid: unicode
- :param keymanager: A Key Manager from where to get recipients' public
- keys.
- :param encrypted_only: Whether the SMTP gateway should send unencrypted
- mail or not.
- :type encrypted_only: bool
- :param outgoing_mail: The outgoing mail to send the message
- :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail
- """
-
- leap_assert_type(encrypted_only, bool)
- # and store them
- self._userid = userid
- self._km = keymanager
- self._outgoing_mail = outgoing_mail
- self._encrypted_only = encrypted_only
+ self._soledad_sessions = soledad_sessions
+ self._keymanager_sessions = keymanager_sessions
+ self._sendmail_opts = sendmail_opts
def buildProtocol(self, addr):
- """
- Return a protocol suitable for the job.
-
- :param addr: An address, e.g. a TCP (host, port).
- :type addr: twisted.internet.interfaces.IAddress
-
- @return: The protocol.
- @rtype: SMTPDelivery
- """
- smtpProtocol = SMTPHeloLocalhost(
- SMTPDelivery(
- self._userid, self._km, self._encrypted_only,
- self._outgoing_mail))
- smtpProtocol.factory = self
- return smtpProtocol
+ p = self.protocol(
+ self._soledad_sessions, self._keymanager_sessions,
+ self._sendmail_opts, encrypted_only=self.encrypted_only)
+ p.factory = self
+ p.host = LOCAL_FQDN
+ p.challengers = {"LOGIN": LOGINCredentials, "PLAIN": PLAINCredentials}
+ return p
#
# SMTPDelivery
#
+@implementer(smtp.IMessageDelivery)
class SMTPDelivery(object):
"""
Validate email addresses and handle message delivery.
"""
- implements(smtp.IMessageDelivery)
-
def __init__(self, userid, keymanager, encrypted_only, outgoing_mail):
"""
Initialize the SMTP delivery object.
@@ -202,20 +302,21 @@ class SMTPDelivery(object):
def found(_):
log.msg("Accepting mail for %s..." % user.dest.addrstr)
emit_async(catalog.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED,
- user.dest.addrstr)
+ self._userid, user.dest.addrstr)
def not_found(failure):
failure.trap(KeyNotFound)
# if key was not found, check config to see if will send anyway
if self._encrypted_only:
- emit_async(catalog.SMTP_RECIPIENT_REJECTED, user.dest.addrstr)
+ emit_async(catalog.SMTP_RECIPIENT_REJECTED, self._userid,
+ user.dest.addrstr)
raise smtp.SMTPBadRcpt(user.dest.addrstr)
log.msg("Warning: will send an unencrypted message (because "
"encrypted_only' is set to False).")
emit_async(
catalog.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED,
- user.dest.addrstr)
+ self._userid, user.dest.addrstr)
def encrypt_func(_):
return lambda: EncryptedMessage(user, self._outgoing_mail)
@@ -307,7 +408,8 @@ class EncryptedMessage(object):
"""
log.msg("Connection lost unexpectedly!")
log.err()
- emit_async(catalog.SMTP_CONNECTION_LOST, self._user.dest.addrstr)
+ emit_async(catalog.SMTP_CONNECTION_LOST, self._userid,
+ self._user.dest.addrstr)
# unexpected loss of connection; don't save
self._lines = []
diff --git a/src/leap/mail/smtp/tests/test_gateway.py b/src/leap/mail/smtp/tests/test_gateway.py
index 0b9a364..df83cf0 100644
--- a/src/leap/mail/smtp/tests/test_gateway.py
+++ b/src/leap/mail/smtp/tests/test_gateway.py
@@ -15,7 +15,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
"""
SMTP gateway tests.
"""
@@ -23,19 +22,18 @@ SMTP gateway tests.
import re
from datetime import datetime
+from twisted.mail import smtp
from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks, fail, succeed, Deferred
from twisted.test import proto_helpers
from mock import Mock
-from leap.mail.smtp.gateway import (
- SMTPFactory
-)
-from leap.mail.tests import (
- TestCaseWithKeyManager,
- ADDRESS,
- ADDRESS_2,
-)
+from leap.mail.smtp.gateway import SMTPFactory, LOCAL_FQDN
+from leap.mail.smtp.gateway import SMTPDelivery
+
+from leap.mail.outgoing.service import outgoingFactory
+from leap.mail.tests import TestCaseWithKeyManager
+from leap.mail.tests import ADDRESS, ADDRESS_2
from leap.keymanager import openpgp, errors
@@ -46,6 +44,52 @@ HOSTNAME_REGEX = "(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*" + \
"([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])"
IP_OR_HOST_REGEX = '(' + IP_REGEX + '|' + HOSTNAME_REGEX + ')'
+TEST_USER = u'anotheruser@leap.se'
+
+
+def getSMTPFactory(soledad_s, keymanager_s, sendmail_opts,
+ encrypted_only=False):
+ factory = UnauthenticatedSMTPFactory
+ factory.encrypted_only = encrypted_only
+ proto = factory(
+ soledad_s, keymanager_s, sendmail_opts).buildProtocol(('127.0.0.1', 0))
+ return proto
+
+
+class UnauthenticatedSMTPServer(smtp.SMTP):
+
+ encrypted_only = False
+
+ def __init__(self, soledads, keyms, opts, encrypted_only=False):
+ smtp.SMTP.__init__(self)
+
+ userid = TEST_USER
+ keym = keyms[userid]
+
+ class Opts:
+ cert = '/tmp/cert'
+ key = '/tmp/cert'
+ hostname = 'remote'
+ port = 666
+
+ outgoing = outgoingFactory(
+ userid, keym, Opts, check_cert=False)
+ avatar = SMTPDelivery(userid, keym, encrypted_only, outgoing)
+ self.delivery = avatar
+
+ def validateFrom(self, helo, origin):
+ return origin
+
+
+class UnauthenticatedSMTPFactory(SMTPFactory):
+ """
+ A Factory that produces a SMTP server that does not authenticate user.
+ Only for tests!
+ """
+ protocol = UnauthenticatedSMTPServer
+ domain = LOCAL_FQDN
+ encrypted_only = False
+
class TestSmtpGateway(TestCaseWithKeyManager):
@@ -85,14 +129,8 @@ class TestSmtpGateway(TestCaseWithKeyManager):
'250 Recipient address accepted',
'354 Continue']
- # XXX this bit can be refactored away in a helper
- # method...
- proto = SMTPFactory(
- u'anotheruser@leap.se',
- self._km,
- self._config['encrypted_only'],
- outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0))
- # snip...
+ user = TEST_USER
+ proto = getSMTPFactory({user: None}, {user: self._km}, {user: None})
transport = proto_helpers.StringTransport()
proto.makeConnection(transport)
reply = ""
@@ -116,12 +154,10 @@ class TestSmtpGateway(TestCaseWithKeyManager):
# mock the key fetching
self._km._fetch_keys_from_server = Mock(
return_value=fail(errors.KeyNotFound()))
- # prepare the SMTP factory
- proto = SMTPFactory(
- u'anotheruser@leap.se',
- self._km,
- self._config['encrypted_only'],
- outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0))
+ user = TEST_USER
+ proto = getSMTPFactory(
+ {user: None}, {user: self._km}, {user: None},
+ encrypted_only=True)
transport = proto_helpers.StringTransport()
proto.makeConnection(transport)
yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport)
@@ -132,7 +168,7 @@ class TestSmtpGateway(TestCaseWithKeyManager):
self.assertEqual(
'550 Cannot receive for specified address\r\n',
reply,
- 'Address should have been rejecetd with appropriate message.')
+ 'Address should have been rejected with appropriate message.')
proto.setTimeout(None)
@inlineCallbacks
@@ -149,11 +185,8 @@ class TestSmtpGateway(TestCaseWithKeyManager):
# mock the key fetching
self._km._fetch_keys_from_server = Mock(
return_value=fail(errors.KeyNotFound()))
- # prepare the SMTP factory with encrypted only equal to false
- proto = SMTPFactory(
- u'anotheruser@leap.se',
- self._km,
- False, outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0))
+ user = TEST_USER
+ proto = getSMTPFactory({user: None}, {user: self._km}, {user: None})
transport = proto_helpers.StringTransport()
proto.makeConnection(transport)
yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport)
diff --git a/src/leap/mail/tests/__init__.py b/src/leap/mail/tests/__init__.py
index 71452d2..8094c11 100644
--- a/src/leap/mail/tests/__init__.py
+++ b/src/leap/mail/tests/__init__.py
@@ -94,6 +94,8 @@ class TestCaseWithKeyManager(unittest.TestCase, BaseLeapTest):
gpgbinary=self.GPG_BINARY_PATH)
self._km._fetcher.put = Mock()
self._km._fetcher.get = Mock(return_value=Response())
+ self._km._async_client.request = Mock(return_value="")
+ self._km._async_client_pinned.request = Mock(return_value="")
d1 = self._km.put_raw_key(PRIVATE_KEY, OpenPGPKey, ADDRESS)
d2 = self._km.put_raw_key(PRIVATE_KEY_2, OpenPGPKey, ADDRESS_2)
diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py
index 9f40ffb..aca406f 100644
--- a/src/leap/mail/tests/test_mail.py
+++ b/src/leap/mail/tests/test_mail.py
@@ -317,12 +317,12 @@ class AccountTestCase(SoledadTestMixin):
"""
Tests for the Account class.
"""
- def get_account(self):
+ def get_account(self, user_id):
store = self._soledad
- return Account(store)
+ return Account(store, user_id)
def test_add_mailbox(self):
- acc = self.get_account()
+ acc = self.get_account('some_user_id')
d = acc.callWhenReady(lambda _: acc.add_mailbox("TestMailbox"))
d.addCallback(lambda _: acc.list_all_mailbox_names())
d.addCallback(self._test_add_mailbox_cb)
@@ -333,7 +333,7 @@ class AccountTestCase(SoledadTestMixin):
self.assertItemsEqual(mboxes, expected)
def test_delete_mailbox(self):
- acc = self.get_account()
+ acc = self.get_account('some_user_id')
d = acc.callWhenReady(lambda _: acc.delete_mailbox("Inbox"))
d.addCallback(lambda _: acc.list_all_mailbox_names())
d.addCallback(self._test_delete_mailbox_cb)
@@ -344,7 +344,7 @@ class AccountTestCase(SoledadTestMixin):
self.assertItemsEqual(mboxes, expected)
def test_rename_mailbox(self):
- acc = self.get_account()
+ acc = self.get_account('some_user_id')
d = acc.callWhenReady(lambda _: acc.add_mailbox("OriginalMailbox"))
d.addCallback(lambda _: acc.rename_mailbox(
"OriginalMailbox", "RenamedMailbox"))
@@ -357,7 +357,7 @@ class AccountTestCase(SoledadTestMixin):
self.assertItemsEqual(mboxes, expected)
def test_get_all_mailboxes(self):
- acc = self.get_account()
+ acc = self.get_account('some_user_id')
d = acc.callWhenReady(lambda _: acc.add_mailbox("OneMailbox"))
d.addCallback(lambda _: acc.add_mailbox("TwoMailbox"))
d.addCallback(lambda _: acc.add_mailbox("ThreeMailbox"))
@@ -374,7 +374,7 @@ class AccountTestCase(SoledadTestMixin):
self.assertItemsEqual(names, expected)
def test_get_collection_by_mailbox(self):
- acc = self.get_account()
+ acc = self.get_account('some_user_id')
d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox("INBOX"))
d.addCallback(self._test_get_collection_by_mailbox_cb)
return d
diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py
index 1c74366..17349e6 100644
--- a/src/leap/mail/walk.py
+++ b/src/leap/mail/walk.py
@@ -17,20 +17,19 @@
"""
Utilities for walking along a message tree.
"""
-import os
-
-from pycryptopp.hash import sha256
+from cryptography.hazmat.backends.multibackend import MultiBackend
+from cryptography.hazmat.backends.openssl.backend import (
+ Backend as OpenSSLBackend)
+from cryptography.hazmat.primitives import hashes
from leap.mail.utils import first
-DEBUG = os.environ.get("BITMASK_MAIL_DEBUG")
-if DEBUG:
- def get_hash(s):
- return sha256.SHA256(s).hexdigest()[:10]
-else:
- def get_hash(s):
- return sha256.SHA256(s).hexdigest()
+def get_hash(s):
+ backend = MultiBackend([OpenSSLBackend()])
+ digest = hashes.Hash(hashes.SHA256(), backend)
+ digest.update(s)
+ return digest.finalize().encode("hex").upper()
"""
@@ -92,7 +91,7 @@ def get_raw_docs(msg, parts):
return (
{
"type": "cnt", # type content they'll be
- "raw": payload if not DEBUG else payload[:100],
+ "raw": payload,
"phash": get_hash(payload),
"content-disposition": first(headers.get(
'content-disposition', '').split(';')),
@@ -168,10 +167,6 @@ def walk_msg_tree(parts, body_phash=None):
inner_headers = parts[1].get(HEADERS, None) if (
len(parts) == 2) else None
- if DEBUG:
- print "parts vector: ", pv
- print
-
# wrappers vector
def getwv(pv):
return [
@@ -209,7 +204,6 @@ def walk_msg_tree(parts, body_phash=None):
last_part = max(main_pmap.keys())
main_pmap[last_part][PART_MAP] = {}
for partind in range(len(pv) - 1):
- print partind + 1, len(parts)
main_pmap[last_part][PART_MAP][partind] = parts[partind + 1]
outer = parts[0]