diff options
| author | Kali Kaneko <kali@leap.se> | 2016-04-18 11:53:18 -0400 |
|---|---|---|
| committer | Kali Kaneko <kali@leap.se> | 2016-04-18 11:53:18 -0400 |
| commit | 6db937b20828bc39ea13836e8a70c056affa593e (patch) | |
| tree | fea16d104b10dfc993079b763e7bbc64da37ed81 | |
| parent | 9ba85bcc7724f1d9abc3ae200326e5f0a8597374 (diff) | |
| parent | d6f260f85f8464c6db6b9e158ecc85cfc02761ac (diff) | |
Merge tag '0.4.1'
Tag version 0.4.1
34 files changed, 2929 insertions, 995 deletions
diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..af315ed --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,28 @@ +0.4.1 - 18 Apr, 2016 ++++++++++++++++++++++ + +Features +~~~~~~~~ +- `#7656 <https://leap.se/code/issues/7656>`_: Emit multi-user aware events. +- `#4008 <https://leap.se/code/issues/4008>`_: Add token-based authentication to local IMAP/SMTP services. +- `#7889 <https://leap.se/code/issues/7889>`_: Use cryptography instead of pycryptopp to reduce dependencies. +- `#7263 <https://leap.se/code/issues/7263>`_: Implement local bounces to notify user of SMTP delivery errors. +- Use twisted.cred to authenticate IMAP/SMTP users. +- Verify plain text signed email. +- Validate signature with attachments. +- Use fingerprint instead of key_id to address keys. + + +Bugfixes +~~~~~~~~ +- `#7861 <https://leap.se/code/issues/7861>`_: Use the right succeed function for passthrough encrypted email. +- `#7898 <https://leap.se/code/issues/7898>`_: Fix IMAP fetch headers +- `#7977 <https://leap.se/code/issues/7977>`_: Decode attached keys so they are recognized by keymanager. +- `#7952 <https://leap.se/code/issues/7952>`_: Specify openssl backend explicitely. +- Fix the get_body logic for corner-cases in which body is None (yet-to-be synced docs, mainly). +- Let the inbox used in IncomingMail notify any subscribed Mailbox. +- Adds user_id to Account (fixes Pixelated mail leakage). + +Misc +~~~~ +- Change IMAPAccount signature, for consistency with a previous Account change. diff --git a/MANIFEST.in b/MANIFEST.in index 83264d4..1821bf4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ include versioneer.py include LICENSE include CHANGELOG include README.rst +include src/leap/mail/_version.py @@ -2,8 +2,8 @@ leap.mail ========= Mail services for the LEAP Client. -.. image:: https://pypip.in/v/leap.mail/badge.png - :target: https://crate.io/packages/leap.mail +.. image:: https://badge.fury.io/py/leap.mail.svg + :target: http://badge.fury.io/py/leap.mail .. image:: https://readthedocs.org/projects/leapmail/badge/?version=latest :target: http://leapmail.readthedocs.org/en/latest/ diff --git a/changes/next-changelog.rst b/changes/next-changelog.rst new file mode 100644 index 0000000..9b2a9d6 --- /dev/null +++ b/changes/next-changelog.rst @@ -0,0 +1,39 @@ +0.4.1 - xxx ++++++++++++++++++++++++++++++++ + +Please add lines to this file, they will be moved to the CHANGELOG.rst during +the next release. + +There are two template lines for each category, use them as reference. + +I've added a new category `Misc` so we can track doc/style/packaging stuff. + +Features +~~~~~~~~ +- `#7656 <https://leap.se/code/issues/7656>`_: Emit multi-user aware events. +- `#4008 <https://leap.se/code/issues/4008>`_: Add token-based authentication to local IMAP/SMTP services. +- `#7889 <https://leap.se/code/issues/7889>`_: Use cryptography instead of pycryptopp to reduce dependencies. +- `#7263 <https://leap.se/code/issues/7263>`_: Implement local bounces to notify user of SMTP delivery errors. +- Use twisted.cred to authenticate IMAP users. + +- `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234. +- New feature without related issue number. + +Bugfixes +~~~~~~~~ +- `#7861 <https://leap.se/code/issues/7861>`_: Use the right succeed function for passthrough encrypted email. +- `#7898 <https://leap.se/code/issues/7898>`_: Fix IMAP fetch headers +- `#7977 <https://leap.se/code/issues/7977>`_: Decode attached keys so they are recognized by keymanager. +- Fix the get_body logic for corner-cases in which body is None (yet-to-be synced docs, mainly). + +- `#1235 <https://leap.se/code/issues/1235>`_: Description for the fixed stuff corresponding with issue #1235. +- Bugfix without related issue number. + +Misc +~~~~ +- `#1236 <https://leap.se/code/issues/1236>`_: Description of the new feature corresponding with issue #1236. +- Some change without issue number. + +Known Issues +~~~~~~~~~~~~ +- `#1236 <https://leap.se/code/issues/1236>`_: Description of the known issue corresponding with issue #1236. diff --git a/pkg/requirements-latest.pip b/pkg/requirements-latest.pip index 846a319..3526bbd 100644 --- a/pkg/requirements-latest.pip +++ b/pkg/requirements-latest.pip @@ -1,9 +1,9 @@ --index-url https://pypi.python.org/simple/ ---allow-external u1db --allow-unverified u1db ---allow-external dirspec --allow-unverified dirspec --e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common' --e 'git+https://github.com/pixelated-project/soledad.git@develop#egg=leap.soledad.common&subdirectory=common/' --e 'git+https://github.com/pixelated-project/soledad.git@develop#egg=leap.soledad.client&subdirectory=client/' --e 'git+https://github.com/pixelated-project/keymanager.git@develop#egg=leap.keymanager' +https://launchpad.net/dirspec/stable-13-10/13.10/+download/dirspec-13.10.tar.gz +https://launchpad.net/ubuntu/+archive/primary/+files/u1db_13.09.orig.tar.bz2 +-e 'git+https://github.com/pixelated/leap_pycommon.git@develop#egg=leap.common' +-e 'git+https://github.com/pixelated/soledad.git@develop#egg=leap.soledad.common&subdirectory=common/' +-e 'git+https://github.com/pixelated/soledad.git@develop#egg=leap.soledad.client&subdirectory=client/' +-e 'git+https://github.com/pixelated/keymanager.git@develop#egg=leap.keymanager' -e . diff --git a/pkg/requirements-leap.pip b/pkg/requirements-leap.pip index feb9f37..134b783 100644 --- a/pkg/requirements-leap.pip +++ b/pkg/requirements-leap.pip @@ -1,3 +1,3 @@ -leap.common>=0.4.3 -leap.soledad.client>=0.7.0 -leap.keymanager>=0.4.0 +leap.common>=0.5.1 +leap.soledad.client>=0.8.0 +leap.keymanager>=0.5.0 @@ -8,3 +8,10 @@ ignore = E731 [flake8] exclude = versioneer.py,_version.py,*.egg,build,docs ignore = E731 + +[versioneer] +VCS = git +style = pep440 +versionfile_source = src/leap/mail/_version.py +versionfile_build = leap/mail/_version.py +tag_prefix = @@ -26,11 +26,6 @@ from pkg import utils import versioneer -versioneer.versionfile_source = 'src/leap/mail/_version.py' -versioneer.versionfile_build = 'leap/mail/_version.py' -versioneer.tag_prefix = '' # tags are like 1.2.0 -versioneer.parentdir_prefix = 'leap.mail-' - trove_classifiers = [ 'Development Status :: 4 - Beta', @@ -54,7 +49,7 @@ DOWNLOAD_BASE = ('https://github.com/leapcode/leap_mail/' 'archive/%s.tar.gz') _versions = versioneer.get_versions() VERSION = _versions['version'] -VERSION_FULL = _versions['full'] +VERSION_REVISION = _versions['full-revisionid'] DOWNLOAD_URL = "" # get the short version for the download url @@ -72,6 +67,22 @@ class freeze_debianver(Command): To be used after merging the development branch onto the debian one. """ user_options = [] + template = r""" +# This file was generated by the `freeze_debianver` command in setup.py +# Using 'versioneer.py' (0.16) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. + +version_version = '{version}' +full_revisionid = '{full_revisionid}' +""" + templatefun = r""" + +def get_versions(default={}, verbose=False): + return {'version': version_version, + 'full-revisionid': full_revisionid} +""" def initialize_options(self): pass @@ -85,24 +96,9 @@ class freeze_debianver(Command): if proceed != "y": print("He. You scared. Aborting.") return - template = r""" -# This file was generated by the `freeze_debianver` command in setup.py -# Using 'versioneer.py' (0.7+) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -version_version = '{version}' -version_full = '{version_full}' -""" - templatefun = r""" - -def get_versions(default={}, verbose=False): - return {'version': version_version, 'full': version_full} -""" - subst_template = template.format( + subst_template = self.template.format( version=VERSION_SHORT, - version_full=VERSION_FULL) + templatefun + version_full=VERSION_REVISION) + self.templatefun with open(versioneer.versionfile_source, 'w') as f: f.write(subst_template) diff --git a/src/leap/mail/_version.py b/src/leap/mail/_version.py index b77d552..954f488 100644 --- a/src/leap/mail/_version.py +++ b/src/leap/mail/_version.py @@ -1,72 +1,157 @@ -import subprocess -import sys -import re -import os.path -IN_LONG_VERSION_PY = True # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (build by setup.py sdist) and build +# feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.7+ (https://github.com/warner/python-versioneer) +# versioneer-0.16 (https://github.com/warner/python-versioneer) -# these strings will be replaced by git during git-archive -git_refnames = "$Format:%d$" -git_full = "$Format:%H$" +"""Git implementation of _version.py.""" +import errno +import os +import re +import subprocess +import sys -def run_command(args, cwd=None, verbose=False): - try: - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError: - e = sys.exc_info()[1] + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "$Format:%d$" + git_full = "$Format:%H$" + keywords = {"refnames": git_refnames, "full": git_full} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "pep440" + cfg.tag_prefix = "" + cfg.parentdir_prefix = "None" + cfg.versionfile_source = "src/leap/mail/_version.py" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None + else: if verbose: - print("unable to run %s" % args[0]) - print(e) + print("unable to find command, tried %s" % (commands,)) return None stdout = p.communicate()[0].strip() - if sys.version >= '3': + if sys.version_info[0] >= 3: stdout = stdout.decode() if p.returncode != 0: if verbose: - print("unable to run %s (error)" % args[0]) + print("unable to run %s (error)" % dispcmd) return None return stdout -def get_expanded_variables(versionfile_source): +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%s', but '%s' doesn't start with " + "prefix '%s'" % (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these - # variables. When used from setup.py, we don't want to import - # _version.py, so we do it with a regexp instead. This function is not - # used from _version.py. - variables = {} + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} try: - f = open(versionfile_source, "r") + f = open(versionfile_abs, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["refnames"] = mo.group(1) + keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["full"] = mo.group(1) + keywords["full"] = mo.group(1) f.close() except EnvironmentError: pass - return variables + return keywords -def versions_from_expanded_variables(variables, tag_prefix, verbose=False): - refnames = variables["refnames"].strip() +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: - print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -82,7 +167,7 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): # "stabilization", as well as "HEAD" and "master". tags = set([r for r in refs if re.search(r'\d', r)]) if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) + print("discarding '%s', no digits" % ",".join(refs-tags)) if verbose: print("likely tags: %s" % ",".join(sorted(tags))) for ref in sorted(tags): @@ -92,114 +177,308 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): if verbose: print("picking %s" % r) return {"version": r, - "full": variables["full"].strip()} - # no suitable tags, so we use the full revision id + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: - print("no suitable tags, using full revision id") - return {"version": variables["full"].strip(), - "full": variables["full"].strip()} - - -def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): - # this runs 'git' from the root of the source tree. That either means - # someone ran a setup.py command (and this code is in versioneer.py, so - # IN_LONG_VERSION_PY=False, thus the containing directory is the root of - # the source tree), or someone ran a project-specific entry point (and - # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the - # containing directory is somewhere deeper in the source tree). This only - # gets called if the git-archive 'subst' variables were *not* expanded, - # and _version.py hasn't already been rewritten with a short version - # string, meaning we're inside a checked out source tree. + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} - try: - here = os.path.abspath(__file__) - except NameError: - # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct - - # versionfile_source is the relative path from the top of the source tree - # (where the .git directory might live) to this file. Invert this to find - # the root from __file__. - root = here - if IN_LONG_VERSION_PY: - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) - else: - root = os.path.dirname(here) + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ if not os.path.exists(os.path.join(root, ".git")): if verbose: print("no .git in %s" % root) - return {} + raise NotThisMethod("no .git directory") - GIT = "git" + GITS = ["git"] if sys.platform == "win32": - GIT = "git.cmd" - stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], - cwd=root) - if stdout is None: - return {} - if not stdout.startswith(tag_prefix): - if verbose: - print("tag '%s' doesn't start with prefix '%s'" % - (stdout, tag_prefix)) - return {} - tag = stdout[len(tag_prefix):] - stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) - if stdout is None: - return {} - full = stdout.strip() - if tag.endswith("-dirty"): - full += "-dirty" - return {"version": tag, "full": full} - - -def versions_from_parentdir(parentdir_prefix, versionfile_source, - verbose=False): - if IN_LONG_VERSION_PY: - # We're running from _version.py. If it's from a source tree - # (execute-in-place), we can work upwards to find the root of the - # tree, and then check the parent directory for a version string. If - # it's in an installed application, there's no hope. - try: - here = os.path.abspath(__file__) - except NameError: - # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) # versionfile_source is the relative path from the top of the source - # tree to _version.py. Invert this to find the root from __file__. - root = here - for i in range(len(versionfile_source.split("/"))): + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): root = os.path.dirname(root) - else: - # we're running from versioneer.py, which means we're running from - # the setup.py in a source tree. sys.argv[0] is setup.py in the root. - here = os.path.abspath(sys.argv[0]) - root = os.path.dirname(here) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree"} - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't " - "start with prefix '%s'" % - (root, dirname, parentdir_prefix)) - return None - return {"version": dirname[len(parentdir_prefix):], "full": ""} - -tag_prefix = "" -parentdir_prefix = "leap-mail" -versionfile_source = "src/leap/mail/_version.py" - - -def get_versions(default={"version": "unknown", "full": ""}, verbose=False): - variables = {"refnames": git_refnames, "full": git_full} - ver = versions_from_expanded_variables(variables, tag_prefix, verbose) - if not ver: - ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) - if not ver: - ver = versions_from_parentdir(parentdir_prefix, versionfile_source, - verbose) - if not ver: - ver = default - return ver + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version"} diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 8de83f7..f4af020 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -22,7 +22,6 @@ import re from collections import defaultdict from email import message_from_string -from pycryptopp.hash import sha256 from twisted.internet import defer from twisted.python import log from zope.interface import implements @@ -687,13 +686,14 @@ class MessageWrapper(object): :rtype: deferred """ body_phash = self.hdoc.body - if not body_phash: - if self.cdocs: - return self.cdocs[1] - d = store.get_doc('C-' + body_phash) - d.addCallback(lambda doc: ContentDocWrapper(**doc.content)) - return d - + if body_phash: + d = store.get_doc('C-' + body_phash) + d.addCallback(lambda doc: ContentDocWrapper(**doc.content)) + return d + elif self.cdocs: + return self.cdocs[1] + else: + return '' # # Mailboxes @@ -1207,7 +1207,7 @@ def _split_into_parts(raw): def _parse_msg(raw): msg = message_from_string(raw) parts = walk.get_parts(msg) - chash = sha256.SHA256(raw).hexdigest() + chash = walk.get_hash(raw) multi = msg.is_multipart() return msg, parts, chash, multi diff --git a/src/leap/mail/cred.py b/src/leap/mail/cred.py new file mode 100644 index 0000000..7eab1f0 --- /dev/null +++ b/src/leap/mail/cred.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# cred.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Credentials handling. +""" + +from zope.interface import implementer +from twisted.cred.checkers import ICredentialsChecker +from twisted.cred.credentials import IUsernamePassword +from twisted.cred.error import UnauthorizedLogin +from twisted.internet import defer + + +@implementer(ICredentialsChecker) +class LocalSoledadTokenChecker(object): + + """ + A Credentials Checker for a LocalSoledad store. + + It checks that: + + 1) The Local SoledadStorage has been correctly unlocked for the given + user. This currently means that the right passphrase has been passed + to the Local SoledadStorage. + + 2) The password passed in the credentials matches whatever token has + been stored in the local encrypted SoledadStorage, associated to the + Protocol that is requesting the authentication. + """ + + credentialInterfaces = (IUsernamePassword,) + service = None + + def __init__(self, soledad_sessions): + """ + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by + userid. + """ + self._soledad_sessions = soledad_sessions + + def requestAvatarId(self, credentials): + if self.service is None: + raise NotImplementedError( + "this checker has not defined its service name") + username, password = credentials.username, credentials.password + d = self.checkSoledadToken(username, password, self.service) + d.addErrback(lambda f: defer.fail(UnauthorizedLogin())) + return d + + def checkSoledadToken(self, username, password, service): + soledad = self._soledad_sessions.get(username) + if not soledad: + return defer.fail(Exception("No soledad")) + + def match_token(token): + if token is None: + raise RuntimeError('no token') + if token == password: + return username + else: + raise RuntimeError('bad token') + + d = soledad.get_or_create_service_token(service) + d.addCallback(match_token) + return d diff --git a/src/leap/mail/errors.py b/src/leap/mail/errors.py new file mode 100644 index 0000000..2f18e87 --- /dev/null +++ b/src/leap/mail/errors.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# errors.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Exceptions for leap.mail +""" + + +class AuthenticationError(Exception): + pass + + +class ConfigurationError(Exception): + pass diff --git a/src/leap/mail/generator.py b/src/leap/mail/generator.py new file mode 100644 index 0000000..bb3f26e --- /dev/null +++ b/src/leap/mail/generator.py @@ -0,0 +1,23 @@ +from email.generator import Generator as EmailGenerator + + +class Generator(EmailGenerator): + """ + Generates output from a Message object tree, keeping signatures. + + This code was extracted from Mailman.Generator.Generator, version 2.1.4: + + Most other Generator will be created not setting the foldheader flag, + as we do not overwrite clone(). The original clone() does not + set foldheaders. + + So you need to set foldheaders if you want the toplevel to fold headers + + TODO: Python 3.3 is patched against this problems. See issue 1590744 on + python bug tracker. + """ + def _write_headers(self, msg): + for h, v in msg.items(): + print >> self._fp, '%s:' % h, + print >> self._fp, v + print >> self._fp diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index cc56fff..459b0ba 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -49,6 +49,7 @@ if PROFILE_CMD: # Soledad IMAP Account ####################################### + class IMAPAccount(object): """ An implementation of an imap4 Account @@ -59,7 +60,7 @@ class IMAPAccount(object): selected = None - def __init__(self, user_id, store, d=defer.Deferred()): + def __init__(self, store, user_id, d=defer.Deferred()): """ Keeps track of the mailboxes and subscriptions handled by this account. @@ -68,13 +69,14 @@ class IMAPAccount(object): You can either pass a deferred to this constructor, or use `callWhenReady` method. - :param user_id: The name of the account (user id, in the form - user@provider). - :type user_id: str - :param store: a Soledad instance. :type store: Soledad + :param user_id: The identifier of the user this account belongs to + (user id, in the form user@provider). + :type user_id: str + + :param d: a deferred that will be fired with this IMAPAccount instance when the account is ready to be used. :type d: defer.Deferred @@ -87,7 +89,7 @@ class IMAPAccount(object): # about user_id, only the client backend. self.user_id = user_id - self.account = Account(store, ready_cb=lambda: d.callback(self)) + self.account = Account(store, user_id, ready_cb=lambda: d.callback(self)) def end_session(self): """ diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index bfc0bfc..d545c00 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -508,7 +508,7 @@ class IMAPMailbox(object): def get_range(messages_asked): return self._filter_msg_seq(messages_asked) - d = defer.maybeDeferred(self._bound_seq, messages_asked, uid) + d = self._bound_seq(messages_asked, uid) if uid: d.addCallback(get_range) d.addErrback(lambda f: log.err(f)) @@ -520,7 +520,7 @@ class IMAPMailbox(object): :param messages_asked: IDs of the messages. :type messages_asked: MessageSet - :rtype: MessageSet + :return: a Deferred that will fire with a MessageSet """ def set_last_uid(last_uid): @@ -543,7 +543,7 @@ class IMAPMailbox(object): d = self.collection.all_uid_iter() d.addCallback(set_last_seq) return d - return messages_asked + return defer.succeed(messages_asked) def _filter_msg_seq(self, messages_asked): """ @@ -713,6 +713,7 @@ class IMAPMailbox(object): d_seq.addCallback(get_flags_for_seq) return d_seq + @defer.inlineCallbacks def fetch_headers(self, messages_asked, uid): """ A fast method to fetch all headers, tricking just the @@ -757,14 +758,15 @@ class IMAPMailbox(object): for key, value in self.headers.items()) - messages_asked = self._bound_seq(messages_asked) - seq_messg = self._filter_msg_seq(messages_asked) + messages_asked = yield self._bound_seq(messages_asked, uid) + seq_messg = yield self._filter_msg_seq(messages_asked) - all_headers = self.messages.all_headers() - result = ((msgid, headersPart( - msgid, all_headers.get(msgid, {}))) - for msgid in seq_messg) - return result + result = [] + for msgid in seq_messg: + msg = yield self.collection.get_message_by_uid(msgid) + headers = headersPart(msgid, msg.get_headers()) + result.append((msgid, headers)) + defer.returnValue(iter(result)) def store(self, messages_asked, flags, mode, uid): """ diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 99e7174..5a63af0 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -20,21 +20,17 @@ LEAP IMAP4 Server Implementation. import StringIO from copy import copy -from twisted import cred -from twisted.internet import reactor from twisted.internet.defer import maybeDeferred from twisted.mail import imap4 from twisted.python import log -from leap.common.check import leap_assert, leap_assert_type -from leap.common.events import emit_async, catalog -from leap.soledad.client import Soledad - # imports for LITERAL+ patch from twisted.internet import defer, interfaces from twisted.mail.imap4 import IllegalClientResponse from twisted.mail.imap4 import LiteralString, LiteralFile +from leap.common.events import emit_async, catalog + def _getContentType(msg): """ @@ -72,25 +68,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): """ An IMAP4 Server with a LEAP Storage Backend. """ - def __init__(self, *args, **kwargs): - # pop extraneous arguments - soledad = kwargs.pop('soledad', None) - uuid = kwargs.pop('uuid', None) - userid = kwargs.pop('userid', None) - - leap_assert(soledad, "need a soledad instance") - leap_assert_type(soledad, Soledad) - leap_assert(uuid, "need a user in the initialization") - - self._userid = userid - - # initialize imap server! - imap4.IMAP4Server.__init__(self, *args, **kwargs) - - # we should initialize the account here, - # but we move it to the factory so we can - # populate the test account properly (and only once - # per session) ############################################################# # @@ -181,10 +158,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): :param line: the line from the server, without the line delimiter. :type line: str """ - if self.theAccount.session_ended is True and self.state != "unauth": - log.msg("Closing the session. State: unauth") - self.state = "unauth" - if "login" in line.lower(): # avoid to log the pass, even though we are using a dummy auth # by now. @@ -208,25 +181,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): self.mbox = None self.state = 'unauth' - def authenticateLogin(self, username, password): - """ - Lookup the account with the given parameters, and deny - the improper combinations. - - :param username: the username that is attempting authentication. - :type username: str - :param password: the password to authenticate with. - :type password: str - """ - # XXX this should use portal: - # return portal.login(cred.credentials.UsernamePassword(user, pass) - if username != self._userid: - # bad username, reject. - raise cred.error.UnauthorizedLogin() - # any dummy password is allowed so far. use realm instead! - emit_async(catalog.IMAP_CLIENT_LOGIN, "1") - return imap4.IAccount, self.theAccount, lambda: None - def do_FETCH(self, tag, messages, query, uid=0): """ Overwritten fetch dispatcher to use the fast fetch_flags @@ -657,7 +611,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): d.addCallback(send_response) return d # XXX patched --------------------------------- - # ----------------------------------------------------------------------- auth_APPEND = (do_APPEND, arg_astring, imap4.IMAP4Server.opt_plist, @@ -733,3 +686,8 @@ class LEAPIMAPServer(imap4.IMAP4Server): ############################################################# # END of Twisted imap4 patch to support LITERAL+ extension ############################################################# + + def authenticateLogin(self, user, passwd): + result = imap4.IMAP4Server.authenticateLogin(self, user, passwd) + emit_async(catalog.IMAP_CLIENT_LOGIN, str(user)) + return result diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac index 2045757..c4d602d 100644 --- a/src/leap/mail/imap/service/imap-server.tac +++ b/src/leap/mail/imap/service/imap-server.tac @@ -1,3 +1,4 @@ +#!/usr/bin/env python # -*- coding: utf-8 -*- # imap-server.tac # Copyright (C) 2013,2014 LEAP @@ -27,6 +28,9 @@ userid = 'user@provider' uuid = 'deadbeefdeadabad' passwd = 'supersecret' # optional, will get prompted if not found. """ + +# TODO -- this .tac file should be deprecated in favor of bitmask.core.bitmaskd + import ConfigParser import getpass import os @@ -112,7 +116,7 @@ tempdir = "/tmp/" print "[~] user:", userid soledad = initialize_soledad(uuid, userid, passwd, secrets, - localdb, gnupg_home, tempdir) + localdb, gnupg_home, tempdir, userid=userid) km_args = (userid, "https://localhost", soledad) km_kwargs = { "token": "", @@ -131,7 +135,8 @@ keymanager = KeyManager(*km_args, **km_kwargs) def getIMAPService(): - factory = imap.LeapIMAPFactory(uuid, userid, soledad) + soledad_sessions = {userid: soledad} + factory = imap.LeapIMAPFactory(soledad_sessions) return internet.TCPServer(port, factory, interface="localhost") diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index a50611b..4663854 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -15,21 +15,24 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -IMAP service initialization +IMAP Service Initialization. """ import logging import os from collections import defaultdict +from twisted.cred.portal import Portal, IRealm +from twisted.mail.imap4 import IAccount +from twisted.internet import defer from twisted.internet import reactor from twisted.internet.error import CannotListenError from twisted.internet.protocol import ServerFactory -from twisted.mail import imap4 from twisted.python import log +from zope.interface import implementer from leap.common.events import emit_async, catalog -from leap.common.check import leap_check +from leap.mail.cred import LocalSoledadTokenChecker from leap.mail.imap.account import IMAPAccount from leap.mail.imap.server import LEAPIMAPServer @@ -41,57 +44,92 @@ DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) if DO_MANHOLE: from leap.mail.imap.service import manhole -DO_PROFILE = os.environ.get("LEAP_PROFILE", None) -if DO_PROFILE: - import cProfile - log.msg("Starting PROFILING...") - - PROFILE_DAT = "/tmp/leap_mail_profile.pstats" - pr = cProfile.Profile() - pr.enable() - # The default port in which imap service will run + IMAP_PORT = 1984 +# +# Credentials Handling +# + + +@implementer(IRealm) +class LocalSoledadIMAPRealm(object): + + _encoding = 'utf-8' + + def __init__(self, soledad_sessions): + """ + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by + userid. + """ + self._soledad_sessions = soledad_sessions + + def requestAvatar(self, avatarId, mind, *interfaces): + if isinstance(avatarId, str): + avatarId = avatarId.decode(self._encoding) + + def gotSoledad(soledad): + for iface in interfaces: + if iface is IAccount: + avatar = IMAPAccount(soledad, avatarId) + return (IAccount, avatar, + getattr(avatar, 'logout', lambda: None)) + raise NotImplementedError(self, interfaces) + + return self.lookupSoledadInstance(avatarId).addCallback(gotSoledad) + + def lookupSoledadInstance(self, userid): + soledad = self._soledad_sessions[userid] + # XXX this should return the instance after whenReady callback + return defer.succeed(soledad) + + +class IMAPTokenChecker(LocalSoledadTokenChecker): + """A credentials checker that will lookup a token for the IMAP service. + For now it will be using the same identifier than SMTPTokenChecker""" + + service = 'mail_auth' + + +class LocalSoledadIMAPServer(LEAPIMAPServer): -class IMAPAuthRealm(object): """ - Dummy authentication realm. Do not use in production! + An IMAP Server that authenticates against a LocalSoledad store. """ - theAccount = None - def requestAvatar(self, avatarId, mind, *interfaces): - return imap4.IAccount, self.theAccount, lambda: None + def __init__(self, soledad_sessions, *args, **kw): + + LEAPIMAPServer.__init__(self, *args, **kw) + + realm = LocalSoledadIMAPRealm(soledad_sessions) + portal = Portal(realm) + checker = IMAPTokenChecker(soledad_sessions) + self.checker = checker + self.portal = portal + portal.registerChecker(checker) class LeapIMAPFactory(ServerFactory): + """ Factory for a IMAP4 server with soledad remote sync and gpg-decryption capabilities. """ - protocol = LEAPIMAPServer - def __init__(self, uuid, userid, soledad): + protocol = LocalSoledadIMAPServer + + def __init__(self, soledad_sessions): """ Initializes the server factory. - :param uuid: user uuid - :type uuid: str - - :param userid: user id (user@provider.org) - :type userid: str - - :param soledad: soledad instance - :type soledad: Soledad + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by + userid. """ - self._uuid = uuid - self._userid = userid - self._soledad = soledad - - theAccount = IMAPAccount(uuid, soledad) - self.theAccount = theAccount + self._soledad_sessions = soledad_sessions self._connections = defaultdict() - # XXX how to pass the store along? def buildProtocol(self, addr): """ @@ -103,13 +141,7 @@ class LeapIMAPFactory(ServerFactory): # TODO should reject anything from addr != localhost, # just in case. log.msg("Building protocol for connection %s" % addr) - imapProtocol = self.protocol( - uuid=self._uuid, - userid=self._userid, - soledad=self._soledad) - imapProtocol.theAccount = self.theAccount - imapProtocol.factory = self - + imapProtocol = self.protocol(self._soledad_sessions) self._connections[addr] = imapProtocol return imapProtocol @@ -123,39 +155,21 @@ class LeapIMAPFactory(ServerFactory): """ Stops imap service (fetcher, factory and port). """ - # mark account as unusable, so any imap command will fail - # with unauth state. - self.theAccount.end_session() - - # TODO should wait for all the pending deferreds, - # the twisted way! - if DO_PROFILE: - log.msg("Stopping PROFILING") - pr.disable() - pr.dump_stats(PROFILE_DAT) - return ServerFactory.doStop(self) -def run_service(store, **kwargs): +def run_service(soledad_sessions, port=IMAP_PORT): """ Main entry point to run the service from the client. - :param store: a soledad instance + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by userid. :returns: the port as returned by the reactor when starts listening, and the factory for the protocol. + :rtype: tuple """ - leap_check(store, "store cannot be None") - # XXX this can also be a ProxiedObject, FIXME - # leap_assert_type(store, Soledad) - - port = kwargs.get('port', IMAP_PORT) - userid = kwargs.get('userid', None) - leap_check(userid is not None, "need an user id") - - uuid = store.uuid - factory = LeapIMAPFactory(uuid, userid, store) + factory = LeapIMAPFactory(soledad_sessions) try: interface = "localhost" @@ -164,6 +178,7 @@ def run_service(store, **kwargs): if os.environ.get("LEAP_DOCKERIZED"): interface = '' + # TODO use Endpoints !!! tport = reactor.listenTCP(port, factory, interface=interface) except CannotListenError: @@ -178,9 +193,9 @@ def run_service(store, **kwargs): # TODO get pass from env var.too. manhole_factory = manhole.getManholeFactory( {'f': factory, - 'a': factory.theAccount, 'gm': factory.theAccount.getMailbox}, "boss", "leap") + # TODO use Endpoints !!! reactor.listenTCP(manhole.MANHOLE_PORT, manhole_factory, interface="127.0.0.1") logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 62c3c41..ccce285 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -575,8 +575,8 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): """ Test login requiring quoting """ - self.server._userid = '{test}user@leap.se' - self.server._password = '{test}password' + self.server.checker.userid = '{test}user@leap.se' + self.server.checker.password = '{test}password' def login(): d = self.client.login('{test}user@leap.se', '{test}password') diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index a34538b..64a0326 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -20,10 +20,15 @@ Common utilities for testing Soledad IMAP Server. from email import parser from mock import Mock +from twisted.cred.checkers import ICredentialsChecker +from twisted.cred.credentials import IUsernamePassword +from twisted.cred.error import UnauthorizedLogin +from twisted.cred.portal import Portal, IRealm from twisted.mail import imap4 from twisted.internet import defer from twisted.protocols import loopback from twisted.python import log +from zope.interface import implementer from leap.mail.adaptors import soledad as soledad_adaptor from leap.mail.imap.account import IMAPAccount @@ -64,6 +69,57 @@ class SimpleClient(imap4.IMAP4Client): self.events.append(['newMessages', exists, recent]) self.transport.loseConnection() +# +# Dummy credentials for tests +# + + +@implementer(IRealm) +class TestRealm(object): + + def __init__(self, account): + self._account = account + + def requestAvatar(self, avatarId, mind, *interfaces): + avatar = self._account + return (imap4.IAccount, avatar, + getattr(avatar, 'logout', lambda: None)) + + +@implementer(ICredentialsChecker) +class TestCredentialsChecker(object): + + credentialInterfaces = (IUsernamePassword,) + + userid = TEST_USER + password = TEST_PASSWD + + def requestAvatarId(self, credentials): + username, password = credentials.username, credentials.password + d = self.checkTestCredentials(username, password) + d.addErrback(lambda f: defer.fail(UnauthorizedLogin())) + return d + + def checkTestCredentials(self, username, password): + if username == self.userid and password == self.password: + return defer.succeed(username) + else: + return defer.fail(Exception("Wrong credentials")) + + +class TestSoledadIMAPServer(LEAPIMAPServer): + + def __init__(self, account, *args, **kw): + + LEAPIMAPServer.__init__(self, *args, **kw) + + realm = TestRealm(account) + portal = Portal(realm) + checker = TestCredentialsChecker() + self.checker = checker + self.portal = portal + portal.registerChecker(checker) + class IMAP4HelperMixin(SoledadTestMixin): """ @@ -77,14 +133,12 @@ class IMAP4HelperMixin(SoledadTestMixin): soledad_adaptor.cleanup_deferred_locks() - UUID = 'deadbeef', USERID = TEST_USER def setup_server(account): - self.server = LEAPIMAPServer( - uuid=UUID, userid=USERID, - contextFactory=self.serverCTX, - soledad=self._soledad) + self.server = TestSoledadIMAPServer( + account=account, + contextFactory=self.serverCTX) self.server.theAccount = account d_server_ready = defer.Deferred() @@ -104,7 +158,7 @@ class IMAP4HelperMixin(SoledadTestMixin): self._soledad.sync = Mock() d = defer.Deferred() - self.acc = IMAPAccount(USERID, self._soledad, d=d) + self.acc = IMAPAccount(self._soledad, USERID, d=d) return d d = super(IMAP4HelperMixin, self).setUp() diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index d8b91ba..c7d194d 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -24,7 +24,6 @@ import time import warnings from email.parser import Parser -from email.generator import Generator from email.utils import parseaddr from email.utils import formatdate from StringIO import StringIO @@ -43,6 +42,7 @@ from leap.common.mail import get_email_charset from leap.keymanager import errors as keymanager_errors from leap.keymanager.openpgp import OpenPGPKey from leap.mail.adaptors import soledad_indexes as fields +from leap.mail.generator import Generator from leap.mail.utils import json_loads, empty from leap.soledad.client import Soledad from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY @@ -233,7 +233,7 @@ class IncomingMail(Service): failure.trap(InvalidAuthTokenError) # if the token is invalid, send an event so the GUI can # disable mail and show an error message. - emit_async(catalog.SOLEDAD_INVALID_AUTH_TOKEN) + emit_async(catalog.SOLEDAD_INVALID_AUTH_TOKEN, self._userid) log.msg('FETCH: syncing soledad...') d = self._soledad.sync() @@ -254,7 +254,7 @@ class IncomingMail(Service): num_mails = len(doclist) if doclist is not None else 0 if num_mails != 0: log.msg("there are %s mails" % (num_mails,)) - emit_async(catalog.MAIL_FETCHED_INCOMING, + emit_async(catalog.MAIL_FETCHED_INCOMING, self._userid, str(num_mails), str(fetched_ts)) return doclist @@ -262,7 +262,7 @@ class IncomingMail(Service): """ Sends unread event to ui. """ - emit_async(catalog.MAIL_UNREAD_MESSAGES, + emit_async(catalog.MAIL_UNREAD_MESSAGES, self._userid, str(self._inbox_collection.count_unseen())) # process incoming mail. @@ -286,7 +286,7 @@ class IncomingMail(Service): deferreds = [] for index, doc in enumerate(doclist): logger.debug("processing doc %d of %d" % (index + 1, num_mails)) - emit_async(catalog.MAIL_MSG_PROCESSING, + emit_async(catalog.MAIL_MSG_PROCESSING, self._userid, str(index), str(num_mails)) keys = doc.content.keys() @@ -336,7 +336,8 @@ class IncomingMail(Service): decrdata = "" success = False - emit_async(catalog.MAIL_MSG_DECRYPTED, "1" if success else "0") + emit_async(catalog.MAIL_MSG_DECRYPTED, self._userid, + "1" if success else "0") return self._process_decrypted_doc(doc, decrdata) d = self._keymanager.decrypt( @@ -393,7 +394,7 @@ class IncomingMail(Service): # ok, this is an incoming message rawmsg = msg.get(self.CONTENT_KEY, None) - if not rawmsg: + if rawmsg is None: return "" return self._maybe_decrypt_msg(rawmsg) @@ -439,6 +440,7 @@ class IncomingMail(Service): fromHeader = msg.get('from', None) senderAddress = None + if (fromHeader is not None and (msg.get_content_type() == MULTIPART_ENCRYPTED or msg.get_content_type() == MULTIPART_SIGNED)): @@ -459,12 +461,14 @@ class IncomingMail(Service): decrmsg.add_header( self.LEAP_SIGNATURE_HEADER, self.LEAP_SIGNATURE_VALID, - pubkey=signkey.key_id) + pubkey=signkey.fingerprint) return decrmsg.as_string() if msg.get_content_type() == MULTIPART_ENCRYPTED: d = self._decrypt_multipart_encrypted_msg( msg, encoding, senderAddress) + elif msg.get_content_type() == MULTIPART_SIGNED: + d = self._verify_signature_not_encrypted_msg(msg, senderAddress) else: d = self._maybe_decrypt_inline_encrypted_msg( msg, encoding, senderAddress) @@ -544,11 +548,8 @@ class IncomingMail(Service): :rtype: Deferred """ log.msg('maybe decrypting inline encrypted msg') - # serialize the original message - buf = StringIO() - g = Generator(buf) - g.flatten(origmsg) - data = buf.getvalue() + + data = self._serialize_msg(origmsg) def decrypted_data(res): decrdata, signkey = res @@ -577,6 +578,46 @@ class IncomingMail(Service): d.addCallback(encode_and_return) return d + def _verify_signature_not_encrypted_msg(self, origmsg, sender_address): + """ + Possibly decrypt an inline OpenPGP encrypted message. + + :param origmsg: The original, possibly encrypted message. + :type origmsg: Message + :param sender_address: The email address of the sender of the message. + :type sender_address: str + + :return: A Deferred that will be fired with a tuple containing a + signed Message and the signing OpenPGPKey if the signature + is valid or InvalidSignature. + :rtype: Deferred + """ + msg = copy.deepcopy(origmsg) + data = self._serialize_msg(msg.get_payload(0)) + detached_sig = self._extract_signature(msg) + d = self._keymanager.verify(data, sender_address, OpenPGPKey, + detached_sig) + + d.addCallback(lambda sign_key: (msg, sign_key)) + d.addErrback(lambda _: (msg, keymanager_errors.InvalidSignature())) + return d + + def _serialize_msg(self, origmsg): + buf = StringIO() + g = Generator(buf) + g.flatten(origmsg) + return buf.getvalue() + + def _extract_signature(self, msg): + body = msg.get_payload(0).get_payload() + + if isinstance(body, str): + body = msg.get_payload(0) + + detached_sig = msg.get_payload(1).get_payload() + msg.set_payload(body) + return detached_sig + def _decryption_error(self, failure, msg): """ Check for known decryption errors @@ -708,7 +749,7 @@ class IncomingMail(Service): for attachment in attachments: if MIME_KEY == attachment.get_content_type(): d = self._keymanager.put_raw_key( - attachment.get_payload(), + attachment.get_payload(decode=True), OpenPGPKey, address=address) d.addCallbacks(log_key_added, failed_put_key) @@ -743,10 +784,11 @@ class IncomingMail(Service): listener(result) def signal_deleted(doc_id): - emit_async(catalog.MAIL_MSG_DELETED_INCOMING) + emit_async(catalog.MAIL_MSG_DELETED_INCOMING, + self._userid) return doc_id - emit_async(catalog.MAIL_MSG_SAVED_LOCALLY) + emit_async(catalog.MAIL_MSG_SAVED_LOCALLY, self._userid) d = self._delete_incoming_message(doc) d.addCallback(signal_deleted) return d diff --git a/src/leap/mail/incoming/tests/test_incoming_mail.py b/src/leap/mail/incoming/tests/test_incoming_mail.py index 6880496..754df9f 100644 --- a/src/leap/mail/incoming/tests/test_incoming_mail.py +++ b/src/leap/mail/incoming/tests/test_incoming_mail.py @@ -77,7 +77,7 @@ subject: independence of cyberspace def setUp(self): def getInbox(_): d = defer.Deferred() - theAccount = IMAPAccount(ADDRESS, self._soledad, d=d) + theAccount = IMAPAccount(self._soledad, ADDRESS, d=d) d.addCallback( lambda _: theAccount.getMailbox(INBOX_NAME)) return d diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index c0e16a6..d3659de 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -29,6 +29,8 @@ import StringIO import time import weakref +from collections import defaultdict + from twisted.internet import defer from twisted.python import log @@ -744,7 +746,7 @@ class MessageCollection(object): :param unseen: number of unseen messages. :type unseen: int """ - emit_async(catalog.MAIL_UNREAD_MESSAGES, str(unseen)) + emit_async(catalog.MAIL_UNREAD_MESSAGES, self.store.uuid, str(unseen)) def copy_msg(self, msg, new_mbox_uuid): """ @@ -924,19 +926,26 @@ class Account(object): adaptor_class = SoledadMailAdaptor - def __init__(self, store, ready_cb=None): + # this is a defaultdict, indexed by userid, that returns a + # WeakValueDictionary mapping to collection instances so that we always + # return a reference to them instead of creating new ones. however, + # being a dictionary of weakrefs values, they automagically vanish + # from the dict when no hard refs is left to them (so they can be + # garbage collected) this is important because the different wrappers + # rely on several kinds of deferredlocks that are kept as class or + # instance variables. + + # We need it to be a class property because we create more than one Account + # object in the current usage pattern (ie, one in the mail service, and + # another one in the IncomingMailService). When we move to a proper service + # tree we can let it be an instance attribute. + _collection_mapping = defaultdict(weakref.WeakValueDictionary) + + def __init__(self, store, user_id, ready_cb=None): self.store = store + self.user_id = user_id self.adaptor = self.adaptor_class() - # this is a mapping to collection instances so that we always - # return a reference to them instead of creating new ones. however, - # being a dictionary of weakrefs values, they automagically vanish - # from the dict when no hard refs is left to them (so they can be - # garbage collected) this is important because the different wrappers - # rely on several kinds of deferredlocks that are kept as class or - # instance variables - self._collection_mapping = weakref.WeakValueDictionary() - self.mbox_indexer = MailboxIndexer(self.store) # This flag is only used from the imap service for the moment. @@ -1069,7 +1078,8 @@ class Account(object): :rtype: deferred :return: a deferred that will fire with a MessageCollection """ - collection = self._collection_mapping.get(name, None) + collection = self._collection_mapping[self.user_id].get( + name, None) if collection: return defer.succeed(collection) @@ -1077,7 +1087,7 @@ class Account(object): def get_collection_for_mailbox(mbox_wrapper): collection = MessageCollection( self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) - self._collection_mapping[name] = collection + self._collection_mapping[self.user_id][name] = collection return collection d = self.adaptor.get_or_create_mbox(self.store, name) diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py index 7cc5a24..335cae4 100644 --- a/src/leap/mail/outgoing/service.py +++ b/src/leap/mail/outgoing/service.py @@ -14,6 +14,14 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +OutgoingMail module. + +The OutgoingMail class allows to send mail, and encrypts/signs it if needed. +""" + +import os.path import re from StringIO import StringIO from copy import deepcopy @@ -35,6 +43,7 @@ from leap.common.events import emit_async, catalog from leap.keymanager.openpgp import OpenPGPKey from leap.keymanager.errors import KeyNotFound, KeyAddressMismatch from leap.mail import __version__ +from leap.mail import errors from leap.mail.utils import validate_address from leap.mail.rfc3156 import MultipartEncrypted from leap.mail.rfc3156 import MultipartSigned @@ -64,14 +73,32 @@ class SSLContextFactory(ssl.ClientContextFactory): return ctx -class OutgoingMail: +def outgoingFactory(userid, keymanager, opts, check_cert=True, bouncer=None): + + cert = unicode(opts.cert) + key = unicode(opts.key) + hostname = str(opts.hostname) + port = opts.port + + if check_cert: + if not os.path.isfile(cert): + raise errors.ConfigurationError( + 'No valid SMTP certificate could be found for %s!' % userid) + + return OutgoingMail( + str(userid), keymanager, cert, key, hostname, port, + bouncer) + + +class OutgoingMail(object): """ - A service for handling encrypted outgoing mail. + Sends Outgoing Mail, encrypting and signing if needed. """ - def __init__(self, from_address, keymanager, cert, key, host, port): + def __init__(self, from_address, keymanager, cert, key, host, port, + bouncer=None): """ - Initialize the mail service. + Initialize the outgoing mail service. :param from_address: The sender address. :type from_address: str @@ -109,6 +136,7 @@ class OutgoingMail: self._cert = cert self._from_address = from_address self._keymanager = keymanager + self._bouncer = bouncer def send_message(self, raw, recipient): """ @@ -121,8 +149,8 @@ class OutgoingMail: :return: a deferred which delivers the message when fired """ d = self._maybe_encrypt_and_sign(raw, recipient) - d.addCallback(self._route_msg) - d.addErrback(self.sendError) + d.addCallback(self._route_msg, raw) + d.addErrback(self.sendError, raw) return d def sendSuccess(self, smtp_sender_result): @@ -134,23 +162,41 @@ class OutgoingMail: :type smtp_sender_result: tuple(int, list(tuple)) """ dest_addrstr = smtp_sender_result[1][0][0] - log.msg('Message sent to %s' % dest_addrstr) - emit_async(catalog.SMTP_SEND_MESSAGE_SUCCESS, dest_addrstr) + fromaddr = self._from_address + log.msg('Message sent from %s to %s' % (fromaddr, dest_addrstr)) + emit_async(catalog.SMTP_SEND_MESSAGE_SUCCESS, + fromaddr, dest_addrstr) - def sendError(self, failure): + def sendError(self, failure, origmsg): """ Callback for an unsuccessfull send. - :param e: The result from the last errback. - :type e: anything + :param failure: The result from the last errback. + :type failure: anything + :param origmsg: the original, unencrypted, raw message, to be passed to + the bouncer. + :type origmsg: str """ - # XXX: need to get the address from the exception to send signal - # emit_async(catalog.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr) + # XXX: need to get the address from the original message to send signal + # emit_async(catalog.SMTP_SEND_MESSAGE_ERROR, self._from_address, + # self._user.dest.addrstr) + + # TODO when we implement outgoing queues/long-term-retries, we could + # examine the error *here* and delay the notification if it's just a + # temporal error. We might want to notify the permanent errors + # differently. + err = failure.value log.err(err) - raise err - def _route_msg(self, encrypt_and_sign_result): + if self._bouncer: + self._bouncer.bounce_message( + err.message, to=self._from_address, + orig=origmsg) + else: + raise err + + def _route_msg(self, encrypt_and_sign_result, raw): """ Sends the msg using the ESMTPSenderFactory. @@ -164,7 +210,8 @@ class OutgoingMail: # we construct a defer to pass to the ESMTPSenderFactory d = defer.Deferred() - d.addCallbacks(self.sendSuccess, self.sendError) + d.addCallback(self.sendSuccess) + d.addErrback(self.sendError, raw) # we don't pass an ssl context factory to the ESMTPSenderFactory # because ssl will be handled by reactor.connectSSL() below. factory = smtp.ESMTPSenderFactory( @@ -178,7 +225,8 @@ class OutgoingMail: requireAuthentication=False, requireTransportSecurity=True) factory.domain = __version__ - emit_async(catalog.SMTP_SEND_MESSAGE_START, recipient.dest.addrstr) + emit_async(catalog.SMTP_SEND_MESSAGE_START, + self._from_address, recipient.dest.addrstr) reactor.connectSSL( self._host, self._port, factory, contextFactory=SSLContextFactory(self._cert, self._key)) @@ -226,7 +274,7 @@ class OutgoingMail: origmsg = Parser().parsestr(raw) if origmsg.get_content_type() == 'multipart/encrypted': - return defer.success((origmsg, recipient)) + return defer.succeed((origmsg, recipient)) from_address = validate_address(self._from_address) username, domain = from_address.split('@') @@ -241,6 +289,7 @@ class OutgoingMail: def signal_encrypt_sign(newmsg): emit_async(catalog.SMTP_END_ENCRYPT_AND_SIGN, + self._from_address, "%s,%s" % (self._from_address, to_address)) return newmsg, recipient @@ -248,7 +297,7 @@ class OutgoingMail: failure.trap(KeyNotFound, KeyAddressMismatch) log.msg('Will send unencrypted message to %s.' % to_address) - emit_async(catalog.SMTP_START_SIGN, self._from_address) + emit_async(catalog.SMTP_START_SIGN, self._from_address, to_address) d = self._sign(message, from_address) d.addCallback(signal_sign) return d @@ -260,6 +309,7 @@ class OutgoingMail: log.msg("Will encrypt the message with %s and sign with %s." % (to_address, from_address)) emit_async(catalog.SMTP_START_ENCRYPT_AND_SIGN, + self._from_address, "%s,%s" % (self._from_address, to_address)) d = self._maybe_attach_key(origmsg, from_address, to_address) d.addCallback(maybe_encrypt_and_sign) @@ -457,7 +507,7 @@ class OutgoingMail: def add_openpgp_header(signkey): username, domain = sign_address.split('@') newmsg.add_header( - 'OpenPGP', 'id=%s' % signkey.key_id, + 'OpenPGP', 'id=%s' % signkey.fingerprint, url='https://%s/key/%s' % (domain, username), preference='signencrypt') return newmsg, origmsg diff --git a/src/leap/mail/outgoing/tests/test_outgoing.py b/src/leap/mail/outgoing/tests/test_outgoing.py index 5518b33..ad7803d 100644 --- a/src/leap/mail/outgoing/tests/test_outgoing.py +++ b/src/leap/mail/outgoing/tests/test_outgoing.py @@ -29,20 +29,19 @@ from twisted.mail.smtp import User from mock import Mock -from leap.mail.smtp.gateway import SMTPFactory from leap.mail.rfc3156 import RFC3156CompliantGenerator from leap.mail.outgoing.service import OutgoingMail -from leap.mail.tests import ( - TestCaseWithKeyManager, - ADDRESS, - ADDRESS_2, - PUBLIC_KEY_2, -) +from leap.mail.tests import TestCaseWithKeyManager +from leap.mail.tests import ADDRESS, ADDRESS_2, PUBLIC_KEY_2 +from leap.mail.smtp.tests.test_gateway import getSMTPFactory + from leap.keymanager import openpgp, errors BEGIN_PUBLIC_KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----" +TEST_USER = u'anotheruser@leap.se' + class TestOutgoingMail(TestCaseWithKeyManager): EMAIL_DATA = ['HELO gateway.leap.se', @@ -73,11 +72,12 @@ class TestOutgoingMail(TestCaseWithKeyManager): self.fromAddr, self._km, self._config['cert'], self._config['key'], self._config['host'], self._config['port']) - self.proto = SMTPFactory( - u'anotheruser@leap.se', - self._km, - self._config['encrypted_only'], - self.outgoing_mail).buildProtocol(('127.0.0.1', 0)) + + user = TEST_USER + + # TODO -- this shouldn't need SMTP to be tested!? or does it? + self.proto = getSMTPFactory( + {user: None}, {user: self._km}, {user: None}) self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS_2) d = TestCaseWithKeyManager.setUp(self) @@ -236,7 +236,7 @@ class TestOutgoingMail(TestCaseWithKeyManager): def _set_sign_used(self, address): def set_sign(key): key.sign_used = True - return self._km.put_key(key, address) + return self._km.put_key(key) d = self._km.get_key(address, openpgp.OpenPGPKey, fetch_remote=False) d.addCallback(set_sign) diff --git a/src/leap/mail/smtp/__init__.py b/src/leap/mail/smtp/__init__.py index 7b62808..9fab70a 100644 --- a/src/leap/mail/smtp/__init__.py +++ b/src/leap/mail/smtp/__init__.py @@ -23,47 +23,35 @@ import os from twisted.internet import reactor from twisted.internet.error import CannotListenError -from leap.mail.outgoing.service import OutgoingMail from leap.common.events import emit_async, catalog + from leap.mail.smtp.gateway import SMTPFactory logger = logging.getLogger(__name__) -def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port, - smtp_cert, smtp_key, encrypted_only): - """ - Setup SMTP gateway to run with Twisted. +SMTP_PORT = 2013 + - This function sets up the SMTP gateway configuration and the Twisted - reactor. +def run_service(soledad_sessions, keymanager_sessions, sendmail_opts, + port=SMTP_PORT): + """ + Main entry point to run the service from the client. - :param port: The port in which to run the server. - :type port: int - :param userid: The user currently logged in - :type userid: str - :param keymanager: A Key Manager from where to get recipients' public - keys. - :type keymanager: leap.common.keymanager.KeyManager - :param smtp_host: The hostname of the remote SMTP server. - :type smtp_host: str - :param smtp_port: The port of the remote SMTP server. - :type smtp_port: int - :param smtp_cert: The client certificate for authentication. - :type smtp_cert: str - :param smtp_key: The client key for authentication. - :type smtp_key: str - :param encrypted_only: Whether the SMTP gateway should send unencrypted - mail or not. - :type encrypted_only: bool + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by userid. + :param keymanager_sessions: a dict-like object, containing instances + of Keymanager, indexed by userid. + :param sendmail_opts: a dict-like object of sendmailOptions. - :returns: tuple of SMTPFactory, twisted.internet.tcp.Port + :returns: the port as returned by the reactor when starts listening, and + the factory for the protocol. + :rtype: tuple """ - # configure the use of this service with twistd - outgoing_mail = OutgoingMail( - userid, keymanager, smtp_cert, smtp_key, smtp_host, smtp_port) - factory = SMTPFactory(userid, keymanager, encrypted_only, outgoing_mail) + factory = SMTPFactory(soledad_sessions, keymanager_sessions, + sendmail_opts) + try: interface = "localhost" # don't bind just to localhost if we are running on docker since we @@ -71,8 +59,10 @@ def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port, if os.environ.get("LEAP_DOCKERIZED"): interface = '' + # TODO Use Endpoints instead -------------------------------- tport = reactor.listenTCP(port, factory, interface=interface) emit_async(catalog.SMTP_SERVICE_STARTED, str(port)) + return factory, tport except CannotListenError: logger.error("STMP Service failed to start: " diff --git a/src/leap/mail/smtp/bounces.py b/src/leap/mail/smtp/bounces.py new file mode 100644 index 0000000..7a4674b --- /dev/null +++ b/src/leap/mail/smtp/bounces.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# bounces.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Deliver bounces to the user Inbox. +""" +import time +from email.message import Message +from email.utils import formatdate + +from leap.mail.constants import INBOX_NAME +from leap.mail.mail import Account + + +# TODO implement localization for this template. + +BOUNCE_TEMPLATE = """This is your local Bitmask Mail Agent running at localhost. + +I'm sorry to have to inform you that your message could not be delivered to one +or more recipients. + +The reasons I got for the error are: + +{raw_error} + +If the problem persists and it's not a network connectivity issue, you might +want to contact your provider ({provider}) with this information (remove any +sensitive data before). + +--- Original message (*before* it was encrypted by bitmask) below ----: + +{orig}""" + + +class Bouncer(object): + """ + Implements a mechanism to deliver bounces to user inbox. + """ + # TODO this should follow RFC 6522, and compose a correct multipart + # attaching the report and the original message. Leaving this for a future + # iteration. + + def __init__(self, inbox_collection): + self._inbox_collection = inbox_collection + + def bounce_message(self, error_data, to, date=None, orig=''): + if not date: + date = formatdate(time.time()) + + raw_data = self._format_msg(error_data, to, date, orig) + d = self._inbox_collection.add_msg( + raw_data, ('\\Recent',), date=date) + return d + + def _format_msg(self, error_data, to, date, orig): + provider = to.split('@')[1] + + msg = Message() + msg.add_header( + 'From', 'bitmask-bouncer@localhost (Bitmask Local Agent)') + msg.add_header('To', to) + msg.add_header('Subject', 'Undelivered Message') + msg.add_header('Date', date) + msg.set_payload(BOUNCE_TEMPLATE.format( + raw_error=error_data, + provider=provider, + orig=orig)) + + return msg.as_string() + + +def bouncerFactory(soledad): + user_id = soledad.uuid + acc = Account(soledad, user_id) + d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox(INBOX_NAME)) + d.addCallback(lambda inbox: Bouncer(inbox)) + return d diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index 45560bf..bd0be6f 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -29,19 +29,27 @@ The following classes comprise the SMTP gateway service: * EncryptedMessage - An implementation of twisted.mail.smtp.IMessage that knows how to encrypt/sign itself before sending. """ +from email.Header import Header from zope.interface import implements +from zope.interface import implementer + +from twisted.cred.portal import Portal, IRealm from twisted.mail import smtp -from twisted.internet.protocol import ServerFactory +from twisted.mail.imap4 import LOGINCredentials, PLAINCredentials +from twisted.internet import defer, protocol from twisted.python import log -from email.Header import Header from leap.common.check import leap_assert_type from leap.common.events import emit_async, catalog -from leap.keymanager.openpgp import OpenPGPKey -from leap.keymanager.errors import KeyNotFound +from leap.mail import errors +from leap.mail.cred import LocalSoledadTokenChecker from leap.mail.utils import validate_address from leap.mail.rfc3156 import RFC3156CompliantGenerator +from leap.mail.outgoing.service import outgoingFactory +from leap.mail.smtp.bounces import bouncerFactory +from leap.keymanager.openpgp import OpenPGPKey +from leap.keymanager.errors import KeyNotFound # replace email generator with a RFC 3156 compliant one. from email import generator @@ -49,84 +57,176 @@ from email import generator generator.Generator = RFC3156CompliantGenerator -# -# Helper utilities -# - LOCAL_FQDN = "bitmask.local" -class SMTPHeloLocalhost(smtp.SMTP): +@implementer(IRealm) +class LocalSMTPRealm(object): + + _encoding = 'utf-8' + + def __init__(self, keymanager_sessions, soledad_sessions, sendmail_opts, + encrypted_only=False): + """ + :param keymanager_sessions: a dict-like object, containing instances + of a Keymanager objects, indexed by + userid. + """ + self._keymanager_sessions = keymanager_sessions + self._soledad_sessions = soledad_sessions + self._sendmail_opts = sendmail_opts + self.encrypted_only = encrypted_only + + def requestAvatar(self, avatarId, mind, *interfaces): + + if isinstance(avatarId, str): + avatarId = avatarId.decode(self._encoding) + + def gotKeymanagerAndSoledad(result): + keymanager, soledad = result + d = bouncerFactory(soledad) + d.addCallback(lambda bouncer: (keymanager, soledad, bouncer)) + return d + + def getMessageDelivery(result): + keymanager, soledad, bouncer = result + # TODO use IMessageDeliveryFactory instead ? + # it could reuse the connections. + if smtp.IMessageDelivery in interfaces: + userid = avatarId + opts = self.getSendingOpts(userid) + + outgoing = outgoingFactory( + userid, keymanager, opts, bouncer=bouncer) + avatar = SMTPDelivery(userid, keymanager, self.encrypted_only, + outgoing) + + return (smtp.IMessageDelivery, avatar, + getattr(avatar, 'logout', lambda: None)) + + raise NotImplementedError(self, interfaces) + + d1 = self.lookupKeymanagerInstance(avatarId) + d2 = self.lookupSoledadInstance(avatarId) + d = defer.gatherResults([d1, d2]) + d.addCallback(gotKeymanagerAndSoledad) + d.addCallback(getMessageDelivery) + return d + + def lookupKeymanagerInstance(self, userid): + print 'getting KM INSTNACE>>>' + try: + keymanager = self._keymanager_sessions[userid] + except: + raise errors.AuthenticationError( + 'No keymanager session found for user %s. Is it authenticated?' + % userid) + # XXX this should return the instance after whenReady callback + return defer.succeed(keymanager) + + def lookupSoledadInstance(self, userid): + try: + soledad = self._soledad_sessions[userid] + except: + raise errors.AuthenticationError( + 'No soledad session found for user %s. Is it authenticated?' + % userid) + # XXX this should return the instance after whenReady callback + return defer.succeed(soledad) + + def getSendingOpts(self, userid): + try: + opts = self._sendmail_opts[userid] + except KeyError: + raise errors.ConfigurationError( + 'No sendingMail options found for user %s' % userid) + return opts + + +class SMTPTokenChecker(LocalSoledadTokenChecker): + """A credentials checker that will lookup a token for the SMTP service. + For now it will be using the same identifier than IMAPTokenChecker""" + + service = 'mail_auth' + + # TODO besides checking for token credential, + # we could also verify the certificate here. + + +class LEAPInitMixin(object): + + """ + A Mixin that takes care of initialization of all the data needed to access + LEAP sessions. """ - An SMTP class that ensures a proper FQDN - for localhost. + def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts, + encrypted_only=False): + realm = LocalSMTPRealm( + keymanager_sessions, soledad_sessions, sendmail_opts, + encrypted_only) + portal = Portal(realm) + + checker = SMTPTokenChecker(soledad_sessions) + self.checker = checker + self.portal = portal + portal.registerChecker(checker) - This avoids a problem in which unproperly configured providers - would complain about the helo not being a fqdn. + +class LocalSMTPServer(smtp.ESMTP, LEAPInitMixin): + """ + The Production ESMTP Server: Authentication Needed. + Authenticates against SMTP Token stored in Local Soledad instance. + The Realm will produce a Delivery Object that handles encryption/signing. """ - def __init__(self, *args): - smtp.SMTP.__init__(self, *args) - self.host = LOCAL_FQDN + # TODO: implement Queue using twisted.mail.mail.MailService + + def __init__(self, soledads, keyms, sendmailopts, *args, **kw): + encrypted_only = kw.pop('encrypted_only', False) + + LEAPInitMixin.__init__(self, soledads, keyms, sendmailopts, + encrypted_only) + smtp.ESMTP.__init__(self, *args, **kw) -class SMTPFactory(ServerFactory): +# TODO implement retries -- see smtp.SenderMixin +class SMTPFactory(protocol.ServerFactory): """ Factory for an SMTP server with encrypted gatewaying capabilities. """ + + protocol = LocalSMTPServer domain = LOCAL_FQDN + timeout = 600 + encrypted_only = False - def __init__(self, userid, keymanager, encrypted_only, outgoing_mail): - """ - Initialize the SMTP factory. + def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts, + deferred=None, retries=3): - :param userid: The user currently logged in - :type userid: unicode - :param keymanager: A Key Manager from where to get recipients' public - keys. - :param encrypted_only: Whether the SMTP gateway should send unencrypted - mail or not. - :type encrypted_only: bool - :param outgoing_mail: The outgoing mail to send the message - :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail - """ - - leap_assert_type(encrypted_only, bool) - # and store them - self._userid = userid - self._km = keymanager - self._outgoing_mail = outgoing_mail - self._encrypted_only = encrypted_only + self._soledad_sessions = soledad_sessions + self._keymanager_sessions = keymanager_sessions + self._sendmail_opts = sendmail_opts def buildProtocol(self, addr): - """ - Return a protocol suitable for the job. - - :param addr: An address, e.g. a TCP (host, port). - :type addr: twisted.internet.interfaces.IAddress - - @return: The protocol. - @rtype: SMTPDelivery - """ - smtpProtocol = SMTPHeloLocalhost( - SMTPDelivery( - self._userid, self._km, self._encrypted_only, - self._outgoing_mail)) - smtpProtocol.factory = self - return smtpProtocol + p = self.protocol( + self._soledad_sessions, self._keymanager_sessions, + self._sendmail_opts, encrypted_only=self.encrypted_only) + p.factory = self + p.host = LOCAL_FQDN + p.challengers = {"LOGIN": LOGINCredentials, "PLAIN": PLAINCredentials} + return p # # SMTPDelivery # +@implementer(smtp.IMessageDelivery) class SMTPDelivery(object): """ Validate email addresses and handle message delivery. """ - implements(smtp.IMessageDelivery) - def __init__(self, userid, keymanager, encrypted_only, outgoing_mail): """ Initialize the SMTP delivery object. @@ -202,20 +302,21 @@ class SMTPDelivery(object): def found(_): log.msg("Accepting mail for %s..." % user.dest.addrstr) emit_async(catalog.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, - user.dest.addrstr) + self._userid, user.dest.addrstr) def not_found(failure): failure.trap(KeyNotFound) # if key was not found, check config to see if will send anyway if self._encrypted_only: - emit_async(catalog.SMTP_RECIPIENT_REJECTED, user.dest.addrstr) + emit_async(catalog.SMTP_RECIPIENT_REJECTED, self._userid, + user.dest.addrstr) raise smtp.SMTPBadRcpt(user.dest.addrstr) log.msg("Warning: will send an unencrypted message (because " "encrypted_only' is set to False).") emit_async( catalog.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, - user.dest.addrstr) + self._userid, user.dest.addrstr) def encrypt_func(_): return lambda: EncryptedMessage(user, self._outgoing_mail) @@ -307,7 +408,8 @@ class EncryptedMessage(object): """ log.msg("Connection lost unexpectedly!") log.err() - emit_async(catalog.SMTP_CONNECTION_LOST, self._user.dest.addrstr) + emit_async(catalog.SMTP_CONNECTION_LOST, self._userid, + self._user.dest.addrstr) # unexpected loss of connection; don't save self._lines = [] diff --git a/src/leap/mail/smtp/tests/test_gateway.py b/src/leap/mail/smtp/tests/test_gateway.py index 0b9a364..df83cf0 100644 --- a/src/leap/mail/smtp/tests/test_gateway.py +++ b/src/leap/mail/smtp/tests/test_gateway.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. - """ SMTP gateway tests. """ @@ -23,19 +22,18 @@ SMTP gateway tests. import re from datetime import datetime +from twisted.mail import smtp from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, fail, succeed, Deferred from twisted.test import proto_helpers from mock import Mock -from leap.mail.smtp.gateway import ( - SMTPFactory -) -from leap.mail.tests import ( - TestCaseWithKeyManager, - ADDRESS, - ADDRESS_2, -) +from leap.mail.smtp.gateway import SMTPFactory, LOCAL_FQDN +from leap.mail.smtp.gateway import SMTPDelivery + +from leap.mail.outgoing.service import outgoingFactory +from leap.mail.tests import TestCaseWithKeyManager +from leap.mail.tests import ADDRESS, ADDRESS_2 from leap.keymanager import openpgp, errors @@ -46,6 +44,52 @@ HOSTNAME_REGEX = "(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*" + \ "([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])" IP_OR_HOST_REGEX = '(' + IP_REGEX + '|' + HOSTNAME_REGEX + ')' +TEST_USER = u'anotheruser@leap.se' + + +def getSMTPFactory(soledad_s, keymanager_s, sendmail_opts, + encrypted_only=False): + factory = UnauthenticatedSMTPFactory + factory.encrypted_only = encrypted_only + proto = factory( + soledad_s, keymanager_s, sendmail_opts).buildProtocol(('127.0.0.1', 0)) + return proto + + +class UnauthenticatedSMTPServer(smtp.SMTP): + + encrypted_only = False + + def __init__(self, soledads, keyms, opts, encrypted_only=False): + smtp.SMTP.__init__(self) + + userid = TEST_USER + keym = keyms[userid] + + class Opts: + cert = '/tmp/cert' + key = '/tmp/cert' + hostname = 'remote' + port = 666 + + outgoing = outgoingFactory( + userid, keym, Opts, check_cert=False) + avatar = SMTPDelivery(userid, keym, encrypted_only, outgoing) + self.delivery = avatar + + def validateFrom(self, helo, origin): + return origin + + +class UnauthenticatedSMTPFactory(SMTPFactory): + """ + A Factory that produces a SMTP server that does not authenticate user. + Only for tests! + """ + protocol = UnauthenticatedSMTPServer + domain = LOCAL_FQDN + encrypted_only = False + class TestSmtpGateway(TestCaseWithKeyManager): @@ -85,14 +129,8 @@ class TestSmtpGateway(TestCaseWithKeyManager): '250 Recipient address accepted', '354 Continue'] - # XXX this bit can be refactored away in a helper - # method... - proto = SMTPFactory( - u'anotheruser@leap.se', - self._km, - self._config['encrypted_only'], - outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) - # snip... + user = TEST_USER + proto = getSMTPFactory({user: None}, {user: self._km}, {user: None}) transport = proto_helpers.StringTransport() proto.makeConnection(transport) reply = "" @@ -116,12 +154,10 @@ class TestSmtpGateway(TestCaseWithKeyManager): # mock the key fetching self._km._fetch_keys_from_server = Mock( return_value=fail(errors.KeyNotFound())) - # prepare the SMTP factory - proto = SMTPFactory( - u'anotheruser@leap.se', - self._km, - self._config['encrypted_only'], - outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) + user = TEST_USER + proto = getSMTPFactory( + {user: None}, {user: self._km}, {user: None}, + encrypted_only=True) transport = proto_helpers.StringTransport() proto.makeConnection(transport) yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport) @@ -132,7 +168,7 @@ class TestSmtpGateway(TestCaseWithKeyManager): self.assertEqual( '550 Cannot receive for specified address\r\n', reply, - 'Address should have been rejecetd with appropriate message.') + 'Address should have been rejected with appropriate message.') proto.setTimeout(None) @inlineCallbacks @@ -149,11 +185,8 @@ class TestSmtpGateway(TestCaseWithKeyManager): # mock the key fetching self._km._fetch_keys_from_server = Mock( return_value=fail(errors.KeyNotFound())) - # prepare the SMTP factory with encrypted only equal to false - proto = SMTPFactory( - u'anotheruser@leap.se', - self._km, - False, outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) + user = TEST_USER + proto = getSMTPFactory({user: None}, {user: self._km}, {user: None}) transport = proto_helpers.StringTransport() proto.makeConnection(transport) yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport) diff --git a/src/leap/mail/tests/__init__.py b/src/leap/mail/tests/__init__.py index 71452d2..8094c11 100644 --- a/src/leap/mail/tests/__init__.py +++ b/src/leap/mail/tests/__init__.py @@ -94,6 +94,8 @@ class TestCaseWithKeyManager(unittest.TestCase, BaseLeapTest): gpgbinary=self.GPG_BINARY_PATH) self._km._fetcher.put = Mock() self._km._fetcher.get = Mock(return_value=Response()) + self._km._async_client.request = Mock(return_value="") + self._km._async_client_pinned.request = Mock(return_value="") d1 = self._km.put_raw_key(PRIVATE_KEY, OpenPGPKey, ADDRESS) d2 = self._km.put_raw_key(PRIVATE_KEY_2, OpenPGPKey, ADDRESS_2) diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py index 9f40ffb..aca406f 100644 --- a/src/leap/mail/tests/test_mail.py +++ b/src/leap/mail/tests/test_mail.py @@ -317,12 +317,12 @@ class AccountTestCase(SoledadTestMixin): """ Tests for the Account class. """ - def get_account(self): + def get_account(self, user_id): store = self._soledad - return Account(store) + return Account(store, user_id) def test_add_mailbox(self): - acc = self.get_account() + acc = self.get_account('some_user_id') d = acc.callWhenReady(lambda _: acc.add_mailbox("TestMailbox")) d.addCallback(lambda _: acc.list_all_mailbox_names()) d.addCallback(self._test_add_mailbox_cb) @@ -333,7 +333,7 @@ class AccountTestCase(SoledadTestMixin): self.assertItemsEqual(mboxes, expected) def test_delete_mailbox(self): - acc = self.get_account() + acc = self.get_account('some_user_id') d = acc.callWhenReady(lambda _: acc.delete_mailbox("Inbox")) d.addCallback(lambda _: acc.list_all_mailbox_names()) d.addCallback(self._test_delete_mailbox_cb) @@ -344,7 +344,7 @@ class AccountTestCase(SoledadTestMixin): self.assertItemsEqual(mboxes, expected) def test_rename_mailbox(self): - acc = self.get_account() + acc = self.get_account('some_user_id') d = acc.callWhenReady(lambda _: acc.add_mailbox("OriginalMailbox")) d.addCallback(lambda _: acc.rename_mailbox( "OriginalMailbox", "RenamedMailbox")) @@ -357,7 +357,7 @@ class AccountTestCase(SoledadTestMixin): self.assertItemsEqual(mboxes, expected) def test_get_all_mailboxes(self): - acc = self.get_account() + acc = self.get_account('some_user_id') d = acc.callWhenReady(lambda _: acc.add_mailbox("OneMailbox")) d.addCallback(lambda _: acc.add_mailbox("TwoMailbox")) d.addCallback(lambda _: acc.add_mailbox("ThreeMailbox")) @@ -374,7 +374,7 @@ class AccountTestCase(SoledadTestMixin): self.assertItemsEqual(names, expected) def test_get_collection_by_mailbox(self): - acc = self.get_account() + acc = self.get_account('some_user_id') d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox("INBOX")) d.addCallback(self._test_get_collection_by_mailbox_cb) return d diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 1c74366..17349e6 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -17,20 +17,19 @@ """ Utilities for walking along a message tree. """ -import os - -from pycryptopp.hash import sha256 +from cryptography.hazmat.backends.multibackend import MultiBackend +from cryptography.hazmat.backends.openssl.backend import ( + Backend as OpenSSLBackend) +from cryptography.hazmat.primitives import hashes from leap.mail.utils import first -DEBUG = os.environ.get("BITMASK_MAIL_DEBUG") -if DEBUG: - def get_hash(s): - return sha256.SHA256(s).hexdigest()[:10] -else: - def get_hash(s): - return sha256.SHA256(s).hexdigest() +def get_hash(s): + backend = MultiBackend([OpenSSLBackend()]) + digest = hashes.Hash(hashes.SHA256(), backend) + digest.update(s) + return digest.finalize().encode("hex").upper() """ @@ -92,7 +91,7 @@ def get_raw_docs(msg, parts): return ( { "type": "cnt", # type content they'll be - "raw": payload if not DEBUG else payload[:100], + "raw": payload, "phash": get_hash(payload), "content-disposition": first(headers.get( 'content-disposition', '').split(';')), @@ -168,10 +167,6 @@ def walk_msg_tree(parts, body_phash=None): inner_headers = parts[1].get(HEADERS, None) if ( len(parts) == 2) else None - if DEBUG: - print "parts vector: ", pv - print - # wrappers vector def getwv(pv): return [ @@ -209,7 +204,6 @@ def walk_msg_tree(parts, body_phash=None): last_part = max(main_pmap.keys()) main_pmap[last_part][PART_MAP] = {} for partind in range(len(pv) - 1): - print partind + 1, len(parts) main_pmap[last_part][PART_MAP][partind] = parts[partind + 1] outer = parts[0] diff --git a/versioneer.py b/versioneer.py index 4e2c0a5..7ed2a21 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1,170 +1,641 @@ -#! /usr/bin/python -"""versioneer.py +# Version: 0.16 -(like a rocketeer, but for versions) +"""The Versioneer - like a rocketeer, but for versions. +The Versioneer +============== + +* like a rocketeer, but for versions! * https://github.com/warner/python-versioneer * Brian Warner * License: Public Domain -* Version: 0.7+ - -This file helps distutils-based projects manage their version number by just -creating version-control tags. - -For developers who work from a VCS-generated tree (e.g. 'git clone' etc), -each 'setup.py version', 'setup.py build', 'setup.py sdist' will compute a -version number by asking your version-control tool about the current -checkout. The version number will be written into a generated _version.py -file of your choosing, where it can be included by your __init__.py - -For users who work from a VCS-generated tarball (e.g. 'git archive'), it will -compute a version number by looking at the name of the directory created when -te tarball is unpacked. This conventionally includes both the name of the -project and a version number. - -For users who work from a tarball built by 'setup.py sdist', it will get a -version number from a previously-generated _version.py file. - -As a result, loading code directly from the source tree will not result in a -real version. If you want real versions from VCS trees (where you frequently -update from the upstream repository, or do new development), you will need to -do a 'setup.py version' after each update, and load code from the build/ -directory. - -You need to provide this code with a few configuration values: - - versionfile_source: - A project-relative pathname into which the generated version strings - should be written. This is usually a _version.py next to your project's - main __init__.py file. If your project uses src/myproject/__init__.py, - this should be 'src/myproject/_version.py'. This file should be checked - in to your VCS as usual: the copy created below by 'setup.py - update_files' will include code that parses expanded VCS keywords in - generated tarballs. The 'build' and 'sdist' commands will replace it with - a copy that has just the calculated version string. - - versionfile_build: - Like versionfile_source, but relative to the build directory instead of - the source directory. These will differ when your setup.py uses - 'package_dir='. If you have package_dir={'myproject': 'src/myproject'}, - then you will probably have versionfile_build='myproject/_version.py' and - versionfile_source='src/myproject/_version.py'. - - tag_prefix: a string, like 'PROJECTNAME-', which appears at the start of all - VCS tags. If your tags look like 'myproject-1.2.0', then you - should use tag_prefix='myproject-'. If you use unprefixed tags - like '1.2.0', this should be an empty string. - - parentdir_prefix: a string, frequently the same as tag_prefix, which - appears at the start of all unpacked tarball filenames. If - your tarball unpacks into 'myproject-1.2.0', this should - be 'myproject-'. - -To use it: - - 1: include this file in the top level of your project - 2: make the following changes to the top of your setup.py: - import versioneer - versioneer.versionfile_source = 'src/myproject/_version.py' - versioneer.versionfile_build = 'myproject/_version.py' - versioneer.tag_prefix = '' # tags are like 1.2.0 - versioneer.parentdir_prefix = 'myproject-' # dirname like 'myproject-1.2.0' - 3: add the following arguments to the setup() call in your setup.py: - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), - 4: run 'setup.py update_files', which will create _version.py, and will - modify your __init__.py to define __version__ (by calling a function - from _version.py) - 5: modify your MANIFEST.in to include versioneer.py - 6: add both versioneer.py and the generated _version.py to your VCS -""" +* Compatible With: python2.6, 2.7, 3.3, 3.4, 3.5, and pypy +* [![Latest Version] +(https://pypip.in/version/versioneer/badge.svg?style=flat) +](https://pypi.python.org/pypi/versioneer/) +* [![Build Status] +(https://travis-ci.org/warner/python-versioneer.png?branch=master) +](https://travis-ci.org/warner/python-versioneer) + +This is a tool for managing a recorded version number in distutils-based +python projects. The goal is to remove the tedious and error-prone "update +the embedded version string" step from your release process. Making a new +release should be as easy as recording a new tag in your version-control +system, and maybe making new tarballs. + + +## Quick Install + +* `pip install versioneer` to somewhere to your $PATH +* add a `[versioneer]` section to your setup.cfg (see below) +* run `versioneer install` in your source tree, commit the results + +## Version Identifiers + +Source trees come from a variety of places: + +* a version-control system checkout (mostly used by developers) +* a nightly tarball, produced by build automation +* a snapshot tarball, produced by a web-based VCS browser, like github's + "tarball from tag" feature +* a release tarball, produced by "setup.py sdist", distributed through PyPI + +Within each source tree, the version identifier (either a string or a number, +this tool is format-agnostic) can come from a variety of places: + +* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows + about recent "tags" and an absolute revision-id +* the name of the directory into which the tarball was unpacked +* an expanded VCS keyword ($Id$, etc) +* a `_version.py` created by some earlier build step + +For released software, the version identifier is closely related to a VCS +tag. Some projects use tag names that include more than just the version +string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool +needs to strip the tag prefix to extract the version identifier. For +unreleased software (between tags), the version identifier should provide +enough information to help developers recreate the same tree, while also +giving them an idea of roughly how old the tree is (after version 1.2, before +version 1.3). Many VCS systems can report a description that captures this, +for example `git describe --tags --dirty --always` reports things like +"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the +0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has +uncommitted changes. + +The version identifier is used for multiple purposes: + +* to allow the module to self-identify its version: `myproject.__version__` +* to choose a name and prefix for a 'setup.py sdist' tarball + +## Theory of Operation + +Versioneer works by adding a special `_version.py` file into your source +tree, where your `__init__.py` can import it. This `_version.py` knows how to +dynamically ask the VCS tool for version information at import time. + +`_version.py` also contains `$Revision$` markers, and the installation +process marks `_version.py` to have this marker rewritten with a tag name +during the `git archive` command. As a result, generated tarballs will +contain enough information to get the proper version. + +To allow `setup.py` to compute a version too, a `versioneer.py` is added to +the top level of your source tree, next to `setup.py` and the `setup.cfg` +that configures it. This overrides several distutils/setuptools commands to +compute the version when invoked, and changes `setup.py build` and `setup.py +sdist` to replace `_version.py` with a small static file that contains just +the generated version data. + +## Installation + +First, decide on values for the following configuration variables: + +* `VCS`: the version control system you use. Currently accepts "git". + +* `style`: the style of version string to be produced. See "Styles" below for + details. Defaults to "pep440", which looks like + `TAG[+DISTANCE.gSHORTHASH[.dirty]]`. + +* `versionfile_source`: + + A project-relative pathname into which the generated version strings should + be written. This is usually a `_version.py` next to your project's main + `__init__.py` file, so it can be imported at runtime. If your project uses + `src/myproject/__init__.py`, this should be `src/myproject/_version.py`. + This file should be checked in to your VCS as usual: the copy created below + by `setup.py setup_versioneer` will include code that parses expanded VCS + keywords in generated tarballs. The 'build' and 'sdist' commands will + replace it with a copy that has just the calculated version string. + + This must be set even if your project does not have any modules (and will + therefore never import `_version.py`), since "setup.py sdist" -based trees + still need somewhere to record the pre-calculated version strings. Anywhere + in the source tree should do. If there is a `__init__.py` next to your + `_version.py`, the `setup.py setup_versioneer` command (described below) + will append some `__version__`-setting assignments, if they aren't already + present. + +* `versionfile_build`: + + Like `versionfile_source`, but relative to the build directory instead of + the source directory. These will differ when your setup.py uses + 'package_dir='. If you have `package_dir={'myproject': 'src/myproject'}`, + then you will probably have `versionfile_build='myproject/_version.py'` and + `versionfile_source='src/myproject/_version.py'`. + + If this is set to None, then `setup.py build` will not attempt to rewrite + any `_version.py` in the built tree. If your project does not have any + libraries (e.g. if it only builds a script), then you should use + `versionfile_build = None`. To actually use the computed version string, + your `setup.py` will need to override `distutils.command.build_scripts` + with a subclass that explicitly inserts a copy of + `versioneer.get_version()` into your script file. See + `test/demoapp-script-only/setup.py` for an example. + +* `tag_prefix`: + + a string, like 'PROJECTNAME-', which appears at the start of all VCS tags. + If your tags look like 'myproject-1.2.0', then you should use + tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this + should be an empty string, using either `tag_prefix=` or `tag_prefix=''`. + +* `parentdir_prefix`: + + a optional string, frequently the same as tag_prefix, which appears at the + start of all unpacked tarball filenames. If your tarball unpacks into + 'myproject-1.2.0', this should be 'myproject-'. To disable this feature, + just omit the field from your `setup.cfg`. + +This tool provides one script, named `versioneer`. That script has one mode, +"install", which writes a copy of `versioneer.py` into the current directory +and runs `versioneer.py setup` to finish the installation. + +To versioneer-enable your project: + +* 1: Modify your `setup.cfg`, adding a section named `[versioneer]` and + populating it with the configuration values you decided earlier (note that + the option names are not case-sensitive): + + ```` + [versioneer] + VCS = git + style = pep440 + versionfile_source = src/myproject/_version.py + versionfile_build = myproject/_version.py + tag_prefix = + parentdir_prefix = myproject- + ```` + +* 2: Run `versioneer install`. This will do the following: + + * copy `versioneer.py` into the top of your source tree + * create `_version.py` in the right place (`versionfile_source`) + * modify your `__init__.py` (if one exists next to `_version.py`) to define + `__version__` (by calling a function from `_version.py`) + * modify your `MANIFEST.in` to include both `versioneer.py` and the + generated `_version.py` in sdist tarballs + + `versioneer install` will complain about any problems it finds with your + `setup.py` or `setup.cfg`. Run it multiple times until you have fixed all + the problems. + +* 3: add a `import versioneer` to your setup.py, and add the following + arguments to the setup() call: + + version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), + +* 4: commit these changes to your VCS. To make sure you won't forget, + `versioneer install` will mark everything it touched for addition using + `git add`. Don't forget to add `setup.py` and `setup.cfg` too. + +## Post-Installation Usage + +Once established, all uses of your tree from a VCS checkout should get the +current version string. All generated tarballs should include an embedded +version string (so users who unpack them will not need a VCS tool installed). + +If you distribute your project through PyPI, then the release process should +boil down to two steps: + +* 1: git tag 1.0 +* 2: python setup.py register sdist upload + +If you distribute it through github (i.e. users use github to generate +tarballs with `git archive`), the process is: + +* 1: git tag 1.0 +* 2: git push; git push --tags + +Versioneer will report "0+untagged.NUMCOMMITS.gHASH" until your tree has at +least one tag in its history. + +## Version-String Flavors + +Code which uses Versioneer can learn about its version string at runtime by +importing `_version` from your main `__init__.py` file and running the +`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can +import the top-level `versioneer.py` and run `get_versions()`. + +Both functions return a dictionary with different flavors of version +information: + +* `['version']`: A condensed version string, rendered using the selected + style. This is the most commonly used value for the project's version + string. The default "pep440" style yields strings like `0.11`, + `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section + below for alternative styles. + +* `['full-revisionid']`: detailed revision identifier. For Git, this is the + full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". + +* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that + this is only accurate if run in a VCS checkout, otherwise it is likely to + be False or None + +* `['error']`: if the version string could not be computed, this will be set + to a string describing the problem, otherwise it will be None. It may be + useful to throw an exception in setup.py if this is set, to avoid e.g. + creating tarballs with a version string of "unknown". + +Some variants are more useful than others. Including `full-revisionid` in a +bug report should allow developers to reconstruct the exact code being tested +(or indicate the presence of local changes that should be shared with the +developers). `version` is suitable for display in an "about" box or a CLI +`--version` output: it can be easily compared against release notes and lists +of bugs fixed in various releases. + +The installer adds the following text to your `__init__.py` to place a basic +version in `YOURPROJECT.__version__`: + + from ._version import get_versions + __version__ = get_versions()['version'] + del get_versions + +## Styles + +The setup.cfg `style=` configuration controls how the VCS information is +rendered into a version string. + +The default style, "pep440", produces a PEP440-compliant string, equal to the +un-prefixed tag name for actual releases, and containing an additional "local +version" section with more detail for in-between builds. For Git, this is +TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags +--dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the +tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and +that this commit is two revisions ("+2") beyond the "0.11" tag. For released +software (exactly equal to a known tag), the identifier will only contain the +stripped tag, e.g. "0.11". + +Other styles are available. See details.md in the Versioneer source tree for +descriptions. + +## Debugging + +Versioneer tries to avoid fatal errors: if something goes wrong, it will tend +to return a version of "0+unknown". To investigate the problem, run `setup.py +version`, which will run the version-lookup code in a verbose mode, and will +display the full contents of `get_versions()` (including the `error` string, +which may help identify what went wrong). + +## Updating Versioneer + +To upgrade your project to a new release of Versioneer, do the following: + +* install the new Versioneer (`pip install -U versioneer` or equivalent) +* edit `setup.cfg`, if necessary, to include any new configuration settings + indicated by the release notes +* re-run `versioneer install` in your source tree, to replace + `SRC/_version.py` +* commit any changed files + +### Upgrading to 0.16 + +Nothing special. + +### Upgrading to 0.15 + +Starting with this version, Versioneer is configured with a `[versioneer]` +section in your `setup.cfg` file. Earlier versions required the `setup.py` to +set attributes on the `versioneer` module immediately after import. The new +version will refuse to run (raising an exception during import) until you +have provided the necessary `setup.cfg` section. + +In addition, the Versioneer package provides an executable named +`versioneer`, and the installation process is driven by running `versioneer +install`. In 0.14 and earlier, the executable was named +`versioneer-installer` and was run without an argument. + +### Upgrading to 0.14 -import os, sys, re -from distutils.core import Command -from distutils.command.sdist import sdist as _sdist -from distutils.command.build import build as _build +0.14 changes the format of the version string. 0.13 and earlier used +hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a +plus-separated "local version" section strings, with dot-separated +components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old +format, but should be ok with the new one. -versionfile_source = None -versionfile_build = None -tag_prefix = None -parentdir_prefix = None +### Upgrading from 0.11 to 0.12 -VCS = "git" -IN_LONG_VERSION_PY = False +Nothing special. +### Upgrading from 0.10 to 0.11 -LONG_VERSION_PY = ''' -IN_LONG_VERSION_PY = True +You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running +`setup.py setup_versioneer`. This will enable the use of additional +version-control systems (SVN, etc) in the future. + +## Future Directions + +This tool is designed to make it easily extended to other version-control +systems: all VCS-specific components are in separate directories like +src/git/ . The top-level `versioneer.py` script is assembled from these +components by running make-versioneer.py . In the future, make-versioneer.py +will take a VCS name as an argument, and will construct a version of +`versioneer.py` that is specific to the given VCS. It might also take the +configuration arguments that are currently provided manually during +installation by editing setup.py . Alternatively, it might go the other +direction and include code from all supported VCS systems, reducing the +number of intermediate scripts. + + +## License + +To make Versioneer easier to embed, all its code is dedicated to the public +domain. The `_version.py` that it creates is also in the public domain. +Specifically, both are released under the Creative Commons "Public Domain +Dedication" license (CC0-1.0), as described in +https://creativecommons.org/publicdomain/zero/1.0/ . + +""" + +from __future__ import print_function +try: + import configparser +except ImportError: + import ConfigParser as configparser +import errno +import json +import os +import re +import subprocess +import sys + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_root(): + """Get the project root directory. + + We require that all commands are run from the project root, i.e. the + directory that contains setup.py, setup.cfg, and versioneer.py . + """ + root = os.path.realpath(os.path.abspath(os.getcwd())) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + # allow 'python path/to/setup.py COMMAND' + root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + err = ("Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND').") + raise VersioneerBadRootError(err) + try: + # Certain runtime workflows (setup.py install/develop in a setuptools + # tree) execute all dependencies in a single python process, so + # "versioneer" may be imported multiple times, and python's shared + # module-import table will cache the first one. So we can't use + # os.path.dirname(__file__), as that will find whichever + # versioneer.py was first imported, even in later projects. + me = os.path.realpath(os.path.abspath(__file__)) + if os.path.splitext(me)[0] != os.path.splitext(versioneer_py)[0]: + print("Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py)) + except NameError: + pass + return root + + +def get_config_from_root(root): + """Read the project setup.cfg file to determine Versioneer config.""" + # This might raise EnvironmentError (if setup.cfg is missing), or + # configparser.NoSectionError (if it lacks a [versioneer] section), or + # configparser.NoOptionError (if it lacks "VCS="). See the docstring at + # the top of versioneer.py for instructions on writing your setup.cfg . + setup_cfg = os.path.join(root, "setup.cfg") + parser = configparser.SafeConfigParser() + with open(setup_cfg, "r") as f: + parser.readfp(f) + VCS = parser.get("versioneer", "VCS") # mandatory + + def get(parser, name): + if parser.has_option("versioneer", name): + return parser.get("versioneer", name) + return None + cfg = VersioneerConfig() + cfg.VCS = VCS + cfg.style = get(parser, "style") or "" + cfg.versionfile_source = get(parser, "versionfile_source") + cfg.versionfile_build = get(parser, "versionfile_build") + cfg.tag_prefix = get(parser, "tag_prefix") + if cfg.tag_prefix in ("''", '""'): + cfg.tag_prefix = "" + cfg.parentdir_prefix = get(parser, "parentdir_prefix") + cfg.verbose = get(parser, "verbose") + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + +# these dictionaries contain VCS-specific tools +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + return None + return stdout +LONG_VERSION_PY['git'] = ''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (build by setup.py sdist) and build +# feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.7+ (https://github.com/warner/python-versioneer) - -# these strings will be replaced by git during git-archive -git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" -git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" +# versioneer-0.16 (https://github.com/warner/python-versioneer) +"""Git implementation of _version.py.""" +import errno +import os +import re import subprocess import sys -def run_command(args, cwd=None, verbose=False): - try: - # remember shell=False, so use git.exe on windows, not just git - p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError: - e = sys.exc_info()[1] + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" + git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" + keywords = {"refnames": git_refnames, "full": git_full} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "%(STYLE)s" + cfg.tag_prefix = "%(TAG_PREFIX)s" + cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" + cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %%s" %% dispcmd) + print(e) + return None + else: if verbose: - print("unable to run %%s" %% args[0]) - print(e) + print("unable to find command, tried %%s" %% (commands,)) return None stdout = p.communicate()[0].strip() - if sys.version >= '3': + if sys.version_info[0] >= 3: stdout = stdout.decode() if p.returncode != 0: if verbose: - print("unable to run %%s (error)" %% args[0]) + print("unable to run %%s (error)" %% dispcmd) return None return stdout -import sys -import re -import os.path +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%%s', but '%%s' doesn't start with " + "prefix '%%s'" %% (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} -def get_expanded_variables(versionfile_source): + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these - # variables. When used from setup.py, we don't want to import - # _version.py, so we do it with a regexp instead. This function is not - # used from _version.py. - variables = {} + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} try: - f = open(versionfile_source,"r") + f = open(versionfile_abs, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["refnames"] = mo.group(1) + keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["full"] = mo.group(1) + keywords["full"] = mo.group(1) f.close() except EnvironmentError: pass - return variables + return keywords + -def versions_from_expanded_variables(variables, tag_prefix, verbose=False): - refnames = variables["refnames"].strip() +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: - print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -189,172 +660,350 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): r = ref[len(tag_prefix):] if verbose: print("picking %%s" %% r) - return { "version": r, - "full": variables["full"].strip() } - # no suitable tags, so we use the full revision id + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: - print("no suitable tags, using full revision id") - return { "version": variables["full"].strip(), - "full": variables["full"].strip() } - -def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): - # this runs 'git' from the root of the source tree. That either means - # someone ran a setup.py command (and this code is in versioneer.py, so - # IN_LONG_VERSION_PY=False, thus the containing directory is the root of - # the source tree), or someone ran a project-specific entry point (and - # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the - # containing directory is somewhere deeper in the source tree). This only - # gets called if the git-archive 'subst' variables were *not* expanded, - # and _version.py hasn't already been rewritten with a short version - # string, meaning we're inside a checked out source tree. + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} - try: - here = os.path.abspath(__file__) - except NameError: - # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct - - # versionfile_source is the relative path from the top of the source tree - # (where the .git directory might live) to this file. Invert this to find - # the root from __file__. - root = here - if IN_LONG_VERSION_PY: - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) - else: - root = os.path.dirname(here) + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ if not os.path.exists(os.path.join(root, ".git")): if verbose: print("no .git in %%s" %% root) - return {} + raise NotThisMethod("no .git directory") - GIT = "git" + GITS = ["git"] if sys.platform == "win32": - GIT = "git.exe" - stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], - cwd=root) - if stdout is None: - return {} - if not stdout.startswith(tag_prefix): - if verbose: - print("tag '%%s' doesn't start with prefix '%%s'" %% (stdout, tag_prefix)) - return {} - tag = stdout[len(tag_prefix):] - stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) - if stdout is None: - return {} - full = stdout.strip() - if tag.endswith("-dirty"): - full += "-dirty" - return {"version": tag, "full": full} - - -def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False): - if IN_LONG_VERSION_PY: - # We're running from _version.py. If it's from a source tree - # (execute-in-place), we can work upwards to find the root of the - # tree, and then check the parent directory for a version string. If - # it's in an installed application, there's no hope. - try: - here = os.path.abspath(__file__) - except NameError: - # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope - # versionfile_source is the relative path from the top of the source - # tree to _version.py. Invert this to find the root from __file__. - root = here - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%%s*" %% tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%%s'" + %% describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%%s' doesn't start with prefix '%%s'" + print(fmt %% (full_tag, tag_prefix)) + pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" + %% (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%%d" %% pieces["distance"] else: - # we're running from versioneer.py, which means we're running from - # the setup.py in a source tree. sys.argv[0] is setup.py in the root. - here = os.path.abspath(sys.argv[0]) - root = os.path.dirname(here) + # exception #1 + rendered = "0.post.dev%%d" %% pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%%s" %% pieces["short"] + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%%s" %% pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%%s', but '%%s' doesn't start with prefix '%%s'" %% - (root, dirname, parentdir_prefix)) - return None - return {"version": dirname[len(parentdir_prefix):], "full": ""} - -tag_prefix = "%(TAG_PREFIX)s" -parentdir_prefix = "%(PARENTDIR_PREFIX)s" -versionfile_source = "%(VERSIONFILE_SOURCE)s" - -def get_versions(default={"version": "unknown", "full": ""}, verbose=False): - variables = { "refnames": git_refnames, "full": git_full } - ver = versions_from_expanded_variables(variables, tag_prefix, verbose) - if not ver: - ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) - if not ver: - ver = versions_from_parentdir(parentdir_prefix, versionfile_source, - verbose) - if not ver: - ver = default - return ver -''' +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + Like 'git describe --tags --dirty --always'. -import subprocess -import sys + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%%s'" %% style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose -def run_command(args, cwd=None, verbose=False): try: - # remember shell=False, so use git.exe on windows, not just git - p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError: - e = sys.exc_info()[1] - if verbose: - print("unable to run %s" % args[0]) - print(e) - return None - stdout = p.communicate()[0].strip() - if sys.version >= '3': - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % args[0]) - return None - return stdout + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree"} -import sys -import re -import os.path + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass -def get_expanded_variables(versionfile_source): + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version"} +''' + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these - # variables. When used from setup.py, we don't want to import - # _version.py, so we do it with a regexp instead. This function is not - # used from _version.py. - variables = {} + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} try: - f = open(versionfile_source,"r") + f = open(versionfile_abs, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["refnames"] = mo.group(1) + keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["full"] = mo.group(1) + keywords["full"] = mo.group(1) f.close() except EnvironmentError: pass - return variables + return keywords + -def versions_from_expanded_variables(variables, tag_prefix, verbose=False): - refnames = variables["refnames"].strip() +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: - print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -379,107 +1028,122 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): r = ref[len(tag_prefix):] if verbose: print("picking %s" % r) - return { "version": r, - "full": variables["full"].strip() } - # no suitable tags, so we use the full revision id + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: - print("no suitable tags, using full revision id") - return { "version": variables["full"].strip(), - "full": variables["full"].strip() } - -def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): - # this runs 'git' from the root of the source tree. That either means - # someone ran a setup.py command (and this code is in versioneer.py, so - # IN_LONG_VERSION_PY=False, thus the containing directory is the root of - # the source tree), or someone ran a project-specific entry point (and - # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the - # containing directory is somewhere deeper in the source tree). This only - # gets called if the git-archive 'subst' variables were *not* expanded, - # and _version.py hasn't already been rewritten with a short version - # string, meaning we're inside a checked out source tree. + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} - try: - here = os.path.abspath(__file__) - except NameError: - # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct - - # versionfile_source is the relative path from the top of the source tree - # (where the .git directory might live) to this file. Invert this to find - # the root from __file__. - root = here - if IN_LONG_VERSION_PY: - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) - else: - root = os.path.dirname(here) + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ if not os.path.exists(os.path.join(root, ".git")): if verbose: print("no .git in %s" % root) - return {} + raise NotThisMethod("no .git directory") - GIT = "git" + GITS = ["git"] if sys.platform == "win32": - GIT = "git.exe" - stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], - cwd=root) - if stdout is None: - return {} - if not stdout.startswith(tag_prefix): - if verbose: - print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix)) - return {} - tag = stdout[len(tag_prefix):] - stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) - if stdout is None: - return {} - full = stdout.strip() - if tag.endswith("-dirty"): - full += "-dirty" - return {"version": tag, "full": full} - - -def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False): - if IN_LONG_VERSION_PY: - # We're running from _version.py. If it's from a source tree - # (execute-in-place), we can work upwards to find the root of the - # tree, and then check the parent directory for a version string. If - # it's in an installed application, there's no hope. - try: - here = os.path.abspath(__file__) - except NameError: - # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope - # versionfile_source is the relative path from the top of the source - # tree to _version.py. Invert this to find the root from __file__. - root = here - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + else: - # we're running from versioneer.py, which means we're running from - # the setup.py in a source tree. sys.argv[0] is setup.py in the root. - here = os.path.abspath(sys.argv[0]) - root = os.path.dirname(here) + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" % - (root, dirname, parentdir_prefix)) - return None - return {"version": dirname[len(parentdir_prefix):], "full": ""} + return pieces -import sys -def do_vcs_install(versionfile_source, ipy): - GIT = "git" +def do_vcs_install(manifest_in, versionfile_source, ipy): + """Git-specific installation logic for Versioneer. + + For Git, this means creating/changing .gitattributes to mark _version.py + for export-time keyword substitution. + """ + GITS = ["git"] if sys.platform == "win32": - GIT = "git.exe" - run_command([GIT, "add", "versioneer.py"]) - run_command([GIT, "add", versionfile_source]) - run_command([GIT, "add", ipy]) + GITS = ["git.cmd", "git.exe"] + files = [manifest_in, versionfile_source] + if ipy: + files.append(ipy) + try: + me = __file__ + if me.endswith(".pyc") or me.endswith(".pyo"): + me = os.path.splitext(me)[0] + ".py" + versioneer_file = os.path.relpath(me) + except NameError: + versioneer_file = "versioneer.py" + files.append(versioneer_file) present = False try: f = open(".gitattributes", "r") @@ -494,135 +1158,487 @@ def do_vcs_install(versionfile_source, ipy): f = open(".gitattributes", "a+") f.write("%s export-subst\n" % versionfile_source) f.close() - run_command([GIT, "add", ".gitattributes"]) + files.append(".gitattributes") + run_command(GITS, ["add", "--"] + files) + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%s', but '%s' doesn't start with " + "prefix '%s'" % (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.7+) from +# This file was generated by 'versioneer.py' (0.16) from # revision-control system data, or from the parent directory name of an # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '%(version)s' -version_full = '%(full)s' -def get_versions(default={}, verbose=False): - return {'version': version_version, 'full': version_full} +import json +import sys + +version_json = ''' +%s +''' # END VERSION_JSON + +def get_versions(): + return json.loads(version_json) """ -DEFAULT = {"version": "unknown", "full": "unknown"} def versions_from_file(filename): - versions = {} + """Try to determine the version from _version.py if present.""" try: - f = open(filename) + with open(filename) as f: + contents = f.read() except EnvironmentError: - return versions - for line in f.readlines(): - mo = re.match("version_version = '([^']+)'", line) - if mo: - versions["version"] = mo.group(1) - mo = re.match("version_full = '([^']+)'", line) - if mo: - versions["full"] = mo.group(1) - f.close() - return versions + raise NotThisMethod("unable to read _version.py") + mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", + contents, re.M | re.S) + if not mo: + raise NotThisMethod("no version_json in _version.py") + return json.loads(mo.group(1)) + def write_to_version_file(filename, versions): - f = open(filename, "w") - f.write(SHORT_VERSION_PY % versions) - f.close() + """Write the given version number to the given _version.py file.""" + os.unlink(filename) + contents = json.dumps(versions, sort_keys=True, + indent=1, separators=(",", ": ")) + with open(filename, "w") as f: + f.write(SHORT_VERSION_PY % contents) + print("set %s to '%s'" % (filename, versions["version"])) -def get_best_versions(versionfile, tag_prefix, parentdir_prefix, - default=DEFAULT, verbose=False): - # returns dict with two keys: 'version' and 'full' - # - # extract version from first of _version.py, 'git describe', parentdir. - # This is meant to work for developers using a source checkout, for users - # of a tarball created by 'setup.py sdist', and for users of a - # tarball/zipball created by 'git archive' or github's download-from-tag - # feature. - - variables = get_expanded_variables(versionfile_source) - if variables: - ver = versions_from_expanded_variables(variables, tag_prefix) - if ver: - if verbose: print("got version from expanded variable %s" % ver) - return ver +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" - ver = versions_from_file(versionfile) - if ver: - if verbose: print("got version from file %s %s" % (versionfile, ver)) - return ver - ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) - if ver: - if verbose: print("got version from git %s" % ver) - return ver +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". - ver = versions_from_parentdir(parentdir_prefix, versionfile_source, verbose) - if ver: - if verbose: print("got version from parentdir %s" % ver) - return ver + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. - if verbose: print("got version from default %s" % ver) - return default - -def get_versions(default=DEFAULT, verbose=False): - assert versionfile_source is not None, "please set versioneer.versionfile_source" - assert tag_prefix is not None, "please set versioneer.tag_prefix" - assert parentdir_prefix is not None, "please set versioneer.parentdir_prefix" - return get_best_versions(versionfile_source, tag_prefix, parentdir_prefix, - default=default, verbose=verbose) -def get_version(verbose=False): - return get_versions(verbose=verbose)["version"] - -class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - def initialize_options(self): + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +class VersioneerBadRootError(Exception): + """The project root directory is unknown or missing key files.""" + + +def get_versions(verbose=False): + """Get the project version from whatever source is available. + + Returns dict with two keys: 'version' and 'full'. + """ + if "versioneer" in sys.modules: + # see the discussion in cmdclass.py:get_cmdclass() + del sys.modules["versioneer"] + + root = get_root() + cfg = get_config_from_root(root) + + assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" + handlers = HANDLERS.get(cfg.VCS) + assert handlers, "unrecognized VCS '%s'" % cfg.VCS + verbose = verbose or cfg.verbose + assert cfg.versionfile_source is not None, \ + "please set versioneer.versionfile_source" + assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" + + versionfile_abs = os.path.join(root, cfg.versionfile_source) + + # extract version from first of: _version.py, VCS command (e.g. 'git + # describe'), parentdir. This is meant to work for developers using a + # source checkout, for users of a tarball created by 'setup.py sdist', + # and for users of a tarball/zipball created by 'git archive' or github's + # download-from-tag feature or the equivalent in other VCSes. + + get_keywords_f = handlers.get("get_keywords") + from_keywords_f = handlers.get("keywords") + if get_keywords_f and from_keywords_f: + try: + keywords = get_keywords_f(versionfile_abs) + ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) + if verbose: + print("got version from expanded keyword %s" % ver) + return ver + except NotThisMethod: + pass + + try: + ver = versions_from_file(versionfile_abs) + if verbose: + print("got version from file %s %s" % (versionfile_abs, ver)) + return ver + except NotThisMethod: pass - def finalize_options(self): + + from_vcs_f = handlers.get("pieces_from_vcs") + if from_vcs_f: + try: + pieces = from_vcs_f(cfg.tag_prefix, root, verbose) + ver = render(pieces, cfg.style) + if verbose: + print("got version from VCS %s" % ver) + return ver + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + if verbose: + print("got version from parentdir %s" % ver) + return ver + except NotThisMethod: pass - def run(self): - ver = get_version(verbose=True) - print("Version is currently: %s" % ver) - - -class cmd_build(_build): - def run(self): - versions = get_versions(verbose=True) - _build.run(self) - # now locate _version.py in the new build/ directory and replace it - # with an updated value - target_versionfile = os.path.join(self.build_lib, versionfile_build) - print("UPDATING %s" % target_versionfile) - os.unlink(target_versionfile) - f = open(target_versionfile, "w") - f.write(SHORT_VERSION_PY % versions) - f.close() -class cmd_sdist(_sdist): - def run(self): - versions = get_versions(verbose=True) - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory (remembering - # that it may be a hardlink) and replace it with an updated value - target_versionfile = os.path.join(base_dir, versionfile_source) - print("UPDATING %s" % target_versionfile) - os.unlink(target_versionfile) - f = open(target_versionfile, "w") - f.write(SHORT_VERSION_PY % self._versioneer_generated_versions) - f.close() + if verbose: + print("unable to compute version") + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, "error": "unable to compute version"} + + +def get_version(): + """Get the short version string for this project.""" + return get_versions()["version"] + + +def get_cmdclass(): + """Get the custom setuptools/distutils subclasses used by Versioneer.""" + if "versioneer" in sys.modules: + del sys.modules["versioneer"] + # this fixes the "python setup.py develop" case (also 'install' and + # 'easy_install .'), in which subdependencies of the main project are + # built (using setup.py bdist_egg) in the same python process. Assume + # a main project A and a dependency B, which use different versions + # of Versioneer. A's setup.py imports A's Versioneer, leaving it in + # sys.modules by the time B's setup.py is executed, causing B to run + # with the wrong versioneer. Setuptools wraps the sub-dep builds in a + # sandbox that restores sys.modules to it's pre-build state, so the + # parent is protected against the child's "import versioneer". By + # removing ourselves from sys.modules here, before the child build + # happens, we protect the child from the parent's versioneer too. + # Also see https://github.com/warner/python-versioneer/issues/52 + + cmds = {} + + # we add "version" to both distutils and setuptools + from distutils.core import Command + + class cmd_version(Command): + description = "report generated version string" + user_options = [] + boolean_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + vers = get_versions(verbose=True) + print("Version: %s" % vers["version"]) + print(" full-revisionid: %s" % vers.get("full-revisionid")) + print(" dirty: %s" % vers.get("dirty")) + if vers["error"]: + print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version + + # we override "build_py" in both distutils and setuptools + # + # most invocation pathways end up running build_py: + # distutils/build -> build_py + # distutils/install -> distutils/build ->.. + # setuptools/bdist_wheel -> distutils/install ->.. + # setuptools/bdist_egg -> distutils/install_lib -> build_py + # setuptools/install -> bdist_egg ->.. + # setuptools/develop -> ? + + # we override different "build_py" commands for both environments + if "setuptools" in sys.modules: + from setuptools.command.build_py import build_py as _build_py + else: + from distutils.command.build_py import build_py as _build_py + + class cmd_build_py(_build_py): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + _build_py.run(self) + # now locate _version.py in the new build/ directory and replace + # it with an updated value + if cfg.versionfile_build: + target_versionfile = os.path.join(self.build_lib, + cfg.versionfile_build) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py + + if "cx_Freeze" in sys.modules: # cx_freeze enabled? + from cx_Freeze.dist import build_exe as _build_exe + + class cmd_build_exe(_build_exe): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + target_versionfile = cfg.versionfile_source + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + _build_exe.run(self) + os.unlink(target_versionfile) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % + {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + cmds["build_exe"] = cmd_build_exe + del cmds["build_py"] + + # we override different "sdist" commands for both environments + if "setuptools" in sys.modules: + from setuptools.command.sdist import sdist as _sdist + else: + from distutils.command.sdist import sdist as _sdist + + class cmd_sdist(_sdist): + def run(self): + versions = get_versions() + self._versioneer_generated_versions = versions + # unless we update this, the command will keep using the old + # version + self.distribution.metadata.version = versions["version"] + return _sdist.run(self) + + def make_release_tree(self, base_dir, files): + root = get_root() + cfg = get_config_from_root(root) + _sdist.make_release_tree(self, base_dir, files) + # now locate _version.py in the new base_dir directory + # (remembering that it may be a hardlink) and replace it with an + # updated value + target_versionfile = os.path.join(base_dir, cfg.versionfile_source) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, + self._versioneer_generated_versions) + cmds["sdist"] = cmd_sdist + + return cmds + + +CONFIG_ERROR = """ +setup.cfg is missing the necessary Versioneer configuration. You need +a section like: + + [versioneer] + VCS = git + style = pep440 + versionfile_source = src/myproject/_version.py + versionfile_build = myproject/_version.py + tag_prefix = + parentdir_prefix = myproject- + +You will also need to edit your setup.py to use the results: + + import versioneer + setup(version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), ...) + +Please read the docstring in ./versioneer.py for configuration instructions, +edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. +""" + +SAMPLE_CONFIG = """ +# See the docstring in versioneer.py for instructions. Note that you must +# re-run 'versioneer.py setup' after changing this section, and commit the +# resulting files. + +[versioneer] +#VCS = git +#style = pep440 +#versionfile_source = +#versionfile_build = +#tag_prefix = +#parentdir_prefix = + +""" INIT_PY_SNIPPET = """ from ._version import get_versions @@ -630,40 +1646,129 @@ __version__ = get_versions()['version'] del get_versions """ -class cmd_update_files(Command): - description = "modify __init__.py and create _version.py" - user_options = [] - boolean_options = [] - def initialize_options(self): - pass - def finalize_options(self): - pass - def run(self): - ipy = os.path.join(os.path.dirname(versionfile_source), "__init__.py") - print(" creating %s" % versionfile_source) - f = open(versionfile_source, "w") - f.write(LONG_VERSION_PY % {"DOLLAR": "$", - "TAG_PREFIX": tag_prefix, - "PARENTDIR_PREFIX": parentdir_prefix, - "VERSIONFILE_SOURCE": versionfile_source, - }) - f.close() + +def do_setup(): + """Main VCS-independent setup function for installing Versioneer.""" + root = get_root() + try: + cfg = get_config_from_root(root) + except (EnvironmentError, configparser.NoSectionError, + configparser.NoOptionError) as e: + if isinstance(e, (EnvironmentError, configparser.NoSectionError)): + print("Adding sample versioneer config to setup.cfg", + file=sys.stderr) + with open(os.path.join(root, "setup.cfg"), "a") as f: + f.write(SAMPLE_CONFIG) + print(CONFIG_ERROR, file=sys.stderr) + return 1 + + print(" creating %s" % cfg.versionfile_source) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), + "__init__.py") + if os.path.exists(ipy): try: - old = open(ipy, "r").read() + with open(ipy, "r") as f: + old = f.read() except EnvironmentError: old = "" if INIT_PY_SNIPPET not in old: print(" appending to %s" % ipy) - f = open(ipy, "a") - f.write(INIT_PY_SNIPPET) - f.close() + with open(ipy, "a") as f: + f.write(INIT_PY_SNIPPET) else: print(" %s unmodified" % ipy) - do_vcs_install(versionfile_source, ipy) + else: + print(" %s doesn't exist, ok" % ipy) + ipy = None + + # Make sure both the top-level "versioneer.py" and versionfile_source + # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so + # they'll be copied into source distributions. Pip won't be able to + # install the package without this. + manifest_in = os.path.join(root, "MANIFEST.in") + simple_includes = set() + try: + with open(manifest_in, "r") as f: + for line in f: + if line.startswith("include "): + for include in line.split()[1:]: + simple_includes.add(include) + except EnvironmentError: + pass + # That doesn't cover everything MANIFEST.in can do + # (http://docs.python.org/2/distutils/sourcedist.html#commands), so + # it might give some false negatives. Appending redundant 'include' + # lines is safe, though. + if "versioneer.py" not in simple_includes: + print(" appending 'versioneer.py' to MANIFEST.in") + with open(manifest_in, "a") as f: + f.write("include versioneer.py\n") + else: + print(" 'versioneer.py' already in MANIFEST.in") + if cfg.versionfile_source not in simple_includes: + print(" appending versionfile_source ('%s') to MANIFEST.in" % + cfg.versionfile_source) + with open(manifest_in, "a") as f: + f.write("include %s\n" % cfg.versionfile_source) + else: + print(" versionfile_source already in MANIFEST.in") -def get_cmdclass(): - return {'version': cmd_version, - 'update_files': cmd_update_files, - 'build': cmd_build, - 'sdist': cmd_sdist, - } + # Make VCS-specific changes. For git, this means creating/changing + # .gitattributes to mark _version.py for export-time keyword + # substitution. + do_vcs_install(manifest_in, cfg.versionfile_source, ipy) + return 0 + + +def scan_setup_py(): + """Validate the contents of setup.py against Versioneer's expectations.""" + found = set() + setters = False + errors = 0 + with open("setup.py", "r") as f: + for line in f.readlines(): + if "import versioneer" in line: + found.add("import") + if "versioneer.get_cmdclass()" in line: + found.add("cmdclass") + if "versioneer.get_version()" in line: + found.add("get_version") + if "versioneer.VCS" in line: + setters = True + if "versioneer.versionfile_source" in line: + setters = True + if len(found) != 3: + print("") + print("Your setup.py appears to be missing some important items") + print("(but I might be wrong). Please make sure it has something") + print("roughly like the following:") + print("") + print(" import versioneer") + print(" setup( version=versioneer.get_version(),") + print(" cmdclass=versioneer.get_cmdclass(), ...)") + print("") + errors += 1 + if setters: + print("You should remove lines like 'versioneer.VCS = ' and") + print("'versioneer.versionfile_source = ' . This configuration") + print("now lives in setup.cfg, and should be removed from setup.py") + print("") + errors += 1 + return errors + +if __name__ == "__main__": + cmd = sys.argv[1] + if cmd == "setup": + errors = do_setup() + errors += scan_setup_py() + if errors: + sys.exit(1) |