diff options
| author | Kali Kaneko <kali@leap.se> | 2016-04-18 12:04:34 -0400 |
|---|---|---|
| committer | Kali Kaneko <kali@leap.se> | 2016-04-18 12:04:34 -0400 |
| commit | 3a25464e92acd3fb0192c2d438a2c2122ca526fd (patch) | |
| tree | 531689dd646f174f3e9ce9acbdf0a621da827210 | |
| parent | 881cab15bd331c4a8f58d13c0a4e35ebfb324442 (diff) | |
| parent | e2a19eaf0ceca35acaedafb3796c66b562e825da (diff) | |
Merge tag '0.8.0'
Tag leap.mx version 0.8.0
# gpg: Signature made Mon 18 Apr 2016 12:03:24 PM BOT
# gpg: using RSA key 1CAF6C5B9F720808
# gpg: Good signature from "Kaliyuga <kaliyuga@riseup.net>" [ultimate]
# gpg: aka "Kali Kaneko (leap communications) <kali@leap.se>" [ultimate]
38 files changed, 2846 insertions, 728 deletions
@@ -9,6 +9,7 @@ dist build eggs +.eggs parts bin var @@ -51,3 +52,5 @@ nosetests.xml *.log *.conf *.config + +_trial_temp @@ -0,0 +1,9 @@ +Isis Agora Lovecruft <isis@torproject.org> +Tomás Touceda <chiiph@leap.se> +Kali Kaneko <kali@leap.se> +drebs <drebs@leap.se> +Ivan Alejandro <ivanalejandro0@gmail.com> +Micah Anderson <micah@riseup.net> +varac <varacanero@zeromail.org> +Bruno Wagner Goncalves <bwagner@thoughtworks.com> +Ruben Pollan <meskio@sindominio.net> diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..31fba49 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,17 @@ +0.8.0 - 18 Apr, 2016 ++++++++++++++++++++++++++++++++ + +Features +~~~~~~~~ +- `#4285 <https://leap.se/code/issues/4285>`_: Add postfix lookup against couchdb for client smtp fingerprint +- `#5959 <https://leap.se/code/issues/5959>`_: Make alias resolver to return *uuid@deliver.local* +- `#7998 <https://leap.se/code/issues/7998>`_: Bounce stalled emails after a timeout. + +Bugfixes +~~~~~~~~ +- `#7253 <https://leap.se/code/issues/7253>`_: Use the original message for encryption. +- `#7961 <https://leap.se/code/issues/7961>`_: Check if the account is enabled. + +Misc +~~~~ +- `#7271 <https://leap.se/code/issues/7271>`_: Document the return codes of the TCP maps. diff --git a/MANIFEST.in b/MANIFEST.in index 3b96cd4..f6e8824 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,3 +4,4 @@ include pkg/utils/* include versioneer.py include LICENSE include CHANGELOG +include src/leap/mx/_version.py @@ -33,6 +33,14 @@ $ python setup.py install $ twistd -ny pkg/mx.tac ~~~ +### Stalled emails + +In case of problems with couchdb and other unknown sources emails can get +stalled in the spool. There is a bouncing mechanism for long stalled emails, +after 5 days the email will get bounced. The timestamp of stalled emails is +hold in memory, restarting leap-mx will erase all timestamps and the stalled +timeout will be reset. + ## Hacking Please see the doc/DESIGN docs. diff --git a/changes/bug_6937_remove-syslog-option b/changes/bug_6937_remove-syslog-option new file mode 100644 index 0000000..d3a65c3 --- /dev/null +++ b/changes/bug_6937_remove-syslog-option @@ -0,0 +1,2 @@ + o Remove logging to syslog for now. We should only do this in the future, + when the platform is ready for it. Closes #6937. diff --git a/changes/feature_6942_use_syslog b/changes/feature_6942_use_syslog new file mode 100644 index 0000000..ffa8f62 --- /dev/null +++ b/changes/feature_6942_use_syslog @@ -0,0 +1 @@ +- Use syslog for logging (Closes: #6859) diff --git a/changes/feature_7272-msg-key-not-found b/changes/feature_7272-msg-key-not-found new file mode 100644 index 0000000..2d82df8 --- /dev/null +++ b/changes/feature_7272-msg-key-not-found @@ -0,0 +1 @@ +- return a more meaningful msg if user exists but has no key (Closes: #7272) diff --git a/changes/feature_7435_unit_testing b/changes/feature_7435_unit_testing new file mode 100644 index 0000000..32778b7 --- /dev/null +++ b/changes/feature_7435_unit_testing @@ -0,0 +1 @@ +- set up unit testing infrastructure (Closes: #7435) diff --git a/changes/feature_7439_remove_provenance b/changes/feature_7439_remove_provenance new file mode 100644 index 0000000..188b9a2 --- /dev/null +++ b/changes/feature_7439_remove_provenance @@ -0,0 +1 @@ +- Don't add X-Leap-Provenance header (Closes: #7439) diff --git a/changes/feature_7565_couchdb_refactor b/changes/feature_7565_couchdb_refactor new file mode 100644 index 0000000..dc6ac0b --- /dev/null +++ b/changes/feature_7565_couchdb_refactor @@ -0,0 +1 @@ +- Update code to the new CouchDatabase soledad code diff --git a/changes/next-changelog.txt b/changes/next-changelog.txt new file mode 100644 index 0000000..2e13a84 --- /dev/null +++ b/changes/next-changelog.txt @@ -0,0 +1,28 @@ +0.8.1 - xxx ++++++++++++++++++++++++++++++++ + +Please add lines to this file, they will be moved to the CHANGELOG.rst during +the next release. + +There are two template lines for each category, use them as reference. + +I've added a new category `Misc` so we can track doc/style/packaging stuff. + +Features +~~~~~~~~ +- `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234. +- New feature without related issue number. + +Bugfixes +~~~~~~~~ +- `#1235 <https://leap.se/code/issues/1235>`_: Description for the fixed stuff corresponding with issue #1235. +- Bugfix without related issue number. + +Misc +~~~~ +- `#1236 <https://leap.se/code/issues/1236>`_: Description of the new feature corresponding with issue #1236. +- Some change without issue number. + +Known Issues +~~~~~~~~~~~~ +- `#1236 <https://leap.se/code/issues/1236>`_: Description of the known issue corresponding with issue #1236. diff --git a/doc/DESIGN.md b/doc/DESIGN.md index e98976d..dbfbc99 100644 --- a/doc/DESIGN.md +++ b/doc/DESIGN.md @@ -145,6 +145,31 @@ virtual transport instead, we should append the domain (eg 123456@example.org). see http://www.postfix.org/ADDRESS_REWRITING_README.html#resolve +#### fingerprint_resolver + +postfix config: + +``` +virtual_alias_map tcp:localhost:2424 +``` + +postfix sends "get 12:34:56:78:90:ab:cd:ef:12:34:56:78:90:ab:cd:ef:12:34:56:78" +providing an smtp fingerprint and fingerprint_resolver returns "200 2016-01-19", +where 2016-01-19 is the expiration date of the given fingerprint. If the +fingerprint does not exists or is expired it will return "500 NOT FOUND SRY". + +#### Return values + +The return codes and content of the tcp maps are: + + +----------------------------------------------------------+ + | virtual_alias_map | check_recipient_access | ++----------------+---------------------+------------------------------------+ +| user not found | 500 "NOT FOUND SRY" | 500 "REJECT" | +| key not found | 200 "<uuid>" | 400 "4.7.13 USER ACCOUNT DISABLED" | +| both found | 200 "<uuid>" | 200 "OK" | ++----------------+---------------------+------------------------------------+ + ### Current status diff --git a/pkg/generate_wheels.sh b/pkg/generate_wheels.sh new file mode 100755 index 0000000..e8096af --- /dev/null +++ b/pkg/generate_wheels.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Generate wheels for dependencies + +if [ "$WHEELHOUSE" = "" ]; then + WHEELHOUSE=$HOME/wheelhouse +fi + +pip wheel --wheel-dir $WHEELHOUSE pip +pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements.pip +if [ -f pkg/requirements-testing.pip ]; then + pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip +fi diff --git a/pkg/leap_mx.init b/pkg/leap-mx.init index 60dddc4..8093c22 100644 --- a/pkg/leap_mx.init +++ b/pkg/leap-mx.init @@ -12,9 +12,10 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin PIDFILE=/var/run/leap_mx.pid RUNDIR=/var/lib/leap_mx/ -FILE=/usr/share/app/leap_mx.tac -LOGFILE=/var/log/leap_mx.log +FILE=/usr/share/app/mx.tac TWISTD_PATH=/usr/bin/twistd +USER=leap-mx +GROUP=leap-mx [ -r /etc/default/leap_mx ] && . /etc/default/leap_mx @@ -32,8 +33,10 @@ case "$1" in --pidfile=$PIDFILE \ --rundir=$RUNDIR \ --python=$FILE \ - --logfile=$LOGFILE \ - --syslog --prefix=leap-mx + --syslog \ + --prefix=leap-mx \ + --uid=$USER \ + --gid=$GROUP echo "." ;; diff --git a/pkg/leap-mx.service b/pkg/leap-mx.service new file mode 100644 index 0000000..5852ee8 --- /dev/null +++ b/pkg/leap-mx.service @@ -0,0 +1,10 @@ +[Unit] +Description=Leap MX +Before=postfix.service + +[Service] +ExecStart=/usr/bin/python /usr/bin/twistd -n --rundir=/var/lib/leap_mx/ --python=/usr/share/app/mx.tac --syslog --prefix=leap-mx --pidfile=/tmp/leap-mx.pid +User=leap-mx + +[Install] +WantedBy=multi-user.target diff --git a/pkg/mx.conf.sample b/pkg/mx.conf.sample index c9ad0f8..a649b73 100644 --- a/pkg/mx.conf.sample +++ b/pkg/mx.conf.sample @@ -14,6 +14,9 @@ port=4242 [check recipient] port=2244 +[fingerprint map] +port=2424 + [bounce] from=<address for the From: of the bounce email without domain> subject=Delivery failure
\ No newline at end of file @@ -24,6 +24,7 @@ from leap.mx import couchdbhelper from leap.mx.mail_receiver import MailReceiver from leap.mx.alias_resolver import AliasResolverFactory from leap.mx.check_recipient_access import CheckRecipientAccessFactory +from leap.mx.fingerprint_resolver import FingerprintResolverFactory try: from twisted.application import service, internet @@ -57,6 +58,7 @@ except ConfigParser.NoSectionError: alias_port = config.getint("alias map", "port") check_recipient_port = config.getint("check recipient", "port") +fingerprint_port = config.getint("fingerprint map", "port") cdb = couchdbhelper.ConnectedCouchDB(server, port=port, @@ -68,27 +70,32 @@ cdb = couchdbhelper.ConnectedCouchDB(server, application = service.Application("LEAP MX") # Alias map -alias_map = internet.TCPServer(alias_port, AliasResolverFactory(couchdb=cdb)) +alias_map = internet.TCPServer( + alias_port, AliasResolverFactory(couchdb=cdb), + interface="localhost") alias_map.setServiceParent(application) # Check recipient access -check_recipient = internet.TCPServer(check_recipient_port, - CheckRecipientAccessFactory(couchdb=cdb)) +check_recipient = internet.TCPServer( + check_recipient_port, CheckRecipientAccessFactory(couchdb=cdb), + interface="localhost") check_recipient.setServiceParent(application) +# Fingerprint map +fingerprint_map = internet.TCPServer( + fingerprint_port, FingerprintResolverFactory(couchdb=cdb), + interface="localhost") +fingerprint_map.setServiceParent(application) + # Mail receiver -mail_couch_url_prefix = "http://%s:%s@%s:%s" % (user, - password, - server, - port) directories = [] for section in config.sections(): - if section in ("couchdb", "alias map", "check recipient", "bounce"): + if section in ("couchdb", "alias map", "check recipient", + "fingerprint map", "bounce"): continue to_watch = config.get(section, "path") recursive = config.getboolean(section, "recursive") directories.append([to_watch, recursive]) -mr = MailReceiver(mail_couch_url_prefix, cdb, directories, bounce_from, - bounce_subject) +mr = MailReceiver(cdb, directories, bounce_from, bounce_subject) mr.setServiceParent(application) diff --git a/pkg/pip_install_requirements.sh b/pkg/pip_install_requirements.sh new file mode 100755 index 0000000..57732e2 --- /dev/null +++ b/pkg/pip_install_requirements.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Update pip and install LEAP base/testing requirements. +# For convenience, $insecure_packages are allowed with insecure flags enabled. +# Use at your own risk. +# See $usage for help + +insecure_packages="" +leap_wheelhouse=https://lizard.leap.se/wheels + +show_help() { + usage="Usage: $0 [--testing] [--use-leap-wheels]\n --testing\t\tInstall dependencies from requirements-testing.pip\n +\t\t\tOtherwise, it will install requirements.pip\n +--use-leap-wheels\tUse wheels from leap.se" + echo -e $usage + + exit 1 +} + +process_arguments() { + testing=false + use_leap_wheels=false + + while [ "$#" -gt 0 ]; do + # From http://stackoverflow.com/a/31443098 + case "$1" in + --help) show_help;; + --testing) testing=true; shift 1;; + --use-leap-wheels) use_leap_wheels=true; shift 1;; + + -h) show_help;; + -*) echo "unknown option: $1" >&2; exit 1;; + esac + done +} + +return_wheelhouse() { + if $use_leap_wheels ; then + WHEELHOUSE=$leap_wheelhouse + elif [ "$WHEELHOUSE" = "" ]; then + WHEELHOUSE=$HOME/wheelhouse + fi + + # Tested with bash and zsh + if [[ $WHEELHOUSE != http* && ! -d "$WHEELHOUSE" ]]; then + mkdir $WHEELHOUSE + fi + + echo "$WHEELHOUSE" +} + +return_install_options() { + wheelhouse=`return_wheelhouse` + install_options="-U --find-links=$wheelhouse" + if $use_leap_wheels ; then + install_options="$install_options --trusted-host lizard.leap.se" + fi + + echo $install_options +} + +return_insecure_flags() { + for insecure_package in $insecure_packages; do + flags="$flags --allow-external $insecure_package --allow-unverified $insecure_package" + done + + echo $flags +} + +return_packages() { + if $testing ; then + packages="-r pkg/requirements-testing.pip" + else + packages="-r pkg/requirements.pip" + fi + + echo $packages +} + +process_arguments $@ +install_options=`return_install_options` +insecure_flags=`return_insecure_flags` +packages=`return_packages` + +pip install -U wheel +pip install $install_options pip +pip install $install_options $insecure_flags $packages diff --git a/pkg/requirements-leap.pip b/pkg/requirements-leap.pip new file mode 100644 index 0000000..aab5fa5 --- /dev/null +++ b/pkg/requirements-leap.pip @@ -0,0 +1,3 @@ +leap.common>=0.5.1 +leap.soledad.common>=0.8.0 +leap.keymanager>=0.5.0 diff --git a/pkg/requirements-testing.pip b/pkg/requirements-testing.pip new file mode 100644 index 0000000..94b8e9c --- /dev/null +++ b/pkg/requirements-testing.pip @@ -0,0 +1,2 @@ +pep8 +setuptools-trial diff --git a/pkg/requirements.pip b/pkg/requirements.pip index ed3ad0d..328b1c3 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -7,7 +7,3 @@ paisley>=0.3.1 # in soledad-common, but we need to declare here # for the time being. couchdb - -leap.common>=0.3.5 -leap.soledad.common>=0.4.5 -leap.keymanager>=0.3.4 diff --git a/pkg/utils/get_authors.sh b/pkg/utils/get_authors.sh new file mode 100755 index 0000000..0169bb1 --- /dev/null +++ b/pkg/utils/get_authors.sh @@ -0,0 +1,2 @@ +#!/bin/sh +git log --format='%aN <%aE>' | awk '{arr[$0]++} END{for (i in arr){print arr[i], i;}}' | sort -rn | cut -d' ' -f2- diff --git a/pkg/utils/reqs.py b/pkg/utils/reqs.py index 5e2324f..251c7e9 100644 --- a/pkg/utils/reqs.py +++ b/pkg/utils/reqs.py @@ -22,6 +22,22 @@ import re import sys +def is_develop_mode(): + """ + Returns True if we're calling the setup script using the argument for + setuptools development mode. + + This avoids messing up with dependency pinning and order, the + responsibility of installing the leap dependencies is left to the + developer. + """ + args = sys.argv + devflags = "setup.py", "develop" + if (args[0], args[1]) == devflags: + return True + return False + + def get_reqs_from_files(reqfiles): """ Returns the contents of the top requirement file listed as a @@ -51,8 +67,8 @@ def parse_requirements(reqfiles=['requirements.txt', if re.match(r'\s*-e\s+', line): pass # do not try to do anything with externals on vcs - #requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', - #line)) + # requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', + # line)) # http://foo.bar/baz/foobar/zipball/master#egg=foobar elif re.match(r'\s*https?:', line): requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..d8a5cc1 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,17 @@ +[aliases] +test = trial + +[pep8] +exclude = versioneer.py,_version.py,*.egg,build,dist,docs +ignore = E731 + +[flake8] +exclude = versioneer.py,_version.py,*.egg,build,dist,docs +ignore = E731 + +[versioneer] +VCS = git +style = pep440 +versionfile_source = src/leap/mx/_version.py +versionfile_build = leap/mx/_version.py +tag_prefix = @@ -20,14 +20,12 @@ setup file for leap.mx import os import re from setuptools import setup, find_packages +from setuptools import Command + +from pkg.utils.reqs import parse_requirements, is_develop_mode import versioneer -versioneer.versionfile_source = 'src/leap/mx/_version.py' -versioneer.versionfile_build = 'leap/mx/_version.py' -versioneer.tag_prefix = '' # tags are like 1.2.0 -versioneer.parentdir_prefix = 'leap.mx-' -from pkg.utils.reqs import parse_requirements trove_classifiers = [ 'Development Status :: 3 - Alpha', @@ -44,11 +42,11 @@ trove_classifiers = [ 'Topic :: Security :: Cryptography', ] -DOWNLOAD_BASE = ('https://github.com/leapcode/leap_mx/' +DOWNLOAD_BASE = ('https://github.com/leapcode/bitmask_client/' 'archive/%s.tar.gz') _versions = versioneer.get_versions() VERSION = _versions['version'] -VERSION_FULL = _versions['full'] +VERSION_REVISION = _versions['full-revisionid'] DOWNLOAD_URL = "" # get the short version for the download url @@ -60,15 +58,29 @@ if len(_version_short) > 0: cmdclass = versioneer.get_cmdclass() -from setuptools import Command - - class freeze_debianver(Command): + """ Freezes the version in a debian branch. To be used after merging the development branch onto the debian one. """ user_options = [] + template = r""" +# This file was generated by the `freeze_debianver` command in setup.py +# Using 'versioneer.py' (0.16) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. + +version_version = '{version}' +full_revisionid = '{full_revisionid}' +""" + templatefun = r""" + +def get_versions(default={}, verbose=False): + return {'version': version_version, + 'full-revisionid': full_revisionid} +""" def initialize_options(self): pass @@ -82,25 +94,11 @@ class freeze_debianver(Command): if proceed != "y": print("He. You scared. Aborting.") return - template = r""" -# This file was generated by the `freeze_debianver` command in setup.py -# Using 'versioneer.py' (0.7+) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -version_version = '{version}' -version_full = '{version_full}' -""" - templatefun = r""" - -def get_versions(default={}, verbose=False): - return {'version': version_version, 'full': version_full} -""" - subst_template = template.format( + subst_template = self.template.format( version=VERSION_SHORT, - version_full=VERSION_FULL) + templatefun - with open(versioneer.versionfile_source, 'w') as f: + full_revisionid=VERSION_REVISION) + self.templatefun + versioneer_cfg = versioneer.get_config_from_root('.') + with open(versioneer_cfg.versionfile_source, 'w') as f: f.write(subst_template) @@ -113,7 +111,25 @@ else: # be automatically # placed by distutils, using whatever interpreter is # available. - data_files = [("/usr/local/bin/", ["pkg/mx.tac"])] + data_files = [("/usr/share/app/", ["pkg/mx.tac"])] + + +requirements = parse_requirements() + +if is_develop_mode(): + print + print ("[WARNING] Skipping leap-specific dependencies " + "because development mode is detected.") + print ("[WARNING] You can install " + "the latest published versions with " + "'pip install -r pkg/requirements-leap.pip'") + print ("[WARNING] Or you can instead do 'python setup.py develop' " + "from the parent folder of each one of them.") + print +else: + requirements += parse_requirements( + reqfiles=["pkg/requirements-leap.pip"]) + setup( name='leap.mx', version=VERSION, @@ -134,8 +150,10 @@ setup( namespace_packages=["leap"], package_dir={'': 'src'}, packages=find_packages('src'), - #test_suite='leap.mx.tests', - install_requires=parse_requirements(), + test_suite='leap.mx.tests', + tests_require=parse_requirements( + reqfiles=['pkg/requirements-testing.pip']), + install_requires=requirements, classifiers=trove_classifiers, data_files=data_files ) diff --git a/src/leap/mx/_version.py b/src/leap/mx/_version.py index 2367de5..ed9fc86 100644 --- a/src/leap/mx/_version.py +++ b/src/leap/mx/_version.py @@ -1,74 +1,157 @@ -IN_LONG_VERSION_PY = True # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (build by setup.py sdist) and build +# feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.7+ (https://github.com/warner/python-versioneer) - -# these strings will be replaced by git during git-archive -git_refnames = "$Format:%d$" -git_full = "$Format:%H$" +# versioneer-0.16 (https://github.com/warner/python-versioneer) +"""Git implementation of _version.py.""" +import errno +import os +import re import subprocess import sys -def run_command(args, cwd=None, verbose=False): - try: - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError: - e = sys.exc_info()[1] + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "$Format:%d$" + git_full = "$Format:%H$" + keywords = {"refnames": git_refnames, "full": git_full} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "pep440" + cfg.tag_prefix = "" + cfg.parentdir_prefix = "None" + cfg.versionfile_source = "src/leap/mx/_version.py" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None + else: if verbose: - print("unable to run %s" % args[0]) - print(e) + print("unable to find command, tried %s" % (commands,)) return None stdout = p.communicate()[0].strip() - if sys.version >= '3': + if sys.version_info[0] >= 3: stdout = stdout.decode() if p.returncode != 0: if verbose: - print("unable to run %s (error)" % args[0]) + print("unable to run %s (error)" % dispcmd) return None return stdout -import sys -import re -import os.path +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%s', but '%s' doesn't start with " + "prefix '%s'" % (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} -def get_expanded_variables(versionfile_source): + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these - # variables. When used from setup.py, we don't want to import - # _version.py, so we do it with a regexp instead. This function is not - # used from _version.py. - variables = {} + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} try: - f = open(versionfile_source,"r") + f = open(versionfile_abs, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["refnames"] = mo.group(1) + keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["full"] = mo.group(1) + keywords["full"] = mo.group(1) f.close() except EnvironmentError: pass - return variables + return keywords + -def versions_from_expanded_variables(variables, tag_prefix, verbose=False): - refnames = variables["refnames"].strip() +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: - print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -93,111 +176,309 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): r = ref[len(tag_prefix):] if verbose: print("picking %s" % r) - return { "version": r, - "full": variables["full"].strip() } - # no suitable tags, so we use the full revision id + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: - print("no suitable tags, using full revision id") - return { "version": variables["full"].strip(), - "full": variables["full"].strip() } - -def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): - # this runs 'git' from the root of the source tree. That either means - # someone ran a setup.py command (and this code is in versioneer.py, so - # IN_LONG_VERSION_PY=False, thus the containing directory is the root of - # the source tree), or someone ran a project-specific entry point (and - # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the - # containing directory is somewhere deeper in the source tree). This only - # gets called if the git-archive 'subst' variables were *not* expanded, - # and _version.py hasn't already been rewritten with a short version - # string, meaning we're inside a checked out source tree. + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} - try: - here = os.path.abspath(__file__) - except NameError: - # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct - - # versionfile_source is the relative path from the top of the source tree - # (where the .git directory might live) to this file. Invert this to find - # the root from __file__. - root = here - if IN_LONG_VERSION_PY: - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) - else: - root = os.path.dirname(here) + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ if not os.path.exists(os.path.join(root, ".git")): if verbose: print("no .git in %s" % root) - return {} + raise NotThisMethod("no .git directory") - GIT = "git" + GITS = ["git"] if sys.platform == "win32": - GIT = "git.cmd" - stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], - cwd=root) - if stdout is None: - return {} - if not stdout.startswith(tag_prefix): - if verbose: - print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix)) - return {} - tag = stdout[len(tag_prefix):] - stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) - if stdout is None: - return {} - full = stdout.strip() - if tag.endswith("-dirty"): - full += "-dirty" - return {"version": tag, "full": full} - - -def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False): - if IN_LONG_VERSION_PY: - # We're running from _version.py. If it's from a source tree - # (execute-in-place), we can work upwards to find the root of the - # tree, and then check the parent directory for a version string. If - # it's in an installed application, there's no hope. - try: - here = os.path.abspath(__file__) - except NameError: - # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) # versionfile_source is the relative path from the top of the source - # tree to _version.py. Invert this to find the root from __file__. - root = here - for i in range(len(versionfile_source.split("/"))): + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): root = os.path.dirname(root) - else: - # we're running from versioneer.py, which means we're running from - # the setup.py in a source tree. sys.argv[0] is setup.py in the root. - here = os.path.abspath(sys.argv[0]) - root = os.path.dirname(here) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree"} - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" % - (root, dirname, parentdir_prefix)) - return None - return {"version": dirname[len(parentdir_prefix):], "full": ""} - -tag_prefix = "" -parentdir_prefix = "leap.mx-" -versionfile_source = "src/leap/mx/_version.py" - -def get_versions(default={"version": "unknown", "full": ""}, verbose=False): - variables = { "refnames": git_refnames, "full": git_full } - ver = versions_from_expanded_variables(variables, tag_prefix, verbose) - if not ver: - ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) - if not ver: - ver = versions_from_parentdir(parentdir_prefix, versionfile_source, - verbose) - if not ver: - ver = default - return ver + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version"} diff --git a/src/leap/mx/alias_resolver.py b/src/leap/mx/alias_resolver.py index bf7a58b..a5b5107 100644 --- a/src/leap/mx/alias_resolver.py +++ b/src/leap/mx/alias_resolver.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- encoding: utf-8 -*- # alias_resolver.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013, 2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -60,6 +60,7 @@ class LEAPPostfixTCPMapAliasServer(postfix.PostfixTCPMapServer): TCP_MAP_CODE_PERMANENT_FAILURE, postfix.quote("NOT FOUND SRY")) else: + uuid += "@deliver.local" # properly encode uuid, otherwise twisted complains when replying if isinstance(uuid, unicode): uuid = uuid.encode("utf8") diff --git a/src/leap/mx/check_recipient_access.py b/src/leap/mx/check_recipient_access.py index f994e78..67bfd04 100644 --- a/src/leap/mx/check_recipient_access.py +++ b/src/leap/mx/check_recipient_access.py @@ -43,7 +43,6 @@ class LEAPPostFixTCPMapAccessServer(postfix.PostfixTCPMapServer): are looked up by the factory, and will return a permanent or a temporary failure in case either the user or the key don't exist, respectivelly. """ - def _cbGot(self, value): """ Return a code and message depending on the result of the factory's @@ -65,7 +64,7 @@ class LEAPPostFixTCPMapAccessServer(postfix.PostfixTCPMapServer): elif pubkey is None: self.sendCode( TCP_MAP_CODE_TEMPORARY_FAILURE, - postfix.quote("4.7.13 USER ACCOUNT DISABLED")) + postfix.quote("4.7.13 NO PUBKEY FOUND")) else: self.sendCode( TCP_MAP_CODE_SUCCESS, @@ -85,4 +84,3 @@ class CheckRecipientAccessFactory(LEAPPostfixTCPMapServerFactory): @property def _query_message(self): return "check recipient access" - diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py index 1752b4e..e9cf4a4 100644 --- a/src/leap/mx/couchdbhelper.py +++ b/src/leap/mx/couchdbhelper.py @@ -23,7 +23,9 @@ maps, user UUIDs, and GPG keyIDs. from paisley import client +from twisted.internet import defer from twisted.python import log +from leap.soledad.common.couch import CouchDatabase class ConnectedCouchDB(client.CouchDB): @@ -50,6 +52,10 @@ class ConnectedCouchDB(client.CouchDB): :param str password: (optional) The password for authorization. :type password: str """ + self._mail_couch_url = "http://%s:%s@%s:%s" % (username, + password, + host, + port) client.CouchDB.__init__(self, host, port=port, @@ -94,9 +100,10 @@ class ConnectedCouchDB(client.CouchDB): pubkey = None if result["rows"]: doc = result["rows"][0]["doc"] - uuid = doc["user_id"] - if "keys" in doc: - pubkey = doc["keys"]["pgp"] + if doc["enabled"]: + uuid = doc["user_id"] + if "keys" in doc: + pubkey = doc["keys"]["pgp"] return uuid, pubkey d.addCallback(_get_uuid_and_pubkey_cbk) @@ -131,3 +138,60 @@ class ConnectedCouchDB(client.CouchDB): d.addCallbacks(_get_pubkey_cbk, log.err) return d + + def getCertExpiry(self, fingerprint): + """ + Query couch and return a deferred that will fire with the expiration + date for the cert with the given fingerprint. + + :param fingerprint: The cert fingerprint + :type fingerprint: str + + :return: A deferred that will fire with the cert expiration date as a + str. + :rtype: Deferred + """ + d = self.openView(docId="Identity", + viewId="cert_expiry_by_fingerprint/", + key=fingerprint, + reduce=False, + include_docs=True) + + def _get_cert_expiry_cbk(result): + try: + expiry = result["rows"][0]["value"] + except (KeyError, IndexError): + expiry = None + return expiry + + d.addCallback(_get_cert_expiry_cbk) + return d + + def put_doc(self, uuid, doc): + """ + Update a document. + + If the document currently has conflicts, put will fail. + If the database specifies a maximum document size and the document + exceeds it, put will fail and raise a DocumentTooBig exception. + + :param uuid: The uuid of a user + :type uuid: str + :param doc: A Document with new content. + :type doc: leap.soledad.common.couch.CouchDocument + + :return: A deferred which fires with the new revision identifier for + the document if the Document object has being updated, or + which fails with CouchDBError if there was any error. + """ + # TODO: that should be implemented with paisley + url = self._mail_couch_url + "/user-%s" % (uuid,) + try: + db = CouchDatabase.open_database(url, create=False) + return defer.succeed(db.put_doc(doc)) + except Exception as e: + return defer.fail(CouchDBError(e.message)) + + +class CouchDBError(Exception): + pass diff --git a/src/leap/mx/fingerprint_resolver.py b/src/leap/mx/fingerprint_resolver.py new file mode 100644 index 0000000..0a0850d --- /dev/null +++ b/src/leap/mx/fingerprint_resolver.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# fingerprint_resolver.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +Classes for resolve expiration date of certs. + +Test this with postmap -v -q "fingerprint" tcp:localhost:2424 +""" + + +from datetime import datetime +from twisted.internet.protocol import ServerFactory +from twisted.protocols import postfix +from twisted.python import log + +from leap.mx.tcp_map import TCP_MAP_CODE_SUCCESS +from leap.mx.tcp_map import TCP_MAP_CODE_PERMANENT_FAILURE + + +class LEAPPostfixTCPMapFingerprintServer(postfix.PostfixTCPMapServer): + """ + A postfix tcp map fingerprint resolver server. + """ + + def _cbGot(self, res): + """ + Return a code and message depending on the result of the factory's + get(). + + :param res: The fingerprint and expiration date of the cert + :type res: (str, str) + """ + fingerprint, expiry = (None, None) + if res is not None: + fingerprint, expiry = res + + if expiry is None: + code = TCP_MAP_CODE_PERMANENT_FAILURE + msg = "NOT FOUND SRY" + elif expiry < datetime.utcnow().strftime("%Y-%m-%d"): + code = TCP_MAP_CODE_PERMANENT_FAILURE + msg = "EXPIRED CERT" + else: + # properly encode expiry, otherwise twisted complains when replying + if isinstance(expiry, unicode): + expiry = expiry.encode("utf8") + code = TCP_MAP_CODE_SUCCESS + msg = fingerprint + " " + expiry + + self.sendCode(code, postfix.quote(msg)) + + +class FingerprintResolverFactory(ServerFactory, object): + """ + A factory for postfix tcp map fingerprint resolver servers. + """ + + protocol = LEAPPostfixTCPMapFingerprintServer + + def __init__(self, couchdb): + """ + Initialize the factory. + + :param couchdb: A CouchDB client. + :type couchdb: leap.mx.couchdbhelper.ConnectedCouchDB + """ + self._cdb = couchdb + + def get(self, fingerprint): + """ + Look up the cert expiration date based on fingerprint. + + :param fingerprint: The cert fingerprint. + :type fingerprint: str + + :return: A deferred that will be fired with the expiration date. + :rtype: Deferred + """ + log.msg("look up: %s" % (fingerprint,)) + d = self._cdb.getCertExpiry(fingerprint.lower()) + d.addCallback(lambda expiry: (fingerprint, expiry)) + d.addErrback(log.err) + return d diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 446fd38..7c5a368 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- encoding: utf-8 -*- # mail_receiver.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013, 2015 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -40,6 +40,7 @@ import signal import json import email.utils +from datetime import datetime, timedelta from email import message_from_string from twisted.application.service import Service, IService @@ -51,7 +52,7 @@ from zope.interface import implements from leap.soledad.common.crypto import EncryptionSchemes from leap.soledad.common.crypto import ENC_JSON_KEY from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.common.couch import CouchDatabase, CouchDocument +from leap.soledad.common.document import ServerDocument from leap.keymanager import openpgp @@ -75,15 +76,16 @@ class MailReceiver(Service): """ RETRY_DIR_WATCH_DELAY = 60 * 5 # 5 minutes - def __init__(self, mail_couch_url, users_cdb, directories, bounce_from, + """ + Time delta to keep stalled emails + """ + MAX_BOUNCE_DELTA = timedelta(days=5) + + def __init__(self, users_cdb, directories, bounce_from, bounce_subject): """ Constructor - :param mail_couch_url: URL prefix for the couchdb where mail - should be stored - :type mail_couch_url: str - :param users_cdb: CouchDB instance from where to get the uuid and pubkey for a user :type users_cdb: ConnectedCouchDB @@ -98,11 +100,11 @@ class MailReceiver(Service): :type bounce_subject: str """ # IService doesn't define an __init__ - self._mail_couch_url = mail_couch_url self._users_cdb = users_cdb self._directories = directories self._bounce_from = bounce_from self._bounce_subject = bounce_subject + self._bounce_timestamp = {} self._processing_skipped = False def startService(self): @@ -171,24 +173,21 @@ class MailReceiver(Service): :param pubkey: public key for the owner of the message :type pubkey: str :param message: message contents - :type message: email.message.Message + :type message: str :return: doc to sync with Soledad or None, None if something went wrong. - :rtype: CouchDocument + :rtype: ServerDocument """ if pubkey is None or len(pubkey) == 0: log.msg("_encrypt_message: Something went wrong, here's all " "I know: %r" % (pubkey,)) return None - # find message's encoding - message_as_string = message.as_string() - - doc = CouchDocument(doc_id=str(pyuuid.uuid4())) + doc = ServerDocument(doc_id=str(pyuuid.uuid4())) # store plain text if pubkey is not available - data = {'incoming': True, 'content': message_as_string} + data = {'incoming': True, 'content': message} if pubkey is None or len(pubkey) == 0: doc.content = { self.INCOMING_KEY: True, @@ -203,16 +202,6 @@ class MailReceiver(Service): with openpgp.TempGPGWrapper(gpgbinary='/usr/bin/gpg') as gpg: gpg.import_keys(pubkey) key = gpg.list_keys().pop() - - # add X-Leap-Provenance header if message is not encrypted - if message.get_content_type() != 'multipart/encrypted' and \ - '-----BEGIN PGP MESSAGE-----' not in \ - message_as_string: - message.add_header( - 'X-Leap-Provenance', - email.utils.formatdate(), - pubkey=key["keyid"]) - data = {'incoming': True, 'content': message.as_string()} doc.content = { self.INCOMING_KEY: True, self.ERROR_DECRYPTING_KEY: False, @@ -225,55 +214,44 @@ class MailReceiver(Service): return doc + @defer.inlineCallbacks def _export_message(self, uuid, doc): """ - Given a UUID and a CouchDocument, it saves it directly in the + Given a UUID and a ServerDocument, it saves it directly in the couchdb that serves as a backend for Soledad, in a db accessible to the recipient of the mail. :param uuid: the mail owner's uuid :type uuid: str - :param doc: CouchDocument that represents the email - :type doc: CouchDocument + :param doc: ServerDocument that represents the email + :type doc: ServerDocument - :return: True if it's ok to remove the message, False - otherwise - :rtype: bool + :return: A Deferred which fires if it's ok to remove the message, + or fails otherwise + :rtype: Deferred """ if uuid is None or doc is None: log.msg("_export_message: Something went wrong, here's all " "I know: %r | %r" % (uuid, doc)) - return False + raise Exception("No uuid or doc") log.msg("Exporting message for %s" % (uuid,)) - - db = CouchDatabase(self._mail_couch_url, "user-%s" % (uuid,)) - db.put_doc(doc) - + yield self._users_cdb.put_doc(uuid, doc) log.msg("Done exporting") - return True - - def _conditional_remove(self, do_remove, filepath): + def _remove(self, filepath): """ - Removes the message if do_remove is True. + Removes the message. - :param do_remove: True if the message should be removed, False - otherwise - :type do_remove: bool :param filepath: path to the mail :type filepath: twisted.python.filepath.FilePath """ - if do_remove: - # remove the original mail - try: - log.msg("Removing %r" % (filepath.path,)) - filepath.remove() - log.msg("Done removing") - except Exception: - log.err() - else: - log.msg("Not removing %r" % (filepath.path,)) + try: + log.msg("Removing %r" % (filepath.path,)) + filepath.remove() + log.msg("Done removing") + except Exception: + log.err() def _get_owner(self, mail): """ @@ -295,7 +273,7 @@ class MailReceiver(Service): return None final_address = delivereds.pop(0) _, addr = email.utils.parseaddr(final_address) - uuid, _ = addr.split("@") + uuid = addr.split("@")[0] return uuid @defer.inlineCallbacks @@ -317,7 +295,7 @@ class MailReceiver(Service): except InvalidReturnPathError: # give up bouncing this message! log.msg("Will not bounce message because of invalid return path.") - yield self._conditional_remove(True, filepath) + yield self._remove(filepath) def sleep(self, secs): """ @@ -395,10 +373,6 @@ class MailReceiver(Service): defer.returnValue(None) log.msg("Mail owner: %s" % (uuid,)) - if uuid is None: - log.msg("BUG: There was no uuid!") - defer.returnValue(None) - pubkey = yield self._users_cdb.getPubkey(uuid) if pubkey is None or len(pubkey) == 0: log.msg( @@ -411,10 +385,31 @@ class MailReceiver(Service): defer.returnValue(None) log.msg("Encrypting message to %s's pubkey" % (uuid,)) - doc = yield self._encrypt_message(pubkey, msg) + try: + doc = yield self._encrypt_message(pubkey, mail_data) - do_remove = yield self._export_message(uuid, doc) - yield self._conditional_remove(do_remove, filepath) + yield self._export_message(uuid, doc) + yield self._remove(filepath) + except Exception as e: + yield self._bounce_with_timeout(filepath, msg, e) + + @defer.inlineCallbacks + def _bounce_with_timeout(self, filepath, msg, error): + if filepath not in self._bounce_timestamp: + self._bounce_timestamp[filepath] = datetime.now() + log.msg("New stalled email {0!r}: {1!r}".format(filepath, error)) + defer.returnValue(None) + + current_delta = datetime.now() - self._bounce_timestamp[filepath] + if current_delta > self.MAX_BOUNCE_DELTA: + log.msg("Bouncing stalled email {0!r}: {1!r}" + .format(filepath, error)) + bounce_reason = "There was a problem in the server and the " \ + "email could not be delivered." + yield self._bounce_message(msg, filepath, bounce_reason) + else: + log.msg("Still stalled email {0!r} for the last {1}: {2!r}" + .format(filepath, str(current_delta), error)) @defer.inlineCallbacks def _process_incoming_email(self, otherself, filepath, mask): @@ -440,4 +435,3 @@ class MailReceiver(Service): except Exception as e: log.msg("Something went wrong while processing {0!r}: {1!r}" .format(filepath, e)) - log.err() diff --git a/src/leap/mx/tcp_map.py b/src/leap/mx/tcp_map.py index 96db70a..07bf51d 100644 --- a/src/leap/mx/tcp_map.py +++ b/src/leap/mx/tcp_map.py @@ -41,7 +41,6 @@ class LEAPPostfixTCPMapServerFactory(ServerFactory, object): __metaclass__ = ABCMeta - def __init__(self, couchdb): """ Initialize the factory. diff --git a/src/leap/mx/tests/__init__.py b/src/leap/mx/tests/__init__.py index 2002c48..13df919 100644 --- a/src/leap/mx/tests/__init__.py +++ b/src/leap/mx/tests/__init__.py @@ -22,6 +22,7 @@ code, using twisted.trial, for testing leap_mx. __all__ = ['test_alias_resolver'] + def run(): """xxx fill me in""" pass diff --git a/src/leap/mx/tests/test_mail_receiver.py b/src/leap/mx/tests/test_mail_receiver.py new file mode 100644 index 0000000..33967ea --- /dev/null +++ b/src/leap/mx/tests/test_mail_receiver.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# test_mail_receiver.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +MailReceiver tests +""" + +import codecs +import json +import os +import os.path +import shutil +import tempfile + +from email.message import Message +from twisted.internet import defer, reactor +from twisted.trial import unittest + +from leap.keymanager import openpgp +from leap.mx.couchdbhelper import CouchDBError +from leap.mx.mail_receiver import MailReceiver + + +BOUNCE_ADDRESS = "bounce@leap.se" +BOUNCE_SUBJECT = "bounce subject" +ADDRESS = "leap@leap.se" +UUID = "13d5203bdd09be1e638bdb1d315251cb" + + +class MailReceiverTestCase(unittest.TestCase): + def setUp(self): + self.directory = tempfile.mkdtemp(prefix="leap_tests-") + os.mkdir(os.path.join(self.directory, "new")) + + self.users_cdb = self.usersCdb() + self.receiver = MailReceiver( + users_cdb=self.users_cdb, + directories=[(self.directory, True)], + bounce_from=BOUNCE_ADDRESS, + bounce_subject=BOUNCE_SUBJECT) + self.receiver.startService() + + def tearDown(self): + self.receiver.stopService() + shutil.rmtree(self.directory) + + def usersCdb(self): + self.pubKey = PUBLIC_KEY + self.docs = [] + self.defer_put_doc = defer.Deferred() + + class UsersCdb(object): + def getPubkey(_, uuid): + return self.pubKey + + def put_doc(_, uuid, doc): + self.docs.append({'uuid': uuid, 'doc': doc}) + if not self.defer_put_doc.called: + reactor.callLater(1, self.defer_put_doc.callback, + (uuid, doc)) + return defer.succeed(None) + + return UsersCdb() + + @defer.inlineCallbacks + def test_single_mail(self): + msg, path = self.addMail("foo bar") + uuid, doc = yield self.defer_put_doc + self.assertEqual(uuid, UUID) + decmsg = self.decryptDoc(doc) + self.assertEqual(msg, decmsg) + self.assertFalse(os.path.exists(path)) + + @defer.inlineCallbacks + def test_put_doc_raises(self): + defer_called = defer.Deferred() + + def put_doc_raise(*args): + defer_called.callback(None) + return defer.fail(CouchDBError()) + + self.users_cdb.put_doc = put_doc_raise + _, path = self.addMail() + yield defer_called + self.assertTrue(os.path.exists(path)) + + @defer.inlineCallbacks + def test_misleading_encoding(self): + msg, path = self.addMail( + "ñáûä", headers={'Content-Transfer-Encoding': '7Bit'}) + uuid, doc = yield self.defer_put_doc + self.assertEqual(uuid, UUID) + decmsg = self.decryptDoc(doc) + self.assertEqual(unicode(msg, "utf-8"), decmsg) + self.assertFalse(os.path.exists(path)) + + def addMail(self, body="", filename="foo", to=ADDRESS, + frm="someone@domain.org", subject="sent subject", + headers={}): + msg = Message() + msg.add_header("To", to) + msg.add_header( + "Delivered-To", UUID + "@deliver.local") + msg.add_header("From", frm) + msg.add_header("Subject", subject) + for header, value in headers.iteritems(): + msg.add_header(header, value) + msg.set_payload(body) + + path = os.path.join(self.directory, "new", filename) + with open(path, "w") as f: + f.write(msg.as_string()) + + return msg.as_string(), path + + def decryptDoc(self, doc): + encdoc = doc.content['_enc_json'] + decdoc = {} + + with openpgp.TempGPGWrapper(gpgbinary='/usr/bin/gpg') as gpg: + gpg.import_keys(PRIVATE_KEY) + decstr = gpg.decrypt(encdoc) + decdoc = json.loads(decstr.data) + + self.assertTrue(decdoc['incoming']) + return decdoc['content'] + + +# key 24D18DDF: public key "Leap Test Key <leap@leap.se>" +KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF" +PUBLIC_KEY = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +mQINBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz +iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO +zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx +irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT +huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs +d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g +wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb +hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv +U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H +T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i +Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB +tBxMZWFwIFRlc3QgS2V5IDxsZWFwQGxlYXAuc2U+iQI3BBMBCAAhBQJQvfnZAhsD +BQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEC9FXigk0Y3fT7EQAKH3IuRniOpb +T/DDIgwwjz3oxB/W0DDMyPXowlhSOuM0rgGfntBpBb3boezEXwL86NPQxNGGruF5 +hkmecSiuPSvOmQlqlS95NGQp6hNG0YaKColh+Q5NTspFXCAkFch9oqUje0LdxfSP +QfV9UpeEvGyPmk1I9EJV/YDmZ4+Djge1d7qhVZInz4Rx1NrSyF/Tc2EC0VpjQFsU +Y9Kb2YBBR7ivG6DBc8ty0jJXi7B4WjkFcUEJviQpMF2dCLdonCehYs1PqsN1N7j+ +eFjQd+hqVMJgYuSGKjvuAEfClM6MQw7+FmFwMyLgK/Ew/DttHEDCri77SPSkOGSI +txCzhTg6798f6mJr7WcXmHX1w1Vcib5FfZ8vTDFVhz/XgAgArdhPo9V6/1dgSSiB +KPQ/spsco6u5imdOhckERE0lnAYvVT6KE81TKuhF/b23u7x+Wdew6kK0EQhYA7wy +7LmlaNXc7rMBQJ9Z60CJ4JDtatBWZ0kNrt2VfdDHVdqBTOpl0CraNUjWE5YMDasr +K2dF5IX8D3uuYtpZnxqg0KzyLg0tzL0tvOL1C2iudgZUISZNPKbS0z0v+afuAAnx +2pTC3uezbh2Jt8SWTLhll4i0P4Ps5kZ6HQUO56O+/Z1cWovX+mQekYFmERySDR9n +3k1uAwLilJmRmepGmvYbB8HloV8HqwgguQINBFC9+dkBEAC0I/xn1uborMgDvBtf +H0sEhwnXBC849/32zic6udB6/3Efk9nzbSpL3FSOuXITZsZgCHPkKarnoQ2ztMcS +sh1ke1C5gQGms75UVmM/nS+2YI4vY8OX/GC/on2vUyncqdH+bR6xH5hx4NbWpfTs +iQHmz5C6zzS/kuabGdZyKRaZHt23WQ7JX/4zpjqbC99DjHcP9BSk7tJ8wI4bkMYD +uFVQdT9O6HwyKGYwUU4sAQRAj7XCTGvVbT0dpgJwH4RmrEtJoHAx4Whg8mJ710E0 +GCmzf2jqkNuOw76ivgk27Kge+Hw00jmJjQhHY0yVbiaoJwcRrPKzaSjEVNgrpgP3 +lXPRGQArgESsIOTeVVHQ8fhK2YtTeCY9rIiO+L0OX2xo9HK7hfHZZWL6rqymXdyS +fhzh/f6IPyHFWnvj7Brl7DR8heMikygcJqv+ed2yx7iLyCUJ10g12I48+aEj1aLe +dP7lna32iY8/Z0SHQLNH6PXO9SlPcq2aFUgKqE75A/0FMk7CunzU1OWr2ZtTLNO1 +WT/13LfOhhuEq9jTyTosn0WxBjJKq18lnhzCXlaw6EAtbA7CUwsD3CTPR56aAXFK +3I7KXOVAqggrvMe5Tpdg5drfYpI8hZovL5aAgb+7Y5ta10TcJdUhS5K3kFAWe/td +U0cmWUMDP1UMSQ5Jg6JIQVWhSwARAQABiQIfBBgBCAAJBQJQvfnZAhsMAAoJEC9F +Xigk0Y3fRwsP/i0ElYCyxeLpWJTwo1iCLkMKz2yX1lFVa9nT1BVTPOQwr/IAc5OX +NdtbJ14fUsKL5pWgW8OmrXtwZm1y4euI1RPWWubG01ouzwnGzv26UcuHeqC5orZj +cOnKtL40y8VGMm8LoicVkRJH8blPORCnaLjdOtmA3rx/v2EXrJpSa3AhOy0ZSRXk +ZSrK68AVNwamHRoBSYyo0AtaXnkPX4+tmO8X8BPfj125IljubvwZPIW9VWR9UqCE +VPfDR1XKegVb6VStIywF7kmrknM1C5qUY28rdZYWgKorw01hBGV4jTW0cqde3N51 +XT1jnIAa+NoXUM9uQoGYMiwrL7vNsLlyyiW5ayDyV92H/rIuiqhFgbJsHTlsm7I8 +oGheR784BagAA1NIKD1qEO9T6Kz9lzlDaeWS5AUKeXrb7ZJLI1TTCIZx5/DxjLqM +Tt/RFBpVo9geZQrvLUqLAMwdaUvDXC2c6DaCPXTh65oCZj/hqzlJHH+RoTWWzKI+ +BjXxgUWF9EmZUBrg68DSmI+9wuDFsjZ51BcqvJwxyfxtTaWhdoYqH/UQS+D1FP3/ +diZHHlzwVwPICzM9ooNTgbrcDzyxRkIVqsVwBq7EtzcvgYUyX53yG25Giy6YQaQ2 +ZtQ/VymwFL3XdUWV6B/hU4PVAFvO3qlOtdJ6TpE+nEWgcWjCv5g7RjXX +=MuOY +-----END PGP PUBLIC KEY BLOCK----- +""" +PRIVATE_KEY = """ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQcYBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz +iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO +zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx +irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT +huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs +d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g +wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb +hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv +U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H +T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i +Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB +AA/+JHtlL39G1wsH9R6UEfUQJGXR9MiIiwZoKcnRB2o8+DS+OLjg0JOh8XehtuCs +E/8oGQKtQqa5bEIstX7IZoYmYFiUQi9LOzIblmp2vxOm+HKkxa4JszWci2/ZmC3t +KtaA4adl9XVnshoQ7pijuCMUKB3naBEOAxd8s9d/JeReGIYkJErdrnVfNk5N71Ds +FmH5Ll3XtEDvgBUQP3nkA6QFjpsaB94FHjL3gDwum/cxzj6pCglcvHOzEhfY0Ddb +J967FozQTaf2JW3O+w3LOqtcKWpq87B7+O61tVidQPSSuzPjCtFF0D2LC9R/Hpky +KTMQ6CaKja4MPhjwywd4QPcHGYSqjMpflvJqi+kYIt8psUK/YswWjnr3r4fbuqVY +VhtiHvnBHQjz135lUqWvEz4hM3Xpnxydx7aRlv5NlevK8+YIO5oFbWbGNTWsPZI5 +jpoFBpSsnR1Q5tnvtNHauvoWV+XN2qAOBTG+/nEbDYH6Ak3aaE9jrpTdYh0CotYF +q7csANsDy3JvkAzeU6WnYpsHHaAjqOGyiZGsLej1UcXPFMosE/aUo4WQhiS8Zx2c +zOVKOi/X5vQ2GdNT9Qolz8AriwzsvFR+bxPzyd8V6ALwDsoXvwEYinYBKK8j0OPv +OOihSR6HVsuP9NUZNU9ewiGzte/+/r6pNXHvR7wTQ8EWLcEIAN6Zyrb0bHZTIlxt +VWur/Ht2mIZrBaO50qmM5RD3T5oXzWXi/pjLrIpBMfeZR9DWfwQwjYzwqi7pxtYx +nJvbMuY505rfnMoYxb4J+cpRXV8MS7Dr1vjjLVUC9KiwSbM3gg6emfd2yuA93ihv +Pe3mffzLIiQa4mRE3wtGcioC43nWuV2K2e1KjxeFg07JhrezA/1Cak505ab/tmvP +4YmjR5c44+yL/YcQ3HdFgs4mV+nVbptRXvRcPpolJsgxPccGNdvHhsoR4gwXMS3F +RRPD2z6x8xeN73Q4KH3bm01swQdwFBZbWVfmUGLxvN7leCdfs9+iFJyqHiCIB6Iv +mQfp8F0IAOwSo8JhWN+V1dwML4EkIrM8wUb4yecNLkyR6TpPH/qXx4PxVMC+vy6x +sCtjeHIwKE+9vqnlhd5zOYh7qYXEJtYwdeDDmDbL8oks1LFfd+FyAuZXY33DLwn0 +cRYsr2OEZmaajqUB3NVmj3H4uJBN9+paFHyFSXrH68K1Fk2o3n+RSf2EiX+eICwI +L6rqoF5sSVUghBWdNegV7qfy4anwTQwrIMGjgU5S6PKW0Dr/3iO5z3qQpGPAj5OW +ATqPWkDICLbObPxD5cJlyyNE2wCA9VVc6/1d6w4EVwSq9h3/WTpATEreXXxTGptd +LNiTA1nmakBYNO2Iyo3djhaqBdWjk+EIAKtVEnJH9FAVwWOvaj1RoZMA5DnDMo7e +SnhrCXl8AL7Z1WInEaybasTJXn1uQ8xY52Ua4b8cbuEKRKzw/70NesFRoMLYoHTO +dyeszvhoDHberpGRTciVmpMu7Hyi33rM31K9epA4ib6QbbCHnxkWOZB+Bhgj1hJ8 +xb4RBYWiWpAYcg0+DAC3w9gfxQhtUlZPIbmbrBmrVkO2GVGUj8kH6k4UV6kUHEGY +HQWQR0HcbKcXW81ZXCCD0l7ROuEWQtTe5Jw7dJ4/QFuqZnPutXVRNOZqpl6eRShw +7X2/a29VXBpmHA95a88rSQsL+qm7Fb3prqRmuMCtrUZgFz7HLSTuUMR867QcTGVh +cCBUZXN0IEtleSA8bGVhcEBsZWFwLnNlPokCNwQTAQgAIQUCUL352QIbAwULCQgH +AwUVCgkICwUWAgMBAAIeAQIXgAAKCRAvRV4oJNGN30+xEACh9yLkZ4jqW0/wwyIM +MI896MQf1tAwzMj16MJYUjrjNK4Bn57QaQW926HsxF8C/OjT0MTRhq7heYZJnnEo +rj0rzpkJapUveTRkKeoTRtGGigqJYfkOTU7KRVwgJBXIfaKlI3tC3cX0j0H1fVKX +hLxsj5pNSPRCVf2A5mePg44HtXe6oVWSJ8+EcdTa0shf03NhAtFaY0BbFGPSm9mA +QUe4rxugwXPLctIyV4uweFo5BXFBCb4kKTBdnQi3aJwnoWLNT6rDdTe4/nhY0Hfo +alTCYGLkhio77gBHwpTOjEMO/hZhcDMi4CvxMPw7bRxAwq4u+0j0pDhkiLcQs4U4 +Ou/fH+pia+1nF5h19cNVXIm+RX2fL0wxVYc/14AIAK3YT6PVev9XYEkogSj0P7Kb +HKOruYpnToXJBERNJZwGL1U+ihPNUyroRf29t7u8flnXsOpCtBEIWAO8Muy5pWjV +3O6zAUCfWetAieCQ7WrQVmdJDa7dlX3Qx1XagUzqZdAq2jVI1hOWDA2rKytnReSF +/A97rmLaWZ8aoNCs8i4NLcy9Lbzi9QtornYGVCEmTTym0tM9L/mn7gAJ8dqUwt7n +s24dibfElky4ZZeItD+D7OZGeh0FDuejvv2dXFqL1/pkHpGBZhEckg0fZ95NbgMC +4pSZkZnqRpr2GwfB5aFfB6sIIJ0HGARQvfnZARAAtCP8Z9bm6KzIA7wbXx9LBIcJ +1wQvOPf99s4nOrnQev9xH5PZ820qS9xUjrlyE2bGYAhz5Cmq56ENs7THErIdZHtQ +uYEBprO+VFZjP50vtmCOL2PDl/xgv6J9r1Mp3KnR/m0esR+YceDW1qX07IkB5s+Q +us80v5LmmxnWcikWmR7dt1kOyV/+M6Y6mwvfQ4x3D/QUpO7SfMCOG5DGA7hVUHU/ +Tuh8MihmMFFOLAEEQI+1wkxr1W09HaYCcB+EZqxLSaBwMeFoYPJie9dBNBgps39o +6pDbjsO+or4JNuyoHvh8NNI5iY0IR2NMlW4mqCcHEazys2koxFTYK6YD95Vz0RkA +K4BErCDk3lVR0PH4StmLU3gmPayIjvi9Dl9saPRyu4Xx2WVi+q6spl3ckn4c4f3+ +iD8hxVp74+wa5ew0fIXjIpMoHCar/nndsse4i8glCddINdiOPPmhI9Wi3nT+5Z2t +9omPP2dEh0CzR+j1zvUpT3KtmhVICqhO+QP9BTJOwrp81NTlq9mbUyzTtVk/9dy3 +zoYbhKvY08k6LJ9FsQYySqtfJZ4cwl5WsOhALWwOwlMLA9wkz0eemgFxStyOylzl +QKoIK7zHuU6XYOXa32KSPIWaLy+WgIG/u2ObWtdE3CXVIUuSt5BQFnv7XVNHJllD +Az9VDEkOSYOiSEFVoUsAEQEAAQAP/1AagnZQZyzHDEgw4QELAspYHCWLXE5aZInX +wTUJhK31IgIXNn9bJ0hFiSpQR2xeMs9oYtRuPOu0P8oOFMn4/z374fkjZy8QVY3e +PlL+3EUeqYtkMwlGNmVw5a/NbNuNfm5Darb7pEfbYd1gPcni4MAYw7R2SG/57GbC +9gucvspHIfOSfBNLBthDzmK8xEKe1yD2eimfc2T7IRYb6hmkYfeds5GsqvGI6mwI +85h4uUHWRc5JOlhVM6yX8hSWx0L60Z3DZLChmc8maWnFXd7C8eQ6P1azJJbW71Ih +7CoK0XW4LE82vlQurSRFgTwfl7wFYszW2bOzCuhHDDtYnwH86Nsu0DC78ZVRnvxn +E8Ke/AJgrdhIOo4UAyR+aZD2+2mKd7/waOUTUrUtTzc7i8N3YXGi/EIaNReBXaq+ +ZNOp24BlFzRp+FCF/pptDW9HjPdiV09x0DgICmeZS4Gq/4vFFIahWctg52NGebT0 +Idxngjj+xDtLaZlLQoOz0n5ByjO/Wi0ANmMv1sMKCHhGvdaSws2/PbMR2r4caj8m +KXpIgdinM/wUzHJ5pZyF2U/qejsRj8Kw8KH/tfX4JCLhiaP/mgeTuWGDHeZQERAT +xPmRFHaLP9/ZhvGNh6okIYtrKjWTLGoXvKLHcrKNisBLSq+P2WeFrlme1vjvJMo/ +jPwLT5o9CADQmcbKZ+QQ1ZM9v99iDZol7SAMZX43JC019sx6GK0u6xouJBcLfeB4 +OXacTgmSYdTa9RM9fbfVpti01tJ84LV2SyL/VJq/enJF4XQPSynT/tFTn1PAor6o +tEAAd8fjKdJ6LnD5wb92SPHfQfXqI84rFEO8rUNIE/1ErT6DYifDzVCbfD2KZdoF +cOSp7TpD77sY1bs74ocBX5ejKtd+aH99D78bJSMM4pSDZsIEwnomkBHTziubPwJb +OwnATy0LmSMAWOw5rKbsh5nfwCiUTM20xp0t5JeXd+wPVWbpWqI2EnkCEN+RJr9i +7dp/ymDQ+Yt5wrsN3NwoyiexPOG91WQVCADdErHsnglVZZq9Z8Wx7KwecGCUurJ2 +H6lKudv5YOxPnAzqZS5HbpZd/nRTMZh2rdXCr5m2YOuewyYjvM757AkmUpM09zJX +MQ1S67/UX2y8/74TcRF97Ncx9HeELs92innBRXoFitnNguvcO6Esx4BTe1OdU6qR +ER3zAmVf22Le9ciXbu24DN4mleOH+OmBx7X2PqJSYW9GAMTsRB081R6EWKH7romQ +waxFrZ4DJzZ9ltyosEJn5F32StyLrFxpcrdLUoEaclZCv2qka7sZvi0EvovDVEBU +e10jOx9AOwf8Gj2ufhquQ6qgVYCzbP+YrodtkFrXRS3IsljIchj1M2ffB/0bfoUs +rtER9pLvYzCjBPg8IfGLw0o754Qbhh/ReplCRTusP/fQMybvCvfxreS3oyEriu/G +GufRomjewZ8EMHDIgUsLcYo2UHZsfF7tcazgxMGmMvazp4r8vpgrvW/8fIN/6Adu +tF+WjWDTvJLFJCe6O+BFJOWrssNrrra1zGtLC1s8s+Wfpe+bGPL5zpHeebGTwH1U +22eqgJArlEKxrfarz7W5+uHZJHSjF/K9ZvunLGD0n9GOPMpji3UO3zeM8IYoWn7E +/EWK1XbjnssNemeeTZ+sDh+qrD7BOi+vCX1IyBxbfqnQfJZvmcPWpruy1UsO+aIC +0GY8Jr3OL69dDQ21jueJAh8EGAEIAAkFAlC9+dkCGwwACgkQL0VeKCTRjd9HCw/+ +LQSVgLLF4ulYlPCjWIIuQwrPbJfWUVVr2dPUFVM85DCv8gBzk5c121snXh9Swovm +laBbw6ate3BmbXLh64jVE9Za5sbTWi7PCcbO/bpRy4d6oLmitmNw6cq0vjTLxUYy +bwuiJxWREkfxuU85EKdouN062YDevH+/YResmlJrcCE7LRlJFeRlKsrrwBU3BqYd +GgFJjKjQC1peeQ9fj62Y7xfwE9+PXbkiWO5u/Bk8hb1VZH1SoIRU98NHVcp6BVvp +VK0jLAXuSauSczULmpRjbyt1lhaAqivDTWEEZXiNNbRyp17c3nVdPWOcgBr42hdQ +z25CgZgyLCsvu82wuXLKJblrIPJX3Yf+si6KqEWBsmwdOWybsjygaF5HvzgFqAAD +U0goPWoQ71PorP2XOUNp5ZLkBQp5etvtkksjVNMIhnHn8PGMuoxO39EUGlWj2B5l +Cu8tSosAzB1pS8NcLZzoNoI9dOHrmgJmP+GrOUkcf5GhNZbMoj4GNfGBRYX0SZlQ +GuDrwNKYj73C4MWyNnnUFyq8nDHJ/G1NpaF2hiof9RBL4PUU/f92JkceXPBXA8gL +Mz2ig1OButwPPLFGQhWqxXAGrsS3Ny+BhTJfnfIbbkaLLphBpDZm1D9XKbAUvdd1 +RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc= +=JTFu +-----END PGP PRIVATE KEY BLOCK----- +""" diff --git a/src/leap/mx/tester.py b/src/leap/mx/tests/tester.py index 05d2d05..05d2d05 100644 --- a/src/leap/mx/tester.py +++ b/src/leap/mx/tests/tester.py diff --git a/versioneer.py b/versioneer.py index 34e4807..7ed2a21 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1,170 +1,641 @@ -#! /usr/bin/python -"""versioneer.py +# Version: 0.16 -(like a rocketeer, but for versions) +"""The Versioneer - like a rocketeer, but for versions. +The Versioneer +============== + +* like a rocketeer, but for versions! * https://github.com/warner/python-versioneer * Brian Warner * License: Public Domain -* Version: 0.7+ - -This file helps distutils-based projects manage their version number by just -creating version-control tags. - -For developers who work from a VCS-generated tree (e.g. 'git clone' etc), -each 'setup.py version', 'setup.py build', 'setup.py sdist' will compute a -version number by asking your version-control tool about the current -checkout. The version number will be written into a generated _version.py -file of your choosing, where it can be included by your __init__.py - -For users who work from a VCS-generated tarball (e.g. 'git archive'), it will -compute a version number by looking at the name of the directory created when -te tarball is unpacked. This conventionally includes both the name of the -project and a version number. - -For users who work from a tarball built by 'setup.py sdist', it will get a -version number from a previously-generated _version.py file. - -As a result, loading code directly from the source tree will not result in a -real version. If you want real versions from VCS trees (where you frequently -update from the upstream repository, or do new development), you will need to -do a 'setup.py version' after each update, and load code from the build/ -directory. - -You need to provide this code with a few configuration values: - - versionfile_source: - A project-relative pathname into which the generated version strings - should be written. This is usually a _version.py next to your project's - main __init__.py file. If your project uses src/myproject/__init__.py, - this should be 'src/myproject/_version.py'. This file should be checked - in to your VCS as usual: the copy created below by 'setup.py - update_files' will include code that parses expanded VCS keywords in - generated tarballs. The 'build' and 'sdist' commands will replace it with - a copy that has just the calculated version string. - - versionfile_build: - Like versionfile_source, but relative to the build directory instead of - the source directory. These will differ when your setup.py uses - 'package_dir='. If you have package_dir={'myproject': 'src/myproject'}, - then you will probably have versionfile_build='myproject/_version.py' and - versionfile_source='src/myproject/_version.py'. - - tag_prefix: a string, like 'PROJECTNAME-', which appears at the start of all - VCS tags. If your tags look like 'myproject-1.2.0', then you - should use tag_prefix='myproject-'. If you use unprefixed tags - like '1.2.0', this should be an empty string. - - parentdir_prefix: a string, frequently the same as tag_prefix, which - appears at the start of all unpacked tarball filenames. If - your tarball unpacks into 'myproject-1.2.0', this should - be 'myproject-'. - -To use it: - - 1: include this file in the top level of your project - 2: make the following changes to the top of your setup.py: - import versioneer - versioneer.versionfile_source = 'src/myproject/_version.py' - versioneer.versionfile_build = 'myproject/_version.py' - versioneer.tag_prefix = '' # tags are like 1.2.0 - versioneer.parentdir_prefix = 'myproject-' # dirname like 'myproject-1.2.0' - 3: add the following arguments to the setup() call in your setup.py: - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), - 4: run 'setup.py update_files', which will create _version.py, and will - modify your __init__.py to define __version__ (by calling a function - from _version.py) - 5: modify your MANIFEST.in to include versioneer.py - 6: add both versioneer.py and the generated _version.py to your VCS -""" +* Compatible With: python2.6, 2.7, 3.3, 3.4, 3.5, and pypy +* [![Latest Version] +(https://pypip.in/version/versioneer/badge.svg?style=flat) +](https://pypi.python.org/pypi/versioneer/) +* [![Build Status] +(https://travis-ci.org/warner/python-versioneer.png?branch=master) +](https://travis-ci.org/warner/python-versioneer) + +This is a tool for managing a recorded version number in distutils-based +python projects. The goal is to remove the tedious and error-prone "update +the embedded version string" step from your release process. Making a new +release should be as easy as recording a new tag in your version-control +system, and maybe making new tarballs. + + +## Quick Install + +* `pip install versioneer` to somewhere to your $PATH +* add a `[versioneer]` section to your setup.cfg (see below) +* run `versioneer install` in your source tree, commit the results + +## Version Identifiers + +Source trees come from a variety of places: + +* a version-control system checkout (mostly used by developers) +* a nightly tarball, produced by build automation +* a snapshot tarball, produced by a web-based VCS browser, like github's + "tarball from tag" feature +* a release tarball, produced by "setup.py sdist", distributed through PyPI + +Within each source tree, the version identifier (either a string or a number, +this tool is format-agnostic) can come from a variety of places: + +* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows + about recent "tags" and an absolute revision-id +* the name of the directory into which the tarball was unpacked +* an expanded VCS keyword ($Id$, etc) +* a `_version.py` created by some earlier build step + +For released software, the version identifier is closely related to a VCS +tag. Some projects use tag names that include more than just the version +string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool +needs to strip the tag prefix to extract the version identifier. For +unreleased software (between tags), the version identifier should provide +enough information to help developers recreate the same tree, while also +giving them an idea of roughly how old the tree is (after version 1.2, before +version 1.3). Many VCS systems can report a description that captures this, +for example `git describe --tags --dirty --always` reports things like +"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the +0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has +uncommitted changes. + +The version identifier is used for multiple purposes: + +* to allow the module to self-identify its version: `myproject.__version__` +* to choose a name and prefix for a 'setup.py sdist' tarball + +## Theory of Operation + +Versioneer works by adding a special `_version.py` file into your source +tree, where your `__init__.py` can import it. This `_version.py` knows how to +dynamically ask the VCS tool for version information at import time. + +`_version.py` also contains `$Revision$` markers, and the installation +process marks `_version.py` to have this marker rewritten with a tag name +during the `git archive` command. As a result, generated tarballs will +contain enough information to get the proper version. + +To allow `setup.py` to compute a version too, a `versioneer.py` is added to +the top level of your source tree, next to `setup.py` and the `setup.cfg` +that configures it. This overrides several distutils/setuptools commands to +compute the version when invoked, and changes `setup.py build` and `setup.py +sdist` to replace `_version.py` with a small static file that contains just +the generated version data. + +## Installation + +First, decide on values for the following configuration variables: + +* `VCS`: the version control system you use. Currently accepts "git". + +* `style`: the style of version string to be produced. See "Styles" below for + details. Defaults to "pep440", which looks like + `TAG[+DISTANCE.gSHORTHASH[.dirty]]`. + +* `versionfile_source`: + + A project-relative pathname into which the generated version strings should + be written. This is usually a `_version.py` next to your project's main + `__init__.py` file, so it can be imported at runtime. If your project uses + `src/myproject/__init__.py`, this should be `src/myproject/_version.py`. + This file should be checked in to your VCS as usual: the copy created below + by `setup.py setup_versioneer` will include code that parses expanded VCS + keywords in generated tarballs. The 'build' and 'sdist' commands will + replace it with a copy that has just the calculated version string. + + This must be set even if your project does not have any modules (and will + therefore never import `_version.py`), since "setup.py sdist" -based trees + still need somewhere to record the pre-calculated version strings. Anywhere + in the source tree should do. If there is a `__init__.py` next to your + `_version.py`, the `setup.py setup_versioneer` command (described below) + will append some `__version__`-setting assignments, if they aren't already + present. + +* `versionfile_build`: + + Like `versionfile_source`, but relative to the build directory instead of + the source directory. These will differ when your setup.py uses + 'package_dir='. If you have `package_dir={'myproject': 'src/myproject'}`, + then you will probably have `versionfile_build='myproject/_version.py'` and + `versionfile_source='src/myproject/_version.py'`. + + If this is set to None, then `setup.py build` will not attempt to rewrite + any `_version.py` in the built tree. If your project does not have any + libraries (e.g. if it only builds a script), then you should use + `versionfile_build = None`. To actually use the computed version string, + your `setup.py` will need to override `distutils.command.build_scripts` + with a subclass that explicitly inserts a copy of + `versioneer.get_version()` into your script file. See + `test/demoapp-script-only/setup.py` for an example. + +* `tag_prefix`: + + a string, like 'PROJECTNAME-', which appears at the start of all VCS tags. + If your tags look like 'myproject-1.2.0', then you should use + tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this + should be an empty string, using either `tag_prefix=` or `tag_prefix=''`. + +* `parentdir_prefix`: + + a optional string, frequently the same as tag_prefix, which appears at the + start of all unpacked tarball filenames. If your tarball unpacks into + 'myproject-1.2.0', this should be 'myproject-'. To disable this feature, + just omit the field from your `setup.cfg`. + +This tool provides one script, named `versioneer`. That script has one mode, +"install", which writes a copy of `versioneer.py` into the current directory +and runs `versioneer.py setup` to finish the installation. + +To versioneer-enable your project: + +* 1: Modify your `setup.cfg`, adding a section named `[versioneer]` and + populating it with the configuration values you decided earlier (note that + the option names are not case-sensitive): + + ```` + [versioneer] + VCS = git + style = pep440 + versionfile_source = src/myproject/_version.py + versionfile_build = myproject/_version.py + tag_prefix = + parentdir_prefix = myproject- + ```` + +* 2: Run `versioneer install`. This will do the following: + + * copy `versioneer.py` into the top of your source tree + * create `_version.py` in the right place (`versionfile_source`) + * modify your `__init__.py` (if one exists next to `_version.py`) to define + `__version__` (by calling a function from `_version.py`) + * modify your `MANIFEST.in` to include both `versioneer.py` and the + generated `_version.py` in sdist tarballs + + `versioneer install` will complain about any problems it finds with your + `setup.py` or `setup.cfg`. Run it multiple times until you have fixed all + the problems. + +* 3: add a `import versioneer` to your setup.py, and add the following + arguments to the setup() call: + + version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), + +* 4: commit these changes to your VCS. To make sure you won't forget, + `versioneer install` will mark everything it touched for addition using + `git add`. Don't forget to add `setup.py` and `setup.cfg` too. + +## Post-Installation Usage + +Once established, all uses of your tree from a VCS checkout should get the +current version string. All generated tarballs should include an embedded +version string (so users who unpack them will not need a VCS tool installed). + +If you distribute your project through PyPI, then the release process should +boil down to two steps: + +* 1: git tag 1.0 +* 2: python setup.py register sdist upload + +If you distribute it through github (i.e. users use github to generate +tarballs with `git archive`), the process is: + +* 1: git tag 1.0 +* 2: git push; git push --tags + +Versioneer will report "0+untagged.NUMCOMMITS.gHASH" until your tree has at +least one tag in its history. + +## Version-String Flavors + +Code which uses Versioneer can learn about its version string at runtime by +importing `_version` from your main `__init__.py` file and running the +`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can +import the top-level `versioneer.py` and run `get_versions()`. + +Both functions return a dictionary with different flavors of version +information: + +* `['version']`: A condensed version string, rendered using the selected + style. This is the most commonly used value for the project's version + string. The default "pep440" style yields strings like `0.11`, + `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section + below for alternative styles. + +* `['full-revisionid']`: detailed revision identifier. For Git, this is the + full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". + +* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that + this is only accurate if run in a VCS checkout, otherwise it is likely to + be False or None + +* `['error']`: if the version string could not be computed, this will be set + to a string describing the problem, otherwise it will be None. It may be + useful to throw an exception in setup.py if this is set, to avoid e.g. + creating tarballs with a version string of "unknown". + +Some variants are more useful than others. Including `full-revisionid` in a +bug report should allow developers to reconstruct the exact code being tested +(or indicate the presence of local changes that should be shared with the +developers). `version` is suitable for display in an "about" box or a CLI +`--version` output: it can be easily compared against release notes and lists +of bugs fixed in various releases. + +The installer adds the following text to your `__init__.py` to place a basic +version in `YOURPROJECT.__version__`: + + from ._version import get_versions + __version__ = get_versions()['version'] + del get_versions + +## Styles + +The setup.cfg `style=` configuration controls how the VCS information is +rendered into a version string. + +The default style, "pep440", produces a PEP440-compliant string, equal to the +un-prefixed tag name for actual releases, and containing an additional "local +version" section with more detail for in-between builds. For Git, this is +TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags +--dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the +tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and +that this commit is two revisions ("+2") beyond the "0.11" tag. For released +software (exactly equal to a known tag), the identifier will only contain the +stripped tag, e.g. "0.11". + +Other styles are available. See details.md in the Versioneer source tree for +descriptions. + +## Debugging + +Versioneer tries to avoid fatal errors: if something goes wrong, it will tend +to return a version of "0+unknown". To investigate the problem, run `setup.py +version`, which will run the version-lookup code in a verbose mode, and will +display the full contents of `get_versions()` (including the `error` string, +which may help identify what went wrong). + +## Updating Versioneer + +To upgrade your project to a new release of Versioneer, do the following: + +* install the new Versioneer (`pip install -U versioneer` or equivalent) +* edit `setup.cfg`, if necessary, to include any new configuration settings + indicated by the release notes +* re-run `versioneer install` in your source tree, to replace + `SRC/_version.py` +* commit any changed files + +### Upgrading to 0.16 + +Nothing special. + +### Upgrading to 0.15 + +Starting with this version, Versioneer is configured with a `[versioneer]` +section in your `setup.cfg` file. Earlier versions required the `setup.py` to +set attributes on the `versioneer` module immediately after import. The new +version will refuse to run (raising an exception during import) until you +have provided the necessary `setup.cfg` section. + +In addition, the Versioneer package provides an executable named +`versioneer`, and the installation process is driven by running `versioneer +install`. In 0.14 and earlier, the executable was named +`versioneer-installer` and was run without an argument. + +### Upgrading to 0.14 -import os, sys, re -from distutils.core import Command -from distutils.command.sdist import sdist as _sdist -from distutils.command.build import build as _build +0.14 changes the format of the version string. 0.13 and earlier used +hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a +plus-separated "local version" section strings, with dot-separated +components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old +format, but should be ok with the new one. -versionfile_source = None -versionfile_build = None -tag_prefix = None -parentdir_prefix = None +### Upgrading from 0.11 to 0.12 -VCS = "git" -IN_LONG_VERSION_PY = False +Nothing special. +### Upgrading from 0.10 to 0.11 -LONG_VERSION_PY = ''' -IN_LONG_VERSION_PY = True +You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running +`setup.py setup_versioneer`. This will enable the use of additional +version-control systems (SVN, etc) in the future. + +## Future Directions + +This tool is designed to make it easily extended to other version-control +systems: all VCS-specific components are in separate directories like +src/git/ . The top-level `versioneer.py` script is assembled from these +components by running make-versioneer.py . In the future, make-versioneer.py +will take a VCS name as an argument, and will construct a version of +`versioneer.py` that is specific to the given VCS. It might also take the +configuration arguments that are currently provided manually during +installation by editing setup.py . Alternatively, it might go the other +direction and include code from all supported VCS systems, reducing the +number of intermediate scripts. + + +## License + +To make Versioneer easier to embed, all its code is dedicated to the public +domain. The `_version.py` that it creates is also in the public domain. +Specifically, both are released under the Creative Commons "Public Domain +Dedication" license (CC0-1.0), as described in +https://creativecommons.org/publicdomain/zero/1.0/ . + +""" + +from __future__ import print_function +try: + import configparser +except ImportError: + import ConfigParser as configparser +import errno +import json +import os +import re +import subprocess +import sys + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_root(): + """Get the project root directory. + + We require that all commands are run from the project root, i.e. the + directory that contains setup.py, setup.cfg, and versioneer.py . + """ + root = os.path.realpath(os.path.abspath(os.getcwd())) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + # allow 'python path/to/setup.py COMMAND' + root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + err = ("Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND').") + raise VersioneerBadRootError(err) + try: + # Certain runtime workflows (setup.py install/develop in a setuptools + # tree) execute all dependencies in a single python process, so + # "versioneer" may be imported multiple times, and python's shared + # module-import table will cache the first one. So we can't use + # os.path.dirname(__file__), as that will find whichever + # versioneer.py was first imported, even in later projects. + me = os.path.realpath(os.path.abspath(__file__)) + if os.path.splitext(me)[0] != os.path.splitext(versioneer_py)[0]: + print("Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py)) + except NameError: + pass + return root + + +def get_config_from_root(root): + """Read the project setup.cfg file to determine Versioneer config.""" + # This might raise EnvironmentError (if setup.cfg is missing), or + # configparser.NoSectionError (if it lacks a [versioneer] section), or + # configparser.NoOptionError (if it lacks "VCS="). See the docstring at + # the top of versioneer.py for instructions on writing your setup.cfg . + setup_cfg = os.path.join(root, "setup.cfg") + parser = configparser.SafeConfigParser() + with open(setup_cfg, "r") as f: + parser.readfp(f) + VCS = parser.get("versioneer", "VCS") # mandatory + + def get(parser, name): + if parser.has_option("versioneer", name): + return parser.get("versioneer", name) + return None + cfg = VersioneerConfig() + cfg.VCS = VCS + cfg.style = get(parser, "style") or "" + cfg.versionfile_source = get(parser, "versionfile_source") + cfg.versionfile_build = get(parser, "versionfile_build") + cfg.tag_prefix = get(parser, "tag_prefix") + if cfg.tag_prefix in ("''", '""'): + cfg.tag_prefix = "" + cfg.parentdir_prefix = get(parser, "parentdir_prefix") + cfg.verbose = get(parser, "verbose") + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + +# these dictionaries contain VCS-specific tools +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + return None + return stdout +LONG_VERSION_PY['git'] = ''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (build by setup.py sdist) and build +# feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.7+ (https://github.com/warner/python-versioneer) - -# these strings will be replaced by git during git-archive -git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" -git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" +# versioneer-0.16 (https://github.com/warner/python-versioneer) +"""Git implementation of _version.py.""" +import errno +import os +import re import subprocess import sys -def run_command(args, cwd=None, verbose=False): - try: - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError: - e = sys.exc_info()[1] + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" + git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" + keywords = {"refnames": git_refnames, "full": git_full} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "%(STYLE)s" + cfg.tag_prefix = "%(TAG_PREFIX)s" + cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" + cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %%s" %% dispcmd) + print(e) + return None + else: if verbose: - print("unable to run %%s" %% args[0]) - print(e) + print("unable to find command, tried %%s" %% (commands,)) return None stdout = p.communicate()[0].strip() - if sys.version >= '3': + if sys.version_info[0] >= 3: stdout = stdout.decode() if p.returncode != 0: if verbose: - print("unable to run %%s (error)" %% args[0]) + print("unable to run %%s (error)" %% dispcmd) return None return stdout -import sys -import re -import os.path +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%%s', but '%%s' doesn't start with " + "prefix '%%s'" %% (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} -def get_expanded_variables(versionfile_source): + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these - # variables. When used from setup.py, we don't want to import - # _version.py, so we do it with a regexp instead. This function is not - # used from _version.py. - variables = {} + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} try: - f = open(versionfile_source,"r") + f = open(versionfile_abs, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["refnames"] = mo.group(1) + keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["full"] = mo.group(1) + keywords["full"] = mo.group(1) f.close() except EnvironmentError: pass - return variables + return keywords + -def versions_from_expanded_variables(variables, tag_prefix, verbose=False): - refnames = variables["refnames"].strip() +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: - print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -189,172 +660,350 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): r = ref[len(tag_prefix):] if verbose: print("picking %%s" %% r) - return { "version": r, - "full": variables["full"].strip() } - # no suitable tags, so we use the full revision id + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: - print("no suitable tags, using full revision id") - return { "version": variables["full"].strip(), - "full": variables["full"].strip() } - -def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): - # this runs 'git' from the root of the source tree. That either means - # someone ran a setup.py command (and this code is in versioneer.py, so - # IN_LONG_VERSION_PY=False, thus the containing directory is the root of - # the source tree), or someone ran a project-specific entry point (and - # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the - # containing directory is somewhere deeper in the source tree). This only - # gets called if the git-archive 'subst' variables were *not* expanded, - # and _version.py hasn't already been rewritten with a short version - # string, meaning we're inside a checked out source tree. + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} - try: - here = os.path.abspath(__file__) - except NameError: - # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct - - # versionfile_source is the relative path from the top of the source tree - # (where the .git directory might live) to this file. Invert this to find - # the root from __file__. - root = here - if IN_LONG_VERSION_PY: - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) - else: - root = os.path.dirname(here) + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ if not os.path.exists(os.path.join(root, ".git")): if verbose: print("no .git in %%s" %% root) - return {} + raise NotThisMethod("no .git directory") - GIT = "git" + GITS = ["git"] if sys.platform == "win32": - GIT = "git.cmd" - stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], - cwd=root) - if stdout is None: - return {} - if not stdout.startswith(tag_prefix): - if verbose: - print("tag '%%s' doesn't start with prefix '%%s'" %% (stdout, tag_prefix)) - return {} - tag = stdout[len(tag_prefix):] - stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) - if stdout is None: - return {} - full = stdout.strip() - if tag.endswith("-dirty"): - full += "-dirty" - return {"version": tag, "full": full} - - -def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False): - if IN_LONG_VERSION_PY: - # We're running from _version.py. If it's from a source tree - # (execute-in-place), we can work upwards to find the root of the - # tree, and then check the parent directory for a version string. If - # it's in an installed application, there's no hope. - try: - here = os.path.abspath(__file__) - except NameError: - # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope - # versionfile_source is the relative path from the top of the source - # tree to _version.py. Invert this to find the root from __file__. - root = here - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%%s*" %% tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%%s'" + %% describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%%s' doesn't start with prefix '%%s'" + print(fmt %% (full_tag, tag_prefix)) + pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" + %% (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%%d" %% pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%%d" %% pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%%s" %% pieces["short"] else: - # we're running from versioneer.py, which means we're running from - # the setup.py in a source tree. sys.argv[0] is setup.py in the root. - here = os.path.abspath(sys.argv[0]) - root = os.path.dirname(here) + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%%s" %% pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%%s', but '%%s' doesn't start with prefix '%%s'" %% - (root, dirname, parentdir_prefix)) - return None - return {"version": dirname[len(parentdir_prefix):], "full": ""} - -tag_prefix = "%(TAG_PREFIX)s" -parentdir_prefix = "%(PARENTDIR_PREFIX)s" -versionfile_source = "%(VERSIONFILE_SOURCE)s" - -def get_versions(default={"version": "unknown", "full": ""}, verbose=False): - variables = { "refnames": git_refnames, "full": git_full } - ver = versions_from_expanded_variables(variables, tag_prefix, verbose) - if not ver: - ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) - if not ver: - ver = versions_from_parentdir(parentdir_prefix, versionfile_source, - verbose) - if not ver: - ver = default - return ver -''' +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + Like 'git describe --tags --dirty --always'. -import subprocess -import sys + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%%s'" %% style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose -def run_command(args, cwd=None, verbose=False): try: - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError: - e = sys.exc_info()[1] - if verbose: - print("unable to run %s" % args[0]) - print(e) - return None - stdout = p.communicate()[0].strip() - if sys.version >= '3': - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % args[0]) - return None - return stdout + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree"} -import sys -import re -import os.path + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass -def get_expanded_variables(versionfile_source): + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version"} +''' + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these - # variables. When used from setup.py, we don't want to import - # _version.py, so we do it with a regexp instead. This function is not - # used from _version.py. - variables = {} + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} try: - f = open(versionfile_source,"r") + f = open(versionfile_abs, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["refnames"] = mo.group(1) + keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["full"] = mo.group(1) + keywords["full"] = mo.group(1) f.close() except EnvironmentError: pass - return variables + return keywords -def versions_from_expanded_variables(variables, tag_prefix, verbose=False): - refnames = variables["refnames"].strip() + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: - print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -379,107 +1028,122 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): r = ref[len(tag_prefix):] if verbose: print("picking %s" % r) - return { "version": r, - "full": variables["full"].strip() } - # no suitable tags, so we use the full revision id + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: - print("no suitable tags, using full revision id") - return { "version": variables["full"].strip(), - "full": variables["full"].strip() } - -def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): - # this runs 'git' from the root of the source tree. That either means - # someone ran a setup.py command (and this code is in versioneer.py, so - # IN_LONG_VERSION_PY=False, thus the containing directory is the root of - # the source tree), or someone ran a project-specific entry point (and - # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the - # containing directory is somewhere deeper in the source tree). This only - # gets called if the git-archive 'subst' variables were *not* expanded, - # and _version.py hasn't already been rewritten with a short version - # string, meaning we're inside a checked out source tree. + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} - try: - here = os.path.abspath(__file__) - except NameError: - # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct - - # versionfile_source is the relative path from the top of the source tree - # (where the .git directory might live) to this file. Invert this to find - # the root from __file__. - root = here - if IN_LONG_VERSION_PY: - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) - else: - root = os.path.dirname(here) + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ if not os.path.exists(os.path.join(root, ".git")): if verbose: print("no .git in %s" % root) - return {} + raise NotThisMethod("no .git directory") - GIT = "git" + GITS = ["git"] if sys.platform == "win32": - GIT = "git.cmd" - stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], - cwd=root) - if stdout is None: - return {} - if not stdout.startswith(tag_prefix): - if verbose: - print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix)) - return {} - tag = stdout[len(tag_prefix):] - stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) - if stdout is None: - return {} - full = stdout.strip() - if tag.endswith("-dirty"): - full += "-dirty" - return {"version": tag, "full": full} - - -def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False): - if IN_LONG_VERSION_PY: - # We're running from _version.py. If it's from a source tree - # (execute-in-place), we can work upwards to find the root of the - # tree, and then check the parent directory for a version string. If - # it's in an installed application, there's no hope. - try: - here = os.path.abspath(__file__) - except NameError: - # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope - # versionfile_source is the relative path from the top of the source - # tree to _version.py. Invert this to find the root from __file__. - root = here - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + else: - # we're running from versioneer.py, which means we're running from - # the setup.py in a source tree. sys.argv[0] is setup.py in the root. - here = os.path.abspath(sys.argv[0]) - root = os.path.dirname(here) + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" % - (root, dirname, parentdir_prefix)) - return None - return {"version": dirname[len(parentdir_prefix):], "full": ""} + return pieces -import sys -def do_vcs_install(versionfile_source, ipy): - GIT = "git" +def do_vcs_install(manifest_in, versionfile_source, ipy): + """Git-specific installation logic for Versioneer. + + For Git, this means creating/changing .gitattributes to mark _version.py + for export-time keyword substitution. + """ + GITS = ["git"] if sys.platform == "win32": - GIT = "git.cmd" - run_command([GIT, "add", "versioneer.py"]) - run_command([GIT, "add", versionfile_source]) - run_command([GIT, "add", ipy]) + GITS = ["git.cmd", "git.exe"] + files = [manifest_in, versionfile_source] + if ipy: + files.append(ipy) + try: + me = __file__ + if me.endswith(".pyc") or me.endswith(".pyo"): + me = os.path.splitext(me)[0] + ".py" + versioneer_file = os.path.relpath(me) + except NameError: + versioneer_file = "versioneer.py" + files.append(versioneer_file) present = False try: f = open(".gitattributes", "r") @@ -489,140 +1153,492 @@ def do_vcs_install(versionfile_source, ipy): present = True f.close() except EnvironmentError: - pass + pass if not present: f = open(".gitattributes", "a+") f.write("%s export-subst\n" % versionfile_source) f.close() - run_command([GIT, "add", ".gitattributes"]) - + files.append(".gitattributes") + run_command(GITS, ["add", "--"] + files) + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%s', but '%s' doesn't start with " + "prefix '%s'" % (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.7+) from +# This file was generated by 'versioneer.py' (0.16) from # revision-control system data, or from the parent directory name of an # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '%(version)s' -version_full = '%(full)s' -def get_versions(default={}, verbose=False): - return {'version': version_version, 'full': version_full} +import json +import sys +version_json = ''' +%s +''' # END VERSION_JSON + + +def get_versions(): + return json.loads(version_json) """ -DEFAULT = {"version": "unknown", "full": "unknown"} def versions_from_file(filename): - versions = {} + """Try to determine the version from _version.py if present.""" try: - f = open(filename) + with open(filename) as f: + contents = f.read() except EnvironmentError: - return versions - for line in f.readlines(): - mo = re.match("version_version = '([^']+)'", line) - if mo: - versions["version"] = mo.group(1) - mo = re.match("version_full = '([^']+)'", line) - if mo: - versions["full"] = mo.group(1) - f.close() - return versions + raise NotThisMethod("unable to read _version.py") + mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", + contents, re.M | re.S) + if not mo: + raise NotThisMethod("no version_json in _version.py") + return json.loads(mo.group(1)) + def write_to_version_file(filename, versions): - f = open(filename, "w") - f.write(SHORT_VERSION_PY % versions) - f.close() + """Write the given version number to the given _version.py file.""" + os.unlink(filename) + contents = json.dumps(versions, sort_keys=True, + indent=1, separators=(",", ": ")) + with open(filename, "w") as f: + f.write(SHORT_VERSION_PY % contents) + print("set %s to '%s'" % (filename, versions["version"])) -def get_best_versions(versionfile, tag_prefix, parentdir_prefix, - default=DEFAULT, verbose=False): - # returns dict with two keys: 'version' and 'full' - # - # extract version from first of _version.py, 'git describe', parentdir. - # This is meant to work for developers using a source checkout, for users - # of a tarball created by 'setup.py sdist', and for users of a - # tarball/zipball created by 'git archive' or github's download-from-tag - # feature. - - variables = get_expanded_variables(versionfile_source) - if variables: - ver = versions_from_expanded_variables(variables, tag_prefix) - if ver: - if verbose: print("got version from expanded variable %s" % ver) - return ver +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" - ver = versions_from_file(versionfile) - if ver: - if verbose: print("got version from file %s %s" % (versionfile, ver)) - return ver - ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) - if ver: - if verbose: print("got version from git %s" % ver) - return ver +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". - ver = versions_from_parentdir(parentdir_prefix, versionfile_source, verbose) - if ver: - if verbose: print("got version from parentdir %s" % ver) - return ver + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. - if verbose: print("got version from default %s" % ver) - return default - -def get_versions(default=DEFAULT, verbose=False): - assert versionfile_source is not None, "please set versioneer.versionfile_source" - assert tag_prefix is not None, "please set versioneer.tag_prefix" - assert parentdir_prefix is not None, "please set versioneer.parentdir_prefix" - return get_best_versions(versionfile_source, tag_prefix, parentdir_prefix, - default=default, verbose=verbose) -def get_version(verbose=False): - return get_versions(verbose=verbose)["version"] - -class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - def initialize_options(self): + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +class VersioneerBadRootError(Exception): + """The project root directory is unknown or missing key files.""" + + +def get_versions(verbose=False): + """Get the project version from whatever source is available. + + Returns dict with two keys: 'version' and 'full'. + """ + if "versioneer" in sys.modules: + # see the discussion in cmdclass.py:get_cmdclass() + del sys.modules["versioneer"] + + root = get_root() + cfg = get_config_from_root(root) + + assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" + handlers = HANDLERS.get(cfg.VCS) + assert handlers, "unrecognized VCS '%s'" % cfg.VCS + verbose = verbose or cfg.verbose + assert cfg.versionfile_source is not None, \ + "please set versioneer.versionfile_source" + assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" + + versionfile_abs = os.path.join(root, cfg.versionfile_source) + + # extract version from first of: _version.py, VCS command (e.g. 'git + # describe'), parentdir. This is meant to work for developers using a + # source checkout, for users of a tarball created by 'setup.py sdist', + # and for users of a tarball/zipball created by 'git archive' or github's + # download-from-tag feature or the equivalent in other VCSes. + + get_keywords_f = handlers.get("get_keywords") + from_keywords_f = handlers.get("keywords") + if get_keywords_f and from_keywords_f: + try: + keywords = get_keywords_f(versionfile_abs) + ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) + if verbose: + print("got version from expanded keyword %s" % ver) + return ver + except NotThisMethod: + pass + + try: + ver = versions_from_file(versionfile_abs) + if verbose: + print("got version from file %s %s" % (versionfile_abs, ver)) + return ver + except NotThisMethod: pass - def finalize_options(self): + + from_vcs_f = handlers.get("pieces_from_vcs") + if from_vcs_f: + try: + pieces = from_vcs_f(cfg.tag_prefix, root, verbose) + ver = render(pieces, cfg.style) + if verbose: + print("got version from VCS %s" % ver) + return ver + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + if verbose: + print("got version from parentdir %s" % ver) + return ver + except NotThisMethod: pass - def run(self): - ver = get_version(verbose=True) - print("Version is currently: %s" % ver) - - -class cmd_build(_build): - def run(self): - versions = get_versions(verbose=True) - _build.run(self) - # now locate _version.py in the new build/ directory and replace it - # with an updated value - target_versionfile = os.path.join(self.build_lib, versionfile_build) - print("UPDATING %s" % target_versionfile) - os.unlink(target_versionfile) - f = open(target_versionfile, "w") - f.write(SHORT_VERSION_PY % versions) - f.close() -class cmd_sdist(_sdist): - def run(self): - versions = get_versions(verbose=True) - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory (remembering - # that it may be a hardlink) and replace it with an updated value - target_versionfile = os.path.join(base_dir, versionfile_source) - print("UPDATING %s" % target_versionfile) - os.unlink(target_versionfile) - f = open(target_versionfile, "w") - f.write(SHORT_VERSION_PY % self._versioneer_generated_versions) - f.close() + if verbose: + print("unable to compute version") + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, "error": "unable to compute version"} + + +def get_version(): + """Get the short version string for this project.""" + return get_versions()["version"] + + +def get_cmdclass(): + """Get the custom setuptools/distutils subclasses used by Versioneer.""" + if "versioneer" in sys.modules: + del sys.modules["versioneer"] + # this fixes the "python setup.py develop" case (also 'install' and + # 'easy_install .'), in which subdependencies of the main project are + # built (using setup.py bdist_egg) in the same python process. Assume + # a main project A and a dependency B, which use different versions + # of Versioneer. A's setup.py imports A's Versioneer, leaving it in + # sys.modules by the time B's setup.py is executed, causing B to run + # with the wrong versioneer. Setuptools wraps the sub-dep builds in a + # sandbox that restores sys.modules to it's pre-build state, so the + # parent is protected against the child's "import versioneer". By + # removing ourselves from sys.modules here, before the child build + # happens, we protect the child from the parent's versioneer too. + # Also see https://github.com/warner/python-versioneer/issues/52 + + cmds = {} + + # we add "version" to both distutils and setuptools + from distutils.core import Command + + class cmd_version(Command): + description = "report generated version string" + user_options = [] + boolean_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + vers = get_versions(verbose=True) + print("Version: %s" % vers["version"]) + print(" full-revisionid: %s" % vers.get("full-revisionid")) + print(" dirty: %s" % vers.get("dirty")) + if vers["error"]: + print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version + + # we override "build_py" in both distutils and setuptools + # + # most invocation pathways end up running build_py: + # distutils/build -> build_py + # distutils/install -> distutils/build ->.. + # setuptools/bdist_wheel -> distutils/install ->.. + # setuptools/bdist_egg -> distutils/install_lib -> build_py + # setuptools/install -> bdist_egg ->.. + # setuptools/develop -> ? + + # we override different "build_py" commands for both environments + if "setuptools" in sys.modules: + from setuptools.command.build_py import build_py as _build_py + else: + from distutils.command.build_py import build_py as _build_py + + class cmd_build_py(_build_py): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + _build_py.run(self) + # now locate _version.py in the new build/ directory and replace + # it with an updated value + if cfg.versionfile_build: + target_versionfile = os.path.join(self.build_lib, + cfg.versionfile_build) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py + + if "cx_Freeze" in sys.modules: # cx_freeze enabled? + from cx_Freeze.dist import build_exe as _build_exe + + class cmd_build_exe(_build_exe): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + target_versionfile = cfg.versionfile_source + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + _build_exe.run(self) + os.unlink(target_versionfile) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % + {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + cmds["build_exe"] = cmd_build_exe + del cmds["build_py"] + + # we override different "sdist" commands for both environments + if "setuptools" in sys.modules: + from setuptools.command.sdist import sdist as _sdist + else: + from distutils.command.sdist import sdist as _sdist + + class cmd_sdist(_sdist): + def run(self): + versions = get_versions() + self._versioneer_generated_versions = versions + # unless we update this, the command will keep using the old + # version + self.distribution.metadata.version = versions["version"] + return _sdist.run(self) + + def make_release_tree(self, base_dir, files): + root = get_root() + cfg = get_config_from_root(root) + _sdist.make_release_tree(self, base_dir, files) + # now locate _version.py in the new base_dir directory + # (remembering that it may be a hardlink) and replace it with an + # updated value + target_versionfile = os.path.join(base_dir, cfg.versionfile_source) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, + self._versioneer_generated_versions) + cmds["sdist"] = cmd_sdist + + return cmds + + +CONFIG_ERROR = """ +setup.cfg is missing the necessary Versioneer configuration. You need +a section like: + + [versioneer] + VCS = git + style = pep440 + versionfile_source = src/myproject/_version.py + versionfile_build = myproject/_version.py + tag_prefix = + parentdir_prefix = myproject- + +You will also need to edit your setup.py to use the results: + + import versioneer + setup(version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), ...) + +Please read the docstring in ./versioneer.py for configuration instructions, +edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. +""" + +SAMPLE_CONFIG = """ +# See the docstring in versioneer.py for instructions. Note that you must +# re-run 'versioneer.py setup' after changing this section, and commit the +# resulting files. + +[versioneer] +#VCS = git +#style = pep440 +#versionfile_source = +#versionfile_build = +#tag_prefix = +#parentdir_prefix = + +""" INIT_PY_SNIPPET = """ from ._version import get_versions @@ -630,40 +1646,129 @@ __version__ = get_versions()['version'] del get_versions """ -class cmd_update_files(Command): - description = "modify __init__.py and create _version.py" - user_options = [] - boolean_options = [] - def initialize_options(self): - pass - def finalize_options(self): - pass - def run(self): - ipy = os.path.join(os.path.dirname(versionfile_source), "__init__.py") - print(" creating %s" % versionfile_source) - f = open(versionfile_source, "w") - f.write(LONG_VERSION_PY % {"DOLLAR": "$", - "TAG_PREFIX": tag_prefix, - "PARENTDIR_PREFIX": parentdir_prefix, - "VERSIONFILE_SOURCE": versionfile_source, - }) - f.close() + +def do_setup(): + """Main VCS-independent setup function for installing Versioneer.""" + root = get_root() + try: + cfg = get_config_from_root(root) + except (EnvironmentError, configparser.NoSectionError, + configparser.NoOptionError) as e: + if isinstance(e, (EnvironmentError, configparser.NoSectionError)): + print("Adding sample versioneer config to setup.cfg", + file=sys.stderr) + with open(os.path.join(root, "setup.cfg"), "a") as f: + f.write(SAMPLE_CONFIG) + print(CONFIG_ERROR, file=sys.stderr) + return 1 + + print(" creating %s" % cfg.versionfile_source) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), + "__init__.py") + if os.path.exists(ipy): try: - old = open(ipy, "r").read() + with open(ipy, "r") as f: + old = f.read() except EnvironmentError: old = "" if INIT_PY_SNIPPET not in old: print(" appending to %s" % ipy) - f = open(ipy, "a") - f.write(INIT_PY_SNIPPET) - f.close() + with open(ipy, "a") as f: + f.write(INIT_PY_SNIPPET) else: print(" %s unmodified" % ipy) - do_vcs_install(versionfile_source, ipy) + else: + print(" %s doesn't exist, ok" % ipy) + ipy = None + + # Make sure both the top-level "versioneer.py" and versionfile_source + # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so + # they'll be copied into source distributions. Pip won't be able to + # install the package without this. + manifest_in = os.path.join(root, "MANIFEST.in") + simple_includes = set() + try: + with open(manifest_in, "r") as f: + for line in f: + if line.startswith("include "): + for include in line.split()[1:]: + simple_includes.add(include) + except EnvironmentError: + pass + # That doesn't cover everything MANIFEST.in can do + # (http://docs.python.org/2/distutils/sourcedist.html#commands), so + # it might give some false negatives. Appending redundant 'include' + # lines is safe, though. + if "versioneer.py" not in simple_includes: + print(" appending 'versioneer.py' to MANIFEST.in") + with open(manifest_in, "a") as f: + f.write("include versioneer.py\n") + else: + print(" 'versioneer.py' already in MANIFEST.in") + if cfg.versionfile_source not in simple_includes: + print(" appending versionfile_source ('%s') to MANIFEST.in" % + cfg.versionfile_source) + with open(manifest_in, "a") as f: + f.write("include %s\n" % cfg.versionfile_source) + else: + print(" versionfile_source already in MANIFEST.in") -def get_cmdclass(): - return {'version': cmd_version, - 'update_files': cmd_update_files, - 'build': cmd_build, - 'sdist': cmd_sdist, - } + # Make VCS-specific changes. For git, this means creating/changing + # .gitattributes to mark _version.py for export-time keyword + # substitution. + do_vcs_install(manifest_in, cfg.versionfile_source, ipy) + return 0 + + +def scan_setup_py(): + """Validate the contents of setup.py against Versioneer's expectations.""" + found = set() + setters = False + errors = 0 + with open("setup.py", "r") as f: + for line in f.readlines(): + if "import versioneer" in line: + found.add("import") + if "versioneer.get_cmdclass()" in line: + found.add("cmdclass") + if "versioneer.get_version()" in line: + found.add("get_version") + if "versioneer.VCS" in line: + setters = True + if "versioneer.versionfile_source" in line: + setters = True + if len(found) != 3: + print("") + print("Your setup.py appears to be missing some important items") + print("(but I might be wrong). Please make sure it has something") + print("roughly like the following:") + print("") + print(" import versioneer") + print(" setup( version=versioneer.get_version(),") + print(" cmdclass=versioneer.get_cmdclass(), ...)") + print("") + errors += 1 + if setters: + print("You should remove lines like 'versioneer.VCS = ' and") + print("'versioneer.versionfile_source = ' . This configuration") + print("now lives in setup.cfg, and should be removed from setup.py") + print("") + errors += 1 + return errors + +if __name__ == "__main__": + cmd = sys.argv[1] + if cmd == "setup": + errors = do_setup() + errors += scan_setup_py() + if errors: + sys.exit(1) |