diff options
author | Vik Bhatti <vik@vikbhatti.com> | 2014-04-08 14:50:52 +0100 |
---|---|---|
committer | Vik Bhatti <vik@vikbhatti.com> | 2014-04-08 14:50:52 +0100 |
commit | 8d9ea35a00f133083e752d199a1102b00827876f (patch) | |
tree | f86cad23c5c5f842442b15a905ea3c92262cf254 | |
parent | 275cd8672ac5a0144f4d96ba39f0db6705786fc4 (diff) | |
parent | c3a5a8f338776255859d50f4b906642f540cb162 (diff) |
Pulled from upstream
-rw-r--r-- | .gitignore | 1 | ||||
-rw-r--r-- | Modulefile | 4 | ||||
-rw-r--r-- | README.md | 13 | ||||
-rw-r--r-- | lib/facter/pip_version.rb | 27 | ||||
-rw-r--r-- | lib/facter/python_version.rb | 40 | ||||
-rw-r--r-- | lib/facter/virtualenv_version.rb | 27 | ||||
-rw-r--r-- | manifests/config.pp | 35 | ||||
-rw-r--r-- | manifests/gunicorn.pp | 4 | ||||
-rw-r--r-- | manifests/init.pp | 22 | ||||
-rw-r--r-- | manifests/install.pp | 54 | ||||
-rw-r--r-- | manifests/pip.pp | 68 | ||||
-rw-r--r-- | manifests/requirements.pp | 13 | ||||
-rw-r--r-- | manifests/virtualenv.pp | 44 | ||||
-rw-r--r-- | templates/gunicorn.erb | 32 |
14 files changed, 313 insertions, 71 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b2bcbe8 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +pkg/
\ No newline at end of file @@ -1,5 +1,5 @@ -name 'puppet-python' -version '1.1.5' +name 'stankevich-python' +version '1.7.1' author 'Sergey Stankevich' license 'Apache License, Version 2.0' @@ -37,6 +37,8 @@ Installs and manages python, python-dev, python-virtualenv and Gunicorn. **gunicorn** - Install Gunicorn. Default: false +**manage_gunicorn** - Allow Installation / Removal of Gunicorn. Default: true + class { 'python': version => 'system', dev => true, @@ -48,7 +50,9 @@ Installs and manages python, python-dev, python-virtualenv and Gunicorn. Installs and manages packages from pip. -**ensure** - present/absent. Default: present +**pkgname** - the name of the package to install. Required. + +**ensure** - present/latest/absent. Default: present **virtualenv** - virtualenv to run pip in. Default: system (no virtualenv) @@ -67,6 +71,7 @@ Installs and manages packages from pip. **uninstall_args** - Array of additional flags to pass to pip during uninstall. Default: none python::pip { 'cx_Oracle': + pkgname => 'cx_Oracle', virtualenv => '/var/www/project1', owner => 'appuser', proxy => 'http://proxy.domain.com:3128', @@ -117,6 +122,10 @@ Creates Python virtualenv. **index** - Base URL of Python package index. Default: none +**cwd** - The directory from which to run the "pip install" command. Default: undef + +**timeout** - The maximum time in seconds the "pip install" command should take. Default: 1800 + python::virtualenv { '/var/www/project1': ensure => present, version => 'system', @@ -126,6 +135,8 @@ Creates Python virtualenv. distribute => false, owner => 'appuser', group => 'apps', + cwd => '/var/www/project1', + timeout => 0, } ### python::gunicorn diff --git a/lib/facter/pip_version.rb b/lib/facter/pip_version.rb new file mode 100644 index 0000000..ce32f98 --- /dev/null +++ b/lib/facter/pip_version.rb @@ -0,0 +1,27 @@ +# Make pip version available as a fact +# Works with pip loaded and without, pip installed using pip and package installed +require 'puppet' +pkg = Puppet::Type.type(:package).new(:name => "python-pip") +Facter.add("pip_version") do + has_weight 100 + setcode do + begin + /^pip (\d+\.\d+\.?\d*).*$/.match(Facter::Util::Resolution.exec('pip --version 2>/dev/null'))[1] + rescue + false + end + end +end + +Facter.add("pip_version") do + has_weight 50 + setcode do + begin + unless [:absent,'purged'].include?(pkg.retrieve[pkg.property(:ensure)]) + /^.*(\d+\.\d+\.\d+).*$/.match(pkg.retrieve[pkg.property(:ensure)])[1] + end + rescue + false + end + end +end diff --git a/lib/facter/python_version.rb b/lib/facter/python_version.rb new file mode 100644 index 0000000..bb9ec98 --- /dev/null +++ b/lib/facter/python_version.rb @@ -0,0 +1,40 @@ +# Make python versions available as facts +# In lists default python and system python versions +require 'puppet' +pkg = Puppet::Type.type(:package).new(:name => "python") + +Facter.add("system_python_version") do + setcode do + begin + unless [:absent,'purged'].include?(pkg.retrieve[pkg.property(:ensure)]) + /^(\d+\.\d+\.\d+).*$/.match(pkg.retrieve[pkg.property(:ensure)])[1] + end + rescue + false + end + end +end + +Facter.add("python_version") do + has_weight 100 + setcode do + begin + /^.*(\d+\.\d+\.\d+)$/.match(Facter::Util::Resolution.exec('python -V 2>&1'))[1] + rescue + false + end + end +end + +Facter.add("python_version") do + has_weight 50 + setcode do + begin + unless [:absent,'purged'].include?(pkg.retrieve[pkg.property(:ensure)]) + /^.*(\d+\.\d+\.\d+).*$/.match(pkg.retrieve[pkg.property(:ensure)])[1] + end + rescue + false + end + end +end diff --git a/lib/facter/virtualenv_version.rb b/lib/facter/virtualenv_version.rb new file mode 100644 index 0000000..04736ed --- /dev/null +++ b/lib/facter/virtualenv_version.rb @@ -0,0 +1,27 @@ +# Make virtualenv version available as a fact +# Works with virualenv loaded and without, pip installed and package installed +require 'puppet' +pkg = Puppet::Type.type(:package).new(:name => "virtualenv") +Facter.add("virtualenv_version") do + has_weight 100 + setcode do + begin + Facter::Util::Resolution.exec('virtualenv --version') + rescue + false + end + end +end + +Facter.add("virtualenv_version") do + has_weight 50 + setcode do + begin + unless [:absent,'purged'].include?(pkg.retrieve[pkg.property(:ensure)]) + /^.*(\d+\.\d+\.\d+).*$/.match(pkg.retrieve[pkg.property(:ensure)])[1] + end + rescue + false + end + end +end diff --git a/manifests/config.pp b/manifests/config.pp index 49e5230..c7196f2 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -1,3 +1,18 @@ +# == Define: python::config +# +# Optionally installs the gunicorn service +# +# === Examples +# +# include python::config +# +# === Authors +# +# Sergey Stankevich +# Ashley Penney +# Fotis Gimian +# + class python::config { Class['python::install'] -> Python::Pip <| |> @@ -6,17 +21,19 @@ class python::config { Python::Virtualenv <| |> -> Python::Pip <| |> - if $python::gunicorn { - Class['python::install'] -> Python::Gunicorn <| |> + if $python::manage_gunicorn { + if $python::gunicorn { + Class['python::install'] -> Python::Gunicorn <| |> - Python::Gunicorn <| |> ~> Service['gunicorn'] + Python::Gunicorn <| |> ~> Service['gunicorn'] - service { 'gunicorn': - ensure => running, - enable => true, - hasrestart => true, - hasstatus => false, - pattern => '/usr/bin/gunicorn', + service { 'gunicorn': + ensure => running, + enable => true, + hasrestart => true, + hasstatus => false, + pattern => '/usr/bin/gunicorn', + } } } diff --git a/manifests/gunicorn.pp b/manifests/gunicorn.pp index 13f4872..159afa3 100644 --- a/manifests/gunicorn.pp +++ b/manifests/gunicorn.pp @@ -37,6 +37,8 @@ # dir => '/var/www/project1/current', # bind => 'unix:/tmp/gunicorn.socket', # environment => 'prod', +# owner => 'www-data', +# group => 'www-data', # template => 'python/gunicorn.erb', # } # @@ -53,6 +55,8 @@ define python::gunicorn ( $dir = false, $bind = false, $environment = false, + $owner = 'www-data', + $group = 'www-data', $template = 'python/gunicorn.erb', ) { diff --git a/manifests/init.pp b/manifests/init.pp index 2a9a44c..4797b11 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -14,11 +14,15 @@ # Install python-dev. Default: false # # [*virtualenv*] -# Install python-virtualenv. Default: false +# Install python-virtualenv. Default: false, also accepts 'pip' which will +# install latest virtualenv from pip rather than package manager # # [*gunicorn*] # Install Gunicorn. Default: false # +# [*manage_gunicorn*] +# Allow Installation / Removal of Gunicorn. Default: true +# # === Examples # # class { 'python': @@ -34,16 +38,18 @@ # Sergey Stankevich # class python ( - $version = 'system', - $pip = false, - $dev = false, - $virtualenv = false, - $gunicorn = false + $version = 'system', + $pip = false, + $dev = false, + $virtualenv = false, + $gunicorn = false, + $manage_gunicorn = true, + $provider = undef ) { # Module compatibility check - $compatible = [ 'Debian', 'Ubuntu', 'CentOS', 'RedHat' ] - if ! ($::operatingsystem in $compatible) { + $compatible = [ 'Debian', 'RedHat'] + if ! ($::osfamily in $compatible) { fail("Module is not compatible with ${::operatingsystem}") } diff --git a/manifests/install.pp b/manifests/install.pp index 9306e3a..1c2cb0b 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -1,17 +1,31 @@ +# == Define: python::install +# +# Installs core python packages +# +# === Examples +# +# include python::install +# +# === Authors +# +# Sergey Stankevich +# Ashley Penney +# Fotis Gimian +# + class python::install { $python = $python::version ? { 'system' => 'python', + 'pypy' => 'pypy', default => "python${python::version}", } - $pythondev = $::operatingsystem ? { - /(?i:RedHat|CentOS|Fedora)/ => "${python}-devel", - /(?i:Debian|Ubuntu)/ => "${python}-dev" + $pythondev = $::osfamily ? { + RedHat => "${python}-devel", + Debian => "${python}-dev" } - package { $python: ensure => present } - $dev_ensure = $python::dev ? { true => present, default => absent, @@ -22,21 +36,33 @@ class python::install { default => absent, } - package { $pythondev: ensure => $dev_ensure } - package { 'python-pip': ensure => $pip_ensure } - $venv_ensure = $python::virtualenv ? { true => present, default => absent, } - package { 'python-virtualenv': ensure => $venv_ensure } - - $gunicorn_ensure = $python::gunicorn ? { - true => present, - default => absent, + # Install latest from pip if pip is the provider + case $python::provider { + pip: { + package { 'virtualenv': ensure => latest, provider => pip } + package { 'pip': ensure => latest, provider => pip } + package { $pythondev: ensure => latest } + package { "python==${python::version}": ensure => latest, provider => pip } + } + default: { + package { 'python-virtualenv': ensure => $venv_ensure } + package { 'python-pip': ensure => $pip_ensure } + package { $pythondev: ensure => $dev_ensure } + package { $python: ensure => present } + } } - package { 'gunicorn': ensure => $gunicorn_ensure } + if $python::manage_gunicorn { + $gunicorn_ensure = $python::gunicorn ? { + true => present, + default => absent, + } + package { 'gunicorn': ensure => $gunicorn_ensure } + } } diff --git a/manifests/pip.pp b/manifests/pip.pp index 98b6fac..0c78903 100644 --- a/manifests/pip.pp +++ b/manifests/pip.pp @@ -4,6 +4,12 @@ # # === Parameters # +# [*name] +# must be unique +# +# [*pkgname] +# name of the package. If pkgname is not specified, use name (title) instead. +# # [*ensure*] # present|absent. Default: present # @@ -35,6 +41,7 @@ # Fotis Gimian # define python::pip ( + $pkgname = $name, $ensure = present, $virtualenv = 'system', $url = false, @@ -70,37 +77,84 @@ define python::pip ( default => "--proxy=${proxy}", } - $grep_regex = $name ? { - /==/ => "^${name}\$", - default => "^${name}==", + # Check if searching by explicit version. + if $ensure =~ /^((19|20)[0-9][0-9]-(0[1-9]|1[1-2])-([0-2][1-9]|3[0-1])|[0-9]+\.[0-9]+(\.[0-9]+)?)$/ { + $grep_regex = "^${pkgname}==${ensure}\$" + } else { + $grep_regex = $pkgname ? { + /==/ => "^${pkgname}\$", + default => "^${pkgname}==", + } } $egg_name = $egg ? { - false => $name, + false => $pkgname, default => $egg } $source = $url ? { - false => $name, + false => $pkgname, default => "${url}#egg=${egg_name}", } + # We need to jump through hoops to make sure we issue the correct pip command + # depending on wheel support and versions. + # + # Pip does not support wheels prior to version 1.4.0 + # Pip wheels require setuptools/distribute > 0.8 + # Python 2.6 and older does not support setuptools/distribute > 0.8 + # Pip >= 1.5 tries to use wheels by default, even if wheel package is not + # installed, in this case the --no-use-wheel flag needs to be passed + # Versions prior to 1.5 don't support the --no-use-wheel flag + # + # To check for this we test for wheel parameter using help and then using + # version, this makes sure we only use wheels if they are supported and + # installed + + case $ensure { + /^((19|20)[0-9][0-9]-(0[1-9]|1[1-2])-([0-2][1-9]|3[0-1])|[0-9]+\.[0-9]+(\.[0-9]+)?)$/: { + # Version formats as per http://guide.python-distribute.org/specification.html#standard-versioning-schemes + # Explicit version. + exec { "pip_install_${name}": + command => "${pip_env} wheel --help > /dev/null 2>&1 && { ${pip_env} wheel --version > /dev/null 2>&1 || wheel_support_flag='--no-use-wheel'; } ; ${pip_env} --log ${cwd}/pip.log install ${install_args} \$wheel_support_flag ${proxy_flag} ${source}==${ensure}", + unless => "${pip_env} freeze | grep -i -e ${grep_regex}", + user => $owner, + environment => $environment, + path => ['/usr/local/bin','/usr/bin','/bin', '/usr/sbin'], + } + } + present: { + # Whatever version is available. exec { "pip_install_${name}": - command => "${pip_env} --log-file ${cwd}/pip.log install ${install_args} ${proxy_flag} ${source}", + command => "${pip_env} wheel --help > /dev/null 2>&1 && { ${pip_env} wheel --version > /dev/null 2>&1 || wheel_support_flag='--no-use-wheel'; } ; ${pip_env} --log ${cwd}/pip.log install ${install_args} \$wheel_support_flag ${proxy_flag} ${source}", unless => "${pip_env} freeze | grep -i -e ${grep_regex}", user => $owner, environment => $environment, + path => ['/usr/local/bin','/usr/bin','/bin', '/usr/sbin'], + } + } + + latest: { + # Latest version. + exec { "pip_install_${name}": + command => "${pip_env} wheel --help > /dev/null 2>&1 && { ${pip_env} wheel --version > /dev/null 2>&1 || wheel_support_flag='--no-use-wheel'; } ; ${pip_env} --log ${cwd}/pip.log install --upgrade \$wheel_support_flag ${proxy_flag} ${source}", + unless => "${pip_env} search ${source} | grep -i INSTALLED | grep -i latest", + user => $owner, + environment => $environment, + path => ['/usr/local/bin','/usr/bin','/bin', '/usr/sbin'], } } default: { + # Anti-action, uninstall. exec { "pip_uninstall_${name}": - command => "echo y | ${pip_env} uninstall ${uninstall_args} ${proxy_flag} ${name}", + command => "echo y | ${pip_env} uninstall ${uninstall_args} ${proxy_flag} ${use_pkgname}", onlyif => "${pip_env} freeze | grep -i -e ${grep_regex}", user => $owner, environment => $environment, + path => ['/usr/local/bin','/usr/bin','/bin', '/usr/sbin'], } } } diff --git a/manifests/requirements.pp b/manifests/requirements.pp index 010a199..11c5f00 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -27,6 +27,11 @@ # [*environment*] # Additional environment variables required to install the packages. Default: none # +# [*forceupdate*] +# Run a pip install requirements even if we don't receive an event from the +# requirements file - Useful for when the requirements file is written as part of a +# resource other than file (E.g vcsrepo) +# # === Examples # # python::requirements { '/var/www/project1/requirements.txt': @@ -47,7 +52,8 @@ define python::requirements ( $group = 'root', $proxy = false, $src = false, - $environment = [] + $environment = [], + $forceupdate = false, ) { if $virtualenv == 'system' and ($owner != 'root' or $group != 'root') { @@ -90,12 +96,13 @@ define python::requirements ( exec { "python_requirements${name}": provider => shell, - command => "${pip_env} --log-file ${cwd}/pip.log install ${proxy_flag} ${src_flag} -r ${requirements}", - refreshonly => true, + command => "${pip_env} --log ${cwd}/pip.log install ${proxy_flag} ${src_flag} -r ${requirements}", + refreshonly => !$forceupdate, timeout => 1800, user => $owner, subscribe => File[$requirements], environment => $environment, + path => ['/usr/local/bin','/usr/bin','/bin', '/usr/sbin'], } } diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index ebc75be..a88042b 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -15,7 +15,7 @@ # # [*systempkgs*] # Copy system site-packages into virtualenv. Default: don't -# +# If virtualenv version < 1.7 this flag has no effect since # [*venv_dir*] # Directory to install virtualenv to. Default: $name # @@ -40,6 +40,12 @@ # [*path*] # Specifies the PATH variable. Default: [ '/bin', '/usr/bin', '/usr/sbin' ] # +# [*cwd*] +# The directory from which to run the "pip install" command. Default: undef +# +# [*timeout*] +# The maximum time in seconds the "pip install" command should take. Default: 1800 +# # === Examples # # python::virtualenv { '/var/www/project1': @@ -71,12 +77,15 @@ define python::virtualenv ( $proxy = false, $environment = [], $path = [ '/bin', '/usr/bin', '/usr/sbin' ] + $cwd = undef, + $timeout = 1800 ) { if $ensure == 'present' { $python = $version ? { 'system' => 'python', + 'pypy' => 'pypy', default => "python${version}", } @@ -90,37 +99,53 @@ define python::virtualenv ( default => "&& export http_proxy=${proxy}", } - $system_pkgs_flag = $systempkgs ? { - false => '', - default => '--system-site-packages', + # Virtualenv versions prior to 1.7 do not support the + # --system-site-packages flag, default off for prior versions + # Prior to version 1.7 the default was equal to --system-site-packages + # and the flag --no-site-packages had to be passed to do the opposite + if (( versioncmp($::virtualenv_version,'1.7') > 0 ) and ( $systempkgs == true )) { + $system_pkgs_flag = '--system-site-packages' + } elsif (( versioncmp($::virtualenv_version,'1.7') < 0 ) and ( $systempkgs == false )) { + $system_pkgs_flag = '--no-site-packages' + } else { + $system_pkgs_flag = '' } $distribute_pkg = $distribute ? { true => 'distribute', - default => '', + default => 'setuptools', } $pypi_index = $index ? { false => '', default => "-i ${index}", } + # Python 2.6 and older does not support setuptools/distribute > 0.8 which + # is required for pip wheel support, pip therefor requires --no-use-wheel flag + # if the # pip version is more recent than 1.4.1 but using an old python or + # setuputils/distribute version + # To check for this we test for wheel parameter using help and then using + # version, this makes sure we only use wheels if they are supported + exec { "python_virtualenv_${venv_dir}": - command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv ${system_pkgs_flag} -p ${python} ${venv_dir} && ${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} --upgrade pip ${distribute_pkg}", + command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv ${system_pkgs_flag} -p ${python} ${venv_dir} && ${venv_dir}/bin/pip wheel --help > /dev/null 2>&1 && { ${venv_dir}/bin/pip wheel --version > /dev/null 2>&1 || wheel_support_flag='--no-use-wheel'; } ; { ${venv_dir}/bin/pip --log ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} \$wheel_support_flag --upgrade pip ${distribute_pkg} || ${venv_dir}/bin/pip --log ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} --upgrade pip ${distribute_pkg} ;}", user => $owner, creates => "${venv_dir}/bin/activate", path => $path, cwd => '/tmp', environment => $environment, + unless => "grep '^[\\t ]*VIRTUAL_ENV=[\\\\'\\\"]*${venv_dir}[\\\"\\\\'][\\t ]*$' ${venv_dir}/bin/activate", #Unless activate exists and VIRTUAL_ENV is correct we re-create the virtualenv } if $requirements { exec { "python_requirements_initial_install_${requirements}_${venv_dir}": - command => "${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} -r ${requirements}", + command => "${venv_dir}/bin/pip wheel --help > /dev/null 2>&1 && { ${venv_dir}/bin/pip wheel --version > /dev/null 2>&1 || wheel_support_flag='--no-use-wheel'; } ; ${venv_dir}/bin/pip --log ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} \$wheel_support_flag -r ${requirements}", refreshonly => true, - timeout => 1800, + timeout => $timeout, user => $owner, subscribe => Exec["python_virtualenv_${venv_dir}"], environment => $environment, + cwd => $cwd } python::requirements { "${requirements}_${venv_dir}": @@ -132,7 +157,6 @@ define python::virtualenv ( require => Exec["python_virtualenv_${venv_dir}"], } } - } elsif $ensure == 'absent' { file { $venv_dir: @@ -141,7 +165,5 @@ define python::virtualenv ( recurse => true, purge => true, } - } - } diff --git a/templates/gunicorn.erb b/templates/gunicorn.erb index 1a96531..10f81fa 100644 --- a/templates/gunicorn.erb +++ b/templates/gunicorn.erb @@ -1,36 +1,36 @@ CONFIG = { -<% if mode == 'django' -%> +<% if @mode == 'django' -%> 'mode': 'django', <% else -%> 'mode': 'wsgi', <% end -%> -<% if virtualenv -%> +<% if @virtualenv -%> 'environment': { -<% if environment -%> - 'ENVIRONMENT': '<%= environment %>', +<% if @environment -%> + 'ENVIRONMENT': '<%= @environment %>', <% end -%> - 'PYTHONPATH': '<%= virtualenv %>' + 'PYTHONPATH': '<%= @virtualenv %>' }, <% end -%> - 'working_dir': '<%= dir %>', - 'user': 'www-data', - 'group': 'www-data', -<% if virtualenv -%> - 'python': '<%= virtualenv %>/bin/python', + 'working_dir': '<%= @dir %>', + 'user': '<%= @owner %>', + 'group': '<%= @group %>', +<% if @virtualenv -%> + 'python': '<%= @virtualenv %>/bin/python', <% else -%> 'python': '/usr/bin/python', <% end -%> 'args': ( -<% if !virtualenv and !bind -%> - '--bind=unix:/tmp/gunicorn-<%= name %>.socket', -<% elsif virtualenv and !bind -%> - '--bind=unix:<%= virtualenv %>/<%= name %>.socket', +<% if !@virtualenv and !@bind -%> + '--bind=unix:/tmp/gunicorn-<%= @name %>.socket', +<% elsif @virtualenv and !@bind -%> + '--bind=unix:<%= @virtualenv %>/<%= @name %>.socket', <% else -%> - '--bind=<%= bind %>', + '--bind=<%= @bind %>', <% end -%> '--workers=<%= @processorcount.to_i*2 %>', '--timeout=30', -<% if mode != 'django' -%> +<% if @mode != 'django' -%> 'app:app', <% end -%> ), |