diff options
Diffstat (limited to 'manifests')
| -rw-r--r-- | manifests/config.pp | 35 | ||||
| -rw-r--r-- | manifests/gunicorn.pp | 4 | ||||
| -rw-r--r-- | manifests/init.pp | 34 | ||||
| -rw-r--r-- | manifests/install.pp | 56 | ||||
| -rw-r--r-- | manifests/pip.pp | 151 | ||||
| -rw-r--r-- | manifests/requirements.pp | 67 | ||||
| -rw-r--r-- | manifests/virtualenv.pp | 114 |
7 files changed, 358 insertions, 103 deletions
diff --git a/manifests/config.pp b/manifests/config.pp index 49e5230..c7196f2 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -1,3 +1,18 @@ +# == Define: python::config +# +# Optionally installs the gunicorn service +# +# === Examples +# +# include python::config +# +# === Authors +# +# Sergey Stankevich +# Ashley Penney +# Fotis Gimian +# + class python::config { Class['python::install'] -> Python::Pip <| |> @@ -6,17 +21,19 @@ class python::config { Python::Virtualenv <| |> -> Python::Pip <| |> - if $python::gunicorn { - Class['python::install'] -> Python::Gunicorn <| |> + if $python::manage_gunicorn { + if $python::gunicorn { + Class['python::install'] -> Python::Gunicorn <| |> - Python::Gunicorn <| |> ~> Service['gunicorn'] + Python::Gunicorn <| |> ~> Service['gunicorn'] - service { 'gunicorn': - ensure => running, - enable => true, - hasrestart => true, - hasstatus => false, - pattern => '/usr/bin/gunicorn', + service { 'gunicorn': + ensure => running, + enable => true, + hasrestart => true, + hasstatus => false, + pattern => '/usr/bin/gunicorn', + } } } diff --git a/manifests/gunicorn.pp b/manifests/gunicorn.pp index 13f4872..159afa3 100644 --- a/manifests/gunicorn.pp +++ b/manifests/gunicorn.pp @@ -37,6 +37,8 @@ # dir => '/var/www/project1/current', # bind => 'unix:/tmp/gunicorn.socket', # environment => 'prod', +# owner => 'www-data', +# group => 'www-data', # template => 'python/gunicorn.erb', # } # @@ -53,6 +55,8 @@ define python::gunicorn ( $dir = false, $bind = false, $environment = false, + $owner = 'www-data', + $group = 'www-data', $template = 'python/gunicorn.erb', ) { diff --git a/manifests/init.pp b/manifests/init.pp index 8e8de9e..65a7e66 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -7,19 +7,27 @@ # [*version*] # Python version to install. Default: system default # +# [*pip*] +# Install python-pip. Default: false +# # [*dev*] # Install python-dev. Default: false # # [*virtualenv*] -# Install python-virtualenv. Default: false +# Install python-virtualenv. Default: false, also accepts 'pip' which will +# install latest virtualenv from pip rather than package manager # # [*gunicorn*] # Install Gunicorn. Default: false # +# [*manage_gunicorn*] +# Allow Installation / Removal of Gunicorn. Default: true +# # === Examples # # class { 'python': # version => 'system', +# pip => true, # dev => true, # virtualenv => true, # gunicorn => true, @@ -30,21 +38,25 @@ # Sergey Stankevich # class python ( - $version = 'system', - $dev = false, - $virtualenv = false, - $gunicorn = false + $version = 'system', + $pip = false, + $dev = false, + $virtualenv = false, + $gunicorn = false, + $manage_gunicorn = true, + $provider = undef ) { # Module compatibility check - $compatible = [ 'Debian', 'Ubuntu', 'CentOS', 'RedHat' ] - if ! ($::operatingsystem in $compatible) { + $compatible = [ 'Debian', 'RedHat'] + if ! ($::osfamily in $compatible) { fail("Module is not compatible with ${::operatingsystem}") } - Class['python::install'] -> Class['python::config'] - - include python::install - include python::config + # Anchor pattern to contain dependencies + anchor { 'python::begin': } -> + class { 'python::install': } -> + class { 'python::config': } -> + anchor { 'python::end': } } diff --git a/manifests/install.pp b/manifests/install.pp index cf0a13e..a8d7d36 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -1,36 +1,68 @@ +# == Define: python::install +# +# Installs core python packages +# +# === Examples +# +# include python::install +# +# === Authors +# +# Sergey Stankevich +# Ashley Penney +# Fotis Gimian +# + class python::install { $python = $python::version ? { 'system' => 'python', + 'pypy' => 'pypy', default => "python${python::version}", } - $pythondev = $::operatingsystem ? { - /(?i:RedHat|CentOS|Fedora)/ => "$python-devel", - /(?i:Debian|Ubuntu)/ => "$python-dev" + $pythondev = $::osfamily ? { + 'RedHat' => "${python}-devel", + 'Debian' => "${python}-dev" } - package { $python: ensure => present } - $dev_ensure = $python::dev ? { true => present, default => absent, } - package { [ $pythondev, 'python-pip' ]: ensure => $dev_ensure } - - $venv_ensure = $python::virtualenv ? { + $pip_ensure = $python::pip ? { true => present, default => absent, } - package { 'python-virtualenv': ensure => $venv_ensure } - - $gunicorn_ensure = $python::gunicorn ? { + $venv_ensure = $python::virtualenv ? { true => present, default => absent, } - package { 'gunicorn': ensure => $gunicorn_ensure } + # Install latest from pip if pip is the provider + case $python::provider { + pip: { + package { 'virtualenv': ensure => latest, provider => pip } + package { 'pip': ensure => latest, provider => pip } + package { $pythondev: ensure => latest } + package { "python==${python::version}": ensure => latest, provider => pip } + } + default: { + package { 'python-virtualenv': ensure => $venv_ensure } + package { 'python-pip': ensure => $pip_ensure } + package { $pythondev: ensure => $dev_ensure } + package { $python: ensure => present } + } + } + + if $python::manage_gunicorn { + $gunicorn_ensure = $python::gunicorn ? { + true => present, + default => absent, + } + package { 'gunicorn': ensure => $gunicorn_ensure } + } } diff --git a/manifests/pip.pp b/manifests/pip.pp index 3a8ec1b..dd2be0d 100644 --- a/manifests/pip.pp +++ b/manifests/pip.pp @@ -4,6 +4,12 @@ # # === Parameters # +# [*name] +# must be unique +# +# [*pkgname] +# name of the package. If pkgname is not specified, use name (title) instead. +# # [*ensure*] # present|absent. Default: present # @@ -13,9 +19,29 @@ # [*url*] # URL to install from. Default: none # +# [*owner*] +# The owner of the virtualenv being manipulated. Default: root +# # [*proxy*] # Proxy server to use for outbound connections. Default: none # +# [*editable*] +# Boolean. If true the package is installed as an editable resource. +# +# [*environment*] +# Additional environment variables required to install the packages. Default: none +# +# [*timeout*] +# The maximum time in seconds the "pip install" command should take. Default: 1800 +# +# [*install_args*] +# String. Any additional installation arguments that will be supplied +# when running pip install. +# +# [*uninstall args*] +# String. Any additional arguments that will be supplied when running +# pip uninstall. +# # === Examples # # python::pip { 'flask': @@ -26,12 +52,21 @@ # === Authors # # Sergey Stankevich +# Fotis Gimian # define python::pip ( - $virtualenv, - $ensure = present, - $url = false, - $proxy = false + $pkgname = $name, + $ensure = present, + $virtualenv = 'system', + $url = false, + $owner = 'root', + $proxy = false, + $egg = false, + $editable = false, + $environment = [], + $install_args = '', + $uninstall_args = '', + $timeout = 1800, ) { # Parameter validation @@ -39,33 +74,123 @@ define python::pip ( fail('python::pip: virtualenv parameter must not be empty') } + if $virtualenv == 'system' and $owner != 'root' { + fail('python::pip: root user must be used when virtualenv is system') + } + + $cwd = $virtualenv ? { + 'system' => '/', + default => $virtualenv, + } + + $pip_env = $virtualenv ? { + 'system' => 'pip', + default => "${virtualenv}/bin/pip", + } + $proxy_flag = $proxy ? { false => '', default => "--proxy=${proxy}", } - $grep_regex = $name ? { - /==/ => "^${name}\$", - default => "^${name}==", + if $editable == true { + $install_editable = ' -e ' + } + else { + $install_editable = '' + } + + #TODO: Do more robust argument checking, but below is a start + if ($ensure == absent) and ($install_args != '') { + fail('python::pip cannot provide install_args with ensure => absent') + } + + if ($ensure == present) and ($uninstall_args != '') { + fail('python::pip cannot provide uninstall_args with ensure => present') + } + + # Check if searching by explicit version. + if $ensure =~ /^((19|20)[0-9][0-9]-(0[1-9]|1[1-2])-([0-2][1-9]|3[0-1])|[0-9]+\.[0-9]+(\.[0-9]+)?)$/ { + $grep_regex = "^${pkgname}==${ensure}\$" + } else { + $grep_regex = $pkgname ? { + /==/ => "^${pkgname}\$", + default => "^${pkgname}==", + } + } + + $egg_name = $egg ? { + false => $pkgname, + default => $egg } $source = $url ? { - false => $name, - default => "${url}#egg=${name}", + false => $pkgname, + default => "${url}#egg=${egg_name}", } + # We need to jump through hoops to make sure we issue the correct pip command + # depending on wheel support and versions. + # + # Pip does not support wheels prior to version 1.4.0 + # Pip wheels require setuptools/distribute > 0.8 + # Python 2.6 and older does not support setuptools/distribute > 0.8 + # Pip >= 1.5 tries to use wheels by default, even if wheel package is not + # installed, in this case the --no-use-wheel flag needs to be passed + # Versions prior to 1.5 don't support the --no-use-wheel flag + # + # To check for this we test for wheel parameter using help and then using + # version, this makes sure we only use wheels if they are supported and + # installed + + case $ensure { + /^((19|20)[0-9][0-9]-(0[1-9]|1[1-2])-([0-2][1-9]|3[0-1])|[0-9]+\.[0-9]+(\.[0-9]+)?)$/: { + # Version formats as per http://guide.python-distribute.org/specification.html#standard-versioning-schemes + # Explicit version. + exec { "pip_install_${name}": + command => "${pip_env} wheel --help > /dev/null 2>&1 && { ${pip_env} wheel --version > /dev/null 2>&1 || wheel_support_flag='--no-use-wheel'; } ; { ${pip_env} --log ${cwd}/pip.log install ${install_args} \$wheel_support_flag ${proxy_flag} ${install_args} ${install_editable} ${source}==${ensure} || ${pip_env} --log ${cwd}/pip.log install ${install_args} ${proxy_flag} ${install_args} ${install_editable} ${source}==${ensure} ;}", + unless => "${pip_env} freeze | grep -i -e ${grep_regex}", + user => $owner, + environment => $environment, + path => ['/usr/local/bin','/usr/bin','/bin', '/usr/sbin'], + timeout => $timeout, + } + } + present: { + # Whatever version is available. + exec { "pip_install_${name}": + command => "${pip_env} wheel --help > /dev/null 2>&1 && { ${pip_env} wheel --version > /dev/null 2>&1 || wheel_support_flag='--no-use-wheel'; } ; { ${pip_env} --log ${cwd}/pip.log install \$wheel_support_flag ${proxy_flag} ${install_args} ${install_editable} ${source} || ${pip_env} --log ${cwd}/pip.log install ${proxy_flag} ${install_args} ${install_editable} ${source} ;}", + unless => "${pip_env} freeze | grep -i -e ${grep_regex}", + user => $owner, + environment => $environment, + path => ['/usr/local/bin','/usr/bin','/bin', '/usr/sbin'], + timeout => $timeout, + } + } + + latest: { + # Latest version. exec { "pip_install_${name}": - command => "${virtualenv}/bin/pip install ${proxy_flag} ${source}", - unless => "${virtualenv}/bin/pip freeze | grep -i -e ${grep_regex}", + command => "${pip_env} wheel --help > /dev/null 2>&1 && { ${pip_env} wheel --version > /dev/null 2>&1 || wheel_support_flag='--no-use-wheel'; } ; { ${pip_env} --log ${cwd}/pip.log install --upgrade \$wheel_support_flag ${proxy_flag} ${uninstall_args} ${install_editable} ${source} || ${pip_env} --log ${cwd}/pip.log install --upgrade ${proxy_flag} ${uninstall_args} ${install_editable} ${source} ;}", + unless => "${pip_env} search ${source} | grep -i INSTALLED | grep -i latest", + user => $owner, + environment => $environment, + path => ['/usr/local/bin','/usr/bin','/bin', '/usr/sbin'], + timeout => $timeout, } } default: { + # Anti-action, uninstall. exec { "pip_uninstall_${name}": - command => "echo y | ${virtualenv}/bin/pip uninstall ${proxy_flag} ${name}", - onlyif => "${virtualenv}/bin/pip freeze | grep -i -e ${grep_regex}", + command => "echo y | ${pip_env} uninstall ${uninstall_args} ${proxy_flag}", + onlyif => "${pip_env} freeze | grep -i -e ${grep_regex}", + user => $owner, + environment => $environment, + path => ['/usr/local/bin','/usr/bin','/bin', '/usr/sbin'], + timeout => $timeout, } } } diff --git a/manifests/requirements.pp b/manifests/requirements.pp index 67d24ce..d9b6242 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -10,9 +10,31 @@ # [*virtualenv*] # virtualenv to run pip in. Default: system-wide # +# [*owner*] +# The owner of the virtualenv being manipulated. Default: root +# +# [*group*] +# The group relating to the virtualenv being manipulated. Default: root +# # [*proxy*] # Proxy server to use for outbound connections. Default: none # +# [*src*] +# Pip --src parameter; if the requirements file contains --editable resources, +# this parameter specifies where they will be installed. See the pip +# documentation for more. Default: none (i.e. use the pip default). +# +# [*environment*] +# Additional environment variables required to install the packages. Default: none +# +# [*forceupdate*] +# Run a pip install requirements even if we don't receive an event from the +# requirements file - Useful for when the requirements file is written as part of a +# resource other than file (E.g vcsrepo) +# +# [*cwd*] +# The directory from which to run the "pip install" command. Default: undef +# # === Examples # # python::requirements { '/var/www/project1/requirements.txt': @@ -24,22 +46,31 @@ # # Sergey Stankevich # Ashley Penney +# Fotis Gimian # define python::requirements ( $requirements = $name, $virtualenv = 'system', - $proxy = false, $owner = 'root', - $group = 'root' + $group = 'root', + $proxy = false, + $src = false, + $environment = [], + $forceupdate = false, + $cwd = undef, ) { - $cwd = $virtualenv ? { + if $virtualenv == 'system' and ($owner != 'root' or $group != 'root') { + fail('python::pip: root user must be used when virtualenv is system') + } + + $rootdir = $virtualenv ? { 'system' => '/', - default => "${virtualenv}/bin/", + default => $virtualenv, } $pip_env = $virtualenv ? { - 'system' => '`which pip`', + 'system' => 'pip', default => "${virtualenv}/bin/pip", } @@ -48,7 +79,10 @@ define python::requirements ( default => "--proxy=${proxy}", } - $req_crc = "${requirements}.sha1" + $src_flag = $src ? { + false => '', + default => "--src=${src}", + } # This will ensure multiple python::virtualenv definitions can share the # the same requirements file. @@ -58,28 +92,21 @@ define python::requirements ( mode => '0644', owner => $owner, group => $group, + audit => content, replace => false, content => '# Puppet will install and/or update pip packages listed here', } } - # SHA1 checksum to detect changes - exec { "python_requirements_check_${name}": - provider => shell, - command => "sha1sum ${requirements} > ${req_crc}", - unless => "sha1sum -c ${req_crc}", - user => $owner, - require => File[$requirements], - } - - exec { "python_requirements_update_${name}": + exec { "python_requirements${name}": provider => shell, - command => "${pip_env} install ${proxy_flag} -Ur ${requirements}", - cwd => $cwd, - refreshonly => true, + command => "${pip_env} --log ${rootdir}/pip.log install ${proxy_flag} ${src_flag} -r ${requirements}", + refreshonly => !$forceupdate, timeout => 1800, + cwd => $cwd, user => $owner, - subscribe => Exec["python_requirements_check_${name}"], + subscribe => File[$requirements], + environment => $environment, } } diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index 9f9e96c..d3bc3b5 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -13,15 +13,39 @@ # [*requirements*] # Path to pip requirements.txt file. Default: none # -# [*proxy*] -# Proxy server to use for outbound connections. Default: none -# # [*systempkgs*] # Copy system site-packages into virtualenv. Default: don't +# If virtualenv version < 1.7 this flag has no effect since +# [*venv_dir*] +# Directory to install virtualenv to. Default: $name # # [*distribute*] # Include distribute in the virtualenv. Default: true # +# [*index*] +# Base URL of Python package index. Default: none (http://pypi.python.org/simple/) +# +# [*owner*] +# The owner of the virtualenv being manipulated. Default: root +# +# [*group*] +# The group relating to the virtualenv being manipulated. Default: root +# +# [*proxy*] +# Proxy server to use for outbound connections. Default: none +# +# [*environment*] +# Additional environment variables required to install the packages. Default: none +# +# [*path*] +# Specifies the PATH variable. Default: [ '/bin', '/usr/bin', '/usr/sbin' ] +# +# [*cwd*] +# The directory from which to run the "pip install" command. Default: undef +# +# [*timeout*] +# The maximum time in seconds the "pip install" command should take. Default: 1800 +# # === Examples # # python::virtualenv { '/var/www/project1': @@ -38,25 +62,30 @@ # Sergey Stankevich # Ashley Penney # Marc Fournier +# Fotis Gimian # define python::virtualenv ( - $ensure = present, - $version = 'system', - $requirements = false, - $proxy = false, - $systempkgs = false, - $distribute = true, - $owner = 'root', - $group = 'root', - $index = false, + $ensure = present, + $version = 'system', + $requirements = false, + $systempkgs = false, + $venv_dir = $name, + $distribute = true, + $index = false, + $owner = 'root', + $group = 'root', + $proxy = false, + $environment = [], + $path = [ '/bin', '/usr/bin', '/usr/sbin' ], + $cwd = undef, + $timeout = 1800 ) { - $venv_dir = $name - if $ensure == 'present' { $python = $version ? { 'system' => 'python', + 'pypy' => 'pypy', default => "python${version}", } @@ -70,46 +99,57 @@ define python::virtualenv ( default => "&& export http_proxy=${proxy}", } - $system_pkgs_flag = $systempkgs ? { - false => '', - default => '--system-site-packages', + # Virtualenv versions prior to 1.7 do not support the + # --system-site-packages flag, default off for prior versions + # Prior to version 1.7 the default was equal to --system-site-packages + # and the flag --no-site-packages had to be passed to do the opposite + if (( versioncmp($::virtualenv_version,'1.7') > 0 ) and ( $systempkgs == true )) { + $system_pkgs_flag = '--system-site-packages' + } elsif (( versioncmp($::virtualenv_version,'1.7') < 0 ) and ( $systempkgs == false )) { + $system_pkgs_flag = '--no-site-packages' + } else { + $system_pkgs_flag = $systempkgs ? { + true => '--system-site-packages', + false => '--no-site-packages', + default => fail('Invalid value for systempkgs. Boolean value is expected') + } } $distribute_pkg = $distribute ? { true => 'distribute', - default => '', + default => 'setuptools', } $pypi_index = $index ? { false => '', default => "-i ${index}", } + # Python 2.6 and older does not support setuptools/distribute > 0.8 which + # is required for pip wheel support, pip therefor requires --no-use-wheel flag + # if the # pip version is more recent than 1.4.1 but using an old python or + # setuputils/distribute version + # To check for this we test for wheel parameter using help and then using + # version, this makes sure we only use wheels if they are supported exec { "python_virtualenv_${venv_dir}": - command => "mkdir -p ${venv_dir} \ - ${proxy_command} \ - && virtualenv -p `which ${python}` ${system_pkgs_flag} ${venv_dir} \ - && ${venv_dir}/bin/pip install ${pypi_index} ${proxy_flag} --upgrade ${distribute_pkg} pip", - user => $owner, - creates => $venv_dir, - path => [ '/bin', '/usr/bin', '/usr/sbin' ], - } - - file{$venv_dir: - ensure => directory, - owner => $owner, - group => $group, - recurse => true, - require => Exec["python_virtualenv_${venv_dir}"], + command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv ${system_pkgs_flag} -p ${python} ${venv_dir} && ${venv_dir}/bin/pip wheel --help > /dev/null 2>&1 && { ${venv_dir}/bin/pip wheel --version > /dev/null 2>&1 || wheel_support_flag='--no-use-wheel'; } ; { ${venv_dir}/bin/pip --log ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} \$wheel_support_flag --upgrade pip ${distribute_pkg} || ${venv_dir}/bin/pip --log ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} --upgrade pip ${distribute_pkg} ;}", + user => $owner, + creates => "${venv_dir}/bin/activate", + path => $path, + cwd => '/tmp', + environment => $environment, + unless => "grep '^[\\t ]*VIRTUAL_ENV=[\\\\'\\\"]*${venv_dir}[\\\"\\\\'][\\t ]*$' ${venv_dir}/bin/activate", #Unless activate exists and VIRTUAL_ENV is correct we re-create the virtualenv } if $requirements { exec { "python_requirements_initial_install_${requirements}_${venv_dir}": - command => "${venv_dir}/bin/pip install ${pypi_index} ${proxy_flag} --requirement ${requirements}", + command => "${venv_dir}/bin/pip wheel --help > /dev/null 2>&1 && { ${venv_dir}/bin/pip wheel --version > /dev/null 2>&1 || wheel_support_flag='--no-use-wheel'; } ; ${venv_dir}/bin/pip --log ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} \$wheel_support_flag -r ${requirements}", refreshonly => true, - timeout => 1800, + timeout => $timeout, user => $owner, subscribe => Exec["python_virtualenv_${venv_dir}"], + environment => $environment, + cwd => $cwd } python::requirements { "${requirements}_${venv_dir}": @@ -118,10 +158,10 @@ define python::virtualenv ( proxy => $proxy, owner => $owner, group => $group, + cwd => $cwd, require => Exec["python_virtualenv_${venv_dir}"], } } - } elsif $ensure == 'absent' { file { $venv_dir: @@ -130,7 +170,5 @@ define python::virtualenv ( recurse => true, purge => true, } - } - } |