Merge pull request #20 from fgimian/master

Stability & performance improvements along with a few new features (take what you wish)

7 files changed, 165 insertions, 68 deletions

diff --git a/Modulefile b/Modulefile
index c7f2f97..3fe4033 100644
--- a/Modulefile
+++ b/Modulefile
@@ -1,9 +1,9 @@
 name         'puppet-python'
-version      '1.0.0'
+version      '1.1.4'
 
-author       'Sergey Stankevich'
+author       'Fotis Gimian'
 license      'Apache License, Version 2.0'
-project_page 'https://github.com/stankevich/puppet-python'
-source       'git://github.com/stankevich/puppet-python'
+project_page 'https://github.com/fgimian/puppet-python'
+source       'git://github.com/fgimian/puppet-python'
 summary      'Puppet module for Python'
 description  'Installs and manages Python, pip, virtualenv, Gunicorn virtual hosts.'
diff --git a/README.md b/README.md
index 5a4c8b5..9e35e36 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,27 @@
-[puppet-python](https://github.com/stankevich/puppet-python)
+[puppet-python](https://github.com/fgimian/puppet-python)
 ======
 
-Puppet module for installing and managing python, pip, virtualenvs and Gunicorn virtual hosts.
+This project was forked from [stankevich's Github page](https://github.com/stankevich/puppet-python)
+
+Puppet module for installing and managing python, pip, virtualenvs and Gunicorn virtual hosts.  This fork makes several fundamental changes to the core of this module, adding some additional features, improving performance and making operations more robust in general.
+
+**Version 1.1.x Notes (fgimian fork)**
+
+Version 1.1.x makes several fundamental changes to the core of this module, adding some additional features, improving performance and making operations more robust in general.
+
+Please note that everal changes have been made in v1.1.x which make manifests incompatible with the previous version.  However, modifying your manifests to suit is trivial.  Please see the notes below.
+
+Currently, the changes you need to make are as follows:
+
+* All pip definitions MUST include the owner field which specifies which user owns the virtualenv that packages will be installed in.  Adding this greatly improves performance and efficiency of this module.
+* You must explicitly specify pip => true in the python class if you want pip installed.  As such, the pip package is now independent of the dev package and so one can exist without the other.
+
+## Installation
+
+``` bash
+cd /etc/puppet/modules
+git clone git://github.com/fgimian/puppet-python.git python
+```
 
 ## Usage
 
@@ -9,13 +29,15 @@ Puppet module for installing and managing python, pip, virtualenvs and Gunicorn
 
 Installs and manages python, python-dev, python-virtualenv and Gunicorn.
 
-**version** — Python version to install. Default: system default
+**version** - Python version to install. Default: system default
 
-**dev** — Install python-dev. Default: false
+**pip** - Install python-pip. Default: false
 
-**virtualenv** — Install python-virtualenv. Default: false
+**dev** - Install python-dev. Default: false
 
-**gunicorn** — Install Gunicorn. Default: false
+**virtualenv** - Install python-virtualenv. Default: false
+
+**gunicorn** - Install Gunicorn. Default: false
 
 	class { 'python':
 	  version    => 'system',
@@ -28,52 +50,65 @@ Installs and manages python, python-dev, python-virtualenv and Gunicorn.
 
 Installs and manages packages from pip.
 
-**ensure** — present/absent. Default: present
+**ensure** - present/absent. Default: present
 
-**virtualenv** — virtualenv to run pip in.
+**virtualenv** - virtualenv to run pip in. Default: system (no virtualenv)
 
-**url** — URL to install from. Default: none
+**url** - URL to install from. Default: none
 
-**proxy** — Proxy server to use for outbound connections. Default: none
+**owner** - The owner of the virtualenv to ensure that packages are installed with the correct permissions (must be specified). Default: root
 
-	python::pip { 'flask':
-	  virtualenv => '/var/www/project1',
-	  proxy      => 'http://proxy.domain.com:3128',
+**proxy** - Proxy server to use for outbound connections. Default: none
+
+**environment** - Additional environment variables required to install the packages. Default: none
+
+	python::pip { 'cx_Oracle':
+	  virtualenv  => '/var/www/project1',
+	  owner       => 'appuser',
+	  proxy       => 'http://proxy.domain.com:3128',
+	  environment => 'ORACLE_HOME=/usr/lib/oracle/11.2/client64',
 	}
 
 ### python::requirements
 
 Installs and manages Python packages from requirements file.
 
-**virtualenv** — virtualenv to run pip in. Default: system-wide
+**virtualenv** - virtualenv to run pip in. Default: system-wide
+
+**proxy** - Proxy server to use for outbound connections. Default: none
+
+**owner** - The owner of the virtualenv to ensure that packages are installed with the correct permissions (must be specified). Default: root
 
-**proxy** — Proxy server to use for outbound connections. Default: none
+**group** - The group that was used to create the virtualenv.  This is used to create the requirements file with correct permissions if it's not present already.
 
 	python::requirements { '/var/www/project1/requirements.txt':
 	  virtualenv => '/var/www/project1',
 	  proxy      => 'http://proxy.domain.com:3128',
+	  owner      => 'appuser',
+	  group      => 'apps',
 	}
 
 ### python::virtualenv
 
 Creates Python virtualenv.
 
-**ensure** — present/absent. Default: present
+**ensure** - present/absent. Default: present
 
-**version** — Python version to use. Default: system default
+**version** - Python version to use. Default: system default
 
-**requirements** — Path to pip requirements.txt file. Default: none
+**requirements** - Path to pip requirements.txt file. Default: none
 
-**proxy** — Proxy server to use for outbound connections. Default: none
+**proxy** - Proxy server to use for outbound connections. Default: none
 
-**systempkgs** — Copy system site-packages into virtualenv. Default: don't
+**systempkgs** - Copy system site-packages into virtualenv. Default: don't
 
-**distribute** — Include distribute in the virtualenv. Default: true
+**distribute** - Include distribute in the virtualenv. Default: true
 
-** owner ** - specify the owner of this virtualenv
+**owner** - Specify the owner of this virtualenv
 
-** group ** - specify the group for this virtualenv
+**group** - Specify the group for this virtualenv
 
+**index** - Base URL of Python package index. Default: none
 
 	python::virtualenv { '/var/www/project1':
 	  ensure       => present,
@@ -90,19 +125,19 @@ Creates Python virtualenv.
 
 Manages Gunicorn virtual hosts.
 
-**ensure** — present/absent. Default: present
+**ensure** - present/absent. Default: present
 
-**virtualenv** — Run in virtualenv, specify directory. Default: disabled
+**virtualenv** - Run in virtualenv, specify directory. Default: disabled
 
-**mode** — Gunicorn mode. wsgi/django. Default: wsgi
+**mode** - Gunicorn mode. wsgi/django. Default: wsgi
 
-**dir** — Application directory.
+**dir** - Application directory.
 
-**bind** — Bind on: 'HOST', 'HOST:PORT', 'unix:PATH'. Default: unix:/tmp/gunicorn-$name.socket or unix:${virtualenv}/${name}.socket
+**bind** - Bind on: 'HOST', 'HOST:PORT', 'unix:PATH'. Default: unix:/tmp/gunicorn-$name.socket or unix:${virtualenv}/${name}.socket
 
-**environment** — Set ENVIRONMENT variable. Default: none
+**environment** - Set ENVIRONMENT variable. Default: none
 
-**template** — Which ERB template to use. Default: python/gunicorn.erb
+**template** - Which ERB template to use. Default: python/gunicorn.erb
 
 	python::gunicorn { 'vhost':
 	  ensure      => present,
@@ -119,3 +154,4 @@ Manages Gunicorn virtual hosts.
 [Sergey Stankevich](https://github.com/stankevich)
 [Ashley Penney](https://github.com/apenney)
 [Marc Fournier](https://github.com/mfournier)
+[Fotis Gimian](https://github.com/fgimian)
diff --git a/manifests/init.pp b/manifests/init.pp
index 8e8de9e..2a9a44c 100644
--- a/manifests/init.pp
+++ b/manifests/init.pp
@@ -7,6 +7,9 @@
 # [*version*]
 #  Python version to install. Default: system default
 #
+# [*pip*]
+#  Install python-pip. Default: false
+#
 # [*dev*]
 #  Install python-dev. Default: false
 #
@@ -20,6 +23,7 @@
 #
 # class { 'python':
 #   version    => 'system',
+#   pip        => true,
 #   dev        => true,
 #   virtualenv => true,
 #   gunicorn   => true,
@@ -31,6 +35,7 @@
 #
 class python (
   $version    = 'system',
+  $pip        = false,
   $dev        = false,
   $virtualenv = false,
   $gunicorn   = false
diff --git a/manifests/install.pp b/manifests/install.pp
index cf0a13e..9306e3a 100644
--- a/manifests/install.pp
+++ b/manifests/install.pp
@@ -6,8 +6,8 @@ class python::install {
   }
 
   $pythondev = $::operatingsystem ? {
-    /(?i:RedHat|CentOS|Fedora)/ => "$python-devel",
-    /(?i:Debian|Ubuntu)/        => "$python-dev"
+    /(?i:RedHat|CentOS|Fedora)/ => "${python}-devel",
+    /(?i:Debian|Ubuntu)/        => "${python}-dev"
   }
 
   package { $python: ensure => present }
@@ -17,7 +17,13 @@ class python::install {
     default => absent,
   }
 
-  package { [ $pythondev, 'python-pip' ]: ensure => $dev_ensure }
+  $pip_ensure = $python::pip ? {
+    true    => present,
+    default => absent,
+  }
+
+  package { $pythondev: ensure => $dev_ensure }
+  package { 'python-pip': ensure => $pip_ensure }
 
   $venv_ensure = $python::virtualenv ? {
     true    => present,
diff --git a/manifests/pip.pp b/manifests/pip.pp
index 3a8ec1b..ddbcd6f 100644
--- a/manifests/pip.pp
+++ b/manifests/pip.pp
@@ -13,9 +13,15 @@
 # [*url*]
 #  URL to install from. Default: none
 #
+# [*owner*]
+#  The owner of the virtualenv being manipulated. Default: root
+#
 # [*proxy*]
 #  Proxy server to use for outbound connections. Default: none
 #
+# [*environment*]
+#  Additional environment variables required to install the packages. Default: none
+#
 # === Examples
 #
 # python::pip { 'flask':
@@ -26,12 +32,15 @@
 # === Authors
 #
 # Sergey Stankevich
+# Fotis Gimian
 #
 define python::pip (
-  $virtualenv,
-  $ensure = present,
-  $url    = false,
-  $proxy  = false
+  $ensure      = present,
+  $virtualenv  = 'system',
+  $url         = false,
+  $owner       = 'root',
+  $proxy       = false,
+  $environment = []
 ) {
 
   # Parameter validation
@@ -39,6 +48,20 @@ define python::pip (
     fail('python::pip: virtualenv parameter must not be empty')
   }
 
+  if $virtualenv == 'system' and $owner != 'root' {
+    fail('python::pip: root user must be used when virtualenv is system')
+  }
+
+  $cwd = $virtualenv ? {
+    'system' => '/',
+    default  => "${virtualenv}",
+  }
+
+  $pip_env = $virtualenv ? {
+    'system' => 'pip',
+    default  => "${virtualenv}/bin/pip",
+  }
+
   $proxy_flag = $proxy ? {
     false    => '',
     default  => "--proxy=${proxy}",
@@ -57,15 +80,19 @@ define python::pip (
   case $ensure {
     present: {
       exec { "pip_install_${name}":
-        command => "${virtualenv}/bin/pip install ${proxy_flag} ${source}",
-        unless  => "${virtualenv}/bin/pip freeze | grep -i -e ${grep_regex}",
+        command     => "$pip_env --log-file ${cwd}/pip.log install ${proxy_flag} ${source}",
+        unless      => "$pip_env freeze | grep -i -e ${grep_regex}",
+        user        => $owner,
+        environment => $environment,
       }
     }
 
     default: {
       exec { "pip_uninstall_${name}":
-        command => "echo y | ${virtualenv}/bin/pip uninstall ${proxy_flag} ${name}",
-        onlyif  => "${virtualenv}/bin/pip freeze | grep -i -e ${grep_regex}",
+        command     => "echo y | $pip_env uninstall ${proxy_flag} ${name}",
+        onlyif      => "$pip_env freeze | grep -i -e ${grep_regex}",
+        user        => $owner,
+        environment => $environment,
       }
     }
   }
diff --git a/manifests/requirements.pp b/manifests/requirements.pp
index 8b94839..67906fc 100644
--- a/manifests/requirements.pp
+++ b/manifests/requirements.pp
@@ -10,9 +10,18 @@
 # [*virtualenv*]
 #  virtualenv to run pip in. Default: system-wide
 #
+# [*owner*]
+#  The owner of the virtualenv being manipulated. Default: root
+#
+# [*group*]
+#  The group relating to the virtualenv being manipulated. Default: root
+#
 # [*proxy*]
 #  Proxy server to use for outbound connections. Default: none
 #
+# [*environment*]
+#  Additional environment variables required to install the packages. Default: none
+#
 # === Examples
 #
 # python::requirements { '/var/www/project1/requirements.txt':
@@ -24,22 +33,28 @@
 #
 # Sergey Stankevich
 # Ashley Penney
+# Fotis Gimian
 #
 define python::requirements (
   $requirements = $name,
   $virtualenv   = 'system',
-  $proxy        = false,
   $owner        = 'root',
-  $group        = 'root'
+  $group        = 'root',
+  $proxy        = false,
+  $environment = []
 ) {
 
+  if $virtualenv == 'system' and ($owner != 'root' or $group != 'root') {
+    fail('python::pip: root user must be used when virtualenv is system')
+  }
+
   $cwd = $virtualenv ? {
     'system' => '/',
-    default  => "${virtualenv}/bin/",
+    default  => "${virtualenv}",
   }
 
   $pip_env = $virtualenv ? {
-    'system' => '`which pip`',
+    'system' => 'pip',
     default  => "${virtualenv}/bin/pip",
   }
 
@@ -64,12 +79,12 @@ define python::requirements (
 
   exec { "python_requirements${name}":
     provider    => shell,
-    command     => "${pip_env} install ${proxy_flag} -r ${requirements}",
-    cwd         => $cwd,
+    command     => "${pip_env} --log-file ${cwd}/pip.log install ${proxy_flag} -r ${requirements}",
     refreshonly => true,
     timeout     => 1800,
     user        => $owner,
     subscribe   => File[$requirements],
+    environment => $environment,
   }
 
 }
diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp
index fc0f45b..354e4d8 100644
--- a/manifests/virtualenv.pp
+++ b/manifests/virtualenv.pp
@@ -13,15 +13,27 @@
 # [*requirements*]
 #  Path to pip requirements.txt file. Default: none
 #
-# [*proxy*]
-#  Proxy server to use for outbound connections. Default: none
-#
 # [*systempkgs*]
 #  Copy system site-packages into virtualenv. Default: don't
 #
 # [*distribute*]
 #  Include distribute in the virtualenv. Default: true
 #
+# [*index*]
+#  Base URL of Python package index. Default: none (http://pypi.python.org/simple/)
+#
+# [*owner*]
+#  The owner of the virtualenv being manipulated. Default: root
+#
+# [*group*]
+#  The group relating to the virtualenv being manipulated. Default: root
+#
+# [*proxy*]
+#  Proxy server to use for outbound connections. Default: none
+#
+# [*environment*]
+#  Additional environment variables required to install the packages. Default: none
+#
 # === Examples
 #
 # python::virtualenv { '/var/www/project1':
@@ -38,17 +50,19 @@
 # Sergey Stankevich
 # Ashley Penney
 # Marc Fournier
+# Fotis Gimian
 #
 define python::virtualenv (
   $ensure       = present,
   $version      = 'system',
   $requirements = false,
-  $proxy        = false,
   $systempkgs   = false,
   $distribute   = true,
+  $index        = false,
   $owner        = 'root',
   $group        = 'root',
-  $index        = false,
+  $proxy        = false,
+  $environment = []
 ) {
 
   $venv_dir = $name
@@ -84,29 +98,23 @@ define python::virtualenv (
         default => "-i ${index}",
     }
 
-
     exec { "python_virtualenv_${venv_dir}":
-      command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv -p `which ${python}` ${system_pkgs_flag} ${venv_dir} && ${venv_dir}/bin/pip install ${pypi_index} ${proxy_flag} --upgrade ${distribute_pkg} pip",
+      command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv ${system_pkgs_flag} ${venv_dir} && ${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} --upgrade ${distribute_pkg} pip",
       user    => $owner,
-      creates => $venv_dir,
+      creates => "${venv_dir}/bin/activate",
       path    => [ '/bin', '/usr/bin', '/usr/sbin' ],
-    }
-
-    file{$venv_dir:
-      ensure  => directory,
-      owner   => $owner,
-      group   => $group,
-      recurse => true,
-      require => Exec["python_virtualenv_${venv_dir}"],
+      cwd     => "/tmp",
+      environment => $environment,
     }
 
     if $requirements {
       exec { "python_requirements_initial_install_${requirements}_${venv_dir}":
-        command     => "${venv_dir}/bin/pip install ${pypi_index} ${proxy_flag} --requirement ${requirements}",
+        command     => "${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} -r ${requirements}",
         refreshonly => true,
         timeout     => 1800,
         user        => $owner,
         subscribe   => Exec["python_virtualenv_${venv_dir}"],
+        environment => $environment,
       }
 
       python::requirements { "${requirements}_${venv_dir}":