From ee1fe7a0236e5fd100bbc229ea034bf7d2b530aa Mon Sep 17 00:00:00 2001 From: "R.I.Pienaar" Date: Thu, 6 May 2010 22:55:02 +0100 Subject: Pull in work from private Subversion repository --- README | 49 ++++++++++++++++ files/concatfragments.sh | 123 +++++++++++++++++++++++++++++++++++++++ files/null/.gitignore | 0 manifests/fragment.pp | 48 ++++++++++++++++ manifests/init.pp | 147 +++++++++++++++++++++++++++++++++++++++++++++++ manifests/setup.pp | 36 ++++++++++++ 6 files changed, 403 insertions(+) create mode 100644 README create mode 100755 files/concatfragments.sh create mode 100644 files/null/.gitignore create mode 100755 manifests/fragment.pp create mode 100755 manifests/init.pp create mode 100755 manifests/setup.pp diff --git a/README b/README new file mode 100644 index 0000000..c5ab5f2 --- /dev/null +++ b/README @@ -0,0 +1,49 @@ +A Puppet module that can construct files from fragments. + +Please see the comments in the various .pp files for details +as well as posts on my blog at www.devco.net + +Released under the Apache 2.0 licence + +KNOWN ISSUES: +- In 0.24.8 you will see inintended notifies, if you build a file + in a run, the next run will also see it as changed. This is due + to how 0.24.8 does the purging of unhandled files, this is improved + in 0.25.x and we cannot work around it in our code. + +- At present you cannot change the ownership of the file by just setting + the property on the resources, we have a work around in mind. + +CHANGELOG: +- 2010/02/19 - initial release +- 2010/03/12 - add support for 0.24.8 and newer + - make the location of sort configurable + - add the ability to add shell comment based warnings to + top of files + - add the ablity to create empty files +- 2010/04/05 - fix parsing of WARN and change code style to match rest + of the code + - Better and safer boolean handling for warn and force + - Don't use hard coded paths in the shell script, set PATH + top of the script + - Use file{} to copy the result and make all fragments owned + by root. This means we can chnage the ownership/group of the + resulting file at any time. + - You can specify ensure => "/some/other/file" in concat::fragment + to include the contents of a symlink into the final file. +- 2010/04/16 - Add more cleaning of the fragment name - removing / from the $name + +CONTRIBUTORS: +Paul Elliot - Provided 0.24.8 support, shell warnings and empty file + creation support. +Chad Netzer - Various patches to improve safety of file operations + - Symlink support +David Schmitt - Patch to remove hard coded paths relying on OS path + - Patch to use file{} to copy the resulting file to the + final destination. This means Puppet client will show + diffs and that hopefully we can change file ownerships + now + + +CONTACT: +R.I.Pienaar - rip@devco.net / www.devco.net / @ripienaar diff --git a/files/concatfragments.sh b/files/concatfragments.sh new file mode 100755 index 0000000..570853e --- /dev/null +++ b/files/concatfragments.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +# Script to concat files to a config file. +# +# Given a directory like this: +# /path/to/conf.d +# |-- fragments +# | |-- 00_named.conf +# | |-- 10_domain.net +# | `-- zz_footer +# +# The script supports a test option that will build the concat file to a temp location and +# use /usr/bin/cmp to verify if it should be run or not. This would result in the concat happening +# twice on each run but gives you the option to have an unless option in your execs to inhibit rebuilds. +# +# Without the test option and the unless combo your services that depend on the final file would end up +# restarting on each run, or in other manifest models some changes might get missed. +# +# OPTIONS: +# -o The file to create from the sources +# -d The directory where the fragments are kept +# -t Test to find out if a build is needed, basically concats the files to a temp +# location and compare with what's in the final location, return codes are designed +# for use with unless on an exec resource +# -w Add a shell style comment at the top of the created file to warn users that it +# is generated by puppet +# -f Enables the creation of empty output files when no fragments are found +# -n Sort the output numerically rather than the default alpha sort +# +# the command: +# +# concatfragments.sh -o /path/to/conffile.cfg -d /path/to/conf.d +# +# creates /path/to/conf.d/fragments.concat and copies the resulting +# file to /path/to/conffile.cfg. The files will be sorted alphabetically +# pass the -n switch to sort numerically. +# +# The script does error checking on the various dirs and files to make +# sure things don't fail. + +OUTFILE="" +WORKDIR="" +TEST="" +FORCE="" +WARN="" +SORTARG="-z" + +PATH=/sbin:/usr/sbin:/bin:/usr/bin + +while getopts "o:s:d:tnwf" options; do + case $options in + o ) OUTFILE=$OPTARG;; + d ) WORKDIR=$OPTARG;; + n ) SORTARG="-zn";; + w ) WARN="true";; + f ) FORCE="true";; + t ) TEST="true";; + * ) echo "Specify output file with -o and fragments directory with -d" + exit 1;; + esac +done + +# do we have -o? +if [ x${OUTFILE} = "x" ]; then + echo "Please specify an output file with -o" + exit 1 +fi + +# do we have -d? +if [ x${WORKDIR} = "x" ]; then + echo "Please fragments directory with -d" + exit 1 +fi + +# can we write to -o? +if [ -a ${OUTFILE} ]; then + if [ ! -w ${OUTFILE} ]; then + echo "Cannot write to ${OUTFILE}" + exit 1 + fi +else + if [ ! -w `dirname ${OUTFILE}` ]; then + echo "Cannot write to `dirname ${OUTFILE}` to create ${OUTFILE}" + exit 1 + fi +fi + +# do we have a fragments subdir inside the work dir? +if [ ! -d "${WORKDIR}/fragments" ] && [ ! -x "${WORKDIR}/fragments" ]; then + echo "Cannot access the fragments directory" + exit 1 +fi + +# are there actually any fragments? +if [ ! "$(ls -A ${WORKDIR}/fragments)" ]; then + if [ x${FORCE} = "x" ]; then + echo "The fragments directory is empty, cowardly refusing to make empty config files" + exit 1 + fi +fi + +cd ${WORKDIR} + +if [ x${WARN} = "x" ]; then + cat /dev/null > "fragments.concat" +else + echo '# This file is managed by Puppet. DO NOT EDIT.' > "fragments.concat" +fi + +# find all the files in the fragments directory, sort them numerically and concat to fragments.concat in the working dir +find fragments/ -type f -follow -print0 |sort ${SORTARG}|xargs -0 cat >>"fragments.concat" + +if [ x${TEST} = "x" ]; then + # This is a real run, copy the file to outfile + cp fragments.concat ${OUTFILE} + RETVAL=$? +else + # Just compare the result to outfile to help the exec decide + cmp ${OUTFILE} fragments.concat + RETVAL=$? +fi + +exit $RETVAL diff --git a/files/null/.gitignore b/files/null/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/manifests/fragment.pp b/manifests/fragment.pp new file mode 100755 index 0000000..7950593 --- /dev/null +++ b/manifests/fragment.pp @@ -0,0 +1,48 @@ +# Puts a file fragment into a directory previous setup using concat +# +# OPTIONS: +# - target The file that these fragments belong to +# - content If present puts the content into the file +# - source If content was not specified, use the source +# - order By default all files gets a 10_ prefix in the directory +# you can set it to anything else using this to influence the +# order of the content in the file +# - ensure Present/Absent +# - mode Mode for the file +# - owner Owner of the file +# - group Owner of the file +define concat::fragment($target, $content='', $source='', $order=10, $ensure = "present", $mode = 0644, $owner = root, $group = root) { + $safe_name = regsubst($name, '/', '_', 'G') + $safe_target_name = regsubst($target, '/', '_', 'G') + $concatdir = $concat::setup::concatdir + $fragdir = "${concatdir}/${safe_target_name}" + + # if content is passed, use that, else if source is passed use that + # if neither passed, but $ensure is in symlink form, make a symlink + case $content { + "": { + case $source { + "": { + case $ensure { + "", "absent", "present", "file", "directory": { + crit("No content or source specified") + } + } + } + default: { File{ source => $source } } + } + } + default: { File{ content => $content } } + } + + file{"${fragdir}/fragments/${order}_${safe_name}": + mode => $mode, + owner => $owner, + group => $group, + ensure => $ensure, + alias => "concat_fragment_${name}", + notify => Exec["concat_${target}"] + } +} + +# vi:tabstop=4:expandtab:ai diff --git a/manifests/init.pp b/manifests/init.pp new file mode 100755 index 0000000..59ca5aa --- /dev/null +++ b/manifests/init.pp @@ -0,0 +1,147 @@ +# A system to construct files using fragments from other files or templates. +# +# This requires at least puppet 0.25 to work correctly as we use some +# enhancements in recursive directory management and regular expressions +# to do the work here. +# +# USAGE: +# The basic use case is as below: +# +# concat{"/etc/named.conf": +# notify => Service["named"] +# } +# +# concat::fragment{"foo.com_config": +# target => "/etc/named.conf", +# order => 10, +# content => template("named_conf_zone.erb") +# } +# +# This will use the template named_conf_zone.erb to build a single +# bit of config up and put it into the fragments dir. The file +# will have an number prefix of 10, you can use the order option +# to control that and thus control the order the final file gets built in. +# +# SETUP: +# The class concat::setup defines a variable $concatdir - you should set this +# to a directory where you want all the temporary files and fragments to be +# stored. Avoid placing this somewhere like /tmp since you should never +# delete files here, puppet will manage them. +# +# If you are on version 0.24.8 or newer you can set $puppetversion to 24 in +# concat::setup to enable a compatible mode, else just leave it on 25 +# +# Before you can use any of the concat features you should include the +# class concat::setup somewhere on your node first. +# +# DETAIL: +# We use a helper shell script called concatfragments.sh that gets placed +# in /usr/local/bin to do the concatenation. While this might seem more +# complex than some of the one-liner alternatives you might find on the net +# we do a lot of error checking and safety checks in the script to avoid +# problems that might be caused by complex escaping errors etc. +# +# LICENSE: +# Apache Version 2 +# +# HISTORY: +# 2010/02/19 - First release based on earlier concat_snippets work +# +# CONTACT: +# R.I.Pienaar +# Volcane on freenode +# @ripienaar on twitter +# www.devco.net + + +# Sets up so that you can use fragments to build a final config file, +# +# OPTIONS: +# - mode The mode of the final file +# - owner Who will own the file +# - group Who will own the file +# - force Enables creating empty files if no fragments are present +# - warn Adds a normal shell style comment top of the file indicating +# that it is built by puppet +# +# ACTIONS: +# - Creates fragment directories if it didn't exist already +# - Executes the concatfragments.sh script to build the final file, this script will create +# directory/fragments.concat and copy it to the final destination. Execution happens only when: +# * The directory changes +# * fragments.concat != final destination, this means rebuilds will happen whenever +# someone changes or deletes the final file. Checking is done using /usr/bin/cmp. +# * The Exec gets notified by something else - like the concat::fragment define +# - Defines a File resource to ensure $mode is set correctly but also to provide another +# means of requiring +# +# ALIASES: +# - The exec can notified using Exec["concat_/path/to/file"] or Exec["concat_/path/to/directory"] +# - The final file can be referened as File["/path/to/file"] or File["concat_/path/to/file"] +define concat($mode = 0644, $owner = "root", $group = "root", $warn = "false", $force = "false") { + $safe_name = regsubst($name, '/', '_', 'G') + $concatdir = $concat::setup::concatdir + $version = $concat::setup::majorversion + $fragdir = "${concatdir}/${safe_name}" + $concat_name = "fragments.concat.out" + + case $warn { + 'true',true,yes,on: { $warnflag = "-w" } + 'false',false,no,off: { $warnflag = "" } + default: { fail("Improper 'warn' value given to concat: $warn") } + } + + case $force { + 'true',true,yes,on: { $forceflag = "-f" } + 'false',false,no,off: { $forceflag = "" } + default: { fail("Improper 'force' value given to concat: $force") } + } + + File{ + owner => root, + group => root, + mode => $mode, + } + + file{$fragdir: + ensure => directory; + + "${fragdir}/fragments": + ensure => directory, + recurse => true, + purge => true, + force => true, + ignore => [".svn", ".git", ".gitignore"], + source => $version ? { + 24 => "puppet:///concat/null", + default => undef, + }, + notify => Exec["concat_${name}"]; + + "${fragdir}/fragments.concat": + ensure => present; + + "${fragdir}/${concat_name}": + ensure => present; + + $name: + source => "${fragdir}/${concat_name}", + owner => $owner, + group => $group, + checksum => md5, + mode => $mode, + ensure => present, + alias => "concat_${name}"; + } + + exec{"concat_${name}": + user => root, + group => root, + notify => File[$name], + subscribe => File[$fragdir], + alias => "concat_${fragdir}", + require => [ File["/usr/local/bin/concatfragments.sh"], File[$fragdir], File["${fragdir}/fragments"], File["${fragdir}/fragments.concat"] ], + unless => "/usr/local/bin/concatfragments.sh -o ${fragdir}/${concat_name} -d ${fragdir} -t ${warnflag} ${forceflag}", + command => "/usr/local/bin/concatfragments.sh -o ${fragdir}/${concat_name} -d ${fragdir} ${warnflag} ${forceflag}", + } +} diff --git a/manifests/setup.pp b/manifests/setup.pp new file mode 100755 index 0000000..9676fb6 --- /dev/null +++ b/manifests/setup.pp @@ -0,0 +1,36 @@ +# Sets up the concat system. +# +# $concatdir should point to a place where you wish the fragments to +# live. This should not be somewhere like /tmp since ideally these files +# should not be deleted ever, puppet should always manage them +# +# $puppetversion should be either 24 or 25 to enable a 24 compatible +# mode, in 24 mode you might see phantom notifies this is a side effect +# of the method we use to clear the fragments directory. +# +# The regular expression below will try to figure out your puppet version +# but this code will only work in 0.24.8 and newer. +# +# It also copies out the concatfragments.sh file to /usr/local/bin +class concat::setup { + $concatdir = "/var/lib/puppet/concat" + $majorversion = regsubst($puppetversion, '^[0-9]+[.]([0-9]+)[.][0-9]+$', '\1') + + file{"/usr/local/bin/concatfragments.sh": + owner => root, + group => root, + mode => 755, + source => $majorversion ? { + 24 => "puppet:///concat/concatfragments.sh", + default => "puppet:///modules/concat/concatfragments.sh" + }; + + $concatdir: + ensure => directory, + owner => root, + group => root, + mode => 755; + } +} + +# vi:tabstop=4:expandtab:ai -- cgit v1.2.3