From 996f68542865b811b1b3ca93119f7ca8cfb73fa0 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 24 May 2016 10:19:21 -0400 Subject: Squashed 'puppet/modules/concat/' content from commit abce128 git-subtree-dir: puppet/modules/concat git-subtree-split: abce1280e07b544d8455f1572dd870bbd2f14892 --- CHANGELOG | 29 +++++++ LICENSE | 14 ++++ Modulefile | 8 ++ README.markdown | 112 +++++++++++++++++++++++++++ Rakefile | 13 ++++ files/concatfragments.sh | 129 +++++++++++++++++++++++++++++++ files/null/.gitignore | 0 lib/facter/concat_basedir.rb | 5 ++ manifests/fragment.pp | 49 ++++++++++++ manifests/init.pp | 178 +++++++++++++++++++++++++++++++++++++++++++ manifests/setup.pp | 49 ++++++++++++ spec/defines/init_spec.rb | 20 +++++ spec/spec_helper.rb | 9 +++ 13 files changed, 615 insertions(+) create mode 100644 CHANGELOG create mode 100644 LICENSE create mode 100644 Modulefile create mode 100644 README.markdown create mode 100644 Rakefile create mode 100755 files/concatfragments.sh create mode 100644 files/null/.gitignore create mode 100644 lib/facter/concat_basedir.rb create mode 100644 manifests/fragment.pp create mode 100644 manifests/init.pp create mode 100644 manifests/setup.pp create mode 100644 spec/defines/init_spec.rb create mode 100644 spec/spec_helper.rb diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 00000000..c506cf1a --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,29 @@ +KNOWN ISSUES: +- In 0.24.8 you will see inintended notifies, if you build a file + in a run, the next run will also see it as changed. This is due + to how 0.24.8 does the purging of unhandled files, this is improved + in 0.25.x and we cannot work around it in our code. + +CHANGELOG: +- 2010/02/19 - initial release +- 2010/03/12 - add support for 0.24.8 and newer + - make the location of sort configurable + - add the ability to add shell comment based warnings to + top of files + - add the ablity to create empty files +- 2010/04/05 - fix parsing of WARN and change code style to match rest + of the code + - Better and safer boolean handling for warn and force + - Don't use hard coded paths in the shell script, set PATH + top of the script + - Use file{} to copy the result and make all fragments owned + by root. This means we can chnage the ownership/group of the + resulting file at any time. + - You can specify ensure => "/some/other/file" in concat::fragment + to include the contents of a symlink into the final file. +- 2010/04/16 - Add more cleaning of the fragment name - removing / from the $name +- 2010/05/22 - Improve documentation and show the use of ensure => +- 2010/07/14 - Add support for setting the filebucket behavior of files +- 2010/10/04 - Make the warning message configurable +- 2010/12/03 - Add flags to make concat work better on Solaris - thanks Jonathan Boyett +- 2011/02/03 - Make the shell script more portable and add a config option for root group diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..6a9e9a19 --- /dev/null +++ b/LICENSE @@ -0,0 +1,14 @@ + Copyright 2012 R.I.Pienaar + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/Modulefile b/Modulefile new file mode 100644 index 00000000..d6ab2bb0 --- /dev/null +++ b/Modulefile @@ -0,0 +1,8 @@ +name 'puppet-concat' +version '0.1.0' +source 'git://github.com/ripienaar/puppet-concat.git' +author 'R.I.Pienaar' +license 'Apache' +summary 'Concat module' +description 'Concat module' +project_page 'http://github.com/ripienaar/puppet-concat' diff --git a/README.markdown b/README.markdown new file mode 100644 index 00000000..8736d57a --- /dev/null +++ b/README.markdown @@ -0,0 +1,112 @@ +What is it? +=========== + +A Puppet module that can construct files from fragments. + +Please see the comments in the various .pp files for details +as well as posts on my blog at http://www.devco.net/ + +Released under the Apache 2.0 licence + +Usage: +------ + +Before you can use any of the concat features you should include the class +concat::setup somewhere on your node first. + +If you wanted a /etc/motd file that listed all the major modules +on the machine. And that would be maintained automatically even +if you just remove the include lines for other modules you could +use code like below, a sample /etc/motd would be: + +
+Puppet modules on this server:
+
+    -- Apache
+    -- MySQL
+
+ +Local sysadmins can also append to the file by just editing /etc/motd.local +their changes will be incorporated into the puppet managed motd. + +
+# class to setup basic motd, include on all nodes
+class motd {
+   include concat::setup
+   $motd = "/etc/motd"
+
+   concat{$motd:
+      owner => root,
+      group => root,
+      mode  => 644
+   }
+
+   concat::fragment{"motd_header":
+      target => $motd,
+      content => "\nPuppet modules on this server:\n\n",
+      order   => 01,
+   }
+
+   # local users on the machine can append to motd by just creating
+   # /etc/motd.local
+   concat::fragment{"motd_local":
+      target => $motd,
+      ensure  => "/etc/motd.local",
+      order   => 15
+   }
+}
+
+# used by other modules to register themselves in the motd
+define motd::register($content="", $order=10) {
+   if $content == "" {
+      $body = $name
+   } else {
+      $body = $content
+   }
+
+   concat::fragment{"motd_fragment_$name":
+      target  => "/etc/motd",
+      content => "    -- $body\n"
+   }
+}
+
+# a sample apache module
+class apache {
+   include apache::install, apache::config, apache::service
+
+   motd::register{"Apache": }
+}
+
+ +Known Issues: +------------- +* In 0.24.8 you will see inintended notifies, if you build a file + in a run, the next run will also see it as changed. This is due + to how 0.24.8 does the purging of unhandled files, this is improved + in 0.25.x and we cannot work around it in our code. +* Since puppet-concat now relies on a fact for the concat directory, + you will need to set up pluginsync = true for at least the first run. + You have this issue if puppet fails to run on the client and you have + a message similar to + "err: Failed to apply catalog: Parameter path failed: File + paths must be fully qualified, not 'undef' at [...]/concat/manifests/setup.pp:44". + +Contributors: +------------- +**Paul Elliot** + + * Provided 0.24.8 support, shell warnings and empty file creation support. + +**Chad Netzer** + + * Various patches to improve safety of file operations + * Symlink support + +**David Schmitt** + + * Patch to remove hard coded paths relying on OS path + * Patch to use file{} to copy the resulting file to the final destination. This means Puppet client will show diffs and that hopefully we can change file ownerships now + +Contact: +-------- +You can contact me on rip@devco.net or follow my blog at http://www.devco.net I am also on twitter as ripienaar diff --git a/Rakefile b/Rakefile new file mode 100644 index 00000000..764aebd2 --- /dev/null +++ b/Rakefile @@ -0,0 +1,13 @@ +require 'rake' +require 'rspec/core/rake_task' + +task :default => [:spec] + +desc "Run all module spec tests (Requires rspec-puppet gem)" +RSpec::Core::RakeTask.new(:spec) + +desc "Build package" +task :build do + system("puppet-module build") +end + diff --git a/files/concatfragments.sh b/files/concatfragments.sh new file mode 100755 index 00000000..c9397975 --- /dev/null +++ b/files/concatfragments.sh @@ -0,0 +1,129 @@ +#!/bin/sh + +# Script to concat files to a config file. +# +# Given a directory like this: +# /path/to/conf.d +# |-- fragments +# | |-- 00_named.conf +# | |-- 10_domain.net +# | `-- zz_footer +# +# The script supports a test option that will build the concat file to a temp location and +# use /usr/bin/cmp to verify if it should be run or not. This would result in the concat happening +# twice on each run but gives you the option to have an unless option in your execs to inhibit rebuilds. +# +# Without the test option and the unless combo your services that depend on the final file would end up +# restarting on each run, or in other manifest models some changes might get missed. +# +# OPTIONS: +# -o The file to create from the sources +# -d The directory where the fragments are kept +# -t Test to find out if a build is needed, basically concats the files to a temp +# location and compare with what's in the final location, return codes are designed +# for use with unless on an exec resource +# -w Add a shell style comment at the top of the created file to warn users that it +# is generated by puppet +# -f Enables the creation of empty output files when no fragments are found +# -n Sort the output numerically rather than the default alpha sort +# +# the command: +# +# concatfragments.sh -o /path/to/conffile.cfg -d /path/to/conf.d +# +# creates /path/to/conf.d/fragments.concat and copies the resulting +# file to /path/to/conffile.cfg. The files will be sorted alphabetically +# pass the -n switch to sort numerically. +# +# The script does error checking on the various dirs and files to make +# sure things don't fail. + +OUTFILE="" +WORKDIR="" +TEST="" +FORCE="" +WARN="" +SORTARG="" + +PATH=/sbin:/usr/sbin:/bin:/usr/bin + +## Well, if there's ever a bad way to do things, Nexenta has it. +## http://nexenta.org/projects/site/wiki/Personalities +unset SUN_PERSONALITY + +while getopts "o:s:d:tnw:f" options; do + case $options in + o ) OUTFILE=$OPTARG;; + d ) WORKDIR=$OPTARG;; + n ) SORTARG="-n";; + w ) WARNMSG="$OPTARG";; + f ) FORCE="true";; + t ) TEST="true";; + * ) echo "Specify output file with -o and fragments directory with -d" + exit 1;; + esac +done + +# do we have -o? +if [ x${OUTFILE} = "x" ]; then + echo "Please specify an output file with -o" + exit 1 +fi + +# do we have -d? +if [ x${WORKDIR} = "x" ]; then + echo "Please fragments directory with -d" + exit 1 +fi + +# can we write to -o? +if [ -f ${OUTFILE} ]; then + if [ ! -w ${OUTFILE} ]; then + echo "Cannot write to ${OUTFILE}" + exit 1 + fi +else + if [ ! -w `dirname ${OUTFILE}` ]; then + echo "Cannot write to `dirname ${OUTFILE}` to create ${OUTFILE}" + exit 1 + fi +fi + +# do we have a fragments subdir inside the work dir? +if [ ! -d "${WORKDIR}/fragments" ] && [ ! -x "${WORKDIR}/fragments" ]; then + echo "Cannot access the fragments directory" + exit 1 +fi + +# are there actually any fragments? +if [ ! "$(ls -A ${WORKDIR}/fragments)" ]; then + if [ x${FORCE} = "x" ]; then + echo "The fragments directory is empty, cowardly refusing to make empty config files" + exit 1 + fi +fi + +cd ${WORKDIR} + +if [ x${WARNMSG} = "x" ]; then + : > "fragments.concat" +else + printf '%s\n' "$WARNMSG" > "fragments.concat" +fi + +# find all the files in the fragments directory, sort them numerically and concat to fragments.concat in the working dir +find fragments/ -type f -follow | sort ${SORTARG} | while read fragfile; do + cat "$fragfile" >> "fragments.concat" +done + +if [ x${TEST} = "x" ]; then + # This is a real run, copy the file to outfile + cp fragments.concat ${OUTFILE} + RETVAL=$? +else + # Just compare the result to outfile to help the exec decide + cmp ${OUTFILE} fragments.concat + RETVAL=$? +fi + +exit $RETVAL diff --git a/files/null/.gitignore b/files/null/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/lib/facter/concat_basedir.rb b/lib/facter/concat_basedir.rb new file mode 100644 index 00000000..02e9c5bf --- /dev/null +++ b/lib/facter/concat_basedir.rb @@ -0,0 +1,5 @@ +Facter.add("concat_basedir") do + setcode do + File.join(Puppet[:vardir],"concat") + end +end diff --git a/manifests/fragment.pp b/manifests/fragment.pp new file mode 100644 index 00000000..943bf671 --- /dev/null +++ b/manifests/fragment.pp @@ -0,0 +1,49 @@ +# Puts a file fragment into a directory previous setup using concat +# +# OPTIONS: +# - target The file that these fragments belong to +# - content If present puts the content into the file +# - source If content was not specified, use the source +# - order By default all files gets a 10_ prefix in the directory +# you can set it to anything else using this to influence the +# order of the content in the file +# - ensure Present/Absent or destination to a file to include another file +# - mode Mode for the file +# - owner Owner of the file +# - group Owner of the file +# - backup Controls the filebucketing behavior of the final file and +# see File type reference for its use. Defaults to 'puppet' +define concat::fragment($target, $content='', $source='', $order=10, $ensure = 'present', $mode = '0644', $owner = $::id, $group = $concat::setup::root_group, $backup = 'puppet') { + $safe_name = regsubst($name, '/', '_', 'G') + $safe_target_name = regsubst($target, '/', '_', 'G') + $concatdir = $concat::setup::concatdir + $fragdir = "${concatdir}/${safe_target_name}" + + # if content is passed, use that, else if source is passed use that + # if neither passed, but $ensure is in symlink form, make a symlink + case $content { + '': { + case $source { + '': { + case $ensure { + '', 'absent', 'present', 'file', 'directory': { + crit('No content, source or symlink specified') + } + } + } + default: { File{ source => $source } } + } + } + default: { File{ content => $content } } + } + + file{"${fragdir}/fragments/${order}_${safe_name}": + ensure => $ensure, + mode => $mode, + owner => $owner, + group => $group, + backup => $backup, + alias => "concat_fragment_${name}", + notify => Exec["concat_${target}"] + } +} diff --git a/manifests/init.pp b/manifests/init.pp new file mode 100644 index 00000000..0b3ed564 --- /dev/null +++ b/manifests/init.pp @@ -0,0 +1,178 @@ +# A system to construct files using fragments from other files or templates. +# +# This requires at least puppet 0.25 to work correctly as we use some +# enhancements in recursive directory management and regular expressions +# to do the work here. +# +# USAGE: +# The basic use case is as below: +# +# concat{"/etc/named.conf": +# notify => Service["named"] +# } +# +# concat::fragment{"foo.com_config": +# target => "/etc/named.conf", +# order => 10, +# content => template("named_conf_zone.erb") +# } +# +# # add a fragment not managed by puppet so local users +# # can add content to managed file +# concat::fragment{"foo.com_user_config": +# target => "/etc/named.conf", +# order => 12, +# ensure => "/etc/named.conf.local" +# } +# +# This will use the template named_conf_zone.erb to build a single +# bit of config up and put it into the fragments dir. The file +# will have an number prefix of 10, you can use the order option +# to control that and thus control the order the final file gets built in. +# +# SETUP: +# The class concat::setup uses the fact concat_basedir to define the variable +# $concatdir, where all the temporary files and fragments will be +# durably stored. The fact concat_basedir will be set up on the client to +# /concat, so you will be able to run different setup/flavours +# of puppet clients. +# However, since this requires the file lib/facter/concat_basedir.rb to be +# deployed on the clients, so you will have to set "pluginsync = true" on +# both the master and client, at least for the first run. +# +# There's some regular expression magic to figure out the puppet version but +# if you're on an older 0.24 version just set $puppetversion = 24 +# +# Before you can use any of the concat features you should include the +# class concat::setup somewhere on your node first. +# +# DETAIL: +# We use a helper shell script called concatfragments.sh that gets placed +# in /concat/bin to do the concatenation. While this might +# seem more complex than some of the one-liner alternatives you might find on +# the net we do a lot of error checking and safety checks in the script to avoid +# problems that might be caused by complex escaping errors etc. +# +# LICENSE: +# Apache Version 2 +# +# LATEST: +# http://github.com/ripienaar/puppet-concat/ +# +# CONTACT: +# R.I.Pienaar +# Volcane on freenode +# @ripienaar on twitter +# www.devco.net + + +# Sets up so that you can use fragments to build a final config file, +# +# OPTIONS: +# - mode The mode of the final file +# - owner Who will own the file +# - group Who will own the file +# - force Enables creating empty files if no fragments are present +# - warn Adds a normal shell style comment top of the file indicating +# that it is built by puppet +# - backup Controls the filebucketing behavior of the final file and +# see File type reference for its use. Defaults to 'puppet' +# +# ACTIONS: +# - Creates fragment directories if it didn't exist already +# - Executes the concatfragments.sh script to build the final file, this script will create +# directory/fragments.concat. Execution happens only when: +# * The directory changes +# * fragments.concat != final destination, this means rebuilds will happen whenever +# someone changes or deletes the final file. Checking is done using /usr/bin/cmp. +# * The Exec gets notified by something else - like the concat::fragment define +# - Copies the file over to the final destination using a file resource +# +# ALIASES: +# - The exec can notified using Exec["concat_/path/to/file"] or Exec["concat_/path/to/directory"] +# - The final file can be referened as File["/path/to/file"] or File["concat_/path/to/file"] +define concat($mode = '0644', $owner = $::id, $group = $concat::setup::root_group, $warn = false, $force = false, $backup = 'puppet', $gnu = undef, $order='alpha') { + $safe_name = regsubst($name, '/', '_', 'G') + $concatdir = $concat::setup::concatdir + $version = $concat::setup::majorversion + $fragdir = "${concatdir}/${safe_name}" + $concat_name = 'fragments.concat.out' + $default_warn_message = '# This file is managed by Puppet. DO NOT EDIT.' + + case $warn { + 'true',true,yes,on: { $warnmsg = $default_warn_message } + 'false',false,no,off: { $warnmsg = '' } + default: { $warnmsg = $warn } + } + + $warnmsg_escaped = regsubst($warnmsg, "'", "'\\\\''", 'G') + $warnflag = $warnmsg_escaped ? { + '' => '', + default => "-w '${warnmsg_escaped}'" + } + + case $force { + 'true',true,yes,on: { $forceflag = '-f' } + 'false',false,no,off: { $forceflag = '' } + default: { fail("Improper 'force' value given to concat: ${force}") } + } + + case $order { + numeric: { $orderflag = '-n' } + alpha: { $orderflag = '' } + default: { fail("Improper 'order' value given to concat: ${order}") } + } + + File{ + owner => $::id, + group => $group, + mode => $mode, + backup => $backup + } + + file{$fragdir: + ensure => directory; + + "${fragdir}/fragments": + ensure => directory, + recurse => true, + purge => true, + force => true, + ignore => ['.svn', '.git', '.gitignore'], + source => $version ? { + 24 => 'puppet:///concat/null', + default => undef, + }, + notify => Exec["concat_${name}"]; + + "${fragdir}/fragments.concat": + ensure => present; + + "${fragdir}/${concat_name}": + ensure => present; + + $name: + ensure => present, + source => "${fragdir}/${concat_name}", + owner => $owner, + group => $group, + checksum => md5, + mode => $mode, + alias => "concat_${name}"; + } + + exec{"concat_${name}": + notify => File[$name], + subscribe => File[$fragdir], + alias => "concat_${fragdir}", + require => [ File[$fragdir], File["${fragdir}/fragments"], File["${fragdir}/fragments.concat"] ], + unless => "${concat::setup::concatdir}/bin/concatfragments.sh -o ${fragdir}/${concat_name} -d ${fragdir} -t ${warnflag} ${forceflag} ${orderflag}", + command => "${concat::setup::concatdir}/bin/concatfragments.sh -o ${fragdir}/${concat_name} -d ${fragdir} ${warnflag} ${forceflag} ${orderflag}", + } + if $::id == 'root' { + Exec["concat_${name}"]{ + user => root, + group => $group, + } + } +} diff --git a/manifests/setup.pp b/manifests/setup.pp new file mode 100644 index 00000000..38aeb964 --- /dev/null +++ b/manifests/setup.pp @@ -0,0 +1,49 @@ +# Sets up the concat system. +# +# $concatdir is where the fragments live and is set on the fact concat_basedir. +# Since puppet should always manage files in $concatdir and they should +# not be deleted ever, /tmp is not an option. +# +# $puppetversion should be either 24 or 25 to enable a 24 compatible +# mode, in 24 mode you might see phantom notifies this is a side effect +# of the method we use to clear the fragments directory. +# +# The regular expression below will try to figure out your puppet version +# but this code will only work in 0.24.8 and newer. +# +# It also copies out the concatfragments.sh file to ${concatdir}/bin +class concat::setup { + $id = $::id + $root_group = $id ? { + root => 0, + default => $id + } + + if $::concat_basedir { + $concatdir = $::concat_basedir + } else { + fail ("\$concat_basedir not defined. Try running again with pluginsync enabled") + } + + $majorversion = regsubst($::puppetversion, '^[0-9]+[.]([0-9]+)[.][0-9]+$', '\1') + + file{"${concatdir}/bin/concatfragments.sh": + owner => $id, + group => $root_group, + mode => '0755', + source => $majorversion ? { + 24 => 'puppet:///concat/concatfragments.sh', + default => 'puppet:///modules/concat/concatfragments.sh' + }; + + [ $concatdir, "${concatdir}/bin" ]: + ensure => directory, + owner => $id, + group => $root_group, + mode => '0750'; + + ## Old versions of this module used a different path. + '/usr/local/bin/concatfragments.sh': + ensure => absent; + } +} diff --git a/spec/defines/init_spec.rb b/spec/defines/init_spec.rb new file mode 100644 index 00000000..d968a26c --- /dev/null +++ b/spec/defines/init_spec.rb @@ -0,0 +1,20 @@ +require 'spec_helper' + +describe 'concat' do + basedir = '/var/lib/puppet/concat' + let(:title) { '/etc/foo.bar' } + let(:facts) { { :concat_basedir => '/var/lib/puppet/concat' } } + let :pre_condition do + 'include concat::setup' + end + it { should contain_file("#{basedir}/_etc_foo.bar").with('ensure' => 'directory') } + it { should contain_file("#{basedir}/_etc_foo.bar/fragments").with('ensure' => 'directory') } + + it { should contain_file("#{basedir}/_etc_foo.bar/fragments.concat").with('ensure' => 'present') } + it { should contain_file("/etc/foo.bar").with('ensure' => 'present') } + it { should contain_exec("concat_/etc/foo.bar").with_command( + "#{basedir}/bin/concatfragments.sh "+ + "-o #{basedir}/_etc_foo.bar/fragments.concat.out "+ + "-d #{basedir}/_etc_foo.bar ") + } +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 00000000..e6e9309b --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,9 @@ +require 'puppet' +require 'rspec' +require 'rspec-puppet' + +RSpec.configure do |c| + c.module_path = File.expand_path(File.join(File.dirname(__FILE__), 'fixtures/modules/')) + # Using an empty site.pp file to avoid: https://github.com/rodjek/rspec-puppet/issues/15 + c.manifest_dir = File.expand_path(File.join(File.dirname(__FILE__), 'fixtures/manifests')) +end -- cgit v1.2.3