summaryrefslogtreecommitdiff
path: root/puppet/modules
diff options
context:
space:
mode:
authorvarac <varacanero@zeromail.org>2014-12-16 22:06:08 +0100
committervarac <varacanero@zeromail.org>2014-12-17 20:01:02 +0100
commitc16c019a3f0f1f4d5ebce9ade59a7386c3c2bb18 (patch)
tree37d5d4701e9652575917cbaa6323ab5eb3204153 /puppet/modules
parentbbd0bb1e0a497a9aca93065c3c6ec20fafde824a (diff)
Check tapicero heartbeat (Bug #6556)
In order to assure tapicero is still working, we need to monitor /var/log/syslog for the last tapicero log msg, which should not be older than the last check_mk_agent run (every 2 mins atm).
Diffstat (limited to 'puppet/modules')
-rw-r--r--puppet/modules/site_check_mk/manifests/agent/tapicero.pp6
-rwxr-xr-xpuppet/modules/site_nagios/files/plugins/check_last_regex_in_log85
-rw-r--r--puppet/modules/site_nagios/manifests/plugins.pp16
3 files changed, 107 insertions, 0 deletions
diff --git a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp
index 369ed00b..ffd11100 100644
--- a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp
+++ b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp
@@ -1,5 +1,7 @@
class site_check_mk::agent::tapicero {
+ include ::site_nagios::plugins
+
concat::fragment { 'syslog_tapicero':
source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/tapicero.cfg',
target => '/etc/check_mk/logwatch.d/syslog.cfg',
@@ -11,6 +13,10 @@ class site_check_mk::agent::tapicero {
'Tapicero_Procs':
line => 'Tapicero_Procs /usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a tapicero',
path => '/etc/check_mk/mrpe.cfg';
+
+ 'Tapicero_Heartbeat':
+ line => 'Tapicero_Heartbeat /usr/local/lib/nagios/plugins/check_last_regex_in_log -f /var/log/syslog -r "tapicero" -w 300 -c 600',
+ path => '/etc/check_mk/mrpe.cfg';
}
}
diff --git a/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log b/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log
new file mode 100755
index 00000000..cf7c03e5
--- /dev/null
+++ b/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log
@@ -0,0 +1,85 @@
+#!/bin/sh
+#
+# depends on nagios-plugins-common for /usr/lib/nagios/plugins/utils.sh
+# this package is installed using leap_platform by the Site_check_mk::Agent::Mrpe
+# class
+
+set -e
+
+usage()
+{
+cat << EOF
+usage: $0 -w <sec> -c <sec> -r <regexp> -f <filename>
+
+OPTIONS:
+ -h Show this message
+ -r <regex> regex to grep for
+ -f <file> logfile to search in
+ -w <sec> warning state after X seconds
+ -c <sec> critical state after x seconds
+
+example: $0 -f /var/log/syslog -r 'tapicero' -w 300 -c 600
+EOF
+}
+
+
+. /usr/lib/nagios/plugins/utils.sh
+
+
+warn=0
+crit=0
+log=''
+regex=''
+
+set -- $(getopt hr:f:w:c: "$@")
+while [ $# -gt 0 ]
+do
+ case "$1" in
+ (-h) usage; exit 0 ;;
+ (-f) log="$2"; shift;;
+ (-r) regex="$2"; shift;;
+ (-w) warn="$2"; shift;;
+ (-c) crit="$2"; shift;;
+ (--) shift; break;;
+ (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;;
+ (*) break;;
+ esac
+ shift
+done
+
+[ $warn -eq 0 -o $crit -eq 0 -o -z "$regex" -o -z "$log" ] && ( usage; exit $STATE_UNKNOWN)
+[ -f "$log" ] || (echo "$log doesn't exist"; exit $STATE_UNKNOWN)
+
+lastmsg=$(tac $log | grep -i $regex | head -1 | cut -d' ' -f 1-3)
+
+if [ -z "$lastmsg" ]
+then
+ summary="\"$regex\" in $log was not found"
+ state=$STATE_CRITICAL
+ state_text='CRITICAL'
+ diff_sec=0
+else
+ lastmsg_sec=$(date '+%s' -d "$lastmsg")
+ now_sec=$(date '+%s')
+
+ diff_sec=$(($now_sec - $lastmsg_sec))
+
+ if [ $diff_sec -lt $warn ]; then
+ state=$STATE_OK
+ state_text='OK'
+ elif [ $diff_sec -lt $crit ]; then
+ state=$STATE_WARNING
+ state_text='WARNING'
+ else
+ state=$STATE_CRITICAL
+ state_text='CRITICAL'
+ fi
+
+ summary="Last occurrence of \"$regex\" in $log was $diff_sec sec ago"
+fi
+
+# check_mk_agent output
+# echo "$state Tapicero_Heatbeat sec=$diff_sec;$warn;$crit;0; $state_text - $summary"
+
+echo "${state_text}: $summary | seconds=${diff_sec};$warn;$crit;0;"
+exit $state
diff --git a/puppet/modules/site_nagios/manifests/plugins.pp b/puppet/modules/site_nagios/manifests/plugins.pp
new file mode 100644
index 00000000..90a01cfb
--- /dev/null
+++ b/puppet/modules/site_nagios/manifests/plugins.pp
@@ -0,0 +1,16 @@
+# Deploy generic plugins useful to all nodes
+# nagios::plugin won't work to deploy a plugin
+# because it complains with:
+# Could not find dependency Package[nagios-plugins] …
+# at /srv/leap/puppet/modules/nagios/manifests/plugin.pp:18
+class site_nagios::plugins {
+
+ file { [
+ '/usr/local/lib', '/usr/local/lib/nagios',
+ '/usr/local/lib/nagios/plugins' ]:
+ ensure => directory;
+ '/usr/local/lib/nagios/plugins/check_last_regex_in_log':
+ source => 'puppet:///modules/site_nagios/plugins/check_last_regex_in_log',
+ mode => '0755';
+ }
+}