Check tapicero heartbeat (Bug #6556)

In order to assure tapicero is still working, we need to monitor
/var/log/syslog for the last tapicero log msg, which should not be older
than the last check_mk_agent run (every 2 mins atm).

3 files changed, 107 insertions, 0 deletions

diff --git a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp
index 369ed00b..ffd11100 100644
--- a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp
+++ b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp
@@ -1,5 +1,7 @@
 class site_check_mk::agent::tapicero {
 
+  include ::site_nagios::plugins
+
   concat::fragment { 'syslog_tapicero':
     source  => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/tapicero.cfg',
     target  => '/etc/check_mk/logwatch.d/syslog.cfg',
@@ -11,6 +13,10 @@ class site_check_mk::agent::tapicero {
     'Tapicero_Procs':
       line => 'Tapicero_Procs  /usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a tapicero',
       path => '/etc/check_mk/mrpe.cfg';
+
+    'Tapicero_Heartbeat':
+      line => 'Tapicero_Heartbeat  /usr/local/lib/nagios/plugins/check_last_regex_in_log -f /var/log/syslog -r "tapicero" -w 300 -c 600',
+      path => '/etc/check_mk/mrpe.cfg';
   }
 
 }
diff --git a/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log b/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log
new file mode 100755
index 00000000..cf7c03e5
--- /dev/null
+++ b/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log
@@ -0,0 +1,85 @@
+#!/bin/sh
+#
+# depends on nagios-plugins-common for /usr/lib/nagios/plugins/utils.sh
+# this package is installed using leap_platform by the Site_check_mk::Agent::Mrpe
+# class
+
+set -e
+
+usage()
+{
+cat << EOF
+usage: $0 -w <sec> -c <sec> -r <regexp> -f <filename>
+
+OPTIONS:
+  -h         Show this message
+  -r <regex> regex to grep for
+  -f <file>  logfile to search in
+  -w <sec>   warning  state after X seconds
+  -c <sec>   critical state after x seconds
+
+example: $0 -f /var/log/syslog -r 'tapicero' -w 300 -c 600
+EOF
+}
+
+
+. /usr/lib/nagios/plugins/utils.sh
+
+
+warn=0
+crit=0
+log=''
+regex=''
+
+set -- $(getopt hr:f:w:c: "$@")
+while [ $# -gt 0 ]
+do
+    case "$1" in
+    (-h) usage; exit 0 ;;
+    (-f) log="$2"; shift;;
+    (-r) regex="$2"; shift;;
+    (-w) warn="$2"; shift;;
+    (-c) crit="$2"; shift;;
+    (--) shift; break;;
+    (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;;
+    (*)  break;;
+    esac
+    shift
+done
+
+[ $warn -eq 0 -o $crit -eq 0 -o -z "$regex" -o -z "$log" ] && ( usage; exit $STATE_UNKNOWN)
+[ -f "$log" ] || (echo "$log doesn't exist"; exit $STATE_UNKNOWN)
+
+lastmsg=$(tac $log | grep -i $regex | head -1 | cut -d' ' -f 1-3)
+
+if [ -z "$lastmsg" ]
+then
+  summary="\"$regex\" in $log was not found"
+  state=$STATE_CRITICAL
+  state_text='CRITICAL'
+  diff_sec=0
+else
+  lastmsg_sec=$(date '+%s' -d "$lastmsg")
+  now_sec=$(date '+%s')
+
+  diff_sec=$(($now_sec - $lastmsg_sec))
+
+  if [ $diff_sec -lt $warn ]; then
+    state=$STATE_OK
+    state_text='OK'
+  elif [ $diff_sec -lt $crit ]; then
+    state=$STATE_WARNING
+    state_text='WARNING'
+  else
+    state=$STATE_CRITICAL
+    state_text='CRITICAL'
+  fi
+
+  summary="Last occurrence of \"$regex\" in $log was $diff_sec sec ago"
+fi
+
+# check_mk_agent output
+# echo "$state Tapicero_Heatbeat sec=$diff_sec;$warn;$crit;0; $state_text - $summary"
+
+echo "${state_text}: $summary | seconds=${diff_sec};$warn;$crit;0;"
+exit $state
diff --git a/puppet/modules/site_nagios/manifests/plugins.pp b/puppet/modules/site_nagios/manifests/plugins.pp
new file mode 100644
index 00000000..90a01cfb
--- /dev/null
+++ b/puppet/modules/site_nagios/manifests/plugins.pp
@@ -0,0 +1,16 @@
+# Deploy generic plugins useful to all nodes
+# nagios::plugin won't work to deploy a plugin
+# because it complains with:
+# Could not find dependency Package[nagios-plugins] …
+# at /srv/leap/puppet/modules/nagios/manifests/plugin.pp:18
+class site_nagios::plugins {
+
+  file { [
+    '/usr/local/lib', '/usr/local/lib/nagios',
+    '/usr/local/lib/nagios/plugins' ]:
+      ensure => directory;
+    '/usr/local/lib/nagios/plugins/check_last_regex_in_log':
+      source => 'puppet:///modules/site_nagios/plugins/check_last_regex_in_log',
+      mode   => '0755';
+  }
+}