diff options
8 files changed, 125 insertions, 6 deletions
diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg index 5cd2a47b..95ddd2ca 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg @@ -14,7 +14,11 @@ I Uncaught error in HTTP request: {exit,normal} I Uncaught error in HTTP request: {exit, # Ignore rexi_EXIT bigcouch error (Bug #6512) - I Error in process <[0-9.]+> on node .* with exit value: {{rexi_EXIT,{killed,\[{couch_db,collect_results + I Error in process <[0-9.]+> on node .* with exit value: {{rexi_EXIT,{(killed|noproc|shutdown),\[{couch_db,collect_results + # Ignore "Generic server terminating" bigcouch message (Feature #6544) + I Generic server <.*> terminating + I {error_report,<.*>, + I {error_info, C Uncaught error in HTTP request: {error, C Response abnormally terminated: {nodedown, C rexi_DOWN,noproc diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg index d58e876d..ac17c0ca 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg @@ -2,6 +2,12 @@ # suddenly hangup before properly establishing # a tls connection I ovpn-.*TLS Error: Unroutable control packet received from - I ovpn-.*TLS Error: TLS key negotiation failed to occur within 60 seconds (check your network connectivity) + I ovpn-.*TLS Error: TLS key negotiation failed to occur within 60 seconds \(check your network connectivity\) I ovpn-.*TLS Error: TLS handshake failed + I ovpn-.*TLS Error: TLS object -> incoming plaintext read error + I ovpn-.*Fatal TLS error \(check_tls_errors_co\), restarting + I ovpn-.*TLS_ERROR: BIO read tls_read_plaintext error: error:140890B2:SSL routines:SSL3_GET_CLIENT_CERTIFICATE:no certificate + + I ovpn-.*SIGUSR1\[soft,tls-error\] received, client-instance restarting + I ovpn-.*VERIFY ERROR: depth=0, error=certificate has expired diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg index d9f0eafc..d00b8a75 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg @@ -1,5 +1,5 @@ # Ignore transient Tapicero errors when creating a db (#6511) - I tapicero.*(Creating database|Checking security of|Writing security to|Uploading design doc to) user-.* failed (\(trying again soon\)|twice due to): (RestClient::Resource Not Found|RestClient::InternalServerError): (404 Resource Not Found|500 Internal Server Error) + I tapicero.*(Creating database|Checking security of|Writing security to|Uploading design doc to) user-.* failed (\(trying again soon\)|(twice )? due to): (RestClient::Resource Not Found|RestClient::InternalServerError): (404 Resource Not Found|500 Internal Server Error) C tapicero.*RestClient::InternalServerError: # possible race condition between multiple tapicero # instances, so we ignore it diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg index b19ac241..71395c50 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg @@ -2,14 +2,16 @@ I Error: Driver 'pcspkr' is already registered, aborting... # ignore postfix errors on lost connection (Bug #6476) I postfix/smtpd.*SSL_accept error from.*lost connection +# ignore postfix too many errors after DATA (#6545) + I postfix/smtpd.*too many errors after DATA from C panic C Oops C Error +# ignore ipv6 icmp errors for now (Bug #6540) + I kernel: .*icmpv6_send: no reply to icmp error C error W generic protection rip W .*Unrecovered read error - auto reallocate failed # 401 Unauthorized error logged by webapp and possible other # applications C Unauthorized -# ignore ipv6 icmp errors for now (Bug #6540) - I kernel: .*icmpv6_send: no reply to icmp error diff --git a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp index 369ed00b..ffd11100 100644 --- a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp +++ b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp @@ -1,5 +1,7 @@ class site_check_mk::agent::tapicero { + include ::site_nagios::plugins + concat::fragment { 'syslog_tapicero': source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/tapicero.cfg', target => '/etc/check_mk/logwatch.d/syslog.cfg', @@ -11,6 +13,10 @@ class site_check_mk::agent::tapicero { 'Tapicero_Procs': line => 'Tapicero_Procs /usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a tapicero', path => '/etc/check_mk/mrpe.cfg'; + + 'Tapicero_Heartbeat': + line => 'Tapicero_Heartbeat /usr/local/lib/nagios/plugins/check_last_regex_in_log -f /var/log/syslog -r "tapicero" -w 300 -c 600', + path => '/etc/check_mk/mrpe.cfg'; } } diff --git a/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log b/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log new file mode 100755 index 00000000..cf7c03e5 --- /dev/null +++ b/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log @@ -0,0 +1,85 @@ +#!/bin/sh +# +# depends on nagios-plugins-common for /usr/lib/nagios/plugins/utils.sh +# this package is installed using leap_platform by the Site_check_mk::Agent::Mrpe +# class + +set -e + +usage() +{ +cat << EOF +usage: $0 -w <sec> -c <sec> -r <regexp> -f <filename> + +OPTIONS: + -h Show this message + -r <regex> regex to grep for + -f <file> logfile to search in + -w <sec> warning state after X seconds + -c <sec> critical state after x seconds + +example: $0 -f /var/log/syslog -r 'tapicero' -w 300 -c 600 +EOF +} + + +. /usr/lib/nagios/plugins/utils.sh + + +warn=0 +crit=0 +log='' +regex='' + +set -- $(getopt hr:f:w:c: "$@") +while [ $# -gt 0 ] +do + case "$1" in + (-h) usage; exit 0 ;; + (-f) log="$2"; shift;; + (-r) regex="$2"; shift;; + (-w) warn="$2"; shift;; + (-c) crit="$2"; shift;; + (--) shift; break;; + (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;; + (*) break;; + esac + shift +done + +[ $warn -eq 0 -o $crit -eq 0 -o -z "$regex" -o -z "$log" ] && ( usage; exit $STATE_UNKNOWN) +[ -f "$log" ] || (echo "$log doesn't exist"; exit $STATE_UNKNOWN) + +lastmsg=$(tac $log | grep -i $regex | head -1 | cut -d' ' -f 1-3) + +if [ -z "$lastmsg" ] +then + summary="\"$regex\" in $log was not found" + state=$STATE_CRITICAL + state_text='CRITICAL' + diff_sec=0 +else + lastmsg_sec=$(date '+%s' -d "$lastmsg") + now_sec=$(date '+%s') + + diff_sec=$(($now_sec - $lastmsg_sec)) + + if [ $diff_sec -lt $warn ]; then + state=$STATE_OK + state_text='OK' + elif [ $diff_sec -lt $crit ]; then + state=$STATE_WARNING + state_text='WARNING' + else + state=$STATE_CRITICAL + state_text='CRITICAL' + fi + + summary="Last occurrence of \"$regex\" in $log was $diff_sec sec ago" +fi + +# check_mk_agent output +# echo "$state Tapicero_Heatbeat sec=$diff_sec;$warn;$crit;0; $state_text - $summary" + +echo "${state_text}: $summary | seconds=${diff_sec};$warn;$crit;0;" +exit $state diff --git a/puppet/modules/site_nagios/manifests/plugins.pp b/puppet/modules/site_nagios/manifests/plugins.pp new file mode 100644 index 00000000..90a01cfb --- /dev/null +++ b/puppet/modules/site_nagios/manifests/plugins.pp @@ -0,0 +1,16 @@ +# Deploy generic plugins useful to all nodes +# nagios::plugin won't work to deploy a plugin +# because it complains with: +# Could not find dependency Package[nagios-plugins] … +# at /srv/leap/puppet/modules/nagios/manifests/plugin.pp:18 +class site_nagios::plugins { + + file { [ + '/usr/local/lib', '/usr/local/lib/nagios', + '/usr/local/lib/nagios/plugins' ]: + ensure => directory; + '/usr/local/lib/nagios/plugins/check_last_regex_in_log': + source => 'puppet:///modules/site_nagios/plugins/check_last_regex_in_log', + mode => '0755'; + } +} diff --git a/puppet/modules/tapicero/manifests/init.pp b/puppet/modules/tapicero/manifests/init.pp index ae5d058e..28711b94 100644 --- a/puppet/modules/tapicero/manifests/init.pp +++ b/puppet/modules/tapicero/manifests/init.pp @@ -95,7 +95,7 @@ class tapicero { vcsrepo { '/srv/leap/tapicero': ensure => present, force => true, - revision => 'origin/develop', + revision => 'origin/version/0.6', provider => git, source => 'https://leap.se/git/tapicero', owner => 'tapicero', |