diff options
58 files changed, 1266 insertions, 50 deletions
diff --git a/.gitmodules b/.gitmodules index cc8afdb5..7005b770 100644 --- a/.gitmodules +++ b/.gitmodules @@ -88,3 +88,6 @@ [submodule "puppet/modules/sysctl"] path = puppet/modules/sysctl url = https://leap.se/git/puppet_sysctl +[submodule "puppet/modules/check_mk"] + path = puppet/modules/check_mk + url = https://leap.se/git/puppet_check_mk diff --git a/provider_base/services/couchdb.json b/provider_base/services/couchdb.json index 0cb044ef..5f1b5381 100644 --- a/provider_base/services/couchdb.json +++ b/provider_base/services/couchdb.json @@ -48,6 +48,9 @@ "password": "= secret :couch_webapp_password", "salt": "= hex_secret :couch_webapp_password_salt, 128" } + }, + "webapp": { + "nagios_test_pw": "= secret :nagios_test_password" } } } diff --git a/provider_base/services/monitor.json b/provider_base/services/monitor.json index 53e6b1f1..03f6c6d1 100644 --- a/provider_base/services/monitor.json +++ b/provider_base/services/monitor.json @@ -1,13 +1,22 @@ { "nagios": { "nagiosadmin_pw": "= secret :nagios_admin_password", - "hosts": "= nodes_like_me[:services => '!monitor'].pick_fields('domain.internal', 'ip_address', 'services', 'openvpn.gateway_address')" + "hosts": "= (self.environment == 'local' ? nodes_like_me : nodes[:environment => '!local']).pick_fields('domain.internal', 'domain.full_suffix', 'ip_address', 'services', 'openvpn.gateway_address', 'ssh.port')" }, - "hosts": "= hosts_file(nodes_like_me[:services => '!monitor'])", + "hosts": "= self.environment == 'local' ? hosts_file(nodes_like_me) : hosts_file(nodes[:environment => '!local'])", "ssh": { "monitor": { "username": "= Leap::Platform.monitor_username", "private_key": "= file(:monitor_priv_key)" } + }, + "x509": { + "use": true, + "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", + "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", + "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'", + "commercial_cert": "= file [:commercial_cert, domain.full_suffix]", + "commercial_key": "= file [:commercial_key, domain.full_suffix]", + "commercial_ca_cert": "= try_file :commercial_ca_cert" } } diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json index d57c3168..7e3f20ba 100644 --- a/provider_base/services/mx.json +++ b/provider_base/services/mx.json @@ -5,7 +5,11 @@ "haproxy": { "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.couch_client)" }, - "couchdb_leap_mx_user": "= global.services[:couchdb].couch.users[:leap_mx]", + "couchdb_leap_mx_user": { + "username": "= global.services[:couchdb].couch.users[:leap_mx].username", + "password": "= secret :couch_leap_mx_password", + "salt": "= hex_secret :couch_leap_mx_password_salt, 128" + }, "mx_nodes": "= nodes['services' => 'mx']['environment' => '!local'].field('ip_address')", "x509": { "use": true, diff --git a/provider_base/services/soledad.json b/provider_base/services/soledad.json index fc349ce1..ed6fbc9f 100644 --- a/provider_base/services/soledad.json +++ b/provider_base/services/soledad.json @@ -2,7 +2,11 @@ "soledad": { "port": 2323, "require_couchdb": "=> assert %(services.include? 'couchdb')", - "couchdb_soledad_user": "= global.services[:couchdb].couch.users[:soledad]" + "couchdb_soledad_user": { + "username": "= global.services[:couchdb].couch.users[:soledad].username", + "password": "= secret :couch_soledad_password", + "salt": "= hex_secret :couch_soledad_password_salt, 128" + } }, "service_type": "public_service" } diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index 08c7c5b0..f7abdffd 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -2,7 +2,11 @@ "webapp": { "admins": [], "modules": ["user", "billing", "help"], - "couchdb_webapp_user": "= global.services[:couchdb].couch.users[:webapp]", + "couchdb_webapp_user": { + "username": "= global.services[:couchdb].couch.users[:webapp].username", + "password": "= secret :couch_webapp_password", + "salt": "= hex_secret :couch_webapp_password_salt, 128" + }, "customization_dir": "= file_path 'webapp'", "client_certificates": "= global.provider.ca.client_certificates", "allow_limited_certs": "= global.provider.service.allow_limited_bandwidth", @@ -15,7 +19,11 @@ "source": "https://leap.se/git/leap_web", "revision": "origin/master" }, - "client_version": "= global.provider.client_version" + "client_version": "= global.provider.client_version", + "nagios_test_user": { + "username": "nagios_test", + "password": "= secret :nagios_test_password" + } }, "stunnel": { "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" @@ -36,7 +44,11 @@ }, "nickserver": { "domain": "= 'nicknym.' + domain.full_suffix", - "couchdb_nickserver_user": "= global.services[:couchdb].couch.users[:nickserver]", + "couchdb_nickserver_user": { + "username": "= global.services[:couchdb].couch.users[:nickserver].username", + "password": "= secret :couch_nickserver_password", + "salt": "= hex_secret :couch_nickserver_password_salt, 128" + }, "port": 6425 }, "dns": { diff --git a/puppet/bin/apply_on_node.sh b/puppet/bin/apply_on_node.sh new file mode 100755 index 00000000..09e5b035 --- /dev/null +++ b/puppet/bin/apply_on_node.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# Script to use on a node for debugging +# Usage: ./apply_on_node.sh <puppet parameters> +# +# Example: ./apply_on_node.sh --debug --verbose + +ROOTDIR='/srv/leap' +PLATFORM="$ROOTDIR" +MODULEPATH="$PLATFORM/puppet/modules" +LOG=/var/log/leap.log + +# example tags to use +#TAGS='--tags=leap_base,leap_service,leap_slow' +#TAGS='--tags=leap_base,leap_slow' +#TAGS='--tags=leap_base,leap_service' + +####### +# Setup +####### + +puppet apply -v --confdir $PLATFORM/puppet --libdir $PLATFORM/puppet/lib --modulepath=$MODULEPATH $PLATFORM/puppet/manifests/setup.pp $TAGS $@ |tee $LOG 2>&1 + +######### +# site.pp +######### + +puppet apply -v --confdir $PLATFORM/puppet --libdir $PLATFORM/puppet/lib --modulepath=$MODULEPATH $PLATFORM/puppet/manifests/site.pp $TAGS $@ |tee $LOG 2>&1 + + diff --git a/puppet/modules/check_mk b/puppet/modules/check_mk new file mode 160000 +Subproject f9e494265f6c7b83ab9ef418e40cc7aac558956 diff --git a/puppet/modules/rubygems b/puppet/modules/rubygems -Subproject 1e5ed3dbef9381bb9d5e2a7b4957bb3f5288d6a +Subproject 7e0240acce5a338e7cc2b55f1f8ba749634c5da diff --git a/puppet/modules/site_apache/manifests/common.pp b/puppet/modules/site_apache/manifests/common.pp new file mode 100644 index 00000000..72f24838 --- /dev/null +++ b/puppet/modules/site_apache/manifests/common.pp @@ -0,0 +1,26 @@ +class site_apache::common { + # installs x509 cert + key and common config + # that both nagios + leap webapp use + + $web_domain = hiera('domain') + $domain_name = $web_domain['name'] + + include x509::variables + include site_config::x509::commercial::cert + include site_config::x509::commercial::key + include site_config::x509::commercial::ca + + Class['Site_config::X509::Commercial::Key'] ~> Service[apache] + Class['Site_config::X509::Commercial::Cert'] ~> Service[apache] + Class['Site_config::X509::Commercial::Ca'] ~> Service[apache] + + include site_apache::module::rewrite + + class { '::apache': no_default_site => true, ssl => true } + + apache::vhost::file { + 'common': + content => template('site_apache/vhosts.d/common.conf.erb') + } + +} diff --git a/puppet/modules/site_apache/templates/vhosts.d/leap_webapp.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb index a001552a..30f0a6b1 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/leap_webapp.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb @@ -23,13 +23,14 @@ RequestHeader set X_FORWARDED_PROTO 'https' <IfModule mod_headers.c> -<% if @webapp['secure'] -%> +<% if (defined? @services) and (@services.include? 'webapp') and (@webapp['secure']) -%> Header always set Strict-Transport-Security "max-age=31536000; includeSubDomains" <% end -%> Header always unset X-Powered-By Header always unset X-Runtime </IfModule> +<% if (defined? @services) and (@services.include? 'webapp') -%> DocumentRoot /srv/leap/webapp/public RewriteEngine On @@ -52,16 +53,20 @@ ExpiresActive On ExpiresDefault "access plus 1 year" </Location> +<% end -%> + - <% if (defined? @services) and (@services.include? 'monitor') -%> +<% if (defined? @services) and (@services.include? 'monitor') -%> <DirectoryMatch (/usr/share/nagios3/htdocs|/usr/lib/cgi-bin/nagios3|/etc/nagios3/stylesheets)> + <% if (defined? @services) and (@services.include? 'webapp') -%> PassengerEnabled off + <% end -%> AllowOverride all # Nagios won't work with setting this option to "DENY", # as set in conf.d/security (#4169). Therefor we allow # it here, only for nagios. Header set X-Frame-Options: "ALLOW" </DirectoryMatch> - <% end -%> +<% end -%> </VirtualHost> diff --git a/puppet/modules/site_apt/manifests/preferences/check_mk.pp b/puppet/modules/site_apt/manifests/preferences/check_mk.pp new file mode 100644 index 00000000..580e0d3f --- /dev/null +++ b/puppet/modules/site_apt/manifests/preferences/check_mk.pp @@ -0,0 +1,9 @@ +class site_apt::preferences::check_mk { + + apt::preferences_snippet { 'check-mk': + package => 'check-mk-*', + release => "${::lsbdistcodename}-backports", + priority => 999; + } + +} diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg new file mode 100644 index 00000000..a1eb1312 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg @@ -0,0 +1,15 @@ +/opt/bigcouch/var/log/bigcouch.log + C Uncaught error in HTTP request: {exit, + C Uncaught error in HTTP request: {exit,normal} + C Uncaught error in HTTP request: {error, + C Response abnormally terminated: {nodedown, + C rexi_DOWN,noproc + C rexi_DOWN,noconnection + C error + C Connection attempt from disallowed node + W Shutting down group server +# ignore requests that are fine + I undefined - -.*200$ + I undefined - -.*201$ + I 127.0.0.1 undefined.* ok + I 127.0.0.1 localhost:5984 .* ok diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/logwatch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/logwatch.cfg new file mode 100644 index 00000000..c4acae40 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/logwatch.cfg @@ -0,0 +1,31 @@ +# This file is managed by Puppet. DO NOT EDIT. + +# logwatch.cfg +# This file configures mk_logwatch. Define your logfiles +# and patterns to be looked for here. + +# Name one or more logfiles +/var/log/messages +# Patterns are indented with one space are prefixed with: +# C: Critical messages +# W: Warning messages +# I: ignore these lines (OK) +# The first match decided. Lines that do not match any pattern +# are ignored + C Fail event detected on md device + I mdadm.*: Rebuild.*event detected + W mdadm\[ + W ata.*hard resetting link + W ata.*soft reset failed (.*FIS failed) + W device-mapper: thin:.*reached low water mark + C device-mapper: thin:.*no free space + +/var/log/auth.log + W sshd.*Corrupted MAC on input + +/var/log/kern.log + C panic + C Oops + W generic protection rip + W .*Unrecovered read error - auto reallocate failed + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg new file mode 100644 index 00000000..54b782d3 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg @@ -0,0 +1,4 @@ +/var/log/soledad.log + C WSGI application error + C error + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog.cfg new file mode 100644 index 00000000..f3505c1c --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog.cfg @@ -0,0 +1,12 @@ +/var/log/syslog +# some general patterns + C panic + C Oops + I Error: Driver 'pcspkr' is already registered, aborting... + C Error + C error + W generic protection rip + W .*Unrecovered read error - auto reallocate failed +# 401 Unauthorized error logged by webapp and possible other +# applications + C Unauthorized diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg new file mode 100644 index 00000000..5f8d5b95 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg @@ -0,0 +1,2 @@ + C /usr/local/bin/couch-doc-update.*failed + C /usr/local/bin/couch-doc-update.*ERROR diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/stunnel.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/stunnel.cfg new file mode 100644 index 00000000..cf7ebca8 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/stunnel.cfg @@ -0,0 +1,5 @@ +# check for stunnel failures + C stunnel:.*Connection refused +# this is a temporary failure and happens very often, so we +# ignore it + I stunnel:.*Connection reset by peer diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg new file mode 100644 index 00000000..9983d27c --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg @@ -0,0 +1,7 @@ + C tapicero.*RestClient::InternalServerError: +# possible race condition between multiple tapicero +# instances, so we ignore it +# see https://leap.se/code/issues/5168 + I tapicero.*RestClient::PreconditionFailed: + C tapicero.*failed + W tapicero.*Couch stream ended unexpectedly. diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/webapp.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/webapp.cfg new file mode 100644 index 00000000..00f9c7fd --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/webapp.cfg @@ -0,0 +1,5 @@ +# check for webapp errors + C webapp.*Could not connect to couch database messages due to 401 Unauthorized: {"error":"unauthorized","reason":"You are not a server admin."} +# ignore RoutingErrors that rails throw when it can't handle a url +# see https://leap.se/code/issues/5173 + I webapp.*ActionController::RoutingError diff --git a/puppet/modules/site_check_mk/files/agent/nagios_plugins/check_unix_open_fds.pl b/puppet/modules/site_check_mk/files/agent/nagios_plugins/check_unix_open_fds.pl new file mode 100755 index 00000000..06163d49 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/nagios_plugins/check_unix_open_fds.pl @@ -0,0 +1,322 @@ +#!/usr/bin/perl -w + +# check_unix_open_fds Nagios Plugin +# +# TComm - Carlos Peris Pla +# +# This nagios plugin is free software, and comes with ABSOLUTELY +# NO WARRANTY. It may be used, redistributed and/or modified under +# the terms of the GNU General Public Licence (see +# http://www.fsf.org/licensing/licenses/gpl.txt). + + +# MODULE DECLARATION + +use strict; +use Nagios::Plugin; + + +# FUNCTION DECLARATION + +sub CreateNagiosManager (); +sub CheckArguments (); +sub PerformCheck (); + + +# CONSTANT DEFINITION + +use constant NAME => 'check_unix_open_fds'; +use constant VERSION => '0.1b'; +use constant USAGE => "Usage:\ncheck_unix_open_fds -w <process_threshold,application_threshold> -c <process_threshold,application_threshold>\n". + "\t\t[-V <version>]\n"; +use constant BLURB => "This plugin checks, in UNIX systems with the command lsof installed and with its SUID bit activated, the number\n". + "of file descriptors opened by an application and its processes.\n"; +use constant LICENSE => "This nagios plugin is free software, and comes with ABSOLUTELY\n". + "no WARRANTY. It may be used, redistributed and/or modified under\n". + "the terms of the GNU General Public Licence\n". + "(see http://www.fsf.org/licensing/licenses/gpl.txt).\n"; +use constant EXAMPLE => "\n\n". + "Example:\n". + "\n". + "check_unix_open_fds -a /usr/local/nagios/bin/ndo2db -w 20,75 -c 25,85\n". + "\n". + "It returns CRITICAL if number of file descriptors opened by ndo2db is higher than 85,\n". + "if not it returns WARNING if number of file descriptors opened by ndo2db is higher \n". + "than 75, if not it returns CRITICAL if number of file descriptors opened by any process\n". + "of ndo2db is higher than 25, if not it returns WARNING if number of file descriptors \n". + "opened by any process of ndo2db is higher than 20.\n". + "In other cases it returns OK if check has been performed succesfully.\n\n"; + + +# VARIABLE DEFINITION + +my $Nagios; +my $Error; +my $PluginResult; +my $PluginOutput; +my @WVRange; +my @CVRange; + + +# MAIN FUNCTION + +# Get command line arguments +$Nagios = &CreateNagiosManager(USAGE, VERSION, BLURB, LICENSE, NAME, EXAMPLE); +eval {$Nagios->getopts}; + +if (!$@) { + # Command line parsed + if (&CheckArguments($Nagios, \$Error, \@WVRange, \@CVRange)) { + # Argument checking passed + $PluginResult = &PerformCheck($Nagios, \$PluginOutput, \@WVRange, \@CVRange) + } + else { + # Error checking arguments + $PluginOutput = $Error; + $PluginResult = UNKNOWN; + } + $Nagios->nagios_exit($PluginResult,$PluginOutput); +} +else { + # Error parsing command line + $Nagios->nagios_exit(UNKNOWN,$@); +} + + + +# FUNCTION DEFINITIONS + +# Creates and configures a Nagios plugin object +# Input: strings (usage, version, blurb, license, name and example) to configure argument parsing functionality +# Return value: reference to a Nagios plugin object + +sub CreateNagiosManager() { + # Create GetOpt object + my $Nagios = Nagios::Plugin->new(usage => $_[0], version => $_[1], blurb => $_[2], license => $_[3], plugin => $_[4], extra => $_[5]); + + # Add argument units + $Nagios->add_arg(spec => 'application|a=s', + help => 'Application path for which you want to check the number of open file descriptors', + required => 1); + + # Add argument warning + $Nagios->add_arg(spec => 'warning|w=s', + help => "Warning thresholds. Format: <process_threshold,application_threshold>", + required => 1); + # Add argument critical + $Nagios->add_arg(spec => 'critical|c=s', + help => "Critical thresholds. Format: <process_threshold,application_threshold>", + required => 1); + + # Return value + return $Nagios; +} + + +# Checks argument values and sets some default values +# Input: Nagios Plugin object +# Output: reference to Error description string, Memory Unit, Swap Unit, reference to WVRange ($_[4]), reference to CVRange ($_[5]) +# Return value: True if arguments ok, false if not + +sub CheckArguments() { + my ($Nagios, $Error, $WVRange, $CVRange) = @_; + my $commas; + my $units; + my $i; + my $firstpos; + my $secondpos; + + # Check Warning thresholds list + $commas = $Nagios->opts->warning =~ tr/,//; + if ($commas !=1){ + ${$Error} = "Invalid Warning list format. One comma is expected."; + return 0; + } + else{ + $i=0; + $firstpos=0; + my $warning=$Nagios->opts->warning; + while ($warning =~ /[,]/g) { + $secondpos=pos $warning; + if ($secondpos - $firstpos==1){ + @{$WVRange}[$i] = "~:"; + } + else{ + @{$WVRange}[$i] = substr $Nagios->opts->warning, $firstpos, ($secondpos-$firstpos-1); + } + $firstpos=$secondpos; + $i++ + } + if (length($Nagios->opts->warning) - $firstpos==0){#La coma es el ultimo elemento del string + @{$WVRange}[$i] = "~:"; + } + else{ + @{$WVRange}[$i] = substr $Nagios->opts->warning, $firstpos, (length($Nagios->opts->warning)-$firstpos); + } + + if (@{$WVRange}[0] !~/^(@?(\d+|(\d+|~):(\d+)?))?$/){ + ${$Error} = "Invalid Process Warning threshold in ${$WVRange[0]}"; + return 0; + }if (@{$WVRange}[1] !~/^(@?(\d+|(\d+|~):(\d+)?))?$/){ + ${$Error} = "Invalid Application Warning threshold in ${$WVRange[1]}"; + return 0; + } + } + + # Check Critical thresholds list + $commas = $Nagios->opts->critical =~ tr/,//; + if ($commas !=1){ + ${$Error} = "Invalid Critical list format. One comma is expected."; + return 0; + } + else{ + $i=0; + $firstpos=0; + my $critical=$Nagios->opts->critical; + while ($critical =~ /[,]/g) { + $secondpos=pos $critical ; + if ($secondpos - $firstpos==1){ + @{$CVRange}[$i] = "~:"; + } + else{ + @{$CVRange}[$i] =substr $Nagios->opts->critical, $firstpos, ($secondpos-$firstpos-1); + } + $firstpos=$secondpos; + $i++ + } + if (length($Nagios->opts->critical) - $firstpos==0){#La coma es el ultimo elemento del string + @{$CVRange}[$i] = "~:"; + } + else{ + @{$CVRange}[$i] = substr $Nagios->opts->critical, $firstpos, (length($Nagios->opts->critical)-$firstpos); + } + + if (@{$CVRange}[0] !~/^(@?(\d+|(\d+|~):(\d+)?))?$/) { + ${$Error} = "Invalid Process Critical threshold in @{$CVRange}[0]"; + return 0; + } + if (@{$CVRange}[1] !~/^(@?(\d+|(\d+|~):(\d+)?))?$/) { + ${$Error} = "Invalid Application Critical threshold in @{$CVRange}[1]"; + return 0; + } + } + + return 1; +} + + +# Performs whole check: +# Input: Nagios Plugin object, reference to Plugin output string, Application, referece to WVRange, reference to CVRange +# Output: Plugin output string +# Return value: Plugin return value + +sub PerformCheck() { + my ($Nagios, $PluginOutput, $WVRange, $CVRange) = @_; + my $Application; + my @AppNameSplitted; + my $ApplicationName; + my $PsCommand; + my $PsResult; + my @PsResultLines; + my $ProcLine; + my $ProcPid; + my $LsofCommand; + my $LsofResult; + my $ProcCount = 0; + my $FDCount = 0; + my $ProcFDAvg = 0; + my $PerProcMaxFD = 0; + my $ProcOKFlag = 0; + my $ProcWarningFlag = 0; + my $ProcCriticalFlag = 0; + my $OKFlag = 0; + my $WarningFlag = 0; + my $CriticalFlag = 0; + my $LastWarningProcFDs = 0; + my $LastWarningProc = -1; + my $LastCriticalProcFDs = 0; + my $LastCriticalProc = -1; + my $ProcPluginReturnValue = UNKNOWN; + my $AppPluginReturnValue = UNKNOWN; + my $PluginReturnValue = UNKNOWN; + my $PerformanceData = ""; + my $PerfdataUnit = "FDs"; + + $Application = $Nagios->opts->application; + $PsCommand = "ps -eaf | grep $Application"; + $PsResult = `$PsCommand`; + @AppNameSplitted = split(/\//, $Application); + $ApplicationName = $AppNameSplitted[$#AppNameSplitted]; + @PsResultLines = split(/\n/, $PsResult); + if ( $#PsResultLines > 1 ) { + foreach my $Proc (split(/\n/, $PsResult)) { + if ($Proc !~ /check_unix_open_fds/ && $Proc !~ / grep /) { + $ProcCount += 1; + $ProcPid = (split(/\s+/, $Proc))[1]; + $LsofCommand = "lsof -p $ProcPid | wc -l"; + $LsofResult = `$LsofCommand`; + $LsofResult = ($LsofResult > 0 ) ? ($LsofResult - 1) : 0; + $FDCount += $LsofResult; + if ($LsofResult >= $PerProcMaxFD) { $PerProcMaxFD = $LsofResult; } + $ProcPluginReturnValue = $Nagios->check_threshold(check => $LsofResult,warning => @{$WVRange}[0],critical => @{$CVRange}[0]); + if ($ProcPluginReturnValue eq OK) { + $ProcOKFlag = 1; + } + elsif ($ProcPluginReturnValue eq WARNING) { + $ProcWarningFlag = 1; + if ($LsofResult >= $LastWarningProcFDs) { + $LastWarningProcFDs = $LsofResult; + $LastWarningProc = $ProcPid; + } + } + #if ($LsofResult >= $PCT) { + elsif ($ProcPluginReturnValue eq CRITICAL) { + $ProcCriticalFlag = 1; + if ($LsofResult >= $LastCriticalProcFDs) { + $LastCriticalProcFDs = $LsofResult; + $LastCriticalProc = $ProcPid; + } + } + } + } + if ($ProcCount) { $ProcFDAvg = int($FDCount / $ProcCount); } + $AppPluginReturnValue = $Nagios->check_threshold(check => $FDCount,warning => @{$WVRange}[1],critical => @{$CVRange}[1]); + #if ($FDCount >= $TWT) { + if ($AppPluginReturnValue eq OK) { $OKFlag = 1; } + elsif ($AppPluginReturnValue eq WARNING) { $WarningFlag = 1; } + elsif ($AppPluginReturnValue eq CRITICAL) { $CriticalFlag = 1; } + + # PluginReturnValue and PluginOutput + if ($CriticalFlag) { + $PluginReturnValue = CRITICAL; + ${$PluginOutput} .= "$ApplicationName handling $FDCount files (critical threshold set to @{$CVRange}[1])"; + } + elsif ($WarningFlag) { + $PluginReturnValue = WARNING; + ${$PluginOutput} .= "$ApplicationName handling $FDCount files (warning threshold set to @{$WVRange}[1])"; + } + elsif ($ProcCriticalFlag) { + $PluginReturnValue = CRITICAL; + ${$PluginOutput} .= "Process ID $LastCriticalProc handling $LastCriticalProcFDs files (critical threshold set to @{$CVRange}[0])"; + } + elsif ($ProcWarningFlag) { + $PluginReturnValue = WARNING; + ${$PluginOutput} .= "Process ID $LastWarningProc handling $LastWarningProcFDs files (warning threshold set to @{$WVRange}[0])"; + } + elsif ($OKFlag && $ProcOKFlag) { + $PluginReturnValue = OK; + ${$PluginOutput} .= "$ApplicationName handling $FDCount files"; + } + } + else { + ${$PluginOutput} .= "No existe la aplicacion $ApplicationName"; + } + + + $PerformanceData .= "ProcCount=$ProcCount$PerfdataUnit FDCount=$FDCount$PerfdataUnit ProcFDAvg=$ProcFDAvg$PerfdataUnit PerProcMaxFD=$PerProcMaxFD$PerfdataUnit"; + + # Output with performance data: + ${$PluginOutput} .= " | $PerformanceData"; + + return $PluginReturnValue; +} diff --git a/puppet/modules/site_check_mk/files/agent/plugins/mk_logwatch.1.2.4 b/puppet/modules/site_check_mk/files/agent/plugins/mk_logwatch.1.2.4 new file mode 100755 index 00000000..3dbca322 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/plugins/mk_logwatch.1.2.4 @@ -0,0 +1,374 @@ +#!/usr/bin/python +# -*- encoding: utf-8; py-indent-offset: 4 -*- +# +------------------------------------------------------------------+ +# | ____ _ _ __ __ _ __ | +# | / ___| |__ ___ ___| | __ | \/ | |/ / | +# | | | | '_ \ / _ \/ __| |/ / | |\/| | ' / | +# | | |___| | | | __/ (__| < | | | | . \ | +# | \____|_| |_|\___|\___|_|\_\___|_| |_|_|\_\ | +# | | +# | Copyright Mathias Kettner 2010 mk@mathias-kettner.de | +# +------------------------------------------------------------------+ +# +# This file is part of Check_MK. +# The official homepage is at http://mathias-kettner.de/check_mk. +# +# check_mk is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation in version 2. check_mk is distributed +# in the hope that it will be useful, but WITHOUT ANY WARRANTY; with- +# out even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more de- +# ails. You should have received a copy of the GNU General Public +# License along with GNU Make; see the file COPYING. If not, write +# to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, +# Boston, MA 02110-1301 USA. + +# Call with -d for debug mode: colored output, no saving of status + +import sys, os, re, time +import glob + +if '-d' in sys.argv[1:] or '--debug' in sys.argv[1:]: + tty_red = '\033[1;31m' + tty_green = '\033[1;32m' + tty_yellow = '\033[1;33m' + tty_blue = '\033[1;34m' + tty_normal = '\033[0m' + debug = True +else: + tty_red = '' + tty_green = '' + tty_yellow = '' + tty_blue = '' + tty_normal = '' + debug = False + +# The configuration file and status file are searched +# in the directory named by the environment variable +# LOGWATCH_DIR. If that is not set, MK_CONFDIR is used. +# If that is not set either, the current directory ist +# used. +logwatch_dir = os.getenv("LOGWATCH_DIR") +if not logwatch_dir: + logwatch_dir = os.getenv("MK_CONFDIR") + if not logwatch_dir: + logwatch_dir = "." + +print "<<<logwatch>>>" + +config_filename = logwatch_dir + "/logwatch.cfg" +status_filename = logwatch_dir + "/logwatch.state" +config_dir = logwatch_dir + "/logwatch.d/*.cfg" + +def is_not_comment(line): + if line.lstrip().startswith('#') or \ + line.strip() == '': + return False + return True + +def parse_filenames(line): + return line.split() + +def parse_pattern(level, pattern): + if level not in [ 'C', 'W', 'I', 'O' ]: + raise(Exception("Invalid pattern line '%s'" % line)) + try: + compiled = re.compile(pattern) + except: + raise(Exception("Invalid regular expression in line '%s'" % line)) + return (level, compiled) + +def read_config(): + config_lines = [ line.rstrip() for line in filter(is_not_comment, file(config_filename).readlines()) ] + # Add config from a logwatch.d folder + for config_file in glob.glob(config_dir): + config_lines += [ line.rstrip() for line in filter(is_not_comment, file(config_file).readlines()) ] + + have_filenames = False + config = [] + + for line in config_lines: + rewrite = False + if line[0].isspace(): # pattern line + if not have_filenames: + raise Exception("Missing logfile names") + level, pattern = line.split(None, 1) + if level == 'A': + cont_list.append(parse_cont_pattern(pattern)) + elif level == 'R': + rewrite_list.append(pattern) + else: + level, compiled = parse_pattern(level, pattern) + cont_list = [] # List of continuation patterns + rewrite_list = [] # List of rewrite patterns + patterns.append((level, compiled, cont_list, rewrite_list)) + else: # filename line + patterns = [] + config.append((parse_filenames(line), patterns)) + have_filenames = True + return config + +def parse_cont_pattern(pattern): + try: + return int(pattern) + except: + try: + return re.compile(pattern) + except: + if debug: + raise + raise Exception("Invalid regular expression in line '%s'" % pattern) + +# structure of statusfile +# # LOGFILE OFFSET INODE +# /var/log/messages|7767698|32455445 +# /var/test/x12134.log|12345|32444355 +def read_status(): + if debug: + return {} + + status = {} + for line in file(status_filename): + # TODO: Remove variants with spaces. rsplit is + # not portable. split fails if logfilename contains + # spaces + inode = -1 + try: + parts = line.split('|') + filename = parts[0] + offset = parts[1] + if len(parts) >= 3: + inode = parts[2] + + except: + try: + filename, offset = line.rsplit(None, 1) + except: + filename, offset = line.split(None, 1) + status[filename] = int(offset), int(inode) + return status + +def save_status(status): + f = file(status_filename, "w") + for filename, (offset, inode) in status.items(): + f.write("%s|%d|%d\n" % (filename, offset, inode)) + +pushed_back_line = None +def next_line(f): + global pushed_back_line + if pushed_back_line != None: + line = pushed_back_line + pushed_back_line = None + return line + else: + try: + line = f.next() + return line + except: + return None + + +def process_logfile(logfile, patterns): + global pushed_back_line + + # Look at which file offset we have finished scanning + # the logfile last time. If we have never seen this file + # before, we set the offset to -1 + offset, prev_inode = status.get(logfile, (-1, -1)) + try: + fl = os.open(logfile, os.O_RDONLY) + inode = os.fstat(fl)[1] # 1 = st_ino + except: + if debug: + raise + print "[[[%s:cannotopen]]]" % logfile + return + + print "[[[%s]]]" % logfile + + # Seek to the current end in order to determine file size + current_end = os.lseek(fl, 0, 2) # os.SEEK_END not available in Python 2.4 + status[logfile] = current_end, inode + + # If we have never seen this file before, we just set the + # current pointer to the file end. We do not want to make + # a fuss about ancient log messages... + if offset == -1: + if not debug: + return + else: + offset = 0 + + + # If the inode of the logfile has changed it has appearently + # been started from new (logfile rotation). At least we must + # assume that. In some rare cases (restore of a backup, etc) + # we are wrong and resend old log messages + if prev_inode >= 0 and inode != prev_inode: + offset = 0 + + # Our previously stored offset is the current end -> + # no new lines in this file + if offset == current_end: + return # nothing new + + # If our offset is beyond the current end, the logfile has been + # truncated or wrapped while keeping the same inode. We assume + # that it contains all new data in that case and restart from + # offset 0. + if offset > current_end: + offset = 0 + + # now seek to offset where interesting data begins + os.lseek(fl, offset, 0) # os.SEEK_SET not available in Python 2.4 + f = os.fdopen(fl) + worst = -1 + outputtxt = "" + lines_parsed = 0 + start_time = time.time() + + while True: + line = next_line(f) + if line == None: + break # End of file + + lines_parsed += 1 + # Check if maximum number of new log messages is exceeded + if opt_maxlines != None and lines_parsed > opt_maxlines: + outputtxt += "%s Maximum number (%d) of new log messages exceeded.\n" % ( + opt_overflow, opt_maxlines) + worst = max(worst, opt_overflow_level) + os.lseek(fl, 0, 2) # Seek to end of file, skip all other messages + break + + # Check if maximum processing time (per file) is exceeded. Check only + # every 100'th line in order to save system calls + if opt_maxtime != None and lines_parsed % 100 == 10 \ + and time.time() - start_time > opt_maxtime: + outputtxt += "%s Maximum parsing time (%.1f sec) of this log file exceeded.\n" % ( + opt_overflow, opt_maxtime) + worst = max(worst, opt_overflow_level) + os.lseek(fl, 0, 2) # Seek to end of file, skip all other messages + break + + level = "." + for lev, pattern, cont_patterns, replacements in patterns: + matches = pattern.search(line[:-1]) + if matches: + level = lev + levelint = {'C': 2, 'W': 1, 'O': 0, 'I': -1, '.': -1}[lev] + worst = max(levelint, worst) + + # Check for continuation lines + for cont_pattern in cont_patterns: + if type(cont_pattern) == int: # add that many lines + for x in range(cont_pattern): + cont_line = next_line(f) + if cont_line == None: # end of file + break + line = line[:-1] + "\1" + cont_line + + else: # pattern is regex + while True: + cont_line = next_line(f) + if cont_line == None: # end of file + break + elif cont_pattern.search(cont_line[:-1]): + line = line[:-1] + "\1" + cont_line + else: + pushed_back_line = cont_line # sorry for stealing this line + break + + # Replacement + for replace in replacements: + line = replace.replace('\\0', line) + "\n" + for nr, group in enumerate(matches.groups()): + line = line.replace('\\%d' % (nr+1), group) + + break # matching rule found and executed + + color = {'C': tty_red, 'W': tty_yellow, 'O': tty_green, 'I': tty_blue, '.': ''}[level] + if debug: + line = line.replace("\1", "\nCONT:") + if level == "I": + level = "." + if opt_nocontext and level == '.': + continue + outputtxt += "%s%s %s%s\n" % (color, level, line[:-1], tty_normal) + + new_offset = os.lseek(fl, 0, 1) # os.SEEK_CUR not available in Python 2.4 + status[logfile] = new_offset, inode + + # output all lines if at least one warning, error or ok has been found + if worst > -1: + sys.stdout.write(outputtxt) + sys.stdout.flush() + +try: + config = read_config() +except Exception, e: + if debug: + raise + print "CANNOT READ CONFIG FILE: %s" % e + sys.exit(1) + +# Simply ignore errors in the status file. In case of a corrupted status file we simply begin +# with an empty status. That keeps the monitoring up and running - even if we might loose a +# message in the extreme case of a corrupted status file. +try: + status = read_status() +except Exception, e: + status = {} + + +# The filename line may contain options like 'maxlines=100' or 'maxtime=10' +for filenames, patterns in config: + # Initialize options with default values + opt_maxlines = None + opt_maxtime = None + opt_regex = None + opt_overflow = 'C' + opt_overflow_level = 2 + opt_nocontext = False + try: + options = [ o.split('=', 1) for o in filenames if '=' in o ] + for key, value in options: + if key == 'maxlines': + opt_maxlines = int(value) + elif key == 'maxtime': + opt_maxtime = float(value) + elif key == 'overflow': + if value not in [ 'C', 'I', 'W', 'O' ]: + raise Exception("Invalid value %s for overflow. Allowed are C, I, O and W" % value) + opt_overflow = value + opt_overflow_level = {'C':2, 'W':1, 'O':0, 'I':0}[value] + elif key == 'regex': + opt_regex = re.compile(value) + elif key == 'iregex': + opt_regex = re.compile(value, re.I) + elif key == 'nocontext': + opt_nocontext = True + else: + raise Exception("Invalid option %s" % key) + except Exception, e: + if debug: + raise + print "INVALID CONFIGURATION: %s" % e + sys.exit(1) + + + for glob in filenames: + if '=' in glob: + continue + logfiles = [ l.strip() for l in os.popen("ls %s 2>/dev/null" % glob).readlines() ] + if opt_regex: + logfiles = [ f for f in logfiles if opt_regex.search(f) ] + if len(logfiles) == 0: + print '[[[%s:missing]]]' % glob + else: + for logfile in logfiles: + process_logfile(logfile, patterns) + +if not debug: + save_status(status) diff --git a/puppet/modules/site_check_mk/manifests/agent.pp b/puppet/modules/site_check_mk/manifests/agent.pp new file mode 100644 index 00000000..efb05b37 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent.pp @@ -0,0 +1,23 @@ +class site_check_mk::agent { + + $ssh_hash = hiera('ssh') + $pubkey = $ssh_hash['authorized_keys']['monitor']['key'] + $type = $ssh_hash['authorized_keys']['monitor']['type'] + + include site_apt::preferences::check_mk + + class { 'check_mk::agent': + agent_package_name => 'check-mk-agent', + agent_logwatch_package_name => 'check-mk-agent-logwatch', + method => 'ssh', + homedir => '/etc/nagios/check_mk', + register_agent => false + } + + file { [ '/srv/leap/nagios', '/srv/leap/nagios/plugins' ]: + ensure => directory, + } + + include site_check_mk::agent::mrpe + include site_check_mk::agent::logwatch +} diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp new file mode 100644 index 00000000..01e2b886 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp @@ -0,0 +1,36 @@ +class site_check_mk::agent::couchdb { + + # watch logs + file { '/etc/check_mk/logwatch.d/bigcouch.cfg': + source => 'puppet:///modules/site_check_mk/agent/logwatch/bigcouch.cfg', + } + concat::fragment { 'syslog_couchdb': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/couchdb.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + + + # check bigcouch processes + file_line { + 'Bigcouch_epmd_procs': + line => 'Bigcouch_epmd_procs /usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/epmd', + path => '/etc/check_mk/mrpe.cfg'; + 'Bigcouch_beam_procs': + line => 'Bigcouch_beam_procs /usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/beam', + path => '/etc/check_mk/mrpe.cfg'; + } + + # check open files for bigcouch proc + include site_check_mk::agent::package::perl_plugin + file { '/srv/leap/nagios/plugins/check_unix_open_fds.pl': + source => 'puppet:///modules/site_check_mk/agent/nagios_plugins/check_unix_open_fds.pl', + mode => '0755' + } + file_line { + 'Bigcouch_open_files': + line => 'Bigcouch_open_files /srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 750,750 -c 1000,1000', + path => '/etc/check_mk/mrpe.cfg'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/haproxy.pp b/puppet/modules/site_check_mk/manifests/agent/haproxy.pp new file mode 100644 index 00000000..e7986db1 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/haproxy.pp @@ -0,0 +1,12 @@ +class site_check_mk::agent::haproxy { + + include site_check_mk::agent::package::nagios_plugins_contrib + + # local nagios plugin checks via mrpe + file_line { + 'haproxy': + line => 'Haproxy /usr/lib/nagios/plugins/check_haproxy -u "http://localhost:8000/haproxy;csv"', + path => '/etc/check_mk/mrpe.cfg'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/logwatch.pp b/puppet/modules/site_check_mk/manifests/agent/logwatch.pp new file mode 100644 index 00000000..38514af2 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/logwatch.pp @@ -0,0 +1,34 @@ +class site_check_mk::agent::logwatch { + # Deploy mk_logwatch 1.2.4 so we can split the config + # into multiple config files in /etc/check_mk/logwatch.d + # see https://leap.se/code/issues/5135 + + file { '/usr/lib/check_mk_agent/plugins/mk_logwatch': + source => 'puppet:///modules/site_check_mk/agent/plugins/mk_logwatch.1.2.4', + mode => '0755' + } + + # only config files that watch a distinct logfile should go in logwatch.d/ + file { '/etc/check_mk/logwatch.d': + ensure => directory, + recurse => true, + purge => true, + } + + # service that share a common logfile (i.e. /var/log/syslog) need to get + # concanated in one file, otherwise the last file sourced will override + # the config before + # see mk_logwatch: "logwatch.cfg overwrites config files in logwatch.d", + # https://leap.se/code/issues/5155 + + # first, we need to deploy a custom logwatch.cfg that doesn't include + # a section about /var/log/syslog + + file { '/etc/check_mk/logwatch.cfg': + source => 'puppet:///modules/site_check_mk/agent/logwatch/logwatch.cfg', + require => Package['check_mk-agent-logwatch'] + } + + include concat::setup + include site_check_mk::agent::logwatch::syslog +} diff --git a/puppet/modules/site_check_mk/manifests/agent/logwatch/syslog.pp b/puppet/modules/site_check_mk/manifests/agent/logwatch/syslog.pp new file mode 100644 index 00000000..ac3dda64 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/logwatch/syslog.pp @@ -0,0 +1,13 @@ +class site_check_mk::agent::logwatch::syslog { + + concat { '/etc/check_mk/logwatch.d/syslog.cfg': + warn => true + } + + concat::fragment { 'syslog_header': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '01'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/mrpe.pp b/puppet/modules/site_check_mk/manifests/agent/mrpe.pp new file mode 100644 index 00000000..37df7f74 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/mrpe.pp @@ -0,0 +1,17 @@ +class site_check_mk::agent::mrpe { + # check_mk can use standard nagios plugins using + # a wrapper called mrpe + # see http://mathias-kettner.de/checkmk_mrpe.html + + package { 'nagios-plugins-basic': + ensure => latest, + } + + file { '/etc/check_mk/mrpe.cfg': + ensure => present + } -> + file_line { 'Apt': + line => 'APT /usr/lib/nagios/plugins/check_apt', + path => '/etc/check_mk/mrpe.cfg', + } +} diff --git a/puppet/modules/site_check_mk/manifests/agent/package/nagios_plugins_contrib.pp b/puppet/modules/site_check_mk/manifests/agent/package/nagios_plugins_contrib.pp new file mode 100644 index 00000000..95a60d17 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/package/nagios_plugins_contrib.pp @@ -0,0 +1,5 @@ +class site_check_mk::agent::package::nagios_plugins_contrib { + package { 'nagios-plugins-contrib': + ensure => installed, + } +} diff --git a/puppet/modules/site_check_mk/manifests/agent/package/perl_plugin.pp b/puppet/modules/site_check_mk/manifests/agent/package/perl_plugin.pp new file mode 100644 index 00000000..4feda375 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/package/perl_plugin.pp @@ -0,0 +1,5 @@ +class site_check_mk::agent::package::perl_plugin { + package { 'libnagios-plugin-perl': + ensure => installed, + } +} diff --git a/puppet/modules/site_check_mk/manifests/agent/soledad.pp b/puppet/modules/site_check_mk/manifests/agent/soledad.pp new file mode 100644 index 00000000..cbae81fe --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/soledad.pp @@ -0,0 +1,14 @@ +class site_check_mk::agent::soledad { + + file { '/etc/check_mk/logwatch.d/soledad.cfg': + source => 'puppet:///modules/site_check_mk/agent/logwatch/soledad.cfg', + } + + # local nagios plugin checks via mrpe + file_line { + 'Soledad_Procs': + line => 'Soledad_Procs /usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a soledad', + path => '/etc/check_mk/mrpe.cfg'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/stunnel.pp b/puppet/modules/site_check_mk/manifests/agent/stunnel.pp new file mode 100644 index 00000000..64022824 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/stunnel.pp @@ -0,0 +1,9 @@ +class site_check_mk::agent::stunnel { + + concat::fragment { 'syslog_stunnel': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/stunnel.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp new file mode 100644 index 00000000..369ed00b --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp @@ -0,0 +1,16 @@ +class site_check_mk::agent::tapicero { + + concat::fragment { 'syslog_tapicero': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/tapicero.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + + # local nagios plugin checks via mrpe + file_line { + 'Tapicero_Procs': + line => 'Tapicero_Procs /usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a tapicero', + path => '/etc/check_mk/mrpe.cfg'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/webapp.pp b/puppet/modules/site_check_mk/manifests/agent/webapp.pp new file mode 100644 index 00000000..48fa0828 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/webapp.pp @@ -0,0 +1,18 @@ +class site_check_mk::agent::webapp { + + # check webapp login + package { 'python-srp': ensure => installed } + file { '/usr/lib/check_mk_agent/local/nagios-webapp_login.py': + ensure => link, + target => '/srv/leap/webapp/test/nagios/webapp_login.py' + } + + + # check syslog + concat::fragment { 'syslog_webapp': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/webapp.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/server.pp b/puppet/modules/site_check_mk/manifests/server.pp new file mode 100644 index 00000000..0c98cd5a --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/server.pp @@ -0,0 +1,56 @@ +class site_check_mk::server { + + $ssh_hash = hiera('ssh') + $pubkey = $ssh_hash['authorized_keys']['monitor']['key'] + $type = $ssh_hash['authorized_keys']['monitor']['type'] + $seckey = $ssh_hash['monitor']['private_key'] + + $nagios_hiera = hiera_hash('nagios') + $nagios_hosts = $nagios_hiera['hosts'] + + $hosts = hiera_hash('hosts') + $all_hosts = inline_template ('<% @hosts.keys.sort.each do |key| -%>"<%= @hosts[key]["domain_internal"] %>", <% end -%>') + + package { 'check-mk-server': + ensure => installed, + } + + # override paths to use the system check_mk rather than OMD + class { 'check_mk::config': + site => '', + etc_dir => '/etc', + nagios_subdir => 'nagios3', + bin_dir => '/usr/bin', + host_groups => undef, + require => Package['check-mk-server'] + } + + Exec['check_mk-reload'] -> Service['nagios'] + + file { + '/etc/check_mk/conf.d/use_ssh.mk': + content => template('site_check_mk/use_ssh.mk'), + notify => Exec['check_mk-refresh']; + '/etc/check_mk/all_hosts_static': + content => $all_hosts, + notify => Exec['check_mk-refresh']; + '/etc/check_mk/.ssh': + ensure => directory; + '/etc/check_mk/.ssh/id_rsa': + content => $seckey, + owner => 'nagios', + mode => '0600'; + '/etc/check_mk/.ssh/id_rsa.pub': + content => "${type} ${pubkey} monitor", + owner => 'nagios', + mode => '0644'; + # check_icmp must be suid root or called by sudo + # see https://leap.se/code/issues/5171 + '/usr/lib/nagios/plugins/check_icmp': + mode => '4755', + require => Package['nagios-plugins-basic']; + } + + + include check_mk::agent::local_checks +} diff --git a/puppet/modules/site_check_mk/templates/use_ssh.mk b/puppet/modules/site_check_mk/templates/use_ssh.mk new file mode 100644 index 00000000..0bebebcf --- /dev/null +++ b/puppet/modules/site_check_mk/templates/use_ssh.mk @@ -0,0 +1,6 @@ +# http://mathias-kettner.de/checkmk_datasource_programs.html +datasource_programs = [ +<% nagios_hosts.sort.each do |name,config| %> + ( "ssh -l root -i /etc/check_mk/.ssh/id_rsa -p <%=config['ssh_port']%> <%=config['domain_internal']%> check_mk_agent", [ "<%=config['domain_internal']%>" ], ),<%- end -%> + +] diff --git a/puppet/modules/site_config/manifests/default.pp b/puppet/modules/site_config/manifests/default.pp index d85d9c8f..53cc60f6 100644 --- a/puppet/modules/site_config/manifests/default.pp +++ b/puppet/modules/site_config/manifests/default.pp @@ -86,4 +86,6 @@ class site_config::default { if defined( '::site_custom') { include ::site_custom } + + include site_check_mk::agent } diff --git a/puppet/modules/site_config/manifests/hosts.pp b/puppet/modules/site_config/manifests/hosts.pp index a3ce0c1f..e5d4dd70 100644 --- a/puppet/modules/site_config/manifests/hosts.pp +++ b/puppet/modules/site_config/manifests/hosts.pp @@ -3,6 +3,7 @@ class site_config::hosts() { $hostname = hiera('name') $domain_hash = hiera('domain') $domain_public = $domain_hash['full_suffix'] + $api = hiera('api', '') file { '/etc/hostname': ensure => present, diff --git a/puppet/modules/site_config/templates/hosts b/puppet/modules/site_config/templates/hosts index c0a2740f..bfcabaa5 100644 --- a/puppet/modules/site_config/templates/hosts +++ b/puppet/modules/site_config/templates/hosts @@ -1,7 +1,8 @@ # This file is managed by puppet, any changes will be overwritten! 127.0.0.1 localhost -127.0.1.1 <%= @hostname %>.<%= @domain_public %> <%= @hostname %> +127.0.1.1 <%= @hostname %>.<%= @domain_public %> <%= @hostname %> <% if (defined? @services) and (@services.include? 'webapp') -%><%= @domain_public %> <%= @api['domain'] %><% end -%> + <%- if @hosts then -%> <% @hosts.keys.sort.each do |name| -%> diff --git a/puppet/modules/site_couchdb/files/designs/messages/Message.json b/puppet/modules/site_couchdb/files/designs/messages/Message.json new file mode 100644 index 00000000..7bcd74c7 --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/messages/Message.json @@ -0,0 +1,18 @@ +{ + "_id": "_design/Message", + "language": "javascript", + "views": { + "by_user_ids_to_show_and_created_at": { + "map": "// not using at moment\n// call with something like Message.by_user_ids_to_show_and_created_at.startkey([user_id, start_date]).endkey([user_id,end_date])\nfunction (doc) {\n if (doc.type === 'Message' && doc.user_ids_to_show && Array.isArray(doc.user_ids_to_show)) {\n doc.user_ids_to_show.forEach(function (userId) {\n emit([userId, doc.created_at], 1);\n });\n }\n}\n", + "reduce": "function(key, values, rereduce) { return sum(values); }" + }, + "by_user_ids_to_show": { + "map": "function (doc) {\n if (doc.type === 'Message' && doc.user_ids_to_show && Array.isArray(doc.user_ids_to_show)) {\n doc.user_ids_to_show.forEach(function (userId) {\n emit(userId, 1);\n });\n }\n}\n", + "reduce": "function(key, values, rereduce) { return sum(values); }" + }, + "all": { + "map": " function(doc) {\n if (doc['type'] == 'Message') {\n emit(doc._id, null);\n }\n }\n" + } + }, + "couchrest-hash": "0967e7cc5bb1e61edc1c085f6f0cecbf" +}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/users/User.json b/puppet/modules/site_couchdb/files/designs/users/User.json index c99666cb..4089ad97 100644 --- a/puppet/modules/site_couchdb/files/designs/users/User.json +++ b/puppet/modules/site_couchdb/files/designs/users/User.json @@ -9,10 +9,14 @@ "all": { "map": " function(doc) {\n if (doc['type'] == 'User') {\n emit(doc._id, null);\n }\n }\n" }, + "by_created_at_and_one_month_warning_not_sent": { + "map": "function (doc) {\n if ((doc['type'] == 'User') && (doc['created_at'] != null) && (doc['one_month_warning_sent'] == null)) {\n emit(doc['created_at'], 1);\n } \n}\n", + "reduce": "function(key, values, rereduce) { return sum(values); }" + }, "by_created_at": { "map": " function(doc) {\n if ((doc['type'] == 'User') && (doc['created_at'] != null)) {\n emit(doc['created_at'], 1);\n }\n }\n", "reduce": "_sum" } }, - "couchrest-hash": "3bdbcd85b928ad911e0c89a8924e015c" + "couchrest-hash": "61840ab3ec0f94ef8bbd6dd208db3b70" }
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/manifests/create_dbs.pp b/puppet/modules/site_couchdb/manifests/create_dbs.pp index b0ebca4d..41500d3a 100644 --- a/puppet/modules/site_couchdb/manifests/create_dbs.pp +++ b/puppet/modules/site_couchdb/manifests/create_dbs.pp @@ -59,4 +59,12 @@ class site_couchdb::create_dbs { members => "{ \"names\": [], \"roles\": [\"users\"] }", require => Couchdb::Query::Setup['localhost'] } + + ## messages db + ## store messages to the clients such as payment reminders + ## r/w: webapp + couchdb::create_db { 'messages': + members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }", + require => Couchdb::Query::Setup['localhost'] + } } diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 137b661f..e4c0211b 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -107,4 +107,7 @@ class site_couchdb { } if $couchdb_backup { include site_couchdb::backup } + + include site_check_mk::agent::couchdb + include site_check_mk::agent::tapicero } diff --git a/puppet/modules/site_haproxy/files/haproxy-stats.cfg b/puppet/modules/site_haproxy/files/haproxy-stats.cfg new file mode 100644 index 00000000..e6335ba2 --- /dev/null +++ b/puppet/modules/site_haproxy/files/haproxy-stats.cfg @@ -0,0 +1,6 @@ +# provide access to stats for the nagios plugin +listen stats 127.0.0.1:8000 + mode http + stats enable + stats uri /haproxy + diff --git a/puppet/modules/site_haproxy/manifests/init.pp b/puppet/modules/site_haproxy/manifests/init.pp index 602e26be..1a681373 100644 --- a/puppet/modules/site_haproxy/manifests/init.pp +++ b/puppet/modules/site_haproxy/manifests/init.pp @@ -22,4 +22,11 @@ class site_haproxy { } } + # monitor haproxy + concat::fragment { 'stats': + target => '/etc/haproxy/haproxy.cfg', + order => '90', + source => 'puppet:///modules/site_haproxy/haproxy-stats.cfg'; + } + include site_check_mk::agent::haproxy } diff --git a/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg b/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg index 753d1610..9bd3da28 100644 --- a/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg +++ b/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg @@ -25,6 +25,9 @@ log_file=/var/log/nagios3/nagios.log # Puppet-managed configuration files cfg_dir=/etc/nagios3/conf.d +# check-mk managed configuration files +cfg_dir=/etc/nagios3/local + # Debian also defaults to using the check commands defined by the debian # nagios-plugins package cfg_dir=/etc/nagios-plugins/config @@ -248,7 +251,7 @@ log_archive_path=/var/log/nagios3/archives # If you want messages logged to the syslog facility, as well as the # Nagios log file set this option to 1. If not, set it to 0. -use_syslog=1 +use_syslog=0 diff --git a/puppet/modules/site_nagios/manifests/add_host.pp b/puppet/modules/site_nagios/manifests/add_host_services.pp index 94352de4..279809d1 100644 --- a/puppet/modules/site_nagios/manifests/add_host.pp +++ b/puppet/modules/site_nagios/manifests/add_host_services.pp @@ -1,17 +1,13 @@ -define site_nagios::add_host ( +define site_nagios::add_host_services ( + $domain_full_suffix, $domain_internal, $ip_address, $services, + $ssh_port, $openvpn_gateway_address='' ) { $nagios_hostname = $domain_internal - # Add Nagios host - nagios_host { $nagios_hostname: - address => $ip_address, - use => 'generic-host', - } - # Add Nagios service # First, we need to turn the serice array into hash, using a "hash template" diff --git a/puppet/modules/site_nagios/manifests/init.pp b/puppet/modules/site_nagios/manifests/init.pp index c3cfa02e..eb08cdcb 100644 --- a/puppet/modules/site_nagios/manifests/init.pp +++ b/puppet/modules/site_nagios/manifests/init.pp @@ -1,6 +1,6 @@ class site_nagios { tag 'leap_service' Class['site_config::default'] -> Class['site_nagios'] - + include site_nagios::server } diff --git a/puppet/modules/site_nagios/manifests/server.pp b/puppet/modules/site_nagios/manifests/server.pp index 3e1ef7e7..9aae8ae8 100644 --- a/puppet/modules/site_nagios/manifests/server.pp +++ b/puppet/modules/site_nagios/manifests/server.pp @@ -7,22 +7,30 @@ class site_nagios::server inherits nagios::base { $nagios_hiera = hiera('nagios') $nagiosadmin_pw = htpasswd_sha1($nagios_hiera['nagiosadmin_pw']) - $hosts = $nagios_hiera['hosts'] + $nagios_hosts = $nagios_hiera['hosts'] include nagios::defaults include nagios::base - #Class ['nagios'] -> Class ['nagios::defaults'] - class {'nagios::apache': + class {'nagios': + # don't manage apache class from nagios, cause we already include + # it in site_apache::common + httpd => 'absent', allow_external_cmd => true, stored_config => false, - #before => Class ['nagios::defaults'] } + file { '/etc/apache2/conf.d/nagios3.conf': + ensure => link, + target => '/usr/share/doc/nagios3-common/examples/apache2.conf', + notify => Service['apache'] + } + + include site_apache::common include site_apache::module::headers File ['nagios_htpasswd'] { source => undef, - content => "nagiosadmin:$nagiosadmin_pw", + content => "nagiosadmin:${nagiosadmin_pw}", mode => '0640', } @@ -35,7 +43,9 @@ class site_nagios::server inherits nagios::base { group => 'nagios', } - create_resources ( site_nagios::add_host, $hosts ) + create_resources ( site_nagios::add_host_services, $nagios_hosts ) + include site_nagios::server::apache + include site_check_mk::server include site_shorewall::monitor } diff --git a/puppet/modules/site_nagios/manifests/server/apache.pp b/puppet/modules/site_nagios/manifests/server/apache.pp new file mode 100644 index 00000000..8dbc7e9b --- /dev/null +++ b/puppet/modules/site_nagios/manifests/server/apache.pp @@ -0,0 +1,7 @@ +class site_nagios::server::apache { + include x509::variables + include site_config::x509::commercial::cert + include site_config::x509::commercial::key + include site_config::x509::commercial::ca + +} diff --git a/puppet/modules/site_nagios/manifests/server/purge.pp b/puppet/modules/site_nagios/manifests/server/purge.pp index 39735cd3..1c12cfb0 100644 --- a/puppet/modules/site_nagios/manifests/server/purge.pp +++ b/puppet/modules/site_nagios/manifests/server/purge.pp @@ -1,7 +1,18 @@ -class site_nagios::server::purge { - exec {'purge_conf.d': - command => '/bin/rm -rf /etc/nagios3/conf.d/*', - onlyif => 'test -e /etc/nagios3/conf.d' +class site_nagios::server::purge inherits nagios::base { + # we don't want to get /etc/nagios3 and /etc/nagios3/conf.d + # purged, cause the check-mk-config-nagios3 package + # places its templates in /etc/nagios3/conf.d/check_mk, + # and check_mk -O updated it's nagios config in /etc/nagios3/conf.d/check_mk + File['nagios_cfgdir'] { + purge => false + } + File['nagios_confd'] { + purge => false } + # only purge find in the /etc/nagios3/conf.d/ dir, not in any subdir + exec {'purge_conf.d': + command => '/usr/bin/find /etc/nagios3/conf.d/ -maxdepth 1 -type f -exec rm {} \;', + onlyif => '/usr/bin/find /etc/nagios3/conf.d/ -maxdepth 1 -type f | grep -q "/etc/nagios3/conf.d"' + } } diff --git a/puppet/modules/site_sshd/manifests/authorized_keys.pp b/puppet/modules/site_sshd/manifests/authorized_keys.pp index c18f691c..f36fe20f 100644 --- a/puppet/modules/site_sshd/manifests/authorized_keys.pp +++ b/puppet/modules/site_sshd/manifests/authorized_keys.pp @@ -1,4 +1,7 @@ define site_sshd::authorized_keys ($keys, $ensure = 'present', $home = '') { + # We use a custom define here to deploy the authorized_keys file + # cause puppet doesn't allow purgin before populating this file + # (see https://tickets.puppetlabs.com/browse/PUP-1174) # This line allows default homedir based on $title variable. # If $home is empty, the default is used. $homedir = $home ? {'' => "/home/${title}", default => $home} diff --git a/puppet/modules/site_sshd/templates/authorized_keys.erb b/puppet/modules/site_sshd/templates/authorized_keys.erb index 3c65e8ab..69f4d8e6 100644 --- a/puppet/modules/site_sshd/templates/authorized_keys.erb +++ b/puppet/modules/site_sshd/templates/authorized_keys.erb @@ -2,5 +2,9 @@ # all manually added keys will be overridden <% keys.sort.each do |user, hash| -%> +<% if user == 'monitor' -%> +command="/usr/bin/check_mk_agent",no-port-forwarding,no-x11-forwarding,no-agent-forwarding,no-pty,no-user-rc, <%=hash['type']-%> <%=hash['key']%> <%=user%> +<% else -%> <%=hash['type']-%> <%=hash['key']%> <%=user%> +<% end -%> <% end -%> diff --git a/puppet/modules/site_stunnel/manifests/clients.pp b/puppet/modules/site_stunnel/manifests/clients.pp index 791fdbc5..837665a3 100644 --- a/puppet/modules/site_stunnel/manifests/clients.pp +++ b/puppet/modules/site_stunnel/manifests/clients.pp @@ -28,4 +28,6 @@ define site_stunnel::clients ( Class['Site_config::X509::Ca'] ]; } + + include site_check_mk::agent::stunnel } diff --git a/puppet/modules/site_webapp/manifests/apache.pp b/puppet/modules/site_webapp/manifests/apache.pp index d327877a..21243d34 100644 --- a/puppet/modules/site_webapp/manifests/apache.pp +++ b/puppet/modules/site_webapp/manifests/apache.pp @@ -7,19 +7,8 @@ class site_webapp::apache { $web_domain = hiera('domain') $domain_name = $web_domain['name'] - include x509::variables - include site_config::x509::commercial::cert - include site_config::x509::commercial::key - include site_config::x509::commercial::ca - - Class['Site_config::X509::Commercial::Key'] ~> Service[apache] - Class['Site_config::X509::Commercial::Cert'] ~> Service[apache] - Class['Site_config::X509::Commercial::Ca'] ~> Service[apache] - - class { '::apache': no_default_site => true, ssl => true } - + include site_apache::common include site_apache::module::headers - include site_apache::module::rewrite include site_apache::module::alias include site_apache::module::expires include site_apache::module::removeip @@ -27,11 +16,6 @@ class site_webapp::apache { class { 'passenger': use_munin => false } apache::vhost::file { - 'leap_webapp': - content => template('site_apache/vhosts.d/leap_webapp.conf.erb') - } - - apache::vhost::file { 'api': content => template('site_apache/vhosts.d/api.conf.erb') } diff --git a/puppet/modules/site_webapp/manifests/init.pp b/puppet/modules/site_webapp/manifests/init.pp index f8216aa4..d02a7261 100644 --- a/puppet/modules/site_webapp/manifests/init.pp +++ b/puppet/modules/site_webapp/manifests/init.pp @@ -157,5 +157,5 @@ class site_webapp { } include site_shorewall::webapp - + include site_check_mk::agent::webapp } diff --git a/puppet/modules/soledad/manifests/init.pp b/puppet/modules/soledad/manifests/init.pp index 7d44c8b4..7cf0b729 100644 --- a/puppet/modules/soledad/manifests/init.pp +++ b/puppet/modules/soledad/manifests/init.pp @@ -26,5 +26,4 @@ class soledad { group => 'soledad', require => User['soledad']; } - } diff --git a/puppet/modules/soledad/manifests/server.pp b/puppet/modules/soledad/manifests/server.pp index 572cad44..1137080f 100644 --- a/puppet/modules/soledad/manifests/server.pp +++ b/puppet/modules/soledad/manifests/server.pp @@ -59,4 +59,5 @@ class soledad::server { } include site_shorewall::soledad + include site_check_mk::agent::soledad } |