Don't use check_mk logwatch to watch bigcouch logs anymore (#7375)

The rationale here is:

- bigcouch/its included erlang version is incredibly noisy and spits out
  warnings/error msgs all the time
- it uses the worst logging format i ever saw, multiple lines directly
  to a file (couch 2.0 uses lager as logging backend which can log to
  syslog)
- trying to sort out the false positives will take too much time,
  and who knows which of them will be resolved in couch 1.6/2.0

Change-Id: Idbe6b37a19cd65ce31a50d4c28eedb4cf15ba3b5

2 files changed, 30 insertions, 5 deletions

diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp
index abfc7ad0..8de5121b 100644
--- a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp
+++ b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp
@@ -1,9 +1,17 @@
+# configure logwatch and nagios checks for couchdb
 class site_check_mk::agent::couchdb {
 
-  # watch logs
-  file { '/etc/check_mk/logwatch.d/bigcouch.cfg':
-    source => 'puppet:///modules/site_check_mk/agent/logwatch/bigcouch.cfg',
-  }
+  # watch bigcouch logs
+  # currently disabled because bigcouch is too noisy
+  # see https://leap.se/code/issues/7375 for more details
+  # and site_config::remove_files for removing leftovers
+  #file { '/etc/check_mk/logwatch.d/bigcouch.cfg':
+  #  source => 'puppet:///modules/site_check_mk/agent/logwatch/bigcouch.cfg',
+  #}
+
+  # check syslog msg from:
+  # - empd
+  # - /usr/local/bin/couch-doc-update
   concat::fragment { 'syslog_couchdb':
     source  => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/couchdb.cfg',
     target  => '/etc/check_mk/logwatch.d/syslog.cfg',
diff --git a/puppet/modules/site_config/manifests/remove_files.pp b/puppet/modules/site_config/manifests/remove_files.pp
index b339e6af..a9a0c8bf 100644
--- a/puppet/modules/site_config/manifests/remove_files.pp
+++ b/puppet/modules/site_config/manifests/remove_files.pp
@@ -46,5 +46,22 @@ class site_config::remove_files {
       onlyif  => "/bin/grep -qe 'leap_mx.log' /etc/check_mk/logwatch.state"
   }
 
-
+  # Don't use check_mk logwatch to watch bigcouch logs anymore
+  # see https://leap.se/code/issues/7375 for more details
+  file { '/etc/check_mk/logwatch.d/bigcouch.cfg':
+    ensure => absent,
+    notify => [
+      Exec['remove_bigcouch_logwatch_spoolfiles'],
+      Exec['remove_bigcouch_logwatch_stateline']
+    ]
+  }
+  # remove leftover bigcouch logwatch spool files
+  exec { 'remove_bigcouch_logwatch_spoolfiles':
+    command     => 'find /var/lib/check_mk/logwatch -name \'\\opt\\bigcouch\\var\\log\\bigcouch.log\' -exec rm {} \;',
+    refreshonly => true,
+  }
+  exec { 'remove_bigcouch_logwatch_stateline':
+    command     => "sed -i '/bigcouch.log/d' /etc/check_mk/logwatch.state",
+    refreshonly => true,
+  }
 }