diff options
| author | Micah Anderson <micah@leap.se> | 2014-04-04 10:37:09 -0400 |
|---|---|---|
| committer | Micah Anderson <micah@leap.se> | 2014-04-04 10:37:09 -0400 |
| commit | 6af957a1c20f75a827655a3cd75e40a03cffe7c4 (patch) | |
| tree | e5995b0c4b53583fd9d16857f66f81137dccbf73 | |
| parent | 7451213d5e0772d0d6cba4613bf66792da495909 (diff) | |
| parent | 1551f785c5c7c515781995928eec7659365d8988 (diff) | |
Merge branch '0.5' into develop
Conflicts:
provider_base/services/tor.json
Change-Id: I826579945a0d93c43384f0fd12c9833762b084cf
101 files changed, 1976 insertions, 237 deletions
diff --git a/.gitmodules b/.gitmodules index cc8afdb5..7005b770 100644 --- a/.gitmodules +++ b/.gitmodules @@ -88,3 +88,6 @@ [submodule "puppet/modules/sysctl"] path = puppet/modules/sysctl url = https://leap.se/git/puppet_sysctl +[submodule "puppet/modules/check_mk"] + path = puppet/modules/check_mk + url = https://leap.se/git/puppet_check_mk diff --git a/bin/run_tests b/bin/run_tests index a44fcdcf..f4fb0157 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -60,6 +60,20 @@ class LeapTest < MiniTest::Unit::TestCase end # + # returns all the test classes, sorted in dependency order. + # + def self.test_classes + classes = ObjectSpace.each_object(Class).select {|test_class| + test_class.ancestors.include?(self) + } + return TestDependencyGraph.new(classes).sorted + end + + def self.tests + self.instance_methods.grep(/^test_/).sort + end + + # # The default pass just does an `assert true`. In our case, we want to make the passes more explicit. # def pass @@ -272,9 +286,16 @@ class LeapRunner < MiniTest::Unit # LeapTest._run # def _run args = [] - suites = LeapTest.send "test_suites" + if $pinned_test_class + suites = [$pinned_test_class] + if $pinned_test_method + options.merge!(:filter => $pinned_test_method.to_s) + end + else + suites = LeapTest.send "test_suites" + suites = TestDependencyGraph.new(suites).sorted + end output.sync = true - suites = TestDependencyGraph.new(suites).sorted results = _run_suites(suites, :test) @test_count = results.inject(0) { |sum, (tc, _)| sum + tc } @assertion_count = results.inject(0) { |sum, (_, ac)| sum + ac } @@ -318,45 +339,70 @@ class LeapRunner < MiniTest::Unit end # - # override default status slightly + # override default status summary # def status(io = self.output) - format = "%d tests, %d assertions, %d passes, %d failures, %d errors, %d skips" - output.puts format % [test_count, assertion_count, passes, failures, errors, skips] + if $output_format == :human + format = "%d tests, %d assertions, %d passes, %d failures, %d errors, %d skips" + output.puts format % [test_count, assertion_count, passes, failures, errors, skips] + end end # # returns a string for a PASS, SKIP, or FAIL error # def report_line(prefix, klass, meth, e=nil, message=nil) + msg_txt = nil if message message = message.sub(/http:\/\/([a-z_]+):([a-zA-Z0-9_]+)@/, "http://\\1:password@") - indent = "\n " - msg_txt = indent + message.split("\n").join(indent) + if $output_format == :human + indent = "\n " + msg_txt = indent + message.split("\n").join(indent) + else + msg_txt = message.gsub("\n", ' ') + end end - if e && message - output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)} [#{File.basename(location(e))}]:#{msg_txt}" - elsif message - output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)}:#{msg_txt}" - else - output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)}" + + if $output_format == :human + if e && msg_txt + output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)} [#{File.basename(location(e))}]:#{msg_txt}" + elsif msg_txt + output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)}:#{msg_txt}" + else + output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)}" + end + # I don't understand at all why, but adding a very tiny sleep here will + sleep(0.0001) # keep lines from being joined together by the logger. output.flush doesn't. + elsif $output_format == :checkmk + code = CHECKMK_CODES[prefix] + msg_txt ||= "Success" if prefix == "PASS" + if e && msg_txt + output.puts "#{code} #{klass.name}/#{machine_readable(meth)} - [#{File.basename(location(e))}]:#{msg_txt}" + elsif msg_txt + output.puts "#{code} #{klass.name}/#{machine_readable(meth)} - #{msg_txt}" + else + output.puts "#{code} #{klass.name}/#{machine_readable(meth)} - no message" + end end - # I don't understand at all why, but adding a very tiny sleep here will - sleep(0.0001) # keep lines from being joined together by the logger. output.flush doesn't. end private + CHECKMK_CODES = {"PASS" => 0, "SKIP" => 1, "FAIL" => 2, "ERROR" => 3} + # # Converts snake_case and CamelCase to something more pleasant for humans to read. # def readable(str) - str.gsub(/([A-Z]+)([A-Z][a-z])/, '\1 \2'). - gsub(/([a-z])([A-Z])/, '\1 \2'). + str. gsub(/_/, ' '). - sub(/^test (\d* )?/i, ''). - downcase.capitalize + sub(/^test (\d* )?/i, '') end + + def machine_readable(str) + str.sub(/^test_(\d+_)?/i, '') + end + end ## @@ -380,7 +426,12 @@ class TestDependencyGraph end def tsort_each_child(test_class_name, &block) - @dependencies[test_class_name].each(&block) + if @dependencies[test_class_name] + @dependencies[test_class_name].each(&block) + else + puts "ERROR: bad dependency, no such class `#{test_class_name}`" + exit(1) + end end def sorted @@ -391,33 +442,90 @@ class TestDependencyGraph end ## -## RUN THE TESTS +## COMMAND LINE ACTIONS ## -# load node data from hiera file -if File.exists?('/etc/leap/hiera.yaml') - $node = YAML.load_file('/etc/leap/hiera.yaml') -else - $node = {"services" => [], "dummy" => true} +def die(test, msg) + if $output_format == :human + puts "ERROR in test `#{test}`: #{msg}" + elsif $output_format == :checkmk + puts "3 #{test} - #{msg}" + end + exit(1) +end + +def print_help + puts ["USAGE: run_tests [OPTIONS]", + " --continue Don't halt on an error, but continue to the next test.", + " --checkmk Print test results in checkmk format (must come before --test).", + " --test TEST Run only the test with name TEST.", + " --list-tests Prints the names of all available tests and exit."].join("\n") + exit(0) +end + +def list_tests + LeapTest.test_classes.each do |test_class| + test_class.tests.each do |test| + puts test_class.name + "/" + test.to_s.sub(/^test_(\d+_)?/, '') + end + end + exit(0) end -# load all test classes -Dir[File.expand_path('../../tests/white-box/*.rb', __FILE__)].each do |test_file| - begin - require test_file - rescue SkipTest +def pin_test_name(name) + test_class, test_name = name.split('/') + $pinned_test_class = LeapTest.test_classes.detect{|c| c.name == test_class} + unless $pinned_test_class + die name, "there is no test class `#{test_class}`" + end + if test_name + $pinned_test_method = $pinned_test_class.tests.detect{|m| m.to_s =~ /^test_(\d+_)?#{Regexp.escape(test_name)}$/} + unless $pinned_test_method + die name, "there is no test `#{test_name}` in class `#{test_class}`" + end end end -# parse command line options -$halt_on_failure = true -loop do - case ARGV[0] - when '--continue' then ARGV.shift; $halt_on_failure = false - else break +def run_tests + MiniTest::Unit.runner = LeapRunner.new + MiniTest::Unit.new.run +end + +## +## MAIN +## + +def main + # load node data from hiera file + if File.exists?('/etc/leap/hiera.yaml') + $node = YAML.load_file('/etc/leap/hiera.yaml') + else + $node = {"services" => [], "dummy" => true} + end + + # load all test classes + this_file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__ + Dir[File.expand_path('../../tests/white-box/*.rb', this_file)].each do |test_file| + begin + require test_file + rescue SkipTest + end + end + + # parse command line options + $halt_on_failure = true + $output_format = :human + loop do + case ARGV[0] + when '--continue' then ARGV.shift; $halt_on_failure = false; + when '--checkmk' then ARGV.shift; $output_format = :checkmk; $halt_on_failure = false + when '--help' then print_help + when '--test' then ARGV.shift; pin_test_name(ARGV.shift) + when '--list-tests' then list_tests + else break + end end + run_tests end -# run some tests already -MiniTest::Unit.runner = LeapRunner.new -MiniTest::Unit.new.run +main()
\ No newline at end of file diff --git a/platform.rb b/platform.rb index ed68e595..689c58b7 100644 --- a/platform.rb +++ b/platform.rb @@ -5,7 +5,7 @@ Leap::Platform.define do self.version = "0.4.0" - self.compatible_cli = "1.4.0".."1.99" + self.compatible_cli = "1.5.0".."1.99" # # the facter facts that should be gathered @@ -27,6 +27,7 @@ Leap::Platform.define do # input config files :common_config => 'common.json', :provider_config => 'provider.json', + :provider_env_config => 'provider.#{arg}.json', :secrets_config => 'secrets.json', :node_config => 'nodes/#{arg}.json', :service_config => 'services/#{arg}.json', diff --git a/provider_base/common.json b/provider_base/common.json index 07a58bba..a4d9c5f2 100644 --- a/provider_base/common.json +++ b/provider_base/common.json @@ -3,10 +3,10 @@ "environment": null, "services": [], "tags": [], - "contacts": "= global.provider.contacts.default", + "contacts": "= provider.contacts.default", "domain": { - "full_suffix": "= global.provider.domain", - "internal_suffix": "= global.provider.domain_internal", + "full_suffix": "= provider.domain", + "internal_suffix": "= provider.domain_internal", "full": "= node.name + '.' + domain.full_suffix", "internal": "= node.name + '.' + domain.internal_suffix", "name": "= node.name + '.' + (dns.public ? domain.full_suffix : domain.internal_suffix)" diff --git a/provider_base/files/service-definitions/provider.json.erb b/provider_base/files/service-definitions/provider.json.erb index 5d4c63a0..97454f81 100644 --- a/provider_base/files/service-definitions/provider.json.erb +++ b/provider_base/files/service-definitions/provider.json.erb @@ -1,6 +1,6 @@ <%= # grab some fields from provider.json - hsh = global.provider.pick( + hsh = provider.pick( :languages, :description, :name, :enrollment_policy, :default_language, :service ) diff --git a/provider_base/files/service-definitions/v1/eip-service.json.erb b/provider_base/files/service-definitions/v1/eip-service.json.erb index feaea25b..3b8976fd 100644 --- a/provider_base/files/service-definitions/v1/eip-service.json.erb +++ b/provider_base/files/service-definitions/v1/eip-service.json.erb @@ -27,6 +27,7 @@ hsh["version"] = 1 locations = {} gateways = [] + configuration = nil nodes_like_me[:services => 'openvpn'].each_node do |node| if node.openvpn.allow_limited && node.openvpn.allow_unlimited gateways << add_gateway(node, locations, :ip => node.openvpn.gateway_address, :limited => false) @@ -36,13 +37,13 @@ elsif node.openvpn.allow_limited gateways << add_gateway(node, locations, :ip => node.openvpn.gateway_address, :limited => true) end + if configuration && node.openvpn.configuration != configuration + log :error, "OpenVPN nodes in the environment `#{node.environment}` have conflicting `openvpn.configuration` values. This will result in bad errors." + end + configuration = node.openvpn.configuration end hsh["gateways"] = gateways.compact hsh["locations"] = locations - hsh["openvpn_configuration"] = { - "tls-cipher" => "DHE-RSA-AES128-SHA", - "auth" => "SHA1", - "cipher" => "AES-128-CBC" - } + hsh["openvpn_configuration"] = configuration JSON.sorted_generate hsh %>
\ No newline at end of file diff --git a/provider_base/provider.json b/provider_base/provider.json index 8864e24d..fa69318b 100644 --- a/provider_base/provider.json +++ b/provider_base/provider.json @@ -23,17 +23,17 @@ ], "default_service_level": 1, "bandwidth_limit": 102400, - "allow_free": "= global.provider.service.levels.select {|l| l['rate'].nil?}.any?", - "allow_paid": "= global.provider.service.levels.select {|l| !l['rate'].nil?}.any?", - "allow_anonymous": "= global.provider.service.levels.select {|l| l['name'] == 'anonymous'}.any?", - "allow_registration": "= global.provider.service.levels.select {|l| l['name'] != 'anonymous'}.any?", - "allow_limited_bandwidth": "= global.provider.service.levels.select {|l| l['bandwidth'] == 'limited'}.any?", - "allow_unlimited_bandwidth": "= global.provider.service.levels.select {|l| l['bandwidth'].nil?}.any?" + "allow_free": "= provider.service.levels.select {|l| l['rate'].nil?}.any?", + "allow_paid": "= provider.service.levels.select {|l| !l['rate'].nil?}.any?", + "allow_anonymous": "= provider.service.levels.select {|l| l['name'] == 'anonymous'}.any?", + "allow_registration": "= provider.service.levels.select {|l| l['name'] != 'anonymous'}.any?", + "allow_limited_bandwidth": "= provider.service.levels.select {|l| l['bandwidth'] == 'limited'}.any?", + "allow_unlimited_bandwidth": "= provider.service.levels.select {|l| l['bandwidth'].nil?}.any?" }, "ca": { - "name": "= global.provider.ca.organization + ' Root CA'", - "organization": "= global.provider.name[global.provider.default_language]", - "organizational_unit": "= 'https://' + global.provider.domain", + "name": "= provider.ca.organization + ' Root CA'", + "organization": "= provider.name[provider.default_language]", + "organizational_unit": "= 'https://' + provider.domain", "bit_size": 4096, "digest": "SHA256", "life_span": "10y", diff --git a/provider_base/services/couchdb.json b/provider_base/services/couchdb.json index 0cb044ef..5f1b5381 100644 --- a/provider_base/services/couchdb.json +++ b/provider_base/services/couchdb.json @@ -48,6 +48,9 @@ "password": "= secret :couch_webapp_password", "salt": "= hex_secret :couch_webapp_password_salt, 128" } + }, + "webapp": { + "nagios_test_pw": "= secret :nagios_test_password" } } } diff --git a/provider_base/services/monitor.json b/provider_base/services/monitor.json index 53e6b1f1..03f6c6d1 100644 --- a/provider_base/services/monitor.json +++ b/provider_base/services/monitor.json @@ -1,13 +1,22 @@ { "nagios": { "nagiosadmin_pw": "= secret :nagios_admin_password", - "hosts": "= nodes_like_me[:services => '!monitor'].pick_fields('domain.internal', 'ip_address', 'services', 'openvpn.gateway_address')" + "hosts": "= (self.environment == 'local' ? nodes_like_me : nodes[:environment => '!local']).pick_fields('domain.internal', 'domain.full_suffix', 'ip_address', 'services', 'openvpn.gateway_address', 'ssh.port')" }, - "hosts": "= hosts_file(nodes_like_me[:services => '!monitor'])", + "hosts": "= self.environment == 'local' ? hosts_file(nodes_like_me) : hosts_file(nodes[:environment => '!local'])", "ssh": { "monitor": { "username": "= Leap::Platform.monitor_username", "private_key": "= file(:monitor_priv_key)" } + }, + "x509": { + "use": true, + "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", + "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", + "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'", + "commercial_cert": "= file [:commercial_cert, domain.full_suffix]", + "commercial_key": "= file [:commercial_key, domain.full_suffix]", + "commercial_ca_cert": "= try_file :commercial_ca_cert" } } diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json index 7e3f20ba..731dee9a 100644 --- a/provider_base/services/mx.json +++ b/provider_base/services/mx.json @@ -10,7 +10,7 @@ "password": "= secret :couch_leap_mx_password", "salt": "= hex_secret :couch_leap_mx_password_salt, 128" }, - "mx_nodes": "= nodes['services' => 'mx']['environment' => '!local'].field('ip_address')", + "mynetworks": "= nodes['environment' => '!local'].map{|name, n| [n.ip_address, (global.facts[name]||{})['ec2_public_ipv4']]}.flatten.compact.uniq", "x509": { "use": true, "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", diff --git a/provider_base/services/openvpn.json b/provider_base/services/openvpn.json index 5d77f946..e5b97ed9 100644 --- a/provider_base/services/openvpn.json +++ b/provider_base/services/openvpn.json @@ -14,10 +14,15 @@ "filter_dns": false, "adblock": false, "user_ips": false, - "allow_limited": "= global.provider.service.allow_limited_bandwidth", - "allow_unlimited": "= global.provider.service.allow_unlimited_bandwidth", - "limited_prefix": "= global.provider.ca.client_certificates.limited_prefix", - "unlimited_prefix": "= global.provider.ca.client_certificates.unlimited_prefix", - "rate_limit": "= openvpn.allow_limited ? global.provider.service.bandwidth_limit : nil" + "allow_limited": "= provider.service.allow_limited_bandwidth", + "allow_unlimited": "= provider.service.allow_unlimited_bandwidth", + "limited_prefix": "= provider.ca.client_certificates.limited_prefix", + "unlimited_prefix": "= provider.ca.client_certificates.unlimited_prefix", + "rate_limit": "= openvpn.allow_limited ? provider.service.bandwidth_limit : nil", + "configuration": { + "tls-cipher": "TLS-DHE-RSA-WITH-AES-128-CBC-SHA", + "auth": "SHA1", + "cipher": "AES-128-CBC" + } } } diff --git a/provider_base/services/static.json b/provider_base/services/static.json new file mode 100644 index 00000000..d9155a84 --- /dev/null +++ b/provider_base/services/static.json @@ -0,0 +1,6 @@ +{ + "static": { + "formats": "=> (self.static.domains||{}).values.collect{|d| (d.locations||{}).values.collect{|l|l['format']}}.flatten.uniq" + }, + "service_type": "public_service" +}
\ No newline at end of file diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index 8974f851..29c0cbf9 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -8,10 +8,10 @@ "salt": "= hex_secret :couch_webapp_password_salt, 128" }, "customization_dir": "= file_path 'webapp'", - "client_certificates": "= global.provider.ca.client_certificates", - "allow_limited_certs": "= global.provider.service.allow_limited_bandwidth", - "allow_unlimited_certs": "= global.provider.service.allow_unlimited_bandwidth", - "allow_anonymous_certs": "= global.provider.service.allow_anonymous", + "client_certificates": "= provider.ca.client_certificates", + "allow_limited_certs": "= provider.service.allow_limited_bandwidth", + "allow_unlimited_certs": "= provider.service.allow_unlimited_bandwidth", + "allow_anonymous_certs": "= provider.service.allow_anonymous", "secret_token": "= secret :webapp_secret_token", "api_version": 1, "secure": false, @@ -19,7 +19,11 @@ "source": "https://leap.se/git/leap_web", "revision": "origin/master" }, - "client_version": "= global.provider.client_version" + "client_version": "= provider.client_version", + "nagios_test_user": { + "username": "nagios_test", + "password": "= secret :nagios_test_password" + } }, "stunnel": { "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" diff --git a/provider_base/tags/development.json b/provider_base/tags/development.json index 6d4f9e25..d9c2c007 100644 --- a/provider_base/tags/development.json +++ b/provider_base/tags/development.json @@ -1,7 +1,7 @@ { "environment": "development", "domain": { - "full_suffix": "= 'dev.' + global.provider.domain", - "internal_suffix": "= 'dev.' + global.provider.domain_internal" + "full_suffix": "= 'dev.' + provider.domain", + "internal_suffix": "= 'dev.' + provider.domain_internal" } }
\ No newline at end of file diff --git a/puppet/bin/apply_on_node.sh b/puppet/bin/apply_on_node.sh new file mode 100755 index 00000000..09e5b035 --- /dev/null +++ b/puppet/bin/apply_on_node.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# Script to use on a node for debugging +# Usage: ./apply_on_node.sh <puppet parameters> +# +# Example: ./apply_on_node.sh --debug --verbose + +ROOTDIR='/srv/leap' +PLATFORM="$ROOTDIR" +MODULEPATH="$PLATFORM/puppet/modules" +LOG=/var/log/leap.log + +# example tags to use +#TAGS='--tags=leap_base,leap_service,leap_slow' +#TAGS='--tags=leap_base,leap_slow' +#TAGS='--tags=leap_base,leap_service' + +####### +# Setup +####### + +puppet apply -v --confdir $PLATFORM/puppet --libdir $PLATFORM/puppet/lib --modulepath=$MODULEPATH $PLATFORM/puppet/manifests/setup.pp $TAGS $@ |tee $LOG 2>&1 + +######### +# site.pp +######### + +puppet apply -v --confdir $PLATFORM/puppet --libdir $PLATFORM/puppet/lib --modulepath=$MODULEPATH $PLATFORM/puppet/manifests/site.pp $TAGS $@ |tee $LOG 2>&1 + + diff --git a/puppet/manifests/site.pp b/puppet/manifests/site.pp index def0a642..ff8f8c6c 100644 --- a/puppet/manifests/site.pp +++ b/puppet/manifests/site.pp @@ -38,3 +38,8 @@ if $services =~ /\bmx\b/ { include site_mx } +if $services =~ /\bstatic\b/ { + include site_static +} + +include site_config::packages::uninstall
\ No newline at end of file diff --git a/puppet/modules/check_mk b/puppet/modules/check_mk new file mode 160000 +Subproject 5c11597a055858b5ddc1ce8f7f8db249f5f1b33 diff --git a/puppet/modules/leap_mx/templates/mx.conf.erb b/puppet/modules/leap_mx/templates/mx.conf.erb index bf1e6421..e05bc150 100644 --- a/puppet/modules/leap_mx/templates/mx.conf.erb +++ b/puppet/modules/leap_mx/templates/mx.conf.erb @@ -1,5 +1,5 @@ [mail1] -path=/var/mail/vmail +path=/var/mail/vmail/Maildir recursive=True [couchdb] @@ -12,4 +12,4 @@ port=<%= @couchdb_port %> port=4242 [check recipient] -port=2244
\ No newline at end of file +port=2244 diff --git a/puppet/modules/site_apache/manifests/common.pp b/puppet/modules/site_apache/manifests/common.pp new file mode 100644 index 00000000..72f24838 --- /dev/null +++ b/puppet/modules/site_apache/manifests/common.pp @@ -0,0 +1,26 @@ +class site_apache::common { + # installs x509 cert + key and common config + # that both nagios + leap webapp use + + $web_domain = hiera('domain') + $domain_name = $web_domain['name'] + + include x509::variables + include site_config::x509::commercial::cert + include site_config::x509::commercial::key + include site_config::x509::commercial::ca + + Class['Site_config::X509::Commercial::Key'] ~> Service[apache] + Class['Site_config::X509::Commercial::Cert'] ~> Service[apache] + Class['Site_config::X509::Commercial::Ca'] ~> Service[apache] + + include site_apache::module::rewrite + + class { '::apache': no_default_site => true, ssl => true } + + apache::vhost::file { + 'common': + content => template('site_apache/vhosts.d/common.conf.erb') + } + +} diff --git a/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb index 5f1f4c1d..3360ac59 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb @@ -10,9 +10,10 @@ Listen 0.0.0.0:<%= api_port %> ServerName <%= api_domain %> SSLEngine on - SSLProtocol -all +SSLv3 +TLSv1 - SSLCipherSuite HIGH:MEDIUM:!aNULL:!SSLv2:!MD5:@STRENGTH + SSLProtocol all -SSLv2 SSLHonorCipherOrder on + SSLCompression off + SSLCipherSuite "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-ECDSA-RC4-SHA:AES128:AES256:RC4-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK" SSLCACertificatePath /etc/ssl/certs SSLCertificateChainFile <%= scope.lookupvar('x509::variables::local_CAs') %>/<%= scope.lookupvar('site_config::params::ca_name') %>.crt diff --git a/puppet/modules/site_apache/templates/vhosts.d/leap_webapp.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb index a001552a..ed430510 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/leap_webapp.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb @@ -11,9 +11,10 @@ ServerAlias www.<%= domain %> SSLEngine on - SSLProtocol -all +SSLv3 +TLSv1 - SSLCipherSuite HIGH:MEDIUM:!aNULL:!SSLv2:!MD5:@STRENGTH + SSLProtocol all -SSLv2 SSLHonorCipherOrder on + SSLCompression off + SSLCipherSuite "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-ECDSA-RC4-SHA:AES128:AES256:RC4-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK" SSLCACertificatePath /etc/ssl/certs SSLCertificateChainFile <%= scope.lookupvar('x509::variables::local_CAs') %>/<%= scope.lookupvar('site_config::params::commercial_ca_name') %>.crt @@ -23,13 +24,14 @@ RequestHeader set X_FORWARDED_PROTO 'https' <IfModule mod_headers.c> -<% if @webapp['secure'] -%> +<% if (defined? @services) and (@services.include? 'webapp') and (@webapp['secure']) -%> Header always set Strict-Transport-Security "max-age=31536000; includeSubDomains" <% end -%> Header always unset X-Powered-By Header always unset X-Runtime </IfModule> +<% if (defined? @services) and (@services.include? 'webapp') -%> DocumentRoot /srv/leap/webapp/public RewriteEngine On @@ -52,16 +54,20 @@ ExpiresActive On ExpiresDefault "access plus 1 year" </Location> +<% end -%> + - <% if (defined? @services) and (@services.include? 'monitor') -%> +<% if (defined? @services) and (@services.include? 'monitor') -%> <DirectoryMatch (/usr/share/nagios3/htdocs|/usr/lib/cgi-bin/nagios3|/etc/nagios3/stylesheets)> + <% if (defined? @services) and (@services.include? 'webapp') -%> PassengerEnabled off + <% end -%> AllowOverride all # Nagios won't work with setting this option to "DENY", # as set in conf.d/security (#4169). Therefor we allow # it here, only for nagios. Header set X-Frame-Options: "ALLOW" </DirectoryMatch> - <% end -%> +<% end -%> </VirtualHost> diff --git a/puppet/modules/site_apt/manifests/preferences/check_mk.pp b/puppet/modules/site_apt/manifests/preferences/check_mk.pp new file mode 100644 index 00000000..580e0d3f --- /dev/null +++ b/puppet/modules/site_apt/manifests/preferences/check_mk.pp @@ -0,0 +1,9 @@ +class site_apt::preferences::check_mk { + + apt::preferences_snippet { 'check-mk': + package => 'check-mk-*', + release => "${::lsbdistcodename}-backports", + priority => 999; + } + +} diff --git a/puppet/modules/site_check_mk/files/agent/local_checks/all_hosts/run_node_tests.sh b/puppet/modules/site_check_mk/files/agent/local_checks/all_hosts/run_node_tests.sh new file mode 100644 index 00000000..1dd0afc9 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/local_checks/all_hosts/run_node_tests.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# +# runs node tests + +/srv/leap/bin/run_tests --checkmk diff --git a/puppet/modules/site_check_mk/files/agent/local_checks/mx/check_leap_mx.sh b/puppet/modules/site_check_mk/files/agent/local_checks/mx/check_leap_mx.sh new file mode 100755 index 00000000..b8687c9a --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/local_checks/mx/check_leap_mx.sh @@ -0,0 +1,33 @@ +#!/bin/bash + + +WARN=1 +CRIT=5 + +# in minutes +MAXAGE=10 + +STATUS[0]='OK' +STATUS[1]='Warning' +STATUS[2]='Critical' +CHECKNAME='Leap_MX_Queue' + +WATCHDIR='/var/mail/vmail/Maildir/new/' + + +total=`find $WATCHDIR -type f -mmin +$MAXAGE | wc -l` + +if [ $total -lt $WARN ] +then + exitcode=0 +else + if [ $total -le $CRIT ] + then + exitcode=1 + else + exitcode=2 + fi +fi + +echo "${exitcode} ${CHECKNAME} stale_files=${total} ${STATUS[exitcode]}: ${total} stale files (>=${MAXAGE} min) in ${WATCHDIR}." + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg new file mode 100644 index 00000000..28f333b0 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg @@ -0,0 +1,20 @@ +/opt/bigcouch/var/log/bigcouch.log nocontext=1 +# ignore requests that are fine + I undefined - -.*200$ + I undefined - -.*201$ + I 127.0.0.1 undefined.* ok + I 127.0.0.1 localhost:5984 .* ok + # https://leap.se/code/issues/5246 + I Shutting down group server + # ignore "Uncaught error in HTTP request: {exit, normal}" error + # it's suppressed in later versions of bigcouch anhow + # see https://leap.se/code/issues/5226 + I Uncaught error in HTTP request: {exit,normal} + I Uncaught error in HTTP request: {exit, + C Uncaught error in HTTP request: {error, + C Response abnormally terminated: {nodedown, + C rexi_DOWN,noproc + C rexi_DOWN,noconnection + C error + C Connection attempt from disallowed node + W Apache CouchDB has started diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/leap_mx.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/leap_mx.cfg new file mode 100644 index 00000000..c71c5392 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/leap_mx.cfg @@ -0,0 +1,4 @@ +/var/log/leap_mx.log + W Don't know how to deliver mail + W No public key, stopping the processing chain + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/logwatch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/logwatch.cfg new file mode 100644 index 00000000..4f16d1bd --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/logwatch.cfg @@ -0,0 +1,31 @@ +# This file is managed by Puppet. DO NOT EDIT. + +# logwatch.cfg +# This file configures mk_logwatch. Define your logfiles +# and patterns to be looked for here. + +# Name one or more logfiles +/var/log/messages +# Patterns are indented with one space are prefixed with: +# C: Critical messages +# W: Warning messages +# I: ignore these lines (OK) +# The first match decided. Lines that do not match any pattern +# are ignored + C Fail event detected on md device + I mdadm.*: Rebuild.*event detected + W mdadm\[ + W ata.*hard resetting link + W ata.*soft reset failed (.*FIS failed) + W device-mapper: thin:.*reached low water mark + C device-mapper: thin:.*no free space + +/var/log/auth.log + W sshd.*Corrupted MAC on input + +/var/log/kern.log + C panic + C Oops + W generic protection rip + W .*Unrecovered read error - auto reallocate failed + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg new file mode 100644 index 00000000..623d1e46 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg @@ -0,0 +1,5 @@ +/var/log/soledad.log + C WSGI application error + C Error + C error + W Timing out client: diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg new file mode 100644 index 00000000..f546135a --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg @@ -0,0 +1,7 @@ + C /usr/local/bin/couch-doc-update.*failed + C /usr/local/bin/couch-doc-update.*ERROR +# on one-node bigcouch setups, we'll get this msg +# a lot, so we ignore it here until we fix +# https://leap.se/code/issues/5244 + I epmd: got partial packet only on file descriptor + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg new file mode 100644 index 00000000..d58e876d --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg @@ -0,0 +1,7 @@ +# ignore openvpn TLS initialization errors when clients +# suddenly hangup before properly establishing +# a tls connection + I ovpn-.*TLS Error: Unroutable control packet received from + I ovpn-.*TLS Error: TLS key negotiation failed to occur within 60 seconds (check your network connectivity) + I ovpn-.*TLS Error: TLS handshake failed + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/stunnel.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/stunnel.cfg new file mode 100644 index 00000000..eb3131f2 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/stunnel.cfg @@ -0,0 +1,9 @@ +# check for stunnel failures +# +# these are temporary failures and happen very often, so we +# ignore them until we tuned stunnel timeouts/logging, +# see https://leap.se/code/issues/5218 + I stunnel:.*Connection reset by peer + I stunnel:.*Peer suddenly disconnected + I stunnel:.*Connection refused + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg new file mode 100644 index 00000000..93ce0311 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg @@ -0,0 +1,8 @@ + C tapicero.*RestClient::InternalServerError: +# possible race condition between multiple tapicero +# instances, so we ignore it +# see https://leap.se/code/issues/5168 + I tapicero.*RestClient::PreconditionFailed: + C tapicero.*Creating database.*failed due to: + C tapicero.*failed + W tapicero.*Couch stream ended unexpectedly. diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/webapp.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/webapp.cfg new file mode 100644 index 00000000..00f9c7fd --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/webapp.cfg @@ -0,0 +1,5 @@ +# check for webapp errors + C webapp.*Could not connect to couch database messages due to 401 Unauthorized: {"error":"unauthorized","reason":"You are not a server admin."} +# ignore RoutingErrors that rails throw when it can't handle a url +# see https://leap.se/code/issues/5173 + I webapp.*ActionController::RoutingError diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_header.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_header.cfg new file mode 100644 index 00000000..f60d752b --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_header.cfg @@ -0,0 +1 @@ +/var/log/syslog diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg new file mode 100644 index 00000000..450b9e90 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg @@ -0,0 +1,11 @@ +# some general patterns + C panic + C Oops + I Error: Driver 'pcspkr' is already registered, aborting... + C Error + C error + W generic protection rip + W .*Unrecovered read error - auto reallocate failed +# 401 Unauthorized error logged by webapp and possible other +# applications + C Unauthorized diff --git a/puppet/modules/site_check_mk/files/agent/nagios_plugins/check_unix_open_fds.pl b/puppet/modules/site_check_mk/files/agent/nagios_plugins/check_unix_open_fds.pl new file mode 100755 index 00000000..06163d49 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/nagios_plugins/check_unix_open_fds.pl @@ -0,0 +1,322 @@ +#!/usr/bin/perl -w + +# check_unix_open_fds Nagios Plugin +# +# TComm - Carlos Peris Pla +# +# This nagios plugin is free software, and comes with ABSOLUTELY +# NO WARRANTY. It may be used, redistributed and/or modified under +# the terms of the GNU General Public Licence (see +# http://www.fsf.org/licensing/licenses/gpl.txt). + + +# MODULE DECLARATION + +use strict; +use Nagios::Plugin; + + +# FUNCTION DECLARATION + +sub CreateNagiosManager (); +sub CheckArguments (); +sub PerformCheck (); + + +# CONSTANT DEFINITION + +use constant NAME => 'check_unix_open_fds'; +use constant VERSION => '0.1b'; +use constant USAGE => "Usage:\ncheck_unix_open_fds -w <process_threshold,application_threshold> -c <process_threshold,application_threshold>\n". + "\t\t[-V <version>]\n"; +use constant BLURB => "This plugin checks, in UNIX systems with the command lsof installed and with its SUID bit activated, the number\n". + "of file descriptors opened by an application and its processes.\n"; +use constant LICENSE => "This nagios plugin is free software, and comes with ABSOLUTELY\n". + "no WARRANTY. It may be used, redistributed and/or modified under\n". + "the terms of the GNU General Public Licence\n". + "(see http://www.fsf.org/licensing/licenses/gpl.txt).\n"; +use constant EXAMPLE => "\n\n". + "Example:\n". + "\n". + "check_unix_open_fds -a /usr/local/nagios/bin/ndo2db -w 20,75 -c 25,85\n". + "\n". + "It returns CRITICAL if number of file descriptors opened by ndo2db is higher than 85,\n". + "if not it returns WARNING if number of file descriptors opened by ndo2db is higher \n". + "than 75, if not it returns CRITICAL if number of file descriptors opened by any process\n". + "of ndo2db is higher than 25, if not it returns WARNING if number of file descriptors \n". + "opened by any process of ndo2db is higher than 20.\n". + "In other cases it returns OK if check has been performed succesfully.\n\n"; + + +# VARIABLE DEFINITION + +my $Nagios; +my $Error; +my $PluginResult; +my $PluginOutput; +my @WVRange; +my @CVRange; + + +# MAIN FUNCTION + +# Get command line arguments +$Nagios = &CreateNagiosManager(USAGE, VERSION, BLURB, LICENSE, NAME, EXAMPLE); +eval {$Nagios->getopts}; + +if (!$@) { + # Command line parsed + if (&CheckArguments($Nagios, \$Error, \@WVRange, \@CVRange)) { + # Argument checking passed + $PluginResult = &PerformCheck($Nagios, \$PluginOutput, \@WVRange, \@CVRange) + } + else { + # Error checking arguments + $PluginOutput = $Error; + $PluginResult = UNKNOWN; + } + $Nagios->nagios_exit($PluginResult,$PluginOutput); +} +else { + # Error parsing command line + $Nagios->nagios_exit(UNKNOWN,$@); +} + + + +# FUNCTION DEFINITIONS + +# Creates and configures a Nagios plugin object +# Input: strings (usage, version, blurb, license, name and example) to configure argument parsing functionality +# Return value: reference to a Nagios plugin object + +sub CreateNagiosManager() { + # Create GetOpt object + my $Nagios = Nagios::Plugin->new(usage => $_[0], version => $_[1], blurb => $_[2], license => $_[3], plugin => $_[4], extra => $_[5]); + + # Add argument units + $Nagios->add_arg(spec => 'application|a=s', + help => 'Application path for which you want to check the number of open file descriptors', + required => 1); + + # Add argument warning + $Nagios->add_arg(spec => 'warning|w=s', + help => "Warning thresholds. Format: <process_threshold,application_threshold>", + required => 1); + # Add argument critical + $Nagios->add_arg(spec => 'critical|c=s', + help => "Critical thresholds. Format: <process_threshold,application_threshold>", + required => 1); + + # Return value + return $Nagios; +} + + +# Checks argument values and sets some default values +# Input: Nagios Plugin object +# Output: reference to Error description string, Memory Unit, Swap Unit, reference to WVRange ($_[4]), reference to CVRange ($_[5]) +# Return value: True if arguments ok, false if not + +sub CheckArguments() { + my ($Nagios, $Error, $WVRange, $CVRange) = @_; + my $commas; + my $units; + my $i; + my $firstpos; + my $secondpos; + + # Check Warning thresholds list + $commas = $Nagios->opts->warning =~ tr/,//; + if ($commas !=1){ + ${$Error} = "Invalid Warning list format. One comma is expected."; + return 0; + } + else{ + $i=0; + $firstpos=0; + my $warning=$Nagios->opts->warning; + while ($warning =~ /[,]/g) { + $secondpos=pos $warning; + if ($secondpos - $firstpos==1){ + @{$WVRange}[$i] = "~:"; + } + else{ + @{$WVRange}[$i] = substr $Nagios->opts->warning, $firstpos, ($secondpos-$firstpos-1); + } + $firstpos=$secondpos; + $i++ + } + if (length($Nagios->opts->warning) - $firstpos==0){#La coma es el ultimo elemento del string + @{$WVRange}[$i] = "~:"; + } + else{ + @{$WVRange}[$i] = substr $Nagios->opts->warning, $firstpos, (length($Nagios->opts->warning)-$firstpos); + } + + if (@{$WVRange}[0] !~/^(@?(\d+|(\d+|~):(\d+)?))?$/){ + ${$Error} = "Invalid Process Warning threshold in ${$WVRange[0]}"; + return 0; + }if (@{$WVRange}[1] !~/^(@?(\d+|(\d+|~):(\d+)?))?$/){ + ${$Error} = "Invalid Application Warning threshold in ${$WVRange[1]}"; + return 0; + } + } + + # Check Critical thresholds list + $commas = $Nagios->opts->critical =~ tr/,//; + if ($commas !=1){ + ${$Error} = "Invalid Critical list format. One comma is expected."; + return 0; + } + else{ + $i=0; + $firstpos=0; + my $critical=$Nagios->opts->critical; + while ($critical =~ /[,]/g) { + $secondpos=pos $critical ; + if ($secondpos - $firstpos==1){ + @{$CVRange}[$i] = "~:"; + } + else{ + @{$CVRange}[$i] =substr $Nagios->opts->critical, $firstpos, ($secondpos-$firstpos-1); + } + $firstpos=$secondpos; + $i++ + } + if (length($Nagios->opts->critical) - $firstpos==0){#La coma es el ultimo elemento del string + @{$CVRange}[$i] = "~:"; + } + else{ + @{$CVRange}[$i] = substr $Nagios->opts->critical, $firstpos, (length($Nagios->opts->critical)-$firstpos); + } + + if (@{$CVRange}[0] !~/^(@?(\d+|(\d+|~):(\d+)?))?$/) { + ${$Error} = "Invalid Process Critical threshold in @{$CVRange}[0]"; + return 0; + } + if (@{$CVRange}[1] !~/^(@?(\d+|(\d+|~):(\d+)?))?$/) { + ${$Error} = "Invalid Application Critical threshold in @{$CVRange}[1]"; + return 0; + } + } + + return 1; +} + + +# Performs whole check: +# Input: Nagios Plugin object, reference to Plugin output string, Application, referece to WVRange, reference to CVRange +# Output: Plugin output string +# Return value: Plugin return value + +sub PerformCheck() { + my ($Nagios, $PluginOutput, $WVRange, $CVRange) = @_; + my $Application; + my @AppNameSplitted; + my $ApplicationName; + my $PsCommand; + my $PsResult; + my @PsResultLines; + my $ProcLine; + my $ProcPid; + my $LsofCommand; + my $LsofResult; + my $ProcCount = 0; + my $FDCount = 0; + my $ProcFDAvg = 0; + my $PerProcMaxFD = 0; + my $ProcOKFlag = 0; + my $ProcWarningFlag = 0; + my $ProcCriticalFlag = 0; + my $OKFlag = 0; + my $WarningFlag = 0; + my $CriticalFlag = 0; + my $LastWarningProcFDs = 0; + my $LastWarningProc = -1; + my $LastCriticalProcFDs = 0; + my $LastCriticalProc = -1; + my $ProcPluginReturnValue = UNKNOWN; + my $AppPluginReturnValue = UNKNOWN; + my $PluginReturnValue = UNKNOWN; + my $PerformanceData = ""; + my $PerfdataUnit = "FDs"; + + $Application = $Nagios->opts->application; + $PsCommand = "ps -eaf | grep $Application"; + $PsResult = `$PsCommand`; + @AppNameSplitted = split(/\//, $Application); + $ApplicationName = $AppNameSplitted[$#AppNameSplitted]; + @PsResultLines = split(/\n/, $PsResult); + if ( $#PsResultLines > 1 ) { + foreach my $Proc (split(/\n/, $PsResult)) { + if ($Proc !~ /check_unix_open_fds/ && $Proc !~ / grep /) { + $ProcCount += 1; + $ProcPid = (split(/\s+/, $Proc))[1]; + $LsofCommand = "lsof -p $ProcPid | wc -l"; + $LsofResult = `$LsofCommand`; + $LsofResult = ($LsofResult > 0 ) ? ($LsofResult - 1) : 0; + $FDCount += $LsofResult; + if ($LsofResult >= $PerProcMaxFD) { $PerProcMaxFD = $LsofResult; } + $ProcPluginReturnValue = $Nagios->check_threshold(check => $LsofResult,warning => @{$WVRange}[0],critical => @{$CVRange}[0]); + if ($ProcPluginReturnValue eq OK) { + $ProcOKFlag = 1; + } + elsif ($ProcPluginReturnValue eq WARNING) { + $ProcWarningFlag = 1; + if ($LsofResult >= $LastWarningProcFDs) { + $LastWarningProcFDs = $LsofResult; + $LastWarningProc = $ProcPid; + } + } + #if ($LsofResult >= $PCT) { + elsif ($ProcPluginReturnValue eq CRITICAL) { + $ProcCriticalFlag = 1; + if ($LsofResult >= $LastCriticalProcFDs) { + $LastCriticalProcFDs = $LsofResult; + $LastCriticalProc = $ProcPid; + } + } + } + } + if ($ProcCount) { $ProcFDAvg = int($FDCount / $ProcCount); } + $AppPluginReturnValue = $Nagios->check_threshold(check => $FDCount,warning => @{$WVRange}[1],critical => @{$CVRange}[1]); + #if ($FDCount >= $TWT) { + if ($AppPluginReturnValue eq OK) { $OKFlag = 1; } + elsif ($AppPluginReturnValue eq WARNING) { $WarningFlag = 1; } + elsif ($AppPluginReturnValue eq CRITICAL) { $CriticalFlag = 1; } + + # PluginReturnValue and PluginOutput + if ($CriticalFlag) { + $PluginReturnValue = CRITICAL; + ${$PluginOutput} .= "$ApplicationName handling $FDCount files (critical threshold set to @{$CVRange}[1])"; + } + elsif ($WarningFlag) { + $PluginReturnValue = WARNING; + ${$PluginOutput} .= "$ApplicationName handling $FDCount files (warning threshold set to @{$WVRange}[1])"; + } + elsif ($ProcCriticalFlag) { + $PluginReturnValue = CRITICAL; + ${$PluginOutput} .= "Process ID $LastCriticalProc handling $LastCriticalProcFDs files (critical threshold set to @{$CVRange}[0])"; + } + elsif ($ProcWarningFlag) { + $PluginReturnValue = WARNING; + ${$PluginOutput} .= "Process ID $LastWarningProc handling $LastWarningProcFDs files (warning threshold set to @{$WVRange}[0])"; + } + elsif ($OKFlag && $ProcOKFlag) { + $PluginReturnValue = OK; + ${$PluginOutput} .= "$ApplicationName handling $FDCount files"; + } + } + else { + ${$PluginOutput} .= "No existe la aplicacion $ApplicationName"; + } + + + $PerformanceData .= "ProcCount=$ProcCount$PerfdataUnit FDCount=$FDCount$PerfdataUnit ProcFDAvg=$ProcFDAvg$PerfdataUnit PerProcMaxFD=$PerProcMaxFD$PerfdataUnit"; + + # Output with performance data: + ${$PluginOutput} .= " | $PerformanceData"; + + return $PluginReturnValue; +} diff --git a/puppet/modules/site_check_mk/files/agent/plugins/mk_logwatch.1.2.4 b/puppet/modules/site_check_mk/files/agent/plugins/mk_logwatch.1.2.4 new file mode 100755 index 00000000..3dbca322 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/plugins/mk_logwatch.1.2.4 @@ -0,0 +1,374 @@ +#!/usr/bin/python +# -*- encoding: utf-8; py-indent-offset: 4 -*- +# +------------------------------------------------------------------+ +# | ____ _ _ __ __ _ __ | +# | / ___| |__ ___ ___| | __ | \/ | |/ / | +# | | | | '_ \ / _ \/ __| |/ / | |\/| | ' / | +# | | |___| | | | __/ (__| < | | | | . \ | +# | \____|_| |_|\___|\___|_|\_\___|_| |_|_|\_\ | +# | | +# | Copyright Mathias Kettner 2010 mk@mathias-kettner.de | +# +------------------------------------------------------------------+ +# +# This file is part of Check_MK. +# The official homepage is at http://mathias-kettner.de/check_mk. +# +# check_mk is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation in version 2. check_mk is distributed +# in the hope that it will be useful, but WITHOUT ANY WARRANTY; with- +# out even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more de- +# ails. You should have received a copy of the GNU General Public +# License along with GNU Make; see the file COPYING. If not, write +# to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, +# Boston, MA 02110-1301 USA. + +# Call with -d for debug mode: colored output, no saving of status + +import sys, os, re, time +import glob + +if '-d' in sys.argv[1:] or '--debug' in sys.argv[1:]: + tty_red = '\033[1;31m' + tty_green = '\033[1;32m' + tty_yellow = '\033[1;33m' + tty_blue = '\033[1;34m' + tty_normal = '\033[0m' + debug = True +else: + tty_red = '' + tty_green = '' + tty_yellow = '' + tty_blue = '' + tty_normal = '' + debug = False + +# The configuration file and status file are searched +# in the directory named by the environment variable +# LOGWATCH_DIR. If that is not set, MK_CONFDIR is used. +# If that is not set either, the current directory ist +# used. +logwatch_dir = os.getenv("LOGWATCH_DIR") +if not logwatch_dir: + logwatch_dir = os.getenv("MK_CONFDIR") + if not logwatch_dir: + logwatch_dir = "." + +print "<<<logwatch>>>" + +config_filename = logwatch_dir + "/logwatch.cfg" +status_filename = logwatch_dir + "/logwatch.state" +config_dir = logwatch_dir + "/logwatch.d/*.cfg" + +def is_not_comment(line): + if line.lstrip().startswith('#') or \ + line.strip() == '': + return False + return True + +def parse_filenames(line): + return line.split() + +def parse_pattern(level, pattern): + if level not in [ 'C', 'W', 'I', 'O' ]: + raise(Exception("Invalid pattern line '%s'" % line)) + try: + compiled = re.compile(pattern) + except: + raise(Exception("Invalid regular expression in line '%s'" % line)) + return (level, compiled) + +def read_config(): + config_lines = [ line.rstrip() for line in filter(is_not_comment, file(config_filename).readlines()) ] + # Add config from a logwatch.d folder + for config_file in glob.glob(config_dir): + config_lines += [ line.rstrip() for line in filter(is_not_comment, file(config_file).readlines()) ] + + have_filenames = False + config = [] + + for line in config_lines: + rewrite = False + if line[0].isspace(): # pattern line + if not have_filenames: + raise Exception("Missing logfile names") + level, pattern = line.split(None, 1) + if level == 'A': + cont_list.append(parse_cont_pattern(pattern)) + elif level == 'R': + rewrite_list.append(pattern) + else: + level, compiled = parse_pattern(level, pattern) + cont_list = [] # List of continuation patterns + rewrite_list = [] # List of rewrite patterns + patterns.append((level, compiled, cont_list, rewrite_list)) + else: # filename line + patterns = [] + config.append((parse_filenames(line), patterns)) + have_filenames = True + return config + +def parse_cont_pattern(pattern): + try: + return int(pattern) + except: + try: + return re.compile(pattern) + except: + if debug: + raise + raise Exception("Invalid regular expression in line '%s'" % pattern) + +# structure of statusfile +# # LOGFILE OFFSET INODE +# /var/log/messages|7767698|32455445 +# /var/test/x12134.log|12345|32444355 +def read_status(): + if debug: + return {} + + status = {} + for line in file(status_filename): + # TODO: Remove variants with spaces. rsplit is + # not portable. split fails if logfilename contains + # spaces + inode = -1 + try: + parts = line.split('|') + filename = parts[0] + offset = parts[1] + if len(parts) >= 3: + inode = parts[2] + + except: + try: + filename, offset = line.rsplit(None, 1) + except: + filename, offset = line.split(None, 1) + status[filename] = int(offset), int(inode) + return status + +def save_status(status): + f = file(status_filename, "w") + for filename, (offset, inode) in status.items(): + f.write("%s|%d|%d\n" % (filename, offset, inode)) + +pushed_back_line = None +def next_line(f): + global pushed_back_line + if pushed_back_line != None: + line = pushed_back_line + pushed_back_line = None + return line + else: + try: + line = f.next() + return line + except: + return None + + +def process_logfile(logfile, patterns): + global pushed_back_line + + # Look at which file offset we have finished scanning + # the logfile last time. If we have never seen this file + # before, we set the offset to -1 + offset, prev_inode = status.get(logfile, (-1, -1)) + try: + fl = os.open(logfile, os.O_RDONLY) + inode = os.fstat(fl)[1] # 1 = st_ino + except: + if debug: + raise + print "[[[%s:cannotopen]]]" % logfile + return + + print "[[[%s]]]" % logfile + + # Seek to the current end in order to determine file size + current_end = os.lseek(fl, 0, 2) # os.SEEK_END not available in Python 2.4 + status[logfile] = current_end, inode + + # If we have never seen this file before, we just set the + # current pointer to the file end. We do not want to make + # a fuss about ancient log messages... + if offset == -1: + if not debug: + return + else: + offset = 0 + + + # If the inode of the logfile has changed it has appearently + # been started from new (logfile rotation). At least we must + # assume that. In some rare cases (restore of a backup, etc) + # we are wrong and resend old log messages + if prev_inode >= 0 and inode != prev_inode: + offset = 0 + + # Our previously stored offset is the current end -> + # no new lines in this file + if offset == current_end: + return # nothing new + + # If our offset is beyond the current end, the logfile has been + # truncated or wrapped while keeping the same inode. We assume + # that it contains all new data in that case and restart from + # offset 0. + if offset > current_end: + offset = 0 + + # now seek to offset where interesting data begins + os.lseek(fl, offset, 0) # os.SEEK_SET not available in Python 2.4 + f = os.fdopen(fl) + worst = -1 + outputtxt = "" + lines_parsed = 0 + start_time = time.time() + + while True: + line = next_line(f) + if line == None: + break # End of file + + lines_parsed += 1 + # Check if maximum number of new log messages is exceeded + if opt_maxlines != None and lines_parsed > opt_maxlines: + outputtxt += "%s Maximum number (%d) of new log messages exceeded.\n" % ( + opt_overflow, opt_maxlines) + worst = max(worst, opt_overflow_level) + os.lseek(fl, 0, 2) # Seek to end of file, skip all other messages + break + + # Check if maximum processing time (per file) is exceeded. Check only + # every 100'th line in order to save system calls + if opt_maxtime != None and lines_parsed % 100 == 10 \ + and time.time() - start_time > opt_maxtime: + outputtxt += "%s Maximum parsing time (%.1f sec) of this log file exceeded.\n" % ( + opt_overflow, opt_maxtime) + worst = max(worst, opt_overflow_level) + os.lseek(fl, 0, 2) # Seek to end of file, skip all other messages + break + + level = "." + for lev, pattern, cont_patterns, replacements in patterns: + matches = pattern.search(line[:-1]) + if matches: + level = lev + levelint = {'C': 2, 'W': 1, 'O': 0, 'I': -1, '.': -1}[lev] + worst = max(levelint, worst) + + # Check for continuation lines + for cont_pattern in cont_patterns: + if type(cont_pattern) == int: # add that many lines + for x in range(cont_pattern): + cont_line = next_line(f) + if cont_line == None: # end of file + break + line = line[:-1] + "\1" + cont_line + + else: # pattern is regex + while True: + cont_line = next_line(f) + if cont_line == None: # end of file + break + elif cont_pattern.search(cont_line[:-1]): + line = line[:-1] + "\1" + cont_line + else: + pushed_back_line = cont_line # sorry for stealing this line + break + + # Replacement + for replace in replacements: + line = replace.replace('\\0', line) + "\n" + for nr, group in enumerate(matches.groups()): + line = line.replace('\\%d' % (nr+1), group) + + break # matching rule found and executed + + color = {'C': tty_red, 'W': tty_yellow, 'O': tty_green, 'I': tty_blue, '.': ''}[level] + if debug: + line = line.replace("\1", "\nCONT:") + if level == "I": + level = "." + if opt_nocontext and level == '.': + continue + outputtxt += "%s%s %s%s\n" % (color, level, line[:-1], tty_normal) + + new_offset = os.lseek(fl, 0, 1) # os.SEEK_CUR not available in Python 2.4 + status[logfile] = new_offset, inode + + # output all lines if at least one warning, error or ok has been found + if worst > -1: + sys.stdout.write(outputtxt) + sys.stdout.flush() + +try: + config = read_config() +except Exception, e: + if debug: + raise + print "CANNOT READ CONFIG FILE: %s" % e + sys.exit(1) + +# Simply ignore errors in the status file. In case of a corrupted status file we simply begin +# with an empty status. That keeps the monitoring up and running - even if we might loose a +# message in the extreme case of a corrupted status file. +try: + status = read_status() +except Exception, e: + status = {} + + +# The filename line may contain options like 'maxlines=100' or 'maxtime=10' +for filenames, patterns in config: + # Initialize options with default values + opt_maxlines = None + opt_maxtime = None + opt_regex = None + opt_overflow = 'C' + opt_overflow_level = 2 + opt_nocontext = False + try: + options = [ o.split('=', 1) for o in filenames if '=' in o ] + for key, value in options: + if key == 'maxlines': + opt_maxlines = int(value) + elif key == 'maxtime': + opt_maxtime = float(value) + elif key == 'overflow': + if value not in [ 'C', 'I', 'W', 'O' ]: + raise Exception("Invalid value %s for overflow. Allowed are C, I, O and W" % value) + opt_overflow = value + opt_overflow_level = {'C':2, 'W':1, 'O':0, 'I':0}[value] + elif key == 'regex': + opt_regex = re.compile(value) + elif key == 'iregex': + opt_regex = re.compile(value, re.I) + elif key == 'nocontext': + opt_nocontext = True + else: + raise Exception("Invalid option %s" % key) + except Exception, e: + if debug: + raise + print "INVALID CONFIGURATION: %s" % e + sys.exit(1) + + + for glob in filenames: + if '=' in glob: + continue + logfiles = [ l.strip() for l in os.popen("ls %s 2>/dev/null" % glob).readlines() ] + if opt_regex: + logfiles = [ f for f in logfiles if opt_regex.search(f) ] + if len(logfiles) == 0: + print '[[[%s:missing]]]' % glob + else: + for logfile in logfiles: + process_logfile(logfile, patterns) + +if not debug: + save_status(status) diff --git a/puppet/modules/site_check_mk/manifests/agent.pp b/puppet/modules/site_check_mk/manifests/agent.pp new file mode 100644 index 00000000..b752182e --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent.pp @@ -0,0 +1,28 @@ +class site_check_mk::agent { + + $ssh_hash = hiera('ssh') + $pubkey = $ssh_hash['authorized_keys']['monitor']['key'] + $type = $ssh_hash['authorized_keys']['monitor']['type'] + + include site_apt::preferences::check_mk + + class { 'check_mk::agent': + agent_package_name => 'check-mk-agent', + agent_logwatch_package_name => 'check-mk-agent-logwatch', + method => 'ssh', + homedir => '/etc/nagios/check_mk', + register_agent => false + } + + file { [ '/srv/leap/nagios', '/srv/leap/nagios/plugins' ]: + ensure => directory, + } + + file { '/usr/lib/check_mk_agent/local/run_node_tests.sh': + source => 'puppet:///modules/site_check_mk/agent/local_checks/all_hosts/run_node_tests.sh', + mode => '0755' + } + + include site_check_mk::agent::mrpe + include site_check_mk::agent::logwatch +} diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp new file mode 100644 index 00000000..01e2b886 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp @@ -0,0 +1,36 @@ +class site_check_mk::agent::couchdb { + + # watch logs + file { '/etc/check_mk/logwatch.d/bigcouch.cfg': + source => 'puppet:///modules/site_check_mk/agent/logwatch/bigcouch.cfg', + } + concat::fragment { 'syslog_couchdb': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/couchdb.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + + + # check bigcouch processes + file_line { + 'Bigcouch_epmd_procs': + line => 'Bigcouch_epmd_procs /usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/epmd', + path => '/etc/check_mk/mrpe.cfg'; + 'Bigcouch_beam_procs': + line => 'Bigcouch_beam_procs /usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/beam', + path => '/etc/check_mk/mrpe.cfg'; + } + + # check open files for bigcouch proc + include site_check_mk::agent::package::perl_plugin + file { '/srv/leap/nagios/plugins/check_unix_open_fds.pl': + source => 'puppet:///modules/site_check_mk/agent/nagios_plugins/check_unix_open_fds.pl', + mode => '0755' + } + file_line { + 'Bigcouch_open_files': + line => 'Bigcouch_open_files /srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 750,750 -c 1000,1000', + path => '/etc/check_mk/mrpe.cfg'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/haproxy.pp b/puppet/modules/site_check_mk/manifests/agent/haproxy.pp new file mode 100644 index 00000000..e7986db1 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/haproxy.pp @@ -0,0 +1,12 @@ +class site_check_mk::agent::haproxy { + + include site_check_mk::agent::package::nagios_plugins_contrib + + # local nagios plugin checks via mrpe + file_line { + 'haproxy': + line => 'Haproxy /usr/lib/nagios/plugins/check_haproxy -u "http://localhost:8000/haproxy;csv"', + path => '/etc/check_mk/mrpe.cfg'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/logwatch.pp b/puppet/modules/site_check_mk/manifests/agent/logwatch.pp new file mode 100644 index 00000000..423cace2 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/logwatch.pp @@ -0,0 +1,36 @@ +class site_check_mk::agent::logwatch { + # Deploy mk_logwatch 1.2.4 so we can split the config + # into multiple config files in /etc/check_mk/logwatch.d + # see https://leap.se/code/issues/5135 + + file { '/usr/lib/check_mk_agent/plugins/mk_logwatch': + source => 'puppet:///modules/site_check_mk/agent/plugins/mk_logwatch.1.2.4', + mode => '0755', + require => Package['check-mk-agent-logwatch'] + } + + # only config files that watch a distinct logfile should go in logwatch.d/ + file { '/etc/check_mk/logwatch.d': + ensure => directory, + recurse => true, + purge => true, + require => Package['check-mk-agent-logwatch'] + } + + # service that share a common logfile (i.e. /var/log/syslog) need to get + # concanated in one file, otherwise the last file sourced will override + # the config before + # see mk_logwatch: "logwatch.cfg overwrites config files in logwatch.d", + # https://leap.se/code/issues/5155 + + # first, we need to deploy a custom logwatch.cfg that doesn't include + # a section about /var/log/syslog + + file { '/etc/check_mk/logwatch.cfg': + source => 'puppet:///modules/site_check_mk/agent/logwatch/logwatch.cfg', + require => Package['check_mk-agent-logwatch'] + } + + include concat::setup + include site_check_mk::agent::logwatch::syslog +} diff --git a/puppet/modules/site_check_mk/manifests/agent/logwatch/syslog.pp b/puppet/modules/site_check_mk/manifests/agent/logwatch/syslog.pp new file mode 100644 index 00000000..c927780d --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/logwatch/syslog.pp @@ -0,0 +1,18 @@ +class site_check_mk::agent::logwatch::syslog { + + concat { '/etc/check_mk/logwatch.d/syslog.cfg': + warn => true + } + + concat::fragment { 'syslog_header': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog_header.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '01'; + } + concat::fragment { 'syslog_tail': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog_tail.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '99'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/mrpe.pp b/puppet/modules/site_check_mk/manifests/agent/mrpe.pp new file mode 100644 index 00000000..6921574f --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/mrpe.pp @@ -0,0 +1,18 @@ +class site_check_mk::agent::mrpe { + # check_mk can use standard nagios plugins using + # a wrapper called mrpe + # see http://mathias-kettner.de/checkmk_mrpe.html + + package { 'nagios-plugins-basic': + ensure => latest, + } + + file { '/etc/check_mk/mrpe.cfg': + ensure => present, + require => Package['check-mk-agent'] + } -> + file_line { 'Apt': + line => 'APT /usr/lib/nagios/plugins/check_apt', + path => '/etc/check_mk/mrpe.cfg', + } +} diff --git a/puppet/modules/site_check_mk/manifests/agent/mx.pp b/puppet/modules/site_check_mk/manifests/agent/mx.pp new file mode 100644 index 00000000..2dd50400 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/mx.pp @@ -0,0 +1,22 @@ +class site_check_mk::agent::mx { + + # watch logs + file { '/etc/check_mk/logwatch.d/leap_mx.cfg': + source => 'puppet:///modules/site_check_mk/agent/logwatch/leap_mx.cfg', + } + + # local nagios plugin checks via mrpe + file_line { + 'Leap_MX_Procs': + line => 'Leap_MX_Procs /usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a leap_mx', + path => '/etc/check_mk/mrpe.cfg'; + } + + + # check stale files in queue dir + file { '/usr/lib/check_mk_agent/local/check_leap_mx.sh': + source => 'puppet:///modules/site_check_mk/agent/local_checks/mx/check_leap_mx.sh', + mode => '0755' + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/openvpn.pp b/puppet/modules/site_check_mk/manifests/agent/openvpn.pp new file mode 100644 index 00000000..919a408d --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/openvpn.pp @@ -0,0 +1,10 @@ +class site_check_mk::agent::openvpn { + + # check syslog + concat::fragment { 'syslog_openpvn': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/openvpn.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/package/nagios_plugins_contrib.pp b/puppet/modules/site_check_mk/manifests/agent/package/nagios_plugins_contrib.pp new file mode 100644 index 00000000..95a60d17 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/package/nagios_plugins_contrib.pp @@ -0,0 +1,5 @@ +class site_check_mk::agent::package::nagios_plugins_contrib { + package { 'nagios-plugins-contrib': + ensure => installed, + } +} diff --git a/puppet/modules/site_check_mk/manifests/agent/package/perl_plugin.pp b/puppet/modules/site_check_mk/manifests/agent/package/perl_plugin.pp new file mode 100644 index 00000000..4feda375 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/package/perl_plugin.pp @@ -0,0 +1,5 @@ +class site_check_mk::agent::package::perl_plugin { + package { 'libnagios-plugin-perl': + ensure => installed, + } +} diff --git a/puppet/modules/site_check_mk/manifests/agent/soledad.pp b/puppet/modules/site_check_mk/manifests/agent/soledad.pp new file mode 100644 index 00000000..cbae81fe --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/soledad.pp @@ -0,0 +1,14 @@ +class site_check_mk::agent::soledad { + + file { '/etc/check_mk/logwatch.d/soledad.cfg': + source => 'puppet:///modules/site_check_mk/agent/logwatch/soledad.cfg', + } + + # local nagios plugin checks via mrpe + file_line { + 'Soledad_Procs': + line => 'Soledad_Procs /usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a soledad', + path => '/etc/check_mk/mrpe.cfg'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/stunnel.pp b/puppet/modules/site_check_mk/manifests/agent/stunnel.pp new file mode 100644 index 00000000..64022824 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/stunnel.pp @@ -0,0 +1,9 @@ +class site_check_mk::agent::stunnel { + + concat::fragment { 'syslog_stunnel': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/stunnel.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp new file mode 100644 index 00000000..369ed00b --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp @@ -0,0 +1,16 @@ +class site_check_mk::agent::tapicero { + + concat::fragment { 'syslog_tapicero': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/tapicero.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + + # local nagios plugin checks via mrpe + file_line { + 'Tapicero_Procs': + line => 'Tapicero_Procs /usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a tapicero', + path => '/etc/check_mk/mrpe.cfg'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/webapp.pp b/puppet/modules/site_check_mk/manifests/agent/webapp.pp new file mode 100644 index 00000000..cb160d83 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/webapp.pp @@ -0,0 +1,24 @@ +class site_check_mk::agent::webapp { + + # check webapp login + soledad sync + package { [ 'python-srp', 'python-requests', 'python-yaml', 'python-u1db' ]: + ensure => installed + } + file { '/usr/lib/check_mk_agent/local/nagios-webapp_login.py': + ensure => link, + target => '/srv/leap/webapp/test/nagios/webapp_login.py' + } + file { '/usr/lib/check_mk_agent/local/soledad_sync.py': + ensure => link, + target => '/srv/leap/webapp/test/nagios/soledad_sync.py' + } + + + # check syslog + concat::fragment { 'syslog_webapp': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/webapp.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/server.pp b/puppet/modules/site_check_mk/manifests/server.pp new file mode 100644 index 00000000..e544ef0d --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/server.pp @@ -0,0 +1,64 @@ +class site_check_mk::server { + + $ssh_hash = hiera('ssh') + $pubkey = $ssh_hash['authorized_keys']['monitor']['key'] + $type = $ssh_hash['authorized_keys']['monitor']['type'] + $seckey = $ssh_hash['monitor']['private_key'] + + $nagios_hiera = hiera_hash('nagios') + $nagios_hosts = $nagios_hiera['hosts'] + + $hosts = hiera_hash('hosts') + $all_hosts = inline_template ('<% @hosts.keys.sort.each do |key| -%>"<%= @hosts[key]["domain_internal"] %>", <% end -%>') + + package { 'check-mk-server': + ensure => installed, + } + + # override paths to use the system check_mk rather than OMD + class { 'check_mk::config': + site => '', + etc_dir => '/etc', + nagios_subdir => 'nagios3', + bin_dir => '/usr/bin', + host_groups => undef, + use_storedconfigs => false, + require => Package['check-mk-server'] + } + + Exec['check_mk-reload'] -> + Exec['check_mk-refresh-inventory-daily'] -> + Service['nagios'] + + file { + '/etc/check_mk/conf.d/use_ssh.mk': + content => template('site_check_mk/use_ssh.mk'), + notify => Exec['check_mk-refresh'], + require => Package['check-mk-server']; + '/etc/check_mk/all_hosts_static': + content => $all_hosts, + notify => Exec['check_mk-refresh'], + require => Package['check-mk-server']; + '/etc/check_mk/.ssh': + ensure => directory, + require => Package['check-mk-server']; + '/etc/check_mk/.ssh/id_rsa': + content => $seckey, + owner => 'nagios', + mode => '0600', + require => Package['check-mk-server']; + '/etc/check_mk/.ssh/id_rsa.pub': + content => "${type} ${pubkey} monitor", + owner => 'nagios', + mode => '0644', + require => Package['check-mk-server']; + # check_icmp must be suid root or called by sudo + # see https://leap.se/code/issues/5171 + '/usr/lib/nagios/plugins/check_icmp': + mode => '4755', + require => Package['nagios-plugins-basic']; + } + + + include check_mk::agent::local_checks +} diff --git a/puppet/modules/site_check_mk/templates/use_ssh.mk b/puppet/modules/site_check_mk/templates/use_ssh.mk new file mode 100644 index 00000000..0bebebcf --- /dev/null +++ b/puppet/modules/site_check_mk/templates/use_ssh.mk @@ -0,0 +1,6 @@ +# http://mathias-kettner.de/checkmk_datasource_programs.html +datasource_programs = [ +<% nagios_hosts.sort.each do |name,config| %> + ( "ssh -l root -i /etc/check_mk/.ssh/id_rsa -p <%=config['ssh_port']%> <%=config['domain_internal']%> check_mk_agent", [ "<%=config['domain_internal']%>" ], ),<%- end -%> + +] diff --git a/puppet/modules/site_config/manifests/default.pp b/puppet/modules/site_config/manifests/default.pp index d85d9c8f..53cc60f6 100644 --- a/puppet/modules/site_config/manifests/default.pp +++ b/puppet/modules/site_config/manifests/default.pp @@ -86,4 +86,6 @@ class site_config::default { if defined( '::site_custom') { include ::site_custom } + + include site_check_mk::agent } diff --git a/puppet/modules/site_config/manifests/hosts.pp b/puppet/modules/site_config/manifests/hosts.pp index a3ce0c1f..e5d4dd70 100644 --- a/puppet/modules/site_config/manifests/hosts.pp +++ b/puppet/modules/site_config/manifests/hosts.pp @@ -3,6 +3,7 @@ class site_config::hosts() { $hostname = hiera('name') $domain_hash = hiera('domain') $domain_public = $domain_hash['full_suffix'] + $api = hiera('api', '') file { '/etc/hostname': ensure => present, diff --git a/puppet/modules/site_config/manifests/packages/base.pp b/puppet/modules/site_config/manifests/packages/base.pp index 9d416043..ae47963c 100644 --- a/puppet/modules/site_config/manifests/packages/base.pp +++ b/puppet/modules/site_config/manifests/packages/base.pp @@ -1,6 +1,5 @@ class site_config::packages::base { - include site_config::params # base set of packages that we want to have installed everywhere package { [ 'etckeeper', 'screen', 'less', 'ntp' ]: @@ -16,18 +15,4 @@ class site_config::packages::base { 'x11-utils', 'xterm' ]: ensure => absent; } - - if $::site_config::params::environment == 'local' or $::services =~ /\bwebapp\b/ { - $dev_packages_ensure = present - } else { - $dev_packages_ensure = absent - } - - # g++ and ruby1.9.1-dev are needed for nickserver/eventmachine (#4079) - # dev_packages are needed for building gems on the webapp node - - package { [ 'build-essential', 'g++', 'g++-4.7', 'gcc', - 'gcc-4.6', 'gcc-4.7', 'cpp', 'cpp-4.6', 'cpp-4.7', 'libc6-dev' ]: - ensure => $dev_packages_ensure - } } diff --git a/puppet/modules/site_config/manifests/packages/build_essential.pp b/puppet/modules/site_config/manifests/packages/build_essential.pp new file mode 100644 index 00000000..c9efaafb --- /dev/null +++ b/puppet/modules/site_config/manifests/packages/build_essential.pp @@ -0,0 +1,8 @@ +# +# include this whenever you want to ensure build-essential package and related compilers are installed. +# +class site_config::packages::build_essential { + if $install_build_essential == undef { + $install_build_essential = true + } +}
\ No newline at end of file diff --git a/puppet/modules/site_config/manifests/packages/uninstall.pp b/puppet/modules/site_config/manifests/packages/uninstall.pp new file mode 100644 index 00000000..2919cc96 --- /dev/null +++ b/puppet/modules/site_config/manifests/packages/uninstall.pp @@ -0,0 +1,20 @@ +# +# this should be included last to allow other modules to set $::install_build_packages +# +class site_config::packages::uninstall { + + if $site_config::packages::build_essential::install_essential == true { + $dev_packages_ensure = present + } else { + $dev_packages_ensure = absent + } + + # generally, dev packages are needed for installing ruby gems with native extensions. + # (nickserver, webapp, etc) + + package { [ 'build-essential', 'g++', 'g++-4.7', 'gcc', + 'gcc-4.6', 'gcc-4.7', 'cpp', 'cpp-4.6', 'cpp-4.7', 'libc6-dev' ]: + ensure => $dev_packages_ensure + } + +}
\ No newline at end of file diff --git a/puppet/modules/site_config/manifests/params.pp b/puppet/modules/site_config/manifests/params.pp index 5bdc0077..012b3ce0 100644 --- a/puppet/modules/site_config/manifests/params.pp +++ b/puppet/modules/site_config/manifests/params.pp @@ -8,6 +8,7 @@ class site_config::params { if $environment == 'local' { $interface = 'eth1' + include site_config::packages::build_essential } elsif hiera('interface','') != '' { $interface = hiera('interface') diff --git a/puppet/modules/site_config/manifests/ruby/dev.pp b/puppet/modules/site_config/manifests/ruby/dev.pp index dbc77ae7..3ea6ca96 100644 --- a/puppet/modules/site_config/manifests/ruby/dev.pp +++ b/puppet/modules/site_config/manifests/ruby/dev.pp @@ -3,4 +3,6 @@ class site_config::ruby::dev inherits site_config::ruby { ruby_version => '1.9.3', install_dev => true } + # building gems locally probably requires build-essential and gcc: + include site_config::packages::build_essential } diff --git a/puppet/modules/site_config/templates/hosts b/puppet/modules/site_config/templates/hosts index c0a2740f..bfcabaa5 100644 --- a/puppet/modules/site_config/templates/hosts +++ b/puppet/modules/site_config/templates/hosts @@ -1,7 +1,8 @@ # This file is managed by puppet, any changes will be overwritten! 127.0.0.1 localhost -127.0.1.1 <%= @hostname %>.<%= @domain_public %> <%= @hostname %> +127.0.1.1 <%= @hostname %>.<%= @domain_public %> <%= @hostname %> <% if (defined? @services) and (@services.include? 'webapp') -%><%= @domain_public %> <%= @api['domain'] %><% end -%> + <%- if @hosts then -%> <% @hosts.keys.sort.each do |name| -%> diff --git a/puppet/modules/site_couchdb/files/designs/messages/Message.json b/puppet/modules/site_couchdb/files/designs/messages/Message.json new file mode 100644 index 00000000..7bcd74c7 --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/messages/Message.json @@ -0,0 +1,18 @@ +{ + "_id": "_design/Message", + "language": "javascript", + "views": { + "by_user_ids_to_show_and_created_at": { + "map": "// not using at moment\n// call with something like Message.by_user_ids_to_show_and_created_at.startkey([user_id, start_date]).endkey([user_id,end_date])\nfunction (doc) {\n if (doc.type === 'Message' && doc.user_ids_to_show && Array.isArray(doc.user_ids_to_show)) {\n doc.user_ids_to_show.forEach(function (userId) {\n emit([userId, doc.created_at], 1);\n });\n }\n}\n", + "reduce": "function(key, values, rereduce) { return sum(values); }" + }, + "by_user_ids_to_show": { + "map": "function (doc) {\n if (doc.type === 'Message' && doc.user_ids_to_show && Array.isArray(doc.user_ids_to_show)) {\n doc.user_ids_to_show.forEach(function (userId) {\n emit(userId, 1);\n });\n }\n}\n", + "reduce": "function(key, values, rereduce) { return sum(values); }" + }, + "all": { + "map": " function(doc) {\n if (doc['type'] == 'Message') {\n emit(doc._id, null);\n }\n }\n" + } + }, + "couchrest-hash": "0967e7cc5bb1e61edc1c085f6f0cecbf" +}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/users/User.json b/puppet/modules/site_couchdb/files/designs/users/User.json index c99666cb..4089ad97 100644 --- a/puppet/modules/site_couchdb/files/designs/users/User.json +++ b/puppet/modules/site_couchdb/files/designs/users/User.json @@ -9,10 +9,14 @@ "all": { "map": " function(doc) {\n if (doc['type'] == 'User') {\n emit(doc._id, null);\n }\n }\n" }, + "by_created_at_and_one_month_warning_not_sent": { + "map": "function (doc) {\n if ((doc['type'] == 'User') && (doc['created_at'] != null) && (doc['one_month_warning_sent'] == null)) {\n emit(doc['created_at'], 1);\n } \n}\n", + "reduce": "function(key, values, rereduce) { return sum(values); }" + }, "by_created_at": { "map": " function(doc) {\n if ((doc['type'] == 'User') && (doc['created_at'] != null)) {\n emit(doc['created_at'], 1);\n }\n }\n", "reduce": "_sum" } }, - "couchrest-hash": "3bdbcd85b928ad911e0c89a8924e015c" + "couchrest-hash": "61840ab3ec0f94ef8bbd6dd208db3b70" }
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/compaction.pp b/puppet/modules/site_couchdb/manifests/bigcouch/compaction.pp new file mode 100644 index 00000000..84aab4ef --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/bigcouch/compaction.pp @@ -0,0 +1,8 @@ +class site_couchdb::bigcouch::compaction { + cron { + 'compact_all_shards': + command => '/srv/leap/couchdb/scripts/bigcouch_compact_all_shards.sh >> /var/log/bigcouch/compaction.log', + hour => 3, + minute => 17; + } +} diff --git a/puppet/modules/site_couchdb/manifests/create_dbs.pp b/puppet/modules/site_couchdb/manifests/create_dbs.pp index b0ebca4d..41500d3a 100644 --- a/puppet/modules/site_couchdb/manifests/create_dbs.pp +++ b/puppet/modules/site_couchdb/manifests/create_dbs.pp @@ -59,4 +59,12 @@ class site_couchdb::create_dbs { members => "{ \"names\": [], \"roles\": [\"users\"] }", require => Couchdb::Query::Setup['localhost'] } + + ## messages db + ## store messages to the clients such as payment reminders + ## r/w: webapp + couchdb::create_db { 'messages': + members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }", + require => Couchdb::Query::Setup['localhost'] + } } diff --git a/puppet/modules/site_couchdb/manifests/designs.pp b/puppet/modules/site_couchdb/manifests/designs.pp index 83d6c8cd..9e88de64 100644 --- a/puppet/modules/site_couchdb/manifests/designs.pp +++ b/puppet/modules/site_couchdb/manifests/designs.pp @@ -12,9 +12,8 @@ class site_couchdb::designs { } exec { '/srv/leap/couchdb/scripts/load_design_documents.sh': - subscribe => File['/srv/leap/couchdb/designs'], - refreshonly => true, - require => Vcsrepo['/srv/leap/couchdb/scripts'] + require => Vcsrepo['/srv/leap/couchdb/scripts'], + refreshonly => false } } diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 137b661f..c67ce8c8 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -94,6 +94,7 @@ class site_couchdb { include site_couchdb::add_users include site_couchdb::designs include site_couchdb::logrotate + include site_couchdb::bigcouch::compaction include site_shorewall::couchdb include site_shorewall::couchdb::bigcouch @@ -107,4 +108,12 @@ class site_couchdb { } if $couchdb_backup { include site_couchdb::backup } + + include site_check_mk::agent::couchdb + include site_check_mk::agent::tapicero + + file { '/var/log/bigcouch': + ensure => directory + } + } diff --git a/puppet/modules/site_couchdb/manifests/stunnel.pp b/puppet/modules/site_couchdb/manifests/stunnel.pp index 87c35f05..91f1e3aa 100644 --- a/puppet/modules/site_couchdb/manifests/stunnel.pp +++ b/puppet/modules/site_couchdb/manifests/stunnel.pp @@ -107,4 +107,6 @@ class site_couchdb::stunnel { } create_resources(site_stunnel::clients, $ednp_clients, $ednp_client_defaults) + + include site_check_mk::agent::stunnel } diff --git a/puppet/modules/site_haproxy/files/haproxy-stats.cfg b/puppet/modules/site_haproxy/files/haproxy-stats.cfg new file mode 100644 index 00000000..e6335ba2 --- /dev/null +++ b/puppet/modules/site_haproxy/files/haproxy-stats.cfg @@ -0,0 +1,6 @@ +# provide access to stats for the nagios plugin +listen stats 127.0.0.1:8000 + mode http + stats enable + stats uri /haproxy + diff --git a/puppet/modules/site_haproxy/manifests/init.pp b/puppet/modules/site_haproxy/manifests/init.pp index 602e26be..1a681373 100644 --- a/puppet/modules/site_haproxy/manifests/init.pp +++ b/puppet/modules/site_haproxy/manifests/init.pp @@ -22,4 +22,11 @@ class site_haproxy { } } + # monitor haproxy + concat::fragment { 'stats': + target => '/etc/haproxy/haproxy.cfg', + order => '90', + source => 'puppet:///modules/site_haproxy/haproxy-stats.cfg'; + } + include site_check_mk::agent::haproxy } diff --git a/puppet/modules/site_mx/manifests/init.pp b/puppet/modules/site_mx/manifests/init.pp index cdb84b54..3949c787 100644 --- a/puppet/modules/site_mx/manifests/init.pp +++ b/puppet/modules/site_mx/manifests/init.pp @@ -15,4 +15,5 @@ class site_mx { include site_shorewall::service::smtp include site_mx::couchdb include leap_mx + include site_check_mk::agent::mx } diff --git a/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg b/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg index 753d1610..e46ebf62 100644 --- a/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg +++ b/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg @@ -1,6 +1,6 @@ ############################################################################## # -# NAGIOS.CFG - Sample Main Config File for Nagios +# NAGIOS.CFG - Sample Main Config File for Nagios # # ############################################################################## @@ -8,7 +8,7 @@ # LOG FILE # This is the main log file where service and host events are logged -# for historical purposes. This should be the first option specified +# for historical purposes. This should be the first option specified # in the config file!!! log_file=/var/log/nagios3/nagios.log @@ -25,6 +25,9 @@ log_file=/var/log/nagios3/nagios.log # Puppet-managed configuration files cfg_dir=/etc/nagios3/conf.d +# check-mk managed configuration files +cfg_dir=/etc/nagios3/local + # Debian also defaults to using the check commands defined by the debian # nagios-plugins package cfg_dir=/etc/nagios-plugins/config @@ -33,7 +36,7 @@ cfg_dir=/etc/nagios-plugins/config # OBJECT CACHE FILE # This option determines where object definitions are cached when -# Nagios starts/restarts. The CGIs read object definitions from +# Nagios starts/restarts. The CGIs read object definitions from # this cache file (rather than looking at the object config files # directly) in order to prevent inconsistencies that can occur # when the config files are modified after Nagios starts. @@ -49,7 +52,7 @@ object_cache_file=/var/cache/nagios3/objects.cache # file. You can then start Nagios with the -u option to have it read # object definitions from this precached file, rather than the standard # object configuration files (see the cfg_file and cfg_dir options above). -# Using a precached object file can speed up the time needed to (re)start +# Using a precached object file can speed up the time needed to (re)start # the Nagios process if you've got a large and/or complex configuration. # Read the documentation section on optimizing Nagios to find our more # about how this feature works. @@ -83,7 +86,7 @@ status_file=/var/cache/nagios3/status.dat # STATUS FILE UPDATE INTERVAL # This option determines the frequency (in seconds) that -# Nagios will periodically dump program, host, and +# Nagios will periodically dump program, host, and # service status data. status_update_interval=10 @@ -91,7 +94,7 @@ status_update_interval=10 # NAGIOS USER -# This determines the effective user that Nagios should run as. +# This determines the effective user that Nagios should run as. # You can either supply a username or a UID. nagios_user=nagios @@ -99,7 +102,7 @@ nagios_user=nagios # NAGIOS GROUP -# This determines the effective group that Nagios should run as. +# This determines the effective group that Nagios should run as. # You can either supply a group name or a GID. nagios_group=nagios @@ -125,7 +128,7 @@ check_external_commands=1 # Nagios to check for external commands every minute. If you specify a # number followed by an "s" (i.e. 15s), this will be interpreted to mean # actual seconds rather than a multiple of the interval_length variable. -# Note: In addition to reading the external command file at regularly +# Note: In addition to reading the external command file at regularly # scheduled intervals, Nagios will also check for external commands after # event handlers are executed. # NOTE: Setting this value to -1 causes Nagios to check the external @@ -140,7 +143,7 @@ command_check_interval=-1 # This is the file that Nagios checks for external command requests. # It is also where the command CGI will write commands that are submitted # by users, so it must be writeable by the user that the web server -# is running as (usually 'nobody'). Permissions should be set at the +# is running as (usually 'nobody'). Permissions should be set at the # directory level instead of on the file, as the file is deleted every # time its contents are processed. # Debian Users: In case you didn't read README.Debian yet, _NOW_ is the @@ -152,9 +155,9 @@ command_file=/var/lib/nagios3/rw/nagios.cmd # EXTERNAL COMMAND BUFFER SLOTS # This settings is used to tweak the number of items or "slots" that -# the Nagios daemon should allocate to the buffer that holds incoming -# external commands before they are processed. As external commands -# are processed by the daemon, they are removed from the buffer. +# the Nagios daemon should allocate to the buffer that holds incoming +# external commands before they are processed. As external commands +# are processed by the daemon, they are removed from the buffer. external_command_buffer_slots=4096 @@ -232,12 +235,12 @@ event_broker_options=-1 # w = Weekly rotation (midnight on Saturday evening) # m = Monthly rotation (midnight last day of month) -log_rotation_method=d +log_rotation_method=n # LOG ARCHIVE PATH -# This is the directory where archived (rotated) log files should be +# This is the directory where archived (rotated) log files should be # placed (assuming you've chosen to do log rotation). log_archive_path=/var/log/nagios3/archives @@ -248,7 +251,7 @@ log_archive_path=/var/log/nagios3/archives # If you want messages logged to the syslog facility, as well as the # Nagios log file set this option to 1. If not, set it to 0. -use_syslog=1 +use_syslog=0 @@ -400,7 +403,7 @@ max_host_check_spread=30 # MAXIMUM CONCURRENT SERVICE CHECKS -# This option allows you to specify the maximum number of +# This option allows you to specify the maximum number of # service checks that can be run in parallel at any given time. # Specifying a value of 1 for this variable essentially prevents # any service checks from being parallelized. A value of 0 @@ -422,7 +425,7 @@ check_result_reaper_frequency=10 # MAX CHECK RESULT REAPER TIME # This is the max amount of time (in seconds) that a single -# check result reaper event will be allowed to run before +# check result reaper event will be allowed to run before # returning control back to Nagios so it can perform other # duties. @@ -436,7 +439,7 @@ max_check_result_reaper_time=30 # service checks that have not yet been processed. # # Note: Make sure that only one instance of Nagios has access -# to this directory! +# to this directory! check_result_path=/var/lib/nagios3/spool/checkresults @@ -445,7 +448,7 @@ check_result_path=/var/lib/nagios3/spool/checkresults # MAX CHECK RESULT FILE AGE # This option determines the maximum age (in seconds) which check -# result files are considered to be valid. Files older than this +# result files are considered to be valid. Files older than this # threshold will be mercilessly deleted without further processing. max_check_result_file_age=3600 @@ -507,14 +510,14 @@ enable_predictive_service_dependency_checks=1 # SOFT STATE DEPENDENCIES -# This option determines whether or not Nagios will use soft state -# information when checking host and service dependencies. Normally -# Nagios will only use the latest hard host or service state when +# This option determines whether or not Nagios will use soft state +# information when checking host and service dependencies. Normally +# Nagios will only use the latest hard host or service state when # checking dependencies. If you want it to use the latest state (regardless -# of whether its a soft or hard state type), enable this option. +# of whether its a soft or hard state type), enable this option. # Values: -# 0 = Don't use soft state dependencies (default) -# 1 = Use soft state dependencies +# 0 = Don't use soft state dependencies (default) +# 1 = Use soft state dependencies soft_state_dependencies=0 @@ -532,7 +535,7 @@ soft_state_dependencies=0 # This option determines whether or not Nagios will attempt to # automatically reschedule active host and service checks to # "smooth" them out over time. This can help balance the load on -# the monitoring server. +# the monitoring server. # WARNING: THIS IS AN EXPERIMENTAL FEATURE - IT CAN DEGRADE # PERFORMANCE, RATHER THAN INCREASE IT, IF USED IMPROPERLY @@ -595,7 +598,7 @@ perfdata_timeout=5 # This setting determines whether or not Nagios will save state # information for services and hosts before it shuts down. Upon # startup Nagios will reload all saved service and host state -# information before starting to monitor. This is useful for +# information before starting to monitor. This is useful for # maintaining long-term data on state statistics, etc, but will # slow Nagios down a bit when it (re)starts. Since its only # a one-time penalty, I think its well worth the additional @@ -607,7 +610,7 @@ retain_state_information=1 # STATE RETENTION FILE # This is the file that Nagios should use to store host and -# service state information before it shuts down. The state +# service state information before it shuts down. The state # information in this file is also read immediately prior to # starting to monitor the network when Nagios is restarted. # This file is used only if the preserve_state_information @@ -630,7 +633,7 @@ retention_update_interval=60 # USE RETAINED PROGRAM STATE -# This setting determines whether or not Nagios will set +# This setting determines whether or not Nagios will set # program status variables based on the values saved in the # retention file. If you want to use retained program status # information, set this value to 1. If not, set this value @@ -657,7 +660,7 @@ use_retained_scheduling_info=1 # program restarts. # # The values of the masks are bitwise ANDs of values specified -# by the "MODATTR_" definitions found in include/common.h. +# by the "MODATTR_" definitions found in include/common.h. # For example, if you do not want the current enabled/disabled state # of flap detection and event handlers for hosts to be retained, you # would use a value of 24 for the host attribute mask... @@ -708,7 +711,7 @@ use_aggressive_host_checking=0 # SERVICE CHECK EXECUTION OPTION # This determines whether or not Nagios will actively execute -# service checks when it initially starts. If this option is +# service checks when it initially starts. If this option is # disabled, checks are not actively made, but Nagios can still # receive and process passive check results that come in. Unless # you're implementing redundant hosts or have a special need for @@ -730,7 +733,7 @@ accept_passive_service_checks=1 # HOST CHECK EXECUTION OPTION # This determines whether or not Nagios will actively execute -# host checks when it initially starts. If this option is +# host checks when it initially starts. If this option is # disabled, checks are not actively made, but Nagios can still # receive and process passive check results that come in. Unless # you're implementing redundant hosts or have a special need for @@ -787,7 +790,7 @@ process_performance_data=0 # These commands are run after every host and service check is # performed. These commands are executed only if the # enable_performance_data option (above) is set to 1. The command -# argument is the short name of a command definition that you +# argument is the short name of a command definition that you # define in your host configuration file. Read the HTML docs for # more information on performance data. @@ -867,7 +870,7 @@ obsess_over_services=0 # OBSESSIVE COMPULSIVE SERVICE PROCESSOR COMMAND # This is the command that is run for every service check that is # processed by Nagios. This command is executed only if the -# obsess_over_services option (above) is set to 1. The command +# obsess_over_services option (above) is set to 1. The command # argument is the short name of a command definition that you # define in your host configuration file. Read the HTML docs for # more information on implementing distributed monitoring. @@ -891,7 +894,7 @@ obsess_over_hosts=0 # OBSESSIVE COMPULSIVE HOST PROCESSOR COMMAND # This is the command that is run for every host check that is # processed by Nagios. This command is executed only if the -# obsess_over_hosts option (above) is set to 1. The command +# obsess_over_hosts option (above) is set to 1. The command # argument is the short name of a command definition that you # define in your host configuration file. Read the HTML docs for # more information on implementing distributed monitoring. @@ -930,9 +933,9 @@ passive_host_checks_are_soft=0 # ORPHANED HOST/SERVICE CHECK OPTIONS -# These options determine whether or not Nagios will periodically +# These options determine whether or not Nagios will periodically # check for orphaned host service checks. Since service checks are -# not rescheduled until the results of their previous execution +# not rescheduled until the results of their previous execution # instance are processed, there exists a possibility that some # checks may never get rescheduled. A similar situation exists for # host checks, although the exact scheduling details differ a bit @@ -1000,9 +1003,9 @@ additional_freshness_latency=15 # FLAP DETECTION OPTION # This option determines whether or not Nagios will try -# and detect hosts and services that are "flapping". +# and detect hosts and services that are "flapping". # Flapping occurs when a host or service changes between -# states too frequently. When Nagios detects that a +# states too frequently. When Nagios detects that a # host or service is flapping, it will temporarily suppress # notifications for that host/service until it stops # flapping. Flap detection is very experimental, so read @@ -1046,7 +1049,7 @@ date_format=iso8601 # the system configured timezone. # # NOTE: In order to display the correct timezone in the CGIs, you -# will also need to alter the Apache directives for the CGI path +# will also need to alter the Apache directives for the CGI path # to include your timezone. Example: # # <Directory "/usr/local/nagios/sbin/"> @@ -1083,7 +1086,7 @@ enable_embedded_perl=1 # This option determines whether or not Nagios will process Perl plugins # and scripts with the embedded Perl interpreter if the plugins/scripts # do not explicitly indicate whether or not it is okay to do so. Read -# the HTML documentation on the embedded Perl interpreter for more +# the HTML documentation on the embedded Perl interpreter for more # information on how this option works. use_embedded_perl_implicitly=1 @@ -1130,7 +1133,7 @@ use_regexp_matching=0 # "TRUE" REGULAR EXPRESSION MATCHING -# This option controls whether or not "true" regular expression +# This option controls whether or not "true" regular expression # matching takes place in the object config files. This option # only has an effect if regular expression matching is enabled # (see above). If this option is DISABLED, regular expression @@ -1183,7 +1186,7 @@ use_large_installation_tweaks=0 # This option determines whether or not Nagios will make all standard # macros available as environment variables when host/service checks # and system commands (event handlers, notifications, etc.) are -# executed. Enabling this option can cause performance issues in +# executed. Enabling this option can cause performance issues in # large installations, as it will consume a bit more memory and (more # importantly) consume more CPU. # Values: 1 - Enable environment variable macros (default) @@ -1224,7 +1227,7 @@ enable_environment_macros=1 # This option determines how much (if any) debugging information will # be written to the debug file. OR values together to log multiple # types of information. -# Values: +# Values: # -1 = Everything # 0 = Nothing # 1 = Functions diff --git a/puppet/modules/site_nagios/manifests/add_host.pp b/puppet/modules/site_nagios/manifests/add_host_services.pp index 94352de4..279809d1 100644 --- a/puppet/modules/site_nagios/manifests/add_host.pp +++ b/puppet/modules/site_nagios/manifests/add_host_services.pp @@ -1,17 +1,13 @@ -define site_nagios::add_host ( +define site_nagios::add_host_services ( + $domain_full_suffix, $domain_internal, $ip_address, $services, + $ssh_port, $openvpn_gateway_address='' ) { $nagios_hostname = $domain_internal - # Add Nagios host - nagios_host { $nagios_hostname: - address => $ip_address, - use => 'generic-host', - } - # Add Nagios service # First, we need to turn the serice array into hash, using a "hash template" diff --git a/puppet/modules/site_nagios/manifests/init.pp b/puppet/modules/site_nagios/manifests/init.pp index c3cfa02e..eb08cdcb 100644 --- a/puppet/modules/site_nagios/manifests/init.pp +++ b/puppet/modules/site_nagios/manifests/init.pp @@ -1,6 +1,6 @@ class site_nagios { tag 'leap_service' Class['site_config::default'] -> Class['site_nagios'] - + include site_nagios::server } diff --git a/puppet/modules/site_nagios/manifests/server.pp b/puppet/modules/site_nagios/manifests/server.pp index 3e1ef7e7..85443917 100644 --- a/puppet/modules/site_nagios/manifests/server.pp +++ b/puppet/modules/site_nagios/manifests/server.pp @@ -1,28 +1,34 @@ class site_nagios::server inherits nagios::base { # First, purge old nagios config (see #1467) - class { 'site_nagios::server::purge': - stage => setup - } + class { 'site_nagios::server::purge': } $nagios_hiera = hiera('nagios') $nagiosadmin_pw = htpasswd_sha1($nagios_hiera['nagiosadmin_pw']) - $hosts = $nagios_hiera['hosts'] + $nagios_hosts = $nagios_hiera['hosts'] include nagios::defaults include nagios::base - #Class ['nagios'] -> Class ['nagios::defaults'] - class {'nagios::apache': + class {'nagios': + # don't manage apache class from nagios, cause we already include + # it in site_apache::common + httpd => 'absent', allow_external_cmd => true, stored_config => false, - #before => Class ['nagios::defaults'] } + file { '/etc/apache2/conf.d/nagios3.conf': + ensure => link, + target => '/usr/share/doc/nagios3-common/examples/apache2.conf', + notify => Service['apache'] + } + + include site_apache::common include site_apache::module::headers File ['nagios_htpasswd'] { source => undef, - content => "nagiosadmin:$nagiosadmin_pw", + content => "nagiosadmin:${nagiosadmin_pw}", mode => '0640', } @@ -35,7 +41,18 @@ class site_nagios::server inherits nagios::base { group => 'nagios', } - create_resources ( site_nagios::add_host, $hosts ) + create_resources ( site_nagios::add_host_services, $nagios_hosts ) + include site_nagios::server::apache + include site_check_mk::server include site_shorewall::monitor + + augeas { + 'logrotate_nagios': + context => '/files/etc/logrotate.d/nagios/rule', + changes => [ 'set file /var/log/nagios3/nagios.log', 'set rotate 7', + 'set schedule daily', 'set compress compress', + 'set missingok missingok', 'set ifempty notifempty', + 'set copytruncate copytruncate' ] + } } diff --git a/puppet/modules/site_nagios/manifests/server/apache.pp b/puppet/modules/site_nagios/manifests/server/apache.pp new file mode 100644 index 00000000..8dbc7e9b --- /dev/null +++ b/puppet/modules/site_nagios/manifests/server/apache.pp @@ -0,0 +1,7 @@ +class site_nagios::server::apache { + include x509::variables + include site_config::x509::commercial::cert + include site_config::x509::commercial::key + include site_config::x509::commercial::ca + +} diff --git a/puppet/modules/site_nagios/manifests/server/purge.pp b/puppet/modules/site_nagios/manifests/server/purge.pp index 39735cd3..6815a703 100644 --- a/puppet/modules/site_nagios/manifests/server/purge.pp +++ b/puppet/modules/site_nagios/manifests/server/purge.pp @@ -1,7 +1,19 @@ -class site_nagios::server::purge { - exec {'purge_conf.d': - command => '/bin/rm -rf /etc/nagios3/conf.d/*', - onlyif => 'test -e /etc/nagios3/conf.d' +class site_nagios::server::purge inherits nagios::base { + # we don't want to get /etc/nagios3 and /etc/nagios3/conf.d + # purged, cause the check-mk-config-nagios3 package + # places its templates in /etc/nagios3/conf.d/check_mk, + # and check_mk -O updated it's nagios config in /etc/nagios3/conf.d/check_mk + File['nagios_cfgdir'] { + purge => false + } + File['nagios_confd'] { + purge => false } + # only purge files in the /etc/nagios3/conf.d/ dir, not in any subdir + exec {'purge_conf.d': + command => '/usr/bin/find /etc/nagios3/conf.d/ -maxdepth 1 -type f -exec rm {} \;', + onlyif => '/usr/bin/find /etc/nagios3/conf.d/ -maxdepth 1 -type f | grep -q "/etc/nagios3/conf.d"', + require => Package['nagios'] + } } diff --git a/puppet/modules/site_openvpn/manifests/init.pp b/puppet/modules/site_openvpn/manifests/init.pp index 42146741..7aec0faa 100644 --- a/puppet/modules/site_openvpn/manifests/init.pp +++ b/puppet/modules/site_openvpn/manifests/init.pp @@ -27,22 +27,23 @@ class site_openvpn { Class['site_config::default'] -> Class['site_openvpn'] - $openvpn_config = hiera('openvpn') - $openvpn_ports = $openvpn_config['ports'] + $openvpn = hiera('openvpn') + $openvpn_ports = $openvpn['ports'] + $openvpn_config = $openvpn['configuration'] if $::ec2_instance_id { $openvpn_gateway_address = $::ipaddress } else { - $openvpn_gateway_address = $openvpn_config['gateway_address'] - if $openvpn_config['second_gateway_address'] { - $openvpn_second_gateway_address = $openvpn_config['second_gateway_address'] + $openvpn_gateway_address = $openvpn['gateway_address'] + if $openvpn['second_gateway_address'] { + $openvpn_second_gateway_address = $openvpn['second_gateway_address'] } else { $openvpn_second_gateway_address = undef } } - $openvpn_allow_unlimited = $openvpn_config['allow_unlimited'] - $openvpn_unlimited_prefix = $openvpn_config['unlimited_prefix'] + $openvpn_allow_unlimited = $openvpn['allow_unlimited'] + $openvpn_unlimited_prefix = $openvpn['unlimited_prefix'] $openvpn_unlimited_tcp_network_prefix = '10.41.0' $openvpn_unlimited_tcp_netmask = '255.255.248.0' $openvpn_unlimited_tcp_cidr = '21' @@ -51,9 +52,9 @@ class site_openvpn { $openvpn_unlimited_udp_cidr = '21' if !$::ec2_instance_id { - $openvpn_allow_limited = $openvpn_config['allow_limited'] - $openvpn_limited_prefix = $openvpn_config['limited_prefix'] - $openvpn_rate_limit = $openvpn_config['rate_limit'] + $openvpn_allow_limited = $openvpn['allow_limited'] + $openvpn_limited_prefix = $openvpn['limited_prefix'] + $openvpn_rate_limit = $openvpn['rate_limit'] $openvpn_limited_tcp_network_prefix = '10.43.0' $openvpn_limited_tcp_netmask = '255.255.248.0' $openvpn_limited_tcp_cidr = '21' @@ -90,7 +91,8 @@ class site_openvpn { tls_remote => "\"${openvpn_unlimited_prefix}\"", server => "${openvpn_unlimited_tcp_network_prefix}.0 ${openvpn_unlimited_tcp_netmask}", push => "\"dhcp-option DNS ${openvpn_unlimited_tcp_network_prefix}.1\"", - management => '127.0.0.1 1000' + management => '127.0.0.1 1000', + config => $openvpn_config } site_openvpn::server_config { 'udp_config': port => '1194', @@ -99,7 +101,8 @@ class site_openvpn { tls_remote => "\"${openvpn_unlimited_prefix}\"", server => "${openvpn_unlimited_udp_network_prefix}.0 ${openvpn_unlimited_udp_netmask}", push => "\"dhcp-option DNS ${openvpn_unlimited_udp_network_prefix}.1\"", - management => '127.0.0.1 1001' + management => '127.0.0.1 1001', + config => $openvpn_config } } else { tidy { '/etc/openvpn/tcp_config.conf': } @@ -114,7 +117,8 @@ class site_openvpn { tls_remote => "\"${openvpn_limited_prefix}\"", server => "${openvpn_limited_tcp_network_prefix}.0 ${openvpn_limited_tcp_netmask}", push => "\"dhcp-option DNS ${openvpn_limited_tcp_network_prefix}.1\"", - management => '127.0.0.1 1002' + management => '127.0.0.1 1002', + config => $openvpn_config } site_openvpn::server_config { 'limited_udp_config': port => '1194', @@ -123,7 +127,8 @@ class site_openvpn { tls_remote => "\"${openvpn_limited_prefix}\"", server => "${openvpn_limited_udp_network_prefix}.0 ${openvpn_limited_udp_netmask}", push => "\"dhcp-option DNS ${openvpn_limited_udp_network_prefix}.1\"", - management => '127.0.0.1 1003' + management => '127.0.0.1 1003', + config => $openvpn_config } } else { tidy { '/etc/openvpn/limited_tcp_config.conf': } @@ -213,4 +218,7 @@ class site_openvpn { target => '/etc/default/openvpn', order => 10; } + + include site_check_mk::agent::openvpn + } diff --git a/puppet/modules/site_openvpn/manifests/server_config.pp b/puppet/modules/site_openvpn/manifests/server_config.pp index befeaef7..6246a836 100644 --- a/puppet/modules/site_openvpn/manifests/server_config.pp +++ b/puppet/modules/site_openvpn/manifests/server_config.pp @@ -54,7 +54,7 @@ define site_openvpn::server_config( $port, $proto, $local, $server, $push, - $management, $tls_remote = undef) { + $management, $config, $tls_remote = undef) { $openvpn_configname = $name @@ -96,15 +96,15 @@ define site_openvpn::server_config( server => $openvpn_configname; "tls-cipher ${openvpn_configname}": key => 'tls-cipher', - value => 'DHE-RSA-AES128-SHA', + value => $config['tls-cipher'], server => $openvpn_configname; "auth ${openvpn_configname}": key => 'auth', - value => 'SHA1', + value => $config['auth'], server => $openvpn_configname; "cipher ${openvpn_configname}": key => 'cipher', - value => 'AES-128-CBC', + value => $config['cipher'], server => $openvpn_configname; "dev ${openvpn_configname}": key => 'dev', diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp index de89c26e..bdfee665 100644 --- a/puppet/modules/site_postfix/manifests/mx.pp +++ b/puppet/modules/site_postfix/manifests/mx.pp @@ -4,7 +4,7 @@ class site_postfix::mx { $domain = $domain_hash['full_suffix'] $host_domain = $domain_hash['full'] $cert_name = hiera('name') - $mynetworks = join(hiera('mx_nodes'), ' ') + $mynetworks = join(hiera('mynetworks'), ' ') $root_mail_recipient = hiera ('contacts') $postfix_smtp_listen = 'all' @@ -31,6 +31,10 @@ class site_postfix::mx { value => 'vmail'; 'smtpd_tls_received_header': value => 'yes'; + # Note: we are setting this here, instead of in site_postfix::mx::smtp_tls + # because the satellites need to have a different value + 'smtp_tls_security_level': + value => 'may'; } include site_postfix::mx::smtpd_checks diff --git a/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp b/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp index aea66f78..83e27376 100644 --- a/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp +++ b/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp @@ -1,11 +1,13 @@ +# Defines which mail addresses shouldn't be available and where they should fwd class site_postfix::mx::reserved_aliases { postfix::mailalias { - [ 'postmaster', 'hostmaster', 'domainadmin', 'certmaster', 'ssladmin', - 'arin-admin', 'administrator', 'webmaster', 'www-data', 'www', - 'nobody', 'sys', 'postgresql', 'mysql', 'bin', 'cron', 'lp', 'games', - 'maildrop', 'abuse', 'noc', 'security', 'usenet', 'news', 'uucp', - 'ftp' ]: + [ 'abuse', 'admin', 'arin-admin', 'administrator', 'bin', 'cron', + 'certmaster', 'domainadmin', 'games', 'ftp', 'hostmaster', 'lp', + 'maildrop', 'mysql', 'news', 'nobody', 'noc', 'postmaster', 'postgresql', + 'security', 'ssladmin', 'sys', 'usenet', 'uucp', 'webmaster', 'www', + 'www-data', + ]: ensure => present, recipient => 'root' } diff --git a/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp b/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp index 3cc7ea72..d9b59f40 100644 --- a/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp +++ b/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp @@ -20,8 +20,6 @@ class site_postfix::mx::smtp_tls { value => 'sha1'; 'smtp_tls_session_cache_database': value => 'btree:${data_directory}/smtp_cache'; - 'smtp_tls_security_level': - value => 'may'; # see issue #4011 'smtp_tls_protocols': value => '!SSLv2, !SSLv3'; diff --git a/puppet/modules/site_postfix/manifests/satellite.pp b/puppet/modules/site_postfix/manifests/satellite.pp index 7be51b22..5725e6b8 100644 --- a/puppet/modules/site_postfix/manifests/satellite.pp +++ b/puppet/modules/site_postfix/manifests/satellite.pp @@ -10,5 +10,38 @@ class site_postfix::satellite { root_mail_recipient => $root_mail_recipient } + # There are special conditions for satellite hosts that will make them not be + # able to contact their relayhost: + # + # 1. they are on openstack/amazon/PC and are on the same cluster as the relay + # host, the MX lookup for the relay host will use the public IP, which cannot + # be contacted + # + # 2. When a domain is used that is not in DNS, because it is internal, + # a testing domain, etc. eg. a .local domain cannot be looked up in DNS + # + # to resolve this, so the satellite can contact the relayhost, we need to set + # the http://www.postfix.org/postconf.5.html#smtp_host_lookup to be 'native' + # which will cause the lookup to use the native naming service + # (nsswitch.conf), which typically defaults to 'files, dns' allowing the + # /etc/hosts to be consulted first, then DNS if the entry doesn't exist. + # + # NOTE: this will make it not possible to enable DANE support through DNSSEC + # with http://www.postfix.org/postconf.5.html#smtp_dns_support_level - but + # this parameter is not available until 2.11. If this ends up being important + # we could also make this an optional parameter for providers without + # dns / local domains + + postfix::config { + 'smtp_host_lookup': + value => 'native'; + + # Note: we are setting this here, instead of in site_postfix::mx::smtp_tls + # because the mx server has to have a different value + 'smtp_tls_security_level': + value => 'encrypt'; + } + include site_postfix::mx::smtp_tls + } diff --git a/puppet/modules/site_sshd/manifests/authorized_keys.pp b/puppet/modules/site_sshd/manifests/authorized_keys.pp index c18f691c..f36fe20f 100644 --- a/puppet/modules/site_sshd/manifests/authorized_keys.pp +++ b/puppet/modules/site_sshd/manifests/authorized_keys.pp @@ -1,4 +1,7 @@ define site_sshd::authorized_keys ($keys, $ensure = 'present', $home = '') { + # We use a custom define here to deploy the authorized_keys file + # cause puppet doesn't allow purgin before populating this file + # (see https://tickets.puppetlabs.com/browse/PUP-1174) # This line allows default homedir based on $title variable. # If $home is empty, the default is used. $homedir = $home ? {'' => "/home/${title}", default => $home} diff --git a/puppet/modules/site_sshd/templates/authorized_keys.erb b/puppet/modules/site_sshd/templates/authorized_keys.erb index 3c65e8ab..69f4d8e6 100644 --- a/puppet/modules/site_sshd/templates/authorized_keys.erb +++ b/puppet/modules/site_sshd/templates/authorized_keys.erb @@ -2,5 +2,9 @@ # all manually added keys will be overridden <% keys.sort.each do |user, hash| -%> +<% if user == 'monitor' -%> +command="/usr/bin/check_mk_agent",no-port-forwarding,no-x11-forwarding,no-agent-forwarding,no-pty,no-user-rc, <%=hash['type']-%> <%=hash['key']%> <%=user%> +<% else -%> <%=hash['type']-%> <%=hash['key']%> <%=user%> +<% end -%> <% end -%> diff --git a/puppet/modules/site_static/README b/puppet/modules/site_static/README new file mode 100644 index 00000000..bc719782 --- /dev/null +++ b/puppet/modules/site_static/README @@ -0,0 +1,3 @@ +Deploy one or more static websites to a node. + +For now, it only supports `amber` based static sites. Should support plain html and jekyll in the future. diff --git a/puppet/modules/site_static/manifests/domain.pp b/puppet/modules/site_static/manifests/domain.pp new file mode 100644 index 00000000..8af2230f --- /dev/null +++ b/puppet/modules/site_static/manifests/domain.pp @@ -0,0 +1,28 @@ +define site_static::domain ( + $locations, + $ca_cert, + $key, + $cert, + $tls_only) { + + $domain = $name + $base_dir = '/srv/static' + + create_resources(site_static::location, $locations) + + x509::cert { $domain: content => $cert } + x509::key { $domain: content => $key } + x509::ca { "${domain}_ca": content => $ca_cert } + + class { '::apache': no_default_site => true, ssl => true } + include site_apache::module::headers + include site_apache::module::alias + include site_apache::module::expires + include site_apache::module::removeip + include site_apache::module::rewrite + + apache::vhost::file { $domain: + content => template('site_static/apache.conf.erb') + } + +} diff --git a/puppet/modules/site_static/manifests/init.pp b/puppet/modules/site_static/manifests/init.pp new file mode 100644 index 00000000..91a4a7a9 --- /dev/null +++ b/puppet/modules/site_static/manifests/init.pp @@ -0,0 +1,17 @@ +class site_static { + tag 'leap_service' + $static = hiera('static') + $domains = $static['domains'] + $formats = $static['formats'] + + if (member($formats, 'amber')) { + include site_config::ruby::dev + rubygems::gem{'amber': } + } + + create_resources(site_static::domain, $domains) + + include site_shorewall::defaults + include site_shorewall::service::http + include site_shorewall::service::https +}
\ No newline at end of file diff --git a/puppet/modules/site_static/manifests/location.pp b/puppet/modules/site_static/manifests/location.pp new file mode 100644 index 00000000..1ba6807e --- /dev/null +++ b/puppet/modules/site_static/manifests/location.pp @@ -0,0 +1,25 @@ +define site_static::location($path, $format, $source) { + + $file_path = "/srv/static/${name}" + + if ($format == 'amber') { + exec {"amber_build_${name}": + cwd => $file_path, + command => 'amber rebuild', + user => 'www-data', + timeout => 600, + subscribe => Vcsrepo[$file_path] + } + } + + vcsrepo { $file_path: + ensure => present, + force => true, + revision => $source['revision'], + provider => $source['type'], + source => $source['repo'], + owner => 'www-data', + group => 'www-data' + } + +} diff --git a/puppet/modules/site_static/templates/apache.conf.erb b/puppet/modules/site_static/templates/apache.conf.erb new file mode 100644 index 00000000..76534911 --- /dev/null +++ b/puppet/modules/site_static/templates/apache.conf.erb @@ -0,0 +1,109 @@ +<%- + ## + ## An apache config for static websites. + ## + def location_directory(name, location) + if location['format'] == 'amber' + File.join(@base_dir, name, 'public') + else + File.join(@base_dir, name) + end + end + document_root = '/var/www' + @locations.each do |name, location| + if location['path'] == '/' + document_root = location_directory(name, location) + end + end +-%> + +<VirtualHost *:80> + ServerName <%= @domain %> + ServerAlias www.<%= @domain %> + RewriteEngine On + RewriteRule ^.*$ https://<%= @domain -%>%{REQUEST_URI} [R=permanent,L] +</VirtualHost> + +<VirtualHost *:443> + ServerName <%= @domain %> + ServerAlias www.<%= @domain %> + + #RewriteLog "/var/log/apache2/rewrite.log" + #RewriteLogLevel 3 + + SSLEngine on + SSLProtocol -all +SSLv3 +TLSv1 + SSLCipherSuite HIGH:MEDIUM:!aNULL:!SSLv2:!MD5:@STRENGTH + SSLHonorCipherOrder on + + Header add Strict-Transport-Security: "max-age=15768000;includeSubdomains" + Header set X-Frame-Options "deny" + + SSLCertificateKeyFile /etc/x509/keys/<%= @domain %>.key + SSLCertificateFile /etc/x509/certs/<%= @domain %>.crt + SSLCertificateChainFile /etc/ssl/certs/<%= @domain %>_ca.pem + + RequestHeader set X_FORWARDED_PROTO 'https' + + DocumentRoot <%= document_root %> + +<%- @locations.each do |name, location| -%> + ## + ## <%= name %> + ## + <%- if location['path'] == '/' -%> + # Location / + <%- else -%> + Alias <%= location['path'] %> <%= location_directory(name, location) %> + <Location <%= location['path'] %>> + <%- end -%> + # remove trailing slashes + RewriteEngine On + RewriteRule ^(.+)/$ /$1 [R=301,L] + + # e.g. /de/blah => /blah/index.de.html + RewriteCond %{DOCUMENT_ROOT}/$2/index.$1.html -f + RewriteRule ^/([a-z]{2})/(.*) /$2/index.$1.html [L] + + # e.g. /de/foo/bar => /foo/bar.de.html + RewriteCond %{DOCUMENT_ROOT}/$2.$1.html -f + RewriteRule ^/([a-z]{2})/(.*) /$2.$1.html [L] + + # e.g. /de => /index.de.html + RewriteCond %{DOCUMENT_ROOT}/index.$1.html -f + RewriteRule ^/([a-z]{2})$ /index.$1.html [L] + + # e.g. /de/img.png => /img.png + RewriteCond %{DOCUMENT_ROOT}/$2 -f + RewriteRule ^/([a-z]{2})/(.*) /$2 [L] + + # Simulate "DirectorySlash On" + # e.g. /foo/bar => /foo/bar/ (so that MultiViews will negotiate correct locale file) + RewriteCond %{DOCUMENT_ROOT}/$1 -d + RewriteRule ^/(.*[^/])$ /$1/ [PT] + <%- if location['path'] == '/' -%> + # end Location / + <%- else -%> + </Location> + <%- end -%> + <Directory <%= location_directory(name, location) %>> + ## + ## PERMISSIONS + ## + AllowOverride None + Order deny,allow + Allow from all + + ## + ## LOCALE SUPPORT (e.g. index.en.html) + ## + LanguagePriority en + ForceLanguagePriority Prefer Fallback + DirectoryIndex index + DirectorySlash Off + Options +MultiViews + </Directory> + +<%- end -%> + +</VirtualHost> diff --git a/puppet/modules/site_stunnel/manifests/clients.pp b/puppet/modules/site_stunnel/manifests/clients.pp index 791fdbc5..837665a3 100644 --- a/puppet/modules/site_stunnel/manifests/clients.pp +++ b/puppet/modules/site_stunnel/manifests/clients.pp @@ -28,4 +28,6 @@ define site_stunnel::clients ( Class['Site_config::X509::Ca'] ]; } + + include site_check_mk::agent::stunnel } diff --git a/puppet/modules/site_tor/manifests/init.pp b/puppet/modules/site_tor/manifests/init.pp index b6e73a23..02368a0e 100644 --- a/puppet/modules/site_tor/manifests/init.pp +++ b/puppet/modules/site_tor/manifests/init.pp @@ -19,12 +19,17 @@ class site_tor { my_family => '$2A431444756B0E7228A7918C85A8DACFF7E3B050', } - tor::daemon::directory { $::hostname: port => 80 } - - include site_shorewall::tor - - if ( $tor_type != 'exit' ) { + if ( $tor_type == 'exit'){ + tor::daemon::directory { $::hostname: port => 80 } + } + else { + tor::daemon::directory { $::hostname: + port => 80, + port_front_page => ''; + } include site_tor::disable_exit } + include site_shorewall::tor + } diff --git a/puppet/modules/site_webapp/manifests/apache.pp b/puppet/modules/site_webapp/manifests/apache.pp index d327877a..21243d34 100644 --- a/puppet/modules/site_webapp/manifests/apache.pp +++ b/puppet/modules/site_webapp/manifests/apache.pp @@ -7,19 +7,8 @@ class site_webapp::apache { $web_domain = hiera('domain') $domain_name = $web_domain['name'] - include x509::variables - include site_config::x509::commercial::cert - include site_config::x509::commercial::key - include site_config::x509::commercial::ca - - Class['Site_config::X509::Commercial::Key'] ~> Service[apache] - Class['Site_config::X509::Commercial::Cert'] ~> Service[apache] - Class['Site_config::X509::Commercial::Ca'] ~> Service[apache] - - class { '::apache': no_default_site => true, ssl => true } - + include site_apache::common include site_apache::module::headers - include site_apache::module::rewrite include site_apache::module::alias include site_apache::module::expires include site_apache::module::removeip @@ -27,11 +16,6 @@ class site_webapp::apache { class { 'passenger': use_munin => false } apache::vhost::file { - 'leap_webapp': - content => template('site_apache/vhosts.d/leap_webapp.conf.erb') - } - - apache::vhost::file { 'api': content => template('site_apache/vhosts.d/api.conf.erb') } diff --git a/puppet/modules/site_webapp/manifests/init.pp b/puppet/modules/site_webapp/manifests/init.pp index f8216aa4..d02a7261 100644 --- a/puppet/modules/site_webapp/manifests/init.pp +++ b/puppet/modules/site_webapp/manifests/init.pp @@ -157,5 +157,5 @@ class site_webapp { } include site_shorewall::webapp - + include site_check_mk::agent::webapp } diff --git a/puppet/modules/soledad/manifests/init.pp b/puppet/modules/soledad/manifests/init.pp index 7d44c8b4..7cf0b729 100644 --- a/puppet/modules/soledad/manifests/init.pp +++ b/puppet/modules/soledad/manifests/init.pp @@ -26,5 +26,4 @@ class soledad { group => 'soledad', require => User['soledad']; } - } diff --git a/puppet/modules/soledad/manifests/server.pp b/puppet/modules/soledad/manifests/server.pp index 572cad44..1137080f 100644 --- a/puppet/modules/soledad/manifests/server.pp +++ b/puppet/modules/soledad/manifests/server.pp @@ -59,4 +59,5 @@ class soledad::server { } include site_shorewall::soledad + include site_check_mk::agent::soledad } diff --git a/tests/order.rb b/tests/order.rb new file mode 100644 index 00000000..ffa6ae4e --- /dev/null +++ b/tests/order.rb @@ -0,0 +1,15 @@ +class LeapCli::Config::Node + # + # returns a list of node names that should be tested before this node. + # make sure to not return ourselves (please no dependency loops!). + # + def test_dependencies + dependents = LeapCli::Config::ObjectList.new + unless services.include?('couchdb') + if services.include?('webapp') || services.include?('mx') || services.include?('soledad') + dependents.merge! nodes_like_me[:services => 'couchdb'] + end + end + dependents.keys.delete_if {|name| self.name == name} + end +end
\ No newline at end of file diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb index 93551367..9d5da94f 100644 --- a/tests/white-box/couchdb.rb +++ b/tests/white-box/couchdb.rb @@ -2,13 +2,13 @@ raise SkipTest unless $node["services"].include?("couchdb") require 'json' -class TestCouchdb < LeapTest - depends_on "TestNetwork" +class CouchDB < LeapTest + depends_on "Network" def setup end - def test_00_daemons_running + def test_00_Are_daemons_running? assert_running 'tapicero' assert_running 'bin/beam' assert_running 'bin/epmd' @@ -18,7 +18,7 @@ class TestCouchdb < LeapTest # # check to make sure we can get welcome response from local couchdb # - def test_01_couch_is_working + def test_01_Is_CouchDB_running? assert_get(couchdb_url) do |body| assert_match /"couchdb":"Welcome"/, body, "Could not get welcome message from #{couchdb_url}. Probably couchdb is not running." end @@ -28,7 +28,7 @@ class TestCouchdb < LeapTest # # compare the configured nodes to the nodes that are actually listed in bigcouch # - def test_02_nodes_are_in_replication_database + def test_02_Is_cluster_membership_ok? url = couchdb_backend_url("/nodes/_all_docs") neighbors = assert_property('couch.bigcouch.neighbors') neighbors << assert_property('domain.full') @@ -47,7 +47,7 @@ class TestCouchdb < LeapTest # # this seems backward to me, so it might be the other way around. # - def test_03_replica_membership_is_kosher + def test_03_Are_configured_nodes_online? url = couchdb_url("/_membership") assert_get(url) do |body| response = JSON.parse(body) @@ -65,7 +65,7 @@ class TestCouchdb < LeapTest end end - def test_04_acl_users_exist + def test_04_Do_ACL_users_exist? acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'tapicero', 'webapp'] url = couchdb_backend_url("/_users/_all_docs") assert_get(url) do |body| @@ -77,7 +77,7 @@ class TestCouchdb < LeapTest pass end - def test_05_required_databases_exist + def test_05_Do_required_databases_exist? dbs_that_should_exist = ["customers","identities","keycache","sessions","shared","tickets","tokens","users"] dbs_that_should_exist.each do |db_name| assert_get(couchdb_url("/"+db_name)) do |body| diff --git a/tests/white-box/network.rb b/tests/white-box/network.rb index 57002beb..955857dc 100644 --- a/tests/white-box/network.rb +++ b/tests/white-box/network.rb @@ -2,12 +2,12 @@ require 'socket' raise SkipTest if $node["dummy"] -class TestNetwork < LeapTest +class Network < LeapTest def setup end - def test_01_can_connect_to_internet + def test_01_Can_connect_to_internet? assert_get('http://www.google.com/images/srpr/logo11w.png') pass end @@ -25,7 +25,7 @@ class TestNetwork < LeapTest # accept: 15984 # connect: "127.0.0.1:5984" # - def test_02_stunnel_is_running + def test_02_Is_stunnel_running? if $node['stunnel'] good_stunnel_pids = [] $node['stunnel'].each do |stunnel_type, stunnel_configs| diff --git a/tests/white-box/openvpn.rb b/tests/white-box/openvpn.rb index 2b1276f4..5eb2bdb5 100644 --- a/tests/white-box/openvpn.rb +++ b/tests/white-box/openvpn.rb @@ -1,12 +1,12 @@ raise SkipTest unless $node["services"].include?("openvpn") -class TestOpenvpn < LeapTest - depends_on "TestNetwork" +class Openvpn < LeapTest + depends_on "Network" def setup end - def test_01_daemons_running + def test_01_Are_daemons_running? assert_running '/usr/sbin/openvpn .* /etc/openvpn/tcp_config.conf' assert_running '/usr/sbin/openvpn .* /etc/openvpn/udp_config.conf' assert_running '/usr/sbin/unbound' diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb index 09e92797..142ac2de 100644 --- a/tests/white-box/webapp.rb +++ b/tests/white-box/webapp.rb @@ -2,8 +2,8 @@ raise SkipTest unless $node["services"].include?("webapp") require 'socket' -class TestWebapp < LeapTest - depends_on "TestNetwork" +class Webapp < LeapTest + depends_on "Network" HAPROXY_CONFIG = '/etc/haproxy/haproxy.cfg' @@ -20,7 +20,7 @@ class TestWebapp < LeapTest # connect: couch1.bitmask.i # connect_port: 15984 # - def test_01_can_contact_couchdb + def test_01_Can_contact_couchdb? assert_property('stunnel.couch_client') $node['stunnel']['couch_client'].values.each do |stunnel_conf| assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.' @@ -45,7 +45,7 @@ class TestWebapp < LeapTest # port: 4000 # weight: 10 # - def test_02_haproxy_is_working + def test_02_Is_haproxy_working? port = file_match(HAPROXY_CONFIG, /^ bind localhost:(\d+)$/) url = "http://localhost:#{port}" assert_get(url) do |body| @@ -54,7 +54,7 @@ class TestWebapp < LeapTest pass end - def test_03_daemons_running + def test_03_Are_daemons_running? assert_running '/usr/sbin/apache2' assert_running '/usr/bin/nickserver' pass |