diff options
23 files changed, 281 insertions, 47 deletions
diff --git a/bin/run_tests b/bin/run_tests index 2336eba8..f4fb0157 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -394,11 +394,9 @@ class LeapRunner < MiniTest::Unit # Converts snake_case and CamelCase to something more pleasant for humans to read. # def readable(str) - str.gsub(/([A-Z]+)([A-Z][a-z])/, '\1 \2'). - gsub(/([a-z])([A-Z])/, '\1 \2'). + str. gsub(/_/, ' '). - sub(/^test (\d* )?/i, ''). - downcase.capitalize + sub(/^test (\d* )?/i, '') end def machine_readable(str) @@ -428,7 +426,12 @@ class TestDependencyGraph end def tsort_each_child(test_class_name, &block) - @dependencies[test_class_name].each(&block) + if @dependencies[test_class_name] + @dependencies[test_class_name].each(&block) + else + puts "ERROR: bad dependency, no such class `#{test_class_name}`" + exit(1) + end end def sorted @@ -476,7 +479,7 @@ def pin_test_name(name) die name, "there is no test class `#{test_class}`" end if test_name - $pinned_test_method = $pinned_test_class.tests.detect{|m| m.to_s =~ /^test_(\d+_)?#{test_name}$/} + $pinned_test_method = $pinned_test_class.tests.detect{|m| m.to_s =~ /^test_(\d+_)?#{Regexp.escape(test_name)}$/} unless $pinned_test_method die name, "there is no test `#{test_name}` in class `#{test_class}`" end diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json index 7e3f20ba..731dee9a 100644 --- a/provider_base/services/mx.json +++ b/provider_base/services/mx.json @@ -10,7 +10,7 @@ "password": "= secret :couch_leap_mx_password", "salt": "= hex_secret :couch_leap_mx_password_salt, 128" }, - "mx_nodes": "= nodes['services' => 'mx']['environment' => '!local'].field('ip_address')", + "mynetworks": "= nodes['environment' => '!local'].map{|name, n| [n.ip_address, (global.facts[name]||{})['ec2_public_ipv4']]}.flatten.compact.uniq", "x509": { "use": true, "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", diff --git a/provider_base/services/static.json b/provider_base/services/static.json new file mode 100644 index 00000000..d9155a84 --- /dev/null +++ b/provider_base/services/static.json @@ -0,0 +1,6 @@ +{ + "static": { + "formats": "=> (self.static.domains||{}).values.collect{|d| (d.locations||{}).values.collect{|l|l['format']}}.flatten.uniq" + }, + "service_type": "public_service" +}
\ No newline at end of file diff --git a/puppet/manifests/site.pp b/puppet/manifests/site.pp index def0a642..ff8f8c6c 100644 --- a/puppet/manifests/site.pp +++ b/puppet/manifests/site.pp @@ -38,3 +38,8 @@ if $services =~ /\bmx\b/ { include site_mx } +if $services =~ /\bstatic\b/ { + include site_static +} + +include site_config::packages::uninstall
\ No newline at end of file diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg new file mode 100644 index 00000000..d58e876d --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg @@ -0,0 +1,7 @@ +# ignore openvpn TLS initialization errors when clients +# suddenly hangup before properly establishing +# a tls connection + I ovpn-.*TLS Error: Unroutable control packet received from + I ovpn-.*TLS Error: TLS key negotiation failed to occur within 60 seconds (check your network connectivity) + I ovpn-.*TLS Error: TLS handshake failed + diff --git a/puppet/modules/site_check_mk/manifests/agent/openvpn.pp b/puppet/modules/site_check_mk/manifests/agent/openvpn.pp new file mode 100644 index 00000000..919a408d --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/openvpn.pp @@ -0,0 +1,10 @@ +class site_check_mk::agent::openvpn { + + # check syslog + concat::fragment { 'syslog_openpvn': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/openvpn.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + +} diff --git a/puppet/modules/site_config/manifests/packages/base.pp b/puppet/modules/site_config/manifests/packages/base.pp index 9d416043..ae47963c 100644 --- a/puppet/modules/site_config/manifests/packages/base.pp +++ b/puppet/modules/site_config/manifests/packages/base.pp @@ -1,6 +1,5 @@ class site_config::packages::base { - include site_config::params # base set of packages that we want to have installed everywhere package { [ 'etckeeper', 'screen', 'less', 'ntp' ]: @@ -16,18 +15,4 @@ class site_config::packages::base { 'x11-utils', 'xterm' ]: ensure => absent; } - - if $::site_config::params::environment == 'local' or $::services =~ /\bwebapp\b/ { - $dev_packages_ensure = present - } else { - $dev_packages_ensure = absent - } - - # g++ and ruby1.9.1-dev are needed for nickserver/eventmachine (#4079) - # dev_packages are needed for building gems on the webapp node - - package { [ 'build-essential', 'g++', 'g++-4.7', 'gcc', - 'gcc-4.6', 'gcc-4.7', 'cpp', 'cpp-4.6', 'cpp-4.7', 'libc6-dev' ]: - ensure => $dev_packages_ensure - } } diff --git a/puppet/modules/site_config/manifests/packages/build_essential.pp b/puppet/modules/site_config/manifests/packages/build_essential.pp new file mode 100644 index 00000000..c9efaafb --- /dev/null +++ b/puppet/modules/site_config/manifests/packages/build_essential.pp @@ -0,0 +1,8 @@ +# +# include this whenever you want to ensure build-essential package and related compilers are installed. +# +class site_config::packages::build_essential { + if $install_build_essential == undef { + $install_build_essential = true + } +}
\ No newline at end of file diff --git a/puppet/modules/site_config/manifests/packages/uninstall.pp b/puppet/modules/site_config/manifests/packages/uninstall.pp new file mode 100644 index 00000000..2919cc96 --- /dev/null +++ b/puppet/modules/site_config/manifests/packages/uninstall.pp @@ -0,0 +1,20 @@ +# +# this should be included last to allow other modules to set $::install_build_packages +# +class site_config::packages::uninstall { + + if $site_config::packages::build_essential::install_essential == true { + $dev_packages_ensure = present + } else { + $dev_packages_ensure = absent + } + + # generally, dev packages are needed for installing ruby gems with native extensions. + # (nickserver, webapp, etc) + + package { [ 'build-essential', 'g++', 'g++-4.7', 'gcc', + 'gcc-4.6', 'gcc-4.7', 'cpp', 'cpp-4.6', 'cpp-4.7', 'libc6-dev' ]: + ensure => $dev_packages_ensure + } + +}
\ No newline at end of file diff --git a/puppet/modules/site_config/manifests/params.pp b/puppet/modules/site_config/manifests/params.pp index 5bdc0077..012b3ce0 100644 --- a/puppet/modules/site_config/manifests/params.pp +++ b/puppet/modules/site_config/manifests/params.pp @@ -8,6 +8,7 @@ class site_config::params { if $environment == 'local' { $interface = 'eth1' + include site_config::packages::build_essential } elsif hiera('interface','') != '' { $interface = hiera('interface') diff --git a/puppet/modules/site_config/manifests/ruby/dev.pp b/puppet/modules/site_config/manifests/ruby/dev.pp index dbc77ae7..3ea6ca96 100644 --- a/puppet/modules/site_config/manifests/ruby/dev.pp +++ b/puppet/modules/site_config/manifests/ruby/dev.pp @@ -3,4 +3,6 @@ class site_config::ruby::dev inherits site_config::ruby { ruby_version => '1.9.3', install_dev => true } + # building gems locally probably requires build-essential and gcc: + include site_config::packages::build_essential } diff --git a/puppet/modules/site_openvpn/manifests/init.pp b/puppet/modules/site_openvpn/manifests/init.pp index abcac41c..7aec0faa 100644 --- a/puppet/modules/site_openvpn/manifests/init.pp +++ b/puppet/modules/site_openvpn/manifests/init.pp @@ -218,4 +218,7 @@ class site_openvpn { target => '/etc/default/openvpn', order => 10; } + + include site_check_mk::agent::openvpn + } diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp index de89c26e..a37c7af9 100644 --- a/puppet/modules/site_postfix/manifests/mx.pp +++ b/puppet/modules/site_postfix/manifests/mx.pp @@ -4,7 +4,7 @@ class site_postfix::mx { $domain = $domain_hash['full_suffix'] $host_domain = $domain_hash['full'] $cert_name = hiera('name') - $mynetworks = join(hiera('mx_nodes'), ' ') + $mynetworks = join(hiera('mynetworks'), ' ') $root_mail_recipient = hiera ('contacts') $postfix_smtp_listen = 'all' diff --git a/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp b/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp index aea66f78..83e27376 100644 --- a/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp +++ b/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp @@ -1,11 +1,13 @@ +# Defines which mail addresses shouldn't be available and where they should fwd class site_postfix::mx::reserved_aliases { postfix::mailalias { - [ 'postmaster', 'hostmaster', 'domainadmin', 'certmaster', 'ssladmin', - 'arin-admin', 'administrator', 'webmaster', 'www-data', 'www', - 'nobody', 'sys', 'postgresql', 'mysql', 'bin', 'cron', 'lp', 'games', - 'maildrop', 'abuse', 'noc', 'security', 'usenet', 'news', 'uucp', - 'ftp' ]: + [ 'abuse', 'admin', 'arin-admin', 'administrator', 'bin', 'cron', + 'certmaster', 'domainadmin', 'games', 'ftp', 'hostmaster', 'lp', + 'maildrop', 'mysql', 'news', 'nobody', 'noc', 'postmaster', 'postgresql', + 'security', 'ssladmin', 'sys', 'usenet', 'uucp', 'webmaster', 'www', + 'www-data', + ]: ensure => present, recipient => 'root' } diff --git a/puppet/modules/site_static/README b/puppet/modules/site_static/README new file mode 100644 index 00000000..bc719782 --- /dev/null +++ b/puppet/modules/site_static/README @@ -0,0 +1,3 @@ +Deploy one or more static websites to a node. + +For now, it only supports `amber` based static sites. Should support plain html and jekyll in the future. diff --git a/puppet/modules/site_static/manifests/domain.pp b/puppet/modules/site_static/manifests/domain.pp new file mode 100644 index 00000000..8af2230f --- /dev/null +++ b/puppet/modules/site_static/manifests/domain.pp @@ -0,0 +1,28 @@ +define site_static::domain ( + $locations, + $ca_cert, + $key, + $cert, + $tls_only) { + + $domain = $name + $base_dir = '/srv/static' + + create_resources(site_static::location, $locations) + + x509::cert { $domain: content => $cert } + x509::key { $domain: content => $key } + x509::ca { "${domain}_ca": content => $ca_cert } + + class { '::apache': no_default_site => true, ssl => true } + include site_apache::module::headers + include site_apache::module::alias + include site_apache::module::expires + include site_apache::module::removeip + include site_apache::module::rewrite + + apache::vhost::file { $domain: + content => template('site_static/apache.conf.erb') + } + +} diff --git a/puppet/modules/site_static/manifests/init.pp b/puppet/modules/site_static/manifests/init.pp new file mode 100644 index 00000000..91a4a7a9 --- /dev/null +++ b/puppet/modules/site_static/manifests/init.pp @@ -0,0 +1,17 @@ +class site_static { + tag 'leap_service' + $static = hiera('static') + $domains = $static['domains'] + $formats = $static['formats'] + + if (member($formats, 'amber')) { + include site_config::ruby::dev + rubygems::gem{'amber': } + } + + create_resources(site_static::domain, $domains) + + include site_shorewall::defaults + include site_shorewall::service::http + include site_shorewall::service::https +}
\ No newline at end of file diff --git a/puppet/modules/site_static/manifests/location.pp b/puppet/modules/site_static/manifests/location.pp new file mode 100644 index 00000000..1ba6807e --- /dev/null +++ b/puppet/modules/site_static/manifests/location.pp @@ -0,0 +1,25 @@ +define site_static::location($path, $format, $source) { + + $file_path = "/srv/static/${name}" + + if ($format == 'amber') { + exec {"amber_build_${name}": + cwd => $file_path, + command => 'amber rebuild', + user => 'www-data', + timeout => 600, + subscribe => Vcsrepo[$file_path] + } + } + + vcsrepo { $file_path: + ensure => present, + force => true, + revision => $source['revision'], + provider => $source['type'], + source => $source['repo'], + owner => 'www-data', + group => 'www-data' + } + +} diff --git a/puppet/modules/site_static/templates/apache.conf.erb b/puppet/modules/site_static/templates/apache.conf.erb new file mode 100644 index 00000000..76534911 --- /dev/null +++ b/puppet/modules/site_static/templates/apache.conf.erb @@ -0,0 +1,109 @@ +<%- + ## + ## An apache config for static websites. + ## + def location_directory(name, location) + if location['format'] == 'amber' + File.join(@base_dir, name, 'public') + else + File.join(@base_dir, name) + end + end + document_root = '/var/www' + @locations.each do |name, location| + if location['path'] == '/' + document_root = location_directory(name, location) + end + end +-%> + +<VirtualHost *:80> + ServerName <%= @domain %> + ServerAlias www.<%= @domain %> + RewriteEngine On + RewriteRule ^.*$ https://<%= @domain -%>%{REQUEST_URI} [R=permanent,L] +</VirtualHost> + +<VirtualHost *:443> + ServerName <%= @domain %> + ServerAlias www.<%= @domain %> + + #RewriteLog "/var/log/apache2/rewrite.log" + #RewriteLogLevel 3 + + SSLEngine on + SSLProtocol -all +SSLv3 +TLSv1 + SSLCipherSuite HIGH:MEDIUM:!aNULL:!SSLv2:!MD5:@STRENGTH + SSLHonorCipherOrder on + + Header add Strict-Transport-Security: "max-age=15768000;includeSubdomains" + Header set X-Frame-Options "deny" + + SSLCertificateKeyFile /etc/x509/keys/<%= @domain %>.key + SSLCertificateFile /etc/x509/certs/<%= @domain %>.crt + SSLCertificateChainFile /etc/ssl/certs/<%= @domain %>_ca.pem + + RequestHeader set X_FORWARDED_PROTO 'https' + + DocumentRoot <%= document_root %> + +<%- @locations.each do |name, location| -%> + ## + ## <%= name %> + ## + <%- if location['path'] == '/' -%> + # Location / + <%- else -%> + Alias <%= location['path'] %> <%= location_directory(name, location) %> + <Location <%= location['path'] %>> + <%- end -%> + # remove trailing slashes + RewriteEngine On + RewriteRule ^(.+)/$ /$1 [R=301,L] + + # e.g. /de/blah => /blah/index.de.html + RewriteCond %{DOCUMENT_ROOT}/$2/index.$1.html -f + RewriteRule ^/([a-z]{2})/(.*) /$2/index.$1.html [L] + + # e.g. /de/foo/bar => /foo/bar.de.html + RewriteCond %{DOCUMENT_ROOT}/$2.$1.html -f + RewriteRule ^/([a-z]{2})/(.*) /$2.$1.html [L] + + # e.g. /de => /index.de.html + RewriteCond %{DOCUMENT_ROOT}/index.$1.html -f + RewriteRule ^/([a-z]{2})$ /index.$1.html [L] + + # e.g. /de/img.png => /img.png + RewriteCond %{DOCUMENT_ROOT}/$2 -f + RewriteRule ^/([a-z]{2})/(.*) /$2 [L] + + # Simulate "DirectorySlash On" + # e.g. /foo/bar => /foo/bar/ (so that MultiViews will negotiate correct locale file) + RewriteCond %{DOCUMENT_ROOT}/$1 -d + RewriteRule ^/(.*[^/])$ /$1/ [PT] + <%- if location['path'] == '/' -%> + # end Location / + <%- else -%> + </Location> + <%- end -%> + <Directory <%= location_directory(name, location) %>> + ## + ## PERMISSIONS + ## + AllowOverride None + Order deny,allow + Allow from all + + ## + ## LOCALE SUPPORT (e.g. index.en.html) + ## + LanguagePriority en + ForceLanguagePriority Prefer Fallback + DirectoryIndex index + DirectorySlash Off + Options +MultiViews + </Directory> + +<%- end -%> + +</VirtualHost> diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb index 93551367..9d5da94f 100644 --- a/tests/white-box/couchdb.rb +++ b/tests/white-box/couchdb.rb @@ -2,13 +2,13 @@ raise SkipTest unless $node["services"].include?("couchdb") require 'json' -class TestCouchdb < LeapTest - depends_on "TestNetwork" +class CouchDB < LeapTest + depends_on "Network" def setup end - def test_00_daemons_running + def test_00_Are_daemons_running? assert_running 'tapicero' assert_running 'bin/beam' assert_running 'bin/epmd' @@ -18,7 +18,7 @@ class TestCouchdb < LeapTest # # check to make sure we can get welcome response from local couchdb # - def test_01_couch_is_working + def test_01_Is_CouchDB_running? assert_get(couchdb_url) do |body| assert_match /"couchdb":"Welcome"/, body, "Could not get welcome message from #{couchdb_url}. Probably couchdb is not running." end @@ -28,7 +28,7 @@ class TestCouchdb < LeapTest # # compare the configured nodes to the nodes that are actually listed in bigcouch # - def test_02_nodes_are_in_replication_database + def test_02_Is_cluster_membership_ok? url = couchdb_backend_url("/nodes/_all_docs") neighbors = assert_property('couch.bigcouch.neighbors') neighbors << assert_property('domain.full') @@ -47,7 +47,7 @@ class TestCouchdb < LeapTest # # this seems backward to me, so it might be the other way around. # - def test_03_replica_membership_is_kosher + def test_03_Are_configured_nodes_online? url = couchdb_url("/_membership") assert_get(url) do |body| response = JSON.parse(body) @@ -65,7 +65,7 @@ class TestCouchdb < LeapTest end end - def test_04_acl_users_exist + def test_04_Do_ACL_users_exist? acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'tapicero', 'webapp'] url = couchdb_backend_url("/_users/_all_docs") assert_get(url) do |body| @@ -77,7 +77,7 @@ class TestCouchdb < LeapTest pass end - def test_05_required_databases_exist + def test_05_Do_required_databases_exist? dbs_that_should_exist = ["customers","identities","keycache","sessions","shared","tickets","tokens","users"] dbs_that_should_exist.each do |db_name| assert_get(couchdb_url("/"+db_name)) do |body| diff --git a/tests/white-box/network.rb b/tests/white-box/network.rb index 53df80dc..14de2eac 100644 --- a/tests/white-box/network.rb +++ b/tests/white-box/network.rb @@ -2,12 +2,12 @@ require 'socket' raise SkipTest if $node["dummy"] -class TestNetwork < LeapTest +class Network < LeapTest def setup end - def test_01_can_connect_to_internet + def test_01_Can_connect_to_internet? assert_get('http://www.google.com/images/srpr/logo11w.png') pass end @@ -25,7 +25,7 @@ class TestNetwork < LeapTest # accept: 15984 # connect: "127.0.0.1:5984" # - def test_02_stunnel_is_running + def test_02_Is_stunnel_running? if $node['stunnel'] good_stunnel_pids = [] $node['stunnel'].each do |stunnel_type, stunnel_configs| diff --git a/tests/white-box/openvpn.rb b/tests/white-box/openvpn.rb index 2b1276f4..5eb2bdb5 100644 --- a/tests/white-box/openvpn.rb +++ b/tests/white-box/openvpn.rb @@ -1,12 +1,12 @@ raise SkipTest unless $node["services"].include?("openvpn") -class TestOpenvpn < LeapTest - depends_on "TestNetwork" +class Openvpn < LeapTest + depends_on "Network" def setup end - def test_01_daemons_running + def test_01_Are_daemons_running? assert_running '/usr/sbin/openvpn .* /etc/openvpn/tcp_config.conf' assert_running '/usr/sbin/openvpn .* /etc/openvpn/udp_config.conf' assert_running '/usr/sbin/unbound' diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb index 09e92797..142ac2de 100644 --- a/tests/white-box/webapp.rb +++ b/tests/white-box/webapp.rb @@ -2,8 +2,8 @@ raise SkipTest unless $node["services"].include?("webapp") require 'socket' -class TestWebapp < LeapTest - depends_on "TestNetwork" +class Webapp < LeapTest + depends_on "Network" HAPROXY_CONFIG = '/etc/haproxy/haproxy.cfg' @@ -20,7 +20,7 @@ class TestWebapp < LeapTest # connect: couch1.bitmask.i # connect_port: 15984 # - def test_01_can_contact_couchdb + def test_01_Can_contact_couchdb? assert_property('stunnel.couch_client') $node['stunnel']['couch_client'].values.each do |stunnel_conf| assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.' @@ -45,7 +45,7 @@ class TestWebapp < LeapTest # port: 4000 # weight: 10 # - def test_02_haproxy_is_working + def test_02_Is_haproxy_working? port = file_match(HAPROXY_CONFIG, /^ bind localhost:(\d+)$/) url = "http://localhost:#{port}" assert_get(url) do |body| @@ -54,7 +54,7 @@ class TestWebapp < LeapTest pass end - def test_03_daemons_running + def test_03_Are_daemons_running? assert_running '/usr/sbin/apache2' assert_running '/usr/bin/nickserver' pass |