diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | Rakefile | 42 | ||||
| -rw-r--r-- | platform.rb | 2 | ||||
| m--------- | puppet/modules/apache | 0 | ||||
| m--------- | puppet/modules/backupninja | 0 | ||||
| m--------- | puppet/modules/bundler | 0 | ||||
| -rw-r--r-- | puppet/modules/clamav/templates/clamav-milter.conf.erb | 1 | ||||
| m--------- | puppet/modules/couchdb | 0 | ||||
| m--------- | puppet/modules/nagios | 0 | ||||
| m--------- | puppet/modules/rubygems | 0 | ||||
| -rw-r--r-- | puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg | 4 | ||||
| -rw-r--r-- | puppet/modules/site_couchdb/files/local.ini | 89 | ||||
| -rw-r--r-- | puppet/modules/site_nagios/manifests/server.pp | 2 | ||||
| -rw-r--r-- | puppet/modules/site_openvpn/manifests/server_config.pp | 8 | ||||
| -rw-r--r-- | puppet/modules/site_static/manifests/location.pp | 13 | ||||
| m--------- | puppet/modules/tor | 0 | ||||
| -rw-r--r-- | tests/helpers/os_helper.rb | 2 | ||||
| -rw-r--r-- | tests/white-box/mx.rb | 145 |
18 files changed, 167 insertions, 142 deletions
@@ -1,2 +1,3 @@ /.vagrant /puppet/modules/site_custom +Gemfile.lock @@ -3,11 +3,12 @@ require 'puppet-lint/tasks/puppet-lint' require 'puppet-syntax/tasks/puppet-syntax' # return list of modules, either -# submodules or custom modules +# submodules, custom or all modules # so we can check each array seperately def modules_pattern (type) submodules = Array.new custom_modules = Array.new + all_modules = Array.new Dir['puppet/modules/*'].sort.each do |m| system("grep -q #{m} .gitmodules") @@ -16,32 +17,41 @@ def modules_pattern (type) else custom_modules << m + '/**/*.pp' end + all_modules << m + '/**/*.pp' end - if type == 'submodule' - submodules - elsif type == 'custom' - custom_modules - else + case type + when 'submodule' + submodules + when 'custom' + custom_modules + when 'all' + all_modules end - end +exclude_paths = ["**/vendor/**/*", "spec/fixtures/**/*", "pkg/**/*" ] - -# redefine lint task with specific configuration +# redefine lint task so we don't lint submoudules for now Rake::Task[:lint].clear -desc "boo" PuppetLint::RakeTask.new :lint do |config| - # Pattern of files to check, defaults to `**/*.pp` - config.pattern = modules_pattern('custom') - config.ignore_paths = ["spec/**/*.pp", "pkg/**/*.pp", "vendor/**/*.pp"] + # only check for custom manifests, not submodules for now + config.pattern = modules_pattern('custom') + config.ignore_paths = exclude_paths config.disable_checks = ['documentation', '80chars'] config.fail_on_warnings = false end # rake syntax::* tasks -PuppetSyntax.exclude_paths = ["**/vendor/**/*"] +PuppetSyntax.exclude_paths = exclude_paths +PuppetSyntax.future_parser = true + +desc "Validate erb templates" +task :templates do + Dir['**/templates/**/*.erb'].each do |template| + sh "erb -P -x -T '-' #{template} | ruby -c" unless template =~ /.*vendor.*/ + end +end -desc "Run all puppet checks required for CI" -task :test => [:lint, :syntax , :validate, :spec] +desc "Run all puppet checks required for CI (syntax , validate, spec, lint)" +task :test => [:syntax , :validate, :templates, :spec, :lint] diff --git a/platform.rb b/platform.rb index 1e19a2a9..61fb50ce 100644 --- a/platform.rb +++ b/platform.rb @@ -45,7 +45,7 @@ Leap::Platform.define do :node_config => 'nodes/#{arg}.json', # input config files, environmentally scoped - :common_env_config => 'commmon.#{arg}.json', + :common_env_config => 'common.#{arg}.json', :provider_env_config => 'provider.#{arg}.json', :service_env_config => 'services/#{arg[0]}.#{arg[1]}.json', :tag_env_config => 'tags/#{arg[0]}.#{arg[1]}.json', diff --git a/puppet/modules/apache b/puppet/modules/apache -Subproject 117bed9a9263c21d253d86b667eb165948efdc2 +Subproject 415e9504f99dca3ccaa4dfd389dde24ad9d0e01 diff --git a/puppet/modules/backupninja b/puppet/modules/backupninja -Subproject 497513547be79f9d3c8e96f1650ec43ee634b27 +Subproject 5268a87c329f895017f8ea6c6abc377a4f9a6a7 diff --git a/puppet/modules/bundler b/puppet/modules/bundler -Subproject b4a4a8434616247156e59b860b47cc6256ead8d +Subproject bacec3e072649be4ade56f7df8506b46ae9c516 diff --git a/puppet/modules/clamav/templates/clamav-milter.conf.erb b/puppet/modules/clamav/templates/clamav-milter.conf.erb index 9bf7099e..50b4c620 100644 --- a/puppet/modules/clamav/templates/clamav-milter.conf.erb +++ b/puppet/modules/clamav/templates/clamav-milter.conf.erb @@ -4,7 +4,6 @@ FixStaleSocket true User clamav MilterSocketGroup clamav MilterSocketMode 666 -AllowSupplementaryGroups true ReadTimeout 120 Foreground false PidFile /var/run/clamav/clamav-milter.pid diff --git a/puppet/modules/couchdb b/puppet/modules/couchdb -Subproject 40d2289f8e10625cd45fdccdf492b5fb6490e66 +Subproject 76ff149a095023611c05bbb00157d06f87b07c0 diff --git a/puppet/modules/nagios b/puppet/modules/nagios -Subproject 68dab01a85996e14efcccf856b623a2caf25782 +Subproject e6fee3c731f68ccf8b6add8ada2162c7ad2b840 diff --git a/puppet/modules/rubygems b/puppet/modules/rubygems -Subproject e704c9fe1c40fea5b10fe3ca2b4f5de825341cc +Subproject 510a3693eab5dc78ed27d3728ee4d3b12334ea1 diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg index 71395c50..7daf0cac 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg @@ -15,3 +15,7 @@ # 401 Unauthorized error logged by webapp and possible other # applications C Unauthorized +# catch abnormal termination of processes (due to segfault/fpe +# signals etc). +# see https://github.com/pixelated/pixelated-user-agent/issues/683 + C systemd.*: main process exited, code=killed, status= diff --git a/puppet/modules/site_couchdb/files/local.ini b/puppet/modules/site_couchdb/files/local.ini index 22aa0177..b921a927 100644 --- a/puppet/modules/site_couchdb/files/local.ini +++ b/puppet/modules/site_couchdb/files/local.ini @@ -1,91 +1,8 @@ -; CouchDB Configuration Settings +; Puppet modified file !! ; Custom settings should be made in this file. They will override settings ; in default.ini, but unlike changes made to default.ini, this file won't be ; overwritten on server upgrade. -[couchdb] -;max_document_size = 4294967296 ; bytes - -[httpd] -;port = 5984 -;bind_address = 127.0.0.1 -; Options for the MochiWeb HTTP server. -;server_options = [{backlog, 128}, {acceptor_pool_size, 16}] -; For more socket options, consult Erlang's module 'inet' man page. -;socket_options = [{recbuf, 262144}, {sndbuf, 262144}, {nodelay, true}] - -; Uncomment next line to trigger basic-auth popup on unauthorized requests. -;WWW-Authenticate = Basic realm="administrator" - -; Uncomment next line to set the configuration modification whitelist. Only -; whitelisted values may be changed via the /_config URLs. To allow the admin -; to change this value over HTTP, remember to include {httpd,config_whitelist} -; itself. Excluding it from the list would require editing this file to update -; the whitelist. -;config_whitelist = [{httpd,config_whitelist}, {log,level}, {etc,etc}] - -[httpd_global_handlers] -;_google = {couch_httpd_proxy, handle_proxy_req, <<"http://www.google.com">>} - -# futon is enabled by default on bigcouch in default.ini -# we need to find another way to disable futon, it won't work disabling it here -# enable futon -#_utils = {couch_httpd_misc_handlers, handle_utils_dir_req, "/usr/share/couchdb/www"} -# disable futon -#_utils = {couch_httpd_misc_handlers, handle_welcome_req, <<"Welcome, Futon is disabled!">>} - -[couch_httpd_auth] -; If you set this to true, you should also uncomment the WWW-Authenticate line -; above. If you don't configure a WWW-Authenticate header, CouchDB will send -; Basic realm="server" in order to prevent you getting logged out. -; require_valid_user = false - -[log] -;level = debug - -[os_daemons] -; For any commands listed here, CouchDB will attempt to ensure that -; the process remains alive while CouchDB runs as well as shut them -; down when CouchDB exits. -;foo = /path/to/command -with args - -[daemons] -; enable SSL support by uncommenting the following line and supply the PEM's below. -; the default ssl port CouchDB listens on is 6984 -;httpsd = {couch_httpd, start_link, [https]} - -[ssl] -;cert_file = /etc/couchdb/server_cert.pem -;key_file = /etc/couchdb/server_key.pem -;password = somepassword -; set to true to validate peer certificates -;verify_ssl_certificates = false -; Path to file containing PEM encoded CA certificates (trusted -; certificates used for verifying a peer certificate). May be omitted if -; you do not want to verify the peer. -;cacert_file = /full/path/to/cacertf -; The verification fun (optionnal) if not specidied, the default -; verification fun will be used. -;verify_fun = {Module, VerifyFun} -;ssl_certificate_max_depth = 1 -; To enable Virtual Hosts in CouchDB, add a vhost = path directive. All requests to -; the Virual Host will be redirected to the path. In the example below all requests -; to http://example.com/ are redirected to /database. -; If you run CouchDB on a specific port, include the port number in the vhost: -; example.com:5984 = /database - -[vhosts] -;example.com = /database/ - -[update_notification] -;unique notifier name=/full/path/to/exe -with "cmd line arg" - -; To create an admin account uncomment the '[admins]' section below and add a -; line in the format 'username = password'. When you next start CouchDB, it -; will change the password to a hash (so that your passwords don't linger -; around in plain-text files). You can add more admin accounts with more -; 'username = password' lines. Don't forget to restart CouchDB after -; changing this. -;[admins] -;admin = mysecretpassword +[compactions] +_default = [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "03:00"}, {to, "05:00"}] diff --git a/puppet/modules/site_nagios/manifests/server.pp b/puppet/modules/site_nagios/manifests/server.pp index aa9b956e..6537124d 100644 --- a/puppet/modules/site_nagios/manifests/server.pp +++ b/puppet/modules/site_nagios/manifests/server.pp @@ -59,7 +59,7 @@ class site_nagios::server inherits nagios::base { include site_webapp::common_vhost include apache::module::headers - File ['nagios_htpasswd'] { + File['nagios_htpasswd'] { source => undef, content => "nagiosadmin:${nagiosadmin_pw}", mode => '0640', diff --git a/puppet/modules/site_openvpn/manifests/server_config.pp b/puppet/modules/site_openvpn/manifests/server_config.pp index 6decc665..15e6fb38 100644 --- a/puppet/modules/site_openvpn/manifests/server_config.pp +++ b/puppet/modules/site_openvpn/manifests/server_config.pp @@ -30,7 +30,7 @@ # auth SHA1 # # dkg: For HMAC digest to authenticate packets, we just want SHA256. OpenVPN lists -# a number of “digest” with names like “RSA-SHA256”, but this are legacy and +# a number of "digest" with names like "RSA-SHA256", but this are legacy and # should be avoided. # # elijah: i am not so sure that the digest algo matters for 'auth' option, because @@ -40,14 +40,14 @@ # cipher AES-128-CBC # # dkg: For the choice of cipher, we need to select an algorithm and a -# cipher mode. OpenVPN defaults to Blowfish, which is a fine algorithm — but +# cipher mode. OpenVPN defaults to Blowfish, which is a fine algorithm - but # our control channel is already relying on AES not being broken; if the # control channel is cracked, then the key material for the tunnel is exposed, # and the choice of algorithm is moot. So it makes more sense to me to rely on # the same cipher here: AES128. As for the cipher mode, OFB seems cleaner to # me, but CBC is more well-tested, and the OpenVPN man page (at least as of -# version 2.2.1) says “CBC is recommended and CFB and OFB should be considered -# advanced modes.” +# version 2.2.1) says "CBC is recommended and CFB and OFB should be considered +# advanced modes." # # note: the default is BF-CBC (blowfish) # diff --git a/puppet/modules/site_static/manifests/location.pp b/puppet/modules/site_static/manifests/location.pp index d116de2f..ab2b7494 100644 --- a/puppet/modules/site_static/manifests/location.pp +++ b/puppet/modules/site_static/manifests/location.pp @@ -23,6 +23,19 @@ define site_static::location($path, $format, $source) { } } + if ($format == 'rack') { + # Run bundler if there is a Gemfile + exec { 'bundler_update': + cwd => $file_path, + command => '/bin/bash -c "/usr/bin/bundle check --path vendor/bundle || /usr/bin/bundle install --path vendor/bundle --without test development debug"', + unless => '/usr/bin/bundle check --path vendor/bundle', + onlyif => 'test -f Gemfile', + user => 'www-data', + timeout => 600, + require => [Class['bundler::install'], Class['site_config::ruby::dev']]; + } + } + vcsrepo { $file_path: ensure => present, force => true, diff --git a/puppet/modules/tor b/puppet/modules/tor -Subproject 8c936c166b6da1ebd0e8d95e56ceee5167357d6 +Subproject 9981a70f7ba1f9e4fe33e4eb46654295287c1fc diff --git a/tests/helpers/os_helper.rb b/tests/helpers/os_helper.rb index da9ac843..9923d5b1 100644 --- a/tests/helpers/os_helper.rb +++ b/tests/helpers/os_helper.rb @@ -32,7 +32,7 @@ class LeapTest # runs the specified command, failing on a non-zero exit status. # def assert_run(command) - output = `#{command}` + output = `#{command} 2>&1` if $?.exitstatus != 0 fail "Error running `#{command}`:\n#{output}" end diff --git a/tests/white-box/mx.rb b/tests/white-box/mx.rb index 6c0982ce..e0cb273a 100644 --- a/tests/white-box/mx.rb +++ b/tests/white-box/mx.rb @@ -1,5 +1,6 @@ raise SkipTest unless service?(:mx) +require 'date' require 'json' require 'net/smtp' @@ -38,38 +39,15 @@ class Mx < LeapTest # using the by_address view for that same document again. # def test_03_Can_query_identities_db? - assert_get(couchdb_url("/identities", couch_url_options)) do |body| + ident = pick_random_identity + address = ident['address'] + url_base = %(/identities/_design/Identity/_view/by_address) + params = %(?include_docs=true&reduce=false&startkey="#{address}"&endkey="#{address}") + assert_get(couchdb_url(url_base+params, couch_url_options)) do |body| assert response = JSON.parse(body) - doc_count = response['doc_count'].to_i - if doc_count <= 1 - # the design document counts as one document. - skip "There are no identity documents yet." - else - # try five times to get a valid doc - for i in 1..5 - offset = rand(doc_count) # pick a random document - count_url = couchdb_url("/identities/_all_docs?include_docs=true&limit=1&skip=#{offset}", couch_url_options) - assert_get(count_url) do |body| - assert response = JSON.parse(body) - record = response['rows'].first - if record['id'] =~ /_design/ - next - else - address = record['doc']['address'] - assert address, "Identity document #{record['id']} is missing an address field. #{record['doc'].inspect}" - url_base = %(/identities/_design/Identity/_view/by_address) - params = %(?include_docs=true&reduce=false&startkey="#{address}"&endkey="#{address}") - assert_get(couchdb_url(url_base+params, couch_url_options)) do |body| - assert response = JSON.parse(body) - assert record = response['rows'].first - assert_equal address, record['doc']['address'] - pass - end - break - end - end - end - end + assert record = response['rows'].first + assert_equal address, record['doc']['address'] + pass end end @@ -92,12 +70,62 @@ class Mx < LeapTest end # + # TODO: test to make sure postmap returned the right result + # + def test_05_Can_postfix_query_leapmx? + ident = pick_random_identity(10, :with_public_key => true) + address = ident["address"] + + # + # virtual alias map: + # + # user@domain => 41c29a80a44f4775513c64ac9cab91b9@deliver.local + # + assert_run("postmap -v -q \"#{address}\" tcp:localhost:4242") + + # + # recipient access map: + # + # user@domain => [OK|REJECT|TEMP_FAIL] + # + # This map is queried by the mail server before delivery to the mail spool + # directory, and should check if the address is able to receive messages. + # Examples of reasons for denying delivery would be that the user is out of + # quota, is user, or have no pgp public key in the server. + # + # NOTE: in the future, when we support quota, we need to make sure that + # we don't randomly pick a user for this test that happens to be over quota. + # + assert_run("postmap -v -q \"#{address}\" tcp:localhost:2244") + + # + # certificate validity map: + # + # fa:2a:70:1f:d8:16:4e:1a:3b:15:c1:67:00:f0 => [200|500] + # + # Determines whether a particular SMTP client cert is authorized + # to relay mail, based on the fingerprint. + # + if ident["cert_fingerprints"] + not_expired = ident["cert_fingerprints"].select {|key, value| + Time.now.utc < DateTime.strptime("2016-01-03", "%F").to_time.utc + } + if not_expired.any? + fingerprint = not_expired.first + assert_run("postmap -v -q #{fingerprint} tcp:localhost:2424") + end + end + + pass + end + + # # The email sent by this test might get bounced back. # In this case, the test will pass, but the bounce message will # get sent to root, so the sysadmin will still figure out pretty # quickly that something is wrong. # - def test_05_Can_deliver_email? + def test_06_Can_deliver_email? addr = [TEST_EMAIL_USER, property('domain.full_suffix')].join('@') bad_addr = [TEST_BAD_USER, property('domain.full_suffix')].join('@') @@ -123,6 +151,59 @@ class Mx < LeapTest } end + # + # returns a random identity record that also has valid address + # and destination fields. + # + # options: + # + # * :with_public_key -- searches only for identities with public keys + # + # note to self: for debugging, here is the curl you want: + # curl --netrc "127.0.0.1:5984/identities/_design/Identity/_view/by_address?startkey=\"xxxx@leap.se\"&endkey=\"xxxx@leap.se\"&reduce=false&include_docs=true" + # + def pick_random_identity(tries=5, options={}) + assert_get(couchdb_url("/identities", couch_url_options)) do |body| + assert response = JSON.parse(body) + doc_count = response['doc_count'].to_i + if doc_count <= 1 + # the design document counts as one document. + skip "There are no identity documents yet." + else + # try repeatedly to get a valid doc + for i in 1..tries + offset = rand(doc_count) # pick a random document + url = couchdb_url("/identities/_all_docs?include_docs=true&limit=1&skip=#{offset}", couch_url_options) + assert_get(url) do |body| + assert response = JSON.parse(body) + record = response['rows'].first + if record['id'] =~ /_design/ + next + elsif record['doc'] && record['doc']['address'] + next if record['doc']['destination'].nil? || record['doc']['destination'].empty? + next if options[:with_public_key] && !record_has_key?(record) + return record['doc'] + else + fail "Identity document #{record['id']} is missing an address field. #{record['doc'].inspect}" + end + end + end + if options[:with_public_key] + skip "Could not find an Identity document with a public key for testing." + else + fail "Failed to find a valid Identity document (with address and destination)." + end + end + end + end + + def record_has_key?(record) + !record['doc']['keys'].nil? && + !record['doc']['keys'].empty? && + !record['doc']['keys']['pgp'].nil? && + !record['doc']['keys']['pgp'].empty? + end + TEST_EMAIL_PUBLIC_KEY=<<HERE -----BEGIN PGP PUBLIC KEY BLOCK----- mI0EVvzIKQEEAN4f8FOGntJGTTD+fFUQS6y/ihn6tYLtyGZZbCOd0t/9kHt/raoR |