From ec24733308676d6822aaeaae6c17b042f7e2bf14 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 7 Jul 2015 16:22:36 -0400 Subject: Clean up left-over files from old way of leap-mx logging, this should stop the logrotate cron errors from happening. (#7058) Change-Id: Iceaeb8c17600fc23d2b1ca075546f8573c145760 --- puppet/modules/site_config/manifests/remove_files.pp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/puppet/modules/site_config/manifests/remove_files.pp b/puppet/modules/site_config/manifests/remove_files.pp index 3f46659c..b339e6af 100644 --- a/puppet/modules/site_config/manifests/remove_files.pp +++ b/puppet/modules/site_config/manifests/remove_files.pp @@ -27,6 +27,10 @@ class site_config::remove_files { path => '/var/log/', recurse => true, matches => 'leap_mx*'; + 'leap_mx_rotate': + path => '/var/log/leap/', + recurse => true, + matches => [ 'mx.log.[0-9]', 'mx.log.[0-9]?', 'mx.log.[6-9]?gz']; '/srv/leap/webapp/public/provider.json':; '/srv/leap/couchdb/designs/tmp_users': recurse => true, -- cgit v1.2.3 From 1e872b71382f94f9c2d63ccbcaa43ca7d6741b42 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 9 Jul 2015 12:18:03 -0700 Subject: use latest amber for static nodes. --- puppet/modules/site_static/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_static/manifests/init.pp b/puppet/modules/site_static/manifests/init.pp index 2a198b57..1e7317a0 100644 --- a/puppet/modules/site_static/manifests/init.pp +++ b/puppet/modules/site_static/manifests/init.pp @@ -46,7 +46,7 @@ class site_static { } if (member($formats, 'amber')) { - rubygems::gem{'amber-0.3.4': } + rubygems::gem{'amber-0.3.7': } } create_resources(site_static::domain, $domains) -- cgit v1.2.3 From 2adbf36f656b68ef09f1d6dafdc9f1b8f99d8080 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 12 Jul 2015 12:14:11 -0400 Subject: Add emacs/vim modelines to Vagrantfile Makes most editors recognize Vagrantfile as a ruby file and do appropriate syntax highlighting --- Vagrantfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Vagrantfile b/Vagrantfile index 18590a8f..2d9b473e 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,3 +1,5 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : Vagrant.configure("2") do |config| config.vm.define :node1 do |config| -- cgit v1.2.3 From f1263dc48510c92366a1203beec84e9fd27e1c46 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 12 Jul 2015 23:06:38 -0400 Subject: vagrant: Increase Memory to 1GB leap_cli running puppet fails from lack of memory with the default amount of RAM --- Vagrantfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Vagrantfile b/Vagrantfile index 18590a8f..16f973cb 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -8,6 +8,7 @@ Vagrant.configure("2") do |config| config.vm.box = "LEAP/wheezy" #config.vm.network :private_network, ip: "10.5.5.102" config.vm.provider "virtualbox" do |v| + v.memory = 1024 v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] v.name = "node1" end -- cgit v1.2.3 From 2bb4f3d3c350aaac569f0c6a8c7935f4c35ecfee Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 12 Jul 2015 23:31:15 -0400 Subject: vagrant: Fix variable shadowing in Vagrantfile I'm unsure if the two levels of config are actually required, but making the most minimal changes possible atm. --- Vagrantfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 16f973cb..fdf92020 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,5 +1,7 @@ -Vagrant.configure("2") do |config| - config.vm.define :node1 do |config| +# -*- mode: ruby -*- +# vi: set ft=ruby : +Vagrant.configure("2") do |vagrant_config| + vagrant_config.vm.define :node1 do |config| # Please verify the sha512 sum of the downloaded box before importing it into vagrant ! # see https://leap.se/en/docs/platform/details/development#Verify.vagrantbox.download -- cgit v1.2.3 From c5f1790602b2a987f7cfb18b0da8e11e692cdd40 Mon Sep 17 00:00:00 2001 From: kwadronaut Date: Tue, 14 Jul 2015 13:55:49 +0000 Subject: bump amber version, taking care of puppet ordering with require. --- puppet/modules/site_static/manifests/init.pp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/puppet/modules/site_static/manifests/init.pp b/puppet/modules/site_static/manifests/init.pp index e37d5ad2..a3fd9c1e 100644 --- a/puppet/modules/site_static/manifests/init.pp +++ b/puppet/modules/site_static/manifests/init.pp @@ -46,10 +46,13 @@ class site_static { } if (member($formats, 'amber')) { + rubygems::gem{'amber-0.3.7': + require => Package['zlib1g-dev'] + } + package { 'zlib1g-dev': - ensure => installed + ensure => installed } - rubygems::gem{'amber-0.3.4': } } create_resources(site_static::domain, $domains) @@ -57,4 +60,4 @@ class site_static { include site_shorewall::defaults include site_shorewall::service::http include site_shorewall::service::https -} +} \ No newline at end of file -- cgit v1.2.3 From d609a948520e38feb79892eec6c80f3915940444 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 21 Jul 2015 10:19:23 -0400 Subject: minor linting fix double quotes and indentation Change-Id: I79c28159d17e6256db3094f413d61dcdc9520dc6 --- puppet/modules/leap_mx/manifests/init.pp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/puppet/modules/leap_mx/manifests/init.pp b/puppet/modules/leap_mx/manifests/init.pp index 6bcdd19a..2986f622 100644 --- a/puppet/modules/leap_mx/manifests/init.pp +++ b/puppet/modules/leap_mx/manifests/init.pp @@ -77,16 +77,16 @@ class leap_mx { } augeas { - "logrotate_mx": - context => "/files/etc/logrotate.d/leap-mx/rule", + 'logrotate_mx': + context => '/files/etc/logrotate.d/leap-mx/rule', changes => [ - "set file /var/log/leap/mx.log", - 'set rotate 5', - 'set schedule daily', - 'set compress compress', - 'set missingok missingok', - 'set ifempty notifempty', - 'set copytruncate copytruncate' - ] + 'set file /var/log/leap/mx.log', + 'set rotate 5', + 'set schedule daily', + 'set compress compress', + 'set missingok missingok', + 'set ifempty notifempty', + 'set copytruncate copytruncate' + ] } } -- cgit v1.2.3 From 7c071c8e5953922040175c9bfdb458effb982847 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 21 Jul 2015 11:00:50 -0400 Subject: Fix leap-mx logrotation to work with twistd (#7058) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We don't want to try and create the log file, twistd will do that. Don’t rename the log file from mx.log to mx.log.0, instead just copy it to mx.log.1, and then clear out mx.log so it’s empty (this is needed because leap-mx might assume that its file descriptor is still valid and continue trying to write to it, without this, leap-mx might lose data because it’ll assume the original log file is still around and continue to write to it, even though it’s gone)It’s a little dangerous because it’s possible that you might lose some logged data between the time that logrotate copies the new log file and truncates the old file (Caveat administrator). Finally, we don't want logrotate to complain if it finds mx.log, its ok if its there. Change-Id: I9952627f4d47e7a89a2915f6b72d82f9e6ca0d8b --- puppet/modules/leap_mx/manifests/init.pp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/puppet/modules/leap_mx/manifests/init.pp b/puppet/modules/leap_mx/manifests/init.pp index 2986f622..284662d2 100644 --- a/puppet/modules/leap_mx/manifests/init.pp +++ b/puppet/modules/leap_mx/manifests/init.pp @@ -83,9 +83,11 @@ class leap_mx { 'set file /var/log/leap/mx.log', 'set rotate 5', 'set schedule daily', + 'clear nocreate', + 'rm create', + 'rm ifempty', 'set compress compress', 'set missingok missingok', - 'set ifempty notifempty', 'set copytruncate copytruncate' ] } -- cgit v1.2.3 From 21af45fd2156e16786e3476c779115662ecb72a7 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 21 Jul 2015 15:47:44 -0400 Subject: Increase tapicero heatbeat nagios checks (#7275) Increase warning/critical thresholds for time between tapicero heartbeat checks so it will emit less false positives Change-Id: I0f97373d88658b7f17b2c4e8c1963198dc3f66ed --- puppet/modules/site_check_mk/manifests/agent/tapicero.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp index ad9962d4..8505b34a 100644 --- a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp +++ b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp @@ -20,7 +20,7 @@ class site_check_mk::agent::tapicero { 'Tapicero_Heartbeat': incl => '/etc/check_mk/mrpe.cfg', lens => 'Spacevars.lns', - changes => 'set Tapicero_Heartbeat \'/usr/local/lib/nagios/plugins/check_last_regex_in_log -f /var/log/leap/tapicero.log -r "tapicero" -w 300 -c 600\'', + changes => 'set Tapicero_Heartbeat \'/usr/local/lib/nagios/plugins/check_last_regex_in_log -f /var/log/leap/tapicero.log -r "tapicero" -w 1200 -c 2400\'', require => File['/etc/check_mk/mrpe.cfg']; } } -- cgit v1.2.3 From 2761fa77394d5a2857812de840e49172d0e486fb Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Thu, 23 Jul 2015 10:10:05 -0400 Subject: update CHANGES.md for the latest set of information Change-Id: I06e29515a28af8688d839fffa01a3dfe7fc8a2fc --- CHANGES.md | 55 ++++++------------------------------------------------- 1 file changed, 6 insertions(+), 49 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6540bd0c..67d8edc1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,9 +1,9 @@ -Platform 0.7.0 +Platform 0.7.1 ----------------------- Compatibility: -* Requires leap_cli version 1.7.2 +* Requires leap_cli version 1.7.4 * Requires bitmask client version >= 0.7 * Previous releases supported cookies when using the provider API. Now, only tokens are supported. @@ -13,55 +13,12 @@ Compatibility: * webapp 0.7 * soledad 0.7 -Commits: https://leap.se/git/leap_platform.git/shortlog/refs/tags/0.7.0 -Issues fixed: https://leap.se/code/versions/168 +Commits: https://leap.se/git/leap_platform.git/shortlog/refs/tags/0.7.1 +Issues fixed: https://leap.se/code/versions/159 Upgrading: -* `gem install leap_cli --version 1.7.2`. -* `cd leap_platform; git pull; git checkout 0.7.0`. +* `gem install leap_cli --version 1.7.4`. +* `cd leap_platform; git pull; git checkout 0.7.1`. * `leap deploy` -* `leap db destroy --db sessions,tokens` You can ignore message about needing to redeploy (since, in this case, we just want to permanently delete those databases). * `leap test` to make sure everything is working - -New features: - -* rotating couchdb databases: CouchDB is not designed to handle ephemeral data, like sessions, because documents are never really deleted (a tombstone document is always kept to record the deletion). To overcome this limitation, we now rotate the `sessions` and `tokens` databases monthly. The new database names are `tokens_XXX` and `sessions_XXX` where XXX is a counter since the epoch that increments every month (not a calendar month, but a month's worth of seconds). Additionally, nagios checks and `leap test run` now will create and destroy test users in the `tmp_users` database, which will get periodically deleted and recreated. -* deployment logging: information on every deploy is logged to `/var/log/leap` on the node, including the user, leap_cli version, and platform version. -* you must now run `leap deploy --downgrade` if you want to deploy an older version over a newer platform version. -* the install source for each custom daemons (e.g. tapicero, etc) can now configured in `common.json`. -* you can configure apt sources in common.json -* improved nagios graphs integration (with pnp4nagios) -* default MTU was reduced to 1400 for better overall compatibility -* install haveged for some minimal entropy on depleted systems -* switch to release branches for webapp, tapicero -* implement weakdh recommendations, and update minimal cipher lists for web TLS connections -* many bug bugfixes, security improvements, and tests - - -Platform 0.6 -------------------------------------- - -Compatibility: - -* Requires leap_cli version 1.6 -* Requires bitmask client version >= 0.5 - -Commits: https://leap.se/git/leap_platform.git/shortlog/refs/tags/0.6.0 - -New features: - -* single node deployment -* include custom puppet modules and manifests -* couch flexibility -* stunnel rework -* new debian repository structure -* dependency pinning -* leap_cli modularization -* improved cert generation -* monitoring improvements such as per-environment tooling and notifications -* tor hidden service support -* switch away from NIST curve and ensure TLSv1 is used -* tests made significantly more robust -* add support for webapp deployment to a subdomain -* many, many bugfixes and stability improvements -- cgit v1.2.3 From 8684aa38ece3271a0eb0f8a1751f6c3297025afa Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 28 Jul 2015 14:35:40 -0400 Subject: Support RBL blocking of incoming mail (#5923) Set zen.spamhaus as the default rbl Change-Id: Ic3537d645c80ba42267bab370a1cf77730382158 --- provider_base/services/mx.json | 1 + puppet/modules/site_postfix/manifests/mx.pp | 1 + puppet/modules/site_postfix/manifests/mx/smtpd_checks.pp | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json index 11293ae8..db2e4795 100644 --- a/provider_base/services/mx.json +++ b/provider_base/services/mx.json @@ -16,6 +16,7 @@ "salt": "= hex_secret :couch_leap_mx_password_salt, 128" }, "mynetworks": "= nodes['environment' => '!local'].map{|name, n| [n.ip_address, (global.facts[name]||{})['ec2_public_ipv4']]}.flatten.compact.uniq", + "rbls": ["zen.spamhaus.org"], "x509": { "use": true, "use_commercial": true, diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp index 49692d24..af0f9f56 100644 --- a/puppet/modules/site_postfix/manifests/mx.pp +++ b/puppet/modules/site_postfix/manifests/mx.pp @@ -8,6 +8,7 @@ class site_postfix::mx { $host_domain = $domain_hash['full'] $cert_name = hiera('name') $mynetworks = join(hiera('mynetworks'), ' ') + $rbls = suffix(prefix(hiera('rbls'), 'reject_rbl_client '), ',') $root_mail_recipient = hiera('contacts') $postfix_smtp_listen = 'all' diff --git a/puppet/modules/site_postfix/manifests/mx/smtpd_checks.pp b/puppet/modules/site_postfix/manifests/mx/smtpd_checks.pp index 0ec40277..1c3e5c92 100644 --- a/puppet/modules/site_postfix/manifests/mx/smtpd_checks.pp +++ b/puppet/modules/site_postfix/manifests/mx/smtpd_checks.pp @@ -6,7 +6,7 @@ class site_postfix::mx::smtpd_checks { 'checks_dir': value => '$config_directory/checks'; 'smtpd_client_restrictions': - value => 'permit_mynetworks,permit'; + value => "${site_postfix::mx::rbls}permit_mynetworks,permit"; 'smtpd_data_restrictions': value => 'permit_mynetworks, reject_unauth_pipelining, permit'; 'smtpd_delay_reject': -- cgit v1.2.3 From 738e77a729813901db8725cacfb15f4150fd49d2 Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 3 Aug 2015 14:44:24 -0700 Subject: webapp: add support for customizing locales --- provider_base/services/webapp.json | 4 +++- puppet/modules/site_webapp/templates/config.yml.erb | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index 941f4f61..081b3718 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -31,7 +31,9 @@ }, "engines": [ "support" - ] + ], + "locales": "= provider.languages", + "default_locale": "= provider.default_language" }, "stunnel": { "clients": { diff --git a/puppet/modules/site_webapp/templates/config.yml.erb b/puppet/modules/site_webapp/templates/config.yml.erb index ccde2d2e..e8853ade 100644 --- a/puppet/modules/site_webapp/templates/config.yml.erb +++ b/puppet/modules/site_webapp/templates/config.yml.erb @@ -2,6 +2,11 @@ <%- cert_options = @webapp['client_certificates'] -%> production: admins: <%= @webapp['admins'].inspect %> + default_locale: :<%= @webapp['default_locale'] %> + available_locales: +<%- @webapp['locales'].each do |locale| -%> + - :<%= locale %> +<%- end -%> domain: <%= @provider_domain %> force_ssl: <%= @webapp['secure'] %> client_ca_key: <%= scope.lookupvar('x509::variables::keys') %>/<%= scope.lookupvar('site_config::params::client_ca_name') %>.key -- cgit v1.2.3 From 478987d564e96c123ae243b1f6e089e00df20fcf Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 3 Aug 2015 14:45:16 -0700 Subject: allow_registration should always be false if enrollment_policy is 'closed' --- provider_base/provider.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider_base/provider.json b/provider_base/provider.json index 60ad2a9e..9b6e73ca 100644 --- a/provider_base/provider.json +++ b/provider_base/provider.json @@ -32,7 +32,7 @@ "allow_free": "= provider.service.levels.select {|l| l['rate'].nil?}.any?", "allow_paid": "= provider.service.levels.select {|l| !l['rate'].nil?}.any?", "allow_anonymous": "= provider.service.levels.select {|l| l['name'] == 'anonymous'}.any?", - "allow_registration": "= provider.service.levels.select {|l| l['name'] != 'anonymous'}.any?", + "allow_registration": "= provider.enrollment_policy != 'closed' && provider.service.levels.select {|l| l['name'] != 'anonymous'}.any?", "allow_limited_bandwidth": "= provider.service.levels.select {|l| l['bandwidth'] == 'limited'}.any?", "allow_unlimited_bandwidth": "= provider.service.levels.select {|l| l['bandwidth'].nil?}.any?" }, -- cgit v1.2.3 From fc082470fa945dd5a394057cbea8ea55831b9fad Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 7 Aug 2015 14:08:09 -0700 Subject: set platform version 0.8, pin to leap_cli 1.8 --- platform.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platform.rb b/platform.rb index 0c3de2a0..61f0dd05 100644 --- a/platform.rb +++ b/platform.rb @@ -4,8 +4,8 @@ # Leap::Platform.define do - self.version = "0.7.1" - self.compatible_cli = "1.7.0".."1.7.99" + self.version = "0.8" + self.compatible_cli = "1.8".."1.99" # # the facter facts that should be gathered -- cgit v1.2.3 From e6f6324ebea9fc46d5d617a08c2eb239349d9f90 Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 7 Aug 2015 18:08:14 -0700 Subject: move 'enabled service' calculation to a macro. --- .../files/service-definitions/provider.json.erb | 5 ----- provider_base/lib/macros.rb | 1 + provider_base/lib/macros/provider.rb | 20 ++++++++++++++++++++ provider_base/provider.json | 3 ++- 4 files changed, 23 insertions(+), 6 deletions(-) create mode 100644 provider_base/lib/macros/provider.rb diff --git a/provider_base/files/service-definitions/provider.json.erb b/provider_base/files/service-definitions/provider.json.erb index be8ae484..e51f9c3a 100644 --- a/provider_base/files/service-definitions/provider.json.erb +++ b/provider_base/files/service-definitions/provider.json.erb @@ -6,11 +6,6 @@ ) hsh['domain'] = domain.full_suffix - # advertise services that are 'user services' and for which there are actually nodes - hsh['services'] ||= global.services[:service_type => :user_service].field(:name).select do |service| - nodes_like_me[:services => service].any? - end - hsh['api_version'] = "1" hsh['api_uri'] = ["https://", api.domain, ':', api.port].join diff --git a/provider_base/lib/macros.rb b/provider_base/lib/macros.rb index ecc3e6ba..fdb9a94e 100644 --- a/provider_base/lib/macros.rb +++ b/provider_base/lib/macros.rb @@ -13,3 +13,4 @@ require_relative 'macros/keys' require_relative 'macros/nodes' require_relative 'macros/secrets' require_relative 'macros/stunnel' +require_relative 'macros/provider' diff --git a/provider_base/lib/macros/provider.rb b/provider_base/lib/macros/provider.rb new file mode 100644 index 00000000..84c4e1b8 --- /dev/null +++ b/provider_base/lib/macros/provider.rb @@ -0,0 +1,20 @@ +# +# These macros are intended only for use in provider.json, although they are +# currently loaded in all .json contexts. +# + +module LeapCli + module Macro + + # + # returns an array of the service names, including only those services that + # are enabled for this environment. + # + def enabled_services + manager.env(self.environment).services[:service_type => :user_service].field(:name).select { |service| + manager.nodes[:environment => self.environment][:services => service].any? + } + end + + end +end diff --git a/provider_base/provider.json b/provider_base/provider.json index 9b6e73ca..81b2ea98 100644 --- a/provider_base/provider.json +++ b/provider_base/provider.json @@ -14,6 +14,7 @@ "languages": ["en"], "default_language": "en", "enrollment_policy": "open", + "services": "= enabled_services", "service": { // bandwidth limit is in Bytes, storage limit is in MB. // for example: @@ -31,7 +32,7 @@ "bandwidth_limit": 102400, "allow_free": "= provider.service.levels.select {|l| l['rate'].nil?}.any?", "allow_paid": "= provider.service.levels.select {|l| !l['rate'].nil?}.any?", - "allow_anonymous": "= provider.service.levels.select {|l| l['name'] == 'anonymous'}.any?", + "allow_anonymous": "= provider.service.levels.select {|l| l['name'] == 'anonymous'}.any? && services.include?('openvpn')", "allow_registration": "= provider.enrollment_policy != 'closed' && provider.service.levels.select {|l| l['name'] != 'anonymous'}.any?", "allow_limited_bandwidth": "= provider.service.levels.select {|l| l['bandwidth'] == 'limited'}.any?", "allow_unlimited_bandwidth": "= provider.service.levels.select {|l| l['bandwidth'].nil?}.any?" -- cgit v1.2.3 From 5e21bb0d2415de0a40adfaa3b149313c459e7947 Mon Sep 17 00:00:00 2001 From: varac Date: Tue, 11 Aug 2015 14:57:58 +0200 Subject: Don't use check_mk logwatch to watch bigcouch logs anymore (#7375) The rationale here is: - bigcouch/its included erlang version is incredibly noisy and spits out warnings/error msgs all the time - it uses the worst logging format i ever saw, multiple lines directly to a file (couch 2.0 uses lager as logging backend which can log to syslog) - trying to sort out the false positives will take too much time, and who knows which of them will be resolved in couch 1.6/2.0 Change-Id: Idbe6b37a19cd65ce31a50d4c28eedb4cf15ba3b5 --- .../modules/site_check_mk/manifests/agent/couchdb.pp | 16 ++++++++++++---- puppet/modules/site_config/manifests/remove_files.pp | 19 ++++++++++++++++++- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp index abfc7ad0..8de5121b 100644 --- a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp @@ -1,9 +1,17 @@ +# configure logwatch and nagios checks for couchdb class site_check_mk::agent::couchdb { - # watch logs - file { '/etc/check_mk/logwatch.d/bigcouch.cfg': - source => 'puppet:///modules/site_check_mk/agent/logwatch/bigcouch.cfg', - } + # watch bigcouch logs + # currently disabled because bigcouch is too noisy + # see https://leap.se/code/issues/7375 for more details + # and site_config::remove_files for removing leftovers + #file { '/etc/check_mk/logwatch.d/bigcouch.cfg': + # source => 'puppet:///modules/site_check_mk/agent/logwatch/bigcouch.cfg', + #} + + # check syslog msg from: + # - empd + # - /usr/local/bin/couch-doc-update concat::fragment { 'syslog_couchdb': source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/couchdb.cfg', target => '/etc/check_mk/logwatch.d/syslog.cfg', diff --git a/puppet/modules/site_config/manifests/remove_files.pp b/puppet/modules/site_config/manifests/remove_files.pp index b339e6af..a9a0c8bf 100644 --- a/puppet/modules/site_config/manifests/remove_files.pp +++ b/puppet/modules/site_config/manifests/remove_files.pp @@ -46,5 +46,22 @@ class site_config::remove_files { onlyif => "/bin/grep -qe 'leap_mx.log' /etc/check_mk/logwatch.state" } - + # Don't use check_mk logwatch to watch bigcouch logs anymore + # see https://leap.se/code/issues/7375 for more details + file { '/etc/check_mk/logwatch.d/bigcouch.cfg': + ensure => absent, + notify => [ + Exec['remove_bigcouch_logwatch_spoolfiles'], + Exec['remove_bigcouch_logwatch_stateline'] + ] + } + # remove leftover bigcouch logwatch spool files + exec { 'remove_bigcouch_logwatch_spoolfiles': + command => 'find /var/lib/check_mk/logwatch -name \'\\opt\\bigcouch\\var\\log\\bigcouch.log\' -exec rm {} \;', + refreshonly => true, + } + exec { 'remove_bigcouch_logwatch_stateline': + command => "sed -i '/bigcouch.log/d' /etc/check_mk/logwatch.state", + refreshonly => true, + } } -- cgit v1.2.3 From c4ddd197a1ca6a3fac70a86a3ed3dc3d4920e3ca Mon Sep 17 00:00:00 2001 From: varac Date: Thu, 13 Aug 2015 07:28:26 +0200 Subject: Increase readability of nagios notification mail subjects (#6847) Change-Id: Ic9af9ef3602abbb51edf1c9d71d4d264b4ace714 --- puppet/modules/site_check_mk/files/extra_host_conf.mk | 6 ------ puppet/modules/site_check_mk/manifests/server.pp | 2 +- puppet/modules/site_check_mk/templates/extra_host_conf.mk | 13 +++++++++++++ 3 files changed, 14 insertions(+), 7 deletions(-) delete mode 100644 puppet/modules/site_check_mk/files/extra_host_conf.mk create mode 100644 puppet/modules/site_check_mk/templates/extra_host_conf.mk diff --git a/puppet/modules/site_check_mk/files/extra_host_conf.mk b/puppet/modules/site_check_mk/files/extra_host_conf.mk deleted file mode 100644 index 2c96f97a..00000000 --- a/puppet/modules/site_check_mk/files/extra_host_conf.mk +++ /dev/null @@ -1,6 +0,0 @@ -# retry 3 times before setting a host into a hard state -# and send out notification -extra_host_conf["max_check_attempts"] = [ - ("4", ALL_HOSTS ) -] - diff --git a/puppet/modules/site_check_mk/manifests/server.pp b/puppet/modules/site_check_mk/manifests/server.pp index 67519513..57f68d3e 100644 --- a/puppet/modules/site_check_mk/manifests/server.pp +++ b/puppet/modules/site_check_mk/manifests/server.pp @@ -54,7 +54,7 @@ class site_check_mk::server { notify => Exec['check_mk-refresh'], require => Package['check-mk-server']; '/etc/check_mk/conf.d/extra_host_conf.mk': - source => 'puppet:///modules/site_check_mk/extra_host_conf.mk', + content => template('site_check_mk/extra_host_conf.mk'), notify => Exec['check_mk-refresh'], require => Package['check-mk-server']; diff --git a/puppet/modules/site_check_mk/templates/extra_host_conf.mk b/puppet/modules/site_check_mk/templates/extra_host_conf.mk new file mode 100644 index 00000000..bc27b514 --- /dev/null +++ b/puppet/modules/site_check_mk/templates/extra_host_conf.mk @@ -0,0 +1,13 @@ +# retry 3 times before setting a host into a hard state +# and send out notification +extra_host_conf["max_check_attempts"] = [ + ("4", ALL_HOSTS ) +] + +# Use hostnames as alias so notification mail subjects +# are more readable and not so long. Alias defaults to +# the fqdn of a host is not changed. +extra_host_conf["alias"] = [ +<% @hosts.keys.sort.each do |key| -%> ( "<%= key.strip %>", ["<%= @hosts[key]['domain_internal']%>"]), +<% end -%> +] -- cgit v1.2.3 From 64787942086b6fbfdf432cd6250f0937c785de1a Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 12 Aug 2015 14:37:21 -0700 Subject: mv commands and macros to lib/leap_cli --- lib/leap_cli/commands/README | 11 + lib/leap_cli/commands/ca.rb | 518 +++++++++++++++++++++++++++++++++++ lib/leap_cli/commands/clean.rb | 16 ++ lib/leap_cli/commands/compile.rb | 384 ++++++++++++++++++++++++++ lib/leap_cli/commands/db.rb | 65 +++++ lib/leap_cli/commands/deploy.rb | 368 +++++++++++++++++++++++++ lib/leap_cli/commands/env.rb | 76 +++++ lib/leap_cli/commands/facts.rb | 100 +++++++ lib/leap_cli/commands/inspect.rb | 144 ++++++++++ lib/leap_cli/commands/list.rb | 132 +++++++++ lib/leap_cli/commands/node.rb | 165 +++++++++++ lib/leap_cli/commands/node_init.rb | 169 ++++++++++++ lib/leap_cli/commands/ssh.rb | 220 +++++++++++++++ lib/leap_cli/commands/test.rb | 74 +++++ lib/leap_cli/commands/user.rb | 136 +++++++++ lib/leap_cli/commands/util.rb | 50 ++++ lib/leap_cli/commands/vagrant.rb | 197 +++++++++++++ lib/leap_cli/macros.rb | 16 ++ lib/leap_cli/macros/core.rb | 94 +++++++ lib/leap_cli/macros/files.rb | 89 ++++++ lib/leap_cli/macros/haproxy.rb | 73 +++++ lib/leap_cli/macros/hosts.rb | 68 +++++ lib/leap_cli/macros/keys.rb | 83 ++++++ lib/leap_cli/macros/nodes.rb | 88 ++++++ lib/leap_cli/macros/provider.rb | 20 ++ lib/leap_cli/macros/secrets.rb | 39 +++ lib/leap_cli/macros/stunnel.rb | 95 +++++++ provider_base/lib/macros.rb | 16 -- provider_base/lib/macros/core.rb | 94 ------- provider_base/lib/macros/files.rb | 89 ------ provider_base/lib/macros/haproxy.rb | 73 ----- provider_base/lib/macros/hosts.rb | 68 ----- provider_base/lib/macros/keys.rb | 83 ------ provider_base/lib/macros/nodes.rb | 88 ------ provider_base/lib/macros/provider.rb | 20 -- provider_base/lib/macros/secrets.rb | 39 --- provider_base/lib/macros/stunnel.rb | 95 ------- 37 files changed, 3490 insertions(+), 665 deletions(-) create mode 100644 lib/leap_cli/commands/README create mode 100644 lib/leap_cli/commands/ca.rb create mode 100644 lib/leap_cli/commands/clean.rb create mode 100644 lib/leap_cli/commands/compile.rb create mode 100644 lib/leap_cli/commands/db.rb create mode 100644 lib/leap_cli/commands/deploy.rb create mode 100644 lib/leap_cli/commands/env.rb create mode 100644 lib/leap_cli/commands/facts.rb create mode 100644 lib/leap_cli/commands/inspect.rb create mode 100644 lib/leap_cli/commands/list.rb create mode 100644 lib/leap_cli/commands/node.rb create mode 100644 lib/leap_cli/commands/node_init.rb create mode 100644 lib/leap_cli/commands/ssh.rb create mode 100644 lib/leap_cli/commands/test.rb create mode 100644 lib/leap_cli/commands/user.rb create mode 100644 lib/leap_cli/commands/util.rb create mode 100644 lib/leap_cli/commands/vagrant.rb create mode 100644 lib/leap_cli/macros.rb create mode 100644 lib/leap_cli/macros/core.rb create mode 100644 lib/leap_cli/macros/files.rb create mode 100644 lib/leap_cli/macros/haproxy.rb create mode 100644 lib/leap_cli/macros/hosts.rb create mode 100644 lib/leap_cli/macros/keys.rb create mode 100644 lib/leap_cli/macros/nodes.rb create mode 100644 lib/leap_cli/macros/provider.rb create mode 100644 lib/leap_cli/macros/secrets.rb create mode 100644 lib/leap_cli/macros/stunnel.rb delete mode 100644 provider_base/lib/macros.rb delete mode 100644 provider_base/lib/macros/core.rb delete mode 100644 provider_base/lib/macros/files.rb delete mode 100644 provider_base/lib/macros/haproxy.rb delete mode 100644 provider_base/lib/macros/hosts.rb delete mode 100644 provider_base/lib/macros/keys.rb delete mode 100644 provider_base/lib/macros/nodes.rb delete mode 100644 provider_base/lib/macros/provider.rb delete mode 100644 provider_base/lib/macros/secrets.rb delete mode 100644 provider_base/lib/macros/stunnel.rb diff --git a/lib/leap_cli/commands/README b/lib/leap_cli/commands/README new file mode 100644 index 00000000..bec78179 --- /dev/null +++ b/lib/leap_cli/commands/README @@ -0,0 +1,11 @@ +This directory contains ruby source files that define the available sub- +commands of the `leap` executable. + +For example, the command: + + leap compile + +Lives in lib/leap_cli/commands/init.rb + +These files use a DSL (called GLI) for defining command suites. +See https://github.com/davetron5000/gli for more information. diff --git a/lib/leap_cli/commands/ca.rb b/lib/leap_cli/commands/ca.rb new file mode 100644 index 00000000..d5c6240d --- /dev/null +++ b/lib/leap_cli/commands/ca.rb @@ -0,0 +1,518 @@ +autoload :OpenSSL, 'openssl' +autoload :CertificateAuthority, 'certificate_authority' +autoload :Date, 'date' +require 'digest/md5' + +module LeapCli; module Commands + + desc "Manage X.509 certificates" + command :cert do |cert| + + cert.desc 'Creates two Certificate Authorities (one for validating servers and one for validating clients).' + cert.long_desc 'See see what values are used in the generation of the certificates (like name and key size), run `leap inspect provider` and look for the "ca" property. To see the details of the created certs, run `leap inspect `.' + cert.command :ca do |ca| + ca.action do |global_options,options,args| + assert_config! 'provider.ca.name' + generate_new_certificate_authority(:ca_key, :ca_cert, provider.ca.name) + generate_new_certificate_authority(:client_ca_key, :client_ca_cert, provider.ca.name + ' (client certificates only!)') + end + end + + cert.desc 'Creates or renews a X.509 certificate/key pair for a single node or all nodes, but only if needed.' + cert.long_desc 'This command will a generate new certificate for a node if some value in the node has changed ' + + 'that is included in the certificate (like hostname or IP address), or if the old certificate will be expiring soon. ' + + 'Sometimes, you might want to force the generation of a new certificate, ' + + 'such as in the cases where you have changed a CA parameter for server certificates, like bit size or digest hash. ' + + 'In this case, use --force. If is empty, this command will apply to all nodes.' + cert.arg_name 'FILTER' + cert.command :update do |update| + update.switch 'force', :desc => 'Always generate new certificates', :negatable => false + update.action do |global_options,options,args| + update_certificates(manager.filter!(args), options) + end + end + + cert.desc 'Creates a Diffie-Hellman parameter file, needed for forward secret OpenVPN ciphers.' # (needed for server-side of some TLS connections) + cert.command :dh do |dh| + dh.action do |global_options,options,args| + long_running do + if cmd_exists?('certtool') + log 0, 'Generating DH parameters (takes a long time)...' + output = assert_run!('certtool --generate-dh-params --sec-param high') + output.sub! /.*(-----BEGIN DH PARAMETERS-----.*-----END DH PARAMETERS-----).*/m, '\1' + output << "\n" + write_file!(:dh_params, output) + else + log 0, 'Generating DH parameters (takes a REALLY long time)...' + output = OpenSSL::PKey::DH.generate(3248).to_pem + write_file!(:dh_params, output) + end + end + end + end + + # + # hints: + # + # inspect CSR: + # openssl req -noout -text -in files/cert/x.csr + # + # generate CSR with openssl to see how it compares: + # openssl req -sha256 -nodes -newkey rsa:2048 -keyout example.key -out example.csr + # + # validate a CSR: + # http://certlogik.com/decoder/ + # + # nice details about CSRs: + # http://www.redkestrel.co.uk/Articles/CSR.html + # + cert.desc "Creates a CSR for use in buying a commercial X.509 certificate." + cert.long_desc "Unless specified, the CSR is created for the provider's primary domain. "+ + "The properties used for this CSR come from `provider.ca.server_certificates`, "+ + "but may be overridden here." + cert.command :csr do |csr| + csr.flag 'domain', :arg_name => 'DOMAIN', :desc => 'Specify what domain to create the CSR for.' + csr.flag ['organization', 'O'], :arg_name => 'ORGANIZATION', :desc => "Override default O in distinguished name." + csr.flag ['unit', 'OU'], :arg_name => 'UNIT', :desc => "Set OU in distinguished name." + csr.flag 'email', :arg_name => 'EMAIL', :desc => "Set emailAddress in distinguished name." + csr.flag ['locality', 'L'], :arg_name => 'LOCALITY', :desc => "Set L in distinguished name." + csr.flag ['state', 'ST'], :arg_name => 'STATE', :desc => "Set ST in distinguished name." + csr.flag ['country', 'C'], :arg_name => 'COUNTRY', :desc => "Set C in distinguished name." + csr.flag :bits, :arg_name => 'BITS', :desc => "Override default certificate bit length" + csr.flag :digest, :arg_name => 'DIGEST', :desc => "Override default signature digest" + csr.action do |global_options,options,args| + assert_config! 'provider.domain' + assert_config! 'provider.name' + assert_config! 'provider.default_language' + assert_config! 'provider.ca.server_certificates.bit_size' + assert_config! 'provider.ca.server_certificates.digest' + domain = options[:domain] || provider.domain + + unless global_options[:force] + assert_files_missing! [:commercial_key, domain], [:commercial_csr, domain], + :msg => 'If you really want to create a new key and CSR, remove these files first or run with --force.' + end + + server_certificates = provider.ca.server_certificates + + # RSA key + keypair = CertificateAuthority::MemoryKeyMaterial.new + bit_size = (options[:bits] || server_certificates.bit_size).to_i + log :generating, "%s bit RSA key" % bit_size do + keypair.generate_key(bit_size) + write_file! [:commercial_key, domain], keypair.private_key.to_pem + end + + # CSR + dn = CertificateAuthority::DistinguishedName.new + dn.common_name = domain + dn.organization = options[:organization] || provider.name[provider.default_language] + dn.ou = options[:organizational_unit] # optional + dn.email_address = options[:email] # optional + dn.country = options[:country] || server_certificates['country'] # optional + dn.state = options[:state] || server_certificates['state'] # optional + dn.locality = options[:locality] || server_certificates['locality'] # optional + + digest = options[:digest] || server_certificates.digest + log :generating, "CSR with #{digest} digest and #{print_dn(dn)}" do + csr = create_csr(dn, keypair, digest) + request = csr.to_x509_csr + write_file! [:commercial_csr, domain], csr.to_pem + end + + # Sign using our own CA, for use in testing but hopefully not production. + # It is not that commerical CAs are so secure, it is just that signing your own certs is + # a total drag for the user because they must click through dire warnings. + #if options[:sign] + log :generating, "self-signed x509 server certificate for testing purposes" do + cert = csr.to_cert + cert.serial_number.number = cert_serial_number(domain) + cert.not_before = yesterday + cert.not_after = yesterday.advance(:years => 1) + cert.parent = ca_root + cert.sign! domain_test_signing_profile + write_file! [:commercial_cert, domain], cert.to_pem + log "please replace this file with the real certificate you get from a CA using #{Path.relative_path([:commercial_csr, domain])}" + end + #end + + # FAKE CA + unless file_exists? :commercial_ca_cert + log :using, "generated CA in place of commercial CA for testing purposes" do + write_file! :commercial_ca_cert, read_file!(:ca_cert) + log "please also replace this file with the CA cert from the commercial authority you use." + end + end + end + end + end + + protected + + # + # will generate new certificates for the specified nodes, if needed. + # + def update_certificates(nodes, options={}) + assert_files_exist! :ca_cert, :ca_key, :msg => 'Run `leap cert ca` to create them' + assert_config! 'provider.ca.server_certificates.bit_size' + assert_config! 'provider.ca.server_certificates.digest' + assert_config! 'provider.ca.server_certificates.life_span' + assert_config! 'common.x509.use' + + nodes.each_node do |node| + warn_if_commercial_cert_will_soon_expire(node) + if !node.x509.use + remove_file!([:node_x509_key, node.name]) + remove_file!([:node_x509_cert, node.name]) + elsif options[:force] || cert_needs_updating?(node) + generate_cert_for_node(node) + end + end + end + + private + + def generate_new_certificate_authority(key_file, cert_file, common_name) + assert_files_missing! key_file, cert_file + assert_config! 'provider.ca.name' + assert_config! 'provider.ca.bit_size' + assert_config! 'provider.ca.life_span' + + root = CertificateAuthority::Certificate.new + + # set subject + root.subject.common_name = common_name + possible = ['country', 'state', 'locality', 'organization', 'organizational_unit', 'email_address'] + provider.ca.keys.each do |key| + if possible.include?(key) + root.subject.send(key + '=', provider.ca[key]) + end + end + + # set expiration + root.not_before = yesterday + root.not_after = yesterday_advance(provider.ca.life_span) + + # generate private key + root.serial_number.number = 1 + root.key_material.generate_key(provider.ca.bit_size) + + # sign self + root.signing_entity = true + root.parent = root + root.sign!(ca_root_signing_profile) + + # save + write_file!(key_file, root.key_material.private_key.to_pem) + write_file!(cert_file, root.to_pem) + end + + # + # returns true if the certs associated with +node+ need to be regenerated. + # + def cert_needs_updating?(node) + if !file_exists?([:node_x509_cert, node.name], [:node_x509_key, node.name]) + return true + else + cert = load_certificate_file([:node_x509_cert, node.name]) + if cert.not_after < Time.now.advance(:months => 2) + log :updating, "cert for node '#{node.name}' because it will expire soon" + return true + end + if cert.subject.common_name != node.domain.full + log :updating, "cert for node '#{node.name}' because domain.full has changed (was #{cert.subject.common_name}, now #{node.domain.full})" + return true + end + cert.openssl_body.extensions.each do |ext| + if ext.oid == "subjectAltName" + ips = [] + dns_names = [] + ext.value.split(",").each do |value| + value.strip! + ips << $1 if value =~ /^IP Address:(.*)$/ + dns_names << $1 if value =~ /^DNS:(.*)$/ + end + dns_names.sort! + if ips.first != node.ip_address + log :updating, "cert for node '#{node.name}' because ip_address has changed (from #{ips.first} to #{node.ip_address})" + return true + elsif dns_names != dns_names_for_node(node) + log :updating, "cert for node '#{node.name}' because domain name aliases have changed\n from: #{dns_names.inspect}\n to: #{dns_names_for_node(node).inspect})" + return true + end + end + end + end + return false + end + + def warn_if_commercial_cert_will_soon_expire(node) + dns_names_for_node(node).each do |domain| + if file_exists?([:commercial_cert, domain]) + cert = load_certificate_file([:commercial_cert, domain]) + path = Path.relative_path([:commercial_cert, domain]) + if cert.not_after < Time.now.utc + log :error, "the commercial certificate '#{path}' has EXPIRED! " + + "You should renew it with `leap cert csr --domain #{domain}`." + elsif cert.not_after < Time.now.advance(:months => 2) + log :warning, "the commercial certificate '#{path}' will expire soon. "+ + "You should renew it with `leap cert csr --domain #{domain}`." + end + end + end + end + + def generate_cert_for_node(node) + return if node.x509.use == false + + cert = CertificateAuthority::Certificate.new + + # set subject + cert.subject.common_name = node.domain.full + cert.serial_number.number = cert_serial_number(node.domain.full) + + # set expiration + cert.not_before = yesterday + cert.not_after = yesterday_advance(provider.ca.server_certificates.life_span) + + # generate key + cert.key_material.generate_key(provider.ca.server_certificates.bit_size) + + # sign + cert.parent = ca_root + cert.sign!(server_signing_profile(node)) + + # save + write_file!([:node_x509_key, node.name], cert.key_material.private_key.to_pem) + write_file!([:node_x509_cert, node.name], cert.to_pem) + end + + # + # yields client key and cert suitable for testing + # + def generate_test_client_cert(prefix=nil) + cert = CertificateAuthority::Certificate.new + cert.serial_number.number = cert_serial_number(provider.domain) + cert.subject.common_name = [prefix, random_common_name(provider.domain)].join + cert.not_before = yesterday + cert.not_after = yesterday.advance(:years => 1) + cert.key_material.generate_key(1024) # just for testing, remember! + cert.parent = client_ca_root + cert.sign! client_test_signing_profile + yield cert.key_material.private_key.to_pem, cert.to_pem + end + + # + # creates a CSR and returns it. + # with the correct extReq attribute so that the CA + # doens't generate certs with extensions we don't want. + # + def create_csr(dn, keypair, digest) + csr = CertificateAuthority::SigningRequest.new + csr.distinguished_name = dn + csr.key_material = keypair + csr.digest = digest + + # define extensions manually (library doesn't support setting these on CSRs) + extensions = [] + extensions << CertificateAuthority::Extensions::BasicConstraints.new.tap {|basic| + basic.ca = false + } + extensions << CertificateAuthority::Extensions::KeyUsage.new.tap {|keyusage| + keyusage.usage = ["digitalSignature", "keyEncipherment"] + } + extensions << CertificateAuthority::Extensions::ExtendedKeyUsage.new.tap {|extkeyusage| + extkeyusage.usage = [ "serverAuth"] + } + + # convert extensions to attribute 'extReq' + # aka "Requested Extensions" + factory = OpenSSL::X509::ExtensionFactory.new + attrval = OpenSSL::ASN1::Set([OpenSSL::ASN1::Sequence( + extensions.map{|e| factory.create_ext(e.openssl_identifier, e.to_s, e.critical)} + )]) + attrs = [ + OpenSSL::X509::Attribute.new("extReq", attrval), + ] + csr.attributes = attrs + + return csr + end + + def ca_root + @ca_root ||= begin + load_certificate_file(:ca_cert, :ca_key) + end + end + + def client_ca_root + @client_ca_root ||= begin + load_certificate_file(:client_ca_cert, :client_ca_key) + end + end + + def load_certificate_file(crt_file, key_file=nil, password=nil) + crt = read_file!(crt_file) + openssl_cert = OpenSSL::X509::Certificate.new(crt) + cert = CertificateAuthority::Certificate.from_openssl(openssl_cert) + if key_file + key = read_file!(key_file) + cert.key_material.private_key = OpenSSL::PKey::RSA.new(key, password) + end + return cert + end + + def ca_root_signing_profile + { + "extensions" => { + "basicConstraints" => {"ca" => true}, + "keyUsage" => { + "usage" => ["critical", "keyCertSign"] + }, + "extendedKeyUsage" => { + "usage" => [] + } + } + } + end + + # + # For keyusage, openvpn server certs can have keyEncipherment or keyAgreement. + # Web browsers seem to break without keyEncipherment. + # For now, I am using digitalSignature + keyEncipherment + # + # * digitalSignature -- for (EC)DHE cipher suites + # "The digitalSignature bit is asserted when the subject public key is used + # with a digital signature mechanism to support security services other + # than certificate signing (bit 5), or CRL signing (bit 6). Digital + # signature mechanisms are often used for entity authentication and data + # origin authentication with integrity." + # + # * keyEncipherment ==> for plain RSA cipher suites + # "The keyEncipherment bit is asserted when the subject public key is used for + # key transport. For example, when an RSA key is to be used for key management, + # then this bit is set." + # + # * keyAgreement ==> for used with DH, not RSA. + # "The keyAgreement bit is asserted when the subject public key is used for key + # agreement. For example, when a Diffie-Hellman key is to be used for key + # management, then this bit is set." + # + # digest options: SHA512, SHA256, SHA1 + # + def server_signing_profile(node) + { + "digest" => provider.ca.server_certificates.digest, + "extensions" => { + "keyUsage" => { + "usage" => ["digitalSignature", "keyEncipherment"] + }, + "extendedKeyUsage" => { + "usage" => ["serverAuth", "clientAuth"] + }, + "subjectAltName" => { + "ips" => [node.ip_address], + "dns_names" => dns_names_for_node(node) + } + } + } + end + + # + # This is used when signing the main cert for the provider's domain + # with our own CA (for testing purposes). Typically, this cert would + # be purchased from a commercial CA, and not signed this way. + # + def domain_test_signing_profile + { + "digest" => "SHA256", + "extensions" => { + "keyUsage" => { + "usage" => ["digitalSignature", "keyEncipherment"] + }, + "extendedKeyUsage" => { + "usage" => ["serverAuth"] + } + } + } + end + + # + # This is used when signing a dummy client certificate that is only to be + # used for testing. + # + def client_test_signing_profile + { + "digest" => "SHA256", + "extensions" => { + "keyUsage" => { + "usage" => ["digitalSignature"] + }, + "extendedKeyUsage" => { + "usage" => ["clientAuth"] + } + } + } + end + + def dns_names_for_node(node) + names = [node.domain.internal, node.domain.full] + if node['dns'] && node.dns['aliases'] && node.dns.aliases.any? + names += node.dns.aliases + end + names.compact! + names.sort! + names.uniq! + return names + end + + # + # For cert serial numbers, we need a non-colliding number less than 160 bits. + # md5 will do nicely, since there is no need for a secure hash, just a short one. + # (md5 is 128 bits) + # + def cert_serial_number(domain_name) + Digest::MD5.hexdigest("#{domain_name} -- #{Time.now}").to_i(16) + end + + # + # for the random common name, we need a text string that will be unique across all certs. + # ruby 1.8 doesn't have a built-in uuid generator, or we would use SecureRandom.uuid + # + def random_common_name(domain_name) + cert_serial_number(domain_name).to_s(36) + end + + # prints CertificateAuthority::DistinguishedName fields + def print_dn(dn) + fields = {} + [:common_name, :locality, :state, :country, :organization, :organizational_unit, :email_address].each do |attr| + fields[attr] = dn.send(attr) if dn.send(attr) + end + fields.inspect + end + + ## + ## TIME HELPERS + ## + ## note: we use 'yesterday' instead of 'today', because times are in UTC, and some people on the planet + ## are behind UTC. + ## + + def yesterday + t = Time.now - 24*24*60 + Time.utc t.year, t.month, t.day + end + + def yesterday_advance(string) + number, unit = string.split(' ') + unless ['years', 'months', 'days', 'hours', 'minutes'].include? unit + bail!("The time property '#{string}' is missing a unit (one of: years, months, days, hours, minutes).") + end + unless number.to_i.to_s == number + bail!("The time property '#{string}' is missing a number.") + end + yesterday.advance(unit.to_sym => number.to_i) + end + +end; end diff --git a/lib/leap_cli/commands/clean.rb b/lib/leap_cli/commands/clean.rb new file mode 100644 index 00000000..a9afff53 --- /dev/null +++ b/lib/leap_cli/commands/clean.rb @@ -0,0 +1,16 @@ +module LeapCli + module Commands + + desc 'Removes all files generated with the "compile" command.' + command :clean do |c| + c.action do |global_options,options,args| + Dir.glob(path([:hiera, '*'])).each do |file| + remove_file! file + end + remove_file! path(:authorized_keys) + remove_file! path(:known_hosts) + end + end + + end +end \ No newline at end of file diff --git a/lib/leap_cli/commands/compile.rb b/lib/leap_cli/commands/compile.rb new file mode 100644 index 00000000..f5895b8b --- /dev/null +++ b/lib/leap_cli/commands/compile.rb @@ -0,0 +1,384 @@ +require 'socket' + +module LeapCli + module Commands + + desc "Compile generated files." + command [:compile, :c] do |c| + c.desc 'Compiles node configuration files into hiera files used for deployment.' + c.arg_name 'ENVIRONMENT', :optional => true + c.command :all do |all| + all.action do |global_options,options,args| + environment = args.first + if !LeapCli.leapfile.environment.nil? && !environment.nil? && environment != LeapCli.leapfile.environment + bail! "You cannot specify an ENVIRONMENT argument while the environment is pinned." + end + if environment + if manager.environment_names.include?(environment) + compile_hiera_files(manager.filter([environment]), false) + else + bail! "There is no environment named `#{environment}`." + end + else + clean_export = LeapCli.leapfile.environment.nil? + compile_hiera_files(manager.filter, clean_export) + end + if file_exists?(:static_web_readme) + compile_provider_json(environment) + end + end + end + + c.desc "Compile a DNS zone file for your provider." + c.command :zone do |zone| + zone.action do |global_options, options, args| + compile_zone_file + end + end + + c.desc "Compile provider.json bootstrap files for your provider." + c.command 'provider.json' do |provider| + provider.action do |global_options, options, args| + compile_provider_json + end + end + + c.desc "Generate a list of firewall rules. These rules are already "+ + "implemented on each node, but you might want the list of all "+ + "rules in case you also have a restrictive network firewall." + c.command :firewall do |zone| + zone.action do |global_options, options, args| + compile_firewall + end + end + + c.default_command :all + end + + protected + + # + # a "clean" export of secrets will also remove keys that are no longer used, + # but this should not be done if we are not examining all possible nodes. + # + def compile_hiera_files(nodes, clean_export) + update_compiled_ssh_configs # must come first + sanity_check(nodes) + manager.export_nodes(nodes) + manager.export_secrets(clean_export) + end + + def update_compiled_ssh_configs + generate_monitor_ssh_keys + update_authorized_keys + update_known_hosts + end + + def sanity_check(nodes) + # confirm that every node has a unique ip address + ips = {} + nodes.pick_fields('ip_address').each do |name, ip_address| + if ips.key?(ip_address) + bail! { + log(:fatal_error, "Every node must have its own IP address.") { + log "Nodes `#{name}` and `#{ips[ip_address]}` are both configured with `#{ip_address}`." + } + } + else + ips[ip_address] = name + end + end + # confirm that the IP address of this machine is not also used for a node. + Socket.ip_address_list.each do |addrinfo| + if !addrinfo.ipv4_private? && ips.key?(addrinfo.ip_address) + ip = addrinfo.ip_address + name = ips[ip] + bail! { + log(:fatal_error, "Something is very wrong. The `leap` command must only be run on your sysadmin machine, not on a provider node.") { + log "This machine has the same IP address (#{ip}) as node `#{name}`." + } + } + end + end + end + + ## + ## SSH + ## + + # + # generates a ssh key pair that is used only by remote monitors + # to connect to nodes and run certain allowed commands. + # + # every node has the public monitor key added to their authorized + # keys, and every monitor node has a copy of the private monitor key. + # + def generate_monitor_ssh_keys + priv_key_file = path(:monitor_priv_key) + pub_key_file = path(:monitor_pub_key) + unless file_exists?(priv_key_file, pub_key_file) + ensure_dir(File.dirname(priv_key_file)) + ensure_dir(File.dirname(pub_key_file)) + cmd = %(ssh-keygen -N '' -C 'monitor' -t rsa -b 4096 -f '%s') % priv_key_file + assert_run! cmd + if file_exists?(priv_key_file, pub_key_file) + log :created, priv_key_file + log :created, pub_key_file + else + log :failed, 'to create monitor ssh keys' + end + end + end + + # + # Compiles the authorized keys file, which gets installed on every during init. + # Afterwards, puppet installs an authorized keys file that is generated differently + # (see authorized_keys() in macros.rb) + # + def update_authorized_keys + buffer = StringIO.new + keys = Dir.glob(path([:user_ssh, '*'])) + if keys.empty? + bail! "You must have at least one public SSH user key configured in order to proceed. See `leap help add-user`." + end + if file_exists?(path(:monitor_pub_key)) + keys << path(:monitor_pub_key) + end + keys.sort.each do |keyfile| + ssh_type, ssh_key = File.read(keyfile).strip.split(" ") + buffer << ssh_type + buffer << " " + buffer << ssh_key + buffer << " " + buffer << Path.relative_path(keyfile) + buffer << "\n" + end + write_file!(:authorized_keys, buffer.string) + end + + # + # generates the known_hosts file. + # + # we do a 'late' binding on the hostnames and ip part of the ssh pub key record in order to allow + # for the possibility that the hostnames or ip has changed in the node configuration. + # + def update_known_hosts + buffer = StringIO.new + buffer << "#\n" + buffer << "# This file is automatically generated by the command `leap`. You should NOT modify this file.\n" + buffer << "# Instead, rerun `leap node init` on whatever node is causing SSH problems.\n" + buffer << "#\n" + manager.nodes.keys.sort.each do |node_name| + node = manager.nodes[node_name] + hostnames = [node.name, node.domain.internal, node.domain.full, node.ip_address].join(',') + pub_key = read_file([:node_ssh_pub_key,node.name]) + if pub_key + buffer << [hostnames, pub_key].join(' ') + buffer << "\n" + end + end + write_file!(:known_hosts, buffer.string) + end + + ## + ## provider.json + ## + + # + # generates static provider.json files that can put into place + # (e.g. https://domain/provider.json) for the cases where the + # webapp domain does not match the provider's domain. + # + def compile_provider_json(environments=nil) + webapp_nodes = manager.nodes[:services => 'webapp'] + write_file!(:static_web_readme, STATIC_WEB_README) + environments ||= manager.environment_names + environments.each do |env| + node = webapp_nodes[:environment => env].values.first + if node + env ||= 'default' + write_file!( + [:static_web_provider_json, env], + node['definition_files']['provider'] + ) + write_file!( + [:static_web_htaccess, env], + HTACCESS_FILE % {:min_version => manager.env(env).provider.client_version['min']} + ) + end + end + end + + HTACCESS_FILE = %[ + + Header set X-Minimum-Client-Version %{min_version} + +] + + STATIC_WEB_README = %[ +This directory contains statically rendered copies of the `provider.json` file +used by the client to "bootstrap" configure itself for use with your service +provider. + +There is a separate provider.json file for each environment, although you +should only need 'production/provider.json' or, if you have no environments +configured, 'default/provider.json'. + +To clarify, this is the public `provider.json` file used by the client, not the +`provider.json` file that is used to configure the provider. + +The provider.json file must be available at `https://domain/provider.json` +(unless this provider is included in the list of providers which are pre- +seeded in client). + +This provider.json file can be served correctly in one of three ways: + +(1) If the property webapp.domain is not configured, then the web app will be + installed at https://domain/ and it will handle serving the provider.json file. + +(2) If one or more nodes have the 'static' service configured for the provider's + domain, then these 'static' nodes will correctly serve provider.json. + +(3) Otherwise, you must copy the provider.json file to your web + server and make it available at '/provider.json'. The example htaccess + file shows what header options should be sent by the web server + with the response. + +This directory is needed for method (3), but not for methods (1) or (2). + +This directory has been created by the command `leap compile provider.json`. +Once created, it will be kept up to date everytime you compile. You may safely +remove this directory if you don't use it. +] + + ## + ## + ## ZONE FILE + ## + + def relative_hostname(fqdn) + @domain_regexp ||= /\.?#{Regexp.escape(provider.domain)}$/ + fqdn.sub(@domain_regexp, '') + end + + # + # serial is any number less than 2^32 (4294967296) + # + def compile_zone_file + hosts_seen = {} + f = $stdout + f.puts ZONE_HEADER % {:domain => provider.domain, :ns => provider.domain, :contact => provider.contacts.default.first.sub('@','.')} + max_width = manager.nodes.values.inject(0) {|max, node| [max, relative_hostname(node.domain.full).length].max } + put_line = lambda do |host, line| + host = '@' if host == '' + f.puts("%-#{max_width}s %s" % [host, line]) + end + + f.puts ORIGIN_HEADER + # 'A' records for primary domain + manager.nodes[:environment => '!local'].each_node do |node| + if node.dns['aliases'] && node.dns.aliases.include?(provider.domain) + put_line.call "", "IN A #{node.ip_address}" + end + end + + # NS records + if provider['dns'] && provider.dns['nameservers'] + provider.dns.nameservers.each do |ns| + put_line.call "", "IN NS #{ns}." + end + end + + # all other records + manager.environment_names.each do |env| + next if env == 'local' + nodes = manager.nodes[:environment => env] + next unless nodes.any? + f.puts ENV_HEADER % (env.nil? ? 'default' : env) + nodes.each_node do |node| + if node.dns.public + hostname = relative_hostname(node.domain.full) + put_line.call relative_hostname(node.domain.full), "IN A #{node.ip_address}" + end + if node.dns['aliases'] + node.dns.aliases.each do |host_alias| + if host_alias != node.domain.full && host_alias != provider.domain + put_line.call relative_hostname(host_alias), "IN A #{node.ip_address}" + end + end + end + if node.services.include? 'mx' + put_line.call relative_hostname(node.domain.full_suffix), "IN MX 10 #{relative_hostname(node.domain.full)}" + end + end + end + end + + ENV_HEADER = %[ +;; +;; ENVIRONMENT %s +;; + +] + + ZONE_HEADER = %[ +;; +;; BIND data file for %{domain} +;; + +$TTL 600 +$ORIGIN %{domain}. + +@ IN SOA %{ns}. %{contact}. ( + 0000 ; serial + 7200 ; refresh ( 24 hours) + 3600 ; retry ( 2 hours) + 1209600 ; expire (1000 hours) + 600 ) ; minimum ( 2 days) +; +] + + ORIGIN_HEADER = %[ +;; +;; ZONE ORIGIN +;; + +] + + ## + ## FIREWALL + ## + + def compile_firewall + manager.nodes.each_node(&:evaluate) + + rules = [["ALLOW TO", "PORTS", "ALLOW FROM"]] + manager.nodes[:environment => '!local'].values.each do |node| + next unless node['firewall'] + node.firewall.each do |name, rule| + if rule.is_a? Hash + rules << add_rule(rule) + elsif rule.is_a? Array + rule.each do |r| + rules << add_rule(r) + end + end + end + end + + max_to = rules.inject(0) {|max, r| [max, r[0].length].max} + max_port = rules.inject(0) {|max, r| [max, r[1].length].max} + max_from = rules.inject(0) {|max, r| [max, r[2].length].max} + rules.each do |rule| + puts "%-#{max_to}s %-#{max_port}s %-#{max_from}s" % rule + end + end + + private + + def add_rule(rule) + [rule["to"], [rule["port"]].compact.join(','), rule["from"]] + end + + end +end \ No newline at end of file diff --git a/lib/leap_cli/commands/db.rb b/lib/leap_cli/commands/db.rb new file mode 100644 index 00000000..e4fd3858 --- /dev/null +++ b/lib/leap_cli/commands/db.rb @@ -0,0 +1,65 @@ +module LeapCli; module Commands + + desc 'Database commands.' + command :db do |db| + db.desc 'Destroy one or more databases. If present, limit to FILTER nodes. For example `leap db destroy --db sessions,tokens testing`.' + db.arg_name 'FILTER', :optional => true + db.command :destroy do |destroy| + destroy.flag :db, :arg_name => "DATABASES", :desc => 'Comma separated list of databases to destroy (no space). Use "--db all" to destroy all databases.', :optional => false + destroy.action do |global_options,options,args| + dbs = (options[:db]||"").split(',') + bail!('No databases specified') if dbs.empty? + nodes = manager.filter(args) + if nodes.any? + nodes = nodes[:services => 'couchdb'] + end + if nodes.any? + unless global_options[:yes] + if dbs.include?('all') + say 'You are about to permanently destroy all database data for nodes [%s].' % nodes.keys.join(', ') + else + say 'You are about to permanently destroy databases [%s] for nodes [%s].' % [dbs.join(', '), nodes.keys.join(', ')] + end + bail! unless agree("Continue? ") + end + if dbs.include?('all') + destroy_all_dbs(nodes) + else + destroy_dbs(nodes, dbs) + end + say 'You must run `leap deploy` in order to create the databases again.' + else + say 'No nodes' + end + end + end + end + + private + + def destroy_all_dbs(nodes) + ssh_connect(nodes) do |ssh| + ssh.run('/etc/init.d/bigcouch stop && test ! -z "$(ls /opt/bigcouch/var/lib/ 2> /dev/null)" && rm -r /opt/bigcouch/var/lib/* && echo "db destroyed" || echo "db already destroyed"') + ssh.run('grep ^seq_dir /etc/leap/tapicero.yaml | cut -f2 -d\" | xargs rm -rv') + end + end + + def destroy_dbs(nodes, dbs) + nodes.each_node do |node| + ssh_connect(node) do |ssh| + dbs.each do |db| + ssh.run(DESTROY_DB_COMMAND % {:db => db}) + end + end + end + end + + DESTROY_DB_COMMAND = %{ +if [ 200 = `curl -ns -w "%%{http_code}" -X GET "127.0.0.1:5984/%{db}" -o /dev/null` ]; then + echo "Result from DELETE /%{db}:" `curl -ns -X DELETE "127.0.0.1:5984/%{db}"`; +else + echo "Skipping db '%{db}': it does not exist or has already been deleted."; +fi +} + +end; end diff --git a/lib/leap_cli/commands/deploy.rb b/lib/leap_cli/commands/deploy.rb new file mode 100644 index 00000000..c2a70afa --- /dev/null +++ b/lib/leap_cli/commands/deploy.rb @@ -0,0 +1,368 @@ +require 'etc' + +module LeapCli + module Commands + + desc 'Apply recipes to a node or set of nodes.' + long_desc 'The FILTER can be the name of a node, service, or tag.' + arg_name 'FILTER' + command [:deploy, :d] do |c| + + c.switch :fast, :desc => 'Makes the deploy command faster by skipping some slow steps. A "fast" deploy can be used safely if you recently completed a normal deploy.', + :negatable => false + + c.switch :sync, :desc => "Sync files, but don't actually apply recipes.", :negatable => false + + c.switch :force, :desc => 'Deploy even if there is a lockfile.', :negatable => false + + c.switch :downgrade, :desc => 'Allows deploy to run with an older platform version.', :negatable => false + + c.switch :dev, :desc => "Development mode: don't run 'git submodule update' before deploy.", :negatable => false + + c.flag :tags, :desc => 'Specify tags to pass through to puppet (overriding the default).', + :arg_name => 'TAG[,TAG]' + + c.flag :port, :desc => 'Override the default SSH port.', + :arg_name => 'PORT' + + c.flag :ip, :desc => 'Override the default SSH IP address.', + :arg_name => 'IPADDRESS' + + c.action do |global,options,args| + + if options[:dev] != true + init_submodules + end + + nodes = manager.filter!(args, :disabled => false) + if nodes.size > 1 + say "Deploying to these nodes: #{nodes.keys.join(', ')}" + if !global[:yes] && !agree("Continue? ") + quit! "OK. Bye." + end + end + + environments = nodes.field('environment').uniq + if environments.empty? + environments = [nil] + end + environments.each do |env| + check_platform_pinning(env, global) + end + # compile hiera files for all the nodes in every environment that is + # being deployed and only those environments. + compile_hiera_files(manager.filter(environments), false) + # update server certificates if needed + update_certificates(nodes) + + ssh_connect(nodes, connect_options(options)) do |ssh| + ssh.leap.log :checking, 'node' do + ssh.leap.check_for_no_deploy + ssh.leap.assert_initialized + end + ssh.leap.log :synching, "configuration files" do + sync_hiera_config(ssh) + sync_support_files(ssh) + end + ssh.leap.log :synching, "puppet manifests" do + sync_puppet_files(ssh) + end + unless options[:sync] + ssh.leap.log :applying, "puppet" do + ssh.puppet.apply(:verbosity => [LeapCli.log_level,5].min, + :tags => tags(options), + :force => options[:force], + :info => deploy_info, + :downgrade => options[:downgrade] + ) + end + end + end + if !Util.exit_status.nil? && Util.exit_status != 0 + log :warning, "puppet did not finish successfully." + end + end + end + + desc 'Display recent deployment history for a set of nodes.' + long_desc 'The FILTER can be the name of a node, service, or tag.' + arg_name 'FILTER' + command [:history, :h] do |c| + c.flag :port, :desc => 'Override the default SSH port.', + :arg_name => 'PORT' + c.flag :ip, :desc => 'Override the default SSH IP address.', + :arg_name => 'IPADDRESS' + c.action do |global,options,args| + nodes = manager.filter!(args) + ssh_connect(nodes, connect_options(options)) do |ssh| + ssh.leap.history + end + end + end + + private + + def forcible_prompt(forced, msg, prompt) + say(msg) + if forced + log :warning, "continuing anyway because of --force" + else + say "hint: use --force to skip this prompt." + quit!("OK. Bye.") unless agree(prompt) + end + end + + # + # The currently activated provider.json could have loaded some pinning + # information for the platform. If this is the case, refuse to deploy + # if there is a mismatch. + # + # For example: + # + # "platform": { + # "branch": "develop" + # "version": "1.0..99" + # "commit": "e1d6280e0a8c565b7fb1a4ed3969ea6fea31a5e2..HEAD" + # } + # + def check_platform_pinning(environment, global_options) + provider = manager.env(environment).provider + return unless provider['platform'] + + if environment.nil? || environment == 'default' + provider_json = 'provider.json' + else + provider_json = 'provider.' + environment + '.json' + end + + # can we have json schema verification already? + unless provider.platform.is_a? Hash + bail!('`platform` attribute in #{provider_json} must be a hash (was %s).' % provider.platform.inspect) + end + + # check version + if provider.platform['version'] + if !Leap::Platform.version_in_range?(provider.platform.version) + forcible_prompt( + global_options[:force], + "The platform is pinned to a version range of '#{provider.platform.version}' "+ + "by the `platform.version` property in #{provider_json}, but the platform "+ + "(#{Path.platform}) has version #{Leap::Platform.version}.", + "Do you really want to deploy from the wrong version? " + ) + end + end + + # check branch + if provider.platform['branch'] + if !is_git_directory?(Path.platform) + forcible_prompt( + global_options[:force], + "The platform is pinned to a particular branch by the `platform.branch` property "+ + "in #{provider_json}, but the platform directory (#{Path.platform}) is not a git repository.", + "Do you really want to deploy anyway? " + ) + end + unless provider.platform.branch == current_git_branch(Path.platform) + forcible_prompt( + global_options[:force], + "The platform is pinned to branch '#{provider.platform.branch}' by the `platform.branch` property "+ + "in #{provider_json}, but the current branch is '#{current_git_branch(Path.platform)}' " + + "(for directory '#{Path.platform}')", + "Do you really want to deploy from the wrong branch? " + ) + end + end + + # check commit + if provider.platform['commit'] + if !is_git_directory?(Path.platform) + forcible_prompt( + global_options[:force], + "The platform is pinned to a particular commit range by the `platform.commit` property "+ + "in #{provider_json}, but the platform directory (#{Path.platform}) is not a git repository.", + "Do you really want to deploy anyway? " + ) + end + current_commit = current_git_commit(Path.platform) + Dir.chdir(Path.platform) do + commit_range = assert_run!("git log --pretty='format:%H' '#{provider.platform.commit}'", + "The platform is pinned to a particular commit range by the `platform.commit` property "+ + "in #{provider_json}, but git was not able to find commits in the range specified "+ + "(#{provider.platform.commit}).") + commit_range = commit_range.split("\n") + if !commit_range.include?(current_commit) && + provider.platform.commit.split('..').first != current_commit + forcible_prompt( + global_options[:force], + "The platform is pinned via the `platform.commit` property in #{provider_json} " + + "to a commit in the range #{provider.platform.commit}, but the current HEAD " + + "(#{current_commit}) is not in that range.", + "Do you really want to deploy from the wrong commit? " + ) + end + end + end + end + + def sync_hiera_config(ssh) + ssh.rsync.update do |server| + node = manager.node(server.host) + hiera_file = Path.relative_path([:hiera, node.name]) + ssh.leap.log hiera_file + ' -> ' + node.name + ':' + Leap::Platform.hiera_path + { + :source => hiera_file, + :dest => Leap::Platform.hiera_path, + :flags => "-rltp --chmod=u+rX,go-rwx" + } + end + end + + # + # sync various support files. + # + def sync_support_files(ssh) + dest_dir = Leap::Platform.files_dir + custom_files = build_custom_file_list + ssh.rsync.update do |server| + node = manager.node(server.host) + files_to_sync = node.file_paths.collect {|path| Path.relative_path(path, Path.provider) } + files_to_sync += custom_files + if files_to_sync.any? + ssh.leap.log(files_to_sync.join(', ') + ' -> ' + node.name + ':' + dest_dir) + { + :chdir => Path.named_path(:files_dir), + :source => ".", + :dest => dest_dir, + :excludes => "*", + :includes => calculate_includes_from_files(files_to_sync, '/files'), + :flags => "-rltp --chmod=u+rX,go-rwx --relative --delete --delete-excluded --copy-links" + } + else + nil + end + end + end + + def sync_puppet_files(ssh) + ssh.rsync.update do |server| + ssh.leap.log(Path.platform + '/[bin,tests,puppet] -> ' + server.host + ':' + Leap::Platform.leap_dir) + { + :dest => Leap::Platform.leap_dir, + :source => '.', + :chdir => Path.platform, + :excludes => '*', + :includes => ['/bin', '/bin/**', '/puppet', '/puppet/**', '/tests', '/tests/**'], + :flags => "-rlt --relative --delete --copy-links" + } + end + end + + # + # ensure submodules are up to date, if the platform is a git + # repository. + # + def init_submodules + return unless is_git_directory?(Path.platform) + Dir.chdir Path.platform do + assert_run! "git submodule sync" + statuses = assert_run! "git submodule status" + statuses.strip.split("\n").each do |status_line| + if status_line =~ /^[\+-]/ + submodule = status_line.split(' ')[1] + log "Updating submodule #{submodule}" + assert_run! "git submodule update --init #{submodule}" + end + end + end + end + + # + # converts an array of file paths into an array + # suitable for --include of rsync + # + # if set, `prefix` is stripped off. + # + def calculate_includes_from_files(files, prefix=nil) + return nil unless files and files.any? + + # prepend '/' (kind of like ^ for rsync) + includes = files.collect {|file| file =~ /^\// ? file : '/' + file } + + # include all sub files of specified directories + includes.size.times do |i| + if includes[i] =~ /\/$/ + includes << includes[i] + '**' + end + end + + # include all parent directories (required because of --exclude '*') + includes.size.times do |i| + path = File.dirname(includes[i]) + while(path != '/') + includes << path unless includes.include?(path) + path = File.dirname(path) + end + end + + if prefix + includes.map! {|path| path.sub(/^#{Regexp.escape(prefix)}\//, '/')} + end + + return includes + end + + def tags(options) + if options[:tags] + tags = options[:tags].split(',') + else + tags = Leap::Platform.default_puppet_tags.dup + end + tags << 'leap_slow' unless options[:fast] + tags.join(',') + end + + # + # a provider might have various customization files that should be sync'ed to the server. + # this method builds that list of files to sync. + # + def build_custom_file_list + custom_files = [] + Leap::Platform.paths.keys.grep(/^custom_/).each do |path| + if file_exists?(path) + relative_path = Path.relative_path(path, Path.provider) + if dir_exists?(path) + custom_files << relative_path + '/' # rsync needs trailing slash + else + custom_files << relative_path + end + end + end + return custom_files + end + + def deploy_info + info = [] + info << "user: %s" % Etc.getpwuid(Process.euid).name + if is_git_directory?(Path.platform) && current_git_branch(Path.platform) != 'master' + info << "platform: %s (%s %s)" % [ + Leap::Platform.version, + current_git_branch(Path.platform), + current_git_commit(Path.platform)[0..4] + ] + else + info << "platform: %s" % Leap::Platform.version + end + if is_git_directory?(LEAP_CLI_BASE_DIR) + info << "leap_cli: %s (%s %s)" % [ + LeapCli::VERSION, + current_git_branch(LEAP_CLI_BASE_DIR), + current_git_commit(LEAP_CLI_BASE_DIR)[0..4] + ] + else + info << "leap_cli: %s" % LeapCli::VERSION + end + info.join(', ') + end + end +end diff --git a/lib/leap_cli/commands/env.rb b/lib/leap_cli/commands/env.rb new file mode 100644 index 00000000..80be2174 --- /dev/null +++ b/lib/leap_cli/commands/env.rb @@ -0,0 +1,76 @@ +module LeapCli + module Commands + + desc "Manipulate and query environment information." + long_desc "The 'environment' node property can be used to isolate sets of nodes into entirely separate environments. "+ + "A node in one environment will never interact with a node from another environment. "+ + "Environment pinning works by modifying your ~/.leaprc file and is dependent on the "+ + "absolute file path of your provider directory (pins don't apply if you move the directory)" + command [:env, :e] do |c| + c.desc "List the available environments. The pinned environment, if any, will be marked with '*'. Will also set the pin if run with an environment argument." + c.arg_name 'ENVIRONMENT', :optional => true + c.command :ls do |ls| + ls.action do |global_options, options, args| + environment = get_env_from_args(args) + if environment + pin(environment) + LeapCli.leapfile.load + end + print_envs + end + end + + c.desc 'Pin the environment to ENVIRONMENT. All subsequent commands will only apply to nodes in this environment.' + c.arg_name 'ENVIRONMENT' + c.command :pin do |pin| + pin.action do |global_options,options,args| + environment = get_env_from_args(args) + if environment + pin(environment) + else + bail! "There is no environment `#{environment}`" + end + end + end + + c.desc "Unpin the environment. All subsequent commands will apply to all nodes." + c.command :unpin do |unpin| + unpin.action do |global_options, options, args| + LeapCli.leapfile.unset('environment') + log 0, :saved, "~/.leaprc, removing environment property." + end + end + + c.default_command :ls + end + + protected + + def get_env_from_args(args) + environment = args.first + if environment == 'default' || (environment && manager.environment_names.include?(environment)) + return environment + else + return nil + end + end + + def pin(environment) + LeapCli.leapfile.set('environment', environment) + log 0, :saved, "~/.leaprc with environment set to #{environment}." + end + + def print_envs + envs = ["default"] + manager.environment_names.compact.sort + envs.each do |env| + if env + if LeapCli.leapfile.environment == env + puts "* #{env}" + else + puts " #{env}" + end + end + end + end + end +end \ No newline at end of file diff --git a/lib/leap_cli/commands/facts.rb b/lib/leap_cli/commands/facts.rb new file mode 100644 index 00000000..11329ccc --- /dev/null +++ b/lib/leap_cli/commands/facts.rb @@ -0,0 +1,100 @@ +# +# Gather facter facts +# + +module LeapCli; module Commands + + desc 'Gather information on nodes.' + command :facts do |facts| + facts.desc 'Query servers to update facts.json.' + facts.long_desc "Queries every node included in FILTER and saves the important information to facts.json" + facts.arg_name 'FILTER' + facts.command :update do |update| + update.action do |global_options,options,args| + update_facts(global_options, options, args) + end + end + end + + protected + + def facter_cmd + 'facter --json ' + Leap::Platform.facts.join(' ') + end + + def remove_node_facts(name) + if file_exists?(:facts) + update_facts_file({name => nil}) + end + end + + def update_node_facts(name, facts) + update_facts_file({name => facts}) + end + + def rename_node_facts(old_name, new_name) + if file_exists?(:facts) + facts = JSON.parse(read_file(:facts) || {}) + facts[new_name] = facts[old_name] + facts[old_name] = nil + update_facts_file(facts, true) + end + end + + # + # if overwrite = true, then ignore existing facts.json. + # + def update_facts_file(new_facts, overwrite=false) + replace_file!(:facts) do |content| + if overwrite || content.nil? || content.empty? + old_facts = {} + else + old_facts = manager.facts + end + facts = old_facts.merge(new_facts) + facts.each do |name, value| + if value.is_a? String + if value == "" + value = nil + else + value = JSON.parse(value) rescue JSON::ParserError + end + end + if value.is_a? Hash + value.delete_if {|key,v| v.nil?} + end + facts[name] = value + end + facts.delete_if do |name, value| + value.nil? || value.empty? + end + if facts.empty? + "{}\n" + else + JSON.sorted_generate(facts) + "\n" + end + end + end + + private + + def update_facts(global_options, options, args) + nodes = manager.filter(args, :local => false, :disabled => false) + new_facts = {} + ssh_connect(nodes) do |ssh| + ssh.leap.run_with_progress(facter_cmd) do |response| + node = manager.node(response[:host]) + if node + new_facts[node.name] = response[:data].strip + else + log :warning, 'Could not find node for hostname %s' % response[:host] + end + end + end + # only overwrite the entire facts file if and only if we are gathering facts + # for all nodes in all environments. + overwrite_existing = args.empty? && LeapCli.leapfile.environment.nil? + update_facts_file(new_facts, overwrite_existing) + end + +end; end \ No newline at end of file diff --git a/lib/leap_cli/commands/inspect.rb b/lib/leap_cli/commands/inspect.rb new file mode 100644 index 00000000..20654fa7 --- /dev/null +++ b/lib/leap_cli/commands/inspect.rb @@ -0,0 +1,144 @@ +module LeapCli; module Commands + + desc 'Prints details about a file. Alternately, the argument FILE can be the name of a node, service or tag.' + arg_name 'FILE' + command [:inspect, :i] do |c| + c.switch 'base', :desc => 'Inspect the FILE from the provider_base (i.e. without local inheritance).', :negatable => false + c.action do |global_options,options,args| + object = args.first + assert! object, 'A file path or node/service/tag name is required' + method = inspection_method(object) + if method && defined?(method) + self.send(method, object, options) + else + log "Sorry, I don't know how to inspect that." + end + end + end + + private + + FTYPE_MAP = { + "PEM certificate" => :inspect_x509_cert, + "PEM RSA private key" => :inspect_x509_key, + "OpenSSH RSA public key" => :inspect_ssh_pub_key, + "PEM certificate request" => :inspect_x509_csr + } + + def inspection_method(object) + if File.exists?(object) + ftype = `file #{object}`.split(':').last.strip + log 2, "file is of type '#{ftype}'" + if FTYPE_MAP[ftype] + FTYPE_MAP[ftype] + elsif File.extname(object) == ".json" + full_path = File.expand_path(object, Dir.pwd) + if path_match?(:node_config, full_path) + :inspect_node + elsif path_match?(:service_config, full_path) + :inspect_service + elsif path_match?(:tag_config, full_path) + :inspect_tag + elsif path_match?(:provider_config, full_path) || path_match?(:provider_env_config, full_path) + :inspect_provider + elsif path_match?(:common_config, full_path) + :inspect_common + else + nil + end + end + elsif manager.nodes[object] + :inspect_node + elsif manager.services[object] + :inspect_service + elsif manager.tags[object] + :inspect_tag + elsif object == "common" + :inspect_common + elsif object == "provider" + :inspect_provider + else + nil + end + end + + # + # inspectors + # + + def inspect_x509_key(file_path, options) + assert_bin! 'openssl' + puts assert_run! 'openssl rsa -in %s -text -check' % file_path + end + + def inspect_x509_cert(file_path, options) + assert_bin! 'openssl' + puts assert_run! 'openssl x509 -in %s -text -noout' % file_path + log 0, :"SHA256 fingerprint", X509.fingerprint("SHA256", file_path) + end + + def inspect_x509_csr(file_path, options) + assert_bin! 'openssl' + puts assert_run! 'openssl req -text -noout -verify -in %s' % file_path + end + + #def inspect_ssh_pub_key(file_path) + #end + + def inspect_node(arg, options) + inspect_json manager.nodes[name(arg)] + end + + def inspect_service(arg, options) + if options[:base] + inspect_json manager.base_services[name(arg)] + else + inspect_json manager.services[name(arg)] + end + end + + def inspect_tag(arg, options) + if options[:base] + inspect_json manager.base_tags[name(arg)] + else + inspect_json manager.tags[name(arg)] + end + end + + def inspect_provider(arg, options) + if options[:base] + inspect_json manager.base_provider + elsif arg =~ /provider\.(.*)\.json/ + inspect_json manager.env($1).provider + else + inspect_json manager.provider + end + end + + def inspect_common(arg, options) + if options[:base] + inspect_json manager.base_common + else + inspect_json manager.common + end + end + + # + # helpers + # + + def name(arg) + File.basename(arg).sub(/\.json$/, '') + end + + def inspect_json(config) + if config + puts JSON.sorted_generate(config) + end + end + + def path_match?(path_symbol, path) + Dir.glob(Path.named_path([path_symbol, '*'])).include?(path) + end + +end; end diff --git a/lib/leap_cli/commands/list.rb b/lib/leap_cli/commands/list.rb new file mode 100644 index 00000000..c562b59b --- /dev/null +++ b/lib/leap_cli/commands/list.rb @@ -0,0 +1,132 @@ +require 'command_line_reporter' + +module LeapCli; module Commands + + desc 'List nodes and their classifications' + long_desc 'Prints out a listing of nodes, services, or tags. ' + + 'If present, the FILTER can be a list of names of nodes, services, or tags. ' + + 'If the name is prefixed with +, this acts like an AND condition. ' + + "For example:\n\n" + + "`leap list node1 node2` matches all nodes named \"node1\" OR \"node2\"\n\n" + + "`leap list openvpn +local` matches all nodes with service \"openvpn\" AND tag \"local\"" + + arg_name 'FILTER', :optional => true + command [:list,:ls] do |c| + c.flag 'print', :desc => 'What attributes to print (optional)' + c.switch 'disabled', :desc => 'Include disabled nodes in the list.', :negatable => false + c.action do |global_options,options,args| + # don't rely on default manager(), because we want to pass custom options to load() + manager = LeapCli::Config::Manager.new + if global_options[:color] + colors = ['cyan', 'white'] + else + colors = [nil, nil] + end + puts + manager.load(:include_disabled => options['disabled'], :continue_on_error => true) + if options['print'] + print_node_properties(manager.filter(args), options['print']) + else + if args.any? + NodeTable.new(manager.filter(args), colors).run + else + environment = LeapCli.leapfile.environment || '_all_' + TagTable.new('SERVICES', manager.env(environment).services, colors).run + TagTable.new('TAGS', manager.env(environment).tags, colors).run + NodeTable.new(manager.filter(), colors).run + end + end + end + end + + private + + def self.print_node_properties(nodes, properties) + properties = properties.split(',') + max_width = nodes.keys.inject(0) {|max,i| [i.size,max].max} + nodes.each_node do |node| + value = properties.collect{|prop| + prop_value = node[prop] + if prop_value.nil? + "null" + elsif prop_value == "" + "empty" + elsif prop_value.is_a? LeapCli::Config::Object + node[prop].dump_json(:compact) # TODO: add option of getting pre-evaluation values. + else + prop_value.to_s + end + }.join(', ') + printf("%#{max_width}s %s\n", node.name, value) + end + puts + end + + class TagTable + include CommandLineReporter + def initialize(heading, tag_list, colors) + @heading = heading + @tag_list = tag_list + @colors = colors + end + def run + tags = @tag_list.keys.select{|tag| tag !~ /^_/}.sort # sorted list of tags, excluding _partials + max_width = [20, (tags+[@heading]).inject(0) {|max,i| [i.size,max].max}].max + table :border => false do + row :color => @colors[0] do + column @heading, :align => 'right', :width => max_width + column "NODES", :width => HighLine::SystemExtensions.terminal_size.first - max_width - 2, :padding => 2 + end + tags.each do |tag| + next if @tag_list[tag].node_list.empty? + row :color => @colors[1] do + column tag + column @tag_list[tag].node_list.keys.sort.join(', ') + end + end + end + vertical_spacing + end + end + + # + # might be handy: HighLine::SystemExtensions.terminal_size.first + # + class NodeTable + include CommandLineReporter + def initialize(node_list, colors) + @node_list = node_list + @colors = colors + end + def run + rows = @node_list.keys.sort.collect do |node_name| + [node_name, @node_list[node_name].services.sort.join(', '), @node_list[node_name].tags.sort.join(', ')] + end + unless rows.any? + puts Paint["no results", :red] + puts + return + end + padding = 2 + max_node_width = [20, (rows.map{|i|i[0]} + ["NODES"] ).inject(0) {|max,i| [i.size,max].max}].max + max_service_width = (rows.map{|i|i[1]} + ["SERVICES"]).inject(0) {|max,i| [i.size+padding+padding,max].max} + max_tag_width = (rows.map{|i|i[2]} + ["TAGS"] ).inject(0) {|max,i| [i.size,max].max} + table :border => false do + row :color => @colors[0] do + column "NODES", :align => 'right', :width => max_node_width + column "SERVICES", :width => max_service_width, :padding => 2 + column "TAGS", :width => max_tag_width + end + rows.each do |r| + row :color => @colors[1] do + column r[0] + column r[1] + column r[2] + end + end + end + vertical_spacing + end + end + +end; end diff --git a/lib/leap_cli/commands/node.rb b/lib/leap_cli/commands/node.rb new file mode 100644 index 00000000..12d6b49d --- /dev/null +++ b/lib/leap_cli/commands/node.rb @@ -0,0 +1,165 @@ +# +# fyi: the `node init` command lives in node_init.rb, +# but all other `node x` commands live here. +# + +autoload :IPAddr, 'ipaddr' + +module LeapCli; module Commands + + ## + ## COMMANDS + ## + + desc 'Node management' + command [:node, :n] do |node| + node.desc 'Create a new configuration file for a node named NAME.' + node.long_desc ["If specified, the optional argument SEED can be used to seed values in the node configuration file.", + "The format is property_name:value.", + "For example: `leap node add web1 ip_address:1.2.3.4 services:webapp`.", + "To set nested properties, property name can contain '.', like so: `leap node add web1 ssh.port:44`", + "Separeate multiple values for a single property with a comma, like so: `leap node add mynode services:webapp,dns`"].join("\n\n") + node.arg_name 'NAME [SEED]' # , :optional => false, :multiple => false + node.command :add do |add| + add.switch :local, :desc => 'Make a local testing node (by automatically assigning the next available local IP address). Local nodes are run as virtual machines on your computer.', :negatable => false + add.action do |global_options,options,args| + # argument sanity checks + name = args.first + assert_valid_node_name!(name, options[:local]) + assert_files_missing! [:node_config, name] + + # create and seed new node + node = Config::Node.new(manager) + if options[:local] + node['ip_address'] = pick_next_vagrant_ip_address + end + seed_node_data(node, args[1..-1]) + validate_ip_address(node) + begin + write_file! [:node_config, name], node.dump_json + "\n" + node['name'] = name + if file_exists? :ca_cert, :ca_key + generate_cert_for_node(manager.reload_node!(node)) + end + rescue LeapCli::ConfigError => exc + remove_node_files(name) + end + end + end + + node.desc 'Renames a node file, and all its related files.' + node.arg_name 'OLD_NAME NEW_NAME' + node.command :mv do |mv| + mv.action do |global_options,options,args| + node = get_node_from_args(args) + new_name = args.last + assert_valid_node_name!(new_name, node.vagrant?) + ensure_dir [:node_files_dir, new_name] + Leap::Platform.node_files.each do |path| + rename_file! [path, node.name], [path, new_name] + end + remove_directory! [:node_files_dir, node.name] + rename_node_facts(node.name, new_name) + end + end + + node.desc 'Removes all the files related to the node named NAME.' + node.arg_name 'NAME' #:optional => false #, :multiple => false + node.command :rm do |rm| + rm.action do |global_options,options,args| + node = get_node_from_args(args) + remove_node_files(node.name) + if node.vagrant? + vagrant_command("destroy --force", [node.name]) + end + remove_node_facts(node.name) + end + end + end + + ## + ## PUBLIC HELPERS + ## + + def get_node_from_args(args, options={}) + node_name = args.first + node = manager.node(node_name) + if node.nil? && options[:include_disabled] + node = manager.disabled_node(node_name) + end + assert!(node, "Node '#{node_name}' not found.") + node + end + + def seed_node_data(node, args) + args.each do |seed| + key, value = seed.split(':') + value = format_seed_value(value) + assert! key =~ /^[0-9a-z\._]+$/, "illegal characters used in property '#{key}'" + if key =~ /\./ + key_parts = key.split('.') + final_key = key_parts.pop + current_object = node + key_parts.each do |key_part| + current_object[key_part] ||= Config::Object.new + current_object = current_object[key_part] + end + current_object[final_key] = value + else + node[key] = value + end + end + end + + def remove_node_files(node_name) + (Leap::Platform.node_files + [:node_files_dir]).each do |path| + remove_file! [path, node_name] + end + end + + # + # conversions: + # + # "x,y,z" => ["x","y","z"] + # + # "22" => 22 + # + # "5.1" => 5.1 + # + def format_seed_value(v) + if v =~ /,/ + v = v.split(',') + v.map! do |i| + i = i.to_i if i.to_i.to_s == i + i = i.to_f if i.to_f.to_s == i + i + end + else + v = v.to_i if v.to_i.to_s == v + v = v.to_f if v.to_f.to_s == v + end + return v + end + + def validate_ip_address(node) + IPAddr.new(node['ip_address']) + rescue ArgumentError + bail! do + if node['ip_address'] + log :invalid, "ip_address #{node['ip_address'].inspect}" + else + log :missing, "ip_address" + end + end + end + + def assert_valid_node_name!(name, local=false) + assert! name, 'No specified.' + if local + assert! name =~ /^[0-9a-z]+$/, "illegal characters used in node name '#{name}' (note: Vagrant does not allow hyphens or underscores)" + else + assert! name =~ /^[0-9a-z-]+$/, "illegal characters used in node name '#{name}' (note: Linux does not allow underscores)" + end + end + +end; end \ No newline at end of file diff --git a/lib/leap_cli/commands/node_init.rb b/lib/leap_cli/commands/node_init.rb new file mode 100644 index 00000000..33f6288d --- /dev/null +++ b/lib/leap_cli/commands/node_init.rb @@ -0,0 +1,169 @@ +# +# Node initialization. +# Most of the fun stuff is in tasks.rb. +# + +module LeapCli; module Commands + + desc 'Node management' + command :node do |node| + node.desc 'Bootstraps a node or nodes, setting up SSH keys and installing prerequisite packages' + node.long_desc "This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, " + + "copying the authorized_keys file, installing packages that are required for deploying, and registering important facts. " + + "Node init must be run before deploying to a server, and the server must be running and available via the network. " + + "This command only needs to be run once, but there is no harm in running it multiple times." + node.arg_name 'FILTER' + node.command :init do |init| + init.switch 'echo', :desc => 'If set, passwords are visible as you type them (default is hidden)', :negatable => false + init.flag :port, :desc => 'Override the default SSH port.', :arg_name => 'PORT' + init.flag :ip, :desc => 'Override the default SSH IP address.', :arg_name => 'IPADDRESS' + + init.action do |global,options,args| + assert! args.any?, 'You must specify a FILTER' + finished = [] + manager.filter!(args).each_node do |node| + is_node_alive(node, options) + save_public_host_key(node, global, options) unless node.vagrant? + update_compiled_ssh_configs + ssh_connect_options = connect_options(options).merge({:bootstrap => true, :echo => options[:echo]}) + ssh_connect(node, ssh_connect_options) do |ssh| + if node.vagrant? + ssh.install_insecure_vagrant_key + end + ssh.install_authorized_keys + ssh.install_prerequisites + unless node.vagrant? + ssh.leap.log(:checking, "SSH host keys") do + ssh.leap.capture(get_ssh_keys_cmd) do |response| + update_local_ssh_host_keys(node, response[:data]) if response[:exitcode] == 0 + end + end + end + ssh.leap.log(:updating, "facts") do + ssh.leap.capture(facter_cmd) do |response| + if response[:exitcode] == 0 + update_node_facts(node.name, response[:data]) + else + log :failed, "to run facter on #{node.name}" + end + end + end + end + finished << node.name + end + log :completed, "initialization of nodes #{finished.join(', ')}" + end + end + end + + private + + ## + ## PRIVATE HELPERS + ## + + def is_node_alive(node, options) + address = options[:ip] || node.ip_address + port = options[:port] || node.ssh.port + log :connecting, "to node #{node.name}" + assert_run! "nc -zw3 #{address} #{port}", + "Failed to reach #{node.name} (address #{address}, port #{port}). You can override the configured IP address and port with --ip or --port." + end + + # + # saves the public ssh host key for node into the provider directory. + # + # see `man sshd` for the format of known_hosts + # + def save_public_host_key(node, global, options) + log :fetching, "public SSH host key for #{node.name}" + address = options[:ip] || node.ip_address + port = options[:port] || node.ssh.port + host_keys = get_public_keys_for_ip(address, port) + pub_key_path = Path.named_path([:node_ssh_pub_key, node.name]) + + if Path.exists?(pub_key_path) + if host_keys.include? SshKey.load(pub_key_path) + log :trusted, "- Public SSH host key for #{node.name} matches previously saved key", :indent => 1 + else + bail! do + log :error, "The public SSH host keys we just fetched for #{node.name} doesn't match what we have saved previously.", :indent => 1 + log "Delete the file #{pub_key_path} if you really want to remove the trusted SSH host key.", :indent => 2 + end + end + else + known_key = host_keys.detect{|k|k.in_known_hosts?(node.name, node.ip_address, node.domain.name)} + if known_key + log :trusted, "- Public SSH host key for #{node.name} is trusted (key found in your ~/.ssh/known_hosts)" + else + public_key = SshKey.pick_best_key(host_keys) + if public_key.nil? + bail!("We got back #{host_keys.size} host keys from #{node.name}, but we can't support any of them.") + else + say(" This is the SSH host key you got back from node \"#{node.name}\"") + say(" Type -- #{public_key.bits} bit #{public_key.type.upcase}") + say(" Fingerprint -- " + public_key.fingerprint) + say(" Public Key -- " + public_key.key) + if !global[:yes] && !agree(" Is this correct? ") + bail! + else + known_key = public_key + end + end + end + puts + write_file! [:node_ssh_pub_key, node.name], known_key.to_s + end + end + + # + # Get the public host keys for a host using ssh-keyscan. + # Return an array of SshKey objects, one for each key. + # + def get_public_keys_for_ip(address, port=22) + assert_bin!('ssh-keyscan') + output = assert_run! "ssh-keyscan -p #{port} #{address}", "Could not get the public host key from #{address}:#{port}. Maybe sshd is not running?" + if output.empty? + bail! :failed, "ssh-keyscan returned empty output." + end + + if output =~ /No route to host/ + bail! :failed, 'ssh-keyscan: no route to %s' % address + else + keys = SshKey.parse_keys(output) + if keys.empty? + bail! "ssh-keyscan got zero host keys back (that we understand)! Output was: #{output}" + else + return keys + end + end + end + + # run on the server to generate a string suitable for passing to SshKey.parse_keys() + def get_ssh_keys_cmd + "/bin/grep ^HostKey /etc/ssh/sshd_config | /usr/bin/awk '{print $2 \".pub\"}' | /usr/bin/xargs /bin/cat" + end + + # + # Sometimes the ssh host keys on the server will be better than what we have + # stored locally. In these cases, ask the user if they want to upgrade. + # + def update_local_ssh_host_keys(node, remote_keys_string) + remote_keys = SshKey.parse_keys(remote_keys_string) + return unless remote_keys.any? + current_key = SshKey.load(Path.named_path([:node_ssh_pub_key, node.name])) + best_key = SshKey.pick_best_key(remote_keys) + return unless best_key && current_key + if current_key != best_key + say(" One of the SSH host keys for node '#{node.name}' is better than what you currently have trusted.") + say(" Current key: #{current_key.summary}") + say(" Better key: #{best_key.summary}") + if agree(" Do you want to use the better key? ") + write_file! [:node_ssh_pub_key, node.name], best_key.to_s + end + else + log(3, "current host key does not need updating") + end + end + +end; end diff --git a/lib/leap_cli/commands/ssh.rb b/lib/leap_cli/commands/ssh.rb new file mode 100644 index 00000000..1a81902c --- /dev/null +++ b/lib/leap_cli/commands/ssh.rb @@ -0,0 +1,220 @@ +module LeapCli; module Commands + + desc 'Log in to the specified node with an interactive shell.' + arg_name 'NAME' #, :optional => false, :multiple => false + command :ssh do |c| + c.flag 'ssh', :desc => "Pass through raw options to ssh (e.g. `--ssh '-F ~/sshconfig'`)." + c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`.' + c.action do |global_options,options,args| + exec_ssh(:ssh, options, args) + end + end + + desc 'Log in to the specified node with an interactive shell using mosh (requires node to have mosh.enabled set to true).' + arg_name 'NAME' + command :mosh do |c| + c.flag 'ssh', :desc => "Pass through raw options to ssh (e.g. `--ssh '-F ~/sshconfig'`)." + c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`.' + c.action do |global_options,options,args| + exec_ssh(:mosh, options, args) + end + end + + desc 'Creates an SSH port forward (tunnel) to the node NAME. REMOTE_PORT is the port on the remote node that the tunnel will connect to. LOCAL_PORT is the optional port on your local machine. For example: `leap tunnel couch1:5984`.' + arg_name '[LOCAL_PORT:]NAME:REMOTE_PORT' + command :tunnel do |c| + c.flag 'ssh', :desc => "Pass through raw options to ssh (e.g. --ssh '-F ~/sshconfig')." + c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`.' + c.action do |global_options,options,args| + local_port, node, remote_port = parse_tunnel_arg(args.first) + options[:ssh] = [options[:ssh], "-N -L 127.0.0.1:#{local_port}:0.0.0.0:#{remote_port}"].join(' ') + log("Forward port localhost:#{local_port} to #{node.name}:#{remote_port}") + if is_port_available?(local_port) + exec_ssh(:ssh, options, [node.name]) + end + end + end + + desc 'Secure copy from FILE1 to FILE2. Files are specified as NODE_NAME:FILE_PATH. For local paths, omit "NODE_NAME:".' + arg_name 'FILE1 FILE2' + command :scp do |c| + c.switch :r, :desc => 'Copy recursively' + c.action do |global_options, options, args| + if args.size != 2 + bail!('You must specificy both FILE1 and FILE2') + end + from, to = args + if (from !~ /:/ && to !~ /:/) || (from =~ /:/ && to =~ /:/) + bail!('One FILE must be remote and the other local.') + end + src_node_name = src_file_path = src_node = nil + dst_node_name = dst_file_path = dst_node = nil + if from =~ /:/ + src_node_name, src_file_path = from.split(':') + src_node = get_node_from_args([src_node_name], :include_disabled => true) + dst_file_path = to + else + dst_node_name, dst_file_path = to.split(':') + dst_node = get_node_from_args([dst_node_name], :include_disabled => true) + src_file_path = from + end + exec_scp(options, src_node, src_file_path, dst_node, dst_file_path) + end + end + + protected + + # + # allow for ssh overrides of all commands that use ssh_connect + # + def connect_options(options) + connect_options = {:ssh_options=>{}} + if options[:port] + connect_options[:ssh_options][:port] = options[:port] + end + if options[:ip] + connect_options[:ssh_options][:host_name] = options[:ip] + end + return connect_options + end + + def ssh_config_help_message + puts "" + puts "Are 'too many authentication failures' getting you down?" + puts "Then we have the solution for you! Add something like this to your ~/.ssh/config file:" + puts " Host *.#{manager.provider.domain}" + puts " IdentityFile ~/.ssh/id_rsa" + puts " IdentitiesOnly=yes" + puts "(replace `id_rsa` with the actual private key filename that you use for this provider)" + end + + require 'socket' + def is_port_available?(port) + TCPServer.open('127.0.0.1', port) {} + true + rescue Errno::EACCES + bail!("You don't have permission to bind to port #{port}.") + rescue Errno::EADDRINUSE + bail!("Local port #{port} is already in use. Specify LOCAL_PORT to pick another.") + rescue Exception => exc + bail!(exc.to_s) + end + + private + + def exec_ssh(cmd, cli_options, args) + node = get_node_from_args(args, :include_disabled => true) + port = node.ssh.port + options = ssh_config(node) + username = 'root' + if LeapCli.log_level >= 3 + options << "-vv" + elsif LeapCli.log_level >= 2 + options << "-v" + end + if cli_options[:port] + port = cli_options[:port] + end + if cli_options[:ssh] + options << cli_options[:ssh] + end + ssh = "ssh -l #{username} -p #{port} #{options.join(' ')}" + if cmd == :ssh + command = "#{ssh} #{node.domain.full}" + elsif cmd == :mosh + command = "MOSH_TITLE_NOPREFIX=1 mosh --ssh \"#{ssh}\" #{node.domain.full}" + end + log 2, command + + # exec the shell command in a subprocess + pid = fork { exec "#{command}" } + + Signal.trap("SIGINT") do + Process.kill("KILL", pid) + Process.wait(pid) + exit(0) + end + + # wait for shell to exit so we can grab the exit status + _, status = Process.waitpid2(pid) + + if status.exitstatus == 255 + ssh_config_help_message + elsif status.exitstatus != 0 + exit(status.exitstatus) + end + end + + def exec_scp(cli_options, src_node, src_file_path, dst_node, dst_file_path) + node = src_node || dst_node + options = ssh_config(node) + port = node.ssh.port + username = 'root' + options << "-r" if cli_options[:r] + scp = "scp -P #{port} #{options.join(' ')}" + if src_node + command = "#{scp} #{username}@#{src_node.domain.full}:#{src_file_path} #{dst_file_path}" + elsif dst_node + command = "#{scp} #{src_file_path} #{username}@#{dst_node.domain.full}:#{dst_file_path}" + end + log 2, command + + # exec the shell command in a subprocess + pid = fork { exec "#{command}" } + + Signal.trap("SIGINT") do + Process.kill("KILL", pid) + Process.wait(pid) + exit(0) + end + + # wait for shell to exit so we can grab the exit status + _, status = Process.waitpid2(pid) + exit(status.exitstatus) + end + + # + # SSH command line -o options. See `man ssh_config` + # + # NOTES: + # + # The option 'HostKeyAlias=#{node.name}' is oddly incompatible with ports in + # known_hosts file, so we must not use this or non-standard ports break. + # + def ssh_config(node) + options = [ + "-o 'HostName=#{node.ip_address}'", + "-o 'GlobalKnownHostsFile=#{path(:known_hosts)}'", + "-o 'UserKnownHostsFile=/dev/null'" + ] + if node.vagrant? + options << "-i #{vagrant_ssh_key_file}" # use the universal vagrant insecure key + options << "-o IdentitiesOnly=yes" # force the use of the insecure vagrant key + options << "-o 'StrictHostKeyChecking=no'" # blindly accept host key and don't save it + # (since userknownhostsfile is /dev/null) + else + options << "-o 'StrictHostKeyChecking=yes'" + end + if !node.supported_ssh_host_key_algorithms.empty? + options << "-o 'HostKeyAlgorithms=#{node.supported_ssh_host_key_algorithms}'" + end + return options + end + + def parse_tunnel_arg(arg) + if arg.count(':') == 1 + node_name, remote = arg.split(':') + local = nil + elsif arg.count(':') == 2 + local, node_name, remote = arg.split(':') + else + bail!('Argument NAME:REMOTE_PORT required.') + end + node = get_node_from_args([node_name], :include_disabled => true) + remote = remote.to_i + local = local || remote + local = local.to_i + return [local, node, remote] + end + +end; end \ No newline at end of file diff --git a/lib/leap_cli/commands/test.rb b/lib/leap_cli/commands/test.rb new file mode 100644 index 00000000..73207b31 --- /dev/null +++ b/lib/leap_cli/commands/test.rb @@ -0,0 +1,74 @@ +module LeapCli; module Commands + + desc 'Run tests.' + command [:test, :t] do |test| + test.desc 'Run the test suit on FILTER nodes.' + test.arg_name 'FILTER', :optional => true + test.command :run do |run| + run.switch 'continue', :desc => 'Continue over errors and failures (default is --no-continue).', :negatable => true + run.action do |global_options,options,args| + test_order = File.join(Path.platform, 'tests/order.rb') + if File.exists?(test_order) + require test_order + end + manager.filter!(args).names_in_test_dependency_order.each do |node_name| + node = manager.nodes[node_name] + begin + ssh_connect(node) do |ssh| + ssh.run(test_cmd(options)) + end + rescue Capistrano::CommandError => exc + if options[:continue] + exit_status(1) + else + bail! + end + end + end + end + end + + test.desc 'Creates files needed to run tests.' + test.command :init do |init| + init.action do |global_options,options,args| + generate_test_client_openvpn_configs + end + end + + test.default_command :run + end + + private + + def test_cmd(options) + if options[:continue] + "#{Leap::Platform.leap_dir}/bin/run_tests --continue" + else + "#{Leap::Platform.leap_dir}/bin/run_tests" + end + end + + # + # generates a whole bunch of openvpn configs that can be used to connect to different openvpn gateways + # + def generate_test_client_openvpn_configs + assert_config! 'provider.ca.client_certificates.unlimited_prefix' + assert_config! 'provider.ca.client_certificates.limited_prefix' + template = read_file! Path.find_file(:test_client_openvpn_template) + manager.environment_names.each do |env| + vpn_nodes = manager.nodes[:environment => env][:services => 'openvpn']['openvpn.allow_limited' => true] + if vpn_nodes.any? + generate_test_client_cert(provider.ca.client_certificates.limited_prefix) do |key, cert| + write_file! [:test_openvpn_config, [env, 'limited'].compact.join('_')], Util.erb_eval(template, binding) + end + end + vpn_nodes = manager.nodes[:environment => env][:services => 'openvpn']['openvpn.allow_unlimited' => true] + if vpn_nodes.any? + generate_test_client_cert(provider.ca.client_certificates.unlimited_prefix) do |key, cert| + write_file! [:test_openvpn_config, [env, 'unlimited'].compact.join('_')], Util.erb_eval(template, binding) + end + end + end + end + +end; end diff --git a/lib/leap_cli/commands/user.rb b/lib/leap_cli/commands/user.rb new file mode 100644 index 00000000..480e9a9c --- /dev/null +++ b/lib/leap_cli/commands/user.rb @@ -0,0 +1,136 @@ + +# +# perhaps we want to verify that the key files are actually the key files we expect. +# we could use 'file' for this: +# +# > file ~/.gnupg/00440025.asc +# ~/.gnupg/00440025.asc: PGP public key block +# +# > file ~/.ssh/id_rsa.pub +# ~/.ssh/id_rsa.pub: OpenSSH RSA public key +# + +module LeapCli + module Commands + + desc 'Adds a new trusted sysadmin by adding public keys to the "users" directory.' + arg_name 'USERNAME' #, :optional => false, :multiple => false + command :'add-user' do |c| + + c.switch 'self', :desc => 'Add yourself as a trusted sysadin by choosing among the public keys available for the current user.', :negatable => false + c.flag 'ssh-pub-key', :desc => 'SSH public key file for this new user' + c.flag 'pgp-pub-key', :desc => 'OpenPGP public key file for this new user' + + c.action do |global_options,options,args| + username = args.first + if !username.any? + if options[:self] + username ||= `whoami`.strip + else + help! "Either USERNAME argument or --self flag is required." + end + end + if Leap::Platform.reserved_usernames.include? username + bail! %(The username "#{username}" is reserved. Sorry, pick another.) + end + + ssh_pub_key = nil + pgp_pub_key = nil + + if options['ssh-pub-key'] + ssh_pub_key = read_file!(options['ssh-pub-key']) + end + if options['pgp-pub-key'] + pgp_pub_key = read_file!(options['pgp-pub-key']) + end + + if options[:self] + ssh_pub_key ||= pick_ssh_key.to_s + pgp_pub_key ||= pick_pgp_key + end + + assert!(ssh_pub_key, 'Sorry, could not find SSH public key.') + + if ssh_pub_key + write_file!([:user_ssh, username], ssh_pub_key) + end + if pgp_pub_key + write_file!([:user_pgp, username], pgp_pub_key) + end + + update_authorized_keys + end + end + + # + # let the the user choose among the ssh public keys that we encounter, or just pick the key if there is only one. + # + def pick_ssh_key + ssh_keys = [] + Dir.glob("#{ENV['HOME']}/.ssh/*.pub").each do |keyfile| + ssh_keys << SshKey.load(keyfile) + end + + if `which ssh-add`.strip.any? + `ssh-add -L 2> /dev/null`.split("\n").compact.each do |line| + key = SshKey.load(line) + if key + key.comment = 'ssh-agent' + ssh_keys << key unless ssh_keys.include?(key) + end + end + end + ssh_keys.compact! + + assert! ssh_keys.any?, 'Sorry, could not find any SSH public key for you. Have you run ssh-keygen?' + + if ssh_keys.length > 1 + key_index = numbered_choice_menu('Choose your SSH public key', ssh_keys.collect(&:summary)) do |line, i| + say("#{i+1}. #{line}") + end + else + key_index = 0 + end + + return ssh_keys[key_index] + end + + # + # let the the user choose among the gpg public keys that we encounter, or just pick the key if there is only one. + # + def pick_pgp_key + begin + require 'gpgme' + rescue LoadError + log "Skipping OpenPGP setup because gpgme is not installed." + return + end + + secret_keys = GPGME::Key.find(:secret) + if secret_keys.empty? + log "Skipping OpenPGP setup because I could not find any OpenPGP keys for you" + return nil + end + + secret_keys.select!{|key| !key.expired} + + if secret_keys.length > 1 + key_index = numbered_choice_menu('Choose your OpenPGP public key', secret_keys) do |key, i| + key_info = key.to_s.split("\n")[0..1].map{|line| line.sub(/^\s*(sec|uid)\s*/,'')}.join(' -- ') + say("#{i+1}. #{key_info}") + end + else + key_index = 0 + end + + key_id = secret_keys[key_index].sha + + # can't use this, it includes signatures: + #puts GPGME::Key.export(key_id, :armor => true, :export_options => :export_minimal) + + # export with signatures removed: + return `gpg --armor --export-options export-minimal --export #{key_id}`.strip + end + + end +end \ No newline at end of file diff --git a/lib/leap_cli/commands/util.rb b/lib/leap_cli/commands/util.rb new file mode 100644 index 00000000..c1da570e --- /dev/null +++ b/lib/leap_cli/commands/util.rb @@ -0,0 +1,50 @@ +module LeapCli; module Commands + + extend self + extend LeapCli::Util + extend LeapCli::Util::RemoteCommand + + def path(name) + Path.named_path(name) + end + + # + # keeps prompting the user for a numbered choice, until they pick a good one or bail out. + # + # block is yielded and is responsible for rendering the choices. + # + def numbered_choice_menu(msg, items, &block) + while true + say("\n" + msg + ':') + items.each_with_index &block + say("q. quit") + index = ask("number 1-#{items.length}> ") + if index.empty? + next + elsif index =~ /q/ + bail! + else + i = index.to_i - 1 + if i < 0 || i >= items.length + bail! + else + return i + end + end + end + end + + + def parse_node_list(nodes) + if nodes.is_a? Config::Object + Config::ObjectList.new(nodes) + elsif nodes.is_a? Config::ObjectList + nodes + elsif nodes.is_a? String + manager.filter!(nodes) + else + bail! "argument error" + end + end + +end; end diff --git a/lib/leap_cli/commands/vagrant.rb b/lib/leap_cli/commands/vagrant.rb new file mode 100644 index 00000000..27c739b1 --- /dev/null +++ b/lib/leap_cli/commands/vagrant.rb @@ -0,0 +1,197 @@ +autoload :IPAddr, 'ipaddr' +require 'fileutils' + +module LeapCli; module Commands + + desc "Manage local virtual machines." + long_desc "This command provides a convient way to manage Vagrant-based virtual machines. If FILTER argument is missing, the command runs on all local virtual machines. The Vagrantfile is automatically generated in 'test/Vagrantfile'. If you want to run vagrant commands manually, cd to 'test'." + command [:local, :l] do |local| + local.desc 'Starts up the virtual machine(s)' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :start do |start| + start.flag(:basebox, + :desc => "The basebox to use. This value is passed to vagrant as the "+ + "`config.vm.box` option. The value here should be the name of an installed box or a "+ + "shorthand name of a box in HashiCorp's Atlas.", + :arg_name => 'BASEBOX', + :default_value => 'LEAP/wheezy' + ) + start.action do |global_options,options,args| + vagrant_command(["up", "sandbox on"], args, options) + end + end + + local.desc 'Shuts down the virtual machine(s)' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :stop do |stop| + stop.action do |global_options,options,args| + if global_options[:yes] + vagrant_command("halt --force", args) + else + vagrant_command("halt", args) + end + end + end + + local.desc 'Destroys the virtual machine(s), reclaiming the disk space' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :destroy do |destroy| + destroy.action do |global_options,options,args| + if global_options[:yes] + vagrant_command("destroy --force", args) + else + vagrant_command("destroy", args) + end + end + end + + local.desc 'Print the status of local virtual machine(s)' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :status do |status| + status.action do |global_options,options,args| + vagrant_command("status", args) + end + end + + local.desc 'Saves the current state of the virtual machine as a new snapshot' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :save do |status| + status.action do |global_options,options,args| + vagrant_command("sandbox commit", args) + end + end + + local.desc 'Resets virtual machine(s) to the last saved snapshot' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :reset do |reset| + reset.action do |global_options,options,args| + vagrant_command("sandbox rollback", args) + end + end + end + + public + + # + # returns the path to a vagrant ssh key file. + # + # if the vagrant.key file is owned by root or ourselves, then + # we need to make sure that it owned by us and not world readable. + # + def vagrant_ssh_key_file + file_path = File.expand_path('../../../vendor/vagrant_ssh_keys/vagrant.key', File.dirname(__FILE__)) + Util.assert_files_exist! file_path + uid = File.new(file_path).stat.uid + if uid == 0 || uid == Process.euid + FileUtils.install file_path, '/tmp/vagrant.key', :mode => 0600 + file_path = '/tmp/vagrant.key' + end + return file_path + end + + protected + + def vagrant_command(cmds, args, options={}) + vagrant_setup(options) + cmds = cmds.to_a + if args.empty? + nodes = [""] + else + nodes = manager.filter(args)[:environment => "local"].field(:name) + end + if nodes.any? + vagrant_dir = File.dirname(Path.named_path(:vagrantfile)) + exec = ["cd #{vagrant_dir}"] + cmds.each do |cmd| + nodes.each do |node| + exec << "vagrant #{cmd} #{node}" + end + end + execute exec.join('; ') + else + bail! "No nodes found. This command only works on nodes with ip_address in the network #{LeapCli.leapfile.vagrant_network}" + end + end + + private + + def vagrant_setup(options) + assert_bin! 'vagrant', 'Vagrant is required for running local virtual machines. Run "sudo apt-get install vagrant".' + + if vagrant_version <= Gem::Version.new('1.0.0') + gem_path = assert_run!('vagrant gem which sahara') + if gem_path.nil? || gem_path.empty? || gem_path =~ /^ERROR/ + log :installing, "vagrant plugin 'sahara'" + assert_run! 'vagrant gem install sahara -v 0.0.13' + end + else + unless assert_run!('vagrant plugin list | grep sahara | cat').chars.any? + log :installing, "vagrant plugin 'sahara'" + assert_run! 'vagrant plugin install sahara' + end + end + create_vagrant_file(options) + end + + def vagrant_version + @vagrant_version ||= Gem::Version.new(assert_run!('vagrant --version').split(' ')[1]) + end + + def execute(cmd) + log 2, :run, cmd + exec cmd + end + + def create_vagrant_file(options) + lines = [] + netmask = IPAddr.new('255.255.255.255').mask(LeapCli.leapfile.vagrant_network.split('/').last).to_s + + basebox = options[:basebox] || 'LEAP/wheezy' + + if vagrant_version <= Gem::Version.new('1.1.0') + lines << %[Vagrant::Config.run do |config|] + manager.each_node do |node| + if node.vagrant? + lines << %[ config.vm.define :#{node.name} do |config|] + lines << %[ config.vm.box = "#{basebox}"] + lines << %[ config.vm.network :hostonly, "#{node.ip_address}", :netmask => "#{netmask}"] + lines << %[ config.vm.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]] + lines << %[ config.vm.customize ["modifyvm", :id, "--name", "#{node.name}"]] + lines << %[ #{leapfile.custom_vagrant_vm_line}] if leapfile.custom_vagrant_vm_line + lines << %[ end] + end + end + else + lines << %[Vagrant.configure("2") do |config|] + manager.each_node do |node| + if node.vagrant? + lines << %[ config.vm.define :#{node.name} do |config|] + lines << %[ config.vm.box = "#{basebox}"] + lines << %[ config.vm.network :private_network, ip: "#{node.ip_address}"] + lines << %[ config.vm.provider "virtualbox" do |v|] + lines << %[ v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]] + lines << %[ v.name = "#{node.name}"] + lines << %[ end] + lines << %[ #{leapfile.custom_vagrant_vm_line}] if leapfile.custom_vagrant_vm_line + lines << %[ end] + end + end + end + + lines << %[end] + lines << "" + write_file! :vagrantfile, lines.join("\n") + end + + def pick_next_vagrant_ip_address + taken_ips = manager.nodes[:environment => "local"].field(:ip_address) + if taken_ips.any? + highest_ip = taken_ips.map{|ip| IPAddr.new(ip)}.max + new_ip = highest_ip.succ + else + new_ip = IPAddr.new(LeapCli.leapfile.vagrant_network).succ.succ + end + return new_ip.to_s + end + +end; end diff --git a/lib/leap_cli/macros.rb b/lib/leap_cli/macros.rb new file mode 100644 index 00000000..fdb9a94e --- /dev/null +++ b/lib/leap_cli/macros.rb @@ -0,0 +1,16 @@ +# +# MACROS +# +# The methods in these files are available in the context of a .json configuration file. +# (The module LeapCli::Macro is included in Config::Object) +# + +require_relative 'macros/core' +require_relative 'macros/files' +require_relative 'macros/haproxy' +require_relative 'macros/hosts' +require_relative 'macros/keys' +require_relative 'macros/nodes' +require_relative 'macros/secrets' +require_relative 'macros/stunnel' +require_relative 'macros/provider' diff --git a/lib/leap_cli/macros/core.rb b/lib/leap_cli/macros/core.rb new file mode 100644 index 00000000..7de50f2f --- /dev/null +++ b/lib/leap_cli/macros/core.rb @@ -0,0 +1,94 @@ +# encoding: utf-8 + +module LeapCli + module Macro + + # + # return a fingerprint for a x509 certificate + # + def fingerprint(filename) + "SHA256: " + X509.fingerprint("SHA256", Path.named_path(filename)) + end + + # + # Creates a hash from the ssh key info in users directory, for use in + # updating authorized_keys file. Additionally, the 'monitor' public key is + # included, which is used by the monitor nodes to run particular commands + # remotely. + # + def authorized_keys + hash = {} + keys = Dir.glob(Path.named_path([:user_ssh, '*'])) + keys.sort.each do |keyfile| + ssh_type, ssh_key = File.read(keyfile, :encoding => 'UTF-8').strip.split(" ") + name = File.basename(File.dirname(keyfile)) + until hash[name].nil? + i ||= 1; name = "#{name}#{i+=1}" + end + hash[name] = { + "type" => ssh_type, + "key" => ssh_key + } + end + ssh_type, ssh_key = File.read(Path.named_path(:monitor_pub_key), :encoding => 'UTF-8').strip.split(" ") + hash[Leap::Platform.monitor_username] = { + "type" => ssh_type, + "key" => ssh_key + } + hash + end + + def assert(assertion) + if instance_eval(assertion) + true + else + raise AssertionFailed.new(assertion), assertion, caller + end + end + + def error(msg) + raise ConfigError.new(@node, msg), msg, caller + end + + # + # applies a JSON partial to this node + # + def apply_partial(partial_path) + manager.partials(partial_path).each do |partial_data| + self.deep_merge!(partial_data) + end + end + + # + # If at first you don't succeed, then it is time to give up. + # + # try{} returns nil if anything in the block throws an exception. + # + # You can wrap something that might fail in `try`, like so. + # + # "= try{ nodes[:services => 'tor'].first.ip_address } " + # + def try(&block) + yield + rescue NoMethodError + rescue ArgumentError + nil + end + + protected + + # + # returns a node list, if argument is not already one + # + def listify(node_list) + if node_list.is_a? Config::ObjectList + node_list + elsif node_list.is_a? Config::Object + Config::ObjectList.new(node_list) + else + raise ArgumentError, 'argument must be a node or node list, not a `%s`' % node_list.class, caller + end + end + + end +end diff --git a/lib/leap_cli/macros/files.rb b/lib/leap_cli/macros/files.rb new file mode 100644 index 00000000..958958bc --- /dev/null +++ b/lib/leap_cli/macros/files.rb @@ -0,0 +1,89 @@ +# encoding: utf-8 + +## +## FILES +## + +module LeapCli + module Macro + + # + # inserts the contents of a file + # + def file(filename, options={}) + if filename.is_a? Symbol + filename = [filename, @node.name] + end + filepath = Path.find_file(filename) + if filepath + if filepath =~ /\.erb$/ + ERB.new(File.read(filepath, :encoding => 'UTF-8'), nil, '%<>').result(binding) + else + File.read(filepath, :encoding => 'UTF-8') + end + else + raise FileMissing.new(Path.named_path(filename), options) + "" + end + end + + # + # like #file, but allow missing files + # + def try_file(filename) + return file(filename) + rescue FileMissing + return nil + end + + # + # returns what the file path will be, once the file is rsynced to the server. + # an internal list of discovered file paths is saved, in order to rsync these files when needed. + # + # notes: + # + # * argument 'path' is relative to Path.provider/files or Path.provider_base/files + # * the path returned by this method is absolute + # * the path stored for use later by rsync is relative to Path.provider + # * if the path does not exist locally, but exists in provider_base, then the default file from + # provider_base is copied locally. this is required for rsync to work correctly. + # + def file_path(path, options={}) + if path.is_a? Symbol + path = [path, @node.name] + elsif path.is_a? String + # ensure it prefixed with files/ + unless path =~ /^files\// + path = "files/" + path + end + end + actual_path = Path.find_file(path) + if actual_path.nil? + if options[:missing] + raise FileMissing.new(Path.named_path(path), options) + else + Util::log 2, :skipping, "file_path(\"#{path}\") because there is no such file." + end + nil + else + if actual_path =~ /^#{Regexp.escape(Path.provider_base)}/ + # if file is under Path.provider_base, we must copy the default file to + # to Path.provider in order for rsync to be able to sync the file. + local_provider_path = actual_path.sub(/^#{Regexp.escape(Path.provider_base)}/, Path.provider) + FileUtils.mkdir_p File.dirname(local_provider_path), :mode => 0700 + FileUtils.install actual_path, local_provider_path, :mode => 0600 + Util.log :created, Path.relative_path(local_provider_path) + actual_path = local_provider_path + end + if File.directory?(actual_path) && actual_path !~ /\/$/ + actual_path += '/' # ensure directories end with /, important for building rsync command + end + relative_path = Path.relative_path(actual_path) + relative_path.sub!(/^files\//, '') # remove "files/" prefix + @node.file_paths << relative_path + File.join(Leap::Platform.files_dir, relative_path) + end + end + + end +end \ No newline at end of file diff --git a/lib/leap_cli/macros/haproxy.rb b/lib/leap_cli/macros/haproxy.rb new file mode 100644 index 00000000..602ae726 --- /dev/null +++ b/lib/leap_cli/macros/haproxy.rb @@ -0,0 +1,73 @@ +# encoding: utf-8 + +## +## HAPROXY +## + +module LeapCli + module Macro + + # + # creates a hash suitable for configuring haproxy. the key is the node name of the server we are proxying to. + # + # * node_list - a hash of nodes for the haproxy servers + # * stunnel_client - contains the mappings to local ports for each server node. + # * non_stunnel_port - in case self is included in node_list, the port to connect to. + # + # 1000 weight is used for nodes in the same location. + # 100 otherwise. + # + def haproxy_servers(node_list, stunnel_clients, non_stunnel_port=nil) + default_weight = 10 + local_weight = 100 + + # record the hosts_file + hostnames(node_list) + + # create a simple map for node name -> local stunnel accept port + accept_ports = stunnel_clients.inject({}) do |hsh, stunnel_entry| + name = stunnel_entry.first.sub /_[0-9]+$/, '' + hsh[name] = stunnel_entry.last['accept_port'] + hsh + end + + # if one the nodes in the node list is ourself, then there will not be a stunnel to it, + # but we need to include it anyway in the haproxy config. + if node_list[self.name] && non_stunnel_port + accept_ports[self.name] = non_stunnel_port + end + + # create the first pass of the servers hash + servers = node_list.values.inject(Config::ObjectList.new) do |hsh, node| + # make sure we have a port to talk to + unless accept_ports[node.name] + error "haproxy needs a local port to talk to when connecting to #{node.name}" + end + weight = default_weight + try { + weight = local_weight if self.location.name == node.location.name + } + hsh[node.name] = Config::Object[ + 'backup', false, + 'host', 'localhost', + 'port', accept_ports[node.name], + 'weight', weight + ] + if node.services.include?('couchdb') + hsh[node.name]['writable'] = node.couch.mode != 'mirror' + end + hsh + end + + # if there are some local servers, make the others backup + if servers.detect{|k,v| v.weight == local_weight} + servers.each do |k,server| + server['backup'] = server['weight'] == default_weight + end + end + + return servers + end + + end +end diff --git a/lib/leap_cli/macros/hosts.rb b/lib/leap_cli/macros/hosts.rb new file mode 100644 index 00000000..8281329f --- /dev/null +++ b/lib/leap_cli/macros/hosts.rb @@ -0,0 +1,68 @@ +# encoding: utf-8 + +module LeapCli + module Macro + + ## + ## HOSTS + ## + + # + # records the list of hosts that are encountered for this node + # + def hostnames(nodes) + @referenced_nodes ||= Config::ObjectList.new + nodes = listify(nodes) + nodes.each_node do |node| + @referenced_nodes[node.name] ||= node + end + return nodes.values.collect {|node| node.domain.name} + end + + # + # Generates entries needed for updating /etc/hosts on a node (as a hash). + # + # Argument `nodes` can be nil or a list of nodes. If nil, only include the + # IPs of the other nodes this @node as has encountered (plus all mx nodes). + # + # Also, for virtual machines, we use the local address if this @node is in + # the same location as the node in question. + # + # We include the ssh public key for each host, so that the hash can also + # be used to generate the /etc/ssh/known_hosts + # + def hosts_file(nodes=nil) + if nodes.nil? + if @referenced_nodes && @referenced_nodes.any? + nodes = @referenced_nodes + nodes = nodes.merge(nodes_like_me[:services => 'mx']) # all nodes always need to communicate with mx nodes. + end + end + return {} unless nodes + hosts = {} + my_location = @node['location'] ? @node['location']['name'] : nil + nodes.each_node do |node| + hosts[node.name] = { + 'ip_address' => node.ip_address, + 'domain_internal' => node.domain.internal, + 'domain_full' => node.domain.full, + 'port' => node.ssh.port + } + node_location = node['location'] ? node['location']['name'] : nil + if my_location == node_location + if facts = @node.manager.facts[node.name] + if facts['ec2_public_ipv4'] + hosts[node.name]['ip_address'] = facts['ec2_public_ipv4'] + end + end + end + host_pub_key = Util::read_file([:node_ssh_pub_key,node.name]) + if host_pub_key + hosts[node.name]['host_pub_key'] = host_pub_key + end + end + hosts + end + + end +end \ No newline at end of file diff --git a/lib/leap_cli/macros/keys.rb b/lib/leap_cli/macros/keys.rb new file mode 100644 index 00000000..0ed7ccd0 --- /dev/null +++ b/lib/leap_cli/macros/keys.rb @@ -0,0 +1,83 @@ +# encoding: utf-8 + +# +# Macro for dealing with cryptographic keys +# + +module LeapCli + module Macro + + # + # return the path to the tor public key + # generating key if it is missing + # + def tor_public_key_path(path_name, key_type) + path = file_path(path_name) + if path.nil? + generate_tor_key(key_type) + file_path(path_name) + else + path + end + end + + # + # return the path to the tor private key + # generating key if it is missing + # + def tor_private_key_path(path_name, key_type) + path = file_path(path_name) + if path.nil? + generate_tor_key(key_type) + file_path(path_name) + else + path + end + end + + # + # Generates a onion_address from a public RSA key file. + # + # path_name is the named path of the Tor public key. + # + # Basically, an onion address is nothing more than a base32 encoding + # of the first 10 bytes of a sha1 digest of the public key. + # + # Additionally, Tor ignores the 22 byte header of the public key + # before taking the sha1 digest. + # + def onion_address(path_name) + require 'base32' + require 'base64' + require 'openssl' + path = Path.find_file([path_name, self.name]) + if path && File.exists?(path) + public_key_str = File.readlines(path).grep(/^[^-]/).join + public_key = Base64.decode64(public_key_str) + public_key = public_key.slice(22..-1) # Tor ignores the 22 byte SPKI header + sha1sum = Digest::SHA1.new.digest(public_key) + Base32.encode(sha1sum.slice(0,10)).downcase + else + LeapCli.log :warning, 'Tor public key file "%s" does not exist' % tor_public_key_path + end + end + + private + + def generate_tor_key(key_type) + if key_type == 'RSA' + require 'certificate_authority' + keypair = CertificateAuthority::MemoryKeyMaterial.new + bit_size = 1024 + LeapCli.log :generating, "%s bit RSA Tor key" % bit_size do + keypair.generate_key(bit_size) + LeapCli::Util.write_file! [:node_tor_priv_key, self.name], keypair.private_key.to_pem + LeapCli::Util.write_file! [:node_tor_pub_key, self.name], keypair.public_key.to_pem + end + else + LeapCli.bail! 'tor.key.type of %s is not yet supported' % key_type + end + end + + end +end diff --git a/lib/leap_cli/macros/nodes.rb b/lib/leap_cli/macros/nodes.rb new file mode 100644 index 00000000..8b961cbc --- /dev/null +++ b/lib/leap_cli/macros/nodes.rb @@ -0,0 +1,88 @@ +# encoding: utf-8 + +## +## node related macros +## + +module LeapCli + module Macro + + # + # the list of all the nodes + # + def nodes + global.nodes + end + + # + # simple alias for global.provider + # + def provider + global.provider + end + + # + # returns a list of nodes that match the same environment + # + # if @node.environment is not set, we return other nodes + # where environment is not set. + # + def nodes_like_me + nodes[:environment => @node.environment] + end + + # + # returns a list of nodes that match the location name + # and environment of @node. + # + def nodes_near_me + if @node['location'] && @node['location']['name'] + nodes_like_me['location.name' => @node.location.name] + else + nodes_like_me['location' => nil] + end + end + + # + # + # picks a node out from the node list in such a way that: + # + # (1) which nodes picked which nodes is saved in secrets.json + # (2) when other nodes call this macro with the same node list, they are guaranteed to get a different node + # (3) if all the nodes in the pick_node list have been picked, remaining nodes are distributed randomly. + # + # if the node_list is empty, an exception is raised. + # if node_list size is 1, then that node is returned and nothing is + # memorized via the secrets.json file. + # + # `label` is needed to distinguish between pools of nodes for different purposes. + # + # TODO: more evenly balance after all the nodes have been picked. + # + def pick_node(label, node_list) + if node_list.any? + if node_list.size == 1 + return node_list.values.first + else + secrets_key = "pick_node(:#{label},#{node_list.keys.sort.join(',')})" + secrets_value = @manager.secrets.retrieve(secrets_key, @node.environment) || {} + secrets_value[@node.name] ||= begin + node_to_pick = nil + node_list.each_node do |node| + next if secrets_value.values.include?(node.name) + node_to_pick = node.name + end + node_to_pick ||= secrets_value.values.shuffle.first # all picked already, so pick a random one. + node_to_pick + end + picked_node_name = secrets_value[@node.name] + @manager.secrets.set(secrets_key, secrets_value, @node.environment) + return node_list[picked_node_name] + end + else + raise ArgumentError.new('pick_node(node_list): node_list cannot be empty') + end + end + + end +end \ No newline at end of file diff --git a/lib/leap_cli/macros/provider.rb b/lib/leap_cli/macros/provider.rb new file mode 100644 index 00000000..84c4e1b8 --- /dev/null +++ b/lib/leap_cli/macros/provider.rb @@ -0,0 +1,20 @@ +# +# These macros are intended only for use in provider.json, although they are +# currently loaded in all .json contexts. +# + +module LeapCli + module Macro + + # + # returns an array of the service names, including only those services that + # are enabled for this environment. + # + def enabled_services + manager.env(self.environment).services[:service_type => :user_service].field(:name).select { |service| + manager.nodes[:environment => self.environment][:services => service].any? + } + end + + end +end diff --git a/lib/leap_cli/macros/secrets.rb b/lib/leap_cli/macros/secrets.rb new file mode 100644 index 00000000..8d1feb55 --- /dev/null +++ b/lib/leap_cli/macros/secrets.rb @@ -0,0 +1,39 @@ +# encoding: utf-8 + +require 'base32' + +module LeapCli + module Macro + + # + # inserts a named secret, generating it if needed. + # + # manager.export_secrets should be called later to capture any newly generated secrets. + # + # +length+ is the character length of the generated password. + # + def secret(name, length=32) + manager.secrets.set(name, @node.environment) { Util::Secret.generate(length) } + end + + # inserts a base32 encoded secret + def base32_secret(name, length=20) + manager.secrets.set(name, @node.environment) { Base32.encode(Util::Secret.generate(length)) } + end + + # Picks a random obfsproxy port from given range + def rand_range(name, range) + manager.secrets.set(name, @node.environment) { rand(range) } + end + + # + # inserts an hexidecimal secret string, generating it if needed. + # + # +bit_length+ is the bits in the secret, (ie length of resulting hex string will be bit_length/4) + # + def hex_secret(name, bit_length=128) + manager.secrets.set(name, @node.environment) { Util::Secret.generate_hex(bit_length) } + end + + end +end \ No newline at end of file diff --git a/lib/leap_cli/macros/stunnel.rb b/lib/leap_cli/macros/stunnel.rb new file mode 100644 index 00000000..f16308c7 --- /dev/null +++ b/lib/leap_cli/macros/stunnel.rb @@ -0,0 +1,95 @@ +## +## STUNNEL +## + +# +# About stunnel +# -------------------------- +# +# The network looks like this: +# +# From the client's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# consumer app -> localhost:accept_port -> connect:connect_port -> ?? +# +# From the server's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# ?? -> *:accept_port -> localhost:connect_port -> service +# + +module LeapCli + module Macro + + # + # stunnel configuration for the client side. + # + # +node_list+ is a ObjectList of nodes running stunnel servers. + # + # +port+ is the real port of the ultimate service running on the servers + # that the client wants to connect to. + # + # * accept_port is the port on localhost to which local clients + # can connect. it is auto generated serially. + # + # * connect_port is the port on the stunnel server to connect to. + # it is auto generated from the +port+ argument. + # + # generates an entry appropriate to be passed directly to + # create_resources(stunnel::service, hiera('..'), defaults) + # + # local ports are automatically generated, starting at 4000 + # and incrementing in sorted order (by node name). + # + def stunnel_client(node_list, port, options={}) + @next_stunnel_port ||= 4000 + node_list = listify(node_list) + hostnames(node_list) # record the hosts + result = Config::ObjectList.new + node_list.each_node do |node| + if node.name != self.name || options[:include_self] + result["#{node.name}_#{port}"] = Config::Object[ + 'accept_port', @next_stunnel_port, + 'connect', node.domain.internal, + 'connect_port', stunnel_port(port), + 'original_port', port + ] + @next_stunnel_port += 1 + end + end + result + end + + # + # generates a stunnel server entry. + # + # +port+ is the real port targeted service. + # + # * `accept_port` is the publicly bound port + # * `connect_port` is the port that the local service is running on. + # + def stunnel_server(port) + { + "accept_port" => stunnel_port(port), + "connect_port" => port + } + end + + private + + # + # maps a real port to a stunnel port (used as the connect_port in the client config + # and the accept_port in the server config) + # + def stunnel_port(port) + port = port.to_i + if port < 50000 + return port + 10000 + else + return port - 10000 + end + end + + end +end \ No newline at end of file diff --git a/provider_base/lib/macros.rb b/provider_base/lib/macros.rb deleted file mode 100644 index fdb9a94e..00000000 --- a/provider_base/lib/macros.rb +++ /dev/null @@ -1,16 +0,0 @@ -# -# MACROS -# -# The methods in these files are available in the context of a .json configuration file. -# (The module LeapCli::Macro is included in Config::Object) -# - -require_relative 'macros/core' -require_relative 'macros/files' -require_relative 'macros/haproxy' -require_relative 'macros/hosts' -require_relative 'macros/keys' -require_relative 'macros/nodes' -require_relative 'macros/secrets' -require_relative 'macros/stunnel' -require_relative 'macros/provider' diff --git a/provider_base/lib/macros/core.rb b/provider_base/lib/macros/core.rb deleted file mode 100644 index 7de50f2f..00000000 --- a/provider_base/lib/macros/core.rb +++ /dev/null @@ -1,94 +0,0 @@ -# encoding: utf-8 - -module LeapCli - module Macro - - # - # return a fingerprint for a x509 certificate - # - def fingerprint(filename) - "SHA256: " + X509.fingerprint("SHA256", Path.named_path(filename)) - end - - # - # Creates a hash from the ssh key info in users directory, for use in - # updating authorized_keys file. Additionally, the 'monitor' public key is - # included, which is used by the monitor nodes to run particular commands - # remotely. - # - def authorized_keys - hash = {} - keys = Dir.glob(Path.named_path([:user_ssh, '*'])) - keys.sort.each do |keyfile| - ssh_type, ssh_key = File.read(keyfile, :encoding => 'UTF-8').strip.split(" ") - name = File.basename(File.dirname(keyfile)) - until hash[name].nil? - i ||= 1; name = "#{name}#{i+=1}" - end - hash[name] = { - "type" => ssh_type, - "key" => ssh_key - } - end - ssh_type, ssh_key = File.read(Path.named_path(:monitor_pub_key), :encoding => 'UTF-8').strip.split(" ") - hash[Leap::Platform.monitor_username] = { - "type" => ssh_type, - "key" => ssh_key - } - hash - end - - def assert(assertion) - if instance_eval(assertion) - true - else - raise AssertionFailed.new(assertion), assertion, caller - end - end - - def error(msg) - raise ConfigError.new(@node, msg), msg, caller - end - - # - # applies a JSON partial to this node - # - def apply_partial(partial_path) - manager.partials(partial_path).each do |partial_data| - self.deep_merge!(partial_data) - end - end - - # - # If at first you don't succeed, then it is time to give up. - # - # try{} returns nil if anything in the block throws an exception. - # - # You can wrap something that might fail in `try`, like so. - # - # "= try{ nodes[:services => 'tor'].first.ip_address } " - # - def try(&block) - yield - rescue NoMethodError - rescue ArgumentError - nil - end - - protected - - # - # returns a node list, if argument is not already one - # - def listify(node_list) - if node_list.is_a? Config::ObjectList - node_list - elsif node_list.is_a? Config::Object - Config::ObjectList.new(node_list) - else - raise ArgumentError, 'argument must be a node or node list, not a `%s`' % node_list.class, caller - end - end - - end -end diff --git a/provider_base/lib/macros/files.rb b/provider_base/lib/macros/files.rb deleted file mode 100644 index 958958bc..00000000 --- a/provider_base/lib/macros/files.rb +++ /dev/null @@ -1,89 +0,0 @@ -# encoding: utf-8 - -## -## FILES -## - -module LeapCli - module Macro - - # - # inserts the contents of a file - # - def file(filename, options={}) - if filename.is_a? Symbol - filename = [filename, @node.name] - end - filepath = Path.find_file(filename) - if filepath - if filepath =~ /\.erb$/ - ERB.new(File.read(filepath, :encoding => 'UTF-8'), nil, '%<>').result(binding) - else - File.read(filepath, :encoding => 'UTF-8') - end - else - raise FileMissing.new(Path.named_path(filename), options) - "" - end - end - - # - # like #file, but allow missing files - # - def try_file(filename) - return file(filename) - rescue FileMissing - return nil - end - - # - # returns what the file path will be, once the file is rsynced to the server. - # an internal list of discovered file paths is saved, in order to rsync these files when needed. - # - # notes: - # - # * argument 'path' is relative to Path.provider/files or Path.provider_base/files - # * the path returned by this method is absolute - # * the path stored for use later by rsync is relative to Path.provider - # * if the path does not exist locally, but exists in provider_base, then the default file from - # provider_base is copied locally. this is required for rsync to work correctly. - # - def file_path(path, options={}) - if path.is_a? Symbol - path = [path, @node.name] - elsif path.is_a? String - # ensure it prefixed with files/ - unless path =~ /^files\// - path = "files/" + path - end - end - actual_path = Path.find_file(path) - if actual_path.nil? - if options[:missing] - raise FileMissing.new(Path.named_path(path), options) - else - Util::log 2, :skipping, "file_path(\"#{path}\") because there is no such file." - end - nil - else - if actual_path =~ /^#{Regexp.escape(Path.provider_base)}/ - # if file is under Path.provider_base, we must copy the default file to - # to Path.provider in order for rsync to be able to sync the file. - local_provider_path = actual_path.sub(/^#{Regexp.escape(Path.provider_base)}/, Path.provider) - FileUtils.mkdir_p File.dirname(local_provider_path), :mode => 0700 - FileUtils.install actual_path, local_provider_path, :mode => 0600 - Util.log :created, Path.relative_path(local_provider_path) - actual_path = local_provider_path - end - if File.directory?(actual_path) && actual_path !~ /\/$/ - actual_path += '/' # ensure directories end with /, important for building rsync command - end - relative_path = Path.relative_path(actual_path) - relative_path.sub!(/^files\//, '') # remove "files/" prefix - @node.file_paths << relative_path - File.join(Leap::Platform.files_dir, relative_path) - end - end - - end -end \ No newline at end of file diff --git a/provider_base/lib/macros/haproxy.rb b/provider_base/lib/macros/haproxy.rb deleted file mode 100644 index 602ae726..00000000 --- a/provider_base/lib/macros/haproxy.rb +++ /dev/null @@ -1,73 +0,0 @@ -# encoding: utf-8 - -## -## HAPROXY -## - -module LeapCli - module Macro - - # - # creates a hash suitable for configuring haproxy. the key is the node name of the server we are proxying to. - # - # * node_list - a hash of nodes for the haproxy servers - # * stunnel_client - contains the mappings to local ports for each server node. - # * non_stunnel_port - in case self is included in node_list, the port to connect to. - # - # 1000 weight is used for nodes in the same location. - # 100 otherwise. - # - def haproxy_servers(node_list, stunnel_clients, non_stunnel_port=nil) - default_weight = 10 - local_weight = 100 - - # record the hosts_file - hostnames(node_list) - - # create a simple map for node name -> local stunnel accept port - accept_ports = stunnel_clients.inject({}) do |hsh, stunnel_entry| - name = stunnel_entry.first.sub /_[0-9]+$/, '' - hsh[name] = stunnel_entry.last['accept_port'] - hsh - end - - # if one the nodes in the node list is ourself, then there will not be a stunnel to it, - # but we need to include it anyway in the haproxy config. - if node_list[self.name] && non_stunnel_port - accept_ports[self.name] = non_stunnel_port - end - - # create the first pass of the servers hash - servers = node_list.values.inject(Config::ObjectList.new) do |hsh, node| - # make sure we have a port to talk to - unless accept_ports[node.name] - error "haproxy needs a local port to talk to when connecting to #{node.name}" - end - weight = default_weight - try { - weight = local_weight if self.location.name == node.location.name - } - hsh[node.name] = Config::Object[ - 'backup', false, - 'host', 'localhost', - 'port', accept_ports[node.name], - 'weight', weight - ] - if node.services.include?('couchdb') - hsh[node.name]['writable'] = node.couch.mode != 'mirror' - end - hsh - end - - # if there are some local servers, make the others backup - if servers.detect{|k,v| v.weight == local_weight} - servers.each do |k,server| - server['backup'] = server['weight'] == default_weight - end - end - - return servers - end - - end -end diff --git a/provider_base/lib/macros/hosts.rb b/provider_base/lib/macros/hosts.rb deleted file mode 100644 index 8281329f..00000000 --- a/provider_base/lib/macros/hosts.rb +++ /dev/null @@ -1,68 +0,0 @@ -# encoding: utf-8 - -module LeapCli - module Macro - - ## - ## HOSTS - ## - - # - # records the list of hosts that are encountered for this node - # - def hostnames(nodes) - @referenced_nodes ||= Config::ObjectList.new - nodes = listify(nodes) - nodes.each_node do |node| - @referenced_nodes[node.name] ||= node - end - return nodes.values.collect {|node| node.domain.name} - end - - # - # Generates entries needed for updating /etc/hosts on a node (as a hash). - # - # Argument `nodes` can be nil or a list of nodes. If nil, only include the - # IPs of the other nodes this @node as has encountered (plus all mx nodes). - # - # Also, for virtual machines, we use the local address if this @node is in - # the same location as the node in question. - # - # We include the ssh public key for each host, so that the hash can also - # be used to generate the /etc/ssh/known_hosts - # - def hosts_file(nodes=nil) - if nodes.nil? - if @referenced_nodes && @referenced_nodes.any? - nodes = @referenced_nodes - nodes = nodes.merge(nodes_like_me[:services => 'mx']) # all nodes always need to communicate with mx nodes. - end - end - return {} unless nodes - hosts = {} - my_location = @node['location'] ? @node['location']['name'] : nil - nodes.each_node do |node| - hosts[node.name] = { - 'ip_address' => node.ip_address, - 'domain_internal' => node.domain.internal, - 'domain_full' => node.domain.full, - 'port' => node.ssh.port - } - node_location = node['location'] ? node['location']['name'] : nil - if my_location == node_location - if facts = @node.manager.facts[node.name] - if facts['ec2_public_ipv4'] - hosts[node.name]['ip_address'] = facts['ec2_public_ipv4'] - end - end - end - host_pub_key = Util::read_file([:node_ssh_pub_key,node.name]) - if host_pub_key - hosts[node.name]['host_pub_key'] = host_pub_key - end - end - hosts - end - - end -end \ No newline at end of file diff --git a/provider_base/lib/macros/keys.rb b/provider_base/lib/macros/keys.rb deleted file mode 100644 index 0ed7ccd0..00000000 --- a/provider_base/lib/macros/keys.rb +++ /dev/null @@ -1,83 +0,0 @@ -# encoding: utf-8 - -# -# Macro for dealing with cryptographic keys -# - -module LeapCli - module Macro - - # - # return the path to the tor public key - # generating key if it is missing - # - def tor_public_key_path(path_name, key_type) - path = file_path(path_name) - if path.nil? - generate_tor_key(key_type) - file_path(path_name) - else - path - end - end - - # - # return the path to the tor private key - # generating key if it is missing - # - def tor_private_key_path(path_name, key_type) - path = file_path(path_name) - if path.nil? - generate_tor_key(key_type) - file_path(path_name) - else - path - end - end - - # - # Generates a onion_address from a public RSA key file. - # - # path_name is the named path of the Tor public key. - # - # Basically, an onion address is nothing more than a base32 encoding - # of the first 10 bytes of a sha1 digest of the public key. - # - # Additionally, Tor ignores the 22 byte header of the public key - # before taking the sha1 digest. - # - def onion_address(path_name) - require 'base32' - require 'base64' - require 'openssl' - path = Path.find_file([path_name, self.name]) - if path && File.exists?(path) - public_key_str = File.readlines(path).grep(/^[^-]/).join - public_key = Base64.decode64(public_key_str) - public_key = public_key.slice(22..-1) # Tor ignores the 22 byte SPKI header - sha1sum = Digest::SHA1.new.digest(public_key) - Base32.encode(sha1sum.slice(0,10)).downcase - else - LeapCli.log :warning, 'Tor public key file "%s" does not exist' % tor_public_key_path - end - end - - private - - def generate_tor_key(key_type) - if key_type == 'RSA' - require 'certificate_authority' - keypair = CertificateAuthority::MemoryKeyMaterial.new - bit_size = 1024 - LeapCli.log :generating, "%s bit RSA Tor key" % bit_size do - keypair.generate_key(bit_size) - LeapCli::Util.write_file! [:node_tor_priv_key, self.name], keypair.private_key.to_pem - LeapCli::Util.write_file! [:node_tor_pub_key, self.name], keypair.public_key.to_pem - end - else - LeapCli.bail! 'tor.key.type of %s is not yet supported' % key_type - end - end - - end -end diff --git a/provider_base/lib/macros/nodes.rb b/provider_base/lib/macros/nodes.rb deleted file mode 100644 index 8b961cbc..00000000 --- a/provider_base/lib/macros/nodes.rb +++ /dev/null @@ -1,88 +0,0 @@ -# encoding: utf-8 - -## -## node related macros -## - -module LeapCli - module Macro - - # - # the list of all the nodes - # - def nodes - global.nodes - end - - # - # simple alias for global.provider - # - def provider - global.provider - end - - # - # returns a list of nodes that match the same environment - # - # if @node.environment is not set, we return other nodes - # where environment is not set. - # - def nodes_like_me - nodes[:environment => @node.environment] - end - - # - # returns a list of nodes that match the location name - # and environment of @node. - # - def nodes_near_me - if @node['location'] && @node['location']['name'] - nodes_like_me['location.name' => @node.location.name] - else - nodes_like_me['location' => nil] - end - end - - # - # - # picks a node out from the node list in such a way that: - # - # (1) which nodes picked which nodes is saved in secrets.json - # (2) when other nodes call this macro with the same node list, they are guaranteed to get a different node - # (3) if all the nodes in the pick_node list have been picked, remaining nodes are distributed randomly. - # - # if the node_list is empty, an exception is raised. - # if node_list size is 1, then that node is returned and nothing is - # memorized via the secrets.json file. - # - # `label` is needed to distinguish between pools of nodes for different purposes. - # - # TODO: more evenly balance after all the nodes have been picked. - # - def pick_node(label, node_list) - if node_list.any? - if node_list.size == 1 - return node_list.values.first - else - secrets_key = "pick_node(:#{label},#{node_list.keys.sort.join(',')})" - secrets_value = @manager.secrets.retrieve(secrets_key, @node.environment) || {} - secrets_value[@node.name] ||= begin - node_to_pick = nil - node_list.each_node do |node| - next if secrets_value.values.include?(node.name) - node_to_pick = node.name - end - node_to_pick ||= secrets_value.values.shuffle.first # all picked already, so pick a random one. - node_to_pick - end - picked_node_name = secrets_value[@node.name] - @manager.secrets.set(secrets_key, secrets_value, @node.environment) - return node_list[picked_node_name] - end - else - raise ArgumentError.new('pick_node(node_list): node_list cannot be empty') - end - end - - end -end \ No newline at end of file diff --git a/provider_base/lib/macros/provider.rb b/provider_base/lib/macros/provider.rb deleted file mode 100644 index 84c4e1b8..00000000 --- a/provider_base/lib/macros/provider.rb +++ /dev/null @@ -1,20 +0,0 @@ -# -# These macros are intended only for use in provider.json, although they are -# currently loaded in all .json contexts. -# - -module LeapCli - module Macro - - # - # returns an array of the service names, including only those services that - # are enabled for this environment. - # - def enabled_services - manager.env(self.environment).services[:service_type => :user_service].field(:name).select { |service| - manager.nodes[:environment => self.environment][:services => service].any? - } - end - - end -end diff --git a/provider_base/lib/macros/secrets.rb b/provider_base/lib/macros/secrets.rb deleted file mode 100644 index 8d1feb55..00000000 --- a/provider_base/lib/macros/secrets.rb +++ /dev/null @@ -1,39 +0,0 @@ -# encoding: utf-8 - -require 'base32' - -module LeapCli - module Macro - - # - # inserts a named secret, generating it if needed. - # - # manager.export_secrets should be called later to capture any newly generated secrets. - # - # +length+ is the character length of the generated password. - # - def secret(name, length=32) - manager.secrets.set(name, @node.environment) { Util::Secret.generate(length) } - end - - # inserts a base32 encoded secret - def base32_secret(name, length=20) - manager.secrets.set(name, @node.environment) { Base32.encode(Util::Secret.generate(length)) } - end - - # Picks a random obfsproxy port from given range - def rand_range(name, range) - manager.secrets.set(name, @node.environment) { rand(range) } - end - - # - # inserts an hexidecimal secret string, generating it if needed. - # - # +bit_length+ is the bits in the secret, (ie length of resulting hex string will be bit_length/4) - # - def hex_secret(name, bit_length=128) - manager.secrets.set(name, @node.environment) { Util::Secret.generate_hex(bit_length) } - end - - end -end \ No newline at end of file diff --git a/provider_base/lib/macros/stunnel.rb b/provider_base/lib/macros/stunnel.rb deleted file mode 100644 index f16308c7..00000000 --- a/provider_base/lib/macros/stunnel.rb +++ /dev/null @@ -1,95 +0,0 @@ -## -## STUNNEL -## - -# -# About stunnel -# -------------------------- -# -# The network looks like this: -# -# From the client's perspective: -# -# |------- stunnel client --------------| |---------- stunnel server -----------------------| -# consumer app -> localhost:accept_port -> connect:connect_port -> ?? -# -# From the server's perspective: -# -# |------- stunnel client --------------| |---------- stunnel server -----------------------| -# ?? -> *:accept_port -> localhost:connect_port -> service -# - -module LeapCli - module Macro - - # - # stunnel configuration for the client side. - # - # +node_list+ is a ObjectList of nodes running stunnel servers. - # - # +port+ is the real port of the ultimate service running on the servers - # that the client wants to connect to. - # - # * accept_port is the port on localhost to which local clients - # can connect. it is auto generated serially. - # - # * connect_port is the port on the stunnel server to connect to. - # it is auto generated from the +port+ argument. - # - # generates an entry appropriate to be passed directly to - # create_resources(stunnel::service, hiera('..'), defaults) - # - # local ports are automatically generated, starting at 4000 - # and incrementing in sorted order (by node name). - # - def stunnel_client(node_list, port, options={}) - @next_stunnel_port ||= 4000 - node_list = listify(node_list) - hostnames(node_list) # record the hosts - result = Config::ObjectList.new - node_list.each_node do |node| - if node.name != self.name || options[:include_self] - result["#{node.name}_#{port}"] = Config::Object[ - 'accept_port', @next_stunnel_port, - 'connect', node.domain.internal, - 'connect_port', stunnel_port(port), - 'original_port', port - ] - @next_stunnel_port += 1 - end - end - result - end - - # - # generates a stunnel server entry. - # - # +port+ is the real port targeted service. - # - # * `accept_port` is the publicly bound port - # * `connect_port` is the port that the local service is running on. - # - def stunnel_server(port) - { - "accept_port" => stunnel_port(port), - "connect_port" => port - } - end - - private - - # - # maps a real port to a stunnel port (used as the connect_port in the client config - # and the accept_port in the server config) - # - def stunnel_port(port) - port = port.to_i - if port < 50000 - return port + 10000 - else - return port - 10000 - end - end - - end -end \ No newline at end of file -- cgit v1.2.3 From 1b4c55b308eda73504769c38972da9b2f1295e99 Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 19 Aug 2015 17:40:46 -0700 Subject: fix vagrant key path --- lib/leap_cli/commands/vagrant.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/leap_cli/commands/vagrant.rb b/lib/leap_cli/commands/vagrant.rb index 27c739b1..1561a658 100644 --- a/lib/leap_cli/commands/vagrant.rb +++ b/lib/leap_cli/commands/vagrant.rb @@ -79,7 +79,7 @@ module LeapCli; module Commands # we need to make sure that it owned by us and not world readable. # def vagrant_ssh_key_file - file_path = File.expand_path('../../../vendor/vagrant_ssh_keys/vagrant.key', File.dirname(__FILE__)) + file_path = Path.vagrant_ssh_key_file Util.assert_files_exist! file_path uid = File.new(file_path).stat.uid if uid == 0 || uid == Process.euid -- cgit v1.2.3 From 3f1c9f42cd93f6ef099124b33663fd9f288eb430 Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 19 Aug 2015 17:41:45 -0700 Subject: allow ca_cert_uri to be configured --- provider_base/files/service-definitions/provider.json.erb | 2 +- provider_base/services/webapp.json | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/provider_base/files/service-definitions/provider.json.erb b/provider_base/files/service-definitions/provider.json.erb index e51f9c3a..a75bea61 100644 --- a/provider_base/files/service-definitions/provider.json.erb +++ b/provider_base/files/service-definitions/provider.json.erb @@ -9,7 +9,7 @@ hsh['api_version'] = "1" hsh['api_uri'] = ["https://", api.domain, ':', api.port].join - hsh['ca_cert_uri'] = 'https://' + webapp.domain + '/ca.crt' + hsh['ca_cert_uri'] = api.ca_cert_uri hsh['ca_cert_fingerprint'] = fingerprint(:ca_cert) hsh.dump_json diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index 081b3718..7b1ca8fb 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -55,7 +55,8 @@ "service_type": "public_service", "api": { "domain": "= 'api.' + webapp.domain", - "port": 4430 + "port": 4430, + "ca_cert_uri": "= 'https://' + webapp.domain + '/ca.crt'" }, "nickserver": { "domain": "= 'nicknym.' + domain.full_suffix", -- cgit v1.2.3 From f232e63b022933a57ac8e86b23d8df6b6bb86930 Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 19 Aug 2015 17:42:37 -0700 Subject: automatically regenerate certs if the ca changes --- lib/leap_cli/commands/ca.rb | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/leap_cli/commands/ca.rb b/lib/leap_cli/commands/ca.rb index d5c6240d..3a0786e2 100644 --- a/lib/leap_cli/commands/ca.rb +++ b/lib/leap_cli/commands/ca.rb @@ -215,6 +215,10 @@ module LeapCli; module Commands return true else cert = load_certificate_file([:node_x509_cert, node.name]) + if !created_by_authority?(cert, ca_root) + log :updating, "cert for node '#{node.name}' because it was signed by an old CA root cert." + return true + end if cert.not_after < Time.now.advance(:months => 2) log :updating, "cert for node '#{node.name}' because it will expire soon" return true @@ -246,6 +250,25 @@ module LeapCli; module Commands return false end + def created_by_authority?(cert, ca) + authority_key_id = cert.extensions["authorityKeyIdentifier"].identifier.sub(/^keyid:/, '') + authority_key_id == public_key_id_for_ca(ca) + end + + # calculate the "key id" for a root CA, that matches the value + # Authority Key Identifier in the x509 extensions of a cert. + def public_key_id_for_ca(ca_cert) + @ca_key_ids ||= {} + @ca_key_ids[ca_cert.object_id] ||= begin + pubkey = ca_cert.key_material.public_key + seq = OpenSSL::ASN1::Sequence([ + OpenSSL::ASN1::Integer.new(pubkey.n), + OpenSSL::ASN1::Integer.new(pubkey.e) + ]) + Digest::SHA1.hexdigest(seq.to_der).upcase.scan(/../).join(':') + end + end + def warn_if_commercial_cert_will_soon_expire(node) dns_names_for_node(node).each do |domain| if file_exists?([:commercial_cert, domain]) -- cgit v1.2.3 From b5fbda1ca3832043e1636ee964a806ff222cb05f Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 21 Aug 2015 17:13:34 -0700 Subject: add support for configurable mail alias maps --- puppet/modules/postfix | 2 +- puppet/modules/site_postfix/manifests/mx.pp | 3 +- .../site_postfix/manifests/mx/reserved_aliases.pp | 15 ------ .../site_postfix/manifests/mx/static_aliases.pp | 58 ++++++++++++++++++++++ .../site_postfix/templates/custom-aliases.erb | 11 ++++ 5 files changed, 72 insertions(+), 17 deletions(-) delete mode 100644 puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp create mode 100644 puppet/modules/site_postfix/manifests/mx/static_aliases.pp create mode 100644 puppet/modules/site_postfix/templates/custom-aliases.erb diff --git a/puppet/modules/postfix b/puppet/modules/postfix index f09cd0ef..53572a89 160000 --- a/puppet/modules/postfix +++ b/puppet/modules/postfix @@ -1 +1 @@ -Subproject commit f09cd0eff2bcab7e12c09ec67be3c918bc83fac5 +Subproject commit 53572a8934fe5b0a3a567cdec10664f288923739 diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp index af0f9f56..334d04d0 100644 --- a/puppet/modules/site_postfix/manifests/mx.pp +++ b/puppet/modules/site_postfix/manifests/mx.pp @@ -51,7 +51,7 @@ class site_postfix::mx { include site_postfix::mx::checks include site_postfix::mx::smtp_tls include site_postfix::mx::smtpd_tls - include site_postfix::mx::reserved_aliases + include site_postfix::mx::static_aliases # greater verbosity for debugging, take out for production #include site_postfix::debug @@ -68,6 +68,7 @@ class site_postfix::mx { preseed => true, root_mail_recipient => $root_mail_recipient, smtp_listen => 'all', + default_alias_maps => false, mastercf_tail => "smtps inet n - - - - smtpd -o smtpd_tls_wrappermode=yes diff --git a/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp b/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp deleted file mode 100644 index 83e27376..00000000 --- a/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp +++ /dev/null @@ -1,15 +0,0 @@ -# Defines which mail addresses shouldn't be available and where they should fwd -class site_postfix::mx::reserved_aliases { - - postfix::mailalias { - [ 'abuse', 'admin', 'arin-admin', 'administrator', 'bin', 'cron', - 'certmaster', 'domainadmin', 'games', 'ftp', 'hostmaster', 'lp', - 'maildrop', 'mysql', 'news', 'nobody', 'noc', 'postmaster', 'postgresql', - 'security', 'ssladmin', 'sys', 'usenet', 'uucp', 'webmaster', 'www', - 'www-data', - ]: - ensure => present, - recipient => 'root' - } - -} diff --git a/puppet/modules/site_postfix/manifests/mx/static_aliases.pp b/puppet/modules/site_postfix/manifests/mx/static_aliases.pp new file mode 100644 index 00000000..786d74c1 --- /dev/null +++ b/puppet/modules/site_postfix/manifests/mx/static_aliases.pp @@ -0,0 +1,58 @@ +# +# Defines static, hard coded aliases that are not in the database. +# + +class site_postfix::mx::static_aliases { + + $mx = hiera('mx') + $aliases = $mx['aliases'] + + # + # Predefined aliases. + # + # Defines which mail addresses shouldn't be available and where they should + # fwd + # + # TODO: reconcile this with the node property webapp.forbidden_usernames + # + # NOTE: if you remove one of these, they will still appear in the + # /etc/aliases file + # + postfix::mailalias { + [ 'abuse', 'admin', 'arin-admin', 'administrator', 'bin', 'cron', + 'certmaster', 'domainadmin', 'games', 'ftp', 'hostmaster', 'lp', + 'maildrop', 'mysql', 'news', 'nobody', 'noc', 'postmaster', 'postgresql', + 'security', 'ssladmin', 'sys', 'usenet', 'uucp', 'webmaster', 'www', + 'www-data', + ]: + ensure => present, + recipient => 'root' + } + + # + # Custom aliases. + # + # This does not use the puppet mailalias resource because we want to be able + # to guarantee the contents of the alias file. This is needed so if you + # remove an alias from the node's config, it will get removed from the alias + # file. + # + + # both alias files must be listed under "alias_database", because once you + # specify one, then `newaliases` no longer will default to updating + # "/etc/aliases.db". + postfix::config { + 'alias_database': + value => "/etc/aliases, /etc/postfix/custom-aliases"; + 'alias_maps': + value => "hash:/etc/aliases, hash:/etc/postfix/custom-aliases"; + } + + file { '/etc/postfix/custom-aliases': + content => template('site_postfix/custom-aliases.erb'), + owner => root, + group => root, + mode => 0600, + notify => Exec['newaliases'] + } +} diff --git a/puppet/modules/site_postfix/templates/custom-aliases.erb b/puppet/modules/site_postfix/templates/custom-aliases.erb new file mode 100644 index 00000000..f261514b --- /dev/null +++ b/puppet/modules/site_postfix/templates/custom-aliases.erb @@ -0,0 +1,11 @@ +# +# This file is managed by puppet. +# +# This is a map of custom, non-standard aliases. The contents of this file +# are derived from the node property `mx.aliases`. +# + +<%- @aliases.keys.sort.each do |from| -%> +"<%= from %>": "<%= [@aliases[from]].flatten.join('", "') %>" +<%- end -%> + -- cgit v1.2.3 From 65c912a7327505b2c99632afa06bba3c6fd4a057 Mon Sep 17 00:00:00 2001 From: varac Date: Thu, 27 Aug 2015 14:08:38 +0200 Subject: updated nagios submodule Change-Id: Iae76f9ca03baf459ae8ea044ea6aecfc73a41b3a --- puppet/modules/nagios | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/nagios b/puppet/modules/nagios index b55f23d4..6c3ca97f 160000 --- a/puppet/modules/nagios +++ b/puppet/modules/nagios @@ -1 +1 @@ -Subproject commit b55f23d4d90c97cec08251544aa9700df86ad0b3 +Subproject commit 6c3ca97f1524e2b6242c27a2c97dbfb78105889f -- cgit v1.2.3 From 4f42910a6792ec3016aa6f9d0792801f75972a62 Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 31 Aug 2015 12:22:20 -0700 Subject: mx: added mx.key_lookup_domain property --- provider_base/services/mx.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json index db2e4795..4d1b3dad 100644 --- a/provider_base/services/mx.json +++ b/provider_base/services/mx.json @@ -1,4 +1,11 @@ { + "mx": { + // provider should define their own custom aliases. + // these are in *addition* to the standard reserved aliases for root and postmaster, etc. + "aliases": {}, + // this is the domain that is used for the OpenPGP header + "key_lookup_domain": "= global.services[:webapp].webapp.domain" + }, "stunnel": { "clients": { "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" -- cgit v1.2.3 From da53a4a723cc05cfa39e066c64a9467d7efad04b Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 3 Sep 2015 20:18:12 -0700 Subject: service definition .json files should not refer to properties inherited from common.json. closes #7423 --- provider_base/services/webapp.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index 7b1ca8fb..dbcfe0a6 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -9,7 +9,7 @@ "owner", "owners", "postmaster", "reply", "robot", "ssladmin", "staff", "support", "tech-support", "tech_support", "techsupport", "ticket", "tickets", "vmail", "www-data"], - "domain": "= domain.full_suffix", + "domain": "= provider.domain", "modules": ["user", "billing", "help"], "couchdb_webapp_user": "= global.services[:couchdb].couch.users[:webapp]", "couchdb_admin_user": "= global.services[:couchdb].couch.users[:admin]", -- cgit v1.2.3 From 9d645a82c7346e8d585c664a82c719647a0d2ffa Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 3 Sep 2015 23:24:43 -0700 Subject: make couchdb.admin.yml only readable by root, make non-admin cron run as webapp user. --- puppet/modules/site_webapp/manifests/couchdb.pp | 16 ++++++++-------- puppet/modules/site_webapp/manifests/cron.pp | 4 ++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/puppet/modules/site_webapp/manifests/couchdb.pp b/puppet/modules/site_webapp/manifests/couchdb.pp index 1dbc745d..5cf7f953 100644 --- a/puppet/modules/site_webapp/manifests/couchdb.pp +++ b/puppet/modules/site_webapp/manifests/couchdb.pp @@ -14,29 +14,29 @@ class site_webapp::couchdb { file { '/srv/leap/webapp/config/couchdb.yml': content => template('site_webapp/couchdb.yml.erb'), - owner => leap-webapp, - group => leap-webapp, + owner => 'leap-webapp', + group => 'leap-webapp', mode => '0600', require => Vcsrepo['/srv/leap/webapp']; '/srv/leap/webapp/config/couchdb.admin.yml': content => template('site_webapp/couchdb.admin.yml.erb'), - owner => leap-webapp, - group => leap-webapp, + owner => 'root', + group => 'root', mode => '0600', require => Vcsrepo['/srv/leap/webapp']; '/srv/leap/webapp/log': ensure => directory, - owner => leap-webapp, - group => leap-webapp, + owner => 'leap-webapp', + group => 'leap-webapp', mode => '0755', require => Vcsrepo['/srv/leap/webapp']; '/srv/leap/webapp/log/production.log': ensure => present, - owner => leap-webapp, - group => leap-webapp, + owner => 'leap-webapp', + group => 'leap-webapp', mode => '0666', require => Vcsrepo['/srv/leap/webapp']; } diff --git a/puppet/modules/site_webapp/manifests/cron.pp b/puppet/modules/site_webapp/manifests/cron.pp index d26ee312..7147a0d2 100644 --- a/puppet/modules/site_webapp/manifests/cron.pp +++ b/puppet/modules/site_webapp/manifests/cron.pp @@ -5,12 +5,14 @@ class site_webapp::cron { 'rotate_databases': command => 'cd /srv/leap/webapp && bundle exec rake db:rotate', environment => 'RAILS_ENV=production', + user => 'root', hour => [0,6,12,18], minute => 0; 'delete_tmp_databases': command => 'cd /srv/leap/webapp && bundle exec rake db:deletetmp', environment => 'RAILS_ENV=production', + user => 'root', hour => 1, minute => 1; @@ -19,6 +21,7 @@ class site_webapp::cron { 'remove_expired_sessions': command => 'cd /srv/leap/webapp && bundle exec rake cleanup:sessions', environment => 'RAILS_ENV=production', + user => 'leap-webapp', hour => 2, minute => 30, ensure => absent; @@ -26,6 +29,7 @@ class site_webapp::cron { 'remove_expired_tokens': command => 'cd /srv/leap/webapp && bundle exec rake cleanup:tokens', environment => 'RAILS_ENV=production', + user => 'leap-webapp', hour => 3, minute => 0; } -- cgit v1.2.3 From ffd340e7b014bc9f35fb6f9365230d483650cc1d Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Thu, 3 Sep 2015 13:03:01 -0400 Subject: rewrite openpgp header to be always correct (#7413) The openpgp header added by the client is sometimes incorrect, because the client doesn't actually know what the proper URL is for the webapp. The server knows, however. Change-Id: I2243b19a6337d8e0be97590e2ca9c9c0b0fffdac --- puppet/modules/site_postfix/manifests/mx.pp | 6 +++++- .../site_postfix/manifests/mx/rewrite_openpgp_header.pp | 11 +++++++++++ .../templates/checks/rewrite_openpgp_headers.erb | 13 +++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 puppet/modules/site_postfix/manifests/mx/rewrite_openpgp_header.pp create mode 100644 puppet/modules/site_postfix/templates/checks/rewrite_openpgp_headers.erb diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp index 334d04d0..2b311e06 100644 --- a/puppet/modules/site_postfix/manifests/mx.pp +++ b/puppet/modules/site_postfix/manifests/mx.pp @@ -52,6 +52,7 @@ class site_postfix::mx { include site_postfix::mx::smtp_tls include site_postfix::mx::smtpd_tls include site_postfix::mx::static_aliases + include site_postfix::mx::rewrite_openpgp_header # greater verbosity for debugging, take out for production #include site_postfix::debug @@ -74,7 +75,10 @@ class site_postfix::mx { -o smtpd_tls_wrappermode=yes -o smtpd_tls_security_level=encrypt -o smtpd_recipient_restrictions=\$smtps_recipient_restrictions - -o smtpd_helo_restrictions=\$smtps_helo_restrictions", + -o smtpd_helo_restrictions=\$smtps_helo_restrictions + -o cleanup_service_name=clean_smtps +clean_smtps unix n - n - 0 cleanup + -o header_checks=pcre:/etc/postfix/checks/rewrite_openpgp_headers", require => [ Class['Site_config::X509::Key'], Class['Site_config::X509::Cert'], diff --git a/puppet/modules/site_postfix/manifests/mx/rewrite_openpgp_header.pp b/puppet/modules/site_postfix/manifests/mx/rewrite_openpgp_header.pp new file mode 100644 index 00000000..71f945b8 --- /dev/null +++ b/puppet/modules/site_postfix/manifests/mx/rewrite_openpgp_header.pp @@ -0,0 +1,11 @@ +class site_postfix::mx::rewrite_openpgp_header { + $mx = hiera('mx') + $correct_domain = $mx['key_lookup_domain'] + + file { '/etc/postfix/checks/rewrite_openpgp_headers': + content => template('site_postfix/checks/rewrite_openpgp_headers.erb'), + mode => '0644', + owner => root, + group => root; + } +} diff --git a/puppet/modules/site_postfix/templates/checks/rewrite_openpgp_headers.erb b/puppet/modules/site_postfix/templates/checks/rewrite_openpgp_headers.erb new file mode 100644 index 00000000..7af14f7d --- /dev/null +++ b/puppet/modules/site_postfix/templates/checks/rewrite_openpgp_headers.erb @@ -0,0 +1,13 @@ +# THIS FILE IS MANAGED BY PUPPET +# +# This will replace the OpenPGP header that the client adds, because it is +# sometimes incorrect (due to the client not always knowing what the proper URL +# is for the webapp). +# e.g. This will rewrite this header: +# OpenPGP: id=4C0E01CD50E2F653; url="https://leap.se/key/elijah"; preference="signencrypt +# with this replacement: +# OpenPGP: id=4C0E01CD50E2F653; url="https://user.leap.se/key/elijah"; preference="signencrypt +# +# Note: whitespace in the pattern is represented by [[:space:]] to avoid these warnings from postmap: +# "record is in "key: value" format; is this an alias file?" and "duplicate entry" +/^(OpenPGP:[[:space:]]id=[[:alnum:]]+;[[:space:]]url="https:\/\/)<%= @domain %>(\/key\/[[:alpha:]]+";.*)/i REPLACE ${1}<%= @correct_domain %>${2} -- cgit v1.2.3 From 492c452801e2679d47b3b09505779a11ed3892cc Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 9 Sep 2015 12:16:36 -0700 Subject: updates to zone compile and tags/development.json to be compatible with the definition of 'domain' in provider.env.json. --- lib/leap_cli/commands/compile.rb | 13 +++++++------ provider_base/tags/development.json | 6 +----- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/lib/leap_cli/commands/compile.rb b/lib/leap_cli/commands/compile.rb index f5895b8b..8f6c7769 100644 --- a/lib/leap_cli/commands/compile.rb +++ b/lib/leap_cli/commands/compile.rb @@ -256,7 +256,7 @@ remove this directory if you don't use it. ## ZONE FILE ## - def relative_hostname(fqdn) + def relative_hostname(fqdn, provider) @domain_regexp ||= /\.?#{Regexp.escape(provider.domain)}$/ fqdn.sub(@domain_regexp, '') end @@ -265,10 +265,11 @@ remove this directory if you don't use it. # serial is any number less than 2^32 (4294967296) # def compile_zone_file + provider = manager.env('default').provider hosts_seen = {} f = $stdout f.puts ZONE_HEADER % {:domain => provider.domain, :ns => provider.domain, :contact => provider.contacts.default.first.sub('@','.')} - max_width = manager.nodes.values.inject(0) {|max, node| [max, relative_hostname(node.domain.full).length].max } + max_width = manager.nodes.values.inject(0) {|max, node| [max, relative_hostname(node.domain.full, provider).length].max } put_line = lambda do |host, line| host = '@' if host == '' f.puts("%-#{max_width}s %s" % [host, line]) @@ -297,18 +298,18 @@ remove this directory if you don't use it. f.puts ENV_HEADER % (env.nil? ? 'default' : env) nodes.each_node do |node| if node.dns.public - hostname = relative_hostname(node.domain.full) - put_line.call relative_hostname(node.domain.full), "IN A #{node.ip_address}" + hostname = relative_hostname(node.domain.full, provider) + put_line.call relative_hostname(node.domain.full, provider), "IN A #{node.ip_address}" end if node.dns['aliases'] node.dns.aliases.each do |host_alias| if host_alias != node.domain.full && host_alias != provider.domain - put_line.call relative_hostname(host_alias), "IN A #{node.ip_address}" + put_line.call relative_hostname(host_alias, provider), "IN A #{node.ip_address}" end end end if node.services.include? 'mx' - put_line.call relative_hostname(node.domain.full_suffix), "IN MX 10 #{relative_hostname(node.domain.full)}" + put_line.call relative_hostname(node.domain.full_suffix, provider), "IN MX 10 #{relative_hostname(node.domain.full, provider)}" end end end diff --git a/provider_base/tags/development.json b/provider_base/tags/development.json index d9c2c007..caf18e9d 100644 --- a/provider_base/tags/development.json +++ b/provider_base/tags/development.json @@ -1,7 +1,3 @@ { - "environment": "development", - "domain": { - "full_suffix": "= 'dev.' + provider.domain", - "internal_suffix": "= 'dev.' + provider.domain_internal" - } + "environment": "development" } \ No newline at end of file -- cgit v1.2.3 From f93e6904fe2abca3cfc7a93a08892dbdcea97327 Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 9 Sep 2015 18:05:19 -0700 Subject: ensure that the webapp has the service levels config it requires. --- lib/leap_cli/macros/provider.rb | 70 ++++++++++++++++++++++++++++++++++++++ provider_base/services/webapp.json | 2 +- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/lib/leap_cli/macros/provider.rb b/lib/leap_cli/macros/provider.rb index 84c4e1b8..4e74da01 100644 --- a/lib/leap_cli/macros/provider.rb +++ b/lib/leap_cli/macros/provider.rb @@ -16,5 +16,75 @@ module LeapCli } end + # + # The webapp will not work unless the service level configuration is precisely defined. + # Here, we take what the sysadmin has specified in provider.json and clean it up to + # ensure it is OK. + # + # It would be better to add support for JSON schema. + # + def service_levels() + levels = {} + provider.service.levels.each do |name, level| + if name =~ /^[0-9]+$/ + name = name.to_i + end + levels[name] = level_cleanup(name, level.clone) + end + levels + end + + private + + def print_warning(name, msg) + if self.environment + provider_str = "provider.json or %s" % ['provider', self.environment, 'json'].join('.') + else + provider_str = "provider.json" + end + LeapCli::log :warning, "In #{provider_str}, you have an incorrect definition for service level '#{name}':" do + LeapCli::log msg + end + end + + def level_cleanup(name, level) + unless level['name'] + print_warning(name, 'required field "name" is missing') + end + unless level['description'] + print_warning(name, 'required field "description" is missing') + end + unless level['bandwidth'].nil? || level['bandwidth'] == 'limited' + print_warning(name, 'field "bandwidth" must be nil or "limited"') + end + unless level['rate'].nil? || level['rate'].is_a?(Hash) + print_warning(name, 'field "rate" must be nil or a hash (e.g. {"USD":10, "EUR":10})') + end + possible_services = enabled_services + if level['services'] + level['services'].each do |service| + unless possible_services.include? service + print_warning(name, "the service '#{service}' does not exist or there are no nodes that provide this service.") + LeapCli::Util::bail! + end + end + else + level['services'] = possible_services + end + level['services'] = remap_services(level['services']) + level + end + + # + # the service names that the webapp uses and that leap_platform uses are different. ugh. + # + SERVICE_MAP = { + "mx" => "email", + "openvpn" => "eip" + } + def remap_services(services) + services.map {|srv| SERVICE_MAP[srv]} + end + end end diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index dbcfe0a6..039b1c0b 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -20,7 +20,7 @@ "allow_anonymous_certs": "= provider.service.allow_anonymous", "allow_registration": "= provider.service.allow_registration", "default_service_level": "= provider.service.default_service_level", - "service_levels": "= provider.service.levels", + "service_levels": "= service_levels()", "secret_token": "= secret :webapp_secret_token", "api_version": 1, "secure": false, -- cgit v1.2.3 From 36540162129243596a5ce1ecc00c999ba5ddc849 Mon Sep 17 00:00:00 2001 From: varac Date: Mon, 4 May 2015 20:09:40 +0200 Subject: moved leap_cli installation to leap module Change-Id: I385f7877d0816456e7c57179511604645a4740bc --- puppet/modules/leap/manifests/cli/install.pp | 33 ++++++++++++++++++++++++++++ vagrant/install-platform.pp | 32 +++++++++------------------ 2 files changed, 43 insertions(+), 22 deletions(-) create mode 100644 puppet/modules/leap/manifests/cli/install.pp diff --git a/puppet/modules/leap/manifests/cli/install.pp b/puppet/modules/leap/manifests/cli/install.pp new file mode 100644 index 00000000..858bd7da --- /dev/null +++ b/puppet/modules/leap/manifests/cli/install.pp @@ -0,0 +1,33 @@ +# installs leap_cli on node +class leap::cli::install ( $source = false ) { + if $source { + # needed for building leap_cli from source + include ::git + include ::site_config::ruby::dev + + vcsrepo { '/srv/leap/cli': + ensure => present, + force => true, + revision => 'develop', + provider => 'git', + source => 'https://leap.se/git/leap_cli.git', + owner => 'root', + group => 'root', + notify => Exec['install_leap_cli'], + require => Package['git'] + } + + exec { 'install_leap_cli': + command => '/usr/bin/rake build && /usr/bin/rake install', + cwd => '/srv/leap/cli', + refreshonly => true, + require => [ Package['ruby-dev'], File['/etc/gemrc'], Package['rake'] ] + } + } + else { + package { 'leap_cli': + ensure => installed, + provider => gem + } + } +} diff --git a/vagrant/install-platform.pp b/vagrant/install-platform.pp index 465ca78a..5ea834b1 100755 --- a/vagrant/install-platform.pp +++ b/vagrant/install-platform.pp @@ -3,34 +3,22 @@ File['/etc/apt/preferences'] -> Exec['refresh_apt'] -> Package <| ( title != 'lsb' ) |> -package { [ 'rsync', 'ruby-hiera-puppet', 'git', 'ruby1.9.1-dev', 'rake', 'jq' ]: - ensure => installed -} - -file { '/etc/gemrc': - content => "---\n:sources:\n - https://rubygems.org/" -} -vcsrepo { '/srv/leap/leap_cli': - ensure => present, - force => true, - revision => 'develop', - provider => 'git', - source => 'https://leap.se/git/leap_cli.git', - owner => 'root', - group => 'root', - notify => Exec['install_leap_cli'], - require => Package['git'] +if $::lsbdistcodename == 'wheezy' { + package { 'ruby-hiera-puppet': + ensure => installed + } } -exec { 'install_leap_cli': - command => '/usr/bin/rake build && /usr/bin/rake install', - cwd => '/srv/leap/leap_cli', - refreshonly => true, - require => [ Package['ruby1.9.1-dev'], File['/etc/gemrc'], Package['rake'] ] +# install leap_cli from source, so it will work with the develop +# branch of leap_platform +class { '::leap::cli::install': + source => true, } file { [ '/srv/leap', '/srv/leap/configuration', '/var/log/leap' ]: ensure => directory } +# install prerequisites for configuring the provider +include ::git -- cgit v1.2.3 From 0d1755e1dd05100128282ae8f31434795ce279d6 Mon Sep 17 00:00:00 2001 From: varac Date: Wed, 9 Sep 2015 11:54:28 +0200 Subject: use vagrant user for configuring provider with leap_cli (new leap_cli version complain when called by root) we don't need to enable ssh pw auth because we're now using the vagrant user that has ssh key-based auth configured already. Change-Id: I5e28e6f5c71724573ff11def5b96142e8eb8b185 --- vagrant/configure-leap.sh | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/vagrant/configure-leap.sh b/vagrant/configure-leap.sh index 9541e194..e0476739 100755 --- a/vagrant/configure-leap.sh +++ b/vagrant/configure-leap.sh @@ -1,13 +1,15 @@ #!/bin/bash -. /vagrant/vagrant/vagrant.config +. /vagrant/vagrant/vagrant.config #OPTS='--no-color' OPTS='' -PROVIDERDIR='/srv/leap/configuration' +USER='vagrant' NODE='node1' -LEAP='/usr/local/bin/leap' +SUDO="sudo -u ${USER}" +PROVIDERDIR="/home/${USER}/leap/configuration" +LEAP="$SUDO /usr/local/bin/leap" echo '===============================================' echo 'configuring leap' @@ -15,19 +17,22 @@ echo '===============================================' # purge $PROVIDERDIR so this script can be run multiple times [ -e $PROVIDERDIR ] && rm -rf $PROVIDERDIR -mkdir $PROVIDERDIR + +mkdir -p $PROVIDERDIR +chown ${USER}:${USER} ${PROVIDERDIR} cd $PROVIDERDIR $LEAP $OPTS new --contacts "$contacts" --domain "$provider_domain" --name "$provider_name" --platform=/vagrant . -echo -e '\n@log = "/var/log/leap/deploy.log"' >> Leapfile +$SUDO echo -e '\n@log = "/var/log/leap/deploy.log"' >> Leapfile -if [ ! -e /root/.ssh/id_rsa ]; then - ssh-keygen -f /root/.ssh/id_rsa -P '' - cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys +if [ ! -e /home/${USER}/.ssh/id_rsa ]; then + $SUDO ssh-keygen -f /home/${USER}/.ssh/id_rsa -P '' + cat /home/${USER}/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys fi -mkdir -p $PROVIDERDIR/files/nodes/$NODE +$SUDO mkdir -p ${PROVIDERDIR}/files/nodes/${NODE} sh -c "cat /etc/ssh/ssh_host_rsa_key.pub | cut -d' ' -f1,2 >> $PROVIDERDIR/files/nodes/$NODE/${NODE}_ssh.pub" +chown ${USER}:${USER} ${PROVIDERDIR}/files/nodes/${NODE}/${NODE}_ssh.pub $LEAP $OPTS add-user --self $LEAP $OPTS cert ca @@ -41,7 +46,7 @@ git init git add . git commit -m'configured provider' -$LEAP $OPTS node init $NODE +$LEAP $OPTS node init $NODE if [ $? -eq 1 ]; then echo 'node init failed' exit 1 @@ -69,9 +74,6 @@ echo 'setting node to demo-mode' echo '===============================================' postconf -e default_transport='error: in demo mode' -sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/g' /etc/ssh/sshd_config -/etc/init.d/ssh reload - # add users: testadmin and testuser with passwords "hallo123" curl -s -k https://localhost/1/users.json -d "user%5Blogin%5D=testuser&user%5Bpassword_salt%5D=7d4880237a038e0e&user%5Bpassword_verifier%5D=b98dc393afcd16e5a40fb57ce9cddfa6a978b84be326196627c111d426cada898cdaf3a6427e98b27daf4b0ed61d278bc856515aeceb2312e50c8f816659fcaa4460d839a1e2d7ffb867d32ac869962061368141c7571a53443d58dc84ca1fca34776894414c1090a93e296db6cef12c2cc3f7a991b05d49728ed358fd868286" curl -s -k https://localhost/1/users.json -d "user%5Blogin%5D=testadmin&user%5Bpassword_salt%5D=ece1c457014d8282&user%5Bpassword_verifier%5D=9654d93ab409edf4ff1543d07e08f321107c3fd00de05c646c637866a94f28b3eb263ea9129dacebb7291b3374cc6f0bf88eb3d231eb3a76eed330a0e8fd2a5c477ed2693694efc1cc23ae83c2ae351a21139701983dd595b6c3225a1bebd2a4e6122f83df87606f1a41152d9890e5a11ac3749b3bfcf4407fc83ef60b4ced68" -- cgit v1.2.3 From 7af30ee28ee492f77244b83b342d0ab8688a28d1 Mon Sep 17 00:00:00 2001 From: varac Date: Wed, 9 Sep 2015 13:51:19 +0200 Subject: Don't exit after failed deploy Sometimes only trivial things fail that doesn't affect basic functionallity. Change-Id: I9d9d1a531a11e6eeee6fd823a51bb02e99771ec2 --- vagrant/configure-leap.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/vagrant/configure-leap.sh b/vagrant/configure-leap.sh index e0476739..332bdddf 100755 --- a/vagrant/configure-leap.sh +++ b/vagrant/configure-leap.sh @@ -53,10 +53,6 @@ if [ $? -eq 1 ]; then fi $LEAP $OPTS -v 2 deploy -if [ $? -eq 1 ]; then - echo 'deploy failed' - exit 1 -fi set +e git add . @@ -82,4 +78,3 @@ echo -e '\n\n\n' echo 'You are now ready to use your provider. Please update your /etc/hosts with following dns overrides:' $LEAP list --print ip_address,domain.full,dns.aliases | sed 's/,//g' | cut -d' ' -f 2- - -- cgit v1.2.3 From d113bf1b2cd3cb6a94fbe20aa711bf9b9b93286f Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Wed, 9 Sep 2015 09:36:59 -0400 Subject: Fix clients being blocked by RBLs (#7431) Valid users submitting mail to be delivered should not be blocked by configured RBLs. Settings in main.cf are valid and used globally, unless they are overridden in master.cf for specific Postfix daemons. We have set in main.cf the smtp_client_restrictions parameter to check for configured rbls, so we need to override that and empty it in order to allow valid clients to send mail, even when their IP is listed in an RBL. Note: most users will typically be connecting via VPN, so their IP would typically be replaced by the VPN gateway one, but there are cases where this is still useful. Change-Id: Ie4171113c78ae2814402a1ed9b5343280cbf79d1 --- puppet/modules/site_postfix/manifests/mx.pp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp index 334d04d0..bff3e291 100644 --- a/puppet/modules/site_postfix/manifests/mx.pp +++ b/puppet/modules/site_postfix/manifests/mx.pp @@ -74,7 +74,8 @@ class site_postfix::mx { -o smtpd_tls_wrappermode=yes -o smtpd_tls_security_level=encrypt -o smtpd_recipient_restrictions=\$smtps_recipient_restrictions - -o smtpd_helo_restrictions=\$smtps_helo_restrictions", + -o smtpd_helo_restrictions=\$smtps_helo_restrictions + -o smtpd_client_restrictions=", require => [ Class['Site_config::X509::Key'], Class['Site_config::X509::Cert'], -- cgit v1.2.3 From 0e13876bd54009bf81e7cab2abcca392ca06e32d Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Thu, 10 Sep 2015 16:04:59 -0400 Subject: Make sure hiera values have valid defaults if they are not specified (#7443) Change-Id: Ib701886ad26c5e39ccd669fadca81404b5c0426a --- puppet/modules/site_postfix/manifests/mx.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp index bff3e291..bc65e370 100644 --- a/puppet/modules/site_postfix/manifests/mx.pp +++ b/puppet/modules/site_postfix/manifests/mx.pp @@ -7,8 +7,8 @@ class site_postfix::mx { $domain = $domain_hash['full_suffix'] $host_domain = $domain_hash['full'] $cert_name = hiera('name') - $mynetworks = join(hiera('mynetworks'), ' ') - $rbls = suffix(prefix(hiera('rbls'), 'reject_rbl_client '), ',') + $mynetworks = join(hiera('mynetworks', ''), ' ') + $rbls = suffix(prefix(hiera('rbls', []), 'reject_rbl_client '), ',') $root_mail_recipient = hiera('contacts') $postfix_smtp_listen = 'all' -- cgit v1.2.3 From 84e6ad1978f8952e28d8935c01b4344c0d62ddbd Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 10 Sep 2015 15:49:26 -0700 Subject: fix various problems with webapp config generation --- puppet/lib/puppet/parser/functions/sorted_yaml.rb | 388 +++++++++++++++++++++ .../modules/site_webapp/templates/config.yml.erb | 67 ++-- 2 files changed, 422 insertions(+), 33 deletions(-) create mode 100644 puppet/lib/puppet/parser/functions/sorted_yaml.rb diff --git a/puppet/lib/puppet/parser/functions/sorted_yaml.rb b/puppet/lib/puppet/parser/functions/sorted_yaml.rb new file mode 100644 index 00000000..fa0db4d2 --- /dev/null +++ b/puppet/lib/puppet/parser/functions/sorted_yaml.rb @@ -0,0 +1,388 @@ +# encoding: UTF-8 +# +# provides sorted_yaml() function, using Ya2YAML. +# see https://github.com/afunai/ya2yaml +# + +class Ya2YAML + # + # Author:: Akira FUNAI + # Copyright:: Copyright (c) 2006-2010 Akira FUNAI + # License:: MIT License + # + + def initialize(opts = {}) + options = opts.dup + options[:indent_size] = 2 if options[:indent_size].to_i <= 0 + options[:minimum_block_length] = 0 if options[:minimum_block_length].to_i <= 0 + options.update( + { + :printable_with_syck => true, + :escape_b_specific => true, + :escape_as_utf8 => true, + } + ) if options[:syck_compatible] + + @options = options + end + + def _ya2yaml(obj) + #raise 'set $KCODE to "UTF8".' if (RUBY_VERSION < '1.9.0') && ($KCODE != 'UTF8') + if (RUBY_VERSION < '1.9.0') + $KCODE = 'UTF8' + end + '--- ' + emit(obj, 1) + "\n" + rescue SystemStackError + raise ArgumentError, "ya2yaml can't handle circular references" + end + + private + + def emit(obj, level) + case obj + when Array + if (obj.length == 0) + '[]' + else + indent = "\n" + s_indent(level - 1) + ### + ### NOTE: a minor modification to normal Ya2YAML... + ### We want arrays to be output in sorted order, not just + ### Hashes. + ### + #obj.collect {|o| + # indent + '- ' + emit(o, level + 1) + #}.join('') + obj.sort {|a,b| a.to_s <=> b.to_s}.collect {|o| + indent + '- ' + emit(o, level + 1) + }.join('') + end + when Hash + if (obj.length == 0) + '{}' + else + indent = "\n" + s_indent(level - 1) + hash_order = @options[:hash_order] + if (hash_order && level == 1) + hash_keys = obj.keys.sort {|x, y| + x_order = hash_order.index(x) ? hash_order.index(x) : Float::MAX + y_order = hash_order.index(y) ? hash_order.index(y) : Float::MAX + o = (x_order <=> y_order) + (o != 0) ? o : (x.to_s <=> y.to_s) + } + elsif @options[:preserve_order] + hash_keys = obj.keys + else + hash_keys = obj.keys.sort {|x, y| x.to_s <=> y.to_s } + end + hash_keys.collect {|k| + key = emit(k, level + 1) + if ( + is_one_plain_line?(key) || + key =~ /\A(#{REX_BOOL}|#{REX_FLOAT}|#{REX_INT}|#{REX_NULL})\z/x + ) + indent + key + ': ' + emit(obj[k], level + 1) + else + indent + '? ' + key + + indent + ': ' + emit(obj[k], level + 1) + end + }.join('') + end + when NilClass + '~' + when String + emit_string(obj, level) + when TrueClass, FalseClass + obj.to_s + when Fixnum, Bignum, Float + obj.to_s + when Date + obj.to_s + when Time + offset = obj.gmtoff + off_hm = sprintf( + '%+.2d:%.2d', + (offset / 3600.0).to_i, + (offset % 3600.0) / 60 + ) + u_sec = (obj.usec != 0) ? sprintf(".%.6d", obj.usec) : '' + obj.strftime("%Y-%m-%d %H:%M:%S#{u_sec} #{off_hm}") + when Symbol + '!ruby/symbol ' + emit_string(obj.to_s, level) + when Range + '!ruby/range ' + obj.to_s + when Regexp + '!ruby/regexp ' + obj.inspect + else + case + when obj.is_a?(Struct) + struct_members = {} + obj.each_pair{|k, v| struct_members[k.to_s] = v } + '!ruby/struct:' + obj.class.to_s.sub(/^(Struct::(.+)|.*)$/, '\2') + ' ' + + emit(struct_members, level + 1) + else + # serialized as a generic object + object_members = {} + obj.instance_variables.each{|k, v| + object_members[k.to_s.sub(/^@/, '')] = obj.instance_variable_get(k) + } + '!ruby/object:' + obj.class.to_s + ' ' + + emit(object_members, level + 1) + end + end + end + + def emit_string(str, level) + (is_string, is_printable, is_one_line, is_one_plain_line) = string_type(str) + if is_string + if is_printable + if is_one_plain_line + emit_simple_string(str, level) + else + (is_one_line || str.length < @options[:minimum_block_length]) ? + emit_quoted_string(str, level) : + emit_block_string(str, level) + end + else + emit_quoted_string(str, level) + end + else + emit_base64_binary(str, level) + end + end + + def emit_simple_string(str, level) + str + end + + def emit_block_string(str, level) + str = normalize_line_break(str) + + indent = s_indent(level) + indentation_indicator = (str =~ /\A /) ? indent.size.to_s : '' + str =~ /(#{REX_NORMAL_LB}*)\z/ + chomping_indicator = case $1.length + when 0 + '-' + when 1 + '' + else + '+' + end + + str.chomp! + str.gsub!(/#{REX_NORMAL_LB}/) { + $1 + indent + } + '|' + indentation_indicator + chomping_indicator + "\n" + indent + str + end + + def emit_quoted_string(str, level) + str = yaml_escape(normalize_line_break(str)) + if (str.length < @options[:minimum_block_length]) + str.gsub!(/#{REX_NORMAL_LB}/) { ESCAPE_SEQ_LB[$1] } + else + str.gsub!(/#{REX_NORMAL_LB}$/) { ESCAPE_SEQ_LB[$1] } + str.gsub!(/(#{REX_NORMAL_LB}+)(.)/) { + trail_c = $3 + $1 + trail_c.sub(/([\t ])/) { ESCAPE_SEQ_WS[$1] } + } + indent = s_indent(level) + str.gsub!(/#{REX_NORMAL_LB}/) { + ESCAPE_SEQ_LB[$1] + "\\\n" + indent + } + end + '"' + str + '"' + end + + def emit_base64_binary(str, level) + indent = "\n" + s_indent(level) + base64 = [str].pack('m') + '!binary |' + indent + base64.gsub(/\n(?!\z)/, indent) + end + + def string_type(str) + if str.respond_to?(:encoding) && (!str.valid_encoding? || str.encoding == Encoding::ASCII_8BIT) + return false, false, false, false + end + (ucs_codes = str.unpack('U*')) rescue ( + # ArgumentError -> binary data + return false, false, false, false + ) + if ( + @options[:printable_with_syck] && + str =~ /\A#{REX_ANY_LB}* | #{REX_ANY_LB}*\z|#{REX_ANY_LB}{2}\z/ + ) + # detour Syck bug + return true, false, nil, false + end + ucs_codes.each {|ucs_code| + return true, false, nil, false unless is_printable?(ucs_code) + } + return true, true, is_one_line?(str), is_one_plain_line?(str) + end + + def is_printable?(ucs_code) + # YAML 1.1 / 4.1.1. + ( + [0x09, 0x0a, 0x0d, 0x85].include?(ucs_code) || + (ucs_code <= 0x7e && ucs_code >= 0x20) || + (ucs_code <= 0xd7ff && ucs_code >= 0xa0) || + (ucs_code <= 0xfffd && ucs_code >= 0xe000) || + (ucs_code <= 0x10ffff && ucs_code >= 0x10000) + ) && + !( + # treat LS/PS as non-printable characters + @options[:escape_b_specific] && + (ucs_code == 0x2028 || ucs_code == 0x2029) + ) + end + + def is_one_line?(str) + str !~ /#{REX_ANY_LB}(?!\z)/ + end + + def is_one_plain_line?(str) + # YAML 1.1 / 4.6.11. + str !~ /^([\-\?:,\[\]\{\}\#&\*!\|>'"%@`\s]|---|\.\.\.)/ && + str !~ /[:\#\s\[\]\{\},]/ && + str !~ /#{REX_ANY_LB}/ && + str !~ /^(#{REX_BOOL}|#{REX_FLOAT}|#{REX_INT}|#{REX_MERGE} + |#{REX_NULL}|#{REX_TIMESTAMP}|#{REX_VALUE})$/x + end + + def s_indent(level) + # YAML 1.1 / 4.2.2. + ' ' * (level * @options[:indent_size]) + end + + def normalize_line_break(str) + # YAML 1.1 / 4.1.4. + str.gsub(/(#{REX_CRLF}|#{REX_CR}|#{REX_NEL})/, "\n") + end + + def yaml_escape(str) + # YAML 1.1 / 4.1.6. + str.gsub(/[^a-zA-Z0-9]/u) {|c| + ucs_code, = (c.unpack('U') rescue [??]) + case + when ESCAPE_SEQ[c] + ESCAPE_SEQ[c] + when is_printable?(ucs_code) + c + when @options[:escape_as_utf8] + c.respond_to?(:bytes) ? + c.bytes.collect {|b| '\\x%.2x' % b }.join : + '\\x' + c.unpack('H2' * c.size).join('\\x') + when ucs_code == 0x2028 || ucs_code == 0x2029 + ESCAPE_SEQ_LB[c] + when ucs_code <= 0x7f + sprintf('\\x%.2x', ucs_code) + when ucs_code <= 0xffff + sprintf('\\u%.4x', ucs_code) + else + sprintf('\\U%.8x', ucs_code) + end + } + end + + module Constants + UCS_0X85 = [0x85].pack('U') # c285@UTF8 Unicode next line + UCS_0XA0 = [0xa0].pack('U') # c2a0@UTF8 Unicode non-breaking space + UCS_0X2028 = [0x2028].pack('U') # e280a8@UTF8 Unicode line separator + UCS_0X2029 = [0x2029].pack('U') # e280a9@UTF8 Unicode paragraph separator + + # non-break characters + ESCAPE_SEQ = { + "\x00" => '\\0', + "\x07" => '\\a', + "\x08" => '\\b', + "\x0b" => '\\v', + "\x0c" => '\\f', + "\x1b" => '\\e', + "\"" => '\\"', + "\\" => '\\\\', + } + + # non-breaking space + ESCAPE_SEQ_NS = { + UCS_0XA0 => '\\_', + } + + # white spaces + ESCAPE_SEQ_WS = { + "\x09" => '\\t', + " " => '\\x20', + } + + # line breaks + ESCAPE_SEQ_LB ={ + "\x0a" => '\\n', + "\x0d" => '\\r', + UCS_0X85 => '\\N', + UCS_0X2028 => '\\L', + UCS_0X2029 => '\\P', + } + + # regexps for line breaks + REX_LF = Regexp.escape("\x0a") + REX_CR = Regexp.escape("\x0d") + REX_CRLF = Regexp.escape("\x0d\x0a") + REX_NEL = Regexp.escape(UCS_0X85) + REX_LS = Regexp.escape(UCS_0X2028) + REX_PS = Regexp.escape(UCS_0X2029) + + REX_ANY_LB = /(#{REX_LF}|#{REX_CR}|#{REX_NEL}|#{REX_LS}|#{REX_PS})/ + REX_NORMAL_LB = /(#{REX_LF}|#{REX_LS}|#{REX_PS})/ + + # regexps for language-Independent types for YAML1.1 + REX_BOOL = / + y|Y|yes|Yes|YES|n|N|no|No|NO + |true|True|TRUE|false|False|FALSE + |on|On|ON|off|Off|OFF + /x + REX_FLOAT = / + [-+]?([0-9][0-9_]*)?\.[0-9.]*([eE][-+][0-9]+)? # (base 10) + |[-+]?[0-9][0-9_]*(:[0-5]?[0-9])+\.[0-9_]* # (base 60) + |[-+]?\.(inf|Inf|INF) # (infinity) + |\.(nan|NaN|NAN) # (not a number) + /x + REX_INT = / + [-+]?0b[0-1_]+ # (base 2) + |[-+]?0[0-7_]+ # (base 8) + |[-+]?(0|[1-9][0-9_]*) # (base 10) + |[-+]?0x[0-9a-fA-F_]+ # (base 16) + |[-+]?[1-9][0-9_]*(:[0-5]?[0-9])+ # (base 60) + /x + REX_MERGE = / + << + /x + REX_NULL = / + ~ # (canonical) + |null|Null|NULL # (English) + | # (Empty) + /x + REX_TIMESTAMP = / + [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] # (ymd) + |[0-9][0-9][0-9][0-9] # (year) + -[0-9][0-9]? # (month) + -[0-9][0-9]? # (day) + ([Tt]|[ \t]+)[0-9][0-9]? # (hour) + :[0-9][0-9] # (minute) + :[0-9][0-9] # (second) + (\.[0-9]*)? # (fraction) + (([ \t]*)Z|[-+][0-9][0-9]?(:[0-9][0-9])?)? # (time zone) + /x + REX_VALUE = / + = + /x + end + + include Constants +end + +module Puppet::Parser::Functions + newfunction(:sorted_yaml, :type => :rvalue, :doc => "This function outputs yaml, but ensures the keys are sorted.") do |argument| + return Ya2YAML.new()._ya2yaml(argument) + end +end diff --git a/puppet/modules/site_webapp/templates/config.yml.erb b/puppet/modules/site_webapp/templates/config.yml.erb index e8853ade..5cb436fc 100644 --- a/puppet/modules/site_webapp/templates/config.yml.erb +++ b/puppet/modules/site_webapp/templates/config.yml.erb @@ -1,33 +1,34 @@ -<%- require 'json' -%> -<%- cert_options = @webapp['client_certificates'] -%> -production: - admins: <%= @webapp['admins'].inspect %> - default_locale: :<%= @webapp['default_locale'] %> - available_locales: -<%- @webapp['locales'].each do |locale| -%> - - :<%= locale %> -<%- end -%> - domain: <%= @provider_domain %> - force_ssl: <%= @webapp['secure'] %> - client_ca_key: <%= scope.lookupvar('x509::variables::keys') %>/<%= scope.lookupvar('site_config::params::client_ca_name') %>.key - client_ca_cert: <%= scope.lookupvar('x509::variables::local_CAs') %>/<%= scope.lookupvar('site_config::params::client_ca_name') %>.crt - secret_token: "<%= @secret_token %>" - client_cert_lifespan: <%= cert_options['life_span'] %> - client_cert_bit_size: <%= cert_options['bit_size'].to_i %> - client_cert_hash: <%= cert_options['digest'] %> - allow_limited_certs: <%= @webapp['allow_limited_certs'].inspect %> - allow_unlimited_certs: <%= @webapp['allow_unlimited_certs'].inspect %> - allow_anonymous_certs: <%= @webapp['allow_anonymous_certs'].inspect %> - limited_cert_prefix: "<%= cert_options['limited_prefix'] %>" - unlimited_cert_prefix: "<%= cert_options['unlimited_prefix'] %>" - minimum_client_version: "<%= @webapp['client_version']['min'] %>" - default_service_level: "<%= @webapp['default_service_level'] %>" - service_levels: <%= scope.function_sorted_json([@webapp['service_levels']]) %> - allow_registration: <%= @webapp['allow_registration'].inspect %> - handle_blacklist: <%= @webapp['forbidden_usernames'].inspect %> -<%- if @webapp['engines'] && @webapp['engines'].any? -%> - engines: -<%- @webapp['engines'].each do |engine| -%> - - <%= engine %> -<%- end -%> -<%- end -%> +<%- +cert_options = @webapp['client_certificates'] +production = { + "admins" => @webapp['admins'], + "default_locale" => @webapp['default_locale'], + "available_locales" => @webapp['locales'], + "domain" => @provider_domain, + "force_ssl" => @webapp['secure'], + "client_ca_key" => "%s/%s.key" % [scope.lookupvar('x509::variables::keys'), scope.lookupvar('site_config::params::client_ca_name')], + "client_ca_cert" => "%s/%s.crt" % [scope.lookupvar('x509::variables::local_CAs'), scope.lookupvar('site_config::params::client_ca_name')], + "secret_token" => @secret_token, + "client_cert_lifespan" => cert_options['life_span'], + "client_cert_bit_size" => cert_options['bit_size'].to_i, + "client_cert_hash" => cert_options['digest'], + "allow_limited_certs" => @webapp['allow_limited_certs'], + "allow_unlimited_certs" => @webapp['allow_unlimited_certs'], + "allow_anonymous_certs" => @webapp['allow_anonymous_certs'], + "limited_cert_prefix" => cert_options['limited_prefix'], + "unlimited_cert_prefix" => cert_options['unlimited_prefix'], + "minimum_client_version" => @webapp['client_version']['min'], + "default_service_level" => @webapp['default_service_level'], + "service_levels" => @webapp['service_levels'], + "allow_registration" => @webapp['allow_registration'], + "handle_blacklist" => @webapp['forbidden_usernames'] +} + +if @webapp['engines'] && @webapp['engines'].any? + production["engines"] = @webapp['engines'] +end +-%> +# +# This file is generated by puppet. This file inherits from defaults.yml. +# +<%= scope.function_sorted_yaml({"production" => production}) %> -- cgit v1.2.3 From 818930af8a05dc44372b99f8e589527050120431 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 10 Sep 2015 22:38:44 -0700 Subject: sshd: let nodes change default AllowTcpForwarding --- provider_base/common.json | 3 +++ puppet/modules/site_sshd/manifests/init.pp | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/provider_base/common.json b/provider_base/common.json index 3d2965d7..e968dd27 100644 --- a/provider_base/common.json +++ b/provider_base/common.json @@ -16,6 +16,9 @@ }, "ssh": { "authorized_keys": "= authorized_keys", + "config": { + "AllowTcpForwarding": "no" + }, "port": 22, "mosh": { "ports": "60000:61000", diff --git a/puppet/modules/site_sshd/manifests/init.pp b/puppet/modules/site_sshd/manifests/init.pp index 1da2f1d5..170be32c 100644 --- a/puppet/modules/site_sshd/manifests/init.pp +++ b/puppet/modules/site_sshd/manifests/init.pp @@ -1,6 +1,7 @@ class site_sshd { - $ssh = hiera_hash('ssh') - $hosts = hiera('hosts', '') + $ssh = hiera_hash('ssh') + $ssh_config = $ssh['config'] + $hosts = hiera('hosts', '') ## ## SETUP AUTHORIZED KEYS @@ -52,11 +53,12 @@ class site_sshd { ## SSHD SERVER CONFIGURATION ## class { '::sshd': - manage_nagios => false, - ports => [ $ssh['port'] ], - use_pam => 'yes', - hardened_ssl => 'yes', - print_motd => 'no', - manage_client => false + manage_nagios => false, + ports => [ $ssh['port'] ], + use_pam => 'yes', + hardened_ssl => 'yes', + print_motd => 'no', + tcp_forwarding => $ssh_config['AllowTcpForwarding'], + manage_client => false } } -- cgit v1.2.3 From 702bf139f407d60e7c297ceb67fc6c30fead1e61 Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 11 Sep 2015 10:34:56 -0700 Subject: switch aliases to use virtual_alias_maps --- puppet/modules/site_postfix/manifests/mx.pp | 9 ++++-- .../site_postfix/manifests/mx/static_aliases.pp | 32 ++++++++-------------- .../site_postfix/templates/custom-aliases.erb | 11 -------- .../site_postfix/templates/virtual-aliases.erb | 22 +++++++++++++++ 4 files changed, 40 insertions(+), 34 deletions(-) delete mode 100644 puppet/modules/site_postfix/templates/custom-aliases.erb create mode 100644 puppet/modules/site_postfix/templates/virtual-aliases.erb diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp index bff3e291..14c8634e 100644 --- a/puppet/modules/site_postfix/manifests/mx.pp +++ b/puppet/modules/site_postfix/manifests/mx.pp @@ -21,16 +21,20 @@ class site_postfix::mx { postfix::config { 'mynetworks': value => "127.0.0.0/8 [::1]/128 [fe80::]/64 ${mynetworks}"; + # Note: mydestination should not include @domain, because this is + # used in virtual alias maps. 'mydestination': - value => "\$myorigin, localhost, localhost.\$mydomain, ${domain}"; + value => "\$myorigin, localhost, localhost.\$mydomain"; 'myhostname': value => $host_domain; 'mailbox_size_limit': value => '0'; 'home_mailbox': value => 'Maildir/'; + # Note: virtual-aliases map will take precedence over leap_mx + # lookup (tcp:localhost) 'virtual_alias_maps': - value => 'tcp:localhost:4242'; + value => 'hash:/etc/postfix/virtual-aliases tcp:localhost:4242'; 'luser_relay': value => 'vmail'; 'smtpd_tls_received_header': @@ -68,7 +72,6 @@ class site_postfix::mx { preseed => true, root_mail_recipient => $root_mail_recipient, smtp_listen => 'all', - default_alias_maps => false, mastercf_tail => "smtps inet n - - - - smtpd -o smtpd_tls_wrappermode=yes diff --git a/puppet/modules/site_postfix/manifests/mx/static_aliases.pp b/puppet/modules/site_postfix/manifests/mx/static_aliases.pp index 786d74c1..d81e05b3 100644 --- a/puppet/modules/site_postfix/manifests/mx/static_aliases.pp +++ b/puppet/modules/site_postfix/manifests/mx/static_aliases.pp @@ -30,29 +30,21 @@ class site_postfix::mx::static_aliases { } # - # Custom aliases. - # - # This does not use the puppet mailalias resource because we want to be able - # to guarantee the contents of the alias file. This is needed so if you - # remove an alias from the node's config, it will get removed from the alias - # file. - # - - # both alias files must be listed under "alias_database", because once you - # specify one, then `newaliases` no longer will default to updating - # "/etc/aliases.db". - postfix::config { - 'alias_database': - value => "/etc/aliases, /etc/postfix/custom-aliases"; - 'alias_maps': - value => "hash:/etc/aliases, hash:/etc/postfix/custom-aliases"; + # Custom static virtual aliases. + # + exec { 'postmap_virtual_aliases': + command => '/usr/sbin/postmap /etc/postfix/virtual-aliases', + refreshonly => true, + user => root, + group => root, + require => Package['postfix'], + subscribe => File['/etc/postfix/virtual-aliases'] } - - file { '/etc/postfix/custom-aliases': - content => template('site_postfix/custom-aliases.erb'), + file { '/etc/postfix/virtual-aliases': + content => template('site_postfix/virtual-aliases.erb'), owner => root, group => root, mode => 0600, - notify => Exec['newaliases'] + require => Package['postfix'] } } diff --git a/puppet/modules/site_postfix/templates/custom-aliases.erb b/puppet/modules/site_postfix/templates/custom-aliases.erb deleted file mode 100644 index f261514b..00000000 --- a/puppet/modules/site_postfix/templates/custom-aliases.erb +++ /dev/null @@ -1,11 +0,0 @@ -# -# This file is managed by puppet. -# -# This is a map of custom, non-standard aliases. The contents of this file -# are derived from the node property `mx.aliases`. -# - -<%- @aliases.keys.sort.each do |from| -%> -"<%= from %>": "<%= [@aliases[from]].flatten.join('", "') %>" -<%- end -%> - diff --git a/puppet/modules/site_postfix/templates/virtual-aliases.erb b/puppet/modules/site_postfix/templates/virtual-aliases.erb new file mode 100644 index 00000000..c474e734 --- /dev/null +++ b/puppet/modules/site_postfix/templates/virtual-aliases.erb @@ -0,0 +1,22 @@ +# +# This file is managed by puppet. +# +# This is a map of custom, non-standard aliases. The contents of this file +# are derived from the node property `mx.aliases`. +# + +# +# enable these virtual domains: +# +<%= @domain %> enabled +<%- @aliases.keys.map {|addr| addr.split('@')[1] }.compact.sort.uniq.each do |virt_domain| -%> +<%= virt_domain %> enabled +<%- end %> + +# +# virtual aliases: +# +<%- @aliases.keys.sort.each do |from| -%> +<%- full_address = from =~ /@/ ? from : from + "@" + @domain -%> +<%= full_address %> <%= [@aliases[from]].flatten.map{|a| a =~ /@/ ? a : a + "@" + @domain}.join(', ') %> +<%- end -%> -- cgit v1.2.3 From ce5baecd3e831a6ab2af33a15a99419fab39d04a Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 14 Sep 2015 12:13:29 -0700 Subject: Added help/warning if running tunnel command without TCP forwarding enabled. --- lib/leap_cli/commands/ssh.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/leap_cli/commands/ssh.rb b/lib/leap_cli/commands/ssh.rb index 1a81902c..3887618e 100644 --- a/lib/leap_cli/commands/ssh.rb +++ b/lib/leap_cli/commands/ssh.rb @@ -27,6 +27,11 @@ module LeapCli; module Commands c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`.' c.action do |global_options,options,args| local_port, node, remote_port = parse_tunnel_arg(args.first) + unless node.ssh.config.AllowTcpForwarding == "yes" + log :warning, "It looks like TCP forwarding is not enabled. "+ + "The tunnel command requires that the node property ssh.config.AllowTcpForwarding "+ + "be set to 'yes'. Add this property to #{node.name}.json, deploy, and then try tunnel again." + end options[:ssh] = [options[:ssh], "-N -L 127.0.0.1:#{local_port}:0.0.0.0:#{remote_port}"].join(' ') log("Forward port localhost:#{local_port} to #{node.name}:#{remote_port}") if is_port_available?(local_port) -- cgit v1.2.3 From 0ace5c6dfaa5367cb5d6f67c04bc703ce8f1404d Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 15 Sep 2015 03:45:34 -0700 Subject: fix incorrect name for vagrant ssh public key file --- lib/leap_cli/commands/vagrant.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/leap_cli/commands/vagrant.rb b/lib/leap_cli/commands/vagrant.rb index 1561a658..9e81b2f8 100644 --- a/lib/leap_cli/commands/vagrant.rb +++ b/lib/leap_cli/commands/vagrant.rb @@ -79,7 +79,7 @@ module LeapCli; module Commands # we need to make sure that it owned by us and not world readable. # def vagrant_ssh_key_file - file_path = Path.vagrant_ssh_key_file + file_path = Path.vagrant_ssh_pub_key_file Util.assert_files_exist! file_path uid = File.new(file_path).stat.uid if uid == 0 || uid == Process.euid -- cgit v1.2.3 From 36fea3b7f448f50d500c0ec1a30b8c745b6f8c4c Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 15 Sep 2015 10:54:24 -0400 Subject: minor linting Change-Id: If92faee5f877301bf23564d5b6e71c4b1263de54 --- puppet/modules/site_postfix/manifests/mx/static_aliases.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_postfix/manifests/mx/static_aliases.pp b/puppet/modules/site_postfix/manifests/mx/static_aliases.pp index d81e05b3..e9118470 100644 --- a/puppet/modules/site_postfix/manifests/mx/static_aliases.pp +++ b/puppet/modules/site_postfix/manifests/mx/static_aliases.pp @@ -44,7 +44,7 @@ class site_postfix::mx::static_aliases { content => template('site_postfix/virtual-aliases.erb'), owner => root, group => root, - mode => 0600, + mode => '0600', require => Package['postfix'] } } -- cgit v1.2.3 From 2d20633ceaa670c58ca575eb2a751605bf5b4156 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 3 Sep 2015 20:18:12 -0700 Subject: service definition .json files should not refer to properties inherited from common.json. closes #7423 --- provider_base/services/webapp.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index 7b1ca8fb..dbcfe0a6 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -9,7 +9,7 @@ "owner", "owners", "postmaster", "reply", "robot", "ssladmin", "staff", "support", "tech-support", "tech_support", "techsupport", "ticket", "tickets", "vmail", "www-data"], - "domain": "= domain.full_suffix", + "domain": "= provider.domain", "modules": ["user", "billing", "help"], "couchdb_webapp_user": "= global.services[:couchdb].couch.users[:webapp]", "couchdb_admin_user": "= global.services[:couchdb].couch.users[:admin]", -- cgit v1.2.3 From 2b1911f17b0ed5ee5ad2384e176b84b84243802f Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 3 Sep 2015 23:24:43 -0700 Subject: make couchdb.admin.yml only readable by root, make non-admin cron run as webapp user. --- puppet/modules/site_webapp/manifests/couchdb.pp | 16 ++++++++-------- puppet/modules/site_webapp/manifests/cron.pp | 4 ++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/puppet/modules/site_webapp/manifests/couchdb.pp b/puppet/modules/site_webapp/manifests/couchdb.pp index 1dbc745d..5cf7f953 100644 --- a/puppet/modules/site_webapp/manifests/couchdb.pp +++ b/puppet/modules/site_webapp/manifests/couchdb.pp @@ -14,29 +14,29 @@ class site_webapp::couchdb { file { '/srv/leap/webapp/config/couchdb.yml': content => template('site_webapp/couchdb.yml.erb'), - owner => leap-webapp, - group => leap-webapp, + owner => 'leap-webapp', + group => 'leap-webapp', mode => '0600', require => Vcsrepo['/srv/leap/webapp']; '/srv/leap/webapp/config/couchdb.admin.yml': content => template('site_webapp/couchdb.admin.yml.erb'), - owner => leap-webapp, - group => leap-webapp, + owner => 'root', + group => 'root', mode => '0600', require => Vcsrepo['/srv/leap/webapp']; '/srv/leap/webapp/log': ensure => directory, - owner => leap-webapp, - group => leap-webapp, + owner => 'leap-webapp', + group => 'leap-webapp', mode => '0755', require => Vcsrepo['/srv/leap/webapp']; '/srv/leap/webapp/log/production.log': ensure => present, - owner => leap-webapp, - group => leap-webapp, + owner => 'leap-webapp', + group => 'leap-webapp', mode => '0666', require => Vcsrepo['/srv/leap/webapp']; } diff --git a/puppet/modules/site_webapp/manifests/cron.pp b/puppet/modules/site_webapp/manifests/cron.pp index d26ee312..7147a0d2 100644 --- a/puppet/modules/site_webapp/manifests/cron.pp +++ b/puppet/modules/site_webapp/manifests/cron.pp @@ -5,12 +5,14 @@ class site_webapp::cron { 'rotate_databases': command => 'cd /srv/leap/webapp && bundle exec rake db:rotate', environment => 'RAILS_ENV=production', + user => 'root', hour => [0,6,12,18], minute => 0; 'delete_tmp_databases': command => 'cd /srv/leap/webapp && bundle exec rake db:deletetmp', environment => 'RAILS_ENV=production', + user => 'root', hour => 1, minute => 1; @@ -19,6 +21,7 @@ class site_webapp::cron { 'remove_expired_sessions': command => 'cd /srv/leap/webapp && bundle exec rake cleanup:sessions', environment => 'RAILS_ENV=production', + user => 'leap-webapp', hour => 2, minute => 30, ensure => absent; @@ -26,6 +29,7 @@ class site_webapp::cron { 'remove_expired_tokens': command => 'cd /srv/leap/webapp && bundle exec rake cleanup:tokens', environment => 'RAILS_ENV=production', + user => 'leap-webapp', hour => 3, minute => 0; } -- cgit v1.2.3 From f8b2a72aa09fca368d3038efa9100a48126f8ae9 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 15 Sep 2015 11:52:20 -0400 Subject: Fix server-status availability to tor hidden services (#7456) Make the server-status information unavailable by putting the vhost on a port that isn't configured as available to the tor hidden-service. Change-Id: Idd3bfefb5b7fc26fb0a8cf48cdf6afc68a4192bb --- puppet/modules/site_apache/manifests/common.pp | 21 +--------------- puppet/modules/site_apache/manifests/common/tls.pp | 6 +++++ puppet/modules/site_nagios/manifests/server.pp | 1 + puppet/modules/site_static/manifests/init.pp | 13 ++++++---- .../modules/site_webapp/files/server-status.conf | 28 ++++++++++++++++++++++ puppet/modules/site_webapp/manifests/apache.pp | 3 ++- .../modules/site_webapp/manifests/common_vhost.pp | 18 ++++++++++++++ .../site_webapp/manifests/hidden_service.pp | 10 ++++++-- 8 files changed, 73 insertions(+), 27 deletions(-) create mode 100644 puppet/modules/site_apache/manifests/common/tls.pp create mode 100644 puppet/modules/site_webapp/files/server-status.conf create mode 100644 puppet/modules/site_webapp/manifests/common_vhost.pp diff --git a/puppet/modules/site_apache/manifests/common.pp b/puppet/modules/site_apache/manifests/common.pp index 2b83ffa5..64beb231 100644 --- a/puppet/modules/site_apache/manifests/common.pp +++ b/puppet/modules/site_apache/manifests/common.pp @@ -1,27 +1,8 @@ class site_apache::common { - # installs x509 cert + key and common config - # that both nagios + leap webapp use - - $web_domain = hiera('domain') - $domain_name = $web_domain['name'] - - include x509::variables - include site_config::x509::commercial::cert - include site_config::x509::commercial::key - include site_config::x509::commercial::ca - - Class['Site_config::X509::Commercial::Key'] ~> Service[apache] - Class['Site_config::X509::Commercial::Cert'] ~> Service[apache] - Class['Site_config::X509::Commercial::Ca'] ~> Service[apache] include site_apache::module::rewrite class { '::apache': no_default_site => true, ssl => true } - apache::vhost::file { - 'common': - content => template('site_apache/vhosts.d/common.conf.erb') - } - - apache::config::include{ 'ssl_common.inc': } + include site_apache::common::tls } diff --git a/puppet/modules/site_apache/manifests/common/tls.pp b/puppet/modules/site_apache/manifests/common/tls.pp new file mode 100644 index 00000000..040868bf --- /dev/null +++ b/puppet/modules/site_apache/manifests/common/tls.pp @@ -0,0 +1,6 @@ +class site_apache::common::tls { + # class to setup common SSL configurations + + apache::config::include{ 'ssl_common.inc': } + +} diff --git a/puppet/modules/site_nagios/manifests/server.pp b/puppet/modules/site_nagios/manifests/server.pp index cb6c8d95..60a471b7 100644 --- a/puppet/modules/site_nagios/manifests/server.pp +++ b/puppet/modules/site_nagios/manifests/server.pp @@ -32,6 +32,7 @@ class site_nagios::server inherits nagios::base { } include site_apache::common + include site_webapp::common_vhost include site_apache::module::headers File ['nagios_htpasswd'] { diff --git a/puppet/modules/site_static/manifests/init.pp b/puppet/modules/site_static/manifests/init.pp index 1efc510b..f69ffba7 100644 --- a/puppet/modules/site_static/manifests/init.pp +++ b/puppet/modules/site_static/manifests/init.pp @@ -9,6 +9,7 @@ class site_static { $domains = $static['domains'] $formats = $static['formats'] $bootstrap = $static['bootstrap_files'] + $tor = hiera('tor', false) if $bootstrap['enabled'] { $bootstrap_domain = $bootstrap['domain'] @@ -27,14 +28,11 @@ class site_static { } } - class { '::apache': no_default_site => true, ssl => true } include site_apache::module::headers include site_apache::module::alias include site_apache::module::expires include site_apache::module::removeip - include site_apache::module::rewrite - apache::config::include{ 'ssl_common.inc': } - + include site_apache::common include site_config::ruby::dev if (member($formats, 'rack')) { @@ -57,6 +55,13 @@ class site_static { create_resources(site_static::domain, $domains) + if $tor { + $hidden_service = $tor['hidden_service'] + if $hidden_service['active'] { + include site_webapp::hidden_service + } + } + include site_shorewall::defaults include site_shorewall::service::http include site_shorewall::service::https diff --git a/puppet/modules/site_webapp/files/server-status.conf b/puppet/modules/site_webapp/files/server-status.conf new file mode 100644 index 00000000..84cb9ae0 --- /dev/null +++ b/puppet/modules/site_webapp/files/server-status.conf @@ -0,0 +1,28 @@ +# Keep track of extended status information for each request +ExtendedStatus On + +# Determine if mod_status displays the first 63 characters of a request or +# the last 63, assuming the request itself is greater than 63 chars. +# Default: Off +#SeeRequestTail On + +Listen 127.0.0.1:8162 +NameVirtualHost 127.0.0.1:8162 + + + + + SetHandler server-status + Order deny,allow + Deny from all + Allow from 127.0.0.1 + + + + + + + # Show Proxy LoadBalancer status in mod_status + ProxyStatus On + + diff --git a/puppet/modules/site_webapp/manifests/apache.pp b/puppet/modules/site_webapp/manifests/apache.pp index 93e172a0..ddd04a91 100644 --- a/puppet/modules/site_webapp/manifests/apache.pp +++ b/puppet/modules/site_webapp/manifests/apache.pp @@ -15,12 +15,13 @@ class site_webapp::apache { include site_apache::module::alias include site_apache::module::expires include site_apache::module::removeip + include site_webapp::common_vhost class { 'passenger': use_munin => false } apache::vhost::file { 'api': - content => template('site_apache/vhosts.d/api.conf.erb') + content => template('site_apache/vhosts.d/api.conf.erb'); } } diff --git a/puppet/modules/site_webapp/manifests/common_vhost.pp b/puppet/modules/site_webapp/manifests/common_vhost.pp new file mode 100644 index 00000000..c57aad57 --- /dev/null +++ b/puppet/modules/site_webapp/manifests/common_vhost.pp @@ -0,0 +1,18 @@ +class site_webapp::common_vhost { + # installs x509 cert + key and common config + # that both nagios + leap webapp use + + include x509::variables + include site_config::x509::commercial::cert + include site_config::x509::commercial::key + include site_config::x509::commercial::ca + + Class['Site_config::X509::Commercial::Key'] ~> Service[apache] + Class['Site_config::X509::Commercial::Cert'] ~> Service[apache] + Class['Site_config::X509::Commercial::Ca'] ~> Service[apache] + + apache::vhost::file { + 'common': + content => template('site_apache/vhosts.d/common.conf.erb') + } +} diff --git a/puppet/modules/site_webapp/manifests/hidden_service.pp b/puppet/modules/site_webapp/manifests/hidden_service.pp index 16b6e2e7..99a756ca 100644 --- a/puppet/modules/site_webapp/manifests/hidden_service.pp +++ b/puppet/modules/site_webapp/manifests/hidden_service.pp @@ -32,12 +32,18 @@ class site_webapp::hidden_service { owner => 'debian-tor', group => 'debian-tor', mode => '0600'; + + '/etc/apache2/mods-enabled/status.conf': + ensure => absent, + notify => Service['apache']; } apache::vhost::file { 'hidden_service': - content => template('site_apache/vhosts.d/hidden_service.conf.erb') + content => template('site_apache/vhosts.d/hidden_service.conf.erb'); + 'server_status': + vhost_source => 'modules/site_webapp/server-status.conf'; } include site_shorewall::tor -} \ No newline at end of file +} -- cgit v1.2.3 From 4241729ba2f5e539ba228616c479bf5a64a7f8e2 Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 15 Sep 2015 11:45:24 -0700 Subject: fix vagrant ssh private key path --- lib/leap_cli/commands/vagrant.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/leap_cli/commands/vagrant.rb b/lib/leap_cli/commands/vagrant.rb index 9e81b2f8..e2dfb8a9 100644 --- a/lib/leap_cli/commands/vagrant.rb +++ b/lib/leap_cli/commands/vagrant.rb @@ -73,13 +73,13 @@ module LeapCli; module Commands public # - # returns the path to a vagrant ssh key file. + # returns the path to a vagrant ssh private key file. # # if the vagrant.key file is owned by root or ourselves, then # we need to make sure that it owned by us and not world readable. # def vagrant_ssh_key_file - file_path = Path.vagrant_ssh_pub_key_file + file_path = Path.vagrant_ssh_priv_key_file Util.assert_files_exist! file_path uid = File.new(file_path).stat.uid if uid == 0 || uid == Process.euid -- cgit v1.2.3 From 2e30fad6b23f591e84acc06263774919eee6ec76 Mon Sep 17 00:00:00 2001 From: kwadronaut Date: Sun, 20 Sep 2015 23:31:10 +0200 Subject: automatic update of submodule apt --- puppet/modules/apt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/apt b/puppet/modules/apt index fca10348..ab90d1d0 160000 --- a/puppet/modules/apt +++ b/puppet/modules/apt @@ -1 +1 @@ -Subproject commit fca103484ddc1f647a54135b6a902edabf459554 +Subproject commit ab90d1d0fe9655d367c637e95dff59e4dbe2dd35 -- cgit v1.2.3 From bbc95640557e200a5a4e463f451ed647692dc0a3 Mon Sep 17 00:00:00 2001 From: Micah Date: Sat, 19 Sep 2015 22:19:44 -0400 Subject: Remove no longer used vhost for leap_webapp (#7475) The configuration /etc/apache/sites-enabled/leap_webapp.conf was never removed after 6255e58bf9ff3489bf2707bc2be9759ec5c7db68 made it obsolete, and because it exists on older systems, it is being used instead of the correct common.conf. This removes it and reloads apache. Change-Id: Ic4c9901f4bba869ecb3dfe5362dfd1971570f89a --- puppet/modules/site_config/manifests/remove_files.pp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/puppet/modules/site_config/manifests/remove_files.pp b/puppet/modules/site_config/manifests/remove_files.pp index a9a0c8bf..e2ab3c2e 100644 --- a/puppet/modules/site_config/manifests/remove_files.pp +++ b/puppet/modules/site_config/manifests/remove_files.pp @@ -23,6 +23,8 @@ class site_config::remove_files { '/etc/logrotate.d/mx':; '/etc/logrotate.d/stunnel':; '/var/log/stunnel4/stunnel.log':; + '/etc/apache/sites-enabled/leap_webapp.conf': + notify => Service['apache']; 'leap_mx': path => '/var/log/', recurse => true, -- cgit v1.2.3 From 4b26c0f30980789844c747e796c12958f51c932c Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 24 Sep 2015 11:25:48 -0400 Subject: fix missing service dependency error this tidy should only happen on webapp nodes Change-Id: I56faac4fa28fde9dcad7ce9a6ed0d684630a556e --- puppet/modules/site_config/manifests/remove_files.pp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/puppet/modules/site_config/manifests/remove_files.pp b/puppet/modules/site_config/manifests/remove_files.pp index e2ab3c2e..51d1ea88 100644 --- a/puppet/modules/site_config/manifests/remove_files.pp +++ b/puppet/modules/site_config/manifests/remove_files.pp @@ -23,8 +23,6 @@ class site_config::remove_files { '/etc/logrotate.d/mx':; '/etc/logrotate.d/stunnel':; '/var/log/stunnel4/stunnel.log':; - '/etc/apache/sites-enabled/leap_webapp.conf': - notify => Service['apache']; 'leap_mx': path => '/var/log/', recurse => true, @@ -39,6 +37,13 @@ class site_config::remove_files { rmdirs => true; } + if member($::services, 'webapp') { + tidy { + '/etc/apache/sites-enabled/leap_webapp.conf': + notify => Service['apache']; + } + } + # leax-mx logged to /var/log/leap_mx.log in the past # we need to use a dumb exec here because file_line doesn't # allow removing lines that match a regex in the current version -- cgit v1.2.3 From 9352ce45a7950ff1175d3a7e5412fc9006691799 Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 7 Aug 2015 14:22:48 -0700 Subject: added firewall information to nodes (needed for `leap compile firewall`) --- lib/leap_cli/macros/stunnel.rb | 13 ++++++++++++- provider_base/common.json | 8 ++++++++ provider_base/services/dns.json | 9 ++++++++- provider_base/services/monitor.json | 7 +++++++ provider_base/services/mx.json | 9 ++++++++- provider_base/services/openvpn.json | 7 +++++++ provider_base/services/soledad.json | 9 ++++++++- provider_base/services/static.json | 9 ++++++++- provider_base/services/webapp.json | 7 +++++++ 9 files changed, 73 insertions(+), 5 deletions(-) diff --git a/lib/leap_cli/macros/stunnel.rb b/lib/leap_cli/macros/stunnel.rb index f16308c7..821bda38 100644 --- a/lib/leap_cli/macros/stunnel.rb +++ b/lib/leap_cli/macros/stunnel.rb @@ -49,12 +49,14 @@ module LeapCli result = Config::ObjectList.new node_list.each_node do |node| if node.name != self.name || options[:include_self] + s_port = stunnel_port(port) result["#{node.name}_#{port}"] = Config::Object[ 'accept_port', @next_stunnel_port, 'connect', node.domain.internal, - 'connect_port', stunnel_port(port), + 'connect_port', s_port, 'original_port', port ] + manager.connections.add(:from => @node.ip_address, :to => node.ip_address, :port => s_port) @next_stunnel_port += 1 end end @@ -76,6 +78,15 @@ module LeapCli } end + # + # lists the ips that connect to this node, on particular ports. + # + def stunnel_firewall + manager.connections.select {|connection| + connection['to'] == @node.ip_address + } + end + private # diff --git a/provider_base/common.json b/provider_base/common.json index e968dd27..9cc7875a 100644 --- a/provider_base/common.json +++ b/provider_base/common.json @@ -50,6 +50,14 @@ "clients": {}, "servers": {} }, + "firewall": { + "ssh": { + "from": "sysadmin", + "to": "= ip_address", + "port": "= ssh.port" + }, + "stunnel": "=> stunnel_firewall" + }, "platform": { "version": "= Leap::Platform.version.to_s", "major_version": "= Leap::Platform.major_version" diff --git a/provider_base/services/dns.json b/provider_base/services/dns.json index 677d9b2c..67948ef8 100644 --- a/provider_base/services/dns.json +++ b/provider_base/services/dns.json @@ -3,5 +3,12 @@ "public": "= nodes['dns.public' => true].fields('domain.name', 'dns.aliases', 'ip_address')", "private": "= nodes['dns.public' => false].fields('domain.name', 'dns.aliases', 'ip_address')" }, - "service_type": "public_service" + "service_type": "public_service", + "firewall": { + "dns": { + "from": "*", + "to": "= ip_address", + "port": "53" + } + } } \ No newline at end of file diff --git a/provider_base/services/monitor.json b/provider_base/services/monitor.json index 10d5ac81..28fb837c 100644 --- a/provider_base/services/monitor.json +++ b/provider_base/services/monitor.json @@ -18,5 +18,12 @@ "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'" + }, + "firewall": { + "monitor": { + "from": "sysadmin", + "to": "= ip_address", + "port": [443, 80] + } } } diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json index 4d1b3dad..d6e9fff9 100644 --- a/provider_base/services/mx.json +++ b/provider_base/services/mx.json @@ -31,5 +31,12 @@ "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'" }, - "service_type": "user_service" + "service_type": "user_service", + "firewall": { + "mx": { + "from": "*", + "to": "= ip_address", + "port": [25, 465] + } + } } diff --git a/provider_base/services/openvpn.json b/provider_base/services/openvpn.json index 11cb0dc2..6f73e31c 100644 --- a/provider_base/services/openvpn.json +++ b/provider_base/services/openvpn.json @@ -34,5 +34,12 @@ "port" : "= rand_range('scramblesuit_port_'+name, 18000..32000)" }, "gateway_address": "= openvpn.gateway_address" + }, + "firewall": { + "vpn": { + "from": "*", + "to": "= openvpn.gateway_address", + "port": "= openvpn.ports + [obfsproxy.scramblesuit.port]" + } } } diff --git a/provider_base/services/soledad.json b/provider_base/services/soledad.json index ed6fbc9f..76f7155f 100644 --- a/provider_base/services/soledad.json +++ b/provider_base/services/soledad.json @@ -8,5 +8,12 @@ "salt": "= hex_secret :couch_soledad_password_salt, 128" } }, - "service_type": "public_service" + "service_type": "public_service", + "firewall": { + "soledad": { + "from": "*", + "to": "= ip_address", + "port": "= soledad.port" + } + } } diff --git a/provider_base/services/static.json b/provider_base/services/static.json index d9f52b36..2f408ec1 100644 --- a/provider_base/services/static.json +++ b/provider_base/services/static.json @@ -9,5 +9,12 @@ "client_version": "= static.bootstrap_files.enabled ? provider.client_version : nil" } }, - "service_type": "public_service" + "service_type": "public_service", + "firewall": { + "static": { + "from": "*", + "to": "= ip_address", + "port": [80, 443] + } + } } \ No newline at end of file diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index 039b1c0b..9e3d751b 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -76,5 +76,12 @@ "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`.'", "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`.'" + }, + "firewall": { + "webapp": { + "from": "*", + "to": "= ip_address", + "port": "= [api.port, 443, 80, nickserver.port]" + } } } -- cgit v1.2.3 From afd8867ba953513c6e08f957e3099f0ff3b1a3a2 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 24 Sep 2015 13:29:15 -0700 Subject: allow certain aliases, like 'abuse', to be publicly forwardable. --- .../site_postfix/manifests/mx/static_aliases.pp | 68 +++++++++++++++++----- .../site_postfix/templates/virtual-aliases.erb | 3 +- 2 files changed, 54 insertions(+), 17 deletions(-) diff --git a/puppet/modules/site_postfix/manifests/mx/static_aliases.pp b/puppet/modules/site_postfix/manifests/mx/static_aliases.pp index e9118470..71c0555a 100644 --- a/puppet/modules/site_postfix/manifests/mx/static_aliases.pp +++ b/puppet/modules/site_postfix/manifests/mx/static_aliases.pp @@ -1,37 +1,75 @@ # # Defines static, hard coded aliases that are not in the database. +# These aliases take precedence over the database aliases. +# +# There are three classes of reserved names: +# +# (1) forbidden_usernames: +# Some usernames are forbidden and cannot be registered. +# this is defined in node property webapp.forbidden_usernames +# This is enforced by the webapp. +# +# (2) public aliases: +# Some aliases for root, and are publicly exposed so that anyone +# can deliver mail to them. For example, postmaster. +# These are implemented in the virtual alias map, which takes +# precedence over the local alias map. +# +# (3) local aliases: +# Some aliases are only available locally: mail can be delivered +# to the alias if the mail originates from the local host, or is +# hostname qualified, but otherwise it will be rejected. +# These are implemented in the local alias map. +# +# The alias for local 'root' is defined elsewhere. In this file, we +# define the virtual 'root@domain' (which can be overwritten by +# defining an entry for root in node property mx.aliases). # class site_postfix::mx::static_aliases { $mx = hiera('mx') - $aliases = $mx['aliases'] + $root_recipients = hiera('contacts') # - # Predefined aliases. - # - # Defines which mail addresses shouldn't be available and where they should - # fwd - # - # TODO: reconcile this with the node property webapp.forbidden_usernames + # LOCAL ALIASES # + # NOTE: if you remove one of these, they will still appear in the # /etc/aliases file - # + $local_aliases = [ + 'admin', 'administrator', 'bin', 'cron', 'games', 'ftp', 'lp', 'maildrop', + 'mysql', 'news', 'nobody', 'noc', 'postgresql', 'ssladmin', 'sys', + 'usenet', 'uucp', 'www', 'www-data' + ] + postfix::mailalias { - [ 'abuse', 'admin', 'arin-admin', 'administrator', 'bin', 'cron', - 'certmaster', 'domainadmin', 'games', 'ftp', 'hostmaster', 'lp', - 'maildrop', 'mysql', 'news', 'nobody', 'noc', 'postmaster', 'postgresql', - 'security', 'ssladmin', 'sys', 'usenet', 'uucp', 'webmaster', 'www', - 'www-data', - ]: + $local_aliases: ensure => present, recipient => 'root' } # - # Custom static virtual aliases. + # PUBLIC ALIASES # + + $public_aliases = $mx['aliases'] + + $default_public_aliases = { + 'root' => $root_recipients, + 'abuse' => 'postmaster', + 'arin-admin' => 'root', + 'certmaster' => 'hostmaster', + 'domainadmin' => 'hostmaster', + 'hostmaster' => 'root', + 'mailer-daemon' => 'postmaster', + 'postmaster' => 'root', + 'security' => 'root', + 'webmaster' => 'hostmaster', + } + + $aliases = merge($default_public_aliases, $public_aliases) + exec { 'postmap_virtual_aliases': command => '/usr/sbin/postmap /etc/postfix/virtual-aliases', refreshonly => true, diff --git a/puppet/modules/site_postfix/templates/virtual-aliases.erb b/puppet/modules/site_postfix/templates/virtual-aliases.erb index c474e734..8373de97 100644 --- a/puppet/modules/site_postfix/templates/virtual-aliases.erb +++ b/puppet/modules/site_postfix/templates/virtual-aliases.erb @@ -1,8 +1,7 @@ # # This file is managed by puppet. # -# This is a map of custom, non-standard aliases. The contents of this file -# are derived from the node property `mx.aliases`. +# These virtual aliases take precedence over all other aliases. # # -- cgit v1.2.3 From 14bcd5b08e3059e44e90f080c29fc6e8054cf193 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 24 Sep 2015 13:34:49 -0700 Subject: do not remove /var/log/leap/mx.log.*, this is where leap_mx is logging. --- puppet/modules/site_config/manifests/remove_files.pp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/puppet/modules/site_config/manifests/remove_files.pp b/puppet/modules/site_config/manifests/remove_files.pp index e2ab3c2e..776c3731 100644 --- a/puppet/modules/site_config/manifests/remove_files.pp +++ b/puppet/modules/site_config/manifests/remove_files.pp @@ -11,6 +11,16 @@ class site_config::remove_files { + # + # Platform 0.8 removals + # + + tidy { + '/etc/apache/sites-enabled/leap_webapp.conf': + notify => Service['apache']; + } + + # # Platform 0.7 removals # @@ -23,16 +33,10 @@ class site_config::remove_files { '/etc/logrotate.d/mx':; '/etc/logrotate.d/stunnel':; '/var/log/stunnel4/stunnel.log':; - '/etc/apache/sites-enabled/leap_webapp.conf': - notify => Service['apache']; 'leap_mx': path => '/var/log/', recurse => true, matches => 'leap_mx*'; - 'leap_mx_rotate': - path => '/var/log/leap/', - recurse => true, - matches => [ 'mx.log.[0-9]', 'mx.log.[0-9]?', 'mx.log.[6-9]?gz']; '/srv/leap/webapp/public/provider.json':; '/srv/leap/couchdb/designs/tmp_users': recurse => true, -- cgit v1.2.3 From 023d62631eff760237dd7493d9fded1c9d8bfc95 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 24 Sep 2015 15:12:46 -0700 Subject: add spf to compile zone, closes #5925 --- lib/leap_cli/commands/compile.rb | 66 +++++++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/lib/leap_cli/commands/compile.rb b/lib/leap_cli/commands/compile.rb index 8f6c7769..c388e5c3 100644 --- a/lib/leap_cli/commands/compile.rb +++ b/lib/leap_cli/commands/compile.rb @@ -265,56 +265,86 @@ remove this directory if you don't use it. # serial is any number less than 2^32 (4294967296) # def compile_zone_file - provider = manager.env('default').provider + provider = manager.env('default').provider hosts_seen = {} - f = $stdout - f.puts ZONE_HEADER % {:domain => provider.domain, :ns => provider.domain, :contact => provider.contacts.default.first.sub('@','.')} - max_width = manager.nodes.values.inject(0) {|max, node| [max, relative_hostname(node.domain.full, provider).length].max } - put_line = lambda do |host, line| - host = '@' if host == '' - f.puts("%-#{max_width}s %s" % [host, line]) - end + lines = [] + + # + # header + # + lines << ZONE_HEADER % {:domain => provider.domain, :ns => provider.domain, :contact => provider.contacts.default.first.sub('@','.')} - f.puts ORIGIN_HEADER + # + # common records + # + lines << ORIGIN_HEADER # 'A' records for primary domain manager.nodes[:environment => '!local'].each_node do |node| if node.dns['aliases'] && node.dns.aliases.include?(provider.domain) - put_line.call "", "IN A #{node.ip_address}" + lines << ["@", "IN A #{node.ip_address}"] end end - # NS records if provider['dns'] && provider.dns['nameservers'] provider.dns.nameservers.each do |ns| - put_line.call "", "IN NS #{ns}." + lines << ["@", "IN NS #{ns}."] end end - # all other records + # environment records manager.environment_names.each do |env| next if env == 'local' nodes = manager.nodes[:environment => env] next unless nodes.any? - f.puts ENV_HEADER % (env.nil? ? 'default' : env) + spf = nil + lines << ENV_HEADER % (env.nil? ? 'default' : env) nodes.each_node do |node| if node.dns.public - hostname = relative_hostname(node.domain.full, provider) - put_line.call relative_hostname(node.domain.full, provider), "IN A #{node.ip_address}" + lines << [relative_hostname(node.domain.full, provider), "IN A #{node.ip_address}"] end if node.dns['aliases'] node.dns.aliases.each do |host_alias| if host_alias != node.domain.full && host_alias != provider.domain - put_line.call relative_hostname(host_alias, provider), "IN A #{node.ip_address}" + lines << [relative_hostname(host_alias, provider), "IN A #{node.ip_address}"] end end end if node.services.include? 'mx' - put_line.call relative_hostname(node.domain.full_suffix, provider), "IN MX 10 #{relative_hostname(node.domain.full, provider)}" + mx_domain = relative_hostname(node.domain.full_suffix, provider) + lines << [mx_domain, "IN MX 10 #{relative_hostname(node.domain.full, provider)}"] + spf ||= [mx_domain, spf_record(node)] end end + lines << spf if spf + end + + # print the lines + max_width = lines.inject(0) {|max, line| line.is_a?(Array) ? [max, line[0].length].max : max} + lines.each do |host, line| + if line.nil? + puts(host) + else + host = '@' if host == '' + puts("%-#{max_width}s %s" % [host, line]) + end end end + # + # allow mail from any mx node, plus the webapp nodes. + # + # TODO: ipv6 + # + def spf_record(node) + ips = node.nodes_like_me['services' => 'webapp'].values.collect {|n| + "ip4:" + n.ip_address + } + # TXT strings may not be longer than 255 characters, although + # you can chain multiple strings together. + strings = "v=spf1 MX #{ips.join(' ')} -all".scan(/.{1,255}/).join('" "') + %(IN TXT "#{strings}") + end + ENV_HEADER = %[ ;; ;; ENVIRONMENT %s -- cgit v1.2.3 From dc87df7ca0eac492fcd65638b8e72b6e02f7bb4f Mon Sep 17 00:00:00 2001 From: varac Date: Mon, 28 Sep 2015 13:29:43 +0200 Subject: [feat] Vagrant: forward leap_web ports 443 ad 80 --- Vagrantfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Vagrantfile b/Vagrantfile index c9c68284..ba5451aa 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -9,6 +9,11 @@ Vagrant.configure("2") do |vagrant_config| config.vm.box = "LEAP/wheezy" #config.vm.network :private_network, ip: "10.5.5.102" + + # forward leap_web ports + config.vm.network "forwarded_port", guest: 80, host:8080 + config.vm.network "forwarded_port", guest: 443, host:4443 + config.vm.provider "virtualbox" do |v| v.memory = 1024 v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] -- cgit v1.2.3 From 25f7aabc528f9c6804c9b7cf7a1cc6335d43a119 Mon Sep 17 00:00:00 2001 From: ankonym Date: Mon, 28 Sep 2015 13:50:08 +0200 Subject: Modify config.yml.erb to include the invite code option --- puppet/modules/site_webapp/templates/config.yml.erb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/puppet/modules/site_webapp/templates/config.yml.erb b/puppet/modules/site_webapp/templates/config.yml.erb index 5cb436fc..19ed6b7b 100644 --- a/puppet/modules/site_webapp/templates/config.yml.erb +++ b/puppet/modules/site_webapp/templates/config.yml.erb @@ -21,7 +21,8 @@ production = { "default_service_level" => @webapp['default_service_level'], "service_levels" => @webapp['service_levels'], "allow_registration" => @webapp['allow_registration'], - "handle_blacklist" => @webapp['forbidden_usernames'] + "handle_blacklist" => @webapp['forbidden_usernames'], + "invite_required" => @webapp['invite_required'] } if @webapp['engines'] && @webapp['engines'].any? -- cgit v1.2.3 From 3224a73ec6b2f06cf4c43f86d5b7673e442043dd Mon Sep 17 00:00:00 2001 From: ankonym Date: Mon, 28 Sep 2015 15:42:08 +0200 Subject: Create invite code db and design docs --- .../files/designs/invite_codes/InviteCode.json | 22 ++++++++++++++++++++++ .../modules/site_couchdb/manifests/create_dbs.pp | 9 +++++++++ puppet/modules/site_couchdb/manifests/designs.pp | 13 +++++++------ 3 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json diff --git a/puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json b/puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json new file mode 100644 index 00000000..006c1ea1 --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json @@ -0,0 +1,22 @@ +{ + "_id": "_design/InviteCode", + "language": "javascript", + "views": { + "by__id": { + "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['_id'] != null)) {\n emit(doc['_id'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_invite_code": { + "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['invite_code'] != null)) {\n emit(doc['invite_code'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_invite_count": { + "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['invite_count'] != null)) {\n emit(doc['invite_count'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "all": { + "map": " function(doc) {\n if (doc['type'] == 'InviteCode') {\n emit(doc._id, null);\n }\n }\n" + } + }, + "couchrest-hash": "83fb8f504520b4a9c7ddbb7928cd0ce3" +} \ No newline at end of file diff --git a/puppet/modules/site_couchdb/manifests/create_dbs.pp b/puppet/modules/site_couchdb/manifests/create_dbs.pp index eea4bbf5..a2d1c655 100644 --- a/puppet/modules/site_couchdb/manifests/create_dbs.pp +++ b/puppet/modules/site_couchdb/manifests/create_dbs.pp @@ -90,4 +90,13 @@ class site_couchdb::create_dbs { members => "{ \"names\": [\"${site_couchdb::couchdb_webapp_user}\"], \"roles\": [\"replication\"] }", require => Couchdb::Query::Setup['localhost'] } + + ## invite_codes db + ## store invite codes for new signups + ## r/w: webapp + couchdb::create_db { 'invite_codes': + members => "{ \"names\": [\"${site_couchdb::couchdb_webapp_user}\"], \"roles\": [\"replication\"] }", + require => Couchdb::Query::Setup['localhost'] + } + } diff --git a/puppet/modules/site_couchdb/manifests/designs.pp b/puppet/modules/site_couchdb/manifests/designs.pp index 1ab1c6a1..e5fd94c6 100644 --- a/puppet/modules/site_couchdb/manifests/designs.pp +++ b/puppet/modules/site_couchdb/manifests/designs.pp @@ -12,12 +12,13 @@ class site_couchdb::designs { } site_couchdb::upload_design { - 'customers': design => 'customers/Customer.json'; - 'identities': design => 'identities/Identity.json'; - 'tickets': design => 'tickets/Ticket.json'; - 'messages': design => 'messages/Message.json'; - 'users': design => 'users/User.json'; - 'tmp_users': design => 'users/User.json'; + 'customers': design => 'customers/Customer.json'; + 'identities': design => 'identities/Identity.json'; + 'tickets': design => 'tickets/Ticket.json'; + 'messages': design => 'messages/Message.json'; + 'users': design => 'users/User.json'; + 'tmp_users': design => 'users/User.json'; + 'invite_codes': design => 'invite_codes/InviteCode.json'; 'shared_docs': db => 'shared', design => 'shared/docs.json'; -- cgit v1.2.3 From 35c122900c52858b25e4ff8117b8f1eff47304a5 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 15 Sep 2015 11:52:20 -0400 Subject: Fix server-status availability to tor hidden services (#7456) Make the server-status information unavailable by putting the vhost on a port that isn't configured as available to the tor hidden-service. Change-Id: Idd3bfefb5b7fc26fb0a8cf48cdf6afc68a4192bb --- puppet/modules/site_apache/manifests/common.pp | 21 +--------------- puppet/modules/site_apache/manifests/common/tls.pp | 6 +++++ puppet/modules/site_nagios/manifests/server.pp | 1 + puppet/modules/site_static/manifests/init.pp | 13 ++++++---- .../modules/site_webapp/files/server-status.conf | 28 ++++++++++++++++++++++ puppet/modules/site_webapp/manifests/apache.pp | 3 ++- .../modules/site_webapp/manifests/common_vhost.pp | 18 ++++++++++++++ .../site_webapp/manifests/hidden_service.pp | 10 ++++++-- 8 files changed, 73 insertions(+), 27 deletions(-) create mode 100644 puppet/modules/site_apache/manifests/common/tls.pp create mode 100644 puppet/modules/site_webapp/files/server-status.conf create mode 100644 puppet/modules/site_webapp/manifests/common_vhost.pp diff --git a/puppet/modules/site_apache/manifests/common.pp b/puppet/modules/site_apache/manifests/common.pp index 2b83ffa5..64beb231 100644 --- a/puppet/modules/site_apache/manifests/common.pp +++ b/puppet/modules/site_apache/manifests/common.pp @@ -1,27 +1,8 @@ class site_apache::common { - # installs x509 cert + key and common config - # that both nagios + leap webapp use - - $web_domain = hiera('domain') - $domain_name = $web_domain['name'] - - include x509::variables - include site_config::x509::commercial::cert - include site_config::x509::commercial::key - include site_config::x509::commercial::ca - - Class['Site_config::X509::Commercial::Key'] ~> Service[apache] - Class['Site_config::X509::Commercial::Cert'] ~> Service[apache] - Class['Site_config::X509::Commercial::Ca'] ~> Service[apache] include site_apache::module::rewrite class { '::apache': no_default_site => true, ssl => true } - apache::vhost::file { - 'common': - content => template('site_apache/vhosts.d/common.conf.erb') - } - - apache::config::include{ 'ssl_common.inc': } + include site_apache::common::tls } diff --git a/puppet/modules/site_apache/manifests/common/tls.pp b/puppet/modules/site_apache/manifests/common/tls.pp new file mode 100644 index 00000000..040868bf --- /dev/null +++ b/puppet/modules/site_apache/manifests/common/tls.pp @@ -0,0 +1,6 @@ +class site_apache::common::tls { + # class to setup common SSL configurations + + apache::config::include{ 'ssl_common.inc': } + +} diff --git a/puppet/modules/site_nagios/manifests/server.pp b/puppet/modules/site_nagios/manifests/server.pp index cb6c8d95..60a471b7 100644 --- a/puppet/modules/site_nagios/manifests/server.pp +++ b/puppet/modules/site_nagios/manifests/server.pp @@ -32,6 +32,7 @@ class site_nagios::server inherits nagios::base { } include site_apache::common + include site_webapp::common_vhost include site_apache::module::headers File ['nagios_htpasswd'] { diff --git a/puppet/modules/site_static/manifests/init.pp b/puppet/modules/site_static/manifests/init.pp index 1efc510b..f69ffba7 100644 --- a/puppet/modules/site_static/manifests/init.pp +++ b/puppet/modules/site_static/manifests/init.pp @@ -9,6 +9,7 @@ class site_static { $domains = $static['domains'] $formats = $static['formats'] $bootstrap = $static['bootstrap_files'] + $tor = hiera('tor', false) if $bootstrap['enabled'] { $bootstrap_domain = $bootstrap['domain'] @@ -27,14 +28,11 @@ class site_static { } } - class { '::apache': no_default_site => true, ssl => true } include site_apache::module::headers include site_apache::module::alias include site_apache::module::expires include site_apache::module::removeip - include site_apache::module::rewrite - apache::config::include{ 'ssl_common.inc': } - + include site_apache::common include site_config::ruby::dev if (member($formats, 'rack')) { @@ -57,6 +55,13 @@ class site_static { create_resources(site_static::domain, $domains) + if $tor { + $hidden_service = $tor['hidden_service'] + if $hidden_service['active'] { + include site_webapp::hidden_service + } + } + include site_shorewall::defaults include site_shorewall::service::http include site_shorewall::service::https diff --git a/puppet/modules/site_webapp/files/server-status.conf b/puppet/modules/site_webapp/files/server-status.conf new file mode 100644 index 00000000..84cb9ae0 --- /dev/null +++ b/puppet/modules/site_webapp/files/server-status.conf @@ -0,0 +1,28 @@ +# Keep track of extended status information for each request +ExtendedStatus On + +# Determine if mod_status displays the first 63 characters of a request or +# the last 63, assuming the request itself is greater than 63 chars. +# Default: Off +#SeeRequestTail On + +Listen 127.0.0.1:8162 +NameVirtualHost 127.0.0.1:8162 + + + + + SetHandler server-status + Order deny,allow + Deny from all + Allow from 127.0.0.1 + + + + + + + # Show Proxy LoadBalancer status in mod_status + ProxyStatus On + + diff --git a/puppet/modules/site_webapp/manifests/apache.pp b/puppet/modules/site_webapp/manifests/apache.pp index 93e172a0..ddd04a91 100644 --- a/puppet/modules/site_webapp/manifests/apache.pp +++ b/puppet/modules/site_webapp/manifests/apache.pp @@ -15,12 +15,13 @@ class site_webapp::apache { include site_apache::module::alias include site_apache::module::expires include site_apache::module::removeip + include site_webapp::common_vhost class { 'passenger': use_munin => false } apache::vhost::file { 'api': - content => template('site_apache/vhosts.d/api.conf.erb') + content => template('site_apache/vhosts.d/api.conf.erb'); } } diff --git a/puppet/modules/site_webapp/manifests/common_vhost.pp b/puppet/modules/site_webapp/manifests/common_vhost.pp new file mode 100644 index 00000000..c57aad57 --- /dev/null +++ b/puppet/modules/site_webapp/manifests/common_vhost.pp @@ -0,0 +1,18 @@ +class site_webapp::common_vhost { + # installs x509 cert + key and common config + # that both nagios + leap webapp use + + include x509::variables + include site_config::x509::commercial::cert + include site_config::x509::commercial::key + include site_config::x509::commercial::ca + + Class['Site_config::X509::Commercial::Key'] ~> Service[apache] + Class['Site_config::X509::Commercial::Cert'] ~> Service[apache] + Class['Site_config::X509::Commercial::Ca'] ~> Service[apache] + + apache::vhost::file { + 'common': + content => template('site_apache/vhosts.d/common.conf.erb') + } +} diff --git a/puppet/modules/site_webapp/manifests/hidden_service.pp b/puppet/modules/site_webapp/manifests/hidden_service.pp index 16b6e2e7..99a756ca 100644 --- a/puppet/modules/site_webapp/manifests/hidden_service.pp +++ b/puppet/modules/site_webapp/manifests/hidden_service.pp @@ -32,12 +32,18 @@ class site_webapp::hidden_service { owner => 'debian-tor', group => 'debian-tor', mode => '0600'; + + '/etc/apache2/mods-enabled/status.conf': + ensure => absent, + notify => Service['apache']; } apache::vhost::file { 'hidden_service': - content => template('site_apache/vhosts.d/hidden_service.conf.erb') + content => template('site_apache/vhosts.d/hidden_service.conf.erb'); + 'server_status': + vhost_source => 'modules/site_webapp/server-status.conf'; } include site_shorewall::tor -} \ No newline at end of file +} -- cgit v1.2.3 From 8b0910f1caf19884b6b46976b72536ee1f570ed5 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 15 Sep 2015 11:52:20 -0400 Subject: Fix server-status availability to tor hidden services (#7456) Make the server-status information unavailable by putting the vhost on a port that isn't configured as available to the tor hidden-service. Change-Id: Idd3bfefb5b7fc26fb0a8cf48cdf6afc68a4192bb --- puppet/modules/site_apache/manifests/common.pp | 21 +--------------- puppet/modules/site_apache/manifests/common/tls.pp | 6 +++++ puppet/modules/site_nagios/manifests/server.pp | 1 + puppet/modules/site_static/manifests/init.pp | 13 ++++++---- .../modules/site_webapp/files/server-status.conf | 28 ++++++++++++++++++++++ puppet/modules/site_webapp/manifests/apache.pp | 3 ++- .../modules/site_webapp/manifests/common_vhost.pp | 18 ++++++++++++++ .../site_webapp/manifests/hidden_service.pp | 10 ++++++-- 8 files changed, 73 insertions(+), 27 deletions(-) create mode 100644 puppet/modules/site_apache/manifests/common/tls.pp create mode 100644 puppet/modules/site_webapp/files/server-status.conf create mode 100644 puppet/modules/site_webapp/manifests/common_vhost.pp diff --git a/puppet/modules/site_apache/manifests/common.pp b/puppet/modules/site_apache/manifests/common.pp index 2b83ffa5..64beb231 100644 --- a/puppet/modules/site_apache/manifests/common.pp +++ b/puppet/modules/site_apache/manifests/common.pp @@ -1,27 +1,8 @@ class site_apache::common { - # installs x509 cert + key and common config - # that both nagios + leap webapp use - - $web_domain = hiera('domain') - $domain_name = $web_domain['name'] - - include x509::variables - include site_config::x509::commercial::cert - include site_config::x509::commercial::key - include site_config::x509::commercial::ca - - Class['Site_config::X509::Commercial::Key'] ~> Service[apache] - Class['Site_config::X509::Commercial::Cert'] ~> Service[apache] - Class['Site_config::X509::Commercial::Ca'] ~> Service[apache] include site_apache::module::rewrite class { '::apache': no_default_site => true, ssl => true } - apache::vhost::file { - 'common': - content => template('site_apache/vhosts.d/common.conf.erb') - } - - apache::config::include{ 'ssl_common.inc': } + include site_apache::common::tls } diff --git a/puppet/modules/site_apache/manifests/common/tls.pp b/puppet/modules/site_apache/manifests/common/tls.pp new file mode 100644 index 00000000..040868bf --- /dev/null +++ b/puppet/modules/site_apache/manifests/common/tls.pp @@ -0,0 +1,6 @@ +class site_apache::common::tls { + # class to setup common SSL configurations + + apache::config::include{ 'ssl_common.inc': } + +} diff --git a/puppet/modules/site_nagios/manifests/server.pp b/puppet/modules/site_nagios/manifests/server.pp index cb6c8d95..60a471b7 100644 --- a/puppet/modules/site_nagios/manifests/server.pp +++ b/puppet/modules/site_nagios/manifests/server.pp @@ -32,6 +32,7 @@ class site_nagios::server inherits nagios::base { } include site_apache::common + include site_webapp::common_vhost include site_apache::module::headers File ['nagios_htpasswd'] { diff --git a/puppet/modules/site_static/manifests/init.pp b/puppet/modules/site_static/manifests/init.pp index 1efc510b..f69ffba7 100644 --- a/puppet/modules/site_static/manifests/init.pp +++ b/puppet/modules/site_static/manifests/init.pp @@ -9,6 +9,7 @@ class site_static { $domains = $static['domains'] $formats = $static['formats'] $bootstrap = $static['bootstrap_files'] + $tor = hiera('tor', false) if $bootstrap['enabled'] { $bootstrap_domain = $bootstrap['domain'] @@ -27,14 +28,11 @@ class site_static { } } - class { '::apache': no_default_site => true, ssl => true } include site_apache::module::headers include site_apache::module::alias include site_apache::module::expires include site_apache::module::removeip - include site_apache::module::rewrite - apache::config::include{ 'ssl_common.inc': } - + include site_apache::common include site_config::ruby::dev if (member($formats, 'rack')) { @@ -57,6 +55,13 @@ class site_static { create_resources(site_static::domain, $domains) + if $tor { + $hidden_service = $tor['hidden_service'] + if $hidden_service['active'] { + include site_webapp::hidden_service + } + } + include site_shorewall::defaults include site_shorewall::service::http include site_shorewall::service::https diff --git a/puppet/modules/site_webapp/files/server-status.conf b/puppet/modules/site_webapp/files/server-status.conf new file mode 100644 index 00000000..84cb9ae0 --- /dev/null +++ b/puppet/modules/site_webapp/files/server-status.conf @@ -0,0 +1,28 @@ +# Keep track of extended status information for each request +ExtendedStatus On + +# Determine if mod_status displays the first 63 characters of a request or +# the last 63, assuming the request itself is greater than 63 chars. +# Default: Off +#SeeRequestTail On + +Listen 127.0.0.1:8162 +NameVirtualHost 127.0.0.1:8162 + + + + + SetHandler server-status + Order deny,allow + Deny from all + Allow from 127.0.0.1 + + + + + + + # Show Proxy LoadBalancer status in mod_status + ProxyStatus On + + diff --git a/puppet/modules/site_webapp/manifests/apache.pp b/puppet/modules/site_webapp/manifests/apache.pp index 93e172a0..ddd04a91 100644 --- a/puppet/modules/site_webapp/manifests/apache.pp +++ b/puppet/modules/site_webapp/manifests/apache.pp @@ -15,12 +15,13 @@ class site_webapp::apache { include site_apache::module::alias include site_apache::module::expires include site_apache::module::removeip + include site_webapp::common_vhost class { 'passenger': use_munin => false } apache::vhost::file { 'api': - content => template('site_apache/vhosts.d/api.conf.erb') + content => template('site_apache/vhosts.d/api.conf.erb'); } } diff --git a/puppet/modules/site_webapp/manifests/common_vhost.pp b/puppet/modules/site_webapp/manifests/common_vhost.pp new file mode 100644 index 00000000..c57aad57 --- /dev/null +++ b/puppet/modules/site_webapp/manifests/common_vhost.pp @@ -0,0 +1,18 @@ +class site_webapp::common_vhost { + # installs x509 cert + key and common config + # that both nagios + leap webapp use + + include x509::variables + include site_config::x509::commercial::cert + include site_config::x509::commercial::key + include site_config::x509::commercial::ca + + Class['Site_config::X509::Commercial::Key'] ~> Service[apache] + Class['Site_config::X509::Commercial::Cert'] ~> Service[apache] + Class['Site_config::X509::Commercial::Ca'] ~> Service[apache] + + apache::vhost::file { + 'common': + content => template('site_apache/vhosts.d/common.conf.erb') + } +} diff --git a/puppet/modules/site_webapp/manifests/hidden_service.pp b/puppet/modules/site_webapp/manifests/hidden_service.pp index 16b6e2e7..99a756ca 100644 --- a/puppet/modules/site_webapp/manifests/hidden_service.pp +++ b/puppet/modules/site_webapp/manifests/hidden_service.pp @@ -32,12 +32,18 @@ class site_webapp::hidden_service { owner => 'debian-tor', group => 'debian-tor', mode => '0600'; + + '/etc/apache2/mods-enabled/status.conf': + ensure => absent, + notify => Service['apache']; } apache::vhost::file { 'hidden_service': - content => template('site_apache/vhosts.d/hidden_service.conf.erb') + content => template('site_apache/vhosts.d/hidden_service.conf.erb'); + 'server_status': + vhost_source => 'modules/site_webapp/server-status.conf'; } include site_shorewall::tor -} \ No newline at end of file +} -- cgit v1.2.3 From 5cc27111151083d0a8e5098505a0024c2d2ea201 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 24 Sep 2015 11:25:48 -0400 Subject: fix missing service dependency error this tidy should only happen on webapp nodes Change-Id: I56faac4fa28fde9dcad7ce9a6ed0d684630a556e --- puppet/modules/site_config/manifests/remove_files.pp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/puppet/modules/site_config/manifests/remove_files.pp b/puppet/modules/site_config/manifests/remove_files.pp index 776c3731..8b2f9541 100644 --- a/puppet/modules/site_config/manifests/remove_files.pp +++ b/puppet/modules/site_config/manifests/remove_files.pp @@ -43,6 +43,13 @@ class site_config::remove_files { rmdirs => true; } + if member($::services, 'webapp') { + tidy { + '/etc/apache/sites-enabled/leap_webapp.conf': + notify => Service['apache']; + } + } + # leax-mx logged to /var/log/leap_mx.log in the past # we need to use a dumb exec here because file_line doesn't # allow removing lines that match a regex in the current version -- cgit v1.2.3 From 4fc7419598a3baf564f063b7330b9cf9115420b5 Mon Sep 17 00:00:00 2001 From: varac Date: Thu, 1 Oct 2015 12:06:02 +0200 Subject: [feat] Create-user-db: use couchdb admin rights - create soledad-admin user - deploy netrc file for userdb creation - Move soledad-server.conf from /etc/leap to /etc/soledad - make soledad-server.conf group-accessible for the soledad group, so the soledad-admin user can read it - Resolves: #7502 --- .../modules/site_config/manifests/remove_files.pp | 1 + puppet/modules/site_couchdb/manifests/setup.pp | 35 +++++++++++++++------- puppet/modules/soledad/manifests/init.pp | 17 +++++++++-- puppet/modules/soledad/manifests/server.pp | 21 ++++++++----- .../soledad/templates/soledad-server.conf.erb | 5 ++-- 5 files changed, 56 insertions(+), 23 deletions(-) diff --git a/puppet/modules/site_config/manifests/remove_files.pp b/puppet/modules/site_config/manifests/remove_files.pp index 776c3731..3532c0f0 100644 --- a/puppet/modules/site_config/manifests/remove_files.pp +++ b/puppet/modules/site_config/manifests/remove_files.pp @@ -41,6 +41,7 @@ class site_config::remove_files { '/srv/leap/couchdb/designs/tmp_users': recurse => true, rmdirs => true; + '/etc/leap/soledad-server.conf':; } # leax-mx logged to /var/log/leap_mx.log in the past diff --git a/puppet/modules/site_couchdb/manifests/setup.pp b/puppet/modules/site_couchdb/manifests/setup.pp index 69bd1c6a..fef48505 100644 --- a/puppet/modules/site_couchdb/manifests/setup.pp +++ b/puppet/modules/site_couchdb/manifests/setup.pp @@ -12,27 +12,40 @@ class site_couchdb::setup { $user = $site_couchdb::couchdb_admin_user - # /etc/couchdb/couchdb-admin.netrc is deployed by couchdb::query::setup - # we symlink to couchdb.netrc for puppet commands. - # we symlink this to /root/.netrc for couchdb_scripts (eg. backup) - # and makes life easier for the admin (i.e. using curl/wget without - # passing credentials) + # setup /etc/couchdb/couchdb-admin.netrc for couchdb admin access + couchdb::query::setup { 'localhost': + user => $user, + pw => $site_couchdb::couchdb_admin_pw + } + + # We symlink /etc/couchdb/couchdb-admin.netrc to /etc/couchdb/couchdb.netrc + # for puppet commands, and to to /root/.netrc for couchdb_scripts + # (eg. backup) and to makes life easier for the admin on the command line + # (i.e. using curl/wget without passing credentials) file { '/etc/couchdb/couchdb.netrc': ensure => link, target => "/etc/couchdb/couchdb-${user}.netrc"; - '/root/.netrc': ensure => link, target => '/etc/couchdb/couchdb.netrc'; + } - '/srv/leap/couchdb': - ensure => directory + # setup /etc/couchdb/couchdb-soledad-admin.netrc file for couchdb admin + # access, accessible only for the soledad-admin user to create soledad + # userdbs + file { '/etc/couchdb/couchdb-soledad-admin.netrc': + content => "machine localhost login ${user} password ${site_couchdb::couchdb_admin_pw}", + mode => '0400', + owner => 'soledad-admin', + group => 'root', + require => [ Package['couchdb'], User['soledad-admin'] ]; } - couchdb::query::setup { 'localhost': - user => $user, - pw => $site_couchdb::couchdb_admin_pw, + # Checkout couchdb_scripts repo + file { + '/srv/leap/couchdb': + ensure => directory } vcsrepo { '/srv/leap/couchdb/scripts': diff --git a/puppet/modules/soledad/manifests/init.pp b/puppet/modules/soledad/manifests/init.pp index 7cf0b729..6a2c328e 100644 --- a/puppet/modules/soledad/manifests/init.pp +++ b/puppet/modules/soledad/manifests/init.pp @@ -1,18 +1,29 @@ +# set up users, group and directories for soledad-server +# although the soledad users are already created by the +# soledad-server package class soledad { group { 'soledad': - ensure => present, - allowdupe => false; + ensure => present, + system => true, } user { 'soledad': ensure => present, - allowdupe => false, + system => true, gid => 'soledad', home => '/srv/leap/soledad', require => Group['soledad']; } + user { 'soledad-admin': + ensure => present, + system => true, + gid => 'soledad', + home => '/srv/leap/soledad', + require => Group['soledad']; + } + file { '/srv/leap/soledad': ensure => directory, diff --git a/puppet/modules/soledad/manifests/server.pp b/puppet/modules/soledad/manifests/server.pp index b71fab69..e437c8f2 100644 --- a/puppet/modules/soledad/manifests/server.pp +++ b/puppet/modules/soledad/manifests/server.pp @@ -1,3 +1,4 @@ +# setup soledad-server class soledad::server { tag 'leap_service' include soledad @@ -22,13 +23,19 @@ class soledad::server { # SOLEDAD CONFIG # - file { '/etc/leap/soledad-server.conf': - content => template('soledad/soledad-server.conf.erb'), - owner => 'soledad', - group => 'soledad', - mode => '0600', - notify => Service['soledad-server'], - require => Class['soledad']; + file { + '/etc/soledad': + ensure => directory, + owner => 'root', + group => 'root', + mode => '0755'; + '/etc/soledad/soledad-server.conf': + content => template('soledad/soledad-server.conf.erb'), + owner => 'soledad', + group => 'soledad', + mode => '0640', + notify => Service['soledad-server'], + require => Class['soledad']; } package { $sources['soledad']['package']: diff --git a/puppet/modules/soledad/templates/soledad-server.conf.erb b/puppet/modules/soledad/templates/soledad-server.conf.erb index 47d1f6e4..42cf44d8 100644 --- a/puppet/modules/soledad/templates/soledad-server.conf.erb +++ b/puppet/modules/soledad/templates/soledad-server.conf.erb @@ -1,3 +1,4 @@ [soledad-server] -couch_url = http://<%= @couchdb_user %>:<%= @couchdb_password %>@<%= @couchdb_host %>:<%= @couchdb_port %> - +couch_url = http://<%= @couchdb_user %>:<%= @couchdb_password %>@<%= @couchdb_host %>:<%= @couchdb_port %> +create_cmd = sudo -u soledad-admin /usr/bin/create-user-db +admin_netrc = /etc/couchdb/couchdb-soledad-admin.netrc -- cgit v1.2.3 From 276b77cdcc0d169b84e046afe8763e2c52ff76fb Mon Sep 17 00:00:00 2001 From: varac Date: Mon, 5 Oct 2015 15:22:25 +0200 Subject: [feat] remove tapicero leftovers Soledad now creates user-dbs, which has been done by tapicero in the past. we need to remove any leftovers from tapicero. --- puppet/manifests/site.pp | 1 - .../files/agent/logwatch/bigcouch.cfg | 2 +- .../files/agent/logwatch/tapicero.cfg | 11 -- .../site_check_mk/manifests/agent/tapicero.pp | 26 ---- puppet/modules/site_config/manifests/default.pp | 7 +- puppet/modules/site_config/manifests/remove.pp | 5 + .../modules/site_config/manifests/remove/files.pp | 74 +++++++++++ .../site_config/manifests/remove/tapicero.pp | 57 +++++++++ .../modules/site_config/manifests/remove_files.pp | 81 ------------ puppet/modules/site_couchdb/manifests/add_users.pp | 11 +- puppet/modules/site_couchdb/manifests/init.pp | 6 - puppet/modules/tapicero/files/tapicero.init | 60 --------- puppet/modules/tapicero/manifests/init.pp | 137 --------------------- .../modules/tapicero/templates/tapicero.yaml.erb | 52 -------- 14 files changed, 143 insertions(+), 387 deletions(-) delete mode 100644 puppet/modules/site_check_mk/files/agent/logwatch/tapicero.cfg delete mode 100644 puppet/modules/site_check_mk/manifests/agent/tapicero.pp create mode 100644 puppet/modules/site_config/manifests/remove.pp create mode 100644 puppet/modules/site_config/manifests/remove/files.pp create mode 100644 puppet/modules/site_config/manifests/remove/tapicero.pp delete mode 100644 puppet/modules/site_config/manifests/remove_files.pp delete mode 100755 puppet/modules/tapicero/files/tapicero.init delete mode 100644 puppet/modules/tapicero/manifests/init.pp delete mode 100644 puppet/modules/tapicero/templates/tapicero.yaml.erb diff --git a/puppet/manifests/site.pp b/puppet/manifests/site.pp index 912234ac..91dd2d3c 100644 --- a/puppet/manifests/site.pp +++ b/puppet/manifests/site.pp @@ -20,7 +20,6 @@ if member($services, 'openvpn') { if member($services, 'couchdb') { include site_couchdb - include tapicero } if member($services, 'webapp') { diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg index 95ddd2ca..0f378a5a 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg @@ -6,7 +6,7 @@ I 127.0.0.1 localhost:5984 .* ok # https://leap.se/code/issues/5246 I Shutting down group server - # ignore bigcouch conflict errors, mainly coming from tapicero creating new users + # ignore bigcouch conflict errors I Error in process.*{{nocatch,conflict} # ignore "Uncaught error in HTTP request: {exit, normal}" error # it's suppressed in later versions of bigcouch anhow diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/tapicero.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/tapicero.cfg deleted file mode 100644 index d98f5094..00000000 --- a/puppet/modules/site_check_mk/files/agent/logwatch/tapicero.cfg +++ /dev/null @@ -1,11 +0,0 @@ -/var/log/leap/tapicero.log -# Ignore transient Tapicero errors when creating a db (#6511) - I tapicero.*(Creating database|Checking security of|Writing security to|Uploading design doc to) user-.* failed (\(trying again soon\)|(twice )?due to): (RestClient::ResourceNotFound|RestClient::InternalServerError): (404 Resource Not Found|500 Internal Server Error) - C tapicero.*RestClient::InternalServerError: -# possible race condition between multiple tapicero -# instances, so we ignore it -# see https://leap.se/code/issues/5168 - I tapicero.*RestClient::PreconditionFailed: - C tapicero.*Creating database.*failed due to: - C tapicero.*failed - W tapicero.*Couch stream ended unexpectedly. diff --git a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp deleted file mode 100644 index 8505b34a..00000000 --- a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp +++ /dev/null @@ -1,26 +0,0 @@ -# sets up tapicero monitoring -class site_check_mk::agent::tapicero { - - include ::site_nagios::plugins - - # watch logs - file { '/etc/check_mk/logwatch.d/tapicero.cfg': - source => 'puppet:///modules/site_check_mk/agent/logwatch/tapicero.cfg', - } - - # local nagios plugin checks via mrpe - augeas { - 'Tapicero_Procs': - incl => '/etc/check_mk/mrpe.cfg', - lens => 'Spacevars.lns', - changes => [ - 'rm /files/etc/check_mk/mrpe.cfg/Tapicero_Procs', - "set Tapicero_Procs \"/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 --ereg-argument-array='^tapicero$'\"" ], - require => File['/etc/check_mk/mrpe.cfg']; - 'Tapicero_Heartbeat': - incl => '/etc/check_mk/mrpe.cfg', - lens => 'Spacevars.lns', - changes => 'set Tapicero_Heartbeat \'/usr/local/lib/nagios/plugins/check_last_regex_in_log -f /var/log/leap/tapicero.log -r "tapicero" -w 1200 -c 2400\'', - require => File['/etc/check_mk/mrpe.cfg']; - } -} diff --git a/puppet/modules/site_config/manifests/default.pp b/puppet/modules/site_config/manifests/default.pp index e69e4b7b..6b10dc19 100644 --- a/puppet/modules/site_config/manifests/default.pp +++ b/puppet/modules/site_config/manifests/default.pp @@ -1,3 +1,4 @@ +# common things to set up on every node class site_config::default { tag 'leap_base' @@ -29,7 +30,7 @@ class site_config::default { # i.e. openstack/aws nodes, vagrant nodes # fix dhclient from changing resolver information - if $::dhcp_enabled == 'true' { + if $::dhcp_enabled == 'true' { include site_config::dhclient } @@ -58,7 +59,9 @@ class site_config::default { # set up core leap files and directories include site_config::files - include site_config::remove_files + + # remove leftovers from previous deploys + include site_config::remove if ! member($services, 'mx') { include site_postfix::satellite diff --git a/puppet/modules/site_config/manifests/remove.pp b/puppet/modules/site_config/manifests/remove.pp new file mode 100644 index 00000000..00502c0a --- /dev/null +++ b/puppet/modules/site_config/manifests/remove.pp @@ -0,0 +1,5 @@ +# remove leftovers from previous deploys +class site_config::remove { + include site_config::remove::files + include site_config::remove::tapicero +} diff --git a/puppet/modules/site_config/manifests/remove/files.pp b/puppet/modules/site_config/manifests/remove/files.pp new file mode 100644 index 00000000..feff7c05 --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/files.pp @@ -0,0 +1,74 @@ +# +# Sometimes when we upgrade the platform, we need to ensure that files that +# the platform previously created will get removed. +# +# These file removals don't need to be kept forever: we only need to remove +# files that are present in the prior platform release. +# +# We can assume that the every node is upgraded from the previous platform +# release. +# + +class site_config::remove::files { + + # + # Platform 0.8 removals + # + + tidy { + '/etc/apache/sites-enabled/leap_webapp.conf': + notify => Service['apache']; + } + + + # + # Platform 0.7 removals + # + + tidy { + '/etc/rsyslog.d/99-tapicero.conf':; + '/etc/rsyslog.d/99-leap-mx.conf':; + '/etc/rsyslog.d/01-webapp.conf':; + '/etc/rsyslog.d/50-stunnel.conf':; + '/etc/logrotate.d/mx':; + '/etc/logrotate.d/stunnel':; + '/var/log/stunnel4/stunnel.log':; + 'leap_mx': + path => '/var/log/', + recurse => true, + matches => 'leap_mx*'; + '/srv/leap/webapp/public/provider.json':; + '/srv/leap/couchdb/designs/tmp_users': + recurse => true, + rmdirs => true; + '/etc/leap/soledad-server.conf':; + } + + # leax-mx logged to /var/log/leap_mx.log in the past + # we need to use a dumb exec here because file_line doesn't + # allow removing lines that match a regex in the current version + # of stdlib, see https://tickets.puppetlabs.com/browse/MODULES-1903 + exec { 'rm_old_leap_mx_log_destination': + command => "/bin/sed -i '/leap_mx.log/d' /etc/check_mk/logwatch.state", + onlyif => "/bin/grep -qe 'leap_mx.log' /etc/check_mk/logwatch.state" + } + + # Don't use check_mk logwatch to watch bigcouch logs anymore + # see https://leap.se/code/issues/7375 for more details + file { '/etc/check_mk/logwatch.d/bigcouch.cfg': + ensure => absent, + notify => [ + Exec['remove_bigcouch_logwatch_spoolfiles'], + Exec['remove_bigcouch_logwatch_stateline'] + ] + } + # remove leftover bigcouch logwatch spool files + exec { 'remove_bigcouch_logwatch_spoolfiles': + command => 'find /var/lib/check_mk/logwatch -name \'\\opt\\bigcouch\\var\\log\\bigcouch.log\' -exec rm {} \;', + refreshonly => true, + } + exec { 'remove_bigcouch_logwatch_stateline': + command => "sed -i '/bigcouch.log/d' /etc/check_mk/logwatch.state", + refreshonly => true, + } +} diff --git a/puppet/modules/site_config/manifests/remove/tapicero.pp b/puppet/modules/site_config/manifests/remove/tapicero.pp new file mode 100644 index 00000000..765f7428 --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/tapicero.pp @@ -0,0 +1,57 @@ +# remove tapicero leftovers from previous deploys +class site_config::remove::tapicero { + + exec { 'kill_tapicero': + onlyif => '/usr/bin/test -s /var/run/tapicero.pid', + command => '/usr/bin/pkill --pidfile /var/run/tapicero.pid' + } + + user { 'tapicero': + ensure => absent; + } + + group { 'tapicero': + ensure => absent, + require => User['tapicero']; + } + + tidy { + '/srv/leap/tapicero': + recurse => true, + require => [ Exec['kill_tapicero'] ]; + '/var/lib/leap/tapicero': + require => [ Exec['kill_tapicero'] ]; + '/var/run/tapicero': + require => [ Exec['kill_tapicero'] ]; + '/etc/leap/tapicero.yaml': + require => [ Exec['kill_tapicero'] ]; + '/etc/init.d/tapicero': + require => [ Exec['kill_tapicero'] ]; + 'tapicero_logs': + path => '/var/log/leap', + recurse => true, + matches => 'tapicero*', + require => [ Exec['kill_tapicero'] ]; + '/etc/check_mk/logwatch.d/tapicero.cfg':; + 'checkmk_logwatch_spool': + path => '/var/lib/check_mk/logwatch', + recurse => true, + matches => '*tapicero.log', + require => [ Exec['kill_tapicero'] ]; + } + + # remove local nagios plugin checks via mrpe + augeas { + 'Tapicero_Procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Tapicero_Procs', + require => File['/etc/check_mk/mrpe.cfg']; + 'Tapicero_Heartbeat': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm Tapicero_Heartbeat', + require => File['/etc/check_mk/mrpe.cfg']; + } + +} diff --git a/puppet/modules/site_config/manifests/remove_files.pp b/puppet/modules/site_config/manifests/remove_files.pp deleted file mode 100644 index 07487d6a..00000000 --- a/puppet/modules/site_config/manifests/remove_files.pp +++ /dev/null @@ -1,81 +0,0 @@ -# -# Sometimes when we upgrade the platform, we need to ensure that files that -# the platform previously created will get removed. -# -# These file removals don't need to be kept forever: we only need to remove -# files that are present in the prior platform release. -# -# We can assume that the every node is upgraded from the previous platform -# release. -# - -class site_config::remove_files { - - # - # Platform 0.8 removals - # - - tidy { - '/etc/apache/sites-enabled/leap_webapp.conf': - notify => Service['apache']; - } - - - # - # Platform 0.7 removals - # - - tidy { - '/etc/rsyslog.d/99-tapicero.conf':; - '/etc/rsyslog.d/99-leap-mx.conf':; - '/etc/rsyslog.d/01-webapp.conf':; - '/etc/rsyslog.d/50-stunnel.conf':; - '/etc/logrotate.d/mx':; - '/etc/logrotate.d/stunnel':; - '/var/log/stunnel4/stunnel.log':; - 'leap_mx': - path => '/var/log/', - recurse => true, - matches => 'leap_mx*'; - '/srv/leap/webapp/public/provider.json':; - '/srv/leap/couchdb/designs/tmp_users': - recurse => true, - rmdirs => true; - '/etc/leap/soledad-server.conf':; - } - - if member($::services, 'webapp') { - tidy { - '/etc/apache/sites-enabled/leap_webapp.conf': - notify => Service['apache']; - } - } - - # leax-mx logged to /var/log/leap_mx.log in the past - # we need to use a dumb exec here because file_line doesn't - # allow removing lines that match a regex in the current version - # of stdlib, see https://tickets.puppetlabs.com/browse/MODULES-1903 - exec { 'rm_old_leap_mx_log_destination': - command => "/bin/sed -i '/leap_mx.log/d' /etc/check_mk/logwatch.state", - onlyif => "/bin/grep -qe 'leap_mx.log' /etc/check_mk/logwatch.state" - } - - # Don't use check_mk logwatch to watch bigcouch logs anymore - # see https://leap.se/code/issues/7375 for more details - file { '/etc/check_mk/logwatch.d/bigcouch.cfg': - ensure => absent, - notify => [ - Exec['remove_bigcouch_logwatch_spoolfiles'], - Exec['remove_bigcouch_logwatch_stateline'] - ] - } - # remove leftover bigcouch logwatch spool files - exec { 'remove_bigcouch_logwatch_spoolfiles': - command => 'find /var/lib/check_mk/logwatch -name \'\\opt\\bigcouch\\var\\log\\bigcouch.log\' -exec rm {} \;', - refreshonly => true, - } - exec { 'remove_bigcouch_logwatch_stateline': - command => "sed -i '/bigcouch.log/d' /etc/check_mk/logwatch.state", - refreshonly => true, - } -} diff --git a/puppet/modules/site_couchdb/manifests/add_users.pp b/puppet/modules/site_couchdb/manifests/add_users.pp index 2f734ed4..c905316b 100644 --- a/puppet/modules/site_couchdb/manifests/add_users.pp +++ b/puppet/modules/site_couchdb/manifests/add_users.pp @@ -1,3 +1,4 @@ +# add couchdb users for all services class site_couchdb::add_users { Class['site_couchdb::create_dbs'] @@ -35,16 +36,6 @@ class site_couchdb::add_users { require => Couchdb::Query::Setup['localhost'] } - ### tapicero couchdb user - ### admin: needs to be able to create user- databases - ### read: users - couchdb::add_user { $site_couchdb::couchdb_tapicero_user: - roles => '["users"]', - pw => $site_couchdb::couchdb_tapicero_pw, - salt => $site_couchdb::couchdb_tapicero_salt, - require => Couchdb::Query::Setup['localhost'] - } - ## webapp couchdb user ## read/write: users, tokens, sessions, tickets, identities, customer couchdb::add_user { $site_couchdb::couchdb_webapp_user: diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 6b6ddd3a..1ec15f00 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -26,11 +26,6 @@ class site_couchdb { $couchdb_soledad_pw = $couchdb_soledad['password'] $couchdb_soledad_salt = $couchdb_soledad['salt'] - $couchdb_tapicero = $couchdb_users['tapicero'] - $couchdb_tapicero_user = $couchdb_tapicero['username'] - $couchdb_tapicero_pw = $couchdb_tapicero['password'] - $couchdb_tapicero_salt = $couchdb_tapicero['salt'] - $couchdb_webapp = $couchdb_users['webapp'] $couchdb_webapp_user = $couchdb_webapp['username'] $couchdb_webapp_pw = $couchdb_webapp['password'] @@ -66,6 +61,5 @@ class site_couchdb { if $couchdb_backup { include site_couchdb::backup } include site_check_mk::agent::couchdb - include site_check_mk::agent::tapicero } diff --git a/puppet/modules/tapicero/files/tapicero.init b/puppet/modules/tapicero/files/tapicero.init deleted file mode 100755 index 7a9af45f..00000000 --- a/puppet/modules/tapicero/files/tapicero.init +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh - -### BEGIN INIT INFO -# Provides: tapicero -# Required-Start: $remote_fs $syslog -# Required-Stop: $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: tapicero initscript -# Description: Controls tapicero daemon -### END INIT INFO - -PATH=/sbin:/usr/sbin:/bin:/usr/bin -BUNDLER=/usr/bin/bundle -NAME=tapicero -HOME="/srv/leap" -DAEMON="${HOME}/${NAME}/bin/${NAME}" -BUNDLE_GEMFILE="${HOME}/${NAME}/Gemfile" - -export BUNDLE_GEMFILE - -# exit if the daemon doesn't exist -[ -x "$DAEMON" ] || exit 0 - -. /lib/init/vars.sh -. /lib/lsb/init-functions - -if [ "$VERBOSE" != no ]; then - OPTIONS="--verbose" -else - OPTIONS="" -fi - -case "$1" in - start) - $BUNDLER exec $DAEMON start $OPTIONS - exit $? - ;; - stop) - $BUNDLER exec $DAEMON stop $OPTIONS - exit $? - ;; - restart) - $BUNDLER exec $DAEMON restart $OPTIONS - exit $? - ;; - reload) - $BUNDLER exec $DAEMON reload $OPTIONS - exit $? - ;; - status) - $BUNDLER exec $DAEMON status $OPTIONS - exit $? - ;; - *) - echo "Usage: /etc/init.d/$NAME {start|stop|reload|restart|status}" - exit 1 -esac - -exit 0 diff --git a/puppet/modules/tapicero/manifests/init.pp b/puppet/modules/tapicero/manifests/init.pp deleted file mode 100644 index ca8488c8..00000000 --- a/puppet/modules/tapicero/manifests/init.pp +++ /dev/null @@ -1,137 +0,0 @@ -class tapicero { - tag 'leap_service' - - $couchdb = hiera('couch') - $couchdb_port = $couchdb['port'] - - $couchdb_users = $couchdb['users'] - - $couchdb_admin_user = $couchdb_users['admin']['username'] - $couchdb_admin_password = $couchdb_users['admin']['password'] - - $couchdb_soledad_user = $couchdb_users['soledad']['username'] - $couchdb_leap_mx_user = $couchdb_users['leap_mx']['username'] - - $couchdb_mode = $couchdb['mode'] - $couchdb_replication = $couchdb['replication'] - - $sources = hiera('sources') - - Class['site_config::default'] -> Class['tapicero'] - - include site_config::ruby::dev - - # - # USER AND GROUP - # - - group { 'tapicero': - ensure => present, - allowdupe => false; - } - - user { 'tapicero': - ensure => present, - allowdupe => false, - gid => 'tapicero', - home => '/srv/leap/tapicero', - require => Group['tapicero']; - } - - # - # TAPICERO FILES - # - - file { - - # - # TAPICERO DIRECTORIES - # - - '/srv/leap/tapicero': - ensure => directory, - owner => 'tapicero', - group => 'tapicero', - require => User['tapicero']; - - '/var/lib/leap/tapicero': - ensure => directory, - owner => 'tapicero', - group => 'tapicero', - require => User['tapicero']; - - # for pid file - '/var/run/tapicero': - ensure => directory, - owner => 'tapicero', - group => 'tapicero', - require => User['tapicero']; - - # - # TAPICERO CONFIG - # - - '/etc/leap/tapicero.yaml': - content => template('tapicero/tapicero.yaml.erb'), - owner => 'tapicero', - group => 'tapicero', - mode => '0600', - notify => Service['tapicero']; - - # - # TAPICERO INIT - # - - '/etc/init.d/tapicero': - source => 'puppet:///modules/tapicero/tapicero.init', - owner => root, - group => 0, - mode => '0755', - require => Vcsrepo['/srv/leap/tapicero']; - } - - # - # TAPICERO CODE - # - - vcsrepo { '/srv/leap/tapicero': - ensure => present, - force => true, - revision => $sources['tapicero']['revision'], - provider => $sources['tapicero']['type'], - source => $sources['tapicero']['source'], - owner => 'tapicero', - group => 'tapicero', - require => [ User['tapicero'], Group['tapicero'] ], - notify => Exec['tapicero_bundler_update'] - } - - exec { 'tapicero_bundler_update': - cwd => '/srv/leap/tapicero', - command => '/bin/bash -c "/usr/bin/bundle check || /usr/bin/bundle install --path vendor/bundle --without test development"', - unless => '/usr/bin/bundle check', - user => 'tapicero', - timeout => 600, - require => [ - Class['bundler::install'], - Vcsrepo['/srv/leap/tapicero'], - Class['site_config::ruby::dev'] ], - notify => Service['tapicero']; - } - - # - # TAPICERO DAEMON - # - - service { 'tapicero': - ensure => running, - enable => true, - hasstatus => false, - hasrestart => true, - require => [ File['/etc/init.d/tapicero'], - File['/var/run/tapicero'], - Couchdb::Add_user[$::site_couchdb::couchdb_tapicero_user] ]; - } - - leap::logfile { 'tapicero': } -} diff --git a/puppet/modules/tapicero/templates/tapicero.yaml.erb b/puppet/modules/tapicero/templates/tapicero.yaml.erb deleted file mode 100644 index 8b08b49c..00000000 --- a/puppet/modules/tapicero/templates/tapicero.yaml.erb +++ /dev/null @@ -1,52 +0,0 @@ -<%- require 'json' -%> - -# -# Default configuration options for Tapicero -# - -# couch connection configuration -connection: - protocol: "http" - host: "localhost" - port: <%= @couchdb_port %> - username: <%= @couchdb_admin_user %> - password: <%= @couchdb_admin_password %> - prefix : "" - suffix : "" - netrc: "/etc/couchdb/couchdb.netrc" - -# file to store the last processed user record in so we can resume after -# a restart: -seq_dir: "/var/lib/leap/tapicero/" - -# Configure log_file like this if you want to log to a file instead of syslog: -#log_file: "/var/log/leap/tapicero.log" -#log_level: debug -log_level: info - -# tapicero specific options -options: - # prefix for per user databases: - db_prefix: "user-" - mode: <%= @couchdb_mode %> -<%- if @couchdb_replication %> - replication: <%= @couchdb_replication.to_json %> -<%- end -%> - - # security settings to be used for the per user databases - security: - admins: - names: - # We explicitly allow the admin user to access per user databases, even - # though admin access ignores per database security we just do this to be - # explicit about this - - <%= @couchdb_admin_user %> - roles: [] - members: - names: - - <%= @couchdb_soledad_user %> - - <%= @couchdb_leap_mx_user %> - roles: - - replication - - -- cgit v1.2.3 From b1e50fc76ddece9944ae253da9bacd485ffea84b Mon Sep 17 00:00:00 2001 From: varac Date: Tue, 6 Oct 2015 13:23:56 +0200 Subject: [feat] Remove tapicero from more places Remove from: - platform white-box tests (couchdb user ACLs, tapicero daemon test) - provider_base/ dir that handles the compilation of the hiera config file - Resolves: #7501 --- lib/leap_cli/commands/db.rb | 1 - provider_base/common.json | 5 ----- provider_base/services/couchdb.json | 5 ----- puppet/modules/site_config/manifests/remove/tapicero.pp | 12 ++++++++---- tests/white-box/couchdb.rb | 3 +-- tests/white-box/webapp.rb | 2 +- 6 files changed, 10 insertions(+), 18 deletions(-) diff --git a/lib/leap_cli/commands/db.rb b/lib/leap_cli/commands/db.rb index e4fd3858..5703e4bd 100644 --- a/lib/leap_cli/commands/db.rb +++ b/lib/leap_cli/commands/db.rb @@ -40,7 +40,6 @@ module LeapCli; module Commands def destroy_all_dbs(nodes) ssh_connect(nodes) do |ssh| ssh.run('/etc/init.d/bigcouch stop && test ! -z "$(ls /opt/bigcouch/var/lib/ 2> /dev/null)" && rm -r /opt/bigcouch/var/lib/* && echo "db destroyed" || echo "db already destroyed"') - ssh.run('grep ^seq_dir /etc/leap/tapicero.yaml | cut -f2 -d\" | xargs rm -rv') end end diff --git a/provider_base/common.json b/provider_base/common.json index 9cc7875a..5821789b 100644 --- a/provider_base/common.json +++ b/provider_base/common.json @@ -83,11 +83,6 @@ "package": "soledad-server", "revision": "latest" }, - "tapicero": { - "type": "git", - "source": "https://leap.se/git/tapicero", - "revision": "origin/version/0.7" - }, "webapp": { "type": "git", "source": "https://leap.se/git/leap_web", diff --git a/provider_base/services/couchdb.json b/provider_base/services/couchdb.json index 8b1386f8..5e65b2ec 100644 --- a/provider_base/services/couchdb.json +++ b/provider_base/services/couchdb.json @@ -31,11 +31,6 @@ "password": "= secret :couch_soledad_password", "salt": "= hex_secret :couch_soledad_password_salt, 128" }, - "tapicero": { - "username": "tapicero", - "password": "= secret :couch_tapicero_password", - "salt": "= hex_secret :couch_tapicero_password_salt, 128" - }, "webapp": { "username": "webapp", "password": "= secret :couch_webapp_password", diff --git a/puppet/modules/site_config/manifests/remove/tapicero.pp b/puppet/modules/site_config/manifests/remove/tapicero.pp index 765f7428..497cf8b2 100644 --- a/puppet/modules/site_config/manifests/remove/tapicero.pp +++ b/puppet/modules/site_config/manifests/remove/tapicero.pp @@ -32,12 +32,14 @@ class site_config::remove::tapicero { recurse => true, matches => 'tapicero*', require => [ Exec['kill_tapicero'] ]; - '/etc/check_mk/logwatch.d/tapicero.cfg':; + '/etc/check_mk/logwatch.d/tapicero.cfg': + notify => Exec['check_mk-refresh']; 'checkmk_logwatch_spool': path => '/var/lib/check_mk/logwatch', recurse => true, matches => '*tapicero.log', - require => [ Exec['kill_tapicero'] ]; + require => Exec['kill_tapicero'], + notify => Exec['check_mk-refresh']; } # remove local nagios plugin checks via mrpe @@ -46,12 +48,14 @@ class site_config::remove::tapicero { incl => '/etc/check_mk/mrpe.cfg', lens => 'Spacevars.lns', changes => 'rm /files/etc/check_mk/mrpe.cfg/Tapicero_Procs', - require => File['/etc/check_mk/mrpe.cfg']; + require => File['/etc/check_mk/mrpe.cfg'], + notify => Exec['check_mk-refresh']; 'Tapicero_Heartbeat': incl => '/etc/check_mk/mrpe.cfg', lens => 'Spacevars.lns', changes => 'rm Tapicero_Heartbeat', - require => File['/etc/check_mk/mrpe.cfg']; + require => File['/etc/check_mk/mrpe.cfg'], + notify => Exec['check_mk-refresh']; } } diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb index 5ee12ff3..edb28eac 100644 --- a/tests/white-box/couchdb.rb +++ b/tests/white-box/couchdb.rb @@ -9,7 +9,6 @@ class CouchDB < LeapTest end def test_00_Are_daemons_running? - assert_running '^tapicero', :single => true if multimaster? assert_running 'bin/beam' assert_running 'bin/epmd' @@ -70,7 +69,7 @@ class CouchDB < LeapTest end def test_04_Do_ACL_users_exist? - acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'tapicero', 'webapp', 'replication'] + acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'webapp', 'replication'] url = couchdb_backend_url("/_users/_all_docs", :username => 'admin') assert_get(url) do |body| response = JSON.parse(body) diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb index 9956eb35..8be6bde2 100644 --- a/tests/white-box/webapp.rb +++ b/tests/white-box/webapp.rb @@ -95,7 +95,7 @@ class Webapp < LeapTest end # - # returns true if the per-user db created by tapicero exists. + # returns true if the per-user db created by soledad-server exists. # we try three times, and give up after that. # def assert_user_db_exists(user) -- cgit v1.2.3 From 2b0386bee6525dda705152031d7125bc30b65269 Mon Sep 17 00:00:00 2001 From: varac Date: Wed, 7 Oct 2015 10:57:24 +0200 Subject: [bug] Fix removal of webapp apache config file Done by including a service-dependend site_config::remove::webapp class. --- puppet/modules/site_config/manifests/remove/files.pp | 17 ----------------- puppet/modules/site_config/manifests/remove/webapp.pp | 7 +++++++ puppet/modules/site_webapp/manifests/init.pp | 4 ++++ 3 files changed, 11 insertions(+), 17 deletions(-) create mode 100644 puppet/modules/site_config/manifests/remove/webapp.pp diff --git a/puppet/modules/site_config/manifests/remove/files.pp b/puppet/modules/site_config/manifests/remove/files.pp index 66647d31..466f50c8 100644 --- a/puppet/modules/site_config/manifests/remove/files.pp +++ b/puppet/modules/site_config/manifests/remove/files.pp @@ -11,16 +11,6 @@ class site_config::remove::files { - # - # Platform 0.8 removals - # - - tidy { - '/etc/apache/sites-enabled/leap_webapp.conf': - notify => Service['apache']; - } - - # # Platform 0.7 removals # @@ -44,13 +34,6 @@ class site_config::remove::files { '/etc/leap/soledad-server.conf':; } - if member($::services, 'webapp') { - tidy { - '/etc/apache/sites-enabled/leap_webapp.conf': - notify => Service['apache']; - } - } - # leax-mx logged to /var/log/leap_mx.log in the past # we need to use a dumb exec here because file_line doesn't # allow removing lines that match a regex in the current version diff --git a/puppet/modules/site_config/manifests/remove/webapp.pp b/puppet/modules/site_config/manifests/remove/webapp.pp new file mode 100644 index 00000000..58f59815 --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/webapp.pp @@ -0,0 +1,7 @@ +# remove leftovers on webapp nodes +class site_config::remove::webapp { + tidy { + '/etc/apache/sites-enabled/leap_webapp.conf': + notify => Service['apache']; + } +} diff --git a/puppet/modules/site_webapp/manifests/init.pp b/puppet/modules/site_webapp/manifests/init.pp index ec94c090..d046b7df 100644 --- a/puppet/modules/site_webapp/manifests/init.pp +++ b/puppet/modules/site_webapp/manifests/init.pp @@ -1,3 +1,4 @@ +# configure webapp service class site_webapp { tag 'leap_service' $definition_files = hiera('definition_files') @@ -26,6 +27,9 @@ class site_webapp { include site_config::x509::client_ca::ca include site_config::x509::client_ca::key + # remove leftovers from previous installations on webapp nodes + include site_config::remove::webapp + group { 'leap-webapp': ensure => present, allowdupe => false; -- cgit v1.2.3 From 89f609c97e43b06403706b81caf7a1c3e116bdf8 Mon Sep 17 00:00:00 2001 From: varac Date: Wed, 7 Oct 2015 11:17:11 +0200 Subject: [bug] Fix missing dependency (tapicero leftovers) We need to remove local check-mk-agent checks on the tapicero nodes, and want to notify the monitoring server to re-inventarize the local checks. This doesn't work when both services run on different hosts, it will fail with: Could not find dependent Exec[check_mk-refresh] for Tidy[checkmk_logwatch_spool] So i remove the notifies, because we will re-inventarize of local checks by a daily cronjob anyway, see #6873. ... - Resolves: #XYZ - Related: #XYZ - Documentation: #XYZ - Releases: XYZ --- puppet/modules/site_config/manifests/remove/tapicero.pp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/puppet/modules/site_config/manifests/remove/tapicero.pp b/puppet/modules/site_config/manifests/remove/tapicero.pp index 497cf8b2..edb4e393 100644 --- a/puppet/modules/site_config/manifests/remove/tapicero.pp +++ b/puppet/modules/site_config/manifests/remove/tapicero.pp @@ -32,14 +32,12 @@ class site_config::remove::tapicero { recurse => true, matches => 'tapicero*', require => [ Exec['kill_tapicero'] ]; - '/etc/check_mk/logwatch.d/tapicero.cfg': - notify => Exec['check_mk-refresh']; + '/etc/check_mk/logwatch.d/tapicero.cfg':; 'checkmk_logwatch_spool': path => '/var/lib/check_mk/logwatch', recurse => true, matches => '*tapicero.log', require => Exec['kill_tapicero'], - notify => Exec['check_mk-refresh']; } # remove local nagios plugin checks via mrpe @@ -48,14 +46,12 @@ class site_config::remove::tapicero { incl => '/etc/check_mk/mrpe.cfg', lens => 'Spacevars.lns', changes => 'rm /files/etc/check_mk/mrpe.cfg/Tapicero_Procs', - require => File['/etc/check_mk/mrpe.cfg'], - notify => Exec['check_mk-refresh']; + require => File['/etc/check_mk/mrpe.cfg']; 'Tapicero_Heartbeat': incl => '/etc/check_mk/mrpe.cfg', lens => 'Spacevars.lns', changes => 'rm Tapicero_Heartbeat', - require => File['/etc/check_mk/mrpe.cfg'], - notify => Exec['check_mk-refresh']; + require => File['/etc/check_mk/mrpe.cfg']; } } -- cgit v1.2.3 From b53bd889d5407fb357329b414bcae929176c0690 Mon Sep 17 00:00:00 2001 From: ankonym Date: Wed, 7 Oct 2015 11:19:26 +0200 Subject: Change webapp tests to work with enabled invite codes This will generate an invite code so test_05_Can_create_and_authenticate_and_delete_user_via_API? will work correctly (when invite codes are required for signups). --- tests/helpers/bonafide_helper.rb | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/helpers/bonafide_helper.rb b/tests/helpers/bonafide_helper.rb index 9b26eaaf..39f761fe 100644 --- a/tests/helpers/bonafide_helper.rb +++ b/tests/helpers/bonafide_helper.rb @@ -32,7 +32,11 @@ class LeapTest def assert_create_user user = SRP::User.new url = api_url("/1/users.json") - assert_post(url, user.to_params) do |body| + + params = user.to_params + params['user[invite_code]'] = generate_invite_code + + assert_post(url, params) do |body| assert response = JSON.parse(body), 'response should be JSON' assert response['ok'], "Creating a user should be successful, got #{response.inspect} instead." end @@ -40,6 +44,13 @@ class LeapTest return user end + def generate_invite_code + if property('webapp.invite_required') + `cd /srv/leap/webapp/ && sudo RAILS_ENV=production bundle exec rake generate_invites[1]`.gsub(/\n/, "") + end + end + + # # attempts to authenticate user. if successful, # user object is updated with id and session token. -- cgit v1.2.3 From dc426cc777aec921772c1d8c2fa014d61aa90d33 Mon Sep 17 00:00:00 2001 From: ankonym Date: Wed, 7 Oct 2015 12:11:21 +0200 Subject: Modify bonafide_helper to improve user creation test with invites Will now use the correct user to generate invite codes and only add invite code parameter when invite codes are enabled --- tests/helpers/bonafide_helper.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/helpers/bonafide_helper.rb b/tests/helpers/bonafide_helper.rb index 39f761fe..1a6dc6fe 100644 --- a/tests/helpers/bonafide_helper.rb +++ b/tests/helpers/bonafide_helper.rb @@ -34,7 +34,10 @@ class LeapTest url = api_url("/1/users.json") params = user.to_params - params['user[invite_code]'] = generate_invite_code + + if property('webapp.invite_required') + params['user[invite_code]'] = generate_invite_code + end assert_post(url, params) do |body| assert response = JSON.parse(body), 'response should be JSON' @@ -45,9 +48,7 @@ class LeapTest end def generate_invite_code - if property('webapp.invite_required') - `cd /srv/leap/webapp/ && sudo RAILS_ENV=production bundle exec rake generate_invites[1]`.gsub(/\n/, "") - end + `cd /srv/leap/webapp/ && sudo -u leap-webapp RAILS_ENV=production bundle exec rake generate_invites[1]`.gsub(/\n/, "") end -- cgit v1.2.3 From acf45b97c1f532553c77685b8dff8559c378b45b Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 7 Oct 2015 17:32:22 -0700 Subject: added `leap db destroy --username USER` command. --- lib/leap_cli/commands/db.rb | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/lib/leap_cli/commands/db.rb b/lib/leap_cli/commands/db.rb index 5703e4bd..5307ac4d 100644 --- a/lib/leap_cli/commands/db.rb +++ b/lib/leap_cli/commands/db.rb @@ -5,15 +5,28 @@ module LeapCli; module Commands db.desc 'Destroy one or more databases. If present, limit to FILTER nodes. For example `leap db destroy --db sessions,tokens testing`.' db.arg_name 'FILTER', :optional => true db.command :destroy do |destroy| - destroy.flag :db, :arg_name => "DATABASES", :desc => 'Comma separated list of databases to destroy (no space). Use "--db all" to destroy all databases.', :optional => false + destroy.flag :db, :arg_name => "DATABASES", :desc => 'Comma separated list of databases to destroy (no space). Use "--db all" to destroy all databases.', :optional => true + destroy.flag :user, :arg_name => "USERS", :desc => 'Comma separated list of usernames. The storage databases for these user(s) will be destroyed.', :optional => true destroy.action do |global_options,options,args| dbs = (options[:db]||"").split(',') - bail!('No databases specified') if dbs.empty? + users = (options[:user]||"").split(',') + if dbs.empty? && users.empty? + bail!('Either --db or --user is required.') + end nodes = manager.filter(args) if nodes.any? nodes = nodes[:services => 'couchdb'] end - if nodes.any? + unless nodes.any? + bail! 'No db nodes selected.' + end + if users.any? + unless global_options[:yes] + say 'You are about to permanently destroy user databases for [%s] for nodes [%s].' % [users.join(', '), nodes.keys.join(', ')] + bail! unless agree("Continue? ") + end + destroy_user_dbs(nodes, users) + elsif dbs.any? unless global_options[:yes] if dbs.include?('all') say 'You are about to permanently destroy all database data for nodes [%s].' % nodes.keys.join(', ') @@ -28,8 +41,6 @@ module LeapCli; module Commands destroy_dbs(nodes, dbs) end say 'You must run `leap deploy` in order to create the databases again.' - else - say 'No nodes' end end end @@ -39,7 +50,7 @@ module LeapCli; module Commands def destroy_all_dbs(nodes) ssh_connect(nodes) do |ssh| - ssh.run('/etc/init.d/bigcouch stop && test ! -z "$(ls /opt/bigcouch/var/lib/ 2> /dev/null)" && rm -r /opt/bigcouch/var/lib/* && echo "db destroyed" || echo "db already destroyed"') + ssh.run('/etc/init.d/bigcouch stop && test ! -z "$(ls /opt/bigcouch/var/lib/ 2> /dev/null)" && rm -r /opt/bigcouch/var/lib/* && echo "All DBs destroyed" || echo "DBs already destroyed"') end end @@ -53,6 +64,16 @@ module LeapCli; module Commands end end + def destroy_user_dbs(nodes, users) + nodes.each_node do |node| + ssh_connect(node) do |ssh| + users.each do |user| + ssh.run(DESTROY_USER_DB_COMMAND % {:user => user}) + end + end + end + end + DESTROY_DB_COMMAND = %{ if [ 200 = `curl -ns -w "%%{http_code}" -X GET "127.0.0.1:5984/%{db}" -o /dev/null` ]; then echo "Result from DELETE /%{db}:" `curl -ns -X DELETE "127.0.0.1:5984/%{db}"`; @@ -61,4 +82,5 @@ else fi } + DESTROY_USER_DB_COMMAND = %{/srv/leap/couchdb/scripts/destroy-user-db --username %{user}} end; end -- cgit v1.2.3 From b748aeffbdd72d50a7665b9c21c96a9750a840c0 Mon Sep 17 00:00:00 2001 From: varac Date: Thu, 8 Oct 2015 14:30:56 +0200 Subject: Update submodule couchdb --- puppet/modules/couchdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/couchdb b/puppet/modules/couchdb index 3c20a316..1f583d9c 160000 --- a/puppet/modules/couchdb +++ b/puppet/modules/couchdb @@ -1 +1 @@ -Subproject commit 3c20a3169e77e5a5f9abc06788c3a7730d5530ca +Subproject commit 1f583d9c9157390850a7737630f60832ced82374 -- cgit v1.2.3 From 33b9876af4af85504107aae20feb57aaab5a17ad Mon Sep 17 00:00:00 2001 From: elijah Date: Sun, 11 Oct 2015 20:36:07 -0700 Subject: russian text requires amber 0.3.8 --- puppet/modules/site_static/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_static/manifests/init.pp b/puppet/modules/site_static/manifests/init.pp index f69ffba7..8df53075 100644 --- a/puppet/modules/site_static/manifests/init.pp +++ b/puppet/modules/site_static/manifests/init.pp @@ -44,7 +44,7 @@ class site_static { } if (member($formats, 'amber')) { - rubygems::gem{'amber-0.3.7': + rubygems::gem{'amber-0.3.8': require => Package['zlib1g-dev'] } -- cgit v1.2.3 From 2053883c81d261b5a9af5011bd6f0e8bc709f8d1 Mon Sep 17 00:00:00 2001 From: ankonym Date: Mon, 12 Oct 2015 11:19:21 +0200 Subject: Fix soledad test when invite codes are enabled This provides an invite code when invite codes are enabled while the test runs (but it does not get deleted yet afterwards) --- tests/helpers/bonafide_helper.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/helpers/bonafide_helper.rb b/tests/helpers/bonafide_helper.rb index 1a6dc6fe..82db3973 100644 --- a/tests/helpers/bonafide_helper.rb +++ b/tests/helpers/bonafide_helper.rb @@ -36,7 +36,8 @@ class LeapTest params = user.to_params if property('webapp.invite_required') - params['user[invite_code]'] = generate_invite_code + @invite_code = generate_invite_code + params['user[invite_code]'] = @invite_code end assert_post(url, params) do |body| -- cgit v1.2.3 From 19e5d23e3fe34199265117e033acfabc3cff9109 Mon Sep 17 00:00:00 2001 From: varac Date: Mon, 12 Oct 2015 16:30:58 +0200 Subject: [feat] Remove tapicero couchdb user - Resolves: #7514 --- puppet/modules/site_config/manifests/remove.pp | 1 - .../site_config/manifests/remove/monitoring.pp | 10 +++++++++ .../site_config/manifests/remove/tapicero.pp | 24 ++++++++++++++++------ puppet/modules/site_couchdb/manifests/init.pp | 3 +++ puppet/modules/site_nagios/manifests/init.pp | 4 ++++ 5 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 puppet/modules/site_config/manifests/remove/monitoring.pp diff --git a/puppet/modules/site_config/manifests/remove.pp b/puppet/modules/site_config/manifests/remove.pp index 00502c0a..b1ad1a2b 100644 --- a/puppet/modules/site_config/manifests/remove.pp +++ b/puppet/modules/site_config/manifests/remove.pp @@ -1,5 +1,4 @@ # remove leftovers from previous deploys class site_config::remove { include site_config::remove::files - include site_config::remove::tapicero } diff --git a/puppet/modules/site_config/manifests/remove/monitoring.pp b/puppet/modules/site_config/manifests/remove/monitoring.pp new file mode 100644 index 00000000..d7095597 --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/monitoring.pp @@ -0,0 +1,10 @@ +# remove leftovers on monitoring nodes +class site_config::remove::monitoring { + + tidy { + 'checkmk_logwatch_spool': + path => '/var/lib/check_mk/logwatch', + recurse => true, + matches => '*tapicero.log' + } +} diff --git a/puppet/modules/site_config/manifests/remove/tapicero.pp b/puppet/modules/site_config/manifests/remove/tapicero.pp index edb4e393..4ce972d0 100644 --- a/puppet/modules/site_config/manifests/remove/tapicero.pp +++ b/puppet/modules/site_config/manifests/remove/tapicero.pp @@ -1,6 +1,23 @@ -# remove tapicero leftovers from previous deploys +# remove tapicero leftovers from previous deploys on couchdb nodes class site_config::remove::tapicero { + # remove tapicero couchdb user + $couchdb_config = hiera('couch') + $couchdb_mode = $couchdb_config['mode'] + + if $couchdb_mode == 'multimaster' + { + $port = 5986 + } else { + $port = 5984 + } + + exec { 'remove_couchdb_user': + onlyif => "/usr/bin/curl -s 127.0.0.1:${port}/_users/org.couchdb.user:tapicero | grep -qv 'not_found'", + command => "/usr/local/bin/couch-doc-update --host 127.0.0.1:${port} --db _users --id org.couchdb.user:tapicero --delete" + } + + exec { 'kill_tapicero': onlyif => '/usr/bin/test -s /var/run/tapicero.pid', command => '/usr/bin/pkill --pidfile /var/run/tapicero.pid' @@ -33,11 +50,6 @@ class site_config::remove::tapicero { matches => 'tapicero*', require => [ Exec['kill_tapicero'] ]; '/etc/check_mk/logwatch.d/tapicero.cfg':; - 'checkmk_logwatch_spool': - path => '/var/lib/check_mk/logwatch', - recurse => true, - matches => '*tapicero.log', - require => Exec['kill_tapicero'], } # remove local nagios plugin checks via mrpe diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 1ec15f00..61aa887e 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -62,4 +62,7 @@ class site_couchdb { include site_check_mk::agent::couchdb + # remove tapicero leftovers on couchdb nodes + include site_config::remove::tapicero + } diff --git a/puppet/modules/site_nagios/manifests/init.pp b/puppet/modules/site_nagios/manifests/init.pp index eb08cdcb..40ae4b86 100644 --- a/puppet/modules/site_nagios/manifests/init.pp +++ b/puppet/modules/site_nagios/manifests/init.pp @@ -1,6 +1,10 @@ +# setup nagios on monitoring node class site_nagios { tag 'leap_service' Class['site_config::default'] -> Class['site_nagios'] include site_nagios::server + + # remove leftovers on monitoring nodes + include site_config::remove::monitoring } -- cgit v1.2.3 From d6b521372243b79105a1513d4559572dfab6db54 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 22 Sep 2015 15:04:33 -0400 Subject: add clamav filtering, with sanesecurity signature updating and provider whitelisting (#3625) Change-Id: I15985ca00ee95bc62855f098a78e364ebbc32616 --- provider_base/services/mx.json | 3 + puppet/modules/clamav/files/01-leap.conf | 58 +++++++++++++++ puppet/modules/clamav/files/clamav-daemon_default | 8 ++ puppet/modules/clamav/files/clamav-milter_default | 14 ++++ puppet/modules/clamav/manifests/daemon.pp | 86 ++++++++++++++++++++++ puppet/modules/clamav/manifests/freshclam.pp | 21 ++++++ puppet/modules/clamav/manifests/init.pp | 8 ++ puppet/modules/clamav/manifests/milter.pp | 48 ++++++++++++ puppet/modules/clamav/manifests/unofficial_sigs.pp | 22 ++++++ .../clamav/templates/clamav-milter.conf.erb | 28 +++++++ puppet/modules/clamav/templates/local.pdb.erb | 1 + .../clamav/templates/whitelisted_addresses.erb | 5 ++ puppet/modules/site_postfix/manifests/mx.pp | 5 ++ 13 files changed, 307 insertions(+) create mode 100644 puppet/modules/clamav/files/01-leap.conf create mode 100644 puppet/modules/clamav/files/clamav-daemon_default create mode 100644 puppet/modules/clamav/files/clamav-milter_default create mode 100644 puppet/modules/clamav/manifests/daemon.pp create mode 100644 puppet/modules/clamav/manifests/freshclam.pp create mode 100644 puppet/modules/clamav/manifests/init.pp create mode 100644 puppet/modules/clamav/manifests/milter.pp create mode 100644 puppet/modules/clamav/manifests/unofficial_sigs.pp create mode 100644 puppet/modules/clamav/templates/clamav-milter.conf.erb create mode 100644 puppet/modules/clamav/templates/local.pdb.erb create mode 100644 puppet/modules/clamav/templates/whitelisted_addresses.erb diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json index d6e9fff9..70acf5cb 100644 --- a/provider_base/services/mx.json +++ b/provider_base/services/mx.json @@ -24,6 +24,9 @@ }, "mynetworks": "= nodes['environment' => '!local'].map{|name, n| [n.ip_address, (global.facts[name]||{})['ec2_public_ipv4']]}.flatten.compact.uniq", "rbls": ["zen.spamhaus.org"], + "clamav": { + "whitelisted_addresses": [] + }, "x509": { "use": true, "use_commercial": true, diff --git a/puppet/modules/clamav/files/01-leap.conf b/puppet/modules/clamav/files/01-leap.conf new file mode 100644 index 00000000..abeeb302 --- /dev/null +++ b/puppet/modules/clamav/files/01-leap.conf @@ -0,0 +1,58 @@ +# If running clamd in "LocalSocket" mode (*NOT* in TCP/IP mode), and +# either "SOcket Cat" (socat) or the "IO::Socket::UNIX" perl module +# are installed on the system, and you want to report whether clamd +# is running or not, uncomment the "clamd_socket" variable below (you +# will be warned if neither socat nor IO::Socket::UNIX are found, but +# the script will still run). You will also need to set the correct +# path to your clamd socket file (if unsure of the path, check the +# "LocalSocket" setting in your clamd.conf file for socket location). +clamd_socket="/run/clamav/clamd.ctl" + +# If you would like to attempt to restart ClamD if detected not running, +# uncomment the next 2 lines. Confirm the path to the "clamd_lock" file +# (usually can be found in the clamd init script) and also enter the clamd +# start command for your particular distro for the "start_clamd" variable +# (the sample start command shown below should work for most linux distros). +# NOTE: these 2 variables are dependant on the "clamd_socket" variable +# shown above - if not enabled, then the following 2 variables will be +# ignored, whether enabled or not. +clamd_lock="/run/clamav/clamd.pid" +start_clamd="service clamav-daemon start" + +ss_dbs=" + junk.ndb + phish.ndb + rogue.hdb + sanesecurity.ftm + scam.ndb + sigwhitelist.ign2 + spamattach.hdb + spamimg.hdb + winnow.attachments.hdb + winnow_bad_cw.hdb + winnow_extended_malware.hdb + winnow_malware.hdb + winnow_malware_links.ndb + malwarehash.hsb + doppelstern.hdb + bofhland_cracked_URL.ndb + bofhland_malware_attach.hdb + bofhland_malware_URL.ndb + bofhland_phishing_URL.ndb + crdfam.clamav.hdb + phishtank.ndb + porcupine.ndb + spear.ndb + spearl.ndb +" + +# ======================== +# SecuriteInfo Database(s) +# ======================== +# Add or remove database file names between quote marks as needed. To +# disable any SecuriteInfo database downloads, remove the appropriate +# lines below. To disable all SecuriteInfo database file downloads, +# comment all of the following lines. +si_dbs="" + +mbl_dbs="" \ No newline at end of file diff --git a/puppet/modules/clamav/files/clamav-daemon_default b/puppet/modules/clamav/files/clamav-daemon_default new file mode 100644 index 00000000..b4cd6a4f --- /dev/null +++ b/puppet/modules/clamav/files/clamav-daemon_default @@ -0,0 +1,8 @@ +# This is a file designed only t0 set special environment variables +# eg TMP or TMPDIR. It is sourced from a shell script, so anything +# put in here must be in variable=value format, suitable for sourcing +# from a shell script. +# Examples: +# export TMPDIR=/dev/shm +export TMP=/var/tmp +export TMPDIR=/var/tmp diff --git a/puppet/modules/clamav/files/clamav-milter_default b/puppet/modules/clamav/files/clamav-milter_default new file mode 100644 index 00000000..5e33e822 --- /dev/null +++ b/puppet/modules/clamav/files/clamav-milter_default @@ -0,0 +1,14 @@ +# +# clamav-milter init options +# + +## SOCKET_RWGROUP +# by default, the socket created by the milter has permissions +# clamav:clamav:755. SOCKET_RWGROUP changes the group and changes the +# permissions to 775 to give read-write access to that group. +# +# If you are using postfix to speak to the milter, you have to give permission +# to the postfix group to write +# +SOCKET_RWGROUP=postfix +export TMPDIR=/var/tmp diff --git a/puppet/modules/clamav/manifests/daemon.pp b/puppet/modules/clamav/manifests/daemon.pp new file mode 100644 index 00000000..9aebf9b0 --- /dev/null +++ b/puppet/modules/clamav/manifests/daemon.pp @@ -0,0 +1,86 @@ +class clamav::daemon { + + $domain_hash = hiera('domain') + $domain = $domain_hash['full_suffix'] + + package { [ 'clamav-daemon', 'arj' ]: + ensure => installed; + } + + service { + 'clamav-daemon': + ensure => running, + name => clamav-daemon, + pattern => '/usr/sbin/clamd', + enable => true, + hasrestart => true, + subscribe => File['/etc/default/clamav-daemon']; + } + + file { + '/var/run/clamav': + ensure => directory, + mode => '0750', + owner => clamav, + group => postfix; + + '/var/lib/clamav': + mode => '0755', + owner => clamav, + group => clamav; + + '/etc/default/clamav-daemon': + source => 'puppet:///modules/clamav/clamav-daemon_default', + mode => '0644', + owner => root, + group => root; + + # this file contains additional domains that we want the clamav + # phishing process to look for (our domain) + '/var/lib/clamav/local.pdb': + content => template('clamav/local.pdb.erb'), + mode => '0644', + owner => clamav, + group => clamav; + } + + file_line { + 'clamav_daemon_tmp': + path => '/etc/clamav/clamd.conf', + line => 'TemporaryDirectory /var/tmp', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + + 'enable_phishscanurls': + path => '/etc/clamav/clamd.conf', + match => 'PhishingScanURLs no', + line => 'PhishingScanURLs yes', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + + 'clamav_LogSyslog_true': + path => '/etc/clamav/clamd.conf', + match => '^LogSyslog false', + line => 'LogSyslog true', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + + 'clamav_MaxThreads': + path => '/etc/clamav/clamd.conf', + match => 'MaxThreads 20', + line => 'MaxThreads 100', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + } + + # remove LogFile line + file_line { + 'clamav_LogFile': + path => '/etc/clamav/clamd.conf', + match => '^LogFile .*', + line => '', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + } + +} diff --git a/puppet/modules/clamav/manifests/freshclam.pp b/puppet/modules/clamav/manifests/freshclam.pp new file mode 100644 index 00000000..b9827ede --- /dev/null +++ b/puppet/modules/clamav/manifests/freshclam.pp @@ -0,0 +1,21 @@ +class clamav::freshclam { + + package { 'clamav-freshclam': ensure => installed } + + service { + 'freshclam': + ensure => running, + enable => true, + name => clamav-freshclam, + pattern => '/usr/bin/freshclam', + hasrestart => true; + } + + file_line { + 'freshclam_notify': + path => '/etc/clamav/freshclam.conf', + line => 'NotifyClamd /etc/clamav/clamd.conf', + notify => Service[freshclam]; + } + +} diff --git a/puppet/modules/clamav/manifests/init.pp b/puppet/modules/clamav/manifests/init.pp new file mode 100644 index 00000000..fa7b553c --- /dev/null +++ b/puppet/modules/clamav/manifests/init.pp @@ -0,0 +1,8 @@ +class clamav { + + include clamav::daemon + include clamav::milter + include clamav::sanesecurity + include clamav::freshclam + +} diff --git a/puppet/modules/clamav/manifests/milter.pp b/puppet/modules/clamav/manifests/milter.pp new file mode 100644 index 00000000..52ddaef1 --- /dev/null +++ b/puppet/modules/clamav/manifests/milter.pp @@ -0,0 +1,48 @@ +class clamav::milter { + + $clamav = hiera('clamav') + $whitelisted_addresses = $clamav['whitelisted_addresses'] + $domain_hash = hiera('domain') + $domain = $domain_hash['full_suffix'] + + package { 'clamav-milter': ensure => installed } + + service { + 'clamav-milter': + ensure => running, + enable => true, + name => clamav-milter, + pattern => '/usr/sbin/clamav-milter', + hasrestart => true, + subscribe => File['/etc/default/clamav-milter']; + } + + file { + '/run/clamav/milter.ctl': + mode => '0666', + owner => clamav, + group => postfix, + require => Class['clamav::daemon']; + + '/etc/clamav/clamav-milter.conf': + content => template('clamav/clamav-milter.conf.erb'), + mode => '0644', + owner => root, + group => root, + subscribe => Service['clamav-milter']; + + '/etc/default/clamav-milter': + source => 'puppet:///modules/clamav/clamav-milter_default', + mode => '0644', + owner => root, + group => root; + + '/etc/clamav/whitelisted_addresses': + content => template('clamav/whitelisted_addresses.erb'), + mode => '0644', + owner => root, + group => root; + + } + +} diff --git a/puppet/modules/clamav/manifests/unofficial_sigs.pp b/puppet/modules/clamav/manifests/unofficial_sigs.pp new file mode 100644 index 00000000..316154d3 --- /dev/null +++ b/puppet/modules/clamav/manifests/unofficial_sigs.pp @@ -0,0 +1,22 @@ +class clamav::unofficial_sigs { + + package { [ 'clamav-unofficial-sigs', 'wget', 'gnupg', + 'socat', 'rsync', 'curl' ]: + ensure => installed + } + + file { + '/var/log/clamav-unofficial-sigs.log': + ensure => file, + owner => clamav, + group => clamav, + require => Package['clamav-unofficial-sigs']; + + '/etc/clamav-unofficial-sigs.conf.d/01-leap.conf': + source => 'puppet:///modules/clamav/01-leap.conf', + mode => '0755', + owner => root, + group => root, + require => Package['clamav-unofficial-sigs']; + } +} diff --git a/puppet/modules/clamav/templates/clamav-milter.conf.erb b/puppet/modules/clamav/templates/clamav-milter.conf.erb new file mode 100644 index 00000000..9bf7099e --- /dev/null +++ b/puppet/modules/clamav/templates/clamav-milter.conf.erb @@ -0,0 +1,28 @@ +# THIS FILE MANAGED BY PUPPET +MilterSocket /var/run/clamav/milter.ctl +FixStaleSocket true +User clamav +MilterSocketGroup clamav +MilterSocketMode 666 +AllowSupplementaryGroups true +ReadTimeout 120 +Foreground false +PidFile /var/run/clamav/clamav-milter.pid +ClamdSocket unix:/var/run/clamav/clamd.ctl +OnClean Accept +OnInfected Reject +OnFail Defer +AddHeader Replace +LogSyslog true +LogFacility LOG_LOCAL6 +LogVerbose yes +LogInfected Basic +LogTime true +LogFileUnlock false +LogClean Off +LogRotate true +SupportMultipleRecipients false +MaxFileSize 10M +TemporaryDirectory /var/tmp +RejectMsg "Message refused due to content violation: %v - contact https://<%= @domain %>/tickets/new if this is in error" +Whitelist /etc/clamav/whitelisted_addresses diff --git a/puppet/modules/clamav/templates/local.pdb.erb b/puppet/modules/clamav/templates/local.pdb.erb new file mode 100644 index 00000000..9ea0584a --- /dev/null +++ b/puppet/modules/clamav/templates/local.pdb.erb @@ -0,0 +1 @@ +H:<%= @domain %> diff --git a/puppet/modules/clamav/templates/whitelisted_addresses.erb b/puppet/modules/clamav/templates/whitelisted_addresses.erb new file mode 100644 index 00000000..9e068ec5 --- /dev/null +++ b/puppet/modules/clamav/templates/whitelisted_addresses.erb @@ -0,0 +1,5 @@ +<%- if @whitelisted_addresses then -%> +<% @whitelisted_addresses.each do |name| -%> +From::<%= name %> +<% end -%> +<% end -%> diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp index 42313d1a..f0a2554a 100644 --- a/puppet/modules/site_postfix/manifests/mx.pp +++ b/puppet/modules/site_postfix/manifests/mx.pp @@ -49,6 +49,10 @@ class site_postfix::mx { # alias map 'local_recipient_maps': value => '$alias_maps'; + 'smtpd_milters': + value => 'unix:/run/clamav/milter.ctl'; + 'milter_default_action': + value => 'accept'; } include site_postfix::mx::smtpd_checks @@ -57,6 +61,7 @@ class site_postfix::mx { include site_postfix::mx::smtpd_tls include site_postfix::mx::static_aliases include site_postfix::mx::rewrite_openpgp_header + include clamav # greater verbosity for debugging, take out for production #include site_postfix::debug -- cgit v1.2.3 From eda35dc4f8a9bb5dab84d917c7a9e9a058ba8d2f Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 13 Oct 2015 11:49:20 -0400 Subject: Update resource_file to not include /private/ as this is not used anymore by the nagios module, and our config template has drifted. Fixes: #7527 Change-Id: I56c3492056fcb95c499cf78b893249adcf0ae67f --- puppet/modules/site_nagios/files/configs/Debian/nagios.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg b/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg index 0d729b8c..981dc12a 100644 --- a/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg +++ b/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg @@ -70,7 +70,7 @@ precached_object_file=/var/lib/nagios3/objects.precache # defined as macros in this file and restrictive permissions (600) # can be placed on this file. -resource_file=/etc/nagios3/private/resource.cfg +resource_file=/etc/nagios3/resource.cfg -- cgit v1.2.3 From 717bd0f1061cbc4cd22a22f87b9b00ddf469f2fc Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 13 Oct 2015 12:01:53 -0400 Subject: Make syslog stop logging the icmpv6_send: no reply to icmp error messages, these are spamming provider's logs and will continue to do so until we have ipv6 working for the VPN (#6540) Change-Id: I80673bb64d8239e478bc042794929640f7a7cc39 --- puppet/modules/site_openvpn/manifests/init.pp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/puppet/modules/site_openvpn/manifests/init.pp b/puppet/modules/site_openvpn/manifests/init.pp index e2a3124e..ede35a9e 100644 --- a/puppet/modules/site_openvpn/manifests/init.pp +++ b/puppet/modules/site_openvpn/manifests/init.pp @@ -229,6 +229,13 @@ class site_openvpn { } leap::logfile { 'openvpn': } + + # Because we currently do not support ipv6 and instead block it (so no leaks + # happen), we get a large number of these messages, so we ignore them (#6540) + rsyslog::snippet { '01-ignore_icmpv6_send': + content => ':msg, contains, "icmpv6_send: no reply to icmp error" ~' + } + include site_check_mk::agent::openvpn } -- cgit v1.2.3 From 1c5e9e2afad7e9225ff5eaa8350268654078cf21 Mon Sep 17 00:00:00 2001 From: varac Date: Tue, 13 Oct 2015 18:13:05 +0200 Subject: updated submodule couchdb --- puppet/modules/couchdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/couchdb b/puppet/modules/couchdb index 1f583d9c..ae53b180 160000 --- a/puppet/modules/couchdb +++ b/puppet/modules/couchdb @@ -1 +1 @@ -Subproject commit 1f583d9c9157390850a7737630f60832ced82374 +Subproject commit ae53b180783016faa4331094a52769ddd57463f8 -- cgit v1.2.3 From 2c8de729a2a4213b8cb312bcb481695ae44f9a62 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 13 Oct 2015 14:01:48 -0400 Subject: Class was renamed, but not properly cared for in the rest of the manifest Change-Id: Ic9f022dcbb9f2096b933c898ae43023e0bf278c6 --- puppet/modules/clamav/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/clamav/manifests/init.pp b/puppet/modules/clamav/manifests/init.pp index fa7b553c..de8fb4dc 100644 --- a/puppet/modules/clamav/manifests/init.pp +++ b/puppet/modules/clamav/manifests/init.pp @@ -2,7 +2,7 @@ class clamav { include clamav::daemon include clamav::milter - include clamav::sanesecurity + include clamav::unofficial_sigs include clamav::freshclam } -- cgit v1.2.3 From 43595b105a21aaccb41c4d9199d87b3dc2d48ab5 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 13 Oct 2015 15:58:12 -0400 Subject: Fix ordering of clamav resources, by requiring the package installation as a pre-requisite Change-Id: Ic9c8cc6ccfb31ce5e56937a2d95de7974707c368 --- puppet/modules/clamav/manifests/daemon.pp | 22 +++++++++++++--------- puppet/modules/clamav/manifests/freshclam.pp | 10 ++++++---- puppet/modules/clamav/manifests/milter.pp | 6 ++++-- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/puppet/modules/clamav/manifests/daemon.pp b/puppet/modules/clamav/manifests/daemon.pp index 9aebf9b0..bf232e2c 100644 --- a/puppet/modules/clamav/manifests/daemon.pp +++ b/puppet/modules/clamav/manifests/daemon.pp @@ -14,20 +14,23 @@ class clamav::daemon { pattern => '/usr/sbin/clamd', enable => true, hasrestart => true, - subscribe => File['/etc/default/clamav-daemon']; + subscribe => File['/etc/default/clamav-daemon'], + require => Package['clamav-daemon']; } file { '/var/run/clamav': - ensure => directory, - mode => '0750', - owner => clamav, - group => postfix; + ensure => directory, + mode => '0750', + owner => clamav, + group => postfix, + require => [Package['postfix'], Package['clamav-daemon']]; '/var/lib/clamav': - mode => '0755', - owner => clamav, - group => clamav; + mode => '0755', + owner => clamav, + group => clamav, + require => Package['clamav-daemon']; '/etc/default/clamav-daemon': source => 'puppet:///modules/clamav/clamav-daemon_default', @@ -41,7 +44,8 @@ class clamav::daemon { content => template('clamav/local.pdb.erb'), mode => '0644', owner => clamav, - group => clamav; + group => clamav, + require => Package['clamav-daemon']; } file_line { diff --git a/puppet/modules/clamav/manifests/freshclam.pp b/puppet/modules/clamav/manifests/freshclam.pp index b9827ede..80c822a4 100644 --- a/puppet/modules/clamav/manifests/freshclam.pp +++ b/puppet/modules/clamav/manifests/freshclam.pp @@ -8,14 +8,16 @@ class clamav::freshclam { enable => true, name => clamav-freshclam, pattern => '/usr/bin/freshclam', - hasrestart => true; + hasrestart => true, + require => Package['clamav-freshclam']; } file_line { 'freshclam_notify': - path => '/etc/clamav/freshclam.conf', - line => 'NotifyClamd /etc/clamav/clamd.conf', - notify => Service[freshclam]; + path => '/etc/clamav/freshclam.conf', + line => 'NotifyClamd /etc/clamav/clamd.conf', + require => Package['clamav-freshclam'], + notify => Service['freshclam']; } } diff --git a/puppet/modules/clamav/manifests/milter.pp b/puppet/modules/clamav/manifests/milter.pp index 52ddaef1..e8a85e3f 100644 --- a/puppet/modules/clamav/manifests/milter.pp +++ b/puppet/modules/clamav/manifests/milter.pp @@ -14,6 +14,7 @@ class clamav::milter { name => clamav-milter, pattern => '/usr/sbin/clamav-milter', hasrestart => true, + require => Package['clamav-milter'], subscribe => File['/etc/default/clamav-milter']; } @@ -29,6 +30,7 @@ class clamav::milter { mode => '0644', owner => root, group => root, + require => Package['clamav-milter'], subscribe => Service['clamav-milter']; '/etc/default/clamav-milter': @@ -41,8 +43,8 @@ class clamav::milter { content => template('clamav/whitelisted_addresses.erb'), mode => '0644', owner => root, - group => root; - + group => root, + require => Package['clamav-milter']; } } -- cgit v1.2.3 From 2443311119a618e544f0f701c4a596690a3fcd05 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 15 Oct 2015 17:12:48 -0400 Subject: switch to ensure_packages to avoid puppet duplicate package definitions (#7530) Change-Id: I398b929fc96cf64e46075266ace0d8d1145b3aac --- puppet/modules/clamav/manifests/unofficial_sigs.pp | 5 +++-- puppet/modules/couchdb | 2 +- puppet/modules/ruby | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/puppet/modules/clamav/manifests/unofficial_sigs.pp b/puppet/modules/clamav/manifests/unofficial_sigs.pp index 316154d3..2d849585 100644 --- a/puppet/modules/clamav/manifests/unofficial_sigs.pp +++ b/puppet/modules/clamav/manifests/unofficial_sigs.pp @@ -1,10 +1,11 @@ class clamav::unofficial_sigs { - package { [ 'clamav-unofficial-sigs', 'wget', 'gnupg', - 'socat', 'rsync', 'curl' ]: + package { 'clamav-unofficial-sigs': ensure => installed } + ensure_packages(['wget', 'gnupg', 'socat', 'rsync', 'curl']) + file { '/var/log/clamav-unofficial-sigs.log': ensure => file, diff --git a/puppet/modules/couchdb b/puppet/modules/couchdb index ae53b180..d077a7b1 160000 --- a/puppet/modules/couchdb +++ b/puppet/modules/couchdb @@ -1 +1 @@ -Subproject commit ae53b180783016faa4331094a52769ddd57463f8 +Subproject commit d077a7b11c95089882e08432c45b883a9097e81d diff --git a/puppet/modules/ruby b/puppet/modules/ruby index e4de25d7..0fb2b398 160000 --- a/puppet/modules/ruby +++ b/puppet/modules/ruby @@ -1 +1 @@ -Subproject commit e4de25d78eefc7df70a35dee22a3e0dc1b7e1d0b +Subproject commit 0fb2b398dbfce59c678d6f4044a55969e42c6d4d -- cgit v1.2.3 From 20f298c48e6df0908dddea696d972c61b0e88bb8 Mon Sep 17 00:00:00 2001 From: varac Date: Sat, 17 Oct 2015 13:32:42 +0200 Subject: [bug] updated submodule couchdb - Tested: [local singlenode, citest] - Resolves: #7530 --- puppet/modules/couchdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/couchdb b/puppet/modules/couchdb index d077a7b1..cdde1e17 160000 --- a/puppet/modules/couchdb +++ b/puppet/modules/couchdb @@ -1 +1 @@ -Subproject commit d077a7b11c95089882e08432c45b883a9097e81d +Subproject commit cdde1e172b3ed2c6c1f203341e75bcef5c3c3491 -- cgit v1.2.3 From add63156286e3b89ae38b8f6975e84817f772373 Mon Sep 17 00:00:00 2001 From: varac Date: Sat, 17 Oct 2015 14:16:32 +0200 Subject: [feat] Added contrib folder for contributed stuff - Added a README.md - added a git commit template - moved offlineimap example config file from vagrant/ to crontib/ --- contrib/README.md | 9 +++++++++ contrib/commit-template.txt | 7 +++++++ contrib/offlineimaprc.example.org | 24 ++++++++++++++++++++++++ vagrant/offlineimaprc.example.org | 24 ------------------------ 4 files changed, 40 insertions(+), 24 deletions(-) create mode 100644 contrib/README.md create mode 100644 contrib/commit-template.txt create mode 100644 contrib/offlineimaprc.example.org delete mode 100644 vagrant/offlineimaprc.example.org diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 00000000..e836bc7e --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,9 @@ +# Contributed Files + +## Commit Template + +to install this commit template, use following cmd (use --global to use it in your global .gitconfig): + + git config [--global] commit.template "~/path_to_leap_platform/contrib/commit-template.txt" + + diff --git a/contrib/commit-template.txt b/contrib/commit-template.txt new file mode 100644 index 00000000..9a1fa81b --- /dev/null +++ b/contrib/commit-template.txt @@ -0,0 +1,7 @@ +#[bug|feat|docs|style|refactor|test|pkg|i18n] + +#- Tested: [local singlenode|local multinode|citest|unstable.bitmask.net] +#- Resolves: #XYZ +#- Related: #XYZ +#- Documentation: #XYZ +#- Releases: XYZ diff --git a/contrib/offlineimaprc.example.org b/contrib/offlineimaprc.example.org new file mode 100644 index 00000000..3d119634 --- /dev/null +++ b/contrib/offlineimaprc.example.org @@ -0,0 +1,24 @@ +# WARNING: Use offlineimap *only* for testing/debugging, +# because it will save the mails *decrypted* locally to +# your disk ! + +[general] +accounts = testuser@example.org + +[Account testuser@example.org] +localrepository = testuser@example.org_local +remoterepository = testuser@example.org_remote + +[Repository testuser@example.org_local] +type = Maildir +localfolders = /tmp/offlineimap.testuser@example.org + +[Repository testuser@example.org_remote] +type = IMAP +remotehost = localhost +remoteuser = testuser@example.org +remoteport = 1984 +ssl = no +remotepass = every_pw_works_here + + diff --git a/vagrant/offlineimaprc.example.org b/vagrant/offlineimaprc.example.org deleted file mode 100644 index 3d119634..00000000 --- a/vagrant/offlineimaprc.example.org +++ /dev/null @@ -1,24 +0,0 @@ -# WARNING: Use offlineimap *only* for testing/debugging, -# because it will save the mails *decrypted* locally to -# your disk ! - -[general] -accounts = testuser@example.org - -[Account testuser@example.org] -localrepository = testuser@example.org_local -remoterepository = testuser@example.org_remote - -[Repository testuser@example.org_local] -type = Maildir -localfolders = /tmp/offlineimap.testuser@example.org - -[Repository testuser@example.org_remote] -type = IMAP -remotehost = localhost -remoteuser = testuser@example.org -remoteport = 1984 -ssl = no -remotepass = every_pw_works_here - - -- cgit v1.2.3 From 9b18f7880aad97320cd5d118c31f04a0afc7c542 Mon Sep 17 00:00:00 2001 From: guido Date: Mon, 19 Oct 2015 15:07:30 -0300 Subject: Redirect to webapp_domain instead of domain This is needed for webapp when running on a subdomain. --- puppet/modules/site_apache/templates/vhosts.d/common.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb index ee5cd707..7f9fd5ab 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb @@ -4,7 +4,7 @@ ServerAlias <%= domain %> ServerAlias www.<%= domain %> RewriteEngine On - RewriteRule ^.*$ https://<%= domain -%>%{REQUEST_URI} [R=permanent,L] + RewriteRule ^.*$ https://<%= webapp_domain -%>%{REQUEST_URI} [R=permanent,L] CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common -- cgit v1.2.3 From 91c638f7d30243f0c5c079659bd3bd1d32a7cc7c Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 19 Oct 2015 20:57:07 -0400 Subject: change apache header set for HSTS to be always, otherwise it wont be set for redirects (#7540) Change-Id: Ic77c64c03a99dad951f42633de04c352bed17c1e --- puppet/modules/site_static/templates/apache.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_static/templates/apache.conf.erb b/puppet/modules/site_static/templates/apache.conf.erb index 4d61cc08..2853c5c7 100644 --- a/puppet/modules/site_static/templates/apache.conf.erb +++ b/puppet/modules/site_static/templates/apache.conf.erb @@ -48,7 +48,7 @@ Include include.d/ssl_common.inc <%- if @tls_only -%> - Header add Strict-Transport-Security: "max-age=15768000;includeSubdomains" + Header always set Strict-Transport-Security: "max-age=15768000;includeSubdomains" <%- end -%> Header set X-Frame-Options "deny" Header always unset X-Powered-By -- cgit v1.2.3 From 1ade690d20618ca5adb0c4a1647b36200197fd26 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 20 Oct 2015 17:17:39 -0400 Subject: Provide tor hidden service configuration for static sites (#7546) Without this configuration, a very basic, and non-functional virtualhost is created, making the hidden service not work Change-Id: Ibe87c6acf5c21cff2388247c4ba320a5b6af7933 --- .../site_apache/templates/vhosts.d/hidden_service.conf.erb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb index 0c6f3b8e..2c8d5eb5 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb @@ -30,4 +30,14 @@ ExpiresDefault "access plus 1 year" <% end -%> + +<% if (defined? @services) and (@services.include? 'static') -%> + DocumentRoot "/srv/static/root/public" + AccessFileName .htaccess + + Alias /provider.json /srv/leap/provider.json + + Header set X-Minimum-Client-Version 0.5 + +<% end -%> -- cgit v1.2.3 From d7f8fde6d48568c912733aca6fa3030ccdfcada7 Mon Sep 17 00:00:00 2001 From: varac Date: Mon, 26 Oct 2015 10:56:31 +0100 Subject: [bug] Disable user-creation and soledad test Until we have a proper fix for #7523, we disable this test. - Tested: [unstable.bitmask.net] - Related: #7523 --- tests/white-box/webapp.rb | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb index 8be6bde2..48507521 100644 --- a/tests/white-box/webapp.rb +++ b/tests/white-box/webapp.rb @@ -41,34 +41,6 @@ class Webapp < LeapTest pass end - def test_05_Can_create_and_authenticate_and_delete_user_via_API? - if property('webapp.allow_registration') - assert_tmp_user - pass - else - skip "New user registrations are disabled." - end - end - - def test_06_Can_sync_Soledad? - return unless property('webapp.allow_registration') - soledad_config = property('definition_files.soledad_service') - if soledad_config && !soledad_config.empty? - soledad_server = pick_soledad_server(soledad_config) - if soledad_server - assert_tmp_user do |user| - assert_user_db_exists(user) - command = File.expand_path "../../helpers/soledad_sync.py", __FILE__ - soledad_url = "https://#{soledad_server}/user-#{user.id}" - assert_run "#{command} #{user.id} #{user.session_token} #{soledad_url}" - pass - end - end - else - skip 'No soledad service configuration' - end - end - private def url_options -- cgit v1.2.3 From 88746d15542dbc4d4f34d810b6362fb3f5481b10 Mon Sep 17 00:00:00 2001 From: varac Date: Thu, 22 Oct 2015 19:10:39 +0200 Subject: Add Jenkins Embeddable Build Status Icon --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b25b26d..84af4324 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ -What is it? +Leap Platform ============================= +[![Build Status](https://squirrel.leap.se:8080/job/platform_develop/badge/icon)](https://squirrel.leap.se:8080/job/platform_develop/) + The LEAP Platform is set of complementary packages and server recipes to automate the maintenance of LEAP services in a hardened Debian environment. Its goal is to make it as painless as possible for sysadmins to deploy and maintain a service provider's infrastructure for secure communication. These recipes define an abstract service provider. It is a set of Puppet modules designed to work together to provide to sysadmins everything they need to manage a service provider infrastructure that provides secure communication services. Getting started -- cgit v1.2.3 From 4449f09cc362ec0cf64881f547cb41e73cbd67f5 Mon Sep 17 00:00:00 2001 From: varac Date: Mon, 26 Oct 2015 13:26:25 +0100 Subject: updated unbound submodule --- puppet/modules/unbound | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/unbound b/puppet/modules/unbound index 00646b0f..9997485b 160000 --- a/puppet/modules/unbound +++ b/puppet/modules/unbound @@ -1 +1 @@ -Subproject commit 00646b0ffc71a86981b05f983c86ace0979d1b6f +Subproject commit 9997485b8a31abbe0cd1943d09995705c2c8146a -- cgit v1.2.3 From 171a5a9a3794224a92244078574aac4b22845266 Mon Sep 17 00:00:00 2001 From: varac Date: Mon, 26 Oct 2015 16:18:17 +0100 Subject: [bug] Add leap_mx username to soledad.conf - Tested: [unstable.pixelated-project.org] - Related: https://github.com/pixelated/pixelated-platform/issues/127 --- provider_base/services/soledad.json | 3 +++ puppet/modules/soledad/manifests/server.pp | 7 ++++--- puppet/modules/soledad/templates/soledad-server.conf.erb | 8 ++++++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/provider_base/services/soledad.json b/provider_base/services/soledad.json index 76f7155f..99390d17 100644 --- a/provider_base/services/soledad.json +++ b/provider_base/services/soledad.json @@ -6,6 +6,9 @@ "username": "= global.services[:couchdb].couch.users[:soledad].username", "password": "= secret :couch_soledad_password", "salt": "= hex_secret :couch_soledad_password_salt, 128" + }, + "couchdb_leap_mx_user": { + "username": "= global.services[:couchdb].couch.users[:leap_mx].username" } }, "service_type": "public_service", diff --git a/puppet/modules/soledad/manifests/server.pp b/puppet/modules/soledad/manifests/server.pp index e437c8f2..1113bd86 100644 --- a/puppet/modules/soledad/manifests/server.pp +++ b/puppet/modules/soledad/manifests/server.pp @@ -4,9 +4,10 @@ class soledad::server { include soledad include site_apt::preferences::twisted - $soledad = hiera('soledad') - $couchdb_user = $soledad['couchdb_soledad_user']['username'] - $couchdb_password = $soledad['couchdb_soledad_user']['password'] + $soledad = hiera('soledad') + $couchdb_user = $soledad['couchdb_soledad_user']['username'] + $couchdb_password = $soledad['couchdb_soledad_user']['password'] + $couchdb_leap_mx_user = $soledad['couchdb_leap_mx_user']['username'] $couchdb_host = 'localhost' $couchdb_port = '5984' diff --git a/puppet/modules/soledad/templates/soledad-server.conf.erb b/puppet/modules/soledad/templates/soledad-server.conf.erb index 42cf44d8..1c6a0d19 100644 --- a/puppet/modules/soledad/templates/soledad-server.conf.erb +++ b/puppet/modules/soledad/templates/soledad-server.conf.erb @@ -2,3 +2,11 @@ couch_url = http://<%= @couchdb_user %>:<%= @couchdb_password %>@<%= @couchdb_host %>:<%= @couchdb_port %> create_cmd = sudo -u soledad-admin /usr/bin/create-user-db admin_netrc = /etc/couchdb/couchdb-soledad-admin.netrc + +[database-security] +members = <%= @couchdb_user %>, <%= @couchdb_leap_mx_user %> +# not needed, but for documentation: +# members_roles = replication +# admins = admin +# admins_roles = replication + -- cgit v1.2.3 From 72bec64f52895153612b5e736274266ebc0ab554 Mon Sep 17 00:00:00 2001 From: varac Date: Fri, 30 Oct 2015 10:31:17 +0100 Subject: [feat] Add soledad::client class for soledad-sync - Restructure soledad class - Include soledad::client class on webapp nodes - Tested: [unstable.bitmask.net] - Related: #7523 --- puppet/modules/site_webapp/manifests/init.pp | 6 ++-- puppet/modules/soledad/manifests/client.pp | 18 ++++++++++++ puppet/modules/soledad/manifests/common.pp | 6 ++-- puppet/modules/soledad/manifests/init.pp | 40 --------------------------- puppet/modules/soledad/manifests/server.pp | 41 ++++++++++++++++++++++++---- 5 files changed, 59 insertions(+), 52 deletions(-) create mode 100644 puppet/modules/soledad/manifests/client.pp delete mode 100644 puppet/modules/soledad/manifests/init.pp diff --git a/puppet/modules/site_webapp/manifests/init.pp b/puppet/modules/site_webapp/manifests/init.pp index d046b7df..837950a8 100644 --- a/puppet/modules/site_webapp/manifests/init.pp +++ b/puppet/modules/site_webapp/manifests/init.pp @@ -167,10 +167,8 @@ class site_webapp { # needed for the soledad-sync check which is run on the - # webapp node (#6520) - package { 'python-u1db': - ensure => latest, - } + # webapp node + include soledad::client leap::logfile { 'webapp': } diff --git a/puppet/modules/soledad/manifests/client.pp b/puppet/modules/soledad/manifests/client.pp new file mode 100644 index 00000000..5700cb09 --- /dev/null +++ b/puppet/modules/soledad/manifests/client.pp @@ -0,0 +1,18 @@ +# setup soledad-client +# currently needed on webapp node to run the soledad-sync test +class soledad::client { + + tag 'leap_service' + include soledad::common + + package { + 'soledad-client': + ensure => latest, + require => [ + Class['site_apt::preferences::twisted'], + Class['site_apt::leap_repo'] ]; + 'python-u1db': + ensure => latest; + } + +} diff --git a/puppet/modules/soledad/manifests/common.pp b/puppet/modules/soledad/manifests/common.pp index 8a1d664a..d66e943c 100644 --- a/puppet/modules/soledad/manifests/common.pp +++ b/puppet/modules/soledad/manifests/common.pp @@ -1,10 +1,10 @@ +# install soledad-common, both needed both soledad-client and soledad-server class soledad::common { - include soledad + include site_apt::preferences::twisted package { 'soledad-common': - ensure => latest, - require => User['soledad'] + ensure => latest; } } diff --git a/puppet/modules/soledad/manifests/init.pp b/puppet/modules/soledad/manifests/init.pp deleted file mode 100644 index 6a2c328e..00000000 --- a/puppet/modules/soledad/manifests/init.pp +++ /dev/null @@ -1,40 +0,0 @@ -# set up users, group and directories for soledad-server -# although the soledad users are already created by the -# soledad-server package -class soledad { - - group { 'soledad': - ensure => present, - system => true, - } - - user { 'soledad': - ensure => present, - system => true, - gid => 'soledad', - home => '/srv/leap/soledad', - require => Group['soledad']; - } - - user { 'soledad-admin': - ensure => present, - system => true, - gid => 'soledad', - home => '/srv/leap/soledad', - require => Group['soledad']; - } - - file { - '/srv/leap/soledad': - ensure => directory, - owner => 'soledad', - group => 'soledad', - require => User['soledad']; - - '/var/lib/soledad': - ensure => directory, - owner => 'soledad', - group => 'soledad', - require => User['soledad']; - } -} diff --git a/puppet/modules/soledad/manifests/server.pp b/puppet/modules/soledad/manifests/server.pp index 1113bd86..5c5a1bb7 100644 --- a/puppet/modules/soledad/manifests/server.pp +++ b/puppet/modules/soledad/manifests/server.pp @@ -1,8 +1,7 @@ # setup soledad-server class soledad::server { tag 'leap_service' - include soledad - include site_apt::preferences::twisted + include soledad::common $soledad = hiera('soledad') $couchdb_user = $soledad['couchdb_soledad_user']['username'] @@ -36,7 +35,17 @@ class soledad::server { group => 'soledad', mode => '0640', notify => Service['soledad-server'], - require => Class['soledad']; + require => [ User['soledad'], Group['soledad'] ]; + '/srv/leap/soledad': + ensure => directory, + owner => 'soledad', + group => 'soledad', + require => [ User['soledad'], Group['soledad'] ]; + '/var/lib/soledad': + ensure => directory, + owner => 'soledad', + group => 'soledad', + require => [ User['soledad'], Group['soledad'] ]; } package { $sources['soledad']['package']: @@ -52,7 +61,7 @@ class soledad::server { group => 'soledad', mode => '0600', notify => Service['soledad-server'], - require => Class['soledad']; + require => [ User['soledad'], Group['soledad'] ]; } service { 'soledad-server': @@ -60,7 +69,7 @@ class soledad::server { enable => true, hasstatus => true, hasrestart => true, - require => Class['soledad'], + require => [ User['soledad'], Group['soledad'] ], subscribe => [ Package['soledad-server'], Class['Site_config::X509::Key'], @@ -70,4 +79,26 @@ class soledad::server { include site_shorewall::soledad include site_check_mk::agent::soledad + + # set up users, group and directories for soledad-server + # although the soledad users are already created by the + # soledad-server package + group { 'soledad': + ensure => present, + system => true, + } + user { + 'soledad': + ensure => present, + system => true, + gid => 'soledad', + home => '/srv/leap/soledad', + require => Group['soledad']; + 'soledad-admin': + ensure => present, + system => true, + gid => 'soledad', + home => '/srv/leap/soledad', + require => Group['soledad']; + } } -- cgit v1.2.3 From cfbe272d17a21c4bff088a87865cbcbefc837e39 Mon Sep 17 00:00:00 2001 From: varac Date: Fri, 30 Oct 2015 11:23:40 +0100 Subject: [feat] Remove bigcouch nagios leftovers When migrating from bigcouch to couchdb, we need to remove leftover nagios tests for bigcouch. - Added new classes: site_check_mk::agent::couchdb::bigcouch and site_check_mk::agent::couchdb::master - Tested: unstable.pixelated-project.org - Resolves: https://github.com/pixelated/pixelated-platform/issues/126 --- .../site_check_mk/manifests/agent/couchdb.pp | 49 ++++--------------- .../manifests/agent/couchdb/bigcouch.pp | 56 ++++++++++++++++++++++ .../manifests/agent/couchdb/master.pp | 23 +++++++++ puppet/modules/site_couchdb/manifests/bigcouch.pp | 3 ++ puppet/modules/site_couchdb/manifests/master.pp | 2 + 5 files changed, 94 insertions(+), 39 deletions(-) create mode 100644 puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp create mode 100644 puppet/modules/site_check_mk/manifests/agent/couchdb/master.pp diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp index 8de5121b..1554fd3c 100644 --- a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp @@ -1,40 +1,18 @@ -# configure logwatch and nagios checks for couchdb +# configure logwatch and nagios checks for couchdb (both bigcouch and plain +# couchdb installations) class site_check_mk::agent::couchdb { - # watch bigcouch logs - # currently disabled because bigcouch is too noisy - # see https://leap.se/code/issues/7375 for more details - # and site_config::remove_files for removing leftovers - #file { '/etc/check_mk/logwatch.d/bigcouch.cfg': - # source => 'puppet:///modules/site_check_mk/agent/logwatch/bigcouch.cfg', - #} - - # check syslog msg from: - # - empd - # - /usr/local/bin/couch-doc-update concat::fragment { 'syslog_couchdb': source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/couchdb.cfg', target => '/etc/check_mk/logwatch.d/syslog.cfg', order => '02'; } - - # check bigcouch processes - augeas { - 'Bigcouch_epmd_procs': - incl => '/etc/check_mk/mrpe.cfg', - lens => 'Spacevars.lns', - changes => [ - 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_epmd_procs', - 'set Bigcouch_epmd_procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/epmd\'' ], - require => File['/etc/check_mk/mrpe.cfg']; - 'Bigcouch_beam_procs': - incl => '/etc/check_mk/mrpe.cfg', - lens => 'Spacevars.lns', - changes => [ - 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_beam_procs', - 'set Bigcouch_beam_procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/beam\'' ], - require => File['/etc/check_mk/mrpe.cfg']; + # check different couchdb stats + file { '/usr/lib/check_mk_agent/local/leap_couch_stats.sh': + source => 'puppet:///modules/site_check_mk/agent/local_checks/couchdb/leap_couch_stats.sh', + mode => '0755', + require => Package['check_mk-agent'] } # check open files for bigcouch proc @@ -44,20 +22,13 @@ class site_check_mk::agent::couchdb { mode => '0755' } augeas { - 'Bigcouch_open_files': + 'Couchdb_open_files': incl => '/etc/check_mk/mrpe.cfg', lens => 'Spacevars.lns', changes => [ - 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_open_files', - 'set Bigcouch_open_files \'/srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 28672,28672 -c 30720,30720\'' ], + 'rm /files/etc/check_mk/mrpe.cfg/Couchdb_open_files', + 'set Couchdb_open_files \'/srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 28672,28672 -c 30720,30720\'' ], require => File['/etc/check_mk/mrpe.cfg']; } - - # check different couchdb stats - file { '/usr/lib/check_mk_agent/local/leap_couch_stats.sh': - source => 'puppet:///modules/site_check_mk/agent/local_checks/couchdb/leap_couch_stats.sh', - mode => '0755', - require => Package['check_mk-agent'] - } } diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp new file mode 100644 index 00000000..073d07a9 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp @@ -0,0 +1,56 @@ +# configure logwatch and nagios checks for bigcouch +class site_check_mk::agent::couchdb::bigcouch { + + # watch bigcouch logs + # currently disabled because bigcouch is too noisy + # see https://leap.se/code/issues/7375 for more details + # and site_config::remove_files for removing leftovers + #file { '/etc/check_mk/logwatch.d/bigcouch.cfg': + # source => 'puppet:///modules/site_check_mk/agent/logwatch/bigcouch.cfg', + #} + + # check syslog msg from: + # - empd + # - /usr/local/bin/couch-doc-update + concat::fragment { 'syslog_bigcouch': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/bigcouch.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + + # check bigcouch processes + augeas { + 'Bigcouch_epmd_procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_epmd_procs', + 'set Bigcouch_epmd_procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/epmd\'' ], + require => File['/etc/check_mk/mrpe.cfg']; + 'Bigcouch_beam_procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_beam_procs', + 'set Bigcouch_beam_procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/beam\'' ], + require => File['/etc/check_mk/mrpe.cfg']; + } + + # check open files for bigcouch proc + include site_check_mk::agent::package::perl_plugin + file { '/srv/leap/nagios/plugins/check_unix_open_fds.pl': + source => 'puppet:///modules/site_check_mk/agent/nagios_plugins/check_unix_open_fds.pl', + mode => '0755' + } + + augeas { + 'Bigcouch_open_files': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_open_files', + 'set Bigcouch_open_files \'/srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 28672,28672 -c 30720,30720\'' ], + require => File['/etc/check_mk/mrpe.cfg']; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb/master.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb/master.pp new file mode 100644 index 00000000..291b87d1 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb/master.pp @@ -0,0 +1,23 @@ +# configure logwatch and nagios checks for plain single couchdb master +class site_check_mk::agent::couchdb::master { + + # remove bigcouch leftovers + augeas { + 'Bigcouch_epmd_procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_epmd_procs', + require => File['/etc/check_mk/mrpe.cfg']; + 'Bigcouch_beam_procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_beam_procs', + require => File['/etc/check_mk/mrpe.cfg']; + 'Bigcouch_open_files': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_open_files', + require => File['/etc/check_mk/mrpe.cfg']; + } + +} diff --git a/puppet/modules/site_couchdb/manifests/bigcouch.pp b/puppet/modules/site_couchdb/manifests/bigcouch.pp index 469a2783..2de3d4d0 100644 --- a/puppet/modules/site_couchdb/manifests/bigcouch.pp +++ b/puppet/modules/site_couchdb/manifests/bigcouch.pp @@ -44,4 +44,7 @@ class site_couchdb::bigcouch { require => Package['couchdb'], notify => Service['couchdb'] } + + include site_check_mk::agent::couchdb::bigcouch + } diff --git a/puppet/modules/site_couchdb/manifests/master.pp b/puppet/modules/site_couchdb/manifests/master.pp index c28eee7d..5dab6325 100644 --- a/puppet/modules/site_couchdb/manifests/master.pp +++ b/puppet/modules/site_couchdb/manifests/master.pp @@ -6,4 +6,6 @@ class site_couchdb::master { chttpd_bind_address => '127.0.0.1', pwhash_alg => $site_couchdb::couchdb_pwhash_alg } + + include site_check_mk::agent::couchdb::master } -- cgit v1.2.3 From 9b135ab96f1e419698e3e638ea871097fe4956e4 Mon Sep 17 00:00:00 2001 From: varac Date: Fri, 30 Oct 2015 18:00:51 +0100 Subject: [bug] Remove duplicte declaration Duplicate declaration: File[/srv/leap/nagios/plugins/check_unix_open_fds.pl] is already declared in file /srv/leap/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp at line 44; cannot redeclare at /srv/leap/puppet/modules/site_check_mk/manifests/agent/couchdb.pp:23 on node rewdevcouch1.rewire.org --- puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp index 073d07a9..82c3ac72 100644 --- a/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp @@ -36,13 +36,6 @@ class site_check_mk::agent::couchdb::bigcouch { require => File['/etc/check_mk/mrpe.cfg']; } - # check open files for bigcouch proc - include site_check_mk::agent::package::perl_plugin - file { '/srv/leap/nagios/plugins/check_unix_open_fds.pl': - source => 'puppet:///modules/site_check_mk/agent/nagios_plugins/check_unix_open_fds.pl', - mode => '0755' - } - augeas { 'Bigcouch_open_files': incl => '/etc/check_mk/mrpe.cfg', -- cgit v1.2.3 From 87ddb4d6505229f36b096188c3e43a19281b540c Mon Sep 17 00:00:00 2001 From: varac Date: Sat, 31 Oct 2015 20:03:28 +0100 Subject: [bug] Add bigcouch syslog snippet for logwatch --- .../modules/site_check_mk/files/agent/logwatch/syslog/bigcouch.cfg | 5 +++++ puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 puppet/modules/site_check_mk/files/agent/logwatch/syslog/bigcouch.cfg diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/bigcouch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/bigcouch.cfg new file mode 100644 index 00000000..f53f0780 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/bigcouch.cfg @@ -0,0 +1,5 @@ +# on one-node bigcouch setups, we'll get this msg +# a lot, so we ignore it here until we fix +# https://leap.se/code/issues/5244 + I epmd: got partial packet only on file descriptor + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg index f546135a..5f8d5b95 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg @@ -1,7 +1,2 @@ C /usr/local/bin/couch-doc-update.*failed C /usr/local/bin/couch-doc-update.*ERROR -# on one-node bigcouch setups, we'll get this msg -# a lot, so we ignore it here until we fix -# https://leap.se/code/issues/5244 - I epmd: got partial packet only on file descriptor - -- cgit v1.2.3 From e97a9d3800b173375a630e18e4b1aa0894eb96e1 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 20 Oct 2015 17:14:21 -0400 Subject: Add basic DKIM support, this requires changes in leap_cli detailed in issue #5924 Change-Id: I6aa1e7751633407d441cbc6436d8426d37dbbfa7 --- puppet/modules/opendkim/manifests/init.pp | 38 +++++++++++++++++++ puppet/modules/opendkim/templates/opendkim.conf | 44 ++++++++++++++++++++++ .../modules/site_config/manifests/x509/dkim/key.pp | 13 +++++++ puppet/modules/site_postfix/manifests/mx.pp | 2 +- 4 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 puppet/modules/opendkim/manifests/init.pp create mode 100644 puppet/modules/opendkim/templates/opendkim.conf create mode 100644 puppet/modules/site_config/manifests/x509/dkim/key.pp diff --git a/puppet/modules/opendkim/manifests/init.pp b/puppet/modules/opendkim/manifests/init.pp new file mode 100644 index 00000000..9e67569e --- /dev/null +++ b/puppet/modules/opendkim/manifests/init.pp @@ -0,0 +1,38 @@ +# configure opendkim service (#5924) +class opendkim { + + $domain_hash = hiera('domain') + $domain = $domain_hash['full_suffix'] + $dkim = hiera('dkim') + $selector = $dkim['dkim_selector'] + + include site_config::x509::dkim::key + $dkim_key = "${x509::variables::keys}/dkim.key" + + ensure_packages(['opendkim', 'libopendkim7', 'libvbr2']) + + # postfix user needs to be in the opendkim group + # in order to access the opendkim socket located at: + # local:/var/run/opendkim/opendkim.sock + user { 'postfix': + groups => 'opendkim'; + } + + service { 'opendkim': + ensure => running, + enable => true, + hasstatus => true, + hasrestart => true, + require => Class['Site_config::X509::Dkim::Key'], + subscribe => File[$dkim_key]; + } + + file { '/etc/opendkim.conf': + ensure => present, + content => template('opendkim/opendkim.conf'), + mode => '0644', + owner => root, + group => root, + notify => Service['opendkim'], + require => Package['opendkim']; +} diff --git a/puppet/modules/opendkim/templates/opendkim.conf b/puppet/modules/opendkim/templates/opendkim.conf new file mode 100644 index 00000000..46ddb7a8 --- /dev/null +++ b/puppet/modules/opendkim/templates/opendkim.conf @@ -0,0 +1,44 @@ +# This is a basic configuration that can easily be adapted to suit a standard +# installation. For more advanced options, see opendkim.conf(5) and/or +# /usr/share/doc/opendkim/examples/opendkim.conf.sample. + +# Log to syslog +Syslog yes +SyslogSuccess yes +LogWhy no +# Required to use local socket with MTAs that access the socket as a non- +# privileged user (e.g. Postfix) +UMask 002 + +Domain <%= @domain %> +SubDomains yes + +# set internal hosts to all the known hosts, like mydomains? + +# can we generate a larger key and get it in dns? +KeyFile <%= @dkim_key %> + +# what selector do we use? +Selector <%= @selector %> + +# Commonly-used options; the commented-out versions show the defaults. +Canonicalization relaxed +#Mode sv +#ADSPDiscard no + +# Always oversign From (sign using actual From and a null From to prevent +# malicious signatures header fields (From and/or others) between the signer +# and the verifier. From is oversigned by default in the Debian pacakge +# because it is often the identity key used by reputation systems and thus +# somewhat security sensitive. +OversignHeaders From + +# List domains to use for RFC 6541 DKIM Authorized Third-Party Signatures +# (ATPS) (experimental) + +#ATPSDomains example.com + +RemoveOldSignatures yes + +Mode sv +BaseDirectory /var/tmp diff --git a/puppet/modules/site_config/manifests/x509/dkim/key.pp b/puppet/modules/site_config/manifests/x509/dkim/key.pp new file mode 100644 index 00000000..c63a7e94 --- /dev/null +++ b/puppet/modules/site_config/manifests/x509/dkim/key.pp @@ -0,0 +1,13 @@ +class site_config::x509::dkim::key { + + ## + ## This is for the DKIM key that is used exclusively for DKIM + ## signing + + $x509 = hiera('x509') + $key = $x509['dkim_key'] + + x509::key { 'dkim': + content => $key + } +} diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp index f0a2554a..edaa506f 100644 --- a/puppet/modules/site_postfix/manifests/mx.pp +++ b/puppet/modules/site_postfix/manifests/mx.pp @@ -50,7 +50,7 @@ class site_postfix::mx { 'local_recipient_maps': value => '$alias_maps'; 'smtpd_milters': - value => 'unix:/run/clamav/milter.ctl'; + value => 'unix:/run/clamav/milter.ctl,unix:/var/run/opendkim/opendkim.sock'; 'milter_default_action': value => 'accept'; } -- cgit v1.2.3 From ed1ff6fa01bf110fc338b7116fdf577aa88a8d46 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 27 Oct 2015 15:27:24 -0400 Subject: Add initial rate-limiting for outgoing SMTP, using postfwd (#5972) Change-Id: I6a6e68908b71d7499eb3ef3c7f0173b3d5b7baa2 --- puppet/modules/postfwd/files/postfwd_default | 19 ++++++++++ puppet/modules/postfwd/manifests/init.pp | 49 +++++++++++++++++++++++++ puppet/modules/postfwd/templates/postfwd.cf.erb | 31 ++++++++++++++++ puppet/modules/site_postfix/manifests/mx.pp | 1 + 4 files changed, 100 insertions(+) create mode 100644 puppet/modules/postfwd/files/postfwd_default create mode 100644 puppet/modules/postfwd/manifests/init.pp create mode 100644 puppet/modules/postfwd/templates/postfwd.cf.erb diff --git a/puppet/modules/postfwd/files/postfwd_default b/puppet/modules/postfwd/files/postfwd_default new file mode 100644 index 00000000..79d0e3de --- /dev/null +++ b/puppet/modules/postfwd/files/postfwd_default @@ -0,0 +1,19 @@ +### This file managed by Puppet +# Global options for postfwd(8). + +# Set to '1' to enable startup (daemon mode) +STARTUP=1 + +# Config file +CONF=/etc/postfix/postfwd.cf +# IP where listen to +INET=127.0.0.1 +# Port where listen to +PORT=10040 +# run as user postfwd +RUNAS="postfw" +# Arguments passed on start (--daemon implied) +# RISEUP disable summary and cache-no-size +#ARGS="--summary=600 --cache=600 --cache-rdomain-only --cache-no-size" +ARGS="--cache=600 --cache-rdomain-only --no-rulestats" + diff --git a/puppet/modules/postfwd/manifests/init.pp b/puppet/modules/postfwd/manifests/init.pp new file mode 100644 index 00000000..b00bb071 --- /dev/null +++ b/puppet/modules/postfwd/manifests/init.pp @@ -0,0 +1,49 @@ +# This class provides rate-limiting for outgoing SMTP, using postfwd +# it is configured with some limits that seem reasonable for a generic +# use-case. Each of the following applies to sasl_authenticated users: +# +# . 150 recipients at a time +# . no more than 50 messages in 60 minutes +# . no more than 250 recipients in 60 minutes. +# +# This class could be easily extended to add overrides to these rules, +# maximum sizes per client, or additional rules +class postfwd { + + ensure_packages(['libnet-server-perl', 'libnet-dns-perl', 'postfwd']) + + file { + '/etc/default/postfwd': + source => 'puppet:///modules/postfwd/postfwd', + mode => '0644', + owner => root, + group => root, + require => Package['postfwd']; + + '/etc/postfix/postfwd.cf': + content => template('postfwd/postfwd.cf.erb'), + mode => '0644', + owner => root, + group => root, + require => File['/etc/postfix']; + } + + exec { + '/etc/init.d/postfwd reload': + refreshonly => true, + subscribe => [ File['/etc/postfix/postfwd.cf'], + File['/etc/default/postfwd'] ]; + } + + service { + 'postfwd': + ensure => running, + name => postfwd, + pattern => '/usr/sbin/postfwd', + enable => true, + hasrestart => true, + hasstatus => false, + require => [ File['/etc/default/postfwd'], + File['/etc/postfix/postfwd.cf']]; + } +} diff --git a/puppet/modules/postfwd/templates/postfwd.cf.erb b/puppet/modules/postfwd/templates/postfwd.cf.erb new file mode 100644 index 00000000..6460994a --- /dev/null +++ b/puppet/modules/postfwd/templates/postfwd.cf.erb @@ -0,0 +1,31 @@ +### This file managed by Puppet +# Before deploying a rule +# 1. test with an additional "sender==test@domain.org;" in the rule so it +# only applies to your test account +# 2. then when ready to test for all users, use WARN and watch the logs +# for a few days and make sure it working the way you like +# 3. Then when ready to deploy for real set a proper error code + +## Overrides - make like the following example +# id=exampleuser; sasl_username==exampleuser; action=dunno + +## Rules that apply to all senders +# Recipient Per Message Limit +# We only receive mail via smtp from sasl authenticated users +# directly. We want to limit to a lower amount to prevent phished accounts +# spamming +id=RCPTSENDER; recipient_count=150; action=REJECT Too many recipients, please try again. Contact http://<%= @domain %>/tickets/new if this is in error. ERROR:RCPTSENDER + +# Message Rate Limit +# This limits sasl authenticated users to no more than 50/60mins +# NOTE: sasl_username needs to be set to something or this check will fail +id=MSGRATE ; sasl_username=!!(^$); action==rate($$sasl_username/100/3600/450 4.7.1 exceeded message rate. Contact Contact http://<%= @domain %>/tickets/new if this is in error. ERROR:MSGRATE) + +# Total Recipient Rate Limit +# This adds up the recipients for all the sasl authenticated users messages +# and can't exceed more than 250/60min +# NOTE: sasl_username needs to be set to something or this check will fail +id=RCPTRATE ; sasl_username=!!(^$); action==rcpt($$sasl_username/500/3600/450 4.7.1 exceeded message rate. Contact http://<%= @domain %>/tickets/new if this is in error. ERROR:RCPTRATE) + +# Size per client Limit +id=SENDSIZE ; state==END_OF_DATA ; client_address==!!(10.0.1.0/24); action==size($$client_address/314572800/3600/450 4.7.1 Sorry you have sent too much data. Contact http://<%= @domain %>/tickets/new if this is in error. ERROR:SENDSIZE) diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp index edaa506f..71d61621 100644 --- a/puppet/modules/site_postfix/manifests/mx.pp +++ b/puppet/modules/site_postfix/manifests/mx.pp @@ -62,6 +62,7 @@ class site_postfix::mx { include site_postfix::mx::static_aliases include site_postfix::mx::rewrite_openpgp_header include clamav + include postfwd # greater verbosity for debugging, take out for production #include site_postfix::debug -- cgit v1.2.3 From ea5b55fb9a4f831c586ba773205d3238e5213260 Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 2 Nov 2015 18:45:13 -0500 Subject: fix postfwd dependency requirement Change-Id: Ied475dd1d555a2388034012f5a799a202dcc6ee7 --- puppet/modules/postfwd/manifests/init.pp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/puppet/modules/postfwd/manifests/init.pp b/puppet/modules/postfwd/manifests/init.pp index b00bb071..1ebc1d53 100644 --- a/puppet/modules/postfwd/manifests/init.pp +++ b/puppet/modules/postfwd/manifests/init.pp @@ -14,7 +14,7 @@ class postfwd { file { '/etc/default/postfwd': - source => 'puppet:///modules/postfwd/postfwd', + source => 'puppet:///modules/postfwd/postfwd_default', mode => '0644', owner => root, group => root, @@ -25,14 +25,7 @@ class postfwd { mode => '0644', owner => root, group => root, - require => File['/etc/postfix']; - } - - exec { - '/etc/init.d/postfwd reload': - refreshonly => true, - subscribe => [ File['/etc/postfix/postfwd.cf'], - File['/etc/default/postfwd'] ]; + require => Package['postfix']; } service { -- cgit v1.2.3 From 7d0b6b25e49a1ccb70c4f502f7dfc58878b900cc Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 2 Nov 2015 21:02:03 -0500 Subject: remove unused postfwd rule Change-Id: I8756c5c3212a3d7e3c44414fdf6bfff5cd29d70f --- puppet/modules/postfwd/templates/postfwd.cf.erb | 3 --- 1 file changed, 3 deletions(-) diff --git a/puppet/modules/postfwd/templates/postfwd.cf.erb b/puppet/modules/postfwd/templates/postfwd.cf.erb index 6460994a..1c45dd03 100644 --- a/puppet/modules/postfwd/templates/postfwd.cf.erb +++ b/puppet/modules/postfwd/templates/postfwd.cf.erb @@ -26,6 +26,3 @@ id=MSGRATE ; sasl_username=!!(^$); action==rate($$sasl_username/100/3600/450 4.7 # and can't exceed more than 250/60min # NOTE: sasl_username needs to be set to something or this check will fail id=RCPTRATE ; sasl_username=!!(^$); action==rcpt($$sasl_username/500/3600/450 4.7.1 exceeded message rate. Contact http://<%= @domain %>/tickets/new if this is in error. ERROR:RCPTRATE) - -# Size per client Limit -id=SENDSIZE ; state==END_OF_DATA ; client_address==!!(10.0.1.0/24); action==size($$client_address/314572800/3600/450 4.7.1 Sorry you have sent too much data. Contact http://<%= @domain %>/tickets/new if this is in error. ERROR:SENDSIZE) -- cgit v1.2.3