From 2a3b4ec1bc522409d4dc8d2e7750344de41acb50 Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 17 Jun 2014 14:48:50 -0700 Subject: allow webapp.json to configure what engines are enabled --- provider_base/services/webapp.json | 5 ++++- puppet/modules/site_webapp/manifests/init.pp | 4 ++-- puppet/modules/site_webapp/templates/config.yml.erb | 6 ++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index bbb52094..a5b1ed30 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -26,7 +26,10 @@ "nagios_test_user": { "username": "nagios_test", "password": "= secret :nagios_test_password" - } + }, + "engines": [ + "support" + ] }, "stunnel": { "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" diff --git a/puppet/modules/site_webapp/manifests/init.pp b/puppet/modules/site_webapp/manifests/init.pp index d6f1d7ae..08618457 100644 --- a/puppet/modules/site_webapp/manifests/init.pp +++ b/puppet/modules/site_webapp/manifests/init.pp @@ -52,8 +52,8 @@ class site_webapp { exec { 'bundler_update': cwd => '/srv/leap/webapp', - command => '/bin/bash -c "/usr/bin/bundle check || /usr/bin/bundle install --path vendor/bundle --without test development"', - unless => '/usr/bin/bundle check', + command => '/bin/bash -c "/usr/bin/bundle check --path vendor/bundle || /usr/bin/bundle install --path vendor/bundle --without test development"', + unless => '/usr/bin/bundle check --path vendor/bundle', user => 'leap-webapp', timeout => 600, require => [ diff --git a/puppet/modules/site_webapp/templates/config.yml.erb b/puppet/modules/site_webapp/templates/config.yml.erb index 6461c5e8..8faf76f4 100644 --- a/puppet/modules/site_webapp/templates/config.yml.erb +++ b/puppet/modules/site_webapp/templates/config.yml.erb @@ -18,3 +18,9 @@ production: minimum_client_version: "<%= @webapp['client_version']['min'] %>" default_service_level: "<%= @webapp['default_service_level'] %>" service_levels: <%= @webapp['service_levels'].to_json %> +<%- if @webapp['engines'] && @webapp['engines'].any? -%> + engines: +<%- @webapp['engines'].each do |engine| -%> + - <%= engine %> +<%- end -%> +<%- end -%> \ No newline at end of file -- cgit v1.2.3 From 1b5f0892bbcb07fa075bfe8c7b083521b38cb73c Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 19 Jun 2014 00:02:54 -0700 Subject: couchdb: generate hiera files suitable for plain couchdb + read-only mirrors --- platform.rb | 2 +- provider_base/services/_couchdb_master.json | 8 ++++++++ provider_base/services/_couchdb_mirror.json | 16 ++++++++++++++++ provider_base/services/_couchdb_multimaster.json | 20 ++++++++++++++++++++ provider_base/services/couchdb.json | 17 ++++------------- provider_base/services/couchdb.rb | 18 ++++++++++++++++++ 6 files changed, 67 insertions(+), 14 deletions(-) create mode 100644 provider_base/services/_couchdb_master.json create mode 100644 provider_base/services/_couchdb_mirror.json create mode 100644 provider_base/services/_couchdb_multimaster.json create mode 100644 provider_base/services/couchdb.rb diff --git a/platform.rb b/platform.rb index cd0cbde0..ac79df65 100644 --- a/platform.rb +++ b/platform.rb @@ -5,7 +5,7 @@ Leap::Platform.define do self.version = "0.5.2" - self.compatible_cli = "1.5.5".."1.99" + self.compatible_cli = "1.5.7".."1.99" # # the facter facts that should be gathered diff --git a/provider_base/services/_couchdb_master.json b/provider_base/services/_couchdb_master.json new file mode 100644 index 00000000..20c6f99b --- /dev/null +++ b/provider_base/services/_couchdb_master.json @@ -0,0 +1,8 @@ +// +// Applied to master couchdb node when there is a single master +// +{ + "couch": { + "mode": "master" + } +} \ No newline at end of file diff --git a/provider_base/services/_couchdb_mirror.json b/provider_base/services/_couchdb_mirror.json new file mode 100644 index 00000000..67004c70 --- /dev/null +++ b/provider_base/services/_couchdb_mirror.json @@ -0,0 +1,16 @@ +// +// Applied to all non-master couchdb nodes +// +{ + "stunnel": { + "couch_client": "= stunnel_client(nodes[couch.replication.masters.keys], couch.port)" + }, + "couch": { + "mode": "mirror", + "replication": { + // for now, pick the first close one, or the first one. + // in the future, maybe use haproxy to balance among all the masters + "masters": "= try{pick_node(:couch_master,nodes_near_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal')} || try{pick_node(:couch_master,nodes_like_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal')}" + } + } +} \ No newline at end of file diff --git a/provider_base/services/_couchdb_multimaster.json b/provider_base/services/_couchdb_multimaster.json new file mode 100644 index 00000000..3eb4835c --- /dev/null +++ b/provider_base/services/_couchdb_multimaster.json @@ -0,0 +1,20 @@ +// +// Only applied to master couchdb nodes when there are multiple masters +// +{ + "stunnel": { + "epmd_server": "= stunnel_server(couch.bigcouch.epmd_port)", + "epmd_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], couch.bigcouch.epmd_port)", + "ednp_server": "= stunnel_server(couch.bigcouch.ednp_port)", + "ednp_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], couch.bigcouch.ednp_port)" + }, + "couch": { + "mode": "multimaster", + "bigcouch": { + "epmd_port": 4369, + "ednp_port": 9002, + "cookie": "= secret :bigcouch_cookie", + "neighbors": "= nodes_like_me['services' => 'couchdb']['couchdb.master' => true].exclude(self).field('domain.full')" + } + } +} diff --git a/provider_base/services/couchdb.json b/provider_base/services/couchdb.json index 5f1b5381..d75fd8de 100644 --- a/provider_base/services/couchdb.json +++ b/provider_base/services/couchdb.json @@ -3,20 +3,11 @@ "use": true }, "stunnel": { - "couch_server": "= stunnel_server(couch.port)", - "epmd_server": "= stunnel_server(couch.bigcouch.epmd_port)", - "epmd_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.bigcouch.epmd_port)", - "ednp_server": "= stunnel_server(couch.bigcouch.ednp_port)", - "ednp_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.bigcouch.ednp_port)" + "couch_server": "= stunnel_server(couch.port)" }, "couch": { + "master": false, "port": 5984, - "bigcouch": { - "epmd_port": 4369, - "ednp_port": 9002, - "cookie": "= secret :bigcouch_cookie", - "neighbors": "= nodes_like_me[:services => :couchdb].exclude(self).field('domain.full')" - }, "users": { "admin": { "username": "admin", @@ -49,8 +40,8 @@ "salt": "= hex_secret :couch_webapp_password_salt, 128" } }, - "webapp": { - "nagios_test_pw": "= secret :nagios_test_password" + "webapp": { + "nagios_test_pw": "= secret :nagios_test_password" } } } diff --git a/provider_base/services/couchdb.rb b/provider_base/services/couchdb.rb new file mode 100644 index 00000000..c8f5d8a7 --- /dev/null +++ b/provider_base/services/couchdb.rb @@ -0,0 +1,18 @@ +# +# custom logic for couchdb json resolution +# + +unless nodes_like_me['services' => 'couchdb']['couch.master' => true].any? + raise 'node `%s`, environment `%s`: there must be at least one node with couch.master set to `true` for this environment.' % [@node.name, @node.environment] +end + +if couch.master + if nodes_like_me['services' => 'couchdb']['couch.master' => true].size > 1 + apply_partial 'services/_couchdb_multimaster.json' + else + apply_partial 'services/_couchdb_master.json' + end +else + apply_partial 'services/_couchdb_mirror.json' +end + -- cgit v1.2.3 From 90a97f5333e82e302f487b060076a8c0ceaf6259 Mon Sep 17 00:00:00 2001 From: Azul Date: Thu, 19 Jun 2014 12:11:14 +0200 Subject: split bigcouch stunnel from plain couch stunnel --- .../site_couchdb/manifests/bigcouch/stunnel.pp | 89 ++++++++++++++++++++++ puppet/modules/site_couchdb/manifests/stunnel.pp | 81 ++------------------ 2 files changed, 95 insertions(+), 75 deletions(-) create mode 100644 puppet/modules/site_couchdb/manifests/bigcouch/stunnel.pp diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/stunnel.pp b/puppet/modules/site_couchdb/manifests/bigcouch/stunnel.pp new file mode 100644 index 00000000..5166ba93 --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/bigcouch/stunnel.pp @@ -0,0 +1,89 @@ +class site_couchdb::bigcouch::stunnel { + + $stunnel = hiera('stunnel') + + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca + + include x509::variables + $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" + $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" + $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" + + + # Erlang Port Mapper Daemon (epmd) stunnel server/clients + $epmd_server = $stunnel['epmd_server'] + $epmd_server_accept = $epmd_server['accept'] + $epmd_server_connect = $epmd_server['connect'] + $epmd_clients = $stunnel['epmd_clients'] + + # Erlang Distributed Node Protocol (ednp) stunnel server/clients + $ednp_server = $stunnel['ednp_server'] + $ednp_server_accept = $ednp_server['accept'] + $ednp_server_connect = $ednp_server['connect'] + $ednp_clients = $stunnel['ednp_clients'] + + + # setup stunnel server for Erlang Port Mapper Daemon (epmd), necessary for + # bigcouch clustering between each bigcouchdb node + stunnel::service { 'epmd_server': + accept => $epmd_server_accept, + connect => $epmd_server_connect, + client => false, + cafile => $ca_path, + key => $key_path, + cert => $cert_path, + verify => '2', + pid => '/var/run/stunnel4/epmd_server.pid', + rndfile => '/var/lib/stunnel4/.rnd', + debuglevel => '4', + require => [ + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Ca'] ]; + } + + # setup stunnel clients for Erlang Port Mapper Daemon (epmd) to connect + # to the above epmd stunnel server. + $epmd_client_defaults = { + 'client' => true, + 'cafile' => $ca_path, + 'key' => $key_path, + 'cert' => $cert_path, + } + + create_resources(site_stunnel::clients, $epmd_clients, $epmd_client_defaults) + + # setup stunnel server for Erlang Distributed Node Protocol (ednp), necessary + # for bigcouch clustering between each bigcouchdb node + stunnel::service { 'ednp_server': + accept => $ednp_server_accept, + connect => $ednp_server_connect, + client => false, + cafile => $ca_path, + key => $key_path, + cert => $cert_path, + verify => '2', + pid => '/var/run/stunnel4/ednp_server.pid', + rndfile => '/var/lib/stunnel4/.rnd', + debuglevel => '4', + require => [ + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Ca'] ]; + } + + # setup stunnel clients for Erlang Distributed Node Protocol (ednp) to connect + # to the above ednp stunnel server. + $ednp_client_defaults = { + 'client' => true, + 'cafile' => $ca_path, + 'key' => $key_path, + 'cert' => $cert_path, + } + + create_resources(site_stunnel::clients, $ednp_clients, $ednp_client_defaults) + + include site_check_mk::agent::stunnel +} diff --git a/puppet/modules/site_couchdb/manifests/stunnel.pp b/puppet/modules/site_couchdb/manifests/stunnel.pp index 91f1e3aa..484a0c00 100644 --- a/puppet/modules/site_couchdb/manifests/stunnel.pp +++ b/puppet/modules/site_couchdb/manifests/stunnel.pp @@ -1,29 +1,21 @@ class site_couchdb::stunnel { $stunnel = hiera('stunnel') + $couchdb_config = hiera('couch') + $couchdb_bigcouch = $couchdb_config['mode'] == "multimaster" $couch_server = $stunnel['couch_server'] $couch_server_accept = $couch_server['accept'] $couch_server_connect = $couch_server['connect'] - # Erlang Port Mapper Daemon (epmd) stunnel server/clients - $epmd_server = $stunnel['epmd_server'] - $epmd_server_accept = $epmd_server['accept'] - $epmd_server_connect = $epmd_server['connect'] - $epmd_clients = $stunnel['epmd_clients'] - - # Erlang Distributed Node Protocol (ednp) stunnel server/clients - $ednp_server = $stunnel['ednp_server'] - $ednp_server_accept = $ednp_server['accept'] - $ednp_server_connect = $ednp_server['connect'] - $ednp_clients = $stunnel['ednp_clients'] - - - include site_config::x509::cert include site_config::x509::key include site_config::x509::ca + if $couchdb_bigcouch { + include site_couchdb::bigcouch::stunnel + } + include x509::variables $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" @@ -47,66 +39,5 @@ class site_couchdb::stunnel { Class['Site_config::X509::Ca'] ]; } - - # setup stunnel server for Erlang Port Mapper Daemon (epmd), necessary for - # bigcouch clustering between each bigcouchdb node - stunnel::service { 'epmd_server': - accept => $epmd_server_accept, - connect => $epmd_server_connect, - client => false, - cafile => $ca_path, - key => $key_path, - cert => $cert_path, - verify => '2', - pid => '/var/run/stunnel4/epmd_server.pid', - rndfile => '/var/lib/stunnel4/.rnd', - debuglevel => '4', - require => [ - Class['Site_config::X509::Key'], - Class['Site_config::X509::Cert'], - Class['Site_config::X509::Ca'] ]; - } - - # setup stunnel clients for Erlang Port Mapper Daemon (epmd) to connect - # to the above epmd stunnel server. - $epmd_client_defaults = { - 'client' => true, - 'cafile' => $ca_path, - 'key' => $key_path, - 'cert' => $cert_path, - } - - create_resources(site_stunnel::clients, $epmd_clients, $epmd_client_defaults) - - # setup stunnel server for Erlang Distributed Node Protocol (ednp), necessary - # for bigcouch clustering between each bigcouchdb node - stunnel::service { 'ednp_server': - accept => $ednp_server_accept, - connect => $ednp_server_connect, - client => false, - cafile => $ca_path, - key => $key_path, - cert => $cert_path, - verify => '2', - pid => '/var/run/stunnel4/ednp_server.pid', - rndfile => '/var/lib/stunnel4/.rnd', - debuglevel => '4', - require => [ - Class['Site_config::X509::Key'], - Class['Site_config::X509::Cert'], - Class['Site_config::X509::Ca'] ]; - } - - # setup stunnel clients for Erlang Distributed Node Protocol (ednp) to connect - # to the above ednp stunnel server. - $ednp_client_defaults = { - 'client' => true, - 'cafile' => $ca_path, - 'key' => $key_path, - 'cert' => $cert_path, - } - - create_resources(site_stunnel::clients, $ednp_clients, $ednp_client_defaults) - include site_check_mk::agent::stunnel } -- cgit v1.2.3 From b286f81dd29404b64494dab3fe1e7a2bec1c9c46 Mon Sep 17 00:00:00 2001 From: Azul Date: Thu, 19 Jun 2014 12:29:30 +0200 Subject: separate bigcouch specifics from init.pp --- puppet/modules/site_couchdb/manifests/bigcouch.pp | 20 ++++++++++++++++++++ puppet/modules/site_couchdb/manifests/init.pp | 21 ++++----------------- 2 files changed, 24 insertions(+), 17 deletions(-) create mode 100644 puppet/modules/site_couchdb/manifests/bigcouch.pp diff --git a/puppet/modules/site_couchdb/manifests/bigcouch.pp b/puppet/modules/site_couchdb/manifests/bigcouch.pp new file mode 100644 index 00000000..a3f6db2c --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/bigcouch.pp @@ -0,0 +1,20 @@ +class site_couchdb::bigcouch { + + $bigcouch_config = $couchdb_config['bigcouch'] + $bigcouch_cookie = $bigcouch_config['cookie'] + + $ednp_port = $bigcouch_config['ednp_port'] + + Class['site_config::default'] + -> Class['site_couchdb::bigcouch::add_nodes'] + -> Class['site_couchdb::bigcouch::settle_cluster'] + + include site_couchdb::bigcouch::add_nodes + include site_couchdb::bigcouch::settle_cluster + include site_couchdb::bigcouch::compaction + include site_shorewall::couchdb::bigcouch + + file { '/var/log/bigcouch': + ensure => directory + } +} diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 3614661d..22d6ef45 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -35,14 +35,10 @@ class site_couchdb { $couchdb_webapp_salt = $couchdb_webapp['salt'] $couchdb_backup = $couchdb_config['backup'] - - $bigcouch_config = $couchdb_config['bigcouch'] - $bigcouch_cookie = $bigcouch_config['cookie'] - - $ednp_port = $bigcouch_config['ednp_port'] + $couchdb_bigcouch = $couchdb_config['mode'] == "multimaster" class { 'couchdb': - bigcouch => true, + bigcouch => $couchdb_bigcouch, admin_pw => $couchdb_admin_pw, admin_salt => $couchdb_admin_salt, bigcouch_cookie => $bigcouch_cookie, @@ -63,8 +59,6 @@ class site_couchdb { -> Class['site_couchdb::stunnel'] -> Service['couchdb'] -> File['/root/.netrc'] - -> Class['site_couchdb::bigcouch::add_nodes'] - -> Class['site_couchdb::bigcouch::settle_cluster'] -> Class['site_couchdb::create_dbs'] -> Class['site_couchdb::add_users'] @@ -95,24 +89,17 @@ class site_couchdb { } include site_couchdb::stunnel - include site_couchdb::bigcouch::add_nodes - include site_couchdb::bigcouch::settle_cluster include site_couchdb::create_dbs include site_couchdb::add_users include site_couchdb::designs include site_couchdb::logrotate - include site_couchdb::bigcouch::compaction - if $couchdb_backup { include site_couchdb::backup } + if $couchdb_bigcouch { include site_couchdb::bigcouch } + if $couchdb_backup { include site_couchdb::backup } include site_shorewall::couchdb - include site_shorewall::couchdb::bigcouch include site_check_mk::agent::couchdb include site_check_mk::agent::tapicero - file { '/var/log/bigcouch': - ensure => directory - } - } -- cgit v1.2.3 From ae3ad84bdf646ddb3c8da9258201307ede65ea41 Mon Sep 17 00:00:00 2001 From: Azul Date: Thu, 19 Jun 2014 20:01:20 +0200 Subject: set mirror option if we are on a couch mirror --- puppet/modules/tapicero/manifests/init.pp | 1 + puppet/modules/tapicero/templates/tapicero.yaml.erb | 2 ++ 2 files changed, 3 insertions(+) diff --git a/puppet/modules/tapicero/manifests/init.pp b/puppet/modules/tapicero/manifests/init.pp index af1a96ac..1db75eb0 100644 --- a/puppet/modules/tapicero/manifests/init.pp +++ b/puppet/modules/tapicero/manifests/init.pp @@ -12,6 +12,7 @@ class tapicero { $couchdb_soledad_user = $couchdb_users['soledad']['username'] $couchdb_leap_mx_user = $couchdb_users['leap_mx']['username'] + $couchdb_mirror = $couchdb['mode'] == 'mirror' Class['site_config::default'] -> Class['tapicero'] diff --git a/puppet/modules/tapicero/templates/tapicero.yaml.erb b/puppet/modules/tapicero/templates/tapicero.yaml.erb index 8e19b22f..3a5f821e 100644 --- a/puppet/modules/tapicero/templates/tapicero.yaml.erb +++ b/puppet/modules/tapicero/templates/tapicero.yaml.erb @@ -24,6 +24,7 @@ log_level: info options: # prefix for per user databases: db_prefix: "user-" + mirror: <%= @couchdb_mirror %> # security settings to be used for the per user databases security: @@ -40,3 +41,4 @@ options: - <%= @couchdb_leap_mx_user %> roles: [] + -- cgit v1.2.3 From 9034a2eb1fdec68d46aa0d1ea2720409a7312f35 Mon Sep 17 00:00:00 2001 From: Azul Date: Thu, 19 Jun 2014 20:02:02 +0200 Subject: first steps towards mirroring couch --- puppet/modules/site_couchdb/manifests/bigcouch.pp | 6 +-- .../site_couchdb/manifests/bigcouch/add_nodes.pp | 2 +- puppet/modules/site_couchdb/manifests/init.pp | 6 ++- puppet/modules/site_couchdb/manifests/mirror.pp | 61 ++++++++++++++++++++++ 4 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 puppet/modules/site_couchdb/manifests/mirror.pp diff --git a/puppet/modules/site_couchdb/manifests/bigcouch.pp b/puppet/modules/site_couchdb/manifests/bigcouch.pp index a3f6db2c..97c8cd12 100644 --- a/puppet/modules/site_couchdb/manifests/bigcouch.pp +++ b/puppet/modules/site_couchdb/manifests/bigcouch.pp @@ -1,9 +1,9 @@ class site_couchdb::bigcouch { - $bigcouch_config = $couchdb_config['bigcouch'] - $bigcouch_cookie = $bigcouch_config['cookie'] + $config = $::site_couchdb::couchdb_config['bigcouch'] + $cookie = $config['cookie'] - $ednp_port = $bigcouch_config['ednp_port'] + $ednp_port = $config['ednp_port'] Class['site_config::default'] -> Class['site_couchdb::bigcouch::add_nodes'] diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp b/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp index 97e85785..c8c43275 100644 --- a/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp +++ b/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp @@ -1,6 +1,6 @@ class site_couchdb::bigcouch::add_nodes { # loop through neighbors array and add nodes - $nodes = $::site_couchdb::bigcouch_config['neighbors'] + $nodes = $::site_couchdb::bigcouch::config['neighbors'] couchdb::bigcouch::add_node { $nodes: require => Couchdb::Query::Setup['localhost'] diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 22d6ef45..0b923c9f 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -35,7 +35,7 @@ class site_couchdb { $couchdb_webapp_salt = $couchdb_webapp['salt'] $couchdb_backup = $couchdb_config['backup'] - $couchdb_bigcouch = $couchdb_config['mode'] == "multimaster" + $couchdb_mode = $couchdb_config['mode'] class { 'couchdb': bigcouch => $couchdb_bigcouch, @@ -94,7 +94,9 @@ class site_couchdb { include site_couchdb::designs include site_couchdb::logrotate - if $couchdb_bigcouch { include site_couchdb::bigcouch } + if $couchdb_mode == "multimaster" { include site_couchdb::bigcouch } + if $couchdb_mode == "mirror" { include site_couchdb::mirror } + if $couchdb_backup { include site_couchdb::backup } include site_shorewall::couchdb diff --git a/puppet/modules/site_couchdb/manifests/mirror.pp b/puppet/modules/site_couchdb/manifests/mirror.pp new file mode 100644 index 00000000..708171e4 --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/mirror.pp @@ -0,0 +1,61 @@ +class site_couchdb::mirror { + + # Couchdb databases + + $from = $site_couchdb::couchdb_config['replication']['masters'][0] + + ### customer database + couchdb::mirror_db { 'customers': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## identities database + couchdb::mirror_db { 'identities': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## keycache database + couchdb::mirror_db { 'keycache': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## sessions database + couchdb::mirror_db { 'sessions': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## shared database + couchdb::mirror_db { 'shared': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## tickets database + couchdb::mirror_db { 'tickets': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## tokens database + couchdb::mirror_db { 'tokens': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## users database + couchdb::mirror_db { 'users': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## messages db + couchdb::mirror_db { 'messages': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + +} -- cgit v1.2.3 From a7380ee34769e8142f4fb9e58825af5fd1342108 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 19 Jun 2014 14:30:22 -0700 Subject: fix typo in _couchdb_multimaster.json --- provider_base/services/_couchdb_multimaster.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider_base/services/_couchdb_multimaster.json b/provider_base/services/_couchdb_multimaster.json index 3eb4835c..ff133b9c 100644 --- a/provider_base/services/_couchdb_multimaster.json +++ b/provider_base/services/_couchdb_multimaster.json @@ -14,7 +14,7 @@ "epmd_port": 4369, "ednp_port": 9002, "cookie": "= secret :bigcouch_cookie", - "neighbors": "= nodes_like_me['services' => 'couchdb']['couchdb.master' => true].exclude(self).field('domain.full')" + "neighbors": "= nodes_like_me['services' => 'couchdb']['couch.master' => true].exclude(self).field('domain.full')" } } } -- cgit v1.2.3 From 80809853298f16ce7f27c5202f81b516cfa11d56 Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 20 Jun 2014 01:58:39 -0700 Subject: new generic system for stunnel: just `include site_stunnel` and stunnel + needed shorewall will be automatically set up. requires new leap_cli --- platform.rb | 4 +- provider_base/common.json | 4 + provider_base/services/_couchdb_mirror.json | 6 +- provider_base/services/_couchdb_multimaster.json | 12 ++- provider_base/services/couchdb.json | 4 +- .../site_couchdb/manifests/bigcouch/stunnel.pp | 89 ---------------------- puppet/modules/site_couchdb/manifests/stunnel.pp | 43 ----------- puppet/modules/site_shorewall/manifests/couchdb.pp | 24 ------ .../site_shorewall/manifests/couchdb/bigcouch.pp | 51 ------------- .../site_shorewall/manifests/couchdb/dnat.pp | 21 ----- .../site_shorewall/manifests/stunnel/client.pp | 40 ++++++++++ .../site_shorewall/manifests/stunnel/server.pp | 22 ++++++ puppet/modules/site_stunnel/manifests/client.pp | 52 +++++++++++++ puppet/modules/site_stunnel/manifests/clients.pp | 55 ++++++------- puppet/modules/site_stunnel/manifests/init.pp | 15 ++++ puppet/modules/site_stunnel/manifests/servers.pp | 53 +++++++++++++ 16 files changed, 227 insertions(+), 268 deletions(-) delete mode 100644 puppet/modules/site_couchdb/manifests/bigcouch/stunnel.pp delete mode 100644 puppet/modules/site_couchdb/manifests/stunnel.pp delete mode 100644 puppet/modules/site_shorewall/manifests/couchdb.pp delete mode 100644 puppet/modules/site_shorewall/manifests/couchdb/bigcouch.pp delete mode 100644 puppet/modules/site_shorewall/manifests/couchdb/dnat.pp create mode 100644 puppet/modules/site_shorewall/manifests/stunnel/client.pp create mode 100644 puppet/modules/site_shorewall/manifests/stunnel/server.pp create mode 100644 puppet/modules/site_stunnel/manifests/client.pp create mode 100644 puppet/modules/site_stunnel/manifests/servers.pp diff --git a/platform.rb b/platform.rb index ac79df65..872a34cb 100644 --- a/platform.rb +++ b/platform.rb @@ -4,8 +4,8 @@ # Leap::Platform.define do - self.version = "0.5.2" - self.compatible_cli = "1.5.7".."1.99" + self.version = "0.5.3" + self.compatible_cli = "1.5.8".."1.99" # # the facter facts that should be gathered diff --git a/provider_base/common.json b/provider_base/common.json index a4d9c5f2..265d2ce4 100644 --- a/provider_base/common.json +++ b/provider_base/common.json @@ -38,5 +38,9 @@ "enabled": true, "mail": { "smarthost": "= nodes_like_me[:services => :mx].exclude(self).field('domain.full')" + }, + "stunnel": { + "clients": {}, + "servers": {} } } diff --git a/provider_base/services/_couchdb_mirror.json b/provider_base/services/_couchdb_mirror.json index 67004c70..a496804d 100644 --- a/provider_base/services/_couchdb_mirror.json +++ b/provider_base/services/_couchdb_mirror.json @@ -3,14 +3,16 @@ // { "stunnel": { - "couch_client": "= stunnel_client(nodes[couch.replication.masters.keys], couch.port)" + "clients": { + "couch_client": "= stunnel_client(nodes[couch.replication.masters.keys], couch.port)" + } }, "couch": { "mode": "mirror", "replication": { // for now, pick the first close one, or the first one. // in the future, maybe use haproxy to balance among all the masters - "masters": "= try{pick_node(:couch_master,nodes_near_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal')} || try{pick_node(:couch_master,nodes_like_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal')}" + "masters": "= try{pick_node(:couch_master,nodes_near_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')} || try{pick_node(:couch_master,nodes_like_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')}" } } } \ No newline at end of file diff --git a/provider_base/services/_couchdb_multimaster.json b/provider_base/services/_couchdb_multimaster.json index ff133b9c..8c433188 100644 --- a/provider_base/services/_couchdb_multimaster.json +++ b/provider_base/services/_couchdb_multimaster.json @@ -3,10 +3,14 @@ // { "stunnel": { - "epmd_server": "= stunnel_server(couch.bigcouch.epmd_port)", - "epmd_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], couch.bigcouch.epmd_port)", - "ednp_server": "= stunnel_server(couch.bigcouch.ednp_port)", - "ednp_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], couch.bigcouch.ednp_port)" + "servers": { + "epmd_server": "= stunnel_server(couch.bigcouch.epmd_port)", + "ednp_server": "= stunnel_server(couch.bigcouch.ednp_port)" + }, + "clients": { + "epmd_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], couch.bigcouch.epmd_port)", + "ednp_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], couch.bigcouch.ednp_port)" + } }, "couch": { "mode": "multimaster", diff --git a/provider_base/services/couchdb.json b/provider_base/services/couchdb.json index d75fd8de..c2482235 100644 --- a/provider_base/services/couchdb.json +++ b/provider_base/services/couchdb.json @@ -3,7 +3,9 @@ "use": true }, "stunnel": { - "couch_server": "= stunnel_server(couch.port)" + "servers": { + "couch_server": "= stunnel_server(couch.port)" + } }, "couch": { "master": false, diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/stunnel.pp b/puppet/modules/site_couchdb/manifests/bigcouch/stunnel.pp deleted file mode 100644 index 5166ba93..00000000 --- a/puppet/modules/site_couchdb/manifests/bigcouch/stunnel.pp +++ /dev/null @@ -1,89 +0,0 @@ -class site_couchdb::bigcouch::stunnel { - - $stunnel = hiera('stunnel') - - include site_config::x509::cert - include site_config::x509::key - include site_config::x509::ca - - include x509::variables - $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" - $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" - $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" - - - # Erlang Port Mapper Daemon (epmd) stunnel server/clients - $epmd_server = $stunnel['epmd_server'] - $epmd_server_accept = $epmd_server['accept'] - $epmd_server_connect = $epmd_server['connect'] - $epmd_clients = $stunnel['epmd_clients'] - - # Erlang Distributed Node Protocol (ednp) stunnel server/clients - $ednp_server = $stunnel['ednp_server'] - $ednp_server_accept = $ednp_server['accept'] - $ednp_server_connect = $ednp_server['connect'] - $ednp_clients = $stunnel['ednp_clients'] - - - # setup stunnel server for Erlang Port Mapper Daemon (epmd), necessary for - # bigcouch clustering between each bigcouchdb node - stunnel::service { 'epmd_server': - accept => $epmd_server_accept, - connect => $epmd_server_connect, - client => false, - cafile => $ca_path, - key => $key_path, - cert => $cert_path, - verify => '2', - pid => '/var/run/stunnel4/epmd_server.pid', - rndfile => '/var/lib/stunnel4/.rnd', - debuglevel => '4', - require => [ - Class['Site_config::X509::Key'], - Class['Site_config::X509::Cert'], - Class['Site_config::X509::Ca'] ]; - } - - # setup stunnel clients for Erlang Port Mapper Daemon (epmd) to connect - # to the above epmd stunnel server. - $epmd_client_defaults = { - 'client' => true, - 'cafile' => $ca_path, - 'key' => $key_path, - 'cert' => $cert_path, - } - - create_resources(site_stunnel::clients, $epmd_clients, $epmd_client_defaults) - - # setup stunnel server for Erlang Distributed Node Protocol (ednp), necessary - # for bigcouch clustering between each bigcouchdb node - stunnel::service { 'ednp_server': - accept => $ednp_server_accept, - connect => $ednp_server_connect, - client => false, - cafile => $ca_path, - key => $key_path, - cert => $cert_path, - verify => '2', - pid => '/var/run/stunnel4/ednp_server.pid', - rndfile => '/var/lib/stunnel4/.rnd', - debuglevel => '4', - require => [ - Class['Site_config::X509::Key'], - Class['Site_config::X509::Cert'], - Class['Site_config::X509::Ca'] ]; - } - - # setup stunnel clients for Erlang Distributed Node Protocol (ednp) to connect - # to the above ednp stunnel server. - $ednp_client_defaults = { - 'client' => true, - 'cafile' => $ca_path, - 'key' => $key_path, - 'cert' => $cert_path, - } - - create_resources(site_stunnel::clients, $ednp_clients, $ednp_client_defaults) - - include site_check_mk::agent::stunnel -} diff --git a/puppet/modules/site_couchdb/manifests/stunnel.pp b/puppet/modules/site_couchdb/manifests/stunnel.pp deleted file mode 100644 index 484a0c00..00000000 --- a/puppet/modules/site_couchdb/manifests/stunnel.pp +++ /dev/null @@ -1,43 +0,0 @@ -class site_couchdb::stunnel { - - $stunnel = hiera('stunnel') - $couchdb_config = hiera('couch') - $couchdb_bigcouch = $couchdb_config['mode'] == "multimaster" - - $couch_server = $stunnel['couch_server'] - $couch_server_accept = $couch_server['accept'] - $couch_server_connect = $couch_server['connect'] - - include site_config::x509::cert - include site_config::x509::key - include site_config::x509::ca - - if $couchdb_bigcouch { - include site_couchdb::bigcouch::stunnel - } - - include x509::variables - $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" - $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" - $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" - - # setup a stunnel server for the webapp to connect to couchdb - stunnel::service { 'couch_server': - accept => $couch_server_accept, - connect => $couch_server_connect, - client => false, - cafile => $ca_path, - key => $key_path, - cert => $cert_path, - verify => '2', - pid => '/var/run/stunnel4/couchserver.pid', - rndfile => '/var/lib/stunnel4/.rnd', - debuglevel => '4', - require => [ - Class['Site_config::X509::Key'], - Class['Site_config::X509::Cert'], - Class['Site_config::X509::Ca'] ]; - } - - include site_check_mk::agent::stunnel -} diff --git a/puppet/modules/site_shorewall/manifests/couchdb.pp b/puppet/modules/site_shorewall/manifests/couchdb.pp deleted file mode 100644 index 73bed62b..00000000 --- a/puppet/modules/site_shorewall/manifests/couchdb.pp +++ /dev/null @@ -1,24 +0,0 @@ -class site_shorewall::couchdb { - - include site_shorewall::defaults - - $stunnel = hiera('stunnel') - $couch_server = $stunnel['couch_server'] - $couch_stunnel_port = $couch_server['accept'] - - # define macro for incoming services - file { '/etc/shorewall/macro.leap_couchdb': - content => "PARAM - - tcp ${couch_stunnel_port}", - notify => Service['shorewall'], - require => Package['shorewall'] - } - - shorewall::rule { - 'net2fw-couchdb': - source => 'net', - destination => '$FW', - action => 'leap_couchdb(ACCEPT)', - order => 200; - } - -} diff --git a/puppet/modules/site_shorewall/manifests/couchdb/bigcouch.pp b/puppet/modules/site_shorewall/manifests/couchdb/bigcouch.pp deleted file mode 100644 index 20740650..00000000 --- a/puppet/modules/site_shorewall/manifests/couchdb/bigcouch.pp +++ /dev/null @@ -1,51 +0,0 @@ -class site_shorewall::couchdb::bigcouch { - - include site_shorewall::defaults - - $stunnel = hiera('stunnel') - - # Erlang Port Mapper Daemon (epmd) stunnel server/clients - $epmd_clients = $stunnel['epmd_clients'] - $epmd_server = $stunnel['epmd_server'] - $epmd_server_port = $epmd_server['accept'] - $epmd_server_connect = $epmd_server['connect'] - - # Erlang Distributed Node Protocol (ednp) stunnel server/clients - $ednp_clients = $stunnel['ednp_clients'] - $ednp_server = $stunnel['ednp_server'] - $ednp_server_port = $ednp_server['accept'] - $ednp_server_connect = $ednp_server['connect'] - - # define macro for incoming services - file { '/etc/shorewall/macro.leap_bigcouch': - content => "PARAM - - tcp ${epmd_server_port},${ednp_server_port}", - notify => Service['shorewall'], - require => Package['shorewall'] - } - - shorewall::rule { - 'net2fw-bigcouch': - source => 'net', - destination => '$FW', - action => 'leap_bigcouch(ACCEPT)', - order => 300; - } - - # setup DNAT rules for each epmd - $epmd_shorewall_dnat_defaults = { - 'source' => '$FW', - 'proto' => 'tcp', - 'destinationport' => regsubst($epmd_server_connect, '^([0-9.]+:)([0-9]+)$', '\2') - } - create_resources(site_shorewall::couchdb::dnat, $epmd_clients, $epmd_shorewall_dnat_defaults) - - # setup DNAT rules for each ednp - $ednp_shorewall_dnat_defaults = { - 'source' => '$FW', - 'proto' => 'tcp', - 'destinationport' => regsubst($ednp_server_connect, '^([0-9.]+:)([0-9]+)$', '\2') - } - create_resources(site_shorewall::couchdb::dnat, $ednp_clients, $ednp_shorewall_dnat_defaults) - -} - diff --git a/puppet/modules/site_shorewall/manifests/couchdb/dnat.pp b/puppet/modules/site_shorewall/manifests/couchdb/dnat.pp deleted file mode 100644 index f1bc9acf..00000000 --- a/puppet/modules/site_shorewall/manifests/couchdb/dnat.pp +++ /dev/null @@ -1,21 +0,0 @@ -define site_shorewall::couchdb::dnat ( - $source, - $connect, - $connect_port, - $accept_port, - $proto, - $destinationport ) -{ - - - shorewall::rule { - "dnat_${name}_${destinationport}": - action => 'DNAT', - source => $source, - destination => "\$FW:127.0.0.1:${accept_port}", - proto => $proto, - destinationport => $destinationport, - originaldest => $connect, - order => 200 - } -} diff --git a/puppet/modules/site_shorewall/manifests/stunnel/client.pp b/puppet/modules/site_shorewall/manifests/stunnel/client.pp new file mode 100644 index 00000000..9a89a244 --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/stunnel/client.pp @@ -0,0 +1,40 @@ +# +# Adds some firewall magic to the stunnel. +# +# Using DNAT, this firewall rule allow a locally running program +# to try to connect to the normal remote IP and remote port of the +# service on another machine, but have this connection magically +# routed through the locally running stunnel client. +# +# The network looks like this: +# +# From the client's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# consumer app -> localhost:accept_port -> connect:connect_port -> localhost:original_port +# +# From the server's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# ?? -> *:accept_port -> localhost:connect_port -> service +# + +define site_shorewall::stunnel::client( + $accept_port, + $connect, + $connect_port, + $original_port) { + + include site_shorewall::defaults + + shorewall::rule { + "stunnel_dnat_${name}": + action => 'DNAT', + source => '$FW', + destination => "\$FW:127.0.0.1:${accept_port}", + proto => 'tcp', + destinationport => $original_port, + originaldest => $connect, + order => 200 + } +} diff --git a/puppet/modules/site_shorewall/manifests/stunnel/server.pp b/puppet/modules/site_shorewall/manifests/stunnel/server.pp new file mode 100644 index 00000000..db3ecd3e --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/stunnel/server.pp @@ -0,0 +1,22 @@ +# +# Allow all incoming connections to stunnel server port +# + +define site_shorewall::stunnel::server($port) { + + include site_shorewall::defaults + + file { "/etc/shorewall/macro.stunnel_server_${name}": + content => "PARAM - - tcp ${port}", + notify => Service['shorewall'], + require => Package['shorewall'] + } + shorewall::rule { + 'net2fw-couchdb': + source => 'net', + destination => '$FW', + action => "stunnel_server_${name}(ACCEPT)", + order => 200; + } + +} \ No newline at end of file diff --git a/puppet/modules/site_stunnel/manifests/client.pp b/puppet/modules/site_stunnel/manifests/client.pp new file mode 100644 index 00000000..12d664b4 --- /dev/null +++ b/puppet/modules/site_stunnel/manifests/client.pp @@ -0,0 +1,52 @@ +# +# Sets up stunnel and firewall configuration for +# a single stunnel client +# +# As a client, we accept connections on localhost, +# and connect to a remote $connect:$connect_port +# + +define site_stunnel::client ( + $accept_port, + $connect_port, + $connect, + $original_port, + $verify = '2', + $pid = $name, + $rndfile = '/var/lib/stunnel4/.rnd', + $debuglevel = '4' ) { + + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca + include x509::variables + $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" + $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" + $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" + + stunnel::service { $name: + accept => "127.0.0.1:${accept_port}", + connect => "${connect}:${connect_port}", + client => true, + cafile => $ca_path, + key => $key_path, + cert => $cert_path, + verify => $verify, + pid => "/var/run/stunnel4/${pid}.pid", + rndfile => $rndfile, + debuglevel => $debuglevel, + subscribe => [ + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Ca'] ]; + } + + site_shorewall::stunnel::client { $name: + accept_port => $accept_port, + connect => $connect, + connect_port => $connect_port, + original_port => $original_port + } + + include site_check_mk::agent::stunnel +} diff --git a/puppet/modules/site_stunnel/manifests/clients.pp b/puppet/modules/site_stunnel/manifests/clients.pp index b75c9ac3..44b31aaa 100644 --- a/puppet/modules/site_stunnel/manifests/clients.pp +++ b/puppet/modules/site_stunnel/manifests/clients.pp @@ -1,33 +1,26 @@ -define site_stunnel::clients ( - $accept_port, - $connect_port, - $connect, - $cafile, - $key, - $cert, - $client = true, - $verify = '2', - $pid = $name, - $rndfile = '/var/lib/stunnel4/.rnd', - $debuglevel = '4' ) { +# +# usage: +# create_resource(site_stunnel::clients, hiera('stunnel')['clients']) +# +# example hiera yaml: +# +# stunnel: +# clients: +# ednp_clients: +# thrips_9002: +# accept_port: 4001 +# connect: thrips.demo.bitmask.i +# connect_port: 19002 +# epmd_clients: +# thrips_4369: +# accept_port: 4000 +# connect: thrips.demo.bitmask.i +# connect_port: 14369 +# +# In the above example, this resource definition is called twice, with $name +# 'ednp_clients' and 'epmd_clients' +# - stunnel::service { $name: - accept => "127.0.0.1:${accept_port}", - connect => "${connect}:${connect_port}", - client => $client, - cafile => $cafile, - key => $key, - cert => $cert, - verify => $verify, - pid => "/var/run/stunnel4/${pid}.pid", - rndfile => $rndfile, - debuglevel => $debuglevel, - subscribe => [ - Class['Site_config::X509::Key'], - Class['Site_config::X509::Cert'], - Class['Site_config::X509::Ca'] ]; - - } - - include site_check_mk::agent::stunnel +define site_stunnel::clients { + create_resources(site_stunnel::client, $site_stunnel::clients[$name]) } diff --git a/puppet/modules/site_stunnel/manifests/init.pp b/puppet/modules/site_stunnel/manifests/init.pp index c7d6acc6..b292f1cd 100644 --- a/puppet/modules/site_stunnel/manifests/init.pp +++ b/puppet/modules/site_stunnel/manifests/init.pp @@ -1,3 +1,8 @@ +# +# If you need something to happen after stunnel is started, +# you can depend on Service['stunnel'] or Class['site_stunnel'] +# + class site_stunnel { # include the generic stunnel module @@ -13,5 +18,15 @@ class site_stunnel { ensure => absent; } } + + $stunnel = hiera('stunnel') + + # add server stunnels + create_resources(site_stunnel::servers, $stunnel['servers']) + + # add client stunnels + $clients = $stunnel['clients'] + $client_sections = keys($clients) + site_stunnel::clients { $client_sections: } } diff --git a/puppet/modules/site_stunnel/manifests/servers.pp b/puppet/modules/site_stunnel/manifests/servers.pp new file mode 100644 index 00000000..4419923f --- /dev/null +++ b/puppet/modules/site_stunnel/manifests/servers.pp @@ -0,0 +1,53 @@ +# +# usage: +# create_resource(site_stunnel::servers, hiera('stunnel')['servers']) +# +# example hiera yaml: +# +# stunnel: +# servers: +# couch_server: +# accept_port: 15984 +# connect_port: 5984 +# + +define site_stunnel::servers ( + $accept_port, + $connect_port, + $verify = '2', + $pid = $name, + $rndfile = '/var/lib/stunnel4/.rnd', + $debuglevel = '4' ) { + + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca + include x509::variables + $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" + $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" + $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" + + stunnel::service { $name: + accept => $accept_port, + connect => "127.0.0.1:${connect_port}", + client => false, + cafile => $ca_path, + key => $key_path, + cert => $cert_path, + verify => $verify, + pid => "/var/run/stunnel4/${pid}.pid", + rndfile => '/var/lib/stunnel4/.rnd', + debuglevel => $debuglevel, + require => [ + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Ca'] ]; + } + + # allow incoming connections on $accept_port + site_shorewall::stunnel::server { $name: + port => $accept_port + } + + include site_check_mk::agent::stunnel +} -- cgit v1.2.3 From 7dc648d43d255999c1ce347a73787bd0c0a3f910 Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 20 Jun 2014 01:59:01 -0700 Subject: tmp comment out error if no master nodes defined --- provider_base/services/couchdb.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider_base/services/couchdb.rb b/provider_base/services/couchdb.rb index c8f5d8a7..c63e3a00 100644 --- a/provider_base/services/couchdb.rb +++ b/provider_base/services/couchdb.rb @@ -3,7 +3,7 @@ # unless nodes_like_me['services' => 'couchdb']['couch.master' => true].any? - raise 'node `%s`, environment `%s`: there must be at least one node with couch.master set to `true` for this environment.' % [@node.name, @node.environment] + #raise 'node `%s`, environment `%s`: there must be at least one node with couch.master set to `true` for this environment.' % [@node.name, @node.environment] end if couch.master -- cgit v1.2.3 From 2f7ee09eb9f184349057802e0d6c0102200d2419 Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 20 Jun 2014 02:00:18 -0700 Subject: site_couchdb: support auto-stunnel setup, split master, bigcouch, and mirror out into separate files. --- puppet/modules/site_couchdb/manifests/add_users.pp | 3 ++ puppet/modules/site_couchdb/manifests/bigcouch.pp | 20 ++++++-- .../modules/site_couchdb/manifests/create_dbs.pp | 3 ++ puppet/modules/site_couchdb/manifests/init.pp | 60 +++------------------- puppet/modules/site_couchdb/manifests/master.pp | 9 ++++ puppet/modules/site_couchdb/manifests/mirror.pp | 15 +++++- puppet/modules/site_couchdb/manifests/setup.pp | 39 ++++++++++++++ 7 files changed, 93 insertions(+), 56 deletions(-) create mode 100644 puppet/modules/site_couchdb/manifests/master.pp create mode 100644 puppet/modules/site_couchdb/manifests/setup.pp diff --git a/puppet/modules/site_couchdb/manifests/add_users.pp b/puppet/modules/site_couchdb/manifests/add_users.pp index f9ea7349..41930b7b 100644 --- a/puppet/modules/site_couchdb/manifests/add_users.pp +++ b/puppet/modules/site_couchdb/manifests/add_users.pp @@ -1,5 +1,8 @@ class site_couchdb::add_users { + Class['site_couchdb::create_dbs'] + -> Class['site_couchdb::add_users'] + # Couchdb users ## leap_mx couchdb user diff --git a/puppet/modules/site_couchdb/manifests/bigcouch.pp b/puppet/modules/site_couchdb/manifests/bigcouch.pp index 97c8cd12..f0aab734 100644 --- a/puppet/modules/site_couchdb/manifests/bigcouch.pp +++ b/puppet/modules/site_couchdb/manifests/bigcouch.pp @@ -1,18 +1,32 @@ class site_couchdb::bigcouch { - $config = $::site_couchdb::couchdb_config['bigcouch'] + $config = $couchdb_config['bigcouch'] $cookie = $config['cookie'] + $ednp_port = $config['ednp_port'] - $ednp_port = $config['ednp_port'] + class { 'couchdb': + admin_pw => $couchdb_admin_pw, + admin_salt => $couchdb_admin_salt, + bigcouch => true, + bigcouch_cookie => $cookie, + ednp_port => $ednp_port, + chttpd_bind_address => '127.0.0.1' + } + # + # stunnel must running correctly before bigcouch dbs can be set up. + # Class['site_config::default'] + -> Class['couchdb::bigcouch::package::cloudant'] + -> Service['shorewall'] + -> Service['stunnel'] + -> Class['site_couchdb::setup'] -> Class['site_couchdb::bigcouch::add_nodes'] -> Class['site_couchdb::bigcouch::settle_cluster'] include site_couchdb::bigcouch::add_nodes include site_couchdb::bigcouch::settle_cluster include site_couchdb::bigcouch::compaction - include site_shorewall::couchdb::bigcouch file { '/var/log/bigcouch': ensure => directory diff --git a/puppet/modules/site_couchdb/manifests/create_dbs.pp b/puppet/modules/site_couchdb/manifests/create_dbs.pp index 41500d3a..f8d8098a 100644 --- a/puppet/modules/site_couchdb/manifests/create_dbs.pp +++ b/puppet/modules/site_couchdb/manifests/create_dbs.pp @@ -1,5 +1,8 @@ class site_couchdb::create_dbs { + Class['site_couchdb::setup'] + -> Class['site_couchdb::create_dbs'] + # Couchdb databases ### customer database diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 0b923c9f..4999b611 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -37,70 +37,26 @@ class site_couchdb { $couchdb_backup = $couchdb_config['backup'] $couchdb_mode = $couchdb_config['mode'] - class { 'couchdb': - bigcouch => $couchdb_bigcouch, - admin_pw => $couchdb_admin_pw, - admin_salt => $couchdb_admin_salt, - bigcouch_cookie => $bigcouch_cookie, - ednp_port => $ednp_port, - chttpd_bind_address => '127.0.0.1' - } - - # ensure that we don't have leftovers from previous installations - # where we installed the cloudant bigcouch package - # https://leap.se/code/issues/4971 - class { 'couchdb::bigcouch::package::cloudant': - ensure => absent - } + if $couchdb_mode == "multimaster" { include site_couchdb::bigcouch } + if $couchdb_mode == "master" { include site_couchdb::master } + if $couchdb_mode == "mirror" { include site_couchdb::mirror } Class['site_config::default'] - -> Class['couchdb::bigcouch::package::cloudant'] -> Service['shorewall'] - -> Class['site_couchdb::stunnel'] - -> Service['couchdb'] - -> File['/root/.netrc'] - -> Class['site_couchdb::create_dbs'] - -> Class['site_couchdb::add_users'] - - # /etc/couchdb/couchdb.netrc is deployed by couchdb::query::setup - # we symlink this to /root/.netrc for couchdb_scripts (eg. backup) - # and makes life easier for the admin (i.e. using curl/wget without - # passing credentials) - file { - '/root/.netrc': - ensure => link, - target => '/etc/couchdb/couchdb.netrc'; - - '/srv/leap/couchdb': - ensure => directory - } + -> Service['stunnel'] + -> Class['couchdb'] + -> Class['site_couchdb::setup'] - couchdb::query::setup { 'localhost': - user => $couchdb_admin_user, - pw => $couchdb_admin_pw, - } + include site_stunnel - vcsrepo { '/srv/leap/couchdb/scripts': - ensure => present, - provider => git, - source => 'https://leap.se/git/couchdb_scripts', - revision => 'origin/master', - require => File['/srv/leap/couchdb'] - } - - include site_couchdb::stunnel + include site_couchdb::setup include site_couchdb::create_dbs include site_couchdb::add_users include site_couchdb::designs include site_couchdb::logrotate - if $couchdb_mode == "multimaster" { include site_couchdb::bigcouch } - if $couchdb_mode == "mirror" { include site_couchdb::mirror } - if $couchdb_backup { include site_couchdb::backup } - include site_shorewall::couchdb - include site_check_mk::agent::couchdb include site_check_mk::agent::tapicero diff --git a/puppet/modules/site_couchdb/manifests/master.pp b/puppet/modules/site_couchdb/manifests/master.pp new file mode 100644 index 00000000..a0a6633d --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/master.pp @@ -0,0 +1,9 @@ +class site_couchdb::master { + + class { 'couchdb': + admin_pw => $site_couchdb::couchdb_admin_pw, + admin_salt => $site_couchdb::couchdb_admin_salt, + chttpd_bind_address => '127.0.0.1' + } + +} \ No newline at end of file diff --git a/puppet/modules/site_couchdb/manifests/mirror.pp b/puppet/modules/site_couchdb/manifests/mirror.pp index 708171e4..a6222f6a 100644 --- a/puppet/modules/site_couchdb/manifests/mirror.pp +++ b/puppet/modules/site_couchdb/manifests/mirror.pp @@ -1,8 +1,21 @@ class site_couchdb::mirror { + class { 'couchdb': + admin_pw => $site_couchdb::couchdb_admin_pw, + admin_salt => $site_couchdb::couchdb_admin_salt, + chttpd_bind_address => '127.0.0.1' + } + # Couchdb databases - $from = $site_couchdb::couchdb_config['replication']['masters'][0] + $masters = $site_couchdb::couchdb_config['replication']['masters'] + $master_node_names = keys($site_couchdb::couchdb_config['replication']['masters']) + $master_node = $masters[$master_node_names[0]] + $from_host = $master_node['domain_internal'] + $from_port = $master_node['couch_port'] + $from = "${from_host}:${from_port}" + + notice("mirror from: ${from}") ### customer database couchdb::mirror_db { 'customers': diff --git a/puppet/modules/site_couchdb/manifests/setup.pp b/puppet/modules/site_couchdb/manifests/setup.pp new file mode 100644 index 00000000..e398356b --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/setup.pp @@ -0,0 +1,39 @@ +# +# An initial setup class. All the other classes depend on this +# +class site_couchdb::setup { + + # ensure that we don't have leftovers from previous installations + # where we installed the cloudant bigcouch package + # https://leap.se/code/issues/4971 + class { 'couchdb::bigcouch::package::cloudant': + ensure => absent + } + + # /etc/couchdb/couchdb.netrc is deployed by couchdb::query::setup + # we symlink this to /root/.netrc for couchdb_scripts (eg. backup) + # and makes life easier for the admin (i.e. using curl/wget without + # passing credentials) + file { + '/root/.netrc': + ensure => link, + target => '/etc/couchdb/couchdb.netrc'; + + '/srv/leap/couchdb': + ensure => directory + } + + couchdb::query::setup { 'localhost': + user => $site_couchdb::couchdb_admin_user, + pw => $site_couchdb::couchdb_admin_pw, + } + + vcsrepo { '/srv/leap/couchdb/scripts': + ensure => present, + provider => git, + source => 'https://leap.se/git/couchdb_scripts', + revision => 'origin/master', + require => File['/srv/leap/couchdb'] + } + +} -- cgit v1.2.3 From f0459aad2e72646de27823ae4fb9011f74a3c16d Mon Sep 17 00:00:00 2001 From: Azul Date: Fri, 20 Jun 2014 19:10:44 +0200 Subject: add replication user --- provider_base/services/couchdb.json | 5 +++++ puppet/modules/site_couchdb/manifests/add_users.pp | 9 +++++++++ puppet/modules/site_couchdb/manifests/create_dbs.pp | 18 +++++++++--------- puppet/modules/site_couchdb/manifests/init.pp | 5 +++++ puppet/modules/site_couchdb/manifests/mirror.pp | 4 +++- 5 files changed, 31 insertions(+), 10 deletions(-) diff --git a/provider_base/services/couchdb.json b/provider_base/services/couchdb.json index c2482235..8b1386f8 100644 --- a/provider_base/services/couchdb.json +++ b/provider_base/services/couchdb.json @@ -40,6 +40,11 @@ "username": "webapp", "password": "= secret :couch_webapp_password", "salt": "= hex_secret :couch_webapp_password_salt, 128" + }, + "replication": { + "username": "replication", + "password": "= secret :couch_replication_password", + "salt": "= hex_secret :couch_replication_password_salt, 128" } }, "webapp": { diff --git a/puppet/modules/site_couchdb/manifests/add_users.pp b/puppet/modules/site_couchdb/manifests/add_users.pp index 41930b7b..0585da27 100644 --- a/puppet/modules/site_couchdb/manifests/add_users.pp +++ b/puppet/modules/site_couchdb/manifests/add_users.pp @@ -54,4 +54,13 @@ class site_couchdb::add_users { require => Couchdb::Query::Setup['localhost'] } + ## replication couchdb user + ## read/write: all databases for replication + couchdb::add_user { $site_couchdb::couchdb_replication_user: + roles => '["repliction"]', + pw => $site_couchdb::couchdb_replication_pw, + salt => $site_couchdb::couchdb_replication_salt, + require => Couchdb::Query::Setup['localhost'] + } + } diff --git a/puppet/modules/site_couchdb/manifests/create_dbs.pp b/puppet/modules/site_couchdb/manifests/create_dbs.pp index f8d8098a..4322f773 100644 --- a/puppet/modules/site_couchdb/manifests/create_dbs.pp +++ b/puppet/modules/site_couchdb/manifests/create_dbs.pp @@ -8,7 +8,7 @@ class site_couchdb::create_dbs { ### customer database ### r/w: webapp, couchdb::create_db { 'customers': - members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }", + members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [\"replication\"] }", require => Couchdb::Query::Setup['localhost'] } @@ -16,35 +16,35 @@ class site_couchdb::create_dbs { ## r: nickserver, leap_mx - needs to be restrict with design document ## r/w: webapp couchdb::create_db { 'identities': - members => "{ \"names\": [], \"roles\": [\"identities\"] }", + members => "{ \"names\": [], \"roles\": [\"replication\", \"identities\"] }", require => Couchdb::Query::Setup['localhost'] } ## keycache database ## r/w: nickserver couchdb::create_db { 'keycache': - members => "{ \"names\": [], \"roles\": [\"keycache\"] }", + members => "{ \"names\": [], \"roles\": [\"replication\", \"keycache\"] }", require => Couchdb::Query::Setup['localhost'] } ## sessions database ## r/w: webapp couchdb::create_db { 'sessions': - members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }", + members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [\"replication\"] }", require => Couchdb::Query::Setup['localhost'] } ## shared database ## r/w: soledad couchdb::create_db { 'shared': - members => "{ \"names\": [\"$site_couchdb::couchdb_soledad_user\"], \"roles\": [] }", + members => "{ \"names\": [\"$site_couchdb::couchdb_soledad_user\"], \"roles\": [\"replication\"] }", require => Couchdb::Query::Setup['localhost'] } ## tickets database ## r/w: webapp couchdb::create_db { 'tickets': - members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }", + members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [\"replication\"] }", require => Couchdb::Query::Setup['localhost'] } @@ -52,14 +52,14 @@ class site_couchdb::create_dbs { ## r: soledad - needs to be restricted with a design document ## r/w: webapp couchdb::create_db { 'tokens': - members => "{ \"names\": [], \"roles\": [\"tokens\"] }", + members => "{ \"names\": [], \"roles\": [\"replication\", \"tokens\"] }", require => Couchdb::Query::Setup['localhost'] } ## users database ## r/w: webapp couchdb::create_db { 'users': - members => "{ \"names\": [], \"roles\": [\"users\"] }", + members => "{ \"names\": [], \"roles\": [\"replication\", \"users\"] }", require => Couchdb::Query::Setup['localhost'] } @@ -67,7 +67,7 @@ class site_couchdb::create_dbs { ## store messages to the clients such as payment reminders ## r/w: webapp couchdb::create_db { 'messages': - members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }", + members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [\"replication\"] }", require => Couchdb::Query::Setup['localhost'] } } diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 4999b611..6f7e974e 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -34,6 +34,11 @@ class site_couchdb { $couchdb_webapp_pw = $couchdb_webapp['password'] $couchdb_webapp_salt = $couchdb_webapp['salt'] + $couchdb_replication = $couchdb_users['replication'] + $couchdb_replication_user= $couchdb_replication['username'] + $couchdb_replication_pw = $couchdb_replication['password'] + $couchdb_replication_salt= $couchdb_replication['salt'] + $couchdb_backup = $couchdb_config['backup'] $couchdb_mode = $couchdb_config['mode'] diff --git a/puppet/modules/site_couchdb/manifests/mirror.pp b/puppet/modules/site_couchdb/manifests/mirror.pp index a6222f6a..df305737 100644 --- a/puppet/modules/site_couchdb/manifests/mirror.pp +++ b/puppet/modules/site_couchdb/manifests/mirror.pp @@ -11,9 +11,11 @@ class site_couchdb::mirror { $masters = $site_couchdb::couchdb_config['replication']['masters'] $master_node_names = keys($site_couchdb::couchdb_config['replication']['masters']) $master_node = $masters[$master_node_names[0]] + $user = $site_couchdb::couchdb_replication_user + $password = $site_couchdb::couchdb_replication_pw $from_host = $master_node['domain_internal'] $from_port = $master_node['couch_port'] - $from = "${from_host}:${from_port}" + $from = "http://${user}:${password}@${from_host}:${from_port}" notice("mirror from: ${from}") -- cgit v1.2.3 From 34ab06778ef23203454c27ae41773568b8aae506 Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 20 Jun 2014 14:34:53 -0700 Subject: stunnel: make site_mx and site_webapp use new site_stunnel --- puppet/modules/site_mx/manifests/couchdb.pp | 23 ----------------------- puppet/modules/site_mx/manifests/init.pp | 2 +- puppet/modules/site_stunnel/manifests/clients.pp | 3 --- puppet/modules/site_stunnel/manifests/servers.pp | 3 --- puppet/modules/site_webapp/manifests/couchdb.pp | 14 -------------- 5 files changed, 1 insertion(+), 44 deletions(-) delete mode 100644 puppet/modules/site_mx/manifests/couchdb.pp diff --git a/puppet/modules/site_mx/manifests/couchdb.pp b/puppet/modules/site_mx/manifests/couchdb.pp deleted file mode 100644 index b1f3bd02..00000000 --- a/puppet/modules/site_mx/manifests/couchdb.pp +++ /dev/null @@ -1,23 +0,0 @@ -class site_mx::couchdb { - - $stunnel = hiera('stunnel') - $couch_client = $stunnel['couch_client'] - $couch_client_connect = $couch_client['connect'] - - include x509::variables - $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" - $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" - $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" - - include site_stunnel - - $couchdb_stunnel_client_defaults = { - 'connect_port' => $couch_client_connect, - 'client' => true, - 'cafile' => $ca_path, - 'key' => $key_path, - 'cert' => $cert_path, - } - - create_resources(site_stunnel::clients, $couch_client, $couchdb_stunnel_client_defaults) -} diff --git a/puppet/modules/site_mx/manifests/init.pp b/puppet/modules/site_mx/manifests/init.pp index c3d38a46..91014ed6 100644 --- a/puppet/modules/site_mx/manifests/init.pp +++ b/puppet/modules/site_mx/manifests/init.pp @@ -8,12 +8,12 @@ class site_mx { include site_config::x509::client_ca::ca include site_config::x509::client_ca::key + include site_stunnel include site_postfix::mx include site_haproxy include site_shorewall::mx include site_shorewall::service::smtp - include site_mx::couchdb include leap_mx include site_check_mk::agent::mx } diff --git a/puppet/modules/site_stunnel/manifests/clients.pp b/puppet/modules/site_stunnel/manifests/clients.pp index 44b31aaa..c0958b5f 100644 --- a/puppet/modules/site_stunnel/manifests/clients.pp +++ b/puppet/modules/site_stunnel/manifests/clients.pp @@ -1,7 +1,4 @@ # -# usage: -# create_resource(site_stunnel::clients, hiera('stunnel')['clients']) -# # example hiera yaml: # # stunnel: diff --git a/puppet/modules/site_stunnel/manifests/servers.pp b/puppet/modules/site_stunnel/manifests/servers.pp index 4419923f..b1da5c59 100644 --- a/puppet/modules/site_stunnel/manifests/servers.pp +++ b/puppet/modules/site_stunnel/manifests/servers.pp @@ -1,7 +1,4 @@ # -# usage: -# create_resource(site_stunnel::servers, hiera('stunnel')['servers']) -# # example hiera yaml: # # stunnel: diff --git a/puppet/modules/site_webapp/manifests/couchdb.pp b/puppet/modules/site_webapp/manifests/couchdb.pp index ff743fba..3ae4d266 100644 --- a/puppet/modules/site_webapp/manifests/couchdb.pp +++ b/puppet/modules/site_webapp/manifests/couchdb.pp @@ -7,10 +7,6 @@ class site_webapp::couchdb { $couchdb_webapp_user = $webapp['couchdb_webapp_user']['username'] $couchdb_webapp_password = $webapp['couchdb_webapp_user']['password'] - $stunnel = hiera('stunnel') - $couch_client = $stunnel['couch_client'] - $couch_client_connect = $couch_client['connect'] - include x509::variables file { @@ -37,14 +33,4 @@ class site_webapp::couchdb { } include site_stunnel - - $couchdb_stunnel_client_defaults = { - 'connect_port' => $couch_client_connect, - 'client' => true, - 'cafile' => "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt", - 'key' => "${x509::variables::keys}/${site_config::params::cert_name}.key", - 'cert' => "${x509::variables::certs}/${site_config::params::cert_name}.crt", - } - - create_resources(site_stunnel::clients, $couch_client, $couchdb_stunnel_client_defaults) } -- cgit v1.2.3 From 73ef324c993f03b2b8f47418f8b2cf7ff97314c6 Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 20 Jun 2014 15:42:51 -0700 Subject: moved json macros to provider_base/lib/macros. requires new unreleased leap_cli --- provider_base/lib/macros.rb | 14 ++++++ provider_base/lib/macros/core.rb | 86 +++++++++++++++++++++++++++++++++ provider_base/lib/macros/files.rb | 79 ++++++++++++++++++++++++++++++ provider_base/lib/macros/haproxy.rb | 68 ++++++++++++++++++++++++++ provider_base/lib/macros/hosts.rb | 63 ++++++++++++++++++++++++ provider_base/lib/macros/nodes.rb | 88 ++++++++++++++++++++++++++++++++++ provider_base/lib/macros/secrets.rb | 39 +++++++++++++++ provider_base/lib/macros/stunnel.rb | 95 +++++++++++++++++++++++++++++++++++++ 8 files changed, 532 insertions(+) create mode 100644 provider_base/lib/macros.rb create mode 100644 provider_base/lib/macros/core.rb create mode 100644 provider_base/lib/macros/files.rb create mode 100644 provider_base/lib/macros/haproxy.rb create mode 100644 provider_base/lib/macros/hosts.rb create mode 100644 provider_base/lib/macros/nodes.rb create mode 100644 provider_base/lib/macros/secrets.rb create mode 100644 provider_base/lib/macros/stunnel.rb diff --git a/provider_base/lib/macros.rb b/provider_base/lib/macros.rb new file mode 100644 index 00000000..854b92b5 --- /dev/null +++ b/provider_base/lib/macros.rb @@ -0,0 +1,14 @@ +# +# MACROS +# +# The methods in these files are available in the context of a .json configuration file. +# (The module LeapCli::Macro is included in Config::Object) +# + +require_relative 'macros/core' +require_relative 'macros/files' +require_relative 'macros/haproxy' +require_relative 'macros/hosts' +require_relative 'macros/nodes' +require_relative 'macros/secrets' +require_relative 'macros/stunnel' diff --git a/provider_base/lib/macros/core.rb b/provider_base/lib/macros/core.rb new file mode 100644 index 00000000..d4d9171f --- /dev/null +++ b/provider_base/lib/macros/core.rb @@ -0,0 +1,86 @@ +# encoding: utf-8 + +module LeapCli + module Macro + + # + # return a fingerprint for a x509 certificate + # + def fingerprint(filename) + "SHA256: " + X509.fingerprint("SHA256", Path.named_path(filename)) + end + + # + # Creates a hash from the ssh key info in users directory, for use in + # updating authorized_keys file. Additionally, the 'monitor' public key is + # included, which is used by the monitor nodes to run particular commands + # remotely. + # + def authorized_keys + hash = {} + keys = Dir.glob(Path.named_path([:user_ssh, '*'])) + keys.sort.each do |keyfile| + ssh_type, ssh_key = File.read(keyfile, :encoding => 'UTF-8').strip.split(" ") + name = File.basename(File.dirname(keyfile)) + hash[name] = { + "type" => ssh_type, + "key" => ssh_key + } + end + ssh_type, ssh_key = File.read(Path.named_path(:monitor_pub_key), :encoding => 'UTF-8').strip.split(" ") + hash[Leap::Platform.monitor_username] = { + "type" => ssh_type, + "key" => ssh_key + } + hash + end + + def assert(assertion) + if instance_eval(assertion) + true + else + raise AssertionFailed.new(assertion) + end + end + + # + # applies a JSON partial to this node + # + def apply_partial(partial_path) + manager.partials(partial_path).each do |partial_data| + self.deep_merge!(partial_data) + end + end + + # + # If at first you don't succeed, then it is time to give up. + # + # try{} returns nil if anything in the block throws an exception. + # + # You can wrap something that might fail in `try`, like so. + # + # "= try{ nodes[:services => 'tor'].first.ip_address } " + # + def try(&block) + yield + rescue NoMethodError + nil + end + + protected + + # + # returns a node list, if argument is not already one + # + def listify(node_list) + if node_list.is_a? Config::ObjectList + node_list + elsif node_list.is_a? Config::Object + Config::ObjectList.new(node_list) + else + raise ArgumentError, 'argument must be a node or node list, not a `%s`' % node_list.class, caller + end + end + + end +end diff --git a/provider_base/lib/macros/files.rb b/provider_base/lib/macros/files.rb new file mode 100644 index 00000000..0a491325 --- /dev/null +++ b/provider_base/lib/macros/files.rb @@ -0,0 +1,79 @@ +# encoding: utf-8 + +## +## FILES +## + +module LeapCli + module Macro + + # + # inserts the contents of a file + # + def file(filename, options={}) + if filename.is_a? Symbol + filename = [filename, @node.name] + end + filepath = Path.find_file(filename) + if filepath + if filepath =~ /\.erb$/ + ERB.new(File.read(filepath, :encoding => 'UTF-8'), nil, '%<>').result(binding) + else + File.read(filepath, :encoding => 'UTF-8') + end + else + raise FileMissing.new(Path.named_path(filename), options) + "" + end + end + + # + # like #file, but allow missing files + # + def try_file(filename) + return file(filename) + rescue FileMissing + return nil + end + + # + # returns what the file path will be, once the file is rsynced to the server. + # an internal list of discovered file paths is saved, in order to rsync these files when needed. + # + # notes: + # + # * argument 'path' is relative to Path.provider/files or Path.provider_base/files + # * the path returned by this method is absolute + # * the path stored for use later by rsync is relative to Path.provider + # * if the path does not exist locally, but exists in provider_base, then the default file from + # provider_base is copied locally. this is required for rsync to work correctly. + # + def file_path(path) + if path.is_a? Symbol + path = [path, @node.name] + end + actual_path = Path.find_file(path) + if actual_path.nil? + Util::log 2, :skipping, "file_path(\"#{path}\") because there is no such file." + nil + else + if actual_path =~ /^#{Regexp.escape(Path.provider_base)}/ + # if file is under Path.provider_base, we must copy the default file to + # to Path.provider in order for rsync to be able to sync the file. + local_provider_path = actual_path.sub(/^#{Regexp.escape(Path.provider_base)}/, Path.provider) + FileUtils.mkdir_p File.dirname(local_provider_path), :mode => 0700 + FileUtils.install actual_path, local_provider_path, :mode => 0600 + Util.log :created, Path.relative_path(local_provider_path) + actual_path = local_provider_path + end + if File.directory?(actual_path) && actual_path !~ /\/$/ + actual_path += '/' # ensure directories end with /, important for building rsync command + end + relative_path = Path.relative_path(actual_path) + @node.file_paths << relative_path + @node.manager.provider.hiera_sync_destination + '/' + relative_path + end + end + + end +end \ No newline at end of file diff --git a/provider_base/lib/macros/haproxy.rb b/provider_base/lib/macros/haproxy.rb new file mode 100644 index 00000000..db0d4db8 --- /dev/null +++ b/provider_base/lib/macros/haproxy.rb @@ -0,0 +1,68 @@ +# encoding: utf-8 + +## +## HAPROXY +## + +module LeapCli + module Macro + + # + # creates a hash suitable for configuring haproxy. the key is the node name of the server we are proxying to. + # + # * node_list - a hash of nodes for the haproxy servers + # * stunnel_client - contains the mappings to local ports for each server node. + # * non_stunnel_port - in case self is included in node_list, the port to connect to. + # + # 1000 weight is used for nodes in the same location. + # 100 otherwise. + # + def haproxy_servers(node_list, stunnel_clients, non_stunnel_port=nil) + default_weight = 10 + local_weight = 100 + + # record the hosts_file + hostnames(node_list) + + # create a simple map for node name -> local stunnel accept port + accept_ports = stunnel_clients.inject({}) do |hsh, stunnel_entry| + name = stunnel_entry.first.sub /_[0-9]+$/, '' + hsh[name] = stunnel_entry.last['accept_port'] + hsh + end + + # if one the nodes in the node list is ourself, then there will not be a stunnel to it, + # but we need to include it anyway in the haproxy config. + if node_list[self.name] && non_stunnel_port + accept_ports[self.name] = non_stunnel_port + end + + # create the first pass of the servers hash + servers = node_list.values.inject(Config::ObjectList.new) do |hsh, node| + weight = default_weight + if self['location'] && node['location'] + if self.location['name'] == node.location['name'] + weight = local_weight + end + end + hsh[node.name] = Config::Object[ + 'backup', false, + 'host', 'localhost', + 'port', accept_ports[node.name] || 0, + 'weight', weight + ] + hsh + end + + # if there are some local servers, make the others backup + if servers.detect{|k,v| v.weight == local_weight} + servers.each do |k,server| + server['backup'] = server['weight'] == default_weight + end + end + + return servers + end + + end +end \ No newline at end of file diff --git a/provider_base/lib/macros/hosts.rb b/provider_base/lib/macros/hosts.rb new file mode 100644 index 00000000..8a4058a5 --- /dev/null +++ b/provider_base/lib/macros/hosts.rb @@ -0,0 +1,63 @@ +# encoding: utf-8 + +module LeapCli + module Macro + + ## + ## HOSTS + ## + + # + # records the list of hosts that are encountered for this node + # + def hostnames(nodes) + @referenced_nodes ||= Config::ObjectList.new + nodes = listify(nodes) + nodes.each_node do |node| + @referenced_nodes[node.name] ||= node + end + return nodes.values.collect {|node| node.domain.name} + end + + # + # Generates entries needed for updating /etc/hosts on a node (as a hash). + # + # Argument `nodes` can be nil or a list of nodes. If nil, only include the + # IPs of the other nodes this @node as has encountered (plus all mx nodes). + # + # Also, for virtual machines, we use the local address if this @node is in + # the same location as the node in question. + # + # We include the ssh public key for each host, so that the hash can also + # be used to generate the /etc/ssh/known_hosts + # + def hosts_file(nodes=nil) + if nodes.nil? + if @referenced_nodes && @referenced_nodes.any? + nodes = @referenced_nodes + nodes = nodes.merge(nodes_like_me[:services => 'mx']) # all nodes always need to communicate with mx nodes. + end + end + return {} unless nodes + hosts = {} + my_location = @node['location'] ? @node['location']['name'] : nil + nodes.each_node do |node| + hosts[node.name] = {'ip_address' => node.ip_address, 'domain_internal' => node.domain.internal, 'domain_full' => node.domain.full} + node_location = node['location'] ? node['location']['name'] : nil + if my_location == node_location + if facts = @node.manager.facts[node.name] + if facts['ec2_public_ipv4'] + hosts[node.name]['ip_address'] = facts['ec2_public_ipv4'] + end + end + end + host_pub_key = Util::read_file([:node_ssh_pub_key,node.name]) + if host_pub_key + hosts[node.name]['host_pub_key'] = host_pub_key + end + end + hosts + end + + end +end \ No newline at end of file diff --git a/provider_base/lib/macros/nodes.rb b/provider_base/lib/macros/nodes.rb new file mode 100644 index 00000000..0c6668a0 --- /dev/null +++ b/provider_base/lib/macros/nodes.rb @@ -0,0 +1,88 @@ +# encoding: utf-8 + +## +## node related macros +## + +module LeapCli + module Macro + + # + # the list of all the nodes + # + def nodes + global.nodes + end + + # + # grab an environment appropriate provider + # + def provider + global.env(@node.environment).provider + end + + # + # returns a list of nodes that match the same environment + # + # if @node.environment is not set, we return other nodes + # where environment is not set. + # + def nodes_like_me + nodes[:environment => @node.environment] + end + + # + # returns a list of nodes that match the location name + # and environment of @node. + # + def nodes_near_me + if @node['location'] && @node['location']['name'] + nodes_like_me['location.name' => @node.location.name] + else + nodes_like_me['location' => nil] + end + end + + # + # + # picks a node out from the node list in such a way that: + # + # (1) which nodes picked which nodes is saved in secrets.json + # (2) when other nodes call this macro with the same node list, they are guaranteed to get a different node + # (3) if all the nodes in the pick_node list have been picked, remaining nodes are distributed randomly. + # + # if the node_list is empty, an exception is raised. + # if node_list size is 1, then that node is returned and nothing is + # memorized via the secrets.json file. + # + # `label` is needed to distinguish between pools of nodes for different purposes. + # + # TODO: more evenly balance after all the nodes have been picked. + # + def pick_node(label, node_list) + if node_list.any? + if node_list.size == 1 + return node_list.values.first + else + secrets_key = "pick_node(:#{label},#{node_list.keys.sort.join(',')})" + secrets_value = @manager.secrets.retrieve(secrets_key, @node.environment) || {} + secrets_value[@node.name] ||= begin + node_to_pick = nil + node_list.each_node do |node| + next if secrets_value.values.include?(node.name) + node_to_pick = node.name + end + node_to_pick ||= secrets_value.values.shuffle.first # all picked already, so pick a random one. + node_to_pick + end + picked_node_name = secrets_value[@node.name] + @manager.secrets.set(secrets_key, secrets_value, @node.environment) + return node_list[picked_node_name] + end + else + raise ArgumentError.new('pick_node(node_list): node_list cannot be empty') + end + end + + end +end \ No newline at end of file diff --git a/provider_base/lib/macros/secrets.rb b/provider_base/lib/macros/secrets.rb new file mode 100644 index 00000000..51bf3971 --- /dev/null +++ b/provider_base/lib/macros/secrets.rb @@ -0,0 +1,39 @@ +# encoding: utf-8 + +require 'base32' + +module LeapCli + module Macro + + # + # inserts a named secret, generating it if needed. + # + # manager.export_secrets should be called later to capture any newly generated secrets. + # + # +length+ is the character length of the generated password. + # + def secret(name, length=32) + @manager.secrets.set(name, Util::Secret.generate(length), @node[:environment]) + end + + # inserts a base32 encoded secret + def base32_secret(name, length=20) + @manager.secrets.set(name, Base32.encode(Util::Secret.generate(length)), @node[:environment]) + end + + # Picks a random obfsproxy port from given range + def rand_range(name, range) + @manager.secrets.set(name, rand(range), @node[:environment]) + end + + # + # inserts an hexidecimal secret string, generating it if needed. + # + # +bit_length+ is the bits in the secret, (ie length of resulting hex string will be bit_length/4) + # + def hex_secret(name, bit_length=128) + @manager.secrets.set(name, Util::Secret.generate_hex(bit_length), @node[:environment]) + end + + end +end \ No newline at end of file diff --git a/provider_base/lib/macros/stunnel.rb b/provider_base/lib/macros/stunnel.rb new file mode 100644 index 00000000..f16308c7 --- /dev/null +++ b/provider_base/lib/macros/stunnel.rb @@ -0,0 +1,95 @@ +## +## STUNNEL +## + +# +# About stunnel +# -------------------------- +# +# The network looks like this: +# +# From the client's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# consumer app -> localhost:accept_port -> connect:connect_port -> ?? +# +# From the server's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# ?? -> *:accept_port -> localhost:connect_port -> service +# + +module LeapCli + module Macro + + # + # stunnel configuration for the client side. + # + # +node_list+ is a ObjectList of nodes running stunnel servers. + # + # +port+ is the real port of the ultimate service running on the servers + # that the client wants to connect to. + # + # * accept_port is the port on localhost to which local clients + # can connect. it is auto generated serially. + # + # * connect_port is the port on the stunnel server to connect to. + # it is auto generated from the +port+ argument. + # + # generates an entry appropriate to be passed directly to + # create_resources(stunnel::service, hiera('..'), defaults) + # + # local ports are automatically generated, starting at 4000 + # and incrementing in sorted order (by node name). + # + def stunnel_client(node_list, port, options={}) + @next_stunnel_port ||= 4000 + node_list = listify(node_list) + hostnames(node_list) # record the hosts + result = Config::ObjectList.new + node_list.each_node do |node| + if node.name != self.name || options[:include_self] + result["#{node.name}_#{port}"] = Config::Object[ + 'accept_port', @next_stunnel_port, + 'connect', node.domain.internal, + 'connect_port', stunnel_port(port), + 'original_port', port + ] + @next_stunnel_port += 1 + end + end + result + end + + # + # generates a stunnel server entry. + # + # +port+ is the real port targeted service. + # + # * `accept_port` is the publicly bound port + # * `connect_port` is the port that the local service is running on. + # + def stunnel_server(port) + { + "accept_port" => stunnel_port(port), + "connect_port" => port + } + end + + private + + # + # maps a real port to a stunnel port (used as the connect_port in the client config + # and the accept_port in the server config) + # + def stunnel_port(port) + port = port.to_i + if port < 50000 + return port + 10000 + else + return port - 10000 + end + end + + end +end \ No newline at end of file -- cgit v1.2.3 From 56917469bd20293dda7e4187c01d822d42ee98cd Mon Sep 17 00:00:00 2001 From: elijah Date: Sat, 21 Jun 2014 02:51:51 -0700 Subject: fix stunnel entries in mx.json and webapp.json --- provider_base/services/mx.json | 9 +++++++-- provider_base/services/webapp.json | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json index 731dee9a..d28f03ed 100644 --- a/provider_base/services/mx.json +++ b/provider_base/services/mx.json @@ -1,9 +1,14 @@ { "stunnel": { - "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" + "clients": { + "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" + } }, "haproxy": { - "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.couch_client)" + "couch": { + "listen_port": 4096, + "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client)" + } }, "couchdb_leap_mx_user": { "username": "= global.services[:couchdb].couch.users[:leap_mx].username", diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index bbb52094..9f319910 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -29,10 +29,15 @@ } }, "stunnel": { - "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" + "clients": { + "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" + } }, "haproxy": { - "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.couch_client, global.services[:couchdb].couch.port)" + "couch": { + "listen_port": 4096, + "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, global.services[:couchdb].couch.port)" + } }, "definition_files": { "provider": "= file :provider_json_template", -- cgit v1.2.3 From 9b55b9a0feac9ec778db74f250d4bc2bb5831e08 Mon Sep 17 00:00:00 2001 From: elijah Date: Sat, 21 Jun 2014 02:52:43 -0700 Subject: haproxy: support read only couchdb mirrors --- provider_base/lib/macros/haproxy.rb | 11 +++--- puppet/modules/site_haproxy/manifests/init.pp | 42 +++++++++++----------- puppet/modules/site_haproxy/templates/couch.erb | 32 +++++++++++++++++ .../modules/site_haproxy/templates/haproxy.cfg.erb | 11 ++++++ .../site_haproxy/templates/haproxy_couchdb.cfg.erb | 23 ------------ 5 files changed, 70 insertions(+), 49 deletions(-) create mode 100644 puppet/modules/site_haproxy/templates/couch.erb create mode 100644 puppet/modules/site_haproxy/templates/haproxy.cfg.erb delete mode 100644 puppet/modules/site_haproxy/templates/haproxy_couchdb.cfg.erb diff --git a/provider_base/lib/macros/haproxy.rb b/provider_base/lib/macros/haproxy.rb index db0d4db8..c0f9ede5 100644 --- a/provider_base/lib/macros/haproxy.rb +++ b/provider_base/lib/macros/haproxy.rb @@ -40,17 +40,18 @@ module LeapCli # create the first pass of the servers hash servers = node_list.values.inject(Config::ObjectList.new) do |hsh, node| weight = default_weight - if self['location'] && node['location'] - if self.location['name'] == node.location['name'] - weight = local_weight - end - end + try { + weight = local_weight if self.location.name == node.location.name + } hsh[node.name] = Config::Object[ 'backup', false, 'host', 'localhost', 'port', accept_ports[node.name] || 0, 'weight', weight ] + if node.services.include?('couchdb') + hsh[node.name]['writable'] = node.couch.mode != 'mirror' + end hsh end diff --git a/puppet/modules/site_haproxy/manifests/init.pp b/puppet/modules/site_haproxy/manifests/init.pp index 6bcf3f5c..b28ce80e 100644 --- a/puppet/modules/site_haproxy/manifests/init.pp +++ b/puppet/modules/site_haproxy/manifests/init.pp @@ -2,25 +2,25 @@ class site_haproxy { $haproxy = hiera('haproxy') class { 'haproxy': - enable => true, - manage_service => true, - global_options => { - 'log' => '127.0.0.1 local0', - 'maxconn' => '4096', - 'stats' => 'socket /var/run/haproxy.sock user haproxy group haproxy', - 'chroot' => '/usr/share/haproxy', - 'user' => 'haproxy', - 'group' => 'haproxy', - 'daemon' => '' - }, - defaults_options => { - 'log' => 'global', - 'retries' => '3', - 'option' => 'redispatch', - 'timeout connect' => '4000', - 'timeout client' => '20000', - 'timeout server' => '20000' - } + enable => true, + manage_service => true, + global_options => { + 'log' => '127.0.0.1 local0', + 'maxconn' => '4096', + 'stats' => 'socket /var/run/haproxy.sock user haproxy group haproxy', + 'chroot' => '/usr/share/haproxy', + 'user' => 'haproxy', + 'group' => 'haproxy', + 'daemon' => '' + }, + defaults_options => { + 'log' => 'global', + 'retries' => '3', + 'option' => 'redispatch', + 'timeout connect' => '4000', + 'timeout client' => '20000', + 'timeout server' => '20000' + } } # monitor haproxy @@ -34,8 +34,8 @@ class site_haproxy { concat::fragment { 'leap_haproxy_webapp_couchdb': target => '/etc/haproxy/haproxy.cfg', order => '20', - content => template('site_haproxy/haproxy_couchdb.cfg.erb'), + content => template('site_haproxy/haproxy.cfg.erb'), } - + include site_check_mk::agent::haproxy } diff --git a/puppet/modules/site_haproxy/templates/couch.erb b/puppet/modules/site_haproxy/templates/couch.erb new file mode 100644 index 00000000..baa31486 --- /dev/null +++ b/puppet/modules/site_haproxy/templates/couch.erb @@ -0,0 +1,32 @@ +frontend couch + bind localhost:<%= @listen_port %> + mode http + option httplog + option dontlognull + option http-server-close # use client keep-alive, but close server connection. + use_backend couch_write if METH_POST + default_backend couch_read + +backend couch_write + mode http + balance roundrobin + option httpchk GET / # health check using simple get to root + option allbackups # balance among all backups, not just one. + default-server inter 3000 fastinter 1000 downinter 1000 rise 2 fall 1 +<%- @servers.sort.each do |name,server| -%> +<%- next unless server['writable'] -%> + # <%=name%> + server couchdb_<%=server['port']%> <%=server['host']%>:<%=server['port']%> <%='backup' if server['backup']%> weight <%=server['weight']%> check +<%- end -%> + +backend couch_read + mode http + balance roundrobin + option httpchk GET / # health check using simple get to root + option allbackups # balance among all backups, not just one. + default-server inter 3000 fastinter 1000 downinter 1000 rise 2 fall 1 +<%- @servers.sort.each do |name,server| -%> + # <%=name%> + server couchdb_<%=server['port']%> <%=server['host']%>:<%=server['port']%> <%='backup' if server['backup']%> weight <%=server['weight']%> check +<%- end -%> + diff --git a/puppet/modules/site_haproxy/templates/haproxy.cfg.erb b/puppet/modules/site_haproxy/templates/haproxy.cfg.erb new file mode 100644 index 00000000..8311b1a5 --- /dev/null +++ b/puppet/modules/site_haproxy/templates/haproxy.cfg.erb @@ -0,0 +1,11 @@ +<%- @haproxy.each do |frontend, options| -%> +<%- if options['servers'] -%> + +## +## <%= frontend %> +## + +<%= scope.function_templatewlv(["site_haproxy/#{frontend}.erb", options]) %> +<%- end -%> +<%- end -%> + diff --git a/puppet/modules/site_haproxy/templates/haproxy_couchdb.cfg.erb b/puppet/modules/site_haproxy/templates/haproxy_couchdb.cfg.erb deleted file mode 100644 index 1fa01b96..00000000 --- a/puppet/modules/site_haproxy/templates/haproxy_couchdb.cfg.erb +++ /dev/null @@ -1,23 +0,0 @@ - -listen bigcouch-in - mode http - balance roundrobin - option httplog - option dontlognull - option httpchk GET / # health check using simple get to root - option http-server-close # use client keep-alive, but close server connection. - option allbackups # balance among all backups, not just one. - - bind localhost:4096 - - default-server inter 3000 fastinter 1000 downinter 1000 rise 2 fall 1 - -<%- if @haproxy['servers'] -%> -<%- @haproxy['servers'].sort.each do |name,server| -%> -<%- backup = server['backup'] ? 'backup' : '' -%> - # <%=name%> - server couchdb_<%=server['port']%> <%=server['host']%>:<%=server['port']%> <%=backup%> weight <%=server['weight']%> check - -<%- end -%> -<%- end -%> - -- cgit v1.2.3 From c20aa4f8c35a4cba982de92105da2566ecdfa1ae Mon Sep 17 00:00:00 2001 From: elijah Date: Sun, 22 Jun 2014 13:42:04 -0700 Subject: run_tests: allow for https in assert_get() --- bin/run_tests | 17 ++++++++++++----- tests/white-box/webapp.rb | 9 +++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/bin/run_tests b/bin/run_tests index 526aa83a..3ba89684 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -127,11 +127,18 @@ class LeapTest < MiniTest::Unit::TestCase if params uri.query = URI.encode_www_form(params) end - response = Net::HTTP.get_response(uri) - if response.is_a?(Net::HTTPSuccess) - yield response.body, response, nil - else - yield nil, response, nil + http = Net::HTTP.new uri.host, uri.port + if uri.scheme == 'https' + http.verify_mode = OpenSSL::SSL::VERIFY_NONE + http.use_ssl = true + end + http.start do |agent| + response = agent.get(uri.request_uri) + if response.is_a?(Net::HTTPSuccess) + yield response.body, response, nil + else + yield nil, response, nil + end end rescue => exc yield nil, nil, exc diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb index 142ac2de..05b86a41 100644 --- a/tests/white-box/webapp.rb +++ b/tests/white-box/webapp.rb @@ -60,4 +60,13 @@ class Webapp < LeapTest pass end + # + # this is technically a black-box test. so, move this when we have support + # for black box tests. + # + def test_04_Can_access_webapp? + assert_get('https://' + $node['webapp']['domain'] + '/') + pass + end + end -- cgit v1.2.3 From 10c76ac1cfa4e94375b456266e4113a4313abe96 Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 23 Jun 2014 02:40:21 -0700 Subject: tests: fixed problem with showing couchdb password in process table, and adding warnings for when ACL is not being respected (which is currently always). closes #5445 --- bin/run_tests | 23 ++++++++++++++++++++--- tests/white-box/couchdb.rb | 35 +++++++++++++++++++++++++++-------- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/bin/run_tests b/bin/run_tests index 3ba89684..d7fc1e4c 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -122,7 +122,7 @@ class LeapTest < MiniTest::Unit::TestCase # # attempts a http GET on the url, yields |body, response, error| # - def get(url, params=nil) + def get(url, params=nil, options=nil) uri = URI(url) if params uri.query = URI.encode_www_form(params) @@ -133,7 +133,11 @@ class LeapTest < MiniTest::Unit::TestCase http.use_ssl = true end http.start do |agent| - response = agent.get(uri.request_uri) + request = Net::HTTP::Get.new uri.request_uri + if options && options[:username] + request.basic_auth options[:username], options[:password] + end + response = agent.request(request) if response.is_a?(Net::HTTPSuccess) yield response.body, response, nil else @@ -146,7 +150,7 @@ class LeapTest < MiniTest::Unit::TestCase def assert_get(url, params=nil, options=nil) options ||= {} - get(url, params) do |body, response, error| + get(url, params, options) do |body, response, error| if body yield body if block_given? elsif response @@ -157,6 +161,19 @@ class LeapTest < MiniTest::Unit::TestCase end end + # + # only a warning for now, should be a failure in the future + # + def assert_auth_fail(url, params, auth_options) + get(url, params, auth_options) do |body, response, error| + unless response.code == 401 + warn "Expected a '401 Unauthorized' response, but got #{response.code} instead (GET #{url} with username '#{auth_options[:username]}')." + return false + end + end + true + end + # # test if a socket can be connected to # diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb index 9d5da94f..8b9789bd 100644 --- a/tests/white-box/couchdb.rb +++ b/tests/white-box/couchdb.rb @@ -33,7 +33,7 @@ class CouchDB < LeapTest neighbors = assert_property('couch.bigcouch.neighbors') neighbors << assert_property('domain.full') neighbors.sort! - assert_get(url) do |body| + assert_get(url, nil, http_basic_auth) do |body| response = JSON.parse(body) nodes_in_db = response['rows'].collect{|row| row['id'].sub(/^bigcouch@/, '')}.sort assert_equal neighbors, nodes_in_db, "The couchdb replication node list is wrong (/nodes/_all_docs)" @@ -49,7 +49,7 @@ class CouchDB < LeapTest # def test_03_Are_configured_nodes_online? url = couchdb_url("/_membership") - assert_get(url) do |body| + assert_get(url, nil, http_basic_auth) do |body| response = JSON.parse(body) nodes_configured_but_not_available = response['cluster_nodes'] - response['all_nodes'] nodes_available_but_not_configured = response['all_nodes'] - response['cluster_nodes'] @@ -68,7 +68,7 @@ class CouchDB < LeapTest def test_04_Do_ACL_users_exist? acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'tapicero', 'webapp'] url = couchdb_backend_url("/_users/_all_docs") - assert_get(url) do |body| + assert_get(url, nil, http_basic_auth) do |body| response = JSON.parse(body) assert_equal 6, response['total_rows'] actual_users = response['rows'].map{|row| row['id'].sub(/^org.couchdb.user:/, '') } @@ -88,6 +88,23 @@ class CouchDB < LeapTest pass end + # + # for now, this just prints warnings, since we are failing these tests. + # + def test_06_Is_ACL_enforced? + ok = assert_auth_fail( + couchdb_url('/users/_all_docs'), + {:limit => 1}, + http_basic_auth('leap_mx') + ) + ok = assert_auth_fail( + couchdb_url('/users/_all_docs'), + {:limit => 1}, + {} + ) && ok + pass if ok + end + private def couchdb_url(path="", port=nil) @@ -95,15 +112,17 @@ class CouchDB < LeapTest assert_property 'couch.port' $node['couch']['port'] end - @password ||= begin - assert_property 'couch.users.admin.password' - $node['couch']['users']['admin']['password'] - end - "http://admin:#{@password}@localhost:#{port || @port}#{path}" + "http://localhost:#{port || @port}#{path}" end def couchdb_backend_url(path="") couchdb_url(path, "5986") # TODO: admin port is hardcoded for now but should be configurable. end + def http_basic_auth(username='admin') + assert_property 'couch.users.' + username + password = $node['couch']['users'][username]['password'] + {:username => username, :password => password} + end + end -- cgit v1.2.3 From 0accf1788b9c7fef05e4436c5015d9099451be05 Mon Sep 17 00:00:00 2001 From: Azul Date: Mon, 23 Jun 2014 21:49:38 +0200 Subject: minor: fix typo in replication user roles --- puppet/modules/site_couchdb/manifests/add_users.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_couchdb/manifests/add_users.pp b/puppet/modules/site_couchdb/manifests/add_users.pp index 0585da27..2f734ed4 100644 --- a/puppet/modules/site_couchdb/manifests/add_users.pp +++ b/puppet/modules/site_couchdb/manifests/add_users.pp @@ -57,7 +57,7 @@ class site_couchdb::add_users { ## replication couchdb user ## read/write: all databases for replication couchdb::add_user { $site_couchdb::couchdb_replication_user: - roles => '["repliction"]', + roles => '["replication"]', pw => $site_couchdb::couchdb_replication_pw, salt => $site_couchdb::couchdb_replication_salt, require => Couchdb::Query::Setup['localhost'] -- cgit v1.2.3 From 9cf230751b824d9134e710739520bc8deda14e8e Mon Sep 17 00:00:00 2001 From: Azul Date: Mon, 23 Jun 2014 21:50:50 +0200 Subject: hand replication credentials to tapicero --- provider_base/services/_couchdb_mirror.json | 7 +++++-- puppet/modules/tapicero/manifests/init.pp | 3 ++- puppet/modules/tapicero/templates/tapicero.yaml.erb | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/provider_base/services/_couchdb_mirror.json b/provider_base/services/_couchdb_mirror.json index a496804d..6a3402bd 100644 --- a/provider_base/services/_couchdb_mirror.json +++ b/provider_base/services/_couchdb_mirror.json @@ -12,7 +12,10 @@ "replication": { // for now, pick the first close one, or the first one. // in the future, maybe use haproxy to balance among all the masters - "masters": "= try{pick_node(:couch_master,nodes_near_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')} || try{pick_node(:couch_master,nodes_like_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')}" + "masters": "= try{pick_node(:couch_master,nodes_near_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')} || try{pick_node(:couch_master,nodes_like_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')}", + "username": "replication", + "password": "= secret :couch_replication_password", + "role": "replication" } } -} \ No newline at end of file +} diff --git a/puppet/modules/tapicero/manifests/init.pp b/puppet/modules/tapicero/manifests/init.pp index 1db75eb0..fd8c1344 100644 --- a/puppet/modules/tapicero/manifests/init.pp +++ b/puppet/modules/tapicero/manifests/init.pp @@ -12,7 +12,8 @@ class tapicero { $couchdb_soledad_user = $couchdb_users['soledad']['username'] $couchdb_leap_mx_user = $couchdb_users['leap_mx']['username'] - $couchdb_mirror = $couchdb['mode'] == 'mirror' + $couchdb_mode = $couchdb['mode'] + $couchdb_replication = $couchdb['replication'] Class['site_config::default'] -> Class['tapicero'] diff --git a/puppet/modules/tapicero/templates/tapicero.yaml.erb b/puppet/modules/tapicero/templates/tapicero.yaml.erb index 3a5f821e..182a6aa6 100644 --- a/puppet/modules/tapicero/templates/tapicero.yaml.erb +++ b/puppet/modules/tapicero/templates/tapicero.yaml.erb @@ -24,7 +24,8 @@ log_level: info options: # prefix for per user databases: db_prefix: "user-" - mirror: <%= @couchdb_mirror %> + mode: <%= @couchdb_mode %> + replication: <%= @couchdb_replication %> # security settings to be used for the per user databases security: -- cgit v1.2.3 From 04d1369ea78c404907dea4aca758e08efe3a9cdd Mon Sep 17 00:00:00 2001 From: Azul Date: Mon, 23 Jun 2014 22:04:59 +0200 Subject: create netrc files for all users with new puppet_couchdb This only works with the latest patch to puppet_couchdb --- puppet/modules/site_couchdb/manifests/mirror.pp | 2 -- puppet/modules/site_couchdb/manifests/setup.pp | 11 +++++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/puppet/modules/site_couchdb/manifests/mirror.pp b/puppet/modules/site_couchdb/manifests/mirror.pp index df305737..1cbd9bcc 100644 --- a/puppet/modules/site_couchdb/manifests/mirror.pp +++ b/puppet/modules/site_couchdb/manifests/mirror.pp @@ -6,8 +6,6 @@ class site_couchdb::mirror { chttpd_bind_address => '127.0.0.1' } - # Couchdb databases - $masters = $site_couchdb::couchdb_config['replication']['masters'] $master_node_names = keys($site_couchdb::couchdb_config['replication']['masters']) $master_node = $masters[$master_node_names[0]] diff --git a/puppet/modules/site_couchdb/manifests/setup.pp b/puppet/modules/site_couchdb/manifests/setup.pp index e398356b..69bd1c6a 100644 --- a/puppet/modules/site_couchdb/manifests/setup.pp +++ b/puppet/modules/site_couchdb/manifests/setup.pp @@ -10,11 +10,18 @@ class site_couchdb::setup { ensure => absent } - # /etc/couchdb/couchdb.netrc is deployed by couchdb::query::setup + $user = $site_couchdb::couchdb_admin_user + + # /etc/couchdb/couchdb-admin.netrc is deployed by couchdb::query::setup + # we symlink to couchdb.netrc for puppet commands. # we symlink this to /root/.netrc for couchdb_scripts (eg. backup) # and makes life easier for the admin (i.e. using curl/wget without # passing credentials) file { + '/etc/couchdb/couchdb.netrc': + ensure => link, + target => "/etc/couchdb/couchdb-${user}.netrc"; + '/root/.netrc': ensure => link, target => '/etc/couchdb/couchdb.netrc'; @@ -24,7 +31,7 @@ class site_couchdb::setup { } couchdb::query::setup { 'localhost': - user => $site_couchdb::couchdb_admin_user, + user => $user, pw => $site_couchdb::couchdb_admin_pw, } -- cgit v1.2.3 From fba004bc8cbee0d9556538342ce78ac1c9d1229b Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 25 Jun 2014 12:49:39 -0700 Subject: more friendly error message in `leap compile` when commercial certificate is missing. --- provider_base/common.json | 6 +++++- provider_base/services/monitor.json | 6 ++---- provider_base/services/mx.json | 6 ++---- provider_base/services/webapp.json | 8 +++----- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/provider_base/common.json b/provider_base/common.json index a4d9c5f2..565633c0 100644 --- a/provider_base/common.json +++ b/provider_base/common.json @@ -25,9 +25,13 @@ "hosts": "=> hosts_file", "x509": { "use": true, + "use_commercial": false, "cert": "= x509.use ? file(:node_x509_cert, :missing => 'x509 certificate for node $node. Run `leap cert update`') : nil", "key": "= x509.use ? file(:node_x509_key, :missing => 'x509 key for node $node. Run `leap cert update`') : nil", - "ca_cert": "= try_file :ca_cert" + "ca_cert": "= try_file :ca_cert", + "commercial_cert": "= x509.use_commercial ? file([:commercial_cert, webapp.domain], :missing => 'commercial x509 certificate for node $node. Add file $file, or run `leap cert csr` to generate a temporary self-signed cert and CSR you can use to purchase a real cert.') : nil", + "commercial_key": "= x509.use_commercial ? file([:commercial_key, webapp.domain], :missing => 'commercial x509 certificate for node $node. Add file $file, or run `leap cert csr` to generate a temporary self-signed cert and CSR you can use to purchase a real cert.') : nil", + "commercial_ca_cert": "= x509.use_commercial ? try_file(:commercial_ca_cert) : nil" }, "service_type": "internal_service", "development": { diff --git a/provider_base/services/monitor.json b/provider_base/services/monitor.json index 03f6c6d1..c24724bf 100644 --- a/provider_base/services/monitor.json +++ b/provider_base/services/monitor.json @@ -12,11 +12,9 @@ }, "x509": { "use": true, + "use_commercial": true, "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", - "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'", - "commercial_cert": "= file [:commercial_cert, domain.full_suffix]", - "commercial_key": "= file [:commercial_key, domain.full_suffix]", - "commercial_ca_cert": "= try_file :commercial_ca_cert" + "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'" } } diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json index 731dee9a..30a19d9a 100644 --- a/provider_base/services/mx.json +++ b/provider_base/services/mx.json @@ -13,12 +13,10 @@ "mynetworks": "= nodes['environment' => '!local'].map{|name, n| [n.ip_address, (global.facts[name]||{})['ec2_public_ipv4']]}.flatten.compact.uniq", "x509": { "use": true, + "use_commercial": true, "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", - "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'", - "commercial_cert": "= file [:commercial_cert, domain.full_suffix]", - "commercial_key": "= file [:commercial_key, domain.full_suffix]", - "commercial_ca_cert": "= try_file :commercial_ca_cert" + "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'" }, "service_type": "user_service" } diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index a5b1ed30..d268a020 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -62,11 +62,9 @@ }, "x509": { "use": true, + "use_commercial": true, "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", - "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", - "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'", - "commercial_cert": "= file [:commercial_cert, webapp.domain]", - "commercial_key": "= file [:commercial_key, webapp.domain]", - "commercial_ca_cert": "= try_file :commercial_ca_cert" + "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`.'", + "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`.'" } } -- cgit v1.2.3 From a9e85045bb312db3dff560c1ca1f40d669cd71bf Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 25 Jun 2014 15:22:02 -0700 Subject: fix commercial cert usage with mx and monitor nodes. --- provider_base/common.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/provider_base/common.json b/provider_base/common.json index 565633c0..1f0c6730 100644 --- a/provider_base/common.json +++ b/provider_base/common.json @@ -29,8 +29,8 @@ "cert": "= x509.use ? file(:node_x509_cert, :missing => 'x509 certificate for node $node. Run `leap cert update`') : nil", "key": "= x509.use ? file(:node_x509_key, :missing => 'x509 key for node $node. Run `leap cert update`') : nil", "ca_cert": "= try_file :ca_cert", - "commercial_cert": "= x509.use_commercial ? file([:commercial_cert, webapp.domain], :missing => 'commercial x509 certificate for node $node. Add file $file, or run `leap cert csr` to generate a temporary self-signed cert and CSR you can use to purchase a real cert.') : nil", - "commercial_key": "= x509.use_commercial ? file([:commercial_key, webapp.domain], :missing => 'commercial x509 certificate for node $node. Add file $file, or run `leap cert csr` to generate a temporary self-signed cert and CSR you can use to purchase a real cert.') : nil", + "commercial_cert": "= x509.use_commercial ? file([:commercial_cert, try{webapp.domain}||domain.full_suffix], :missing => 'commercial x509 certificate for node $node. Add file $file, or run `leap cert csr` to generate a temporary self-signed cert and CSR you can use to purchase a real cert.') : nil", + "commercial_key": "= x509.use_commercial ? file([:commercial_key, try{webapp.domain}||domain.full_suffix], :missing => 'commercial x509 certificate for node $node. Add file $file, or run `leap cert csr` to generate a temporary self-signed cert and CSR you can use to purchase a real cert.') : nil", "commercial_ca_cert": "= x509.use_commercial ? try_file(:commercial_ca_cert) : nil" }, "service_type": "internal_service", -- cgit v1.2.3 From edf1306a418203df297d034544d51f5cdcd053c7 Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 25 Jun 2014 17:21:49 -0700 Subject: run_tests: clean up assert_get() --- bin/run_tests | 17 +++++++++-------- tests/white-box/couchdb.rb | 42 +++++++++++++++++++++++------------------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/bin/run_tests b/bin/run_tests index d7fc1e4c..2ee027f4 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -122,7 +122,7 @@ class LeapTest < MiniTest::Unit::TestCase # # attempts a http GET on the url, yields |body, response, error| # - def get(url, params=nil, options=nil) + def get(url, params=nil) uri = URI(url) if params uri.query = URI.encode_www_form(params) @@ -134,8 +134,8 @@ class LeapTest < MiniTest::Unit::TestCase end http.start do |agent| request = Net::HTTP::Get.new uri.request_uri - if options && options[:username] - request.basic_auth options[:username], options[:password] + if uri.user + request.basic_auth uri.user, uri.password end response = agent.request(request) if response.is_a?(Net::HTTPSuccess) @@ -150,7 +150,7 @@ class LeapTest < MiniTest::Unit::TestCase def assert_get(url, params=nil, options=nil) options ||= {} - get(url, params, options) do |body, response, error| + get(url, params) do |body, response, error| if body yield body if block_given? elsif response @@ -164,10 +164,11 @@ class LeapTest < MiniTest::Unit::TestCase # # only a warning for now, should be a failure in the future # - def assert_auth_fail(url, params, auth_options) - get(url, params, auth_options) do |body, response, error| - unless response.code == 401 - warn "Expected a '401 Unauthorized' response, but got #{response.code} instead (GET #{url} with username '#{auth_options[:username]}')." + def assert_auth_fail(url, params) + uri = URI(url) + get(url, params) do |body, response, error| + unless response.code.to_s == "401" + warn "Expected a '401 Unauthorized' response, but got #{response.code} instead (GET #{uri.request_uri} with username '#{uri.user}')." return false end end diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb index 8b9789bd..a6ad0f6a 100644 --- a/tests/white-box/couchdb.rb +++ b/tests/white-box/couchdb.rb @@ -33,7 +33,7 @@ class CouchDB < LeapTest neighbors = assert_property('couch.bigcouch.neighbors') neighbors << assert_property('domain.full') neighbors.sort! - assert_get(url, nil, http_basic_auth) do |body| + assert_get(url) do |body| response = JSON.parse(body) nodes_in_db = response['rows'].collect{|row| row['id'].sub(/^bigcouch@/, '')}.sort assert_equal neighbors, nodes_in_db, "The couchdb replication node list is wrong (/nodes/_all_docs)" @@ -48,8 +48,8 @@ class CouchDB < LeapTest # this seems backward to me, so it might be the other way around. # def test_03_Are_configured_nodes_online? - url = couchdb_url("/_membership") - assert_get(url, nil, http_basic_auth) do |body| + url = couchdb_url("/_membership", :user => 'admin') + assert_get(url) do |body| response = JSON.parse(body) nodes_configured_but_not_available = response['cluster_nodes'] - response['all_nodes'] nodes_available_but_not_configured = response['all_nodes'] - response['cluster_nodes'] @@ -68,7 +68,7 @@ class CouchDB < LeapTest def test_04_Do_ACL_users_exist? acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'tapicero', 'webapp'] url = couchdb_backend_url("/_users/_all_docs") - assert_get(url, nil, http_basic_auth) do |body| + assert_get(url) do |body| response = JSON.parse(body) assert_equal 6, response['total_rows'] actual_users = response['rows'].map{|row| row['id'].sub(/^org.couchdb.user:/, '') } @@ -93,36 +93,40 @@ class CouchDB < LeapTest # def test_06_Is_ACL_enforced? ok = assert_auth_fail( - couchdb_url('/users/_all_docs'), - {:limit => 1}, - http_basic_auth('leap_mx') + couchdb_url('/users/_all_docs', :user => 'leap_mx'), + {:limit => 1} ) ok = assert_auth_fail( - couchdb_url('/users/_all_docs'), - {:limit => 1}, - {} + couchdb_url('/users/_all_docs', :user => 'leap_mx'), + {:limit => 1} ) && ok pass if ok end + def test_07_What? + pass + end + private - def couchdb_url(path="", port=nil) + def couchdb_url(path="", options=nil) + options||={} @port ||= begin assert_property 'couch.port' $node['couch']['port'] end - "http://localhost:#{port || @port}#{path}" + url = 'http://' + if options[:user] + assert_property 'couch.users.' + options[:user] + password = $node['couch']['users'][options[:user]]['password'] + url += "%s:%s@" % [options[:user], password] + end + url += "localhost:#{options[:port] || @port}#{path}" + url end def couchdb_backend_url(path="") - couchdb_url(path, "5986") # TODO: admin port is hardcoded for now but should be configurable. - end - - def http_basic_auth(username='admin') - assert_property 'couch.users.' + username - password = $node['couch']['users'][username]['password'] - {:username => username, :password => password} + couchdb_url(path, :port => "5986") # TODO: admin port is hardcoded for now but should be configurable. end end -- cgit v1.2.3 From db669a6911c55d9a5675fb4b42f4de5728f34c76 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 19 Jun 2014 00:02:54 -0700 Subject: couchdb: generate hiera files suitable for plain couchdb + read-only mirrors --- platform.rb | 2 +- provider_base/services/_couchdb_master.json | 8 ++++++++ provider_base/services/_couchdb_mirror.json | 16 ++++++++++++++++ provider_base/services/_couchdb_multimaster.json | 20 ++++++++++++++++++++ provider_base/services/couchdb.json | 17 ++++------------- provider_base/services/couchdb.rb | 18 ++++++++++++++++++ 6 files changed, 67 insertions(+), 14 deletions(-) create mode 100644 provider_base/services/_couchdb_master.json create mode 100644 provider_base/services/_couchdb_mirror.json create mode 100644 provider_base/services/_couchdb_multimaster.json create mode 100644 provider_base/services/couchdb.rb diff --git a/platform.rb b/platform.rb index cd0cbde0..ac79df65 100644 --- a/platform.rb +++ b/platform.rb @@ -5,7 +5,7 @@ Leap::Platform.define do self.version = "0.5.2" - self.compatible_cli = "1.5.5".."1.99" + self.compatible_cli = "1.5.7".."1.99" # # the facter facts that should be gathered diff --git a/provider_base/services/_couchdb_master.json b/provider_base/services/_couchdb_master.json new file mode 100644 index 00000000..20c6f99b --- /dev/null +++ b/provider_base/services/_couchdb_master.json @@ -0,0 +1,8 @@ +// +// Applied to master couchdb node when there is a single master +// +{ + "couch": { + "mode": "master" + } +} \ No newline at end of file diff --git a/provider_base/services/_couchdb_mirror.json b/provider_base/services/_couchdb_mirror.json new file mode 100644 index 00000000..67004c70 --- /dev/null +++ b/provider_base/services/_couchdb_mirror.json @@ -0,0 +1,16 @@ +// +// Applied to all non-master couchdb nodes +// +{ + "stunnel": { + "couch_client": "= stunnel_client(nodes[couch.replication.masters.keys], couch.port)" + }, + "couch": { + "mode": "mirror", + "replication": { + // for now, pick the first close one, or the first one. + // in the future, maybe use haproxy to balance among all the masters + "masters": "= try{pick_node(:couch_master,nodes_near_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal')} || try{pick_node(:couch_master,nodes_like_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal')}" + } + } +} \ No newline at end of file diff --git a/provider_base/services/_couchdb_multimaster.json b/provider_base/services/_couchdb_multimaster.json new file mode 100644 index 00000000..ff133b9c --- /dev/null +++ b/provider_base/services/_couchdb_multimaster.json @@ -0,0 +1,20 @@ +// +// Only applied to master couchdb nodes when there are multiple masters +// +{ + "stunnel": { + "epmd_server": "= stunnel_server(couch.bigcouch.epmd_port)", + "epmd_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], couch.bigcouch.epmd_port)", + "ednp_server": "= stunnel_server(couch.bigcouch.ednp_port)", + "ednp_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], couch.bigcouch.ednp_port)" + }, + "couch": { + "mode": "multimaster", + "bigcouch": { + "epmd_port": 4369, + "ednp_port": 9002, + "cookie": "= secret :bigcouch_cookie", + "neighbors": "= nodes_like_me['services' => 'couchdb']['couch.master' => true].exclude(self).field('domain.full')" + } + } +} diff --git a/provider_base/services/couchdb.json b/provider_base/services/couchdb.json index 5f1b5381..d75fd8de 100644 --- a/provider_base/services/couchdb.json +++ b/provider_base/services/couchdb.json @@ -3,20 +3,11 @@ "use": true }, "stunnel": { - "couch_server": "= stunnel_server(couch.port)", - "epmd_server": "= stunnel_server(couch.bigcouch.epmd_port)", - "epmd_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.bigcouch.epmd_port)", - "ednp_server": "= stunnel_server(couch.bigcouch.ednp_port)", - "ednp_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.bigcouch.ednp_port)" + "couch_server": "= stunnel_server(couch.port)" }, "couch": { + "master": false, "port": 5984, - "bigcouch": { - "epmd_port": 4369, - "ednp_port": 9002, - "cookie": "= secret :bigcouch_cookie", - "neighbors": "= nodes_like_me[:services => :couchdb].exclude(self).field('domain.full')" - }, "users": { "admin": { "username": "admin", @@ -49,8 +40,8 @@ "salt": "= hex_secret :couch_webapp_password_salt, 128" } }, - "webapp": { - "nagios_test_pw": "= secret :nagios_test_password" + "webapp": { + "nagios_test_pw": "= secret :nagios_test_password" } } } diff --git a/provider_base/services/couchdb.rb b/provider_base/services/couchdb.rb new file mode 100644 index 00000000..c8f5d8a7 --- /dev/null +++ b/provider_base/services/couchdb.rb @@ -0,0 +1,18 @@ +# +# custom logic for couchdb json resolution +# + +unless nodes_like_me['services' => 'couchdb']['couch.master' => true].any? + raise 'node `%s`, environment `%s`: there must be at least one node with couch.master set to `true` for this environment.' % [@node.name, @node.environment] +end + +if couch.master + if nodes_like_me['services' => 'couchdb']['couch.master' => true].size > 1 + apply_partial 'services/_couchdb_multimaster.json' + else + apply_partial 'services/_couchdb_master.json' + end +else + apply_partial 'services/_couchdb_mirror.json' +end + -- cgit v1.2.3 From 324dce6b8c6a911701fd4a4a7b383f22cc810c9a Mon Sep 17 00:00:00 2001 From: Azul Date: Thu, 19 Jun 2014 12:11:14 +0200 Subject: split bigcouch stunnel from plain couch stunnel --- .../site_couchdb/manifests/bigcouch/stunnel.pp | 89 ++++++++++++++++++++++ puppet/modules/site_couchdb/manifests/stunnel.pp | 81 ++------------------ 2 files changed, 95 insertions(+), 75 deletions(-) create mode 100644 puppet/modules/site_couchdb/manifests/bigcouch/stunnel.pp diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/stunnel.pp b/puppet/modules/site_couchdb/manifests/bigcouch/stunnel.pp new file mode 100644 index 00000000..5166ba93 --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/bigcouch/stunnel.pp @@ -0,0 +1,89 @@ +class site_couchdb::bigcouch::stunnel { + + $stunnel = hiera('stunnel') + + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca + + include x509::variables + $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" + $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" + $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" + + + # Erlang Port Mapper Daemon (epmd) stunnel server/clients + $epmd_server = $stunnel['epmd_server'] + $epmd_server_accept = $epmd_server['accept'] + $epmd_server_connect = $epmd_server['connect'] + $epmd_clients = $stunnel['epmd_clients'] + + # Erlang Distributed Node Protocol (ednp) stunnel server/clients + $ednp_server = $stunnel['ednp_server'] + $ednp_server_accept = $ednp_server['accept'] + $ednp_server_connect = $ednp_server['connect'] + $ednp_clients = $stunnel['ednp_clients'] + + + # setup stunnel server for Erlang Port Mapper Daemon (epmd), necessary for + # bigcouch clustering between each bigcouchdb node + stunnel::service { 'epmd_server': + accept => $epmd_server_accept, + connect => $epmd_server_connect, + client => false, + cafile => $ca_path, + key => $key_path, + cert => $cert_path, + verify => '2', + pid => '/var/run/stunnel4/epmd_server.pid', + rndfile => '/var/lib/stunnel4/.rnd', + debuglevel => '4', + require => [ + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Ca'] ]; + } + + # setup stunnel clients for Erlang Port Mapper Daemon (epmd) to connect + # to the above epmd stunnel server. + $epmd_client_defaults = { + 'client' => true, + 'cafile' => $ca_path, + 'key' => $key_path, + 'cert' => $cert_path, + } + + create_resources(site_stunnel::clients, $epmd_clients, $epmd_client_defaults) + + # setup stunnel server for Erlang Distributed Node Protocol (ednp), necessary + # for bigcouch clustering between each bigcouchdb node + stunnel::service { 'ednp_server': + accept => $ednp_server_accept, + connect => $ednp_server_connect, + client => false, + cafile => $ca_path, + key => $key_path, + cert => $cert_path, + verify => '2', + pid => '/var/run/stunnel4/ednp_server.pid', + rndfile => '/var/lib/stunnel4/.rnd', + debuglevel => '4', + require => [ + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Ca'] ]; + } + + # setup stunnel clients for Erlang Distributed Node Protocol (ednp) to connect + # to the above ednp stunnel server. + $ednp_client_defaults = { + 'client' => true, + 'cafile' => $ca_path, + 'key' => $key_path, + 'cert' => $cert_path, + } + + create_resources(site_stunnel::clients, $ednp_clients, $ednp_client_defaults) + + include site_check_mk::agent::stunnel +} diff --git a/puppet/modules/site_couchdb/manifests/stunnel.pp b/puppet/modules/site_couchdb/manifests/stunnel.pp index 91f1e3aa..484a0c00 100644 --- a/puppet/modules/site_couchdb/manifests/stunnel.pp +++ b/puppet/modules/site_couchdb/manifests/stunnel.pp @@ -1,29 +1,21 @@ class site_couchdb::stunnel { $stunnel = hiera('stunnel') + $couchdb_config = hiera('couch') + $couchdb_bigcouch = $couchdb_config['mode'] == "multimaster" $couch_server = $stunnel['couch_server'] $couch_server_accept = $couch_server['accept'] $couch_server_connect = $couch_server['connect'] - # Erlang Port Mapper Daemon (epmd) stunnel server/clients - $epmd_server = $stunnel['epmd_server'] - $epmd_server_accept = $epmd_server['accept'] - $epmd_server_connect = $epmd_server['connect'] - $epmd_clients = $stunnel['epmd_clients'] - - # Erlang Distributed Node Protocol (ednp) stunnel server/clients - $ednp_server = $stunnel['ednp_server'] - $ednp_server_accept = $ednp_server['accept'] - $ednp_server_connect = $ednp_server['connect'] - $ednp_clients = $stunnel['ednp_clients'] - - - include site_config::x509::cert include site_config::x509::key include site_config::x509::ca + if $couchdb_bigcouch { + include site_couchdb::bigcouch::stunnel + } + include x509::variables $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" @@ -47,66 +39,5 @@ class site_couchdb::stunnel { Class['Site_config::X509::Ca'] ]; } - - # setup stunnel server for Erlang Port Mapper Daemon (epmd), necessary for - # bigcouch clustering between each bigcouchdb node - stunnel::service { 'epmd_server': - accept => $epmd_server_accept, - connect => $epmd_server_connect, - client => false, - cafile => $ca_path, - key => $key_path, - cert => $cert_path, - verify => '2', - pid => '/var/run/stunnel4/epmd_server.pid', - rndfile => '/var/lib/stunnel4/.rnd', - debuglevel => '4', - require => [ - Class['Site_config::X509::Key'], - Class['Site_config::X509::Cert'], - Class['Site_config::X509::Ca'] ]; - } - - # setup stunnel clients for Erlang Port Mapper Daemon (epmd) to connect - # to the above epmd stunnel server. - $epmd_client_defaults = { - 'client' => true, - 'cafile' => $ca_path, - 'key' => $key_path, - 'cert' => $cert_path, - } - - create_resources(site_stunnel::clients, $epmd_clients, $epmd_client_defaults) - - # setup stunnel server for Erlang Distributed Node Protocol (ednp), necessary - # for bigcouch clustering between each bigcouchdb node - stunnel::service { 'ednp_server': - accept => $ednp_server_accept, - connect => $ednp_server_connect, - client => false, - cafile => $ca_path, - key => $key_path, - cert => $cert_path, - verify => '2', - pid => '/var/run/stunnel4/ednp_server.pid', - rndfile => '/var/lib/stunnel4/.rnd', - debuglevel => '4', - require => [ - Class['Site_config::X509::Key'], - Class['Site_config::X509::Cert'], - Class['Site_config::X509::Ca'] ]; - } - - # setup stunnel clients for Erlang Distributed Node Protocol (ednp) to connect - # to the above ednp stunnel server. - $ednp_client_defaults = { - 'client' => true, - 'cafile' => $ca_path, - 'key' => $key_path, - 'cert' => $cert_path, - } - - create_resources(site_stunnel::clients, $ednp_clients, $ednp_client_defaults) - include site_check_mk::agent::stunnel } -- cgit v1.2.3 From d2f59e4cfab5b9fd164d24416b241b14ecfd9307 Mon Sep 17 00:00:00 2001 From: Azul Date: Thu, 19 Jun 2014 12:29:30 +0200 Subject: separate bigcouch specifics from init.pp --- puppet/modules/site_couchdb/manifests/bigcouch.pp | 20 ++++++++++++++++++++ puppet/modules/site_couchdb/manifests/init.pp | 21 ++++----------------- 2 files changed, 24 insertions(+), 17 deletions(-) create mode 100644 puppet/modules/site_couchdb/manifests/bigcouch.pp diff --git a/puppet/modules/site_couchdb/manifests/bigcouch.pp b/puppet/modules/site_couchdb/manifests/bigcouch.pp new file mode 100644 index 00000000..a3f6db2c --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/bigcouch.pp @@ -0,0 +1,20 @@ +class site_couchdb::bigcouch { + + $bigcouch_config = $couchdb_config['bigcouch'] + $bigcouch_cookie = $bigcouch_config['cookie'] + + $ednp_port = $bigcouch_config['ednp_port'] + + Class['site_config::default'] + -> Class['site_couchdb::bigcouch::add_nodes'] + -> Class['site_couchdb::bigcouch::settle_cluster'] + + include site_couchdb::bigcouch::add_nodes + include site_couchdb::bigcouch::settle_cluster + include site_couchdb::bigcouch::compaction + include site_shorewall::couchdb::bigcouch + + file { '/var/log/bigcouch': + ensure => directory + } +} diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 3614661d..22d6ef45 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -35,14 +35,10 @@ class site_couchdb { $couchdb_webapp_salt = $couchdb_webapp['salt'] $couchdb_backup = $couchdb_config['backup'] - - $bigcouch_config = $couchdb_config['bigcouch'] - $bigcouch_cookie = $bigcouch_config['cookie'] - - $ednp_port = $bigcouch_config['ednp_port'] + $couchdb_bigcouch = $couchdb_config['mode'] == "multimaster" class { 'couchdb': - bigcouch => true, + bigcouch => $couchdb_bigcouch, admin_pw => $couchdb_admin_pw, admin_salt => $couchdb_admin_salt, bigcouch_cookie => $bigcouch_cookie, @@ -63,8 +59,6 @@ class site_couchdb { -> Class['site_couchdb::stunnel'] -> Service['couchdb'] -> File['/root/.netrc'] - -> Class['site_couchdb::bigcouch::add_nodes'] - -> Class['site_couchdb::bigcouch::settle_cluster'] -> Class['site_couchdb::create_dbs'] -> Class['site_couchdb::add_users'] @@ -95,24 +89,17 @@ class site_couchdb { } include site_couchdb::stunnel - include site_couchdb::bigcouch::add_nodes - include site_couchdb::bigcouch::settle_cluster include site_couchdb::create_dbs include site_couchdb::add_users include site_couchdb::designs include site_couchdb::logrotate - include site_couchdb::bigcouch::compaction - if $couchdb_backup { include site_couchdb::backup } + if $couchdb_bigcouch { include site_couchdb::bigcouch } + if $couchdb_backup { include site_couchdb::backup } include site_shorewall::couchdb - include site_shorewall::couchdb::bigcouch include site_check_mk::agent::couchdb include site_check_mk::agent::tapicero - file { '/var/log/bigcouch': - ensure => directory - } - } -- cgit v1.2.3 From 7ee86658b0655ded592eecbaa8b1c5b841d8f846 Mon Sep 17 00:00:00 2001 From: Azul Date: Thu, 19 Jun 2014 20:01:20 +0200 Subject: set mirror option if we are on a couch mirror --- puppet/modules/tapicero/manifests/init.pp | 1 + puppet/modules/tapicero/templates/tapicero.yaml.erb | 2 ++ 2 files changed, 3 insertions(+) diff --git a/puppet/modules/tapicero/manifests/init.pp b/puppet/modules/tapicero/manifests/init.pp index af1a96ac..1db75eb0 100644 --- a/puppet/modules/tapicero/manifests/init.pp +++ b/puppet/modules/tapicero/manifests/init.pp @@ -12,6 +12,7 @@ class tapicero { $couchdb_soledad_user = $couchdb_users['soledad']['username'] $couchdb_leap_mx_user = $couchdb_users['leap_mx']['username'] + $couchdb_mirror = $couchdb['mode'] == 'mirror' Class['site_config::default'] -> Class['tapicero'] diff --git a/puppet/modules/tapicero/templates/tapicero.yaml.erb b/puppet/modules/tapicero/templates/tapicero.yaml.erb index 8e19b22f..3a5f821e 100644 --- a/puppet/modules/tapicero/templates/tapicero.yaml.erb +++ b/puppet/modules/tapicero/templates/tapicero.yaml.erb @@ -24,6 +24,7 @@ log_level: info options: # prefix for per user databases: db_prefix: "user-" + mirror: <%= @couchdb_mirror %> # security settings to be used for the per user databases security: @@ -40,3 +41,4 @@ options: - <%= @couchdb_leap_mx_user %> roles: [] + -- cgit v1.2.3 From 6df59b9f579134a9521aafb71727a98fdc92e19a Mon Sep 17 00:00:00 2001 From: Azul Date: Thu, 19 Jun 2014 20:02:02 +0200 Subject: first steps towards mirroring couch --- puppet/modules/site_couchdb/manifests/bigcouch.pp | 6 +-- .../site_couchdb/manifests/bigcouch/add_nodes.pp | 2 +- puppet/modules/site_couchdb/manifests/init.pp | 6 ++- puppet/modules/site_couchdb/manifests/mirror.pp | 61 ++++++++++++++++++++++ 4 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 puppet/modules/site_couchdb/manifests/mirror.pp diff --git a/puppet/modules/site_couchdb/manifests/bigcouch.pp b/puppet/modules/site_couchdb/manifests/bigcouch.pp index a3f6db2c..97c8cd12 100644 --- a/puppet/modules/site_couchdb/manifests/bigcouch.pp +++ b/puppet/modules/site_couchdb/manifests/bigcouch.pp @@ -1,9 +1,9 @@ class site_couchdb::bigcouch { - $bigcouch_config = $couchdb_config['bigcouch'] - $bigcouch_cookie = $bigcouch_config['cookie'] + $config = $::site_couchdb::couchdb_config['bigcouch'] + $cookie = $config['cookie'] - $ednp_port = $bigcouch_config['ednp_port'] + $ednp_port = $config['ednp_port'] Class['site_config::default'] -> Class['site_couchdb::bigcouch::add_nodes'] diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp b/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp index 97e85785..c8c43275 100644 --- a/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp +++ b/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp @@ -1,6 +1,6 @@ class site_couchdb::bigcouch::add_nodes { # loop through neighbors array and add nodes - $nodes = $::site_couchdb::bigcouch_config['neighbors'] + $nodes = $::site_couchdb::bigcouch::config['neighbors'] couchdb::bigcouch::add_node { $nodes: require => Couchdb::Query::Setup['localhost'] diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 22d6ef45..0b923c9f 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -35,7 +35,7 @@ class site_couchdb { $couchdb_webapp_salt = $couchdb_webapp['salt'] $couchdb_backup = $couchdb_config['backup'] - $couchdb_bigcouch = $couchdb_config['mode'] == "multimaster" + $couchdb_mode = $couchdb_config['mode'] class { 'couchdb': bigcouch => $couchdb_bigcouch, @@ -94,7 +94,9 @@ class site_couchdb { include site_couchdb::designs include site_couchdb::logrotate - if $couchdb_bigcouch { include site_couchdb::bigcouch } + if $couchdb_mode == "multimaster" { include site_couchdb::bigcouch } + if $couchdb_mode == "mirror" { include site_couchdb::mirror } + if $couchdb_backup { include site_couchdb::backup } include site_shorewall::couchdb diff --git a/puppet/modules/site_couchdb/manifests/mirror.pp b/puppet/modules/site_couchdb/manifests/mirror.pp new file mode 100644 index 00000000..708171e4 --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/mirror.pp @@ -0,0 +1,61 @@ +class site_couchdb::mirror { + + # Couchdb databases + + $from = $site_couchdb::couchdb_config['replication']['masters'][0] + + ### customer database + couchdb::mirror_db { 'customers': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## identities database + couchdb::mirror_db { 'identities': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## keycache database + couchdb::mirror_db { 'keycache': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## sessions database + couchdb::mirror_db { 'sessions': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## shared database + couchdb::mirror_db { 'shared': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## tickets database + couchdb::mirror_db { 'tickets': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## tokens database + couchdb::mirror_db { 'tokens': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## users database + couchdb::mirror_db { 'users': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## messages db + couchdb::mirror_db { 'messages': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + +} -- cgit v1.2.3 From 49f0c54a05f6b542367f8ef4538316ba2eaac6cd Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 20 Jun 2014 01:58:39 -0700 Subject: new generic system for stunnel: just `include site_stunnel` and stunnel + needed shorewall will be automatically set up. requires new leap_cli --- platform.rb | 4 +- provider_base/common.json | 4 + provider_base/services/_couchdb_mirror.json | 6 +- provider_base/services/_couchdb_multimaster.json | 12 ++- provider_base/services/couchdb.json | 4 +- .../site_couchdb/manifests/bigcouch/stunnel.pp | 89 ---------------------- puppet/modules/site_couchdb/manifests/stunnel.pp | 43 ----------- puppet/modules/site_shorewall/manifests/couchdb.pp | 24 ------ .../site_shorewall/manifests/couchdb/bigcouch.pp | 51 ------------- .../site_shorewall/manifests/couchdb/dnat.pp | 21 ----- .../site_shorewall/manifests/stunnel/client.pp | 40 ++++++++++ .../site_shorewall/manifests/stunnel/server.pp | 22 ++++++ puppet/modules/site_stunnel/manifests/client.pp | 52 +++++++++++++ puppet/modules/site_stunnel/manifests/clients.pp | 55 ++++++------- puppet/modules/site_stunnel/manifests/init.pp | 15 ++++ puppet/modules/site_stunnel/manifests/servers.pp | 53 +++++++++++++ 16 files changed, 227 insertions(+), 268 deletions(-) delete mode 100644 puppet/modules/site_couchdb/manifests/bigcouch/stunnel.pp delete mode 100644 puppet/modules/site_couchdb/manifests/stunnel.pp delete mode 100644 puppet/modules/site_shorewall/manifests/couchdb.pp delete mode 100644 puppet/modules/site_shorewall/manifests/couchdb/bigcouch.pp delete mode 100644 puppet/modules/site_shorewall/manifests/couchdb/dnat.pp create mode 100644 puppet/modules/site_shorewall/manifests/stunnel/client.pp create mode 100644 puppet/modules/site_shorewall/manifests/stunnel/server.pp create mode 100644 puppet/modules/site_stunnel/manifests/client.pp create mode 100644 puppet/modules/site_stunnel/manifests/servers.pp diff --git a/platform.rb b/platform.rb index ac79df65..872a34cb 100644 --- a/platform.rb +++ b/platform.rb @@ -4,8 +4,8 @@ # Leap::Platform.define do - self.version = "0.5.2" - self.compatible_cli = "1.5.7".."1.99" + self.version = "0.5.3" + self.compatible_cli = "1.5.8".."1.99" # # the facter facts that should be gathered diff --git a/provider_base/common.json b/provider_base/common.json index 1f0c6730..87af2152 100644 --- a/provider_base/common.json +++ b/provider_base/common.json @@ -42,5 +42,9 @@ "enabled": true, "mail": { "smarthost": "= nodes_like_me[:services => :mx].exclude(self).field('domain.full')" + }, + "stunnel": { + "clients": {}, + "servers": {} } } diff --git a/provider_base/services/_couchdb_mirror.json b/provider_base/services/_couchdb_mirror.json index 67004c70..a496804d 100644 --- a/provider_base/services/_couchdb_mirror.json +++ b/provider_base/services/_couchdb_mirror.json @@ -3,14 +3,16 @@ // { "stunnel": { - "couch_client": "= stunnel_client(nodes[couch.replication.masters.keys], couch.port)" + "clients": { + "couch_client": "= stunnel_client(nodes[couch.replication.masters.keys], couch.port)" + } }, "couch": { "mode": "mirror", "replication": { // for now, pick the first close one, or the first one. // in the future, maybe use haproxy to balance among all the masters - "masters": "= try{pick_node(:couch_master,nodes_near_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal')} || try{pick_node(:couch_master,nodes_like_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal')}" + "masters": "= try{pick_node(:couch_master,nodes_near_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')} || try{pick_node(:couch_master,nodes_like_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')}" } } } \ No newline at end of file diff --git a/provider_base/services/_couchdb_multimaster.json b/provider_base/services/_couchdb_multimaster.json index ff133b9c..8c433188 100644 --- a/provider_base/services/_couchdb_multimaster.json +++ b/provider_base/services/_couchdb_multimaster.json @@ -3,10 +3,14 @@ // { "stunnel": { - "epmd_server": "= stunnel_server(couch.bigcouch.epmd_port)", - "epmd_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], couch.bigcouch.epmd_port)", - "ednp_server": "= stunnel_server(couch.bigcouch.ednp_port)", - "ednp_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], couch.bigcouch.ednp_port)" + "servers": { + "epmd_server": "= stunnel_server(couch.bigcouch.epmd_port)", + "ednp_server": "= stunnel_server(couch.bigcouch.ednp_port)" + }, + "clients": { + "epmd_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], couch.bigcouch.epmd_port)", + "ednp_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], couch.bigcouch.ednp_port)" + } }, "couch": { "mode": "multimaster", diff --git a/provider_base/services/couchdb.json b/provider_base/services/couchdb.json index d75fd8de..c2482235 100644 --- a/provider_base/services/couchdb.json +++ b/provider_base/services/couchdb.json @@ -3,7 +3,9 @@ "use": true }, "stunnel": { - "couch_server": "= stunnel_server(couch.port)" + "servers": { + "couch_server": "= stunnel_server(couch.port)" + } }, "couch": { "master": false, diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/stunnel.pp b/puppet/modules/site_couchdb/manifests/bigcouch/stunnel.pp deleted file mode 100644 index 5166ba93..00000000 --- a/puppet/modules/site_couchdb/manifests/bigcouch/stunnel.pp +++ /dev/null @@ -1,89 +0,0 @@ -class site_couchdb::bigcouch::stunnel { - - $stunnel = hiera('stunnel') - - include site_config::x509::cert - include site_config::x509::key - include site_config::x509::ca - - include x509::variables - $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" - $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" - $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" - - - # Erlang Port Mapper Daemon (epmd) stunnel server/clients - $epmd_server = $stunnel['epmd_server'] - $epmd_server_accept = $epmd_server['accept'] - $epmd_server_connect = $epmd_server['connect'] - $epmd_clients = $stunnel['epmd_clients'] - - # Erlang Distributed Node Protocol (ednp) stunnel server/clients - $ednp_server = $stunnel['ednp_server'] - $ednp_server_accept = $ednp_server['accept'] - $ednp_server_connect = $ednp_server['connect'] - $ednp_clients = $stunnel['ednp_clients'] - - - # setup stunnel server for Erlang Port Mapper Daemon (epmd), necessary for - # bigcouch clustering between each bigcouchdb node - stunnel::service { 'epmd_server': - accept => $epmd_server_accept, - connect => $epmd_server_connect, - client => false, - cafile => $ca_path, - key => $key_path, - cert => $cert_path, - verify => '2', - pid => '/var/run/stunnel4/epmd_server.pid', - rndfile => '/var/lib/stunnel4/.rnd', - debuglevel => '4', - require => [ - Class['Site_config::X509::Key'], - Class['Site_config::X509::Cert'], - Class['Site_config::X509::Ca'] ]; - } - - # setup stunnel clients for Erlang Port Mapper Daemon (epmd) to connect - # to the above epmd stunnel server. - $epmd_client_defaults = { - 'client' => true, - 'cafile' => $ca_path, - 'key' => $key_path, - 'cert' => $cert_path, - } - - create_resources(site_stunnel::clients, $epmd_clients, $epmd_client_defaults) - - # setup stunnel server for Erlang Distributed Node Protocol (ednp), necessary - # for bigcouch clustering between each bigcouchdb node - stunnel::service { 'ednp_server': - accept => $ednp_server_accept, - connect => $ednp_server_connect, - client => false, - cafile => $ca_path, - key => $key_path, - cert => $cert_path, - verify => '2', - pid => '/var/run/stunnel4/ednp_server.pid', - rndfile => '/var/lib/stunnel4/.rnd', - debuglevel => '4', - require => [ - Class['Site_config::X509::Key'], - Class['Site_config::X509::Cert'], - Class['Site_config::X509::Ca'] ]; - } - - # setup stunnel clients for Erlang Distributed Node Protocol (ednp) to connect - # to the above ednp stunnel server. - $ednp_client_defaults = { - 'client' => true, - 'cafile' => $ca_path, - 'key' => $key_path, - 'cert' => $cert_path, - } - - create_resources(site_stunnel::clients, $ednp_clients, $ednp_client_defaults) - - include site_check_mk::agent::stunnel -} diff --git a/puppet/modules/site_couchdb/manifests/stunnel.pp b/puppet/modules/site_couchdb/manifests/stunnel.pp deleted file mode 100644 index 484a0c00..00000000 --- a/puppet/modules/site_couchdb/manifests/stunnel.pp +++ /dev/null @@ -1,43 +0,0 @@ -class site_couchdb::stunnel { - - $stunnel = hiera('stunnel') - $couchdb_config = hiera('couch') - $couchdb_bigcouch = $couchdb_config['mode'] == "multimaster" - - $couch_server = $stunnel['couch_server'] - $couch_server_accept = $couch_server['accept'] - $couch_server_connect = $couch_server['connect'] - - include site_config::x509::cert - include site_config::x509::key - include site_config::x509::ca - - if $couchdb_bigcouch { - include site_couchdb::bigcouch::stunnel - } - - include x509::variables - $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" - $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" - $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" - - # setup a stunnel server for the webapp to connect to couchdb - stunnel::service { 'couch_server': - accept => $couch_server_accept, - connect => $couch_server_connect, - client => false, - cafile => $ca_path, - key => $key_path, - cert => $cert_path, - verify => '2', - pid => '/var/run/stunnel4/couchserver.pid', - rndfile => '/var/lib/stunnel4/.rnd', - debuglevel => '4', - require => [ - Class['Site_config::X509::Key'], - Class['Site_config::X509::Cert'], - Class['Site_config::X509::Ca'] ]; - } - - include site_check_mk::agent::stunnel -} diff --git a/puppet/modules/site_shorewall/manifests/couchdb.pp b/puppet/modules/site_shorewall/manifests/couchdb.pp deleted file mode 100644 index 73bed62b..00000000 --- a/puppet/modules/site_shorewall/manifests/couchdb.pp +++ /dev/null @@ -1,24 +0,0 @@ -class site_shorewall::couchdb { - - include site_shorewall::defaults - - $stunnel = hiera('stunnel') - $couch_server = $stunnel['couch_server'] - $couch_stunnel_port = $couch_server['accept'] - - # define macro for incoming services - file { '/etc/shorewall/macro.leap_couchdb': - content => "PARAM - - tcp ${couch_stunnel_port}", - notify => Service['shorewall'], - require => Package['shorewall'] - } - - shorewall::rule { - 'net2fw-couchdb': - source => 'net', - destination => '$FW', - action => 'leap_couchdb(ACCEPT)', - order => 200; - } - -} diff --git a/puppet/modules/site_shorewall/manifests/couchdb/bigcouch.pp b/puppet/modules/site_shorewall/manifests/couchdb/bigcouch.pp deleted file mode 100644 index 20740650..00000000 --- a/puppet/modules/site_shorewall/manifests/couchdb/bigcouch.pp +++ /dev/null @@ -1,51 +0,0 @@ -class site_shorewall::couchdb::bigcouch { - - include site_shorewall::defaults - - $stunnel = hiera('stunnel') - - # Erlang Port Mapper Daemon (epmd) stunnel server/clients - $epmd_clients = $stunnel['epmd_clients'] - $epmd_server = $stunnel['epmd_server'] - $epmd_server_port = $epmd_server['accept'] - $epmd_server_connect = $epmd_server['connect'] - - # Erlang Distributed Node Protocol (ednp) stunnel server/clients - $ednp_clients = $stunnel['ednp_clients'] - $ednp_server = $stunnel['ednp_server'] - $ednp_server_port = $ednp_server['accept'] - $ednp_server_connect = $ednp_server['connect'] - - # define macro for incoming services - file { '/etc/shorewall/macro.leap_bigcouch': - content => "PARAM - - tcp ${epmd_server_port},${ednp_server_port}", - notify => Service['shorewall'], - require => Package['shorewall'] - } - - shorewall::rule { - 'net2fw-bigcouch': - source => 'net', - destination => '$FW', - action => 'leap_bigcouch(ACCEPT)', - order => 300; - } - - # setup DNAT rules for each epmd - $epmd_shorewall_dnat_defaults = { - 'source' => '$FW', - 'proto' => 'tcp', - 'destinationport' => regsubst($epmd_server_connect, '^([0-9.]+:)([0-9]+)$', '\2') - } - create_resources(site_shorewall::couchdb::dnat, $epmd_clients, $epmd_shorewall_dnat_defaults) - - # setup DNAT rules for each ednp - $ednp_shorewall_dnat_defaults = { - 'source' => '$FW', - 'proto' => 'tcp', - 'destinationport' => regsubst($ednp_server_connect, '^([0-9.]+:)([0-9]+)$', '\2') - } - create_resources(site_shorewall::couchdb::dnat, $ednp_clients, $ednp_shorewall_dnat_defaults) - -} - diff --git a/puppet/modules/site_shorewall/manifests/couchdb/dnat.pp b/puppet/modules/site_shorewall/manifests/couchdb/dnat.pp deleted file mode 100644 index f1bc9acf..00000000 --- a/puppet/modules/site_shorewall/manifests/couchdb/dnat.pp +++ /dev/null @@ -1,21 +0,0 @@ -define site_shorewall::couchdb::dnat ( - $source, - $connect, - $connect_port, - $accept_port, - $proto, - $destinationport ) -{ - - - shorewall::rule { - "dnat_${name}_${destinationport}": - action => 'DNAT', - source => $source, - destination => "\$FW:127.0.0.1:${accept_port}", - proto => $proto, - destinationport => $destinationport, - originaldest => $connect, - order => 200 - } -} diff --git a/puppet/modules/site_shorewall/manifests/stunnel/client.pp b/puppet/modules/site_shorewall/manifests/stunnel/client.pp new file mode 100644 index 00000000..9a89a244 --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/stunnel/client.pp @@ -0,0 +1,40 @@ +# +# Adds some firewall magic to the stunnel. +# +# Using DNAT, this firewall rule allow a locally running program +# to try to connect to the normal remote IP and remote port of the +# service on another machine, but have this connection magically +# routed through the locally running stunnel client. +# +# The network looks like this: +# +# From the client's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# consumer app -> localhost:accept_port -> connect:connect_port -> localhost:original_port +# +# From the server's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# ?? -> *:accept_port -> localhost:connect_port -> service +# + +define site_shorewall::stunnel::client( + $accept_port, + $connect, + $connect_port, + $original_port) { + + include site_shorewall::defaults + + shorewall::rule { + "stunnel_dnat_${name}": + action => 'DNAT', + source => '$FW', + destination => "\$FW:127.0.0.1:${accept_port}", + proto => 'tcp', + destinationport => $original_port, + originaldest => $connect, + order => 200 + } +} diff --git a/puppet/modules/site_shorewall/manifests/stunnel/server.pp b/puppet/modules/site_shorewall/manifests/stunnel/server.pp new file mode 100644 index 00000000..db3ecd3e --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/stunnel/server.pp @@ -0,0 +1,22 @@ +# +# Allow all incoming connections to stunnel server port +# + +define site_shorewall::stunnel::server($port) { + + include site_shorewall::defaults + + file { "/etc/shorewall/macro.stunnel_server_${name}": + content => "PARAM - - tcp ${port}", + notify => Service['shorewall'], + require => Package['shorewall'] + } + shorewall::rule { + 'net2fw-couchdb': + source => 'net', + destination => '$FW', + action => "stunnel_server_${name}(ACCEPT)", + order => 200; + } + +} \ No newline at end of file diff --git a/puppet/modules/site_stunnel/manifests/client.pp b/puppet/modules/site_stunnel/manifests/client.pp new file mode 100644 index 00000000..12d664b4 --- /dev/null +++ b/puppet/modules/site_stunnel/manifests/client.pp @@ -0,0 +1,52 @@ +# +# Sets up stunnel and firewall configuration for +# a single stunnel client +# +# As a client, we accept connections on localhost, +# and connect to a remote $connect:$connect_port +# + +define site_stunnel::client ( + $accept_port, + $connect_port, + $connect, + $original_port, + $verify = '2', + $pid = $name, + $rndfile = '/var/lib/stunnel4/.rnd', + $debuglevel = '4' ) { + + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca + include x509::variables + $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" + $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" + $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" + + stunnel::service { $name: + accept => "127.0.0.1:${accept_port}", + connect => "${connect}:${connect_port}", + client => true, + cafile => $ca_path, + key => $key_path, + cert => $cert_path, + verify => $verify, + pid => "/var/run/stunnel4/${pid}.pid", + rndfile => $rndfile, + debuglevel => $debuglevel, + subscribe => [ + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Ca'] ]; + } + + site_shorewall::stunnel::client { $name: + accept_port => $accept_port, + connect => $connect, + connect_port => $connect_port, + original_port => $original_port + } + + include site_check_mk::agent::stunnel +} diff --git a/puppet/modules/site_stunnel/manifests/clients.pp b/puppet/modules/site_stunnel/manifests/clients.pp index b75c9ac3..44b31aaa 100644 --- a/puppet/modules/site_stunnel/manifests/clients.pp +++ b/puppet/modules/site_stunnel/manifests/clients.pp @@ -1,33 +1,26 @@ -define site_stunnel::clients ( - $accept_port, - $connect_port, - $connect, - $cafile, - $key, - $cert, - $client = true, - $verify = '2', - $pid = $name, - $rndfile = '/var/lib/stunnel4/.rnd', - $debuglevel = '4' ) { +# +# usage: +# create_resource(site_stunnel::clients, hiera('stunnel')['clients']) +# +# example hiera yaml: +# +# stunnel: +# clients: +# ednp_clients: +# thrips_9002: +# accept_port: 4001 +# connect: thrips.demo.bitmask.i +# connect_port: 19002 +# epmd_clients: +# thrips_4369: +# accept_port: 4000 +# connect: thrips.demo.bitmask.i +# connect_port: 14369 +# +# In the above example, this resource definition is called twice, with $name +# 'ednp_clients' and 'epmd_clients' +# - stunnel::service { $name: - accept => "127.0.0.1:${accept_port}", - connect => "${connect}:${connect_port}", - client => $client, - cafile => $cafile, - key => $key, - cert => $cert, - verify => $verify, - pid => "/var/run/stunnel4/${pid}.pid", - rndfile => $rndfile, - debuglevel => $debuglevel, - subscribe => [ - Class['Site_config::X509::Key'], - Class['Site_config::X509::Cert'], - Class['Site_config::X509::Ca'] ]; - - } - - include site_check_mk::agent::stunnel +define site_stunnel::clients { + create_resources(site_stunnel::client, $site_stunnel::clients[$name]) } diff --git a/puppet/modules/site_stunnel/manifests/init.pp b/puppet/modules/site_stunnel/manifests/init.pp index c7d6acc6..b292f1cd 100644 --- a/puppet/modules/site_stunnel/manifests/init.pp +++ b/puppet/modules/site_stunnel/manifests/init.pp @@ -1,3 +1,8 @@ +# +# If you need something to happen after stunnel is started, +# you can depend on Service['stunnel'] or Class['site_stunnel'] +# + class site_stunnel { # include the generic stunnel module @@ -13,5 +18,15 @@ class site_stunnel { ensure => absent; } } + + $stunnel = hiera('stunnel') + + # add server stunnels + create_resources(site_stunnel::servers, $stunnel['servers']) + + # add client stunnels + $clients = $stunnel['clients'] + $client_sections = keys($clients) + site_stunnel::clients { $client_sections: } } diff --git a/puppet/modules/site_stunnel/manifests/servers.pp b/puppet/modules/site_stunnel/manifests/servers.pp new file mode 100644 index 00000000..4419923f --- /dev/null +++ b/puppet/modules/site_stunnel/manifests/servers.pp @@ -0,0 +1,53 @@ +# +# usage: +# create_resource(site_stunnel::servers, hiera('stunnel')['servers']) +# +# example hiera yaml: +# +# stunnel: +# servers: +# couch_server: +# accept_port: 15984 +# connect_port: 5984 +# + +define site_stunnel::servers ( + $accept_port, + $connect_port, + $verify = '2', + $pid = $name, + $rndfile = '/var/lib/stunnel4/.rnd', + $debuglevel = '4' ) { + + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca + include x509::variables + $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" + $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" + $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" + + stunnel::service { $name: + accept => $accept_port, + connect => "127.0.0.1:${connect_port}", + client => false, + cafile => $ca_path, + key => $key_path, + cert => $cert_path, + verify => $verify, + pid => "/var/run/stunnel4/${pid}.pid", + rndfile => '/var/lib/stunnel4/.rnd', + debuglevel => $debuglevel, + require => [ + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Ca'] ]; + } + + # allow incoming connections on $accept_port + site_shorewall::stunnel::server { $name: + port => $accept_port + } + + include site_check_mk::agent::stunnel +} -- cgit v1.2.3 From 0fbb8b1c2ddcabc23c19229ea89a2070964fc7ab Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 20 Jun 2014 01:59:01 -0700 Subject: tmp comment out error if no master nodes defined --- provider_base/services/couchdb.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider_base/services/couchdb.rb b/provider_base/services/couchdb.rb index c8f5d8a7..c63e3a00 100644 --- a/provider_base/services/couchdb.rb +++ b/provider_base/services/couchdb.rb @@ -3,7 +3,7 @@ # unless nodes_like_me['services' => 'couchdb']['couch.master' => true].any? - raise 'node `%s`, environment `%s`: there must be at least one node with couch.master set to `true` for this environment.' % [@node.name, @node.environment] + #raise 'node `%s`, environment `%s`: there must be at least one node with couch.master set to `true` for this environment.' % [@node.name, @node.environment] end if couch.master -- cgit v1.2.3 From 2bd603b9532fac70a25add8661acc94acb8598f8 Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 20 Jun 2014 02:00:18 -0700 Subject: site_couchdb: support auto-stunnel setup, split master, bigcouch, and mirror out into separate files. --- puppet/modules/site_couchdb/manifests/add_users.pp | 3 ++ puppet/modules/site_couchdb/manifests/bigcouch.pp | 20 ++++++-- .../modules/site_couchdb/manifests/create_dbs.pp | 3 ++ puppet/modules/site_couchdb/manifests/init.pp | 60 +++------------------- puppet/modules/site_couchdb/manifests/master.pp | 9 ++++ puppet/modules/site_couchdb/manifests/mirror.pp | 18 ++++++- puppet/modules/site_couchdb/manifests/setup.pp | 39 ++++++++++++++ 7 files changed, 96 insertions(+), 56 deletions(-) create mode 100644 puppet/modules/site_couchdb/manifests/master.pp create mode 100644 puppet/modules/site_couchdb/manifests/setup.pp diff --git a/puppet/modules/site_couchdb/manifests/add_users.pp b/puppet/modules/site_couchdb/manifests/add_users.pp index f9ea7349..41930b7b 100644 --- a/puppet/modules/site_couchdb/manifests/add_users.pp +++ b/puppet/modules/site_couchdb/manifests/add_users.pp @@ -1,5 +1,8 @@ class site_couchdb::add_users { + Class['site_couchdb::create_dbs'] + -> Class['site_couchdb::add_users'] + # Couchdb users ## leap_mx couchdb user diff --git a/puppet/modules/site_couchdb/manifests/bigcouch.pp b/puppet/modules/site_couchdb/manifests/bigcouch.pp index 97c8cd12..f0aab734 100644 --- a/puppet/modules/site_couchdb/manifests/bigcouch.pp +++ b/puppet/modules/site_couchdb/manifests/bigcouch.pp @@ -1,18 +1,32 @@ class site_couchdb::bigcouch { - $config = $::site_couchdb::couchdb_config['bigcouch'] + $config = $couchdb_config['bigcouch'] $cookie = $config['cookie'] + $ednp_port = $config['ednp_port'] - $ednp_port = $config['ednp_port'] + class { 'couchdb': + admin_pw => $couchdb_admin_pw, + admin_salt => $couchdb_admin_salt, + bigcouch => true, + bigcouch_cookie => $cookie, + ednp_port => $ednp_port, + chttpd_bind_address => '127.0.0.1' + } + # + # stunnel must running correctly before bigcouch dbs can be set up. + # Class['site_config::default'] + -> Class['couchdb::bigcouch::package::cloudant'] + -> Service['shorewall'] + -> Service['stunnel'] + -> Class['site_couchdb::setup'] -> Class['site_couchdb::bigcouch::add_nodes'] -> Class['site_couchdb::bigcouch::settle_cluster'] include site_couchdb::bigcouch::add_nodes include site_couchdb::bigcouch::settle_cluster include site_couchdb::bigcouch::compaction - include site_shorewall::couchdb::bigcouch file { '/var/log/bigcouch': ensure => directory diff --git a/puppet/modules/site_couchdb/manifests/create_dbs.pp b/puppet/modules/site_couchdb/manifests/create_dbs.pp index 41500d3a..f8d8098a 100644 --- a/puppet/modules/site_couchdb/manifests/create_dbs.pp +++ b/puppet/modules/site_couchdb/manifests/create_dbs.pp @@ -1,5 +1,8 @@ class site_couchdb::create_dbs { + Class['site_couchdb::setup'] + -> Class['site_couchdb::create_dbs'] + # Couchdb databases ### customer database diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 0b923c9f..4999b611 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -37,70 +37,26 @@ class site_couchdb { $couchdb_backup = $couchdb_config['backup'] $couchdb_mode = $couchdb_config['mode'] - class { 'couchdb': - bigcouch => $couchdb_bigcouch, - admin_pw => $couchdb_admin_pw, - admin_salt => $couchdb_admin_salt, - bigcouch_cookie => $bigcouch_cookie, - ednp_port => $ednp_port, - chttpd_bind_address => '127.0.0.1' - } - - # ensure that we don't have leftovers from previous installations - # where we installed the cloudant bigcouch package - # https://leap.se/code/issues/4971 - class { 'couchdb::bigcouch::package::cloudant': - ensure => absent - } + if $couchdb_mode == "multimaster" { include site_couchdb::bigcouch } + if $couchdb_mode == "master" { include site_couchdb::master } + if $couchdb_mode == "mirror" { include site_couchdb::mirror } Class['site_config::default'] - -> Class['couchdb::bigcouch::package::cloudant'] -> Service['shorewall'] - -> Class['site_couchdb::stunnel'] - -> Service['couchdb'] - -> File['/root/.netrc'] - -> Class['site_couchdb::create_dbs'] - -> Class['site_couchdb::add_users'] - - # /etc/couchdb/couchdb.netrc is deployed by couchdb::query::setup - # we symlink this to /root/.netrc for couchdb_scripts (eg. backup) - # and makes life easier for the admin (i.e. using curl/wget without - # passing credentials) - file { - '/root/.netrc': - ensure => link, - target => '/etc/couchdb/couchdb.netrc'; - - '/srv/leap/couchdb': - ensure => directory - } + -> Service['stunnel'] + -> Class['couchdb'] + -> Class['site_couchdb::setup'] - couchdb::query::setup { 'localhost': - user => $couchdb_admin_user, - pw => $couchdb_admin_pw, - } + include site_stunnel - vcsrepo { '/srv/leap/couchdb/scripts': - ensure => present, - provider => git, - source => 'https://leap.se/git/couchdb_scripts', - revision => 'origin/master', - require => File['/srv/leap/couchdb'] - } - - include site_couchdb::stunnel + include site_couchdb::setup include site_couchdb::create_dbs include site_couchdb::add_users include site_couchdb::designs include site_couchdb::logrotate - if $couchdb_mode == "multimaster" { include site_couchdb::bigcouch } - if $couchdb_mode == "mirror" { include site_couchdb::mirror } - if $couchdb_backup { include site_couchdb::backup } - include site_shorewall::couchdb - include site_check_mk::agent::couchdb include site_check_mk::agent::tapicero diff --git a/puppet/modules/site_couchdb/manifests/master.pp b/puppet/modules/site_couchdb/manifests/master.pp new file mode 100644 index 00000000..a0a6633d --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/master.pp @@ -0,0 +1,9 @@ +class site_couchdb::master { + + class { 'couchdb': + admin_pw => $site_couchdb::couchdb_admin_pw, + admin_salt => $site_couchdb::couchdb_admin_salt, + chttpd_bind_address => '127.0.0.1' + } + +} \ No newline at end of file diff --git a/puppet/modules/site_couchdb/manifests/mirror.pp b/puppet/modules/site_couchdb/manifests/mirror.pp index 708171e4..f3b43cc2 100644 --- a/puppet/modules/site_couchdb/manifests/mirror.pp +++ b/puppet/modules/site_couchdb/manifests/mirror.pp @@ -1,8 +1,24 @@ class site_couchdb::mirror { + Class['site_couchdb::add_users'] + -> Class['site_couchdb::mirror'] + + class { 'couchdb': + admin_pw => $site_couchdb::couchdb_admin_pw, + admin_salt => $site_couchdb::couchdb_admin_salt, + chttpd_bind_address => '127.0.0.1' + } + # Couchdb databases - $from = $site_couchdb::couchdb_config['replication']['masters'][0] + $masters = $site_couchdb::couchdb_config['replication']['masters'] + $master_node_names = keys($site_couchdb::couchdb_config['replication']['masters']) + $master_node = $masters[$master_node_names[0]] + $from_host = $master_node['domain_internal'] + $from_port = $master_node['couch_port'] + $from = "${from_host}:${from_port}" + + notice("mirror from: ${from}") ### customer database couchdb::mirror_db { 'customers': diff --git a/puppet/modules/site_couchdb/manifests/setup.pp b/puppet/modules/site_couchdb/manifests/setup.pp new file mode 100644 index 00000000..e398356b --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/setup.pp @@ -0,0 +1,39 @@ +# +# An initial setup class. All the other classes depend on this +# +class site_couchdb::setup { + + # ensure that we don't have leftovers from previous installations + # where we installed the cloudant bigcouch package + # https://leap.se/code/issues/4971 + class { 'couchdb::bigcouch::package::cloudant': + ensure => absent + } + + # /etc/couchdb/couchdb.netrc is deployed by couchdb::query::setup + # we symlink this to /root/.netrc for couchdb_scripts (eg. backup) + # and makes life easier for the admin (i.e. using curl/wget without + # passing credentials) + file { + '/root/.netrc': + ensure => link, + target => '/etc/couchdb/couchdb.netrc'; + + '/srv/leap/couchdb': + ensure => directory + } + + couchdb::query::setup { 'localhost': + user => $site_couchdb::couchdb_admin_user, + pw => $site_couchdb::couchdb_admin_pw, + } + + vcsrepo { '/srv/leap/couchdb/scripts': + ensure => present, + provider => git, + source => 'https://leap.se/git/couchdb_scripts', + revision => 'origin/master', + require => File['/srv/leap/couchdb'] + } + +} -- cgit v1.2.3 From a8f6415b0869018fd8d4ac947814529e8e85ace2 Mon Sep 17 00:00:00 2001 From: Azul Date: Fri, 20 Jun 2014 19:10:44 +0200 Subject: add replication user --- provider_base/services/couchdb.json | 5 +++++ puppet/modules/site_couchdb/manifests/add_users.pp | 9 +++++++++ puppet/modules/site_couchdb/manifests/create_dbs.pp | 18 +++++++++--------- puppet/modules/site_couchdb/manifests/init.pp | 5 +++++ puppet/modules/site_couchdb/manifests/mirror.pp | 4 +++- 5 files changed, 31 insertions(+), 10 deletions(-) diff --git a/provider_base/services/couchdb.json b/provider_base/services/couchdb.json index c2482235..8b1386f8 100644 --- a/provider_base/services/couchdb.json +++ b/provider_base/services/couchdb.json @@ -40,6 +40,11 @@ "username": "webapp", "password": "= secret :couch_webapp_password", "salt": "= hex_secret :couch_webapp_password_salt, 128" + }, + "replication": { + "username": "replication", + "password": "= secret :couch_replication_password", + "salt": "= hex_secret :couch_replication_password_salt, 128" } }, "webapp": { diff --git a/puppet/modules/site_couchdb/manifests/add_users.pp b/puppet/modules/site_couchdb/manifests/add_users.pp index 41930b7b..0585da27 100644 --- a/puppet/modules/site_couchdb/manifests/add_users.pp +++ b/puppet/modules/site_couchdb/manifests/add_users.pp @@ -54,4 +54,13 @@ class site_couchdb::add_users { require => Couchdb::Query::Setup['localhost'] } + ## replication couchdb user + ## read/write: all databases for replication + couchdb::add_user { $site_couchdb::couchdb_replication_user: + roles => '["repliction"]', + pw => $site_couchdb::couchdb_replication_pw, + salt => $site_couchdb::couchdb_replication_salt, + require => Couchdb::Query::Setup['localhost'] + } + } diff --git a/puppet/modules/site_couchdb/manifests/create_dbs.pp b/puppet/modules/site_couchdb/manifests/create_dbs.pp index f8d8098a..4322f773 100644 --- a/puppet/modules/site_couchdb/manifests/create_dbs.pp +++ b/puppet/modules/site_couchdb/manifests/create_dbs.pp @@ -8,7 +8,7 @@ class site_couchdb::create_dbs { ### customer database ### r/w: webapp, couchdb::create_db { 'customers': - members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }", + members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [\"replication\"] }", require => Couchdb::Query::Setup['localhost'] } @@ -16,35 +16,35 @@ class site_couchdb::create_dbs { ## r: nickserver, leap_mx - needs to be restrict with design document ## r/w: webapp couchdb::create_db { 'identities': - members => "{ \"names\": [], \"roles\": [\"identities\"] }", + members => "{ \"names\": [], \"roles\": [\"replication\", \"identities\"] }", require => Couchdb::Query::Setup['localhost'] } ## keycache database ## r/w: nickserver couchdb::create_db { 'keycache': - members => "{ \"names\": [], \"roles\": [\"keycache\"] }", + members => "{ \"names\": [], \"roles\": [\"replication\", \"keycache\"] }", require => Couchdb::Query::Setup['localhost'] } ## sessions database ## r/w: webapp couchdb::create_db { 'sessions': - members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }", + members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [\"replication\"] }", require => Couchdb::Query::Setup['localhost'] } ## shared database ## r/w: soledad couchdb::create_db { 'shared': - members => "{ \"names\": [\"$site_couchdb::couchdb_soledad_user\"], \"roles\": [] }", + members => "{ \"names\": [\"$site_couchdb::couchdb_soledad_user\"], \"roles\": [\"replication\"] }", require => Couchdb::Query::Setup['localhost'] } ## tickets database ## r/w: webapp couchdb::create_db { 'tickets': - members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }", + members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [\"replication\"] }", require => Couchdb::Query::Setup['localhost'] } @@ -52,14 +52,14 @@ class site_couchdb::create_dbs { ## r: soledad - needs to be restricted with a design document ## r/w: webapp couchdb::create_db { 'tokens': - members => "{ \"names\": [], \"roles\": [\"tokens\"] }", + members => "{ \"names\": [], \"roles\": [\"replication\", \"tokens\"] }", require => Couchdb::Query::Setup['localhost'] } ## users database ## r/w: webapp couchdb::create_db { 'users': - members => "{ \"names\": [], \"roles\": [\"users\"] }", + members => "{ \"names\": [], \"roles\": [\"replication\", \"users\"] }", require => Couchdb::Query::Setup['localhost'] } @@ -67,7 +67,7 @@ class site_couchdb::create_dbs { ## store messages to the clients such as payment reminders ## r/w: webapp couchdb::create_db { 'messages': - members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }", + members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [\"replication\"] }", require => Couchdb::Query::Setup['localhost'] } } diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 4999b611..6f7e974e 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -34,6 +34,11 @@ class site_couchdb { $couchdb_webapp_pw = $couchdb_webapp['password'] $couchdb_webapp_salt = $couchdb_webapp['salt'] + $couchdb_replication = $couchdb_users['replication'] + $couchdb_replication_user= $couchdb_replication['username'] + $couchdb_replication_pw = $couchdb_replication['password'] + $couchdb_replication_salt= $couchdb_replication['salt'] + $couchdb_backup = $couchdb_config['backup'] $couchdb_mode = $couchdb_config['mode'] diff --git a/puppet/modules/site_couchdb/manifests/mirror.pp b/puppet/modules/site_couchdb/manifests/mirror.pp index f3b43cc2..2a44b1e9 100644 --- a/puppet/modules/site_couchdb/manifests/mirror.pp +++ b/puppet/modules/site_couchdb/manifests/mirror.pp @@ -14,9 +14,11 @@ class site_couchdb::mirror { $masters = $site_couchdb::couchdb_config['replication']['masters'] $master_node_names = keys($site_couchdb::couchdb_config['replication']['masters']) $master_node = $masters[$master_node_names[0]] + $user = $site_couchdb::couchdb_replication_user + $password = $site_couchdb::couchdb_replication_pw $from_host = $master_node['domain_internal'] $from_port = $master_node['couch_port'] - $from = "${from_host}:${from_port}" + $from = "http://${user}:${password}@${from_host}:${from_port}" notice("mirror from: ${from}") -- cgit v1.2.3 From bc42e9bd3a86bb858ef853cf333242c81874209b Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 20 Jun 2014 14:34:53 -0700 Subject: stunnel: make site_mx and site_webapp use new site_stunnel --- puppet/modules/site_mx/manifests/couchdb.pp | 23 ---------------------- puppet/modules/site_mx/manifests/init.pp | 2 +- .../site_shorewall/manifests/stunnel/server.pp | 2 +- puppet/modules/site_stunnel/manifests/clients.pp | 3 --- puppet/modules/site_stunnel/manifests/servers.pp | 3 --- puppet/modules/site_webapp/manifests/couchdb.pp | 14 ------------- 6 files changed, 2 insertions(+), 45 deletions(-) delete mode 100644 puppet/modules/site_mx/manifests/couchdb.pp diff --git a/puppet/modules/site_mx/manifests/couchdb.pp b/puppet/modules/site_mx/manifests/couchdb.pp deleted file mode 100644 index b1f3bd02..00000000 --- a/puppet/modules/site_mx/manifests/couchdb.pp +++ /dev/null @@ -1,23 +0,0 @@ -class site_mx::couchdb { - - $stunnel = hiera('stunnel') - $couch_client = $stunnel['couch_client'] - $couch_client_connect = $couch_client['connect'] - - include x509::variables - $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" - $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" - $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" - - include site_stunnel - - $couchdb_stunnel_client_defaults = { - 'connect_port' => $couch_client_connect, - 'client' => true, - 'cafile' => $ca_path, - 'key' => $key_path, - 'cert' => $cert_path, - } - - create_resources(site_stunnel::clients, $couch_client, $couchdb_stunnel_client_defaults) -} diff --git a/puppet/modules/site_mx/manifests/init.pp b/puppet/modules/site_mx/manifests/init.pp index c3d38a46..91014ed6 100644 --- a/puppet/modules/site_mx/manifests/init.pp +++ b/puppet/modules/site_mx/manifests/init.pp @@ -8,12 +8,12 @@ class site_mx { include site_config::x509::client_ca::ca include site_config::x509::client_ca::key + include site_stunnel include site_postfix::mx include site_haproxy include site_shorewall::mx include site_shorewall::service::smtp - include site_mx::couchdb include leap_mx include site_check_mk::agent::mx } diff --git a/puppet/modules/site_shorewall/manifests/stunnel/server.pp b/puppet/modules/site_shorewall/manifests/stunnel/server.pp index db3ecd3e..798cd631 100644 --- a/puppet/modules/site_shorewall/manifests/stunnel/server.pp +++ b/puppet/modules/site_shorewall/manifests/stunnel/server.pp @@ -12,7 +12,7 @@ define site_shorewall::stunnel::server($port) { require => Package['shorewall'] } shorewall::rule { - 'net2fw-couchdb': + "net2fw-stunnel-server-${name}": source => 'net', destination => '$FW', action => "stunnel_server_${name}(ACCEPT)", diff --git a/puppet/modules/site_stunnel/manifests/clients.pp b/puppet/modules/site_stunnel/manifests/clients.pp index 44b31aaa..c0958b5f 100644 --- a/puppet/modules/site_stunnel/manifests/clients.pp +++ b/puppet/modules/site_stunnel/manifests/clients.pp @@ -1,7 +1,4 @@ # -# usage: -# create_resource(site_stunnel::clients, hiera('stunnel')['clients']) -# # example hiera yaml: # # stunnel: diff --git a/puppet/modules/site_stunnel/manifests/servers.pp b/puppet/modules/site_stunnel/manifests/servers.pp index 4419923f..b1da5c59 100644 --- a/puppet/modules/site_stunnel/manifests/servers.pp +++ b/puppet/modules/site_stunnel/manifests/servers.pp @@ -1,7 +1,4 @@ # -# usage: -# create_resource(site_stunnel::servers, hiera('stunnel')['servers']) -# # example hiera yaml: # # stunnel: diff --git a/puppet/modules/site_webapp/manifests/couchdb.pp b/puppet/modules/site_webapp/manifests/couchdb.pp index ff743fba..3ae4d266 100644 --- a/puppet/modules/site_webapp/manifests/couchdb.pp +++ b/puppet/modules/site_webapp/manifests/couchdb.pp @@ -7,10 +7,6 @@ class site_webapp::couchdb { $couchdb_webapp_user = $webapp['couchdb_webapp_user']['username'] $couchdb_webapp_password = $webapp['couchdb_webapp_user']['password'] - $stunnel = hiera('stunnel') - $couch_client = $stunnel['couch_client'] - $couch_client_connect = $couch_client['connect'] - include x509::variables file { @@ -37,14 +33,4 @@ class site_webapp::couchdb { } include site_stunnel - - $couchdb_stunnel_client_defaults = { - 'connect_port' => $couch_client_connect, - 'client' => true, - 'cafile' => "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt", - 'key' => "${x509::variables::keys}/${site_config::params::cert_name}.key", - 'cert' => "${x509::variables::certs}/${site_config::params::cert_name}.crt", - } - - create_resources(site_stunnel::clients, $couch_client, $couchdb_stunnel_client_defaults) } -- cgit v1.2.3 From d3919f44368e3e9e60736fd90f31118f520747a8 Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 20 Jun 2014 15:42:51 -0700 Subject: moved json macros to provider_base/lib/macros. requires new unreleased leap_cli --- provider_base/lib/macros.rb | 14 ++++++ provider_base/lib/macros/core.rb | 86 +++++++++++++++++++++++++++++++++ provider_base/lib/macros/files.rb | 79 ++++++++++++++++++++++++++++++ provider_base/lib/macros/haproxy.rb | 68 ++++++++++++++++++++++++++ provider_base/lib/macros/hosts.rb | 63 ++++++++++++++++++++++++ provider_base/lib/macros/nodes.rb | 88 ++++++++++++++++++++++++++++++++++ provider_base/lib/macros/secrets.rb | 39 +++++++++++++++ provider_base/lib/macros/stunnel.rb | 95 +++++++++++++++++++++++++++++++++++++ 8 files changed, 532 insertions(+) create mode 100644 provider_base/lib/macros.rb create mode 100644 provider_base/lib/macros/core.rb create mode 100644 provider_base/lib/macros/files.rb create mode 100644 provider_base/lib/macros/haproxy.rb create mode 100644 provider_base/lib/macros/hosts.rb create mode 100644 provider_base/lib/macros/nodes.rb create mode 100644 provider_base/lib/macros/secrets.rb create mode 100644 provider_base/lib/macros/stunnel.rb diff --git a/provider_base/lib/macros.rb b/provider_base/lib/macros.rb new file mode 100644 index 00000000..854b92b5 --- /dev/null +++ b/provider_base/lib/macros.rb @@ -0,0 +1,14 @@ +# +# MACROS +# +# The methods in these files are available in the context of a .json configuration file. +# (The module LeapCli::Macro is included in Config::Object) +# + +require_relative 'macros/core' +require_relative 'macros/files' +require_relative 'macros/haproxy' +require_relative 'macros/hosts' +require_relative 'macros/nodes' +require_relative 'macros/secrets' +require_relative 'macros/stunnel' diff --git a/provider_base/lib/macros/core.rb b/provider_base/lib/macros/core.rb new file mode 100644 index 00000000..d4d9171f --- /dev/null +++ b/provider_base/lib/macros/core.rb @@ -0,0 +1,86 @@ +# encoding: utf-8 + +module LeapCli + module Macro + + # + # return a fingerprint for a x509 certificate + # + def fingerprint(filename) + "SHA256: " + X509.fingerprint("SHA256", Path.named_path(filename)) + end + + # + # Creates a hash from the ssh key info in users directory, for use in + # updating authorized_keys file. Additionally, the 'monitor' public key is + # included, which is used by the monitor nodes to run particular commands + # remotely. + # + def authorized_keys + hash = {} + keys = Dir.glob(Path.named_path([:user_ssh, '*'])) + keys.sort.each do |keyfile| + ssh_type, ssh_key = File.read(keyfile, :encoding => 'UTF-8').strip.split(" ") + name = File.basename(File.dirname(keyfile)) + hash[name] = { + "type" => ssh_type, + "key" => ssh_key + } + end + ssh_type, ssh_key = File.read(Path.named_path(:monitor_pub_key), :encoding => 'UTF-8').strip.split(" ") + hash[Leap::Platform.monitor_username] = { + "type" => ssh_type, + "key" => ssh_key + } + hash + end + + def assert(assertion) + if instance_eval(assertion) + true + else + raise AssertionFailed.new(assertion) + end + end + + # + # applies a JSON partial to this node + # + def apply_partial(partial_path) + manager.partials(partial_path).each do |partial_data| + self.deep_merge!(partial_data) + end + end + + # + # If at first you don't succeed, then it is time to give up. + # + # try{} returns nil if anything in the block throws an exception. + # + # You can wrap something that might fail in `try`, like so. + # + # "= try{ nodes[:services => 'tor'].first.ip_address } " + # + def try(&block) + yield + rescue NoMethodError + nil + end + + protected + + # + # returns a node list, if argument is not already one + # + def listify(node_list) + if node_list.is_a? Config::ObjectList + node_list + elsif node_list.is_a? Config::Object + Config::ObjectList.new(node_list) + else + raise ArgumentError, 'argument must be a node or node list, not a `%s`' % node_list.class, caller + end + end + + end +end diff --git a/provider_base/lib/macros/files.rb b/provider_base/lib/macros/files.rb new file mode 100644 index 00000000..0a491325 --- /dev/null +++ b/provider_base/lib/macros/files.rb @@ -0,0 +1,79 @@ +# encoding: utf-8 + +## +## FILES +## + +module LeapCli + module Macro + + # + # inserts the contents of a file + # + def file(filename, options={}) + if filename.is_a? Symbol + filename = [filename, @node.name] + end + filepath = Path.find_file(filename) + if filepath + if filepath =~ /\.erb$/ + ERB.new(File.read(filepath, :encoding => 'UTF-8'), nil, '%<>').result(binding) + else + File.read(filepath, :encoding => 'UTF-8') + end + else + raise FileMissing.new(Path.named_path(filename), options) + "" + end + end + + # + # like #file, but allow missing files + # + def try_file(filename) + return file(filename) + rescue FileMissing + return nil + end + + # + # returns what the file path will be, once the file is rsynced to the server. + # an internal list of discovered file paths is saved, in order to rsync these files when needed. + # + # notes: + # + # * argument 'path' is relative to Path.provider/files or Path.provider_base/files + # * the path returned by this method is absolute + # * the path stored for use later by rsync is relative to Path.provider + # * if the path does not exist locally, but exists in provider_base, then the default file from + # provider_base is copied locally. this is required for rsync to work correctly. + # + def file_path(path) + if path.is_a? Symbol + path = [path, @node.name] + end + actual_path = Path.find_file(path) + if actual_path.nil? + Util::log 2, :skipping, "file_path(\"#{path}\") because there is no such file." + nil + else + if actual_path =~ /^#{Regexp.escape(Path.provider_base)}/ + # if file is under Path.provider_base, we must copy the default file to + # to Path.provider in order for rsync to be able to sync the file. + local_provider_path = actual_path.sub(/^#{Regexp.escape(Path.provider_base)}/, Path.provider) + FileUtils.mkdir_p File.dirname(local_provider_path), :mode => 0700 + FileUtils.install actual_path, local_provider_path, :mode => 0600 + Util.log :created, Path.relative_path(local_provider_path) + actual_path = local_provider_path + end + if File.directory?(actual_path) && actual_path !~ /\/$/ + actual_path += '/' # ensure directories end with /, important for building rsync command + end + relative_path = Path.relative_path(actual_path) + @node.file_paths << relative_path + @node.manager.provider.hiera_sync_destination + '/' + relative_path + end + end + + end +end \ No newline at end of file diff --git a/provider_base/lib/macros/haproxy.rb b/provider_base/lib/macros/haproxy.rb new file mode 100644 index 00000000..db0d4db8 --- /dev/null +++ b/provider_base/lib/macros/haproxy.rb @@ -0,0 +1,68 @@ +# encoding: utf-8 + +## +## HAPROXY +## + +module LeapCli + module Macro + + # + # creates a hash suitable for configuring haproxy. the key is the node name of the server we are proxying to. + # + # * node_list - a hash of nodes for the haproxy servers + # * stunnel_client - contains the mappings to local ports for each server node. + # * non_stunnel_port - in case self is included in node_list, the port to connect to. + # + # 1000 weight is used for nodes in the same location. + # 100 otherwise. + # + def haproxy_servers(node_list, stunnel_clients, non_stunnel_port=nil) + default_weight = 10 + local_weight = 100 + + # record the hosts_file + hostnames(node_list) + + # create a simple map for node name -> local stunnel accept port + accept_ports = stunnel_clients.inject({}) do |hsh, stunnel_entry| + name = stunnel_entry.first.sub /_[0-9]+$/, '' + hsh[name] = stunnel_entry.last['accept_port'] + hsh + end + + # if one the nodes in the node list is ourself, then there will not be a stunnel to it, + # but we need to include it anyway in the haproxy config. + if node_list[self.name] && non_stunnel_port + accept_ports[self.name] = non_stunnel_port + end + + # create the first pass of the servers hash + servers = node_list.values.inject(Config::ObjectList.new) do |hsh, node| + weight = default_weight + if self['location'] && node['location'] + if self.location['name'] == node.location['name'] + weight = local_weight + end + end + hsh[node.name] = Config::Object[ + 'backup', false, + 'host', 'localhost', + 'port', accept_ports[node.name] || 0, + 'weight', weight + ] + hsh + end + + # if there are some local servers, make the others backup + if servers.detect{|k,v| v.weight == local_weight} + servers.each do |k,server| + server['backup'] = server['weight'] == default_weight + end + end + + return servers + end + + end +end \ No newline at end of file diff --git a/provider_base/lib/macros/hosts.rb b/provider_base/lib/macros/hosts.rb new file mode 100644 index 00000000..8a4058a5 --- /dev/null +++ b/provider_base/lib/macros/hosts.rb @@ -0,0 +1,63 @@ +# encoding: utf-8 + +module LeapCli + module Macro + + ## + ## HOSTS + ## + + # + # records the list of hosts that are encountered for this node + # + def hostnames(nodes) + @referenced_nodes ||= Config::ObjectList.new + nodes = listify(nodes) + nodes.each_node do |node| + @referenced_nodes[node.name] ||= node + end + return nodes.values.collect {|node| node.domain.name} + end + + # + # Generates entries needed for updating /etc/hosts on a node (as a hash). + # + # Argument `nodes` can be nil or a list of nodes. If nil, only include the + # IPs of the other nodes this @node as has encountered (plus all mx nodes). + # + # Also, for virtual machines, we use the local address if this @node is in + # the same location as the node in question. + # + # We include the ssh public key for each host, so that the hash can also + # be used to generate the /etc/ssh/known_hosts + # + def hosts_file(nodes=nil) + if nodes.nil? + if @referenced_nodes && @referenced_nodes.any? + nodes = @referenced_nodes + nodes = nodes.merge(nodes_like_me[:services => 'mx']) # all nodes always need to communicate with mx nodes. + end + end + return {} unless nodes + hosts = {} + my_location = @node['location'] ? @node['location']['name'] : nil + nodes.each_node do |node| + hosts[node.name] = {'ip_address' => node.ip_address, 'domain_internal' => node.domain.internal, 'domain_full' => node.domain.full} + node_location = node['location'] ? node['location']['name'] : nil + if my_location == node_location + if facts = @node.manager.facts[node.name] + if facts['ec2_public_ipv4'] + hosts[node.name]['ip_address'] = facts['ec2_public_ipv4'] + end + end + end + host_pub_key = Util::read_file([:node_ssh_pub_key,node.name]) + if host_pub_key + hosts[node.name]['host_pub_key'] = host_pub_key + end + end + hosts + end + + end +end \ No newline at end of file diff --git a/provider_base/lib/macros/nodes.rb b/provider_base/lib/macros/nodes.rb new file mode 100644 index 00000000..0c6668a0 --- /dev/null +++ b/provider_base/lib/macros/nodes.rb @@ -0,0 +1,88 @@ +# encoding: utf-8 + +## +## node related macros +## + +module LeapCli + module Macro + + # + # the list of all the nodes + # + def nodes + global.nodes + end + + # + # grab an environment appropriate provider + # + def provider + global.env(@node.environment).provider + end + + # + # returns a list of nodes that match the same environment + # + # if @node.environment is not set, we return other nodes + # where environment is not set. + # + def nodes_like_me + nodes[:environment => @node.environment] + end + + # + # returns a list of nodes that match the location name + # and environment of @node. + # + def nodes_near_me + if @node['location'] && @node['location']['name'] + nodes_like_me['location.name' => @node.location.name] + else + nodes_like_me['location' => nil] + end + end + + # + # + # picks a node out from the node list in such a way that: + # + # (1) which nodes picked which nodes is saved in secrets.json + # (2) when other nodes call this macro with the same node list, they are guaranteed to get a different node + # (3) if all the nodes in the pick_node list have been picked, remaining nodes are distributed randomly. + # + # if the node_list is empty, an exception is raised. + # if node_list size is 1, then that node is returned and nothing is + # memorized via the secrets.json file. + # + # `label` is needed to distinguish between pools of nodes for different purposes. + # + # TODO: more evenly balance after all the nodes have been picked. + # + def pick_node(label, node_list) + if node_list.any? + if node_list.size == 1 + return node_list.values.first + else + secrets_key = "pick_node(:#{label},#{node_list.keys.sort.join(',')})" + secrets_value = @manager.secrets.retrieve(secrets_key, @node.environment) || {} + secrets_value[@node.name] ||= begin + node_to_pick = nil + node_list.each_node do |node| + next if secrets_value.values.include?(node.name) + node_to_pick = node.name + end + node_to_pick ||= secrets_value.values.shuffle.first # all picked already, so pick a random one. + node_to_pick + end + picked_node_name = secrets_value[@node.name] + @manager.secrets.set(secrets_key, secrets_value, @node.environment) + return node_list[picked_node_name] + end + else + raise ArgumentError.new('pick_node(node_list): node_list cannot be empty') + end + end + + end +end \ No newline at end of file diff --git a/provider_base/lib/macros/secrets.rb b/provider_base/lib/macros/secrets.rb new file mode 100644 index 00000000..51bf3971 --- /dev/null +++ b/provider_base/lib/macros/secrets.rb @@ -0,0 +1,39 @@ +# encoding: utf-8 + +require 'base32' + +module LeapCli + module Macro + + # + # inserts a named secret, generating it if needed. + # + # manager.export_secrets should be called later to capture any newly generated secrets. + # + # +length+ is the character length of the generated password. + # + def secret(name, length=32) + @manager.secrets.set(name, Util::Secret.generate(length), @node[:environment]) + end + + # inserts a base32 encoded secret + def base32_secret(name, length=20) + @manager.secrets.set(name, Base32.encode(Util::Secret.generate(length)), @node[:environment]) + end + + # Picks a random obfsproxy port from given range + def rand_range(name, range) + @manager.secrets.set(name, rand(range), @node[:environment]) + end + + # + # inserts an hexidecimal secret string, generating it if needed. + # + # +bit_length+ is the bits in the secret, (ie length of resulting hex string will be bit_length/4) + # + def hex_secret(name, bit_length=128) + @manager.secrets.set(name, Util::Secret.generate_hex(bit_length), @node[:environment]) + end + + end +end \ No newline at end of file diff --git a/provider_base/lib/macros/stunnel.rb b/provider_base/lib/macros/stunnel.rb new file mode 100644 index 00000000..f16308c7 --- /dev/null +++ b/provider_base/lib/macros/stunnel.rb @@ -0,0 +1,95 @@ +## +## STUNNEL +## + +# +# About stunnel +# -------------------------- +# +# The network looks like this: +# +# From the client's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# consumer app -> localhost:accept_port -> connect:connect_port -> ?? +# +# From the server's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# ?? -> *:accept_port -> localhost:connect_port -> service +# + +module LeapCli + module Macro + + # + # stunnel configuration for the client side. + # + # +node_list+ is a ObjectList of nodes running stunnel servers. + # + # +port+ is the real port of the ultimate service running on the servers + # that the client wants to connect to. + # + # * accept_port is the port on localhost to which local clients + # can connect. it is auto generated serially. + # + # * connect_port is the port on the stunnel server to connect to. + # it is auto generated from the +port+ argument. + # + # generates an entry appropriate to be passed directly to + # create_resources(stunnel::service, hiera('..'), defaults) + # + # local ports are automatically generated, starting at 4000 + # and incrementing in sorted order (by node name). + # + def stunnel_client(node_list, port, options={}) + @next_stunnel_port ||= 4000 + node_list = listify(node_list) + hostnames(node_list) # record the hosts + result = Config::ObjectList.new + node_list.each_node do |node| + if node.name != self.name || options[:include_self] + result["#{node.name}_#{port}"] = Config::Object[ + 'accept_port', @next_stunnel_port, + 'connect', node.domain.internal, + 'connect_port', stunnel_port(port), + 'original_port', port + ] + @next_stunnel_port += 1 + end + end + result + end + + # + # generates a stunnel server entry. + # + # +port+ is the real port targeted service. + # + # * `accept_port` is the publicly bound port + # * `connect_port` is the port that the local service is running on. + # + def stunnel_server(port) + { + "accept_port" => stunnel_port(port), + "connect_port" => port + } + end + + private + + # + # maps a real port to a stunnel port (used as the connect_port in the client config + # and the accept_port in the server config) + # + def stunnel_port(port) + port = port.to_i + if port < 50000 + return port + 10000 + else + return port - 10000 + end + end + + end +end \ No newline at end of file -- cgit v1.2.3 From 73674f928756321a6b35f06a62a0ff1cf0ff479b Mon Sep 17 00:00:00 2001 From: elijah Date: Sat, 21 Jun 2014 02:51:51 -0700 Subject: fix stunnel entries in mx.json and webapp.json --- provider_base/services/mx.json | 9 +++++++-- provider_base/services/webapp.json | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json index 30a19d9a..1f0e613e 100644 --- a/provider_base/services/mx.json +++ b/provider_base/services/mx.json @@ -1,9 +1,14 @@ { "stunnel": { - "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" + "clients": { + "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" + } }, "haproxy": { - "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.couch_client)" + "couch": { + "listen_port": 4096, + "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client)" + } }, "couchdb_leap_mx_user": { "username": "= global.services[:couchdb].couch.users[:leap_mx].username", diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index d268a020..1b550af9 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -32,10 +32,15 @@ ] }, "stunnel": { - "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" + "clients": { + "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" + } }, "haproxy": { - "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.couch_client, global.services[:couchdb].couch.port)" + "couch": { + "listen_port": 4096, + "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, global.services[:couchdb].couch.port)" + } }, "definition_files": { "provider": "= file :provider_json_template", -- cgit v1.2.3 From 26cbd9a70c4b19b591a6f865812f0ad98de2668c Mon Sep 17 00:00:00 2001 From: elijah Date: Sat, 21 Jun 2014 02:52:43 -0700 Subject: haproxy: support read only couchdb mirrors --- provider_base/lib/macros/haproxy.rb | 11 +++--- puppet/modules/site_haproxy/manifests/init.pp | 42 +++++++++++----------- puppet/modules/site_haproxy/templates/couch.erb | 32 +++++++++++++++++ .../modules/site_haproxy/templates/haproxy.cfg.erb | 11 ++++++ .../site_haproxy/templates/haproxy_couchdb.cfg.erb | 23 ------------ 5 files changed, 70 insertions(+), 49 deletions(-) create mode 100644 puppet/modules/site_haproxy/templates/couch.erb create mode 100644 puppet/modules/site_haproxy/templates/haproxy.cfg.erb delete mode 100644 puppet/modules/site_haproxy/templates/haproxy_couchdb.cfg.erb diff --git a/provider_base/lib/macros/haproxy.rb b/provider_base/lib/macros/haproxy.rb index db0d4db8..c0f9ede5 100644 --- a/provider_base/lib/macros/haproxy.rb +++ b/provider_base/lib/macros/haproxy.rb @@ -40,17 +40,18 @@ module LeapCli # create the first pass of the servers hash servers = node_list.values.inject(Config::ObjectList.new) do |hsh, node| weight = default_weight - if self['location'] && node['location'] - if self.location['name'] == node.location['name'] - weight = local_weight - end - end + try { + weight = local_weight if self.location.name == node.location.name + } hsh[node.name] = Config::Object[ 'backup', false, 'host', 'localhost', 'port', accept_ports[node.name] || 0, 'weight', weight ] + if node.services.include?('couchdb') + hsh[node.name]['writable'] = node.couch.mode != 'mirror' + end hsh end diff --git a/puppet/modules/site_haproxy/manifests/init.pp b/puppet/modules/site_haproxy/manifests/init.pp index 6bcf3f5c..b28ce80e 100644 --- a/puppet/modules/site_haproxy/manifests/init.pp +++ b/puppet/modules/site_haproxy/manifests/init.pp @@ -2,25 +2,25 @@ class site_haproxy { $haproxy = hiera('haproxy') class { 'haproxy': - enable => true, - manage_service => true, - global_options => { - 'log' => '127.0.0.1 local0', - 'maxconn' => '4096', - 'stats' => 'socket /var/run/haproxy.sock user haproxy group haproxy', - 'chroot' => '/usr/share/haproxy', - 'user' => 'haproxy', - 'group' => 'haproxy', - 'daemon' => '' - }, - defaults_options => { - 'log' => 'global', - 'retries' => '3', - 'option' => 'redispatch', - 'timeout connect' => '4000', - 'timeout client' => '20000', - 'timeout server' => '20000' - } + enable => true, + manage_service => true, + global_options => { + 'log' => '127.0.0.1 local0', + 'maxconn' => '4096', + 'stats' => 'socket /var/run/haproxy.sock user haproxy group haproxy', + 'chroot' => '/usr/share/haproxy', + 'user' => 'haproxy', + 'group' => 'haproxy', + 'daemon' => '' + }, + defaults_options => { + 'log' => 'global', + 'retries' => '3', + 'option' => 'redispatch', + 'timeout connect' => '4000', + 'timeout client' => '20000', + 'timeout server' => '20000' + } } # monitor haproxy @@ -34,8 +34,8 @@ class site_haproxy { concat::fragment { 'leap_haproxy_webapp_couchdb': target => '/etc/haproxy/haproxy.cfg', order => '20', - content => template('site_haproxy/haproxy_couchdb.cfg.erb'), + content => template('site_haproxy/haproxy.cfg.erb'), } - + include site_check_mk::agent::haproxy } diff --git a/puppet/modules/site_haproxy/templates/couch.erb b/puppet/modules/site_haproxy/templates/couch.erb new file mode 100644 index 00000000..baa31486 --- /dev/null +++ b/puppet/modules/site_haproxy/templates/couch.erb @@ -0,0 +1,32 @@ +frontend couch + bind localhost:<%= @listen_port %> + mode http + option httplog + option dontlognull + option http-server-close # use client keep-alive, but close server connection. + use_backend couch_write if METH_POST + default_backend couch_read + +backend couch_write + mode http + balance roundrobin + option httpchk GET / # health check using simple get to root + option allbackups # balance among all backups, not just one. + default-server inter 3000 fastinter 1000 downinter 1000 rise 2 fall 1 +<%- @servers.sort.each do |name,server| -%> +<%- next unless server['writable'] -%> + # <%=name%> + server couchdb_<%=server['port']%> <%=server['host']%>:<%=server['port']%> <%='backup' if server['backup']%> weight <%=server['weight']%> check +<%- end -%> + +backend couch_read + mode http + balance roundrobin + option httpchk GET / # health check using simple get to root + option allbackups # balance among all backups, not just one. + default-server inter 3000 fastinter 1000 downinter 1000 rise 2 fall 1 +<%- @servers.sort.each do |name,server| -%> + # <%=name%> + server couchdb_<%=server['port']%> <%=server['host']%>:<%=server['port']%> <%='backup' if server['backup']%> weight <%=server['weight']%> check +<%- end -%> + diff --git a/puppet/modules/site_haproxy/templates/haproxy.cfg.erb b/puppet/modules/site_haproxy/templates/haproxy.cfg.erb new file mode 100644 index 00000000..8311b1a5 --- /dev/null +++ b/puppet/modules/site_haproxy/templates/haproxy.cfg.erb @@ -0,0 +1,11 @@ +<%- @haproxy.each do |frontend, options| -%> +<%- if options['servers'] -%> + +## +## <%= frontend %> +## + +<%= scope.function_templatewlv(["site_haproxy/#{frontend}.erb", options]) %> +<%- end -%> +<%- end -%> + diff --git a/puppet/modules/site_haproxy/templates/haproxy_couchdb.cfg.erb b/puppet/modules/site_haproxy/templates/haproxy_couchdb.cfg.erb deleted file mode 100644 index 1fa01b96..00000000 --- a/puppet/modules/site_haproxy/templates/haproxy_couchdb.cfg.erb +++ /dev/null @@ -1,23 +0,0 @@ - -listen bigcouch-in - mode http - balance roundrobin - option httplog - option dontlognull - option httpchk GET / # health check using simple get to root - option http-server-close # use client keep-alive, but close server connection. - option allbackups # balance among all backups, not just one. - - bind localhost:4096 - - default-server inter 3000 fastinter 1000 downinter 1000 rise 2 fall 1 - -<%- if @haproxy['servers'] -%> -<%- @haproxy['servers'].sort.each do |name,server| -%> -<%- backup = server['backup'] ? 'backup' : '' -%> - # <%=name%> - server couchdb_<%=server['port']%> <%=server['host']%>:<%=server['port']%> <%=backup%> weight <%=server['weight']%> check - -<%- end -%> -<%- end -%> - -- cgit v1.2.3 From 5075fdeee3c8b70d39a2f6105d8e1e33c6843eb4 Mon Sep 17 00:00:00 2001 From: Azul Date: Mon, 23 Jun 2014 21:49:38 +0200 Subject: minor: fix typo in replication user roles --- puppet/modules/site_couchdb/manifests/add_users.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_couchdb/manifests/add_users.pp b/puppet/modules/site_couchdb/manifests/add_users.pp index 0585da27..2f734ed4 100644 --- a/puppet/modules/site_couchdb/manifests/add_users.pp +++ b/puppet/modules/site_couchdb/manifests/add_users.pp @@ -57,7 +57,7 @@ class site_couchdb::add_users { ## replication couchdb user ## read/write: all databases for replication couchdb::add_user { $site_couchdb::couchdb_replication_user: - roles => '["repliction"]', + roles => '["replication"]', pw => $site_couchdb::couchdb_replication_pw, salt => $site_couchdb::couchdb_replication_salt, require => Couchdb::Query::Setup['localhost'] -- cgit v1.2.3 From 813f840cceb284c38dcedea1577d125e62e280f0 Mon Sep 17 00:00:00 2001 From: Azul Date: Mon, 23 Jun 2014 21:50:50 +0200 Subject: hand replication credentials to tapicero --- provider_base/services/_couchdb_mirror.json | 7 +++++-- puppet/modules/tapicero/manifests/init.pp | 3 ++- puppet/modules/tapicero/templates/tapicero.yaml.erb | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/provider_base/services/_couchdb_mirror.json b/provider_base/services/_couchdb_mirror.json index a496804d..6a3402bd 100644 --- a/provider_base/services/_couchdb_mirror.json +++ b/provider_base/services/_couchdb_mirror.json @@ -12,7 +12,10 @@ "replication": { // for now, pick the first close one, or the first one. // in the future, maybe use haproxy to balance among all the masters - "masters": "= try{pick_node(:couch_master,nodes_near_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')} || try{pick_node(:couch_master,nodes_like_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')}" + "masters": "= try{pick_node(:couch_master,nodes_near_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')} || try{pick_node(:couch_master,nodes_like_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')}", + "username": "replication", + "password": "= secret :couch_replication_password", + "role": "replication" } } -} \ No newline at end of file +} diff --git a/puppet/modules/tapicero/manifests/init.pp b/puppet/modules/tapicero/manifests/init.pp index 1db75eb0..fd8c1344 100644 --- a/puppet/modules/tapicero/manifests/init.pp +++ b/puppet/modules/tapicero/manifests/init.pp @@ -12,7 +12,8 @@ class tapicero { $couchdb_soledad_user = $couchdb_users['soledad']['username'] $couchdb_leap_mx_user = $couchdb_users['leap_mx']['username'] - $couchdb_mirror = $couchdb['mode'] == 'mirror' + $couchdb_mode = $couchdb['mode'] + $couchdb_replication = $couchdb['replication'] Class['site_config::default'] -> Class['tapicero'] diff --git a/puppet/modules/tapicero/templates/tapicero.yaml.erb b/puppet/modules/tapicero/templates/tapicero.yaml.erb index 3a5f821e..182a6aa6 100644 --- a/puppet/modules/tapicero/templates/tapicero.yaml.erb +++ b/puppet/modules/tapicero/templates/tapicero.yaml.erb @@ -24,7 +24,8 @@ log_level: info options: # prefix for per user databases: db_prefix: "user-" - mirror: <%= @couchdb_mirror %> + mode: <%= @couchdb_mode %> + replication: <%= @couchdb_replication %> # security settings to be used for the per user databases security: -- cgit v1.2.3 From 7778b20479d4d6789948dc24904ef9302980d983 Mon Sep 17 00:00:00 2001 From: Azul Date: Mon, 23 Jun 2014 22:04:59 +0200 Subject: create netrc files for all users with new puppet_couchdb This only works with the latest patch to puppet_couchdb --- puppet/modules/site_couchdb/manifests/mirror.pp | 2 -- puppet/modules/site_couchdb/manifests/setup.pp | 11 +++++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/puppet/modules/site_couchdb/manifests/mirror.pp b/puppet/modules/site_couchdb/manifests/mirror.pp index 2a44b1e9..abe35c4c 100644 --- a/puppet/modules/site_couchdb/manifests/mirror.pp +++ b/puppet/modules/site_couchdb/manifests/mirror.pp @@ -9,8 +9,6 @@ class site_couchdb::mirror { chttpd_bind_address => '127.0.0.1' } - # Couchdb databases - $masters = $site_couchdb::couchdb_config['replication']['masters'] $master_node_names = keys($site_couchdb::couchdb_config['replication']['masters']) $master_node = $masters[$master_node_names[0]] diff --git a/puppet/modules/site_couchdb/manifests/setup.pp b/puppet/modules/site_couchdb/manifests/setup.pp index e398356b..69bd1c6a 100644 --- a/puppet/modules/site_couchdb/manifests/setup.pp +++ b/puppet/modules/site_couchdb/manifests/setup.pp @@ -10,11 +10,18 @@ class site_couchdb::setup { ensure => absent } - # /etc/couchdb/couchdb.netrc is deployed by couchdb::query::setup + $user = $site_couchdb::couchdb_admin_user + + # /etc/couchdb/couchdb-admin.netrc is deployed by couchdb::query::setup + # we symlink to couchdb.netrc for puppet commands. # we symlink this to /root/.netrc for couchdb_scripts (eg. backup) # and makes life easier for the admin (i.e. using curl/wget without # passing credentials) file { + '/etc/couchdb/couchdb.netrc': + ensure => link, + target => "/etc/couchdb/couchdb-${user}.netrc"; + '/root/.netrc': ensure => link, target => '/etc/couchdb/couchdb.netrc'; @@ -24,7 +31,7 @@ class site_couchdb::setup { } couchdb::query::setup { 'localhost': - user => $site_couchdb::couchdb_admin_user, + user => $user, pw => $site_couchdb::couchdb_admin_pw, } -- cgit v1.2.3 From 6cd7beed954e594110f04dbd0637cb17168ba4dd Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 25 Jun 2014 18:07:13 -0700 Subject: fix couchdb tests for plain couch --- tests/white-box/couchdb.rb | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb index a6ad0f6a..58a24731 100644 --- a/tests/white-box/couchdb.rb +++ b/tests/white-box/couchdb.rb @@ -10,8 +10,10 @@ class CouchDB < LeapTest def test_00_Are_daemons_running? assert_running 'tapicero' - assert_running 'bin/beam' - assert_running 'bin/epmd' + if multimaster? + assert_running 'bin/beam' + assert_running 'bin/epmd' + end pass end @@ -29,6 +31,7 @@ class CouchDB < LeapTest # compare the configured nodes to the nodes that are actually listed in bigcouch # def test_02_Is_cluster_membership_ok? + skip "not a multimaster node" unless multimater? url = couchdb_backend_url("/nodes/_all_docs") neighbors = assert_property('couch.bigcouch.neighbors') neighbors << assert_property('domain.full') @@ -48,6 +51,7 @@ class CouchDB < LeapTest # this seems backward to me, so it might be the other way around. # def test_03_Are_configured_nodes_online? + skip "not a multimaster node" unless multimater? url = couchdb_url("/_membership", :user => 'admin') assert_get(url) do |body| response = JSON.parse(body) @@ -66,11 +70,11 @@ class CouchDB < LeapTest end def test_04_Do_ACL_users_exist? - acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'tapicero', 'webapp'] + acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'tapicero', 'webapp', 'replication'] url = couchdb_backend_url("/_users/_all_docs") assert_get(url) do |body| response = JSON.parse(body) - assert_equal 6, response['total_rows'] + assert_equal acl_users.count, response['total_rows'] actual_users = response['rows'].map{|row| row['id'].sub(/^org.couchdb.user:/, '') } assert_equal acl_users.sort, actual_users.sort end @@ -126,7 +130,16 @@ class CouchDB < LeapTest end def couchdb_backend_url(path="") - couchdb_url(path, :port => "5986") # TODO: admin port is hardcoded for now but should be configurable. + # TODO: admin port is hardcoded for now but should be configurable. + couchdb_url(path, multimaster? && "5986") + end + + def multimaster? + mode == "multimaster" + end + + def mode + assert_property('couch.mode') end end -- cgit v1.2.3 From 3a5a235a975ae25ced89b60c247b6d1d20174e8b Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 25 Jun 2014 18:07:28 -0700 Subject: update couchdb submodule --- puppet/modules/couchdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/couchdb b/puppet/modules/couchdb index c8f5443e..8bc5ed43 160000 --- a/puppet/modules/couchdb +++ b/puppet/modules/couchdb @@ -1 +1 @@ -Subproject commit c8f5443e0998d3d3d43505ff5a6fdf8c438d6c24 +Subproject commit 8bc5ed434c124457b7467140152602c67a9547c5 -- cgit v1.2.3 From d6eabb09f978f1501b8b797d28e949a2e00ac82e Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 25 Jun 2014 23:10:29 -0700 Subject: lint site_couchdb --- puppet/modules/site_couchdb/manifests/init.pp | 80 +++++++++++++-------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 6f7e974e..5a4fb936 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -1,46 +1,46 @@ class site_couchdb { tag 'leap_service' - $couchdb_config = hiera('couch') - $couchdb_users = $couchdb_config['users'] - - $couchdb_admin = $couchdb_users['admin'] - $couchdb_admin_user = $couchdb_admin['username'] - $couchdb_admin_pw = $couchdb_admin['password'] - $couchdb_admin_salt = $couchdb_admin['salt'] - - $couchdb_leap_mx = $couchdb_users['leap_mx'] - $couchdb_leap_mx_user = $couchdb_leap_mx['username'] - $couchdb_leap_mx_pw = $couchdb_leap_mx['password'] - $couchdb_leap_mx_salt = $couchdb_leap_mx['salt'] - - $couchdb_nickserver = $couchdb_users['nickserver'] - $couchdb_nickserver_user = $couchdb_nickserver['username'] - $couchdb_nickserver_pw = $couchdb_nickserver['password'] - $couchdb_nickserver_salt = $couchdb_nickserver['salt'] - - $couchdb_soledad = $couchdb_users['soledad'] - $couchdb_soledad_user = $couchdb_soledad['username'] - $couchdb_soledad_pw = $couchdb_soledad['password'] - $couchdb_soledad_salt = $couchdb_soledad['salt'] - - $couchdb_tapicero = $couchdb_users['tapicero'] - $couchdb_tapicero_user = $couchdb_tapicero['username'] - $couchdb_tapicero_pw = $couchdb_tapicero['password'] - $couchdb_tapicero_salt = $couchdb_tapicero['salt'] - - $couchdb_webapp = $couchdb_users['webapp'] - $couchdb_webapp_user = $couchdb_webapp['username'] - $couchdb_webapp_pw = $couchdb_webapp['password'] - $couchdb_webapp_salt = $couchdb_webapp['salt'] - - $couchdb_replication = $couchdb_users['replication'] - $couchdb_replication_user= $couchdb_replication['username'] - $couchdb_replication_pw = $couchdb_replication['password'] - $couchdb_replication_salt= $couchdb_replication['salt'] - - $couchdb_backup = $couchdb_config['backup'] - $couchdb_mode = $couchdb_config['mode'] + $couchdb_config = hiera('couch') + $couchdb_users = $couchdb_config['users'] + + $couchdb_admin = $couchdb_users['admin'] + $couchdb_admin_user = $couchdb_admin['username'] + $couchdb_admin_pw = $couchdb_admin['password'] + $couchdb_admin_salt = $couchdb_admin['salt'] + + $couchdb_leap_mx = $couchdb_users['leap_mx'] + $couchdb_leap_mx_user = $couchdb_leap_mx['username'] + $couchdb_leap_mx_pw = $couchdb_leap_mx['password'] + $couchdb_leap_mx_salt = $couchdb_leap_mx['salt'] + + $couchdb_nickserver = $couchdb_users['nickserver'] + $couchdb_nickserver_user = $couchdb_nickserver['username'] + $couchdb_nickserver_pw = $couchdb_nickserver['password'] + $couchdb_nickserver_salt = $couchdb_nickserver['salt'] + + $couchdb_soledad = $couchdb_users['soledad'] + $couchdb_soledad_user = $couchdb_soledad['username'] + $couchdb_soledad_pw = $couchdb_soledad['password'] + $couchdb_soledad_salt = $couchdb_soledad['salt'] + + $couchdb_tapicero = $couchdb_users['tapicero'] + $couchdb_tapicero_user = $couchdb_tapicero['username'] + $couchdb_tapicero_pw = $couchdb_tapicero['password'] + $couchdb_tapicero_salt = $couchdb_tapicero['salt'] + + $couchdb_webapp = $couchdb_users['webapp'] + $couchdb_webapp_user = $couchdb_webapp['username'] + $couchdb_webapp_pw = $couchdb_webapp['password'] + $couchdb_webapp_salt = $couchdb_webapp['salt'] + + $couchdb_replication = $couchdb_users['replication'] + $couchdb_replication_user = $couchdb_replication['username'] + $couchdb_replication_pw = $couchdb_replication['password'] + $couchdb_replication_salt = $couchdb_replication['salt'] + + $couchdb_backup = $couchdb_config['backup'] + $couchdb_mode = $couchdb_config['mode'] if $couchdb_mode == "multimaster" { include site_couchdb::bigcouch } if $couchdb_mode == "master" { include site_couchdb::master } -- cgit v1.2.3 From 8d5c14b26b6f95285d9ed2b17c61d18d5de005e6 Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 25 Jun 2014 23:54:16 -0700 Subject: fix tests for new 0.6 format --- tests/white-box/couchdb.rb | 4 ++-- tests/white-box/network.rb | 33 +++++++++++++++------------------ 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb index 58a24731..74bd47bf 100644 --- a/tests/white-box/couchdb.rb +++ b/tests/white-box/couchdb.rb @@ -31,7 +31,7 @@ class CouchDB < LeapTest # compare the configured nodes to the nodes that are actually listed in bigcouch # def test_02_Is_cluster_membership_ok? - skip "not a multimaster node" unless multimater? + return unless multimaster? url = couchdb_backend_url("/nodes/_all_docs") neighbors = assert_property('couch.bigcouch.neighbors') neighbors << assert_property('domain.full') @@ -51,7 +51,7 @@ class CouchDB < LeapTest # this seems backward to me, so it might be the other way around. # def test_03_Are_configured_nodes_online? - skip "not a multimaster node" unless multimater? + return unless multimaster? url = couchdb_url("/_membership", :user => 'admin') assert_get(url) do |body| response = JSON.parse(body) diff --git a/tests/white-box/network.rb b/tests/white-box/network.rb index e0b0339d..118861a7 100644 --- a/tests/white-box/network.rb +++ b/tests/white-box/network.rb @@ -28,29 +28,26 @@ class Network < LeapTest def test_02_Is_stunnel_running? if $node['stunnel'] good_stunnel_pids = [] - $node['stunnel'].each do |stunnel_type, stunnel_configs| - if stunnel_type =~ /_clients?$/ - stunnel_configs.each do |stunnel_name, stunnel_conf| - config_file_name = "/etc/stunnel/#{stunnel_name}.conf" - processes = pgrep(config_file_name) - assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`" - good_stunnel_pids += processes.map{|ps| ps[:pid]} - assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.' - assert_tcp_socket('localhost', port) - end - elsif stunnel_type =~ /_server$/ - config_file_name = "/etc/stunnel/#{stunnel_type}.conf" + $node['stunnel']['clients'].each do |stunnel_type, stunnel_configs| + stunnel_configs.each do |stunnel_name, stunnel_conf| + config_file_name = "/etc/stunnel/#{stunnel_name}.conf" processes = pgrep(config_file_name) assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`" good_stunnel_pids += processes.map{|ps| ps[:pid]} - assert accept = stunnel_configs['accept'], "Field `accept` must be present in property `stunnel.#{stunnel_type}`" - assert_tcp_socket('localhost', accept) - assert connect = stunnel_configs['connect'], "Field `connect` must be present in property `stunnel.#{stunnel_type}`" - assert_tcp_socket(*connect.split(':')) - else - skip "Unknown stunnel type `#{stunnel_type}`" + assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.' + assert_tcp_socket('localhost', port) end end + $node['stunnel']['servers'].each do |stunnel_name, stunnel_conf| + config_file_name = "/etc/stunnel/#{stunnel_name}.conf" + processes = pgrep(config_file_name) + assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`" + good_stunnel_pids += processes.map{|ps| ps[:pid]} + assert accept_port = stunnel_conf['accept_port'], "Field `accept` must be present in property `stunnel.servers.#{stunnel_name}`" + assert_tcp_socket('localhost', accept_port) + assert connect_port = stunnel_conf['connect_port'], "Field `connect` must be present in property `stunnel.servers.#{stunnel_name}`" + assert_tcp_socket('localhost', connect_port) + end all_stunnel_pids = pgrep('/usr/bin/stunnel').collect{|process| process[:pid]}.uniq assert_equal good_stunnel_pids.sort, all_stunnel_pids.sort, "There should not be any extra stunnel processes that are not configured in /etc/stunnel" pass -- cgit v1.2.3 From 89669fbc6d43590f73055c00ee0bb415d5c8eb3e Mon Sep 17 00:00:00 2001 From: Christoph Date: Thu, 26 Jun 2014 15:14:46 +0200 Subject: reorder /etc/hosts now "hostname -f" results in the correct hostname. Fixes #5835 --- puppet/modules/site_config/manifests/hosts.pp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/puppet/modules/site_config/manifests/hosts.pp b/puppet/modules/site_config/manifests/hosts.pp index 6982d37b..e43ad45e 100644 --- a/puppet/modules/site_config/manifests/hosts.pp +++ b/puppet/modules/site_config/manifests/hosts.pp @@ -10,10 +10,9 @@ class site_config::hosts() { } else { $dns_aliases = $dns['aliases'] } - $my_hostnames = unique(sort(concat( - [$hostname, $domain_hash['full'], $domain_hash['internal']], - $dns_aliases - ))) + $my_hostnames = unique(concat( + $dns_aliases, [$hostname, $domain_hash['full'], $domain_hash['internal']] + )) file { '/etc/hostname': ensure => present, -- cgit v1.2.3 From 787eb366d80cee7c89348e11c1befa86255bbe85 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 26 Jun 2014 13:57:33 -0700 Subject: make try{} macro also catch ArgumentErrors --- provider_base/lib/macros/core.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/provider_base/lib/macros/core.rb b/provider_base/lib/macros/core.rb index d4d9171f..609111ac 100644 --- a/provider_base/lib/macros/core.rb +++ b/provider_base/lib/macros/core.rb @@ -64,6 +64,7 @@ module LeapCli def try(&block) yield rescue NoMethodError + rescue ArgumentError nil end -- cgit v1.2.3 From 15ec7cbcb2b9a4c230c4b8a7f7b720c7dc047c61 Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 27 Jun 2014 23:30:58 -0700 Subject: added error() macro. --- provider_base/lib/macros/core.rb | 6 +++++- provider_base/services/couchdb.rb | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/provider_base/lib/macros/core.rb b/provider_base/lib/macros/core.rb index 609111ac..2ab2e71b 100644 --- a/provider_base/lib/macros/core.rb +++ b/provider_base/lib/macros/core.rb @@ -39,10 +39,14 @@ module LeapCli if instance_eval(assertion) true else - raise AssertionFailed.new(assertion) + raise AssertionFailed.new(assertion), assertion, caller end end + def error(msg) + raise ConfigError.new(@node, msg), msg, caller + end + # # applies a JSON partial to this node # diff --git a/provider_base/services/couchdb.rb b/provider_base/services/couchdb.rb index c63e3a00..81f366e1 100644 --- a/provider_base/services/couchdb.rb +++ b/provider_base/services/couchdb.rb @@ -3,7 +3,7 @@ # unless nodes_like_me['services' => 'couchdb']['couch.master' => true].any? - #raise 'node `%s`, environment `%s`: there must be at least one node with couch.master set to `true` for this environment.' % [@node.name, @node.environment] + error('there must be at least one node with couch.master set to `true` for environment `%s`.' % @node.environment) end if couch.master -- cgit v1.2.3 From 82dbbf823d6637082f63e55ed1d2f57a11e0d481 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 26 Jun 2014 02:34:08 -0700 Subject: puppet_command: set hostname manually, not via puppet. --- bin/puppet_command | 55 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/bin/puppet_command b/bin/puppet_command index a6cd5a69..a9d39066 100755 --- a/bin/puppet_command +++ b/bin/puppet_command @@ -7,12 +7,15 @@ # (exit codes, lockfile, multiple manifests, etc) # +require 'pty' +require 'yaml' + PUPPET_BIN = '/usr/bin/puppet' PUPPET_DIRECTORY = '/srv/leap' PUPPET_PARAMETERS = '--color=false --detailed-exitcodes --libdir=puppet/lib --confdir=puppet' SITE_MANIFEST = 'puppet/manifests/site.pp' -SETUP_MANIFEST = 'puppet/manifests/setup.pp' DEFAULT_TAGS = 'leap_base,leap_service' +HIERA_FILE = '/etc/leap/hiera.yaml' def main process_command_line_arguments @@ -54,21 +57,37 @@ def apply end def set_hostname - exit_code = puppet_apply(:manifest => SETUP_MANIFEST, :tags => '') do |line| - # todo: replace setup.pp with https://github.com/lutter/ruby-augeas - # or try this: http://www.puppetcookbook.com/posts/override-a-facter-fact.html - if (line !~ /Finished catalog run/ || @verbosity > 2) && - (line !~ /dnsdomainname: Name or service not known/) && - (line !~ /warning: Could not retrieve fact fqdn/) - puts line + unless File.exists?(HIERA_FILE) + puts("ERROR: Cannot set hostname without #{HIERA_FILE}") + exit(1) + end + hostname = YAML.load_file(HIERA_FILE)['name'] + if hostname.nil? || hostname.empty? + puts('ERROR: NAME argument required') + exit(1) + end + current_hostname_file = File.read('/etc/hostname') rescue nil + current_hostname = `/bin/hostname`.strip + + # set /etc/hostname + if current_hostname_file != hostname + File.open('/etc/hostname', 'w', 0611, :encoding => 'ascii') do |f| + f.write hostname + end + if File.read('/etc/hostname') == hostname + puts "Set /etc/hostname to #{hostname}" + else + puts "ERROR: failed to update /etc/hostname" end end - if exit_code == 2 - puts "Hostname updated." - elsif exit_code == 4 || exit_code == 6 - puts "ERROR: could not update hostname." - elsif exit_code == 0 && @verbosity > 1 - puts "No change to hostname." + + # call /bin/hostname + if current_hostname != hostname + if run("/bin/hostname #{hostname}") == 0 + puts "Set hostname to #{hostname}" + else + puts "ERROR: failed to call `/bin/hostname #{hostname}`" + end end end @@ -157,24 +176,18 @@ end ## this only works under ruby 1.9 ## -require "pty" - def run(cmd) puts cmd if @verbosity >= 3 PTY.spawn("#{cmd}") do |output, input, pid| begin while line = output.gets do yield line - #$stdout.puts line - #$stdout.flush end rescue Errno::EIO end Process.wait(pid) # only works in ruby 1.9, required to capture the exit status. end - status = $?.exitstatus - #yield status if block_given? - return status + return $?.exitstatus rescue PTY::ChildExited end -- cgit v1.2.3 From 54fcafe131c411a49e4277cd0d14c6ea20044203 Mon Sep 17 00:00:00 2001 From: irregulator Date: Tue, 20 May 2014 23:20:58 +0300 Subject: Initial commit for obfsproxy server feature in platform --- provider_base/services/obfsproxy.json | 10 +++ puppet/manifests/site.pp | 5 ++ puppet/modules/obfsproxy/files/obfsproxy_daemon | 99 ++++++++++++++++++++++ puppet/modules/obfsproxy/manifests/init.pp | 64 ++++++++++++++ puppet/modules/obfsproxy/templates/etc_conf.erb | 11 +++ .../site_apt/manifests/preferences/obfsproxy.pp | 9 ++ puppet/modules/site_obfsproxy/README | 0 puppet/modules/site_obfsproxy/manifests/init.pp | 28 ++++++ 8 files changed, 226 insertions(+) create mode 100644 provider_base/services/obfsproxy.json create mode 100755 puppet/modules/obfsproxy/files/obfsproxy_daemon create mode 100644 puppet/modules/obfsproxy/manifests/init.pp create mode 100644 puppet/modules/obfsproxy/templates/etc_conf.erb create mode 100644 puppet/modules/site_apt/manifests/preferences/obfsproxy.pp create mode 100644 puppet/modules/site_obfsproxy/README create mode 100644 puppet/modules/site_obfsproxy/manifests/init.pp diff --git a/provider_base/services/obfsproxy.json b/provider_base/services/obfsproxy.json new file mode 100644 index 00000000..954ae868 --- /dev/null +++ b/provider_base/services/obfsproxy.json @@ -0,0 +1,10 @@ +{ + "obfsproxy": { + "scramblesuit": { + "password": "= base32_secret :scramblesuit_password", + //"port" : "= rand(11..5555)" + "port" : "= obfs_port :scramblesuit_port, 18000..32000" + }, + "gateway_address": "= nodes[:services => 'openvpn'].field('openvpn.gateway_address')[0]" + } +} diff --git a/puppet/manifests/site.pp b/puppet/manifests/site.pp index 9afa5dfd..8e00fbbd 100644 --- a/puppet/manifests/site.pp +++ b/puppet/manifests/site.pp @@ -10,6 +10,7 @@ notice("Services for ${fqdn}: ${services_str}") if member($services, 'openvpn') { include site_openvpn + include site_obfsproxy } if member($services, 'couchdb') { @@ -42,4 +43,8 @@ if member($services, 'static') { include site_static } +if $services =~ /\bobfsproxy\b/ { + include site_obfsproxy +} + include site_config::packages::uninstall diff --git a/puppet/modules/obfsproxy/files/obfsproxy_daemon b/puppet/modules/obfsproxy/files/obfsproxy_daemon new file mode 100755 index 00000000..f5914980 --- /dev/null +++ b/puppet/modules/obfsproxy/files/obfsproxy_daemon @@ -0,0 +1,99 @@ +#!/bin/sh + +### BEGIN INIT INFO +# Provides: obfsproxy daemon +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: obfsproxy daemon +# Description: obfsproxy daemon +### END INIT INFO + +. /lib/lsb/init-functions + +DAEMON=/usr/bin/obfsproxy +NAME=obfsproxy +DESC="obfsproxy daemon" +USER=obfsproxy +PIDFILE=/var/run/obfsproxy.pid +CONF=/etc/obfsproxy.conf + +# If the daemon is not there, then exit. +test -x $DAEMON || exit 0 + +if [ -f $CONF ] ; then + . $CONF +else + echo "Obfsproxy configuration file is missing, aborting..." + exit +fi + +DAEMONARGS=" --log-min-severity=$LOG $TRANSPORT $PARAM \ + --dest=$DEST_IP:$DEST_PORT server 0.0.0.0:$PORT" + +start_obfsproxy() { + start-stop-daemon --start --quiet --oknodo -m --pidfile $PIDFILE \ + -b -c $USER --startas $DAEMON --$DAEMONARGS +} + +stop_obfsproxy() { + start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE +} + +status_obfsproxy() { + status_of_proc -p $PIDFILE $DAEMON $NAME && status="0" || status="$?" +} + +case $1 in + start) + if [ -e $PIDFILE ]; then + status_obfsproxy + if [ $status = "0" ]; then + exit + fi + fi + log_begin_msg "Starting $DESC" + start_obfsproxy + log_end_msg $? + ;; + stop) + if [ -e $PIDFILE ]; then + status_obfsproxy + if [ $status = "0" ]; then + log_begin_msg "Stopping $DESC" + stop_obfsproxy + rm -f $PIDFILE + log_end_msg $? + fi + else + log_daemon_msg "$NAME is not running" + log_end_msg $? + fi + ;; + restart) + $0 stop && sleep 2 && $0 start + ;; + status) +# if [ -e $PIDFILE ]; then +# #status_of_proc -p $PIDFILE $DAEMON "$NAME " && exit 0 || exit $? +# status_obfsproxy +# else +# log_daemon_msg "$NAME is not running" +# log_end_msg 0 +# fi + status_obfsproxy + ;; + reload) + if [ -e $PIDFILE ]; then + start-stop-daemon --stop --signal USR1 --quiet --pidfile $PIDFILE --name $NAME + log_success_msg "$DESC reloaded successfully" + else + log_failure_msg "$PIDFILE does not exists" + fi + ;; + *) + echo "Usage: $0 {start|stop|restart|reload|status}" + exit 2 + ;; +esac diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp new file mode 100644 index 00000000..4deebb62 --- /dev/null +++ b/puppet/modules/obfsproxy/manifests/init.pp @@ -0,0 +1,64 @@ +class obfsproxy ( + $transport, + $port, + $param, + $dest_ip, + $dest_port +){ + + user { obfsproxy: + ensure => present, + system => true, + gid => obfsproxy, + } + + group { obfsproxy: + ensure => present, + system => true, + } + +# file { '/etc/default/obfsproxy': +# path => '/etc/default/obfsproxy', +# owner => 'root', +# group => 'root', +# mode => '0750', +# content => template('obfsproxy/etc_default_conf.erb'), +# } + + file { '/etc/init.d/obfsproxy': + path => '/etc/init.d/obfsproxy', + ensure => present, + source => 'puppet:///modules/obfsproxy/obfsproxy_daemon', + owner => 'root', + group => 'root', + mode => '0755', + require => File['/etc/obfsproxy.conf'], + subscribe => File['/etc/obfsproxy.conf'], + #content => template('obfsproxy/etc_init_d.erb'), + } + + file { '/etc/obfsproxy.conf': + path => '/etc/obfsproxy.conf', + ensure => present, + owner => 'root', + group => 'root', + mode => '0750', + content => template('obfsproxy/etc_conf.erb'), + } + + package { "obfsproxy": + ensure => present, + } + + service { "obfsproxy": + ensure => running, + status => '/usr/sbin/service obfsproxy status + | grep "is running"', + require => [ + Package["obfsproxy"], + File["/etc/init.d/obfsproxy"] ] + } + + +} + diff --git a/puppet/modules/obfsproxy/templates/etc_conf.erb b/puppet/modules/obfsproxy/templates/etc_conf.erb new file mode 100644 index 00000000..3313b326 --- /dev/null +++ b/puppet/modules/obfsproxy/templates/etc_conf.erb @@ -0,0 +1,11 @@ +TRANSPORT=<%= @transport %> +PORT=<%= @port %> +DEST_IP=<%= @dest_ip %> +DEST_PORT=<%= @dest_port %> +<% if @transport == "scramblesuit" %> +PARAM=--password=<%= @param %> +<% else %> +PARAM=<%= @param %> +<% end %> +LOG=info + diff --git a/puppet/modules/site_apt/manifests/preferences/obfsproxy.pp b/puppet/modules/site_apt/manifests/preferences/obfsproxy.pp new file mode 100644 index 00000000..081086e5 --- /dev/null +++ b/puppet/modules/site_apt/manifests/preferences/obfsproxy.pp @@ -0,0 +1,9 @@ +class site_apt::preferences::obfsproxy { + + apt::preferences_snippet { 'obfsproxy': + package => 'obfsproxy', + release => "${::lsbdistcodename}-backports", + priority => 999; + } + +} diff --git a/puppet/modules/site_obfsproxy/README b/puppet/modules/site_obfsproxy/README new file mode 100644 index 00000000..e69de29b diff --git a/puppet/modules/site_obfsproxy/manifests/init.pp b/puppet/modules/site_obfsproxy/manifests/init.pp new file mode 100644 index 00000000..23a8dd30 --- /dev/null +++ b/puppet/modules/site_obfsproxy/manifests/init.pp @@ -0,0 +1,28 @@ +class site_obfsproxy { + tag 'leap_service' + Class['site_config::default'] -> Class['site_obfsproxy'] + + $transport = 'scramblesuit' + + $obfsproxy = hiera('obfsproxy') + $scramblesuit = $obfsproxy['scramblesuit'] + $scram_pass = $scramblesuit['password'] + $scram_port = $scramblesuit['port'] + $dest_ip = $obfsproxy['gateway_address'] + $dest_port = '443' + + include site_apt::preferences::twisted + include site_apt::preferences::obfsproxy + + class { 'obfsproxy': + transport => $transport, + port => $scram_port, + param => $scram_pass, + dest_ip => $dest_ip, + dest_port => $dest_port, + } + +} + + + -- cgit v1.2.3 From 156c2e1194c65d2f7813b946ac8baa90ffdf1f39 Mon Sep 17 00:00:00 2001 From: irregulator Date: Wed, 21 May 2014 20:42:46 +0300 Subject: Make shorewall accept incoming traffic for obfsproxy server --- puppet/modules/site_obfsproxy/manifests/init.pp | 2 ++ .../modules/site_shorewall/manifests/obfsproxy.pp | 24 ++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 puppet/modules/site_shorewall/manifests/obfsproxy.pp diff --git a/puppet/modules/site_obfsproxy/manifests/init.pp b/puppet/modules/site_obfsproxy/manifests/init.pp index 23a8dd30..276b30db 100644 --- a/puppet/modules/site_obfsproxy/manifests/init.pp +++ b/puppet/modules/site_obfsproxy/manifests/init.pp @@ -22,6 +22,8 @@ class site_obfsproxy { dest_port => $dest_port, } + include site_shorewall::obfsproxy + } diff --git a/puppet/modules/site_shorewall/manifests/obfsproxy.pp b/puppet/modules/site_shorewall/manifests/obfsproxy.pp new file mode 100644 index 00000000..68fb9b9f --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/obfsproxy.pp @@ -0,0 +1,24 @@ +class site_shorewall::obfsproxy { + + include site_shorewall::defaults + + $obfsproxy = hiera('obfsproxy') + $scramblesuit = $obfsproxy['scramblesuit'] + $scram_port = $scramblesuit['port'] + + # define macro for incoming services + file { '/etc/shorewall/macro.leap_obfsproxy': + content => "PARAM - - tcp $scram_port ", + notify => Service['shorewall'], + require => Package['shorewall'] + } + + shorewall::rule { + 'net2fw-obfs': + source => 'net', + destination => '$FW', + action => 'leap_obfsproxy(ACCEPT)', + order => 200; + } + +} -- cgit v1.2.3 From 94e0791cff9a3ce47e66c56a921e41b83b52b3d9 Mon Sep 17 00:00:00 2001 From: irregulator Date: Wed, 21 May 2014 21:52:14 +0300 Subject: Add data directory to save scramblesuit's state. Also clean up a little the obfsproxy puppet class, create appropriate directories, restrict permissions. --- puppet/modules/obfsproxy/files/obfsproxy_daemon | 7 +++--- puppet/modules/obfsproxy/manifests/init.pp | 30 ++++++++++++++++--------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/puppet/modules/obfsproxy/files/obfsproxy_daemon b/puppet/modules/obfsproxy/files/obfsproxy_daemon index f5914980..4c9bcedc 100755 --- a/puppet/modules/obfsproxy/files/obfsproxy_daemon +++ b/puppet/modules/obfsproxy/files/obfsproxy_daemon @@ -16,8 +16,9 @@ DAEMON=/usr/bin/obfsproxy NAME=obfsproxy DESC="obfsproxy daemon" USER=obfsproxy +DATDIR=/etc/obfsproxy PIDFILE=/var/run/obfsproxy.pid -CONF=/etc/obfsproxy.conf +CONF=$DATDIR/obfsproxy.conf # If the daemon is not there, then exit. test -x $DAEMON || exit 0 @@ -29,8 +30,8 @@ else exit fi -DAEMONARGS=" --log-min-severity=$LOG $TRANSPORT $PARAM \ - --dest=$DEST_IP:$DEST_PORT server 0.0.0.0:$PORT" +DAEMONARGS=" --log-min-severity=$LOG --data-dir=$DATDIR $TRANSPORT \ + $PARAM --dest=$DEST_IP:$DEST_PORT server 0.0.0.0:$PORT" start_obfsproxy() { start-stop-daemon --start --quiet --oknodo -m --pidfile $PIDFILE \ diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp index 4deebb62..c15a0dc8 100644 --- a/puppet/modules/obfsproxy/manifests/init.pp +++ b/puppet/modules/obfsproxy/manifests/init.pp @@ -6,13 +6,16 @@ class obfsproxy ( $dest_port ){ - user { obfsproxy: + $user = 'obfsproxy' + $conf = '/etc/obfsproxy/obfsproxy.conf' + + user { $user: ensure => present, system => true, - gid => obfsproxy, + gid => $user, } - group { obfsproxy: + group { $user: ensure => present, system => true, } @@ -31,19 +34,26 @@ class obfsproxy ( source => 'puppet:///modules/obfsproxy/obfsproxy_daemon', owner => 'root', group => 'root', - mode => '0755', - require => File['/etc/obfsproxy.conf'], - subscribe => File['/etc/obfsproxy.conf'], - #content => template('obfsproxy/etc_init_d.erb'), + mode => '0750', + require => File[$conf], + subscribe => File[$conf], } - file { '/etc/obfsproxy.conf': - path => '/etc/obfsproxy.conf', + file { $conf : + path => $conf, ensure => present, owner => 'root', group => 'root', - mode => '0750', + mode => '0600', content => template('obfsproxy/etc_conf.erb'), + require => File['/etc/obfsproxy'], + } + + file { '/etc/obfsproxy': + ensure => directory, + owner => $user, + group => $user, + mode => '0700', } package { "obfsproxy": -- cgit v1.2.3 From cfcc589c6465dab8a4d3923d6c81623ecfbeb8c1 Mon Sep 17 00:00:00 2001 From: irregulator Date: Thu, 22 May 2014 03:57:28 +0300 Subject: Reflect change in leap_cli, use rand_range macro --- provider_base/services/obfsproxy.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/provider_base/services/obfsproxy.json b/provider_base/services/obfsproxy.json index 954ae868..792a4183 100644 --- a/provider_base/services/obfsproxy.json +++ b/provider_base/services/obfsproxy.json @@ -2,8 +2,7 @@ "obfsproxy": { "scramblesuit": { "password": "= base32_secret :scramblesuit_password", - //"port" : "= rand(11..5555)" - "port" : "= obfs_port :scramblesuit_port, 18000..32000" + "port" : "= rand_range :scramblesuit_port, 18000..32000" }, "gateway_address": "= nodes[:services => 'openvpn'].field('openvpn.gateway_address')[0]" } -- cgit v1.2.3 From 7c9dd9ee9653c854badaf4f1d21d7dd833e3e620 Mon Sep 17 00:00:00 2001 From: irregulator Date: Thu, 22 May 2014 20:44:51 +0300 Subject: Move obfsproxy_daemon to obfsproxy_init --- puppet/modules/obfsproxy/files/obfsproxy_daemon | 100 ------------------------ puppet/modules/obfsproxy/files/obfsproxy_init | 100 ++++++++++++++++++++++++ puppet/modules/obfsproxy/manifests/init.pp | 2 +- 3 files changed, 101 insertions(+), 101 deletions(-) delete mode 100755 puppet/modules/obfsproxy/files/obfsproxy_daemon create mode 100755 puppet/modules/obfsproxy/files/obfsproxy_init diff --git a/puppet/modules/obfsproxy/files/obfsproxy_daemon b/puppet/modules/obfsproxy/files/obfsproxy_daemon deleted file mode 100755 index 4c9bcedc..00000000 --- a/puppet/modules/obfsproxy/files/obfsproxy_daemon +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/sh - -### BEGIN INIT INFO -# Provides: obfsproxy daemon -# Required-Start: $remote_fs $syslog -# Required-Stop: $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: obfsproxy daemon -# Description: obfsproxy daemon -### END INIT INFO - -. /lib/lsb/init-functions - -DAEMON=/usr/bin/obfsproxy -NAME=obfsproxy -DESC="obfsproxy daemon" -USER=obfsproxy -DATDIR=/etc/obfsproxy -PIDFILE=/var/run/obfsproxy.pid -CONF=$DATDIR/obfsproxy.conf - -# If the daemon is not there, then exit. -test -x $DAEMON || exit 0 - -if [ -f $CONF ] ; then - . $CONF -else - echo "Obfsproxy configuration file is missing, aborting..." - exit -fi - -DAEMONARGS=" --log-min-severity=$LOG --data-dir=$DATDIR $TRANSPORT \ - $PARAM --dest=$DEST_IP:$DEST_PORT server 0.0.0.0:$PORT" - -start_obfsproxy() { - start-stop-daemon --start --quiet --oknodo -m --pidfile $PIDFILE \ - -b -c $USER --startas $DAEMON --$DAEMONARGS -} - -stop_obfsproxy() { - start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE -} - -status_obfsproxy() { - status_of_proc -p $PIDFILE $DAEMON $NAME && status="0" || status="$?" -} - -case $1 in - start) - if [ -e $PIDFILE ]; then - status_obfsproxy - if [ $status = "0" ]; then - exit - fi - fi - log_begin_msg "Starting $DESC" - start_obfsproxy - log_end_msg $? - ;; - stop) - if [ -e $PIDFILE ]; then - status_obfsproxy - if [ $status = "0" ]; then - log_begin_msg "Stopping $DESC" - stop_obfsproxy - rm -f $PIDFILE - log_end_msg $? - fi - else - log_daemon_msg "$NAME is not running" - log_end_msg $? - fi - ;; - restart) - $0 stop && sleep 2 && $0 start - ;; - status) -# if [ -e $PIDFILE ]; then -# #status_of_proc -p $PIDFILE $DAEMON "$NAME " && exit 0 || exit $? -# status_obfsproxy -# else -# log_daemon_msg "$NAME is not running" -# log_end_msg 0 -# fi - status_obfsproxy - ;; - reload) - if [ -e $PIDFILE ]; then - start-stop-daemon --stop --signal USR1 --quiet --pidfile $PIDFILE --name $NAME - log_success_msg "$DESC reloaded successfully" - else - log_failure_msg "$PIDFILE does not exists" - fi - ;; - *) - echo "Usage: $0 {start|stop|restart|reload|status}" - exit 2 - ;; -esac diff --git a/puppet/modules/obfsproxy/files/obfsproxy_init b/puppet/modules/obfsproxy/files/obfsproxy_init new file mode 100755 index 00000000..4c9bcedc --- /dev/null +++ b/puppet/modules/obfsproxy/files/obfsproxy_init @@ -0,0 +1,100 @@ +#!/bin/sh + +### BEGIN INIT INFO +# Provides: obfsproxy daemon +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: obfsproxy daemon +# Description: obfsproxy daemon +### END INIT INFO + +. /lib/lsb/init-functions + +DAEMON=/usr/bin/obfsproxy +NAME=obfsproxy +DESC="obfsproxy daemon" +USER=obfsproxy +DATDIR=/etc/obfsproxy +PIDFILE=/var/run/obfsproxy.pid +CONF=$DATDIR/obfsproxy.conf + +# If the daemon is not there, then exit. +test -x $DAEMON || exit 0 + +if [ -f $CONF ] ; then + . $CONF +else + echo "Obfsproxy configuration file is missing, aborting..." + exit +fi + +DAEMONARGS=" --log-min-severity=$LOG --data-dir=$DATDIR $TRANSPORT \ + $PARAM --dest=$DEST_IP:$DEST_PORT server 0.0.0.0:$PORT" + +start_obfsproxy() { + start-stop-daemon --start --quiet --oknodo -m --pidfile $PIDFILE \ + -b -c $USER --startas $DAEMON --$DAEMONARGS +} + +stop_obfsproxy() { + start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE +} + +status_obfsproxy() { + status_of_proc -p $PIDFILE $DAEMON $NAME && status="0" || status="$?" +} + +case $1 in + start) + if [ -e $PIDFILE ]; then + status_obfsproxy + if [ $status = "0" ]; then + exit + fi + fi + log_begin_msg "Starting $DESC" + start_obfsproxy + log_end_msg $? + ;; + stop) + if [ -e $PIDFILE ]; then + status_obfsproxy + if [ $status = "0" ]; then + log_begin_msg "Stopping $DESC" + stop_obfsproxy + rm -f $PIDFILE + log_end_msg $? + fi + else + log_daemon_msg "$NAME is not running" + log_end_msg $? + fi + ;; + restart) + $0 stop && sleep 2 && $0 start + ;; + status) +# if [ -e $PIDFILE ]; then +# #status_of_proc -p $PIDFILE $DAEMON "$NAME " && exit 0 || exit $? +# status_obfsproxy +# else +# log_daemon_msg "$NAME is not running" +# log_end_msg 0 +# fi + status_obfsproxy + ;; + reload) + if [ -e $PIDFILE ]; then + start-stop-daemon --stop --signal USR1 --quiet --pidfile $PIDFILE --name $NAME + log_success_msg "$DESC reloaded successfully" + else + log_failure_msg "$PIDFILE does not exists" + fi + ;; + *) + echo "Usage: $0 {start|stop|restart|reload|status}" + exit 2 + ;; +esac diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp index c15a0dc8..e62bfcd8 100644 --- a/puppet/modules/obfsproxy/manifests/init.pp +++ b/puppet/modules/obfsproxy/manifests/init.pp @@ -31,7 +31,7 @@ class obfsproxy ( file { '/etc/init.d/obfsproxy': path => '/etc/init.d/obfsproxy', ensure => present, - source => 'puppet:///modules/obfsproxy/obfsproxy_daemon', + source => 'puppet:///modules/obfsproxy/obfsproxy_init', owner => 'root', group => 'root', mode => '0750', -- cgit v1.2.3 From f8694b037dfd22382dc2abd8afefd947d3531974 Mon Sep 17 00:00:00 2001 From: irregulator Date: Thu, 22 May 2014 20:46:06 +0300 Subject: Change exit status code if config file is missing --- puppet/modules/obfsproxy/files/obfsproxy_init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/obfsproxy/files/obfsproxy_init b/puppet/modules/obfsproxy/files/obfsproxy_init index 4c9bcedc..2496bba7 100755 --- a/puppet/modules/obfsproxy/files/obfsproxy_init +++ b/puppet/modules/obfsproxy/files/obfsproxy_init @@ -27,7 +27,7 @@ if [ -f $CONF ] ; then . $CONF else echo "Obfsproxy configuration file is missing, aborting..." - exit + exit 2 fi DAEMONARGS=" --log-min-severity=$LOG --data-dir=$DATDIR $TRANSPORT \ -- cgit v1.2.3 From f4b56483c6e80774f746cd1fbf7d92573dd0f51d Mon Sep 17 00:00:00 2001 From: irregulator Date: Thu, 22 May 2014 20:47:23 +0300 Subject: Remove commented lines from init script status section --- puppet/modules/obfsproxy/files/obfsproxy_init | 7 ------- 1 file changed, 7 deletions(-) diff --git a/puppet/modules/obfsproxy/files/obfsproxy_init b/puppet/modules/obfsproxy/files/obfsproxy_init index 2496bba7..5223ec9d 100755 --- a/puppet/modules/obfsproxy/files/obfsproxy_init +++ b/puppet/modules/obfsproxy/files/obfsproxy_init @@ -76,13 +76,6 @@ case $1 in $0 stop && sleep 2 && $0 start ;; status) -# if [ -e $PIDFILE ]; then -# #status_of_proc -p $PIDFILE $DAEMON "$NAME " && exit 0 || exit $? -# status_obfsproxy -# else -# log_daemon_msg "$NAME is not running" -# log_end_msg 0 -# fi status_obfsproxy ;; reload) -- cgit v1.2.3 From ae75dccbb6a65ee22b6185dcd8c0fedd14e35d0f Mon Sep 17 00:00:00 2001 From: irregulator Date: Thu, 22 May 2014 20:49:12 +0300 Subject: Remove commented lines from obfsproxy puppet module class --- puppet/modules/obfsproxy/manifests/init.pp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp index e62bfcd8..d0212c64 100644 --- a/puppet/modules/obfsproxy/manifests/init.pp +++ b/puppet/modules/obfsproxy/manifests/init.pp @@ -20,14 +20,6 @@ class obfsproxy ( system => true, } -# file { '/etc/default/obfsproxy': -# path => '/etc/default/obfsproxy', -# owner => 'root', -# group => 'root', -# mode => '0750', -# content => template('obfsproxy/etc_default_conf.erb'), -# } - file { '/etc/init.d/obfsproxy': path => '/etc/init.d/obfsproxy', ensure => present, -- cgit v1.2.3 From 1a0161da0ff420d26732b492898ebf0074b2292c Mon Sep 17 00:00:00 2001 From: irregulator Date: Thu, 22 May 2014 20:52:44 +0300 Subject: Line up equal signs, change double to single quotes --- puppet/modules/obfsproxy/manifests/init.pp | 8 ++++---- puppet/modules/site_obfsproxy/manifests/init.pp | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp index d0212c64..456fe1a7 100644 --- a/puppet/modules/obfsproxy/manifests/init.pp +++ b/puppet/modules/obfsproxy/manifests/init.pp @@ -48,17 +48,17 @@ class obfsproxy ( mode => '0700', } - package { "obfsproxy": + package { 'obfsproxy': ensure => present, } - service { "obfsproxy": + service { 'obfsproxy': ensure => running, status => '/usr/sbin/service obfsproxy status | grep "is running"', require => [ - Package["obfsproxy"], - File["/etc/init.d/obfsproxy"] ] + Package['obfsproxy'], + File['/etc/init.d/obfsproxy'] ] } diff --git a/puppet/modules/site_obfsproxy/manifests/init.pp b/puppet/modules/site_obfsproxy/manifests/init.pp index 276b30db..6509fec8 100644 --- a/puppet/modules/site_obfsproxy/manifests/init.pp +++ b/puppet/modules/site_obfsproxy/manifests/init.pp @@ -4,12 +4,12 @@ class site_obfsproxy { $transport = 'scramblesuit' - $obfsproxy = hiera('obfsproxy') + $obfsproxy = hiera('obfsproxy') $scramblesuit = $obfsproxy['scramblesuit'] - $scram_pass = $scramblesuit['password'] - $scram_port = $scramblesuit['port'] - $dest_ip = $obfsproxy['gateway_address'] - $dest_port = '443' + $scram_pass = $scramblesuit['password'] + $scram_port = $scramblesuit['port'] + $dest_ip = $obfsproxy['gateway_address'] + $dest_port = '443' include site_apt::preferences::twisted include site_apt::preferences::obfsproxy -- cgit v1.2.3 From 2f318f0be937f0bace467640f4011ba422a736b7 Mon Sep 17 00:00:00 2001 From: irregulator Date: Thu, 22 May 2014 14:34:55 +0300 Subject: Pick gateway address either from self or another openvpn node --- provider_base/services/obfsproxy.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/provider_base/services/obfsproxy.json b/provider_base/services/obfsproxy.json index 792a4183..d6771148 100644 --- a/provider_base/services/obfsproxy.json +++ b/provider_base/services/obfsproxy.json @@ -4,6 +4,11 @@ "password": "= base32_secret :scramblesuit_password", "port" : "= rand_range :scramblesuit_port, 18000..32000" }, - "gateway_address": "= nodes[:services => 'openvpn'].field('openvpn.gateway_address')[0]" + "gateway_address": "= self['openvpn'] ? openvpn.gateway_address : nodes_like_me[:services => 'openvpn'].field('openvpn.gateway_address').shuffle.first" + // Later, if we add a SafeNil class that looks and acts like nil + // but will allow you to call methods on it (each returning another SafeNil) + // without throwing an exception, we could do: + // "gateway_address": "= self['openvpn'] ? openvpn.gateway_address : (nodes_like_me[:services => 'openvpn']['location.name' => location.name].field('openvpn.gateway_address').shuffle.first || nodes_like_me[:services => 'openvpn'].field('openvpn.gateway_address').shuffle.first)" + // Perhaps we should also create a macro. } } -- cgit v1.2.3 From 7a54923591125894440b9ff7020e4b413a1c6fb5 Mon Sep 17 00:00:00 2001 From: irregulator Date: Fri, 23 May 2014 17:28:32 +0300 Subject: Address logging for obfsproxy daemon Create obfsproxy directory in /var/log, specify log file when obfsproxy is spawned by init script, create a logrotate configuration for obfsproxy's logs. --- puppet/modules/obfsproxy/files/obfsproxy_init | 5 +++-- puppet/modules/obfsproxy/files/obfsproxy_logrotate | 14 ++++++++++++++ puppet/modules/obfsproxy/manifests/init.pp | 16 ++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 puppet/modules/obfsproxy/files/obfsproxy_logrotate diff --git a/puppet/modules/obfsproxy/files/obfsproxy_init b/puppet/modules/obfsproxy/files/obfsproxy_init index 5223ec9d..7a7e7609 100755 --- a/puppet/modules/obfsproxy/files/obfsproxy_init +++ b/puppet/modules/obfsproxy/files/obfsproxy_init @@ -19,6 +19,7 @@ USER=obfsproxy DATDIR=/etc/obfsproxy PIDFILE=/var/run/obfsproxy.pid CONF=$DATDIR/obfsproxy.conf +LOGFILE=/var/log/obfsproxy/log # If the daemon is not there, then exit. test -x $DAEMON || exit 0 @@ -30,8 +31,8 @@ else exit 2 fi -DAEMONARGS=" --log-min-severity=$LOG --data-dir=$DATDIR $TRANSPORT \ - $PARAM --dest=$DEST_IP:$DEST_PORT server 0.0.0.0:$PORT" +DAEMONARGS=" --log-min-severity=$LOG --log-file=$LOGFILE --data-dir=$DATDIR \ + $TRANSPORT $PARAM --dest=$DEST_IP:$DEST_PORT server 0.0.0.0:$PORT" start_obfsproxy() { start-stop-daemon --start --quiet --oknodo -m --pidfile $PIDFILE \ diff --git a/puppet/modules/obfsproxy/files/obfsproxy_logrotate b/puppet/modules/obfsproxy/files/obfsproxy_logrotate new file mode 100644 index 00000000..623bbab1 --- /dev/null +++ b/puppet/modules/obfsproxy/files/obfsproxy_logrotate @@ -0,0 +1,14 @@ +/var/log/obfsproxy/log { + weekly + missingok + rotate 10 + compress + delaycompress + notifempty + create 600 obfsproxy obfsproxy + postrotate + if [ -f /var/run/obfsproxy.pid ]; then + /etc/init.d/obfsproxy restart > /dev/null + fi + endscript +} diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp index 456fe1a7..9ba2d0fd 100644 --- a/puppet/modules/obfsproxy/manifests/init.pp +++ b/puppet/modules/obfsproxy/manifests/init.pp @@ -48,6 +48,22 @@ class obfsproxy ( mode => '0700', } + file { '/var/log/obfsproxy': + ensure => directory, + owner => $user, + group => $user, + mode => '0750', + } + + file { '/etc/logrotate.d/obfsproxy': + ensure => present, + source => 'puppet:///modules/obfsproxy/obfsproxy_logrotate', + owner => 'root', + group => 'root', + mode => '0644', + require => File['/var/log/obfsproxy'], + } + package { 'obfsproxy': ensure => present, } -- cgit v1.2.3 From fedbb6dccf7bd78b0b2a507a817dacaef0b67ac3 Mon Sep 17 00:00:00 2001 From: irregulator Date: Fri, 23 May 2014 17:45:13 +0300 Subject: Be able to specify log_level parameter for obfsproxy log_level sets minimum logging severity of obfsproxy daemon, can be error, warning, info, debug. Defaults to info. --- puppet/modules/obfsproxy/manifests/init.pp | 3 ++- puppet/modules/obfsproxy/templates/etc_conf.erb | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp index 9ba2d0fd..1ee44d6f 100644 --- a/puppet/modules/obfsproxy/manifests/init.pp +++ b/puppet/modules/obfsproxy/manifests/init.pp @@ -3,7 +3,8 @@ class obfsproxy ( $port, $param, $dest_ip, - $dest_port + $dest_port, + $log_level = 'info' ){ $user = 'obfsproxy' diff --git a/puppet/modules/obfsproxy/templates/etc_conf.erb b/puppet/modules/obfsproxy/templates/etc_conf.erb index 3313b326..d9938e1a 100644 --- a/puppet/modules/obfsproxy/templates/etc_conf.erb +++ b/puppet/modules/obfsproxy/templates/etc_conf.erb @@ -7,5 +7,5 @@ PARAM=--password=<%= @param %> <% else %> PARAM=<%= @param %> <% end %> -LOG=info +LOG=<%= @log_level %> -- cgit v1.2.3 From 49c4235477ab11118f8fc92a6f554b36121b36b2 Mon Sep 17 00:00:00 2001 From: irregulator Date: Sat, 24 May 2014 16:39:29 +0300 Subject: Change logrotate's frequency and number of log files to keep --- puppet/modules/obfsproxy/files/obfsproxy_logrotate | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/puppet/modules/obfsproxy/files/obfsproxy_logrotate b/puppet/modules/obfsproxy/files/obfsproxy_logrotate index 623bbab1..e776fcd3 100644 --- a/puppet/modules/obfsproxy/files/obfsproxy_logrotate +++ b/puppet/modules/obfsproxy/files/obfsproxy_logrotate @@ -1,7 +1,7 @@ /var/log/obfsproxy/log { - weekly + daily missingok - rotate 10 + rotate 3 compress delaycompress notifempty -- cgit v1.2.3 From 4ad025d9d7b0c1999bf34e0acd3ca12c88358d05 Mon Sep 17 00:00:00 2001 From: irregulator Date: Sat, 24 May 2014 17:41:46 +0300 Subject: Simplify init script, let puppet service resource use init status --- puppet/modules/obfsproxy/files/obfsproxy_init | 9 ++++----- puppet/modules/obfsproxy/manifests/init.pp | 2 -- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/puppet/modules/obfsproxy/files/obfsproxy_init b/puppet/modules/obfsproxy/files/obfsproxy_init index 7a7e7609..b1297738 100755 --- a/puppet/modules/obfsproxy/files/obfsproxy_init +++ b/puppet/modules/obfsproxy/files/obfsproxy_init @@ -44,14 +44,14 @@ stop_obfsproxy() { } status_obfsproxy() { - status_of_proc -p $PIDFILE $DAEMON $NAME && status="0" || status="$?" + status_of_proc -p $PIDFILE $DAEMON $NAME } case $1 in start) if [ -e $PIDFILE ]; then status_obfsproxy - if [ $status = "0" ]; then + if [ $? = "0" ]; then exit fi fi @@ -62,15 +62,14 @@ case $1 in stop) if [ -e $PIDFILE ]; then status_obfsproxy - if [ $status = "0" ]; then + if [ $? = "0" ]; then log_begin_msg "Stopping $DESC" stop_obfsproxy rm -f $PIDFILE log_end_msg $? fi else - log_daemon_msg "$NAME is not running" - log_end_msg $? + status_obfsproxy fi ;; restart) diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp index 1ee44d6f..b45a60a1 100644 --- a/puppet/modules/obfsproxy/manifests/init.pp +++ b/puppet/modules/obfsproxy/manifests/init.pp @@ -71,8 +71,6 @@ class obfsproxy ( service { 'obfsproxy': ensure => running, - status => '/usr/sbin/service obfsproxy status - | grep "is running"', require => [ Package['obfsproxy'], File['/etc/init.d/obfsproxy'] ] -- cgit v1.2.3 From 58347eddee416410e3ad3c8c4edc2b0e40a3d26c Mon Sep 17 00:00:00 2001 From: irregulator Date: Sat, 24 May 2014 18:08:31 +0300 Subject: Subscribe obfsproxy service resource to conf file --- puppet/modules/obfsproxy/manifests/init.pp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp index b45a60a1..4a0221af 100644 --- a/puppet/modules/obfsproxy/manifests/init.pp +++ b/puppet/modules/obfsproxy/manifests/init.pp @@ -39,7 +39,6 @@ class obfsproxy ( group => 'root', mode => '0600', content => template('obfsproxy/etc_conf.erb'), - require => File['/etc/obfsproxy'], } file { '/etc/obfsproxy': @@ -70,8 +69,9 @@ class obfsproxy ( } service { 'obfsproxy': - ensure => running, - require => [ + ensure => running, + subscribe => File[$conf], + require => [ Package['obfsproxy'], File['/etc/init.d/obfsproxy'] ] } -- cgit v1.2.3 From db9290a2b1b406e8231c0df569ae47c0a74ec12a Mon Sep 17 00:00:00 2001 From: irregulator Date: Sat, 24 May 2014 19:26:05 +0300 Subject: Move log files to var/log instead of var/log/obfsproxy --- puppet/modules/obfsproxy/files/obfsproxy_init | 2 +- puppet/modules/obfsproxy/files/obfsproxy_logrotate | 2 +- puppet/modules/obfsproxy/manifests/init.pp | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/puppet/modules/obfsproxy/files/obfsproxy_init b/puppet/modules/obfsproxy/files/obfsproxy_init index b1297738..629fea9f 100755 --- a/puppet/modules/obfsproxy/files/obfsproxy_init +++ b/puppet/modules/obfsproxy/files/obfsproxy_init @@ -19,7 +19,7 @@ USER=obfsproxy DATDIR=/etc/obfsproxy PIDFILE=/var/run/obfsproxy.pid CONF=$DATDIR/obfsproxy.conf -LOGFILE=/var/log/obfsproxy/log +LOGFILE=/var/log/obfsproxy.log # If the daemon is not there, then exit. test -x $DAEMON || exit 0 diff --git a/puppet/modules/obfsproxy/files/obfsproxy_logrotate b/puppet/modules/obfsproxy/files/obfsproxy_logrotate index e776fcd3..e5679d0c 100644 --- a/puppet/modules/obfsproxy/files/obfsproxy_logrotate +++ b/puppet/modules/obfsproxy/files/obfsproxy_logrotate @@ -1,4 +1,4 @@ -/var/log/obfsproxy/log { +/var/log/obfsproxy.log { daily missingok rotate 3 diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp index 4a0221af..9750932f 100644 --- a/puppet/modules/obfsproxy/manifests/init.pp +++ b/puppet/modules/obfsproxy/manifests/init.pp @@ -48,11 +48,11 @@ class obfsproxy ( mode => '0700', } - file { '/var/log/obfsproxy': - ensure => directory, - owner => $user, - group => $user, - mode => '0750', + file { '/var/log/obfsproxy.log': + ensure => present, + owner => $user, + group => $user, + mode => '0640', } file { '/etc/logrotate.d/obfsproxy': @@ -61,7 +61,7 @@ class obfsproxy ( owner => 'root', group => 'root', mode => '0644', - require => File['/var/log/obfsproxy'], + require => File['/var/log/obfsproxy.log'], } package { 'obfsproxy': -- cgit v1.2.3 From 436d98b3781aa66c78b3ec77fa7d47652a92f590 Mon Sep 17 00:00:00 2001 From: irregulator Date: Sat, 24 May 2014 19:33:08 +0300 Subject: Remove initscript subscription to conf file --- puppet/modules/obfsproxy/manifests/init.pp | 1 - 1 file changed, 1 deletion(-) diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp index 9750932f..ddb198bb 100644 --- a/puppet/modules/obfsproxy/manifests/init.pp +++ b/puppet/modules/obfsproxy/manifests/init.pp @@ -29,7 +29,6 @@ class obfsproxy ( group => 'root', mode => '0750', require => File[$conf], - subscribe => File[$conf], } file { $conf : -- cgit v1.2.3 From 86035bf6936812f5b01ac7d5e3b6d026124e156e Mon Sep 17 00:00:00 2001 From: irregulator Date: Tue, 27 May 2014 20:20:51 +0300 Subject: Use the try method to pick vpn gateway address in obfsproxy.json --- provider_base/services/obfsproxy.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/provider_base/services/obfsproxy.json b/provider_base/services/obfsproxy.json index d6771148..b14a0db9 100644 --- a/provider_base/services/obfsproxy.json +++ b/provider_base/services/obfsproxy.json @@ -4,11 +4,6 @@ "password": "= base32_secret :scramblesuit_password", "port" : "= rand_range :scramblesuit_port, 18000..32000" }, - "gateway_address": "= self['openvpn'] ? openvpn.gateway_address : nodes_like_me[:services => 'openvpn'].field('openvpn.gateway_address').shuffle.first" - // Later, if we add a SafeNil class that looks and acts like nil - // but will allow you to call methods on it (each returning another SafeNil) - // without throwing an exception, we could do: - // "gateway_address": "= self['openvpn'] ? openvpn.gateway_address : (nodes_like_me[:services => 'openvpn']['location.name' => location.name].field('openvpn.gateway_address').shuffle.first || nodes_like_me[:services => 'openvpn'].field('openvpn.gateway_address').shuffle.first)" - // Perhaps we should also create a macro. + "gateway_address": "= try{openvpn.gateway_address} || try{nodes_like_me[:services => 'openvpn']['location.name' => location.name].field('openvpn.gateway_address').shuffle.first} || try{nodes_like_me[:services => 'openvpn'].field('openvpn.gateway_address').shuffle.first}" } } -- cgit v1.2.3 From 08f4c51cbbf9a4307375278ab42d31aa65d57645 Mon Sep 17 00:00:00 2001 From: irregulator Date: Wed, 28 May 2014 15:25:21 +0300 Subject: Include obfsproxy descriptors in openvpn.json This is needed so as obfsproxy service is automatically deployed along with eip service. --- provider_base/services/openvpn.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/provider_base/services/openvpn.json b/provider_base/services/openvpn.json index 090afcd6..1ce397d0 100644 --- a/provider_base/services/openvpn.json +++ b/provider_base/services/openvpn.json @@ -26,5 +26,12 @@ "keepalive": "10 30", "tun-ipv6": true } + }, + "obfsproxy": { + "scramblesuit": { + "password": "= base32_secret :scramblesuit_password", + "port" : "= rand_range :scramblesuit_port, 18000..32000" + }, + "gateway_address": "= try{openvpn.gateway_address} || try{nodes_like_me[:services => 'openvpn']['location.name' => location.name].field('openvpn.gateway_address').shuffle.first} || try{nodes_like_me[:services => 'openvpn'].field('openvpn.gateway_address').shuffle.first}" } } -- cgit v1.2.3 From 791e22b136910ecfa204eb78be747baed2b02590 Mon Sep 17 00:00:00 2001 From: irregulator Date: Wed, 28 May 2014 17:35:12 +0300 Subject: Make obfsproxy daemon bind to specific address rather than 0.0.0.0 If obfsproxy is spawned alongside eip service, make it listen to the gateway_adress IP. If obfsproxy is running standalone listen to ip_address. --- puppet/modules/obfsproxy/files/obfsproxy_init | 2 +- puppet/modules/obfsproxy/manifests/init.pp | 1 + puppet/modules/obfsproxy/templates/etc_conf.erb | 1 + puppet/modules/site_obfsproxy/manifests/init.pp | 19 ++++++++++++++----- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/puppet/modules/obfsproxy/files/obfsproxy_init b/puppet/modules/obfsproxy/files/obfsproxy_init index 629fea9f..69dbab41 100755 --- a/puppet/modules/obfsproxy/files/obfsproxy_init +++ b/puppet/modules/obfsproxy/files/obfsproxy_init @@ -32,7 +32,7 @@ else fi DAEMONARGS=" --log-min-severity=$LOG --log-file=$LOGFILE --data-dir=$DATDIR \ - $TRANSPORT $PARAM --dest=$DEST_IP:$DEST_PORT server 0.0.0.0:$PORT" + $TRANSPORT $PARAM --dest=$DEST_IP:$DEST_PORT server $BINDADDR:$PORT" start_obfsproxy() { start-stop-daemon --start --quiet --oknodo -m --pidfile $PIDFILE \ diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp index ddb198bb..35d47d13 100644 --- a/puppet/modules/obfsproxy/manifests/init.pp +++ b/puppet/modules/obfsproxy/manifests/init.pp @@ -1,5 +1,6 @@ class obfsproxy ( $transport, + $bind_address, $port, $param, $dest_ip, diff --git a/puppet/modules/obfsproxy/templates/etc_conf.erb b/puppet/modules/obfsproxy/templates/etc_conf.erb index d9938e1a..10f6a7f7 100644 --- a/puppet/modules/obfsproxy/templates/etc_conf.erb +++ b/puppet/modules/obfsproxy/templates/etc_conf.erb @@ -8,4 +8,5 @@ PARAM=--password=<%= @param %> PARAM=<%= @param %> <% end %> LOG=<%= @log_level %> +BINDADDR=<%= @bind_address %> diff --git a/puppet/modules/site_obfsproxy/manifests/init.pp b/puppet/modules/site_obfsproxy/manifests/init.pp index 6509fec8..40b7fba8 100644 --- a/puppet/modules/site_obfsproxy/manifests/init.pp +++ b/puppet/modules/site_obfsproxy/manifests/init.pp @@ -11,15 +11,24 @@ class site_obfsproxy { $dest_ip = $obfsproxy['gateway_address'] $dest_port = '443' + if $::services =~ /\bopenvpn\b/ { + $openvpn = hiera('openvpn') + $bind_address = $openvpn['gateway_address'] + } + elsif $::services =~ /\bobfsproxy\b/ { + $bind_address = hiera('ip_address') + } + include site_apt::preferences::twisted include site_apt::preferences::obfsproxy class { 'obfsproxy': - transport => $transport, - port => $scram_port, - param => $scram_pass, - dest_ip => $dest_ip, - dest_port => $dest_port, + transport => $transport, + bind_address => $bind_address, + port => $scram_port, + param => $scram_pass, + dest_ip => $dest_ip, + dest_port => $dest_port, } include site_shorewall::obfsproxy -- cgit v1.2.3 From 02963cea38c916256a6c9c959c58ed5a222f1767 Mon Sep 17 00:00:00 2001 From: irregulator Date: Fri, 30 May 2014 04:04:24 +0300 Subject: Attach node's name to scramblesuit password and port secrets This makes every node with obfsproxy service have unique port and password for scramblesuit pluggable transport. --- provider_base/services/obfsproxy.json | 4 ++-- provider_base/services/openvpn.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/provider_base/services/obfsproxy.json b/provider_base/services/obfsproxy.json index b14a0db9..ab92c11f 100644 --- a/provider_base/services/obfsproxy.json +++ b/provider_base/services/obfsproxy.json @@ -1,8 +1,8 @@ { "obfsproxy": { "scramblesuit": { - "password": "= base32_secret :scramblesuit_password", - "port" : "= rand_range :scramblesuit_port, 18000..32000" + "password": "= base32_secret('scramblesuit_password_'+name)", + "port" : "= rand_range('scramblesuit_port_'+name, 18000..32000)" }, "gateway_address": "= try{openvpn.gateway_address} || try{nodes_like_me[:services => 'openvpn']['location.name' => location.name].field('openvpn.gateway_address').shuffle.first} || try{nodes_like_me[:services => 'openvpn'].field('openvpn.gateway_address').shuffle.first}" } diff --git a/provider_base/services/openvpn.json b/provider_base/services/openvpn.json index 1ce397d0..c62fa04b 100644 --- a/provider_base/services/openvpn.json +++ b/provider_base/services/openvpn.json @@ -29,8 +29,8 @@ }, "obfsproxy": { "scramblesuit": { - "password": "= base32_secret :scramblesuit_password", - "port" : "= rand_range :scramblesuit_port, 18000..32000" + "password": "= base32_secret('scramblesuit_password_'+name)", + "port" : "= rand_range('scramblesuit_port_'+name, 18000..32000)" }, "gateway_address": "= try{openvpn.gateway_address} || try{nodes_like_me[:services => 'openvpn']['location.name' => location.name].field('openvpn.gateway_address').shuffle.first} || try{nodes_like_me[:services => 'openvpn'].field('openvpn.gateway_address').shuffle.first}" } -- cgit v1.2.3 From abb89aca59915223ec3b6ca999d3a15ba8ede594 Mon Sep 17 00:00:00 2001 From: irregulator Date: Tue, 3 Jun 2014 17:35:14 +0300 Subject: Explicitly set apt preferences for obfsproxy to wheezy-backports --- puppet/modules/site_apt/manifests/preferences/obfsproxy.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_apt/manifests/preferences/obfsproxy.pp b/puppet/modules/site_apt/manifests/preferences/obfsproxy.pp index 081086e5..75b01956 100644 --- a/puppet/modules/site_apt/manifests/preferences/obfsproxy.pp +++ b/puppet/modules/site_apt/manifests/preferences/obfsproxy.pp @@ -2,7 +2,7 @@ class site_apt::preferences::obfsproxy { apt::preferences_snippet { 'obfsproxy': package => 'obfsproxy', - release => "${::lsbdistcodename}-backports", + release => 'wheezy-backports', priority => 999; } -- cgit v1.2.3 From ee8064a8281c3f933aeea219baec822ec8f52b84 Mon Sep 17 00:00:00 2001 From: irregulator Date: Tue, 3 Jun 2014 17:37:52 +0300 Subject: Remove unneeded newlines from obfsproxy.conf --- puppet/modules/obfsproxy/templates/etc_conf.erb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/puppet/modules/obfsproxy/templates/etc_conf.erb b/puppet/modules/obfsproxy/templates/etc_conf.erb index 10f6a7f7..8959ef78 100644 --- a/puppet/modules/obfsproxy/templates/etc_conf.erb +++ b/puppet/modules/obfsproxy/templates/etc_conf.erb @@ -2,11 +2,10 @@ TRANSPORT=<%= @transport %> PORT=<%= @port %> DEST_IP=<%= @dest_ip %> DEST_PORT=<%= @dest_port %> -<% if @transport == "scramblesuit" %> +<% if @transport == "scramblesuit" -%> PARAM=--password=<%= @param %> -<% else %> +<% else -%> PARAM=<%= @param %> -<% end %> +<% end -%> LOG=<%= @log_level %> BINDADDR=<%= @bind_address %> - -- cgit v1.2.3 From aa3e39bc8342b6800129965efad72527b53596df Mon Sep 17 00:00:00 2001 From: irregulator Date: Tue, 3 Jun 2014 17:41:46 +0300 Subject: Add User resource requirement for obfsproxy service, log, etc dir --- puppet/modules/obfsproxy/manifests/init.pp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp index 35d47d13..a23cfa58 100644 --- a/puppet/modules/obfsproxy/manifests/init.pp +++ b/puppet/modules/obfsproxy/manifests/init.pp @@ -42,10 +42,11 @@ class obfsproxy ( } file { '/etc/obfsproxy': - ensure => directory, - owner => $user, - group => $user, - mode => '0700', + ensure => directory, + owner => $user, + group => $user, + mode => '0700', + require => User[$user], } file { '/var/log/obfsproxy.log': @@ -53,6 +54,7 @@ class obfsproxy ( owner => $user, group => $user, mode => '0640', + require => User[$user], } file { '/etc/logrotate.d/obfsproxy': @@ -73,7 +75,9 @@ class obfsproxy ( subscribe => File[$conf], require => [ Package['obfsproxy'], - File['/etc/init.d/obfsproxy'] ] + File['/etc/init.d/obfsproxy'], + User[$user], + Group[$user]] } -- cgit v1.2.3 From e184143d3066f02968c8bb1035e0e02bae44d587 Mon Sep 17 00:00:00 2001 From: irregulator Date: Tue, 3 Jun 2014 17:47:50 +0300 Subject: Add apt preferences requirement for obfsproxy package resource --- puppet/modules/obfsproxy/manifests/init.pp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp index a23cfa58..61714fdf 100644 --- a/puppet/modules/obfsproxy/manifests/init.pp +++ b/puppet/modules/obfsproxy/manifests/init.pp @@ -67,7 +67,8 @@ class obfsproxy ( } package { 'obfsproxy': - ensure => present, + ensure => present, + require => Class['site_apt::preferences::obfsproxy'], } service { 'obfsproxy': -- cgit v1.2.3 From 87e997658dae6655aa0a3f2da7dc8737ec9041bc Mon Sep 17 00:00:00 2001 From: irregulator Date: Tue, 3 Jun 2014 21:36:09 +0300 Subject: A vpn node picks its openvpn.gateway as obfsproxy gateway address --- provider_base/services/openvpn.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider_base/services/openvpn.json b/provider_base/services/openvpn.json index c62fa04b..1906244c 100644 --- a/provider_base/services/openvpn.json +++ b/provider_base/services/openvpn.json @@ -32,6 +32,6 @@ "password": "= base32_secret('scramblesuit_password_'+name)", "port" : "= rand_range('scramblesuit_port_'+name, 18000..32000)" }, - "gateway_address": "= try{openvpn.gateway_address} || try{nodes_like_me[:services => 'openvpn']['location.name' => location.name].field('openvpn.gateway_address').shuffle.first} || try{nodes_like_me[:services => 'openvpn'].field('openvpn.gateway_address').shuffle.first}" + "gateway_address": "= openvpn.gateway_address" } } -- cgit v1.2.3 From 0e0ebab964339446ab59cffead5a5546c5dcb18e Mon Sep 17 00:00:00 2001 From: irregulator Date: Mon, 30 Jun 2014 13:19:16 +0300 Subject: Check appropriately if obfsproxy is included in services --- puppet/manifests/site.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/manifests/site.pp b/puppet/manifests/site.pp index 8e00fbbd..57942d99 100644 --- a/puppet/manifests/site.pp +++ b/puppet/manifests/site.pp @@ -43,7 +43,7 @@ if member($services, 'static') { include site_static } -if $services =~ /\bobfsproxy\b/ { +if member($services, 'obfsproxy') { include site_obfsproxy } -- cgit v1.2.3 From 7e278f92f34e3809d380be724f0c306430791b10 Mon Sep 17 00:00:00 2001 From: irregulator Date: Tue, 1 Jul 2014 01:49:56 +0300 Subject: Use new macro pick_node to pick vpn gateway for obfsproxy.json --- provider_base/services/obfsproxy.json | 2 +- puppet/modules/obfsproxy/files/obfsproxy_init | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/provider_base/services/obfsproxy.json b/provider_base/services/obfsproxy.json index ab92c11f..979d0ef9 100644 --- a/provider_base/services/obfsproxy.json +++ b/provider_base/services/obfsproxy.json @@ -4,6 +4,6 @@ "password": "= base32_secret('scramblesuit_password_'+name)", "port" : "= rand_range('scramblesuit_port_'+name, 18000..32000)" }, - "gateway_address": "= try{openvpn.gateway_address} || try{nodes_like_me[:services => 'openvpn']['location.name' => location.name].field('openvpn.gateway_address').shuffle.first} || try{nodes_like_me[:services => 'openvpn'].field('openvpn.gateway_address').shuffle.first}" + "gateway_address": "= try{pick_node(:obfs_gateway,nodes_near_me['services' => 'openvpn']).pick_fields('openvpn.gateway_address')} || try{pick_node(:obfs_gateway,nodes_like_me['services' => 'openvpn']).pick_fields('openvpn.gateway_address')}" } } diff --git a/puppet/modules/obfsproxy/files/obfsproxy_init b/puppet/modules/obfsproxy/files/obfsproxy_init index 69dbab41..01c8013a 100755 --- a/puppet/modules/obfsproxy/files/obfsproxy_init +++ b/puppet/modules/obfsproxy/files/obfsproxy_init @@ -83,7 +83,7 @@ case $1 in start-stop-daemon --stop --signal USR1 --quiet --pidfile $PIDFILE --name $NAME log_success_msg "$DESC reloaded successfully" else - log_failure_msg "$PIDFILE does not exists" + log_failure_msg "$PIDFILE does not exist" fi ;; *) -- cgit v1.2.3 From 9ab38e0551fe3210f57be2889e70db4aa2b4cc2f Mon Sep 17 00:00:00 2001 From: Folker Bernitt Date: Thu, 10 Jul 2014 17:54:36 +0200 Subject: Added allow_registration to webapp config.yml. - See issue #5217 - See companion change in leap_web --- provider_base/services/webapp.json | 1 + puppet/modules/site_webapp/templates/config.yml.erb | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index 1b550af9..3af0dade 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -13,6 +13,7 @@ "allow_limited_certs": "= provider.service.allow_limited_bandwidth", "allow_unlimited_certs": "= provider.service.allow_unlimited_bandwidth", "allow_anonymous_certs": "= provider.service.allow_anonymous", + "allow_registration": "= provider.service.allow_registration", "default_service_level": "= provider.service.default_service_level", "service_levels": "= provider.service.levels", "secret_token": "= secret :webapp_secret_token", diff --git a/puppet/modules/site_webapp/templates/config.yml.erb b/puppet/modules/site_webapp/templates/config.yml.erb index 8faf76f4..ef139404 100644 --- a/puppet/modules/site_webapp/templates/config.yml.erb +++ b/puppet/modules/site_webapp/templates/config.yml.erb @@ -18,9 +18,10 @@ production: minimum_client_version: "<%= @webapp['client_version']['min'] %>" default_service_level: "<%= @webapp['default_service_level'] %>" service_levels: <%= @webapp['service_levels'].to_json %> + allow_registration: <%= @provider['allow_registration'].inspect %> <%- if @webapp['engines'] && @webapp['engines'].any? -%> engines: <%- @webapp['engines'].each do |engine| -%> - <%= engine %> <%- end -%> -<%- end -%> \ No newline at end of file +<%- end -%> -- cgit v1.2.3 From 3e8721d4c4e0ad8ea0daae3c35c79f7130afda6e Mon Sep 17 00:00:00 2001 From: Azul Date: Mon, 14 Jul 2014 19:50:26 +0200 Subject: update couchdb puppet module --- puppet/modules/couchdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/couchdb b/puppet/modules/couchdb index 8bc5ed43..f01b3586 160000 --- a/puppet/modules/couchdb +++ b/puppet/modules/couchdb @@ -1 +1 @@ -Subproject commit 8bc5ed434c124457b7467140152602c67a9547c5 +Subproject commit f01b3586215bdc10f0067fa0f6d940be8e88bcea -- cgit v1.2.3 From 3206634d0d17064ecd4b18cc8e2e47051e422bf3 Mon Sep 17 00:00:00 2001 From: Azul Date: Mon, 14 Jul 2014 19:50:41 +0200 Subject: proper json for tapicero config --- puppet/modules/tapicero/templates/tapicero.yaml.erb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/puppet/modules/tapicero/templates/tapicero.yaml.erb b/puppet/modules/tapicero/templates/tapicero.yaml.erb index 182a6aa6..d6ea56fa 100644 --- a/puppet/modules/tapicero/templates/tapicero.yaml.erb +++ b/puppet/modules/tapicero/templates/tapicero.yaml.erb @@ -1,3 +1,5 @@ +<%- require 'json' -%> + # # Default configuration options for Tapicero # @@ -25,7 +27,9 @@ options: # prefix for per user databases: db_prefix: "user-" mode: <%= @couchdb_mode %> - replication: <%= @couchdb_replication %> +<%- if @couchdb_replication %> + replication: <%= @couchdb_replication.to_json %> +<%- end -%> # security settings to be used for the per user databases security: -- cgit v1.2.3 From e97d0195fff2286d35c18866167b74cc37d58130 Mon Sep 17 00:00:00 2001 From: Azul Date: Mon, 14 Jul 2014 21:40:54 +0200 Subject: fix couch tests to use admin credentials --- tests/white-box/couchdb.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb index 74bd47bf..6d3a7452 100644 --- a/tests/white-box/couchdb.rb +++ b/tests/white-box/couchdb.rb @@ -71,7 +71,7 @@ class CouchDB < LeapTest def test_04_Do_ACL_users_exist? acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'tapicero', 'webapp', 'replication'] - url = couchdb_backend_url("/_users/_all_docs") + url = couchdb_backend_url("/_users/_all_docs", :user => 'admin') assert_get(url) do |body| response = JSON.parse(body) assert_equal acl_users.count, response['total_rows'] @@ -84,7 +84,8 @@ class CouchDB < LeapTest def test_05_Do_required_databases_exist? dbs_that_should_exist = ["customers","identities","keycache","sessions","shared","tickets","tokens","users"] dbs_that_should_exist.each do |db_name| - assert_get(couchdb_url("/"+db_name)) do |body| + url = couchdb_url("/"+db_name, :user => 'admin') + assert_get(url) do |body| assert response = JSON.parse(body) assert_equal db_name, response['db_name'] end @@ -129,9 +130,10 @@ class CouchDB < LeapTest url end - def couchdb_backend_url(path="") + def couchdb_backend_url(path="", options={}) # TODO: admin port is hardcoded for now but should be configurable. - couchdb_url(path, multimaster? && "5986") + options = {port: multimaster? && "5986"}.merge options + couchdb_url(path, options) end def multimaster? -- cgit v1.2.3 From ed292e8ef362e1c5a638480eda02b2ddfde936a7 Mon Sep 17 00:00:00 2001 From: Azul Date: Tue, 15 Jul 2014 13:11:54 +0200 Subject: adopt webapp test to new hiera couch clients format --- tests/white-box/webapp.rb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb index 05b86a41..7df57fd7 100644 --- a/tests/white-box/webapp.rb +++ b/tests/white-box/webapp.rb @@ -14,15 +14,16 @@ class Webapp < LeapTest # example properties: # # stunnel: - # couch_client: - # couch1_5984: - # accept_port: 4000 - # connect: couch1.bitmask.i - # connect_port: 15984 + # clients: + # couch_client: + # couch1_5984: + # accept_port: 4000 + # connect: couch1.bitmask.i + # connect_port: 15984 # def test_01_Can_contact_couchdb? - assert_property('stunnel.couch_client') - $node['stunnel']['couch_client'].values.each do |stunnel_conf| + assert_property('stunnel.clients.couch_client') + $node['stunnel']['clients']['couch_client'].values.each do |stunnel_conf| assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.' local_stunnel_url = "http://localhost:#{port}" remote_ip_address = TCPSocket.gethostbyname(stunnel_conf['connect']).last -- cgit v1.2.3 From ac5781ef6edaf03f06fa980478726aa7d11653c0 Mon Sep 17 00:00:00 2001 From: Azul Date: Tue, 15 Jul 2014 13:13:28 +0200 Subject: haproxy default to couch_write, couch_read on GET METH_POST probably does not catch PUT, DESTROY etc. So instead we now use the master as the default and only use the replications for GET and HEAD requests. --- puppet/modules/site_haproxy/templates/couch.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/puppet/modules/site_haproxy/templates/couch.erb b/puppet/modules/site_haproxy/templates/couch.erb index baa31486..f42e8368 100644 --- a/puppet/modules/site_haproxy/templates/couch.erb +++ b/puppet/modules/site_haproxy/templates/couch.erb @@ -4,8 +4,8 @@ frontend couch option httplog option dontlognull option http-server-close # use client keep-alive, but close server connection. - use_backend couch_write if METH_POST - default_backend couch_read + use_backend couch_read if METH_GET + default_backend couch_write backend couch_write mode http -- cgit v1.2.3 From d341c90c1493a78ed0ee2e216797651ff0aebfa9 Mon Sep 17 00:00:00 2001 From: Azul Date: Wed, 16 Jul 2014 10:32:27 +0200 Subject: haproxy connects to a local couch if available When running a service that requires couch (webapp or mx) on a node that also had couch running the haproxy was confused because it did not have an stunnel port for the local couch. Emit a more useful error and fixed this for webapp and mx --- provider_base/lib/macros/haproxy.rb | 8 ++++++-- provider_base/services/mx.json | 2 +- provider_base/services/webapp.json | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/provider_base/lib/macros/haproxy.rb b/provider_base/lib/macros/haproxy.rb index c0f9ede5..602ae726 100644 --- a/provider_base/lib/macros/haproxy.rb +++ b/provider_base/lib/macros/haproxy.rb @@ -39,6 +39,10 @@ module LeapCli # create the first pass of the servers hash servers = node_list.values.inject(Config::ObjectList.new) do |hsh, node| + # make sure we have a port to talk to + unless accept_ports[node.name] + error "haproxy needs a local port to talk to when connecting to #{node.name}" + end weight = default_weight try { weight = local_weight if self.location.name == node.location.name @@ -46,7 +50,7 @@ module LeapCli hsh[node.name] = Config::Object[ 'backup', false, 'host', 'localhost', - 'port', accept_ports[node.name] || 0, + 'port', accept_ports[node.name], 'weight', weight ] if node.services.include?('couchdb') @@ -66,4 +70,4 @@ module LeapCli end end -end \ No newline at end of file +end diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json index 1f0e613e..32a93638 100644 --- a/provider_base/services/mx.json +++ b/provider_base/services/mx.json @@ -7,7 +7,7 @@ "haproxy": { "couch": { "listen_port": 4096, - "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client)" + "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, 5984)" } }, "couchdb_leap_mx_user": { diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index 1b550af9..c1e3791f 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -39,7 +39,7 @@ "haproxy": { "couch": { "listen_port": 4096, - "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, global.services[:couchdb].couch.port)" + "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, global.services[:couchdb].couch.port, 5984)" } }, "definition_files": { -- cgit v1.2.3 From f7edf6d31a7ffbbc66ab778edec85f3cad4e6c82 Mon Sep 17 00:00:00 2001 From: Azul Date: Tue, 29 Jul 2014 14:53:50 +0200 Subject: fix haproxy_servers call with couchdb default port --- provider_base/services/mx.json | 2 +- provider_base/services/webapp.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json index 32a93638..11293ae8 100644 --- a/provider_base/services/mx.json +++ b/provider_base/services/mx.json @@ -7,7 +7,7 @@ "haproxy": { "couch": { "listen_port": 4096, - "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, 5984)" + "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, global.services[:couchdb].couch.port)" } }, "couchdb_leap_mx_user": { diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index 1af95022..3af0dade 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -40,7 +40,7 @@ "haproxy": { "couch": { "listen_port": 4096, - "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, global.services[:couchdb].couch.port, 5984)" + "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, global.services[:couchdb].couch.port)" } }, "definition_files": { -- cgit v1.2.3 From 42327f4881022424bed354356756ab5815d5ba3f Mon Sep 17 00:00:00 2001 From: Azul Date: Wed, 30 Jul 2014 16:47:47 +0200 Subject: add replication role to user databases with tapicero This way the replication has read access on the source and write access on the target. --- puppet/modules/tapicero/templates/tapicero.yaml.erb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/puppet/modules/tapicero/templates/tapicero.yaml.erb b/puppet/modules/tapicero/templates/tapicero.yaml.erb index d6ea56fa..510450ad 100644 --- a/puppet/modules/tapicero/templates/tapicero.yaml.erb +++ b/puppet/modules/tapicero/templates/tapicero.yaml.erb @@ -40,10 +40,11 @@ options: # explicit about this - <%= @couchdb_admin_user %> roles: [] - readers: + members: names: - <%= @couchdb_soledad_user %> - <%= @couchdb_leap_mx_user %> - roles: [] + roles: + - replication -- cgit v1.2.3 From 2e5d1a8cf2228a87d382bbf8a58d3f485b1ead65 Mon Sep 17 00:00:00 2001 From: Azul Date: Fri, 1 Aug 2014 11:01:49 +0200 Subject: minor: fix typo in webapp config @provider -> @webapp --- puppet/modules/site_webapp/templates/config.yml.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_webapp/templates/config.yml.erb b/puppet/modules/site_webapp/templates/config.yml.erb index ef139404..9205438b 100644 --- a/puppet/modules/site_webapp/templates/config.yml.erb +++ b/puppet/modules/site_webapp/templates/config.yml.erb @@ -18,7 +18,7 @@ production: minimum_client_version: "<%= @webapp['client_version']['min'] %>" default_service_level: "<%= @webapp['default_service_level'] %>" service_levels: <%= @webapp['service_levels'].to_json %> - allow_registration: <%= @provider['allow_registration'].inspect %> + allow_registration: <%= @webapp['allow_registration'].inspect %> <%- if @webapp['engines'] && @webapp['engines'].any? -%> engines: <%- @webapp['engines'].each do |engine| -%> -- cgit v1.2.3 From a54b82ff7cdae2e44bc3c159473ca03e283f0746 Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 22 Aug 2014 02:20:13 -0700 Subject: default to multimaster if no nodes are defined as master --- provider_base/services/couchdb.rb | 62 ++++++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/provider_base/services/couchdb.rb b/provider_base/services/couchdb.rb index 81f366e1..3bee3a67 100644 --- a/provider_base/services/couchdb.rb +++ b/provider_base/services/couchdb.rb @@ -1,18 +1,60 @@ +####################################################################### +### +### NOTE! +### +### Currently, mirrors do not work! The only thing that works is all +### nodes multimaster or a single master. +### +####################################################################### # # custom logic for couchdb json resolution +# ============================================ +# +# There are three modes for a node: +# +# Multimaster +# ----------- +# +# Multimaster uses bigcouch (soon to use couchdb in replication mode +# similar to bigcouch). +# +# Use "multimaster" mode when: +# +# * multiple nodes are marked couch.master +# * OR no nodes are marked couch.master +# +# Master +# ------ +# +# Master uses plain couchdb that is readable and writable. +# +# Use "master" mode when: +# +# * Exactly one node, this one, is marked as master. +# +# Mirror +# ------ +# +# Mirror creates a read-only copy of the database. It uses plain coucdhb +# with legacy couchdb replication (http based). +# +# This does not currently work, because http replication can't handle +# the number of user databases. +# +# Use "mirror" mode when: +# +# * some nodes are marked couch.master +# * AND this node is not a master # -unless nodes_like_me['services' => 'couchdb']['couch.master' => true].any? - error('there must be at least one node with couch.master set to `true` for environment `%s`.' % @node.environment) -end +master_count = nodes_like_me['services' => 'couchdb']['couch.master' => true].size -if couch.master - if nodes_like_me['services' => 'couchdb']['couch.master' => true].size > 1 - apply_partial 'services/_couchdb_multimaster.json' - else - apply_partial 'services/_couchdb_master.json' - end +if master_count == 0 + apply_partial 'services/_couchdb_multimaster.json' +elsif couch.master && master_count > 1 + apply_partial 'services/_couchdb_multimaster.json' +elsif couch.master && master_count == 1 + apply_partial 'services/_couchdb_master.json' else apply_partial 'services/_couchdb_mirror.json' end - -- cgit v1.2.3 From 72bad39f8a21c3be33b17134d2e3ca11f5e0d58f Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 2 Sep 2014 22:02:41 -0700 Subject: tests: make warnings not produce a non-zero exit code, add 'ignore' command to tests, make shorewall optional. --- bin/run_tests | 23 +++++++++++++++++++---- tests/white-box/network.rb | 40 ++++++++++++++++++++-------------------- 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/bin/run_tests b/bin/run_tests index 2ee027f4..e026b5f7 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -41,6 +41,7 @@ end ## # this class is raised if a test file wants to be skipped entirely. +# (to skip an individual test, MiniTest::Skip is used instead) class SkipTest < Exception end @@ -62,6 +63,8 @@ end class LeapTest < MiniTest::Unit::TestCase class Pass < MiniTest::Assertion end + class Ignore < MiniTest::Assertion + end def initialize(name) super(name) @@ -100,6 +103,13 @@ class LeapTest < MiniTest::Unit::TestCase raise LeapTest::Pass end + # + # Called when the test should be silently ignored. + # + def ignore + raise LeapTest::Ignore + end + # # the default fail() is part of the kernel and it just throws a runtime exception. for tests, # we want the same behavior as assert(false) @@ -332,6 +342,7 @@ class LeapRunner < MiniTest::Unit def initialize @passes = 0 @warnings = 0 + @ignores = 0 super end @@ -372,9 +383,12 @@ class LeapRunner < MiniTest::Unit case e when MiniTest::Skip then @skips += 1 - #if @verbose - report_line("SKIP", klass, meth, e, e.message) - #end + report_line("SKIP", klass, meth, e, e.message) + when LeapTest::Ignore then + @ignores += 1 + if @verbose + report_line("IGNORE", klass, meth, e, e.message) + end when LeapTest::Pass then @passes += 1 report_line("PASS", klass, meth) @@ -414,7 +428,8 @@ class LeapRunner < MiniTest::Unit elsif @failures > 0 :failure elsif @warnings > 0 - :warning + # :warning << warnings don't warrant a non-zero exit code. + :success else :success end diff --git a/tests/white-box/network.rb b/tests/white-box/network.rb index 118861a7..f2041710 100644 --- a/tests/white-box/network.rb +++ b/tests/white-box/network.rb @@ -26,35 +26,35 @@ class Network < LeapTest # connect: "127.0.0.1:5984" # def test_02_Is_stunnel_running? - if $node['stunnel'] - good_stunnel_pids = [] - $node['stunnel']['clients'].each do |stunnel_type, stunnel_configs| - stunnel_configs.each do |stunnel_name, stunnel_conf| - config_file_name = "/etc/stunnel/#{stunnel_name}.conf" - processes = pgrep(config_file_name) - assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`" - good_stunnel_pids += processes.map{|ps| ps[:pid]} - assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.' - assert_tcp_socket('localhost', port) - end - end - $node['stunnel']['servers'].each do |stunnel_name, stunnel_conf| + ignore unless $node['stunnel'] + good_stunnel_pids = [] + $node['stunnel']['clients'].each do |stunnel_type, stunnel_configs| + stunnel_configs.each do |stunnel_name, stunnel_conf| config_file_name = "/etc/stunnel/#{stunnel_name}.conf" processes = pgrep(config_file_name) assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`" good_stunnel_pids += processes.map{|ps| ps[:pid]} - assert accept_port = stunnel_conf['accept_port'], "Field `accept` must be present in property `stunnel.servers.#{stunnel_name}`" - assert_tcp_socket('localhost', accept_port) - assert connect_port = stunnel_conf['connect_port'], "Field `connect` must be present in property `stunnel.servers.#{stunnel_name}`" - assert_tcp_socket('localhost', connect_port) + assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.' + assert_tcp_socket('localhost', port) end - all_stunnel_pids = pgrep('/usr/bin/stunnel').collect{|process| process[:pid]}.uniq - assert_equal good_stunnel_pids.sort, all_stunnel_pids.sort, "There should not be any extra stunnel processes that are not configured in /etc/stunnel" - pass end + $node['stunnel']['servers'].each do |stunnel_name, stunnel_conf| + config_file_name = "/etc/stunnel/#{stunnel_name}.conf" + processes = pgrep(config_file_name) + assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`" + good_stunnel_pids += processes.map{|ps| ps[:pid]} + assert accept_port = stunnel_conf['accept_port'], "Field `accept` must be present in property `stunnel.servers.#{stunnel_name}`" + assert_tcp_socket('localhost', accept_port) + assert connect_port = stunnel_conf['connect_port'], "Field `connect` must be present in property `stunnel.servers.#{stunnel_name}`" + assert_tcp_socket('localhost', connect_port) + end + all_stunnel_pids = pgrep('/usr/bin/stunnel').collect{|process| process[:pid]}.uniq + assert_equal good_stunnel_pids.sort, all_stunnel_pids.sort, "There should not be any extra stunnel processes that are not configured in /etc/stunnel" + pass end def test_03_Is_shorewall_running? + ignore unless File.exists?('/sbin/shorewall') assert_run('/sbin/shorewall status') pass end -- cgit v1.2.3 From 77fe62679399c8a780106a50f579f0194be46e6b Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Mon, 15 Sep 2014 16:29:36 -0400 Subject: tests: make warnings not produce a non-zero exit code Change-Id: I60d51728128b95c77d52ab4e8c61966cfa59ff2f --- bin/run_tests | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/run_tests b/bin/run_tests index 2ee027f4..586a8ca1 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -414,7 +414,8 @@ class LeapRunner < MiniTest::Unit elsif @failures > 0 :failure elsif @warnings > 0 - :warning + # :warning << warnings don't warrant a non-zero exit code. + :success else :success end -- cgit v1.2.3 From a91da714d50b63f4f0b28047000633302cd9a8ea Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Mon, 15 Sep 2014 16:37:08 -0400 Subject: tests: make shorewall optional Change-Id: I1703ff7b3dafe5d0562a7c34c1851ebfedc569a8 --- tests/white-box/network.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/white-box/network.rb b/tests/white-box/network.rb index 0d98c314..f2041710 100644 --- a/tests/white-box/network.rb +++ b/tests/white-box/network.rb @@ -54,6 +54,7 @@ class Network < LeapTest end def test_03_Is_shorewall_running? + ignore unless File.exists?('/sbin/shorewall') assert_run('/sbin/shorewall status') pass end -- cgit v1.2.3 From 171fc59783a10dc341c435f47313271b89a12c0d Mon Sep 17 00:00:00 2001 From: varac Date: Wed, 17 Sep 2014 11:09:30 +0200 Subject: disable ACL enforcement, because it's a known issue with bigcouch otherwise it will only confuse the user see https://leap.se/code/issues/6030 for more details --- tests/white-box/couchdb.rb | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb index 6d3a7452..a5adb2bf 100644 --- a/tests/white-box/couchdb.rb +++ b/tests/white-box/couchdb.rb @@ -93,20 +93,24 @@ class CouchDB < LeapTest pass end + # disable ACL enforcement, because it's a known issue with bigcouch + # and will only confuse the user + # see https://leap.se/code/issues/6030 for more details # - # for now, this just prints warnings, since we are failing these tests. - # - def test_06_Is_ACL_enforced? - ok = assert_auth_fail( - couchdb_url('/users/_all_docs', :user => 'leap_mx'), - {:limit => 1} - ) - ok = assert_auth_fail( - couchdb_url('/users/_all_docs', :user => 'leap_mx'), - {:limit => 1} - ) && ok - pass if ok - end + ## for now, this just prints warnings, since we are failing these tests. + ## + + #def test_06_Is_ACL_enforced? + # ok = assert_auth_fail( + # couchdb_url('/users/_all_docs', :user => 'leap_mx'), + # {:limit => 1} + # ) + # ok = assert_auth_fail( + # couchdb_url('/users/_all_docs', :user => 'leap_mx'), + # {:limit => 1} + # ) && ok + # pass if ok + #end def test_07_What? pass -- cgit v1.2.3 From d6c078f4beecefe42c971cc5802e79f42396ebab Mon Sep 17 00:00:00 2001 From: varac Date: Wed, 17 Sep 2014 11:44:17 +0200 Subject: Increase wait-for-couch timeout (Bug #3735) Site_couchdb::Bigcouch::Settle_cluster/Exec[wait_for_couch_nodes] waits 60s for all nodes to be member of the cluster. Because we deploy to multiple nodes in parallel, not all nodes are ready at the same time, so we increased the timeout from 60s to 120s. --- puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp b/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp index aa843e2e..820b5be2 100644 --- a/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp +++ b/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp @@ -1,11 +1,11 @@ class site_couchdb::bigcouch::settle_cluster { exec { 'wait_for_couch_nodes': - command => '/srv/leap/bin/run_tests --test CouchDB/Are_configured_nodes_online? --retry 6 --wait 10' + command => '/srv/leap/bin/run_tests --test CouchDB/Are_configured_nodes_online? --retry 12 --wait 10' } exec { 'settle_cluster_membership': - command => '/srv/leap/bin/run_tests --test CouchDB/Is_cluster_membership_ok? --retry 6 --wait 10', + command => '/srv/leap/bin/run_tests --test CouchDB/Is_cluster_membership_ok? --retry 12 --wait 10', require => Exec['wait_for_couch_nodes'] } } -- cgit v1.2.3 From 9cc0f28e9223c76da9cd491d3faa2dd1b18e3fc2 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Wed, 17 Sep 2014 11:14:46 -0400 Subject: update rsyslog module to fix #6019 Change-Id: I8c64a0c530d44e55963060d52d31a0da1a88615c --- puppet/modules/rsyslog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/rsyslog b/puppet/modules/rsyslog index 20fbda6b..b8ef11c2 160000 --- a/puppet/modules/rsyslog +++ b/puppet/modules/rsyslog @@ -1 +1 @@ -Subproject commit 20fbda6b91472e656331a9c64630fb207e9f5789 +Subproject commit b8ef11c23949d12732ad5cdaebb3023ff39a297a -- cgit v1.2.3 From 63783c1dc0a1e1749810162af169f0ffc0a237d5 Mon Sep 17 00:00:00 2001 From: Christoph Date: Wed, 17 Sep 2014 17:30:38 +0200 Subject: allow outgoing port 3142 for apt-cacher proxy --- puppet/modules/site_config/templates/ipv4firewall_up.rules.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/puppet/modules/site_config/templates/ipv4firewall_up.rules.erb b/puppet/modules/site_config/templates/ipv4firewall_up.rules.erb index 928a2b31..7bde189f 100644 --- a/puppet/modules/site_config/templates/ipv4firewall_up.rules.erb +++ b/puppet/modules/site_config/templates/ipv4firewall_up.rules.erb @@ -18,6 +18,7 @@ -A OUTPUT -p tcp -m state --state NEW,ESTABLISHED --sport <%= @ssh_port %> -j ACCEPT -A OUTPUT -p tcp -m state --state NEW,ESTABLISHED --dport 80 -j ACCEPT -A OUTPUT -p tcp -m state --state NEW,ESTABLISHED --dport 443 -j ACCEPT +-A OUTPUT -p tcp -m state --state NEW,ESTABLISHED --dport 3142 -j ACCEPT -A OUTPUT -p udp -m udp --dport 53 -j ACCEPT -A OUTPUT -p udp -m udp --dport 123 -j ACCEPT -A OUTPUT -m limit --limit 5/min -j LOG --log-prefix "iptables denied: " --log-level 7 -- cgit v1.2.3 From a3dfbb9b44d0b3cdeaa451adf63ac870ca7fe1d7 Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 17 Sep 2014 14:15:22 -0700 Subject: override facter fact for fqdn --- bin/puppet_command | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/bin/puppet_command b/bin/puppet_command index a9d39066..e1752ec2 100755 --- a/bin/puppet_command +++ b/bin/puppet_command @@ -57,13 +57,9 @@ def apply end def set_hostname - unless File.exists?(HIERA_FILE) - puts("ERROR: Cannot set hostname without #{HIERA_FILE}") - exit(1) - end - hostname = YAML.load_file(HIERA_FILE)['name'] + hostname = hiera_file['name'] if hostname.nil? || hostname.empty? - puts('ERROR: NAME argument required') + puts('ERROR: "name" missing from hiera file') exit(1) end current_hostname_file = File.read('/etc/hostname') rescue nil @@ -75,7 +71,7 @@ def set_hostname f.write hostname end if File.read('/etc/hostname') == hostname - puts "Set /etc/hostname to #{hostname}" + puts "Changed /etc/hostname to #{hostname}" else puts "ERROR: failed to update /etc/hostname" end @@ -84,9 +80,9 @@ def set_hostname # call /bin/hostname if current_hostname != hostname if run("/bin/hostname #{hostname}") == 0 - puts "Set hostname to #{hostname}" + puts "Changed hostname to #{hostname}" else - puts "ERROR: failed to call `/bin/hostname #{hostname}`" + puts "ERROR: call to `/bin/hostname #{hostname}` returned an error." end end end @@ -97,9 +93,25 @@ end def puppet_apply(options={}, &block) options = {:verbosity => @verbosity, :tags => @tags}.merge(options) manifest = options[:manifest] || SITE_MANIFEST + fqdn = hiera_file['domain']['name'] Dir.chdir(PUPPET_DIRECTORY) do - return run("#{PUPPET_BIN} apply #{custom_parameters(options)} #{PUPPET_PARAMETERS} #{manifest}", &block) + return run("FACTER_fqdn='#{fqdn}' #{PUPPET_BIN} apply #{custom_parameters(options)} #{PUPPET_PARAMETERS} #{manifest}", &block) + end +end + +# +# Return a ruby object representing the contents of the hiera yaml file. +# +def hiera_file + unless File.exists?(HIERA_FILE) + puts("ERROR: hiera file '#{HIERA_FILE}' does not exist.") + exit(1) end + $hiera_contents ||= YAML.load_file(HIERA_FILE) + return $hiera_contents +rescue Exception => exc + puts("ERROR: problem reading hiera file '#{HIERA_FILE}' (#{exc})") + exit(1) end def custom_parameters(options) -- cgit v1.2.3 From 0d85c95328916425d8e4e29189b63a40c8567eea Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Mon, 22 Sep 2014 14:57:24 -0400 Subject: stop logging user-agent in apache, fixes #6129 Change-Id: I66384ae4a723be063790362f70e57228a0f1539b --- puppet/modules/site_apache/templates/vhosts.d/api.conf.erb | 2 ++ puppet/modules/site_apache/templates/vhosts.d/common.conf.erb | 2 ++ 2 files changed, 4 insertions(+) diff --git a/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb index 3360ac59..74cd1ced 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb @@ -2,12 +2,14 @@ ServerName <%= api_domain %> RewriteEngine On RewriteRule ^.*$ https://<%= api_domain -%>:<%= api_port -%>%{REQUEST_URI} [R=permanent,L] + CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common Listen 0.0.0.0:<%= api_port %> > ServerName <%= api_domain %> + CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common SSLEngine on SSLProtocol all -SSLv2 diff --git a/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb index ed430510..0e08529c 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb @@ -3,12 +3,14 @@ ServerAlias www.<%= domain %> RewriteEngine On RewriteRule ^.*$ https://<%= domain -%>%{REQUEST_URI} [R=permanent,L] + CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common ServerName <%= domain_name %> ServerAlias <%= domain %> ServerAlias www.<%= domain %> + CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common SSLEngine on SSLProtocol all -SSLv2 -- cgit v1.2.3 From af606967d39227390f5ef8403c64d693c1dfd76d Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 23 Sep 2014 14:48:48 -0700 Subject: couch: for neighbors, use 'couch.mode' instead of 'couch.master' (which might be false even for multimaster). closes #6064 --- provider_base/services/_couchdb_multimaster.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/provider_base/services/_couchdb_multimaster.json b/provider_base/services/_couchdb_multimaster.json index 8c433188..0f340e00 100644 --- a/provider_base/services/_couchdb_multimaster.json +++ b/provider_base/services/_couchdb_multimaster.json @@ -8,8 +8,8 @@ "ednp_server": "= stunnel_server(couch.bigcouch.ednp_port)" }, "clients": { - "epmd_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], couch.bigcouch.epmd_port)", - "ednp_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], couch.bigcouch.ednp_port)" + "epmd_clients": "= stunnel_client(nodes_like_me['services' => 'couchdb']['couch.mode' => 'multimaster'], couch.bigcouch.epmd_port)", + "ednp_clients": "= stunnel_client(nodes_like_me['services' => 'couchdb']['couch.mode' => 'multimaster'], couch.bigcouch.ednp_port)" } }, "couch": { @@ -18,7 +18,7 @@ "epmd_port": 4369, "ednp_port": 9002, "cookie": "= secret :bigcouch_cookie", - "neighbors": "= nodes_like_me['services' => 'couchdb']['couch.master' => true].exclude(self).field('domain.full')" + "neighbors": "= nodes_like_me['services' => 'couchdb']['couch.mode' => 'multimaster'].exclude(self).field('domain.full')" } } } -- cgit v1.2.3 From 9da2a36155d3b96e0dc41cac3dd38f8b6c50efd2 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Mon, 22 Sep 2014 14:57:24 -0400 Subject: stop logging user-agent in apache, fixes #6129 Change-Id: I66384ae4a723be063790362f70e57228a0f1539b --- puppet/modules/site_apache/templates/vhosts.d/api.conf.erb | 2 ++ puppet/modules/site_apache/templates/vhosts.d/common.conf.erb | 2 ++ 2 files changed, 4 insertions(+) diff --git a/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb index 3360ac59..74cd1ced 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb @@ -2,12 +2,14 @@ ServerName <%= api_domain %> RewriteEngine On RewriteRule ^.*$ https://<%= api_domain -%>:<%= api_port -%>%{REQUEST_URI} [R=permanent,L] + CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common Listen 0.0.0.0:<%= api_port %> > ServerName <%= api_domain %> + CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common SSLEngine on SSLProtocol all -SSLv2 diff --git a/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb index ed430510..0e08529c 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb @@ -3,12 +3,14 @@ ServerAlias www.<%= domain %> RewriteEngine On RewriteRule ^.*$ https://<%= domain -%>%{REQUEST_URI} [R=permanent,L] + CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common ServerName <%= domain_name %> ServerAlias <%= domain %> ServerAlias www.<%= domain %> + CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common SSLEngine on SSLProtocol all -SSLv2 -- cgit v1.2.3 From 308ac3cb420cbce7a6c67e03e887db4723aa8169 Mon Sep 17 00:00:00 2001 From: varac Date: Wed, 24 Sep 2014 13:55:20 +0200 Subject: remove /etc/apt/preferences.d/fixed_rsyslog_anon_package (#6138) This was a leftover from earlier versions, where we installed rsyslog from the leap debian package repo. Change-Id: I88a852f08b5aff3bd7b591b6220ac354463a9786 --- puppet/modules/site_apt/manifests/preferences/rsyslog.pp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/puppet/modules/site_apt/manifests/preferences/rsyslog.pp b/puppet/modules/site_apt/manifests/preferences/rsyslog.pp index 132a6e24..bfeaa7da 100644 --- a/puppet/modules/site_apt/manifests/preferences/rsyslog.pp +++ b/puppet/modules/site_apt/manifests/preferences/rsyslog.pp @@ -1,9 +1,13 @@ class site_apt::preferences::rsyslog { - apt::preferences_snippet { 'rsyslog_anon_depends': - package => 'libestr0 librelp0 rsyslog*', - priority => '999', - pin => 'release a=wheezy-backports', - before => Class['rsyslog::install'] + apt::preferences_snippet { + 'rsyslog_anon_depends': + package => 'libestr0 librelp0 rsyslog*', + priority => '999', + pin => 'release a=wheezy-backports', + before => Class['rsyslog::install']; + + 'fixed_rsyslog_anon_package': + ensure => absent; } } -- cgit v1.2.3 From 8f8862aab798cbf0b2dbb690a154cd54dd5d6592 Mon Sep 17 00:00:00 2001 From: irregulator Date: Thu, 25 Sep 2014 16:46:57 +0300 Subject: Use member function instead of regexp to check services array --- puppet/modules/site_obfsproxy/manifests/init.pp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/puppet/modules/site_obfsproxy/manifests/init.pp b/puppet/modules/site_obfsproxy/manifests/init.pp index 40b7fba8..6275ebee 100644 --- a/puppet/modules/site_obfsproxy/manifests/init.pp +++ b/puppet/modules/site_obfsproxy/manifests/init.pp @@ -11,13 +11,13 @@ class site_obfsproxy { $dest_ip = $obfsproxy['gateway_address'] $dest_port = '443' - if $::services =~ /\bopenvpn\b/ { - $openvpn = hiera('openvpn') - $bind_address = $openvpn['gateway_address'] - } - elsif $::services =~ /\bobfsproxy\b/ { - $bind_address = hiera('ip_address') - } + if member($::services, 'openvpn') { + $openvpn = hiera('openvpn') + $bind_address = $openvpn['gateway_address'] + } + elsif member($::services, 'obfsproxy') { + $bind_address = hiera('ip_address') + } include site_apt::preferences::twisted include site_apt::preferences::obfsproxy -- cgit v1.2.3 From 343572ab04686c65c10fd49a5d09314ca99b3d75 Mon Sep 17 00:00:00 2001 From: Christoph Kluenter Date: Thu, 25 Sep 2014 16:01:37 +0200 Subject: allow all outgoing traffic as discussed on #leap --- .../modules/site_config/templates/ipv4firewall_up.rules.erb | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/puppet/modules/site_config/templates/ipv4firewall_up.rules.erb b/puppet/modules/site_config/templates/ipv4firewall_up.rules.erb index 7bde189f..b0c2b7ad 100644 --- a/puppet/modules/site_config/templates/ipv4firewall_up.rules.erb +++ b/puppet/modules/site_config/templates/ipv4firewall_up.rules.erb @@ -2,7 +2,7 @@ *filter :INPUT DROP [0:0] :FORWARD DROP [0:0] -:OUTPUT DROP [0:0] +:OUTPUT ACCEPT [0:0] -A INPUT -i lo -j ACCEPT -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT -A INPUT -p tcp -m state --state NEW,ESTABLISHED --dport 22 -j ACCEPT @@ -11,15 +11,4 @@ -A INPUT -p icmp -m icmp --icmp-type 8 -m state --state NEW,RELATED,ESTABLISHED -j ACCEPT -A INPUT -p icmp -m icmp --icmp-type 0 -m state --state RELATED,ESTABLISHED -j ACCEPT -A INPUT -m limit --limit 5/min -j LOG --log-prefix "iptables denied: " --log-level 7 --A OUTPUT -o lo -j ACCEPT --A OUTPUT -p icmp -m icmp --icmp-type 0 -m state --state RELATED,ESTABLISHED -j ACCEPT --A OUTPUT -p icmp -m icmp --icmp-type 8 -m state --state NEW,RELATED,ESTABLISHED -j ACCEPT --A OUTPUT -p tcp -m state --state NEW,ESTABLISHED --sport 22 -j ACCEPT --A OUTPUT -p tcp -m state --state NEW,ESTABLISHED --sport <%= @ssh_port %> -j ACCEPT --A OUTPUT -p tcp -m state --state NEW,ESTABLISHED --dport 80 -j ACCEPT --A OUTPUT -p tcp -m state --state NEW,ESTABLISHED --dport 443 -j ACCEPT --A OUTPUT -p tcp -m state --state NEW,ESTABLISHED --dport 3142 -j ACCEPT --A OUTPUT -p udp -m udp --dport 53 -j ACCEPT --A OUTPUT -p udp -m udp --dport 123 -j ACCEPT --A OUTPUT -m limit --limit 5/min -j LOG --log-prefix "iptables denied: " --log-level 7 COMMIT -- cgit v1.2.3 From 027c20e2b8f779086d1480048152fe06d044b216 Mon Sep 17 00:00:00 2001 From: varac Date: Tue, 7 Oct 2014 13:55:58 +0200 Subject: every environment is defined as nagios hostsgroup (#5216) Change-Id: I6508ce0d06b37a1c5601a0e981a59f7fda47f76a --- provider_base/services/monitor.json | 1 + puppet/modules/site_check_mk/manifests/server.pp | 13 +++++++++---- puppet/modules/site_check_mk/templates/hostgroups.mk | 4 ++++ puppet/modules/site_nagios/manifests/server.pp | 9 ++++++--- puppet/modules/site_nagios/manifests/server/hostgroup.pp | 3 +++ 5 files changed, 23 insertions(+), 7 deletions(-) create mode 100644 puppet/modules/site_check_mk/templates/hostgroups.mk create mode 100644 puppet/modules/site_nagios/manifests/server/hostgroup.pp diff --git a/provider_base/services/monitor.json b/provider_base/services/monitor.json index c24724bf..56ca015b 100644 --- a/provider_base/services/monitor.json +++ b/provider_base/services/monitor.json @@ -1,6 +1,7 @@ { "nagios": { "nagiosadmin_pw": "= secret :nagios_admin_password", + "domains_internal": "= global.tags.field('domain.internal_suffix').compact.uniq", "hosts": "= (self.environment == 'local' ? nodes_like_me : nodes[:environment => '!local']).pick_fields('domain.internal', 'domain.full_suffix', 'ip_address', 'services', 'openvpn.gateway_address', 'ssh.port')" }, "hosts": "= self.environment == 'local' ? hosts_file(nodes_like_me) : hosts_file(nodes[:environment => '!local'])", diff --git a/puppet/modules/site_check_mk/manifests/server.pp b/puppet/modules/site_check_mk/manifests/server.pp index e544ef0d..aa24d96c 100644 --- a/puppet/modules/site_check_mk/manifests/server.pp +++ b/puppet/modules/site_check_mk/manifests/server.pp @@ -5,11 +5,12 @@ class site_check_mk::server { $type = $ssh_hash['authorized_keys']['monitor']['type'] $seckey = $ssh_hash['monitor']['private_key'] - $nagios_hiera = hiera_hash('nagios') - $nagios_hosts = $nagios_hiera['hosts'] + $nagios_hiera = hiera_hash('nagios') + $nagios_hosts = $nagios_hiera['hosts'] - $hosts = hiera_hash('hosts') - $all_hosts = inline_template ('<% @hosts.keys.sort.each do |key| -%>"<%= @hosts[key]["domain_internal"] %>", <% end -%>') + $hosts = hiera_hash('hosts') + $all_hosts = inline_template ('<% @hosts.keys.sort.each do |key| -%>"<%= @hosts[key]["domain_internal"] %>", <% end -%>') + $domains_internal = $nagios_hiera['domains_internal'] package { 'check-mk-server': ensure => installed, @@ -35,6 +36,10 @@ class site_check_mk::server { content => template('site_check_mk/use_ssh.mk'), notify => Exec['check_mk-refresh'], require => Package['check-mk-server']; + '/etc/check_mk/conf.d/hostgroups.mk': + content => template('site_check_mk/hostgroups.mk'), + notify => Exec['check_mk-refresh'], + require => Package['check-mk-server']; '/etc/check_mk/all_hosts_static': content => $all_hosts, notify => Exec['check_mk-refresh'], diff --git a/puppet/modules/site_check_mk/templates/hostgroups.mk b/puppet/modules/site_check_mk/templates/hostgroups.mk new file mode 100644 index 00000000..79b7f92f --- /dev/null +++ b/puppet/modules/site_check_mk/templates/hostgroups.mk @@ -0,0 +1,4 @@ +host_groups = [ + <% @domains_internal.each do |domain| %>( '<%= domain %>', [<% @nagios_hosts.keys.sort.each do |key| -%><% if @nagios_hosts[key]['domain_internal'] == key+'.'+domain -%>'<%= key %>.<%= domain %>', <% end -%><% end -%>] ), + <% end -%> +] diff --git a/puppet/modules/site_nagios/manifests/server.pp b/puppet/modules/site_nagios/manifests/server.pp index 85443917..8cc1ae24 100644 --- a/puppet/modules/site_nagios/manifests/server.pp +++ b/puppet/modules/site_nagios/manifests/server.pp @@ -3,9 +3,10 @@ class site_nagios::server inherits nagios::base { # First, purge old nagios config (see #1467) class { 'site_nagios::server::purge': } - $nagios_hiera = hiera('nagios') - $nagiosadmin_pw = htpasswd_sha1($nagios_hiera['nagiosadmin_pw']) - $nagios_hosts = $nagios_hiera['hosts'] + $nagios_hiera = hiera('nagios') + $nagiosadmin_pw = htpasswd_sha1($nagios_hiera['nagiosadmin_pw']) + $nagios_hosts = $nagios_hiera['hosts'] + $domains_internal = $nagios_hiera['domains_internal'] include nagios::defaults include nagios::base @@ -55,4 +56,6 @@ class site_nagios::server inherits nagios::base { 'set missingok missingok', 'set ifempty notifempty', 'set copytruncate copytruncate' ] } + + ::site_nagios::server::hostgroup { $domains_internal: } } diff --git a/puppet/modules/site_nagios/manifests/server/hostgroup.pp b/puppet/modules/site_nagios/manifests/server/hostgroup.pp new file mode 100644 index 00000000..035ba7d1 --- /dev/null +++ b/puppet/modules/site_nagios/manifests/server/hostgroup.pp @@ -0,0 +1,3 @@ +define site_nagios::server::hostgroup { + nagios_hostgroup { $name: } +} -- cgit v1.2.3 From 36a1332f5f6e43591a47b89ec505541f84381667 Mon Sep 17 00:00:00 2001 From: varac Date: Tue, 7 Oct 2014 14:16:54 +0200 Subject: include different nagios::defaults classes manually (#5216) nagios::defaults will include nagios::defaults::hostgroups which add "all" and "centos_servers" hostgroups which we don't want. Change-Id: If42faa11c167fb7305ebbb21dc358a8813afaa25 --- puppet/modules/site_nagios/manifests/server.pp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/puppet/modules/site_nagios/manifests/server.pp b/puppet/modules/site_nagios/manifests/server.pp index 8cc1ae24..b195c880 100644 --- a/puppet/modules/site_nagios/manifests/server.pp +++ b/puppet/modules/site_nagios/manifests/server.pp @@ -8,8 +8,14 @@ class site_nagios::server inherits nagios::base { $nagios_hosts = $nagios_hiera['hosts'] $domains_internal = $nagios_hiera['domains_internal'] - include nagios::defaults include nagios::base + include nagios::defaults::commands + include nagios::defaults::contactgroups + include nagios::defaults::contacts + include nagios::defaults::templates + include nagios::defaults::timeperiods + include nagios::defaults::plugins + class {'nagios': # don't manage apache class from nagios, cause we already include # it in site_apache::common -- cgit v1.2.3 From 189bd4b704ba685640ca01afe90f592e7b33567a Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Wed, 15 Oct 2014 17:07:45 -0400 Subject: Disable SSLv3, and RC4 ciphers Change-Id: I7214aa4334e3d817dd1b6d8dce43523e3d955b5d --- puppet/modules/site_apache/templates/vhosts.d/api.conf.erb | 4 ++-- puppet/modules/site_apache/templates/vhosts.d/common.conf.erb | 4 ++-- puppet/modules/site_nickserver/templates/nickserver-proxy.conf.erb | 5 +++-- puppet/modules/site_static/templates/apache.conf.erb | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb index 74cd1ced..e4732289 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb @@ -12,10 +12,10 @@ Listen 0.0.0.0:<%= api_port %> CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common SSLEngine on - SSLProtocol all -SSLv2 + SSLProtocol all -SSLv2 -SSLv3 SSLHonorCipherOrder on SSLCompression off - SSLCipherSuite "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-ECDSA-RC4-SHA:AES128:AES256:RC4-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK" + SSLCipherSuite "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128:AES256:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK" SSLCACertificatePath /etc/ssl/certs SSLCertificateChainFile <%= scope.lookupvar('x509::variables::local_CAs') %>/<%= scope.lookupvar('site_config::params::ca_name') %>.crt diff --git a/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb index 0e08529c..a9733a97 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb @@ -13,10 +13,10 @@ CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common SSLEngine on - SSLProtocol all -SSLv2 + SSLProtocol all -SSLv2 -SSLv3 SSLHonorCipherOrder on SSLCompression off - SSLCipherSuite "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-ECDSA-RC4-SHA:AES128:AES256:RC4-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK" + SSLCipherSuite "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128:AES256:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK" SSLCACertificatePath /etc/ssl/certs SSLCertificateChainFile <%= scope.lookupvar('x509::variables::local_CAs') %>/<%= scope.lookupvar('site_config::params::commercial_ca_name') %>.crt diff --git a/puppet/modules/site_nickserver/templates/nickserver-proxy.conf.erb b/puppet/modules/site_nickserver/templates/nickserver-proxy.conf.erb index ae06410e..56a8d9f6 100644 --- a/puppet/modules/site_nickserver/templates/nickserver-proxy.conf.erb +++ b/puppet/modules/site_nickserver/templates/nickserver-proxy.conf.erb @@ -9,9 +9,10 @@ Listen 0.0.0.0:<%= @nickserver_port -%> ServerAlias <%= @address_domain %> SSLEngine on - SSLProtocol -all +SSLv3 +TLSv1 - SSLCipherSuite HIGH:MEDIUM:!aNULL:!SSLv2:!MD5:@STRENGTH + SSLProtocol all -SSLv2 -SSLv3 SSLHonorCipherOrder on + SSLCompression off + SSLCipherSuite "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128:AES256:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK" SSLCACertificatePath /etc/ssl/certs SSLCertificateChainFile <%= scope.lookupvar('x509::variables::local_CAs') %>/<%= scope.lookupvar('site_config::params::ca_name') %>.crt diff --git a/puppet/modules/site_static/templates/apache.conf.erb b/puppet/modules/site_static/templates/apache.conf.erb index 07ac481d..9b516a10 100644 --- a/puppet/modules/site_static/templates/apache.conf.erb +++ b/puppet/modules/site_static/templates/apache.conf.erb @@ -46,10 +46,10 @@ #RewriteLogLevel 3 SSLEngine on - SSLProtocol all -SSLv2 + SSLProtocol all -SSLv2 -SSLv3 SSLHonorCipherOrder on SSLCompression off - SSLCipherSuite "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-ECDSA-RC4-SHA:AES128:AES256:RC4-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK" + SSLCipherSuite "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128:AES256:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK" <%- if @tls_only -%> Header add Strict-Transport-Security: "max-age=15768000;includeSubdomains" -- cgit v1.2.3 From 4faf77c2527312b85d836600c01c0d2d9f0b460e Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 20 Oct 2014 14:27:53 -0700 Subject: bumped default server certificate bit size to 4096 --- provider_base/provider.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider_base/provider.json b/provider_base/provider.json index 743964ee..4f71c7d5 100644 --- a/provider_base/provider.json +++ b/provider_base/provider.json @@ -44,7 +44,7 @@ "digest": "SHA256", "life_span": "10y", "server_certificates": { - "bit_size": 2048, + "bit_size": 4096, "digest": "SHA256", "life_span": "1y" }, -- cgit v1.2.3 From f6ffad33042aa6580ec00ef23836291861c1ae17 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Mon, 20 Oct 2014 21:44:37 -0400 Subject: implement custom puppet support (#6201, #6226) change puppet command to include in the --modulepath /srv/leap/files/puppet/modules If a provider places puppet code under files/puppet it will be sync'd over to all the nodes, once leap cli #6225 is merged. The custom puppet entry point is in class 'custom' which can be put into files/puppet/modules/custom/manifests/init.pp Change-Id: I74879c6ee056b03cd4691aa81a7668b60383bdad --- bin/puppet_command | 5 ++++- puppet/modules/site_config/manifests/default.pp | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/bin/puppet_command b/bin/puppet_command index 5e690bef..cdb0b027 100755 --- a/bin/puppet_command +++ b/bin/puppet_command @@ -14,6 +14,8 @@ PUPPET_BIN = '/usr/bin/puppet' PUPPET_DIRECTORY = '/srv/leap' PUPPET_PARAMETERS = '--color=false --detailed-exitcodes --libdir=puppet/lib --confdir=puppet' SITE_MANIFEST = 'puppet/manifests/site.pp' +SITE_MODULES = 'puppet/modules' +CUSTOM_MODULES = ':files/puppet/modules' DEFAULT_TAGS = 'leap_base,leap_service' HIERA_FILE = '/etc/leap/hiera.yaml' @@ -93,10 +95,11 @@ end def puppet_apply(options={}, &block) options = {:verbosity => @verbosity, :tags => @tags}.merge(options) manifest = options[:manifest] || SITE_MANIFEST + modulepath = options[:module_path] || SITE_MODULES + CUSTOM_MODULES fqdn = hiera_file['domain']['name'] domain = hiera_file['domain']['full_suffix'] Dir.chdir(PUPPET_DIRECTORY) do - return run("FACTER_fqdn='#{fqdn}' FACTER_domain='#{domain}' #{PUPPET_BIN} apply #{custom_parameters(options)} #{PUPPET_PARAMETERS} #{manifest}", &block) + return run("FACTER_fqdn='#{fqdn}' FACTER_domain='#{domain}' #{PUPPET_BIN} apply #{custom_parameters(options)} --modulepath='#{modulepath}' #{PUPPET_PARAMETERS} #{manifest}", &block) end end diff --git a/puppet/modules/site_config/manifests/default.pp b/puppet/modules/site_config/manifests/default.pp index fc2179de..790b5a16 100644 --- a/puppet/modules/site_config/manifests/default.pp +++ b/puppet/modules/site_config/manifests/default.pp @@ -59,10 +59,10 @@ class site_config::default { include site_postfix::satellite } - # if class site_custom exists, include it. + # if class custom exists, include it. # possibility for users to define custom puppet recipes - if defined( '::site_custom') { - include ::site_custom + if defined( '::custom') { + include ::custom } include site_check_mk::agent -- cgit v1.2.3 From 51a1e1c6db33ed2868ec74728f854237e3dfd86a Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 21 Oct 2014 15:14:23 -0700 Subject: update platform to take advantage of new platform.rb. requires leap_cli 1.6 --- platform.rb | 18 +++++++++++++++++- provider_base/common.json | 4 ++++ provider_base/lib/macros/files.rb | 2 +- provider_base/provider.json | 1 - 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/platform.rb b/platform.rb index 270dd25a..35017559 100644 --- a/platform.rb +++ b/platform.rb @@ -5,15 +5,24 @@ Leap::Platform.define do self.version = "0.5.4.1" - self.compatible_cli = "1.5.8".."1.99" + self.compatible_cli = "1.6.0".."1.99" # # the facter facts that should be gathered # self.facts = ["ec2_local_ipv4", "ec2_public_ipv4"] + # + # absolute paths on the destination server + # + self.hiera_path = '/etc/leap/hiera.yaml' + self.leap_dir = '/srv/leap' + self.files_dir = '/srv/leap/files' + self.init_path = '/srv/leap/initialized' + # # the named paths for this platform + # (relative to the provider directory) # self.paths = { # directories @@ -43,6 +52,11 @@ Leap::Platform.define do :soledad_service_json_template => 'files/service-definitions/#{arg}/soledad-service.json.erb', :smtp_service_json_template => 'files/service-definitions/#{arg}/smtp-service.json.erb', + # custom puppet + :custom_puppet_dir => 'files/puppet', + :custom_puppet_modules_dir => 'files/puppet/modules', + :custom_puppet_manifests_dir => 'files/puppet/manifests', + # output files :facts => 'facts.json', :user_ssh => 'users/#{arg}/#{arg}_ssh.pub', @@ -85,5 +99,7 @@ Leap::Platform.define do self.monitor_username = 'monitor' self.reserved_usernames = ['monitor'] + + self.default_puppet_tags = ['leap_base','leap_service'] end diff --git a/provider_base/common.json b/provider_base/common.json index 87af2152..649db0d9 100644 --- a/provider_base/common.json +++ b/provider_base/common.json @@ -46,5 +46,9 @@ "stunnel": { "clients": {}, "servers": {} + }, + "platform": { + "version": "= Leap::Platform.version.to_s", + "major_version": "= Leap::Platform.major_version" } } diff --git a/provider_base/lib/macros/files.rb b/provider_base/lib/macros/files.rb index 0a491325..b3ba4a06 100644 --- a/provider_base/lib/macros/files.rb +++ b/provider_base/lib/macros/files.rb @@ -71,7 +71,7 @@ module LeapCli end relative_path = Path.relative_path(actual_path) @node.file_paths << relative_path - @node.manager.provider.hiera_sync_destination + '/' + relative_path + File.join(Leap::Platform.files_dir, relative_path) end end diff --git a/provider_base/provider.json b/provider_base/provider.json index 4f71c7d5..9ef0f76a 100644 --- a/provider_base/provider.json +++ b/provider_base/provider.json @@ -56,7 +56,6 @@ "unlimited_prefix": "UNLIMITED" } }, - "hiera_sync_destination": "/etc/leap", "client_version": { "min": "0.5", "max": null -- cgit v1.2.3 From b76445c07688ba1b4e8940189f8538a741de92d4 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 21 Oct 2014 20:27:06 -0400 Subject: modify the leap repository contents so they pick the correct repository, based on the hiera value 'major_version' (#6251) Change-Id: I10532ef83e3aa2d35d9c0be241952a35e366bba4 --- platform.rb | 2 +- puppet/modules/site_apt/manifests/leap_repo.pp | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/platform.rb b/platform.rb index 35017559..06a49b7e 100644 --- a/platform.rb +++ b/platform.rb @@ -4,7 +4,7 @@ # Leap::Platform.define do - self.version = "0.5.4.1" + self.version = "0.6" self.compatible_cli = "1.6.0".."1.99" # diff --git a/puppet/modules/site_apt/manifests/leap_repo.pp b/puppet/modules/site_apt/manifests/leap_repo.pp index 6b3d9919..2d4ba0e1 100644 --- a/puppet/modules/site_apt/manifests/leap_repo.pp +++ b/puppet/modules/site_apt/manifests/leap_repo.pp @@ -1,6 +1,9 @@ class site_apt::leap_repo { + $platform = hiera_hash('platform') + $major_version = $platform['major_version'] + apt::sources_list { 'leap.list': - content => 'deb http://deb.leap.se/debian stable main', + content => "deb http://deb.leap.se/${major_version} wheezy main\n", before => Exec[refresh_apt] } -- cgit v1.2.3 From 937c61b74bbd99f9955cbee426fb35e96050eea6 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Mon, 27 Oct 2014 09:23:23 -0400 Subject: Change stunnel default sslversion to be TLSv1, instead of the default SSLv3 (#6261) Change-Id: I7ab5a6455e434f8359169d31febed8b92f84bbcc --- puppet/modules/site_stunnel/manifests/client.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/puppet/modules/site_stunnel/manifests/client.pp b/puppet/modules/site_stunnel/manifests/client.pp index 12d664b4..76815174 100644 --- a/puppet/modules/site_stunnel/manifests/client.pp +++ b/puppet/modules/site_stunnel/manifests/client.pp @@ -35,6 +35,7 @@ define site_stunnel::client ( pid => "/var/run/stunnel4/${pid}.pid", rndfile => $rndfile, debuglevel => $debuglevel, + sslversion => 'TLSv1', subscribe => [ Class['Site_config::X509::Key'], Class['Site_config::X509::Cert'], -- cgit v1.2.3 From f14cfe9601ab60b30af7baf126cb223eacec3593 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 28 Oct 2014 12:18:27 -0400 Subject: upgrade unattended-upgrades on deploy (#6245) unattended-upgrades is not able to upgrade itself in certain situations, such as when the conffile prompt is generated due to the config being changed. We want to set this package as latest in the platform so that it is upgraded on every deploy (we deploy the config anyway). Change-Id: I8c99bfb1b001079f0e1a4ffbf048e0e867633335 --- puppet/modules/site_apt/manifests/init.pp | 4 ++-- puppet/modules/site_apt/manifests/unattended_upgrades.pp | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 puppet/modules/site_apt/manifests/unattended_upgrades.pp diff --git a/puppet/modules/site_apt/manifests/init.pp b/puppet/modules/site_apt/manifests/init.pp index 9facf4cc..633ccf1e 100644 --- a/puppet/modules/site_apt/manifests/init.pp +++ b/puppet/modules/site_apt/manifests/init.pp @@ -1,4 +1,4 @@ -class site_apt { +class site_apt { class { 'apt': custom_key_dir => 'puppet:///modules/site_apt/keys' @@ -11,7 +11,7 @@ class site_apt { content => 'Acquire::PDiffs "false";'; } - include ::apt::unattended_upgrades + include ::site_apt::unattended_upgrades apt::sources_list { 'secondary.list.disabled': content => template('site_apt/secondary.list'); diff --git a/puppet/modules/site_apt/manifests/unattended_upgrades.pp b/puppet/modules/site_apt/manifests/unattended_upgrades.pp new file mode 100644 index 00000000..daebffab --- /dev/null +++ b/puppet/modules/site_apt/manifests/unattended_upgrades.pp @@ -0,0 +1,10 @@ +class site_apt::unattended_upgrades inherits apt::unattended_upgrades { + # override unattended-upgrades package resource to make sure + # that it is upgraded on every deploy (#6245) + + include ::apt::unattended_upgrades + + Package['unattended-upgrades'] { + ensure => latest + } +} -- cgit v1.2.3 From d3e24760b33d6ae20f153d3c144d7d443fb0b69e Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 29 Oct 2014 15:20:54 -0700 Subject: added webapp.forbidden_usernames property to allow configuration of usernames to block. --- provider_base/services/webapp.json | 1 + puppet/modules/site_webapp/templates/config.yml.erb | 1 + 2 files changed, 2 insertions(+) diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index 3af0dade..44b5fa14 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -1,6 +1,7 @@ { "webapp": { "admins": [], + "forbidden_usernames": ["admin", "administrator", "arin-admin", "certmaster", "contact", "info", "maildrop", "postmaster", "ssladmin", "www-data"], "domain": "= domain.full_suffix", "modules": ["user", "billing", "help"], "couchdb_webapp_user": { diff --git a/puppet/modules/site_webapp/templates/config.yml.erb b/puppet/modules/site_webapp/templates/config.yml.erb index 9205438b..0c75f3ca 100644 --- a/puppet/modules/site_webapp/templates/config.yml.erb +++ b/puppet/modules/site_webapp/templates/config.yml.erb @@ -19,6 +19,7 @@ production: default_service_level: "<%= @webapp['default_service_level'] %>" service_levels: <%= @webapp['service_levels'].to_json %> allow_registration: <%= @webapp['allow_registration'].inspect %> + handle_blacklist: <%= @webapp['forbidden_usernames'].inspect %> <%- if @webapp['engines'] && @webapp['engines'].any? -%> engines: <%- @webapp['engines'].each do |engine| -%> -- cgit v1.2.3 From 1d4670f8b9b4c1f3d4cd8017a3f6145ccdd41312 Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 31 Oct 2014 00:01:57 -0700 Subject: add support for property tor.key --- platform.rb | 12 +++--- provider_base/lib/macros.rb | 1 + provider_base/lib/macros/files.rb | 14 ++++++- provider_base/lib/macros/keys.rb | 78 +++++++++++++++++++++++++++++++++++++++ provider_base/services/tor.json | 9 ++++- 5 files changed, 106 insertions(+), 8 deletions(-) create mode 100644 provider_base/lib/macros/keys.rb diff --git a/platform.rb b/platform.rb index 06a49b7e..c37b6d29 100644 --- a/platform.rb +++ b/platform.rb @@ -5,7 +5,7 @@ Leap::Platform.define do self.version = "0.6" - self.compatible_cli = "1.6.0".."1.99" + self.compatible_cli = "1.6.1".."1.99" # # the facter facts that should be gathered @@ -77,10 +77,12 @@ Leap::Platform.define do :vagrantfile => 'test/Vagrantfile', # node output files - :hiera => 'hiera/#{arg}.yaml', - :node_ssh_pub_key => 'files/nodes/#{arg}/#{arg}_ssh.pub', - :node_x509_key => 'files/nodes/#{arg}/#{arg}.key', - :node_x509_cert => 'files/nodes/#{arg}/#{arg}.crt', + :hiera => 'hiera/#{arg}.yaml', + :node_ssh_pub_key => 'files/nodes/#{arg}/#{arg}_ssh.pub', + :node_x509_key => 'files/nodes/#{arg}/#{arg}.key', + :node_x509_cert => 'files/nodes/#{arg}/#{arg}.crt', + :node_tor_priv_key => 'files/nodes/#{arg}/tor.key', + :node_tor_pub_key => 'files/nodes/#{arg}/tor.pub', # testing files :test_client_key => 'test/cert/client.key', diff --git a/provider_base/lib/macros.rb b/provider_base/lib/macros.rb index 854b92b5..ecc3e6ba 100644 --- a/provider_base/lib/macros.rb +++ b/provider_base/lib/macros.rb @@ -9,6 +9,7 @@ require_relative 'macros/core' require_relative 'macros/files' require_relative 'macros/haproxy' require_relative 'macros/hosts' +require_relative 'macros/keys' require_relative 'macros/nodes' require_relative 'macros/secrets' require_relative 'macros/stunnel' diff --git a/provider_base/lib/macros/files.rb b/provider_base/lib/macros/files.rb index b3ba4a06..958958bc 100644 --- a/provider_base/lib/macros/files.rb +++ b/provider_base/lib/macros/files.rb @@ -48,13 +48,22 @@ module LeapCli # * if the path does not exist locally, but exists in provider_base, then the default file from # provider_base is copied locally. this is required for rsync to work correctly. # - def file_path(path) + def file_path(path, options={}) if path.is_a? Symbol path = [path, @node.name] + elsif path.is_a? String + # ensure it prefixed with files/ + unless path =~ /^files\// + path = "files/" + path + end end actual_path = Path.find_file(path) if actual_path.nil? - Util::log 2, :skipping, "file_path(\"#{path}\") because there is no such file." + if options[:missing] + raise FileMissing.new(Path.named_path(path), options) + else + Util::log 2, :skipping, "file_path(\"#{path}\") because there is no such file." + end nil else if actual_path =~ /^#{Regexp.escape(Path.provider_base)}/ @@ -70,6 +79,7 @@ module LeapCli actual_path += '/' # ensure directories end with /, important for building rsync command end relative_path = Path.relative_path(actual_path) + relative_path.sub!(/^files\//, '') # remove "files/" prefix @node.file_paths << relative_path File.join(Leap::Platform.files_dir, relative_path) end diff --git a/provider_base/lib/macros/keys.rb b/provider_base/lib/macros/keys.rb new file mode 100644 index 00000000..0d46acb5 --- /dev/null +++ b/provider_base/lib/macros/keys.rb @@ -0,0 +1,78 @@ +# encoding: utf-8 + +# +# Macro for dealing with cryptographic keys +# + +module LeapCli + module Macro + + # + # return the path to the tor public key + # generating key if it is missing + # + def tor_public_key_path(path_name, key_type) + path = file_path(path_name) + if path.nil? + generate_tor_key(key_type) + end + return path + end + + # + # return the path to the tor private key + # generating key if it is missing + # + def tor_private_key_path(path_name, key_type) + path = file_path(path_name) + if path.nil? + generate_tor_key(key_type) + end + return path + end + + # + # on the command line an onion address can be created + # from an rsa public key using this: + # + # base64 -d < ./pubkey | sha1sum | awk '{print $1}' | + # perl -e '$l=<>; chomp $l; print pack("H*", $l)' | + # python -c 'import base64, sys; t=sys.stdin.read(); print base64.b32encode(t[:10]).lower()' + # + # path_name is the named path of the tor public key. + # + def onion_address(path_name) + require 'base32' + require 'base64' + require 'openssl' + path = Path.find_file([path_name, self.name]) + if path && File.exists?(path) + public_key_str = File.readlines(path).grep(/^[^-]/).join + public_key = Base64.decode64(public_key_str) + sha1sum_string = Digest::SHA1.new.hexdigest(public_key) + sha1sum_binary = [sha1sum_string].pack('H*') + Base32.encode(sha1sum_binary.slice(0,10)).downcase + else + LeapCli.log :warning, 'Tor public key file "%s" does not exist' % tor_public_key_path + end + end + + private + + def generate_tor_key(key_type) + if key_type == 'RSA' + require 'certificate_authority' + keypair = CertificateAuthority::MemoryKeyMaterial.new + bit_size = 1024 + LeapCli.log :generating, "%s bit RSA Tor key" % bit_size do + keypair.generate_key(bit_size) + LeapCli::Util.write_file! [:node_tor_priv_key, self.name], keypair.private_key.to_pem + LeapCli::Util.write_file! [:node_tor_pub_key, self.name], keypair.public_key.to_pem + end + else + LeapCli.bail! 'tor.key.type of %s is not yet supported' % key_type + end + end + + end +end diff --git a/provider_base/services/tor.json b/provider_base/services/tor.json index fc365a19..87fb9682 100644 --- a/provider_base/services/tor.json +++ b/provider_base/services/tor.json @@ -3,6 +3,13 @@ "bandwidth_rate": 6550, "contacts": "= [provider.contacts['tor'] || provider.contacts.default].flatten", "nickname": "= (self.name + secret(:tor_family)).sub('_','')[0..18]", - "family": "= nodes[:services => 'tor'][:environment => '!local'].field('tor.nickname').join(',')" + "family": "= nodes[:services => 'tor'][:environment => '!local'].field('tor.nickname').join(',')", + "hidden_service": null, + "key": { + "type": "RSA", + "public": "= tor_public_key_path(:node_tor_pub_key, tor.key.type) if tor.hidden_service", + "private": "= tor_private_key_path(:node_tor_priv_key, tor.key.type) if tor.hidden_service", + "address": "= onion_address(:node_tor_pub_key) if tor.hidden_service" + } } } -- cgit v1.2.3 From 84957fbd0f1e4aa26303b6488d9ec7df8af08ab7 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Fri, 31 Oct 2014 13:15:13 -0400 Subject: Fix deprecated dynamic lookups of variables in site_couchdb (#6286) Change-Id: I318944a6872a53ff9c533704514da339426d9401 --- puppet/modules/site_couchdb/manifests/bigcouch.pp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/puppet/modules/site_couchdb/manifests/bigcouch.pp b/puppet/modules/site_couchdb/manifests/bigcouch.pp index f0aab734..e3cba4ba 100644 --- a/puppet/modules/site_couchdb/manifests/bigcouch.pp +++ b/puppet/modules/site_couchdb/manifests/bigcouch.pp @@ -1,12 +1,12 @@ class site_couchdb::bigcouch { - $config = $couchdb_config['bigcouch'] + $config = $::site_couchdb::couchdb_config['bigcouch'] $cookie = $config['cookie'] $ednp_port = $config['ednp_port'] class { 'couchdb': - admin_pw => $couchdb_admin_pw, - admin_salt => $couchdb_admin_salt, + admin_pw => $::site_couchdb::couchdb_admin_pw, + admin_salt => $::site_couchdb::couchdb_admin_salt, bigcouch => true, bigcouch_cookie => $cookie, ednp_port => $ednp_port, -- cgit v1.2.3 From 5787c97b6f73dacae7f01adeff203287007c381d Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Sat, 1 Nov 2014 10:36:48 -0400 Subject: stop using bad nist curve for ssh host key (#6294) update port parameter in site_sshd to be an array, otherwise puppet errors about it being a Fixnum with new sshd module Change-Id: I854d042edb98817169eef5e758d04d60d3c71dd5 --- puppet/modules/site_sshd/manifests/init.pp | 2 +- puppet/modules/sshd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/puppet/modules/site_sshd/manifests/init.pp b/puppet/modules/site_sshd/manifests/init.pp index 9a05b6ed..1da2f1d5 100644 --- a/puppet/modules/site_sshd/manifests/init.pp +++ b/puppet/modules/site_sshd/manifests/init.pp @@ -53,7 +53,7 @@ class site_sshd { ## class { '::sshd': manage_nagios => false, - ports => $ssh['port'], + ports => [ $ssh['port'] ], use_pam => 'yes', hardened_ssl => 'yes', print_motd => 'no', diff --git a/puppet/modules/sshd b/puppet/modules/sshd index 5c23b332..4652fbca 160000 --- a/puppet/modules/sshd +++ b/puppet/modules/sshd @@ -1 +1 @@ -Subproject commit 5c23b33200fc6229ada7f4e13672b5da0d4bdd8e +Subproject commit 4652fbcae0aadcded5d390e71882aec1b1b738ba -- cgit v1.2.3 From 38d7c80fae4efc1d365ec9f982cb025dc67a4386 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Sun, 2 Nov 2014 20:53:34 -0500 Subject: add missing TLSv1 sslversion parameter to site_stunnel::serviers Change-Id: I48dc8135943393bd11c7181853985f4a5799011e --- puppet/modules/site_stunnel/manifests/servers.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/puppet/modules/site_stunnel/manifests/servers.pp b/puppet/modules/site_stunnel/manifests/servers.pp index b1da5c59..8d537644 100644 --- a/puppet/modules/site_stunnel/manifests/servers.pp +++ b/puppet/modules/site_stunnel/manifests/servers.pp @@ -35,6 +35,7 @@ define site_stunnel::servers ( pid => "/var/run/stunnel4/${pid}.pid", rndfile => '/var/lib/stunnel4/.rnd', debuglevel => $debuglevel, + sslversion => 'TLSv1', require => [ Class['Site_config::X509::Key'], Class['Site_config::X509::Cert'], -- cgit v1.2.3 From 18f5d6ea49446f214cbb764ea223f427aafd641e Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Sun, 2 Nov 2014 21:47:36 -0500 Subject: change ordering hints to use refresh_stunnel exec instead of service (#6287) In a multi-node couch deployment, it was observed that the Service['stunnel'] would be activated, and then later a stunnel::client was created which would trigger an Exec['refresh_stunnel']. Because of this, and the ordering hints that were in place, the service would get started, and then the couchdb databases, users, designs, etc. were being put into place and then a stunnel client was created, triggering the refresh_stunnel exec, which would cause an interruption in the connectivity and result in failures. This change replaces the Service['stunnel'] hint with the the Exec['refresh_stunnel'] to make sure that the stunnels are fully setup before attempting couch operations. Change-Id: I33ddd24884b3c23a1df5555ca53ca65cd703da50 --- puppet/modules/couchdb | 2 +- puppet/modules/site_couchdb/manifests/bigcouch.pp | 2 +- puppet/modules/site_couchdb/manifests/init.pp | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/puppet/modules/couchdb b/puppet/modules/couchdb index f01b3586..4c0d5673 160000 --- a/puppet/modules/couchdb +++ b/puppet/modules/couchdb @@ -1 +1 @@ -Subproject commit f01b3586215bdc10f0067fa0f6d940be8e88bcea +Subproject commit 4c0d5673df02fe42e1bbadfee7d4ea1ca1f88e98 diff --git a/puppet/modules/site_couchdb/manifests/bigcouch.pp b/puppet/modules/site_couchdb/manifests/bigcouch.pp index e3cba4ba..d71c00c5 100644 --- a/puppet/modules/site_couchdb/manifests/bigcouch.pp +++ b/puppet/modules/site_couchdb/manifests/bigcouch.pp @@ -19,7 +19,7 @@ class site_couchdb::bigcouch { Class['site_config::default'] -> Class['couchdb::bigcouch::package::cloudant'] -> Service['shorewall'] - -> Service['stunnel'] + -> Exec['refresh_stunnel'] -> Class['site_couchdb::setup'] -> Class['site_couchdb::bigcouch::add_nodes'] -> Class['site_couchdb::bigcouch::settle_cluster'] diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 5a4fb936..a11f6309 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -42,13 +42,13 @@ class site_couchdb { $couchdb_backup = $couchdb_config['backup'] $couchdb_mode = $couchdb_config['mode'] - if $couchdb_mode == "multimaster" { include site_couchdb::bigcouch } - if $couchdb_mode == "master" { include site_couchdb::master } - if $couchdb_mode == "mirror" { include site_couchdb::mirror } + if $couchdb_mode == 'multimaster' { include site_couchdb::bigcouch } + if $couchdb_mode == 'master' { include site_couchdb::master } + if $couchdb_mode == 'mirror' { include site_couchdb::mirror } Class['site_config::default'] -> Service['shorewall'] - -> Service['stunnel'] + -> Exec['refresh_stunnel'] -> Class['couchdb'] -> Class['site_couchdb::setup'] -- cgit v1.2.3 From c150fa3eb79d822850205c0178de9bb5f422ae01 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 4 Nov 2014 11:34:42 -0500 Subject: add local 50unattended-upgrades to fix unattended-upgrades not upgrading leap packages (#4425) Change-Id: I78c00c4410ff9f712206f95854d8803e43acb286 --- .../modules/site_apt/files/Debian/50unattended-upgrades | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 puppet/modules/site_apt/files/Debian/50unattended-upgrades diff --git a/puppet/modules/site_apt/files/Debian/50unattended-upgrades b/puppet/modules/site_apt/files/Debian/50unattended-upgrades new file mode 100644 index 00000000..f2f574fc --- /dev/null +++ b/puppet/modules/site_apt/files/Debian/50unattended-upgrades @@ -0,0 +1,16 @@ +// this file is managed by puppet ! + +Unattended-Upgrade::Allowed-Origins { + "${distro_id}:stable"; + "${distro_id}:${distro_codename}-security"; + "${distro_id}:${distro_codename}-updates"; + "${distro_id} Backports:${distro_codename}-backports"; + "leap.se:stable"; +}; + +APT::Periodic::Update-Package-Lists "1"; +APT::Periodic::Download-Upgradeable-Packages "1"; +APT::Periodic::Unattended-Upgrade "1"; + +Unattended-Upgrade::Mail "root"; +Unattended-Upgrade::MailOnlyOnError "true"; -- cgit v1.2.3 From 90b672ed58982b232b1c96febcd9736ae5fc4faf Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 4 Nov 2014 12:25:54 -0800 Subject: tor - to activate hidden service, now set tor.hidden_service.active = true --- provider_base/lib/macros/keys.rb | 8 ++++++-- provider_base/services/tor.json | 12 ++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/provider_base/lib/macros/keys.rb b/provider_base/lib/macros/keys.rb index 0d46acb5..ea4c3df2 100644 --- a/provider_base/lib/macros/keys.rb +++ b/provider_base/lib/macros/keys.rb @@ -15,8 +15,10 @@ module LeapCli path = file_path(path_name) if path.nil? generate_tor_key(key_type) + file_path(path_name) + else + path end - return path end # @@ -27,8 +29,10 @@ module LeapCli path = file_path(path_name) if path.nil? generate_tor_key(key_type) + file_path(path_name) + else + path end - return path end # diff --git a/provider_base/services/tor.json b/provider_base/services/tor.json index 87fb9682..55d3d2ee 100644 --- a/provider_base/services/tor.json +++ b/provider_base/services/tor.json @@ -4,12 +4,12 @@ "contacts": "= [provider.contacts['tor'] || provider.contacts.default].flatten", "nickname": "= (self.name + secret(:tor_family)).sub('_','')[0..18]", "family": "= nodes[:services => 'tor'][:environment => '!local'].field('tor.nickname').join(',')", - "hidden_service": null, - "key": { - "type": "RSA", - "public": "= tor_public_key_path(:node_tor_pub_key, tor.key.type) if tor.hidden_service", - "private": "= tor_private_key_path(:node_tor_priv_key, tor.key.type) if tor.hidden_service", - "address": "= onion_address(:node_tor_pub_key) if tor.hidden_service" + "hidden_service": { + "active": null, + "key_type": "RSA", + "public_key": "= tor_public_key_path(:node_tor_pub_key, tor.hidden_service.key_type) if tor.hidden_service.active", + "private_key": "= tor_private_key_path(:node_tor_priv_key, tor.hidden_service.key_type) if tor.hidden_service.active", + "address": "= onion_address(:node_tor_pub_key) if tor.hidden_service.active" } } } -- cgit v1.2.3 From 3c06efc2529188c175209d09f8db1932dc4295b8 Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 4 Nov 2014 12:25:54 -0800 Subject: tor - to activate hidden service, now set tor.hidden_service.active = true --- provider_base/lib/macros/keys.rb | 8 ++++++-- provider_base/services/tor.json | 12 ++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/provider_base/lib/macros/keys.rb b/provider_base/lib/macros/keys.rb index 0d46acb5..ea4c3df2 100644 --- a/provider_base/lib/macros/keys.rb +++ b/provider_base/lib/macros/keys.rb @@ -15,8 +15,10 @@ module LeapCli path = file_path(path_name) if path.nil? generate_tor_key(key_type) + file_path(path_name) + else + path end - return path end # @@ -27,8 +29,10 @@ module LeapCli path = file_path(path_name) if path.nil? generate_tor_key(key_type) + file_path(path_name) + else + path end - return path end # diff --git a/provider_base/services/tor.json b/provider_base/services/tor.json index 87fb9682..55d3d2ee 100644 --- a/provider_base/services/tor.json +++ b/provider_base/services/tor.json @@ -4,12 +4,12 @@ "contacts": "= [provider.contacts['tor'] || provider.contacts.default].flatten", "nickname": "= (self.name + secret(:tor_family)).sub('_','')[0..18]", "family": "= nodes[:services => 'tor'][:environment => '!local'].field('tor.nickname').join(',')", - "hidden_service": null, - "key": { - "type": "RSA", - "public": "= tor_public_key_path(:node_tor_pub_key, tor.key.type) if tor.hidden_service", - "private": "= tor_private_key_path(:node_tor_priv_key, tor.key.type) if tor.hidden_service", - "address": "= onion_address(:node_tor_pub_key) if tor.hidden_service" + "hidden_service": { + "active": null, + "key_type": "RSA", + "public_key": "= tor_public_key_path(:node_tor_pub_key, tor.hidden_service.key_type) if tor.hidden_service.active", + "private_key": "= tor_private_key_path(:node_tor_priv_key, tor.hidden_service.key_type) if tor.hidden_service.active", + "address": "= onion_address(:node_tor_pub_key) if tor.hidden_service.active" } } } -- cgit v1.2.3 From 16c985a1b8e692c0e0f76a30b7ec052c9dc269bd Mon Sep 17 00:00:00 2001 From: guido Date: Tue, 28 Oct 2014 21:03:52 -0300 Subject: Adds support for Tor hidden service on webapp (Feature #6273) Change-Id: I56250e05e3a933deacd0b6e02192e712d3fd9fd5 --- .../templates/vhosts.d/hidden_service.conf.erb | 33 +++++++++++++++++ .../site_webapp/manifests/hidden_service.pp | 43 ++++++++++++++++++++++ puppet/modules/site_webapp/manifests/init.pp | 6 +++ 3 files changed, 82 insertions(+) create mode 100644 puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb create mode 100644 puppet/modules/site_webapp/manifests/hidden_service.pp diff --git a/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb new file mode 100644 index 00000000..0c6f3b8e --- /dev/null +++ b/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb @@ -0,0 +1,33 @@ + + ServerName <%= tor_domain %> + + + Header always unset X-Powered-By + Header always unset X-Runtime + + +<% if (defined? @services) and (@services.include? 'webapp') -%> + DocumentRoot /srv/leap/webapp/public + + RewriteEngine On + # Check for maintenance file and redirect all requests + RewriteCond %{DOCUMENT_ROOT}/system/maintenance.html -f + RewriteCond %{SCRIPT_FILENAME} !maintenance.html + RewriteCond %{REQUEST_URI} !/images/maintenance.jpg + RewriteRule ^.*$ %{DOCUMENT_ROOT}/system/maintenance.html [L] + + # http://www.modrails.com/documentation/Users%20guide%20Apache.html#_passengerallowencodedslashes_lt_on_off_gt + AllowEncodedSlashes on + PassengerAllowEncodedSlashes on + PassengerFriendlyErrorPages off + SetEnv TMPDIR /var/tmp + + # Allow rails assets to be cached for a very long time (since the URLs change whenever the content changes) + + Header unset ETag + FileETag None + ExpiresActive On + ExpiresDefault "access plus 1 year" + +<% end -%> + diff --git a/puppet/modules/site_webapp/manifests/hidden_service.pp b/puppet/modules/site_webapp/manifests/hidden_service.pp new file mode 100644 index 00000000..ac0e8a37 --- /dev/null +++ b/puppet/modules/site_webapp/manifests/hidden_service.pp @@ -0,0 +1,43 @@ +class site_webapp::hidden_service { + $tor = hiera('tor') + $hidden_service = $tor['hidden_service'] + $tor_domain = "${hidden_service['address']}.onion" + + include site_apache::common + include site_apache::module::headers + include site_apache::module::alias + include site_apache::module::expires + include site_apache::module::removeip + + include tor::daemon + tor::daemon::hidden_service { 'webapp': ports => '80 127.0.0.1:80' } + + file { + '/var/lib/tor/webapp/': + ensure => directory, + owner => 'debian-tor', + group => 'debian-tor', + mode => '2700'; + + '/var/lib/tor/webapp/private_key': + ensure => present, + source => '/srv/leap/files/nodes/web/tor.key', + owner => 'debian-tor', + group => 'debian-tor', + mode => '0600'; + + '/var/lib/tor/webapp/hostname': + ensure => present, + content => $tor_domain, + owner => 'debian-tor', + group => 'debian-tor', + mode => '0600'; + } + + apache::vhost::file { + 'hidden_service': + content => template('site_apache/vhosts.d/hidden_service.conf.erb') + } + + include site_shorewall::tor +} \ No newline at end of file diff --git a/puppet/modules/site_webapp/manifests/init.pp b/puppet/modules/site_webapp/manifests/init.pp index 17b010f3..12c69a39 100644 --- a/puppet/modules/site_webapp/manifests/init.pp +++ b/puppet/modules/site_webapp/manifests/init.pp @@ -10,6 +10,8 @@ class site_webapp { $webapp = hiera('webapp') $api_version = $webapp['api_version'] $secret_token = $webapp['secret_token'] + $tor = hiera('tor') + $hidden_service = $tor['hidden_service'] Class['site_config::default'] -> Class['site_webapp'] @@ -157,6 +159,10 @@ class site_webapp { notify => Service['apache']; } + if $hidden_service['active'] { + include site_webapp::hidden_service + } + include site_shorewall::webapp include site_check_mk::agent::webapp } -- cgit v1.2.3 From d0658888a0af6b06b03fd172306ea3f346c50081 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 4 Nov 2014 20:45:39 -0500 Subject: revert 5787c97b6f73dacae7f01adeff203287007c381d: stop using bad nist curve for ssh host key (#6294) We need to transition smoother (see #6319) Change-Id: I8bee032aef9502a7d4b701b99719fbfb3b7169da --- puppet/modules/sshd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/sshd b/puppet/modules/sshd index 4652fbca..750a4977 160000 --- a/puppet/modules/sshd +++ b/puppet/modules/sshd @@ -1 +1 @@ -Subproject commit 4652fbcae0aadcded5d390e71882aec1b1b738ba +Subproject commit 750a497758d94c2f5a6cad23cecc3dbde2d2f92f -- cgit v1.2.3 From 3ec9b173c092f1b582285c3e3573259d289c400e Mon Sep 17 00:00:00 2001 From: guido Date: Thu, 6 Nov 2014 12:45:32 -0300 Subject: Better check for tor hidden service on a webapp node. Change-Id: I92f69b6fa30aae953243ae19096e2998810c9ac6 --- puppet/modules/site_webapp/manifests/init.pp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/puppet/modules/site_webapp/manifests/init.pp b/puppet/modules/site_webapp/manifests/init.pp index 12c69a39..752993c1 100644 --- a/puppet/modules/site_webapp/manifests/init.pp +++ b/puppet/modules/site_webapp/manifests/init.pp @@ -10,8 +10,7 @@ class site_webapp { $webapp = hiera('webapp') $api_version = $webapp['api_version'] $secret_token = $webapp['secret_token'] - $tor = hiera('tor') - $hidden_service = $tor['hidden_service'] + $tor = hiera('tor', false) Class['site_config::default'] -> Class['site_webapp'] @@ -159,8 +158,11 @@ class site_webapp { notify => Service['apache']; } - if $hidden_service['active'] { - include site_webapp::hidden_service + if $tor { + $hidden_service = $tor['hidden_service'] + if $hidden_service['active'] { + include site_webapp::hidden_service + } } include site_shorewall::webapp -- cgit v1.2.3 From 180f32512a4c47444ea9e4f36d7376a894a83a4b Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Sat, 8 Nov 2014 00:44:30 -0500 Subject: Don't configure the tor DirPort options if the node is not an exit (#6335) Change-Id: I4c7fb20b6da6f6a5bb2dd5af70511a28d4581174 --- puppet/modules/site_tor/manifests/init.pp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/puppet/modules/site_tor/manifests/init.pp b/puppet/modules/site_tor/manifests/init.pp index e62cb12d..58f9e971 100644 --- a/puppet/modules/site_tor/manifests/init.pp +++ b/puppet/modules/site_tor/manifests/init.pp @@ -24,10 +24,6 @@ class site_tor { tor::daemon::directory { $::hostname: port => 80 } } else { - tor::daemon::directory { $::hostname: - port => 80, - port_front_page => ''; - } include site_tor::disable_exit } -- cgit v1.2.3 From fe23f66f0cff5af71c10aeefdbb0b1131d871219 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Sat, 8 Nov 2014 00:45:14 -0500 Subject: Only enable the tor DirPort options on an exit if the node isn't also a webapp node (#6336) Change-Id: Ib70bbd8fe7b94b7a1bfb09390d5dd1c535f2da16 --- puppet/modules/site_tor/manifests/init.pp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/puppet/modules/site_tor/manifests/init.pp b/puppet/modules/site_tor/manifests/init.pp index 58f9e971..8f68a4e4 100644 --- a/puppet/modules/site_tor/manifests/init.pp +++ b/puppet/modules/site_tor/manifests/init.pp @@ -21,7 +21,10 @@ class site_tor { } if ( $tor_type == 'exit'){ - tor::daemon::directory { $::hostname: port => 80 } + # Only enable the daemon directory if the node isn't also a webapp node + if ! member($::services, 'webapp') { + tor::daemon::directory { $::hostname: port => 80 } + } } else { include site_tor::disable_exit -- cgit v1.2.3 From 51d581583ca354232f6ccbfb771c1cad00ec2db3 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Sat, 8 Nov 2014 00:46:00 -0500 Subject: minor linting, arrow lining up Change-Id: Ibd08529b7d1c4fc22bcd0ca36e518afa5b8f6d24 --- puppet/modules/site_tor/manifests/init.pp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/puppet/modules/site_tor/manifests/init.pp b/puppet/modules/site_tor/manifests/init.pp index 8f68a4e4..9944bb2b 100644 --- a/puppet/modules/site_tor/manifests/init.pp +++ b/puppet/modules/site_tor/manifests/init.pp @@ -13,11 +13,11 @@ class site_tor { class { 'tor::daemon': } tor::daemon::relay { $nickname: - port => 9001, - address => $address, - contact_info => obfuscate_email($contact_emails), - bandwidth_rate => $bandwidth_rate, - my_family => $family + port => 9001, + address => $address, + contact_info => obfuscate_email($contact_emails), + bandwidth_rate => $bandwidth_rate, + my_family => $family } if ( $tor_type == 'exit'){ -- cgit v1.2.3 From 9299574b45de02d417e7237ba49b0222002bbc21 Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 15 Oct 2014 15:28:54 -0700 Subject: tests - added test that creates user, authenticates, deletes user --- bin/run_tests | 224 ++++------------------------------------ tests/README.md | 27 +++-- tests/helpers/couchdb_helper.rb | 103 ++++++++++++++++++ tests/helpers/files_helper.rb | 54 ++++++++++ tests/helpers/http_helper.rb | 145 ++++++++++++++++++++++++++ tests/helpers/network_helper.rb | 79 ++++++++++++++ tests/helpers/os_helper.rb | 34 ++++++ tests/helpers/srp_helper.rb | 171 ++++++++++++++++++++++++++++++ tests/order.rb | 4 + tests/white-box/couchdb.rb | 58 ++++++----- tests/white-box/mx.rb | 50 +++++++++ tests/white-box/webapp.rb | 141 ++++++++++++++++++------- 12 files changed, 811 insertions(+), 279 deletions(-) create mode 100644 tests/helpers/couchdb_helper.rb create mode 100644 tests/helpers/files_helper.rb create mode 100644 tests/helpers/http_helper.rb create mode 100644 tests/helpers/network_helper.rb create mode 100644 tests/helpers/os_helper.rb create mode 100644 tests/helpers/srp_helper.rb create mode 100644 tests/white-box/mx.rb diff --git a/bin/run_tests b/bin/run_tests index e026b5f7..b3e60fcc 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -14,7 +14,6 @@ require 'minitest/unit' require 'yaml' require 'tsort' -require 'net/http' ## ## EXIT CODES @@ -114,7 +113,10 @@ class LeapTest < MiniTest::Unit::TestCase # the default fail() is part of the kernel and it just throws a runtime exception. for tests, # we want the same behavior as assert(false) # - def fail(msg=nil) + def fail(msg=nil, exception=nil) + if DEBUG && exception && exception.respond_to?(:backtrace) + msg += MiniTest::filter_backtrace(exception.backtrace).join "\n" + end assert(false, msg) end @@ -129,207 +131,6 @@ class LeapTest < MiniTest::Unit::TestCase :alpha end - # - # attempts a http GET on the url, yields |body, response, error| - # - def get(url, params=nil) - uri = URI(url) - if params - uri.query = URI.encode_www_form(params) - end - http = Net::HTTP.new uri.host, uri.port - if uri.scheme == 'https' - http.verify_mode = OpenSSL::SSL::VERIFY_NONE - http.use_ssl = true - end - http.start do |agent| - request = Net::HTTP::Get.new uri.request_uri - if uri.user - request.basic_auth uri.user, uri.password - end - response = agent.request(request) - if response.is_a?(Net::HTTPSuccess) - yield response.body, response, nil - else - yield nil, response, nil - end - end - rescue => exc - yield nil, nil, exc - end - - def assert_get(url, params=nil, options=nil) - options ||= {} - get(url, params) do |body, response, error| - if body - yield body if block_given? - elsif response - fail ["Expected a 200 status code from #{url}, but got #{response.code} instead.", options[:error_msg]].compact.join("\n") - else - fail ["Expected a response from #{url}, but got \"#{error}\" instead.", options[:error_msg]].compact.join("\n") - end - end - end - - # - # only a warning for now, should be a failure in the future - # - def assert_auth_fail(url, params) - uri = URI(url) - get(url, params) do |body, response, error| - unless response.code.to_s == "401" - warn "Expected a '401 Unauthorized' response, but got #{response.code} instead (GET #{uri.request_uri} with username '#{uri.user}')." - return false - end - end - true - end - - # - # test if a socket can be connected to - # - - # - # tcp connection helper with timeout - # - def try_tcp_connect(host, port, timeout = 5) - addr = Socket.getaddrinfo(host, nil) - sockaddr = Socket.pack_sockaddr_in(port, addr[0][3]) - - Socket.new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0).tap do |socket| - socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) - begin - socket.connect_nonblock(sockaddr) - rescue IO::WaitReadable - if IO.select([socket], nil, nil, timeout) == nil - raise "Connection timeout" - else - socket.connect_nonblock(sockaddr) - end - rescue IO::WaitWritable - if IO.select(nil, [socket], nil, timeout) == nil - raise "Connection timeout" - else - socket.connect_nonblock(sockaddr) - end - end - return socket - end - end - - def try_tcp_write(socket, timeout = 5) - begin - socket.write_nonblock("\0") - rescue IO::WaitReadable - if IO.select([socket], nil, nil, timeout) == nil - raise "Write timeout" - else - retry - end - rescue IO::WaitWritable - if IO.select(nil, [socket], nil, timeout) == nil - raise "Write timeout" - else - retry - end - end - end - - def try_tcp_read(socket, timeout = 5) - begin - socket.read_nonblock(1) - rescue IO::WaitReadable - if IO.select([socket], nil, nil, timeout) == nil - raise "Read timeout" - else - retry - end - rescue IO::WaitWritable - if IO.select(nil, [socket], nil, timeout) == nil - raise "Read timeout" - else - retry - end - end - end - - def assert_tcp_socket(host, port, msg=nil) - begin - socket = try_tcp_connect(host, port, 1) - #try_tcp_write(socket,1) - #try_tcp_read(socket,1) - rescue StandardError => exc - fail ["Failed to open socket #{host}:#{port}", exc].join("\n") - ensure - socket.close if socket - end - end - - # - # Matches the regexp in the file, and returns the first matched string (or fails if no match). - # - def file_match(filename, regexp) - if match = File.read(filename).match(regexp) - match.captures.first - else - fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}." - end - end - - # - # Matches the regexp in the file, and returns array of matched strings (or fails if no match). - # - def file_matches(filename, regexp) - if match = File.read(filename).match(regexp) - match.captures - else - fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}." - end - end - - # - # checks to make sure the given property path exists in $node (e.g. hiera.yaml) - # and returns the value - # - def assert_property(property) - latest = $node - property.split('.').each do |segment| - latest = latest[segment] - fail "Required node property `#{property}` is missing." if latest.nil? - end - return latest - end - - # - # works like pgrep command line - # return an array of hashes like so [{:pid => "1234", :process => "ls"}] - # - def pgrep(match) - output = `pgrep --full --list-name '#{match}'` - output.each_line.map{|line| - pid = line.split(' ')[0] - process = line.gsub(/(#{pid} |\n)/, '') - if process =~ /pgrep --full --list-name/ - nil - else - {:pid => pid, :process => process} - end - }.compact - end -end - -def assert_running(process) - assert pgrep(process).any?, "No running process for #{process}" -end - -# -# runs the specified command, failing on a non-zero exit status. -# -def assert_run(command) - output = `#{command}` - if $?.exitstatus != 0 - fail "Error running `#{command}`:\n#{output}" - end end # @@ -441,7 +242,7 @@ class LeapRunner < MiniTest::Unit def report_line(prefix, klass, meth, e=nil, message=nil) msg_txt = nil if message - message = message.sub(/http:\/\/([a-z_]+):([a-zA-Z0-9_]+)@/, "http://\\1:password@") + message = message.sub(/http:\/\/([a-z_]+):([a-zA-Z0-9_]+)@/, "http://\\1:REDACTED@") if $output_format == :human indent = "\n " msg_txt = indent + message.split("\n").join(indent) @@ -556,7 +357,8 @@ def print_help " --test TEST Run only the test with name TEST.", " --list-tests Prints the names of all available tests and exit.", " --retry COUNT If the tests don't pass, retry COUNT additional times (default is zero)", - " --wait SECONDS Wait for SECONDS between retries (default is 5)"].join("\n") + " --wait SECONDS Wait for SECONDS between retries (default is 5)", + " --debug Print out full stack trace on errors"].join("\n") exit(0) end @@ -615,6 +417,9 @@ def main # load all test classes this_file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__ + Dir[File.expand_path('../../tests/helpers/*.rb', this_file)].each do |helper| + require helper + end Dir[File.expand_path('../../tests/white-box/*.rb', this_file)].each do |test_file| begin require test_file @@ -636,10 +441,19 @@ def main when '--list-tests' then list_tests when '--retry' then ARGV.shift; $retry = ARGV.shift.to_i when '--wait' then ARGV.shift; $wait = ARGV.shift.to_i + when '--debug' then ARGV.shift + when '-d' then ARGV.shift else break end end run_tests end +if ARGV.include?('--debug') || ARGV.include?('-d') + DEBUG=true + require 'debugger' +else + DEBUG=false +end + main() diff --git a/tests/README.md b/tests/README.md index debbf700..814c25b1 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,12 +1,25 @@ -This directory contains to kinds of tests: +Tests +--------------------------------- -White Box Tests -================================ +tests/white-box/ -These tests are run on the server as superuser. They are for troubleshooting any problems with the internal setup of the server. + These tests are run on the server as superuser. They are for + troubleshooting any problems with the internal setup of the server. -Black Box Tests -================================ +tests/black-box/ + + These test are run the user's local machine. They are for troubleshooting + any external problems with the service exposed by the server. + +Additional Files +--------------------------------- + +tests/helpers/ + + Utility functions made available to all tests. + +tests/order.rb + + Configuration file to specify which nodes should be tested in which order. -These test are run the user's local machine. They are for troubleshooting any external problems with the service exposed by the server. diff --git a/tests/helpers/couchdb_helper.rb b/tests/helpers/couchdb_helper.rb new file mode 100644 index 00000000..d4d3c0e0 --- /dev/null +++ b/tests/helpers/couchdb_helper.rb @@ -0,0 +1,103 @@ +class LeapTest + + # + # generates a couchdb url for when couchdb is running + # remotely and is available via stunnel. + # + # example properties: + # + # stunnel: + # clients: + # couch_client: + # couch1_5984: + # accept_port: 4000 + # connect: couch1.bitmask.i + # connect_port: 15984 + # + def couchdb_urls_via_stunnel(path="", options=nil) + if options && options[:username] && options[:password] + userpart = "%{username}:%{password}@" % options + else + userpart = "" + end + assert_property('stunnel.clients.couch_client').values.collect do |stunnel_conf| + assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.' + URLString.new("http://#{userpart}localhost:#{port}#{path}").tap {|url| + remote_ip_address = TCPSocket.gethostbyname(stunnel_conf['connect']).last + url.memo = "(via stunnel to %s:%s, aka %s)" % [stunnel_conf['connect'], stunnel_conf['connect_port'], remote_ip_address] + } + end + end + + # + # generates a couchdb url for accessing couchdb via haproxy + # + # example properties: + # + # haproxy: + # couch: + # listen_port: 4096 + # servers: + # panda: + # backup: false + # host: localhost + # port: 4000 + # weight: 100 + # writable: true + # + def couchdb_url_via_haproxy(path="", options=nil) + if options && options[:username] && options[:password] + userpart = "%{username}:%{password}@" % options + else + userpart = "" + end + port = assert_property('haproxy.couch.listen_port') + return URLString.new("http://#{userpart}localhost:#{port}#{path}").tap { |url| + url.memo = '(via haproxy)' + } + end + + # + # generates a couchdb url for when couchdb is running locally. + # + # example properties: + # + # couch: + # port: 5984 + # + def couchdb_url_via_localhost(path="", options=nil) + port = (options && options[:port]) || assert_property('couch.port') + if options && options[:username] + password = property("couch.users.%{username}.password" % options) + userpart = "%s:%s@" % [options[:username], password] + else + userpart = "" + end + return URLString.new("http://#{userpart}localhost:#{port}#{path}").tap { |url| + url.memo = '(via direct localhost connection)' + } + end + + # + # returns a single url for accessing couchdb + # + def couchdb_url(path="", options=nil) + if property('couch.port') + couchdb_url_via_localhost(path, options) + elsif property('stunnel.clients.couch_client') + couchdb_urls_via_stunnel(path, options).first + end + end + + # + # returns an array of urls for accessing couchdb + # + def couchdb_urls(path="", options=nil) + if property('couch.port') + [couchdb_url_via_localhost(path, options)] + elsif property('stunnel.clients.couch_client') + couchdb_urls_via_stunnel(path, options) + end + end + +end \ No newline at end of file diff --git a/tests/helpers/files_helper.rb b/tests/helpers/files_helper.rb new file mode 100644 index 00000000..d6795889 --- /dev/null +++ b/tests/helpers/files_helper.rb @@ -0,0 +1,54 @@ +class LeapTest + + # + # Matches the regexp in the file, and returns the first matched string (or fails if no match). + # + def file_match(filename, regexp) + if match = File.read(filename).match(regexp) + match.captures.first + else + fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}." + end + end + + # + # Matches the regexp in the file, and returns array of matched strings (or fails if no match). + # + def file_matches(filename, regexp) + if match = File.read(filename).match(regexp) + match.captures + else + fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}." + end + end + + # + # checks to make sure the given property path exists in $node (e.g. hiera.yaml) + # and returns the value + # + def assert_property(property) + latest = $node + property.split('.').each do |segment| + latest = latest[segment] + fail "Required node property `#{property}` is missing." if latest.nil? + end + return latest + end + + # + # a handy function to get the value of a long property path + # without needing to test the existance individually of each part + # in the tree. + # + # e.g. property("stunnel.clients.couch_client") + # + def property(property) + latest = $node + property.split('.').each do |segment| + latest = latest[segment] + return nil if latest.nil? + end + return latest + end + +end \ No newline at end of file diff --git a/tests/helpers/http_helper.rb b/tests/helpers/http_helper.rb new file mode 100644 index 00000000..c941ef63 --- /dev/null +++ b/tests/helpers/http_helper.rb @@ -0,0 +1,145 @@ +require 'net/http' + +class LeapTest + + # + # In order to easily provide detailed error messages, it is useful + # to append a memo to a url string that details what this url is for + # (e.g. stunnel, haproxy, etc). + # + # So, the url happens to be a UrlString, the memo field is used + # if there is an error in assert_get. + # + class URLString < String + attr_accessor :memo + end + + # + # aliases for http_send() + # + def get(url, params=nil, options=nil, &block) + http_send("GET", url, params, options, &block) + end + def delete(url, params=nil, options=nil, &block) + http_send("DELETE", url, params, options, &block) + end + def post(url, params=nil, options=nil, &block) + http_send("POST", url, params, options, &block) + end + def put(url, params=nil, options=nil, &block) + http_send("PUT", url, params, options, &block) + end + + # + # send a GET, DELETE, POST, or PUT + # yields |body, response, error| + # + def http_send(method, url, params=nil, options=nil) + options ||= {} + response = nil + + # build uri + uri = URI(url) + if params && (method == 'GET' || method == 'DELETE') + uri.query = URI.encode_www_form(params) + end + + # build http + http = Net::HTTP.new uri.host, uri.port + if uri.scheme == 'https' + http.verify_mode = OpenSSL::SSL::VERIFY_NONE + http.use_ssl = true + end + + # build request + request = build_request(method, uri, params, options) + + # make http request + http.start do |agent| + response = agent.request(request) + yield response.body, response, nil + end + rescue => exc + yield nil, response, exc + end + + # + # Aliases for assert_http_send() + # + def assert_get(url, params=nil, options=nil, &block) + assert_http_send("GET", url, params, options, &block) + end + def assert_delete(url, params=nil, options=nil, &block) + assert_http_send("DELETE", url, params, options, &block) + end + def assert_post(url, params=nil, options=nil, &block) + assert_http_send("POST", url, params, options, &block) + end + def assert_put(url, params=nil, options=nil, &block) + assert_http_send("PUT", url, params, options, &block) + end + + # + # calls http_send, yielding results if successful or failing with + # descriptive infor otherwise. + # + def assert_http_send(method, url, params=nil, options=nil, &block) + options ||= {} + error_msg = options[:error_msg] || (url.respond_to?(:memo) ? url.memo : nil) + http_send(method, url, params, options) do |body, response, error| + if body && response && response.code.to_i >= 200 && response.code.to_i < 300 + if block + yield(body) if block.arity == 1 + yield(response, body) if block.arity == 2 + end + elsif response + fail ["Expected a 200 status code from #{url}, but got #{response.code} instead.", error_msg, body].compact.join("\n") + else + fail ["Expected a response from #{url}, but got \"#{error}\" instead.", error_msg, body].compact.join("\n"), error + end + end + end + + # + # only a warning for now, should be a failure in the future + # + def assert_auth_fail(url, params) + uri = URI(url) + get(url, params) do |body, response, error| + unless response.code.to_s == "401" + warn "Expected a '401 Unauthorized' response, but got #{response.code} instead (GET #{uri.request_uri} with username '#{uri.user}')." + return false + end + end + true + end + + private + + def build_request(method, uri, params, options) + request = case method + when "GET" then Net::HTTP::Get.new(uri.request_uri) + when "DELETE" then Net::HTTP::Delete.new(uri.request_uri) + when "POST" then Net::HTTP::Post.new(uri.request_uri) + when "PUT" then Net::HTTP::Put.new(uri.request_uri) + end + if uri.user + request.basic_auth uri.user, uri.password + end + if params && (method == 'POST' || method == 'PUT') + if options[:format] == :json || options[:format] == 'json' + request["Content-Type"] = "application/json" + request.body = params.to_json + else + request.set_form_data(params) if params + end + end + if options[:headers] + options[:headers].each do |key, value| + request[key] = value + end + end + request + end + +end \ No newline at end of file diff --git a/tests/helpers/network_helper.rb b/tests/helpers/network_helper.rb new file mode 100644 index 00000000..ff92d382 --- /dev/null +++ b/tests/helpers/network_helper.rb @@ -0,0 +1,79 @@ +class LeapTest + + # + # tcp connection helper with timeout + # + def try_tcp_connect(host, port, timeout = 5) + addr = Socket.getaddrinfo(host, nil) + sockaddr = Socket.pack_sockaddr_in(port, addr[0][3]) + + Socket.new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0).tap do |socket| + socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) + begin + socket.connect_nonblock(sockaddr) + rescue IO::WaitReadable + if IO.select([socket], nil, nil, timeout) == nil + raise "Connection timeout" + else + socket.connect_nonblock(sockaddr) + end + rescue IO::WaitWritable + if IO.select(nil, [socket], nil, timeout) == nil + raise "Connection timeout" + else + socket.connect_nonblock(sockaddr) + end + end + return socket + end + end + + def try_tcp_write(socket, timeout = 5) + begin + socket.write_nonblock("\0") + rescue IO::WaitReadable + if IO.select([socket], nil, nil, timeout) == nil + raise "Write timeout" + else + retry + end + rescue IO::WaitWritable + if IO.select(nil, [socket], nil, timeout) == nil + raise "Write timeout" + else + retry + end + end + end + + def try_tcp_read(socket, timeout = 5) + begin + socket.read_nonblock(1) + rescue IO::WaitReadable + if IO.select([socket], nil, nil, timeout) == nil + raise "Read timeout" + else + retry + end + rescue IO::WaitWritable + if IO.select(nil, [socket], nil, timeout) == nil + raise "Read timeout" + else + retry + end + end + end + + def assert_tcp_socket(host, port, msg=nil) + begin + socket = try_tcp_connect(host, port, 1) + #try_tcp_write(socket,1) + #try_tcp_read(socket,1) + rescue StandardError => exc + fail ["Failed to open socket #{host}:#{port}", exc].join("\n") + ensure + socket.close if socket + end + end + +end \ No newline at end of file diff --git a/tests/helpers/os_helper.rb b/tests/helpers/os_helper.rb new file mode 100644 index 00000000..529e899f --- /dev/null +++ b/tests/helpers/os_helper.rb @@ -0,0 +1,34 @@ +class LeapTest + + # + # works like pgrep command line + # return an array of hashes like so [{:pid => "1234", :process => "ls"}] + # + def pgrep(match) + output = `pgrep --full --list-name '#{match}'` + output.each_line.map{|line| + pid = line.split(' ')[0] + process = line.gsub(/(#{pid} |\n)/, '') + if process =~ /pgrep --full --list-name/ + nil + else + {:pid => pid, :process => process} + end + }.compact + end + + def assert_running(process) + assert pgrep(process).any?, "No running process for #{process}" + end + + # + # runs the specified command, failing on a non-zero exit status. + # + def assert_run(command) + output = `#{command}` + if $?.exitstatus != 0 + fail "Error running `#{command}`:\n#{output}" + end + end + +end \ No newline at end of file diff --git a/tests/helpers/srp_helper.rb b/tests/helpers/srp_helper.rb new file mode 100644 index 00000000..9f4d7f5b --- /dev/null +++ b/tests/helpers/srp_helper.rb @@ -0,0 +1,171 @@ +# +# Here are some very stripped down helper methods for SRP, useful only for +# testing the client side. +# + +require 'digest' +require 'openssl' +require 'securerandom' + +module SRP + + ## + ## UTIL + ## + + module Util + PRIME_N = <<-EOS.split.join.hex +115b8b692e0e045692cf280b436735c77a5a9e8a9e7ed56c965f87db5b2a2ece3 + EOS + BIG_PRIME_N = <<-EOS.split.join.hex # 1024 bits modulus (N) +eeaf0ab9adb38dd69c33f80afa8fc5e86072618775ff3c0b9ea2314c9c25657 +6d674df7496ea81d3383b4813d692c6e0e0d5d8e250b98be48e495c1d6089da +d15dc7d7b46154d6b6ce8ef4ad69b15d4982559b297bcf1885c529f566660e5 +7ec68edbc3c05726cc02fd4cbf4976eaa9afd5138fe8376435b9fc61d2fc0eb +06e3 + EOS + GENERATOR = 2 # g + + def hn_xor_hg + byte_xor_hex(sha256_int(BIG_PRIME_N), sha256_int(GENERATOR)) + end + + # a^n (mod m) + def modpow(a, n, m = BIG_PRIME_N) + r = 1 + while true + r = r * a % m if n[0] == 1 + n >>= 1 + return r if n == 0 + a = a * a % m + end + end + + # Hashes the (long) int args + def sha256_int(*args) + sha256_hex(*args.map{|a| "%02x" % a}) + end + + # Hashes the hex args + def sha256_hex(*args) + h = args.map{|a| a.length.odd? ? "0#{a}" : a }.join('') + sha256_str([h].pack('H*')) + end + + def sha256_str(s) + Digest::SHA2.hexdigest(s) + end + + def bigrand(bytes) + OpenSSL::Random.random_bytes(bytes).unpack("H*")[0] + end + + def multiplier + @muliplier ||= calculate_multiplier + end + + protected + + def calculate_multiplier + sha256_int(BIG_PRIME_N, GENERATOR).hex + end + + def byte_xor_hex(a, b) + a = [a].pack('H*') + b = [b].pack('H*') + a.bytes.each_with_index.map do |a_byte, i| + (a_byte ^ (b[i].ord || 0)).chr + end.join + end + end + + ## + ## SESSION + ## + + class Session + include SRP::Util + attr_accessor :user + attr_accessor :bb + + def initialize(user, aa=nil) + @user = user + @a = bigrand(32).hex + end + + def m + @m ||= sha256_hex(n_xor_g_long, login_hash, @user.salt.to_s(16), aa, bb, k) + end + + def aa + @aa ||= modpow(GENERATOR, @a).to_s(16) # A = g^a (mod N) + end + + protected + + # client: K = H( (B - kg^x) ^ (a + ux) ) + def client_secret + base = bb.hex + base -= modpow(GENERATOR, @user.private_key) * multiplier + base = base % BIG_PRIME_N + modpow(base, @user.private_key * u.hex + @a) + end + + def k + @k ||= sha256_int(client_secret) + end + + def n_xor_g_long + @n_xor_g_long ||= hn_xor_hg.bytes.map{|b| "%02x" % b.ord}.join + end + + def login_hash + @login_hash ||= sha256_str(@user.username) + end + + def u + @u ||= sha256_hex(aa, bb) + end + end + + ## + ## Dummy USER + ## + + class User + include SRP::Util + + attr_accessor :username + attr_accessor :password + attr_accessor :salt + attr_accessor :verifier + + def initialize + @username = "test_user_" + SecureRandom.urlsafe_base64(10).downcase.gsub(/[_-]/, '') + @password = "password_" + SecureRandom.urlsafe_base64(10) + @salt = bigrand(4).hex + @verifier = modpow(GENERATOR, private_key) + end + + def private_key + @private_key ||= calculate_private_key + end + + def to_params + { + 'user[login]' => @username, + 'user[password_verifier]' => @verifier.to_s(16), + 'user[password_salt]' => @salt.to_s(16) + } + end + + private + + def calculate_private_key + shex = '%x' % [@salt] + inner = sha256_str([@username, @password].join(':')) + sha256_hex(shex, inner).hex + end + end + +end diff --git a/tests/order.rb b/tests/order.rb index ffa6ae4e..4468686f 100644 --- a/tests/order.rb +++ b/tests/order.rb @@ -3,6 +3,10 @@ class LeapCli::Config::Node # returns a list of node names that should be tested before this node. # make sure to not return ourselves (please no dependency loops!). # + # NOTE: this method determines the order that nodes are tested in. To specify + # the order of tests on a particular node, each test can call class method + # LeapTest.depends_on(). + # def test_dependencies dependents = LeapCli::Config::ObjectList.new unless services.include?('couchdb') diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb index a5adb2bf..d438b193 100644 --- a/tests/white-box/couchdb.rb +++ b/tests/white-box/couchdb.rb @@ -52,7 +52,7 @@ class CouchDB < LeapTest # def test_03_Are_configured_nodes_online? return unless multimaster? - url = couchdb_url("/_membership", :user => 'admin') + url = couchdb_url("/_membership", :username => 'admin') assert_get(url) do |body| response = JSON.parse(body) nodes_configured_but_not_available = response['cluster_nodes'] - response['all_nodes'] @@ -71,7 +71,7 @@ class CouchDB < LeapTest def test_04_Do_ACL_users_exist? acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'tapicero', 'webapp', 'replication'] - url = couchdb_backend_url("/_users/_all_docs", :user => 'admin') + url = couchdb_backend_url("/_users/_all_docs", :username => 'admin') assert_get(url) do |body| response = JSON.parse(body) assert_equal acl_users.count, response['total_rows'] @@ -84,7 +84,7 @@ class CouchDB < LeapTest def test_05_Do_required_databases_exist? dbs_that_should_exist = ["customers","identities","keycache","sessions","shared","tickets","tokens","users"] dbs_that_should_exist.each do |db_name| - url = couchdb_url("/"+db_name, :user => 'admin') + url = couchdb_url("/"+db_name, :username => 'admin') assert_get(url) do |body| assert response = JSON.parse(body) assert_equal db_name, response['db_name'] @@ -102,50 +102,54 @@ class CouchDB < LeapTest #def test_06_Is_ACL_enforced? # ok = assert_auth_fail( - # couchdb_url('/users/_all_docs', :user => 'leap_mx'), + # couchdb_url('/users/_all_docs', :username => 'leap_mx'), # {:limit => 1} # ) # ok = assert_auth_fail( - # couchdb_url('/users/_all_docs', :user => 'leap_mx'), + # couchdb_url('/users/_all_docs', :username => 'leap_mx'), # {:limit => 1} # ) && ok # pass if ok #end - def test_07_What? + def test_07_Can_records_be_created? + token = Token.new + url = couchdb_url("/tokens", :username => 'admin') + assert_post(url, token, :format => :json) do |body| + assert response = JSON.parse(body), "POST response should be JSON" + assert response["ok"], "POST response should be OK" + assert_delete(File.join(url, response["id"]), :rev => response["rev"]) do |body| + assert response = JSON.parse(body), "DELETE response should be JSON" + assert response["ok"], "DELETE response should be OK" + end + end pass end private - def couchdb_url(path="", options=nil) - options||={} - @port ||= begin - assert_property 'couch.port' - $node['couch']['port'] - end - url = 'http://' - if options[:user] - assert_property 'couch.users.' + options[:user] - password = $node['couch']['users'][options[:user]]['password'] - url += "%s:%s@" % [options[:user], password] - end - url += "localhost:#{options[:port] || @port}#{path}" - url + def multimaster? + mode == "multimaster" end + def mode + assert_property('couch.mode') + end + + # TODO: admin port is hardcoded for now but should be configurable. def couchdb_backend_url(path="", options={}) - # TODO: admin port is hardcoded for now but should be configurable. options = {port: multimaster? && "5986"}.merge options couchdb_url(path, options) end - def multimaster? - mode == "multimaster" - end - - def mode - assert_property('couch.mode') + require 'securerandom' + require 'digest/sha2' + class Token < Hash + def initialize + self['token'] = SecureRandom.urlsafe_base64(32).gsub(/^_*/, '') + self['_id'] = Digest::SHA512.hexdigest(self['token']) + self['last_seen_at'] = Time.now + end end end diff --git a/tests/white-box/mx.rb b/tests/white-box/mx.rb new file mode 100644 index 00000000..da780de4 --- /dev/null +++ b/tests/white-box/mx.rb @@ -0,0 +1,50 @@ +raise SkipTest unless $node["services"].include?("mx") + +require 'json' + +class Mx < LeapTest + depends_on "Network" + + def setup + end + + def test_01_Can_contact_couchdb? + dbs = ["identities"] + dbs.each do |db_name| + couchdb_urls("/"+db_name, url_options).each do |url| + assert_get(url) do |body| + assert response = JSON.parse(body) + assert_equal db_name, response['db_name'] + end + end + end + pass + end + + def test_02_Can_contact_couchdb_via_haproxy? + if property('haproxy.couch') + url = couchdb_url_via_haproxy("", url_options) + assert_get(url) do |body| + assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message." + end + pass + end + end + + def test_03_Are_MX_daemons_running? + assert_running 'leap_mx' + assert_running '/usr/lib/postfix/master' + assert_running '/usr/sbin/unbound' + pass + end + + private + + def url_options + { + :username => property('couchdb_leap_mx_user.username'), + :password => property('couchdb_leap_mx_user.password') + } + end + +end diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb index 7df57fd7..0fea1c7f 100644 --- a/tests/white-box/webapp.rb +++ b/tests/white-box/webapp.rb @@ -1,58 +1,29 @@ raise SkipTest unless $node["services"].include?("webapp") -require 'socket' +require 'json' class Webapp < LeapTest depends_on "Network" - HAPROXY_CONFIG = '/etc/haproxy/haproxy.cfg' - def setup end - # - # example properties: - # - # stunnel: - # clients: - # couch_client: - # couch1_5984: - # accept_port: 4000 - # connect: couch1.bitmask.i - # connect_port: 15984 - # def test_01_Can_contact_couchdb? - assert_property('stunnel.clients.couch_client') - $node['stunnel']['clients']['couch_client'].values.each do |stunnel_conf| - assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.' - local_stunnel_url = "http://localhost:#{port}" - remote_ip_address = TCPSocket.gethostbyname(stunnel_conf['connect']).last - msg = "(stunnel to %s:%s, aka %s)" % [stunnel_conf['connect'], stunnel_conf['connect_port'], remote_ip_address] - assert_get(local_stunnel_url, nil, error_msg: msg) do |body| - assert_match /"couchdb":"Welcome"/, body, "Request to #{local_stunnel_url} should return couchdb welcome message." - end + url = couchdb_url("", url_options) + assert_get(url) do |body| + assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message." end pass end - # - # example properties: - # - # haproxy: - # servers: - # couch1: - # backup: false - # host: localhost - # port: 4000 - # weight: 10 - # - def test_02_Is_haproxy_working? - port = file_match(HAPROXY_CONFIG, /^ bind localhost:(\d+)$/) - url = "http://localhost:#{port}" - assert_get(url) do |body| - assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message." + def test_02_Can_contact_couchdb_via_haproxy? + if property('haproxy.couch') + url = couchdb_url_via_haproxy("", url_options) + assert_get(url) do |body| + assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message." + end + pass end - pass end def test_03_Are_daemons_running? @@ -70,4 +41,94 @@ class Webapp < LeapTest pass end + def test_05_Can_create_user? + @@user = nil + user = SRP::User.new + url = api_url("/1/users.json") + assert_post(url, user.to_params) do |body| + assert response = JSON.parse(body), 'response should be JSON' + assert response['ok'], 'creating a user should be successful' + end + @@user = user + pass + end + + def test_06_Can_authenticate? + @@user_id = nil + @@session_token = nil + if @@user.nil? + skip "Depends on user creation" + else + url = api_url("/1/sessions.json") + session = SRP::Session.new(@@user) + params = {'login' => @@user.username, 'A' => session.aa} + assert_post(url, params) do |response, body| + cookie = response['Set-Cookie'].split(';').first + assert(response = JSON.parse(body), 'response should be JSON') + assert(bb = response["B"]) + session.bb = bb + url = api_url("/1/sessions/login.json") + params = {'client_auth' => session.m, 'A' => session.aa} + options = {:headers => {'Cookie' => cookie}} + assert_put(url, params, options) do |body| + assert(response = JSON.parse(body), 'response should be JSON') + assert(response['M2'], 'response should include M2') + assert(@@session_token = response['token'], 'response should include token') + assert(@@user_id = response['id'], 'response should include user id') + end + end + pass + end + end + + def test_07_Can_delete_user? + if @@user_id.nil? || @@session_token.nil? + skip "Depends on authentication" + else + url = api_url("/1/users/#{@@user_id}.json") + options = {:headers => { + "Authorization" => "Token token=\"#{@@session_token}\"" + }} + delete(url, {}, options) do |body, response, error| + if response.code.to_i != 200 + skip "It appears the web api is too old to support deleting users" + else + assert(response = JSON.parse(body), 'response should be JSON') + assert(response["success"], 'delete should be a success') + pass + end + end + end + end + + private + + def url_options + { + :username => property('couchdb_webapp_user.username'), + :password => property('couchdb_webapp_user.password') + } + end + + def api_url(path) + "https://%{domain}:%{port}#{path}" % { + :domain => property('api.domain'), + :port => property('api.port') + } + end + + # + # I tried, but couldn't get this working: + # # + # # get an CSRF authenticity token + # # + # url = api_url("/") + # csrf_token = nil + # assert_get(url) do |body| + # lines = body.split("\n").grep(/csrf-token/) + # assert lines.any?, 'failed to find csrf-token' + # csrf_token = lines.first.split('"')[1] + # assert csrf_token, 'failed to find csrf-token' + # end + end -- cgit v1.2.3 From b9d2030beb890e8dccbbe42bfcc430a2c2702a92 Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 10 Nov 2014 20:43:24 -0800 Subject: openvpn - support customizing --fragment, and set default to 1400 --- .../files/service-definitions/v1/eip-service.json.erb | 4 ++++ provider_base/services/openvpn.json | 3 ++- puppet/modules/site_openvpn/manifests/server_config.pp | 12 ++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/provider_base/files/service-definitions/v1/eip-service.json.erb b/provider_base/files/service-definitions/v1/eip-service.json.erb index 3b8976fd..0ecd002a 100644 --- a/provider_base/files/service-definitions/v1/eip-service.json.erb +++ b/provider_base/files/service-definitions/v1/eip-service.json.erb @@ -42,6 +42,10 @@ end configuration = node.openvpn.configuration end + configuration = configuration.dup + if configuration['fragment'] && configuration['fragment'] == 1500 + configuration.delete('fragment') + end hsh["gateways"] = gateways.compact hsh["locations"] = locations hsh["openvpn_configuration"] = configuration diff --git a/provider_base/services/openvpn.json b/provider_base/services/openvpn.json index 1906244c..127f5890 100644 --- a/provider_base/services/openvpn.json +++ b/provider_base/services/openvpn.json @@ -24,7 +24,8 @@ "auth": "SHA1", "cipher": "AES-128-CBC", "keepalive": "10 30", - "tun-ipv6": true + "tun-ipv6": true, + "fragment": 1400 } }, "obfsproxy": { diff --git a/puppet/modules/site_openvpn/manifests/server_config.pp b/puppet/modules/site_openvpn/manifests/server_config.pp index 97cf2842..466f6d00 100644 --- a/puppet/modules/site_openvpn/manifests/server_config.pp +++ b/puppet/modules/site_openvpn/manifests/server_config.pp @@ -85,6 +85,18 @@ define site_openvpn::server_config( key => 'tcp-nodelay', server => $openvpn_configname; } + } elsif $proto == 'udp' { + if $config['fragment'] != 1500 { + openvpn::option { + "fragment ${openvpn_configname}": + key => 'fragment', + value => $config['fragment'], + server => $openvpn_configname; + "mssfix ${openvpn_configname}": + key => 'mssfix', + server => $openvpn_configname; + } + } } openvpn::option { -- cgit v1.2.3 From e18853b16969cb164613003edfab9a5b5800e099 Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 10 Nov 2014 20:56:38 -0800 Subject: change default openvpn fragment size back to 1500 so we don't break backward compatibility with older clients --- provider_base/services/openvpn.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider_base/services/openvpn.json b/provider_base/services/openvpn.json index 127f5890..11cb0dc2 100644 --- a/provider_base/services/openvpn.json +++ b/provider_base/services/openvpn.json @@ -25,7 +25,7 @@ "cipher": "AES-128-CBC", "keepalive": "10 30", "tun-ipv6": true, - "fragment": 1400 + "fragment": 1500 } }, "obfsproxy": { -- cgit v1.2.3 From 7521958cc6c210d65009aa87c6c7297fd9be3dd2 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Sat, 15 Nov 2014 13:36:51 -0500 Subject: don't enable Tor DirPort if openvpn is running on port 80 (Bug #6377) We need to check the openvpn hiera value, which may or may not be set. If it is not set, then we need to not lookup the $openvpn['ports]' values or we will get an error because it wont be the correct type. If we do have it, then $openvpn_ports gets set with the hash, otherwise it gets set to an empty hash (otherwise puppet will complain when we try to query the member() later with "member(): Requires array to work with"). Finally, if it is set to port 80, we don't include the tor::daemon::directory Change-Id: Ic366c72e966cae9d611e8fe5aa7ea7943be51241 --- puppet/modules/site_tor/manifests/init.pp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/puppet/modules/site_tor/manifests/init.pp b/puppet/modules/site_tor/manifests/init.pp index 9944bb2b..d14e813d 100644 --- a/puppet/modules/site_tor/manifests/init.pp +++ b/puppet/modules/site_tor/manifests/init.pp @@ -11,6 +11,14 @@ class site_tor { $address = hiera('ip_address') + $openvpn = hiera('openvpn', undef) + if $openvpn { + $openvpn_ports = $openvpn['ports'] + } + else { + $openvpn_ports = [] + } + class { 'tor::daemon': } tor::daemon::relay { $nickname: port => 9001, @@ -22,7 +30,8 @@ class site_tor { if ( $tor_type == 'exit'){ # Only enable the daemon directory if the node isn't also a webapp node - if ! member($::services, 'webapp') { + # or running openvpn on port 80 + if ! member($::services, 'webapp') and ! member($openvpn_ports, '80') { tor::daemon::directory { $::hostname: port => 80 } } } -- cgit v1.2.3 From 6200be6fbf1f05665158e9ce8b218433f2440240 Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 19 Nov 2014 11:37:45 -0800 Subject: test if soledad daemon is running --- bin/run_tests | 8 ++++++++ tests/white-box/couchdb.rb | 2 +- tests/white-box/mx.rb | 2 +- tests/white-box/openvpn.rb | 4 ++-- tests/white-box/soledad.rb | 17 +++++++++++++++++ tests/white-box/webapp.rb | 2 +- 6 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 tests/white-box/soledad.rb diff --git a/bin/run_tests b/bin/run_tests index b3e60fcc..4addc0c8 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -35,6 +35,14 @@ def bail(code, msg=nil) end end +## +## UTILITY +## + +def service?(service) + $node["services"].include?(service.to_s) +end + ## ## EXCEPTIONS ## diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb index d438b193..2788f4f7 100644 --- a/tests/white-box/couchdb.rb +++ b/tests/white-box/couchdb.rb @@ -1,4 +1,4 @@ -raise SkipTest unless $node["services"].include?("couchdb") +raise SkipTest unless service?(:couchdb) require 'json' diff --git a/tests/white-box/mx.rb b/tests/white-box/mx.rb index da780de4..794a9a41 100644 --- a/tests/white-box/mx.rb +++ b/tests/white-box/mx.rb @@ -1,4 +1,4 @@ -raise SkipTest unless $node["services"].include?("mx") +raise SkipTest unless service?(:mx) require 'json' diff --git a/tests/white-box/openvpn.rb b/tests/white-box/openvpn.rb index 5eb2bdb5..23a40426 100644 --- a/tests/white-box/openvpn.rb +++ b/tests/white-box/openvpn.rb @@ -1,6 +1,6 @@ -raise SkipTest unless $node["services"].include?("openvpn") +raise SkipTest unless service?(:openvpn) -class Openvpn < LeapTest +class OpenVPN < LeapTest depends_on "Network" def setup diff --git a/tests/white-box/soledad.rb b/tests/white-box/soledad.rb new file mode 100644 index 00000000..5a13e4a6 --- /dev/null +++ b/tests/white-box/soledad.rb @@ -0,0 +1,17 @@ +raise SkipTest unless service?(:soledad) + +require 'json' + +class Soledad < LeapTest + depends_on "Network" + depends_on "CouchDB" if service?(:couchdb) + + def setup + end + + def test_00_Is_Soledad_running? + assert_running 'soledad' + pass + end + +end diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb index 0fea1c7f..2aa87403 100644 --- a/tests/white-box/webapp.rb +++ b/tests/white-box/webapp.rb @@ -1,4 +1,4 @@ -raise SkipTest unless $node["services"].include?("webapp") +raise SkipTest unless service?(:webapp) require 'json' -- cgit v1.2.3 From be18ba31fadd2e587672adc44175dd106187ceba Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Thu, 20 Nov 2014 13:13:33 -0500 Subject: minor linting Change-Id: I6d04cc7e028e86ee0012d96d7ef075fdd7ecef19 --- puppet/modules/site_shorewall/manifests/dnat_rule.pp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/puppet/modules/site_shorewall/manifests/dnat_rule.pp b/puppet/modules/site_shorewall/manifests/dnat_rule.pp index aa298408..49b929f2 100644 --- a/puppet/modules/site_shorewall/manifests/dnat_rule.pp +++ b/puppet/modules/site_shorewall/manifests/dnat_rule.pp @@ -4,7 +4,7 @@ define site_shorewall::dnat_rule { if $port != 1194 { if $site_openvpn::openvpn_allow_unlimited { shorewall::rule { - "dnat_tcp_port_$port": + "dnat_tcp_port_${port}": action => 'DNAT', source => 'net', destination => "\$FW:${site_openvpn::unlimited_gateway_address}:1194", @@ -13,7 +13,7 @@ define site_shorewall::dnat_rule { order => 100; } shorewall::rule { - "dnat_udp_port_$port": + "dnat_udp_port_${port}": action => 'DNAT', source => 'net', destination => "\$FW:${site_openvpn::unlimited_gateway_address}:1194", @@ -24,7 +24,7 @@ define site_shorewall::dnat_rule { } if $site_openvpn::openvpn_allow_limited { shorewall::rule { - "dnat_free_tcp_port_$port": + "dnat_free_tcp_port_${port}": action => 'DNAT', source => 'net', destination => "\$FW:${site_openvpn::limited_gateway_address}:1194", @@ -33,7 +33,7 @@ define site_shorewall::dnat_rule { order => 100; } shorewall::rule { - "dnat_free_udp_port_$port": + "dnat_free_udp_port_${port}": action => 'DNAT', source => 'net', destination => "\$FW:${site_openvpn::limited_gateway_address}:1194", -- cgit v1.2.3 From e334f10447303209ac3802436437670f45511603 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Thu, 20 Nov 2014 13:13:55 -0500 Subject: specify the destination IP for DNAT rules for gateway addresses on port 443 (#6388) Previously the DNAT rule would redirect the incoming port 443 requests to openvpn, which was the wrong thing to do on the primary IP (but the right thing to do on the openvpn gateway IPs). This manifested in the webapp not being available when it was also configured as a service on the node. Change-Id: Ic8c6b6c0389859fab168a7df687351e11263277a --- puppet/modules/site_shorewall/manifests/dnat_rule.pp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/puppet/modules/site_shorewall/manifests/dnat_rule.pp b/puppet/modules/site_shorewall/manifests/dnat_rule.pp index 49b929f2..f9fbe950 100644 --- a/puppet/modules/site_shorewall/manifests/dnat_rule.pp +++ b/puppet/modules/site_shorewall/manifests/dnat_rule.pp @@ -10,6 +10,7 @@ define site_shorewall::dnat_rule { destination => "\$FW:${site_openvpn::unlimited_gateway_address}:1194", proto => 'tcp', destinationport => $port, + originaldest => $site_openvpn::unlimited_gateway_address, order => 100; } shorewall::rule { @@ -19,6 +20,7 @@ define site_shorewall::dnat_rule { destination => "\$FW:${site_openvpn::unlimited_gateway_address}:1194", proto => 'udp', destinationport => $port, + originaldest => $site_openvpn::unlimited_gateway_address, order => 100; } } @@ -30,6 +32,7 @@ define site_shorewall::dnat_rule { destination => "\$FW:${site_openvpn::limited_gateway_address}:1194", proto => 'tcp', destinationport => $port, + originaldest => $site_openvpn::unlimited_gateway_address, order => 100; } shorewall::rule { @@ -39,6 +42,7 @@ define site_shorewall::dnat_rule { destination => "\$FW:${site_openvpn::limited_gateway_address}:1194", proto => 'udp', destinationport => $port, + originaldest => $site_openvpn::unlimited_gateway_address, order => 100; } } -- cgit v1.2.3 From 896dd69710fa24a0235fc70081a71f35adbf9af1 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Thu, 20 Nov 2014 15:22:09 -0500 Subject: Make sure that stunnel restarts when cert/key change (#6181) Change-Id: I5085247a87018e18e73833119ac73225afbfea1e --- puppet/modules/site_stunnel/manifests/client.pp | 6 +----- puppet/modules/site_stunnel/manifests/init.pp | 2 ++ puppet/modules/site_stunnel/manifests/override_service.pp | 13 +++++++++++++ puppet/modules/site_stunnel/manifests/servers.pp | 6 +----- 4 files changed, 17 insertions(+), 10 deletions(-) create mode 100644 puppet/modules/site_stunnel/manifests/override_service.pp diff --git a/puppet/modules/site_stunnel/manifests/client.pp b/puppet/modules/site_stunnel/manifests/client.pp index 76815174..3b10ecb8 100644 --- a/puppet/modules/site_stunnel/manifests/client.pp +++ b/puppet/modules/site_stunnel/manifests/client.pp @@ -35,11 +35,7 @@ define site_stunnel::client ( pid => "/var/run/stunnel4/${pid}.pid", rndfile => $rndfile, debuglevel => $debuglevel, - sslversion => 'TLSv1', - subscribe => [ - Class['Site_config::X509::Key'], - Class['Site_config::X509::Cert'], - Class['Site_config::X509::Ca'] ]; + sslversion => 'TLSv1'; } site_shorewall::stunnel::client { $name: diff --git a/puppet/modules/site_stunnel/manifests/init.pp b/puppet/modules/site_stunnel/manifests/init.pp index b292f1cd..2e0cf5b8 100644 --- a/puppet/modules/site_stunnel/manifests/init.pp +++ b/puppet/modules/site_stunnel/manifests/init.pp @@ -28,5 +28,7 @@ class site_stunnel { $clients = $stunnel['clients'] $client_sections = keys($clients) site_stunnel::clients { $client_sections: } + + include site_stunnel::override_service } diff --git a/puppet/modules/site_stunnel/manifests/override_service.pp b/puppet/modules/site_stunnel/manifests/override_service.pp new file mode 100644 index 00000000..96187048 --- /dev/null +++ b/puppet/modules/site_stunnel/manifests/override_service.pp @@ -0,0 +1,13 @@ +class site_stunnel::override_service inherits stunnel::debian { + + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca + + Service[stunnel] { + subscribe => [ + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Ca'] ] + } +} diff --git a/puppet/modules/site_stunnel/manifests/servers.pp b/puppet/modules/site_stunnel/manifests/servers.pp index 8d537644..b6fac319 100644 --- a/puppet/modules/site_stunnel/manifests/servers.pp +++ b/puppet/modules/site_stunnel/manifests/servers.pp @@ -35,11 +35,7 @@ define site_stunnel::servers ( pid => "/var/run/stunnel4/${pid}.pid", rndfile => '/var/lib/stunnel4/.rnd', debuglevel => $debuglevel, - sslversion => 'TLSv1', - require => [ - Class['Site_config::X509::Key'], - Class['Site_config::X509::Cert'], - Class['Site_config::X509::Ca'] ]; + sslversion => 'TLSv1'; } # allow incoming connections on $accept_port -- cgit v1.2.3 From dff949811324215278ab7e4c2db5de63d8a6218b Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Thu, 20 Nov 2014 15:31:27 -0500 Subject: Make sure openvpn is restarted when cert/key change (#6405) I reformatted the section below for consistency. Change-Id: I18f5e23850e0c1ab4b1f2ee467d5af54ae9ff303 --- puppet/modules/site_openvpn/manifests/init.pp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/puppet/modules/site_openvpn/manifests/init.pp b/puppet/modules/site_openvpn/manifests/init.pp index b6331f12..a8d2044d 100644 --- a/puppet/modules/site_openvpn/manifests/init.pp +++ b/puppet/modules/site_openvpn/manifests/init.pp @@ -148,13 +148,17 @@ class site_openvpn { exec { 'restart_openvpn': command => '/etc/init.d/openvpn restart', refreshonly => true, - subscribe => File['/etc/openvpn'], + subscribe => [ + File['/etc/openvpn'], + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Ca'] ], require => [ - Package['openvpn'], - File['/etc/openvpn'], - Class['Site_config::X509::Key'], - Class['Site_config::X509::Cert'], - Class['Site_config::X509::Ca_bundle'] ]; + Package['openvpn'], + File['/etc/openvpn'], + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Ca_bundle'] ]; } cron { 'add_gateway_ips.sh': -- cgit v1.2.3 From 503a316e67da2b4e0dbae7db28bbd0574fa3739f Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Thu, 20 Nov 2014 16:27:35 -0500 Subject: ship a modified runit config for bigcouch that raises the open file descriptor limits to account for bigcouch sync spikes (#4935) Change-Id: I242fba31f961b6139ec641e1708b170f5c0d009b --- puppet/modules/site_couchdb/files/runit_config | 6 ++++++ puppet/modules/site_couchdb/manifests/bigcouch.pp | 10 ++++++++++ 2 files changed, 16 insertions(+) create mode 100644 puppet/modules/site_couchdb/files/runit_config diff --git a/puppet/modules/site_couchdb/files/runit_config b/puppet/modules/site_couchdb/files/runit_config new file mode 100644 index 00000000..169b4832 --- /dev/null +++ b/puppet/modules/site_couchdb/files/runit_config @@ -0,0 +1,6 @@ +#!/bin/bash +exec 2>&1 +export HOME=/home/bigcouch +ulimit -H -n 32768 +ulimit -S -n 32768 +exec chpst -u bigcouch /opt/bigcouch/bin/bigcouch diff --git a/puppet/modules/site_couchdb/manifests/bigcouch.pp b/puppet/modules/site_couchdb/manifests/bigcouch.pp index d71c00c5..16593ec7 100644 --- a/puppet/modules/site_couchdb/manifests/bigcouch.pp +++ b/puppet/modules/site_couchdb/manifests/bigcouch.pp @@ -31,4 +31,14 @@ class site_couchdb::bigcouch { file { '/var/log/bigcouch': ensure => directory } + + file { '/etc/sv/bigcouch/run': + ensure => present, + source => 'puppet:///modules/site_couchdb/runit_config', + owner => root, + group => root, + mode => '0755', + require => Package['couchdb'], + notify => Service['couchdb'] + } } -- cgit v1.2.3 From 6ebd8cee202c81260c151e7903ff2f16518ffa41 Mon Sep 17 00:00:00 2001 From: varac Date: Thu, 20 Nov 2014 23:00:50 +0100 Subject: Fix Check_mk notifications (Bug #6403) Let check_mk put all hosts into the same "admin" contactgroup, which is defined as default contactgroup by nagios. Change-Id: I13b434925711ef2037de0cf6e919ce39a8255a94 --- puppet/modules/site_check_mk/files/host_contactgroups.mk | 3 +++ puppet/modules/site_check_mk/manifests/server.pp | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 puppet/modules/site_check_mk/files/host_contactgroups.mk diff --git a/puppet/modules/site_check_mk/files/host_contactgroups.mk b/puppet/modules/site_check_mk/files/host_contactgroups.mk new file mode 100644 index 00000000..e89323fb --- /dev/null +++ b/puppet/modules/site_check_mk/files/host_contactgroups.mk @@ -0,0 +1,3 @@ +host_contactgroups = [ + ( "admins", ALL_HOSTS ), +] diff --git a/puppet/modules/site_check_mk/manifests/server.pp b/puppet/modules/site_check_mk/manifests/server.pp index aa24d96c..388ae94b 100644 --- a/puppet/modules/site_check_mk/manifests/server.pp +++ b/puppet/modules/site_check_mk/manifests/server.pp @@ -40,6 +40,10 @@ class site_check_mk::server { content => template('site_check_mk/hostgroups.mk'), notify => Exec['check_mk-refresh'], require => Package['check-mk-server']; + '/etc/check_mk/conf.d/host_contactgroups.mk': + source => 'puppet:///modules/site_check_mk/host_contactgroups.mk', + notify => Exec['check_mk-refresh'], + require => Package['check-mk-server']; '/etc/check_mk/all_hosts_static': content => $all_hosts, notify => Exec['check_mk-refresh'], @@ -64,6 +68,5 @@ class site_check_mk::server { require => Package['nagios-plugins-basic']; } - include check_mk::agent::local_checks } -- cgit v1.2.3 From fbfc88d8e4ff032fd8155e264ba75668b426fdb2 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Sun, 23 Nov 2014 10:30:10 -0500 Subject: fix dependency on x509 ca_bundle class (#6410) Change-Id: Ia1e7009240d61464d7ba45ad07291664f6a3b768 --- puppet/modules/site_openvpn/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_openvpn/manifests/init.pp b/puppet/modules/site_openvpn/manifests/init.pp index a8d2044d..d6f9150b 100644 --- a/puppet/modules/site_openvpn/manifests/init.pp +++ b/puppet/modules/site_openvpn/manifests/init.pp @@ -152,7 +152,7 @@ class site_openvpn { File['/etc/openvpn'], Class['Site_config::X509::Key'], Class['Site_config::X509::Cert'], - Class['Site_config::X509::Ca'] ], + Class['Site_config::X509::Ca_bundle'] ], require => [ Package['openvpn'], File['/etc/openvpn'], -- cgit v1.2.3 From 95d30570f681ba2f43bbe7ffe344b92439be1ae9 Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 24 Nov 2014 12:11:32 -0800 Subject: fixed bug when there is no vpn service --- .../files/service-definitions/v1/eip-service.json.erb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/provider_base/files/service-definitions/v1/eip-service.json.erb b/provider_base/files/service-definitions/v1/eip-service.json.erb index 0ecd002a..4bd220df 100644 --- a/provider_base/files/service-definitions/v1/eip-service.json.erb +++ b/provider_base/files/service-definitions/v1/eip-service.json.erb @@ -42,12 +42,14 @@ end configuration = node.openvpn.configuration end - configuration = configuration.dup - if configuration['fragment'] && configuration['fragment'] == 1500 - configuration.delete('fragment') + if gateways.any? + configuration = configuration.dup + if configuration['fragment'] && configuration['fragment'] == 1500 + configuration.delete('fragment') + end + hsh["gateways"] = gateways.compact + hsh["locations"] = locations + hsh["openvpn_configuration"] = configuration end - hsh["gateways"] = gateways.compact - hsh["locations"] = locations - hsh["openvpn_configuration"] = configuration JSON.sorted_generate hsh %> \ No newline at end of file -- cgit v1.2.3 From de51b83384d97a67cdbdf1992ba9ad771a292c5d Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 24 Nov 2014 14:17:43 -0800 Subject: bind webapp to version/0.6 branch --- provider_base/services/webapp.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index 44b5fa14..67744f99 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -22,7 +22,7 @@ "secure": false, "git": { "source": "https://leap.se/git/leap_web", - "revision": "origin/master" + "revision": "origin/version/0.6" }, "client_version": "= provider.client_version", "nagios_test_user": { -- cgit v1.2.3 From 83267a00f272a6f3f52a66eca44d9ca85924cc8e Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 25 Nov 2014 16:25:43 -0800 Subject: include a host information in ssh_config for ever possible host a given node might communicate with. this includes port and host key algorithm. closes #6432 --- provider_base/lib/macros/hosts.rb | 7 ++++++- puppet/modules/site_sshd/templates/ssh_config.erb | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/provider_base/lib/macros/hosts.rb b/provider_base/lib/macros/hosts.rb index 8a4058a5..8281329f 100644 --- a/provider_base/lib/macros/hosts.rb +++ b/provider_base/lib/macros/hosts.rb @@ -42,7 +42,12 @@ module LeapCli hosts = {} my_location = @node['location'] ? @node['location']['name'] : nil nodes.each_node do |node| - hosts[node.name] = {'ip_address' => node.ip_address, 'domain_internal' => node.domain.internal, 'domain_full' => node.domain.full} + hosts[node.name] = { + 'ip_address' => node.ip_address, + 'domain_internal' => node.domain.internal, + 'domain_full' => node.domain.full, + 'port' => node.ssh.port + } node_location = node['location'] ? node['location']['name'] : nil if my_location == node_location if facts = @node.manager.facts[node.name] diff --git a/puppet/modules/site_sshd/templates/ssh_config.erb b/puppet/modules/site_sshd/templates/ssh_config.erb index 7e967413..36c0b6d5 100644 --- a/puppet/modules/site_sshd/templates/ssh_config.erb +++ b/puppet/modules/site_sshd/templates/ssh_config.erb @@ -21,3 +21,20 @@ Host * StrictHostKeyChecking no <% end -%> +# +# Tell SSH what host key algorithm we should use. I don't understand why this +# is needed, since the man page says that "if hostkeys are known for the +# destination host then [HostKeyAlgorithms default] is modified to prefer +# their algorithms." +# + +<% @hosts.sort.each do |name, host| -%> +Host <%= name %> <%= host['domain_full'] %> <%= host['domain_internal'] %> <%= host['ip_address'] %> +<% if host['host_pub_key'] -%> +HostKeyAlgorithms <%= host['host_pub_key'].split(" ").first %> +<% end -%> +<% if host['port'] -%> +Port <%= host['port'] %> +<% end -%> + +<% end -%> -- cgit v1.2.3 From e7c1e86a4a63def83ff205319e0950e4d3421180 Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 25 Nov 2014 16:52:24 -0800 Subject: atomic tests for webapp api --- tests/white-box/webapp.rb | 82 ++++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 47 deletions(-) diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb index 2aa87403..7fbab1af 100644 --- a/tests/white-box/webapp.rb +++ b/tests/white-box/webapp.rb @@ -41,62 +41,50 @@ class Webapp < LeapTest pass end - def test_05_Can_create_user? - @@user = nil - user = SRP::User.new + def test_05_Can_create_and_authenticate_and_delete_user_via_API? + @user = SRP::User.new + @session_token = nil + @user_id = nil + + # create user url = api_url("/1/users.json") - assert_post(url, user.to_params) do |body| + assert_post(url, @user.to_params) do |body| assert response = JSON.parse(body), 'response should be JSON' assert response['ok'], 'creating a user should be successful' end - @@user = user - pass - end - def test_06_Can_authenticate? - @@user_id = nil - @@session_token = nil - if @@user.nil? - skip "Depends on user creation" - else - url = api_url("/1/sessions.json") - session = SRP::Session.new(@@user) - params = {'login' => @@user.username, 'A' => session.aa} - assert_post(url, params) do |response, body| - cookie = response['Set-Cookie'].split(';').first + # authenticate + url = api_url("/1/sessions.json") + session = SRP::Session.new(@user) + params = {'login' => @user.username, 'A' => session.aa} + assert_post(url, params) do |response, body| + cookie = response['Set-Cookie'].split(';').first + assert(response = JSON.parse(body), 'response should be JSON') + assert(bb = response["B"]) + session.bb = bb + url = api_url("/1/sessions/login.json") + params = {'client_auth' => session.m, 'A' => session.aa} + options = {:headers => {'Cookie' => cookie}} + assert_put(url, params, options) do |body| assert(response = JSON.parse(body), 'response should be JSON') - assert(bb = response["B"]) - session.bb = bb - url = api_url("/1/sessions/login.json") - params = {'client_auth' => session.m, 'A' => session.aa} - options = {:headers => {'Cookie' => cookie}} - assert_put(url, params, options) do |body| - assert(response = JSON.parse(body), 'response should be JSON') - assert(response['M2'], 'response should include M2') - assert(@@session_token = response['token'], 'response should include token') - assert(@@user_id = response['id'], 'response should include user id') - end + assert(response['M2'], 'response should include M2') + assert(@session_token = response['token'], 'response should include token') + assert(@user_id = response['id'], 'response should include user id') end - pass end - end - def test_07_Can_delete_user? - if @@user_id.nil? || @@session_token.nil? - skip "Depends on authentication" - else - url = api_url("/1/users/#{@@user_id}.json") - options = {:headers => { - "Authorization" => "Token token=\"#{@@session_token}\"" - }} - delete(url, {}, options) do |body, response, error| - if response.code.to_i != 200 - skip "It appears the web api is too old to support deleting users" - else - assert(response = JSON.parse(body), 'response should be JSON') - assert(response["success"], 'delete should be a success') - pass - end + # delete + url = api_url("/1/users/#{@user_id}.json") + options = {:headers => { + "Authorization" => "Token token=\"#{@session_token}\"" + }} + delete(url, {}, options) do |body, response, error| + if response.code.to_i != 200 + skip "It appears the web api is too old to support deleting users" + else + assert(response = JSON.parse(body), 'response should be JSON') + assert(response["success"], 'delete should be a success') + pass end end end -- cgit v1.2.3 From fb02557925a464488c2996df3625c051c172f8ad Mon Sep 17 00:00:00 2001 From: elijah Date: Sun, 30 Nov 2014 22:18:41 -0800 Subject: pin tapicero to version/0.6 --- puppet/modules/tapicero/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/tapicero/manifests/init.pp b/puppet/modules/tapicero/manifests/init.pp index 2bf72004..28711b94 100644 --- a/puppet/modules/tapicero/manifests/init.pp +++ b/puppet/modules/tapicero/manifests/init.pp @@ -95,7 +95,7 @@ class tapicero { vcsrepo { '/srv/leap/tapicero': ensure => present, force => true, - revision => 'origin/master', + revision => 'origin/version/0.6', provider => git, source => 'https://leap.se/git/tapicero', owner => 'tapicero', -- cgit v1.2.3 From 69067b64942fad72bd0ec9eeab7f161ef4a16ee3 Mon Sep 17 00:00:00 2001 From: elijah Date: Sun, 30 Nov 2014 22:19:20 -0800 Subject: minor: ensure there is only one tapicero process --- tests/helpers/os_helper.rb | 8 ++++++-- tests/white-box/couchdb.rb | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/helpers/os_helper.rb b/tests/helpers/os_helper.rb index 529e899f..aad67dda 100644 --- a/tests/helpers/os_helper.rb +++ b/tests/helpers/os_helper.rb @@ -17,8 +17,12 @@ class LeapTest }.compact end - def assert_running(process) - assert pgrep(process).any?, "No running process for #{process}" + def assert_running(process, options={}) + processes = pgrep(process) + assert processes.any?, "No running process for #{process}" + if options[:single] + assert processes.length == 1, "More than one process for #{process}" + end end # diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb index 2788f4f7..450c4201 100644 --- a/tests/white-box/couchdb.rb +++ b/tests/white-box/couchdb.rb @@ -9,7 +9,7 @@ class CouchDB < LeapTest end def test_00_Are_daemons_running? - assert_running 'tapicero' + assert_running '^tapicero', :single => true if multimaster? assert_running 'bin/beam' assert_running 'bin/epmd' -- cgit v1.2.3 From 1de9dd7b9297cdd67f8bf51352e5dcea08f5fb29 Mon Sep 17 00:00:00 2001 From: varac Date: Fri, 28 Nov 2014 22:21:15 +0100 Subject: ignore ntp check because it's flapping to often (Bug #6407) Change-Id: I52e19bbdfcf6576bd9c247d99aace47eb86c8116 --- puppet/modules/site_check_mk/files/ignored_services.mk | 3 +++ puppet/modules/site_check_mk/manifests/server.pp | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 puppet/modules/site_check_mk/files/ignored_services.mk diff --git a/puppet/modules/site_check_mk/files/ignored_services.mk b/puppet/modules/site_check_mk/files/ignored_services.mk new file mode 100644 index 00000000..35dc4433 --- /dev/null +++ b/puppet/modules/site_check_mk/files/ignored_services.mk @@ -0,0 +1,3 @@ +ignored_services = [ + ( ALL_HOSTS, [ "NTP Time" ] ) +] diff --git a/puppet/modules/site_check_mk/manifests/server.pp b/puppet/modules/site_check_mk/manifests/server.pp index 388ae94b..93107b04 100644 --- a/puppet/modules/site_check_mk/manifests/server.pp +++ b/puppet/modules/site_check_mk/manifests/server.pp @@ -41,7 +41,11 @@ class site_check_mk::server { notify => Exec['check_mk-refresh'], require => Package['check-mk-server']; '/etc/check_mk/conf.d/host_contactgroups.mk': - source => 'puppet:///modules/site_check_mk/host_contactgroups.mk', + source => 'puppet:///modules/site_check_mk/host_contactgroups.mk', + notify => Exec['check_mk-refresh'], + require => Package['check-mk-server']; + '/etc/check_mk/conf.d/ignored_services.mk': + source => 'puppet:///modules/site_check_mk/ignored_services.mk', notify => Exec['check_mk-refresh'], require => Package['check-mk-server']; '/etc/check_mk/all_hosts_static': @@ -61,6 +65,7 @@ class site_check_mk::server { owner => 'nagios', mode => '0644', require => Package['check-mk-server']; + # check_icmp must be suid root or called by sudo # see https://leap.se/code/issues/5171 '/usr/lib/nagios/plugins/check_icmp': -- cgit v1.2.3 From 989c203002c812b1e6089232409aab08c80aec5d Mon Sep 17 00:00:00 2001 From: varac Date: Fri, 28 Nov 2014 22:22:41 +0100 Subject: Increase nagios max_checks_attempts from 1 to 4 so notifications won't be sent out on first failed check_mk check (Bug #6461) Change-Id: I1bd47b3c3d17508488a4db90d74118006d85a03a --- puppet/modules/site_check_mk/files/extra_service_conf.mk | 3 +++ puppet/modules/site_check_mk/manifests/server.pp | 6 ++++++ 2 files changed, 9 insertions(+) create mode 100644 puppet/modules/site_check_mk/files/extra_service_conf.mk diff --git a/puppet/modules/site_check_mk/files/extra_service_conf.mk b/puppet/modules/site_check_mk/files/extra_service_conf.mk new file mode 100644 index 00000000..51f348f1 --- /dev/null +++ b/puppet/modules/site_check_mk/files/extra_service_conf.mk @@ -0,0 +1,3 @@ +extra_service_conf["max_check_attempts"] = [ + ("4", ALL_HOSTS , ALL_SERVICES ) +] diff --git a/puppet/modules/site_check_mk/manifests/server.pp b/puppet/modules/site_check_mk/manifests/server.pp index 93107b04..b384923c 100644 --- a/puppet/modules/site_check_mk/manifests/server.pp +++ b/puppet/modules/site_check_mk/manifests/server.pp @@ -48,10 +48,16 @@ class site_check_mk::server { source => 'puppet:///modules/site_check_mk/ignored_services.mk', notify => Exec['check_mk-refresh'], require => Package['check-mk-server']; + '/etc/check_mk/conf.d/extra_service_conf.mk': + source => 'puppet:///modules/site_check_mk/extra_service_conf.mk', + notify => Exec['check_mk-refresh'], + require => Package['check-mk-server']; + '/etc/check_mk/all_hosts_static': content => $all_hosts, notify => Exec['check_mk-refresh'], require => Package['check-mk-server']; + '/etc/check_mk/.ssh': ensure => directory, require => Package['check-mk-server']; -- cgit v1.2.3 From 77b04f641582d6995449305100b7a93d61b219d9 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Mon, 1 Dec 2014 13:29:20 -0500 Subject: Increase the nagios alert thresholds for bigcouch open file descriptors (#6473) Change-Id: I2549d781427fffc865c2bdcd1e950d60dad509fd --- puppet/modules/site_check_mk/manifests/agent/couchdb.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp index 01e2b886..ee0268a3 100644 --- a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp @@ -29,7 +29,7 @@ class site_check_mk::agent::couchdb { } file_line { 'Bigcouch_open_files': - line => 'Bigcouch_open_files /srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 750,750 -c 1000,1000', + line => 'Bigcouch_open_files /srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 28672,28672 -c 30720,30720', path => '/etc/check_mk/mrpe.cfg'; } -- cgit v1.2.3 From a3af7019efe6f5ff3daeeff474a8c31b1b3318c8 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 2 Dec 2014 10:13:11 -0500 Subject: minor linting Change-Id: Idf550ed004bcb42d6e19ac0a2c5286f52a390935 --- puppet/modules/site_nagios/manifests/add_service.pp | 2 +- puppet/modules/site_nagios/manifests/server.pp | 2 +- puppet/modules/site_postfix/manifests/mx.pp | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/puppet/modules/site_nagios/manifests/add_service.pp b/puppet/modules/site_nagios/manifests/add_service.pp index 1b67d14e..97c0b30d 100644 --- a/puppet/modules/site_nagios/manifests/add_service.pp +++ b/puppet/modules/site_nagios/manifests/add_service.pp @@ -9,7 +9,7 @@ define site_nagios::add_service ( nagios_service { "${name}_ssh": use => 'generic-service', - check_command => "check_ssh_port!$ssh_port", + check_command => "check_ssh_port!${ssh_port}", service_description => 'SSH', host_name => $hostname; "${name}_cert": diff --git a/puppet/modules/site_nagios/manifests/server.pp b/puppet/modules/site_nagios/manifests/server.pp index b195c880..37bd77fb 100644 --- a/puppet/modules/site_nagios/manifests/server.pp +++ b/puppet/modules/site_nagios/manifests/server.pp @@ -16,7 +16,7 @@ class site_nagios::server inherits nagios::base { include nagios::defaults::timeperiods include nagios::defaults::plugins - class {'nagios': + class { 'nagios': # don't manage apache class from nagios, cause we already include # it in site_apache::common httpd => 'absent', diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp index bdfee665..81f10b77 100644 --- a/puppet/modules/site_postfix/manifests/mx.pp +++ b/puppet/modules/site_postfix/manifests/mx.pp @@ -1,12 +1,12 @@ class site_postfix::mx { - $domain_hash = hiera ('domain') + $domain_hash = hiera('domain') $domain = $domain_hash['full_suffix'] $host_domain = $domain_hash['full'] $cert_name = hiera('name') $mynetworks = join(hiera('mynetworks'), ' ') - $root_mail_recipient = hiera ('contacts') + $root_mail_recipient = hiera('contacts') $postfix_smtp_listen = 'all' include site_config::x509::cert -- cgit v1.2.3 From 172ad4bc7dd5b82629bfa776ec4164f902569fdb Mon Sep 17 00:00:00 2001 From: guido Date: Tue, 2 Dec 2014 11:40:04 -0300 Subject: Use $hostname to locate tor.key. Fixes #6478 Change-Id: Ibbe3687d5a773b444f6e9145bf235aaeea637e1d --- puppet/modules/site_webapp/manifests/hidden_service.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_webapp/manifests/hidden_service.pp b/puppet/modules/site_webapp/manifests/hidden_service.pp index ac0e8a37..16b6e2e7 100644 --- a/puppet/modules/site_webapp/manifests/hidden_service.pp +++ b/puppet/modules/site_webapp/manifests/hidden_service.pp @@ -21,7 +21,7 @@ class site_webapp::hidden_service { '/var/lib/tor/webapp/private_key': ensure => present, - source => '/srv/leap/files/nodes/web/tor.key', + source => "/srv/leap/files/nodes/${::hostname}/tor.key", owner => 'debian-tor', group => 'debian-tor', mode => '0600'; -- cgit v1.2.3 From 8578cf78aca62636cd80bf1302d2946f2d62dbfe Mon Sep 17 00:00:00 2001 From: guido Date: Tue, 2 Dec 2014 11:41:46 -0300 Subject: Use include to avoid redeclaration of class { 'tor::daemon': }. Fixes #6479 Change-Id: Ibd1b1eef7afca10cf2a2d56a24e703636d6a52c6 --- puppet/modules/site_tor/manifests/init.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/puppet/modules/site_tor/manifests/init.pp b/puppet/modules/site_tor/manifests/init.pp index d14e813d..80ccc5d3 100644 --- a/puppet/modules/site_tor/manifests/init.pp +++ b/puppet/modules/site_tor/manifests/init.pp @@ -18,8 +18,8 @@ class site_tor { else { $openvpn_ports = [] } - - class { 'tor::daemon': } + + include tor::daemon tor::daemon::relay { $nickname: port => 9001, address => $address, -- cgit v1.2.3 From 5c300d2399398a7384728ef11b00642b8cfe5128 Mon Sep 17 00:00:00 2001 From: varac Date: Tue, 2 Dec 2014 20:21:26 +0100 Subject: Ignore bigcouch conflict errors, mainly coming from tapicero creating new users (Feature #6481) There are potentially many tapicero daemons running, and they all try to do the same thing at the same time. It is basically designed to create race conditions. All tapicero daemons try to create the user db at the same time. Only one of them wins the race and actually creates it. We need to fix this later (see https://leap.se/code/issues/6480) but for now, we ignore them because conflict errors should be handled by the applictation anyway. Change-Id: I91095b1901d238e3d199954ba3716023d3fd49c1 --- puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg index 28f333b0..d274a676 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg @@ -6,6 +6,8 @@ I 127.0.0.1 localhost:5984 .* ok # https://leap.se/code/issues/5246 I Shutting down group server + # ignore bigcouch conflict errors, mainly coming from tapicero creating new users + I Error in process.*{{nocatch,conflict} # ignore "Uncaught error in HTTP request: {exit, normal}" error # it's suppressed in later versions of bigcouch anhow # see https://leap.se/code/issues/5226 -- cgit v1.2.3 From a84272aa77715a4029ebd06b38b7a5ad05e6acd0 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 2 Dec 2014 11:26:35 -0500 Subject: Change nagios mail To: Header to contain the actual platform environment's contact email (Bug #6466) Change-Id: Ib86ae771e0ac3b6f329a517a8a31c9ec54d33a05 --- provider_base/services/monitor.json | 3 ++- .../modules/site_check_mk/files/host_contactgroups.mk | 3 --- puppet/modules/site_check_mk/manifests/server.pp | 3 ++- .../site_check_mk/templates/host_contactgroups.mk | 17 +++++++++++++++++ puppet/modules/site_check_mk/templates/hostgroups.mk | 17 +++++++++++++++-- .../modules/site_nagios/manifests/add_host_services.pp | 5 ++++- puppet/modules/site_nagios/manifests/add_service.pp | 11 +++++++---- puppet/modules/site_nagios/manifests/server.pp | 9 +++++---- .../site_nagios/manifests/server/add_contacts.pp | 16 ++++++++++++++++ .../site_nagios/manifests/server/contactgroup.pp | 6 ++++++ .../modules/site_nagios/manifests/server/hostgroup.pp | 2 +- 11 files changed, 75 insertions(+), 17 deletions(-) delete mode 100644 puppet/modules/site_check_mk/files/host_contactgroups.mk create mode 100644 puppet/modules/site_check_mk/templates/host_contactgroups.mk create mode 100644 puppet/modules/site_nagios/manifests/server/add_contacts.pp create mode 100644 puppet/modules/site_nagios/manifests/server/contactgroup.pp diff --git a/provider_base/services/monitor.json b/provider_base/services/monitor.json index 56ca015b..a68ee8d8 100644 --- a/provider_base/services/monitor.json +++ b/provider_base/services/monitor.json @@ -2,7 +2,8 @@ "nagios": { "nagiosadmin_pw": "= secret :nagios_admin_password", "domains_internal": "= global.tags.field('domain.internal_suffix').compact.uniq", - "hosts": "= (self.environment == 'local' ? nodes_like_me : nodes[:environment => '!local']).pick_fields('domain.internal', 'domain.full_suffix', 'ip_address', 'services', 'openvpn.gateway_address', 'ssh.port')" + "environments": "= Hash[ global.environment_names.select{|e|e!='local'}.map{|e| [e||'default',{'contact_emails'=>global.env(e).provider.contacts.default}]} ]", + "hosts": "= (self.environment == 'local' ? nodes_like_me : nodes[:environment => '!local']).pick_fields('environment', 'domain.internal', 'domain.full_suffix', 'ip_address', 'services', 'openvpn.gateway_address', 'ssh.port')" }, "hosts": "= self.environment == 'local' ? hosts_file(nodes_like_me) : hosts_file(nodes[:environment => '!local'])", "ssh": { diff --git a/puppet/modules/site_check_mk/files/host_contactgroups.mk b/puppet/modules/site_check_mk/files/host_contactgroups.mk deleted file mode 100644 index e89323fb..00000000 --- a/puppet/modules/site_check_mk/files/host_contactgroups.mk +++ /dev/null @@ -1,3 +0,0 @@ -host_contactgroups = [ - ( "admins", ALL_HOSTS ), -] diff --git a/puppet/modules/site_check_mk/manifests/server.pp b/puppet/modules/site_check_mk/manifests/server.pp index b384923c..1a866b9c 100644 --- a/puppet/modules/site_check_mk/manifests/server.pp +++ b/puppet/modules/site_check_mk/manifests/server.pp @@ -11,6 +11,7 @@ class site_check_mk::server { $hosts = hiera_hash('hosts') $all_hosts = inline_template ('<% @hosts.keys.sort.each do |key| -%>"<%= @hosts[key]["domain_internal"] %>", <% end -%>') $domains_internal = $nagios_hiera['domains_internal'] + $environments = $nagios_hiera['environments'] package { 'check-mk-server': ensure => installed, @@ -41,7 +42,7 @@ class site_check_mk::server { notify => Exec['check_mk-refresh'], require => Package['check-mk-server']; '/etc/check_mk/conf.d/host_contactgroups.mk': - source => 'puppet:///modules/site_check_mk/host_contactgroups.mk', + content => template('site_check_mk/host_contactgroups.mk'), notify => Exec['check_mk-refresh'], require => Package['check-mk-server']; '/etc/check_mk/conf.d/ignored_services.mk': diff --git a/puppet/modules/site_check_mk/templates/host_contactgroups.mk b/puppet/modules/site_check_mk/templates/host_contactgroups.mk new file mode 100644 index 00000000..6a534967 --- /dev/null +++ b/puppet/modules/site_check_mk/templates/host_contactgroups.mk @@ -0,0 +1,17 @@ +<% + contact_groups = [] + @environments.keys.sort.each do |env_name| + hosts = "" + @nagios_hosts.keys.sort.each do |hostname| + hostdata = @nagios_hosts[hostname] + domain_internal = hostdata['domain_internal'] + if hostdata['environment'] == env_name + hosts << '"' + domain_internal + '", ' + end + end + contact_groups << ' ( "%s", [%s] )' % [env_name, hosts] + end +%> +host_contactgroups = [ +<%= contact_groups.join(",\n") %> +] diff --git a/puppet/modules/site_check_mk/templates/hostgroups.mk b/puppet/modules/site_check_mk/templates/hostgroups.mk index 79b7f92f..7158dcd1 100644 --- a/puppet/modules/site_check_mk/templates/hostgroups.mk +++ b/puppet/modules/site_check_mk/templates/hostgroups.mk @@ -1,4 +1,17 @@ +<% + host_groups = [] + @environments.keys.sort.each do |env_name| + hosts = "" + @nagios_hosts.keys.sort.each do |hostname| + hostdata = @nagios_hosts[hostname] + domain_internal = hostdata['domain_internal'] + if hostdata['environment'] == env_name + hosts << '"' + domain_internal + '", ' + end + end + host_groups << ' ( "%s", [%s] )' % [env_name, hosts] + end +%> host_groups = [ - <% @domains_internal.each do |domain| %>( '<%= domain %>', [<% @nagios_hosts.keys.sort.each do |key| -%><% if @nagios_hosts[key]['domain_internal'] == key+'.'+domain -%>'<%= key %>.<%= domain %>', <% end -%><% end -%>] ), - <% end -%> +<%= host_groups.join(",\n") %> ] diff --git a/puppet/modules/site_nagios/manifests/add_host_services.pp b/puppet/modules/site_nagios/manifests/add_host_services.pp index 279809d1..236702e2 100644 --- a/puppet/modules/site_nagios/manifests/add_host_services.pp +++ b/puppet/modules/site_nagios/manifests/add_host_services.pp @@ -4,7 +4,9 @@ define site_nagios::add_host_services ( $ip_address, $services, $ssh_port, - $openvpn_gateway_address='' ) { + $environment, + $openvpn_gateway_address='', + ) { $nagios_hostname = $domain_internal @@ -16,6 +18,7 @@ define site_nagios::add_host_services ( 'hostname' => $nagios_hostname, 'ip_address' => $ip_address, 'openvpn_gw' => $openvpn_gateway_address, + 'environment' => $environment } $dynamic_parameters = { 'service' => '%s' diff --git a/puppet/modules/site_nagios/manifests/add_service.pp b/puppet/modules/site_nagios/manifests/add_service.pp index 97c0b30d..72cd038a 100644 --- a/puppet/modules/site_nagios/manifests/add_service.pp +++ b/puppet/modules/site_nagios/manifests/add_service.pp @@ -1,5 +1,5 @@ define site_nagios::add_service ( - $hostname, $ip_address, $openvpn_gw = '', $service) { + $hostname, $ip_address, $service, $environment, $openvpn_gw = '') { $ssh = hiera_hash('ssh') $ssh_port = $ssh['port'] @@ -11,17 +11,20 @@ define site_nagios::add_service ( use => 'generic-service', check_command => "check_ssh_port!${ssh_port}", service_description => 'SSH', - host_name => $hostname; + host_name => $hostname, + contact_groups => $environment; "${name}_cert": use => 'generic-service', check_command => 'check_https_cert', service_description => 'Website Certificate', - host_name => $hostname; + host_name => $hostname, + contact_groups => $environment; "${name}_website": use => 'generic-service', check_command => 'check_https', service_description => 'Website', - host_name => $hostname + host_name => $hostname, + contact_groups => $environment; } } default: {} diff --git a/puppet/modules/site_nagios/manifests/server.pp b/puppet/modules/site_nagios/manifests/server.pp index 37bd77fb..068ee419 100644 --- a/puppet/modules/site_nagios/manifests/server.pp +++ b/puppet/modules/site_nagios/manifests/server.pp @@ -6,12 +6,11 @@ class site_nagios::server inherits nagios::base { $nagios_hiera = hiera('nagios') $nagiosadmin_pw = htpasswd_sha1($nagios_hiera['nagiosadmin_pw']) $nagios_hosts = $nagios_hiera['hosts'] - $domains_internal = $nagios_hiera['domains_internal'] + $nagios_contacts = hiera('contacts') + $environment = $nagios_hiera['environments'] include nagios::base include nagios::defaults::commands - include nagios::defaults::contactgroups - include nagios::defaults::contacts include nagios::defaults::templates include nagios::defaults::timeperiods include nagios::defaults::plugins @@ -63,5 +62,7 @@ class site_nagios::server inherits nagios::base { 'set copytruncate copytruncate' ] } - ::site_nagios::server::hostgroup { $domains_internal: } + create_resources ( site_nagios::server::hostgroup, $environment ) + create_resources ( site_nagios::server::contactgroup, $environment ) + create_resources ( site_nagios::server::add_contacts, $environment ) } diff --git a/puppet/modules/site_nagios/manifests/server/add_contacts.pp b/puppet/modules/site_nagios/manifests/server/add_contacts.pp new file mode 100644 index 00000000..db507abf --- /dev/null +++ b/puppet/modules/site_nagios/manifests/server/add_contacts.pp @@ -0,0 +1,16 @@ +define site_nagios::server::add_contacts ($contact_emails) { + + $environment = $name + + nagios_contact { + $environment: + alias => $environment, + service_notification_period => '24x7', + host_notification_period => '24x7', + service_notification_options => 'w,u,c,r', + host_notification_options => 'd,r', + service_notification_commands => 'notify-service-by-email', + host_notification_commands => 'notify-host-by-email', + email => join($contact_emails, ', ') + } +} diff --git a/puppet/modules/site_nagios/manifests/server/contactgroup.pp b/puppet/modules/site_nagios/manifests/server/contactgroup.pp new file mode 100644 index 00000000..188c54f1 --- /dev/null +++ b/puppet/modules/site_nagios/manifests/server/contactgroup.pp @@ -0,0 +1,6 @@ +define site_nagios::server::contactgroup ($contact_emails) { + + nagios_contactgroup { $name: + members => $name + } +} diff --git a/puppet/modules/site_nagios/manifests/server/hostgroup.pp b/puppet/modules/site_nagios/manifests/server/hostgroup.pp index 035ba7d1..6f85ca6d 100644 --- a/puppet/modules/site_nagios/manifests/server/hostgroup.pp +++ b/puppet/modules/site_nagios/manifests/server/hostgroup.pp @@ -1,3 +1,3 @@ -define site_nagios::server::hostgroup { +define site_nagios::server::hostgroup ($contact_emails) { nagios_hostgroup { $name: } } -- cgit v1.2.3 From acd6dd2848f38ba1b728f263f297bc6d2ad9e4bb Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 2 Dec 2014 23:47:35 -0800 Subject: add soledad sync test to `leap test`. closes #6431 --- tests/helpers/bonafide_helper.rb | 87 ++++++++++++++++++++++++++++++++++++++++ tests/helpers/soledad_sync.py | 76 +++++++++++++++++++++++++++++++++++ tests/helpers/srp_helper.rb | 6 +-- tests/white-box/webapp.rb | 87 +++++++++++++++++++--------------------- 4 files changed, 206 insertions(+), 50 deletions(-) create mode 100644 tests/helpers/bonafide_helper.rb create mode 100755 tests/helpers/soledad_sync.py diff --git a/tests/helpers/bonafide_helper.rb b/tests/helpers/bonafide_helper.rb new file mode 100644 index 00000000..c84ea142 --- /dev/null +++ b/tests/helpers/bonafide_helper.rb @@ -0,0 +1,87 @@ +# +# helper for the communication with the provider API for creating, authenticating, and deleting accounts. +# + +class LeapTest + + def assert_tmp_user + user = assert_create_user + assert_authenticate_user(user) + yield user if block_given? + rescue + # ^^ ensure here would eat any failed assertions + assert_delete_user(user) + raise + end + + def api_url(path) + api = property('api') + "https://%{domain}:%{port}#{path}" % { + :domain => api['domain'], + :port => api['port'] + } + end + + # + # attempts to create a user account via the API, + # returning the user object if successful. + # + def assert_create_user + user = SRP::User.new + url = api_url("/1/users.json") + assert_post(url, user.to_params) do |body| + assert response = JSON.parse(body), 'response should be JSON' + assert response['ok'], 'creating a user should be successful' + end + user.ok = true + return user + end + + # + # attempts to authenticate user. if successful, + # user object is updated with id and session token. + # + def assert_authenticate_user(user) + url = api_url("/1/sessions.json") + session = SRP::Session.new(user) + params = {'login' => user.username, 'A' => session.aa} + assert_post(url, params) do |response, body| + cookie = response['Set-Cookie'].split(';').first + assert(response = JSON.parse(body), 'response should be JSON') + assert(session.bb = response["B"], 'response should include "B"') + url = api_url("/1/sessions/login.json") + params = {'client_auth' => session.m, 'A' => session.aa} + options = {:headers => {'Cookie' => cookie}} + assert_put(url, params, options) do |body| + assert(response = JSON.parse(body), 'response should be JSON') + assert(response['M2'], 'response should include M2') + user.session_token = response['token'] + user.id = response['id'] + assert(user.session_token, 'response should include token') + assert(user.id, 'response should include user id') + end + end + end + + # + # attempts to destroy a user account via the API. + # + def assert_delete_user(user) + if user && user.ok && user.id && user.session_token + url = api_url("/1/users/#{user.id}.json") + options = {:headers => { + "Authorization" => "Token token=\"#{user.session_token}\"" + }} + delete(url, {}, options) do |body, response, error| + if response.code.to_i != 200 + skip "It appears the web api is too old to support deleting users" + else + assert(response = JSON.parse(body), 'response should be JSON') + assert(response["success"], 'delete should be a success') + pass + end + end + end + end + +end diff --git a/tests/helpers/soledad_sync.py b/tests/helpers/soledad_sync.py new file mode 100755 index 00000000..54b6f34b --- /dev/null +++ b/tests/helpers/soledad_sync.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +# +# Test Soledad sync +# +# This script performs a slightly modified U1DB sync to the Soledad server and +# returns whether that sync was successful or not. +# +# It takes three arguments: +# +# uuid -- uuid of the user to sync +# token -- a valid session token +# server -- the url of the soledad server we should connect to +# +# For example: +# +# soledad_sync.py f6bef0586fcfdb8705e26a58f2d9e580 uYO-4ucEJFksJ6afjmcYwIyap2vW7bv6uLxk0w_RfCc https://199.119.112.9:2323/user-f6bef0586fcfdb8705e26a58f2d9e580 +# + +import os +import sys +import tempfile +import shutil +import u1db + +from u1db.remote.http_target import HTTPSyncTarget + +# +# monkey patch U1DB's HTTPSyncTarget to perform token based auth +# + +def set_token_credentials(self, uuid, token): + self._creds = {'token': (uuid, token)} + +def _sign_request(self, method, url_query, params): + uuid, token = self._creds['token'] + auth = '%s:%s' % (uuid, token) + return [('Authorization', 'Token %s' % auth.encode('base64')[:-1])] + +HTTPSyncTarget.set_token_credentials = set_token_credentials +HTTPSyncTarget._sign_request = _sign_request + +# +# Create a temporary local u1db replica and attempt to sync to it. +# Returns a failure message if something went wrong. +# + +def soledad_sync(uuid, token, server): + tempdir = tempfile.mkdtemp() + try: + db = u1db.open(os.path.join(tempdir, '%s.db' % uuid), True) + creds = {'token': {'uuid': uuid, 'token': token}} + db.sync(server, creds=creds, autocreate=False) + finally: + shutil.rmtree(tempdir) + +# +# exit codes: +# +# 0 - OK +# 1 - WARNING +# 2 - ERROR +# + +if __name__ == '__main__': + try: + uuid, token, server = sys.argv[1:] + result = soledad_sync(uuid, token, server) + if result is None: + exit(0) + else: + print(result) + exit(1) + except Exception as exc: + print(exc.message or str(exc)) + exit(2) diff --git a/tests/helpers/srp_helper.rb b/tests/helpers/srp_helper.rb index 9f4d7f5b..0aa1b1fb 100644 --- a/tests/helpers/srp_helper.rb +++ b/tests/helpers/srp_helper.rb @@ -135,16 +135,14 @@ d15dc7d7b46154d6b6ce8ef4ad69b15d4982559b297bcf1885c529f566660e5 class User include SRP::Util - attr_accessor :username - attr_accessor :password - attr_accessor :salt - attr_accessor :verifier + attr_accessor :username, :password, :salt, :verifier, :id, :session_token, :ok def initialize @username = "test_user_" + SecureRandom.urlsafe_base64(10).downcase.gsub(/[_-]/, '') @password = "password_" + SecureRandom.urlsafe_base64(10) @salt = bigrand(4).hex @verifier = modpow(GENERATOR, private_key) + @ok = false end def private_key diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb index 7fbab1af..d2419ab4 100644 --- a/tests/white-box/webapp.rb +++ b/tests/white-box/webapp.rb @@ -42,50 +42,23 @@ class Webapp < LeapTest end def test_05_Can_create_and_authenticate_and_delete_user_via_API? - @user = SRP::User.new - @session_token = nil - @user_id = nil - - # create user - url = api_url("/1/users.json") - assert_post(url, @user.to_params) do |body| - assert response = JSON.parse(body), 'response should be JSON' - assert response['ok'], 'creating a user should be successful' - end - - # authenticate - url = api_url("/1/sessions.json") - session = SRP::Session.new(@user) - params = {'login' => @user.username, 'A' => session.aa} - assert_post(url, params) do |response, body| - cookie = response['Set-Cookie'].split(';').first - assert(response = JSON.parse(body), 'response should be JSON') - assert(bb = response["B"]) - session.bb = bb - url = api_url("/1/sessions/login.json") - params = {'client_auth' => session.m, 'A' => session.aa} - options = {:headers => {'Cookie' => cookie}} - assert_put(url, params, options) do |body| - assert(response = JSON.parse(body), 'response should be JSON') - assert(response['M2'], 'response should include M2') - assert(@session_token = response['token'], 'response should include token') - assert(@user_id = response['id'], 'response should include user id') - end - end + assert_tmp_user + pass + end - # delete - url = api_url("/1/users/#{@user_id}.json") - options = {:headers => { - "Authorization" => "Token token=\"#{@session_token}\"" - }} - delete(url, {}, options) do |body, response, error| - if response.code.to_i != 200 - skip "It appears the web api is too old to support deleting users" - else - assert(response = JSON.parse(body), 'response should be JSON') - assert(response["success"], 'delete should be a success') + def test_06_Can_sync_Soledad? + soledad_config = property('definition_files.soledad_service') + if soledad_config && !soledad_config.empty? + soledad_server = pick_soledad_server(soledad_config) + assert_tmp_user do |user| + assert user_db_exists?(user), "Could not find user db for test user #{user.username}" + command = File.expand_path "../../helpers/soledad_sync.py", __FILE__ + soledad_url = "https://#{soledad_server}/user-#{user.id}" + assert_run "#{command} #{user.id} #{user.session_token} #{soledad_url}" pass end + else + skip 'No soledad service configuration' end end @@ -98,11 +71,33 @@ class Webapp < LeapTest } end - def api_url(path) - "https://%{domain}:%{port}#{path}" % { - :domain => property('api.domain'), - :port => property('api.port') - } + # + # pick a random soledad server. + # I am not sure why, but using IP address directly does not work. + # + def pick_soledad_server(soledad_config_json_str) + soledad_config = JSON.parse(soledad_config_json_str) + host_name = soledad_config['hosts'].keys.shuffle.first + hostname = soledad_config['hosts'][host_name]['hostname'] + port = soledad_config['hosts'][host_name]['port'] + return "#{hostname}:#{port}" + end + + # + # returns true if the per-user db created by tapicero exists. + # we try three times, and give up after that. + # + def user_db_exists?(user) + 3.times do + sleep 0.1 + get(couchdb_url("/user-#{user.id}/_design/docs")) do |body, response, error| + if response.code.to_i == 200 + return true + end + end + sleep 0.2 + end + return false end # -- cgit v1.2.3 From 049b29370e406cb424254f3be9a283f83f9e92d8 Mon Sep 17 00:00:00 2001 From: varac Date: Thu, 4 Dec 2014 13:56:41 +0100 Subject: remove webapp python tests, because they are integrated into the platform now (Bug #6489) Change-Id: Iaec748a173b6e11bb3ab3c11ca152809817644f9 --- .../modules/site_check_mk/manifests/agent/webapp.pp | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/puppet/modules/site_check_mk/manifests/agent/webapp.pp b/puppet/modules/site_check_mk/manifests/agent/webapp.pp index 64f5ea6d..88c3da30 100644 --- a/puppet/modules/site_check_mk/manifests/agent/webapp.pp +++ b/puppet/modules/site_check_mk/manifests/agent/webapp.pp @@ -1,20 +1,11 @@ class site_check_mk::agent::webapp { - # check webapp login + soledad sync - package { [ 'python-srp', 'python-requests', 'python-yaml', 'python-u1db' ]: - ensure => installed + # remove leftovers of webapp python checks + file { + [ '/usr/lib/check_mk_agent/local/nagios-webapp_login.py', + '/usr/lib/check_mk_agent/local/soledad_sync.py' ]: + ensure => absent } - file { '/usr/lib/check_mk_agent/local/nagios-webapp_login.py': - ensure => link, - target => '/srv/leap/webapp/test/nagios/webapp_login.py', - require => Package['check_mk-agent'] - } - file { '/usr/lib/check_mk_agent/local/soledad_sync.py': - ensure => link, - target => '/srv/leap/webapp/test/nagios/soledad_sync.py', - require => Package['check_mk-agent'] - } - # check syslog concat::fragment { 'syslog_webapp': -- cgit v1.2.3 From 7ca1a6feb2f881f2a99b624c266f0779d2402ff9 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 4 Dec 2014 15:16:25 -0800 Subject: tests - better errors, ensure tmp users are deleted, remove bad 'pass()' call that made tmp_user tests always succeed. --- bin/run_tests | 6 +++--- tests/helpers/bonafide_helper.rb | 22 +++++++++++----------- tests/helpers/soledad_sync.py | 2 ++ tests/helpers/srp_helper.rb | 4 +++- tests/white-box/webapp.rb | 10 ++++++---- 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/bin/run_tests b/bin/run_tests index 4addc0c8..44384379 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -49,15 +49,15 @@ end # this class is raised if a test file wants to be skipped entirely. # (to skip an individual test, MiniTest::Skip is used instead) -class SkipTest < Exception +class SkipTest < StandardError end # raised if --no-continue and there is an error -class TestError < Exception +class TestError < StandardError end # raised if --no-continue and there is a failure -class TestFailure < Exception +class TestFailure < StandardError end ## diff --git a/tests/helpers/bonafide_helper.rb b/tests/helpers/bonafide_helper.rb index c84ea142..d96b3977 100644 --- a/tests/helpers/bonafide_helper.rb +++ b/tests/helpers/bonafide_helper.rb @@ -8,10 +8,13 @@ class LeapTest user = assert_create_user assert_authenticate_user(user) yield user if block_given? - rescue - # ^^ ensure here would eat any failed assertions assert_delete_user(user) - raise + rescue StandardError, MiniTest::Assertion => exc + begin + assert_delete_user(user) + rescue + end + raise exc end def api_url(path) @@ -67,19 +70,16 @@ class LeapTest # attempts to destroy a user account via the API. # def assert_delete_user(user) - if user && user.ok && user.id && user.session_token + if user && user.ok && user.id && user.session_token && !user.deleted url = api_url("/1/users/#{user.id}.json") options = {:headers => { "Authorization" => "Token token=\"#{user.session_token}\"" }} + user.deleted = true delete(url, {}, options) do |body, response, error| - if response.code.to_i != 200 - skip "It appears the web api is too old to support deleting users" - else - assert(response = JSON.parse(body), 'response should be JSON') - assert(response["success"], 'delete should be a success') - pass - end + assert response.code.to_i == 200, "Unable to delete user: HTTP response from API should have code 200, was #{response.code} #{error} #{body}" + assert(response = JSON.parse(body), 'Delete response should be JSON') + assert(response["success"], 'Deleting user should be a success') end end end diff --git a/tests/helpers/soledad_sync.py b/tests/helpers/soledad_sync.py index 54b6f34b..2fb865fc 100755 --- a/tests/helpers/soledad_sync.py +++ b/tests/helpers/soledad_sync.py @@ -19,6 +19,7 @@ import os import sys +import traceback import tempfile import shutil import u1db @@ -73,4 +74,5 @@ if __name__ == '__main__': exit(1) except Exception as exc: print(exc.message or str(exc)) + traceback.print_exc(file=sys.stdout) exit(2) diff --git a/tests/helpers/srp_helper.rb b/tests/helpers/srp_helper.rb index 0aa1b1fb..5d30b459 100644 --- a/tests/helpers/srp_helper.rb +++ b/tests/helpers/srp_helper.rb @@ -6,6 +6,7 @@ require 'digest' require 'openssl' require 'securerandom' +require 'base64' module SRP @@ -135,7 +136,7 @@ d15dc7d7b46154d6b6ce8ef4ad69b15d4982559b297bcf1885c529f566660e5 class User include SRP::Util - attr_accessor :username, :password, :salt, :verifier, :id, :session_token, :ok + attr_accessor :username, :password, :salt, :verifier, :id, :session_token, :ok, :deleted def initialize @username = "test_user_" + SecureRandom.urlsafe_base64(10).downcase.gsub(/[_-]/, '') @@ -143,6 +144,7 @@ d15dc7d7b46154d6b6ce8ef4ad69b15d4982559b297bcf1885c529f566660e5 @salt = bigrand(4).hex @verifier = modpow(GENERATOR, private_key) @ok = false + @deleted = false end def private_key diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb index d2419ab4..d5d41833 100644 --- a/tests/white-box/webapp.rb +++ b/tests/white-box/webapp.rb @@ -51,7 +51,7 @@ class Webapp < LeapTest if soledad_config && !soledad_config.empty? soledad_server = pick_soledad_server(soledad_config) assert_tmp_user do |user| - assert user_db_exists?(user), "Could not find user db for test user #{user.username}" + assert_user_db_exists(user) command = File.expand_path "../../helpers/soledad_sync.py", __FILE__ soledad_url = "https://#{soledad_server}/user-#{user.id}" assert_run "#{command} #{user.id} #{user.session_token} #{soledad_url}" @@ -87,17 +87,19 @@ class Webapp < LeapTest # returns true if the per-user db created by tapicero exists. # we try three times, and give up after that. # - def user_db_exists?(user) + def assert_user_db_exists(user) + last_body, last_response, last_error = nil 3.times do sleep 0.1 get(couchdb_url("/user-#{user.id}/_design/docs")) do |body, response, error| + last_body, last_response, last_error = body, response, error if response.code.to_i == 200 - return true + return end end sleep 0.2 end - return false + assert false, "Could not find user db for test user #{user.username}\nuuid=#{user.id}\nHTTP #{last_response.code} #{last_error} #{last_body}" end # -- cgit v1.2.3 From e517ef53f15d24f22c31ff44eaa37601e4d5ec14 Mon Sep 17 00:00:00 2001 From: varac Date: Tue, 9 Dec 2014 09:57:44 +0100 Subject: add 'local' contactgroup to local environmet monitoring node Change-Id: I1618a8c7f2f7c905b354dbe363fc91b690725479 --- provider_base/services/monitor.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider_base/services/monitor.json b/provider_base/services/monitor.json index a68ee8d8..ff3a27a3 100644 --- a/provider_base/services/monitor.json +++ b/provider_base/services/monitor.json @@ -2,7 +2,7 @@ "nagios": { "nagiosadmin_pw": "= secret :nagios_admin_password", "domains_internal": "= global.tags.field('domain.internal_suffix').compact.uniq", - "environments": "= Hash[ global.environment_names.select{|e|e!='local'}.map{|e| [e||'default',{'contact_emails'=>global.env(e).provider.contacts.default}]} ]", + "environments": "= Hash[ nagios.hosts.values.map{|h|h['environment']}.uniq.map{|e| [e||'default',{'contact_emails'=>global.env(e).provider.contacts.default}]} ]", "hosts": "= (self.environment == 'local' ? nodes_like_me : nodes[:environment => '!local']).pick_fields('environment', 'domain.internal', 'domain.full_suffix', 'ip_address', 'services', 'openvpn.gateway_address', 'ssh.port')" }, "hosts": "= self.environment == 'local' ? hosts_file(nodes_like_me) : hosts_file(nodes[:environment => '!local'])", -- cgit v1.2.3 From dd888ede2897d56e1b7273160b99f990cd4bcf31 Mon Sep 17 00:00:00 2001 From: varac Date: Tue, 9 Dec 2014 10:07:12 +0100 Subject: logwatch: ignore postfix errors on lost connection (Bug #6476) Change-Id: I0b1eec11a3b3da39d65572b6bee8b3ce892e08ac --- puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg index 450b9e90..60881e22 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg @@ -1,7 +1,9 @@ # some general patterns + I Error: Driver 'pcspkr' is already registered, aborting... +# ignore postfix errors on lost connection (Bug #6476) + I postfix/smtpd.*SSL_accept error from.*lost connection C panic C Oops - I Error: Driver 'pcspkr' is already registered, aborting... C Error C error W generic protection rip -- cgit v1.2.3 From 5f2369f1152d63f8a1969f1826140161cc9cd241 Mon Sep 17 00:00:00 2001 From: varac Date: Tue, 9 Dec 2014 12:54:36 +0100 Subject: Deploy leap ca cert for smtp tls config (Bug #6485) Change-Id: I029ffabd33299a5b42e5f262e372eafb6272d094 --- puppet/modules/site_postfix/manifests/mx/smtp_tls.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp b/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp index d9b59f40..d56f6b54 100644 --- a/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp +++ b/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp @@ -1,5 +1,6 @@ class site_postfix::mx::smtp_tls { + include site_config::x509::ca include x509::variables $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" -- cgit v1.2.3 From 723caa71b527132bc58ed5daeacbabdbf59b7199 Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 9 Dec 2014 11:11:50 -0800 Subject: added http verb to test error messages --- tests/helpers/http_helper.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/helpers/http_helper.rb b/tests/helpers/http_helper.rb index c941ef63..0b13b754 100644 --- a/tests/helpers/http_helper.rb +++ b/tests/helpers/http_helper.rb @@ -93,9 +93,9 @@ class LeapTest yield(response, body) if block.arity == 2 end elsif response - fail ["Expected a 200 status code from #{url}, but got #{response.code} instead.", error_msg, body].compact.join("\n") + fail ["Expected a 200 status code from #{method} #{url}, but got #{response.code} instead.", error_msg, body].compact.join("\n") else - fail ["Expected a response from #{url}, but got \"#{error}\" instead.", error_msg, body].compact.join("\n"), error + fail ["Expected a response from #{method} #{url}, but got \"#{error}\" instead.", error_msg, body].compact.join("\n"), error end end end -- cgit v1.2.3 From f48d4dd084376e813cd94c3ef56c3363d83c6f85 Mon Sep 17 00:00:00 2001 From: varac Date: Tue, 9 Dec 2014 21:43:42 +0100 Subject: updated submodule check_mk in order to purge old checks from check_mk_objects.cfg (Feature #5142) Change-Id: Ib0283806b5485a9d15f0aa7e09142989367dae20 --- puppet/modules/check_mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/check_mk b/puppet/modules/check_mk index 5c11597a..205859d8 160000 --- a/puppet/modules/check_mk +++ b/puppet/modules/check_mk @@ -1 +1 @@ -Subproject commit 5c11597a055858b5ddc1ce8f7f8db249f5f1b336 +Subproject commit 205859d87884ac4ceee6d1365548e7dc55640bfa -- cgit v1.2.3 From 3c62d05813568dc389ce54754c1f4f055c899fdc Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 9 Dec 2014 16:48:53 -0500 Subject: add dependency on the stunnel service so refresh_stunnel is not run until the service has been started (#6495) Change-Id: Id48fedb5731117b68b7386c4ce22516333d94081 --- puppet/modules/stunnel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/stunnel b/puppet/modules/stunnel index ec49fd93..b0dc7c84 160000 --- a/puppet/modules/stunnel +++ b/puppet/modules/stunnel @@ -1 +1 @@ -Subproject commit ec49fd93c2469bc5c13f7e6a7d25468613e1b84f +Subproject commit b0dc7c84b5f55aec12d7d65da812037913d9dbee -- cgit v1.2.3 From 2097a686cbe4958a0224e16806ae09ff5fe13933 Mon Sep 17 00:00:00 2001 From: varac Date: Tue, 9 Dec 2014 23:31:48 +0100 Subject: Soledad sync check needs python-u1db package installed (Bug #6520) Change-Id: I8a6c27434f548f24d9dba1a969699200ab307477 --- puppet/modules/site_webapp/manifests/init.pp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/puppet/modules/site_webapp/manifests/init.pp b/puppet/modules/site_webapp/manifests/init.pp index 752993c1..9f97d2c5 100644 --- a/puppet/modules/site_webapp/manifests/init.pp +++ b/puppet/modules/site_webapp/manifests/init.pp @@ -165,6 +165,13 @@ class site_webapp { } } + + # needed for the soledad-sync check which is run on the + # webapp node (#6520) + package { 'python-u1db': + ensure => latest, + } + include site_shorewall::webapp include site_check_mk::agent::webapp } -- cgit v1.2.3 From a8334795fd393fac6e5055085fd7ab94b3b3295a Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 9 Dec 2014 21:03:02 -0500 Subject: Ignore rexi_EXIT bigcouch error (Bug #6512) Change-Id: I03842b65329aabb012cc2c7514007e174cbd8fc0 --- puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg index d274a676..5cd2a47b 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg @@ -13,6 +13,8 @@ # see https://leap.se/code/issues/5226 I Uncaught error in HTTP request: {exit,normal} I Uncaught error in HTTP request: {exit, + # Ignore rexi_EXIT bigcouch error (Bug #6512) + I Error in process <[0-9.]+> on node .* with exit value: {{rexi_EXIT,{killed,\[{couch_db,collect_results C Uncaught error in HTTP request: {error, C Response abnormally terminated: {nodedown, C rexi_DOWN,noproc -- cgit v1.2.3 From a70488de29e8721191de1f6a504dcc35e5d74770 Mon Sep 17 00:00:00 2001 From: varac Date: Wed, 10 Dec 2014 11:55:24 +0100 Subject: ignore transient Tapicero errors when creating a db (Bug #6511) Change-Id: I0939070482fad4f99f03e41094a3df42ff5063e4 --- puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg index 93ce0311..3aec4ad0 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg @@ -3,6 +3,8 @@ # instances, so we ignore it # see https://leap.se/code/issues/5168 I tapicero.*RestClient::PreconditionFailed: +# Ignore transient Tapicero errors when creating a db (#6511) + I tapicero.*Creating database user-[[:alnum:]]* failed \(trying again soon\): RestClient::InternalServerError: 500 Internal Server Error C tapicero.*Creating database.*failed due to: C tapicero.*failed W tapicero.*Couch stream ended unexpectedly. -- cgit v1.2.3 From e134a6f3b17c501df2db447bedbbb83c20f596e5 Mon Sep 17 00:00:00 2001 From: varac Date: Sun, 7 Dec 2014 13:21:35 +0100 Subject: Fix internal domain name generation (#6477) Before, under certain circumstances, the internal domain names were capped from i.e. 'dev.example.com' to 'dev.i' Change-Id: Ibd4998a7ba128dcbce5ccb9d0a52006ed4431f01 --- provider_base/provider.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider_base/provider.json b/provider_base/provider.json index 9ef0f76a..77437935 100644 --- a/provider_base/provider.json +++ b/provider_base/provider.json @@ -1,6 +1,6 @@ { "domain": "REQUIRED", - "domain_internal": "= domain.sub(/\\..*$/,'.i')", + "domain_internal": "= domain.sub(/\\.[^\\.]*$/, '.i')", "name": { "en": "REQUIRED" }, -- cgit v1.2.3 From 7871852a39bf59947b25184e6c6df365ba3b5052 Mon Sep 17 00:00:00 2001 From: varac Date: Wed, 10 Dec 2014 15:52:50 +0100 Subject: https://leap.se/code/issues/6477#note-11 Change-Id: I3094be3ef60108f4f2cad5239b0b2f288b39620d --- provider_base/services/monitor.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/provider_base/services/monitor.json b/provider_base/services/monitor.json index ff3a27a3..10d5ac81 100644 --- a/provider_base/services/monitor.json +++ b/provider_base/services/monitor.json @@ -1,9 +1,9 @@ { "nagios": { "nagiosadmin_pw": "= secret :nagios_admin_password", - "domains_internal": "= global.tags.field('domain.internal_suffix').compact.uniq", + "domains_internal": "= nagios.hosts.values.map{|h|h['domain_internal_suffix']}.uniq", "environments": "= Hash[ nagios.hosts.values.map{|h|h['environment']}.uniq.map{|e| [e||'default',{'contact_emails'=>global.env(e).provider.contacts.default}]} ]", - "hosts": "= (self.environment == 'local' ? nodes_like_me : nodes[:environment => '!local']).pick_fields('environment', 'domain.internal', 'domain.full_suffix', 'ip_address', 'services', 'openvpn.gateway_address', 'ssh.port')" + "hosts": "= (self.environment == 'local' ? nodes_like_me : nodes[:environment => '!local']).pick_fields('environment', 'domain.internal', 'domain.internal_suffix', 'domain.full_suffix', 'ip_address', 'services', 'openvpn.gateway_address', 'ssh.port')" }, "hosts": "= self.environment == 'local' ? hosts_file(nodes_like_me) : hosts_file(nodes[:environment => '!local'])", "ssh": { -- cgit v1.2.3 From 9076e8ccf62bd4cf25cd22af98d3536aff361b51 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Wed, 10 Dec 2014 10:48:10 -0500 Subject: update ffa53ef321bbfd771afff1ccb230d1b5e4f9ab00 to fix ordering requirement in logwatch, remove extended regexp character class and also ignore "Writing security" lines Change-Id: I7d33725db06a40361a3b04f9591adeb6a025bf77 --- puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg index 3aec4ad0..c998322c 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg @@ -1,10 +1,10 @@ +# Ignore transient Tapicero errors when creating a db (#6511) + I tapicero.*(Creating database|Writing security to|Uploading design doc to) user-.* failed (\(trying again soon\)|twice due to): RestClient::InternalServerError: 500 Internal Server Error C tapicero.*RestClient::InternalServerError: # possible race condition between multiple tapicero # instances, so we ignore it # see https://leap.se/code/issues/5168 I tapicero.*RestClient::PreconditionFailed: -# Ignore transient Tapicero errors when creating a db (#6511) - I tapicero.*Creating database user-[[:alnum:]]* failed \(trying again soon\): RestClient::InternalServerError: 500 Internal Server Error C tapicero.*Creating database.*failed due to: C tapicero.*failed W tapicero.*Couch stream ended unexpectedly. -- cgit v1.2.3 From c02a397a2ce13a8917d647ab849225295253e228 Mon Sep 17 00:00:00 2001 From: varac Date: Wed, 10 Dec 2014 20:22:40 +0100 Subject: Fix "invalid parameter domain_internal_suffix" on monitor node (#6477) leap_platform was modified so the nagios.internal_domains contain the domain name (with the tld replaced by an "i" for internal), see https://leap.se/code/issues/6477#note-11. in order to achieve this the easy way, each host got added a domain_internal_suffix value, which can be iterated over to get all nagios.internal_domains. Because we use `create_resources ( site_nagios::add_host_services, $nagios_hosts )` in site_nagios::server to deploy the services, the site_nagios::add_host_services define needs to have a domain_internal_suffix parameter added. Change-Id: I6b83b3f291a1a611b5b92d5ba3ed82597a42bba7 --- puppet/modules/site_nagios/manifests/add_host_services.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/puppet/modules/site_nagios/manifests/add_host_services.pp b/puppet/modules/site_nagios/manifests/add_host_services.pp index 236702e2..bd968e6f 100644 --- a/puppet/modules/site_nagios/manifests/add_host_services.pp +++ b/puppet/modules/site_nagios/manifests/add_host_services.pp @@ -1,6 +1,7 @@ define site_nagios::add_host_services ( $domain_full_suffix, $domain_internal, + $domain_internal_suffix, $ip_address, $services, $ssh_port, -- cgit v1.2.3 From 8c0fa3991f03c31fa2b4d61f20c824b86e1efa1c Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Thu, 11 Dec 2014 15:55:00 -0500 Subject: logwatch: ignore ipv6 icmp errors (Bug #6540) Change-Id: I198c5245c7e73d6dd7a7d9725fac1eb9a8f425a5 --- puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg index 60881e22..b19ac241 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg @@ -11,3 +11,5 @@ # 401 Unauthorized error logged by webapp and possible other # applications C Unauthorized +# ignore ipv6 icmp errors for now (Bug #6540) + I kernel: .*icmpv6_send: no reply to icmp error -- cgit v1.2.3 From c6104cc1ffaca59544f4b4966a640be1ebcfa662 Mon Sep 17 00:00:00 2001 From: varac Date: Thu, 11 Dec 2014 22:12:02 +0100 Subject: Increase time between two check_mk_agent runs (Bug #6539) right now, check_mk_agent is run every minute on each host. The soledad sync test depends on tapicero, and in between finishing the soledad test and removing the testuser db, and the start of another test there's only 13s Change-Id: I5b22ba02470cce799a12043d21091c0c9b8e0b5f --- puppet/modules/site_check_mk/files/extra_service_conf.mk | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/puppet/modules/site_check_mk/files/extra_service_conf.mk b/puppet/modules/site_check_mk/files/extra_service_conf.mk index 51f348f1..03d1ea76 100644 --- a/puppet/modules/site_check_mk/files/extra_service_conf.mk +++ b/puppet/modules/site_check_mk/files/extra_service_conf.mk @@ -1,3 +1,13 @@ +# retry 3 times before setting a service into a hard state +# and send out notification extra_service_conf["max_check_attempts"] = [ ("4", ALL_HOSTS , ALL_SERVICES ) ] + +# run check_mk_agent every 2 minutes if it terminates +# successfully. +# see https://leap.se/code/issues/6539 for the rationale +extra_service_conf["normal_check_interval"] = [ + ("2", ALL_HOSTS , "Check_MK" ) +] + -- cgit v1.2.3 From 0687c83543da29f86e94c46fb3fe872ab19cb709 Mon Sep 17 00:00:00 2001 From: varac Date: Thu, 11 Dec 2014 22:26:28 +0100 Subject: Increase max_check_attempts for hosts checks (Bug #6535) Change-Id: I10ec569821f329e3bd10ac87242db102e9c82246 --- puppet/modules/site_check_mk/files/extra_host_conf.mk | 6 ++++++ puppet/modules/site_check_mk/manifests/server.pp | 4 ++++ 2 files changed, 10 insertions(+) create mode 100644 puppet/modules/site_check_mk/files/extra_host_conf.mk diff --git a/puppet/modules/site_check_mk/files/extra_host_conf.mk b/puppet/modules/site_check_mk/files/extra_host_conf.mk new file mode 100644 index 00000000..2c96f97a --- /dev/null +++ b/puppet/modules/site_check_mk/files/extra_host_conf.mk @@ -0,0 +1,6 @@ +# retry 3 times before setting a host into a hard state +# and send out notification +extra_host_conf["max_check_attempts"] = [ + ("4", ALL_HOSTS ) +] + diff --git a/puppet/modules/site_check_mk/manifests/server.pp b/puppet/modules/site_check_mk/manifests/server.pp index 1a866b9c..171f1576 100644 --- a/puppet/modules/site_check_mk/manifests/server.pp +++ b/puppet/modules/site_check_mk/manifests/server.pp @@ -53,6 +53,10 @@ class site_check_mk::server { source => 'puppet:///modules/site_check_mk/extra_service_conf.mk', notify => Exec['check_mk-refresh'], require => Package['check-mk-server']; + '/etc/check_mk/conf.d/extra_host_conf.mk': + source => 'puppet:///modules/site_check_mk/extra_host_conf.mk', + notify => Exec['check_mk-refresh'], + require => Package['check-mk-server']; '/etc/check_mk/all_hosts_static': content => $all_hosts, -- cgit v1.2.3 From 68688b583bab080342e6710943230fcd46e18938 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Thu, 11 Dec 2014 19:01:16 -0500 Subject: Move kernel ipv6 log message up before the 'C error' line to it is caught (#6540) Change-Id: I1fe8d4cf60532dfe01cfb3a014c4cbeb4acdc479 --- puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg index b19ac241..b8f47434 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg @@ -5,11 +5,11 @@ C panic C Oops C Error +# ignore ipv6 icmp errors for now (Bug #6540) + I kernel: .*icmpv6_send: no reply to icmp error C error W generic protection rip W .*Unrecovered read error - auto reallocate failed # 401 Unauthorized error logged by webapp and possible other # applications C Unauthorized -# ignore ipv6 icmp errors for now (Bug #6540) - I kernel: .*icmpv6_send: no reply to icmp error -- cgit v1.2.3 From a64b2081b872823f2d539b48151bc0d323b42453 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Thu, 11 Dec 2014 21:22:49 -0500 Subject: Ignore additional tapicero message (#6542): tapicero[921]: Checking security of user-1b3b1fb78db851190fa72dac01207b8d failed (trying again soon): RestClient::ResourceNotFound: 404 Resource Not Found: {"error":"not_found","reason":"Database does not exist."}") tapicero recovers from this error Change-Id: Ic105823ddc282512000e6d7445539428581eb997 --- puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg index c998322c..d9f0eafc 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg @@ -1,5 +1,5 @@ # Ignore transient Tapicero errors when creating a db (#6511) - I tapicero.*(Creating database|Writing security to|Uploading design doc to) user-.* failed (\(trying again soon\)|twice due to): RestClient::InternalServerError: 500 Internal Server Error + I tapicero.*(Creating database|Checking security of|Writing security to|Uploading design doc to) user-.* failed (\(trying again soon\)|twice due to): (RestClient::Resource Not Found|RestClient::InternalServerError): (404 Resource Not Found|500 Internal Server Error) C tapicero.*RestClient::InternalServerError: # possible race condition between multiple tapicero # instances, so we ignore it -- cgit v1.2.3 From 8d31cbc8e52f75384e9a5c3e3630a02effe063a8 Mon Sep 17 00:00:00 2001 From: elijah Date: Sun, 14 Dec 2014 19:45:50 -0800 Subject: pin tapicero to origin/develop --- puppet/modules/tapicero/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/tapicero/manifests/init.pp b/puppet/modules/tapicero/manifests/init.pp index 28711b94..32354823 100644 --- a/puppet/modules/tapicero/manifests/init.pp +++ b/puppet/modules/tapicero/manifests/init.pp @@ -95,7 +95,7 @@ class tapicero { vcsrepo { '/srv/leap/tapicero': ensure => present, force => true, - revision => 'origin/version/0.6', + revision => 'origin/develop' provider => git, source => 'https://leap.se/git/tapicero', owner => 'tapicero', -- cgit v1.2.3 From 42ce1aa4fdeaa54058acbbd8530031ce2e516373 Mon Sep 17 00:00:00 2001 From: elijah Date: Sun, 14 Dec 2014 20:06:09 -0800 Subject: fix tapicero typo --- puppet/modules/tapicero/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/tapicero/manifests/init.pp b/puppet/modules/tapicero/manifests/init.pp index 32354823..ae5d058e 100644 --- a/puppet/modules/tapicero/manifests/init.pp +++ b/puppet/modules/tapicero/manifests/init.pp @@ -95,7 +95,7 @@ class tapicero { vcsrepo { '/srv/leap/tapicero': ensure => present, force => true, - revision => 'origin/develop' + revision => 'origin/develop', provider => git, source => 'https://leap.se/git/tapicero', owner => 'tapicero', -- cgit v1.2.3 From aedd6a57b5bf2a5d53cb19d181f90d78918ddf75 Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 15 Dec 2014 16:06:27 -0800 Subject: bugfix: identities of test users must be also destroyed. closes #6550. --- tests/helpers/bonafide_helper.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/helpers/bonafide_helper.rb b/tests/helpers/bonafide_helper.rb index d96b3977..faf8c1f4 100644 --- a/tests/helpers/bonafide_helper.rb +++ b/tests/helpers/bonafide_helper.rb @@ -75,12 +75,24 @@ class LeapTest options = {:headers => { "Authorization" => "Token token=\"#{user.session_token}\"" }} + params = { + :identities => 'destroy' + } user.deleted = true - delete(url, {}, options) do |body, response, error| + delete(url, params, options) do |body, response, error| + assert error.nil?, "Error deleting user: #{error}" assert response.code.to_i == 200, "Unable to delete user: HTTP response from API should have code 200, was #{response.code} #{error} #{body}" assert(response = JSON.parse(body), 'Delete response should be JSON') assert(response["success"], 'Deleting user should be a success') end + domain = property('domain.full_suffix') + identities_url = couchdb_url("/identities/_design/Identity/_view/by_address?key=%22#{user.username}@#{domain}%22") + get(identities_url) do |body, response, error| + assert error.nil?, "Error checking identities db: #{error}" + assert response.code.to_i == 200, "Unable to check that user identity was deleted: HTTP response from API should have code 200, was #{response.code} #{error} #{body}" + assert(response = JSON.parse(body), 'Couch response should be JSON') + assert response['rows'].empty?, "Identity should have been deleted for test user #{user.username} (id #{user.id}), but was not! Response was: #{body}." + end end end -- cgit v1.2.3 From 6df0eb2921db5ea32662036a9fe297cdfb5042ee Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Thu, 11 Dec 2014 19:01:16 -0500 Subject: Move kernel ipv6 log message up before the 'C error' line to it is caught (#6540) Change-Id: I1fe8d4cf60532dfe01cfb3a014c4cbeb4acdc479 --- puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg index b19ac241..b8f47434 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg @@ -5,11 +5,11 @@ C panic C Oops C Error +# ignore ipv6 icmp errors for now (Bug #6540) + I kernel: .*icmpv6_send: no reply to icmp error C error W generic protection rip W .*Unrecovered read error - auto reallocate failed # 401 Unauthorized error logged by webapp and possible other # applications C Unauthorized -# ignore ipv6 icmp errors for now (Bug #6540) - I kernel: .*icmpv6_send: no reply to icmp error -- cgit v1.2.3 From 970af8cf4a7670c01cd06b45a09f010d91949a2f Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 16 Dec 2014 10:56:41 -0500 Subject: ignore additional bigcouch error messages (#6512) Change-Id: Ie51fb485bcae9a9467c465bdd1b4a5785023db04 --- puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg index 5cd2a47b..3d5ada42 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg @@ -14,7 +14,7 @@ I Uncaught error in HTTP request: {exit,normal} I Uncaught error in HTTP request: {exit, # Ignore rexi_EXIT bigcouch error (Bug #6512) - I Error in process <[0-9.]+> on node .* with exit value: {{rexi_EXIT,{killed,\[{couch_db,collect_results + I Error in process <[0-9.]+> on node .* with exit value: {{rexi_EXIT,{(killed|noproc|shutdown),\[{couch_db,collect_results C Uncaught error in HTTP request: {error, C Response abnormally terminated: {nodedown, C rexi_DOWN,noproc -- cgit v1.2.3 From 8c89257f8b600d30e47f9321ef957af719b4ea21 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 16 Dec 2014 11:06:28 -0500 Subject: Ignore postfix "too many errors after DATA" logwatch msg (Bug #6545) Change-Id: I0abeb88f7b6548e5742bd3d99b2f4e5d9c6cf421 --- puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg index b19ac241..71395c50 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg @@ -2,14 +2,16 @@ I Error: Driver 'pcspkr' is already registered, aborting... # ignore postfix errors on lost connection (Bug #6476) I postfix/smtpd.*SSL_accept error from.*lost connection +# ignore postfix too many errors after DATA (#6545) + I postfix/smtpd.*too many errors after DATA from C panic C Oops C Error +# ignore ipv6 icmp errors for now (Bug #6540) + I kernel: .*icmpv6_send: no reply to icmp error C error W generic protection rip W .*Unrecovered read error - auto reallocate failed # 401 Unauthorized error logged by webapp and possible other # applications C Unauthorized -# ignore ipv6 icmp errors for now (Bug #6540) - I kernel: .*icmpv6_send: no reply to icmp error -- cgit v1.2.3 From 9e3be523477be6cf65a1ef1c83382eb7d74d4636 Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 16 Dec 2014 10:59:35 -0800 Subject: repin tapicero to version/0.6 --- puppet/modules/tapicero/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/tapicero/manifests/init.pp b/puppet/modules/tapicero/manifests/init.pp index ae5d058e..28711b94 100644 --- a/puppet/modules/tapicero/manifests/init.pp +++ b/puppet/modules/tapicero/manifests/init.pp @@ -95,7 +95,7 @@ class tapicero { vcsrepo { '/srv/leap/tapicero': ensure => present, force => true, - revision => 'origin/develop', + revision => 'origin/version/0.6', provider => git, source => 'https://leap.se/git/tapicero', owner => 'tapicero', -- cgit v1.2.3 From 8d6c05adf08eb2ddc29c4481ea06fcf85dda7b26 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 16 Dec 2014 20:00:01 -0500 Subject: Ignore "Generic server terminating" bigcouch message (Feature #6544) Change-Id: I73defd7964501e4eabe7dd05c02887e7aeb2f063 --- puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg index 5cd2a47b..434d5503 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg @@ -15,6 +15,10 @@ I Uncaught error in HTTP request: {exit, # Ignore rexi_EXIT bigcouch error (Bug #6512) I Error in process <[0-9.]+> on node .* with exit value: {{rexi_EXIT,{killed,\[{couch_db,collect_results + # Ignore "Generic server terminating" bigcouch message (Feature #6544) + I Generic server <.*> terminating + I {error_report,<.*>, + I {error_info, C Uncaught error in HTTP request: {error, C Response abnormally terminated: {nodedown, C rexi_DOWN,noproc -- cgit v1.2.3 From c16c019a3f0f1f4d5ebce9ade59a7386c3c2bb18 Mon Sep 17 00:00:00 2001 From: varac Date: Tue, 16 Dec 2014 22:06:08 +0100 Subject: Check tapicero heartbeat (Bug #6556) In order to assure tapicero is still working, we need to monitor /var/log/syslog for the last tapicero log msg, which should not be older than the last check_mk_agent run (every 2 mins atm). --- .../site_check_mk/manifests/agent/tapicero.pp | 6 ++ .../files/plugins/check_last_regex_in_log | 85 ++++++++++++++++++++++ puppet/modules/site_nagios/manifests/plugins.pp | 16 ++++ 3 files changed, 107 insertions(+) create mode 100755 puppet/modules/site_nagios/files/plugins/check_last_regex_in_log create mode 100644 puppet/modules/site_nagios/manifests/plugins.pp diff --git a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp index 369ed00b..ffd11100 100644 --- a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp +++ b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp @@ -1,5 +1,7 @@ class site_check_mk::agent::tapicero { + include ::site_nagios::plugins + concat::fragment { 'syslog_tapicero': source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/tapicero.cfg', target => '/etc/check_mk/logwatch.d/syslog.cfg', @@ -11,6 +13,10 @@ class site_check_mk::agent::tapicero { 'Tapicero_Procs': line => 'Tapicero_Procs /usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a tapicero', path => '/etc/check_mk/mrpe.cfg'; + + 'Tapicero_Heartbeat': + line => 'Tapicero_Heartbeat /usr/local/lib/nagios/plugins/check_last_regex_in_log -f /var/log/syslog -r "tapicero" -w 300 -c 600', + path => '/etc/check_mk/mrpe.cfg'; } } diff --git a/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log b/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log new file mode 100755 index 00000000..cf7c03e5 --- /dev/null +++ b/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log @@ -0,0 +1,85 @@ +#!/bin/sh +# +# depends on nagios-plugins-common for /usr/lib/nagios/plugins/utils.sh +# this package is installed using leap_platform by the Site_check_mk::Agent::Mrpe +# class + +set -e + +usage() +{ +cat << EOF +usage: $0 -w -c -r -f + +OPTIONS: + -h Show this message + -r regex to grep for + -f logfile to search in + -w warning state after X seconds + -c critical state after x seconds + +example: $0 -f /var/log/syslog -r 'tapicero' -w 300 -c 600 +EOF +} + + +. /usr/lib/nagios/plugins/utils.sh + + +warn=0 +crit=0 +log='' +regex='' + +set -- $(getopt hr:f:w:c: "$@") +while [ $# -gt 0 ] +do + case "$1" in + (-h) usage; exit 0 ;; + (-f) log="$2"; shift;; + (-r) regex="$2"; shift;; + (-w) warn="$2"; shift;; + (-c) crit="$2"; shift;; + (--) shift; break;; + (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;; + (*) break;; + esac + shift +done + +[ $warn -eq 0 -o $crit -eq 0 -o -z "$regex" -o -z "$log" ] && ( usage; exit $STATE_UNKNOWN) +[ -f "$log" ] || (echo "$log doesn't exist"; exit $STATE_UNKNOWN) + +lastmsg=$(tac $log | grep -i $regex | head -1 | cut -d' ' -f 1-3) + +if [ -z "$lastmsg" ] +then + summary="\"$regex\" in $log was not found" + state=$STATE_CRITICAL + state_text='CRITICAL' + diff_sec=0 +else + lastmsg_sec=$(date '+%s' -d "$lastmsg") + now_sec=$(date '+%s') + + diff_sec=$(($now_sec - $lastmsg_sec)) + + if [ $diff_sec -lt $warn ]; then + state=$STATE_OK + state_text='OK' + elif [ $diff_sec -lt $crit ]; then + state=$STATE_WARNING + state_text='WARNING' + else + state=$STATE_CRITICAL + state_text='CRITICAL' + fi + + summary="Last occurrence of \"$regex\" in $log was $diff_sec sec ago" +fi + +# check_mk_agent output +# echo "$state Tapicero_Heatbeat sec=$diff_sec;$warn;$crit;0; $state_text - $summary" + +echo "${state_text}: $summary | seconds=${diff_sec};$warn;$crit;0;" +exit $state diff --git a/puppet/modules/site_nagios/manifests/plugins.pp b/puppet/modules/site_nagios/manifests/plugins.pp new file mode 100644 index 00000000..90a01cfb --- /dev/null +++ b/puppet/modules/site_nagios/manifests/plugins.pp @@ -0,0 +1,16 @@ +# Deploy generic plugins useful to all nodes +# nagios::plugin won't work to deploy a plugin +# because it complains with: +# Could not find dependency Package[nagios-plugins] … +# at /srv/leap/puppet/modules/nagios/manifests/plugin.pp:18 +class site_nagios::plugins { + + file { [ + '/usr/local/lib', '/usr/local/lib/nagios', + '/usr/local/lib/nagios/plugins' ]: + ensure => directory; + '/usr/local/lib/nagios/plugins/check_last_regex_in_log': + source => 'puppet:///modules/site_nagios/plugins/check_last_regex_in_log', + mode => '0755'; + } +} -- cgit v1.2.3 From 547354d07972130fc41cc79a80085731b696f887 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Wed, 17 Dec 2014 15:21:09 -0500 Subject: Ignore Soledad "Timing out client" warning (Bug #6566) Change-Id: I6d3fa5028ba6eaca7b21a7e850136ef980f6e782 --- puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg index 623d1e46..3af5045b 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg @@ -2,4 +2,5 @@ C WSGI application error C Error C error - W Timing out client: +# Removed this line because we determined it was better to ignore it (#6566) +# W Timing out client: -- cgit v1.2.3 From 429d49824988873870911bf3425d4ca1118e8605 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Wed, 17 Dec 2014 16:04:37 -0500 Subject: Update to logwatch ignore for tapicero Change-Id: I1d8cedfeb1153312c13f7f182c7ac3b031647dd4 --- puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg index d9f0eafc..d00b8a75 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg @@ -1,5 +1,5 @@ # Ignore transient Tapicero errors when creating a db (#6511) - I tapicero.*(Creating database|Checking security of|Writing security to|Uploading design doc to) user-.* failed (\(trying again soon\)|twice due to): (RestClient::Resource Not Found|RestClient::InternalServerError): (404 Resource Not Found|500 Internal Server Error) + I tapicero.*(Creating database|Checking security of|Writing security to|Uploading design doc to) user-.* failed (\(trying again soon\)|(twice )? due to): (RestClient::Resource Not Found|RestClient::InternalServerError): (404 Resource Not Found|500 Internal Server Error) C tapicero.*RestClient::InternalServerError: # possible race condition between multiple tapicero # instances, so we ignore it -- cgit v1.2.3 From 82abc7fa7b9dd7275365d6a185b51803a34368a3 Mon Sep 17 00:00:00 2001 From: varac Date: Wed, 17 Dec 2014 22:17:08 +0100 Subject: Check_mk logwatch: ignore openvpn warnings (Feature #6568) Change-Id: I0d30afbcc6dcb90c6716f7c6bb0bca3e6ae0964a --- .../modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg index d58e876d..ac17c0ca 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg @@ -2,6 +2,12 @@ # suddenly hangup before properly establishing # a tls connection I ovpn-.*TLS Error: Unroutable control packet received from - I ovpn-.*TLS Error: TLS key negotiation failed to occur within 60 seconds (check your network connectivity) + I ovpn-.*TLS Error: TLS key negotiation failed to occur within 60 seconds \(check your network connectivity\) I ovpn-.*TLS Error: TLS handshake failed + I ovpn-.*TLS Error: TLS object -> incoming plaintext read error + I ovpn-.*Fatal TLS error \(check_tls_errors_co\), restarting + I ovpn-.*TLS_ERROR: BIO read tls_read_plaintext error: error:140890B2:SSL routines:SSL3_GET_CLIENT_CERTIFICATE:no certificate + + I ovpn-.*SIGUSR1\[soft,tls-error\] received, client-instance restarting + I ovpn-.*VERIFY ERROR: depth=0, error=certificate has expired -- cgit v1.2.3 From b49fc6c4a864c6e42a4fd4deae25c0ab2eb564ea Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 17 Dec 2014 16:22:29 -0800 Subject: tests: don't run soledad test if there are no soledad nodes --- tests/helpers/bonafide_helper.rb | 2 +- tests/white-box/webapp.rb | 24 +++++++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/tests/helpers/bonafide_helper.rb b/tests/helpers/bonafide_helper.rb index faf8c1f4..9b26eaaf 100644 --- a/tests/helpers/bonafide_helper.rb +++ b/tests/helpers/bonafide_helper.rb @@ -34,7 +34,7 @@ class LeapTest url = api_url("/1/users.json") assert_post(url, user.to_params) do |body| assert response = JSON.parse(body), 'response should be JSON' - assert response['ok'], 'creating a user should be successful' + assert response['ok'], "Creating a user should be successful, got #{response.inspect} instead." end user.ok = true return user diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb index d5d41833..9f104899 100644 --- a/tests/white-box/webapp.rb +++ b/tests/white-box/webapp.rb @@ -50,12 +50,14 @@ class Webapp < LeapTest soledad_config = property('definition_files.soledad_service') if soledad_config && !soledad_config.empty? soledad_server = pick_soledad_server(soledad_config) - assert_tmp_user do |user| - assert_user_db_exists(user) - command = File.expand_path "../../helpers/soledad_sync.py", __FILE__ - soledad_url = "https://#{soledad_server}/user-#{user.id}" - assert_run "#{command} #{user.id} #{user.session_token} #{soledad_url}" - pass + if soledad_server + assert_tmp_user do |user| + assert_user_db_exists(user) + command = File.expand_path "../../helpers/soledad_sync.py", __FILE__ + soledad_url = "https://#{soledad_server}/user-#{user.id}" + assert_run "#{command} #{user.id} #{user.session_token} #{soledad_url}" + pass + end end else skip 'No soledad service configuration' @@ -78,9 +80,13 @@ class Webapp < LeapTest def pick_soledad_server(soledad_config_json_str) soledad_config = JSON.parse(soledad_config_json_str) host_name = soledad_config['hosts'].keys.shuffle.first - hostname = soledad_config['hosts'][host_name]['hostname'] - port = soledad_config['hosts'][host_name]['port'] - return "#{hostname}:#{port}" + if host_name + hostname = soledad_config['hosts'][host_name]['hostname'] + port = soledad_config['hosts'][host_name]['port'] + return "#{hostname}:#{port}" + else + return nil + end end # -- cgit v1.2.3 From d024a172ee4b2dc7fca4fe4251c930ff437f1f1b Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Thu, 18 Dec 2014 14:07:04 -0500 Subject: update tapicero logwatch messages to remove extra space Change-Id: I0149ac2e767531d9724b57b9e3bdae7943f954ff --- puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg index d00b8a75..e5721eea 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg @@ -1,5 +1,5 @@ # Ignore transient Tapicero errors when creating a db (#6511) - I tapicero.*(Creating database|Checking security of|Writing security to|Uploading design doc to) user-.* failed (\(trying again soon\)|(twice )? due to): (RestClient::Resource Not Found|RestClient::InternalServerError): (404 Resource Not Found|500 Internal Server Error) + I tapicero.*(Creating database|Checking security of|Writing security to|Uploading design doc to) user-.* failed (\(trying again soon\)|(twice )?due to): (RestClient::Resource Not Found|RestClient::InternalServerError): (404 Resource Not Found|500 Internal Server Error) C tapicero.*RestClient::InternalServerError: # possible race condition between multiple tapicero # instances, so we ignore it -- cgit v1.2.3 From 784cc8dcc03f63587206d62b5d49843eb974dbe1 Mon Sep 17 00:00:00 2001 From: guido Date: Thu, 18 Dec 2014 19:37:51 -0300 Subject: Install icli package and configure ncli aliases (solves #6475) Change-Id: I0f9397593bc4f00b64b626a159be09ab5ef694d6 --- puppet/modules/site_nagios/manifests/server.pp | 1 + .../modules/site_nagios/manifests/server/icli.pp | 26 ++++++++++++++++++++++ .../modules/site_nagios/templates/icli_aliases.erb | 7 ++++++ 3 files changed, 34 insertions(+) create mode 100644 puppet/modules/site_nagios/manifests/server/icli.pp create mode 100644 puppet/modules/site_nagios/templates/icli_aliases.erb diff --git a/puppet/modules/site_nagios/manifests/server.pp b/puppet/modules/site_nagios/manifests/server.pp index 068ee419..092ca503 100644 --- a/puppet/modules/site_nagios/manifests/server.pp +++ b/puppet/modules/site_nagios/manifests/server.pp @@ -52,6 +52,7 @@ class site_nagios::server inherits nagios::base { include site_nagios::server::apache include site_check_mk::server include site_shorewall::monitor + include site_nagios::server::icli augeas { 'logrotate_nagios': diff --git a/puppet/modules/site_nagios/manifests/server/icli.pp b/puppet/modules/site_nagios/manifests/server/icli.pp new file mode 100644 index 00000000..26fba725 --- /dev/null +++ b/puppet/modules/site_nagios/manifests/server/icli.pp @@ -0,0 +1,26 @@ +# Install icli package and configure ncli aliases +class site_nagios::server::icli { + $nagios_hiera = hiera('nagios') + $environments = $nagios_hiera['environments'] + + package { 'icli': + ensure => installed; + } + + file { '/root/.bashrc': + ensure => present; + } + + file_line { 'icli aliases': + path => '/root/.bashrc', + line => 'source /root/.icli_aliases'; + } + + file { '/root/.icli_aliases': + content => template("${module_name}/icli_aliases.erb"), + mode => '0644', + owner => root, + group => 0, + require => Package['icli']; + } +} \ No newline at end of file diff --git a/puppet/modules/site_nagios/templates/icli_aliases.erb b/puppet/modules/site_nagios/templates/icli_aliases.erb new file mode 100644 index 00000000..f1428f9e --- /dev/null +++ b/puppet/modules/site_nagios/templates/icli_aliases.erb @@ -0,0 +1,7 @@ +alias ncli='icli -c /var/cache/nagios3/objects.cache -f /var/cache/nagios3/status.dat -F /var/lib/nagios3/rw/nagios.cmd' +alias ncli_problems='ncli -z '!o,!A'' + +<% @environments.keys.sort.each do |env_name| %> +alias ncli_<%= env_name %>='ncli -z '!o,!A' -g <%= env_name %>' +alias ncli_<%= env_name %>_recheck='ncli -s Check_MK -g <%= env_name %> -r' +<% end -%> \ No newline at end of file -- cgit v1.2.3 From 6ccce65c46f36dba8f8d572e31558b6963d48b41 Mon Sep 17 00:00:00 2001 From: guido Date: Fri, 19 Dec 2014 12:51:40 -0300 Subject: Add x509 files to static node allowing postfix to work (solves #6577) + minor lint. https://leap.se/code/issues/6577 Change-Id: Iefefbf3e8fc5c13cdd7e302627504a76b856e725 --- puppet/modules/site_static/manifests/init.pp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/puppet/modules/site_static/manifests/init.pp b/puppet/modules/site_static/manifests/init.pp index 6e347d35..aed9775e 100644 --- a/puppet/modules/site_static/manifests/init.pp +++ b/puppet/modules/site_static/manifests/init.pp @@ -1,5 +1,10 @@ class site_static { tag 'leap_service' + + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca_bundle + $static = hiera('static') $domains = $static['domains'] $formats = $static['formats'] @@ -33,7 +38,7 @@ class site_static { include site_apt::preferences::passenger class { 'passenger': use_munin => false, - require => Class['site_apt::preferences::passenger'] + require => Class['site_apt::preferences::passenger'] } } -- cgit v1.2.3 From 1f9f5f11cbf5ee3e9929203ceb2053f5ca164f5a Mon Sep 17 00:00:00 2001 From: elijah Date: Sun, 21 Dec 2014 16:45:17 -0800 Subject: fixed bug that prevented all users/*/*_ssh.pub keys from getting added to authorized_keys --- provider_base/lib/macros/core.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/provider_base/lib/macros/core.rb b/provider_base/lib/macros/core.rb index 2ab2e71b..7de50f2f 100644 --- a/provider_base/lib/macros/core.rb +++ b/provider_base/lib/macros/core.rb @@ -22,6 +22,9 @@ module LeapCli keys.sort.each do |keyfile| ssh_type, ssh_key = File.read(keyfile, :encoding => 'UTF-8').strip.split(" ") name = File.basename(File.dirname(keyfile)) + until hash[name].nil? + i ||= 1; name = "#{name}#{i+=1}" + end hash[name] = { "type" => ssh_type, "key" => ssh_key -- cgit v1.2.3 From 9eb4c55ef0f477afd9c0d74aff62c2bb74c16e8e Mon Sep 17 00:00:00 2001 From: elijah Date: Sun, 21 Dec 2014 20:53:38 -0800 Subject: correctly generate .onion addresses. closes #6559 --- provider_base/lib/macros/keys.rb | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/provider_base/lib/macros/keys.rb b/provider_base/lib/macros/keys.rb index ea4c3df2..0ed7ccd0 100644 --- a/provider_base/lib/macros/keys.rb +++ b/provider_base/lib/macros/keys.rb @@ -36,14 +36,15 @@ module LeapCli end # - # on the command line an onion address can be created - # from an rsa public key using this: + # Generates a onion_address from a public RSA key file. # - # base64 -d < ./pubkey | sha1sum | awk '{print $1}' | - # perl -e '$l=<>; chomp $l; print pack("H*", $l)' | - # python -c 'import base64, sys; t=sys.stdin.read(); print base64.b32encode(t[:10]).lower()' + # path_name is the named path of the Tor public key. # - # path_name is the named path of the tor public key. + # Basically, an onion address is nothing more than a base32 encoding + # of the first 10 bytes of a sha1 digest of the public key. + # + # Additionally, Tor ignores the 22 byte header of the public key + # before taking the sha1 digest. # def onion_address(path_name) require 'base32' @@ -53,9 +54,9 @@ module LeapCli if path && File.exists?(path) public_key_str = File.readlines(path).grep(/^[^-]/).join public_key = Base64.decode64(public_key_str) - sha1sum_string = Digest::SHA1.new.hexdigest(public_key) - sha1sum_binary = [sha1sum_string].pack('H*') - Base32.encode(sha1sum_binary.slice(0,10)).downcase + public_key = public_key.slice(22..-1) # Tor ignores the 22 byte SPKI header + sha1sum = Digest::SHA1.new.digest(public_key) + Base32.encode(sha1sum.slice(0,10)).downcase else LeapCli.log :warning, 'Tor public key file "%s" does not exist' % tor_public_key_path end -- cgit v1.2.3 From fc9a8af17d927085486052a53233401c42b0caab Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 23 Dec 2014 16:33:49 -0500 Subject: update doc directory with current state from leap_doc Change-Id: I5b427b0e875b0e640051227cd1cdd51a52c72ac5 --- doc/commands.md | 285 ------------------------ doc/config.md | 229 ------------------- doc/details/couchdb.md | 57 +++++ doc/details/development.md | 314 ++++++++++++++++++++++++++ doc/details/en.haml | 5 + doc/details/faq.md | 65 ++++++ doc/details/under-the-hood.md | 26 +++ doc/details/webapp.md | 282 +++++++++++++++++++++++ doc/development.md | 272 ----------------------- doc/en.md | 2 +- doc/faq.md | 53 ----- doc/guide.md | 266 ---------------------- doc/guide/commands.md | 419 +++++++++++++++++++++++++++++++++++ doc/guide/config.md | 263 ++++++++++++++++++++++ doc/guide/en.haml | 6 + doc/guide/keys-and-certificates.md | 176 +++++++++++++++ doc/guide/miscellaneous.md | 14 ++ doc/guide/nodes.md | 169 ++++++++++++++ doc/known-issues.md | 64 ------ doc/quick-start.md | 397 --------------------------------- doc/troubleshooting.md | 147 ------------ doc/troubleshooting/en.haml | 5 + doc/troubleshooting/known-issues.md | 77 +++++++ doc/troubleshooting/tests.md | 33 +++ doc/troubleshooting/where-to-look.md | 249 +++++++++++++++++++++ doc/tutorials/en.haml | 5 + doc/tutorials/quick-start.md | 389 ++++++++++++++++++++++++++++++++ doc/tutorials/single-node.md | 343 ++++++++++++++++++++++++++++ doc/under-the-hood.md | 37 ---- 29 files changed, 2898 insertions(+), 1751 deletions(-) delete mode 100644 doc/commands.md delete mode 100644 doc/config.md create mode 100644 doc/details/couchdb.md create mode 100644 doc/details/development.md create mode 100644 doc/details/en.haml create mode 100644 doc/details/faq.md create mode 100644 doc/details/under-the-hood.md create mode 100644 doc/details/webapp.md delete mode 100644 doc/development.md delete mode 100644 doc/faq.md delete mode 100644 doc/guide.md create mode 100644 doc/guide/commands.md create mode 100644 doc/guide/config.md create mode 100644 doc/guide/en.haml create mode 100644 doc/guide/keys-and-certificates.md create mode 100644 doc/guide/miscellaneous.md create mode 100644 doc/guide/nodes.md delete mode 100644 doc/known-issues.md delete mode 100644 doc/quick-start.md delete mode 100644 doc/troubleshooting.md create mode 100644 doc/troubleshooting/en.haml create mode 100644 doc/troubleshooting/known-issues.md create mode 100644 doc/troubleshooting/tests.md create mode 100644 doc/troubleshooting/where-to-look.md create mode 100644 doc/tutorials/en.haml create mode 100644 doc/tutorials/quick-start.md create mode 100644 doc/tutorials/single-node.md delete mode 100644 doc/under-the-hood.md diff --git a/doc/commands.md b/doc/commands.md deleted file mode 100644 index b176541f..00000000 --- a/doc/commands.md +++ /dev/null @@ -1,285 +0,0 @@ -@title = 'Command Line Reference' - -The command "leap" can be used to manage a bevy of servers running the LEAP platform from the comfort of your own home. - - -# Global Options - -* `--log FILE` -Override default log file -Default Value: None - -* `-v|--verbose LEVEL` -Verbosity level 0..2 -Default Value: 1 - -* `--help` -Show this message - -* `--version` -Display version number and exit - -* `--yes` -Skip prompts and assume "yes" - - -# leap add-user USERNAME - -Adds a new trusted sysadmin - - - -**Options** - -* `--pgp-pub-key arg` -OpenPGP public key file for this new user -Default Value: None - -* `--ssh-pub-key arg` -SSH public key file for this new user -Default Value: None - -* `--self` -lets you choose among your public keys - - -# leap cert - -Manage X.509 certificates - - - -## leap cert ca - -Creates two Certificate Authorities (one for validating servers and one for validating clients). - -See see what values are used in the generation of the certificates (like name and key size), run `leap inspect provider` and look for the "ca" property. To see the details of the created certs, run `leap inspect `. - -## leap cert csr - -Creates a CSR for use in buying a commercial X.509 certificate. - -The CSR created is for the for the provider's primary domain. The properties used for this CSR come from `provider.ca.server_certificates`. - -## leap cert dh - -Creates a Diffie-Hellman parameter file. - - - -## leap cert update - -Creates or renews a X.509 certificate/key pair for a single node or all nodes, but only if needed. - -This command will a generate new certificate for a node if some value in the node has changed that is included in the certificate (like hostname or IP address), or if the old certificate will be expiring soon. Sometimes, you might want to force the generation of a new certificate, such as in the cases where you have changed a CA parameter for server certificates, like bit size or digest hash. In this case, use --force. If is empty, this command will apply to all nodes. - -**Options** - -* `--force` -Always generate new certificates - - -# leap clean - -Removes all files generated with the "compile" command. - - - -# leap compile - -Compiles node configuration files into hiera files used for deployment. - - - -# leap deploy FILTER - -Apply recipes to a node or set of nodes. - -The FILTER can be the name of a node, service, or tag. - -**Options** - -* `--tags TAG[,TAG]` -Specify tags to pass through to puppet (overriding the default). -Default Value: leap_base,leap_service - -* `--fast` -Makes the deploy command faster by skipping some slow steps. A "fast" deploy can be used safely if you recently completed a normal deploy. - - -# leap help command - -Shows a list of commands or help for one command - -Gets help for the application or its commands. Can also list the commands in a way helpful to creating a bash-style completion function - -**Options** - -* `-c` -List commands one per line, to assist with shell completion - - -# leap inspect FILE - -Prints details about a file. Alternately, the argument FILE can be the name of a node, service or tag. - - - -# leap list [FILTER] - -List nodes and their classifications - -Prints out a listing of nodes, services, or tags. If present, the FILTER can be a list of names of nodes, services, or tags. If the name is prefixed with +, this acts like an AND condition. For example: - -`leap list node1 node2` matches all nodes named "node1" OR "node2" - -`leap list openvpn +local` matches all nodes with service "openvpn" AND tag "local" - -**Options** - -* `--print arg` -What attributes to print (optional) -Default Value: None - - -# leap local - -Manage local virtual machines. - -This command provides a convient way to manage Vagrant-based virtual machines. If FILTER argument is missing, the command runs on all local virtual machines. The Vagrantfile is automatically generated in 'test/Vagrantfile'. If you want to run vagrant commands manually, cd to 'test'. - -## leap local destroy [FILTER] - -Destroys the virtual machine(s), reclaiming the disk space - - - -## leap local reset [FILTER] - -Resets virtual machine(s) to the last saved snapshot - - - -## leap local save [FILTER] - -Saves the current state of the virtual machine as a new snapshot - - - -## leap local start [FILTER] - -Starts up the virtual machine(s) - - - -## leap local status [FILTER] - -Print the status of local virtual machine(s) - - - -## leap local stop [FILTER] - -Shuts down the virtual machine(s) - - - -# leap new DIRECTORY - -Creates a new provider instance in the specified directory, creating it if necessary. - - - -**Options** - -* `--contacts arg` -Default email address contacts. -Default Value: None - -* `--domain arg` -The primary domain of the provider. -Default Value: None - -* `--name arg` -The name of the provider. -Default Value: None - -* `--platform arg` -File path of the leap_platform directory. -Default Value: None - - -# leap node - -Node management - - - -## leap node add NAME [SEED] - -Create a new configuration file for a node named NAME. - -If specified, the optional argument SEED can be used to seed values in the node configuration file. - -The format is property_name:value. - -For example: `leap node add web1 ip_address:1.2.3.4 services:webapp`. - -To set nested properties, property name can contain '.', like so: `leap node add web1 ssh.port:44` - -Separeate multiple values for a single property with a comma, like so: `leap node add mynode services:webapp,dns` - -**Options** - -* `--local` -Make a local testing node (by automatically assigning the next available local IP address). Local nodes are run as virtual machines on your computer. - - -## leap node init FILTER - -Bootstraps a node or nodes, setting up SSH keys and installing prerequisite packages - -This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, copying the authorized_keys file, and installing packages that are required for deploying. Node init must be run before deploying to a server, and the server must be running and available via the network. This command only needs to be run once, but there is no harm in running it multiple times. - -**Options** - -* `--echo` -If set, passwords are visible as you type them (default is hidden) - - -## leap node mv OLD_NAME NEW_NAME - -Renames a node file, and all its related files. - - - -## leap node rm NAME - -Removes all the files related to the node named NAME. - - - -# leap ssh NAME - -Log in to the specified node with an interactive shell. - - - -# leap test - -Run tests. - - - -## leap test init - -Creates files needed to run tests. - - - -## leap test run - -Run tests. - - -Default Command: run diff --git a/doc/config.md b/doc/config.md deleted file mode 100644 index d0b1f6a7..00000000 --- a/doc/config.md +++ /dev/null @@ -1,229 +0,0 @@ -@title = "Configuration Files" - -Leapfile -------------------------------------------- - -A `Leapfile` defines options for the `leap` command and lives at the root of your provider directory. `Leapfile` is evaluated as ruby, so you can include whatever weird logic you want in this file. In particular, there are several variables you can set that modify the behavior of leap. For example: - - @platform_directory_path = '../leap_platform' - @log = '/var/log/leap.log' - -Additionally, you can create a `~/.leaprc` file that is loaded after `Leapfile` and is evaluated the same way. - -Platform options: - -* `@platform_directory_path` (required). This must be set to the path where `leap_platform` lives. The path may be relative. -* `@platform_branch`. If set, a check is preformed before running any command to ensure that the currently checked out branch of `leap_platform` matches the value set for `@platform_branch`. This is useful if you have a stable branch of your provider that you want to ensure runs off the master branch of `leap_platform`. -* `@allow_production_deploy`. By default, you can only deploy to production nodes if the current branch is 'master' or if the provider directory is not a git repository. This option allows you to override this behavior. - -Vagrant options: - -* `@vagrant_network`. Allows you to override the default network used for local nodes. It should include a netmask like `@vagrant_network = '10.0.0.0/24'`. -* `@custom_vagrant_vm_line`. Insert arbitrary text into the auto-generated Vagrantfile. For example, `@custom_vagrant_vm_line = "config.vm.boot_mode = :gui"`. - -Logging options: - -* `@log`. If set, all command invocation and results are logged to the specified file. This is the same as the switch `--log FILE`, except that the command line switch will override the value in the Leapfile. - - -Configuration files -------------------------------------------- - -All configuration files, other than `Leapfile`, are in the JSON format. For example: - - { - "key1": "value1", - "key2": "value2" - } - -Keys should match `/[a-z0-9_]/` - -Unlike traditional JSON, comments are allowed. If the first non-whitespace characters are `//` then the line is treated as a comment. - - // this is a comment - { - // this is a comment - "key": "value" // this is an error - } - -Options in the configuration files might be nested hashes, arrays, numbers, strings, or boolean. Numbers and boolean values should **not** be quoted. For example: - - { - "openvpn": { - "ip_address": "1.1.1.1", - "protocols": ["tcp", "udp"], - "ports": [80, 53], - "options": { - "public_ip": false, - "adblock": true - } - } - } - -If the value string is prefixed with an '=' character, the result is evaluated as ruby. For example: - - { - "domain": { - "public": "domain.org" - } - "api_domain": "= 'api.' + domain.public" - } - -In this case, the property "api_domain" will be set to "api.domain.org". So long as you do not create unresolvable circular dependencies, you can reference other properties in evaluated ruby that are themselves evaluated ruby. - -See "Macros" below for information on the special macros available to the evaluated ruby. - -TIP: In rare cases, you might want to force the evaluation of a value to happen in a later pass after most of the other properties have been evaluated. To do this, prefix the value string with "=>" instead of "=". - -Node inheritance ----------------------------------------- - -Every node inherits from common.json and also any of the services or tags attached to the node. Additionally, the `leap_platform` contains a directory `provider_base` that defines the default values for tags, services and common.json. - -Suppose you have a node configuration for `bitmask/nodes/willamette.json` like so: - - { - "services": "webapp", - "tags": ["production", "northwest-us"], - "ip_address": "1.1.1.1" - } - -This node will have hostname "willamette" and it will inherit from the following files (in this order): - -1. common.json - - load defaults: `provider_base/common.json` - - load provider: `bitmask/common.json` -2. service "webapp" - - load defaults: `provider_base/services/webapp.json` - - load provider: `bitmask/services/webapp.json` -3. tag "production" - - load defaults: `provider_base/tags/production.json` - - load provider: `bitmask/tags/production.json` -4. tag "northwest-us" - - load: `bitmask/tags/northwest-us.json` -5. finally, load node "willamette" - - load: `bitmask/nodes/willamette.json` - -The `provider_base` directory is under the `leap_platform` specified in the file `Leapfile`. - -To see all the variables a node has inherited, you could run `leap inspect willamette`. - -Common configuration options ----------------------------------------- - -You can use the command `leap inspect` to see what options are available for a provider, node, service, or tag configuration. For example: - -* `leap inspect common` -- show the options inherited by all nodes. -* `leap inspect --base common` -- show the common.json from `provider_base` without the local `common.json` inheritance applied. -* `leap inspect webapp` -- show all the options available for the service `webapp`. - -Here are some of the more important options you should be aware of: - -* `ip_address` -- Required for all nodes, no default. -* `ssh.port` -- The SSH port you want the node's OpenSSH server to bind to. This is also the default when trying to connect to a node, but if the node currently has OpenSSH running on a different port then run deploy with `--port` to override the `ssh.port` configuration value. -* `mosh.enabled` -- If set to `true`, then mosh will be installed on the server. The default is `false`. - -Macros ----------------------------------------- - -When using evaluated ruby in a JSON configuration file, there are several special macros that are available. These are evaluated in the context of a node (available as the variable `self`). - -The following methods are available to the evaluated ruby: - -`variable.variable` - - > Any variable defined or inherited by a particular node configuration is available by just referencing it using either hash notation or object field notation (e.g. `['domain']['public']` or `domain.public`). Circular references are not allowed, but otherwise it is OK to nest evaluated values in other evaluated values. If a value has not been defined, the hash notation will return nil but the field notation will raise an exception. Properties of services, tags, and the global provider can all be referenced the same way. For example, `global.services['openvpn'].x509.dh`. - -`nodes` - - > A hash of all nodes. This list can be filtered. - -`nodes_like_me` - - > A hash of nodes that have the same deployment tags as the current node (e.g. 'production' or 'local'). - -`global.services` - - > A hash of all services, e.g. `global.services['openvpn']` would return the "openvpn" service. - -`global.tags` - - > A hash of all tags, e.g. `global.tags['production']` would return the "production" tag. - - `global.provider` - - > Can be used to access variables defined in `provider.json`, e.g. `global.provider.contacts.default`. - -`file(filename)` - - > Inserts the full contents of the file. If the file is an erb template, it is rendered. The filename can either be one of the pre-defined file symbols, or it can be a path relative to the "files" directory in your provider instance. E.g, `file :ca_cert` or `files 'ca/ca.crt'`. - -`file_path(filename)` - - > Ensures that the file will get rsynced to the node as an individual file. The value returned by `file_path` is the full path where this file will ultimately live when deploy to the node. e.g. `file_path :ca_cert` or `file_path 'branding/images/logo.png'`. - -`secret(:symbol)` - - > Returns the value of a secret in secrets.json (or creates it if necessary). E.g. `secret :couch_admin_password` - -`hosts_file` - - > Returns a data structure that puppet will use to generate /etc/hosts. Care is taken to use the local IP of other hosts when needed. - -`known_hosts_file` - - > Returns the lines needed in a SSH `known_hosts` file. - -`stunnel_client(node_list, port, options={})` - - > Returns a stunnel configuration data structure for the client side. Argument `node_list` is an `ObjectList` of nodes running stunnel servers. Argument `port` is the real port of the ultimate service running on the servers that the client wants to connect to. - -`stunnel_server(port)` - - > Generates a stunnel server entry. The `port` is the real port targeted service. - -Hash tables ------------------------------------------ - -The macros `nodes`, `nodes_like_me`, `global.services`, and `global.tags` all return a hash table of configuration objects (either nodes, services, or tags). There are several ways to filter and process these hash tables: - -Access an element by name: - - nodes['vpn1'] # returns node named 'vpn1' - global.services['openvpn'] # returns service named 'openvpn' - -Create a new hash table by applying filters: - - nodes[:public_dns => true] # all nodes where public_dns == true - nodes[:services => 'openvpn', :services => 'tor'] # openvpn OR tor - nodes[:services => 'openvpn'][:tags => 'production'] # openvpn AND production - nodes[:name => "!bob"] # all nodes that are NOT named "bob" - -Create an array of values by selecting a single field: - - nodes.field('location.name') - ==> ['seattle', 'istanbul'] - -Create an array of hashes by selecting multiple fields: - - nodes.fields('domain.full', 'ip_address') - ==> [ - {'domain_full' => 'red.bitmask.net', 'ip_address' => '1.1.1.1'}, - {'domain_full' => 'blue.bitmask.net', 'ip_address' => '1.1.1.2'}, - ] - -Create a new hash table of hashes, with only certain fields: - - nodes.pick_fields('domain.full', 'ip_address') - ==> { - "red" => {'domain_full' => 'red.bitmask.net', 'ip_address' => '1.1.1.1'}, - "blue => {'domain_full' => 'blue.bitmask.net', 'ip_address' => '1.1.1.2'}, - } - -With `pick_fields`, if there is only one field, it will generate a simple hash table: - - nodes.pick_fields('ip_address') - ==> { - "red" => '1.1.1.1', - "blue => '1.1.1.2', - } diff --git a/doc/details/couchdb.md b/doc/details/couchdb.md new file mode 100644 index 00000000..afecf169 --- /dev/null +++ b/doc/details/couchdb.md @@ -0,0 +1,57 @@ +@title = "CouchDB" + +Rebalance Cluster +================= + +Bigcouch currently does not have automatic rebalancing. +It will probably be added after merging into couchdb. +If you add a node, or remove one node from the cluster, + +. make sure you have a backup of all DBs ! + + /srv/leap/couchdb/scripts/couchdb_dumpall.sh + + +. delete all dbs +. shut down old node +. check the couchdb members + + curl -s —netrc-file /etc/couchdb/couchdb.netrc -X GET http://127.0.0.1:5986/nodes/_all_docs + curl -s —netrc-file /etc/couchdb/couchdb.netrc http://127.0.0.1:5984/_membership + + +. remove bigcouch from all nodes + + apt-get --purge remove bigcouch + + +. deploy to all couch nodes + + leap deploy development +couchdb + +. most likely, deploy will fail because bigcouch will complain about not all nodes beeing connected. Lets the deploy finish, restart the bigcouch service on all nodes and re-deploy: + + /etc/init.d/bigcouch restart + + +. restore the backup + + /srv/leap/couchdb/scripts/couchdb_restoreall.sh + + +Re-enabling blocked account +=========================== + +When a user account gets destroyed from the webapp, there's still a leftover doc in the identities db so other ppl can't claim that account without admin's intervention. Here's how you delete that doc and therefore enable registration for that particular account again: + +. grep the identities db for the email address: + + curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET http://127.0.0.1:5984/identities/_all_docs?include_docs=true|grep test_127@bitmask.net + + +. lookup "id" and "rev" to delete the doc: + + curl -s --netrc-file /etc/couchdb/couchdb.netrc -X DELETE 'http://127.0.0.1:5984/identities/b25cf10f935b58088f0d547fca823265?rev=2-715a9beba597a2ab01851676f12c3e4a' + + + diff --git a/doc/details/development.md b/doc/details/development.md new file mode 100644 index 00000000..97f207ce --- /dev/null +++ b/doc/details/development.md @@ -0,0 +1,314 @@ +@title = "Development Environment" +@summary = "Setting up an environment for modifying the leap_platform." +@toc = true + +If you are wanting to make local changes to your provider, or want to contribute some fixes back to LEAP, we recommend that you follow this guide to build up a development environment to test your changes first. Using this method, you can quickly test your changes without deploying them to your production environment, while benefitting from the convenience of reverting to known good states in order to retry things from scratch. + +This page will walk you through setting up nodes using [Vagrant](http://www.vagrantup.com/) for convenient deployment testing, snapshotting known good states, and reverting to previous snapshots. + +Requirements +============ + +* Be a real machine with virtualization support in the CPU (VT-x or AMD-V). In other words, not a virtual machine. +* Have at least 4gb of RAM. +* Have a fast internet connection (because you will be downloading a lot of big files, like virtual machine images). +* You should do everything described below as an unprivileged user, and only run those commands as root that are noted with *sudo* in front of them. Other than those commands, there is no need for privileged access to your machine, and in fact things may not work correctly. + +Install prerequisites +-------------------------------- + +For development purposes, you will need everything that you need for deploying the LEAP platform: + +* LEAP cli +* A provider instance + +You will also need to setup a virtualized Vagrant environment, to do so please make sure you have the following +pre-requisites installed: + +*Debian & Ubuntu* + +Install core prerequisites: + + sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make + +Install Vagrant in order to be able to test with local virtual machines (typically optional, but required for this tutorial). You probably want a more recent version directly from [vagrant.](https://www.vagrantup.com/downloads.htm) + + sudo apt-get install vagrant virtualbox + + +*Mac OS X 10.9 (Mavericks)* + +Install Homebrew package manager from http://brew.sh/ and enable the [System Duplicates Repository](https://github.com/Homebrew/homebrew/wiki/Interesting-Taps-&-Branches) (needed to update old software versions delivered by Apple) with + + brew tap homebrew/dupes + +Update OpenSSH to support ECDSA keys. Follow [this guide](http://www.dctrwatson.com/2013/07/how-to-update-openssh-on-mac-os-x/) to let your system use the Homebrew binary. + + brew install openssh --with-brewed-openssl --with-keychain-support + +The certtool provided by Apple it's really old, install the one provided by GnuTLS and shadow the system's default. + + sudo brew install gnutls + ln -sf /usr/local/bin/gnutls-certtool /usr/local/bin/certool + +Install the Vagrant and VirtualBox packages for OS X from their respective Download pages. + +* http://www.vagrantup.com/downloads.html +* https://www.virtualbox.org/wiki/Downloads + + +2. Install + + +Adding development nodes to your provider +========================================= + +Now you will add local-only Vagrant development nodes to your provider. + +You do not need to setup a different provider instance for development, in fact it is more convenient if you do not, but you can if you wish. If you do not have a provider already, you will need to create one and configure it before continuing (it is recommended you go through the [Quick Start](quick-start) before continuing down this path). + + +Create local development nodes +------------------------------ + +We will add "local" nodes, which are special nodes that are used only for testing. These nodes exist only as virtual machines on your computer, and cannot be accessed from the outside. Each "node" is a server that can have one or more services attached to it. We recommend that you create different nodes for different services to better isolate issues. + +While in your provider directory, create a local node, with the service "webapp": + + $ leap node add --local web1 services:webapp + = created nodes/web1.json + = created files/nodes/web1/ + = created files/nodes/web1/web1.key + = created files/nodes/web1/web1.crt + +This command creates a node configuration file in `nodes/web1.json` with the webapp service. + +Starting local development nodes +-------------------------------- + +In order to test the node "web1" we need to start it. Starting a node for the first time will spin up a virtual machine. The first time you do this will take some time because it will need to download a VM image (about 700mb). After you've downloaded the base image, you will not need to download it again, and instead you will re-use the downloaded image (until you need to update the image). + +NOTE: Many people have difficulties getting Vagrant working. If the following commands do not work, please see the Vagrant section below to troubleshoot your Vagrant install before proceeding. + + $ leap local start web1 + = created test/ + = created test/Vagrantfile + = installing vagrant plugin 'sahara' + Bringing machine 'web1' up with 'virtualbox' provider... + [web1] Box 'leap-wheezy' was not found. Fetching box from specified URL for + the provider 'virtualbox'. Note that if the URL does not have + a box for this provider, you should interrupt Vagrant now and add + the box yourself. Otherwise Vagrant will attempt to download the + full box prior to discovering this error. + Downloading or copying the box... + Progress: 3% (Rate: 560k/s, Estimated time remaining: 0:13:36) + ... + Bringing machine 'web1' up with 'virtualbox' provider... + [web1] Importing base box 'leap-wheezy'... + 0%...10%...20%...30%...40%...50%...60%...70%...80%...90%...100% + +Now the virtual machine 'web1' is running. You can add another local node using the same process. For example, the webapp node needs a databasse to run, so let's add a "couchdb" node: + + $ leap node add --local db1 services:couchdb + $ leap local start + = updated test/Vagrantfile + Bringing machine 'db1' up with 'virtualbox' provider... + [db1] Importing base box 'leap-wheezy'... + [db1] Matching MAC address for NAT networking... + [db1] Setting the name of the VM... + [db1] Clearing any previously set forwarded ports... + [db1] Fixed port collision for 22 => 2222. Now on port 2202. + [db1] Creating shared folders metadata... + [db1] Clearing any previously set network interfaces... + [db1] Preparing network interfaces based on configuration... + [db1] Forwarding ports... + [db1] -- 22 => 2202 (adapter 1) + [db1] Running any VM customizations... + [db1] Booting VM... + [db1] Waiting for VM to boot. This can take a few minutes. + [db1] VM booted and ready for use! + [db1] Configuring and enabling network interfaces... + [db1] Mounting shared folders... + [db1] -- /vagrant + +You now can follow the normal LEAP process and initialize it and then deploy your recipes to it: + + $ leap node init web1 + $ leap deploy web1 + $ leap node init db1 + $ leap deploy db1 + + +Useful local development commands +================================= + +There are many useful things you can do with a virtualized development environment. + +Listing what machines are running +--------------------------------- + +Now you have the two virtual machines "web1" and "db1" running, you can see the running machines as follows: + + $ leap local status + Current machine states: + + db1 running (virtualbox) + web1 running (virtualbox) + + This environment represents multiple VMs. The VMs are all listed + above with their current state. For more information about a specific + VM, run `vagrant status NAME`. + +Stopping machines +----------------- + +It is not recommended that you leave your virtual machines running when you are not using them. They consume memory and other resources! To stop your machines, simply do the following: + + $ leap local stop web1 db1 + +Connecting to machines +---------------------- + +You can connect to your local nodes just like you do with normal LEAP nodes, by running 'leap ssh node'. + +However, if you cannot connect to your local node, because the networking is not setup properly, or you have deployed a firewall that locks you out, you may need to access the graphical console. + +In order to do that, you will need to configure Vagrant to launch a graphical console and then you can login as root there to diagnose the networking problem. To do this, add the following to your $HOME/.leaprc: + + @custom_vagrant_vm_line = 'config.vm.provider "virtualbox" do |v| + v.gui = true + end' + +and then start, or restart, your local Vagrant node. You should get a VirtualBox graphical interface presented to you showing you the bootup and eventually the login. + +Snapshotting machines +--------------------- + +A very useful feature of local Vagrant development nodes is the ability to snapshot the current state and then revert to that when you need. + +For example, perhaps the base image is a little bit out of date and you want to get the packages updated to the latest before continuing. You can do that simply by starting the node, connecting to it and updating the packages and then snapshotting the node: + + $ leap local start web1 + $ leap ssh web1 + web1# apt-get -u dist-upgrade + web1# exit + $ leap local save web1 + +Now you can deploy to web1 and if you decide you want to revert to the state before deployment, you simply have to reset the node to your previous save: + + $ leap local reset web1 + +More information +---------------- + +See `leap help local` for a complete list of local-only commands and how they can be used. + + +Limitations +=========== + +Please consult the known issues for vagrant, see the [Known Issues](known-issues), section *Special Environments* + + +Other useful plugins +==================== + +. The vagrant-cachier (plugin http://fgrehm.viewdocs.io/vagrant-cachier/) lets you cache .deb packages on your hosts so they are not downloaded by multiple machines over and over again, after resetting to a previous state. + +Troubleshooting Vagrant +======================= + +To troubleshoot vagrant issues, try going through these steps: + +* Try plain vagrant using the [Getting started guide](http://docs.vagrantup.com/v2/getting-started/index.html). +* If that fails, make sure that you can run virtual machines (VMs) in plain virtualbox (Virtualbox GUI or VBoxHeadless). + We don't suggest a sepecial howto for that, [this one](http://www.thegeekstuff.com/2012/02/virtualbox-install-create-vm/) seems pretty decent, or you follow the [Oracale Virtualbox User Manual](http://www.virtualbox.org/manual/UserManual.html). There's also specific documentation for [Debian](https://wiki.debian.org/VirtualBox) and for [Ubuntu](https://help.ubuntu.com/community/VirtualBox). If you succeeded, try again if you now can start vagrant nodes using plain vagrant (see first step). +* If plain vagrant works for you, you're very close to using vagrant with leap ! If you encounter any problems now, please [contact us](https://leap.se/en/about-us/contact) or use our [issue tracker](https://leap.se/code) + +Known working combinations +-------------------------- + +Please consider that using other combinations might work for you as well, these are just the combinations we tried and worked for us: + + +Debian Wheezy +------------- + +* `virtualbox-4.2 4.2.16-86992~Debian~wheezy` from Oracle and `vagrant 1.2.2` from vagrantup.com + + +Ubuntu Raring 13.04 +------------------- + +* `virtualbox 4.2.10-dfsg-0ubuntu2.1` from Ubuntu raring and `vagrant 1.2.2` from vagrantup.com + +Mac OS X 10.9 +------------- + +* `VirtualBox 4.3.10` from virtualbox.org and `vagrant 1.5.4` from vagrantup.com + + +Using Vagrant with libvirt/kvm +============================== + +Vagrant can be used with different providers/backends, one of them is [vagrant-libvirt](https://github.com/pradels/vagrant-libvirt). Here are the steps how to use it. Be sure to use a recent vagrant version for the vagrant-libvirt plugin (>= 1.5, which can only be fetched from http://www.vagrantup.com/downloads.html at this moment). + +Install vagrant-libvirt plugin and add box +------------------------------------------ + sudo apt-get install libvirt-bin libvirt-dev + # you need to assign the new 'libvirtd' group to your user in a running x session, or logout and login again: + newgrp libvirtd + # to build the vagrant-libvirt plugin you need the following packages: + sudo apt-get install ruby-dev libxslt-dev libxml2-dev libvirt-dev + vagrant plugin install vagrant-libvirt + vagrant plugin install sahara + vagrant box add leap-wheezy https://downloads.leap.se/platform/vagrant/libvirt/leap-wheezy.box --provider libvirt + +Remove Virtualbox +----------------- + sudo apt-get remove virtualbox* + +Debugging +--------- + +If you get an error in any of the above commands, try to get some debugging information, it will often tell you what is wrong. In order to get debugging logs, you simply need to re-run the command that produced the error but prepend the command with VAGRANT_LOG=info, for example: + VAGRANT_LOG=info vagrant box add leap-wheezy https://downloads.leap.se/platform/vagrant/libvirt/leap-wheezy.box + +Start it +-------- + +Use this example Vagrantfile: + + Vagrant.configure("2") do |config| + config.vm.define :testvm do |testvm| + testvm.vm.box = "leap-wheezy" + testvm.vm.network :private_network, :ip => '10.6.6.201' + end + + config.vm.provider :libvirt do |libvirt| + libvirt.connect_via_ssh = false + end + end + +Then: + + vagrant up --provider=libvirt + +If everything works, you should export libvirt as the VAGRANT_DEFAULT_PROVIDER: + + export VAGRANT_DEFAULT_PROVIDER="libvirt" + +Now you should be able to use the `leap local` commands. + +Known Issues +------------ + +* 'Call to virConnectOpen failed: internal error: Unable to locate libvirtd daemon in /usr/sbin (to override, set $LIBVIRTD_PATH to the name of the libvirtd binary)' - you don't have the libvirtd daemon running or installed, be sure you installed the 'libvirt-bin' package and it is running +* 'Call to virConnectOpen failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': Permission denied' - you need to be in the libvirt group to access the socket, do 'sudo adduser libvirt' and then re-login to your session +* if each call to vagrant ends up with a segfault, it may be because you still have virtualbox around. if so, remove virtualbox to keep only libvirt + KVM. according to https://github.com/pradels/vagrant-libvirt/issues/75 having two virtualization engines installed simultaneously can lead to such weird issues. +* see the [vagrant-libvirt issue list on github](https://github.com/pradels/vagrant-libvirt/issues) +* be sure to use vagrant-libvirt >= 0.0.11 and sahara >= 0.0.16 (which are the latest stable gems you would get with `vagrant plugin install [vagrant-libvirt|sahara]`) for proper libvirt support +* for shared folder support, you need nfs-kernel-server installed on the host machine and set up sudo to allow unpriviledged users to modify /etc/exports. See [vagrant-libvirt#synced-folders](https://github.com/pradels/vagrant-libvirt#synced-folders) + + + sudo apt-get install nfs-kernel-server diff --git a/doc/details/en.haml b/doc/details/en.haml new file mode 100644 index 00000000..2f59f3f2 --- /dev/null +++ b/doc/details/en.haml @@ -0,0 +1,5 @@ +- @title = "Details" + +%h1.first Platform Details + += child_summaries \ No newline at end of file diff --git a/doc/details/faq.md b/doc/details/faq.md new file mode 100644 index 00000000..57afb6c4 --- /dev/null +++ b/doc/details/faq.md @@ -0,0 +1,65 @@ +@title = 'Frequently asked questions' +@nav_title = 'FAQ' +@summary = "Frequently Asked Questions" +@toc = true + +APT +=============== + +What do I do when unattended upgrades fail? +-------------------------------------------------- + +When you receive notification e-mails with a subject of 'unattended-upgrades result for $machinename', that means that some package couldn't be automatically upgraded and needs manual interaction. The reasons vary, so you have to be careful. Most often you can simply login to the affected machine and run `apt-get dist-upgrade`. + +Puppet +====== + +Where do i find the time a server was last deployed ? +----------------------------------------------------- + +The puppet state file on the node indicates the last puppetrun: + + ls -la /var/lib/puppet/state/state.yaml + +What resources are touched by puppet/leap_platform (services/packages/files etc.) ? +----------------------------------------------------------------------------------- + +Log into your server and issue: + + grep -v '!ruby/sym' /var/lib/puppet/state/state.yaml | sed 's/\"//' | sort + + +How can i customize the leap_platform puppet manifests ? +-------------------------------------------------------- + +You can create custom puppet modules under `files/puppet`. +The custom puppet entry point is in class 'custom' which can be put into +`files/puppet/modules/custom/manifests/init.pp`. This class gets automatically included +by site_config::default, which is applied to all nodes. + +Of cause you can also create a different git branch and change whatever you want, if you are +familiar wit git. + +Facter +====== + +How can i see custom facts distributed by leap_platform on a node ? +------------------------------------------------------------------- + +On the server, export the FACTERLIB env. variable to include the path of the custom fact in question: + + export FACTERLIB=/var/lib/puppet/lib/facter:/srv/leap/puppet/modules/stdlib/lib/facter/ + facter + + +Etc +=== + +How do i change the domain of my provider ? +------------------------------------------- + +* First of all, you need to have access to the nameserver config of your new domain. +* Update domain in provider.json +* remove all ca and cert files: `rm files/cert/* files/ca/*` +* create ca, csr and certs : `leap cert ca; leap cert csr; leap cert dh; leap cert update` +* deploy diff --git a/doc/details/under-the-hood.md b/doc/details/under-the-hood.md new file mode 100644 index 00000000..dcbddb3e --- /dev/null +++ b/doc/details/under-the-hood.md @@ -0,0 +1,26 @@ +@title = "Under the hood" +@summary = "Various implementation details." + +This page contains various details on the how the platform is implemented. You can safely ignore this page, although it may be useful if you plan to make modifications to the platform. + +Puppet Details +====================================== + +Tags +---- + +Tags are beeing used to deploy different classes. + +* leap_base: site_config::default (configure hostname + resolver, sshd, ) +* leap_slow: site_config::slow (slow: apt-get update, apt-get dist-upgrade) +* leap_service: cofigure platform service (openvpn, couchdb, etc.) + +You can pass any combination of tags, i.e. use + +* "--tags leap_base,leap_slow,leap_service" (DEFAULT): Deploy all +* "--tags leap_service": Only deploy service(s) (useful for debugging/development) +* "--tags leap_base": Only deploy basic configuration (again, useful for debugging/development) + +See http://docs.puppetlabs.com/puppet/2.7/reference/lang_tags.html for puppet tag usage. + + diff --git a/doc/details/webapp.md b/doc/details/webapp.md new file mode 100644 index 00000000..2b078af4 --- /dev/null +++ b/doc/details/webapp.md @@ -0,0 +1,282 @@ +@title = 'LEAP Web' +@summary = 'The web component of the LEAP Platform, providing user management, support desk, documentation and more.' +@toc = true + +Introduction +=================== + +"LEAP Web" is the webapp component of the LEAP Platform, providing the following services: + +* REST API for user registration. +* Admin interface to manage users. +* Client certificate distribution and renewal. +* User support help tickets. +* Billing +* Customizable and Localized user documentation + +This web application is written in Ruby on Rails 3, using CouchDB as the backend data store. + +It is licensed under the GNU Affero General Public License (version 3.0 or higher). See http://www.gnu.org/licenses/agpl-3.0.html for more information. + +Known problems +==================== + +* Client certificates are generated without a CSR. The problem is that this makes the web + application extremely vulnerable to denial of service attacks. This was not an issue until we + started to allow the possibility of anonymously fetching a client certificate without + authenticating first. + +* By its very nature, the user database is vulnerable to enumeration attacks. These are + very hard to prevent, because our protocol is designed to allow query of a user database via + proxy in order to provide network perspective. + +Integration +=========== + +LEAP web is part of the leap platform. Most of the time it will be customized and deployed in that context. This section describes the integration of LEAP web in the wider framework. The Development section focusses on development of LEAP web itself. + +Configuration & Customization +------------------------------ + +The customization of the webapp for a leap provider happens via two means: + * configuration settings in services/webapp.json + * custom files in files/webapp + +### Configuration Settings + +The webapp ships with a fairly large set of default settings for all environments. They are stored in config/defaults.yml. During deploy the platform creates config/config.yml from the settings in services/webapp.json. These settings will overwrite the defaults. + +### Custom Files + +Any file placed in files/webapp in the providers repository will overwrite the content of config/customization in the webapp. These files will override files of the same name. + +This mechanism allows customizing basically all aspects of the webapp. +See files/webapp/README.md in the providers repository for more. + +### Provider Information ### + +The leap client fetches provider information via json files from the server. The platform prepares that information and stores it in the webapp in public/1/config/*.json. (1 being the current API version). + +Provider Documentation +------------- + +LEAP web already comes with a bit of user documentation. It mostly resides in app/views/pages and thus can be overwritten by adding files to files/webapp/views/pages in the provider repository. You probably want to add your own Terms of Services and Privacy Policy here. +The webapp will render haml, erb and markdown templates and pick translated content from localized files such as privacy_policy.es.md. In order to add or remove languages you have to modify the available_locales setting in the config. (See Configuration Settings above) + +Development +=========== + +Installation +--------------------------- + +Typically, this application is installed automatically as part of the LEAP Platform. To install it manually for testing or development, follow these instructions: + +### TL;DR ### + +Install git, ruby 1.9, rubygems and couchdb on your system. Then run + + gem install bundler + git clone https://leap.se/git/leap_web + cd leap_web + git submodule update --init + bundle install --binstubs + bin/rails server + +### Install system requirements + +First of all you need to install ruby, git and couchdb. On debian based systems this would be achieved by something like + + sudo apt-get install git ruby1.9.3 rubygems couchdb + +We install most gems we depend upon through [bundler](http://gembundler.com). So first install bundler + + sudo gem install bundler + +On Debian Wheezy or later, there is a Debian package for bundler, so you can alternately run ``sudo apt-get install bundler``. + +### Download source + +Simply clone the git repository: + + git clone git://leap.se/leap_web + cd leap_web + +### SRP Submodule + +We currently use a git submodule to include srp-js. This will soon be replaced by a ruby gem. but for now you need to run + + git submodule update --init + +### Install required ruby libraries + + cd leap_web + bundle + +Typically, you run ``bundle`` as a normal user and it will ask you for a sudo password when it is time to install the required gems. If you don't have sudo, run ``bundle`` as root. + +Configuration +---------------------------- + +The configuration file `config/defaults.yml` providers good defaults for most +values. You can override these defaults by creating a file `config/config.yml`. + +There are a few values you should make sure to modify: + + production: + admins: ["myusername","otherusername"] + domain: example.net + force_ssl: true + secret_token: "4be2f60fafaf615bd4a13b96bfccf2c2c905898dad34..." + client_ca_key: "/etc/ssl/ca.key" + client_ca_cert: "/etc/ssl/ca.crt" + ca_key_password: nil + +* `admins` is an array of usernames that are granted special admin privilege. +* `domain` is your fully qualified domain name. +* `force_ssl`, if set to true, will require secure cookies and turn on HSTS. Don't do this if you are using a self-signed server certificate. +* `secret_token`, used for cookie security, you can create one with `rake secret`. Should be at least 30 characters. +* `client_ca_key`, the private key of the CA used to generate client certificates. +* `client_ca_cert`, the public certificate the CA used to generate client certificates. +* `ca_key_password`, used to unlock the client_ca_key, if needed. + +### Provider Settings + +The leap client fetches provider information via json files from the server. +If you want to use that functionality please add your provider files the public/1/config directory. (1 being the current API version). + +Running +----------------------------- + + cd leap_web + bin/rails server + +You will find Leap Web running on `localhost:3000` + +Testing +-------------------------------- + +To run all tests + + rake test + +To run an individual test: + + rake test TEST=certs/test/unit/client_certificate_test.rb + or + ruby -Itest certs/test/unit/client_certificate_test.rb + +Engines +--------------------- + +Leap Web includes some Engines. All things in `app` will overwrite the engine behaviour. You can clone the leap web repository and add your customizations to the `app` directory. Including leap_web as a gem is currently not supported. It should not require too much work though and we would be happy to include the changes required. + +If you have no use for one of the engines you can remove it from the Gemfile. Engines should really be plugins - no other engines should depend upon them. If you need functionality in different engines it should probably go into the toplevel. + +# Deployment # + +We strongly recommend using the LEAP platform for deploy. Most of the things documented here are automated as part of the platform. If you want to research how the platform deploys or work on your own mechanism this section is for you. + +These instructions are targeting a Debian GNU/Linux system. You might need to change the commands to match your own needs. + +## Server Preperation ## + +### Dependencies ## + +The following packages need to be installed: + +* git +* ruby1.9 +* rubygems1.9 +* couchdb (if you want to use a local couch) + +### Setup Capistrano ### + +We use puppet to deploy. But we also ship an untested config/deploy.rb.example. Edit it to match your needs if you want to use capistrano. + +run `cap deploy:setup` to create the directory structure. + +run `cap deploy` to deploy to the server. + +## Customized Files ## + +Please make sure your deploy includes the following files: + +* public/1/config/*.json (see Provider Settings section) +* config/couchdb.yml + +## Couch Security ## + +We recommend against using an admin user for running the webapp. To avoid this couch design documents need to be created ahead of time and the auto update mechanism needs to be disabled. +Take a look at test/setup_couch.sh for an example of securing the couch. + +## Design Documents ## + +After securing the couch design documents need to be deployed with admin permissions. There are two ways of doing this: + * rake couchrest:migrate_with_proxies + * dump the documents as files with `rake couchrest:dump` and deploy them + to the couch by hand or with the platform. + +### CouchRest::Migrate ### + +The before_script block in .travis.yml illustrates how to do this: + + mv test/config/couchdb.yml.admin config/couchdb.yml # use admin privileges + bundle exec rake couchrest:migrate_with_proxies # run the migrations + bundle exec rake couchrest:migrate_with_proxies # looks like this needs to run twice + mv test/config/couchdb.yml.user config/couchdb.yml # drop admin privileges + +### Deploy design docs from CouchRest::Dump ### + +First of all we get the design docs as files: + + # put design docs in /tmp/design + bundle exec rake couchrest:dump + +Then we add them to files/design in the site_couchdb module in leap_platform so they get deployed with the couch. You could also upload them using curl or sth. similar. + +# Troubleshooting # + +Here are some less common issues you might run into when installing Leap Web. + +## Cannot find Bundler ## + +### Error Messages ### + +`bundle: command not found` + +### Solution ### + +Make sure bundler is installed. `gem list bundler` should list `bundler`. +You also need to be able to access the `bundler` executable in your PATH. + +## Outdated version of rubygems ## + +### Error Messages ### + +`bundler requires rubygems >= 1.3.6` + +### Solution ### + +`gem update --system` will install the latest rubygems + +## Missing development tools ## + +Some required gems will compile C extensions. They need a bunch of utils for this. + +### Error Messages ### + +`make: Command not found` + +### Solution ### + +Install the required tools. For linux the `build-essential` package provides most of them. For Mac OS you probably want the XCode Commandline tools. + +## Missing libraries and headers ## + +Some gem dependencies might not compile because they lack the needed c libraries. + +### Solution ### + +Install the libraries in question including their development files. + + diff --git a/doc/development.md b/doc/development.md deleted file mode 100644 index 7a761418..00000000 --- a/doc/development.md +++ /dev/null @@ -1,272 +0,0 @@ -@title = "Development Environment" -@toc = true - -If you are wanting to make local changes to your provider, or want to contribute some fixes back to LEAP, we recommend that you follow this guide to build up a development environment to test your changes first. Using this method, you can quickly test your changes without deploying them to your production environment, while benefitting from the convenience of reverting to known good states in order to retry things from scratch. - -This page will walk you through setting up nodes using [Vagrant](http://www.vagrantup.com/) for convenient deployment testing, snapshotting known good states, and reverting to previous snapshots. - -Requirements -============ - -* Be a real machine with virtualization support in the CPU (VT-x or AMD-V). In other words, not a virtual machine. -* Have at least 4gb of RAM. -* Have a fast internet connection (because you will be downloading a lot of big files, like virtual machine images). - -Install prerequisites --------------------------------- - -For development purposes, you will need everything that you need for deploying the LEAP platform: - -* LEAP cli -* A provider instance - -You will also need to setup a virtualized Vagrant environment, to do so please make sure you have the following -pre-requisites installed: - -*Debian & Ubuntu* - -Install core prerequisites: - - sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make - -Install Vagrant in order to be able to test with local virtual machines (typically optional, but required for this tutorial): - - sudo apt-get install vagrant virtualbox - - - - -Adding development nodes to your provider -========================================= - -Now you will add local-only Vagrant development nodes to your provider. - -You do not need to setup a different provider instance for development, in fact it is more convenient if you do not, but you can if you wish. If you do not have a provider already, you will need to create one and configure it before continuing (it is recommended you go through the [Quick Start](quick-start) before continuing down this path). - - -Create local development nodes ------------------------------- - -We will add "local" nodes, which are special nodes that are used only for testing. These nodes exist only as virtual machines on your computer, and cannot be accessed from the outside. Each "node" is a server that can have one or more services attached to it. We recommend that you create different nodes for different services to better isolate issues. - -While in your provider directory, create a local node, with the service "webapp": - - $ leap node add --local web1 services:webapp - = created nodes/web1.json - = created files/nodes/web1/ - = created files/nodes/web1/web1.key - = created files/nodes/web1/web1.crt - -This command creates a node configuration file in `nodes/web1.json` with the webapp service. - -Starting local development nodes --------------------------------- - -In order to test the node "web1" we need to start it. Starting a node for the first time will spin up a virtual machine. The first time you do this will take some time because it will need to download a VM image (about 700mb). After you've downloaded the base image, you will not need to download it again, and instead you will re-use the downloaded image (until you need to update the image). - -NOTE: Many people have difficulties getting Vagrant working. If the following commands do not work, please visit the [Vagrant page](vagrant) to troubleshoot your Vagrant install before proceeding. - - $ leap local start web - = created test/ - = created test/Vagrantfile - = installing vagrant plugin 'sahara' - Bringing machine 'web1' up with 'virtualbox' provider... - [web1] Box 'leap-wheezy' was not found. Fetching box from specified URL for - the provider 'virtualbox'. Note that if the URL does not have - a box for this provider, you should interrupt Vagrant now and add - the box yourself. Otherwise Vagrant will attempt to download the - full box prior to discovering this error. - Downloading or copying the box... - Progress: 3% (Rate: 560k/s, Estimated time remaining: 0:13:36) - ... - Bringing machine 'web1' up with 'virtualbox' provider... - [web1] Importing base box 'leap-wheezy'... - 0%...10%...20%...30%...40%...50%...60%...70%...80%...90%...100% - -Now the virtual machine 'web1' is running. You can add another local node using the same process. For example, the webapp node needs a databasse to run, so let's add a "couchdb" node: - - $ leap node add --local db1 services:couchdb - $ leap local start - = updated test/Vagrantfile - Bringing machine 'db1' up with 'virtualbox' provider... - [db1] Importing base box 'leap-wheezy'... - [db1] Matching MAC address for NAT networking... - [db1] Setting the name of the VM... - [db1] Clearing any previously set forwarded ports... - [db1] Fixed port collision for 22 => 2222. Now on port 2202. - [db1] Creating shared folders metadata... - [db1] Clearing any previously set network interfaces... - [db1] Preparing network interfaces based on configuration... - [db1] Forwarding ports... - [db1] -- 22 => 2202 (adapter 1) - [db1] Running any VM customizations... - [db1] Booting VM... - [db1] Waiting for VM to boot. This can take a few minutes. - [db1] VM booted and ready for use! - [db1] Configuring and enabling network interfaces... - [db1] Mounting shared folders... - [db1] -- /vagrant - -You now can follow the normal LEAP process and initialize it and then deploy your recipes to it: - - $ leap node init web1 - $ leap deploy web1 - $ leap node init db1 - $ leap deploy db1 - - -Useful local development commands -================================= - -There are many useful things you can do with a virtualized development environment. - -Listing what machines are running ---------------------------------- - -Now you have the two virtual machines "web1" and "db1" running, you can see the running machines as follows: - - $ leap local status - Current machine states: - - db1 running (virtualbox) - web1 running (virtualbox) - - This environment represents multiple VMs. The VMs are all listed - above with their current state. For more information about a specific - VM, run `vagrant status NAME`. - -Stopping machines ------------------ - -It is not recommended that you leave your virtual machines running when you are not using them. They consume memory and other resources! To stop your machines, simply do the following: - - $ leap local stop web1 db1 - -Connecting to machines ----------------------- - -You can connect to your local nodes just like you do with normal LEAP nodes, by running 'leap ssh node'. - -However, if you cannot connect to your local node, because the networking is not setup properly, or you have deployed a firewall that locks you out, you may need to access the graphical console. - -In order to do that, you will need to configure Vagrant to launch a graphical console and then you can login as root there to diagnose the networking problem. To do this, add the following to you -$HOME/.leaprc: - - @custom_vagrant_vm_line = 'config.vm.boot_mode = :gui' - -and then start, or restart, your local Vagrant node. You should get a VirtualBox graphical interface presented to you showing you the bootup and eventually the login. - -Snapshotting machines ---------------------- - -A very useful feature of local Vagrant development nodes is the ability to snapshot the current state and then revert to that when you need. - -For example, perhaps the base image is a little bit out of date and you want to get the packages updated to the latest before continuing. You can do that simply by starting the node, connecting to it and updating the packages and then snapshotting the node: - - $ leap local start web1 - $ leap ssh web1 - web1# apt-get -u dist-upgrade - web1# exit - $ leap local save web1 - -Now you can deploy to web1 and if you decide you want to revert to the state before deployment, you simply have to reset the node to your previous save: - - $ leap local reset web1 - -More information ----------------- - -See `leap help local` for a complete list of local-only commands and how they can be used. - - -Limitations -=========== - -Please consult the known issues for vagrant, see the [Known Issues](known-issues), section *Special Environments* - - -Troubleshooting Vagrant -======================= - -To troubleshoot vagrant issues, try going through these steps: - -* Try plain vagrant using the [Getting started guide](http://docs.vagrantup.com/v2/getting-started/index.html). -* If that fails, make sure that you can run virtual machines (VMs) in plain virtualbox (Virtualbox GUI or VBoxHeadless). - We don't suggest a sepecial howto for that, [this one](http://www.thegeekstuff.com/2012/02/virtualbox-install-create-vm/) seems pretty decent, or you follow the [Oracale Virtualbox User Manual](http://www.virtualbox.org/manual/UserManual.html). There's also specific documentation for [Debian](https://wiki.debian.org/VirtualBox) and for [Ubuntu](https://help.ubuntu.com/community/VirtualBox). If you succeeded, try again if you now can start vagrant nodes using plain vagrant (see first step). -* If plain vagrant works for you, you're very close to using vagrant with leap ! If you encounter any problems now, please [contact us](https://leap.se/en/about-us/contact) or use our [issue tracker](https://leap.se/code) - -Known working combinations --------------------------- - -Please consider that using other combinations might work for you as well, these are just the combinations we tried and worked for us: - - -Debian Wheezy -------------- - -* `virtualbox-4.2 4.2.16-86992~Debian~wheezy` from Oracle and `vagrant 1.2.2` from vagrantup.com - - -Ubuntu Raring 13.04 -------------------- - -* `virtualbox 4.2.10-dfsg-0ubuntu2.1` from Ubuntu raring and `vagrant 1.2.2` from vagrantup.com - - -Using Vagrant with libvirt/kvm -============================== - -Vagrant can be used with different providers/backends, one of them is [vagrant-libvirt](https://github.com/pradels/vagrant-libvirt). Here are the steps how to use it. Be sure to use a recent vagrant version (>= 1.3). - -Install vagrant-libvirt plugin and add box ------------------------------------------- - sudo apt-get install libvirt-bin libvirt-dev - vagrant plugin install vagrant-libvirt - vagrant plugin install sahara - vagrant box add leap-wheezy https://downloads.leap.se/leap-debian-libvirt.box - - -Debugging ---------- - -If you get an error in any of the above commands, try to get some debugging information, it will often tell you what is wrong. In order to get debugging logs, you simply need to re-run the command that produced the error but prepend the command with VAGRANT_LOG=info, for example: - VAGRANT_LOG=info vagrant box add leap-wheezy https://downloads.leap.se/leap-debian-libvirt.box - -Start it --------- - -Use this example Vagrantfile: - - Vagrant.configure("2") do |config| - config.vm.define :testvm do |testvm| - testvm.vm.box = "leap-wheezy" - testvm.vm.network :private_network, :ip => '10.6.6.201' - end - - config.vm.provider :libvirt do |libvirt| - libvirt.connect_via_ssh = false - end - end - -Then: - - vagrant up --provider=libvirt - -If everything works, you should export libvirt as the VAGRANT_DEFAULT_PROVIDER: - - export VAGRANT_DEFAULT_PROVIDER="libvirt" - -Now you should be able to use the `leap local` commands. - -Known Issues ------------- - -* 'Call to virConnectOpen failed: internal error: Unable to locate libvirtd daemon in /usr/sbin (to override, set $LIBVIRTD_PATH to the name of the libvirtd binary)' - you don't have the libvirtd daemon running or installed, be sure you installed the 'libvirt-bin' package and it is running -* 'Call to virConnectOpen failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': Permission denied' - you need to be in the libvirt group to access the socket, do 'sudo adduser libvirt' and then re-login to your session -* see the [vagrant-libvirt issue list on github](https://github.com/pradels/vagrant-libvirt/issues) -* be sure to use vagrant-libvirt >= 0.0.11 and sahara >= 0.0.16 (which are the latest stable gems you would get with `vagrant plugin install [vagrant-libvirt|sahara]`) for proper libvirt support diff --git a/doc/en.md b/doc/en.md index f1a1fc17..d0dcfcc9 100644 --- a/doc/en.md +++ b/doc/en.md @@ -71,7 +71,7 @@ Getting started We recommend reading the platform documentation in the following order: -1. [Quick start tutorial](platform/quick-start). +1. [Quick start tutorial](tutorials/quick-start). 2. [Platform Guide](platform/guide). 3. [Configuration format](platform/config). 4. The `leap` [command reference](platform/commands). diff --git a/doc/faq.md b/doc/faq.md deleted file mode 100644 index 2654ce80..00000000 --- a/doc/faq.md +++ /dev/null @@ -1,53 +0,0 @@ -@title = 'Frequently asked questions' -@nav_title = 'FAQ' -@toc = true - -Puppet -====== - -Where do i find the time a server was last deployed ? ------------------------------------------------------ - -The puppet state file on the node indicates the last puppetrun: - - ls -la /var/lib/puppet/state/state.yaml - -What resources are touched by puppet/leap_platform (services/packages/files etc.) ? ------------------------------------------------------------------------------------ - -Log into your server and issue: - - grep -v '!ruby/sym' /var/lib/puppet/state/state.yaml | sed 's/\"//' | sort - - -How can i customize the leap_platform puppet manifests ? --------------------------------------------------------- - -You can create a custom module `site_custom`. The class `site_custom::setup` will get -included in the first part of the deploy process, and `site_custom` during the second part. -Of cause you can also create a different git branch and change whatever you want, if you are -familiar wit git. - -Facter -====== - -How can i see custom facts distributed by leap_platform on a node ? -------------------------------------------------------------------- - -On the server, export the FACTERLIB env. variable to include the path of the custom fact in question: - - export FACTERLIB=/var/lib/puppet/lib/facter:/srv/leap/puppet/modules/stdlib/lib/facter/ - facter - - -Etc -=== - -How do i change the domain of my provider ? -------------------------------------------- - -* First of all, you need to have access to the nameserver config of your new domain. -* Update domain in provider.json -* remove all ca and cert files: `rm files/cert/* files/ca/*` -* create ca, csr and certs : `leap cert ca; leap cert csr; leap cert dh; leap cert update` -* deploy diff --git a/doc/guide.md b/doc/guide.md deleted file mode 100644 index 52c3b2fa..00000000 --- a/doc/guide.md +++ /dev/null @@ -1,266 +0,0 @@ -@title = "LEAP Platform Guide" -@nav_title = "Guide" - -Services -================================ - -Every node has one or more services that determines the node's function within your provider's infrastructure. - -When adding a new node to your provider, you should ask yourself four questions: - -* **many or few?** Some services benefit from having many nodes, while some services are best run on only one or two nodes. -* **required or optional?** Some services are required, while others can be left out. -* **who does the node communicate with?** Some services communicate very heavily with other particular services. Nodes running these services should be close together. -* **public or private?** Some services communicate with the public internet, while others only need to communicate with other nodes in the infrastructure. - -Brief overview of the services: - -* **webapp**: The web application. Runs both webapp control panel for users and admins as well as the REST API that the client uses. Needs to communicate heavily with `couchdb` nodes. You need at least one, good to have two for redundancy. The webapp does not get a lot of traffic, so you will not need many. -* **couchdb**: The database for users and user data. You can get away with just one, but for proper redundancy you should have at least three. Communicates heavily with `webapp` and `mx` nodes. -* **soledad**: Handles the data syncing with clients. Typically combined with `couchdb` service, since it communicates heavily with couchdb. (not currently in stable release) -* **mx**: Incoming and outgoing MX servers. Communicates with the public internet, clients, and `couchdb` nodes. (not currently in stable release) -* **openvpn**: OpenVPN gateway for clients. You need at least one, but want as many as needed to support the bandwidth your users are doing. The `openvpn` nodes are autonomous and don't need to communicate with any other nodes. Often combined with `tor` service. -* **monitor**: Internal service to monitor all the other nodes. Currently, you can have zero or one `monitor` nodes. -* **tor**: Sets up a tor exit node, unconnected to any other service. -* **dns**: Not yet implemented. - -Locations -================================ - -All nodes should have a `location.name` specified, and optionally additional information about the location, like the time zone. This location information is used for two things: - -* Determine which nodes can, or must, communicate with one another via a local network. The way some virtualization environments work, like OpenStack, requires that nodes communicate via the local network if they are on the same network. -* Allows the client to prefer connections to nodes that are closer in physical proximity to the user. This is particularly important for OpenVPN nodes. - -The location stanza in a node's config file looks like this: - - { - "location": { - "id": "ankara", - "name": "Ankara", - "country_code": "TR", - "timezone": "+2", - "hemisphere": "N" - } - } - -The fields: - -* `id`: An internal handle to use for this location. If two nodes have match `location.id`, then they are treated as being on a local network with one another. This value defaults to downcase and underscore of `location.name`. -* `name`: Can be anything, might be displayed to the user in the client if they choose to manually select a gateway. -* `country_code`: The [ISO 3166-1](https://en.wikipedia.org/wiki/ISO_3166-1) two letter country code. -* `timezone`: The timezone expressed as an offset from UTC (in standard time, not daylight savings). You can look up the timezone using this [handy map](http://www.timeanddate.com/time/map/). -* `hemisphere`: This should be "S" for all servers in South America, Africa, or Australia. Otherwise, this should be "N". - -These location options are very imprecise, but good enough for most usage. The client often does not know its own location precisely either. Instead, the client makes an educated guess at location based on the OS's timezone and locale. - -If you have multiple nodes in a single location, it is best to use a tag for the location. For example: - -`tags/ankara.json`: - - { - "location": { - "name": "Ankara", - "country_code": "TR", - "timezone": "+2", - "hemisphere": "N" - } - } - -`nodes/vpngateway.json`: - - { - "services": "openvpn", - "tags": ["production", "ankara"], - "ip_address": "1.1.1.1", - "openvpn": { - "gateway_address": "1.1.1.2" - } - } - -Unless you are using OpenStack or AWS, setting `location` for nodes is not required. It is, however, highly recommended. - -Working with SSH -================================ - -Whenever the `leap` command nees to push changes to a node or gather information from a node, it tunnels this command over SSH. Another way to put this: the security of your servers rests entirely on SSH. Because of this, it is important that you understand how `leap` uses SSH. - -SSH related files -------------------------------- - -Assuming your provider directory is called 'provider': - -* `provider/nodes/crow/crow_ssh.pub` -- The public SSH host key for node 'crow'. -* `provider/users/alice/alice_ssh.pub` -- The public SSH user key for user 'alice'. Anyone with the private key that corresponds to this public key will have root access to all nodes. -* `provider/files/ssh/known_hosts` -- An autogenerated known_hosts, built from combining `provider/nodes/*/*_ssh.pub`. You must not edit this file directly. If you need to change it, remove or change one of the files that is used to generate `known_hosts` and then run `leap compile`. -* `provider/files/ssh/authorized_keys` -- An autogenerated list of all the user SSH keys with root access to the notes. It is created from `provider/users/*/*_ssh.pub`. You must not edit this file directly. If you need to change it, remove or change one of the files that is used to generate `authorized_keys` and then run `leap compile`. - -All of these files should be committed to source control. - -If you rename, remove, or add a node with `leap node [mv|add|rm]` the SSH key files and the `known_hosts` file will get properly updated. - -SSH and local nodes ------------------------------ - -Local nodes are run as Vagrant virtual machines. The `leap` command handles SSH slightly differently for these nodes. - -Basically, all the SSH security is turned off for local nodes. Since local nodes only exist for a short time on your computer and can't be reached from the internet, this is not a problem. - -Specifically, for local nodes: - -1. `known_hosts` is never updated with local node keys, since the SSH public key of a local node is different for each user. -2. `leap` entirely skips the checking of host keys when connecting with a local node. -3. `leap` adds the public Vagrant SSH key to the list of SSH keys for a user. The public Vagrant SSH key is a shared and insecure key that has root access to most Vagrant virtual machines. - -When SSH host key changes -------------------------------- - -If the host key for a node has changed, you will get an error "WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED". - -To fix this, you need to remove the file `files/nodes/stompy/stompy_ssh.pub` and run `leap node init stompy`, where the node's name is 'stompy'. **Only do this if you are ABSOLUTELY CERTAIN that the node's SSH host key has changed**. - -Changing the SSH port --------------------------------- - -Suppose you have a node `blinky` that has SSH listening on port 22 and you want to make it port 2200. - -First, modify the configuration for `blinky` to specify the variable `ssh.port` as 2200. Usually, this is done in `common.json` or in a tag file. - -For example, you could put this in `tags/production.json`: - - { - "ssh": { - "port": 2200 - } - } - -Run `leap compile` and open `hiera/blinky.yaml` to confirm that `ssh.port` is set to 2200. The port number must be specified as a number, not a string (no quotes). - -Then, you need to deploy this change so that SSH will bind to 2200. You cannot simply run `leap deploy blinky` because this command will default to using the variable `ssh.port` which is now `2200` but SSH on the node is still bound to 22. - -So, you manually override the port in the deploy command, using the old port: - - leap deploy --port 22 blinky - -Afterwards, SSH on `blinky` should be listening on port 2200 and you can just run `leap deploy blinky` from then on. - -X.509 Certificates -================================ - -Configuration options -------------------------------------------- - -The `ca` option in provider.json provides settings used when generating CAs and certificates. The defaults are as follows: - - { - "ca": { - "name": "= global.provider.ca.organization + ' Root CA'", - "organization": "= global.provider.name[global.provider.default_language]", - "organizational_unit": "= 'https://' + global.provider.domain", - "bit_size": 4096, - "digest": "SHA256", - "life_span": "10y", - "server_certificates": { - "bit_size": 2048, - "digest": "SHA256", - "life_span": "1y" - }, - "client_certificates": { - "bit_size": 2048, - "digest": "SHA256", - "life_span": "2m", - "limited_prefix": "LIMITED", - "unlimited_prefix": "UNLIMITED" - } - } - } - -You should not need to override these defaults in your own provider.json, but you can if you want to. To see what values are used for your provider, run `leap inspect provider.json`. - -NOTE: A certificate `bit_size` greater than 2048 will probably not be recognized by most commercial CAs. - -Certificate Authorities ------------------------------------------ - -There are three x.509 certificate authorities (CA) associated with your provider: - -1. **Commercial CA:** It is strongly recommended that you purchase a commercial cert for your primary domain. The goal of platform is to not depend on the commercial CA system, but it does increase security and usability if you purchase a certificate. The cert for the commercial CA must live at `files/cert/commercial_ca.crt`. -2. **Server CA:** This is a self-signed CA responsible for signing all the **server** certificates. The private key lives at `files/ca/ca.key` and the public cert lives at `files/ca/ca.crt`. The key is very sensitive information and must be kept private. The public cert is distributed publicly. -3. **Client CA:** This is a self-signed CA responsible for signing all the **client** certificates. The private key lives at `files/ca/client_ca.key` and the public cert lives at `files/ca/client_ca.crt`. Neither file is distribute publicly. It is not a big deal if the private key for the client CA is compromised, you can just generate a new one and re-deploy. - -To generate both the Server CA and the Client CA, run the command: - - leap cert ca - -Server certificates ------------------------------------ - -Most every server in your service provider will have a x.509 certificate, generated by the `leap` command using the Server CA. Whenever you modify any settings of a node that might affect it's certificate (like changing the IP address, hostname, or settings in provider.json), you can magically regenerate all the certs that need to be regenerated with this command: - - leap cert update - -Run `leap help cert update` for notes on usage options. - -Because the server certificates are generated locally on your personal machine, the private key for the Server CA need never be put on any server. It is up to you to keep this file secure. - -Client certificates --------------------------------- - -Every leap client gets its own time-limited client certificate. This cert is use to connect to the OpenVPN gateway (and probably other things in the future). It is generated on the fly by the webapp using the Client CA. - -To make this work, the private key of the Client CA is made available to the webapp. This might seem bad, but compromise of the Client CA simply allows the attacker to use the OpenVPN gateways without paying. In the future, we plan to add a command to automatically regenerate the Client CA periodically. - -There are two types of client certificates: limited and unlimited. A client using a limited cert will have its bandwidth limited to the rate specified by `provider.service.bandwidth_limit` (in Bytes per second). An unlimited cert is given to the user if they authenticate and the user's service level matches one configured in `provider.service.levels` without bandwidth limits. Otherwise, the user is given a limited client cert. - -Commercial certificates ------------------------------------ - -We strongly recommend that you use a commercial signed server certificate for your primary domain (in other words, a certificate with a common name matching whatever you have configured for `provider.domain`). This provides several benefits: - -1. When users visit your website, they don't get a scary notice that something is wrong. -2. When a user runs the LEAP client, selecting your service provider will not cause a warning message. -3. When other providers first discover your provider, they are more likely to trust your provider key if it is fetched over a commercially verified link. - -The LEAP platform is designed so that it assumes you are using a commercial cert for the primary domain of your provider, but all other servers are assumed to use non-commercial certs signed by the Server CA you create. - -To generate a CSR, run: - - leap cert csr - -This command will generate the CSR and private key matching `provider.domain` (you can change the domain with `--domain=DOMAIN` switch). It also generates a server certificate signed with the Server CA. You should delete this certificate and replace it with a real one once it is created by your commercial CA. - -The related commercial cert files are: - - files/ - certs/ - domain.org.crt # Server certificate for domain.org, obtained by commercial CA. - domain.org.csr # Certificate signing request - domain.org.key # Private key for you certificate - commercial_ca.crt # The CA cert obtained from the commercial CA. - -The private key file is extremely sensitive and care should be taken with its provenance. - -If your commercial CA has a chained CA cert, you should be OK if you just put the **last** cert in the chain into the `commercial_ca.crt` file. This only works if the other CAs in the chain have certs in the debian package `ca-certificates`, which is the case for almost all CAs. - -If you want to add additional fields to the CSR, like country, city, or locality, you can configure these values in provider.json like so: - - "ca": { - "server_certificates": { - "country": "US", - "state": "Washington", - "locality": "Seattle" - } - } - -If they are not present, the CSR will be created without them. - -Facts -============================== - -There are a few cases when we must gather internal data from a node before we can successfully deploy to other nodes. This is what `facts.json` is for. It stores a snapshot of certain facts about each node, as needed. Entries in `facts.json` are updated automatically when you initialize, rename, or remove a node. To manually force a full update of `facts.json`, run: - - leap facts update FILTER - -Run `leap help facts update` for more information. - -The file `facts.json` should be committed to source control. You might not have a `facts.json` if one is not required for your provider. diff --git a/doc/guide/commands.md b/doc/guide/commands.md new file mode 100644 index 00000000..0cee709a --- /dev/null +++ b/doc/guide/commands.md @@ -0,0 +1,419 @@ +@title = 'Command Line Reference' +@summary = "A copy of leap --help" + +The command "leap" can be used to manage a bevy of servers running the LEAP platform from the comfort of your own home. + + +# Global Options + +* `--log FILE` +Override default log file +Default Value: None + +* `-v|--verbose LEVEL` +Verbosity level 0..5 +Default Value: 1 + +* `--[no-]color` +Disable colors in output + +* `--debug` +Enable debugging library (leap_cli development only) + +* `--help` +Show this message + +* `--version` +Display version number and exit + +* `--yes` +Skip prompts and assume "yes" + + +# leap add-user USERNAME + +Adds a new trusted sysadmin by adding public keys to the "users" directory. + + + +**Options** + +* `--pgp-pub-key arg` +OpenPGP public key file for this new user +Default Value: None + +* `--ssh-pub-key arg` +SSH public key file for this new user +Default Value: None + +* `--self` +Add yourself as a trusted sysadin by choosing among the public keys available for the current user. + + +# leap cert + +Manage X.509 certificates + + + +## leap cert ca + +Creates two Certificate Authorities (one for validating servers and one for validating clients). + +See see what values are used in the generation of the certificates (like name and key size), run `leap inspect provider` and look for the "ca" property. To see the details of the created certs, run `leap inspect `. + +## leap cert csr + +Creates a CSR for use in buying a commercial X.509 certificate. + +Unless specified, the CSR is created for the provider's primary domain. The properties used for this CSR come from `provider.ca.server_certificates`. + +**Options** + +* `--domain DOMAIN` +Specify what domain to create the CSR for. +Unless specified, the CSR is created for the provider's primary domain. The properties used for this CSR come from `provider.ca.server_certificates`. +Default Value: None + + +## leap cert dh + +Creates a Diffie-Hellman parameter file. + + + +## leap cert update FILTER + +Creates or renews a X.509 certificate/key pair for a single node or all nodes, but only if needed. + +This command will a generate new certificate for a node if some value in the node has changed that is included in the certificate (like hostname or IP address), or if the old certificate will be expiring soon. Sometimes, you might want to force the generation of a new certificate, such as in the cases where you have changed a CA parameter for server certificates, like bit size or digest hash. In this case, use --force. If is empty, this command will apply to all nodes. + +**Options** + +* `--force` +Always generate new certificates + + +# leap clean + +Removes all files generated with the "compile" command. + + + +# leap compile + +Compile generated files. + + + +## leap compile all [ENVIRONMENT] + +Compiles node configuration files into hiera files used for deployment. + + + +## leap compile zone + +Compile a DNS zone file for your provider. + + +Default Command: all + +# leap db + +Database commands. + + + +## leap db destroy [FILTER] + +Destroy all the databases. If present, limit to FILTER nodes. + + + +# leap deploy FILTER + +Apply recipes to a node or set of nodes. + +The FILTER can be the name of a node, service, or tag. + +**Options** + +* `--ip IPADDRESS` +Override the default SSH IP address. +Default Value: None + +* `--port PORT` +Override the default SSH port. +Default Value: None + +* `--tags TAG[,TAG]` +Specify tags to pass through to puppet (overriding the default). +Default Value: leap_base,leap_service + +* `--dev` +Development mode: don't run 'git submodule update' before deploy. + +* `--fast` +Makes the deploy command faster by skipping some slow steps. A "fast" deploy can be used safely if you recently completed a normal deploy. + +* `--force` +Deploy even if there is a lockfile. + +* `--[no-]sync` +Sync files, but don't actually apply recipes. + + +# leap env + +Manipulate and query environment information. + +The 'environment' node property can be used to isolate sets of nodes into entirely separate environments. A node in one environment will never interact with a node from another environment. Environment pinning works by modifying your ~/.leaprc file and is dependent on the absolute file path of your provider directory (pins don't apply if you move the directory) + +## leap env ls + +List the available environments. The pinned environment, if any, will be marked with '*'. + + + +## leap env pin ENVIRONMENT + +Pin the environment to ENVIRONMENT. All subsequent commands will only apply to nodes in this environment. + + + +## leap env unpin + +Unpin the environment. All subsequent commands will apply to all nodes. + + +Default Command: ls + +# leap facts + +Gather information on nodes. + + + +## leap facts update FILTER + +Query servers to update facts.json. + +Queries every node included in FILTER and saves the important information to facts.json + +# leap help command + +Shows a list of commands or help for one command + +Gets help for the application or its commands. Can also list the commands in a way helpful to creating a bash-style completion function + +**Options** + +* `-c` +List commands one per line, to assist with shell completion + + +# leap inspect FILE + +Prints details about a file. Alternately, the argument FILE can be the name of a node, service or tag. + + + +**Options** + +* `--base` +Inspect the FILE from the provider_base (i.e. without local inheritance). + + +# leap list [FILTER] + +List nodes and their classifications + +Prints out a listing of nodes, services, or tags. If present, the FILTER can be a list of names of nodes, services, or tags. If the name is prefixed with +, this acts like an AND condition. For example: + +`leap list node1 node2` matches all nodes named "node1" OR "node2" + +`leap list openvpn +local` matches all nodes with service "openvpn" AND tag "local" + +**Options** + +* `--print arg` +What attributes to print (optional) +Default Value: None + +* `--disabled` +Include disabled nodes in the list. + + +# leap local + +Manage local virtual machines. + +This command provides a convient way to manage Vagrant-based virtual machines. If FILTER argument is missing, the command runs on all local virtual machines. The Vagrantfile is automatically generated in 'test/Vagrantfile'. If you want to run vagrant commands manually, cd to 'test'. + +## leap local destroy [FILTER] + +Destroys the virtual machine(s), reclaiming the disk space + + + +## leap local reset [FILTER] + +Resets virtual machine(s) to the last saved snapshot + + + +## leap local save [FILTER] + +Saves the current state of the virtual machine as a new snapshot + + + +## leap local start [FILTER] + +Starts up the virtual machine(s) + + + +## leap local status [FILTER] + +Print the status of local virtual machine(s) + + + +## leap local stop [FILTER] + +Shuts down the virtual machine(s) + + + +# leap mosh NAME + +Log in to the specified node with an interactive shell using mosh (requires node to have mosh.enabled set to true). + + + +# leap new DIRECTORY + +Creates a new provider instance in the specified directory, creating it if necessary. + + + +**Options** + +* `--contacts arg` +Default email address contacts. +Default Value: None + +* `--domain arg` +The primary domain of the provider. +Default Value: None + +* `--name arg` +The name of the provider. +Default Value: None + +* `--platform arg` +File path of the leap_platform directory. +Default Value: None + + +# leap node + +Node management + + + +## leap node add NAME [SEED] + +Create a new configuration file for a node named NAME. + +If specified, the optional argument SEED can be used to seed values in the node configuration file. + +The format is property_name:value. + +For example: `leap node add web1 ip_address:1.2.3.4 services:webapp`. + +To set nested properties, property name can contain '.', like so: `leap node add web1 ssh.port:44` + +Separeate multiple values for a single property with a comma, like so: `leap node add mynode services:webapp,dns` + +**Options** + +* `--local` +Make a local testing node (by automatically assigning the next available local IP address). Local nodes are run as virtual machines on your computer. + + +## leap node init FILTER + +Bootstraps a node or nodes, setting up SSH keys and installing prerequisite packages + +This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, copying the authorized_keys file, installing packages that are required for deploying, and registering important facts. Node init must be run before deploying to a server, and the server must be running and available via the network. This command only needs to be run once, but there is no harm in running it multiple times. + +**Options** + +* `--ip IPADDRESS` +Override the default SSH IP address. +Default Value: None + +* `--port PORT` +Override the default SSH port. +Default Value: None + +* `--echo` +If set, passwords are visible as you type them (default is hidden) + + +## leap node mv OLD_NAME NEW_NAME + +Renames a node file, and all its related files. + + + +## leap node rm NAME + +Removes all the files related to the node named NAME. + + + +# leap ssh NAME + +Log in to the specified node with an interactive shell. + + + +**Options** + +* `--port arg` +Override ssh port for remote host +Default Value: None + +* `--ssh arg` +Pass through raw options to ssh (e.g. --ssh '-F ~/sshconfig') +Default Value: None + + +# leap test + +Run tests. + + + +## leap test init + +Creates files needed to run tests. + + + +## leap test run + +Run tests. + + + +**Options** + +* `--[no-]continue` +Continue over errors and failures (default is --no-continue). + +Default Command: run diff --git a/doc/guide/config.md b/doc/guide/config.md new file mode 100644 index 00000000..be67e6bd --- /dev/null +++ b/doc/guide/config.md @@ -0,0 +1,263 @@ +@title = "Configuration Files" +@summary = "How to edit configuration files." + +Files +------------------------------------------- + +Here are a list of some of the common files that make up a provider. Except for Leapfile and provider.json, the files are optional. Unless otherwise specified, all file names are relative to the 'provider directory' root (where the Leapfile is). + +`Leapfile` -- If present, this file tells `leap` that the directory is a provider directory. This file is usually empty, but can contain global options. + +`~/.leaprc` -- Evaluated the same as Leapfile, but not committed to source control. + +`provider.json` -- Global options related to this provider. + +`provider.ENVIRONMENT.json` -- Global options for the provider that are applied to only a single environment. + +`common.json` -- All nodes inherit from this file. + +`secrets.json` -- An automatically generated file that contains any randomly generated strings needed in order to deploy. These strings are often secret and should be protected, although any need for a random string or number that is remembered will produce another entry in this file. This file is automatically generated and refreshed each time you run `leap compile` or `leap deploy`. If an entry is no longer needed, it will get removed. If you want to change a secret, you can remove this file and have it regenerated, or remove the particular line item and just those items will be created anew. + +`facts.json` -- If some of your servers are running on AWS or OpenStack, you will need to discover certain properties about how networking is configured on these machines in order for a full deploy to work. In these cases, make sure to run `leap facts update` to periodically regenerate the facts.json file. + +`nodes/NAME.json` -- The configuration file for node called NAME. + +`services/SERVICE.json` -- The properties in this configuration file are applied to any node that includes SERVICE in its `services` property. + +`services/SERVICE.ENVIRONMENT.json` -- The properties in this configuration file are applied to any node that includes SERVICE in its services and has environment equal to ENVIRONMENT. + +`services/TAG.json` -- The properties in this configuration file are applied to any node that has includes TAG in its `tags` property. + +`services/TAG.ENVIRONMENT.json` -- The properties in this configuration file are applied to any node that has includes TAG in its `tags` property and has `environment` property equal to ENVIRONMENT. + +`files/*` -- Various static files used by the platform (e.g. keys, certificates, webapp customization, etc). + +`users/USER/` -- A directory that stores the public keys of the sysadmin with name USER. This person will have root access to all the servers. + + +Leapfile +------------------------------------------- + +A `Leapfile` defines options for the `leap` command and lives at the root of your provider directory. `Leapfile` is evaluated as ruby, so you can include whatever weird logic you want in this file. In particular, there are several variables you can set that modify the behavior of leap. For example: + + @platform_directory_path = '../leap_platform' + @log = '/var/log/leap.log' + +Additionally, you can create a `~/.leaprc` file that is loaded after `Leapfile` and is evaluated the same way. + +Platform options: + +* `@platform_directory_path` (required). This must be set to the path where `leap_platform` lives. The path may be relative. + +Vagrant options: + +* `@vagrant_network`. Allows you to override the default network used for local nodes. It should include a netmask like `@vagrant_network = '10.0.0.0/24'`. +* `@custom_vagrant_vm_line`. Insert arbitrary text into the auto-generated Vagrantfile. For example, `@custom_vagrant_vm_line = "config.vm.boot_mode = :gui"`. + +Logging options: + +* `@log`. If set, all command invocation and results are logged to the specified file. This is the same as the switch `--log FILE`, except that the command line switch will override the value in the Leapfile. + + +JSON format +------------------------------------------- + +All configuration files, other than `Leapfile`, are in the JSON format. For example: + + { + "key1": "value1", + "key2": "value2" + } + +Keys should match `/[a-z0-9_]/` + +Unlike traditional JSON, comments are allowed. If the first non-whitespace characters are `//` then the line is treated as a comment. + + // this is a comment + { + // this is a comment + "key": "value" // this is an error + } + +Options in the configuration files might be nested hashes, arrays, numbers, strings, or boolean. Numbers and boolean values should **not** be quoted. For example: + + { + "openvpn": { + "ip_address": "1.1.1.1", + "protocols": ["tcp", "udp"], + "ports": [80, 53], + "options": { + "public_ip": false, + "adblock": true + } + } + } + +If the value string is prefixed with an '=' character, the result is evaluated as ruby. For example: + + { + "domain": { + "public": "domain.org" + } + "api_domain": "= 'api.' + domain.public" + } + +In this case, the property "api_domain" will be set to "api.domain.org". So long as you do not create unresolvable circular dependencies, you can reference other properties in evaluated ruby that are themselves evaluated ruby. + +See "Macros" below for information on the special macros available to the evaluated ruby. + +TIP: In rare cases, you might want to force the evaluation of a value to happen in a later pass after most of the other properties have been evaluated. To do this, prefix the value string with "=>" instead of "=". + +Node inheritance +---------------------------------------- + +Every node inherits from common.json and also any of the services or tags attached to the node. Additionally, the `leap_platform` contains a directory `provider_base` that defines the default values for tags, services and common.json. + +Suppose you have a node configuration for `bitmask/nodes/willamette.json` like so: + + { + "services": "webapp", + "tags": ["production", "northwest-us"], + "ip_address": "1.1.1.1" + } + +This node will have hostname "willamette" and it will inherit from the following files (in this order): + +1. common.json + - load defaults: `provider_base/common.json` + - load provider: `bitmask/common.json` +2. service "webapp" + - load defaults: `provider_base/services/webapp.json` + - load provider: `bitmask/services/webapp.json` +3. tag "production" + - load defaults: `provider_base/tags/production.json` + - load provider: `bitmask/tags/production.json` +4. tag "northwest-us" + - load: `bitmask/tags/northwest-us.json` +5. finally, load node "willamette" + - load: `bitmask/nodes/willamette.json` + +The `provider_base` directory is under the `leap_platform` specified in the file `Leapfile`. + +To see all the variables a node has inherited, you could run `leap inspect willamette`. + +Common configuration options +---------------------------------------- + +You can use the command `leap inspect` to see what options are available for a provider, node, service, or tag configuration. For example: + +* `leap inspect common` -- show the options inherited by all nodes. +* `leap inspect --base common` -- show the common.json from `provider_base` without the local `common.json` inheritance applied. +* `leap inspect webapp` -- show all the options available for the service `webapp`. + +Here are some of the more important options you should be aware of: + +* `ip_address` -- Required for all nodes, no default. +* `ssh.port` -- The SSH port you want the node's OpenSSH server to bind to. This is also the default when trying to connect to a node, but if the node currently has OpenSSH running on a different port then run deploy with `--port` to override the `ssh.port` configuration value. +* `mosh.enabled` -- If set to `true`, then mosh will be installed on the server. The default is `false`. + +Macros +---------------------------------------- + +When using evaluated ruby in a JSON configuration file, there are several special macros that are available. These are evaluated in the context of a node (available as the variable `self`). + +The following methods are available to the evaluated ruby: + +`variable.variable` + + > Any variable defined or inherited by a particular node configuration is available by just referencing it using either hash notation or object field notation (e.g. `['domain']['public']` or `domain.public`). Circular references are not allowed, but otherwise it is OK to nest evaluated values in other evaluated values. If a value has not been defined, the hash notation will return nil but the field notation will raise an exception. Properties of services, tags, and the global provider can all be referenced the same way. For example, `global.services['openvpn'].x509.dh`. + +`nodes` + + > A hash of all nodes. This list can be filtered. + +`nodes_like_me` + + > A hash of nodes that have the same deployment tags as the current node (e.g. 'production' or 'local'). + +`global.services` + + > A hash of all services, e.g. `global.services['openvpn']` would return the "openvpn" service. + +`global.tags` + + > A hash of all tags, e.g. `global.tags['production']` would return the "production" tag. + + `global.provider` + + > Can be used to access variables defined in `provider.json`, e.g. `global.provider.contacts.default`. + +`file(filename)` + + > Inserts the full contents of the file. If the file is an erb template, it is rendered. The filename can either be one of the pre-defined file symbols, or it can be a path relative to the "files" directory in your provider instance. E.g, `file :ca_cert` or `files 'ca/ca.crt'`. + +`file_path(filename)` + + > Ensures that the file will get rsynced to the node as an individual file. The value returned by `file_path` is the full path where this file will ultimately live when deploy to the node. e.g. `file_path :ca_cert` or `file_path 'branding/images/logo.png'`. + +`secret(:symbol)` + + > Returns the value of a secret in secrets.json (or creates it if necessary). E.g. `secret :couch_admin_password` + +`hosts_file` + + > Returns a data structure that puppet will use to generate /etc/hosts. Care is taken to use the local IP of other hosts when needed. + +`known_hosts_file` + + > Returns the lines needed in a SSH `known_hosts` file. + +`stunnel_client(node_list, port, options={})` + + > Returns a stunnel configuration data structure for the client side. Argument `node_list` is an `ObjectList` of nodes running stunnel servers. Argument `port` is the real port of the ultimate service running on the servers that the client wants to connect to. + +`stunnel_server(port)` + + > Generates a stunnel server entry. The `port` is the real port targeted service. + +Hash tables +----------------------------------------- + +The macros `nodes`, `nodes_like_me`, `global.services`, and `global.tags` all return a hash table of configuration objects (either nodes, services, or tags). There are several ways to filter and process these hash tables: + +Access an element by name: + + nodes['vpn1'] # returns node named 'vpn1' + global.services['openvpn'] # returns service named 'openvpn' + +Create a new hash table by applying filters: + + nodes[:public_dns => true] # all nodes where public_dns == true + nodes[:services => 'openvpn', 'location.country_code' => 'US'] # openvpn service OR in the US. + nodes[[:services, 'openvpn'], [:services, 'tor']] # services equal to openvpn OR tor + nodes[:services => 'openvpn'][:tags => 'production'] # openvpn AND production + nodes[:name => "!bob"] # all nodes that are NOT named "bob" + +Create an array of values by selecting a single field: + + nodes.field('location.name') + ==> ['seattle', 'istanbul'] + +Create an array of hashes by selecting multiple fields: + + nodes.fields('domain.full', 'ip_address') + ==> [ + {'domain_full' => 'red.bitmask.net', 'ip_address' => '1.1.1.1'}, + {'domain_full' => 'blue.bitmask.net', 'ip_address' => '1.1.1.2'}, + ] + +Create a new hash table of hashes, with only certain fields: + + nodes.pick_fields('domain.full', 'ip_address') + ==> { + "red" => {'domain_full' => 'red.bitmask.net', 'ip_address' => '1.1.1.1'}, + "blue => {'domain_full' => 'blue.bitmask.net', 'ip_address' => '1.1.1.2'}, + } + +With `pick_fields`, if there is only one field, it will generate a simple hash table: + + nodes.pick_fields('ip_address') + ==> { + "red" => '1.1.1.1', + "blue => '1.1.1.2', + } diff --git a/doc/guide/en.haml b/doc/guide/en.haml new file mode 100644 index 00000000..4c9bd69f --- /dev/null +++ b/doc/guide/en.haml @@ -0,0 +1,6 @@ +- @nav_title = "Guide" +- @title = "Platform Guide" + +%h1.first Platform Guide + += child_summaries \ No newline at end of file diff --git a/doc/guide/keys-and-certificates.md b/doc/guide/keys-and-certificates.md new file mode 100644 index 00000000..bd7f3495 --- /dev/null +++ b/doc/guide/keys-and-certificates.md @@ -0,0 +1,176 @@ +@title = "Keys and Certificates" +@summary = "Working with SSH keys, secrets, and X.509 certificates." + +Working with SSH +================================ + +Whenever the `leap` command nees to push changes to a node or gather information from a node, it tunnels this command over SSH. Another way to put this: the security of your servers rests entirely on SSH. Because of this, it is important that you understand how `leap` uses SSH. + +SSH related files +------------------------------- + +Assuming your provider directory is called 'provider': + +* `provider/nodes/crow/crow_ssh.pub` -- The public SSH host key for node 'crow'. +* `provider/users/alice/alice_ssh.pub` -- The public SSH user key for user 'alice'. Anyone with the private key that corresponds to this public key will have root access to all nodes. +* `provider/files/ssh/known_hosts` -- An autogenerated known_hosts, built from combining `provider/nodes/*/*_ssh.pub`. You must not edit this file directly. If you need to change it, remove or change one of the files that is used to generate `known_hosts` and then run `leap compile`. +* `provider/files/ssh/authorized_keys` -- An autogenerated list of all the user SSH keys with root access to the notes. It is created from `provider/users/*/*_ssh.pub`. You must not edit this file directly. If you need to change it, remove or change one of the files that is used to generate `authorized_keys` and then run `leap compile`. + +All of these files should be committed to source control. + +If you rename, remove, or add a node with `leap node [mv|add|rm]` the SSH key files and the `known_hosts` file will get properly updated. + +SSH and local nodes +----------------------------- + +Local nodes are run as Vagrant virtual machines. The `leap` command handles SSH slightly differently for these nodes. + +Basically, all the SSH security is turned off for local nodes. Since local nodes only exist for a short time on your computer and can't be reached from the internet, this is not a problem. + +Specifically, for local nodes: + +1. `known_hosts` is never updated with local node keys, since the SSH public key of a local node is different for each user. +2. `leap` entirely skips the checking of host keys when connecting with a local node. +3. `leap` adds the public Vagrant SSH key to the list of SSH keys for a user. The public Vagrant SSH key is a shared and insecure key that has root access to most Vagrant virtual machines. + +When SSH host key changes +------------------------------- + +If the host key for a node has changed, you will get an error "WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED". + +To fix this, you need to remove the file `files/nodes/stompy/stompy_ssh.pub` and run `leap node init stompy`, where the node's name is 'stompy'. **Only do this if you are ABSOLUTELY CERTAIN that the node's SSH host key has changed**. + +Changing the SSH port +-------------------------------- + +Suppose you have a node `blinky` that has SSH listening on port 22 and you want to make it port 2200. + +First, modify the configuration for `blinky` to specify the variable `ssh.port` as 2200. Usually, this is done in `common.json` or in a tag file. + +For example, you could put this in `tags/production.json`: + + { + "ssh": { + "port": 2200 + } + } + +Run `leap compile` and open `hiera/blinky.yaml` to confirm that `ssh.port` is set to 2200. The port number must be specified as a number, not a string (no quotes). + +Then, you need to deploy this change so that SSH will bind to 2200. You cannot simply run `leap deploy blinky` because this command will default to using the variable `ssh.port` which is now `2200` but SSH on the node is still bound to 22. + +So, you manually override the port in the deploy command, using the old port: + + leap deploy --port 22 blinky + +Afterwards, SSH on `blinky` should be listening on port 2200 and you can just run `leap deploy blinky` from then on. + +X.509 Certificates +================================ + +Configuration options +------------------------------------------- + +The `ca` option in provider.json provides settings used when generating CAs and certificates. The defaults are as follows: + + { + "ca": { + "name": "= global.provider.ca.organization + ' Root CA'", + "organization": "= global.provider.name[global.provider.default_language]", + "organizational_unit": "= 'https://' + global.provider.domain", + "bit_size": 4096, + "digest": "SHA256", + "life_span": "10y", + "server_certificates": { + "bit_size": 2048, + "digest": "SHA256", + "life_span": "1y" + }, + "client_certificates": { + "bit_size": 2048, + "digest": "SHA256", + "life_span": "2m", + "limited_prefix": "LIMITED", + "unlimited_prefix": "UNLIMITED" + } + } + } + +You should not need to override these defaults in your own provider.json, but you can if you want to. To see what values are used for your provider, run `leap inspect provider.json`. + +NOTE: A certificate `bit_size` greater than 2048 will probably not be recognized by most commercial CAs. + +Certificate Authorities +----------------------------------------- + +There are three x.509 certificate authorities (CA) associated with your provider: + +1. **Commercial CA:** It is strongly recommended that you purchase a commercial cert for your primary domain. The goal of platform is to not depend on the commercial CA system, but it does increase security and usability if you purchase a certificate. The cert for the commercial CA must live at `files/cert/commercial_ca.crt`. +2. **Server CA:** This is a self-signed CA responsible for signing all the **server** certificates. The private key lives at `files/ca/ca.key` and the public cert lives at `files/ca/ca.crt`. The key is very sensitive information and must be kept private. The public cert is distributed publicly. +3. **Client CA:** This is a self-signed CA responsible for signing all the **client** certificates. The private key lives at `files/ca/client_ca.key` and the public cert lives at `files/ca/client_ca.crt`. Neither file is distribute publicly. It is not a big deal if the private key for the client CA is compromised, you can just generate a new one and re-deploy. + +To generate both the Server CA and the Client CA, run the command: + + leap cert ca + +Server certificates +----------------------------------- + +Most every server in your service provider will have a x.509 certificate, generated by the `leap` command using the Server CA. Whenever you modify any settings of a node that might affect it's certificate (like changing the IP address, hostname, or settings in provider.json), you can magically regenerate all the certs that need to be regenerated with this command: + + leap cert update + +Run `leap help cert update` for notes on usage options. + +Because the server certificates are generated locally on your personal machine, the private key for the Server CA need never be put on any server. It is up to you to keep this file secure. + +Client certificates +-------------------------------- + +Every leap client gets its own time-limited client certificate. This cert is use to connect to the OpenVPN gateway (and probably other things in the future). It is generated on the fly by the webapp using the Client CA. + +To make this work, the private key of the Client CA is made available to the webapp. This might seem bad, but compromise of the Client CA simply allows the attacker to use the OpenVPN gateways without paying. In the future, we plan to add a command to automatically regenerate the Client CA periodically. + +There are two types of client certificates: limited and unlimited. A client using a limited cert will have its bandwidth limited to the rate specified by `provider.service.bandwidth_limit` (in Bytes per second). An unlimited cert is given to the user if they authenticate and the user's service level matches one configured in `provider.service.levels` without bandwidth limits. Otherwise, the user is given a limited client cert. + +Commercial certificates +----------------------------------- + +We strongly recommend that you use a commercial signed server certificate for your primary domain (in other words, a certificate with a common name matching whatever you have configured for `provider.domain`). This provides several benefits: + +1. When users visit your website, they don't get a scary notice that something is wrong. +2. When a user runs the LEAP client, selecting your service provider will not cause a warning message. +3. When other providers first discover your provider, they are more likely to trust your provider key if it is fetched over a commercially verified link. + +The LEAP platform is designed so that it assumes you are using a commercial cert for the primary domain of your provider, but all other servers are assumed to use non-commercial certs signed by the Server CA you create. + +To generate a CSR, run: + + leap cert csr + +This command will generate the CSR and private key matching `provider.domain` (you can change the domain with `--domain=DOMAIN` switch). It also generates a server certificate signed with the Server CA. You should delete this certificate and replace it with a real one once it is created by your commercial CA. + +The related commercial cert files are: + + files/ + certs/ + domain.org.crt # Server certificate for domain.org, obtained by commercial CA. + domain.org.csr # Certificate signing request + domain.org.key # Private key for you certificate + commercial_ca.crt # The CA cert obtained from the commercial CA. + +The private key file is extremely sensitive and care should be taken with its provenance. + +If your commercial CA has a chained CA cert, you should be OK if you just put the **last** cert in the chain into the `commercial_ca.crt` file. This only works if the other CAs in the chain have certs in the debian package `ca-certificates`, which is the case for almost all CAs. + +If you want to add additional fields to the CSR, like country, city, or locality, you can configure these values in provider.json like so: + + "ca": { + "server_certificates": { + "country": "US", + "state": "Washington", + "locality": "Seattle" + } + } + +If they are not present, the CSR will be created without them. \ No newline at end of file diff --git a/doc/guide/miscellaneous.md b/doc/guide/miscellaneous.md new file mode 100644 index 00000000..c38c007c --- /dev/null +++ b/doc/guide/miscellaneous.md @@ -0,0 +1,14 @@ +@title = "Miscellaneous" +@summary = "Miscellaneous commands you may need to know." + +Facts +============================== + +There are a few cases when we must gather internal data from a node before we can successfully deploy to other nodes. This is what `facts.json` is for. It stores a snapshot of certain facts about each node, as needed. Entries in `facts.json` are updated automatically when you initialize, rename, or remove a node. To manually force a full update of `facts.json`, run: + + leap facts update FILTER + +Run `leap help facts update` for more information. + +The file `facts.json` should be committed to source control. You might not have a `facts.json` if one is not required for your provider. + diff --git a/doc/guide/nodes.md b/doc/guide/nodes.md new file mode 100644 index 00000000..bc48ff32 --- /dev/null +++ b/doc/guide/nodes.md @@ -0,0 +1,169 @@ +@title = "Nodes" +@summary = "Working with nodes, services, tags, and locations." + +Node types +================================ + +Every node has one or more services that determines the node's function within your provider's infrastructure. + +When adding a new node to your provider, you should ask yourself four questions: + +* **many or few?** Some services benefit from having many nodes, while some services are best run on only one or two nodes. +* **required or optional?** Some services are required, while others can be left out. +* **who does the node communicate with?** Some services communicate very heavily with other particular services. Nodes running these services should be close together. +* **public or private?** Some services communicate with the public internet, while others only need to communicate with other nodes in the infrastructure. + +Brief overview of the services: + +* **webapp**: The web application. Runs both webapp control panel for users and admins as well as the REST API that the client uses. Needs to communicate heavily with `couchdb` nodes. You need at least one, good to have two for redundancy. The webapp does not get a lot of traffic, so you will not need many. +* **couchdb**: The database for users and user data. You can get away with just one, but for proper redundancy you should have at least three. Communicates heavily with `webapp`, `mx`, and `soledad` nodes. +* **soledad**: Handles the data syncing with clients. Typically combined with `couchdb` service, since it communicates heavily with couchdb. +* **mx**: Incoming and outgoing MX servers. Communicates with the public internet, clients, and `couchdb` nodes. +* **openvpn**: OpenVPN gateway for clients. You need at least one, but want as many as needed to support the bandwidth your users are doing. The `openvpn` nodes are autonomous and don't need to communicate with any other nodes. Often combined with `tor` service. +* **monitor**: Internal service to monitor all the other nodes. Currently, you can have zero or one `monitor` nodes. +* **tor**: Sets up a tor exit node, unconnected to any other service. +* **dns**: Not yet implemented. + +Webapp +----------------------------------- + +The webapp node is responsible for both the user face web application and the API that the client interacts with. + +Some users can be "admins" with special powers to answer tickets and close accounts. To make an account into an administrator, you need to configure the `webapp.admins` property with an array of user names. + +For example, to make users `alice` and `bob` into admins, create a file `services/webapp.json` with the following content: + + { + "webapp": { + "admins": ["bob", "alice"] + } + } + +And then redeploy to all webapp nodes: + + leap deploy webapp + +By putting this in `services/webapp.json`, you will ensure that all webapp nodes inherit the value for `webapp.admins`. + +Services +================================ + +What nodes do you need for a provider that offers particular services? + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Node TypeVPN ServiceEmail Service
webapprequiredrequired
couchdbrequiredrequired
soledadnot usedrequired
mxnot usedrequired
openvpnrequirednot used
monitoroptionaloptional
toroptionaloptional
+ +Locations +================================ + +All nodes should have a `location.name` specified, and optionally additional information about the location, like the time zone. This location information is used for two things: + +* Determine which nodes can, or must, communicate with one another via a local network. The way some virtualization environments work, like OpenStack, requires that nodes communicate via the local network if they are on the same network. +* Allows the client to prefer connections to nodes that are closer in physical proximity to the user. This is particularly important for OpenVPN nodes. + +The location stanza in a node's config file looks like this: + + { + "location": { + "id": "ankara", + "name": "Ankara", + "country_code": "TR", + "timezone": "+2", + "hemisphere": "N" + } + } + +The fields: + +* `id`: An internal handle to use for this location. If two nodes have match `location.id`, then they are treated as being on a local network with one another. This value defaults to downcase and underscore of `location.name`. +* `name`: Can be anything, might be displayed to the user in the client if they choose to manually select a gateway. +* `country_code`: The [ISO 3166-1](https://en.wikipedia.org/wiki/ISO_3166-1) two letter country code. +* `timezone`: The timezone expressed as an offset from UTC (in standard time, not daylight savings). You can look up the timezone using this [handy map](http://www.timeanddate.com/time/map/). +* `hemisphere`: This should be "S" for all servers in South America, Africa, or Australia. Otherwise, this should be "N". + +These location options are very imprecise, but good enough for most usage. The client often does not know its own location precisely either. Instead, the client makes an educated guess at location based on the OS's timezone and locale. + +If you have multiple nodes in a single location, it is best to use a tag for the location. For example: + +`tags/ankara.json`: + + { + "location": { + "name": "Ankara", + "country_code": "TR", + "timezone": "+2", + "hemisphere": "N" + } + } + +`nodes/vpngateway.json`: + + { + "services": "openvpn", + "tags": ["production", "ankara"], + "ip_address": "1.1.1.1", + "openvpn": { + "gateway_address": "1.1.1.2" + } + } + +Unless you are using OpenStack or AWS, setting `location` for nodes is not required. It is, however, highly recommended. + +Disabling Nodes +===================================== + +There are two ways to temporarily disable a node: + +**Option 1: enabled == false** + +If a node has a property `enabled` set to false, then the `leap` command will skip over the node and pretend that it does not exist. For example: + + { + "ip_address": "1.1.1.1", + "services": ["openvpn"], + "enabled": false + } + +**Options 2: no-deploy** + +If the file `/etc/leap/no-deploy` exists on a node, then when you run the commmand `leap deploy` it will halt and prevent a deploy from going through (if the node was going to be included in the deploy). diff --git a/doc/known-issues.md b/doc/known-issues.md deleted file mode 100644 index 960eaad7..00000000 --- a/doc/known-issues.md +++ /dev/null @@ -1,64 +0,0 @@ -@title = 'Leap Platform Release Notes' -@nav_title = 'Known issues' -@summary = 'Known issues in the Leap Platform.' -@toc = true - -Here you can find documentation about known issues and potential work-arounds in the current Leap Platform release. - -0.2.2 -===== - -In this release the following issues are known, work-arounds are noted when available. - -General Issues --------------- - -. This release does *not* anonymize your logs (see: https://leap.se/code/issues/1897) - -. This release does *not* setup email relaying, so admins will not receive important email notifications. Email service will be part of the next release (see: https://leap.se/code/issues/1683 https://leap.se/code/issues/1905) - -. Your openvpn gateway address will be added on the /24 network, and is not configurable in this release (see: https://leap.se/code/issues/1863) - -. You must not add a node with an underscore in the name, you also cannot use a hyphen for a vagrant node (see: https://leap.se/code/issues/3087) - -. The nagios website check reports success when the webapp is not functioning but apache is up (see: https://leap.se/code/issues/1629) - -User setup and ssh ------------------- - -. if you aren't using a single ssh key, but have different ones, you will need to define the following at the top of your ~/.ssh/config: - HostName - IdentityFile - - (see: https://leap.se/code/issues/2946 and https://leap.se/code/issues/3002) - -. If the ssh host key changes, you need to run node init again (see: https://leap.se/en/docs/platform/guide#Working.with.SSH) - -. At the moment, only ECDSA ssh host keys are supported. If you get the following error: `= FAILED ssh-keyscan: no hostkey alg (must be missing an ecdsa public host key)` then you should confirm that you have the following line defined in your server's **/etc/ssh/sshd_config**: `HostKey /etc/ssh/ssh_host_ecdsa_key`. If that file doesn't exist, run `ssh-keygen -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key -N ""` in order to create it. If you made a change to your sshd_config, then you need to run `/etc/init.d/ssh restart` (see: https://leap.se/code/issues/2373) - -. To remove an admin's access to your servers, please remove the directory for that user under the `users/` subdirectory in your provider directory and then remove that user's ssh keys from files/ssh/authorized_keys. When finished you *must* run a `leap deploy` to update that information on the servers. - -. At the moment, it is only possible to add an admin who will have access to all LEAP servers (see: https://leap.se/code/issues/2280) - -. leap add-user --self allows only one key - if you run that command twice with different keys, you will just replace the key with the second key. To add a second key, add it manually to files/ssh/authorized_keys (see: https://leap.se/code/issues/866) - - -Deploying ---------- - -. If you have any errors during a run, please try to deploy again as this often solves non-deterministic issues that were not uncovered in our testing. Please re-deploy with `leap -v2 deploy` to get more verbose logs and capture the complete output to provide to us for debugging. - -. If when deploying your debian mirror fails for some reason, network anomoly or the mirror itself is out of date, then platform deployment will not succeed properly. Check the mirror is up and try to deploy again when it is resolved (see: https://leap.se/code/issues/1091) - -. Deployment gives 'error: in `%`: too few arguments (ArgumentError)' - this is because you attempted to do a deploy before initializing a node, please initialize the node first and then do a deploy afterwards (see: https://leap.se/code/issues/2550) - -. This release has no ability to custom configure apt sources or proxies (see: https://leap.se/code/issues/1971) - -. When running a deploy at a verbosity level of 2 and above, you will notice puppet deprecation warnings, these are known and we are working on fixing them - -Special Environments --------------------- - -. When deploying to OpenStack release "nova" or newer, you will need to do an initial deploy, then when it has finished run `leap facts update` and then deploy again (see: https://leap.se/code/issues/3020) - -. It is not possible to actually use the EIP openvpn server on vagrant nodes (see: https://leap.se/code/issues/2401) diff --git a/doc/quick-start.md b/doc/quick-start.md deleted file mode 100644 index 0bce271a..00000000 --- a/doc/quick-start.md +++ /dev/null @@ -1,397 +0,0 @@ -@title = 'LEAP Platform Quick Start' -@nav_title = 'Quick Start' - -Quick Start -=========== - -This tutorial walks you through the initial process of creating and deploying a minimal service provider running the [LEAP platform](platform). This Quick Start guide will guide you through building a three node OpenVPN provider. -If you are curious how this will look like without trying it out yourself, you can watch our [recorded screencasts](http://shelr.tv/users/524415e69660807910000021). - -Our goal ------------------- - -We are going to create a minimal LEAP provider offering OpenVPN service. This basic setup can be expanded by adding more OpenVPN nodes to increase capacity, or more webapp and couchdb nodes to increase availability (performance wise, a single couchdb and a single webapp are more than enough for most usage, since they are only lightly used, but you might want redundancy). - -Our goal is something like this: - - $ leap list - NODES SERVICES TAGS - couch1 couchdb - web1 webapp - vpn1 openvpn - -NOTE: You won't be able to run that `leap list` command yet, not until we actually create the node configurations. - -Requirements ------------- - -In order to complete this Quick Start, you will need a few things: - -* You will need three real or paravirtualized virtual machines (KVM, Xen, Openstack, Amazon, but not Vagrant - sorry) that have a basic Debian Stable installed. If you allocate 10G to each node, that should be plenty. -* You should be able to SSH into them remotely, and know their IP addresses and their SSH host keys -* You will need four different IPs, one for each node, and a second one for the VPN gateway -* The ability to create/modify DNS entries for your domain is preferable, but not needed. If you don't have access to DNS, you can workaround this by modifying your local resolver, i.e. editing `/etc/hosts`. -* You need to be aware that this process will make changes to your systems, so please be sure that these machines are a basic install with nothing configured or running for other purposes -* Your machines will need to be connected to the internet, and not behind a restrictive firewall. -* You should work locally on your laptop/workstation (one that you trust and that is ideally full-disk encrypted) while going through this guide. This is important because the provider configurations you are creating contain sensitive data that should not reside on a remote machine. The leap cli utility will login to your servers and configure the services. - -All the commands in this tutorial are run on your sysadmin machine. In order to complete the tutorial, the sysadmin will do the following: - -* Install pre-requisites -* Install the LEAP command-line utility -* Check out the LEAP platform -* Create a provider and its certificates -* Setup the provider's nodes and the services that will reside on those nodes -* Initialize the nodes -* Deploy the LEAP platform to the nodes -* Test that things worked correctly -* Some additional commands - -We will walk you through each of these steps. - - -Prepare your environment -======================== - -There are a few things you need to setup before you can get going. Just some packages, the LEAP cli and the platform. - -Install pre-requisites --------------------------------- - -*Debian & Ubuntu* - -Install core prerequisites: - - $ sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make bzip2 - - - -NOTE: leap_cli should work with ruby1.8, but has only been tested using ruby1.9. - - -Install the LEAP command-line utility ---------------------- - - - -Install `leap` command from source: - - $ git clone https://leap.se/git/leap_cli - $ cd leap_cli - $ rake build - -Then, install as root user (recommended): - - $ sudo rake install - -Or, install as unprivileged user: - - $ rake install - # watch out for the directory leap is installed to, then i.e. - $ sudo ln -s ~/.gem/ruby/1.9.1/bin/leap /usr/local/bin/leap - -With both methods, you can use now /usr/local/bin/leap, which in most cases will be in your $PATH. - -If you have successfully installed the LEAP cli, then you should be able to do the following: - - $ leap --help - -and be presented with the command-line help options. If you receive an error when doing this, please read through the README.md in the LEAP cli source to try and resolve any problems before going forwards. - - -Check out the platform ----------------------- - -The LEAP Platform is a series of puppet recipes and modules that will be used to configure your provider. You will need a local copy of the platform that will be used to setup your nodes and manage your services. To begin with, you will not need to modify the LEAP Platform. - -First we'll create a directory for LEAP things, and then we'll check out the platform code and initalize the modules: - - $ mkdir ~/leap - $ cd ~/leap - $ git clone https://leap.se/git/leap_platform.git - $ cd leap_platform - $ git submodule sync; git submodule update --init - - -Provider Setup -============== - -A provider instance is a directory tree, usually stored in git, that contains everything you need to manage an infrastructure for a service provider. In this case, we create one for example.org and call the instance directory 'example'. - - $ mkdir -p ~/leap/example - -Bootstrap the provider ------------------------ - -Now, we will initialize this directory to make it a provider instance. Your provider instance will need to know where it can find the local copy of the git repository leap_platform, which we setup in the previous step. - - $ cd ~/leap/example - $ leap new . - -NOTES: - . make sure you include that trailing dot! - -The `leap new` command will ask you for several required values: - -* domain: The primary domain name of your service provider. In this tutorial, we will be using "example.org". -* name: The name of your service provider (we use "Example"). -* contact emails: A comma separated list of email addresses that should be used for important service provider contacts (for things like postmaster aliases, Tor contact emails, etc). -* platform: The directory where you have a copy of the `leap_platform` git repository checked out. - -You could also have passed these configuration options on the command-line, like so: - - $ leap new --contacts your@email.here --domain leap.example.org --name Example --platform=~/leap/leap_platform . - -You may want to poke around and see what is in the files we just created. For example: - - $ cat provider.json - -Optionally, commit your provider directory using the version control software you fancy. For example: - - $ git init - $ git add . - $ git commit -m "initial provider commit" - -Now add yourself as a privileged sysadmin who will have access to deploy to servers: - - $ leap add-user --self - -NOTE: in most cases, `leap` must be run from within a provider instance directory tree (e.g. ~/leap/example). - -Create provider certificates ----------------------------- - -Create two certificate authorities, one for server certs and one for client -certs (note: you only need to run this one command to get both): - - $ leap cert ca - -Create a temporary cert for your main domain (you should replace with a real commercial cert at some point) - - $ leap cert csr - -To see details about the keys and certs that the prior two commands created, you can use `leap inspect` like so: - - $ leap inspect files/ca/ca.crt - -Create the Diffie-Hellman parameters file, needed for forward secret OpenVPN ciphers: - - $ leap cert dh - -NOTE: the files `files/ca/*.key` are extremely sensitive and must be carefully protected. The other key files are much less sensitive and can simply be regenerated if needed. - - -Edit provider.json configuration --------------------------------------- - -There are a few required settings in provider.json. At a minimum, you must have: - - { - "domain": "example.org", - "name": "Example", - "contacts": { - "default": "email1@example.org" - } - } - -For a full list of possible settings, you can use `leap inspect` to see how provider.json is evaluated after including the inherited defaults: - - $ leap inspect provider.json - - -Setup the provider's nodes and services ---------------------------------------- - -A "node" is a server that is part of your infrastructure. Every node can have one or more services associated with it. Some nodes are "local" and used only for testing, see [Development](developmet) for more information. - -Create a node, with the service "webapp": - - $ leap node add web1 ip_address:x.x.x.w services:webapp tags:production - -NOTE: replace x.x.x.w with the actual IP address of this node - -This created a node configuration file in `nodes/web1.json`, but it did not do anything else. It also added the 'tag' called 'production' to this node. Tags allow us to conveniently group nodes together. When creating nodes, you should give them the tag 'production' if the node is to be used in your production infrastructure. - -The web application and the VPN nodes require a database, so lets create the database server node: - - $ leap node add couch1 ip_address:x.x.x.x services:couchdb tags:production - -NOTE: replace x.x.x.x with the actual IP address of this node - -Now we need the VPN gateway, so lets create that node: - - $ leap node add vpn1 ip_address:x.x.x.y openvpn.gateway_address:x.x.x.z services:openvpn tags:production - -NOTE: replace x.x.x.y with the IP address of the machine, and x.x.x.z with the second IP. openvpn gateways must be assigned two IP addresses, one for the host itself and one for the openvpn gateway. We do this to prevent incoming and outgoing VPN traffic on the same IP. Without this, the client might send some traffic to other VPN users in the clear, bypassing the VPN. - - -Setup DNS ---------- - -Now that you have the nodes configured, you should create the DNS entries for these nodes. - -Set up your DNS with these hostnames: - - $ leap list --print ip_address,domain.full,dns.aliases - couch1 x.x.x.w, couch1.example.org, null - web1 x.x.x.x, web1.example.org, api.example.org, nicknym.example.org - vpn1 x.x.x.y, vpn1.example.org, null - -Alternately, you can adapt this zone file snippet: - - $ leap compile zone - -If you cannot edit your DNS zone file, you can still test your provider by adding entries to your local resolver hosts file (`/etc/hosts` for linux): - - x.x.x.w couch1.example.org - x.x.x.x web1.example.org api.example.org example.org - x.x.x.y vpn1.example.org - -Please don't forget about these entries, they will override DNS queries if you setup your DNS later. - - -Initialize the nodes --------------------- - -Node initialization only needs to be done once, but there is no harm in doing it multiple times: - - $ leap node init production - -This will initialize all nodes with the tag "production". When `leap node init` is run, you will be prompted to verify the fingerprint of the SSH host key and to provide the root password of the server(s). You should only need to do this once. - -If you prefer, you can initalize each node, one at a time: - - $ leap node init web1 - $ leap node init couch1 - $ leap node init vpn1 - -Deploy the LEAP platform to the nodes --------------------- - -Now you should deploy the platform recipes to the nodes. Deployment can take a while to run, especially on the first run, as it needs to update the packages on the new machine: - - $ leap deploy web1 - -Watch the output for any errors (in red), if everything worked fine, you should now have your first running node. If you do have errors, try doing the deploy again. - -However, to deploy our three-node openvpn setup, we need the database and LEAP web application requires a database to run, so let's deploy to the couchdb and openvpn nodes: - - $ leap deploy couch1 - $ leap deploy vpn1 - -NOTE: the output from deploying can be quite busy, so we often do them each node one by one. - -What is going on here? --------------------------------------------- - -First, some background terminology: - -* **puppet**: Puppet is a system for automating deployment and management of servers (called nodes). -* **hiera files**: In puppet, you can use something called a 'hiera file' to seed a node with a few configuration values. In LEAP, we go all out and put *every* configuration value needed for a node in the hiera file, and automatically compile a custom hiera file for each node. - -When you run `leap deploy`, a bunch of things happen, in this order: - -1. **Compile hiera files**: The hiera configuration file for each node is compiled in YAML format and saved in the directory `hiera`. The source material for this hiera file consists of all the JSON configuration files imported or inherited by the node's JSON config file. -* **Copy required files to node**: All the files needed for puppet to run are rsync'ed to each node. This includes the entire leap_platform directory, as well as the node's hiera file and other files needed by puppet to set up the node (keys, binary files, etc). -* **Puppet is run**: Once the node is ready, leap connects to the node via ssh and runs `puppet apply`. Puppet is applied locally on the node, without a daemon or puppetmaster. - -You can run `leap -v2 deploy` to see exactly what commands are being executed. - - - - -Test that things worked correctly -================================= - -You should now have three machines with the LEAP platform deployed to them, one for the web application, one for the database and one for the OpenVPN gateway. - - -Access the web application --------------------------------------------- - -In order to connect to the web application in your browser, you need to point your domain at the IP address of the web application node (named web1 in this example). - -There are a lot of different ways to do this, but one easy way is to modify your `/etc/hosts` file. First, find the IP address of the webapp node: - - $ leap list webapp --print ip_address - -Then modify `/etc/hosts` like so: - - x.x.x.w leap.example.org - -Replacing 'leap.example.org' with whatever you specified as the `domain` in the `leap new` command. - -Next, you can connect to the web application either using a web browser or via the API using the LEAP client. To use a browser, connect to https://leap.example.org (replacing that with your domain). Your browser will complain about an untrusted cert, but for now just bypass this. From there, you should be able to register a new user and login. - -Use the VPN ------------ - -You should be able to simply test that the OpenVPN gateway works properly by doing the following: - - $ leap test init - $ sudo openvpn test/openvpn/unlimited.ovpn - -Or, you can use the LEAP client (called "bitmask") to connect to your new provider, create a user and then connect to the VPN. - - -Additional information -====================== - -It is useful to know a few additional things. - -Useful commands ---------------- - -Here are a few useful commands you can run on your new local nodes: - -* `leap ssh web1` -- SSH into node web1 (requires `leap node init web1` first). -* `leap list` -- list all nodes. -* `leap list production` -- list only those nodes with the tag 'production' -* `leap list --print ip_address` -- list a particular attribute of all nodes. -* `leap cert update` -- generate new certificates if needed. - -See the full command reference for more information. - -Node filters -------------------------------------------- - -Many of the `leap` commands take a "node filter". You can use a node filter to target a command at one or more nodes. - -A node filter consists of one or more keywords, with an optional "+" before each keyword. - -* keywords can be a node name, a service type, or a tag. -* the "+" before the keyword constructs an AND condition -* otherwise, multiple keywords together construct an OR condition - -Examples: - -* `leap list openvpn` -- list all nodes with service openvpn. -* `leap list openvpn +production` -- only nodes of service type openvpn AND tag production. -* `leap deploy webapp openvpn` -- deploy to all webapp OR openvpn nodes. -* `leap node init vpn1` -- just init the node named vpn1. - -Keep track of your provider configurations ------------------------------------------- - -You should commit your provider changes to your favorite VCS whenever things change. This way you can share your configurations with other admins, all they have to do is to pull the changes to stay up to date. Every time you make a change to your provider, such as adding nodes, services, generating certificates, etc. you should add those to your VCS, commit them and push them to where your repository is hosted. - -Note that your provider directory contains secrets! Those secrets include passwords for various services. You do not want to have those passwords readable by the world, so make sure that wherever you are hosting your repository, it is not public for the world to read. - -What's next ------------------------------------ - -Read the [LEAP platform guide](guide) to learn about planning and securing your infrastructure. - diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md deleted file mode 100644 index bb2fc4b5..00000000 --- a/doc/troubleshooting.md +++ /dev/null @@ -1,147 +0,0 @@ -@title = 'Troubleshooting Guide' -@nav_title = 'Troubleshooting' -@toc = true - - -General -======= - -* Please increase verbosity when debugging / filing issues in our issue tracker. You can do this with adding i.e. `-v 5` after the `leap` cmd, i.e. `leap -v 2 deploy`. - -Webapp node -=========== - -Places to look for errors -------------------------- - -* `/var/log/apache2/error.log` -* `/srv/leap/webapp/log/production.log` -* `/var/log/syslog` (watch out for stunnel issues) - -Is haproxy ok ? ---------------- - - - curl -s -X GET "http://127.0.0.1:4096" - -Is couchdb accessible through stunnel ? ---------------------------------------- - - - curl -s -X GET "http://127.0.0.1:4000" - - -Check couchdb acl ------------------ - - - mkdir /etc/couchdb - cat /srv/leap/webapp/config/couchdb.yml.admin # see username and password - echo "machine 127.0.0.1 login admin password " > /etc/couchdb/couchdb-admin.netrc - chmod 600 /etc/couchdb/couchdb-admin.netrc - - curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4096" - curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4096/_all_dbs" - - -Couchdb node -============ - -Places to look for errors -------------------------- - -* `/opt/bigcouch/var/log/bigcouch.log` -* `/var/log/syslog` (watch out for stunnel issues) - - -Bigcouch membership -------------------- - -* All nodes configured for the provider should appear here: - - - curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET 'http://127.0.0.1:5986/nodes/_all_docs' - -* All configured nodes should show up under "cluster_nodes", and the ones online and communicating with each other should appear under "all_nodes". This example output shows the configured cluster nodes `couch1.bitmask.net` and `couch2.bitmask.net`, but `couch2.bitmask.net` is currently not accessible from `couch1.bitmask.net` - - - curl -s --netrc-file /etc/couchdb/couchdb.netrc 'http://127.0.0.1:5984/_membership' - {"all_nodes":["bigcouch@couch1.bitmask.net"],"cluster_nodes":["bigcouch@couch1.bitmask.net","bigcouch@couch2.bitmask.net"]} - - - -Databases ---------- - -* Following output shows all neccessary DBs that should be present. Note that the `user-0123456....` DBs are the data stores for a particular user. - - - curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET 'http://127.0.0.1:5984/_all_dbs' - ["customers","identities","sessions","shared","tickets","tokens","user-0","user-9d34680b01074c75c2ec58c7321f540c","user-9d34680b01074c75c2ec58c7325fb7ff","users"] - - - -Design Documents ----------------- - -* Is User `_design doc` available ? - - - curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET "http://127.0.0.1:5984/users/_design/User" - - - -MX node -======= - -Places to look for errors -------------------------- - -* `/var/log/mail.log` -* `/var/log/leap_mx.log` -* `/var/log/syslog` (watch out for stunnel issues) - - -Query leap-mx -------------- - -* for useraccount - - - postmap -v -q "joe@dev.bitmask.net" tcp:localhost:2244 - ... - postmap: dict_tcp_lookup: send: get jow@dev.bitmask.net - postmap: dict_tcp_lookup: recv: 200 - ... - -* for mailalias - - - postmap -v -q "joe@dev.bitmask.net" tcp:localhost:4242 - ... - postmap: dict_tcp_lookup: send: get joe@dev.bitmask.net - postmap: dict_tcp_lookup: recv: 200 f01bc1c70de7d7d80bc1ad77d987e73a - postmap: dict_tcp_lookup: found: f01bc1c70de7d7d80bc1ad77d987e73a - f01bc1c70de7d7d80bc1ad77d987e73a - ... - - - -Mailspool ---------- - -* Any file in the mailspool longer for a few seconds ? - - - ls -la /var/mail/vmail/Maildir/cur/ - - -VPN node -======== - -Places to look for errors -------------------------- - -* `/var/log/syslog` (watch out for openvpn issues) - - diff --git a/doc/troubleshooting/en.haml b/doc/troubleshooting/en.haml new file mode 100644 index 00000000..a4b44939 --- /dev/null +++ b/doc/troubleshooting/en.haml @@ -0,0 +1,5 @@ +- @title = "Troubleshooting" + +%h1.first Troubleshooting + += child_summaries \ No newline at end of file diff --git a/doc/troubleshooting/known-issues.md b/doc/troubleshooting/known-issues.md new file mode 100644 index 00000000..b924fa4b --- /dev/null +++ b/doc/troubleshooting/known-issues.md @@ -0,0 +1,77 @@ +@title = 'Leap Platform Release Notes' +@nav_title = 'Known issues' +@summary = 'Known issues in the Leap Platform.' +@toc = true + +Here you can find documentation about known issues and potential work-arounds in the current Leap Platform release. + +0.6.0 +===== + +openvpn +------- +. On deployment to a openvpn node, if the following happens: + + - err: /Stage[main]/Site_openvpn/Service[openvpn]/ensure: change from stopped to running failed: Could not start Service[openvpn]: Execution of '/etc/init.d/openvpn start' returned 1: at /srv/leap/puppet/modules/site_openvpn/manifests/init.pp:189 + +this is likely the result of a kernel upgrade that happened during the deployment, requiring that the machine be restarted before this service can start. To confirm this, login to the node (leap ssh ) and look at the end of the /var/log/daemon.log: + + # tail /var/log/daemon.log + Nov 22 19:04:15 snail ovpn-udp_config[16173]: ERROR: Cannot open TUN/TAP dev /dev/net/tun: No such device (errno=19) + Nov 22 19:04:15 snail ovpn-udp_config[16173]: Exiting due to fatal error + +if you see this error, simply restart the node. + +CouchDB +------- +. You can't deploy new couchdb nodes after one or more have been deployed. Make *sure* that you configure and deploy all your couchdb nodes when starting the provider. The problem is that we dont not have a clean way of adding couch nodes after initial creation of the databases, so any nodes added after result in improperly synchronized data. See Bug [#5601](https://leap.se/code/issues/5601) for more information. + +. In some scenarios, such as when certain components are unavailable, the couchdb syncing will be broken. When things are brought back to normal, shortly after restart, the nodes will attempt to resync all their data, and can fail to complete this process because they run out of file descriptors. A symptom of this is the webapp wont allow you to register or login, the /opt/bigcouch/var/log/bigcouch.log is huge with a lot of errors that include (over multiple lines): {error, emfile}}. We have raised the limits for available file descriptors to bigcouch to try and accommodate for this situation, but if you still experience it, you may need to increase your /etc/sv/bigcouch/run ulimit values and restart bigcouch while monitoring the open file descriptors. We hope that in the next platform release, a newer couchdb will be better at handling these resources. + +You can also see the number of file descriptors in use by doing: + + # watch -n1 -d lsof -p `pidof beam`|wc -l + +User setup and ssh +------------------ + +. if you aren't using a single ssh key, but have different ones, you will need to define the following at the top of your ~/.ssh/config: + HostName + IdentityFile + + (see: https://leap.se/code/issues/2946 and https://leap.se/code/issues/3002) + +. If the ssh host key changes, you need to run node init again (see: https://leap.se/en/docs/platform/guide#Working.with.SSH) + +. To remove an admin's access to your servers, please remove the directory for that user under the `users/` subdirectory in your provider directory and then remove that user's ssh keys from files/ssh/authorized_keys. When finished you *must* run a `leap deploy` to update that information on the servers. + +. At the moment, it is only possible to add an admin who will have access to all LEAP servers (see: https://leap.se/code/issues/2280) + +. leap add-user --self allows only one key - if you run that command twice with different keys, you will just replace the key with the second key. To add a second key, add it manually to files/ssh/authorized_keys (see: https://leap.se/code/issues/866) + + +Deploying +--------- + +. If you have any errors during a run, please try to deploy again as this often solves non-deterministic issues that were not uncovered in our testing. Please re-deploy with `leap -v2 deploy` to get more verbose logs and capture the complete output to provide to us for debugging. + +. If when deploying your debian mirror fails for some reason, network anomoly or the mirror itself is out of date, then platform deployment will not succeed properly. Check the mirror is up and try to deploy again when it is resolved (see: https://leap.se/code/issues/1091) + +. Deployment gives 'error: in `%`: too few arguments (ArgumentError)' - this is because you attempted to do a deploy before initializing a node, please initialize the node first and then do a deploy afterwards (see: https://leap.se/code/issues/2550) + +. This release has no ability to custom configure apt sources or proxies (see: https://leap.se/code/issues/1971) + +. When running a deploy at a verbosity level of 2 and above, you will notice puppet deprecation warnings, these are known and we are working on fixing them + +IPv6 +---- + +As of this release, IPv6 is not supported by the VPN configuration. If IPv6 is detected on your network as a client, it is blocked and instead it should revert to IPv4. We plan on adding IPv6 support in an upcoming release. + + +Special Environments +-------------------- + +. When deploying to OpenStack release "nova" or newer, you will need to do an initial deploy, then when it has finished run `leap facts update` and then deploy again (see: https://leap.se/code/issues/3020) + +. It is not possible to actually use the EIP openvpn server on vagrant nodes (see: https://leap.se/code/issues/2401) diff --git a/doc/troubleshooting/tests.md b/doc/troubleshooting/tests.md new file mode 100644 index 00000000..84064043 --- /dev/null +++ b/doc/troubleshooting/tests.md @@ -0,0 +1,33 @@ +@title = 'Tests and Monitoring' +@summary = 'Testing and monitoring your infrastructure.' +@toc = true + +## Troubleshooting Tests + +At any time, you can run troubleshooting tests on the nodes of your provider infrastructure to check to see if things seem to be working correctly. If there is a problem, these tests should help you narrow down precisely where the problem is. + +To run tests on FILTER node list: + + leap test run FILTER + +Alternately, you can run test on all nodes (probably only useful if you have pinned the environment): + + leap test + +## Monitoring + +In order to set up a monitoring node, you simply add a `monitor` service tag to the node configuration file. It could be combined with any other service, but we propose that you add it to the webapp node, as this already is public accessible via HTTPS. + +After deploying, this node will regularly poll every node to ask for the status of various health checks. These health checks include the checks run with `leap test`, plus many others. + +We use [Nagios](http://www.nagios.org/) together with [Check MK agent](https://en.wikipedia.org/wiki/Check_MK) for running checks on remote hosts. + +You can log into the monitoring web interface via [https://MONITORNODE/nagios3/](https://MONITORNODE/nagios3/). The username is `nagiosadmin` and the password is found in the secrets.json file in your provider directory. + +### Log Monitoring + +At the moment, we use [check-mk-agent-logwatch](https://mathias-kettner.de/checkmk_check_logwatch.html) for searching logs for irregularities. +Logs are parsed for patterns using a blacklist, and are stored in `/var/lib/check_mk/logwatch/`. + +In order to "acknowledge" a log warning, you need to log in to the monitoring server, and delete the corresponding file in `/var/lib/check_mk/logwatch/`. This should be done via the nagios webinterface in the future. + diff --git a/doc/troubleshooting/where-to-look.md b/doc/troubleshooting/where-to-look.md new file mode 100644 index 00000000..fbd95931 --- /dev/null +++ b/doc/troubleshooting/where-to-look.md @@ -0,0 +1,249 @@ +@title = 'Where to look for errors' +@nav_title = 'Where to look' +@toc = true + + +General +======= + +* Please increase verbosity when debugging / filing issues in our issue tracker. You can do this with adding i.e. `-v 5` after the `leap` cmd, i.e. `leap -v 2 deploy`. + +Webapp +====== + +Places to look for errors +------------------------- + +* `/var/log/apache2/error.log` +* `/srv/leap/webapp/log/production.log` +* `/var/log/syslog` (watch out for stunnel issues) +* `/var/log/leap/*` + +Is haproxy ok ? +--------------- + + + curl -s -X GET "http://127.0.0.1:4096" + +Is couchdb accessible through stunnel ? +--------------------------------------- + +* Depending on how many couch nodes you have, increase the port for every test + (see /etc/haproxy/haproxy.cfg for the server/port mapping): + + + curl -s -X GET "http://127.0.0.1:4000" + curl -s -X GET "http://127.0.0.1:4001" + ... + + +Check couchdb acl as admin +-------------------------- + + mkdir /etc/couchdb + cat /srv/leap/webapp/config/couchdb.yml.admin # see username and password + echo "machine 127.0.0.1 login admin password " > /etc/couchdb/couchdb-admin.netrc + chmod 600 /etc/couchdb/couchdb-admin.netrc + + curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4096" + curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4096/_all_dbs" + +Check couchdb acl as unpriviledged user +--------------------------------------- + + cat /srv/leap/webapp/config/couchdb.yml # see username and password + echo "machine 127.0.0.1 login webapp password " > /etc/couchdb/couchdb-webapp.netrc + chmod 600 /etc/couchdb/couchdb-webapp.netrc + + curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4096" + curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4096/_all_dbs" + + +Check client config files +------------------------- + + https://example.net/provider.json + https://example.net/1/config/smtp-service.json + https://example.net/1/config/soledad-service.json + https://example.net/1/config/eip-service.json + + +Soledad +======= + + /var/log/soledad.log + + +Couchdb +======= + +Places to look for errors +------------------------- + +* `/opt/bigcouch/var/log/bigcouch.log` +* `/var/log/syslog` (watch out for stunnel issues) + + + +Bigcouch membership +------------------- + +* All nodes configured for the provider should appear here: + +
+    curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET 'http://127.0.0.1:5986/nodes/_all_docs'
+
+ +* All configured nodes should show up under "cluster_nodes", and the ones online and communicating with each other should appear under "all_nodes". This example output shows the configured cluster nodes `couch1.bitmask.net` and `couch2.bitmask.net`, but `couch2.bitmask.net` is currently not accessible from `couch1.bitmask.net` + + +
+    curl -s --netrc-file /etc/couchdb/couchdb.netrc 'http://127.0.0.1:5984/_membership'
+    {"all_nodes":["bigcouch@couch1.bitmask.net"],"cluster_nodes":["bigcouch@couch1.bitmask.net","bigcouch@couch2.bitmask.net"]}
+
+ +* Sometimes a `/etc/init.d/bigcouch restart` on all nodes is needed, to register new nodes + +Databases +--------- + +* Following output shows all neccessary DBs that should be present. Note that the `user-0123456....` DBs are the data stores for a particular user. + +
+    curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET 'http://127.0.0.1:5984/_all_dbs'
+    ["customers","identities","sessions","shared","tickets","tokens","user-0","user-9d34680b01074c75c2ec58c7321f540c","user-9d34680b01074c75c2ec58c7325fb7ff","users"]
+
+ + + + +Design Documents +---------------- + +* Is User `_design doc` available ? + + +
+    curl -s --netrc-file /etc/couchdb/couchdb.netrc -X  GET "http://127.0.0.1:5984/users/_design/User"
+
+ +Is couchdb cluster backend accessible through stunnel ? +------------------------------------------------------- + +* Find out how many connections are set up for the couchdb cluster backend: + +
+    grep "accept = 127.0.0.1" /etc/stunnel/*
+
+ + +* Now connect to all of those local endpoints to see if they up. All these tests should return "localhost [127.0.0.1] 4000 (?) open" + +
+    nc -v 127.0.0.1 4000
+    nc -v 127.0.0.1 4001
+    ...
+
+ + +MX +== + +Places to look for errors +------------------------- + +* `/var/log/mail.log` +* `/var/log/leap_mx.log` +* `/var/log/syslog` (watch out for stunnel issues) + +Is couchdb accessible through stunnel ? +--------------------------------------- + +* Depending on how many couch nodes you have, increase the port for every test + (see /etc/haproxy/haproxy.cfg for the server/port mapping): + + + curl -s -X GET "http://127.0.0.1:4000" + curl -s -X GET "http://127.0.0.1:4001" + ... + +Query leap-mx +------------- + +* for useraccount + + +
+    postmap -v -q  "joe@dev.bitmask.net" tcp:localhost:2244
+    ...
+    postmap: dict_tcp_lookup: send: get jow@dev.bitmask.net
+    postmap: dict_tcp_lookup: recv: 200
+    ...
+
+ +* for mailalias + + +
+    postmap -v -q  "joe@dev.bitmask.net" tcp:localhost:4242
+    ...
+    postmap: dict_tcp_lookup: send: get joe@dev.bitmask.net
+    postmap: dict_tcp_lookup: recv: 200 f01bc1c70de7d7d80bc1ad77d987e73a
+    postmap: dict_tcp_lookup: found: f01bc1c70de7d7d80bc1ad77d987e73a
+    f01bc1c70de7d7d80bc1ad77d987e73a
+    ...
+
+ + +Check couchdb acl as unpriviledged user +--------------------------------------- + + + + cat /etc/leap/mx.conf # see username and password + echo "machine 127.0.0.1 login leap_mx password " > /etc/couchdb/couchdb-leap_mx.netrc + chmod 600 /etc/couchdb/couchdb-leap_mx.netrc + + curl -s --netrc-file /etc/couchdb/couchdb-leap_mx.netrc -X GET "http://127.0.0.1:4096/_all_dbs" # pick one "user-" db + curl -s --netrc-file /etc/couchdb/couchdb-leap_mx.netrc -X GET "http://127.0.0.1:4096/user-de9c77a3d7efbc779c6c20da88e8fb9c" + + +* you may check multiple times, cause 127.0.0.1:4096 is haproxy load-balancing the different couchdb nodes + + +Mailspool +--------- + +* Any file in the leap_mx mailspool longer for a few seconds ? + + + +
+    ls -la /var/mail/vmail/Maildir/cur/
+
+ +* Any mails in postfix mailspool longer than a few seconds ? + +
+    mailq
+
+ + + +Testing mail delivery +--------------------- + + swaks -f alice@example.org -t bob@example.net -s mx1.example.net --port 25 + swaks -f varac@cdev.bitmask.net -t varac@cdev.bitmask.net -s chipmonk.cdev.bitmask.net --port 465 --tlsc + swaks -f alice@example.org -t bob@example.net -s mx1.example.net --port 587 --tls + + +VPN +=== + +Places to look for errors +------------------------- + +* `/var/log/syslog` (watch out for openvpn issues) + + diff --git a/doc/tutorials/en.haml b/doc/tutorials/en.haml new file mode 100644 index 00000000..72a8a1fc --- /dev/null +++ b/doc/tutorials/en.haml @@ -0,0 +1,5 @@ +- @title = "Tutorials" + +%h1.first Platform Tutorials + += child_summaries \ No newline at end of file diff --git a/doc/tutorials/quick-start.md b/doc/tutorials/quick-start.md new file mode 100644 index 00000000..f45574df --- /dev/null +++ b/doc/tutorials/quick-start.md @@ -0,0 +1,389 @@ +@title = 'LEAP Platform Quick Start' +@nav_title = 'Quick Start' +@summary = 'Three node OpenVPN provider.' + +Quick Start +=========== + +This tutorial walks you through the initial process of creating and deploying a minimal service provider running the [LEAP platform](platform). This Quick Start guide will guide you through building a three node OpenVPN provider. + +Our goal +------------------ + +We are going to create a minimal LEAP provider offering OpenVPN service. This basic setup can be expanded by adding more OpenVPN nodes to increase capacity, or more webapp and couchdb nodes to increase availability (performance wise, a single couchdb and a single webapp are more than enough for most usage, since they are only lightly used, but you might want redundancy). Please note: currently it is not possible to safely add additional couchdb nodes at a later point. They should all be added in the beginning, so please consider carefully if you would like more before proceeding. + +Our goal is something like this: + + $ leap list + NODES SERVICES TAGS + cheetah couchdb production + wildebeest webapp production + ostrich openvpn production + +NOTE: You won't be able to run that `leap list` command yet, not until we actually create the node configurations. + +Requirements +------------ + +In order to complete this Quick Start, you will need a few things: + +* You will need three real or paravirtualized virtual machines (KVM, Xen, Openstack, Amazon, but not Vagrant - sorry) that have a basic Debian Stable installed. If you allocate 20G of disk space to each node for the system, after this process is completed, you will have used less than 10% of that disk space. If you allocate 2 CPUs and 8G of memory to each node, that should be more than enough to begin with. +* You should be able to SSH into them remotely, and know their root password, IP addresses and their SSH host keys +* You will need four different IPs. Each node gets a primary IP, and the OpenVPN gateway additionally needs a gateway IP. +* The ability to create/modify DNS entries for your domain is preferable, but not needed. If you don't have access to DNS, you can workaround this by modifying your local resolver, i.e. editing `/etc/hosts`. +* You need to be aware that this process will make changes to your systems, so please be sure that these machines are a basic install with nothing configured or running for other purposes +* Your machines will need to be connected to the internet, and not behind a restrictive firewall. +* You should work locally on your laptop/workstation (one that you trust and that is ideally full-disk encrypted) while going through this guide. This is important because the provider configurations you are creating contain sensitive data that should not reside on a remote machine. The `leap` command will login to your servers and configure the services. +* You should do everything described below as an unprivileged user, and only run those commands as root that are noted with *sudo* in front of them. Other than those commands, there is no need for privileged access to your machine, and in fact things may not work correctly. + +All the commands in this tutorial are run on your sysadmin machine. In order to complete the tutorial, the sysadmin will do the following: + +* Install pre-requisites +* Install the LEAP command-line utility +* Check out the LEAP platform +* Create a provider and its certificates +* Setup the provider's nodes and the services that will reside on those nodes +* Initialize the nodes +* Deploy the LEAP platform to the nodes +* Test that things worked correctly +* Some additional commands + +We will walk you through each of these steps. + + +Prepare your environment +======================== + +There are a few things you need to setup before you can get going. Just some packages, the LEAP cli and the platform. + +Install pre-requisites +-------------------------------- + +*Debian & Ubuntu* + +Install core prerequisites: + + $ sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make bzip2 + + + +NOTE: leap_cli requires ruby 1.9 or later. + + +Install the LEAP command-line utility +------------------------------------------------- + +Install the `leap` command from rubygems.org: + + $ sudo gem install leap_cli + +Alternately, you can install `leap` from source: + + $ git clone https://leap.se/git/leap_cli + $ cd leap_cli + $ rake build + $ sudo rake install + +You can also install from source as an unprivileged user, if you want. For example, instead of `sudo rake install` you can do something like this: + + $ rake install + # watch out for the directory leap is installed to, then i.e. + $ sudo ln -s ~/.gem/ruby/1.9.1/bin/leap /usr/local/bin/leap + +With either `rake install` or `sudo rake install`, you can use now /usr/local/bin/leap, which in most cases will be in your $PATH. + +If you have successfully installed the `leap` command, then you should be able to do the following: + + $ leap --help + +This will list the command-line help options. If you receive an error when doing this, please read through the README.md in the `leap_cli` source to try and resolve any problems before going forwards. + +Check out the platform +-------------------------- + +The LEAP Platform is a series of puppet recipes and modules that will be used to configure your provider. You will need a local copy of the platform that will be used to setup your nodes and manage your services. To begin with, you will not need to modify the LEAP Platform. + +First we'll create a directory for LEAP things, and then we'll check out the platform code and initalize the modules: + + $ mkdir ~/leap + $ cd ~/leap + $ git clone https://leap.se/git/leap_platform.git + $ cd leap_platform + $ git submodule sync; git submodule update --init + + +Provider Setup +============== + +A provider instance is a directory tree, usually stored in git, that contains everything you need to manage an infrastructure for a service provider. In this case, we create one for example.org and call the instance directory 'example'. + + $ mkdir -p ~/leap/example + +Bootstrap the provider +----------------------- + +Now, we will initialize this directory to make it a provider instance. Your provider instance will need to know where it can find the local copy of the git repository leap_platform, which we setup in the previous step. + + $ cd ~/leap/example + $ leap new . + +NOTES: + . make sure you include that trailing dot! + +The `leap new` command will ask you for several required values: + +* domain: The primary domain name of your service provider. In this tutorial, we will be using "example.org". +* name: The name of your service provider (we use "Example"). +* contact emails: A comma separated list of email addresses that should be used for important service provider contacts (for things like postmaster aliases, Tor contact emails, etc). +* platform: The directory where you have a copy of the `leap_platform` git repository checked out. + +You could also have passed these configuration options on the command-line, like so: + + $ leap new --contacts your@email.here --domain leap.example.org --name Example --platform=~/leap/leap_platform . + +You may want to poke around and see what is in the files we just created. For example: + + $ cat provider.json + +Optionally, commit your provider directory using the version control software you fancy. For example: + + $ git init + $ git add . + $ git commit -m "initial provider commit" + +Now add yourself as a privileged sysadmin who will have access to deploy to servers: + + $ leap add-user --self + +NOTE: in most cases, `leap` must be run from within a provider instance directory tree (e.g. ~/leap/example). + +Create provider certificates +---------------------------- + +Create two certificate authorities, one for server certs and one for client +certs (note: you only need to run this one command to get both): + + $ leap cert ca + +Create a temporary cert for your main domain (you should replace with a real commercial cert at some point) + + $ leap cert csr + +To see details about the keys and certs that the prior two commands created, you can use `leap inspect` like so: + + $ leap inspect files/ca/ca.crt + +Create the Diffie-Hellman parameters file, needed for forward secret OpenVPN ciphers: + + $ leap cert dh + +NOTE: the files `files/ca/*.key` are extremely sensitive and must be carefully protected. The other key files are much less sensitive and can simply be regenerated if needed. + + +Edit provider.json configuration +-------------------------------------- + +There are a few required settings in provider.json. At a minimum, you must have: + + { + "domain": "example.org", + "name": "Example", + "contacts": { + "default": "email1@example.org" + } + } + +For a full list of possible settings, you can use `leap inspect` to see how provider.json is evaluated after including the inherited defaults: + + $ leap inspect provider.json + + +Setup the provider's nodes and services +--------------------------------------- + +A "node" is a server that is part of your infrastructure. Every node can have one or more services associated with it. Some nodes are "local" and used only for testing, see [Development](development) for more information. + +Create a node, with the service "webapp": + + $ leap node add wildebeest ip_address:x.x.x.w services:webapp tags:production + +NOTE: replace x.x.x.w with the actual IP address of this node + +This created a node configuration file in `nodes/wildebeest.json`, but it did not do anything else. It also added the 'tag' called 'production' to this node. Tags allow us to conveniently group nodes together. When creating nodes, you should give them the tag 'production' if the node is to be used in your production infrastructure. + +The web application and the VPN nodes require a database, so lets create the database server node: + + $ leap node add cheetah ip_address:x.x.x.x services:couchdb tags:production + +NOTE: replace x.x.x.x with the actual IP address of this node + +Now we need the OpenVPN gateway, so lets create that node: + + $ leap node add ostrich ip_address:x.x.x.y openvpn.gateway_address:x.x.x.z services:openvpn tags:production + +NOTE: replace x.x.x.y with the IP address of the machine, and x.x.x.z with the second IP. openvpn gateways must be assigned two IP addresses, one for the host itself and one for the openvpn gateway. We do this to prevent incoming and outgoing VPN traffic on the same IP. Without this, the client might send some traffic to other VPN users in the clear, bypassing the VPN. + + +Setup DNS +--------- + +Now that you have the nodes configured, you should create the DNS entries for these nodes. + +Set up your DNS with these hostnames: + + $ leap list --print ip_address,domain.full,dns.aliases + cheetah x.x.x.w, cheetah.example.org, null + wildebeest x.x.x.x, wildebeest.example.org, api.example.org + ostrich x.x.x.y, ostrich.example.org, null + +Alternately, you can adapt this zone file snippet: + + $ leap compile zone + +If you cannot edit your DNS zone file, you can still test your provider by adding entries to your local resolver hosts file (`/etc/hosts` for linux): + + x.x.x.w cheetah.example.org + x.x.x.x wildebeest.example.org api.example.org example.org + x.x.x.y ostrich.example.org + +Please don't forget about these entries, they will override DNS queries if you setup your DNS later. + + +Initialize the nodes +-------------------- + +Node initialization only needs to be done once, but there is no harm in doing it multiple times: + + $ leap node init production + +This will initialize all nodes with the tag "production". When `leap node init` is run, you will be prompted to verify the fingerprint of the SSH host key and to provide the root password of the server(s). You should only need to do this once. + +If you prefer, you can initalize each node, one at a time: + + $ leap node init wildebeest + $ leap node init cheetah + $ leap node init ostrich + +Deploy the LEAP platform to the nodes +-------------------- + +Now you should deploy the platform recipes to the nodes. [Deployment can take a while to run](http://xkcd.com/303/), especially on the first run, as it needs to update the packages on the new machine. + +*Important notes:* currently nodes must be deployed in a certain order. The underlying couch database node(s) must be deployed first, and then all other nodes. Also you need to configure and deploy all of the couchdb nodes that you plan to use at this time, as currently you cannot add more of them later later ([See](https://leap.se/es/docs/platform/known-issues#CouchDB.Sync)). + + $ leap deploy cheetah + +Watch the output for any errors (in red), if everything worked fine, you should now have your first running node. If you do have errors, try doing the deploy again. + +However, to deploy our three-node openvpn setup, we need the database and LEAP web application requires a database to run, so let's deploy to the couchdb and openvpn nodes: + + $ leap deploy wildebeest + $ leap deploy ostrich + + +What is going on here? +-------------------------------------------- + +First, some background terminology: + +* **puppet**: Puppet is a system for automating deployment and management of servers (called nodes). +* **hiera files**: In puppet, you can use something called a 'hiera file' to seed a node with a few configuration values. In LEAP, we go all out and put *every* configuration value needed for a node in the hiera file, and automatically compile a custom hiera file for each node. + +When you run `leap deploy`, a bunch of things happen, in this order: + +1. **Compile hiera files**: The hiera configuration file for each node is compiled in YAML format and saved in the directory `hiera`. The source material for this hiera file consists of all the JSON configuration files imported or inherited by the node's JSON config file. +* **Copy required files to node**: All the files needed for puppet to run are rsync'ed to each node. This includes the entire leap_platform directory, as well as the node's hiera file and other files needed by puppet to set up the node (keys, binary files, etc). +* **Puppet is run**: Once the node is ready, leap connects to the node via ssh and runs `puppet apply`. Puppet is applied locally on the node, without a daemon or puppetmaster. + +You can run `leap -v2 deploy` to see exactly what commands are being executed. + + +Test that things worked correctly +================================= + +You should now have three machines with the LEAP platform deployed to them, one for the web application, one for the database and one for the OpenVPN gateway. + +To run troubleshooting tests: + + leap test + +If you want to confirm for yourself that things are working, you can perform the following manual tests. + +### Access the web application + +In order to connect to the web application in your browser, you need to point your domain at the IP address of the web application node (named wildebeest in this example). + +There are a lot of different ways to do this, but one easy way is to modify your `/etc/hosts` file. First, find the IP address of the webapp node: + + $ leap list webapp --print ip_address + +Then modify `/etc/hosts` like so: + + x.x.x.w leap.example.org + +Replacing 'leap.example.org' with whatever you specified as the `domain` in the `leap new` command. + +Next, you can connect to the web application either using a web browser or via the API using the LEAP client. To use a browser, connect to https://leap.example.org (replacing that with your domain). Your browser will complain about an untrusted cert, but for now just bypass this. From there, you should be able to register a new user and login. + +### Use the VPN + +You should be able to simply test that the OpenVPN gateway works properly by doing the following: + + $ leap test init + $ sudo openvpn test/openvpn/production_unlimited.ovpn + +Or, you can use the LEAP client (called "bitmask") to connect to your new provider, create a user and then connect to the VPN. + + +Additional information +====================== + +It is useful to know a few additional things. + +Useful commands +--------------- + +Here are a few useful commands you can run on your new local nodes: + +* `leap ssh wildebeest` -- SSH into node wildebeest (requires `leap node init wildebeest` first). +* `leap list` -- list all nodes. +* `leap list production` -- list only those nodes with the tag 'production' +* `leap list --print ip_address` -- list a particular attribute of all nodes. +* `leap cert update` -- generate new certificates if needed. + +See the full command reference for more information. + +Node filters +------------------------------------------- + +Many of the `leap` commands take a "node filter". You can use a node filter to target a command at one or more nodes. + +A node filter consists of one or more keywords, with an optional "+" before each keyword. + +* keywords can be a node name, a service type, or a tag. +* the "+" before the keyword constructs an AND condition +* otherwise, multiple keywords together construct an OR condition + +Examples: + +* `leap list openvpn` -- list all nodes with service openvpn. +* `leap list openvpn +production` -- only nodes of service type openvpn AND tag production. +* `leap deploy webapp openvpn` -- deploy to all webapp OR openvpn nodes. +* `leap node init ostrich` -- just init the node named ostrich. + +Keep track of your provider configurations +------------------------------------------ + +You should commit your provider changes to your favorite VCS whenever things change. This way you can share your configurations with other admins, all they have to do is to pull the changes to stay up to date. Every time you make a change to your provider, such as adding nodes, services, generating certificates, etc. you should add those to your VCS, commit them and push them to where your repository is hosted. + +Note that your provider directory contains secrets! Those secrets include passwords for various services. You do not want to have those passwords readable by the world, so make sure that wherever you are hosting your repository, it is not public for the world to read. + +What's next +----------------------------------- + +Read the [LEAP platform guide](guide) to learn about planning and securing your infrastructure. + diff --git a/doc/tutorials/single-node.md b/doc/tutorials/single-node.md new file mode 100644 index 00000000..02d35c7a --- /dev/null +++ b/doc/tutorials/single-node.md @@ -0,0 +1,343 @@ +@title = 'Single node tutorial' +@nav_title = 'Single node' +@summary = 'A single node email provider.' + +Quick Start - Single node setup +=============================== + +This tutorial walks you through the initial process of creating and deploying a minimal service provider running the [LEAP platform](platform). +We will guide you through building a single node mail provider. + +Our goal +------------------ + +We are going to create a minimal LEAP provider offering Email service. This basic setup can be expanded by adding more webapp and couchdb nodes to increase availability (performance wise, a single couchdb and a single webapp are more than enough for most usage, since they are only lightly used, but you might want redundancy). Please note: currently it is not possible to safely add additional couchdb nodes at a later point. They should all be added in the beginning, so please consider carefully if you would like more before proceeding. + +Our goal is something like this: + + $ leap list + NODES SERVICES TAGS + node1 couchdb, mx, soledad, webapp local + +NOTE: You won't be able to run that `leap list` command yet, not until we actually create the node configurations. + +Requirements +------------ + +In order to complete this Quick Start, you will need a few things: + +* You will need one real or paravirtualized virtual machine (Vagrant, KVM, Xen, Openstack, Amazon, …) that have a basic Debian Stable installed. +* You should be able to SSH into them remotely, and know their root password, IP addresses and their SSH host keys +* The ability to create/modify DNS entries for your domain is preferable, but not needed. If you don't have access to DNS, you can workaround this by modifying your local resolver, i.e. editing `/etc/hosts`. +* You need to be aware that this process will make changes to your systems, so please be sure that these machines are a basic install with nothing configured or running for other purposes +* Your machines will need to be connected to the internet, and not behind a restrictive firewall. +* You should work locally on your laptop/workstation (one that you trust and that is ideally full-disk encrypted) while going through this guide. This is important because the provider configurations you are creating contain sensitive data that should not reside on a remote machine. The leap cli utility will login to your servers and configure the services. +* You should do everything described below as an unprivileged user, and only run those commands as root that are noted with *sudo* in front of them. Other than those commands, there is no need for privileged access to your machine, and in fact things may not work correctly. + +All the commands in this tutorial are run on your sysadmin machine. In order to complete the tutorial, the sysadmin will do the following: + +* Install pre-requisites +* Install the LEAP command-line utility +* Check out the LEAP platform +* Create a provider and its certificates +* Setup the provider's nodes and the services that will reside on those nodes +* Initialize the nodes +* Deploy the LEAP platform to the nodes +* Test that things worked correctly +* Some additional commands + +We will walk you through each of these steps. + + +Prepare your environment +======================== + +There are a few things you need to setup before you can get going. Just some packages, the LEAP cli and the platform. + +Install pre-requisites +-------------------------------- + +*Debian & Ubuntu* + +Install core prerequisites: + + $ sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make bzip2 + + + +NOTE: leap_cli should work with ruby1.8, but has only been tested using ruby1.9. + + +Install the LEAP command-line utility +------------------------------------------------- + +Install the `leap` command from rubygems.org: + + $ sudo gem install leap_cli + +Alternately, you can install `leap` from source: + + $ git clone https://leap.se/git/leap_cli + $ cd leap_cli + $ rake build + $ sudo rake install + +You can also install from source as an unprivileged user, if you want. For example, instead of `sudo rake install` you can do something like this: + + $ rake install + # watch out for the directory leap is installed to, then i.e. + $ sudo ln -s ~/.gem/ruby/1.9.1/bin/leap /usr/local/bin/leap + +With either `rake install` or `sudo rake install`, you can use now /usr/local/bin/leap, which in most cases will be in your $PATH. + +If you have successfully installed the `leap` command, then you should be able to do the following: + + $ leap --help + +This will list the command-line help options. If you receive an error when doing this, please read through the README.md in the `leap_cli` source to try and resolve any problems before going forwards. + +Check out the platform +-------------------------- + +The LEAP Platform is a series of puppet recipes and modules that will be used to configure your provider. You will need a local copy of the platform that will be used to setup your nodes and manage your services. To begin with, you will not need to modify the LEAP Platform. +Until we have a up to date stable release we recommend using the `develop` branch of the platform and leap_cli for all features of LEAP. + +First we'll create a directory for LEAP things, and then we'll check out the platform code and initalize the modules: + + $ mkdir ~/leap + $ cd ~/leap + $ git clone https://leap.se/git/leap_platform.git + $ cd leap_platform + $ git checkout develop + $ git submodule sync; git submodule update --init + + +Provider Setup +============== + +A provider instance is a directory tree, usually stored in git, that contains everything you need to manage an infrastructure for a service provider. In this case, we create one for example.org and call the instance directory 'example'. + + $ mkdir -p ~/leap/example + +Bootstrap the provider +----------------------- + +Now, we will initialize this directory to make it a provider instance. Your provider instance will need to know where it can find the local copy of the git repository leap_platform, which we setup in the previous step. + + $ cd ~/leap/example + $ leap new . + +NOTES: + . make sure you include that trailing dot! + +The `leap new` command will ask you for several required values: + +* domain: The primary domain name of your service provider. In this tutorial, we will be using "example.org". +* name: The name of your service provider (we use "Example"). +* contact emails: A comma separated list of email addresses that should be used for important service provider contacts (for things like postmaster aliases, Tor contact emails, etc). +* platform: The directory where you have a copy of the `leap_platform` git repository checked out. + +You could also have passed these configuration options on the command-line, like so: + + $ leap new --contacts your@email.here --domain leap.example.org --name Example --platform=~/leap/leap_platform . + +You may want to poke around and see what is in the files we just created. For example: + + $ cat provider.json + +Optionally, commit your provider directory using the version control software you fancy. For example: + + $ git init + $ git add . + $ git commit -m "initial provider commit" + +Now add yourself as a privileged sysadmin who will have access to deploy to servers: + + $ leap add-user --self + +NOTE: in most cases, `leap` must be run from within a provider instance directory tree (e.g. ~/leap/example). + +Create provider certificates +---------------------------- + +Create two certificate authorities, one for server certs and one for client +certs (note: you only need to run this one command to get both): + + $ leap cert ca + +Create a temporary cert for your main domain (you should replace with a real commercial cert at some point) + + $ leap cert csr + +To see details about the keys and certs that the prior two commands created, you can use `leap inspect` like so: + + $ leap inspect files/ca/ca.crt + +NOTE: the files `files/ca/*.key` are extremely sensitive and must be carefully protected. The other key files are much less sensitive and can simply be regenerated if needed. + + +Edit provider.json configuration +-------------------------------------- + +There are a few required settings in provider.json. At a minimum, you must have: + + { + "domain": "example.org", + "name": "Example", + "contacts": { + "default": "email1@example.org" + } + } + +For a full list of possible settings, you can use `leap inspect` to see how provider.json is evaluated after including the inherited defaults: + + $ leap inspect provider.json + + +Setup the provider's node and services +-------------------------------------- + +A "node" is a server that is part of your infrastructure. Every node can have one or more services associated with it. Some nodes are "local" and used only for testing, see [Development](development) for more information. + +Create a node, with all the services needed for Email - "couchdb", "mx", "soledad" and "webapp": + + $ leap node add node1 ip_address:x.x.x.w services:couchdb,mx,soledad,webapp tags:production + +NOTE: replace x.x.x.w with the actual IP address of this node + +This created a node configuration file in `nodes/node1.json`, but it did not do anything else. It also added the 'tag' called 'production' to this node. Tags allow us to conveniently group nodes together. When creating nodes, you should give them the tag 'production' if the node is to be used in your production infrastructure. + +Setup DNS +--------- + +Now that you have the node configured, you should create the DNS entrie for this node. + +Set up your DNS with these hostnames: + + $ leap list --print ip_address,domain.full,dns.aliases + node1 x.x.x.w, node1.example.org, example.org, api.example.org, nicknym.example.org + +Alternately, you can adapt this zone file snippet: + + $ leap compile zone + +If you cannot edit your DNS zone file, you can still test your provider by adding this entry to your local resolver hosts file (`/etc/hosts` for linux): + + x.x.x.w node1.example.org example.org api.example.org nicknym.example.org + +Please don't forget about these entries, they will override DNS queries if you setup your DNS later. + + +Initialize the nodes +-------------------- + +Node initialization only needs to be done once, but there is no harm in doing it multiple times: + + $ leap node init production + +This will initialize the node with the tag "production". When `leap node init` is run, you will be prompted to verify the fingerprint of the SSH host key and to provide the root password of the server. You should only need to do this once. + + +Deploy the LEAP platform to the nodes +-------------------- + +Now you should deploy the platform recipes to the nodes. [Deployment can take a while to run](http://xkcd.com/303/), especially on the first run, as it needs to update the packages on the new machine. + + $ leap deploy + +Watch the output for any errors (in red), if everything worked fine, you should now have your first running node. If you do have errors, try doing the deploy again. + +NOTE: the output from deploying can be quite busy, so we often do them each node one by one. + +What is going on here? +-------------------------------------------- + +First, some background terminology: + +* **puppet**: Puppet is a system for automating deployment and management of servers (called nodes). +* **hiera files**: In puppet, you can use something called a 'hiera file' to seed a node with a few configuration values. In LEAP, we go all out and put *every* configuration value needed for a node in the hiera file, and automatically compile a custom hiera file for each node. + +When you run `leap deploy`, a bunch of things happen, in this order: + +1. **Compile hiera files**: The hiera configuration file for each node is compiled in YAML format and saved in the directory `hiera`. The source material for this hiera file consists of all the JSON configuration files imported or inherited by the node's JSON config file. +* **Copy required files to node**: All the files needed for puppet to run are rsync'ed to each node. This includes the entire leap_platform directory, as well as the node's hiera file and other files needed by puppet to set up the node (keys, binary files, etc). +* **Puppet is run**: Once the node is ready, leap connects to the node via ssh and runs `puppet apply`. Puppet is applied locally on the node, without a daemon or puppetmaster. + +You can run `leap -v2 deploy` to see exactly what commands are being executed. + + + + +Test that things worked correctly +================================= + +You should now one machine with the LEAP platform email service deployed to it. + + +Access the web application +-------------------------------------------- + +In order to connect to the web application in your browser, you need to point your domain at the IP address of your new node. + +Next, you can connect to the web application either using a web browser or via the API using the LEAP client. To use a browser, connect to https://leap.example.org (replacing that with your domain). Your browser will complain about an untrusted cert, but for now just bypass this. From there, you should be able to register a new user and login. + +Testing with leap_cli +--------------------- + +Use the test command to run a set of different tests: + + leap test + + +Additional information +====================== + +It is useful to know a few additional things. + +Useful commands +--------------- + +Here are a few useful commands you can run on your new local nodes: + +* `leap ssh web1` -- SSH into node web1 (requires `leap node init web1` first). +* `leap list` -- list all nodes. +* `leap list production` -- list only those nodes with the tag 'production' +* `leap list --print ip_address` -- list a particular attribute of all nodes. +* `leap cert update` -- generate new certificates if needed. + +See the full command reference for more information. + +Node filters +------------------------------------------- + +Many of the `leap` commands take a "node filter". You can use a node filter to target a command at one or more nodes. + +A node filter consists of one or more keywords, with an optional "+" before each keyword. + +* keywords can be a node name, a service type, or a tag. +* the "+" before the keyword constructs an AND condition +* otherwise, multiple keywords together construct an OR condition + +Examples: + +* `leap list openvpn` -- list all nodes with service openvpn. +* `leap list openvpn +production` -- only nodes of service type openvpn AND tag production. +* `leap deploy webapp openvpn` -- deploy to all webapp OR openvpn nodes. +* `leap node init vpn1` -- just init the node named vpn1. + +Keep track of your provider configurations +------------------------------------------ + +You should commit your provider changes to your favorite VCS whenever things change. This way you can share your configurations with other admins, all they have to do is to pull the changes to stay up to date. Every time you make a change to your provider, such as adding nodes, services, generating certificates, etc. you should add those to your VCS, commit them and push them to where your repository is hosted. + +Note that your provider directory contains secrets! Those secrets include passwords for various services. You do not want to have those passwords readable by the world, so make sure that wherever you are hosting your repository, it is not public for the world to read. + +What's next +----------------------------------- + +Read the [LEAP platform guide](guide) to learn about planning and securing your infrastructure. + diff --git a/doc/under-the-hood.md b/doc/under-the-hood.md deleted file mode 100644 index 080a153e..00000000 --- a/doc/under-the-hood.md +++ /dev/null @@ -1,37 +0,0 @@ -@title = "Under the hood" - -This page contains various details on the how the platform is implemented. You can safely ignore this page, although it may be useful if you plan to make modifications to the platform. - -Puppet Details -====================================== - -Run stages ----------- - -We use two run stages for resource ordering: - -* initial: configure hostname, apt-get update + apt-get dist-upgrade -* main: everything else - -Stage initial is run before stage main. - -see http://docs.puppetlabs.com/puppet/2.7/reference/lang_run_stages.html for run stage documentation. - -Tags ----- - -Tags are beeing used to deploy different classes. - -* leap_base: site_config::default (configure hostname + resolver, sshd, ) -* leap_slow: site_config::slow (slow: apt-get update, apt-get dist-upgrade) -* leap_service: cofigure platform service (openvpn, couchdb, etc.) - -You can pass any combination of tags, i.e. use - -* "--tags leap_base,leap_slow,leap_service" (DEFAULT): Deploy all -* "--tags leap_service": Only deploy service(s) (useful for debugging/development) -* "--tags leap_base": Only deploy basic configuration (again, useful for debugging/development) - -See http://docs.puppetlabs.com/puppet/2.7/reference/lang_tags.html for puppet tag usage. - - -- cgit v1.2.3