Merge branch 'develop' (0.5.0)

Conflicts:
	.gitignore

Change-Id: I778f3e1f1f4832f5894bc149ead67e9a4becf304

10 files changed, 324 insertions, 80 deletions

diff --git a/puppet/modules/site_couchdb/manifests/add_users.pp b/puppet/modules/site_couchdb/manifests/add_users.pp
new file mode 100644
index 00000000..f9ea7349
--- /dev/null
+++ b/puppet/modules/site_couchdb/manifests/add_users.pp
@@ -0,0 +1,54 @@
+class site_couchdb::add_users {
+
+  # Couchdb users
+
+  ## leap_mx couchdb user
+  ## read: identities
+  ## write access to user-<uuid>
+  couchdb::add_user { $site_couchdb::couchdb_leap_mx_user:
+    roles   => '["identities"]',
+    pw      => $site_couchdb::couchdb_leap_mx_pw,
+    salt    => $site_couchdb::couchdb_leap_mx_salt,
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## nickserver couchdb user
+  ## r: identities
+  ## r/w: keycache
+  couchdb::add_user { $site_couchdb::couchdb_nickserver_user:
+    roles   => '["identities","keycache"]',
+    pw      => $site_couchdb::couchdb_nickserver_pw,
+    salt    => $site_couchdb::couchdb_nickserver_salt,
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## soledad couchdb user
+  ## r/w: user-<uuid>, shared
+  ## read: tokens
+  couchdb::add_user { $site_couchdb::couchdb_soledad_user:
+    roles   => '["tokens"]',
+    pw      => $site_couchdb::couchdb_soledad_pw,
+    salt    => $site_couchdb::couchdb_soledad_salt,
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ### tapicero couchdb user
+  ### admin: needs to be able to create user-<uuid> databases
+  ### read: users
+  couchdb::add_user { $site_couchdb::couchdb_tapicero_user:
+    roles   => '["users"]',
+    pw      => $site_couchdb::couchdb_tapicero_pw,
+    salt    => $site_couchdb::couchdb_tapicero_salt,
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## webapp couchdb user
+  ## read/write: users, tokens, sessions, tickets, identities, customer
+  couchdb::add_user { $site_couchdb::couchdb_webapp_user:
+    roles   => '["tokens","identities","users"]',
+    pw      => $site_couchdb::couchdb_webapp_pw,
+    salt    => $site_couchdb::couchdb_webapp_salt,
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+}
diff --git a/puppet/modules/site_couchdb/manifests/backup.pp b/puppet/modules/site_couchdb/manifests/backup.pp
new file mode 100644
index 00000000..8b5aa6ea
--- /dev/null
+++ b/puppet/modules/site_couchdb/manifests/backup.pp
@@ -0,0 +1,23 @@
+class site_couchdb::backup {
+
+  # general backupninja config
+  backupninja::config { 'backupninja_config':
+    usecolors     => false,
+  }
+
+  # dump all DBs locally to /var/backups/couchdb once a day
+  backupninja::sh { 'couchdb_backup':
+    command_string => "cd /srv/leap/couchdb/scripts \n./couchdb_dumpall.sh"
+  }
+
+  # Deploy /etc/leap/couchdb_scripts_defaults.conf so we can exclude
+  # some databases
+
+  file { '/etc/leap/couchdb_scripts_defaults.conf':
+    source => 'puppet:///modules/site_couchdb/couchdb_scripts_defaults.conf',
+    mode   => '0644',
+    owner  => 'root',
+    group  => 'root',
+  }
+
+}
diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp b/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp
index 241a4914..97e85785 100644
--- a/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp
+++ b/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp
@@ -1,5 +1,8 @@
 class site_couchdb::bigcouch::add_nodes {
   # loop through neighbors array and add nodes
   $nodes = $::site_couchdb::bigcouch_config['neighbors']
-  couchdb::bigcouch::add_node { $nodes: }
+
+  couchdb::bigcouch::add_node { $nodes:
+    require => Couchdb::Query::Setup['localhost']
+  }
 }
diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/compaction.pp b/puppet/modules/site_couchdb/manifests/bigcouch/compaction.pp
new file mode 100644
index 00000000..84aab4ef
--- /dev/null
+++ b/puppet/modules/site_couchdb/manifests/bigcouch/compaction.pp
@@ -0,0 +1,8 @@
+class site_couchdb::bigcouch::compaction {
+  cron {
+    'compact_all_shards':
+      command     => '/srv/leap/couchdb/scripts/bigcouch_compact_all_shards.sh >> /var/log/bigcouch/compaction.log',
+      hour        => 3,
+      minute      => 17;
+  }
+}
diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp b/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp
new file mode 100644
index 00000000..aa843e2e
--- /dev/null
+++ b/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp
@@ -0,0 +1,11 @@
+class site_couchdb::bigcouch::settle_cluster {
+
+  exec { 'wait_for_couch_nodes':
+    command => '/srv/leap/bin/run_tests --test CouchDB/Are_configured_nodes_online? --retry 6 --wait 10'
+  }
+
+  exec { 'settle_cluster_membership':
+    command => '/srv/leap/bin/run_tests --test CouchDB/Is_cluster_membership_ok? --retry 6 --wait 10',
+    require => Exec['wait_for_couch_nodes']
+  }
+}
diff --git a/puppet/modules/site_couchdb/manifests/create_dbs.pp b/puppet/modules/site_couchdb/manifests/create_dbs.pp
new file mode 100644
index 00000000..41500d3a
--- /dev/null
+++ b/puppet/modules/site_couchdb/manifests/create_dbs.pp
@@ -0,0 +1,70 @@
+class site_couchdb::create_dbs {
+
+  # Couchdb databases
+
+  ### customer database
+  ### r/w: webapp,
+  couchdb::create_db { 'customers':
+    members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }",
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## identities database
+  ## r: nickserver, leap_mx - needs to be restrict with design document
+  ## r/w: webapp
+  couchdb::create_db { 'identities':
+    members => "{ \"names\": [], \"roles\": [\"identities\"] }",
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## keycache database
+  ## r/w: nickserver
+  couchdb::create_db { 'keycache':
+    members => "{ \"names\": [], \"roles\": [\"keycache\"] }",
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## sessions database
+  ## r/w: webapp
+  couchdb::create_db { 'sessions':
+    members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }",
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## shared database
+  ## r/w: soledad
+  couchdb::create_db { 'shared':
+    members => "{ \"names\": [\"$site_couchdb::couchdb_soledad_user\"], \"roles\": [] }",
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## tickets database
+  ## r/w: webapp
+  couchdb::create_db { 'tickets':
+    members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }",
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## tokens database
+  ## r: soledad - needs to be restricted with a design document
+  ## r/w: webapp
+  couchdb::create_db { 'tokens':
+    members => "{ \"names\": [], \"roles\": [\"tokens\"] }",
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## users database
+  ## r/w: webapp
+  couchdb::create_db { 'users':
+    members => "{ \"names\": [], \"roles\": [\"users\"] }",
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## messages db
+  ## store messages to the clients such as payment reminders
+  ## r/w: webapp
+  couchdb::create_db { 'messages':
+    members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }",
+    require => Couchdb::Query::Setup['localhost']
+  }
+}
diff --git a/puppet/modules/site_couchdb/manifests/designs.pp b/puppet/modules/site_couchdb/manifests/designs.pp
new file mode 100644
index 00000000..9e88de64
--- /dev/null
+++ b/puppet/modules/site_couchdb/manifests/designs.pp
@@ -0,0 +1,20 @@
+class site_couchdb::designs {
+
+  Class['site_couchdb::create_dbs']
+    -> Class['site_couchdb::designs']
+
+  file { '/srv/leap/couchdb/designs':
+    ensure  => directory,
+    source  => 'puppet:///modules/site_couchdb/designs',
+    recurse => true,
+    purge   => true,
+    mode    => '0755'
+  }
+
+  exec { '/srv/leap/couchdb/scripts/load_design_documents.sh':
+    require     => Vcsrepo['/srv/leap/couchdb/scripts'],
+    refreshonly => false
+  }
+
+}
+
diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp
index 802f3224..3614661d 100644
--- a/puppet/modules/site_couchdb/manifests/init.pp
+++ b/puppet/modules/site_couchdb/manifests/init.pp
@@ -1,83 +1,118 @@
 class site_couchdb {
   tag 'leap_service'
 
-  $x509                   = hiera('x509')
-  $key                    = $x509['key']
-  $cert                   = $x509['cert']
-  $ca                     = $x509['ca_cert']
-
-  $couchdb_config         = hiera('couch')
-  $couchdb_users          = $couchdb_config['users']
-  $couchdb_admin          = $couchdb_users['admin']
-  $couchdb_admin_user     = $couchdb_admin['username']
-  $couchdb_admin_pw       = $couchdb_admin['password']
-  $couchdb_admin_salt     = $couchdb_admin['salt']
-  $couchdb_webapp         = $couchdb_users['webapp']
-  $couchdb_webapp_user    = $couchdb_webapp['username']
-  $couchdb_webapp_pw      = $couchdb_webapp['password']
-  $couchdb_webapp_salt    = $couchdb_webapp['salt']
-  $couchdb_soledad        = $couchdb_users['soledad']
-  $couchdb_soledad_user   = $couchdb_soledad['username']
-  $couchdb_soledad_pw     = $couchdb_soledad['password']
-  $couchdb_soledad_salt   = $couchdb_soledad['salt']
-
-  $bigcouch_config        = $couchdb_config['bigcouch']
-  $bigcouch_cookie        = $bigcouch_config['cookie']
-
-  $ednp_port              = $bigcouch_config['ednp_port']
+  $couchdb_config          = hiera('couch')
+  $couchdb_users           = $couchdb_config['users']
+
+  $couchdb_admin           = $couchdb_users['admin']
+  $couchdb_admin_user      = $couchdb_admin['username']
+  $couchdb_admin_pw        = $couchdb_admin['password']
+  $couchdb_admin_salt      = $couchdb_admin['salt']
+
+  $couchdb_leap_mx         = $couchdb_users['leap_mx']
+  $couchdb_leap_mx_user    = $couchdb_leap_mx['username']
+  $couchdb_leap_mx_pw      = $couchdb_leap_mx['password']
+  $couchdb_leap_mx_salt    = $couchdb_leap_mx['salt']
+
+  $couchdb_nickserver      = $couchdb_users['nickserver']
+  $couchdb_nickserver_user = $couchdb_nickserver['username']
+  $couchdb_nickserver_pw   = $couchdb_nickserver['password']
+  $couchdb_nickserver_salt = $couchdb_nickserver['salt']
+
+  $couchdb_soledad         = $couchdb_users['soledad']
+  $couchdb_soledad_user    = $couchdb_soledad['username']
+  $couchdb_soledad_pw      = $couchdb_soledad['password']
+  $couchdb_soledad_salt    = $couchdb_soledad['salt']
+
+  $couchdb_tapicero        = $couchdb_users['tapicero']
+  $couchdb_tapicero_user   = $couchdb_tapicero['username']
+  $couchdb_tapicero_pw     = $couchdb_tapicero['password']
+  $couchdb_tapicero_salt   = $couchdb_tapicero['salt']
+
+  $couchdb_webapp          = $couchdb_users['webapp']
+  $couchdb_webapp_user     = $couchdb_webapp['username']
+  $couchdb_webapp_pw       = $couchdb_webapp['password']
+  $couchdb_webapp_salt     = $couchdb_webapp['salt']
+
+  $couchdb_backup          = $couchdb_config['backup']
+
+  $bigcouch_config         = $couchdb_config['bigcouch']
+  $bigcouch_cookie         = $bigcouch_config['cookie']
+
+  $ednp_port               = $bigcouch_config['ednp_port']
 
   class { 'couchdb':
-    bigcouch        => true,
-    admin_pw        => $couchdb_admin_pw,
-    admin_salt      => $couchdb_admin_salt,
-    bigcouch_cookie => $bigcouch_cookie,
-    ednp_port       => $ednp_port
+    bigcouch            => true,
+    admin_pw            => $couchdb_admin_pw,
+    admin_salt          => $couchdb_admin_salt,
+    bigcouch_cookie     => $bigcouch_cookie,
+    ednp_port           => $ednp_port,
+    chttpd_bind_address => '127.0.0.1'
+  }
+
+  # ensure that we don't have leftovers from previous installations
+  # where we installed the cloudant bigcouch package
+  # https://leap.se/code/issues/4971
+  class { 'couchdb::bigcouch::package::cloudant':
+    ensure => absent
   }
 
-  class { 'couchdb::bigcouch::package::cloudant': }
+  Class['site_config::default']
+    -> Class['couchdb::bigcouch::package::cloudant']
+    -> Service['shorewall']
+    -> Class['site_couchdb::stunnel']
+    -> Service['couchdb']
+    -> File['/root/.netrc']
+    -> Class['site_couchdb::bigcouch::add_nodes']
+    -> Class['site_couchdb::bigcouch::settle_cluster']
+    -> Class['site_couchdb::create_dbs']
+    -> Class['site_couchdb::add_users']
 
-  Class ['couchdb::bigcouch::package::cloudant']
-    -> Service ['couchdb']
-    -> Class ['site_couchdb::bigcouch::add_nodes']
-    -> Couchdb::Create_db['users']
-    -> Couchdb::Create_db['tokens']
-    -> Couchdb::Add_user[$couchdb_webapp_user]
-    -> Couchdb::Add_user[$couchdb_soledad_user]
+  # /etc/couchdb/couchdb.netrc is deployed by couchdb::query::setup
+  # we symlink this to /root/.netrc for couchdb_scripts (eg. backup)
+  # and makes life easier for the admin (i.e. using curl/wget without
+  # passing credentials)
+  file {
+    '/root/.netrc':
+      ensure  => link,
+      target  => '/etc/couchdb/couchdb.netrc';
 
-  class { 'site_couchdb::stunnel':
-    key  => $key,
-    cert => $cert,
-    ca   => $ca
+    '/srv/leap/couchdb':
+      ensure => directory
   }
 
-  class { 'site_couchdb::bigcouch::add_nodes': }
-
   couchdb::query::setup { 'localhost':
     user  => $couchdb_admin_user,
     pw    => $couchdb_admin_pw,
   }
 
-  # Populate couchdb
-  couchdb::add_user { $couchdb_webapp_user:
-    roles => '["auth"]',
-    pw    => $couchdb_webapp_pw,
-    salt  => $couchdb_webapp_salt
+  vcsrepo { '/srv/leap/couchdb/scripts':
+    ensure   => present,
+    provider => git,
+    source   => 'https://leap.se/git/couchdb_scripts',
+    revision => 'origin/master',
+    require  => File['/srv/leap/couchdb']
   }
 
-  couchdb::add_user { $couchdb_soledad_user:
-    roles => '["auth"]',
-    pw    => $couchdb_soledad_pw,
-    salt  => $couchdb_soledad_salt
-  }
-
-  couchdb::create_db { 'users':
-    readers => "{ \"names\": [\"$couchdb_webapp_user\"], \"roles\": [] }"
-  }
+  include site_couchdb::stunnel
+  include site_couchdb::bigcouch::add_nodes
+  include site_couchdb::bigcouch::settle_cluster
+  include site_couchdb::create_dbs
+  include site_couchdb::add_users
+  include site_couchdb::designs
+  include site_couchdb::logrotate
+  include site_couchdb::bigcouch::compaction
 
-  couchdb::create_db { 'tokens':
-    readers => "{ \"names\": [], \"roles\": [\"auth\"] }"
-  }
+  if $couchdb_backup { include site_couchdb::backup }
 
   include site_shorewall::couchdb
   include site_shorewall::couchdb::bigcouch
+
+  include site_check_mk::agent::couchdb
+  include site_check_mk::agent::tapicero
+
+  file { '/var/log/bigcouch':
+    ensure => directory
+  }
+
 }
diff --git a/puppet/modules/site_couchdb/manifests/logrotate.pp b/puppet/modules/site_couchdb/manifests/logrotate.pp
new file mode 100644
index 00000000..e1039d49
--- /dev/null
+++ b/puppet/modules/site_couchdb/manifests/logrotate.pp
@@ -0,0 +1,12 @@
+class site_couchdb::logrotate {
+
+  augeas {
+    'logrotate_bigcouch':
+      context => '/files/etc/logrotate.d/bigcouch/rule',
+      changes => [ 'set file /opt/bigcouch/var/log/*.log', 'set rotate 7',
+                   'set schedule daily', 'set compress compress',
+                   'set missingok missingok', 'set ifempty notifempty',
+                   'set copytruncate copytruncate' ]
+  }
+
+}
diff --git a/puppet/modules/site_couchdb/manifests/stunnel.pp b/puppet/modules/site_couchdb/manifests/stunnel.pp
index d982013e..91f1e3aa 100644
--- a/puppet/modules/site_couchdb/manifests/stunnel.pp
+++ b/puppet/modules/site_couchdb/manifests/stunnel.pp
@@ -1,4 +1,4 @@
-class site_couchdb::stunnel ($key, $cert, $ca) {
+class site_couchdb::stunnel {
 
   $stunnel              = hiera('stunnel')
 
@@ -18,22 +18,16 @@ class site_couchdb::stunnel ($key, $cert, $ca) {
   $ednp_server_connect  = $ednp_server['connect']
   $ednp_clients         = $stunnel['ednp_clients']
 
+
+
+  include site_config::x509::cert
+  include site_config::x509::key
+  include site_config::x509::ca
+
   include x509::variables
-  $cert_name = 'leap_couchdb'
-  $ca_name   = 'leap_ca'
-  $ca_path   = "${x509::variables::local_CAs}/${ca_name}.crt"
-  $cert_path = "${x509::variables::certs}/${cert_name}.crt"
-  $key_path  = "${x509::variables::keys}/${cert_name}.key"
-
-  # basic setup: ensure cert, key, ca files are in place, and some generic
-  # stunnel things are done
-  class { 'site_stunnel::setup':
-    cert_name => $cert_name,
-    key       => $key,
-    cert      => $cert,
-    ca_name   => $ca_name,
-    ca        => $ca
-  }
+  $ca_path   = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt"
+  $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt"
+  $key_path  = "${x509::variables::keys}/${site_config::params::cert_name}.key"
 
   # setup a stunnel server for the webapp to connect to couchdb
   stunnel::service { 'couch_server':
@@ -46,7 +40,11 @@ class site_couchdb::stunnel ($key, $cert, $ca) {
     verify     => '2',
     pid        => '/var/run/stunnel4/couchserver.pid',
     rndfile    => '/var/lib/stunnel4/.rnd',
-    debuglevel => '4'
+    debuglevel => '4',
+    require    => [
+      Class['Site_config::X509::Key'],
+      Class['Site_config::X509::Cert'],
+      Class['Site_config::X509::Ca'] ];
   }
 
 
@@ -62,7 +60,11 @@ class site_couchdb::stunnel ($key, $cert, $ca) {
     verify     => '2',
     pid        => '/var/run/stunnel4/epmd_server.pid',
     rndfile    => '/var/lib/stunnel4/.rnd',
-    debuglevel => '4'
+    debuglevel => '4',
+    require    => [
+      Class['Site_config::X509::Key'],
+      Class['Site_config::X509::Cert'],
+      Class['Site_config::X509::Ca'] ];
   }
 
   # setup stunnel clients for Erlang Port Mapper Daemon (epmd) to connect
@@ -88,7 +90,11 @@ class site_couchdb::stunnel ($key, $cert, $ca) {
     verify     => '2',
     pid        => '/var/run/stunnel4/ednp_server.pid',
     rndfile    => '/var/lib/stunnel4/.rnd',
-    debuglevel => '4'
+    debuglevel => '4',
+    require    => [
+      Class['Site_config::X509::Key'],
+      Class['Site_config::X509::Cert'],
+      Class['Site_config::X509::Ca'] ];
   }
 
   # setup stunnel clients for Erlang Distributed Node Protocol (ednp) to connect
@@ -101,4 +107,6 @@ class site_couchdb::stunnel ($key, $cert, $ca) {
   }
 
   create_resources(site_stunnel::clients, $ednp_clients, $ednp_client_defaults)
+
+  include site_check_mk::agent::stunnel
 }