From 18a3a1953802021afccf7105876de7839f152189 Mon Sep 17 00:00:00 2001 From: varac Date: Fri, 20 May 2016 08:13:11 +0200 Subject: [feat] Automatic couchdb db compaction Automatic background couchdb db compaction frees a huge amount of diskspace. - Resolves: #8118 --- puppet/modules/site_couchdb/files/local.ini | 89 +---------------------------- 1 file changed, 3 insertions(+), 86 deletions(-) (limited to 'puppet/modules/site_couchdb') diff --git a/puppet/modules/site_couchdb/files/local.ini b/puppet/modules/site_couchdb/files/local.ini index 22aa0177..b921a927 100644 --- a/puppet/modules/site_couchdb/files/local.ini +++ b/puppet/modules/site_couchdb/files/local.ini @@ -1,91 +1,8 @@ -; CouchDB Configuration Settings +; Puppet modified file !! ; Custom settings should be made in this file. They will override settings ; in default.ini, but unlike changes made to default.ini, this file won't be ; overwritten on server upgrade. -[couchdb] -;max_document_size = 4294967296 ; bytes - -[httpd] -;port = 5984 -;bind_address = 127.0.0.1 -; Options for the MochiWeb HTTP server. -;server_options = [{backlog, 128}, {acceptor_pool_size, 16}] -; For more socket options, consult Erlang's module 'inet' man page. -;socket_options = [{recbuf, 262144}, {sndbuf, 262144}, {nodelay, true}] - -; Uncomment next line to trigger basic-auth popup on unauthorized requests. -;WWW-Authenticate = Basic realm="administrator" - -; Uncomment next line to set the configuration modification whitelist. Only -; whitelisted values may be changed via the /_config URLs. To allow the admin -; to change this value over HTTP, remember to include {httpd,config_whitelist} -; itself. Excluding it from the list would require editing this file to update -; the whitelist. -;config_whitelist = [{httpd,config_whitelist}, {log,level}, {etc,etc}] - -[httpd_global_handlers] -;_google = {couch_httpd_proxy, handle_proxy_req, <<"http://www.google.com">>} - -# futon is enabled by default on bigcouch in default.ini -# we need to find another way to disable futon, it won't work disabling it here -# enable futon -#_utils = {couch_httpd_misc_handlers, handle_utils_dir_req, "/usr/share/couchdb/www"} -# disable futon -#_utils = {couch_httpd_misc_handlers, handle_welcome_req, <<"Welcome, Futon is disabled!">>} - -[couch_httpd_auth] -; If you set this to true, you should also uncomment the WWW-Authenticate line -; above. If you don't configure a WWW-Authenticate header, CouchDB will send -; Basic realm="server" in order to prevent you getting logged out. -; require_valid_user = false - -[log] -;level = debug - -[os_daemons] -; For any commands listed here, CouchDB will attempt to ensure that -; the process remains alive while CouchDB runs as well as shut them -; down when CouchDB exits. -;foo = /path/to/command -with args - -[daemons] -; enable SSL support by uncommenting the following line and supply the PEM's below. -; the default ssl port CouchDB listens on is 6984 -;httpsd = {couch_httpd, start_link, [https]} - -[ssl] -;cert_file = /etc/couchdb/server_cert.pem -;key_file = /etc/couchdb/server_key.pem -;password = somepassword -; set to true to validate peer certificates -;verify_ssl_certificates = false -; Path to file containing PEM encoded CA certificates (trusted -; certificates used for verifying a peer certificate). May be omitted if -; you do not want to verify the peer. -;cacert_file = /full/path/to/cacertf -; The verification fun (optionnal) if not specidied, the default -; verification fun will be used. -;verify_fun = {Module, VerifyFun} -;ssl_certificate_max_depth = 1 -; To enable Virtual Hosts in CouchDB, add a vhost = path directive. All requests to -; the Virual Host will be redirected to the path. In the example below all requests -; to http://example.com/ are redirected to /database. -; If you run CouchDB on a specific port, include the port number in the vhost: -; example.com:5984 = /database - -[vhosts] -;example.com = /database/ - -[update_notification] -;unique notifier name=/full/path/to/exe -with "cmd line arg" - -; To create an admin account uncomment the '[admins]' section below and add a -; line in the format 'username = password'. When you next start CouchDB, it -; will change the password to a hash (so that your passwords don't linger -; around in plain-text files). You can add more admin accounts with more -; 'username = password' lines. Don't forget to restart CouchDB after -; changing this. -;[admins] -;admin = mysecretpassword +[compactions] +_default = [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "03:00"}, {to, "05:00"}] -- cgit v1.2.3 From d0ff379fe2a43d7968b8828c8b31af5254f6f85b Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 28 Jun 2016 13:16:47 -0400 Subject: Remove bigcouch (#8056) Change-Id: I0c6e27298c63bd37de1410985d054799818c22a4 --- puppet/modules/site_couchdb/files/runit_config | 6 --- puppet/modules/site_couchdb/manifests/bigcouch.pp | 50 ---------------------- .../site_couchdb/manifests/bigcouch/add_nodes.pp | 8 ---- .../site_couchdb/manifests/bigcouch/compaction.pp | 8 ---- .../manifests/bigcouch/settle_cluster.pp | 11 ----- puppet/modules/site_couchdb/manifests/init.pp | 9 +--- puppet/modules/site_couchdb/manifests/logrotate.pp | 14 ------ puppet/modules/site_couchdb/manifests/plain.pp | 2 - puppet/modules/site_couchdb/manifests/setup.pp | 7 --- 9 files changed, 1 insertion(+), 114 deletions(-) delete mode 100644 puppet/modules/site_couchdb/files/runit_config delete mode 100644 puppet/modules/site_couchdb/manifests/bigcouch.pp delete mode 100644 puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp delete mode 100644 puppet/modules/site_couchdb/manifests/bigcouch/compaction.pp delete mode 100644 puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp delete mode 100644 puppet/modules/site_couchdb/manifests/logrotate.pp (limited to 'puppet/modules/site_couchdb') diff --git a/puppet/modules/site_couchdb/files/runit_config b/puppet/modules/site_couchdb/files/runit_config deleted file mode 100644 index 169b4832..00000000 --- a/puppet/modules/site_couchdb/files/runit_config +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -exec 2>&1 -export HOME=/home/bigcouch -ulimit -H -n 32768 -ulimit -S -n 32768 -exec chpst -u bigcouch /opt/bigcouch/bin/bigcouch diff --git a/puppet/modules/site_couchdb/manifests/bigcouch.pp b/puppet/modules/site_couchdb/manifests/bigcouch.pp deleted file mode 100644 index 2de3d4d0..00000000 --- a/puppet/modules/site_couchdb/manifests/bigcouch.pp +++ /dev/null @@ -1,50 +0,0 @@ -# sets up bigcouch on couchdb node -class site_couchdb::bigcouch { - - $config = $::site_couchdb::couchdb_config['bigcouch'] - $cookie = $config['cookie'] - $ednp_port = $config['ednp_port'] - - class { 'couchdb': - admin_pw => $::site_couchdb::couchdb_admin_pw, - admin_salt => $::site_couchdb::couchdb_admin_salt, - bigcouch => true, - bigcouch_cookie => $cookie, - ednp_port => $ednp_port, - chttpd_bind_address => '127.0.0.1' - } - - # - # stunnel must running correctly before bigcouch dbs can be set up. - # - Class['site_config::default'] - -> Class['site_config::resolvconf'] - -> Class['couchdb::bigcouch::package::cloudant'] - -> Service['shorewall'] - -> Exec['refresh_stunnel'] - -> Class['site_couchdb::setup'] - -> Class['site_couchdb::bigcouch::add_nodes'] - -> Class['site_couchdb::bigcouch::settle_cluster'] - -> Class['site_couchdb::create_dbs'] - - include site_couchdb::bigcouch::add_nodes - include site_couchdb::bigcouch::settle_cluster - include site_couchdb::bigcouch::compaction - - file { '/var/log/bigcouch': - ensure => directory - } - - file { '/etc/sv/bigcouch/run': - ensure => present, - source => 'puppet:///modules/site_couchdb/runit_config', - owner => root, - group => root, - mode => '0755', - require => Package['couchdb'], - notify => Service['couchdb'] - } - - include site_check_mk::agent::couchdb::bigcouch - -} diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp b/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp deleted file mode 100644 index c8c43275..00000000 --- a/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp +++ /dev/null @@ -1,8 +0,0 @@ -class site_couchdb::bigcouch::add_nodes { - # loop through neighbors array and add nodes - $nodes = $::site_couchdb::bigcouch::config['neighbors'] - - couchdb::bigcouch::add_node { $nodes: - require => Couchdb::Query::Setup['localhost'] - } -} diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/compaction.pp b/puppet/modules/site_couchdb/manifests/bigcouch/compaction.pp deleted file mode 100644 index 84aab4ef..00000000 --- a/puppet/modules/site_couchdb/manifests/bigcouch/compaction.pp +++ /dev/null @@ -1,8 +0,0 @@ -class site_couchdb::bigcouch::compaction { - cron { - 'compact_all_shards': - command => '/srv/leap/couchdb/scripts/bigcouch_compact_all_shards.sh >> /var/log/bigcouch/compaction.log', - hour => 3, - minute => 17; - } -} diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp b/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp deleted file mode 100644 index 820b5be2..00000000 --- a/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp +++ /dev/null @@ -1,11 +0,0 @@ -class site_couchdb::bigcouch::settle_cluster { - - exec { 'wait_for_couch_nodes': - command => '/srv/leap/bin/run_tests --test CouchDB/Are_configured_nodes_online? --retry 12 --wait 10' - } - - exec { 'settle_cluster_membership': - command => '/srv/leap/bin/run_tests --test CouchDB/Is_cluster_membership_ok? --retry 12 --wait 10', - require => Exec['wait_for_couch_nodes'] - } -} diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index c4fe6277..554bf813 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -1,4 +1,4 @@ -# entry class for configuring couchdb/bigcouch node +# entry class for configuring couchdb node # couchdb node class site_couchdb { tag 'leap_service' @@ -39,12 +39,6 @@ class site_couchdb { $couchdb_backup = $couchdb_config['backup'] $couchdb_mode = $couchdb_config['mode'] - # ensure bigcouch has been purged from the system: - # TODO: remove this check in 0.9 release - if file('/opt/bigcouch/bin/bigcouch', '/dev/null') != '' { - fail 'ERROR: BigCouch appears to be installed. Make sure you have migrated to CouchDB before proceeding. See https://leap.se/upgrade-0-8' - } - include site_couchdb::plain Class['site_config::default'] @@ -60,7 +54,6 @@ class site_couchdb { include site_couchdb::create_dbs include site_couchdb::add_users include site_couchdb::designs - include site_couchdb::logrotate if $couchdb_backup { include site_couchdb::backup } diff --git a/puppet/modules/site_couchdb/manifests/logrotate.pp b/puppet/modules/site_couchdb/manifests/logrotate.pp deleted file mode 100644 index bb8843bb..00000000 --- a/puppet/modules/site_couchdb/manifests/logrotate.pp +++ /dev/null @@ -1,14 +0,0 @@ -# configure couchdb logrotation -class site_couchdb::logrotate { - - augeas { - 'logrotate_bigcouch': - context => '/files/etc/logrotate.d/bigcouch/rule', - changes => [ - 'set file /opt/bigcouch/var/log/*.log', 'set rotate 7', - 'set schedule daily', 'set compress compress', - 'set missingok missingok', 'set ifempty notifempty', - 'set copytruncate copytruncate' ] - } - -} diff --git a/puppet/modules/site_couchdb/manifests/plain.pp b/puppet/modules/site_couchdb/manifests/plain.pp index b40fc100..710ff7ca 100644 --- a/puppet/modules/site_couchdb/manifests/plain.pp +++ b/puppet/modules/site_couchdb/manifests/plain.pp @@ -6,8 +6,6 @@ class site_couchdb::plain { chttpd_bind_address => '127.0.0.1' } - include site_check_mk::agent::couchdb::plain - # remove bigcouch leftovers from previous installations include ::site_config::remove::bigcouch diff --git a/puppet/modules/site_couchdb/manifests/setup.pp b/puppet/modules/site_couchdb/manifests/setup.pp index 710d3c1c..7477d24c 100644 --- a/puppet/modules/site_couchdb/manifests/setup.pp +++ b/puppet/modules/site_couchdb/manifests/setup.pp @@ -3,13 +3,6 @@ # class site_couchdb::setup { - # ensure that we don't have leftovers from previous installations - # where we installed the cloudant bigcouch package - # https://leap.se/code/issues/4971 - class { 'couchdb::bigcouch::package::cloudant': - ensure => absent - } - $user = $site_couchdb::couchdb_admin_user # setup /etc/couchdb/couchdb-admin.netrc for couchdb admin access -- cgit v1.2.3 From f3f78ebaf5f3fd3233bc35596fefb51f6e5ed9d9 Mon Sep 17 00:00:00 2001 From: varac Date: Mon, 13 Jun 2016 20:11:23 +0200 Subject: Notify Exec[shorewall_check] not Service[shorew..] Latest shorewall module does `shorewall check` (executed by `Exec[shorewall_check]`) so every related resource change must notify this Exec instead of `Service[shorewall]` as before. --- puppet/modules/site_couchdb/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'puppet/modules/site_couchdb') diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 554bf813..5a73ae87 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -42,7 +42,7 @@ class site_couchdb { include site_couchdb::plain Class['site_config::default'] - -> Service['shorewall'] + -> Exec['shorewall_check'] -> Exec['refresh_stunnel'] -> Class['couchdb'] -> Class['site_couchdb::setup'] -- cgit v1.2.3 From 26537fa81d6f97c6643fa41e5bfc5b8d0151049e Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 21 Jul 2016 21:10:54 -0700 Subject: fix couchdb's backupninja --- puppet/modules/site_couchdb/manifests/backup.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'puppet/modules/site_couchdb') diff --git a/puppet/modules/site_couchdb/manifests/backup.pp b/puppet/modules/site_couchdb/manifests/backup.pp index 8b5aa6ea..a9771776 100644 --- a/puppet/modules/site_couchdb/manifests/backup.pp +++ b/puppet/modules/site_couchdb/manifests/backup.pp @@ -1,8 +1,8 @@ class site_couchdb::backup { # general backupninja config - backupninja::config { 'backupninja_config': - usecolors => false, + class { 'backupninja': + usecolors => false } # dump all DBs locally to /var/backups/couchdb once a day -- cgit v1.2.3 From 5dfa7c46d09beaa15efec7248719833e9b6a9e20 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 2 Aug 2016 14:50:18 -0400 Subject: Set TCP_NODELAY option for couchdb (#8264) Mochiweb in couchdb by default sets the TCP socket option SO_NODELAY to false. This means that small data sent to the TCP socket, like the reply to a document write request (or reading a very small document), will not be sent immediately to the network - TCP will buffer it for a while hoping that it will be asked to send more data through the same socket and then send all the data at once for increased performance. Setting this increases the couchdb speed significantly. Change-Id: Ib493ef061ff62c9bdee501e44ce2b55990fe14b7 --- puppet/modules/site_couchdb/files/local.ini | 3 +++ 1 file changed, 3 insertions(+) (limited to 'puppet/modules/site_couchdb') diff --git a/puppet/modules/site_couchdb/files/local.ini b/puppet/modules/site_couchdb/files/local.ini index b921a927..a6f4d981 100644 --- a/puppet/modules/site_couchdb/files/local.ini +++ b/puppet/modules/site_couchdb/files/local.ini @@ -6,3 +6,6 @@ [compactions] _default = [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "03:00"}, {to, "05:00"}] + +[httpd] +socket_options = [{nodelay, true}] -- cgit v1.2.3 From 73ca4fe4b3bad52b1d4c6c950d06b16e2db014ae Mon Sep 17 00:00:00 2001 From: varac Date: Wed, 12 Oct 2016 16:10:30 +0200 Subject: Lint site_couchdb::setup --- puppet/modules/site_couchdb/manifests/setup.pp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'puppet/modules/site_couchdb') diff --git a/puppet/modules/site_couchdb/manifests/setup.pp b/puppet/modules/site_couchdb/manifests/setup.pp index 7477d24c..84659745 100644 --- a/puppet/modules/site_couchdb/manifests/setup.pp +++ b/puppet/modules/site_couchdb/manifests/setup.pp @@ -17,11 +17,11 @@ class site_couchdb::setup { # (i.e. using curl/wget without passing credentials) file { '/etc/couchdb/couchdb.netrc': - ensure => link, - target => "/etc/couchdb/couchdb-${user}.netrc"; + ensure => link, + target => "/etc/couchdb/couchdb-${user}.netrc"; '/root/.netrc': - ensure => link, - target => '/etc/couchdb/couchdb.netrc'; + ensure => link, + target => '/etc/couchdb/couchdb.netrc'; } # setup /etc/couchdb/couchdb-soledad-admin.netrc file for couchdb admin -- cgit v1.2.3 From 0b279d8a66293bdf5fe20a77b557055a95a66a46 Mon Sep 17 00:00:00 2001 From: varac Date: Wed, 12 Oct 2016 18:41:35 +0200 Subject: Setup couch for soledad before starting soledad When the soledad couch user is not present, soledad-server refuses to start, so we need to ensure that couch is setup correctly before starting soledad-server. see https://leap.se/code/issues/8535 --- puppet/modules/site_couchdb/manifests/add_users.pp | 3 ++- puppet/modules/site_couchdb/manifests/create_dbs.pp | 3 ++- puppet/modules/site_couchdb/manifests/setup.pp | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) (limited to 'puppet/modules/site_couchdb') diff --git a/puppet/modules/site_couchdb/manifests/add_users.pp b/puppet/modules/site_couchdb/manifests/add_users.pp index c905316b..f12c5a5e 100644 --- a/puppet/modules/site_couchdb/manifests/add_users.pp +++ b/puppet/modules/site_couchdb/manifests/add_users.pp @@ -33,7 +33,8 @@ class site_couchdb::add_users { roles => '["tokens"]', pw => $site_couchdb::couchdb_soledad_pw, salt => $site_couchdb::couchdb_soledad_salt, - require => Couchdb::Query::Setup['localhost'] + require => Couchdb::Query::Setup['localhost'], + notify => Service['soledad-server']; } ## webapp couchdb user diff --git a/puppet/modules/site_couchdb/manifests/create_dbs.pp b/puppet/modules/site_couchdb/manifests/create_dbs.pp index a2d1c655..ddfb7d65 100644 --- a/puppet/modules/site_couchdb/manifests/create_dbs.pp +++ b/puppet/modules/site_couchdb/manifests/create_dbs.pp @@ -44,7 +44,8 @@ class site_couchdb::create_dbs { ## r/w: soledad couchdb::create_db { 'shared': members => "{ \"names\": [\"${site_couchdb::couchdb_soledad_user}\"], \"roles\": [\"replication\"] }", - require => Couchdb::Query::Setup['localhost'] + require => Couchdb::Query::Setup['localhost'], + notify => Service['soledad-server']; } ## tickets database diff --git a/puppet/modules/site_couchdb/manifests/setup.pp b/puppet/modules/site_couchdb/manifests/setup.pp index 84659745..a749c628 100644 --- a/puppet/modules/site_couchdb/manifests/setup.pp +++ b/puppet/modules/site_couchdb/manifests/setup.pp @@ -33,7 +33,8 @@ class site_couchdb::setup { mode => '0400', owner => 'soledad-admin', group => 'root', - require => [ Package['couchdb'], User['soledad-admin'] ]; + require => [ Package['couchdb'], User['soledad-admin'] ], + notify => Service['soledad-server']; } } -- cgit v1.2.3