diff options
| -rw-r--r-- | CHANGES.md | 68 | ||||
| -rw-r--r-- | README.md | 10 | ||||
| -rwxr-xr-x | bin/puppet_command | 32 | ||||
| -rwxr-xr-x | puppet/modules/site_check_mk/files/agent/local_checks/couchdb/leap_couch_stats.sh | 106 | ||||
| -rw-r--r-- | puppet/modules/site_check_mk/files/extra_service_conf.mk | 8 | ||||
| -rw-r--r-- | puppet/modules/site_check_mk/manifests/agent/couchdb.pp | 7 | ||||
| -rw-r--r-- | puppet/modules/site_config/manifests/syslog.pp | 20 | ||||
| -rw-r--r-- | puppet/modules/site_static/manifests/domain.pp | 15 | ||||
| -rw-r--r-- | puppet/modules/tapicero/templates/tapicero.yaml.erb | 2 |
9 files changed, 237 insertions, 31 deletions
diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 00000000..4382463b --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,68 @@ +Platform 0.7 +------------------------------------- + +CouchDB improvements: CouchDB is not designed to handle ephemeral data, like +sessions, because documents are never really deleted (a tombstone document is +always kept to record the deletion). To overcome this limitation, we now +rotate the `sessions` and `tokens` databases monthly. The new database names +are `tokens_XXX` and `sessions_XXX` where XXX is counter since the epoch that +increments every month (not a calendar month, but a month's worth of seconds). + +Additionally, nagios monitor and `leap test run` now will create and destroy +test users in the `tmp_users` database, which will get periodically deleted +and recreated. + +Compatibility: + +* requires leap_cli version 1.7 +* requires bitmask client version >= 0.7 + +Commits: https://leap.se/git/leap_platform.git/shortlog/refs/tags/0.7.0 + +Upgrading: + +* `gem install leap_cli --version 1.7` or run leap_cli from current master branch. +* `cd leap_platform; git pull; git checkout 0.7.0` or checkout current master branch. +* `leap deploy` +* `leap db destroy --db sessions,tokens` You can ignore message about needing + to redeploy (since, in this case, we just want to permanently delete those + databases). + +New features: + +* rotating couchdb databases +* deployment logging: information on every deploy is logged to + `/var/log/leap`, including the user, leap_cli version, and platform version. +* you must now run `leap deploy --downgrade` if you want to deploy an older + version over a newer platform version. +* the install source each custom daemons (e.g. tapicero, etc) is now + configured on `common.json`. +* you can configure apt sources in common.json +* many bug fixes + +Platform 0.6 +------------------------------------- + +Compatibility: + +* requires leap_cli version 1.6 +* requires bitmask client version >= 0.5 + +Commits: https://leap.se/git/leap_platform.git/shortlog/refs/tags/0.6.0 + +New features: + +* single node deployment +* include custom puppet modules and manifests +* couch flexibility +* stunnel rework +* new debian repository structure +* dependency pinning +* leap_cli modularization +* improved cert generation +* monitoring improvements such as per-environment tooling and notifications +* tor hidden service support +* switch away from NIST curve and ensure TLSv1 is used +* tests made significantly more robust +* add support for webapp deployment to a subdomain +* many, many bugfixes and stability improvements @@ -42,7 +42,7 @@ You can't deploy new couchdb nodes after one or more have been deployed. Make *s User setup and ssh ------------------ -. if you aren't using a single ssh key, but have different ones, you will need to define the following at the top of your ~/.ssh/config: +. if you aren't using a single ssh key, but have different ones, you will need to define the following at the top of your ~/.ssh/config: HostName <ip address> IdentityFile <path to identity file> @@ -52,7 +52,7 @@ User setup and ssh . At the moment, only ECDSA ssh host keys are supported. If you get the following error: `= FAILED ssh-keyscan: no hostkey alg (must be missing an ecdsa public host key)` then you should confirm that you have the following line defined in your server's **/etc/ssh/sshd_config**: `HostKey /etc/ssh/ssh_host_ecdsa_key`. If that file doesn't exist, run `ssh-keygen -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key -N ""` in order to create it. If you made a change to your sshd_config, then you need to run `/etc/init.d/ssh restart` (see: https://leap.se/code/issues/2373) -. To remove an admin's access to your servers, please remove the directory for that user under the `users/` subdirectory in your provider directory and then remove that user's ssh keys from files/ssh/authorized_keys. When finished you *must* run a `leap deploy` to update that information on the servers. +. To remove an admin's access to your servers, please remove the directory for that user under the `users/` subdirectory in your provider directory and then remove that user's ssh keys from files/ssh/authorized_keys. When finished you *must* run a `leap deploy` to update that information on the servers. . At the moment, it is only possible to add an admin who will have access to all LEAP servers (see: https://leap.se/code/issues/2280) @@ -85,12 +85,10 @@ In order to validate the syntax and style guide compliance before you commit, see https://github.com/pixelated-project/puppet-git-hooks#installation -Changelog +Changes ========= -For a changelog of the current branch: - - git log +Read CHANGES.md or run `git log`. Authors and Credits =================== diff --git a/bin/puppet_command b/bin/puppet_command index bc4bfc5b..1e74522a 100755 --- a/bin/puppet_command +++ b/bin/puppet_command @@ -24,6 +24,7 @@ HIERA_FILE = '/etc/leap/hiera.yaml' LOG_DIR = '/var/log/leap' DEPLOY_LOG = '/var/log/leap/deploy.log' SUMMARY_LOG = '/var/log/leap/deploy-summary.log' +SUMMARY_LOG_1 = '/var/log/leap/deploy-summary.log.1' APPLY_START_STR = "STARTING APPLY" APPLY_FINISH_STR = "APPLY COMPLETE" @@ -132,7 +133,7 @@ def puppet_apply(options={}, &block) options = {:verbosity => @verbosity, :tags => @tags}.merge(options) manifest = options[:manifest] || SITE_MANIFEST modulepath = options[:module_path] || SITE_MODULES + CUSTOM_MODULES - fqdn = hiera_file['domain']['name'] + fqdn = hiera_file['domain']['full'] domain = hiera_file['domain']['full_suffix'] Dir.chdir(PUPPET_DIRECTORY) do return run("FACTER_fqdn='#{fqdn}' FACTER_domain='#{domain}' #{PUPPET_BIN} apply #{custom_parameters(options)} --modulepath='#{modulepath}' #{PUPPET_PARAMETERS} #{manifest}", &block) @@ -161,18 +162,23 @@ end # PLATFORM_RE = /\{.*platform: ([0-9\.]+)[ ,\}].*[\}$]/ def platform_version_check! - if @info["platform"] - new_version = @info["platform"].split(' ').first - if File.exists?(SUMMARY_LOG) && new_version - most_recent_line = `tail '#{SUMMARY_LOG}'`.split("\n").grep(PLATFORM_RE).last - if most_recent_line - prior_version = most_recent_line.match(PLATFORM_RE)[1] - if Gem::Version.new(prior_version) > Gem::Version.new(new_version) - log("ERROR: You are attempting to deploy platform v#{new_version} but this node uses v#{prior_version}.") - log(" Run with --downgrade if you really want to deploy an older platform version.") - exit(0) - end - end + return unless @info["platform"] + new_version = @info["platform"].split(' ').first + return unless new_version + if File.exists?(SUMMARY_LOG) && File.size(SUMMARY_LOG) != 0 + file = SUMMARY_LOG + elsif File.exists?(SUMMARY_LOG_1) && File.size(SUMMARY_LOG_1) != 0 + file = SUMMARY_LOG_1 + else + return + end + most_recent_line = `tail '#{file}'`.split("\n").grep(PLATFORM_RE).last + if most_recent_line + prior_version = most_recent_line.match(PLATFORM_RE)[1] + if Gem::Version.new(prior_version) > Gem::Version.new(new_version) + log("ERROR: You are attempting to deploy platform v#{new_version} but this node uses v#{prior_version}.") + log(" Run with --downgrade if you really want to deploy an older platform version.") + exit(0) end end end diff --git a/puppet/modules/site_check_mk/files/agent/local_checks/couchdb/leap_couch_stats.sh b/puppet/modules/site_check_mk/files/agent/local_checks/couchdb/leap_couch_stats.sh new file mode 100755 index 00000000..51a8ac52 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/local_checks/couchdb/leap_couch_stats.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# +# todo: +# - thresholds +# - couch response time +# - make CURL/URL/DBLIST_EXCLUDE vars configurable +# - move load_nagios_utils() to helper library so we can use it from multiple scripts + +start_time=$(date +%s.%N) + +CURL='curl -s --netrc-file /etc/couchdb/couchdb.netrc' +URL='http://127.0.0.1:5984' +TMPFILE=$(mktemp) +DBLIST_EXCLUDE='user-' +PREFIX='Couchdb_' + + +load_nagios_utils () { + # load the nagios utils + # in debian, the package nagios-plugins-common installs utils.sh to /usr/lib/nagios/plugins/utils.sh + utilsfn= + for d in $PROGPATH /usr/lib/nagios/plugins /usr/lib64/nagios/plugins /usr/local/nagios/libexec /opt/nagios-plugins/libexec . ; do + if [ -f "$d/utils.sh" ]; then + utilsfn=$d/utils.sh; + fi + done + if [ "$utilsfn" = "" ]; then + echo "UNKNOWN - cannot find utils.sh (part of nagios plugins)"; + exit 3; + fi + . "$utilsfn"; + STATE[$STATE_OK]='OK' + STATE[$STATE_WARNING]='Warning' + STATE[$STATE_CRITICAL]='Critical' + STATE[$STATE_UNKNOWN]='Unknown' + STATE[$STATE_DEPENDENT]='Dependend' +} + +get_global_stats_perf () { + trap "localexit=3" ERR + local localexit db_count + localexit=0 + + # get a list of all dbs + $CURL -X GET $URL/_all_dbs | json_pp | egrep -v '(\[|\])' > $TMPFILE + + db_count=$( wc -l < $TMPFILE) + excluded_db_count=$( grep -c "$DBLIST_EXCLUDE" $TMPFILE ) + + echo "db_count=$db_count|excluded_db_count=$excluded_db_count" + return ${localexit} +} + +db_stats () { + trap "localexit=3" ERR + local db db_stats doc_count del_doc_count localexit + localexit=0 + + db=$1 + perf="$perf|${db}_docs=$( $CURL -s -X GET ${URL}/$db | json_pp |grep 'doc_count' | sed 's/[^0-9]//g' )" + db_stats=$( $CURL -s -X GET ${URL}/$db | json_pp ) + + doc_count=$( echo "$db_stats" | grep 'doc_count' | grep -v 'deleted_doc_count' | sed 's/[^0-9]//g' ) + del_doc_count=$( echo "$db_stats" | grep 'doc_del_count' | sed 's/[^0-9]//g' ) + + # don't divide by zero + if [ $del_doc_count -eq 0 ] + then + del_doc_perc=0 + else + del_doc_perc=$(( del_doc_count * 100 / doc_count )) + fi + + bytes=$( echo "$db_stats" | grep disk_size | sed 's/[^0-9]//g' ) + disk_size=$( echo "scale = 2; $bytes / 1024 / 1024" | bc -l ) + + echo -n "${localexit} ${PREFIX}${db}_database ${db}_docs=$doc_count|${db}_deleted_docs=$del_doc_count|${db}_deleted_docs_percentage=${del_doc_perc}%" + printf "|${db}_disksize_mb=%02.2fmb ${STATE[localexit]}: database $db\n" "$disk_size" + + return ${localexit} +} + +# main + +load_nagios_utils + +# per-db stats +# get a list of all dbs +$CURL -X GET $URL/_all_dbs | json_pp | egrep -v '(\[|\])' > $TMPFILE + +# get list of dbs to check +dbs=$( grep -v "${DBLIST_EXCLUDE}" $TMPFILE | tr -d '\n"' | sed 's/,/ /g' ) + +for db in $dbs +do + db_stats "$db" +done + +# show global couchdb stats +global_stats_perf=$(get_global_stats_perf) +exitcode=$? + +end_time=$(date +%s.%N) +duration=$( echo "scale = 2; $end_time - $start_time" | bc -l ) + +printf "${exitcode} ${PREFIX}global_stats ${global_stats_perf}|script_duration=%02.2fs ${STATE[exitcode]}: global couchdb status\n" "$duration" diff --git a/puppet/modules/site_check_mk/files/extra_service_conf.mk b/puppet/modules/site_check_mk/files/extra_service_conf.mk index 61b0dd39..a4c6e769 100644 --- a/puppet/modules/site_check_mk/files/extra_service_conf.mk +++ b/puppet/modules/site_check_mk/files/extra_service_conf.mk @@ -5,14 +5,10 @@ extra_service_conf["max_check_attempts"] = [ ] # -# run check_mk_agent every 2 minutes if it terminates successfully. +# run check_mk_agent every 10 minutes if it terminates successfully. # see https://leap.se/code/issues/6539 for the rationale # -# update: temporarily set interval to 60 minutes until we solve the -# issue with the users db getting bloated with deleted -# test users. -# extra_service_conf["normal_check_interval"] = [ - ("60", ALL_HOSTS , "Check_MK" ) + ("10", ALL_HOSTS , "Check_MK" ) ] diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp index 2dfe16fa..abfc7ad0 100644 --- a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp @@ -45,4 +45,11 @@ class site_check_mk::agent::couchdb { require => File['/etc/check_mk/mrpe.cfg']; } + + # check different couchdb stats + file { '/usr/lib/check_mk_agent/local/leap_couch_stats.sh': + source => 'puppet:///modules/site_check_mk/agent/local_checks/couchdb/leap_couch_stats.sh', + mode => '0755', + require => Package['check_mk-agent'] + } } diff --git a/puppet/modules/site_config/manifests/syslog.pp b/puppet/modules/site_config/manifests/syslog.pp index 1b0c1e05..83b49c8e 100644 --- a/puppet/modules/site_config/manifests/syslog.pp +++ b/puppet/modules/site_config/manifests/syslog.pp @@ -15,8 +15,24 @@ action(type="mmanon" ipv4.bits="32" mode="rewrite")' augeas { 'logrotate_leap_deploy': context => '/files/etc/logrotate.d/leap_deploy/rule', - changes => [ 'set file /var/log/leap/deploy*.log', 'set rotate 7', - 'set schedule daily', 'set compress compress', + changes => [ 'set file /var/log/leap/deploy.log', + 'set rotate 5', + 'set size 1M', + 'set compress compress', + 'set missingok missingok', + 'set copytruncate copytruncate' ]; + + # NOTE: + # the puppet_command script requires the option delaycompress + # be set on the summary log file. + + 'logrotate_leap_deploy_summary': + context => '/files/etc/logrotate.d/leap_deploy_summary/rule', + changes => [ 'set file /var/log/leap/deploy-summary.log', + 'set rotate 5', + 'set size 100k', + 'set delaycompress delaycompress', + 'set compress compress', 'set missingok missingok', 'set copytruncate copytruncate' ] } diff --git a/puppet/modules/site_static/manifests/domain.pp b/puppet/modules/site_static/manifests/domain.pp index 6941b1a3..b9177f25 100644 --- a/puppet/modules/site_static/manifests/domain.pp +++ b/puppet/modules/site_static/manifests/domain.pp @@ -12,9 +12,18 @@ define site_static::domain ( create_resources(site_static::location, $locations) - x509::cert { $domain: content => $cert } - x509::key { $domain: content => $key } - x509::ca { "${domain}_ca": content => $ca_cert } + x509::cert { $domain: + content => $cert, + notify => Service[apache] + } + x509::key { $domain: + content => $key, + notify => Service[apache] + } + x509::ca { "${domain}_ca": + content => $ca_cert, + notify => Service[apache] + } apache::vhost::file { $domain: content => template('site_static/apache.conf.erb') diff --git a/puppet/modules/tapicero/templates/tapicero.yaml.erb b/puppet/modules/tapicero/templates/tapicero.yaml.erb index 40048a5e..fb3b93aa 100644 --- a/puppet/modules/tapicero/templates/tapicero.yaml.erb +++ b/puppet/modules/tapicero/templates/tapicero.yaml.erb @@ -17,7 +17,7 @@ connection: # file to store the last processed user record in so we can resume after # a restart: -seq_file: "/var/lib/leap/tapicero/tapicero.seq" +seq_dir: "/var/lib/leap/tapicero/" # Configure log_file like this if you want to log to a file instead of syslog: # log_file: "/var/leap/log/tapicero.log" |