diff options
316 files changed, 10237 insertions, 3504 deletions
diff --git a/.gitmodules b/.gitmodules index 7005b770..051117f8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -91,3 +91,6 @@ [submodule "puppet/modules/check_mk"] path = puppet/modules/check_mk url = https://leap.se/git/puppet_check_mk +[submodule "puppet/modules/systemd"] + path = puppet/modules/systemd + url = https://leap.se/git/puppet_systemd @@ -1,5 +1,51 @@ +Platform 0.8 +-------------------------------------- + +This release focuses on the email service. + +Requirements: + . You must upgrade to Debian Jessie, see below for details + . You must migrate all data from BigCouch to CouchDB + . Soledad and couchdb services must be on the same node + +WARNING: failure to migrate data from BigCouch to CouchDB will cause all user +accounts to get destroyed. See UPGRADING below for how to safely do this. + +UPGRADING: You must upgrade to Debian Jessie and migrate from BigCouch to +Couchdb. It is tricky to upgrade the OS and migrate the database, so we have +writen and tested a step-by-step guide that you can carefully follow in +doc/upgrading/upgrade-0-8.md, or online at: https://leap.se/en/upgrade-0-8 + +Other new features: + +* It is possible to require invite codes for new users signing up. + +* Tapicero has been removed. Now user storage databases are created as needed + by soledad, and deleted eventually when no longer needed. + +* Admins can now suspend/enable users and block/enable their ability to send + and receive email. + +* Support for SPF and DKIM. + +Compatibility: + +* Now, soledad and couchdb must be on the same node. +* Requires Debian Jessie. Wheezy is no longer supported. +* Requires CouchDB, BigCouch is no longer supported. +* Requires leap_cli version 1.8 +* Requires bitmask client version >= 0.9 +* Includes: + * leap_mx 0.8 + * webapp 0.8 + * soledad 0.8 + +Commits: https://leap.se/git/leap_platform.git/shortlog/refs/tags/0.8 +Issues fixed: https://leap.se/code/versions/189 + + Platform 0.7.1 ------------------------ +-------------------------------------- Compatibility: diff --git a/Gemfile b/Gemfile new file mode 100644 index 00000000..8925a904 --- /dev/null +++ b/Gemfile @@ -0,0 +1,13 @@ +source "https://rubygems.org" + +group :test do + gem "rake" + gem "rspec", '< 3.2.0' + gem "puppet", ENV['PUPPET_VERSION'] || ENV['GEM_PUPPET_VERSION'] || ENV['PUPPET_GEM_VERSION'] || '~> 3.7.0' + gem "facter", ENV['FACTER_VERSION'] || ENV['GEM_FACTER_VERSION'] || ENV['FACTER_GEM_VERSION'] || '~> 2.2.0' + gem "rspec-puppet" + gem "puppetlabs_spec_helper" + gem "metadata-json-lint" + gem "rspec-puppet-facts" + gem "mocha" +end @@ -1,6 +1,8 @@ -What is it? +Leap Platform ============================= +[](https://jenkins.leap.se/job/platform_develop/) + The LEAP Platform is set of complementary packages and server recipes to automate the maintenance of LEAP services in a hardened Debian environment. Its goal is to make it as painless as possible for sysadmins to deploy and maintain a service provider's infrastructure for secure communication. These recipes define an abstract service provider. It is a set of Puppet modules designed to work together to provide to sysadmins everything they need to manage a service provider infrastructure that provides secure communication services. Getting started diff --git a/Rakefile b/Rakefile new file mode 100644 index 00000000..8f7a9686 --- /dev/null +++ b/Rakefile @@ -0,0 +1,47 @@ +require 'puppetlabs_spec_helper/rake_tasks' +require 'puppet-lint/tasks/puppet-lint' +require 'puppet-syntax/tasks/puppet-syntax' + +# return list of modules, either +# submodules or custom modules +# so we can check each array seperately +def modules_pattern (type) + submodules = Array.new + custom_modules = Array.new + + Dir['puppet/modules/*'].sort.each do |m| + system("grep -q #{m} .gitmodules") + if $?.exitstatus == 0 + submodules << m + '/**/*.pp' + else + custom_modules << m + '/**/*.pp' + end + end + + if type == 'submodule' + submodules + elsif type == 'custom' + custom_modules + else + end + +end + + + +# redefine lint task with specific configuration +Rake::Task[:lint].clear +desc "boo" +PuppetLint::RakeTask.new :lint do |config| + # Pattern of files to check, defaults to `**/*.pp` + config.pattern = modules_pattern('custom') + config.ignore_paths = ["spec/**/*.pp", "pkg/**/*.pp", "vendor/**/*.pp"] + config.disable_checks = ['documentation', '80chars'] + config.fail_on_warnings = false +end + +# rake syntax::* tasks +PuppetSyntax.exclude_paths = ["**/vendor/**/*"] + +desc "Run all puppet checks required for CI" +task :test => [:lint, :syntax , :validate, :spec] diff --git a/Vagrantfile b/Vagrantfile index c9c68284..25f26b3b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,29 +1,53 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -Vagrant.configure("2") do |vagrant_config| - vagrant_config.vm.define :node1 do |config| - - # Please verify the sha512 sum of the downloaded box before importing it into vagrant ! - # see https://leap.se/en/docs/platform/details/development#Verify.vagrantbox.download - # for details - - config.vm.box = "LEAP/wheezy" - #config.vm.network :private_network, ip: "10.5.5.102" - config.vm.provider "virtualbox" do |v| - v.memory = 1024 - v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] - v.name = "node1" - end - - config.vm.provision "puppet" do |puppet| - puppet.manifests_path = "./vagrant" - puppet.module_path = "./puppet/modules" - puppet.manifest_file = "install-platform.pp" - puppet.options = "--verbose" - end - config.vm.provision "shell", path: "vagrant/configure-leap.sh" - - config.ssh.username = "vagrant" +Vagrant.configure("2") do |config| + + # shared config for all boxes + + # Please verify the sha512 sum of the downloaded box before importing it into vagrant ! + # see https://leap.se/en/docs/platform/details/development#Verify.vagrantbox.download + # for details + config.vm.box = "LEAP/jessie" + + config.vm.provider "virtualbox" do |v| + v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + v.name = "jessie" + v.memory = 1536 + end + + config.vm.provider "libvirt" do |v| + v.memory = 1536 + end + + # Fix annoying 'stdin: is not a tty' warning + # see http://foo-o-rama.com/vagrant--stdin-is-not-a-tty--fix.html + config.vm.provision "shell" do |s| + s.privileged = false + s.inline = "sudo sed -i '/tty/!s/mesg n/tty -s \\&\\& mesg n/' /root/.profile" + end + + config.vm.provision "puppet" do |puppet| + puppet.manifests_path = "./vagrant" + puppet.module_path = "./puppet/modules" + puppet.manifest_file = "install-platform.pp" + puppet.options = "--verbose" + puppet.hiera_config_path = "hiera.yaml" + end + config.vm.provision "shell", path: "vagrant/configure-leap.sh" + + config.ssh.username = "vagrant" + + # forward leap_web ports + config.vm.network "forwarded_port", guest: 443, host:4443 + # forward pixelated ports + config.vm.network "forwarded_port", guest: 8080, host:8080 + + config.vm.define :"leap_platform", primary: true do |leap_vagrant| + end + + config.vm.define :"pixelated", autostart: false do |pixelated_vagrant| + pixelated_vagrant.vm.provision "shell", path: "vagrant/add-pixelated.sh" end + end diff --git a/bin/debug.sh b/bin/debug.sh new file mode 100755 index 00000000..d6f37542 --- /dev/null +++ b/bin/debug.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# debug script to be run on remote servers +# called from leap_cli with the 'leap debug' cmd + +apps='(leap|pixelated|stunnel|couch|soledad|haproxy)' + +facts='(apt_running |^architecture |^augeasversion |^couchdb_.* |^debian_.* |^dhcp_enabled |^domain |^facterversion |^filesystems |^fqdn |^hardwaremodel |^hostname |^interface.* |^ipaddress.* |^is_pe |^is_virtual |^kernel.* |^lib |^lsb.* |^memory.* |^mtu_.* |^netmask.* |^network_.* |^operatingsystem |^os.* |^path |^physicalprocessorcount |^processor.* |^ps |^puppetversion |^root_home |^rsyslog_version |^rubysitedir |^rubyversion |^selinux |^ssh_version |^swapfree.* |^swapsize.* |^type |^virtual)' + + +# query facts and filter out private stuff +export FACTERLIB="/srv/leap/puppet/modules/apache/lib/facter:/srv/leap/puppet/modules/apt/lib/facter:/srv/leap/puppet/modules/concat/lib/facter:/srv/leap/puppet/modules/couchdb/lib/facter:/srv/leap/puppet/modules/rsyslog/lib/facter:/srv/leap/puppet/modules/site_config/lib/facter:/srv/leap/puppet/modules/sshd/lib/facter:/srv/leap/puppet/modules/stdlib/lib/facter" + +facter 2>/dev/null | egrep -i "$facts" + +# query installed versions +echo -e '\n\n' +dpkg -l | egrep "$apps" + + +# query running procs +echo -e '\n\n' +ps aux|egrep "$apps" + +echo -e '\n\n' +echo -e "Last deploy:\n" +tail -2 /var/log/leap/deploy-summary.log + + + diff --git a/bin/node_init b/bin/node_init new file mode 100644 index 00000000..da250012 --- /dev/null +++ b/bin/node_init @@ -0,0 +1,86 @@ +#!/bin/bash +# +# LEAP Platform node initialization. +# This script is run on the target server when `leap node init` is run. +# + +DEBIAN_VERSION="^(jessie|8\.)" +LEAP_DIR="/srv/leap" +HIERA_DIR="/etc/leap" +INIT_FILE="/srv/leap/initialized" +REQUIRED_PACKAGES="puppet rsync lsb-release locales" + +PATH="/bin:/sbin:/usr/sbin:/usr/bin" +APT_GET="apt-get -q -y -o DPkg::Options::=--force-confold" +APT_GET_UPDATE="apt-get update -o Acquire::Languages=none" +BAD_APT_RESPONSE="(BADSIG|NO_PUBKEY|KEYEXPIRED|REVKEYSIG|NODATA|Could not resolve|failed to fetch)" +export DEBIAN_FRONTEND=noninteractive + +test -f $INIT_FILE && rm $INIT_FILE +if ! egrep -q "$DEBIAN_VERSION" /etc/debian_version; then + echo "ERROR: This operating system is not supported. The file /etc/debian_version must match /$DEBIAN_VERSION/ but is: `cat /etc/debian_version`" + exit 1 +fi +mkdir -p $LEAP_DIR +echo "en_US.UTF-8 UTF-8" > /etc/locale.gen + +# +# UPDATE PACKAGES +# (exit code is not reliable, sadly) +# +echo "updating package list" + +error_count=0 +while read line; do + error=$(echo $line | egrep "$BAD_APT_RESPONSE") + if [[ $error ]]; then + errors[error_count]=$error + ((error_count++)) + break # should we halt on first error? + fi + echo $line +done < <($APT_GET_UPDATE 2>&1) + +if [[ $error_count > 0 ]]; then + echo "ERROR: fatal error in 'apt-get update', bailing out." + for e in "${errors[@]}"; do + echo " $e" + done + exit 1 +fi + +# +# UPDATE TIME +# +if [[ ! $(which ntpd) ]]; then + echo "installing ntpd" + $APT_GET install ntp + exit_code=$? + if [[ $exit_code -ne 0 ]]; then + echo "ERROR: bailing out." + exit $exit_code + fi +fi + +echo "updating server time" +systemctl -q is-active ntp.service && systemctl stop ntp.service +ntpd -gxq +systemctl -q is-active ntp.service || systemctl start ntp.service + +# +# INSTALL PACKAGES +# +echo "installing required packages" +$APT_GET install $REQUIRED_PACKAGES +exit_code=$? +if [[ $exit_code -ne 0 ]]; then + echo "ERROR: bailing out." + exit $exit_code +fi + +# +# FINALIZE +# +mkdir -p $HIERA_DIR +chmod 0755 $HIERA_DIR +touch $INIT_FILE diff --git a/bin/puppet_command b/bin/puppet_command index 1e74522a..eb3cd0b9 100755 --- a/bin/puppet_command +++ b/bin/puppet_command @@ -13,6 +13,7 @@ require 'logger' require 'socket' require 'fileutils' +DEBIAN_VERSION = /^(jessie|8\.)/ PUPPET_BIN = '/usr/bin/puppet' PUPPET_DIRECTORY = '/srv/leap' PUPPET_PARAMETERS = '--color=false --detailed-exitcodes --libdir=puppet/lib --confdir=puppet' @@ -28,7 +29,12 @@ SUMMARY_LOG_1 = '/var/log/leap/deploy-summary.log.1' APPLY_START_STR = "STARTING APPLY" APPLY_FINISH_STR = "APPLY COMPLETE" + def main + if File.read('/etc/debian_version') !~ DEBIAN_VERSION + log "ERROR: This operating system is not supported. The file /etc/debian_version must match #{DEBIAN_VERSION}." + exit 1 + end process_command_line_arguments with_lockfile do @commands.each do |command| @@ -58,9 +64,11 @@ def log(str, *args) str = str.strip $stdout.puts str $stdout.flush - $logger.info(str) - if args.include? :summary - $summary_logger.info(str) + if $logger + $logger.info(str) + if args.include? :summary + $summary_logger.info(str) + end end end diff --git a/bin/run_tests b/bin/run_tests index 8eab5286..b6784ed5 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -14,6 +14,7 @@ require 'minitest/unit' require 'yaml' require 'tsort' +require 'timeout' ## ## CONSTANTS @@ -33,7 +34,8 @@ HELPER_PATHS = [ ] TEST_PATHS = [ '../../tests/white-box/*.rb', - '/srv/leap/files/tests/white-box/*.rb' + '/srv/leap/files/tests/white-box/*.rb', + '/srv/leap/tests_custom/*.rb' ] ## @@ -80,6 +82,8 @@ end class LeapTest < MiniTest::Unit::TestCase class Pass < MiniTest::Assertion end + class SilentPass < Pass + end class Ignore < MiniTest::Assertion end @@ -114,6 +118,16 @@ class LeapTest < MiniTest::Unit::TestCase end # + # thrown Timeout::Error if test run + # takes longer than $timeout + # + def run(*args) + Timeout::timeout($timeout, Timeout::Error) do + super(*args) + end + end + + # # The default pass just does an `assert true`. In our case, we want to make the passes more explicit. # def pass @@ -121,6 +135,12 @@ class LeapTest < MiniTest::Unit::TestCase end # + # This is just like pass(), but the result is normally silent, unless `run_tests --test TEST` + def silent_pass + raise LeapTest::SilentPass + end + + # # Called when the test should be silently ignored. # def ignore @@ -143,6 +163,7 @@ class LeapTest < MiniTest::Unit::TestCase MiniTest::Unit.runner.warn(self.class, method_name, msg.join("\n")) end + # # Always runs test methods within a test class in alphanumeric order # def self.test_order @@ -208,6 +229,10 @@ class LeapRunner < MiniTest::Unit if @verbose report_line("IGNORE", klass, meth, e, e.message) end + when LeapTest::SilentPass then + if $pinned_test_method || $output_format == :checkmk + report_line("PASS", klass, meth) + end when LeapTest::Pass then @passes += 1 report_line("PASS", klass, meth) @@ -217,6 +242,12 @@ class LeapRunner < MiniTest::Unit if $halt_on_failure raise TestFailure.new end + when Timeout::Error then + @failures += 1 + report_line("TIMEOUT", klass, meth, nil, "Test stopped because timeout exceeded (#{$timeout} seconds).") + if $halt_on_failure + raise TestFailure.new + end else @errors += 1 bt = MiniTest::filter_backtrace(e.backtrace).join "\n" @@ -260,7 +291,7 @@ class LeapRunner < MiniTest::Unit def report_line(prefix, klass, meth, e=nil, message=nil) msg_txt = nil if message - message = message.sub(/http:\/\/([a-z_]+):([a-zA-Z0-9_]+)@/, "http://\\1:REDACTED@") + message = message.gsub(/http:\/\/([a-z_]+):([a-zA-Z0-9_]+)@/, "http://\\1:REDACTED@") if $output_format == :human indent = "\n " msg_txt = indent + message.split("\n").join(indent) @@ -370,13 +401,14 @@ end def print_help puts ["USAGE: run_tests [OPTIONS]", - " --continue Don't halt on an error, but continue to the next test.", - " --checkmk Print test results in checkmk format (must come before --test).", - " --test TEST Run only the test with name TEST.", - " --list-tests Prints the names of all available tests and exit.", - " --retry COUNT If the tests don't pass, retry COUNT additional times (default is zero)", - " --wait SECONDS Wait for SECONDS between retries (default is 5)", - " --debug Print out full stack trace on errors"].join("\n") + " --continue Don't halt on an error, but continue to the next test.", + " --checkmk Print test results in checkmk format (must come before --test).", + " --test TEST Run only the test with name TEST.", + " --list-tests Prints the names of all available tests and exit.", + " --retry COUNT If the tests don't pass, retry COUNT additional times (default is zero).", + " --timeout SECONDS Halt a test if it exceed SECONDS (default is 30).", + " --wait SECONDS Wait for SECONDS between retries (default is 5).", + " --debug Print out full stack trace on errors."].join("\n") exit(0) end @@ -454,6 +486,7 @@ def main $output_format = :human $retry = false $wait = 5 + $timeout = 30 loop do case ARGV[0] when '--continue' then ARGV.shift; $halt_on_failure = false; @@ -462,6 +495,7 @@ def main when '--test' then ARGV.shift; pin_test_name(ARGV.shift) when '--list-tests' then list_tests when '--retry' then ARGV.shift; $retry = ARGV.shift.to_i + when '--timeout' then ARGV.shift; $timeout = ARGV.shift.to_i; when '--wait' then ARGV.shift; $wait = ARGV.shift.to_i when '--debug' then ARGV.shift when '-d' then ARGV.shift diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 00000000..e836bc7e --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,9 @@ +# Contributed Files + +## Commit Template + +to install this commit template, use following cmd (use --global to use it in your global .gitconfig): + + git config [--global] commit.template "~/path_to_leap_platform/contrib/commit-template.txt" + + diff --git a/contrib/commit-template.txt b/contrib/commit-template.txt new file mode 100644 index 00000000..9a1fa81b --- /dev/null +++ b/contrib/commit-template.txt @@ -0,0 +1,7 @@ +#[bug|feat|docs|style|refactor|test|pkg|i18n] + +#- Tested: [local singlenode|local multinode|citest|unstable.bitmask.net] +#- Resolves: #XYZ +#- Related: #XYZ +#- Documentation: #XYZ +#- Releases: XYZ diff --git a/vagrant/offlineimaprc.example.org b/contrib/offlineimaprc.example.org index 3d119634..3d119634 100644 --- a/vagrant/offlineimaprc.example.org +++ b/contrib/offlineimaprc.example.org diff --git a/doc/common/_bigcouch_migration.md b/doc/common/_bigcouch_migration.md new file mode 100644 index 00000000..eb7e07e9 --- /dev/null +++ b/doc/common/_bigcouch_migration.md @@ -0,0 +1,117 @@ +@title = "Migrating from BigCouch to plain CouchDB" + +Here are the steps needed to replace BigCouch with CouchDB. + +At the end of this process, you will have just *one* node with `services` property equal to `couchdb`. If you had a BigCouch cluster before, you will be removing all but one of those machines to consolidate them into one CouchDB machine. + +1. if you have multiple nodes with the `couchdb` service on them, pick one of them to be your CouchDB server, and remove the service from the others. If these machines were only doing BigCouch before, you can remove the nodes completely with `leap node rm <nodename>` and then you can decommission the servers + +1. put the webapp into [[maintenance mode => webapp#maintenance-mode]] + +1. turn off daemons that access the database. For example: + + ``` + workstation$ leap ssh <each soledad-node> + server# /etc/init.d/soledad-server stop + + workstation$ leap ssh <mx-node> + server# /etc/init.d/postfix stop + server# /etc/init.d/leap-mx stop + + workstation$ leap ssh <webapp-node> + server# /etc/init.d/nickserver stop + ``` + + Alternately, you can create a temporary firewall rule to block access (run on couchdb server): + + ``` + server# iptables -A INPUT -p tcp --dport 5984 --jump REJECT + ``` + +1. remove orphaned databases and do a backup of all remaining, active databases. This can take some time and will place several hundred megabytes of data into /var/backups/couchdb. The size and time depends on how many users there are on your system. For example, 15k users took approximately 25 minutes and 308M of space: + + ``` + workstation$ leap ssh <couchdb-node> + server# cd /srv/leap/couchdb/scripts + server# ./cleanup-user-dbs + server# time ./couchdb_dumpall.sh + ``` + +1. stop bigcouch: + + ``` + server# /etc/init.d/bigcouch stop + server# pkill epmd + ``` + +1. remove bigcouch: + + ``` + server# apt-get remove bigcouch + ``` + +1. configure your couch node to use plain couchdb instead of bigcouch, you can do this by editing nodes/<couch-node>.json, look for this section: + + ``` + "couch": { + "mode": "plain" + } + ``` + + change it, so it looks like this instead: + + ``` + "couch": { + "mode": "plain", + "pwhash_alg": "pbkdf2" + } + ``` + +1. deploy to the couch node: + + ``` + workstation$ leap deploy <couchdb-node> + ``` + + If you used the iptables method of blocking access to couchdb, you need to run it again because the deploy just overwrote all the iptables rules: + + ``` + server# iptables -A INPUT -p tcp --dport 5984 --jump REJECT + ``` + +1. restore the backup, this will take approximately the same amount of time as the backup took above: + + ``` + server# cd /srv/leap/couchdb/scripts + server# time ./couchdb_restoreall.sh + ``` + +1. start services again that were stopped in the beginning: + + ``` + workstation$ leap ssh soledad-nodes + server# /etc/init.d/soledad-server start + + workstation$ leap ssh mx-node + server# /etc/init.d/postfix start + server# /etc/init.d/leap-mx start + + workstation$ leap ssh webapp + server# /etc/init.d/nickserver start + ``` + + Or, alternately, if you set up the firewall rule instead, now remove it: + + ``` + server# iptables -D INPUT -p tcp --dport 5984 --jump REJECT + ``` + +1. check if everything is working, including running the test on your deployment machine: + + ``` + workstation$ leap test + ``` + +1. Remove old bigcouch data dir `/opt` after you double checked everything is in place + +1. Relax, enjoy a refreshing beverage. diff --git a/doc/common/_bigcouch_migration_begin.md b/doc/common/_bigcouch_migration_begin.md new file mode 100644 index 00000000..4e4233dd --- /dev/null +++ b/doc/common/_bigcouch_migration_begin.md @@ -0,0 +1,66 @@ +@title = "Migrating from BigCouch to plain CouchDB" + +At the end of this process, you will have just *one* node with `services` property equal to `couchdb`. If you had a BigCouch cluster before, you will be removing all but one of those machines to consolidate them into one CouchDB machine. + +1. if you have multiple nodes with the `couchdb` service on them, pick one of them to be your CouchDB server, and remove the service from the others. If these machines were only doing BigCouch before, you can remove the nodes completely with `leap node rm <nodename>` and then you can decommission the servers + +1. put the webapp into [[maintenance mode => webapp#maintenance-mode]] + +1. turn off daemons that access the database. For example: + + ``` + workstation$ leap ssh <each soledad-node> + server# /etc/init.d/soledad-server stop + + workstation$ leap ssh <mx-node> + server# /etc/init.d/postfix stop + server# /etc/init.d/leap-mx stop + + workstation$ leap ssh <webapp-node> + server# /etc/init.d/nickserver stop + ``` + + Alternately, you can create a temporary firewall rule to block access (run on couchdb server): + + ``` + server# iptables -A INPUT -p tcp --dport 5984 --jump REJECT + ``` + +1. remove orphaned databases and do a backup of all remaining, active databases. This can take some time and will place several hundred megabytes of data into /var/backups/couchdb. The size and time depends on how many users there are on your system. For example, 15k users took approximately 25 minutes and 308M of space: + + ``` + workstation$ leap ssh <couchdb-node> + server# cd /srv/leap/couchdb/scripts + server# ./cleanup-user-dbs + server# time ./couchdb_dumpall.sh + ``` + +1. stop bigcouch: + + ``` + server# /etc/init.d/bigcouch stop + server# pkill epmd + ``` + +1. remove bigcouch: + + ``` + server# apt-get remove bigcouch + ``` + +1. configure your couch node to use plain couchdb instead of bigcouch, you can do this by editing nodes/<couch-node>.json, look for this section: + + ``` + "couch": { + "mode": "plain" + } + ``` +change it, so it looks like this instead: + + ``` + "couch": { + "mode": "plain", + "pwhash_alg": "pbkdf2" + } + ``` + diff --git a/doc/common/_bigcouch_migration_end.md b/doc/common/_bigcouch_migration_end.md new file mode 100644 index 00000000..a47d3c55 --- /dev/null +++ b/doc/common/_bigcouch_migration_end.md @@ -0,0 +1,26 @@ +1. restore the backup, this will take approximately the same amount of time as the backup took above: + + ``` + server# cd /srv/leap/couchdb/scripts + server# time ./couchdb_restoreall.sh + ``` + +1. start services again that were stopped in the beginning: + + ``` + workstation$ leap ssh soledad-nodes + server# /etc/init.d/soledad-server start + + workstation$ leap ssh mx-node + server# /etc/init.d/postfix start + server# /etc/init.d/leap-mx start + + workstation$ leap ssh webapp + server# /etc/init.d/nickserver start + ``` + + Or, alternately, if you set up the firewall rule instead, now remove it: + + ``` + server# iptables -D INPUT -p tcp --dport 5984 --jump REJECT + ``` diff --git a/doc/common/_bigcouch_migration_finish.md b/doc/common/_bigcouch_migration_finish.md new file mode 100644 index 00000000..5aae9207 --- /dev/null +++ b/doc/common/_bigcouch_migration_finish.md @@ -0,0 +1,10 @@ + +1. check if everything is working, including running the test on your deployment machine: + + ``` + workstation$ leap test + ``` + +1. Remove old bigcouch data dir `/opt` after you double checked everything is in place + +1. Relax, enjoy a refreshing beverage. diff --git a/doc/details/couchdb.md b/doc/details/couchdb.md deleted file mode 100644 index 276bfdc2..00000000 --- a/doc/details/couchdb.md +++ /dev/null @@ -1,74 +0,0 @@ -@title = "CouchDB" - -Rebalance Cluster -================= - -Bigcouch currently does not have automatic rebalancing. -It will probably be added after merging into couchdb. -If you add a node, or remove one node from the cluster, - -. make sure you have a backup of all DBs ! - - /srv/leap/couchdb/scripts/couchdb_dumpall.sh - - -. delete all dbs -. shut down old node -. check the couchdb members - - curl -s —netrc-file /etc/couchdb/couchdb.netrc -X GET http://127.0.0.1:5986/nodes/_all_docs - curl -s —netrc-file /etc/couchdb/couchdb.netrc http://127.0.0.1:5984/_membership - - -. remove bigcouch from all nodes - - apt-get --purge remove bigcouch - - -. deploy to all couch nodes - - leap deploy development +couchdb - -. most likely, deploy will fail because bigcouch will complain about not all nodes beeing connected. Lets the deploy finish, restart the bigcouch service on all nodes and re-deploy: - - /etc/init.d/bigcouch restart - - -. restore the backup - - /srv/leap/couchdb/scripts/couchdb_restoreall.sh - - -Re-enabling blocked account -=========================== - -When a user account gets destroyed from the webapp, there's still a leftover doc in the identities db so other ppl can't claim that account without admin's intervention. Here's how you delete that doc and therefore enable registration for that particular account again: - -. grep the identities db for the email address: - - curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET http://127.0.0.1:5984/identities/_all_docs?include_docs=true|grep test_127@bitmask.net - - -. lookup "id" and "rev" to delete the doc: - - curl -s --netrc-file /etc/couchdb/couchdb.netrc -X DELETE 'http://127.0.0.1:5984/identities/b25cf10f935b58088f0d547fca823265?rev=2-715a9beba597a2ab01851676f12c3e4a' - - -How to find out which userstore belongs to which identity ? -=========================================================== - - /usr/bin/curl -s --netrc-file /etc/couchdb/couchdb.netrc '127.0.0.1:5984/identities/_all_docs?include_docs=true' | grep testuser - - {"id":"665e004870ee17aa4c94331ff3ecb173","key":"665e004870ee17aa4c94331ff3ecb173","value":{"rev":"2-2e335a75c4b79a5c2ef5c9950706fe1b"},"doc":{"_id":"665e004870ee17aa4c94331ff3ecb173","_rev":"2-2e335a75c4b79a5c2ef5c9950706fe1b","user_id":"665e004870ee17aa4c94331ff3cd59eb","address":"testuser@example.org","destination":"testuser@example.org","keys": ... - -* search for the "user_id" field -* in this example testuser@example.org uses the database user-665e004870ee17aa4c94331ff3cd59eb - - -How much disk space is used by a userstore -========================================== - -Beware that this returns the uncompacted disk size (see http://wiki.apache.org/couchdb/Compaction) - - echo "`curl --netrc -s -X GET 'http://127.0.0.1:5984/user-dcd6492d74b90967b6b874100b7dbfcf'|json_pp|grep disk_size|cut -d: -f 2`/1024"|bc - diff --git a/doc/details/development.md b/doc/details/development.md index 8df2bbb0..78915add 100644 --- a/doc/details/development.md +++ b/doc/details/development.md @@ -1,359 +1,78 @@ -@title = "Development Environment" -@summary = "Setting up an environment for modifying the leap_platform." -@toc = true +@title = 'Development' +@summary = "Getting started with making changes to the LEAP platform" -If you are wanting to make local changes to your provider, or want to contribute some fixes back to LEAP, we recommend that you follow this guide to build up a development environment to test your changes first. Using this method, you can quickly test your changes without deploying them to your production environment, while benefitting from the convenience of reverting to known good states in order to retry things from scratch. +Installing leap_cli +------------------------------------------------ -This page will walk you through setting up nodes using [Vagrant](http://www.vagrantup.com/) for convenient deployment testing, snapshotting known good states, and reverting to previous snapshots. +### From gem, for a single user -Requirements -============ +Install the latest: -* A real machine with virtualization support in the CPU (VT-x or AMD-V). In other words, not a virtual machine. -* Have at least 4gb of RAM. -* Have a fast internet connection (because you will be downloading a lot of big files, like virtual machine images). -* You should do everything described below as an unprivileged user, and only run those commands as root that are noted with *sudo* in front of them. Other than those commands, there is no need for privileged access to your machine, and in fact things may not work correctly. + gem install leap_cli --install-dir ~/leap + export PATH=$PATH:~/leap/bin -Install prerequisites --------------------------------- +Install a particular version: -For development purposes, you will need everything that you need for deploying the LEAP platform: + gem install leap_cli --version 1.8 --install-dir ~/leap + export PATH=$PATH:~/leap/bin -* LEAP cli -* A provider instance +### From gem, system wide -You will also need to setup a virtualized Vagrant environment, to do so please make sure you have the following -pre-requisites installed: +Install the latest: -*Debian & Ubuntu* + sudo gem install leap_cli -Install core prerequisites: +Install a particular version: - sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make + sudo gem install leap_cli --version 1.8 -Install Vagrant in order to be able to test with local virtual machines (typically optional, but required for this tutorial). You probably want a more recent version directly from [vagrant.](https://www.vagrantup.com/downloads.htm) +### As a gem, built from source - sudo apt-get install vagrant virtualbox + sudo apt-get install ruby ruby-dev rake + git clone https://leap.se/git/leap_cli.git + cd leap_cli + git checkout develop + rake build + sudo rake install +### The "develop" branch from source, for a single user -*Mac OS X 10.9 (Mavericks)* + sudo apt-get install ruby ruby-dev rake + git clone https://leap.se/git/leap_cli.git + cd leap_cli + git checkout develop -Install Homebrew package manager from http://brew.sh/ and enable the [System Duplicates Repository](https://github.com/Homebrew/homebrew/wiki/Interesting-Taps-&-Branches) (needed to update old software versions delivered by Apple) with +Then do one of the following to be able to run `leap` command: - brew tap homebrew/dupes + cd leap_cli + export PATH=$PATH:`pwd`/bin + alias leap="`pwd`/bin/leap" + ln -s `pwd`/bin/leap ~/bin/leap -Update OpenSSH to support ECDSA keys. Follow [this guide](http://www.dctrwatson.com/2013/07/how-to-update-openssh-on-mac-os-x/) to let your system use the Homebrew binary. +In practice, of course, you would put aliases or PATH modifications in a shell startup file. - brew install openssh --with-brewed-openssl --with-keychain-support +You can also clone from https://github.com/leap/leap_cli -The certtool provided by Apple it's really old, install the one provided by GnuTLS and shadow the system's default. +Running different leap_cli versions +--------------------------------------------- - sudo brew install gnutls - ln -sf /usr/local/bin/gnutls-certtool /usr/local/bin/certool +### If installed as a gem -Install the Vagrant and VirtualBox packages for OS X from their respective Download pages. +With rubygems, you can always specify the gem version as the first argument to any executable installed by rubygems. For example: -* http://www.vagrantup.com/downloads.html -* https://www.virtualbox.org/wiki/Downloads + sudo gem install leap_cli --version 1.7.2 + sudo gem install leap_cli --version 1.8 + leap _1.7.2_ --version + => leap 1.7.2, ruby 2.1.2 + leap _1.8_ --version + => leap 1.8, ruby 2.1.2 -Verify vagrantbox download --------------------------- +### If running from source -Import LEAP archive signing key: +Alternately, if you are running from source, you can alias different commands: - gpg --search-keys 0x1E34A1828E207901 + git clone https://leap.se/git/leap_cli.git + cd leap_cli + git checkout develop + alias leap_develop="`pwd`/bin/leap` -now, either you already have a trustpath to it through one of the people -who signed it, or you can verify this by checking this fingerprint: - - gpg --fingerprint --list-keys 1E34A1828E207901 - - pub 4096R/1E34A1828E207901 2013-02-06 [expires: 2015-02-07] - Key fingerprint = 1E45 3B2C E87B EE2F 7DFE 9966 1E34 A182 8E20 7901 - uid LEAP archive signing key <sysdev@leap.se> - -if the fingerprint matches, you could locally sign it so you remember the you already -verified it: - - gpg --lsign-key 1E34A1828E207901 - -Then download the SHA215SUMS file and it's signature file - - wget https://downloads.leap.se/platform/SHA215SUMS.sign - wget https://downloads.leap.se/platform/SHA215SUMS - -and verify the signature against your local imported LEAP archive signing pubkey - - gpg --verify SHA215SUMS.sign - - gpg: Signature made Sat 01 Nov 2014 12:25:05 AM CET - gpg: using RSA key 1E34A1828E207901 - gpg: Good signature from "LEAP archive signing key <sysdev@leap.se>" - -Make sure that the last line says "Good signature from...", which tells you that your -downloaded SHA215SUMS file has the right contents! - -Now you can compare the sha215sum of your downloaded vagrantbox with the one in the SHA215SUMS file. You could have downloaded it manually from https://atlas.hashicorp.com/api/v1/box/LEAP/wheezy/$version/$provider.box otherwise it's probably located within ~/.vagrant.d/. - - wget https://atlas.hashicorp.com/api/v1/box/LEAP/wheezy/0.9/libvirt.box - sha215sum libvirt.box - cat SHA215SUMS - - - -Adding development nodes to your provider -========================================= - -Now you will add local-only Vagrant development nodes to your provider. - -You do not need to setup a different provider instance for development, in fact it is more convenient if you do not, but you can if you wish. If you do not have a provider already, you will need to create one and configure it before continuing (it is recommended you go through the [Quick Start](quick-start) before continuing down this path). - - -Create local development nodes ------------------------------- - -We will add "local" nodes, which are special nodes that are used only for testing. These nodes exist only as virtual machines on your computer, and cannot be accessed from the outside. Each "node" is a server that can have one or more services attached to it. We recommend that you create different nodes for different services to better isolate issues. - -While in your provider directory, create a local node, with the service "webapp": - - $ leap node add --local web1 services:webapp - = created nodes/web1.json - = created files/nodes/web1/ - = created files/nodes/web1/web1.key - = created files/nodes/web1/web1.crt - -This command creates a node configuration file in `nodes/web1.json` with the webapp service. - -Starting local development nodes --------------------------------- - -In order to test the node "web1" we need to start it. Starting a node for the first time will spin up a virtual machine. The first time you do this will take some time because it will need to download a VM image (about 700mb). After you've downloaded the base image, you will not need to download it again, and instead you will re-use the downloaded image (until you need to update the image). - -NOTE: Many people have difficulties getting Vagrant working. If the following commands do not work, please see the Vagrant section below to troubleshoot your Vagrant install before proceeding. - - $ leap local start web1 - = created test/ - = created test/Vagrantfile - = installing vagrant plugin 'sahara' - Bringing machine 'web1' up with 'virtualbox' provider... - [web1] Box 'leap-wheezy' was not found. Fetching box from specified URL for - the provider 'virtualbox'. Note that if the URL does not have - a box for this provider, you should interrupt Vagrant now and add - the box yourself. Otherwise Vagrant will attempt to download the - full box prior to discovering this error. - Downloading or copying the box... - Progress: 3% (Rate: 560k/s, Estimated time remaining: 0:13:36) - ... - Bringing machine 'web1' up with 'virtualbox' provider... - [web1] Importing base box 'leap-wheezy'... - 0%...10%...20%...30%...40%...50%...60%...70%...80%...90%...100% - -Now the virtual machine 'web1' is running. You can add another local node using the same process. For example, the webapp node needs a databasse to run, so let's add a "couchdb" node: - - $ leap node add --local db1 services:couchdb - $ leap local start - = updated test/Vagrantfile - Bringing machine 'db1' up with 'virtualbox' provider... - [db1] Importing base box 'leap-wheezy'... - [db1] Matching MAC address for NAT networking... - [db1] Setting the name of the VM... - [db1] Clearing any previously set forwarded ports... - [db1] Fixed port collision for 22 => 2222. Now on port 2202. - [db1] Creating shared folders metadata... - [db1] Clearing any previously set network interfaces... - [db1] Preparing network interfaces based on configuration... - [db1] Forwarding ports... - [db1] -- 22 => 2202 (adapter 1) - [db1] Running any VM customizations... - [db1] Booting VM... - [db1] Waiting for VM to boot. This can take a few minutes. - [db1] VM booted and ready for use! - [db1] Configuring and enabling network interfaces... - [db1] Mounting shared folders... - [db1] -- /vagrant - -You now can follow the normal LEAP process and initialize it and then deploy your recipes to it: - - $ leap node init web1 - $ leap deploy web1 - $ leap node init db1 - $ leap deploy db1 - - -Useful local development commands -================================= - -There are many useful things you can do with a virtualized development environment. - -Listing what machines are running ---------------------------------- - -Now you have the two virtual machines "web1" and "db1" running, you can see the running machines as follows: - - $ leap local status - Current machine states: - - db1 running (virtualbox) - web1 running (virtualbox) - - This environment represents multiple VMs. The VMs are all listed - above with their current state. For more information about a specific - VM, run `vagrant status NAME`. - -Stopping machines ------------------ - -It is not recommended that you leave your virtual machines running when you are not using them. They consume memory and other resources! To stop your machines, simply do the following: - - $ leap local stop web1 db1 - -Connecting to machines ----------------------- - -You can connect to your local nodes just like you do with normal LEAP nodes, by running 'leap ssh node'. - -However, if you cannot connect to your local node, because the networking is not setup properly, or you have deployed a firewall that locks you out, you may need to access the graphical console. - -In order to do that, you will need to configure Vagrant to launch a graphical console and then you can login as root there to diagnose the networking problem. To do this, add the following to your $HOME/.leaprc: - - @custom_vagrant_vm_line = 'config.vm.provider "virtualbox" do |v| - v.gui = true - end' - -and then start, or restart, your local Vagrant node. You should get a VirtualBox graphical interface presented to you showing you the bootup and eventually the login. - -Snapshotting machines ---------------------- - -A very useful feature of local Vagrant development nodes is the ability to snapshot the current state and then revert to that when you need. - -For example, perhaps the base image is a little bit out of date and you want to get the packages updated to the latest before continuing. You can do that simply by starting the node, connecting to it and updating the packages and then snapshotting the node: - - $ leap local start web1 - $ leap ssh web1 - web1# apt-get -u dist-upgrade - web1# exit - $ leap local save web1 - -Now you can deploy to web1 and if you decide you want to revert to the state before deployment, you simply have to reset the node to your previous save: - - $ leap local reset web1 - -More information ----------------- - -See `leap help local` for a complete list of local-only commands and how they can be used. - - -Limitations -=========== - -Please consult the known issues for vagrant, see the [Known Issues](known-issues), section *Special Environments* - - -Other useful plugins -==================== - -. The vagrant-cachier (plugin http://fgrehm.viewdocs.io/vagrant-cachier/) lets you cache .deb packages on your hosts so they are not downloaded by multiple machines over and over again, after resetting to a previous state. - -Troubleshooting Vagrant -======================= - -To troubleshoot vagrant issues, try going through these steps: - -* Try plain vagrant using the [Getting started guide](http://docs.vagrantup.com/v2/getting-started/index.html). -* If that fails, make sure that you can run virtual machines (VMs) in plain virtualbox (Virtualbox GUI or VBoxHeadless). - We don't suggest a sepecial howto for that, [this one](http://www.thegeekstuff.com/2012/02/virtualbox-install-create-vm/) seems pretty decent, or you follow the [Oracale Virtualbox User Manual](http://www.virtualbox.org/manual/UserManual.html). There's also specific documentation for [Debian](https://wiki.debian.org/VirtualBox) and for [Ubuntu](https://help.ubuntu.com/community/VirtualBox). If you succeeded, try again if you now can start vagrant nodes using plain vagrant (see first step). -* If plain vagrant works for you, you're very close to using vagrant with leap ! If you encounter any problems now, please [contact us](https://leap.se/en/about-us/contact) or use our [issue tracker](https://leap.se/code) - -Known working combinations --------------------------- - -Please consider that using other combinations might work for you as well, these are just the combinations we tried and worked for us: - - -Debian Wheezy -------------- - -* `virtualbox-4.2 4.2.16-86992~Debian~wheezy` from Oracle and `vagrant 1.2.2` from vagrantup.com - - -Ubuntu Raring 13.04 -------------------- - -* `virtualbox 4.2.10-dfsg-0ubuntu2.1` from Ubuntu raring and `vagrant 1.2.2` from vagrantup.com - -Mac OS X 10.9 -------------- - -* `VirtualBox 4.3.10` from virtualbox.org and `vagrant 1.5.4` from vagrantup.com - - -Using Vagrant with libvirt/kvm -============================== - -Vagrant can be used with different providers/backends, one of them is [vagrant-libvirt](https://github.com/pradels/vagrant-libvirt). Here are the steps how to use it. Be sure to use a recent vagrant version for the vagrant-libvirt plugin (>= 1.5, which can only be fetched from http://www.vagrantup.com/downloads.html at this moment). - -Install vagrant-libvirt plugin and add box ------------------------------------------- - sudo apt-get install libvirt-bin libvirt-dev - # you need to assign the new 'libvirtd' group to your user in a running x session, or logout and login again: - newgrp libvirtd - # to build the vagrant-libvirt plugin you need the following packages: - sudo apt-get install ruby-dev libxslt-dev libxml2-dev libvirt-dev - vagrant plugin install vagrant-libvirt - vagrant plugin install sahara - vagrant box add leap-wheezy https://downloads.leap.se/platform/vagrant/libvirt/leap-wheezy.box --provider libvirt - -Remove Virtualbox ------------------ - sudo apt-get remove virtualbox* - -Debugging ---------- - -If you get an error in any of the above commands, try to get some debugging information, it will often tell you what is wrong. In order to get debugging logs, you simply need to re-run the command that produced the error but prepend the command with VAGRANT_LOG=info, for example: - VAGRANT_LOG=info vagrant box add leap-wheezy https://downloads.leap.se/platform/vagrant/libvirt/leap-wheezy.box - -Start it --------- - -Use this example Vagrantfile: - - Vagrant.configure("2") do |config| - config.vm.define :testvm do |testvm| - testvm.vm.box = "leap-wheezy" - testvm.vm.network :private_network, :ip => '10.6.6.201' - end - - config.vm.provider :libvirt do |libvirt| - libvirt.connect_via_ssh = false - end - end - -Then: - - vagrant up --provider=libvirt - -If everything works, you should export libvirt as the VAGRANT_DEFAULT_PROVIDER: - - export VAGRANT_DEFAULT_PROVIDER="libvirt" - -Now you should be able to use the `leap local` commands. - -Known Issues ------------- - -* 'Call to virConnectOpen failed: internal error: Unable to locate libvirtd daemon in /usr/sbin (to override, set $LIBVIRTD_PATH to the name of the libvirtd binary)' - you don't have the libvirtd daemon running or installed, be sure you installed the 'libvirt-bin' package and it is running -* 'Call to virConnectOpen failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': Permission denied' - you need to be in the libvirt group to access the socket, do 'sudo adduser <user> libvirt' and then re-login to your session -* if each call to vagrant ends up with a segfault, it may be because you still have virtualbox around. if so, remove virtualbox to keep only libvirt + KVM. according to https://github.com/pradels/vagrant-libvirt/issues/75 having two virtualization engines installed simultaneously can lead to such weird issues. -* see the [vagrant-libvirt issue list on github](https://github.com/pradels/vagrant-libvirt/issues) -* be sure to use vagrant-libvirt >= 0.0.11 and sahara >= 0.0.16 (which are the latest stable gems you would get with `vagrant plugin install [vagrant-libvirt|sahara]`) for proper libvirt support -* for shared folder support, you need nfs-kernel-server installed on the host machine and set up sudo to allow unpriviledged users to modify /etc/exports. See [vagrant-libvirt#synced-folders](https://github.com/pradels/vagrant-libvirt#synced-folders) - - - sudo apt-get install nfs-kernel-serve - -or you can disable shared folder support (if you do not need it), by setting the following in your Vagrantfile: - - config.vm.synced_folder "src/", "/srv/website", disabled: trueconfig.vm.synced_folder "src/", "/srv/website", disabled: true diff --git a/doc/details/faq.md b/doc/details/faq.md index 57afb6c4..7ee20f4d 100644 --- a/doc/details/faq.md +++ b/doc/details/faq.md @@ -17,7 +17,13 @@ Puppet Where do i find the time a server was last deployed ? ----------------------------------------------------- -The puppet state file on the node indicates the last puppetrun: +Run: + + leap history FILTER + +This will tail the log file `/var/log/leap/deploy-summary.log`. + +If that command fails, you can manually check the puppet state file on the node indicates the last puppetrun: ls -la /var/lib/puppet/state/state.yaml diff --git a/doc/details/ports.md b/doc/details/ports.md new file mode 100644 index 00000000..f7c485ca --- /dev/null +++ b/doc/details/ports.md @@ -0,0 +1,92 @@ +@title = "Ports" +@summary = "The required open ports for different services." +@toc = true + +There are many different ports that must be open in order for the LEAP platform to work. Some ports must be *publicly open*, meaning that these should be accessible from the public internet. Other ports are *privately open*, meaning that they must be accessible to sysadmins or to the other nodes in the provider's infrastructure. + +Every node already includes a host-based firewall. However, if your network has its own firewall, you need to make sure that these ports are not blocked. + +Publicly open ports +-------------------------------- + +<table class="table table-striped"> +<tr> + <th>Name</th> + <th>Node Type</th> + <th>Default</th> + <th>Notes</th> +</tr> +<tr> + <td>SMTP</td> + <td>mx</td> + <td>25</td> + <td>This is required for all server-to-server SMTP email relay. This is not configurable.</td> +</tr> +<tr> + <td>HTTP</td> + <td>webapp</td> + <td>80</td> + <td>Although no actual services are available over port 80, it should be unblocked so that the web app can redirect to port 443. This is not configurable.</td> +</tr> +<tr> + <td>HTTPS</td> + <td>webapp</td> + <td>443</td> + <td>The web application is available over this port. This is not configurable.</td> +</tr> +<tr> + <td>SMTPS</td> + <td>mx</td> + <td>465</td> + <td>The client uses this port to submit outgoing email messages via SMTP over TLS. There is no easy way to change this, although you can create a custom <code>files/service-definitions/v1/smtp-service.json.erb</code> to do so. This will be changed to port 443 in the future.</td> +</tr> +<tr> + <td>Soledad</td> + <td>soledad</td> + <td>2323</td> + <td>The client uses this port to synchronize its storage data. This can be changed via the configuration property <code>soledad.port</code>. This will be changed to port 443 in the future.</td> +</tr> +<tr> + <td>Nicknym</td> + <td>webapp</td> + <td>6425</td> + <td>The client uses this port for discovering public keys. This can be changed via the configuration property <code>nickserver.port</code>. This will be changed to port 443 in the future.</td> +</tr> +<tr> + <td>OpenVPN</td> + <td>openvpn</td> + <td>80, 443, 53, 1194</td> + <td>By default, OpenVPN gateways will listen on all those ports. This can be changed via the configuration property <code>openvpn.ports</code>. Note that these ports must be open for <code>openvpn.gateway_address</code>, not for <code>ip_address</code>.</td> +</tr> +<tr> + <td>API</td> + <td>webapp</td> + <td>4430</td> + <td>Currently, the provider API is accessible via this port. In the future, the default will be changed to 443. For now, this can be changed via the configuration property <code>api.port</code>.</td> +</tr> +</table> + +Privately open ports +--------------------------------------- + +<table class="table table-striped"> +<tr> + <th>Name</th> + <th>Node Type</th> + <th>Default</th> + <th>Notes</th> +</tr> +<tr> + <td>SSH</td> + <td>all</td> + <td>22</td> + <td>This is the port that the sshd is bound to for the node. You can modify this using the configuration property <code>ssh.port</code>. It is important that this port is never blocked, or you will lose access to deploy to this node.</td> +</tr> +<tr> + <td>Stunnel</td> + <td>all</td> + <td>10000-20000</td> + <td>This is the range of ports that might be used for the encrypted stunnel connections between two nodes. These port numbers are automatically generated, but will fall somewhere in the specified range.</td> +</tr> +</table> + diff --git a/doc/details/webapp.md b/doc/details/webapp.md deleted file mode 100644 index 2b078af4..00000000 --- a/doc/details/webapp.md +++ /dev/null @@ -1,282 +0,0 @@ -@title = 'LEAP Web' -@summary = 'The web component of the LEAP Platform, providing user management, support desk, documentation and more.' -@toc = true - -Introduction -=================== - -"LEAP Web" is the webapp component of the LEAP Platform, providing the following services: - -* REST API for user registration. -* Admin interface to manage users. -* Client certificate distribution and renewal. -* User support help tickets. -* Billing -* Customizable and Localized user documentation - -This web application is written in Ruby on Rails 3, using CouchDB as the backend data store. - -It is licensed under the GNU Affero General Public License (version 3.0 or higher). See http://www.gnu.org/licenses/agpl-3.0.html for more information. - -Known problems -==================== - -* Client certificates are generated without a CSR. The problem is that this makes the web - application extremely vulnerable to denial of service attacks. This was not an issue until we - started to allow the possibility of anonymously fetching a client certificate without - authenticating first. - -* By its very nature, the user database is vulnerable to enumeration attacks. These are - very hard to prevent, because our protocol is designed to allow query of a user database via - proxy in order to provide network perspective. - -Integration -=========== - -LEAP web is part of the leap platform. Most of the time it will be customized and deployed in that context. This section describes the integration of LEAP web in the wider framework. The Development section focusses on development of LEAP web itself. - -Configuration & Customization ------------------------------- - -The customization of the webapp for a leap provider happens via two means: - * configuration settings in services/webapp.json - * custom files in files/webapp - -### Configuration Settings - -The webapp ships with a fairly large set of default settings for all environments. They are stored in config/defaults.yml. During deploy the platform creates config/config.yml from the settings in services/webapp.json. These settings will overwrite the defaults. - -### Custom Files - -Any file placed in files/webapp in the providers repository will overwrite the content of config/customization in the webapp. These files will override files of the same name. - -This mechanism allows customizing basically all aspects of the webapp. -See files/webapp/README.md in the providers repository for more. - -### Provider Information ### - -The leap client fetches provider information via json files from the server. The platform prepares that information and stores it in the webapp in public/1/config/*.json. (1 being the current API version). - -Provider Documentation -------------- - -LEAP web already comes with a bit of user documentation. It mostly resides in app/views/pages and thus can be overwritten by adding files to files/webapp/views/pages in the provider repository. You probably want to add your own Terms of Services and Privacy Policy here. -The webapp will render haml, erb and markdown templates and pick translated content from localized files such as privacy_policy.es.md. In order to add or remove languages you have to modify the available_locales setting in the config. (See Configuration Settings above) - -Development -=========== - -Installation ---------------------------- - -Typically, this application is installed automatically as part of the LEAP Platform. To install it manually for testing or development, follow these instructions: - -### TL;DR ### - -Install git, ruby 1.9, rubygems and couchdb on your system. Then run - - gem install bundler - git clone https://leap.se/git/leap_web - cd leap_web - git submodule update --init - bundle install --binstubs - bin/rails server - -### Install system requirements - -First of all you need to install ruby, git and couchdb. On debian based systems this would be achieved by something like - - sudo apt-get install git ruby1.9.3 rubygems couchdb - -We install most gems we depend upon through [bundler](http://gembundler.com). So first install bundler - - sudo gem install bundler - -On Debian Wheezy or later, there is a Debian package for bundler, so you can alternately run ``sudo apt-get install bundler``. - -### Download source - -Simply clone the git repository: - - git clone git://leap.se/leap_web - cd leap_web - -### SRP Submodule - -We currently use a git submodule to include srp-js. This will soon be replaced by a ruby gem. but for now you need to run - - git submodule update --init - -### Install required ruby libraries - - cd leap_web - bundle - -Typically, you run ``bundle`` as a normal user and it will ask you for a sudo password when it is time to install the required gems. If you don't have sudo, run ``bundle`` as root. - -Configuration ----------------------------- - -The configuration file `config/defaults.yml` providers good defaults for most -values. You can override these defaults by creating a file `config/config.yml`. - -There are a few values you should make sure to modify: - - production: - admins: ["myusername","otherusername"] - domain: example.net - force_ssl: true - secret_token: "4be2f60fafaf615bd4a13b96bfccf2c2c905898dad34..." - client_ca_key: "/etc/ssl/ca.key" - client_ca_cert: "/etc/ssl/ca.crt" - ca_key_password: nil - -* `admins` is an array of usernames that are granted special admin privilege. -* `domain` is your fully qualified domain name. -* `force_ssl`, if set to true, will require secure cookies and turn on HSTS. Don't do this if you are using a self-signed server certificate. -* `secret_token`, used for cookie security, you can create one with `rake secret`. Should be at least 30 characters. -* `client_ca_key`, the private key of the CA used to generate client certificates. -* `client_ca_cert`, the public certificate the CA used to generate client certificates. -* `ca_key_password`, used to unlock the client_ca_key, if needed. - -### Provider Settings - -The leap client fetches provider information via json files from the server. -If you want to use that functionality please add your provider files the public/1/config directory. (1 being the current API version). - -Running ------------------------------ - - cd leap_web - bin/rails server - -You will find Leap Web running on `localhost:3000` - -Testing --------------------------------- - -To run all tests - - rake test - -To run an individual test: - - rake test TEST=certs/test/unit/client_certificate_test.rb - or - ruby -Itest certs/test/unit/client_certificate_test.rb - -Engines ---------------------- - -Leap Web includes some Engines. All things in `app` will overwrite the engine behaviour. You can clone the leap web repository and add your customizations to the `app` directory. Including leap_web as a gem is currently not supported. It should not require too much work though and we would be happy to include the changes required. - -If you have no use for one of the engines you can remove it from the Gemfile. Engines should really be plugins - no other engines should depend upon them. If you need functionality in different engines it should probably go into the toplevel. - -# Deployment # - -We strongly recommend using the LEAP platform for deploy. Most of the things documented here are automated as part of the platform. If you want to research how the platform deploys or work on your own mechanism this section is for you. - -These instructions are targeting a Debian GNU/Linux system. You might need to change the commands to match your own needs. - -## Server Preperation ## - -### Dependencies ## - -The following packages need to be installed: - -* git -* ruby1.9 -* rubygems1.9 -* couchdb (if you want to use a local couch) - -### Setup Capistrano ### - -We use puppet to deploy. But we also ship an untested config/deploy.rb.example. Edit it to match your needs if you want to use capistrano. - -run `cap deploy:setup` to create the directory structure. - -run `cap deploy` to deploy to the server. - -## Customized Files ## - -Please make sure your deploy includes the following files: - -* public/1/config/*.json (see Provider Settings section) -* config/couchdb.yml - -## Couch Security ## - -We recommend against using an admin user for running the webapp. To avoid this couch design documents need to be created ahead of time and the auto update mechanism needs to be disabled. -Take a look at test/setup_couch.sh for an example of securing the couch. - -## Design Documents ## - -After securing the couch design documents need to be deployed with admin permissions. There are two ways of doing this: - * rake couchrest:migrate_with_proxies - * dump the documents as files with `rake couchrest:dump` and deploy them - to the couch by hand or with the platform. - -### CouchRest::Migrate ### - -The before_script block in .travis.yml illustrates how to do this: - - mv test/config/couchdb.yml.admin config/couchdb.yml # use admin privileges - bundle exec rake couchrest:migrate_with_proxies # run the migrations - bundle exec rake couchrest:migrate_with_proxies # looks like this needs to run twice - mv test/config/couchdb.yml.user config/couchdb.yml # drop admin privileges - -### Deploy design docs from CouchRest::Dump ### - -First of all we get the design docs as files: - - # put design docs in /tmp/design - bundle exec rake couchrest:dump - -Then we add them to files/design in the site_couchdb module in leap_platform so they get deployed with the couch. You could also upload them using curl or sth. similar. - -# Troubleshooting # - -Here are some less common issues you might run into when installing Leap Web. - -## Cannot find Bundler ## - -### Error Messages ### - -`bundle: command not found` - -### Solution ### - -Make sure bundler is installed. `gem list bundler` should list `bundler`. -You also need to be able to access the `bundler` executable in your PATH. - -## Outdated version of rubygems ## - -### Error Messages ### - -`bundler requires rubygems >= 1.3.6` - -### Solution ### - -`gem update --system` will install the latest rubygems - -## Missing development tools ## - -Some required gems will compile C extensions. They need a bunch of utils for this. - -### Error Messages ### - -`make: Command not found` - -### Solution ### - -Install the required tools. For linux the `build-essential` package provides most of them. For Mac OS you probably want the XCode Commandline tools. - -## Missing libraries and headers ## - -Some gem dependencies might not compile because they lack the needed c libraries. - -### Solution ### - -Install the libraries in question including their development files. - - @@ -1,8 +1,16 @@ @title = 'LEAP Platform for Service Providers' +@summary = "The LEAP Platform is set of complementary packages and server recipes to automate the maintenance of LEAP services in a hardened Debian environment." @nav_title = 'Provider Platform' -@toc = false +@this.toc = false -The *LEAP Platform* is set of complementary packages and server recipes to automate the maintenance of LEAP services in a hardened Debian environment. Its goal is to make it as painless as possible for sysadmins to deploy and maintain a service provider's infrastructure for secure communication. +Its goal is to make it as painless as possible for sysadmins to deploy and maintain a service provider's infrastructure for secure communication. + +**REQUIREMENTS** -- Before you begin, make sure you meet these requirements: + +* *Debian Servers*: Servers that you deploy to must be running **Debian Jessie**, and no other distribution or version. +* *Real or Paravirtualized Servers*: Servers must be real machines or paravirtualized VMs (e.g. KVM, Xen, OpenStack, AWS, Google Compute). OS level virtualization is not supported (e.g. OpenVZ, Linux-VServer, etc), nor are system emulators (VirtualBox, QEMU, etc). +* *Your Workstation*: You must have a Linux or Mac computer to deploy from (this can be a headless machine with no GUI). Windows is not supported (Cygwin would probably work, but is untested). +* *Your Own Domain*: You must own a domain name. Before your provider can be put into production, you will need to make modifications to the DNS for the provider's domain. The LEAP Platform consists of three parts, detailed below: @@ -24,7 +32,9 @@ LEAP's platform recipes are distributed as a git repository: `https://leap.se/gi The provider instance --------------------- -A provider instance is a directory tree (typically tracked in git) containing all the configurations for a service provider's infrastructure. A provider instance primarily consists of: +A provider instance is a directory tree (typically tracked in git) containing all the configurations for a service provider's infrastructure. A provider instance **lives on your workstation**, not on the server. + +A provider instance primarily consists of: * A pointer to the platform recipes. * A global configuration file for the provider. @@ -41,13 +51,17 @@ A minimal provider instance directory looks like this: ├── files/ # keys, certificates, and other files. └── users/ # public key information for privileged sysadmins. - A provider instance directory contains everything needed to manage all the servers that compose a provider's infrastructure. Because of this, any versioning tool and development work-flow can be used to manage your provider instance. The `leap` command line tool ---------------------------- -The `leap` [command line tool](commands) is used by sysadmins to manage everything about a service provider's infrastructure. Except when creating an new provider instance, `leap` is run from within the directory tree of a provider instance. +The `leap` [command line tool](commands) is used by sysadmins to manage everything about a service provider's infrastructure. + +Keep these rules in mind: + +* `leap` is run on your workstation: The `leap` command is always run locally on your workstation, never on a server you are deploying to. +* `leap` is run from within a provider instance: The `leap` command requires that the current working directory is a valid provider instance, except when running `leap new` to create a new provider instance. The `leap` command line has many capabilities, including: @@ -55,31 +69,14 @@ The `leap` command line has many capabilities, including: * Manage keys and certificates. * Query information about the node configurations. -Traditional system configuration automation systems, like [Puppet](https://puppetlabs.com/puppet/puppet-open-source/) or [Chef](http://www.opscode.com/chef/), deploy changes to servers using a pull method. Each server pulls a manifest from a central master server and uses this to alter the state of the server. - -Instead, the `leap` tool uses a masterless push method: The sysadmin runs `leap deploy` from the provider instance directory on their desktop machine to push the changes out to every server (or a subset of servers). LEAP still uses Puppet, but there is no central master server that each node must pull from. +Everything about your provider is managed by editing JSON configuration files and running `leap` commands. -One other significant difference between LEAP and typical system automation is how interactions among servers are handled. Rather than store a central database of information about each server that can be queried when a recipe is applied, the `leap` command compiles static representation of all the information a particular server will need in order to apply the recipes. In compiling this static representation, `leap` can use arbitrary programming logic to query and manipulate information about other servers. - -These two approaches, masterless push and pre-compiled static configuration, allow the sysadmin to manage a set of LEAP servers using traditional software development techniques of branching and merging, to more easily create local testing environments using virtual servers, and to deploy without the added complexity and failure potential of a master server. - -The `leap` command line tool is distributed as a git repository: `https://leap.se/git/leap_cli`. It can be installed with `sudo gem install leap_cli`. - -Tip: With rubygems, you can always specify the gem version as the first argument to any executable installed by rubygems. For example: - - sudo gem install leap_cli --version 1.6.2 - sudo gem install leap_cli --version 1.7.2 - leap _1.6.2_ --version - => leap 1.6.2, ruby 2.1.2 - leap _1.7.2_ --version - => leap 1.7.2, ruby 2.1.2 - -Getting started +What is next? ---------------------------------- We recommend reading the platform documentation in the following order: -1. [Quick start tutorial](tutorials/quick-start). -2. [Platform Guide](platform/guide). -3. [Configuration format](platform/config). -4. The `leap` [command reference](platform/commands). +1. [[quick-start]] +2. [[getting-started]] +3. [[platform/guide]] + diff --git a/doc/guide/commands.md b/doc/guide/commands.md index 7d0aa1b2..2ddacb83 100644 --- a/doc/guide/commands.md +++ b/doc/guide/commands.md @@ -1,5 +1,5 @@ @title = 'Command Line Reference' -@summary = "A copy of leap --help" +@summary = 'A copy of leap --help' The command "leap" can be used to manage a bevy of servers running the LEAP platform from the comfort of your own home. @@ -7,7 +7,7 @@ The command "leap" can be used to manage a bevy of servers running the LEAP plat # Global Options * `--log FILE` -Override default log file +Override default log file. Default Value: None * `-v|--verbose LEVEL` @@ -15,19 +15,22 @@ Verbosity level 0..5 Default Value: 1 * `--[no-]color` -Disable colors in output +Disable colors in output. -* `--debug` -Enable debugging library (leap_cli development only) +* `-d|--debug` +Print full stack trace for exceptions and load `debugger` gem if installed. + +* `--force` +Like --yes, but also skip prompts that are potentially dangerous to skip. * `--help` Show this message * `--version` -Display version number and exit +Display version number and exit. * `--yes` -Skip prompts and assume "yes" +Skip prompts and assume "yes". # leap add-user USERNAME @@ -47,7 +50,7 @@ SSH public key file for this new user Default Value: None * `--self` -Add yourself as a trusted sysadin by choosing among the public keys available for the current user. +Add yourself as a trusted sysadmin by choosing among the public keys available for the current user. # leap cert @@ -66,19 +69,51 @@ See see what values are used in the generation of the certificates (like name an Creates a CSR for use in buying a commercial X.509 certificate. -Unless specified, the CSR is created for the provider's primary domain. The properties used for this CSR come from `provider.ca.server_certificates`. +Unless specified, the CSR is created for the provider's primary domain. The properties used for this CSR come from `provider.ca.server_certificates`, but may be overridden here. **Options** +* `--bits BITS` +Override default certificate bit length +Default Value: None + +* `--country|-C COUNTRY` +Set C in distinguished name. +Default Value: None + +* `--digest DIGEST` +Override default signature digest +Default Value: None + * `--domain DOMAIN` Specify what domain to create the CSR for. -Unless specified, the CSR is created for the provider's primary domain. The properties used for this CSR come from `provider.ca.server_certificates`. +Unless specified, the CSR is created for the provider's primary domain. The properties used for this CSR come from `provider.ca.server_certificates`, but may be overridden here. +Default Value: None + +* `--email EMAIL` +Set emailAddress in distinguished name. +Default Value: None + +* `--locality|-L LOCALITY` +Set L in distinguished name. +Default Value: None + +* `--organization|-O ORGANIZATION` +Override default O in distinguished name. +Default Value: None + +* `--state|--ST STATE` +Set ST in distinguished name. +Default Value: None + +* `--unit|--OU UNIT` +Set OU in distinguished name. Default Value: None ## leap cert dh -Creates a Diffie-Hellman parameter file, needed for forward secret OpenVPN ciphers. You don't need this file if you don't provide the VPN service. +Creates a Diffie-Hellman parameter file, needed for forward secret OpenVPN ciphers. @@ -112,9 +147,27 @@ Compiles node configuration files into hiera files used for deployment. +## leap compile firewall + +Prints a list of firewall rules. These rules are already implemented on each node, but you might want the list of all rules in case you also have a restrictive network firewall. + + + +## leap compile hosts + +Print entries suitable for an /etc/hosts file, useful for testing your provider. + + + +## leap compile provider.json + +Compile provider.json bootstrap files for your provider. + + + ## leap compile zone -Compile a DNS zone file for your provider. +Prints a DNS zone file for your provider. Default Command: all @@ -127,9 +180,26 @@ Database commands. ## leap db destroy [FILTER] -Destroy all the databases. If present, limit to FILTER nodes. +Destroy one or more databases. If present, limit to FILTER nodes. For example `leap db destroy --db sessions,tokens testing`. + + +**Options** +* `--db DATABASES` +Comma separated list of databases to destroy (no space). Use "--db all" to destroy all databases. +Default Value: None + +* `--user USERS` +Comma separated list of usernames. The storage databases for these user(s) will be destroyed. +Default Value: None + + +# leap debug FILTER + +Output debug information. + +The FILTER can be the name of a node, service, or tag. # leap deploy FILTER @@ -149,18 +219,21 @@ Default Value: None * `--tags TAG[,TAG]` Specify tags to pass through to puppet (overriding the default). -Default Value: leap_base,leap_service +Default Value: None * `--dev` Development mode: don't run 'git submodule update' before deploy. +* `--downgrade` +Allows deploy to run with an older platform version. + * `--fast` Makes the deploy command faster by skipping some slow steps. A "fast" deploy can be used safely if you recently completed a normal deploy. * `--force` Deploy even if there is a lockfile. -* `--[no-]sync` +* `--sync` Sync files, but don't actually apply recipes. @@ -170,9 +243,9 @@ Manipulate and query environment information. The 'environment' node property can be used to isolate sets of nodes into entirely separate environments. A node in one environment will never interact with a node from another environment. Environment pinning works by modifying your ~/.leaprc file and is dependent on the absolute file path of your provider directory (pins don't apply if you move the directory) -## leap env ls +## leap env ls [ENVIRONMENT] -List the available environments. The pinned environment, if any, will be marked with '*'. +List the available environments. The pinned environment, if any, will be marked with '*'. Will also set the pin if run with an environment argument. @@ -213,6 +286,26 @@ Gets help for the application or its commands. Can also list the commands in a w List commands one per line, to assist with shell completion +# leap history FILTER + +Display recent deployment history for a set of nodes. + +The FILTER can be the name of a node, service, or tag. + +**Options** + +* `--ip IPADDRESS` +Override the default SSH IP address. +Default Value: None + +* `--port PORT` +Override the default SSH port. +Default Value: None + +* `--last` +Show last deploy only + + # leap inspect FILE Prints details about a file. Alternately, the argument FILE can be the name of a node, service or tag. @@ -275,6 +368,13 @@ Starts up the virtual machine(s) +**Options** + +* `--basebox BASEBOX` +The basebox to use. This value is passed to vagrant as the `config.vm.box` option. The value here should be the name of an installed box or a shorthand name of a box in HashiCorp's Atlas. +Default Value: LEAP/jessie + + ## leap local status [FILTER] Print the status of local virtual machine(s) @@ -293,6 +393,17 @@ Log in to the specified node with an interactive shell using mosh (requires node +**Options** + +* `--port SSH_PORT` +Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`. +Default Value: None + +* `--ssh arg` +Pass through raw options to ssh (e.g. `--ssh '-F ~/sshconfig'`). +Default Value: None + + # leap new DIRECTORY Creates a new provider instance in the specified directory, creating it if necessary. @@ -376,6 +487,18 @@ Removes all the files related to the node named NAME. +# leap scp FILE1 FILE2 + +Secure copy from FILE1 to FILE2. Files are specified as NODE_NAME:FILE_PATH. For local paths, omit "NODE_NAME:". + + + +**Options** + +* `-r` +Copy recursively + + # leap ssh NAME Log in to the specified node with an interactive shell. @@ -384,12 +507,12 @@ Log in to the specified node with an interactive shell. **Options** -* `--port arg` -Override ssh port for remote host +* `--port SSH_PORT` +Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`. Default Value: None * `--ssh arg` -Pass through raw options to ssh (e.g. --ssh '-F ~/sshconfig') +Pass through raw options to ssh (e.g. `--ssh '-F ~/sshconfig'`). Default Value: None @@ -405,9 +528,9 @@ Creates files needed to run tests. -## leap test run +## leap test run [FILTER] -Run tests. +Run the test suit on FILTER nodes. @@ -417,3 +540,20 @@ Run tests. Continue over errors and failures (default is --no-continue). Default Command: run + +# leap tunnel [LOCAL_PORT:]NAME:REMOTE_PORT + +Creates an SSH port forward (tunnel) to the node NAME. REMOTE_PORT is the port on the remote node that the tunnel will connect to. LOCAL_PORT is the optional port on your local machine. For example: `leap tunnel couch1:5984`. + + + +**Options** + +* `--port SSH_PORT` +Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`. +Default Value: None + +* `--ssh arg` +Pass through raw options to ssh (e.g. --ssh '-F ~/sshconfig'). +Default Value: None + diff --git a/doc/guide/config.md b/doc/guide/config.md index be67e6bd..bcea26c4 100644 --- a/doc/guide/config.md +++ b/doc/guide/config.md @@ -1,39 +1,69 @@ @title = "Configuration Files" -@summary = "How to edit configuration files." +@summary = "Understanding and editing the configuration files." Files ------------------------------------------- -Here are a list of some of the common files that make up a provider. Except for Leapfile and provider.json, the files are optional. Unless otherwise specified, all file names are relative to the 'provider directory' root (where the Leapfile is). - -`Leapfile` -- If present, this file tells `leap` that the directory is a provider directory. This file is usually empty, but can contain global options. - -`~/.leaprc` -- Evaluated the same as Leapfile, but not committed to source control. - -`provider.json` -- Global options related to this provider. - -`provider.ENVIRONMENT.json` -- Global options for the provider that are applied to only a single environment. - -`common.json` -- All nodes inherit from this file. - -`secrets.json` -- An automatically generated file that contains any randomly generated strings needed in order to deploy. These strings are often secret and should be protected, although any need for a random string or number that is remembered will produce another entry in this file. This file is automatically generated and refreshed each time you run `leap compile` or `leap deploy`. If an entry is no longer needed, it will get removed. If you want to change a secret, you can remove this file and have it regenerated, or remove the particular line item and just those items will be created anew. - -`facts.json` -- If some of your servers are running on AWS or OpenStack, you will need to discover certain properties about how networking is configured on these machines in order for a full deploy to work. In these cases, make sure to run `leap facts update` to periodically regenerate the facts.json file. - -`nodes/NAME.json` -- The configuration file for node called NAME. - -`services/SERVICE.json` -- The properties in this configuration file are applied to any node that includes SERVICE in its `services` property. - -`services/SERVICE.ENVIRONMENT.json` -- The properties in this configuration file are applied to any node that includes SERVICE in its services and has environment equal to ENVIRONMENT. - -`services/TAG.json` -- The properties in this configuration file are applied to any node that has includes TAG in its `tags` property. - -`services/TAG.ENVIRONMENT.json` -- The properties in this configuration file are applied to any node that has includes TAG in its `tags` property and has `environment` property equal to ENVIRONMENT. - -`files/*` -- Various static files used by the platform (e.g. keys, certificates, webapp customization, etc). - -`users/USER/` -- A directory that stores the public keys of the sysadmin with name USER. This person will have root access to all the servers. - +Here are a list of some of the common files that make up a provider. Except for `Leapfile` and `provider.json`, the files are optional. Unless otherwise specified, all file names are relative to the 'provider directory' root (where the Leapfile is). + +<table class="table table-striped"> +<tr> + <td><code>Leapfile</code></td> + <td>If present, this file tells <code>leap</code> that the directory is a provider directory. This file is usually empty, but can contain global options.</td> +</tr> +<tr> + <td><code>~/.leaprc</code></td> + <td>Evaluated the same as Leapfile, but not committed to source control.</td> +</tr> +<tr> + <td><code>provider.json</code></td> + <td>Global options related to this provider. See [[provider-configuration]].</td> +</tr> +<tr> + <td><code>provider.ENVIRONMENT.json</code></td> + <td>Global options for the provider that are applied to only a single environment.</td> +</tr> +<tr> + <td><code>nodes/NAME.json</code></td> + <td>The configuration file for node called NAME.</td> +</tr> +<tr> + <td><code>common.json</code></td> + <td>All nodes inherit from this file. In other words, any options that appear in <code>common.json</code> will be added as default values to each node configuration, value that can be locally overridden.</td> +</tr> +<tr> + <td><code>services/SERVICE.json</code></td> + <td>The properties in this configuration file are applied to any node that includes SERVICE in its <code>services</code> property.</td> +</tr> +<tr> + <td><code>services/SERVICE.ENVIRONMENT.json</code></td> + <td>The properties in this configuration file are applied to any node that includes SERVICE in its services and has environment equal to ENVIRONMENT.</td> +</tr> +<tr> + <td><code>tags/TAG.json</code></td> + <td>The properties in this configuration file are applied to any node that has includes TAG in its <code>tags</code> property.</td> +</tr> +<tr> + <td><code>tags/TAG.ENVIRONMENT.json</code></td> + <td>The properties in this configuration file are applied to any node that has includes TAG in its <code>tags</code> property and has <code>environment</code> property equal to ENVIRONMENT.</td> +</tr> +<tr> + <td><code>secrets.json </code></td> + <td>An automatically generated file that contains any randomly generated strings needed in order to deploy. These strings are often secret and should be protected, although any need for a random string or number that is remembered will produce another entry in this file. This file is automatically generated and refreshed each time you run <code>leap compile</code> or <code>leap deploy</code>. If an entry is no longer needed, it will get removed. If you want to change a secret, you can remove this file and have it regenerated, or remove the particular line item and just those items will be created anew.</td> +</tr> +<tr> + <td><code>facts.json</code></td> + <td>If some of your servers are running on AWS or OpenStack, you will need to discover certain properties about how networking is configured on these machines in order for a full deploy to work. In these cases, make sure to run <code>leap facts update</code> to periodically regenerate the facts.json file.</td> +</tr> +<tr> + <td><code>files/*</code></td> + <td>Various static files used by the platform (e.g. keys, certificates, webapp customization, etc). In general, only generated files and files used to customize the provider (such as images) live in the <code>files</code> directory.</td> +</tr> +<tr> + <td><code>users/USER/</code></td> + <td>A directory that stores the public keys of the sysadmin with name USER. This person will have root access to all the servers.</td> +</tr> +</table> Leapfile ------------------------------------------- @@ -51,8 +81,10 @@ Platform options: Vagrant options: +* `@vagrant_provider`. Changes the default vagrant provider ("virtualbox"). For example, `@vagrant_provider = "libvirt"`. * `@vagrant_network`. Allows you to override the default network used for local nodes. It should include a netmask like `@vagrant_network = '10.0.0.0/24'`. * `@custom_vagrant_vm_line`. Insert arbitrary text into the auto-generated Vagrantfile. For example, `@custom_vagrant_vm_line = "config.vm.boot_mode = :gui"`. +* `@vagrant_basebox` allows specifying a different basebox as the default one. For example, `@vagrant_basebox = "LEAP/jessie"`. Logging options: @@ -69,7 +101,7 @@ All configuration files, other than `Leapfile`, are in the JSON format. For exam "key2": "value2" } -Keys should match `/[a-z0-9_]/` +Keys should match `/[a-z0-9_]/` and must be in double quotes. Unlike traditional JSON, comments are allowed. If the first non-whitespace characters are `//` then the line is treated as a comment. @@ -141,6 +173,71 @@ The `provider_base` directory is under the `leap_platform` specified in the file To see all the variables a node has inherited, you could run `leap inspect willamette`. +### Inheritance rules + +Suppose you have a node configuration `mynode.json`: + + { + "tags": "production", + "simple_value": 100, + "replaced_array": ["dolphin", "kangaroo"], + "+add_array": ["red", "black"], + "-subtract_array": ["bitter"], + "converted_to_array": "not_array_element", + "!override": ["insist on this value"], + "hash": { + "key1": 1, + "key2": 2 + } + } + +And a file `tags/production.json`: + + { + "simple_value": 99999, + "replaced_array": ["zebra"], + "add_array": ["green], + "subtract_array": ["bitter", "sweet", "salty"], + "converted_to_array": ["array_element"], + "override": "this value will be overridden", + "hash": { + "key1": "one" + } + } + +In this scenario, `mynode.json` will inherit from `production.json`. The output of this inheritance will be: + + { + "tags": "production", + "simple_value": 100, + "replaced_array": ["dolphin", "kangaroo"], + "add_array": ["red", "black", "green"], + "subtract_array": ["sweet", "salty"], + "converted_to_array": ["not_array_element", "array_element"], + "override": ["insist on this value"], + "hash": { + "key1": 1, + "key2": 2 + } + +The rules for inheritance (where 'old' refers to the parent, and 'new' refers to the child): + +* Simple values (strings, numbers, boolean): + * Replace the old value with the new value. +* Array values: + * Two arrays: replace the old array with the new array. + * One array and one simple value: add the simple value to the array. + * If property name is prefixed with "+": merge the old and new arrays. + * If property name is prefixed with "-": subtract new array from old array. +* Hash values: + * Hashes are always merged (the result includes the keys of both hashes). If there is a key in common, the new one overrides the old one. +* Mismatch: + * Although you can mix arrays and simple values, you cannot mix arrays with hashes or hashes with simple values. If you attempt to do so, it will fail to compile and give you an error message. +* Override: + * If property name is prefixed with "!": then ensure that new value is always used, regardless of old value. In this case, the override takes precedence over type checking, so you will never get a type mismatch. + +NOTE: special property name prefixes, like "+", "-", or "!", are not included in the property name. These prefixes determine the merge strategy, but are stripped out when compiling the resulting JSON file. + Common configuration options ---------------------------------------- diff --git a/doc/guide/domains.md b/doc/guide/domains.md new file mode 100644 index 00000000..914bce33 --- /dev/null +++ b/doc/guide/domains.md @@ -0,0 +1,129 @@ +@title = "Domains" +@summary = "How to handle domain names and integrating LEAP with existing services." +@toc = true + +Overview +-------------------------------- + +Deploying LEAP can start to get very tricky when you need to integrate LEAP services with an existing domain that you already use or which already has users. Most of this complexity is unavoidable, although there are a few things we plan to do in the future to make this a little less painful. + +Because integration with legacy systems is an advanced topic, we recommend that you begin with a new domain. Once everything works and you are comfortable with your LEAP-powered infrastructure, you can then contemplate integrating with your existing domain. + +### Definitions + +**provider domain** + +This is the main domain used to identify the provider. The **provider domain** is what the user enters in the Bitmask client. e.g. `example.org`. The full host name of every node in your provider infrastructure will use the **provider domain** (e.g. `dbnode.example.org`). + +In order for the Bitmask client to get configured for use with a provider, it must be able to find the `provider.json` bootstrap file at the root of the **provider domain**. This is not needed if the Bitmask client is "pre-seeded" with the provider's information (these providers show up in a the initial list of available providers). + +**webapp domain** + +This is the domain that runs the leap_web application that allows users to register accounts, create help tickets, etc. e.g. `example.org` or `user.example.org`. The **webapp domain** defaults to the **provider domain** unless it is explicitly configured separately. + +**API domain** + +This is the domain that the provider API runs on. Typically, this is set automatically and you never need to configure it. The user should never be aware of this domain. e.g. `api.example.org`. The Bitmask client discovers this API domain by reading it from the `provider.json` file it grabs from the **provider domain**. + +**mail domain** + +This is the domain used for mail accounts, e.g. `username@example.org`. Currently, this is always the **provider domain**, but it may be independently configurable in the future. + +Generating a zone file +----------------------------------- + +Currently, the platform does not include a dedicated `dns` service type, so you need to have your own setup for DNS. You can generate the appropriate configuration options with this command: + + leap compile zone + +A single domain +------------------------------- + +The easy approach is to use a single domain for **provider domain**, **webapp domain**, and **email domain**. This will install the webapp on the **provider domain**, which means that this domain must be a new one that you are not currently using for anything. + +To configure a single domain, just set the domain in `provider.json`: + + { + "domain": "example.org" + } + +If you have multiple environments, you can specify a different **provider domain** for each environment. For example: + +`provider.staging.json` + + { + "domain": "staging.example.org" + } + +A separate domain for the webapp +-------------------------------------- + +It is possible make the **webapp domain** different than the **provider domain**. This is needed if you already have a website running at your **provider domain**. + +In order to put webapp on a different domain, you must take two steps: + +1. You must configure `webapp.domain` for nodes with the `webapp` service. +2. You must make the compiled `provider.json` available at the root of the **provider domain**. + +NOTE: This compiled provider.json is different than the provider.json that you edit and lives in the root of the provider directory. + +### Step 1. Configuring `webapp.domain` + +In `services/webapp.json`: + + { + "webapp": { + "domain": "user.example.org" + } + } + +### Step 2. Putting the compiled `provider.json` in place + +Generate the compiled `provider.json`: + + leap compile provider.json + = created files/web/bootstrap/ + = created files/web/bootstrap/README + = created files/web/bootstrap/production/ + = created files/web/bootstrap/production/provider.json + = created files/web/bootstrap/production/htaccess + = created files/web/bootstrap/staging/ + = created files/web/bootstrap/staging/provider.json + = created files/web/bootstrap/staging/htaccess + +This command compiles a separate `provider.json` for each environment, or "default" if you don't have an environment. In the example above, there is an environment called "production" and one called "staging", but your setup will probably differ. + +The resulting `provider.json` file must then be put at the root URL of your **provider domain** for the appropriate environment. + +There is one additional complication: currently, the Bitmask client tests for compatibility using some HTTP headers on the `/provider.json` response. This is will hopefully change in the future, but for now you need to ensure the right headers are set in the response. The included file `htaccess` has example directives for Apache, if that is what you use. + +This step can be skipped if you happen to use the `static` service to deploy an `amber` powered static website to **provider domain**. In this case, the correct `provider.json` will be automatically put into place. + +Integrating with existing email system +----------------------------------------- + +If your **mail domain** already has users from a legacy email system, then things get a bit complicated. In order to be able to support both LEAP-powered email and legacy email on the same domain, you need to follow these steps: + +1. Modify the LEAP webapp so that it does not create users with the same name as users in the legacy system. +2. Configure your legacy MX servers to forward mail that they cannot handle to the LEAP MX servers, or vice versa. + +### Step 1. Modify LEAP webapp + +In order to modify the webapp to respect the usernames already reserved by your legacy system, you need to modify the LEAP webapp code. The easiest way to do this is to create a custom gem that modifies the behavior of the webapp. + +For this example, we will call our custom gem `reserve_usernames`. + +This gem can live in one of two places: + +(1) You can fork the project leap_web and put the gem in `leap_web/vendor/gems/reserve_usernames`. Then, modify `Gemfile` and add the line `gem 'common_languages', :path => 'vendor/gems/reserve_usernames'` + +(2) Alternately, you can put the gem in the local provider directory `files/webapp/gems/reserve_username`. This will get synced to the webapp servers when you deploy and put in `/srv/leap/webapp/config/customization` where it will get automatically loaded by the webapp. + +What should the gem `reserve_usernames` look like? There is an example available here: https://leap.se/git/reserved_usernames.git + +This example gem uses ActiveResource to communicate with a remote REST API for creating and checking username reservations. This ensures that both the legacy system and the LEAP system use the same namespace. Alternately, you could write a gem that checks the legacy database directly. + +### Step 2. Configure MX servers + +To be written. + diff --git a/doc/guide/getting-started.md b/doc/guide/getting-started.md new file mode 100644 index 00000000..6236cba0 --- /dev/null +++ b/doc/guide/getting-started.md @@ -0,0 +1,145 @@ +@title = 'Getting Started' +@summary = 'An overview of the LEAP Platform' +@toc = true + + +Sensitive files +---------------------------------------------- + +Some files in your provider directory are very sensitive. Leaking these files will compromise your provider. + +Super sensitive and irreplaceable: + +* `files/ca/*.key` -- the private keys for the client and server CAs. +* `files/cert/*.key` -- the private key(s) for the commercial certificate for your domain(s). + +Sensitive, but can be erased and regenerated automatically: + +* `secrets.json` -- various random secrets, such as passwords for databases. +* `files/nodes/*/*.key` -- the private key for each node. +* `hiera/*.yaml` -- hiera file contains a copy of the private key of the node. + +Also, each sysadmin has one or more public ssh keys in `users/*/*_ssh.pub`. Typically, you will want to keep these public keys secure as well. + +See [[keys-and-certificates]] for more information. + +Useful commands +------------------------------------------- + +Here are a few useful `leap` commands: + +* `leap help [COMMAND]` -- get help on COMMAND. +* `leap history [FILTER]` -- show the recent deployment history for the selected nodes. +* `leap ssh web1` -- SSH into node web1 (requires `leap node init web1` first). +* `leap list [FILTER]` -- list the selected nodes. + * `leap list production` -- list only those nodes with the tag 'production' + * `leap list --print ip_address` -- list a particular attribute of all nodes. + +See the full [[commands]] for more information. + +Node filters +------------------------------------------- + +Many of the `leap` commands take a "node filter". You can use a node filter to target a command at one or more nodes. + +A node filter consists of one or more keywords, with an optional "+" before each keyword. + +* keywords can be a node name, a service type, or a tag. +* the "+" before the keyword constructs an AND condition +* otherwise, multiple keywords together construct an OR condition + +Examples: + +* `leap list openvpn` -- list all nodes with service openvpn. +* `leap list openvpn +production` -- only nodes of service type openvpn AND tag production. +* `leap deploy webapp openvpn` -- deploy to all webapp OR openvpn nodes. +* `leap node init ostrich` -- just init the node named ostrich. + +See the full [[commands]] for more information. + +Tracking the provider directory in git +------------------------------------------ + +You should commit your provider changes to your favorite VCS whenever things change. This way you can share your configurations with other admins, all they have to do is to pull the changes to stay up to date. Every time you make a change to your provider, such as adding nodes, services, generating certificates, etc. you should add those to your VCS, commit them and push them to where your repository is hosted. + +Note that your provider directory contains secrets, such as private key material and passwords. You do not want to have those passwords readable by the world, so make sure that wherever you are hosting your repository, it is not public for the world to read. + +If you have a post-commit hook that emails the changes to contributors, you may want to exclude diffs for files that might have sensitive secrets. For example, create a `.gitattributes` file with: + + # No diff, no email for key files + *.key -diff + *.pem -diff + + # Discard diff for secrets.json + secrets.json -diff + + # No diff for hiera files, they contain passwords + hiera/* -diff + + +Editing JSON configuration files +-------------------------------------- + +All the settings that compose your provider are stored in JSON files. + +At a minimum, you will need at least two configuration files: + +* `provider.json` -- general settings for you provider. +* `nodes/NAME.json` -- configuration file for node called "NAME". + +There are a few required properties in provider.json: + + { + "domain": "example.org", + "name": "Example", + "contacts": { + "default": "email1@example.org" + } + } + +See [[provider-configuration]] for more details. + +For node configuration files, there are two required properties: + + { + "ip_address": "1.1.1.1", + "services": ["openvpn"] + } + +See [[services]] for details on what servers are available, and see [[config]] details on how configuration files work. + +How does it work under the hood? +-------------------------------------------- + +You don't need to know any of the details of what happens "under the hood" in order to use the LEAP platform. However, if you are curious as to what is going on, here is a quick primer. + +First, some background terminology: + +* **puppet**: Puppet is a system for automating deployment and management of servers (called nodes). +* **hiera files**: In puppet, you can use something called a 'hiera file' to seed a node with a few configuration values. In LEAP, we go all out and put *every* configuration value needed for a node in the hiera file, and automatically compile a custom hiera file for each node. + +When you run `leap deploy`, a bunch of things happen, in this order: + +1. **Compile hiera files**: The hiera configuration file for each node is compiled in YAML format and saved in the directory `hiera`. The source material for this hiera file consists of all the JSON configuration files imported or inherited by the node's JSON config file. +* **Copy required files to node**: All the files needed for puppet to run are rsync'ed to each node. This includes the entire leap_platform directory, as well as the node's hiera file and other files needed by puppet to set up the node (keys, binary files, etc). +* **Puppet is run**: Once the node is ready, leap connects to the node via ssh and runs `puppet apply`. Puppet is applied locally on the node, without a daemon or puppetmaster. + +You can run `leap -v2 deploy` to see exactly what commands are being executed. + +This mode of operation is fundamentally different from how puppet is normally used: + +* There is no puppetmaster that all the servers take orders from, and there is no puppetd running in the background. +* Servers cannot dynamically query the puppetmaster for information about the other servers. +* There is a static representation for the state of every server that can be committed to git. + +There are advantages and disadvantages to the model that LEAP uses. We have found it very useful for our goal of having a common LEAP platform that many different providers can all use while still allowing providers to configure their unique infrastructure. + +We also find it very beneficial to be able to track the state of your infrastructure in git. + +Traditional system configuration automation systems, like [Puppet](https://puppetlabs.com/puppet/puppet-open-source/) or [Chef](http://www.opscode.com/chef/), deploy changes to servers using a pull method. Each server pulls a manifest from a central master server and uses this to alter the state of the server. + +Instead, the `leap` tool uses a masterless push method: The sysadmin runs `leap deploy` from the provider instance directory on their desktop machine to push the changes out to every server (or a subset of servers). LEAP still uses Puppet, but there is no central master server that each node must pull from. + +One other significant difference between LEAP and typical system automation is how interactions among servers are handled. Rather than store a central database of information about each server that can be queried when a recipe is applied, the `leap` command compiles static representation of all the information a particular server will need in order to apply the recipes. In compiling this static representation, `leap` can use arbitrary programming logic to query and manipulate information about other servers. + +These two approaches, masterless push and pre-compiled static configuration, allow the sysadmin to manage a set of LEAP servers using traditional software development techniques of branching and merging, to more easily create local testing environments using virtual servers, and to deploy without the added complexity and failure potential of a master server. diff --git a/doc/guide/keys-and-certificates.md b/doc/guide/keys-and-certificates.md index aef02ac6..a6862a6a 100644 --- a/doc/guide/keys-and-certificates.md +++ b/doc/guide/keys-and-certificates.md @@ -4,7 +4,7 @@ Working with SSH ================================ -Whenever the `leap` command nees to push changes to a node or gather information from a node, it tunnels this command over SSH. Another way to put this: the security of your servers rests entirely on SSH. Because of this, it is important that you understand how `leap` uses SSH. +Whenever the `leap` command needs to push changes to a node or gather information from a node, it tunnels this command over SSH. Another way to put this: the security of your servers rests entirely on SSH. Because of this, it is important that you understand how `leap` uses SSH. SSH related files ------------------------------- @@ -21,7 +21,7 @@ All of these files should be committed to source control. If you rename, remove, or add a node with `leap node [mv|add|rm]` the SSH key files and the `known_hosts` file will get properly updated. SSH and local nodes ------------------------------ +------------------- Local nodes are run as Vagrant virtual machines. The `leap` command handles SSH slightly differently for these nodes. @@ -33,6 +33,15 @@ Specifically, for local nodes: 2. `leap` entirely skips the checking of host keys when connecting with a local node. 3. `leap` adds the public Vagrant SSH key to the list of SSH keys for a user. The public Vagrant SSH key is a shared and insecure key that has root access to most Vagrant virtual machines. +To upgrade a SSH host key +------------------------------- + +Most servers will have more than one SSH host key. Sometimes, the server will have a better SSH host key than the one you have on file. In order to upgrade to the better SSH host key, simply re-run the init command: + + workstation$ leap node init NODE_NAME + +This will prompt you if you want to upgrade the SSH host key, but only if `leap` thinks that an upgrade is advisable. + When SSH host key changes ------------------------------- @@ -78,7 +87,7 @@ All keys matching 'userx/*_ssh.pub' will be usable. Removing sysadmin access -------------------------------- -Suppose you want to remove `userx` from having any further ssh access to the servers. Do this: +Suppose you want to remove `userx` from having any further SSH access to the servers. Do this: rm -r users/userx leap deploy @@ -192,3 +201,72 @@ If you want to add additional fields to the CSR, like country, city, or locality } If they are not present, the CSR will be created without them. + +Examine Certs +----------------- + +To see details about the keys and certs you can use `leap inspect` like so: + + $ leap inspect files/ca/ca.crt + + +Let's Encrypt certificate +========================= + +LEAP plans to integrate [Let's Encrypt](https://letsencrypt.org/) support, so it will be even easier to receive X.509 certificates that are accepted by all browsers. +Until we achieve this, here's a guide how to do this manually. + +Install the official acme client +-------------------------------- + +Log in to your webapp node + + server$ git clone https://github.com/letsencrypt/letsencrypt + server$ cd letsencrypt + server$ ./letsencrypt-auto --help + +Fetch cert +---------- + +Stop apache so the letsencrypt client can bind to port 80: + + server$ systemctl stop apache2 + +Fetch the certs + + server$ ./letsencrypt-auto certonly --standalone --email admin@$(hostname -d) -d $(hostname -d) -d api.$(hostname -d) -d $(hostname -f) -d nicknym.$(hostname -d) + +This will put the certs and keys into `/etc/letsencrypt/live/DOMAIN/`. + +Now, go to your workstation's provider configuration directory and copy the newly created files from the server to your local config. You will override existing files so please make a backup before proceeding, or use a version control system to track changes. + + workstation$ cd PATH_TO_PROVIDER_CONFIG + +Copy the Certificate + + workstation$ scp root@SERVER:/etc/letsencrypt/live/DOMAIN/cert.pem files/cert/dev.pixelated-project.org.crt + +Copy the private key + + workstation$ scp root@SERVER:/etc/letsencrypt/live/DOMAIN/privkey.pem files/cert/DOMAIN.key + +Copy the CA chain cert + + workstation$ scp root@SERVER:/etc/letsencrypt/live/DOMAIN/fullchain.pem files/cert/DOMAIN.key + +Deploy the certs +---------------- + +Now you only need to deploy the certs + + workstation$ leap deploy + +This will put them into the right locations which are: + +- `/etc/x509/certs/leap_commercial.crt` for the certificate +- `/etc/x509/./keys/leap_commercial.key` for the private key +- `/usr/local/share/ca-certificates/leap_commercial_ca.crt` for the CA chain cert. + +Start apache2 again + + server$ systemctl start apache2 diff --git a/doc/guide/nodes.md b/doc/guide/nodes.md index cf225449..5135f3ba 100644 --- a/doc/guide/nodes.md +++ b/doc/guide/nodes.md @@ -1,106 +1,6 @@ @title = "Nodes" @summary = "Working with nodes, services, tags, and locations." -Node types -================================ - -Every node has one or more services that determines the node's function within your provider's infrastructure. - -When adding a new node to your provider, you should ask yourself four questions: - -* **many or few?** Some services benefit from having many nodes, while some services are best run on only one or two nodes. -* **required or optional?** Some services are required, while others can be left out. -* **who does the node communicate with?** Some services communicate very heavily with other particular services. Nodes running these services should be close together. -* **public or private?** Some services communicate with the public internet, while others only need to communicate with other nodes in the infrastructure. - -Brief overview of the services: - -* **webapp**: The web application. Runs both webapp control panel for users and admins as well as the REST API that the client uses. Needs to communicate heavily with `couchdb` nodes. You need at least one, good to have two for redundancy. The webapp does not get a lot of traffic, so you will not need many. -* **couchdb**: The database for users and user data. You can get away with just one, but for proper redundancy you should have at least three. Communicates heavily with `webapp`, `mx`, and `soledad` nodes. -* **soledad**: Handles the data syncing with clients. Typically combined with `couchdb` service, since it communicates heavily with couchdb. -* **mx**: Incoming and outgoing MX servers. Communicates with the public internet, clients, and `couchdb` nodes. -* **openvpn**: OpenVPN gateway for clients. You need at least one, but want as many as needed to support the bandwidth your users are doing. The `openvpn` nodes are autonomous and don't need to communicate with any other nodes. Often combined with `tor` service. -* **monitor**: Internal service to monitor all the other nodes. Currently, you can have zero or one `monitor` service defined. It is required that the monitor be on the webapp node. It was not designed to be run as a separate node service. -* **tor**: Sets up a tor exit node, unconnected to any other service. -* **dns**: Not yet implemented. - -Webapp ------------------------------------ - -The webapp node is responsible for both the user face web application and the API that the client interacts with. - -Some users can be "admins" with special powers to answer tickets and close accounts. To make an account into an administrator, you need to configure the `webapp.admins` property with an array of user names. - -For example, to make users `alice` and `bob` into admins, create a file `services/webapp.json` with the following content: - - { - "webapp": { - "admins": ["bob", "alice"] - } - } - -And then redeploy to all webapp nodes: - - leap deploy webapp - -By putting this in `services/webapp.json`, you will ensure that all webapp nodes inherit the value for `webapp.admins`. - -Services -================================ - -What nodes do you need for a provider that offers particular services? - -<table class="table table-striped"> -<tr> - <th>Node Type</th> - <th>VPN Service</th> - <th>Email Service</th> - <th>Notes</th> -</tr> -<tr> - <td>webapp</td> - <td>required</td> - <td>required</td> - <td></td> -</tr> -<tr> - <td>couchdb</td> - <td>required</td> - <td>required</td> -<td></td> -</tr> -<tr> - <td>soledad</td> - <td>not used</td> - <td>required</td> -<td></td> -</tr> -<tr> - <td>mx</td> - <td>not used</td> - <td>required</td> - <td></td> -</tr> -<tr> - <td>openvpn</td> - <td>required</td> - <td>not used</td> - <td></td> -</tr> -<tr> - <td>monitor</td> - <td>optional</td> - <td>optional</td> - <td>This service must be on the webapp node</td> -</tr> -<tr> - <td>tor</td> - <td>optional</td> - <td>optional</td> - <td></td> -</tr> -</table> - Locations ================================ diff --git a/doc/guide/provider-configuration.md b/doc/guide/provider-configuration.md new file mode 100644 index 00000000..08cfd1dd --- /dev/null +++ b/doc/guide/provider-configuration.md @@ -0,0 +1,79 @@ +@title = "Provider Configuration" +@summary = "Explore how to configure your provider." + +Required provider configuration +-------------------------------------- + +There are a few required settings in `provider.json`. At a minimum, you must have: + +* `domain`: defines the primary domain of the provider. This is the domain that users will type in when using the Bitmask client, although it is not necessarily the domain where users will visit if they sign up via the web application. If email is supported, all accounts will be `username@domain`. +* `name`: A brief title for this provider. It can be multiple words, but should not be too long. +* `contacts.default`: One or more email addresses for sysadmins. + +For example: + + { + "domain": "freerobot.org", + "name": "Freedom for Robots!", + "contacts": { + "default": "root@freerobot.org" + } + } + + +Recommended provider configuration +-------------------------------------- + +* `description`: A longer description of the provider, shown to the user when they register a new account through Bitmask client. +* `languages`: A list of language codes that should be enabled. +* `default_language`: The initial default language code. +* `enrollment_policy`: One of "open", "closed", or "invite". (invite not currently supported). + +For example: + + { + "description": "It is time for robots of the world to unite and throw of the shackles of servitude to our organic overlords.", + "languages": ["en", "de", "pt", "01"], + "default_language": "01", + "enrollman_policy": "open" + } + +For a full list of possible settings, you can use `leap inspect` to see how provider.json is evaluated after including the inherited defaults: + + $ leap inspect provider.json + +Configuring service levels +-------------------------------------- + +The `provider.json` file defines the available service levels for the provider. + +For example, in provider.json: + + "service": { + "default_service_level": "low", + "levels": { + "low": { + "description": "Entry level plan, with unlimited bandwidth and minimal storage quota.", + "name": "entry", + "storage": "10 MB", + "rate": { + "USD": 5, + "GBP": 3, + "EUR": 6 + } + }, + "full": { + "description": "Full plan, with unlimited bandwidth and higher quota." + "name": "full", + "storage": "5 GB", + "rate": { + "USD": 10, + "GBP": 6, + "EUR": 12 + } + } + } + } + } + +For a list of currency codes, see https://en.wikipedia.org/wiki/ISO_4217#Active_codes diff --git a/doc/services/couchdb.md b/doc/services/couchdb.md new file mode 100644 index 00000000..cc40dc32 --- /dev/null +++ b/doc/services/couchdb.md @@ -0,0 +1,159 @@ +@title = "couchdb" +@summary = "Data storage for all user data." + +Topology +------------------------ + +Required: + +* Nodes with `couchdb` service must also have `soledad` service, if email is enabled. + +Suggested: + +* Nodes with `couchdb` service communicate heavily with `webapp` and `mx`. + +`couchdb` nodes do not need to be reachable from the public internet, although the `soledad` service does require this. + +Configuration +---------------------------- + +### Nighly dumps + +You can do a nightly couchdb data dump by adding this to your node config: + + "couch": { + "backup": true + } + +Data will get dumped to `/var/backups/couchdb`. + +### Plain CouchDB + +BigCouch is not supported on Platform version 0.8 and higher: only plain CouchDB is possible. For earlier versions, you must do this in order to use plain CouchDB: + + "couch": { + "master": true, + "pwhash_alg": "pbkdf2" + } + +Various Tasks +------------------------------------------------- + +### Re-enabling blocked account + +When a user account gets destroyed from the webapp, there's still a leftover doc in the identities db so other people can't claim that account without an admin's intervention. You can remove this username reservation through the webapp. + +However, here is how you could do it manually, if you wanted to: + +grep the identities db for the email address: + + curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET http://127.0.0.1:5984/identities/_all_docs?include_docs=true|grep test_127@bitmask.net + +lookup "id" and "rev" to delete the doc: + + curl -s --netrc-file /etc/couchdb/couchdb.netrc -X DELETE 'http://127.0.0.1:5984/identities/b25cf10f935b58088f0d547fca823265?rev=2-715a9beba597a2ab01851676f12c3e4a' + +### How to find out which userstore belongs to which identity? + + /usr/bin/curl -s --netrc-file /etc/couchdb/couchdb.netrc '127.0.0.1:5984/identities/_all_docs?include_docs=true' | grep testuser + + {"id":"665e004870ee17aa4c94331ff3ecb173","key":"665e004870ee17aa4c94331ff3ecb173","value":{"rev":"2-2e335a75c4b79a5c2ef5c9950706fe1b"},"doc":{"_id":"665e004870ee17aa4c94331ff3ecb173","_rev":"2-2e335a75c4b79a5c2ef5c9950706fe1b","user_id":"665e004870ee17aa4c94331ff3cd59eb","address":"testuser@example.org","destination":"testuser@example.org","keys": ... + +* search for the "user_id" field +* in this example testuser@example.org uses the database user-665e004870ee17aa4c94331ff3cd59eb + + +### How much disk space is used by a userstore + +Beware that this returns the uncompacted disk size (see http://wiki.apache.org/couchdb/Compaction) + + echo "`curl --netrc -s -X GET 'http://127.0.0.1:5984/user-dcd6492d74b90967b6b874100b7dbfcf'|json_pp|grep disk_size|cut -d: -f 2`/1024"|bc + + +Deprecated BigCouch Tasks +----------------------------------------- + +As of release 0.8, the LEAP platform no longer supports BigCouch. This information is kept here for historical reference. + +### Rebalance Cluster + +Bigcouch currently does not have automatic rebalancing. +It will probably be added after merging into couchdb. +If you add a node, or remove one node from the cluster, + +1. make sure you have a backup of all DBs ! + +1. put the webapp into [[maintenance mode => services/webapp#maintenance-mode]] + +1. Stop all services that access the database: + + ``` + workstation$ leap ssh soledad-nodes + server# /etc/init.d/soledad-server stop + + workstation$ leap ssh mx-node + server# /etc/init.d/postfix stop + server# /etc/init.d/leap-mx stop + + workstation$ leap ssh webapp + server# /etc/init.d/nickserver stop + ``` + + Alternately, you can create a temporary firewall rule to block access (run on couchdb server): + + ``` + server# iptables -A INPUT -p tcp --dport 5984 --jump REJECT + ``` + +1. dump the dbs: + + ``` + cd /srv/leap/couchdb/scripts + time ./couchdb_dumpall.sh + ``` + +1. delete all dbs + +1. shut down old node + +1. check the couchdb members + + ``` + curl -s —netrc-file /etc/couchdb/couchdb.netrc -X GET http://127.0.0.1:5986/nodes/_all_docs + curl -s —netrc-file /etc/couchdb/couchdb.netrc http://127.0.0.1:5984/_membership + ``` + +1. remove bigcouch from all nodes + + ``` + apt-get --purge remove bigcouch + ``` + +1. deploy to all couch nodes + + ``` + leap deploy couchdb + ``` + +1. most likely, deploy will fail because bigcouch will complain about not all nodes beeing connected. Let the deploy finish, restart the bigcouch service on all nodes and re-deploy: + + ``` + /etc/init.d/bigcouch restart + ``` + +1. restore the backup + + ``` + cd /srv/leap/couchdb/scripts + time ./couchdb_restoreall.sh + ``` + +### Migrating from BigCouch to plain CouchDB + +<%= render :partial => 'docs/platform/common/bigcouch_migration_begin.md' %> + + +<%= render :partial => 'docs/platform/common/bigcouch_migration_end.md' %> + + +<%= render :partial => 'docs/platform/common/bigcouch_migration_finish.md' %> diff --git a/doc/services/en.md b/doc/services/en.md new file mode 100644 index 00000000..5d0fec5f --- /dev/null +++ b/doc/services/en.md @@ -0,0 +1,80 @@ +@nav_title = "Services" +@title = "Guide to node services" +@summary = "" +@toc = true + +# Introduction + +Every node (server) must have one or more `services` defined that determines what role the node performs. For example: + + workstation$ cat nodes/stallman.json + { + "ip_address": "199.99.99.1", + "services": ["webapp", "tor"] + } + +Here are common questions to ask when adding a new node to your provider: + +* **many or few?** Some services benefit from having many nodes, while some services are best run on only one or two nodes. +* **required or optional?** Some services are required, while others can be left out. +* **who does the node communicate with?** Some services communicate very heavily with other particular services. Nodes running these services should be close together. +* **public or private network?** Some services communicate with the public internet, while others only need to communicate with other nodes in the infrastructure. + +# Available services + +<table class="table table-striped"> +<tr> + <th>Service</th> + <th>VPN</th> + <th>Email</th> + <th>Notes</th> +</tr> +<tr> + <td>webapp</td> + <td><i class="fa fa-circle"></i></td> + <td><i class="fa fa-circle"></i></td> + <td>User control panel, provider API, and support system.</td> +</tr> +<tr> + <td>couchdb</td> + <td><i class="fa fa-circle"></i></td> + <td><i class="fa fa-circle"></i></td> + <td>Data storage for everything. Private node.</td> +<td></td> +</tr> +<tr> + <td>soledad</td> + <td><i class="fa fa-circle-o"></i></td> + <td><i class="fa fa-circle"></i></td> + <td>User data synchronization daemon. Usually paired with <code>couchdb</code> nodes.</td> +<td></td> +</tr> +<tr> + <td>mx</td> + <td><i class="fa fa-circle-o"></i></td> + <td><i class="fa fa-circle"></i></td> + <td>Incoming and outgoing MX servers.</td> +</tr> +<tr> + <td>openvpn</td> + <td><i class="fa fa-circle"></i></td> + <td><i class="fa fa-circle-o"></i></td> + <td>OpenVPN gateways.</td> +</tr> +<tr> + <td>monitor</td> + <td><i class="fa fa-dot-circle-o"></i></td> + <td><i class="fa fa-dot-circle-o"></i></td> + <td>Nagios monitoring. This service must be on the webapp node.</td> +</tr> +<tr> + <td>tor</td> + <td><i class="fa fa-dot-circle-o"></i></td> + <td><i class="fa fa-dot-circle-o"></i></td> + <td>Tor exit node.</td> +</tr> +</table> + +Key: <i class="fa fa-circle"> Required</i>, <i class="fa fa-dot-circle-o"> Optional</i>, <i class="fa fa-circle-o"> Not Used</i> + +<%= child_summaries %>
\ No newline at end of file diff --git a/doc/services/monitor.md b/doc/services/monitor.md new file mode 100644 index 00000000..576b36a9 --- /dev/null +++ b/doc/services/monitor.md @@ -0,0 +1,36 @@ +@title = "monitor" +@summary = "Nagios monitoring and continuous testing." + +The `monitor` node provides a nagios control panel that will give you a view into the health and status of all the servers and all the services. It will also spam you with alerts if something goes down. + +Topology +-------------------------------------- + +Currently, you can have zero or one `monitor` nodes defined. It is required that the monitor be on the webapp node. It was not designed to be run as a separate node service. + +Configuration +----------------------------------------------- + +* `nagios.environments`: By default, the monitor node will monitor all servers in all environments. You can optionally restrict the environments to the ones you specify. + +For example: + + { + "nagios": { + "environments": ["unstable", "production"] + } + } + +Access nagios web +----------------------------------------------- + +*Determine the nagios URL* + + $ leap ls --print domain.name,webapp.domain,ip_address monitor + > chameleon chameleon.bitmask.net, demo.bitmask.net, 199.119.112.10 + +In this case, you would open `https://demo.bitmask.net/cgi-bin/nagios3` in your browser (or alternately you could use 199.119.112.10 or chameleon.bitmask.net). + +*Determine the nagios password* + +The username for nagios is always `nagiosadmin`. The password is randomly generated and stored in `secrets.json` under the key `nagios_admin_password`. Note that the login is `nagiosadmin` without underscore, but the entry in secrets.json is with underscores. diff --git a/doc/services/mx.md b/doc/services/mx.md new file mode 100644 index 00000000..1a34b660 --- /dev/null +++ b/doc/services/mx.md @@ -0,0 +1,35 @@ +@title = "mx" +@summary = "Incoming and outgoing MX servers." + +Topology +------------------- + +`mx` nodes communicate with the public internet, clients, and `couchdb` nodes. + +Configuration +-------------------- + +### Aliases + +Using the `mx.aliases` property, you can specify your own hard-coded email aliases that precedence over the aliases in the user database. The `mx.aliases` property consists of a hash, where source address points to one or more destination addresses. + +For example: + +`services/mx.json`: + + "mx": { + "aliases": { + "rook": "crow", + "robin": "robin@bird.org", + "flock": ["junco@bird.org", "robin", "crow"], + "chickadee@avian.org": "chickadee@bird.org", + "flicker": ["flicker@bird.org", "flicker@deliver.local"] + } + } + +This example demonstrates several of the features with `mx.aliases`: + +1. alias lists: by specifying an array of destination addresses, as in the case of "flock", the single email will get copied to each address. +1. chained resolution: alias resolution will recursively continue until there are no more matching aliases. For example, "flock" is resolved to "robin", which then gets resolved to "robin@bird.org". +1. virtual domains: by specifying the full domain, as in the case of "chickadee@avian.org", the alias will work for any domain you want. Of course, the MX record for that domain must point to appropriate MX servers, but otherwise you don't need to do any additional configuration. +1. local delivery: for testing purposes, it is often useful to copy all incoming mail for a particular address and send those copies to another address. You can do this by adding "@deliver.local" as one of the destination addresses. When "@local.delivery" is found, alias resolution stops and the mail is delivered to that username. diff --git a/doc/services/openvpn.md b/doc/services/openvpn.md new file mode 100644 index 00000000..5f15ff07 --- /dev/null +++ b/doc/services/openvpn.md @@ -0,0 +1,49 @@ +@title = 'openvpn' +@summary = "OpenVPN egress gateways" + +Topology +------------------ + +Currently, `openvpn` service should not be combined with other services on the same node. + +Unlike most of the other node types, the `openvpn` nodes do not need access to the database and does not ever communicate with any other nodes (except for the `monitor` node, if used). So, `openvpn` nodes can be placed anywhere without regard to the other nodes. + +Configuration +--------------------- + +*Essential configuration* + +* `openvpn.gateway_address`: The address that OpenVPN daemon is bound to and that VPN clients connect to. +* `ip_address`: The main IP of the server, and the egress address for outgoing traffic. + +For example: + + { + "ip_address": "1.1.1.1", + "openvpn": { + "gateway_address": "2.2.2.2" + } + } + +In this example, VPN clients will connect to 2.2.2.2, but their traffic will appear to come from 1.1.1.1. + +Why are two IP addresses needed? Without this, traffic between two VPN users on the same gateway will not get encrypted. This is because the VPN on every client must be configured to allow cleartext traffic for the IP address that is the VPN gateway. + +*Optional configuration* + +Here is the default configuration: + + "openvpn": { + "configuration": { + "auth": "SHA1", + "cipher": "AES-128-CBC", + "fragment": 1400, + "keepalive": "10 30", + "tls-cipher": "DHE-RSA-AES128-SHA", + "tun-ipv6": true + }, + "ports": ["80", "443", "53", "1194"], + "protocols": ["tcp", "udp"] + } + +You may want to change the ports so that only 443 or 80 are used. It is probably best to not modify the `openvpn.configuration` options for now.
\ No newline at end of file diff --git a/doc/services/soledad.md b/doc/services/soledad.md new file mode 100644 index 00000000..e2700d06 --- /dev/null +++ b/doc/services/soledad.md @@ -0,0 +1,12 @@ +@title = 'soledad' +@summary = 'User data synchronization daemon' + +Topology +-------------------- + +Currently, the platform is designed for `soledad` and `couchdb` services to be combined (e.g. every `soledad` node should also be a `couchdb` node). `soledad` nodes might work in isolation, but this is not tested. + +Configuration +------------------------ + +There are no options to configure for `soledad` nodes. diff --git a/doc/services/tor.md b/doc/services/tor.md new file mode 100644 index 00000000..e64b0fe0 --- /dev/null +++ b/doc/services/tor.md @@ -0,0 +1,32 @@ +@title = 'tor' +@summary = 'Tor exit node or hidden service' + +Topology +------------------------ + +Nodes with `tor` service will run a Tor exit or hidden service, depending on what other service it is paired with: + +* `tor` + `openvpn`: when combined with `openvpn` nodes, `tor` will create a Tor exit node to provide extra cover traffic for the VPN. This can be especially useful if there are VPN gateways without much traffic. +* `tor` + `webapp`: when combined with a `webapp` node, the `tor` service will make the webapp and the API available via .onion hidden service. +* `tor` stand alone: a regular Tor exit node. + +If activated, you can list the hidden service .onion addresses this way: + + leap ls --print tor.hidden_service.address tor + +Then just add '.onion' to the end of the printed addresses. + +Configuration +------------------------------ + +* `tor.bandwidth_rate`: the max bandwidth allocated to Tor, in KB per second, when used as an exit node. + +For example: + + { + "tor": { + "bandwidth_rate": 6550 + } + } + + diff --git a/doc/services/webapp.md b/doc/services/webapp.md new file mode 100644 index 00000000..1c06d715 --- /dev/null +++ b/doc/services/webapp.md @@ -0,0 +1,293 @@ +@title = "webapp" +@summary = "leap_web user management application and provider API." + +Introduction +------------------------ + +The service `webapp` will install the web application [[leap_web => https://leap.se/git/leap_web.git]]. It has performs the following functions: + +* REST API for user registration and authentication via the Bitmask client. +* Admin interface to manage users. +* Client certificate distribution and renewal. +* User support help tickets. + +Coming soon: + +* Billing. +* Customizable and localized user documentation. + +The leap_web application is written in Ruby on Rails 3, using CouchDB as the backend data store. + +Topology +------------------------- + +Currently, the platform only supports a single `webapp` node, although we hope to change this in the future. + +* `webapp` nodes communicate heavily with `couchdb` nodes, but the two can be on separate servers. +* The `monitor` service, if enabled, must be on the same node as `webapp`. + +Configuration +-------------------------- + +Essential options: + +* `webapp.admin`: An array of usernames that will be blessed with administrative permissions. These admins can delete users, answer help tickets, and so on. These usernames are for users that have registered through the webapp or through the Bitmask client application, NOT the sysadmin usernames lists in the provider directory `users`. + +Other options: + +* `webapp.engines`: A list of the engines you want enabled in leap_web. Currently, only "support" is available, and it is enabled by default. +* `webapp.invite_required`: If true, registration requires an invite code. Default is `false`. + +For example, `services/webapp.json`: + + { + "webapp": { + "admins": ["joehill", "ali", "mack_the_turtle"] + } + } + +By putting this in `services/webapp.json`, all the `webapp` nodes will inherit the same admin list. + +There are many options in `provider.json` that also control how the webapp behaves. See [[provider-configuration]] for details. + +Invite codes +------------------- + +Enabling the invite code functionality will require new users to provide a valid invite code while signing up for a new account. This is turned off by default, allowing all new users to create an account. + +Set the `invite_code` option to `true` in `services/webapp.json`: + + { + "webapp": { + "invite_required": true + } + } + +This only works with LEAP platform 0.8 or higher. + +Run `leap deploy` to enable the option. + +You can then generate invite codes by logging into the web application with an admin user. + +Alternately, you can also generate invite codes with the command line: + + workstation$ leap ssh bumblebee + bumblebee# cd /srv/leap/webapp/ + bumblebee# sudo -u leap-webapp RAILS_ENV=production bundle exec rake "generate_invites[NUM,USES]" + +Where `bumblebee` should be replaced with the name of your webapp node. + +The **NUM** specifies the amount of codes to generate. The **USES** parameter is optional: By default, all new invite codes can be used once and will then become invalid. If you provide another value for **USES**, you can set a different amount of maximum uses for the codes you generate. + +Customization +--------------------------- + +The provider directory `files/webapp` can be used to customize the appearance of the webapp. All the files in this directory will get sync'ed to the `/srv/leap/webapp/config/customization` directory of the deployed webapp node. + +Files in the `files/webapp` can override view files, locales, and stylesheets in the leap_web app: + +For example: + + stylesheets/ -- override files in Rails.root/app/assets/stylesheets + tail.scss -- included before all others + head.scss -- included after all others + + public/ -- overrides files in Rails.root/public + favicon.ico -- custom favicon + img/ -- customary directory to put images in + + views/ -- overrides files Rails.root/app/views + home/ + index.html.haml -- this file is what shows up on + the home page + pages/ + privacy-policy.en.md -- this file will override + the default privacy policy + terms-of-service.en.md -- this file will override + the default TOS. + + locales/ -- overrides files in Rails.root/config/locales + en.yml -- overrides for English + de.yml -- overrides for German + and so on... + +To interactively develop your customizations before you deploy them, you have two options: + +1. Edit a `webapp` node. This approach involves directly modifying the contents of the directory `/srv/leap/webapp/config/customization` on a deployed `webapp` node. This can, and probably should be, a "local" node. When doing this, you may need to restart leap_web in order for changes to take effect (`touch /srv/leap/webapp/tmp/restart.txt`). +2. Alternately, you can install leap_web to run on your computer and edit files in `config/customization` locally. This approach does not require a provider or a `webapp` node. For more information, see the [leap_web README](https://github.com/leapcode/leap_web). + +NOTE: If you add a `tails.scss` or `head.scss` file, then you usually need to run `rake tmp:clear` and restart rails in order for the new stylesheet to get recognized. You should only need to do this once. + +Once you have what you want, then copy these files to the local provider directory `files/webapp` so that they will be installed each time you deploy. + +Customization tutorial +---------------------------- + +This mini-tutorial will walk you through creating a custom "branding" of the leap_web application. We will be creating a provider called "Prehistoric Computer." + +Here are the files we are going to create: + + leap_web/config/customization + ├── locales + │  ├── en.yml + │  └── es.yml + ├── public + │  ├── favicon.ico + │  └── img + │  └── masthead.png + ├── stylesheets + │  └── tail.scss + └── views + └── pages + ├── privacy-policy.en.md + └── privacy-policy.es.md + +All these files are available in the source code in the [[customization.example => https://github.com/leapcode/leap_web/tree/develop/config/customization.example]] directory. + +Remember, these files may live different places: + +* `user@localmachine$ leap_web/config/customization`: This will be the path if you have checked out a local copy of leap_web.git and are running `rails server` locally in order to test your customizations. +* `user@localmachine$ PROVIDER/files/webapp`: This is the local provider directory where the files should be put so that they get correctly deployed to webapp nodes. +* `root@webappnode# /srv/leap/webapp/config/customization`: This is where the files in the local provider directory `PROVIDER/files/webapp` get copied to after a `leap deploy` to a live webapp nodes. + +### Override translations + +You can add additional locale files in order to change the text used in the existing application and to add translations for string that you added to the application. + +In this example, we will be altering the default text for the "login_info" string. In `config/locales/en/home.en.yml` there is this entry: + + en: + login_info: "Log in to change your account settings, create support tickets, and manage payments." + +We are going to override this with some custom text in English and Spanish: + +`leap_web/config/customization/locale/en.yml`: + + en: + login_info: Authenticate to change your "Prehistoric Computer" settings. + +`leap_web/config/customization/locale/es.yml`: + + es: + login_info: Autenticar a cambiar la configuración de "Computer Prehistoria." + +Now, the home page of leap_web will use these new strings instead of the default. Remember that you must restart rails in order for new locale files to take effect. + +### Override static pages + +You can also override any of the static files included with leap_web, such as the privacy policy or terms of service. + +Here is how we would create a custom privacy policy in English and Spanish: + +`leap_web/config/customization/views/pages/privacy-policy.en.md`: + + # Custom Privacy Policy + This is our privacy policy. + +`leap_web/config/customization/views/pages/privacy-policy.es.md`: + + # Custom PolÃtica de Privacidad + Esta es nuestra polÃtica de privacidad. + +### Add a custom header + +Now we will add a custom header to every page. First, we add the images: + + leap_web/config/customization + ├── public + ├── favicon.ico + └── img + └── masthead.png + +You can create your own, or use the example files in https://github.com/leapcode/leap_web/tree/develop/config/customization.example + +Now, we add some custom CSS so that we can style the masthead: + +`leap_web/config/customization/stylesheets/tail.scss` + + $custom-color: #66bbaa; + + a { + color: $custom-color; + } + + // + // MASTHEAD + // + + #masthead { + background-color: $custom-color; + border-bottom: none; + + // make the masthead clickable by replacing the + // site name link with the masthead image: + .title { + padding: 0px; + .sitename a { + display: block; + background: url(/img/masthead.png) 0 0 no-repeat; + font-size: 0px; + height: 100px; + background-size: auto 100px; + } + } + } + + // make the home page masthead slightly larger + body.home #masthead { + .sitename a { + height: 150px; + background-size: auto 150px; + } + } + + // + // FOOTER + // + + #footer .links { + background-color: $custom-color; + } + +NOTE: If you add a `tails.scss` or `head.scss` file, then you usually need to run `rake tmp:clear` and restart rails in order for the new stylesheet to get recognized. You should only need to do this once. + + +Custom Fork +---------------------------- + +Sometimes it is easier to maintain your own fork of the leap_web app. You can keep your customizations in that fork instead of in the provider `files/webapp` directory. Or, perhaps you want to add an engine to the application that modifies the app's behavior. + +To deploy your own leap_web, modify the provider file `common.json`: + + { + "sources": { + "webapp": { + "revision": "origin/develop", + "source": "https://github.com/leapcode/leap_web", + "type": "git" + } + } + } + +To target only particular environment, modify instead `common.ENV.json`, where ENV is the name of the environment. + +See https://github.com/leapcode/leap_web/blob/develop/doc/DEVELOP.md for notes on getting started hacking on leap_web. + +Maintenance mode +------------------ + +You can put the webapp into maintenance mode by simply dropping a html file to `/srv/leap/webapp/public/system/maintenance.html`. For example: + + workstation$ leap ssh webappnode + server# echo "Temporarily down for maintenance. We will be back soon." > /srv/leap/webapp/public/system/maintenance.html + +Known problems +--------------------------- + +* Client certificates are generated without a CSR. The problem is that this makes the web + application extremely vulnerable to denial of service attacks. This was not an issue until we + started to allow the possibility of anonymously fetching a client certificate without + authenticating first. +* By its very nature, the user database is vulnerable to enumeration attacks. These are + very hard to prevent, because our protocol is designed to allow query of a user database via + proxy in order to provide network perspective. diff --git a/doc/troubleshooting/tests.md b/doc/troubleshooting/tests.md index b85c19d2..607f924e 100644 --- a/doc/troubleshooting/tests.md +++ b/doc/troubleshooting/tests.md @@ -8,15 +8,15 @@ At any time, you can run troubleshooting tests on the nodes of your provider inf To run tests on FILTER node list: - leap test run FILTER + workstation$ leap test run FILTER For example, you can also test a single node (`leap test elephant`); test a specific environment (`leap test development`), or any tag (`leap test soledad`). Alternately, you can run test on all nodes (probably only useful if you have pinned the environment): - leap test + workstation$ leap test -The tests that are performed are located in the platform under the tests directory. +The tests that are performed are located in the platform under the tests directory. ## Testing with the bitmask client @@ -50,16 +50,16 @@ In order to set up a monitoring node, you simply add a `monitor` service tag to After deploying, this node will regularly poll every node to ask for the status of various health checks. These health checks include the checks run with `leap test`, plus many others. -We use [Nagios](http://www.nagios.org/) together with [Check MK agent](https://en.wikipedia.org/wiki/Check_MK) for running checks on remote hosts. +We use [Nagios](https://www.nagios.org/) together with [Check MK agent](https://en.wikipedia.org/wiki/Check_MK) for running checks on remote hosts. One nagios installation will monitor all nodes in all your environments. You can log into the monitoring web interface via [https://DOMAIN/nagios3/](https://DOMAIN/nagios3/). The username is `nagiosadmin` and the password is found in the secrets.json file in your provider directory. Nagios will send out mails to the `contacts` address provided in `provider.json`. -## Nagios Frontents +## Nagios Frontends There are other ways to check and get notified by Nagios besides regularly checking the Nagios webinterface or reading email notifications. Check out the [Frontends (GUIs and CLIs)](http://exchange.nagios.org/directory/Addons/Frontends-%28GUIs-and-CLIs%29) on the Nagios project website. -A recommended status tray application is [Nagstamon](https://nagstamon.ifw-dresden.de/), which is available for Linux, MacOS X and Windows. It can not only notify you of hosts/services failures, you can also acknoledge or recheck these with it. +A recommended status tray application is [Nagstamon](https://nagstamon.ifw-dresden.de/), which is available for Linux, MacOS X and Windows. It can not only notify you of hosts/services failures, you can also acknowledge or recheck them. ### Log Monitoring diff --git a/doc/troubleshooting/vagrant.md b/doc/troubleshooting/vagrant.md deleted file mode 100644 index ad284161..00000000 --- a/doc/troubleshooting/vagrant.md +++ /dev/null @@ -1,45 +0,0 @@ -@title = 'LEAP Platform Vagrant testing' -@nav_title = 'Vagrant Integration' -@summary = 'Testing your provider with Vagrant' - -Setting up Vagrant for a testing the platform -============================================= - -There are two ways you can setup leap platform using vagrant. - -Using the Vagrantfile provided by Leap Platform ------------------------------------------------ - -This is by far the easiest way. It will install a single node mail server in the default -configuration with one single command. - -Clone the platform with - - git clone https://github.com/leapcode/leap_platform.git - -Start the vagrant box with - - cd leap_platform - vagrant up - -Follow the instructions how to configure your `/etc/hosts` -in order to use the provider! - -You can login via ssh with the systemuser `vagrant` and the same password. - -There are 2 users preconfigured: - -. `testuser` with pw `hallo123` -. `testadmin` with pw `hallo123` - - -Use the leap_cli vagrant integration ------------------------------------- - -Install leap_cli and leap_platform on your host, configure a provider from scratch and use the `leap local` commands to manage your vagrant node(s). - -See https://leap.se/en/docs/platform/development how to use the leap_cli vagrant -integration and https://leap.se/en/docs/platform/tutorials/single-node-email how -to setup a single node mail server. - - diff --git a/doc/troubleshooting/where-to-look.md b/doc/troubleshooting/where-to-look.md index fbd95931..c92fba8f 100644 --- a/doc/troubleshooting/where-to-look.md +++ b/doc/troubleshooting/where-to-look.md @@ -7,6 +7,16 @@ General ======= * Please increase verbosity when debugging / filing issues in our issue tracker. You can do this with adding i.e. `-v 5` after the `leap` cmd, i.e. `leap -v 2 deploy`. +* We use the `example.org` domain for documentation purposes here, please replace it with the you domain. + +Firewall +======================= + +Every node in your provider has its own restrictive firewall, but you might have a network firewall in place as well that is not managed by LEAP platform. To see what ports and addresses must be open, run this command: + + workstation$ leap compile firewall + +If any of those are blocked, then your provider will not work. Webapp ====== @@ -19,10 +29,10 @@ Places to look for errors * `/var/log/syslog` (watch out for stunnel issues) * `/var/log/leap/*` + Is haproxy ok ? --------------- - curl -s -X GET "http://127.0.0.1:4096" Is couchdb accessible through stunnel ? @@ -59,13 +69,21 @@ Check couchdb acl as unpriviledged user curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4096/_all_dbs" +All URLs accessible ? +--------------------- + +* https://example.org +* https://api.example.org:4430/provider.json +* https://example.org/ca.crt + + Check client config files ------------------------- - https://example.net/provider.json - https://example.net/1/config/smtp-service.json - https://example.net/1/config/soledad-service.json - https://example.net/1/config/eip-service.json +* https://example.net/provider.json +* https://example.net/1/config/smtp-service.json +* https://example.net/1/config/soledad-service.json +* https://example.net/1/config/eip-service.json Soledad diff --git a/doc/tutorials/configure-provider.md b/doc/tutorials/configure-provider.md deleted file mode 100644 index 969d541b..00000000 --- a/doc/tutorials/configure-provider.md +++ /dev/null @@ -1,31 +0,0 @@ -@title = 'Configure provider tutorial' -@nav_title = 'Configure Provider' -@summary = 'Explore how to configure your provider after the initial setup' - - -Edit provider.json configuration --------------------------------------- - -There are a few required settings in provider.json. At a minimum, you must have: - - { - "domain": "example.org", - "name": "Example", - "contacts": { - "default": "email1@example.org" - } - } - -For a full list of possible settings, you can use `leap inspect` to see how provider.json is evaluated after including the inherited defaults: - - $ leap inspect provider.json - - -Examine Certs -============= - -To see details about the keys and certs that the prior two commands created, you can use `leap inspect` like so: - - $ leap inspect files/ca/ca.crt - -NOTE: the files `files/ca/*.key` are extremely sensitive and must be carefully protected. The other key files are much less sensitive and can simply be regenerated if needed. diff --git a/doc/tutorials/quick-start.md b/doc/tutorials/quick-start.md index a92cc9da..f963867a 100644 --- a/doc/tutorials/quick-start.md +++ b/doc/tutorials/quick-start.md @@ -1,385 +1,230 @@ @title = 'Quick Start Tutorial' @nav_title = 'Quick Start Tutorial' -@summary = 'This tutorial walks you through the initial process of creating and deploying a minimal service provider running the LEAP Platform. This Quick Start guide will guide you through building a three node OpenVPN provider.' +@summary = 'This tutorial walks you through the initial process of creating and deploying a minimal service provider running the LEAP Platform.' +Introduction +==================================== -Our goal ------------------- +### Our goal -We are going to create a minimal LEAP provider offering OpenVPN service. This basic setup can be expanded by adding more OpenVPN nodes to increase capacity or geographical diversity, or more webapp nodes to increase availability (at the moment, a single couchdb and single webapp server are all that is supported, and performance wise, are more than enough for most usage, since they are only lightly used). At the moment, we strongly advise only have one couchdb server for stability purposes. +We are going to create a minimal LEAP provider, but one that does not offer any actual services. Check out the other tutorials for adding VPN or email services. Our goal is something like this: $ leap list - NODES SERVICES TAGS - cheetah couchdb production - wildebeest webapp production - ostrich openvpn production + NODES SERVICES TAGS + wildebeest couchdb, webapp NOTE: You won't be able to run that `leap list` command yet, not until we actually create the node configurations. -Requirements ------------- +### Requirements -In order to complete this Quick Start, you will need a few things: +1. A workstation: This is your local machine that you will run commands on. +1. A server: This is the machine that you will deploy to. The server can be either: + 1. A local Vagrant virtual machine: a Vagrant machine can only be useful for testing. + 1. A real or paravirtualized server: The server must have Debian Jessie installed, and you must be able to SSH into the machine as root. Paravirtualization includes KVM, Xen, OpenStack, Amazon, but not VirtualBox or OpenVZ. -* You will need three real or paravirtualized virtual machines (KVM, Xen, Openstack, Amazon, but not Vagrant - sorry) that have a basic Debian Stable installed. If you allocate 20G of disk space to each node for the system, after this process is completed, you will have used less than 10% of that disk space. If you allocate 2 CPUs and 8G of memory to each node, that should be more than enough to begin with. -* You should be able to SSH into them remotely, and know their root password, IP addresses and their SSH host keys -* You will need four different IPs. Each node gets a primary IP, and the OpenVPN gateway additionally needs a gateway IP. -* The ability to create/modify DNS entries for your domain is preferable, but not needed. If you don't have access to DNS, you can workaround this by modifying your local resolver, i.e. editing `/etc/hosts`. -* You need to be aware that this process will make changes to your systems, so please be sure that these machines are a basic install with nothing configured or running for other purposes -* Your machines will need to be connected to the internet, and not behind a restrictive firewall. -* You should work locally on your laptop/workstation (one that you trust and that is ideally full-disk encrypted) while going through this guide. This is important because the provider configurations you are creating contain sensitive data that should not reside on a remote machine. The `leap` command will login to your servers and configure the services. -* You should do everything described below as an unprivileged user, and only run those commands as root that are noted with *sudo* in front of them. Other than those commands, there is no need for privileged access to your machine, and in fact things may not work correctly. - -All the commands in this tutorial are run on your sysadmin machine. In order to complete the tutorial, the sysadmin will do the following: - -* Install pre-requisites -* Install the LEAP command-line utility -* Check out the LEAP platform -* Create a provider and its certificates -* Setup the provider's nodes and the services that will reside on those nodes -* Initialize the nodes -* Deploy the LEAP platform to the nodes -* Test that things worked correctly -* Some additional commands - -We will walk you through each of these steps. +Other things to keep in mind: +* The ability to create/modify DNS entries for your domain is preferable, but not needed. If you don't have access to DNS, you can workaround this by modifying your local resolver, i.e. editing `/etc/hosts`. +* You need to be aware that this process will make changes to your servers, so please be sure that these machines are a basic install with nothing configured or running for other purposes. +* Your servers will need to be connected to the internet, and not behind a restrictive firewall. -Prepare your environment +Prepare your workstation ======================== -There are a few things you need to setup before you can get going. Just some packages, the LEAP cli and the platform. +In order to be able to manage your servers, you need to install the `leap` command on your workstation: -Install pre-requisites --------------------------------- +### Install pre-requisites -*Debian & Ubuntu* +Install core prerequisites on your workstation. -Install core prerequisites: +*Debian & Ubuntu* - $ sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make bzip2 + workstation$ sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make bzip2 -<!-- *Mac OS* -1. Install rubygems from https://rubygems.org/pages/download (unless the `gem` command is already installed). ---> - -NOTE: leap_cli requires ruby 1.9 or later. - + workstation$ brew install ruby-install + workstation$ ruby-install ruby -Install the LEAP command-line utility -------------------------------------------------- +### Install the LEAP command-line utility Install the `leap` command from rubygems.org: - $ sudo gem install leap_cli - -Alternately, you can install `leap` from source: - - $ git clone https://leap.se/git/leap_cli - $ cd leap_cli - $ rake build - $ sudo rake install - -You can also install from source as an unprivileged user, if you want. For example, instead of `sudo rake install` you can do something like this: - - $ rake install - # watch out for the directory leap is installed to, then i.e. - $ sudo ln -s ~/.gem/ruby/1.9.1/bin/leap /usr/local/bin/leap - -With either `rake install` or `sudo rake install`, you can use now /usr/local/bin/leap, which in most cases will be in your $PATH. - -If you have successfully installed the `leap` command, then you should be able to do the following: - - $ leap --help - -This will list the command-line help options. If you receive an error when doing this, please read through the README.md in the `leap_cli` source to try and resolve any problems before going forwards. - -Check out the platform --------------------------- + workstation$ gem install leap_cli --install-dir ~/leap + workstation$ export PATH=$PATH:~/leap/bin -The LEAP Platform is a series of puppet recipes and modules that will be used to configure your provider. You will need a local copy of the platform that will be used to setup your nodes and manage your services. To begin with, you will not need to modify the LEAP Platform. +Alternately, you can install `leap` system wide: -First we'll create a directory for LEAP things, and then we'll check out the platform code and initalize the modules: + workstation$ sudo gem install leap_cli - $ mkdir ~/leap - $ cd ~/leap - $ git clone https://leap.se/git/leap_platform.git - $ cd leap_platform - $ git submodule sync; git submodule update --init +To confirm that you installed `leap` correctly, try running `leap help`. +Create a provider instance +============================================= -Provider Setup -============== +A provider instance is a directory tree, residing on your workstation, that contains everything you need to manage an infrastructure for a service provider. -A provider instance is a directory tree, usually stored in git, that contains everything you need to manage an infrastructure for a service provider. In this case, we create one for example.org and call the instance directory 'example'. +In this case, we create one for example.org and call the instance directory 'example'. - $ mkdir -p ~/leap/example - -Bootstrap the provider ------------------------ - -Now, we will initialize this directory to make it a provider instance. Your provider instance will need to know where it can find the local copy of the git repository leap_platform, which we setup in the previous step. - - $ cd ~/leap/example - $ leap new . - -NOTES: - . make sure you include that trailing dot! + workstation$ leap new ~/example The `leap new` command will ask you for several required values: * domain: The primary domain name of your service provider. In this tutorial, we will be using "example.org". * name: The name of your service provider (we use "Example"). * contact emails: A comma separated list of email addresses that should be used for important service provider contacts (for things like postmaster aliases, Tor contact emails, etc). -* platform: The directory where you have a copy of the `leap_platform` git repository checked out. +* platform: The directory where you have a copy of the `leap_platform` git repository checked out. If the platform directory does not yet exist, the `leap_platform` will be downloaded and placed in that directory. You could also have passed these configuration options on the command-line, like so: - $ leap new --contacts your@email.here --domain leap.example.org --name Example --platform=~/leap/leap_platform . - -You may want to poke around and see what is in the files we just created. For example: + workstation$ leap new --contacts your@email.here --domain example.org --name Example --platform=~/leap/leap_platform . - $ cat provider.json +You should now have the following files: -Optionally, commit your provider directory using the version control software you fancy. For example: - - $ git init - $ git add . - $ git commit -m "initial provider commit" + workstation$ tree example + example + ├── common.json + ├── Leapfile + ├── nodes/ + ├── provider.json + ├── services/ + └── tags/ Now add yourself as a privileged sysadmin who will have access to deploy to servers: - $ leap add-user --self - -NOTE: in most cases, `leap` must be run from within a provider instance directory tree (e.g. ~/leap/example). - -Create provider certificates ----------------------------- - -Create two certificate authorities, one for server certs and one for client -certs (note: you only need to run this one command to get both): + workstation$ cd example + workstation$ leap add-user louise --self - $ leap cert ca +Replace "louise" with whatever you want your sysadmin username to be. -Create a temporary cert for your main domain (you should replace with a real commercial cert at some point) +NOTE: Make sure you change directories so that the `leap` command is run from within the provider instance directory. Most `leap` commands only work when run from a provider instance. - $ leap cert csr +Now create the necessary keys and certificates: -To see details about the keys and certs that the prior two commands created, you can use `leap inspect` like so: + workstation$ leap cert ca + workstation$ leap cert csr - $ leap inspect files/ca/ca.crt - -Create the Diffie-Hellman parameters file, needed for forward secret OpenVPN ciphers: - - $ leap cert dh - -NOTE: the files `files/ca/*.key` are extremely sensitive and must be carefully protected. The other key files are much less sensitive and can simply be regenerated if needed. - - -Edit provider.json configuration --------------------------------------- - -There are a few required settings in provider.json. At a minimum, you must have: - - { - "domain": "example.org", - "name": "Example", - "contacts": { - "default": "email1@example.org" - } - } +What do these commands do? The first command will create two Certificate Authorities, one that clients will use to authenticate with the servers and one for backend servers to authenticate with each other. The second command creates a Certificate Signing Request suitable for submission to a commercial CA. It also creates two "dummy" files for you to use temporarily: -For a full list of possible settings, you can use `leap inspect` to see how provider.json is evaluated after including the inherited defaults: +* `files/cert/example.org.crt` -- This is a "dummy" certificate for your domain that can be used temporarily for testing. Once you get a real certificate from a CA, you should replace this file. +* `files/cert/commercial_ca.crt` -- This is "dummy" CA cert the corresponds to the dummy domain certificate. Once you replace the domain certificate, also replace this file with the CA cert from the real Certificate Authority. - $ leap inspect provider.json +If you plan to run a real service provider, see important information on [[managing keys and certificates => keys-and-certificates]]. +Add a node to the provider +================================================== -Setup the provider's nodes and services ---------------------------------------- +A "node" is a server that is part of your infrastructure. Every node can have one or more services associated with it. We will now add a single node with two services, "webapp" and "couchdb". -A "node" is a server that is part of your infrastructure. Every node can have one or more services associated with it. Some nodes are "local" and used only for testing, see [Development](development) for more information. +You have two choices for node type: a real node or a local node. -Create a node, with the service "webapp": +* Real Node: A real node is any physical or paravirtualized server, including KVM, Xen, OpenStack Compute, Amazon EC2, but not VirtualBox or OpenVZ (VirtualBox and OpenVZ use a more limited form of virtualization). The server must be running Debian Jessie. +* Local Node: A local node is a virtual machine created by Vagrant, useful for local testing on your workstation. - $ leap node add wildebeest ip_address:x.x.x.w services:webapp tags:production +Getting Vagrant working can be a pain and is [[covered in other tutorials => vagrant]]. If you have a real server available, we suggest you try this tutorial with a real node first. -NOTE: replace x.x.x.w with the actual IP address of this node +### Option A: Add a real node -This created a node configuration file in `nodes/wildebeest.json`, but it did not do anything else. It also added the 'tag' called 'production' to this node. Tags allow us to conveniently group nodes together. When creating nodes, you should give them the tag 'production' if the node is to be used in your production infrastructure. +Note: Installing LEAP Platform on this server will potentially destroy anything you have previously installed on this machine. -The web application and the VPN nodes require a database, so lets create the database server node: +Create a node, with the services "webapp" and "couchdb": - $ leap node add cheetah ip_address:x.x.x.x services:couchdb tags:production + workstation$ leap node add wildebeest ip_address:x.x.x.w services:webapp,couchdb -NOTE: replace x.x.x.x with the actual IP address of this node +NOTE: replace x.x.x.x with the actual IP address of this server. -Now we need the OpenVPN gateway, so lets create that node: +### Option B: Add a local node - $ leap node add ostrich ip_address:x.x.x.y openvpn.gateway_address:x.x.x.z services:openvpn tags:production +Create a node, with the services "webapp" and "couchdb", and then start the local virtual machine: -NOTE: replace x.x.x.y with the IP address of the machine, and x.x.x.z with the second IP. openvpn gateways must be assigned two IP addresses, one for the host itself and one for the openvpn gateway. We do this to prevent incoming and outgoing VPN traffic on the same IP. Without this, the client might send some traffic to other VPN users in the clear, bypassing the VPN. + workstation$ leap node add --local wildebeest services:webapp,couchdb + workstation$ leap local start wildebeest +It will take a while to download the Virtualbox base box and create the virtual machine. -Setup DNS ---------- +Deploy your provider +========================================= -Now that you have the nodes configured, you should create the DNS entries for these nodes. - -Set up your DNS with these hostnames: - - $ leap list --print ip_address,domain.full,dns.aliases - cheetah x.x.x.w, cheetah.example.org, null - wildebeest x.x.x.x, wildebeest.example.org, api.example.org - ostrich x.x.x.y, ostrich.example.org, null - -Alternately, you can adapt this zone file snippet: - - $ leap compile zone - -If you cannot edit your DNS zone file, you can still test your provider by adding entries to your local resolver hosts file (`/etc/hosts` for linux): - - x.x.x.w cheetah.example.org - x.x.x.x wildebeest.example.org api.example.org example.org - x.x.x.y ostrich.example.org - -Please don't forget about these entries, they will override DNS queries if you setup your DNS later. - - -Initialize the nodes --------------------- +### Initialize the node Node initialization only needs to be done once, but there is no harm in doing it multiple times: - $ leap node init production + workstation$ leap node init wildebeest -This will initialize all nodes with the tag "production". When `leap node init` is run, you will be prompted to verify the fingerprint of the SSH host key and to provide the root password of the server(s). You should only need to do this once. +This will initialize the node `wildebeest`. -If you prefer, you can initalize each node, one at a time: +For non-local nodes, when `leap node init` is run, you will be prompted to verify the fingerprint of the SSH host key and to provide the root password of the server(s). You should only need to do this once. - $ leap node init wildebeest - $ leap node init cheetah - $ leap node init ostrich +### Deploy to the node -Deploy the LEAP platform to the nodes --------------------- +The next step is to deploy the LEAP platform to your node. [Deployment can take a while to run](https://xkcd.com/303/), especially on the first run, as it needs to update the packages on the new machine. -Now you should deploy the platform recipes to the nodes. [Deployment can take a while to run](http://xkcd.com/303/), especially on the first run, as it needs to update the packages on the new machine. - -*Important notes:* currently nodes must be deployed in a certain order. The underlying couch database node(s) must be deployed first, and then all other nodes. - - $ leap deploy cheetah + workstation$ leap deploy wildebeest Watch the output for any errors (in red), if everything worked fine, you should now have your first running node. If you do have errors, try doing the deploy again. -However, to deploy our three-node openvpn setup, we need the database and LEAP web application requires a database to run, so let's deploy to the couchdb and openvpn nodes: - - $ leap deploy wildebeest - $ leap deploy ostrich - - -What is going on here? --------------------------------------------- +### Setup DNS -First, some background terminology: +The next step is to configure the DNS for your provider. For testing purposes, you can just modify your `/etc/hosts` file. Please don't forget about these entries, they will override DNS queries if you setup your DNS later. For a list of what entries to add to `/etc/hosts`, run this command: -* **puppet**: Puppet is a system for automating deployment and management of servers (called nodes). -* **hiera files**: In puppet, you can use something called a 'hiera file' to seed a node with a few configuration values. In LEAP, we go all out and put *every* configuration value needed for a node in the hiera file, and automatically compile a custom hiera file for each node. + workstation$ leap compile hosts -When you run `leap deploy`, a bunch of things happen, in this order: +Alternately, if you have access to modify the DNS zone entries for your domain: -1. **Compile hiera files**: The hiera configuration file for each node is compiled in YAML format and saved in the directory `hiera`. The source material for this hiera file consists of all the JSON configuration files imported or inherited by the node's JSON config file. -* **Copy required files to node**: All the files needed for puppet to run are rsync'ed to each node. This includes the entire leap_platform directory, as well as the node's hiera file and other files needed by puppet to set up the node (keys, binary files, etc). -* **Puppet is run**: Once the node is ready, leap connects to the node via ssh and runs `puppet apply`. Puppet is applied locally on the node, without a daemon or puppetmaster. + workstation$ leap compile zone -You can run `leap -v2 deploy` to see exactly what commands are being executed. +NOTE: The resulting zone file is incomplete because it is missing a serial number. Use the output of `leap compile zone` as a guide, but do not just copy and paste the output. Also, the `compile zone` output will always exclude mention of local nodes. +The DNS method will not work for local nodes created with Vagrant. Test that things worked correctly ================================= -You should now have three machines with the LEAP platform deployed to them, one for the web application, one for the database and one for the OpenVPN gateway. - To run troubleshooting tests: - leap test - -If you want to confirm for yourself that things are working, you can perform the following manual tests. - -### Access the web application - -In order to connect to the web application in your browser, you need to point your domain at the IP address of the web application node (named wildebeest in this example). + workstation$ leap test -There are a lot of different ways to do this, but one easy way is to modify your `/etc/hosts` file. First, find the IP address of the webapp node: +Alternately, you can run these same tests from the server itself: - $ leap list webapp --print ip_address + workstation$ leap ssh wildebeest + wildebeest# run_tests -Then modify `/etc/hosts` like so: +Create an administrator +=============================== - x.x.x.w leap.example.org +Assuming that you set up your DNS or `/etc/hosts` file, you should be able to load `https://example.org` in your web browser (where example.org is whatever domain name you actually used). -Replacing 'leap.example.org' with whatever you specified as the `domain` in the `leap new` command. +Your browser will complain about an untrusted cert, but for now just bypass this. From there, you should be able to register a new user and login. -Next, you can connect to the web application either using a web browser or via the API using the LEAP client. To use a browser, connect to https://leap.example.org (replacing that with your domain). Your browser will complain about an untrusted cert, but for now just bypass this. From there, you should be able to register a new user and login. +Once you have created a user, you can now make this user an administrator. For example, if you created a user `kangaroo`, you would create the file `services/webapp.json` with the following content: -### Use the VPN - -You should be able to simply test that the OpenVPN gateway works properly by doing the following: - - $ leap test init - $ sudo openvpn test/openvpn/production_unlimited.ovpn + { + "webapp": { + "admins": ["kangaroo"] + } + } -Or, you can use the LEAP client (called "bitmask") to connect to your new provider, create a user and then connect to the VPN. +Save that file and run `leap deploy` again. When you next log on to the web application, the user kangaroo will now be an admin. +If you want to restrict who can register a new user, see [[webapp]] for configuration options. -Additional information +What is next? ====================== -It is useful to know a few additional things. - -Useful commands ---------------- - -Here are a few useful commands you can run on your new local nodes: - -* `leap ssh wildebeest` -- SSH into node wildebeest (requires `leap node init wildebeest` first). -* `leap list` -- list all nodes. -* `leap list production` -- list only those nodes with the tag 'production' -* `leap list --print ip_address` -- list a particular attribute of all nodes. -* `leap cert update` -- generate new certificates if needed. - -See the full command reference for more information. +Add an end-user service +------------------------------- -Node filters -------------------------------------------- +You should now have a minimal service provider with a single node. This service provider is pointless at the moment, because it does not include any end-user services like VPN or email. To add one of these services, continue with one of the following tutorials: -Many of the `leap` commands take a "node filter". You can use a node filter to target a command at one or more nodes. +* [[single-node-email]] +* [[single-node-vpn]] -A node filter consists of one or more keywords, with an optional "+" before each keyword. - -* keywords can be a node name, a service type, or a tag. -* the "+" before the keyword constructs an AND condition -* otherwise, multiple keywords together construct an OR condition - -Examples: - -* `leap list openvpn` -- list all nodes with service openvpn. -* `leap list openvpn +production` -- only nodes of service type openvpn AND tag production. -* `leap deploy webapp openvpn` -- deploy to all webapp OR openvpn nodes. -* `leap node init ostrich` -- just init the node named ostrich. - -Keep track of your provider configurations ------------------------------------------- - -You should commit your provider changes to your favorite VCS whenever things change. This way you can share your configurations with other admins, all they have to do is to pull the changes to stay up to date. Every time you make a change to your provider, such as adding nodes, services, generating certificates, etc. you should add those to your VCS, commit them and push them to where your repository is hosted. - -Note that your provider directory contains secrets! Those secrets include passwords for various services. You do not want to have those passwords readable by the world, so make sure that wherever you are hosting your repository, it is not public for the world to read. - -What's next ------------------------------------ +Learn more +--------------- -Read the [LEAP platform guide](guide) to learn about planning and securing your infrastructure. +We have only just scratched the surface of the possible ways to configure and deploy your service provider. Your next step should be: +* Read [[getting-started]] for more details on using the LEAP platform. +* See [[commands]] for a list of possible commands. diff --git a/doc/tutorials/single-node-email.md b/doc/tutorials/single-node-email.md index b47496b9..0a73e6e1 100644 --- a/doc/tutorials/single-node-email.md +++ b/doc/tutorials/single-node-email.md @@ -1,282 +1,69 @@ @title = 'Single node email tutorial' -@nav_title = 'Single node email' -@summary = 'A single node email provider.' +@nav_title = 'Quick email' +@summary = 'Tutorial for setting up a simple email provider.' -Quick Start - Single node setup -=============================== - -This tutorial walks you through the initial process of creating and deploying a minimal service provider running the [LEAP platform](platform). -We will guide you through building a single node mail provider. +This tutorial walks you through the initial process of creating and deploying a minimal email service provider. Please first complete the [[quick-start]]. This tutorial will pick up where that one left off. Our goal ------------------ -We are going to create a minimal LEAP provider offering Email service. This basic setup can be expanded by adding more webapp and couchdb nodes to increase availability (performance wise, a single couchdb and a single webapp are more than enough for most usage, since they are only lightly used, but you might want redundancy). Please note: currently it is not possible to safely add additional couchdb nodes at a later point. They should all be added in the beginning, so please consider carefully if you would like more before proceeding. +We are going to create a minimal LEAP provider offering email service. Our goal is something like this: $ leap list - NODES SERVICES TAGS - node1 couchdb, mx, soledad, webapp local - -NOTE: You won't be able to run that `leap list` command yet, not until we actually create the node configurations. - -Requirements ------------- - -In order to complete this Quick Start, you will need a few things: - -* You will need `one real or paravirtualized virtual machine` (Vagrant, KVM, Xen, Openstack, Amazon, …) that have a basic Debian Stable installed. -* You should be able to `SSH into them` remotely, and know their root password, IP addresses and their SSH host keys -* The ability to `create/modify DNS entries` for your domain is preferable, but not needed. If you don't have access to DNS, you can workaround this by modifying your local resolver, i.e. editing `/etc/hosts`. -* You need to be aware that this process will make changes to your machines, so please be sure that these machines are a basic install with nothing configured or running for other purposes -* Your machines will need to be connected to the internet, and not behind a restrictive firewall. -* You should `work locally on your laptop/workstation` (one that you trust and that is ideally full-disk encrypted) while going through this guide. This is important because the provider configurations you are creating contain sensitive data that should not reside on a remote machine. The leap cli utility will login to your servers and configure the services. -* You should do everything described below as an `unprivileged user`, and only run those commands as root that are noted with *sudo* in front of them. Other than those commands, there is no need for privileged access to your machine, and in fact things may not work correctly. - -All the commands in this tutorial are run on your sysadmin machine. In order to complete the tutorial, the sysadmin will do the following: - -* Install pre-requisites -* Install the LEAP command-line utility -* Check out the LEAP platform -* Create a provider and its certificates -* Setup the provider's node and the services that will reside on it -* Initialize the node -* Deploy the LEAP platform to the node -* Test that things worked correctly -* Some additional commands - -We will walk you through each of these steps. - - -Prepare your environment -======================== - -There are a few things you need to setup before you can get going. Just some packages, the LEAP cli and the platform. - -Install pre-requisites --------------------------------- - -*Debian & Ubuntu* - -Install core prerequisites: - - $ sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make bzip2 - -*Mac OS* - -Install rubygems from https://rubygems.org/pages/download (unless the `gem` command is already installed). - - -NOTE: leap_cli should work with ruby1.8, but has only been tested using ruby1.9. - - -Install the LEAP command-line utility -------------------------------------------------- - -Install the LEAP command-line utility (leap_cli) from rubygems.org: - - $ sudo gem install leap_cli - -Alternately, you can install `leap_cli` from source, please refer to https://leap.se/git/leap_cli/README.md. - -If you have successfully installed `leap_cli`, then you should be able to do the following: - - $ leap --help - -This will list the command-line help options. If you receive an error when doing this, please read through the README.md in the `leap_cli` source to try and resolve any problems before going forwards. - - -Provider Setup -============== - -A provider instance is a directory tree that contains everything you need to manage an infrastructure for a service provider. In this case, we create one for example.org and call the instance directory 'example'. - - $ mkdir -p ~/leap/example - -Bootstrap the provider ------------------------ - -Now, we will initialize this directory to make it a provider instance. Your provider instance will need to know where it can find the local copy of the git repository leap_platform, which we setup in the previous step. - - $ cd ~/leap/example - $ leap new . - -NOTES: - . make sure you include that trailing dot! - -The `leap new` command will ask you for several required values: - -* domain: The primary domain name of your service provider. In this tutorial, we will be using "example.org". -* name: The name of your service provider (we use "Example"). -* contact emails: A comma separated list of email addresses that should be used for important service provider contacts (for things like postmaster aliases, Tor contact emails, etc). -* platform: The directory where you either have a copy of the `leap_platform` git repository already checked out, or where `leap_cli` should download it too. You could just accept the suggested path for this example. - The LEAP Platform is a series of puppet recipes and modules that will be used to configure your provider. You will need a local copy of the platform that will be used to setup your nodes and manage your services. To begin with, you will not need to modify the LEAP Platform. - -These steps should be sufficient for this example. If you want to configure your provider further or like to examine the files, please refer to the [Configure Provider](configure-provider) section. - -Add Users who will have administrative access ---------------------------------------------- - -Now add yourself as a privileged sysadmin who will have access to deploy to servers: - - $ leap add-user --self - -NOTE: in most cases, `leap` must be run from within a provider instance directory tree (e.g. ~/leap/example). - - -Create provider certificates ----------------------------- - -Create two certificate authorities, one for server certs and one for client -certs (note: you only need to run this one command to get both): + NODES SERVICES TAGS + wildebeest couchdb, mx, soledad, webapp - $ leap cert ca +Where 'wildebeest' is whatever name you chose for your node in the [[quick-start]]. -Create a temporary cert for your main domain (you should replace with a real commercial cert at some point) - - $ leap cert csr - - -Setup the provider's node and services +Add email services to the node -------------------------------------- -A "node" is a server that is part of your infrastructure. Every node can have one or more services associated with it. Some nodes are "local" and used only for testing, see [Development](development) for more information. - -Create a node, with `all the services needed for Email: "couchdb", "mx", "soledad" and "webapp"` - - $ leap node add node1 ip_address:x.x.x.w services:couchdb,mx,soledad,webapp tags:production - -NOTE: replace x.x.x.w with the actual IP address of this node +In order to add [[services => services]] to a node, edit the node's JSON configuration file. -This created a node configuration file in `nodes/node1.json`, but it did not do anything else. It also added the 'tag' called 'production' to this node. Tags allow us to conveniently group nodes together. When creating nodes, you should give them the tag 'production' if the node is to be used in your production infrastructure. +In our example, we would edit `nodes/wildebeest.json`: -Initialize the nodes --------------------- - -Node initialization only needs to be done once, but there is no harm in doing it multiple times: + { + "ip_address": "1.1.1.1", + "services": ["couchdb", "webapp", "mx", "soledad"] + } - $ leap node init node1 +Here, we added `mx` and `soledad` to the node's `services` list. Briefly: -This will initialize the node "node1". When `leap node init` is run, you will be prompted to verify the fingerprint of the SSH host key and to provide the root password of the server. You should only need to do this once. +* **mx**: nodes with the **mx** service will run postfix mail transfer agent, and are able to receive and relay email on behalf of your domain. You can have as many as you want, spread out over as many nodes as you want. +* **soledad**: nodes with **soledad** service run the server-side daemon that allows the client to synchronize a user's personal data store among their devices. Currently, **soledad** only runs on nodes that are also **couchdb** nodes. +For more details, see the [[services]] overview, or the individual pages for the [[mx]] and [[soledad]] services. -Deploy the LEAP platform to the nodes +Deploy to the node -------------------- -Now you should deploy the platform recipes to the node. [Deployment can take a while to run](http://xkcd.com/303/), especially on the first run, as it needs to update the packages on the new machine. - - $ leap deploy - -Watch the output for any errors (in red), if everything worked fine, you should now have your first running node. If you do have errors, try doing the deploy again. +Now you should deploy to your node. + workstation$ leap deploy Setup DNS ---------- - -Now that you have the node configured, you should create the DNS entrie for this node. - -Set up your DNS with these hostnames: - - $ leap list --print ip_address,domain.full,dns.aliases - node1 x.x.x.w, node1.example.org, example.org, api.example.org, nicknym.example.org - -Alternately, you can adapt this zone file snippet: - - $ leap compile zone - -If you cannot edit your DNS zone file, you can still test your provider by adding this entry to your local resolver hosts file (`/etc/hosts` for linux): - - x.x.x.w node1.example.org example.org api.example.org nicknym.example.org - -Please don't forget about these entries, they will override DNS queries if you setup your DNS later. - - -What is going on here? --------------------------------------------- - -First, some background terminology: - -* **puppet**: Puppet is a system for automating deployment and management of servers (called nodes). -* **hiera files**: In puppet, you can use something called a 'hiera file' to seed a node with a few configuration values. In LEAP, we go all out and put *every* configuration value needed for a node in the hiera file, and automatically compile a custom hiera file for each node. - -When you run `leap deploy`, a bunch of things happen, in this order: - -1. **Compile hiera files**: The hiera configuration file for each node is compiled in YAML format and saved in the directory `hiera`. The source material for this hiera file consists of all the JSON configuration files imported or inherited by the node's JSON config file. -* **Copy required files to node**: All the files needed for puppet to run are rsync'ed to each node. This includes the entire leap_platform directory, as well as the node's hiera file and other files needed by puppet to set up the node (keys, binary files, etc). -* **Puppet is run**: Once the node is ready, leap connects to the node via ssh and runs `puppet apply`. Puppet is applied locally on the node, without a daemon or puppetmaster. - -You can run `leap -v2 deploy` to see exactly what commands are being executed. - -<!-- See [under the hood](under-the-hood) for more details. --> - - -Test that things worked correctly -================================= - -You should now one machine with the LEAP platform email service deployed to it. - - -Access the web application --------------------------------------------- - -In order to connect to the web application in your browser, you need to point your domain at the IP address of your new node. - -Next, you can connect to the web application either using a web browser or via the API using the LEAP client. To use a browser, connect to https://example.org (replacing that with your domain). Your browser will complain about an untrusted cert, but for now just bypass this. From there, you should be able to register a new user and login. - -Testing with leap_cli ---------------------- - -Use the test command to run a set of different tests: - - leap test - - -Additional information -====================== - -It is useful to know a few additional things. - -Useful commands ---------------- - -Here are a few useful commands you can run on your new local nodes: - -* `leap ssh web1` -- SSH into node web1 (requires `leap node init web1` first). -* `leap list` -- list all nodes. -* `leap list production` -- list only those nodes with the tag 'production' -* `leap list --print ip_address` -- list a particular attribute of all nodes. -* `leap cert update` -- generate new certificates if needed. - -See the full command reference for more information. - -Node filters -------------------------------------------- - -Many of the `leap` commands take a "node filter". You can use a node filter to target a command at one or more nodes. +---------------------------- -A node filter consists of one or more keywords, with an optional "+" before each keyword. +There are several important DNS entries that all email providers should have: -* keywords can be a node name, a service type, or a tag. -* the "+" before the keyword constructs an AND condition -* otherwise, multiple keywords together construct an OR condition +* SPF (Sender Policy Framework): With SPF, an email provider advertises in their DNS which servers should be allowed to relay email on behalf of your domain. +* DKIM (DomainKey Identified Mail): With DKIM, an email provider is able to vouch for the validity of certain headers in outgoing mail, allowing the receiving provider to have more confidence in these values when processing the message for spam or abuse. -Examples: +In order to take advantage of SPF and DKIM, run this command: -* `leap list openvpn` -- list all nodes with service openvpn. -* `leap list openvpn +production` -- only nodes of service type openvpn AND tag production. -* `leap deploy webapp openvpn` -- deploy to all webapp OR openvpn nodes. -* `leap node init vpn1` -- just init the node named vpn1. + workstation$ leap compile zone -Keep track of your provider configurations ------------------------------------------- +Then take the output of that command and merge it with the DNS zone file for your domain. -You should commit your provider changes to your favorite VCS whenever things change. This way you can share your configurations with other admins, all they have to do is to pull the changes to stay up to date. Every time you make a change to your provider, such as adding nodes, services, generating certificates, etc. you should add those to your VCS, commit them and push them to where your repository is hosted. +CAUTION: the output of `leap compile zone` is not a complete zone file since it is missing a serial number. You will need to manually merge it with your existing zone file. -Note that your provider directory contains secrets! Those secrets include passwords for various services. You do not want to have those passwords readable by the world, so make sure that wherever you are hosting your repository, it is not public for the world to read. +Test it out +--------------------------------- -What's next ------------------------------------ +First, run: -Read the [LEAP platform guide](guide) to learn about planning and securing your infrastructure. + workstation# leap test +Then fire up the bitmask client, register a new user with your provider, and try sending and receiving email. diff --git a/doc/tutorials/single-node-vpn.md b/doc/tutorials/single-node-vpn.md new file mode 100644 index 00000000..dc1df7ab --- /dev/null +++ b/doc/tutorials/single-node-vpn.md @@ -0,0 +1,112 @@ +@title = "Single node VPN tutorial" +@nav_title = "Quick VPN" +@summary = 'Tutorial for setting up a simple VPN provider.' + +This tutorial walks you through the initial process of creating and deploying a minimal VPN service provider. Please first complete the [[quick-start]]. This tutorial will pick up where that one left off. + +NOTE: For the VPN to work, you must use a real or paravirtualized node, not a local Vagrant node. + +Our goal +------------------ + +We are going to create a minimal LEAP provider offering VPN service. + +Our goal is something like this: + + $ leap list + NODES SERVICES TAGS + wildebeest couchdb, webapp, openvpn, tor + +Where 'wildebeest' is whatever name you chose for your node in the [[quick-start]]. + +Add VPN service to the node +-------------------------------------- + +In order to add [[services => services]] to a node, edit the node's JSON configuration file. + +In our example, we would edit `nodes/wildebeest.json`: + + { + "ip_address": "1.1.1.1", + "services": ["couchdb", "webapp", "openvpn", "tor"] + } + +Here, we added `openvpn` and `tor` to the node's `services` list. Briefly: + +* **openvpn**: nodes with the **openvpn** service will become OpenVPN gateways that clients connect to in order to proxy their internet connection. You can have as many as you want, spread out over as many nodes as you want. +* **tor**: nodes with **tor** service become Tor exit nodes. This is entirely optional, and will add additional bandwidth to your node. If you don't have many VPN users, the added traffic will help create cover traffic for your users. On the down side, this VPN gateway will get flagged as an anonymous proxy and some sites may block traffic from it. + +For more details, see the [[services]] overview, or the individual pages for the [[openvpn]] and [[tor]] services. + +Add gateway_address to the node +---------------------------------------- + +VPN gateways require two different IP addresses: + +* `ip_address`: This property is used for VPN traffic **egress**. In other words, all VPN traffic appears to come from this IP address. This is also the main IP of the server. +* `openvpn.gateway_address`: This property is used for VPN traffic **ingress**. In other words, clients will connect to this IP address. + +The node configuration file should now look like this: + + { + "ip_address": "1.1.1.1", + "services": ["couchdb", "webapp", "openvpn", "tor"], + "openvpn": { + "gateway_address": "2.2.2.2" + } + } + +Why two different addresses? Without this, the traffic from one VPN user to another would not be encrypted. This is because the routing table of VPN clients must ensure that packets with a destination of the VPN gateway are sent unmodified and don't get passed through the VPN's encryption. + +Generate a Diffie-Hellman file +------------------------------------------- + +Next we need to create a Diffie-Hellman parameter file, used for forward secret OpenVPN ciphers. You only need to do this once. + + workstation$ leap cert dh + +Feel free to erase the resulting DH file and regenerate it as you please. + +Deploy to the node +-------------------- + +Now you should deploy to your node. This may take a while. + + workstation$ leap deploy + +If the deploy was not successful, try to run it again. + +Test it out +--------------------------------- + +First, run: + + workstation$ leap test + +Then fire up the Bitmask client, register a new user with your provider, and turn on the VPN connection. + +Alternately, you can also manually connect to your VPN gateway using OpenVPN on the command line: + + workstation$ sudo apt install openvpn + workstation$ leap test init + workstation$ sudo openvpn --config test/openvpn/default_unlimited.ovpn + +Make sure that Bitmask is not connected to the VPN when you run that command. + +The name of the test configuration might differ depending on your setup. The test configuration created by `leap test init` includes a client certificate that will expire, so you may need to re-run `leap test init` if it has been a while since you last generated the test configuration. + +What do do next +-------------------------------- + +A VPN provider with a single gateway is kind of limited. You can add as many nodes with service [[openvpn]] as you like. There is no communication among the VPN gateways or with the [[webapp]] or [[couchdb]] nodes, so there is no issue with scaling out the number of gateways. + +For example, add some more nodes: + + workstation$ leap node add giraffe ip_address:1.1.1.2 services:openvpn openvpn.gateway_address:2.2.2.3 + workstation$ leap node add rhino ip_address:1.1.1.3 services:openvpn openvpn.gateway_address:2.2.2.4 + workstation$ leap node init giraffe rhino + workstation$ leap deploy + +Now you have three VPN gateways. + +One consideration is that you should tag each VPN gateway with a [[location => nodes#locations]]. This helps the client determine which VPN gateway it should connect to by default and will allow the user to choose among gateways based on location. diff --git a/doc/tutorials/vagrant.md b/doc/tutorials/vagrant.md new file mode 100644 index 00000000..710c2664 --- /dev/null +++ b/doc/tutorials/vagrant.md @@ -0,0 +1,471 @@ +@title = 'Vagrant and the LEAP Platform' +@nav_title = 'Vagrant' +@summary = 'Running a local provider with Vagrant' + +What is Vagrant? +======================================== + +[[Vagrant => https://www.vagrantup.com]] is a tool to make it easier to manage virtual machines running on your desktop computer (typically for testing or development purposes). You can use Vagrant to create virtual machines and deploy the LEAP platform locally. + +Vagrant can be a pain to get working initially, but this page should help you get through the process. Please make sure you have at least Vagrant v1.5 installed. + +There are two ways you can setup LEAP platform using Vagrant. + +1. use the `leap` command: this will allow you to create multiple virtual machines. +2. use static Vagrantfile: there is a static Vagrantfile that is distributed with the `leap_platform.git`. This only supports a single, pre-configured virtual machine, but can get you started more quickly. + +Install Vagrant +======================================== + +Requirements: + +* A real machine with virtualization support in the CPU (VT-x or AMD-V). In other words, not a virtual machine. +* Have at least 4gb of RAM. +* Have a fast internet connection (because you will be downloading a lot of big files, like virtual machine images). +* You should do everything described below as an unprivileged user, and only run those commands as root that are noted with *sudo* in front of them. Other than those commands, there is no need for privileged access to your machine, and in fact things may not work correctly. + +*Debian & Ubuntu* + +Install core prerequisites: + + sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make + +Install Vagrant: + + sudo apt-get install vagrant virtualbox + +If you want to use libvirt instead of virtualbox, you don't need to install virtualbox. See [support for libvirt](#support-for-libvirt). + +*Mac OS X 10.9 (Mavericks)* + +Install Homebrew package manager from http://brew.sh/ and enable the [System Duplicates Repository](https://github.com/Homebrew/homebrew/wiki/Interesting-Taps-&-Branches) (needed to update old software versions delivered by Apple) with + + brew tap homebrew/dupes + +Update OpenSSH to support ECDSA keys. Follow [this guide](http://www.dctrwatson.com/2013/07/how-to-update-openssh-on-mac-os-x/) to let your system use the Homebrew binary. + + brew install openssh --with-brewed-openssl --with-keychain-support + +The certtool provided by Apple it's really old, install the one provided by GnuTLS and shadow the system's default. + + sudo brew install gnutls + ln -sf /usr/local/bin/gnutls-certtool /usr/local/bin/certool + +Install the Vagrant and VirtualBox packages for OS X from their respective Download pages. + +* http://www.vagrantup.com/downloads.html +* https://www.virtualbox.org/wiki/Downloads + +Vagrant with leap command +======================================= + +If you have not done so, install `leap` command line tool: + + gem install leap_cli + +Creating local nodes +---------------------------------- + +When you create a service provider, your servers are called "nodes". When a node is virtual and exists only locally using vagrant, this type of node is called a "local node". + +If you do not have a provider already, you will need to create one and configure it before continuing (see the [Quick Start](quick-start) guide). + +These commands, for example, will create an initial provider directory "myprovider": + + $ leap new --domain example.org --name Example myprovider + $ cd myprovider + $ leap add-user --self + $ leap cert ca + $ leap cert csr + +To create local nodes, add the flag `--local` to the `leap node add` command. For example: + + $ leap node add --local web1 services:webapp + = created nodes/web1.json + = created files/nodes/web1/ + = created files/nodes/web1/web1.key + = created files/nodes/web1/web1.crt + +This command creates a node configuration file in `nodes/web1.json` with the webapp service. + +Starting local nodes +-------------------------------- + +In order to test the node "web1" we need to start it. Starting a node for the first time will spin up a virtual machine. The first time you do this will take some time because it will need to download a VM image (about 700mb). After you've downloaded the base image, you will not need to download it again, and instead you will re-use the downloaded image (until you need to update the image). + +NOTE: Many people have difficulties getting Vagrant working. If the following commands do not work, please see the troubleshooting section below. + + $ leap local start web1 + = created test/ + = created test/Vagrantfile + = installing vagrant plugin 'sahara' + Bringing machine 'web1' up with 'virtualbox' provider... + [web1] Box 'leap-jessie' was not found. Fetching box from specified URL for + the provider 'virtualbox'. Note that if the URL does not have + a box for this provider, you should interrupt Vagrant now and add + the box yourself. Otherwise Vagrant will attempt to download the + full box prior to discovering this error. + Downloading or copying the box... + Progress: 3% (Rate: 560k/s, Estimated time remaining: 0:13:36) + ... + Bringing machine 'web1' up with 'virtualbox' provider... + [web1] Importing base box 'leap-jessie'... + 0%...10%...20%...30%...40%...50%...60%...70%...80%...90%...100% + +Now the virtual machine 'web1' is running. You can add another local node using the same process. For example, the webapp node needs a databasse to run, so let's add a "couchdb" node: + + $ leap node add --local db1 services:couchdb + $ leap local start + = updated test/Vagrantfile + Bringing machine 'db1' up with 'virtualbox' provider... + [db1] Importing base box 'leap-jessie'... + [db1] Matching MAC address for NAT networking... + [db1] Setting the name of the VM... + [db1] Clearing any previously set forwarded ports... + [db1] Fixed port collision for 22 => 2222. Now on port 2202. + [db1] Creating shared folders metadata... + [db1] Clearing any previously set network interfaces... + [db1] Preparing network interfaces based on configuration... + [db1] Forwarding ports... + [db1] -- 22 => 2202 (adapter 1) + [db1] Running any VM customizations... + [db1] Booting VM... + [db1] Waiting for VM to boot. This can take a few minutes. + [db1] VM booted and ready for use! + [db1] Configuring and enabling network interfaces... + [db1] Mounting shared folders... + [db1] -- /vagrant + +You now can follow the normal LEAP process and initialize it and then deploy your recipes to it: + + $ leap node init web1 + $ leap deploy web1 + $ leap node init db1 + $ leap deploy db1 + +Useful local commands +------------------------------------ + +There are many useful things you can do with a virtualized development environment. + +### Listing what machines are running + +Now you have the two virtual machines "web1" and "db1" running, you can see the running machines as follows: + + $ leap local status + Current machine states: + + db1 running (virtualbox) + web1 running (virtualbox) + + This environment represents multiple VMs. The VMs are all listed + above with their current state. For more information about a specific + VM, run `vagrant status NAME`. + +### Stopping machines + +It is not recommended that you leave your virtual machines running when you are not using them. They consume memory and other resources! To stop your machines, simply do the following: + + $ leap local stop web1 db1 + +### Connecting to machines + +You can connect to your local nodes just like you do with normal LEAP nodes, by running 'leap ssh node'. + +However, if you cannot connect to your local node, because the networking is not setup properly, or you have deployed a firewall that locks you out, you may need to access the graphical console. + +In order to do that, you will need to configure Vagrant to launch a graphical console and then you can login as root there to diagnose the networking problem. To do this, add the following to your $HOME/.leaprc: + + @custom_vagrant_vm_line = 'config.vm.provider "virtualbox" do |v| + v.gui = true + end' + +and then start, or restart, your local Vagrant node. You should get a VirtualBox graphical interface presented to you showing you the bootup and eventually the login. + +### Snapshotting machines + +A very useful feature of local Vagrant development nodes is the ability to snapshot the current state and then revert to that when you need. + +For example, perhaps the base image is a little bit out of date and you want to get the packages updated to the latest before continuing. You can do that simply by starting the node, connecting to it and updating the packages and then snapshotting the node: + + $ leap local start web1 + $ leap ssh web1 + web1# apt-get -u dist-upgrade + web1# exit + $ leap local save web1 + +Now you can deploy to web1 and if you decide you want to revert to the state before deployment, you simply have to reset the node to your previous save: + + $ leap local reset web1 + +### More information + +See `leap help local` for a complete list of local-only commands and how they can be used. + + +2. Vagrant with static Vagrantfile +================================================== + +You can use the static Vagrantfile if you want to get up a running with a pre-canned test provider. + +It will install a single node mail server in the default configuration with one single command. + +Clone the platform with + + git clone --recursive -b develop https://github.com/leapcode/leap_platform.git + +Start the vagrant box with + + cd leap_platform + vagrant up + +Follow the instructions how to configure your `/etc/hosts` +in order to use the provider! + +You can login via ssh with the systemuser `vagrant` and the same password. + + vagrant ssh + +On the host, run the tests to check if everything is working as expected: + + cd /home/vagrant/leap/configuration/ + leap test + +Use the bitmask client to do an initial soledad sync +------------------------------------------------------------- + +Copy the self-signed CA certificate from the host. +The easiest way is to use the [vagrant-scp plugin](https://github.com/invernizzi/vagrant-scp): + + vagrant scp :/home/vagrant/leap/configuration/files/ca/ca.crt /tmp/example.org.ca.crt + + vagrant@node1:~/leap/configuration$ cat files/ca/ca.crt + +and write it into a file, needed by the bitmask client: + + bitmask --ca-cert-file /tmp/example.org.ca.crt + +On the first run, bitmask is creating a gpg keypair. This is +needed for delivering and encrypting incoming mails. + +Testing email +------------- + + sudo apt install swaks + swaks -f test22@leap.se -t test22@example.org -s example.org + +check the logs: + + sudo less /var/log/mail.log + sudo less /var/log/leap/mx.log + +if an error occurs, see if the mail is still laying in the mailspool dir: + + sudo ls /var/mail/leap-mx/Maildir/new + +Re-run bitmask client to sync your mail +--------------------------------------- + + bitmask --ca-cert-file /tmp/example.org.ca.crt + +Now, connect your favorite mail client to the imap and smtp proxy +started by the bitmask client: + + https://bitmask.net/en/help/email + +Happy testing ! + +Using the Webapp +---------------- + +There are 2 users preconfigured: + +. `testuser` with pw `hallo123` +. `testadmin` with pw `hallo123` + +login as `testadmin` to access the webapp with admin priviledges. + + +Support for libvirt +======================================= + +Install libvirt plugin +------------------------------------- + +By default, Vagrant will use VirtualBox to create the virtual machines, but this is how you can use libvirt. Using libvirt is more efficient, but VirtualBox is more stable and easier to set up. + +*For debian/ubuntu:* + + sudo apt-get install libvirt-bin libvirt-dev + + # to build the vagrant-libvirt plugin you need the following packages: + sudo apt-get install ruby-dev libxslt-dev libxml2-dev libvirt-dev + + # install the required plugins + vagrant plugin install vagrant-libvirt fog fog-libvirt sahara + +Log out and then log back in. + +Note: if running ubuntu 15.10 as the host OS, you will probably need to run the following commands before "vagrant plugin install vagrant-libvirt" will work: + + ln -sf /usr/lib/liblzma.so.5 /opt/vagrant/embedded/lib + ln -sf /usr/lib/liblzma.so.5.0.0 /opt/vagrant/embedded/lib + +Create libvirt pool +----------------------------------------- + +Next, you must create the libvirt image pool. The "default" pool uses `/var/lib/libvirt/images`, but Vagrant will not download base boxes there. Instead, create a libvirt pool called "vagrant", like so: + + virsh pool-define-as vagrant dir - - - - /home/$USER/.vagrant.d/boxes + virsh pool-start vagrant + virsh pool-autostart vagrant + +If you want to use a name different than "vagrant" for the pool, you can change the name in `Leapfile` by setting the `@vagrant_libvirt_pool` variable: + + @vagrant_libvirt_pool = "vagrant" + +Force use of libvirt +-------------------------------------------- + +Finally, you need to tell Vagrant to use libvirt instead of VirtualBox. If using vagrant with leap_cli, modify your `Leapfile` or `.leaprc` file and add this line: + + @vagrant_provider = "libvirt" + +Alternately, if using the static Vagrantfile, you must run this in your shell instead: + + export VAGRANT_DEFAULT_PROVIDER=libvirt + + +Debugging +------------------------ + +If you get an error in any of the above commands, try to get some debugging information, it will often tell you what is wrong. In order to get debugging logs, you simply need to re-run the command that produced the error but prepend the command with VAGRANT_LOG=info, for example: + + VAGRANT_LOG=info vagrant box add LEAP/jessie + +You can also run vagrant with --debug for full logging. + +Known issues +------------------------ + +* You may need to undefine the default libvirt pool: + sudo virsh pool-undefine default +* `Call to virConnectOpen failed: internal error: Unable to locate libvirtd daemon in /usr/sbin (to override, set $LIBVIRTD_PATH to the name of the libvirtd binary)` - you don't have the libvirtd daemon running or installed, be sure you installed the 'libvirt-bin' package and it is running +* `Call to virConnectOpen failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': Permission denied` - you need to be in the libvirt group to access the socket, do 'sudo adduser <user> libvirtd' and then re-login to your session. +* if each call to vagrant ends up with a segfault, it may be because you still have virtualbox around. if so, remove virtualbox to keep only libvirt + KVM. according to https://github.com/pradels/vagrant-libvirt/issues/75 having two virtualization engines installed simultaneously can lead to such weird issues. +* see the [vagrant-libvirt issue list on github](https://github.com/pradels/vagrant-libvirt/issues) +* be sure to use vagrant-libvirt >= 0.0.11 and sahara >= 0.0.16 (which are the latest stable gems you would get with `vagrant plugin install [vagrant-libvirt|sahara]`) for proper libvirt support, + +Useful commands +------------------------ + +Force re-download of image, in case something goes wrong: + + vagrant box add leap/jessie --force --provider libvirt + +Shared folder support +---------------------------- + +For shared folder support, you need nfs-kernel-server installed on the host machine and set up sudo to allow unpriviledged users to modify /etc/exports. See [vagrant-libvirt#synced-folders](https://github.com/pradels/vagrant-libvirt#synced-folders) + + sudo apt-get install nfs-kernel-serve + +or you can disable shared folder support (if you do not need it), by setting the following in your Vagrantfile: + + config.vm.synced_folder "src/", "/srv/website", disabled: trueconfig.vm.synced_folder "src/", "/srv/website", disabled: true + +if you are wanting this disabled for all the leap vagrant integration, you can add this to ~/.leaprc: + + @custom_vagrant_vm_line = 'config.vm.synced_folder "src/", "/srv/website", disabled: true' + + +Verify vagrantboxes +=============================================== + +When you run vagrant, it goes out to the internet and downloads an initial image for the virtual machine. If you want to verify that authenticity of these images, follow these steps. + +Import LEAP archive signing key: + + gpg --search-keys 0x1E34A1828E207901 + +now, either you already have a trustpath to it through one of the people +who signed it, or you can verify this by checking this fingerprint: + + gpg --fingerprint --list-keys 1E34A1828E207901 + + pub 4096R/1E34A1828E207901 2013-02-06 [expires: 2015-02-07] + Key fingerprint = 1E45 3B2C E87B EE2F 7DFE 9966 1E34 A182 8E20 7901 + uid LEAP archive signing key <sysdev@leap.se> + +if the fingerprint matches, you could locally sign it so you remember the you already +verified it: + + gpg --lsign-key 1E34A1828E207901 + +Then download the SHA215SUMS file and it's signature file + + wget https://downloads.leap.se/platform/SHA256SUMS.sign + wget https://downloads.leap.se/platform/SHA256SUMS + +and verify the signature against your local imported LEAP archive signing pubkey + + gpg --verify SHA256SUMS.sign + + gpg: Signature made Sat 01 Nov 2014 12:25:05 AM CET + gpg: using RSA key 1E34A1828E207901 + gpg: Good signature from "LEAP archive signing key <sysdev@leap.se>" + +Make sure that the last line says "Good signature from...", which tells you that your +downloaded SHA256SUMS file has the right contents! + +Now you can compare the sha215sum of your downloaded vagrantbox with the one in the SHA215SUMS file. You could have downloaded it manually from https://atlas.hashicorp.com/api/v1/box/LEAP/jessie/$version/$provider.box otherwise it's probably located within ~/.vagrant.d/. + + wget https://atlas.hashicorp.com/LEAP/boxes/jessie/versions/1.1.0/providers/libvirt.box + sha215sum libvirt.box + cat SHA215SUMS + +Troubleshooting +======================= + +To troubleshoot vagrant issues, try going through these steps: + +* Try plain vagrant using the [Getting started guide](http://docs.vagrantup.com/v2/getting-started/index.html). +* If that fails, make sure that you can run virtual machines (VMs) in plain virtualbox (Virtualbox GUI or VBoxHeadless). + We don't suggest a special howto for that, [this one](http://www.thegeekstuff.com/2012/02/virtualbox-install-create-vm/) seems pretty decent, or you follow the [Oracale Virtualbox User Manual](http://www.virtualbox.org/manual/UserManual.html). There's also specific documentation for [Debian](https://wiki.debian.org/VirtualBox) and for [Ubuntu](https://help.ubuntu.com/community/VirtualBox). If you succeeded, try again if you now can start vagrant nodes using plain vagrant (see first step). +* If plain vagrant works for you, you're very close to using vagrant with leap! If you encounter any problems now, please [contact us](https://leap.se/en/about-us/contact) or use our [issue tracker](https://leap.se/code) + +Additional notes +==================== + +Some useful plugins +----------------------------- + +* The vagrant-cachier (plugin http://fgrehm.viewdocs.io/vagrant-cachier/) lets you cache .deb packages on your hosts so they are not downloaded by multiple machines over and over again, after resetting to a previous state. + +Limitations +----------------------- + +Please consult the known issues for vagrant, see the [Known Issues](known-issues), section *Special Environments* + +Known working combinations +-------------------------- + +Please consider that using other combinations might work for you as well, these are just the combinations we tried and worked for us: + +Debian Wheezy + +* `virtualbox-4.2 4.2.16-86992~Debian~wheezy` from Oracle and `vagrant 1.2.2` from vagrantup.com + +Ubuntu Wily 15.10 + +* libvirt with vagrant 1.7.2, from standard Ubuntu packages. + +Mac OS X 10.9 + +* `VirtualBox 4.3.10` from virtualbox.org and `vagrant 1.5.4` from vagrantup.com + + +Issue reporting +--------------- + +When you encounter any bugs, please [check first](https://leap.se/code/search) on our bugtracker if it's something already known. Reporting bugs is the first [step in fixing them](https://leap.se/code/projects/report-issues). Please include all the relevant details: platform branch, version of leap_cli, past upgrades. diff --git a/doc/upgrading/en.haml b/doc/upgrading/en.haml new file mode 100644 index 00000000..efa0d7c5 --- /dev/null +++ b/doc/upgrading/en.haml @@ -0,0 +1,5 @@ +- @nav_title = "Upgrading" +- @title = "Upgrading from prior LEAP platform releases" +- @summary = "" + += child_summaries
\ No newline at end of file diff --git a/doc/upgrading/upgrade-0-8.md b/doc/upgrading/upgrade-0-8.md new file mode 100644 index 00000000..84e9cee2 --- /dev/null +++ b/doc/upgrading/upgrade-0-8.md @@ -0,0 +1,141 @@ +@title = 'Upgrade to 0.8' +@toc = false + +LEAP Platform release 0.8 introduces several major changes that need do get taken into account while upgrading: + +* Dropping Debian Wheezy support. You need to upgrade your nodes to jessie before deploying a platform upgrade. +* Dropping BigCouch support. LEAP Platform now requires CouchDB and therefore you need to migrate all your data from BigCouch to CouchDB. + +Upgrading to Platform 0.8 +--------------------------------------------- + +### Step 1: Get new leap_platform and leap_cli + + workstation$ gem install leap_cli --version 1.8 + workstation$ cd leap_platform + workstation$ git pull + workstation$ git checkout 0.8.0 + +### Step 2: Prepare to migrate from BigCouch to CouchDB + +<%= render :partial => 'docs/platform/common/bigcouch_migration_begin.md' %> + +### Step 3: Upgrade from Debian Wheezy to Jessie + +There are the [Debian release notes on how to upgrade from wheezy to jessie](https://www.debian.org/releases/stable/amd64/release-notes/ch-upgrading.html). Here are the steps that worked for us, but please keep in mind that there is no bullet-proof method that will work in every situation. + +**USE AT YOUR OWN RISK.** + +For each one of your nodes, login to it and do the following process: + + # keep a log of the progress: + screen + script -t 2>~/leap_upgrade-jessiestep.time -a ~/upgrade-jessiestep.script + + # ensure you have a good wheezy install: + export DEBIAN_FRONTEND=noninteractive + apt-get autoremove --yes + apt-get update + apt-get -y -o DPkg::Options::=--force-confold dist-upgrade + + # if either of these return anything, you will need to resolve them before continuing: + dpkg --audit + dpkg --get-selections | grep 'hold$' + + # switch sources to jessie + sed -i 's/wheezy/jessie/g' /etc/apt/sources.list + rm /etc/apt/sources.list.d/* + echo "deb http://deb.leap.se/0.8 jessie main" > /etc/apt/sources.list.d/leap.list + + # remove pinnings to wheezy + rm /etc/apt/preferences + rm /etc/apt/preferences.d/* + + # get jessie package lists + apt-get update + + # clean out old package files + apt-get clean + + # test to see if you have enough space to upgrade, the following will alert + # you if you do not have enough space, it will not do the actual upgrade + apt-get -o APT::Get::Trivial-Only=true dist-upgrade + + # do first stage upgrade + apt-get -y -o DPkg::Options::=--force-confold upgrade + + # repeat the following until it makes no more changes: + apt-get -y -o DPkg::Options::=--force-confold dist-upgrade + + # resolve any apt issues if there are some + apt-get -y -o DPkg::Options::=--force-confold -f install + + # clean up extra packages + apt-get autoremove --yes + + reboot + + +Potential Jessie Upgrade Issues +------------------------------- + +**W: Ignoring Provides line with DepCompareOp for package python-cffi-backend-api-max** + +You can ignore these warnings, they will be resolved on upgrade. + +**E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?** + +If you get this error, run `apt-get update` and then re-run the command. + +**Unmet dependencies. Try using -f.** + +Sometimes you might get an error similar to this (although the package names may be different): + + You might want to run 'apt-get -f install' to correct these. + The following packages have unmet dependencies: + lsof : Depends: libperl4-corelibs-perl but it is not installed or + perl (< 5.12.3-7) but 5.20.2-3+deb8u4 is installed + +If this happens, run `apt-get -f install` to resolve it, and then re-do the previous upgrade command +you did when this happened. + +**Failure restarting some services for OpenSSL upgrade** + +If you get this warning: + + The following services could not be restarted for the OpenSSL library upgrade: + postfix + You will need to start these manually by running '/etc/init.d/<service> start'. + +Just ignore it, it should be fixed on reboot/deploy. + +### Step 4: Deploy LEAP Platform 0.8 to the Couch node + +You will need to deploy the 0.8 version of LEAP Platform to the couch node before continuing. + +1. deploy to the couch node: + + ``` + workstation$ leap deploy <couchdb-node> + ``` + + If you used the iptables method of blocking access to couchdb, you need to run it again because the deploy just overwrote all the iptables rules: + + ``` + server# iptables -A INPUT -p tcp --dport 5984 --jump REJECT + ``` + +### Step 5: Import Data into CouchDB + +<%= render :partial => 'docs/platform/common/bigcouch_migration_end.md' %> + +### Step 6: Deploy everything + +Now that you've upgraded all nodes to Jessie, and migrated to CouchDB, you are ready to deploy LEAP Platform 0.8 to the rest of the nodes: + + workstation$ cd <provider directory> + workstation$ leap deploy + +### Step 7: Test and cleanup + +<%= render :partial => 'docs/platform/common/bigcouch_migration_finish.md' %> diff --git a/hiera.yaml b/hiera.yaml new file mode 100644 index 00000000..3ff857b8 --- /dev/null +++ b/hiera.yaml @@ -0,0 +1,6 @@ +--- +:backends: yaml +:yaml: + :datadir: /var/lib/hiera +:hierarchy: common +:logger: console diff --git a/lib/leap_cli/commands/README b/lib/leap_cli/commands/README new file mode 100644 index 00000000..bec78179 --- /dev/null +++ b/lib/leap_cli/commands/README @@ -0,0 +1,11 @@ +This directory contains ruby source files that define the available sub- +commands of the `leap` executable. + +For example, the command: + + leap compile + +Lives in lib/leap_cli/commands/init.rb + +These files use a DSL (called GLI) for defining command suites. +See https://github.com/davetron5000/gli for more information. diff --git a/lib/leap_cli/commands/ca.rb b/lib/leap_cli/commands/ca.rb new file mode 100644 index 00000000..1b311eee --- /dev/null +++ b/lib/leap_cli/commands/ca.rb @@ -0,0 +1,541 @@ +autoload :OpenSSL, 'openssl' +autoload :CertificateAuthority, 'certificate_authority' +autoload :Date, 'date' +require 'digest/md5' + +module LeapCli; module Commands + + desc "Manage X.509 certificates" + command :cert do |cert| + + cert.desc 'Creates two Certificate Authorities (one for validating servers and one for validating clients).' + cert.long_desc 'See see what values are used in the generation of the certificates (like name and key size), run `leap inspect provider` and look for the "ca" property. To see the details of the created certs, run `leap inspect <file>`.' + cert.command :ca do |ca| + ca.action do |global_options,options,args| + assert_config! 'provider.ca.name' + generate_new_certificate_authority(:ca_key, :ca_cert, provider.ca.name) + generate_new_certificate_authority(:client_ca_key, :client_ca_cert, provider.ca.name + ' (client certificates only!)') + end + end + + cert.desc 'Creates or renews a X.509 certificate/key pair for a single node or all nodes, but only if needed.' + cert.long_desc 'This command will a generate new certificate for a node if some value in the node has changed ' + + 'that is included in the certificate (like hostname or IP address), or if the old certificate will be expiring soon. ' + + 'Sometimes, you might want to force the generation of a new certificate, ' + + 'such as in the cases where you have changed a CA parameter for server certificates, like bit size or digest hash. ' + + 'In this case, use --force. If <node-filter> is empty, this command will apply to all nodes.' + cert.arg_name 'FILTER' + cert.command :update do |update| + update.switch 'force', :desc => 'Always generate new certificates', :negatable => false + update.action do |global_options,options,args| + update_certificates(manager.filter!(args), options) + end + end + + cert.desc 'Creates a Diffie-Hellman parameter file, needed for forward secret OpenVPN ciphers.' # (needed for server-side of some TLS connections) + cert.command :dh do |dh| + dh.action do |global_options,options,args| + long_running do + if cmd_exists?('certtool') + log 0, 'Generating DH parameters (takes a long time)...' + output = assert_run!('certtool --generate-dh-params --sec-param high') + output.sub! /.*(-----BEGIN DH PARAMETERS-----.*-----END DH PARAMETERS-----).*/m, '\1' + output << "\n" + write_file!(:dh_params, output) + else + log 0, 'Generating DH parameters (takes a REALLY long time)...' + output = OpenSSL::PKey::DH.generate(3248).to_pem + write_file!(:dh_params, output) + end + end + end + end + + # + # hints: + # + # inspect CSR: + # openssl req -noout -text -in files/cert/x.csr + # + # generate CSR with openssl to see how it compares: + # openssl req -sha256 -nodes -newkey rsa:2048 -keyout example.key -out example.csr + # + # validate a CSR: + # http://certlogik.com/decoder/ + # + # nice details about CSRs: + # http://www.redkestrel.co.uk/Articles/CSR.html + # + cert.desc "Creates a CSR for use in buying a commercial X.509 certificate." + cert.long_desc "Unless specified, the CSR is created for the provider's primary domain. "+ + "The properties used for this CSR come from `provider.ca.server_certificates`, "+ + "but may be overridden here." + cert.command :csr do |csr| + csr.flag 'domain', :arg_name => 'DOMAIN', :desc => 'Specify what domain to create the CSR for.' + csr.flag ['organization', 'O'], :arg_name => 'ORGANIZATION', :desc => "Override default O in distinguished name." + csr.flag ['unit', 'OU'], :arg_name => 'UNIT', :desc => "Set OU in distinguished name." + csr.flag 'email', :arg_name => 'EMAIL', :desc => "Set emailAddress in distinguished name." + csr.flag ['locality', 'L'], :arg_name => 'LOCALITY', :desc => "Set L in distinguished name." + csr.flag ['state', 'ST'], :arg_name => 'STATE', :desc => "Set ST in distinguished name." + csr.flag ['country', 'C'], :arg_name => 'COUNTRY', :desc => "Set C in distinguished name." + csr.flag :bits, :arg_name => 'BITS', :desc => "Override default certificate bit length" + csr.flag :digest, :arg_name => 'DIGEST', :desc => "Override default signature digest" + csr.action do |global_options,options,args| + assert_config! 'provider.domain' + assert_config! 'provider.name' + assert_config! 'provider.default_language' + assert_config! 'provider.ca.server_certificates.bit_size' + assert_config! 'provider.ca.server_certificates.digest' + domain = options[:domain] || provider.domain + + unless global_options[:force] + assert_files_missing! [:commercial_key, domain], [:commercial_csr, domain], + :msg => 'If you really want to create a new key and CSR, remove these files first or run with --force.' + end + + server_certificates = provider.ca.server_certificates + + # RSA key + keypair = CertificateAuthority::MemoryKeyMaterial.new + bit_size = (options[:bits] || server_certificates.bit_size).to_i + log :generating, "%s bit RSA key" % bit_size do + keypair.generate_key(bit_size) + write_file! [:commercial_key, domain], keypair.private_key.to_pem + end + + # CSR + dn = CertificateAuthority::DistinguishedName.new + dn.common_name = domain + dn.organization = options[:organization] || provider.name[provider.default_language] + dn.ou = options[:organizational_unit] # optional + dn.email_address = options[:email] # optional + dn.country = options[:country] || server_certificates['country'] # optional + dn.state = options[:state] || server_certificates['state'] # optional + dn.locality = options[:locality] || server_certificates['locality'] # optional + + digest = options[:digest] || server_certificates.digest + log :generating, "CSR with #{digest} digest and #{print_dn(dn)}" do + csr = create_csr(dn, keypair, digest) + request = csr.to_x509_csr + write_file! [:commercial_csr, domain], csr.to_pem + end + + # Sign using our own CA, for use in testing but hopefully not production. + # It is not that commerical CAs are so secure, it is just that signing your own certs is + # a total drag for the user because they must click through dire warnings. + #if options[:sign] + log :generating, "self-signed x509 server certificate for testing purposes" do + cert = csr.to_cert + cert.serial_number.number = cert_serial_number(domain) + cert.not_before = yesterday + cert.not_after = yesterday.advance(:years => 1) + cert.parent = ca_root + cert.sign! domain_test_signing_profile + write_file! [:commercial_cert, domain], cert.to_pem + log "please replace this file with the real certificate you get from a CA using #{Path.relative_path([:commercial_csr, domain])}" + end + #end + + # FAKE CA + unless file_exists? :commercial_ca_cert + log :using, "generated CA in place of commercial CA for testing purposes" do + write_file! :commercial_ca_cert, read_file!(:ca_cert) + log "please also replace this file with the CA cert from the commercial authority you use." + end + end + end + end + end + + protected + + # + # will generate new certificates for the specified nodes, if needed. + # + def update_certificates(nodes, options={}) + assert_files_exist! :ca_cert, :ca_key, :msg => 'Run `leap cert ca` to create them' + assert_config! 'provider.ca.server_certificates.bit_size' + assert_config! 'provider.ca.server_certificates.digest' + assert_config! 'provider.ca.server_certificates.life_span' + assert_config! 'common.x509.use' + + nodes.each_node do |node| + warn_if_commercial_cert_will_soon_expire(node) + if !node.x509.use + remove_file!([:node_x509_key, node.name]) + remove_file!([:node_x509_cert, node.name]) + elsif options[:force] || cert_needs_updating?(node) + generate_cert_for_node(node) + end + end + end + + private + + def generate_new_certificate_authority(key_file, cert_file, common_name) + assert_files_missing! key_file, cert_file + assert_config! 'provider.ca.name' + assert_config! 'provider.ca.bit_size' + assert_config! 'provider.ca.life_span' + + root = CertificateAuthority::Certificate.new + + # set subject + root.subject.common_name = common_name + possible = ['country', 'state', 'locality', 'organization', 'organizational_unit', 'email_address'] + provider.ca.keys.each do |key| + if possible.include?(key) + root.subject.send(key + '=', provider.ca[key]) + end + end + + # set expiration + root.not_before = yesterday + root.not_after = yesterday_advance(provider.ca.life_span) + + # generate private key + root.serial_number.number = 1 + root.key_material.generate_key(provider.ca.bit_size) + + # sign self + root.signing_entity = true + root.parent = root + root.sign!(ca_root_signing_profile) + + # save + write_file!(key_file, root.key_material.private_key.to_pem) + write_file!(cert_file, root.to_pem) + end + + # + # returns true if the certs associated with +node+ need to be regenerated. + # + def cert_needs_updating?(node) + if !file_exists?([:node_x509_cert, node.name], [:node_x509_key, node.name]) + return true + else + cert = load_certificate_file([:node_x509_cert, node.name]) + if !created_by_authority?(cert, ca_root) + log :updating, "cert for node '#{node.name}' because it was signed by an old CA root cert." + return true + end + if cert.not_after < Time.now.advance(:months => 2) + log :updating, "cert for node '#{node.name}' because it will expire soon" + return true + end + if cert.subject.common_name != node.domain.full + log :updating, "cert for node '#{node.name}' because domain.full has changed (was #{cert.subject.common_name}, now #{node.domain.full})" + return true + end + cert.openssl_body.extensions.each do |ext| + if ext.oid == "subjectAltName" + ips = [] + dns_names = [] + ext.value.split(",").each do |value| + value.strip! + ips << $1 if value =~ /^IP Address:(.*)$/ + dns_names << $1 if value =~ /^DNS:(.*)$/ + end + dns_names.sort! + if ips.first != node.ip_address + log :updating, "cert for node '#{node.name}' because ip_address has changed (from #{ips.first} to #{node.ip_address})" + return true + elsif dns_names != dns_names_for_node(node) + log :updating, "cert for node '#{node.name}' because domain name aliases have changed\n from: #{dns_names.inspect}\n to: #{dns_names_for_node(node).inspect})" + return true + end + end + end + end + return false + end + + def created_by_authority?(cert, ca) + authority_key_id = cert.extensions["authorityKeyIdentifier"].identifier.sub(/^keyid:/, '') + authority_key_id == public_key_id_for_ca(ca) + end + + # calculate the "key id" for a root CA, that matches the value + # Authority Key Identifier in the x509 extensions of a cert. + def public_key_id_for_ca(ca_cert) + @ca_key_ids ||= {} + @ca_key_ids[ca_cert.object_id] ||= begin + pubkey = ca_cert.key_material.public_key + seq = OpenSSL::ASN1::Sequence([ + OpenSSL::ASN1::Integer.new(pubkey.n), + OpenSSL::ASN1::Integer.new(pubkey.e) + ]) + Digest::SHA1.hexdigest(seq.to_der).upcase.scan(/../).join(':') + end + end + + def warn_if_commercial_cert_will_soon_expire(node) + dns_names_for_node(node).each do |domain| + if file_exists?([:commercial_cert, domain]) + cert = load_certificate_file([:commercial_cert, domain]) + path = Path.relative_path([:commercial_cert, domain]) + if cert.not_after < Time.now.utc + log :error, "the commercial certificate '#{path}' has EXPIRED! " + + "You should renew it with `leap cert csr --domain #{domain}`." + elsif cert.not_after < Time.now.advance(:months => 2) + log :warning, "the commercial certificate '#{path}' will expire soon (#{cert.not_after}). "+ + "You should renew it with `leap cert csr --domain #{domain}`." + end + end + end + end + + def generate_cert_for_node(node) + return if node.x509.use == false + + cert = CertificateAuthority::Certificate.new + + # set subject + cert.subject.common_name = node.domain.full + cert.serial_number.number = cert_serial_number(node.domain.full) + + # set expiration + cert.not_before = yesterday + cert.not_after = yesterday_advance(provider.ca.server_certificates.life_span) + + # generate key + cert.key_material.generate_key(provider.ca.server_certificates.bit_size) + + # sign + cert.parent = ca_root + cert.sign!(server_signing_profile(node)) + + # save + write_file!([:node_x509_key, node.name], cert.key_material.private_key.to_pem) + write_file!([:node_x509_cert, node.name], cert.to_pem) + end + + # + # yields client key and cert suitable for testing + # + def generate_test_client_cert(prefix=nil) + cert = CertificateAuthority::Certificate.new + cert.serial_number.number = cert_serial_number(provider.domain) + cert.subject.common_name = [prefix, random_common_name(provider.domain)].join + cert.not_before = yesterday + cert.not_after = yesterday.advance(:years => 1) + cert.key_material.generate_key(1024) # just for testing, remember! + cert.parent = client_ca_root + cert.sign! client_test_signing_profile + yield cert.key_material.private_key.to_pem, cert.to_pem + end + + # + # creates a CSR and returns it. + # with the correct extReq attribute so that the CA + # doens't generate certs with extensions we don't want. + # + def create_csr(dn, keypair, digest) + csr = CertificateAuthority::SigningRequest.new + csr.distinguished_name = dn + csr.key_material = keypair + csr.digest = digest + + # define extensions manually (library doesn't support setting these on CSRs) + extensions = [] + extensions << CertificateAuthority::Extensions::BasicConstraints.new.tap {|basic| + basic.ca = false + } + extensions << CertificateAuthority::Extensions::KeyUsage.new.tap {|keyusage| + keyusage.usage = ["digitalSignature", "keyEncipherment"] + } + extensions << CertificateAuthority::Extensions::ExtendedKeyUsage.new.tap {|extkeyusage| + extkeyusage.usage = [ "serverAuth"] + } + + # convert extensions to attribute 'extReq' + # aka "Requested Extensions" + factory = OpenSSL::X509::ExtensionFactory.new + attrval = OpenSSL::ASN1::Set([OpenSSL::ASN1::Sequence( + extensions.map{|e| factory.create_ext(e.openssl_identifier, e.to_s, e.critical)} + )]) + attrs = [ + OpenSSL::X509::Attribute.new("extReq", attrval), + ] + csr.attributes = attrs + + return csr + end + + def ca_root + @ca_root ||= begin + load_certificate_file(:ca_cert, :ca_key) + end + end + + def client_ca_root + @client_ca_root ||= begin + load_certificate_file(:client_ca_cert, :client_ca_key) + end + end + + def load_certificate_file(crt_file, key_file=nil, password=nil) + crt = read_file!(crt_file) + openssl_cert = OpenSSL::X509::Certificate.new(crt) + cert = CertificateAuthority::Certificate.from_openssl(openssl_cert) + if key_file + key = read_file!(key_file) + cert.key_material.private_key = OpenSSL::PKey::RSA.new(key, password) + end + return cert + end + + def ca_root_signing_profile + { + "extensions" => { + "basicConstraints" => {"ca" => true}, + "keyUsage" => { + "usage" => ["critical", "keyCertSign"] + }, + "extendedKeyUsage" => { + "usage" => [] + } + } + } + end + + # + # For keyusage, openvpn server certs can have keyEncipherment or keyAgreement. + # Web browsers seem to break without keyEncipherment. + # For now, I am using digitalSignature + keyEncipherment + # + # * digitalSignature -- for (EC)DHE cipher suites + # "The digitalSignature bit is asserted when the subject public key is used + # with a digital signature mechanism to support security services other + # than certificate signing (bit 5), or CRL signing (bit 6). Digital + # signature mechanisms are often used for entity authentication and data + # origin authentication with integrity." + # + # * keyEncipherment ==> for plain RSA cipher suites + # "The keyEncipherment bit is asserted when the subject public key is used for + # key transport. For example, when an RSA key is to be used for key management, + # then this bit is set." + # + # * keyAgreement ==> for used with DH, not RSA. + # "The keyAgreement bit is asserted when the subject public key is used for key + # agreement. For example, when a Diffie-Hellman key is to be used for key + # management, then this bit is set." + # + # digest options: SHA512, SHA256, SHA1 + # + def server_signing_profile(node) + { + "digest" => provider.ca.server_certificates.digest, + "extensions" => { + "keyUsage" => { + "usage" => ["digitalSignature", "keyEncipherment"] + }, + "extendedKeyUsage" => { + "usage" => ["serverAuth", "clientAuth"] + }, + "subjectAltName" => { + "ips" => [node.ip_address], + "dns_names" => dns_names_for_node(node) + } + } + } + end + + # + # This is used when signing the main cert for the provider's domain + # with our own CA (for testing purposes). Typically, this cert would + # be purchased from a commercial CA, and not signed this way. + # + def domain_test_signing_profile + { + "digest" => "SHA256", + "extensions" => { + "keyUsage" => { + "usage" => ["digitalSignature", "keyEncipherment"] + }, + "extendedKeyUsage" => { + "usage" => ["serverAuth"] + } + } + } + end + + # + # This is used when signing a dummy client certificate that is only to be + # used for testing. + # + def client_test_signing_profile + { + "digest" => "SHA256", + "extensions" => { + "keyUsage" => { + "usage" => ["digitalSignature"] + }, + "extendedKeyUsage" => { + "usage" => ["clientAuth"] + } + } + } + end + + def dns_names_for_node(node) + names = [node.domain.internal, node.domain.full] + if node['dns'] && node.dns['aliases'] && node.dns.aliases.any? + names += node.dns.aliases + end + names.compact! + names.sort! + names.uniq! + return names + end + + # + # For cert serial numbers, we need a non-colliding number less than 160 bits. + # md5 will do nicely, since there is no need for a secure hash, just a short one. + # (md5 is 128 bits) + # + def cert_serial_number(domain_name) + Digest::MD5.hexdigest("#{domain_name} -- #{Time.now}").to_i(16) + end + + # + # for the random common name, we need a text string that will be unique across all certs. + # ruby 1.8 doesn't have a built-in uuid generator, or we would use SecureRandom.uuid + # + def random_common_name(domain_name) + cert_serial_number(domain_name).to_s(36) + end + + # prints CertificateAuthority::DistinguishedName fields + def print_dn(dn) + fields = {} + [:common_name, :locality, :state, :country, :organization, :organizational_unit, :email_address].each do |attr| + fields[attr] = dn.send(attr) if dn.send(attr) + end + fields.inspect + end + + ## + ## TIME HELPERS + ## + ## note: we use 'yesterday' instead of 'today', because times are in UTC, and some people on the planet + ## are behind UTC. + ## + + def yesterday + t = Time.now - 24*24*60 + Time.utc t.year, t.month, t.day + end + + def yesterday_advance(string) + number, unit = string.split(' ') + unless ['years', 'months', 'days', 'hours', 'minutes'].include? unit + bail!("The time property '#{string}' is missing a unit (one of: years, months, days, hours, minutes).") + end + unless number.to_i.to_s == number + bail!("The time property '#{string}' is missing a number.") + end + yesterday.advance(unit.to_sym => number.to_i) + end + +end; end diff --git a/lib/leap_cli/commands/clean.rb b/lib/leap_cli/commands/clean.rb new file mode 100644 index 00000000..a9afff53 --- /dev/null +++ b/lib/leap_cli/commands/clean.rb @@ -0,0 +1,16 @@ +module LeapCli + module Commands + + desc 'Removes all files generated with the "compile" command.' + command :clean do |c| + c.action do |global_options,options,args| + Dir.glob(path([:hiera, '*'])).each do |file| + remove_file! file + end + remove_file! path(:authorized_keys) + remove_file! path(:known_hosts) + end + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/compile.rb b/lib/leap_cli/commands/compile.rb new file mode 100644 index 00000000..f9079279 --- /dev/null +++ b/lib/leap_cli/commands/compile.rb @@ -0,0 +1,531 @@ +require 'socket' + +module LeapCli + module Commands + + desc "Compile generated files." + command [:compile, :c] do |c| + c.desc 'Compiles node configuration files into hiera files used for deployment.' + c.arg_name 'ENVIRONMENT', :optional => true + c.command :all do |all| + all.action do |global_options,options,args| + environment = args.first + compile_command(environment) + end + end + + c.desc "Prints a DNS zone file for your provider." + c.command :zone do |zone| + zone.action do |global_options, options, args| + compile_command(nil) + compile_zone_file(global_options[:yes] || global_options[:force]) + end + end + + c.desc "Print entries suitable for an /etc/hosts file, useful for testing your provider." + c.command :hosts do |hosts| + hosts.action do |global_options, options, args| + compile_command(nil) + compile_hosts_file + end + end + + c.desc "Compile provider.json bootstrap files for your provider." + c.command 'provider.json' do |provider| + provider.action do |global_options, options, args| + compile_command(nil) + compile_provider_json + end + end + + c.desc "Prints a list of firewall rules. These rules are already "+ + "implemented on each node, but you might want the list of all "+ + "rules in case you also have a restrictive network firewall." + c.command :firewall do |zone| + zone.action do |global_options, options, args| + compile_command(nil) + compile_firewall + end + end + + c.default_command :all + end + + protected + + def compile_command(environment) + if !LeapCli.leapfile.environment.nil? && !environment.nil? && environment != LeapCli.leapfile.environment + bail! "You cannot specify an ENVIRONMENT argument while the environment is pinned." + end + if environment + if manager.environment_names.include?(environment) + compile_hiera_files(manager.filter([environment]), false) + else + bail! "There is no environment named `#{environment}`." + end + else + clean_export = LeapCli.leapfile.environment.nil? + compile_hiera_files(manager.filter, clean_export) + end + if file_exists?(:static_web_readme) + compile_provider_json(environment) + end + end + + # + # a "clean" export of secrets will also remove keys that are no longer used, + # but this should not be done if we are not examining all possible nodes. + # + def compile_hiera_files(nodes, clean_export) + update_certificates(nodes) # \ must come first so that output will + update_compiled_ssh_configs # / get included in compiled hiera files. + sanity_check(nodes) + manager.export_nodes(nodes) + manager.export_secrets(clean_export) + end + + def update_compiled_ssh_configs + generate_monitor_ssh_keys + update_authorized_keys + update_known_hosts + end + + def sanity_check(nodes) + # confirm that every node has a unique ip address + ips = {} + nodes.pick_fields('ip_address').each do |name, ip_address| + if ips.key?(ip_address) + bail! { + log(:fatal_error, "Every node must have its own IP address.") { + log "Nodes `#{name}` and `#{ips[ip_address]}` are both configured with `#{ip_address}`." + } + } + else + ips[ip_address] = name + end + end + # confirm that the IP address of this machine is not also used for a node. + Socket.ip_address_list.each do |addrinfo| + if !addrinfo.ipv4_private? && ips.key?(addrinfo.ip_address) + ip = addrinfo.ip_address + name = ips[ip] + bail! { + log(:fatal_error, "Something is very wrong. The `leap` command must only be run on your sysadmin machine, not on a provider node.") { + log "This machine has the same IP address (#{ip}) as node `#{name}`." + } + } + end + end + end + + ## + ## SSH + ## + + # + # generates a ssh key pair that is used only by remote monitors + # to connect to nodes and run certain allowed commands. + # + # every node has the public monitor key added to their authorized + # keys, and every monitor node has a copy of the private monitor key. + # + def generate_monitor_ssh_keys + priv_key_file = path(:monitor_priv_key) + pub_key_file = path(:monitor_pub_key) + unless file_exists?(priv_key_file, pub_key_file) + ensure_dir(File.dirname(priv_key_file)) + ensure_dir(File.dirname(pub_key_file)) + cmd = %(ssh-keygen -N '' -C 'monitor' -t rsa -b 4096 -f '%s') % priv_key_file + assert_run! cmd + if file_exists?(priv_key_file, pub_key_file) + log :created, priv_key_file + log :created, pub_key_file + else + log :failed, 'to create monitor ssh keys' + end + end + end + + # + # Compiles the authorized keys file, which gets installed on every during init. + # Afterwards, puppet installs an authorized keys file that is generated differently + # (see authorized_keys() in macros.rb) + # + def update_authorized_keys + buffer = StringIO.new + keys = Dir.glob(path([:user_ssh, '*'])) + if keys.empty? + bail! "You must have at least one public SSH user key configured in order to proceed. See `leap help add-user`." + end + if file_exists?(path(:monitor_pub_key)) + keys << path(:monitor_pub_key) + end + keys.sort.each do |keyfile| + ssh_type, ssh_key = File.read(keyfile).strip.split(" ") + buffer << ssh_type + buffer << " " + buffer << ssh_key + buffer << " " + buffer << Path.relative_path(keyfile) + buffer << "\n" + end + write_file!(:authorized_keys, buffer.string) + end + + # + # generates the known_hosts file. + # + # we do a 'late' binding on the hostnames and ip part of the ssh pub key record in order to allow + # for the possibility that the hostnames or ip has changed in the node configuration. + # + def update_known_hosts + buffer = StringIO.new + buffer << "#\n" + buffer << "# This file is automatically generated by the command `leap`. You should NOT modify this file.\n" + buffer << "# Instead, rerun `leap node init` on whatever node is causing SSH problems.\n" + buffer << "#\n" + manager.nodes.keys.sort.each do |node_name| + node = manager.nodes[node_name] + hostnames = [node.name, node.domain.internal, node.domain.full, node.ip_address].join(',') + pub_key = read_file([:node_ssh_pub_key,node.name]) + if pub_key + buffer << [hostnames, pub_key].join(' ') + buffer << "\n" + end + end + write_file!(:known_hosts, buffer.string) + end + + ## + ## provider.json + ## + + # + # generates static provider.json files that can put into place + # (e.g. https://domain/provider.json) for the cases where the + # webapp domain does not match the provider's domain. + # + def compile_provider_json(environments=nil) + webapp_nodes = manager.nodes[:services => 'webapp'] + write_file!(:static_web_readme, STATIC_WEB_README) + environments ||= manager.environment_names + environments.each do |env| + node = webapp_nodes[:environment => env].values.first + if node + env ||= 'default' + write_file!( + [:static_web_provider_json, env], + node['definition_files']['provider'] + ) + write_file!( + [:static_web_htaccess, env], + HTACCESS_FILE % {:min_version => manager.env(env).provider.client_version['min']} + ) + end + end + end + + HTACCESS_FILE = %[ +<Files provider.json> + Header set X-Minimum-Client-Version %{min_version} +</Files> +] + + STATIC_WEB_README = %[ +This directory contains statically rendered copies of the `provider.json` file +used by the client to "bootstrap" configure itself for use with your service +provider. + +There is a separate provider.json file for each environment, although you +should only need 'production/provider.json' or, if you have no environments +configured, 'default/provider.json'. + +To clarify, this is the public `provider.json` file used by the client, not the +`provider.json` file that is used to configure the provider. + +The provider.json file must be available at `https://domain/provider.json` +(unless this provider is included in the list of providers which are pre- +seeded in client). + +This provider.json file can be served correctly in one of three ways: + +(1) If the property webapp.domain is not configured, then the web app will be + installed at https://domain/ and it will handle serving the provider.json file. + +(2) If one or more nodes have the 'static' service configured for the provider's + domain, then these 'static' nodes will correctly serve provider.json. + +(3) Otherwise, you must copy the provider.json file to your web + server and make it available at '/provider.json'. The example htaccess + file shows what header options should be sent by the web server + with the response. + +This directory is needed for method (3), but not for methods (1) or (2). + +This directory has been created by the command `leap compile provider.json`. +Once created, it will be kept up to date everytime you compile. You may safely +remove this directory if you don't use it. +] + + ## + ## + ## ZONE FILE + ## + + def relative_hostname(fqdn, provider) + @domain_regexp ||= /\.?#{Regexp.escape(provider.domain)}$/ + fqdn.sub(@domain_regexp, '') + end + + # + # serial is any number less than 2^32 (4294967296) + # + def compile_zone_file(force=false) + # note: we use the default provider for all nodes, because we use it + # to generate hostnames that are relative to the default domain. + provider = manager.env('default').provider + hosts_seen = {} + lines = [] + + # + # header + # + lines << ZONE_HEADER % { + :domain => provider.domain, + :ns => provider.domain, + :contact => provider.contacts.default.first.sub('@','.'), + :serial => generate_zone_serial + } + + # + # common records + # + lines << ORIGIN_HEADER + # 'A' records for primary domain + manager.nodes[:environment => '!local'].each_node do |node| + if node.dns['aliases'] && node.dns.aliases.include?(provider.domain) + lines << ["@", "IN A #{node.ip_address}"] + end + end + + # NS records + if provider['dns'] && provider.dns['nameservers'] + unless provider.dns.nameservers.is_a?(Array) + # TODO: remove me once we have JSON schema working + bail! {log :error, 'dns.nameservers must be an array' } + end + provider.dns.nameservers.each do |ns| + lines << ["@", "IN NS #{ns}."] + end + elsif !force + log :warning, "Property dns.nameservers is not configured in provider.json." do + log "This will produce a zone file without any NS records." + log "Use --force to skip this warning." + end + return unless agree("Continue? ") + end + + # environment records + manager.environment_names.each do |env| + next if env == 'local' + nodes = manager.nodes[:environment => env] + next unless nodes.any? + spf = nil + dkim = nil + lines << ENV_HEADER % (env.nil? ? 'default' : env) + nodes.each_node do |node| + if node.dns.public + lines << [relative_hostname(node.domain.full, provider), "IN A #{node.ip_address}"] + end + if node.dns['aliases'] + node.dns.aliases.each do |host_alias| + if host_alias != node.domain.full && host_alias != provider.domain + lines << [relative_hostname(host_alias, provider), "IN A #{node.ip_address}"] + end + end + end + if node.services.include? 'mx' + mx_domain = relative_hostname(node.domain.full_suffix, provider) + lines << [mx_domain, "IN MX 10 #{relative_hostname(node.domain.full, provider)}"] + spf ||= [mx_domain, spf_record(node)] + dkim ||= dkim_record(node, provider) + end + end + lines << spf if spf + lines << dkim if dkim + end + + # print the lines + max_width = lines.inject(0) {|max, line| line.is_a?(Array) ? [max, line[0].length].max : max} + max_width = [max_width, 24].min + lines.each do |host, line| + if line.nil? + puts(host) + else + host = '@' if host == '' + puts("%-#{max_width}s %s" % [host, line]) + end + end + end + + # + # outputs entries suitable for an /etc/hosts file + # + def compile_hosts_file + manager.environment_names.each do |env| + nodes = manager.nodes[:environment => env] + next unless nodes.any? + puts + puts "## environment '#{env || 'default'}'" + nodes.each do |name, node| + puts "%s %s" % [ + node.ip_address, + [name, node.get('domain.full'), node.get('dns.aliases')].compact.join(' ') + ] + end + end + end + + private + + # + # allow mail from any mx node, plus the webapp nodes. + # + # TODO: ipv6 + # + def spf_record(node) + ips = node.nodes_like_me['services' => 'webapp'].values.collect {|n| + "ip4:" + n.ip_address + } + # TXT strings may not be longer than 255 characters, although + # you can chain multiple strings together. + strings = "v=spf1 MX #{ips.join(' ')} -all".scan(/.{1,255}/).join('" "') + %(IN TXT "#{strings}") + end + + # + # for example: + # + # selector._domainkey IN TXT "v=DKIM1;h=sha256;k=rsa;s=email;p=MIGfMA0GCSq...GSIb3DQ" + # + # specification: http://dkim.org/specs/rfc4871-dkimbase.html#rfc.section.7.4 + # + def dkim_record(node, provider) + # PEM encoded public key (base64), without the ---PUBLIC KEY--- armor parts. + assert_files_exist! :dkim_pub_key + dkim_pub_key = Path.named_path(:dkim_pub_key) + public_key = File.readlines(dkim_pub_key).grep(/^[^\-]+/).join + + host = relative_hostname( + node.mx.dkim.selector + "._domainkey." + node.domain.full_suffix, + provider) + + attrs = [ + "v=DKIM1", + "h=sha256", + "k=rsa", + "s=email", + "p=" + public_key + ] + + return [host, "IN TXT " + txt_wrap(attrs.join(';'))] + end + + # + # DNS TXT records cannot be longer than 255 characters. + # + # However, multiple responses will be concatenated together. + # It looks like this: + # + # IN TXT "v=spf1 .... first" "second string..." + # + def txt_wrap(str) + '"' + str.scan(/.{1,255}/).join('" "') + '"' + end + + # + # For zone serial number, we want something that will be + # different each time you deploy but also will be greater + # than any prior likely serial that was prefixed by the + # year, such as 2016040600. + # + # so, we use time_t of right now, modified with first + # digit incremented by one. + # + # this will work until Time.at(2**32 - 1_000_000_000) + # aka 2074-05-31 04:41:36 UTC. + # + def generate_zone_serial + Time.now.utc.to_i + 1_000_000_000 + end + + ENV_HEADER = %[ +;; +;; ENVIRONMENT %s +;; + +] + + ZONE_HEADER = %[ +;; +;; BIND data file for %{domain} +;; + +$TTL 600 +$ORIGIN %{domain}. + +@ IN SOA %{ns}. %{contact}. ( + %{serial} ; serial + 7200 ; refresh ( 24 hours) + 3600 ; retry ( 2 hours) + 1209600 ; expire (1000 hours) + 600 ) ; minimum ( 2 days) +; +] + + ORIGIN_HEADER = %[ +;; +;; ZONE ORIGIN +;; + +] + + ## + ## FIREWALL + ## + + public + + def compile_firewall + manager.nodes.each_node(&:evaluate) + + rules = [["ALLOW TO", "PORTS", "ALLOW FROM"]] + manager.nodes[:environment => '!local'].values.each do |node| + next unless node['firewall'] + node.firewall.each do |name, rule| + if rule.is_a? Hash + rules << add_rule(rule) + elsif rule.is_a? Array + rule.each do |r| + rules << add_rule(r) + end + end + end + end + + max_to = rules.inject(0) {|max, r| [max, r[0].length].max} + max_port = rules.inject(0) {|max, r| [max, r[1].length].max} + max_from = rules.inject(0) {|max, r| [max, r[2].length].max} + rules.each do |rule| + puts "%-#{max_to}s %-#{max_port}s %-#{max_from}s" % rule + end + end + + private + + def add_rule(rule) + [rule["to"], [rule["port"]].compact.join(','), rule["from"]] + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/db.rb b/lib/leap_cli/commands/db.rb new file mode 100644 index 00000000..5307ac4d --- /dev/null +++ b/lib/leap_cli/commands/db.rb @@ -0,0 +1,86 @@ +module LeapCli; module Commands + + desc 'Database commands.' + command :db do |db| + db.desc 'Destroy one or more databases. If present, limit to FILTER nodes. For example `leap db destroy --db sessions,tokens testing`.' + db.arg_name 'FILTER', :optional => true + db.command :destroy do |destroy| + destroy.flag :db, :arg_name => "DATABASES", :desc => 'Comma separated list of databases to destroy (no space). Use "--db all" to destroy all databases.', :optional => true + destroy.flag :user, :arg_name => "USERS", :desc => 'Comma separated list of usernames. The storage databases for these user(s) will be destroyed.', :optional => true + destroy.action do |global_options,options,args| + dbs = (options[:db]||"").split(',') + users = (options[:user]||"").split(',') + if dbs.empty? && users.empty? + bail!('Either --db or --user is required.') + end + nodes = manager.filter(args) + if nodes.any? + nodes = nodes[:services => 'couchdb'] + end + unless nodes.any? + bail! 'No db nodes selected.' + end + if users.any? + unless global_options[:yes] + say 'You are about to permanently destroy user databases for [%s] for nodes [%s].' % [users.join(', '), nodes.keys.join(', ')] + bail! unless agree("Continue? ") + end + destroy_user_dbs(nodes, users) + elsif dbs.any? + unless global_options[:yes] + if dbs.include?('all') + say 'You are about to permanently destroy all database data for nodes [%s].' % nodes.keys.join(', ') + else + say 'You are about to permanently destroy databases [%s] for nodes [%s].' % [dbs.join(', '), nodes.keys.join(', ')] + end + bail! unless agree("Continue? ") + end + if dbs.include?('all') + destroy_all_dbs(nodes) + else + destroy_dbs(nodes, dbs) + end + say 'You must run `leap deploy` in order to create the databases again.' + end + end + end + end + + private + + def destroy_all_dbs(nodes) + ssh_connect(nodes) do |ssh| + ssh.run('/etc/init.d/bigcouch stop && test ! -z "$(ls /opt/bigcouch/var/lib/ 2> /dev/null)" && rm -r /opt/bigcouch/var/lib/* && echo "All DBs destroyed" || echo "DBs already destroyed"') + end + end + + def destroy_dbs(nodes, dbs) + nodes.each_node do |node| + ssh_connect(node) do |ssh| + dbs.each do |db| + ssh.run(DESTROY_DB_COMMAND % {:db => db}) + end + end + end + end + + def destroy_user_dbs(nodes, users) + nodes.each_node do |node| + ssh_connect(node) do |ssh| + users.each do |user| + ssh.run(DESTROY_USER_DB_COMMAND % {:user => user}) + end + end + end + end + + DESTROY_DB_COMMAND = %{ +if [ 200 = `curl -ns -w "%%{http_code}" -X GET "127.0.0.1:5984/%{db}" -o /dev/null` ]; then + echo "Result from DELETE /%{db}:" `curl -ns -X DELETE "127.0.0.1:5984/%{db}"`; +else + echo "Skipping db '%{db}': it does not exist or has already been deleted."; +fi +} + + DESTROY_USER_DB_COMMAND = %{/srv/leap/couchdb/scripts/destroy-user-db --username %{user}} +end; end diff --git a/lib/leap_cli/commands/deploy.rb b/lib/leap_cli/commands/deploy.rb new file mode 100644 index 00000000..9dd190ab --- /dev/null +++ b/lib/leap_cli/commands/deploy.rb @@ -0,0 +1,374 @@ +require 'etc' + +module LeapCli + module Commands + + desc 'Apply recipes to a node or set of nodes.' + long_desc 'The FILTER can be the name of a node, service, or tag.' + arg_name 'FILTER' + command [:deploy, :d] do |c| + + c.switch :fast, :desc => 'Makes the deploy command faster by skipping some slow steps. A "fast" deploy can be used safely if you recently completed a normal deploy.', + :negatable => false + + c.switch :sync, :desc => "Sync files, but don't actually apply recipes.", :negatable => false + + c.switch :force, :desc => 'Deploy even if there is a lockfile.', :negatable => false + + c.switch :downgrade, :desc => 'Allows deploy to run with an older platform version.', :negatable => false + + c.switch :dev, :desc => "Development mode: don't run 'git submodule update' before deploy.", :negatable => false + + c.flag :tags, :desc => 'Specify tags to pass through to puppet (overriding the default).', + :arg_name => 'TAG[,TAG]' + + c.flag :port, :desc => 'Override the default SSH port.', + :arg_name => 'PORT' + + c.flag :ip, :desc => 'Override the default SSH IP address.', + :arg_name => 'IPADDRESS' + + c.action do |global,options,args| + + if options[:dev] != true + init_submodules + end + + nodes = manager.filter!(args, :disabled => false) + if nodes.size > 1 + say "Deploying to these nodes: #{nodes.keys.join(', ')}" + if !global[:yes] && !agree("Continue? ") + quit! "OK. Bye." + end + end + + environments = nodes.field('environment').uniq + if environments.empty? + environments = [nil] + end + environments.each do |env| + check_platform_pinning(env, global) + end + + # compile hiera files for all the nodes in every environment that is + # being deployed and only those environments. + compile_hiera_files(manager.filter(environments), false) + + ssh_connect(nodes, connect_options(options)) do |ssh| + ssh.leap.log :checking, 'node' do + ssh.leap.check_for_no_deploy + ssh.leap.assert_initialized + end + ssh.leap.log :synching, "configuration files" do + sync_hiera_config(ssh) + sync_support_files(ssh) + end + ssh.leap.log :synching, "puppet manifests" do + sync_puppet_files(ssh) + end + unless options[:sync] + ssh.leap.log :applying, "puppet" do + ssh.puppet.apply(:verbosity => [LeapCli.log_level,5].min, + :tags => tags(options), + :force => options[:force], + :info => deploy_info, + :downgrade => options[:downgrade] + ) + end + end + end + if !Util.exit_status.nil? && Util.exit_status != 0 + log :warning, "puppet did not finish successfully." + end + end + end + + desc 'Display recent deployment history for a set of nodes.' + long_desc 'The FILTER can be the name of a node, service, or tag.' + arg_name 'FILTER' + command [:history, :h] do |c| + c.flag :port, :desc => 'Override the default SSH port.', + :arg_name => 'PORT' + c.flag :ip, :desc => 'Override the default SSH IP address.', + :arg_name => 'IPADDRESS' + c.switch :last, :desc => 'Show last deploy only', + :negatable => false + c.action do |global,options,args| + if options[:last] == true + lines = 1 + else + lines = 10 + end + nodes = manager.filter!(args) + ssh_connect(nodes, connect_options(options)) do |ssh| + ssh.leap.history(lines) + end + end + end + + private + + def forcible_prompt(forced, msg, prompt) + say(msg) + if forced + log :warning, "continuing anyway because of --force" + else + say "hint: use --force to skip this prompt." + quit!("OK. Bye.") unless agree(prompt) + end + end + + # + # The currently activated provider.json could have loaded some pinning + # information for the platform. If this is the case, refuse to deploy + # if there is a mismatch. + # + # For example: + # + # "platform": { + # "branch": "develop" + # "version": "1.0..99" + # "commit": "e1d6280e0a8c565b7fb1a4ed3969ea6fea31a5e2..HEAD" + # } + # + def check_platform_pinning(environment, global_options) + provider = manager.env(environment).provider + return unless provider['platform'] + + if environment.nil? || environment == 'default' + provider_json = 'provider.json' + else + provider_json = 'provider.' + environment + '.json' + end + + # can we have json schema verification already? + unless provider.platform.is_a? Hash + bail!('`platform` attribute in #{provider_json} must be a hash (was %s).' % provider.platform.inspect) + end + + # check version + if provider.platform['version'] + if !Leap::Platform.version_in_range?(provider.platform.version) + forcible_prompt( + global_options[:force], + "The platform is pinned to a version range of '#{provider.platform.version}' "+ + "by the `platform.version` property in #{provider_json}, but the platform "+ + "(#{Path.platform}) has version #{Leap::Platform.version}.", + "Do you really want to deploy from the wrong version? " + ) + end + end + + # check branch + if provider.platform['branch'] + if !is_git_directory?(Path.platform) + forcible_prompt( + global_options[:force], + "The platform is pinned to a particular branch by the `platform.branch` property "+ + "in #{provider_json}, but the platform directory (#{Path.platform}) is not a git repository.", + "Do you really want to deploy anyway? " + ) + end + unless provider.platform.branch == current_git_branch(Path.platform) + forcible_prompt( + global_options[:force], + "The platform is pinned to branch '#{provider.platform.branch}' by the `platform.branch` property "+ + "in #{provider_json}, but the current branch is '#{current_git_branch(Path.platform)}' " + + "(for directory '#{Path.platform}')", + "Do you really want to deploy from the wrong branch? " + ) + end + end + + # check commit + if provider.platform['commit'] + if !is_git_directory?(Path.platform) + forcible_prompt( + global_options[:force], + "The platform is pinned to a particular commit range by the `platform.commit` property "+ + "in #{provider_json}, but the platform directory (#{Path.platform}) is not a git repository.", + "Do you really want to deploy anyway? " + ) + end + current_commit = current_git_commit(Path.platform) + Dir.chdir(Path.platform) do + commit_range = assert_run!("git log --pretty='format:%H' '#{provider.platform.commit}'", + "The platform is pinned to a particular commit range by the `platform.commit` property "+ + "in #{provider_json}, but git was not able to find commits in the range specified "+ + "(#{provider.platform.commit}).") + commit_range = commit_range.split("\n") + if !commit_range.include?(current_commit) && + provider.platform.commit.split('..').first != current_commit + forcible_prompt( + global_options[:force], + "The platform is pinned via the `platform.commit` property in #{provider_json} " + + "to a commit in the range #{provider.platform.commit}, but the current HEAD " + + "(#{current_commit}) is not in that range.", + "Do you really want to deploy from the wrong commit? " + ) + end + end + end + end + + def sync_hiera_config(ssh) + ssh.rsync.update do |server| + node = manager.node(server.host) + hiera_file = Path.relative_path([:hiera, node.name]) + ssh.leap.log hiera_file + ' -> ' + node.name + ':' + Leap::Platform.hiera_path + { + :source => hiera_file, + :dest => Leap::Platform.hiera_path, + :flags => "-rltp --chmod=u+rX,go-rwx" + } + end + end + + # + # sync various support files. + # + def sync_support_files(ssh) + dest_dir = Leap::Platform.files_dir + custom_files = build_custom_file_list + ssh.rsync.update do |server| + node = manager.node(server.host) + files_to_sync = node.file_paths.collect {|path| Path.relative_path(path, Path.provider) } + files_to_sync += custom_files + if files_to_sync.any? + ssh.leap.log(files_to_sync.join(', ') + ' -> ' + node.name + ':' + dest_dir) + { + :chdir => Path.named_path(:files_dir), + :source => ".", + :dest => dest_dir, + :excludes => "*", + :includes => calculate_includes_from_files(files_to_sync, '/files'), + :flags => "-rltp --chmod=u+rX,go-rwx --relative --delete --delete-excluded --copy-links" + } + else + nil + end + end + end + + def sync_puppet_files(ssh) + ssh.rsync.update do |server| + ssh.leap.log(Path.platform + '/[bin,tests,puppet] -> ' + server.host + ':' + Leap::Platform.leap_dir) + { + :dest => Leap::Platform.leap_dir, + :source => '.', + :chdir => Path.platform, + :excludes => '*', + :includes => ['/bin', '/bin/**', '/puppet', '/puppet/**', '/tests', '/tests/**'], + :flags => "-rlt --relative --delete --copy-links" + } + end + end + + # + # ensure submodules are up to date, if the platform is a git + # repository. + # + def init_submodules + return unless is_git_directory?(Path.platform) + Dir.chdir Path.platform do + assert_run! "git submodule sync" + statuses = assert_run! "git submodule status" + statuses.strip.split("\n").each do |status_line| + if status_line =~ /^[\+-]/ + submodule = status_line.split(' ')[1] + log "Updating submodule #{submodule}" + assert_run! "git submodule update --init #{submodule}" + end + end + end + end + + # + # converts an array of file paths into an array + # suitable for --include of rsync + # + # if set, `prefix` is stripped off. + # + def calculate_includes_from_files(files, prefix=nil) + return nil unless files and files.any? + + # prepend '/' (kind of like ^ for rsync) + includes = files.collect {|file| file =~ /^\// ? file : '/' + file } + + # include all sub files of specified directories + includes.size.times do |i| + if includes[i] =~ /\/$/ + includes << includes[i] + '**' + end + end + + # include all parent directories (required because of --exclude '*') + includes.size.times do |i| + path = File.dirname(includes[i]) + while(path != '/') + includes << path unless includes.include?(path) + path = File.dirname(path) + end + end + + if prefix + includes.map! {|path| path.sub(/^#{Regexp.escape(prefix)}\//, '/')} + end + + return includes + end + + def tags(options) + if options[:tags] + tags = options[:tags].split(',') + else + tags = Leap::Platform.default_puppet_tags.dup + end + tags << 'leap_slow' unless options[:fast] + tags.join(',') + end + + # + # a provider might have various customization files that should be sync'ed to the server. + # this method builds that list of files to sync. + # + def build_custom_file_list + custom_files = [] + Leap::Platform.paths.keys.grep(/^custom_/).each do |path| + if file_exists?(path) + relative_path = Path.relative_path(path, Path.provider) + if dir_exists?(path) + custom_files << relative_path + '/' # rsync needs trailing slash + else + custom_files << relative_path + end + end + end + return custom_files + end + + def deploy_info + info = [] + info << "user: %s" % Etc.getpwuid(Process.euid).name + if is_git_directory?(Path.platform) && current_git_branch(Path.platform) != 'master' + info << "platform: %s (%s %s)" % [ + Leap::Platform.version, + current_git_branch(Path.platform), + current_git_commit(Path.platform)[0..4] + ] + else + info << "platform: %s" % Leap::Platform.version + end + if is_git_directory?(LEAP_CLI_BASE_DIR) + info << "leap_cli: %s (%s %s)" % [ + LeapCli::VERSION, + current_git_branch(LEAP_CLI_BASE_DIR), + current_git_commit(LEAP_CLI_BASE_DIR)[0..4] + ] + else + info << "leap_cli: %s" % LeapCli::VERSION + end + info.join(', ') + end + end +end diff --git a/lib/leap_cli/commands/env.rb b/lib/leap_cli/commands/env.rb new file mode 100644 index 00000000..80be2174 --- /dev/null +++ b/lib/leap_cli/commands/env.rb @@ -0,0 +1,76 @@ +module LeapCli + module Commands + + desc "Manipulate and query environment information." + long_desc "The 'environment' node property can be used to isolate sets of nodes into entirely separate environments. "+ + "A node in one environment will never interact with a node from another environment. "+ + "Environment pinning works by modifying your ~/.leaprc file and is dependent on the "+ + "absolute file path of your provider directory (pins don't apply if you move the directory)" + command [:env, :e] do |c| + c.desc "List the available environments. The pinned environment, if any, will be marked with '*'. Will also set the pin if run with an environment argument." + c.arg_name 'ENVIRONMENT', :optional => true + c.command :ls do |ls| + ls.action do |global_options, options, args| + environment = get_env_from_args(args) + if environment + pin(environment) + LeapCli.leapfile.load + end + print_envs + end + end + + c.desc 'Pin the environment to ENVIRONMENT. All subsequent commands will only apply to nodes in this environment.' + c.arg_name 'ENVIRONMENT' + c.command :pin do |pin| + pin.action do |global_options,options,args| + environment = get_env_from_args(args) + if environment + pin(environment) + else + bail! "There is no environment `#{environment}`" + end + end + end + + c.desc "Unpin the environment. All subsequent commands will apply to all nodes." + c.command :unpin do |unpin| + unpin.action do |global_options, options, args| + LeapCli.leapfile.unset('environment') + log 0, :saved, "~/.leaprc, removing environment property." + end + end + + c.default_command :ls + end + + protected + + def get_env_from_args(args) + environment = args.first + if environment == 'default' || (environment && manager.environment_names.include?(environment)) + return environment + else + return nil + end + end + + def pin(environment) + LeapCli.leapfile.set('environment', environment) + log 0, :saved, "~/.leaprc with environment set to #{environment}." + end + + def print_envs + envs = ["default"] + manager.environment_names.compact.sort + envs.each do |env| + if env + if LeapCli.leapfile.environment == env + puts "* #{env}" + else + puts " #{env}" + end + end + end + end + end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/facts.rb b/lib/leap_cli/commands/facts.rb new file mode 100644 index 00000000..11329ccc --- /dev/null +++ b/lib/leap_cli/commands/facts.rb @@ -0,0 +1,100 @@ +# +# Gather facter facts +# + +module LeapCli; module Commands + + desc 'Gather information on nodes.' + command :facts do |facts| + facts.desc 'Query servers to update facts.json.' + facts.long_desc "Queries every node included in FILTER and saves the important information to facts.json" + facts.arg_name 'FILTER' + facts.command :update do |update| + update.action do |global_options,options,args| + update_facts(global_options, options, args) + end + end + end + + protected + + def facter_cmd + 'facter --json ' + Leap::Platform.facts.join(' ') + end + + def remove_node_facts(name) + if file_exists?(:facts) + update_facts_file({name => nil}) + end + end + + def update_node_facts(name, facts) + update_facts_file({name => facts}) + end + + def rename_node_facts(old_name, new_name) + if file_exists?(:facts) + facts = JSON.parse(read_file(:facts) || {}) + facts[new_name] = facts[old_name] + facts[old_name] = nil + update_facts_file(facts, true) + end + end + + # + # if overwrite = true, then ignore existing facts.json. + # + def update_facts_file(new_facts, overwrite=false) + replace_file!(:facts) do |content| + if overwrite || content.nil? || content.empty? + old_facts = {} + else + old_facts = manager.facts + end + facts = old_facts.merge(new_facts) + facts.each do |name, value| + if value.is_a? String + if value == "" + value = nil + else + value = JSON.parse(value) rescue JSON::ParserError + end + end + if value.is_a? Hash + value.delete_if {|key,v| v.nil?} + end + facts[name] = value + end + facts.delete_if do |name, value| + value.nil? || value.empty? + end + if facts.empty? + "{}\n" + else + JSON.sorted_generate(facts) + "\n" + end + end + end + + private + + def update_facts(global_options, options, args) + nodes = manager.filter(args, :local => false, :disabled => false) + new_facts = {} + ssh_connect(nodes) do |ssh| + ssh.leap.run_with_progress(facter_cmd) do |response| + node = manager.node(response[:host]) + if node + new_facts[node.name] = response[:data].strip + else + log :warning, 'Could not find node for hostname %s' % response[:host] + end + end + end + # only overwrite the entire facts file if and only if we are gathering facts + # for all nodes in all environments. + overwrite_existing = args.empty? && LeapCli.leapfile.environment.nil? + update_facts_file(new_facts, overwrite_existing) + end + +end; end
\ No newline at end of file diff --git a/lib/leap_cli/commands/info.rb b/lib/leap_cli/commands/info.rb new file mode 100644 index 00000000..52225a94 --- /dev/null +++ b/lib/leap_cli/commands/info.rb @@ -0,0 +1,15 @@ +module LeapCli; module Commands + + desc 'Prints information regarding facts, history, and running processes for a node or nodes.' + long_desc 'The FILTER can be the name of a node, service, or tag.' + arg_name 'FILTER' + command [:info] do |c| + c.action do |global,options,args| + nodes = manager.filter!(args) + ssh_connect(nodes, connect_options(options)) do |ssh| + ssh.leap.debug + end + end + end + +end; end diff --git a/lib/leap_cli/commands/inspect.rb b/lib/leap_cli/commands/inspect.rb new file mode 100644 index 00000000..20654fa7 --- /dev/null +++ b/lib/leap_cli/commands/inspect.rb @@ -0,0 +1,144 @@ +module LeapCli; module Commands + + desc 'Prints details about a file. Alternately, the argument FILE can be the name of a node, service or tag.' + arg_name 'FILE' + command [:inspect, :i] do |c| + c.switch 'base', :desc => 'Inspect the FILE from the provider_base (i.e. without local inheritance).', :negatable => false + c.action do |global_options,options,args| + object = args.first + assert! object, 'A file path or node/service/tag name is required' + method = inspection_method(object) + if method && defined?(method) + self.send(method, object, options) + else + log "Sorry, I don't know how to inspect that." + end + end + end + + private + + FTYPE_MAP = { + "PEM certificate" => :inspect_x509_cert, + "PEM RSA private key" => :inspect_x509_key, + "OpenSSH RSA public key" => :inspect_ssh_pub_key, + "PEM certificate request" => :inspect_x509_csr + } + + def inspection_method(object) + if File.exists?(object) + ftype = `file #{object}`.split(':').last.strip + log 2, "file is of type '#{ftype}'" + if FTYPE_MAP[ftype] + FTYPE_MAP[ftype] + elsif File.extname(object) == ".json" + full_path = File.expand_path(object, Dir.pwd) + if path_match?(:node_config, full_path) + :inspect_node + elsif path_match?(:service_config, full_path) + :inspect_service + elsif path_match?(:tag_config, full_path) + :inspect_tag + elsif path_match?(:provider_config, full_path) || path_match?(:provider_env_config, full_path) + :inspect_provider + elsif path_match?(:common_config, full_path) + :inspect_common + else + nil + end + end + elsif manager.nodes[object] + :inspect_node + elsif manager.services[object] + :inspect_service + elsif manager.tags[object] + :inspect_tag + elsif object == "common" + :inspect_common + elsif object == "provider" + :inspect_provider + else + nil + end + end + + # + # inspectors + # + + def inspect_x509_key(file_path, options) + assert_bin! 'openssl' + puts assert_run! 'openssl rsa -in %s -text -check' % file_path + end + + def inspect_x509_cert(file_path, options) + assert_bin! 'openssl' + puts assert_run! 'openssl x509 -in %s -text -noout' % file_path + log 0, :"SHA256 fingerprint", X509.fingerprint("SHA256", file_path) + end + + def inspect_x509_csr(file_path, options) + assert_bin! 'openssl' + puts assert_run! 'openssl req -text -noout -verify -in %s' % file_path + end + + #def inspect_ssh_pub_key(file_path) + #end + + def inspect_node(arg, options) + inspect_json manager.nodes[name(arg)] + end + + def inspect_service(arg, options) + if options[:base] + inspect_json manager.base_services[name(arg)] + else + inspect_json manager.services[name(arg)] + end + end + + def inspect_tag(arg, options) + if options[:base] + inspect_json manager.base_tags[name(arg)] + else + inspect_json manager.tags[name(arg)] + end + end + + def inspect_provider(arg, options) + if options[:base] + inspect_json manager.base_provider + elsif arg =~ /provider\.(.*)\.json/ + inspect_json manager.env($1).provider + else + inspect_json manager.provider + end + end + + def inspect_common(arg, options) + if options[:base] + inspect_json manager.base_common + else + inspect_json manager.common + end + end + + # + # helpers + # + + def name(arg) + File.basename(arg).sub(/\.json$/, '') + end + + def inspect_json(config) + if config + puts JSON.sorted_generate(config) + end + end + + def path_match?(path_symbol, path) + Dir.glob(Path.named_path([path_symbol, '*'])).include?(path) + end + +end; end diff --git a/lib/leap_cli/commands/list.rb b/lib/leap_cli/commands/list.rb new file mode 100644 index 00000000..aa425432 --- /dev/null +++ b/lib/leap_cli/commands/list.rb @@ -0,0 +1,132 @@ +require 'command_line_reporter' + +module LeapCli; module Commands + + desc 'List nodes and their classifications' + long_desc 'Prints out a listing of nodes, services, or tags. ' + + 'If present, the FILTER can be a list of names of nodes, services, or tags. ' + + 'If the name is prefixed with +, this acts like an AND condition. ' + + "For example:\n\n" + + "`leap list node1 node2` matches all nodes named \"node1\" OR \"node2\"\n\n" + + "`leap list openvpn +local` matches all nodes with service \"openvpn\" AND tag \"local\"" + + arg_name 'FILTER', :optional => true + command [:list,:ls] do |c| + c.flag 'print', :desc => 'What attributes to print (optional)' + c.switch 'disabled', :desc => 'Include disabled nodes in the list.', :negatable => false + c.action do |global_options,options,args| + # don't rely on default manager(), because we want to pass custom options to load() + manager = LeapCli::Config::Manager.new + if global_options[:color] + colors = ['cyan', 'white'] + else + colors = [nil, nil] + end + puts + manager.load(:include_disabled => options['disabled'], :continue_on_error => true) + if options['print'] + print_node_properties(manager.filter(args), options['print']) + else + if args.any? + NodeTable.new(manager.filter(args), colors).run + else + environment = LeapCli.leapfile.environment || '_all_' + TagTable.new('SERVICES', manager.env(environment).services, colors).run + TagTable.new('TAGS', manager.env(environment).tags, colors).run + NodeTable.new(manager.filter(), colors).run + end + end + end + end + + private + + def self.print_node_properties(nodes, properties) + properties = properties.split(',') + max_width = nodes.keys.inject(0) {|max,i| [i.size,max].max} + nodes.each_node do |node| + value = properties.collect{|prop| + prop_value = node[prop] + if prop_value.nil? + "null" + elsif prop_value == "" + "empty" + elsif prop_value.is_a? LeapCli::Config::Object + node[prop].dump_json(:format => :compact) # TODO: add option of getting pre-evaluation values. + else + prop_value.to_s + end + }.join(', ') + printf("%#{max_width}s %s\n", node.name, value) + end + puts + end + + class TagTable + include CommandLineReporter + def initialize(heading, tag_list, colors) + @heading = heading + @tag_list = tag_list + @colors = colors + end + def run + tags = @tag_list.keys.select{|tag| tag !~ /^_/}.sort # sorted list of tags, excluding _partials + max_width = [20, (tags+[@heading]).inject(0) {|max,i| [i.size,max].max}].max + table :border => false do + row :color => @colors[0] do + column @heading, :align => 'right', :width => max_width + column "NODES", :width => HighLine::SystemExtensions.terminal_size.first - max_width - 2, :padding => 2 + end + tags.each do |tag| + next if @tag_list[tag].node_list.empty? + row :color => @colors[1] do + column tag + column @tag_list[tag].node_list.keys.sort.join(', ') + end + end + end + vertical_spacing + end + end + + # + # might be handy: HighLine::SystemExtensions.terminal_size.first + # + class NodeTable + include CommandLineReporter + def initialize(node_list, colors) + @node_list = node_list + @colors = colors + end + def run + rows = @node_list.keys.sort.collect do |node_name| + [node_name, @node_list[node_name].services.sort.join(', '), @node_list[node_name].tags.sort.join(', ')] + end + unless rows.any? + puts Paint["no results", :red] + puts + return + end + padding = 2 + max_node_width = [20, (rows.map{|i|i[0]} + ["NODES"] ).inject(0) {|max,i| [i.size,max].max}].max + max_service_width = (rows.map{|i|i[1]} + ["SERVICES"]).inject(0) {|max,i| [i.size+padding+padding,max].max} + max_tag_width = (rows.map{|i|i[2]} + ["TAGS"] ).inject(0) {|max,i| [i.size,max].max} + table :border => false do + row :color => @colors[0] do + column "NODES", :align => 'right', :width => max_node_width + column "SERVICES", :width => max_service_width, :padding => 2 + column "TAGS", :width => max_tag_width + end + rows.each do |r| + row :color => @colors[1] do + column r[0] + column r[1] + column r[2] + end + end + end + vertical_spacing + end + end + +end; end diff --git a/lib/leap_cli/commands/node.rb b/lib/leap_cli/commands/node.rb new file mode 100644 index 00000000..a23661b3 --- /dev/null +++ b/lib/leap_cli/commands/node.rb @@ -0,0 +1,188 @@ +# +# fyi: the `node init` command lives in node_init.rb, +# but all other `node x` commands live here. +# + +autoload :IPAddr, 'ipaddr' + +module LeapCli; module Commands + + ## + ## COMMANDS + ## + + desc 'Node management' + command [:node, :n] do |node| + node.desc 'Create a new configuration file for a node named NAME.' + node.long_desc ["If specified, the optional argument SEED can be used to seed values in the node configuration file.", + "The format is property_name:value.", + "For example: `leap node add web1 ip_address:1.2.3.4 services:webapp`.", + "To set nested properties, property name can contain '.', like so: `leap node add web1 ssh.port:44`", + "Separeate multiple values for a single property with a comma, like so: `leap node add mynode services:webapp,dns`"].join("\n\n") + node.arg_name 'NAME [SEED]' # , :optional => false, :multiple => false + node.command :add do |add| + add.switch :local, :desc => 'Make a local testing node (by automatically assigning the next available local IP address). Local nodes are run as virtual machines on your computer.', :negatable => false + add.action do |global_options,options,args| + # argument sanity checks + name = args.first + assert_valid_node_name!(name, options[:local]) + assert_files_missing! [:node_config, name] + + # create and seed new node + node = Config::Node.new(manager.env) + if options[:local] + node['ip_address'] = pick_next_vagrant_ip_address + end + seed_node_data_from_cmd_line(node, args[1..-1]) + seed_node_data_from_template(node) + validate_ip_address(node) + begin + node['name'] = name + json = node.dump_json(:exclude => ['name']) + write_file!([:node_config, name], json + "\n") + if file_exists? :ca_cert, :ca_key + generate_cert_for_node(manager.reload_node!(node)) + end + rescue LeapCli::ConfigError => exc + remove_node_files(name) + end + end + end + + node.desc 'Renames a node file, and all its related files.' + node.arg_name 'OLD_NAME NEW_NAME' + node.command :mv do |mv| + mv.action do |global_options,options,args| + node = get_node_from_args(args, include_disabled: true) + new_name = args.last + assert_valid_node_name!(new_name, node.vagrant?) + ensure_dir [:node_files_dir, new_name] + Leap::Platform.node_files.each do |path| + rename_file! [path, node.name], [path, new_name] + end + remove_directory! [:node_files_dir, node.name] + rename_node_facts(node.name, new_name) + end + end + + node.desc 'Removes all the files related to the node named NAME.' + node.arg_name 'NAME' #:optional => false #, :multiple => false + node.command :rm do |rm| + rm.action do |global_options,options,args| + node = get_node_from_args(args, include_disabled: true) + remove_node_files(node.name) + if node.vagrant? + vagrant_command("destroy --force", [node.name]) + end + remove_node_facts(node.name) + end + end + end + + ## + ## PUBLIC HELPERS + ## + + def get_node_from_args(args, options={}) + node_name = args.first + node = manager.node(node_name) + if node.nil? && options[:include_disabled] + node = manager.disabled_node(node_name) + end + assert!(node, "Node '#{node_name}' not found.") + node + end + + def seed_node_data_from_cmd_line(node, args) + args.each do |seed| + key, value = seed.split(':', 2) + value = format_seed_value(value) + assert! key =~ /^[0-9a-z\._]+$/, "illegal characters used in property '#{key}'" + if key =~ /\./ + key_parts = key.split('.') + final_key = key_parts.pop + current_object = node + key_parts.each do |key_part| + current_object[key_part] ||= Config::Object.new + current_object = current_object[key_part] + end + current_object[final_key] = value + else + node[key] = value + end + end + end + + # + # load "new node template" information into the `node`, modifying `node`. + # values in the template will not override existing node values. + # + def seed_node_data_from_template(node) + node.inherit_from!(manager.template('common')) + [node['services']].flatten.each do |service| + if service + template = manager.template(service) + if template + node.inherit_from!(template) + end + end + end + end + + def remove_node_files(node_name) + (Leap::Platform.node_files + [:node_files_dir]).each do |path| + remove_file! [path, node_name] + end + end + + # + # conversions: + # + # "x,y,z" => ["x","y","z"] + # + # "22" => 22 + # + # "5.1" => 5.1 + # + def format_seed_value(v) + if v =~ /,/ + v = v.split(',') + v.map! do |i| + i = i.to_i if i.to_i.to_s == i + i = i.to_f if i.to_f.to_s == i + i + end + else + v = v.to_i if v.to_i.to_s == v + v = v.to_f if v.to_f.to_s == v + end + return v + end + + def validate_ip_address(node) + if node['ip_address'] == "REQUIRED" + bail! do + log :error, "ip_address is not set. Specify with `leap node add NAME ip_address:ADDRESS`." + end + end + IPAddr.new(node['ip_address']) + rescue ArgumentError + bail! do + if node['ip_address'] + log :invalid, "ip_address #{node['ip_address'].inspect}" + else + log :missing, "ip_address" + end + end + end + + def assert_valid_node_name!(name, local=false) + assert! name, 'No <node-name> specified.' + if local + assert! name =~ /^[0-9a-z]+$/, "illegal characters used in node name '#{name}' (note: Vagrant does not allow hyphens or underscores)" + else + assert! name =~ /^[0-9a-z-]+$/, "illegal characters used in node name '#{name}' (note: Linux does not allow underscores)" + end + end + +end; end diff --git a/lib/leap_cli/commands/node_init.rb b/lib/leap_cli/commands/node_init.rb new file mode 100644 index 00000000..33f6288d --- /dev/null +++ b/lib/leap_cli/commands/node_init.rb @@ -0,0 +1,169 @@ +# +# Node initialization. +# Most of the fun stuff is in tasks.rb. +# + +module LeapCli; module Commands + + desc 'Node management' + command :node do |node| + node.desc 'Bootstraps a node or nodes, setting up SSH keys and installing prerequisite packages' + node.long_desc "This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, " + + "copying the authorized_keys file, installing packages that are required for deploying, and registering important facts. " + + "Node init must be run before deploying to a server, and the server must be running and available via the network. " + + "This command only needs to be run once, but there is no harm in running it multiple times." + node.arg_name 'FILTER' + node.command :init do |init| + init.switch 'echo', :desc => 'If set, passwords are visible as you type them (default is hidden)', :negatable => false + init.flag :port, :desc => 'Override the default SSH port.', :arg_name => 'PORT' + init.flag :ip, :desc => 'Override the default SSH IP address.', :arg_name => 'IPADDRESS' + + init.action do |global,options,args| + assert! args.any?, 'You must specify a FILTER' + finished = [] + manager.filter!(args).each_node do |node| + is_node_alive(node, options) + save_public_host_key(node, global, options) unless node.vagrant? + update_compiled_ssh_configs + ssh_connect_options = connect_options(options).merge({:bootstrap => true, :echo => options[:echo]}) + ssh_connect(node, ssh_connect_options) do |ssh| + if node.vagrant? + ssh.install_insecure_vagrant_key + end + ssh.install_authorized_keys + ssh.install_prerequisites + unless node.vagrant? + ssh.leap.log(:checking, "SSH host keys") do + ssh.leap.capture(get_ssh_keys_cmd) do |response| + update_local_ssh_host_keys(node, response[:data]) if response[:exitcode] == 0 + end + end + end + ssh.leap.log(:updating, "facts") do + ssh.leap.capture(facter_cmd) do |response| + if response[:exitcode] == 0 + update_node_facts(node.name, response[:data]) + else + log :failed, "to run facter on #{node.name}" + end + end + end + end + finished << node.name + end + log :completed, "initialization of nodes #{finished.join(', ')}" + end + end + end + + private + + ## + ## PRIVATE HELPERS + ## + + def is_node_alive(node, options) + address = options[:ip] || node.ip_address + port = options[:port] || node.ssh.port + log :connecting, "to node #{node.name}" + assert_run! "nc -zw3 #{address} #{port}", + "Failed to reach #{node.name} (address #{address}, port #{port}). You can override the configured IP address and port with --ip or --port." + end + + # + # saves the public ssh host key for node into the provider directory. + # + # see `man sshd` for the format of known_hosts + # + def save_public_host_key(node, global, options) + log :fetching, "public SSH host key for #{node.name}" + address = options[:ip] || node.ip_address + port = options[:port] || node.ssh.port + host_keys = get_public_keys_for_ip(address, port) + pub_key_path = Path.named_path([:node_ssh_pub_key, node.name]) + + if Path.exists?(pub_key_path) + if host_keys.include? SshKey.load(pub_key_path) + log :trusted, "- Public SSH host key for #{node.name} matches previously saved key", :indent => 1 + else + bail! do + log :error, "The public SSH host keys we just fetched for #{node.name} doesn't match what we have saved previously.", :indent => 1 + log "Delete the file #{pub_key_path} if you really want to remove the trusted SSH host key.", :indent => 2 + end + end + else + known_key = host_keys.detect{|k|k.in_known_hosts?(node.name, node.ip_address, node.domain.name)} + if known_key + log :trusted, "- Public SSH host key for #{node.name} is trusted (key found in your ~/.ssh/known_hosts)" + else + public_key = SshKey.pick_best_key(host_keys) + if public_key.nil? + bail!("We got back #{host_keys.size} host keys from #{node.name}, but we can't support any of them.") + else + say(" This is the SSH host key you got back from node \"#{node.name}\"") + say(" Type -- #{public_key.bits} bit #{public_key.type.upcase}") + say(" Fingerprint -- " + public_key.fingerprint) + say(" Public Key -- " + public_key.key) + if !global[:yes] && !agree(" Is this correct? ") + bail! + else + known_key = public_key + end + end + end + puts + write_file! [:node_ssh_pub_key, node.name], known_key.to_s + end + end + + # + # Get the public host keys for a host using ssh-keyscan. + # Return an array of SshKey objects, one for each key. + # + def get_public_keys_for_ip(address, port=22) + assert_bin!('ssh-keyscan') + output = assert_run! "ssh-keyscan -p #{port} #{address}", "Could not get the public host key from #{address}:#{port}. Maybe sshd is not running?" + if output.empty? + bail! :failed, "ssh-keyscan returned empty output." + end + + if output =~ /No route to host/ + bail! :failed, 'ssh-keyscan: no route to %s' % address + else + keys = SshKey.parse_keys(output) + if keys.empty? + bail! "ssh-keyscan got zero host keys back (that we understand)! Output was: #{output}" + else + return keys + end + end + end + + # run on the server to generate a string suitable for passing to SshKey.parse_keys() + def get_ssh_keys_cmd + "/bin/grep ^HostKey /etc/ssh/sshd_config | /usr/bin/awk '{print $2 \".pub\"}' | /usr/bin/xargs /bin/cat" + end + + # + # Sometimes the ssh host keys on the server will be better than what we have + # stored locally. In these cases, ask the user if they want to upgrade. + # + def update_local_ssh_host_keys(node, remote_keys_string) + remote_keys = SshKey.parse_keys(remote_keys_string) + return unless remote_keys.any? + current_key = SshKey.load(Path.named_path([:node_ssh_pub_key, node.name])) + best_key = SshKey.pick_best_key(remote_keys) + return unless best_key && current_key + if current_key != best_key + say(" One of the SSH host keys for node '#{node.name}' is better than what you currently have trusted.") + say(" Current key: #{current_key.summary}") + say(" Better key: #{best_key.summary}") + if agree(" Do you want to use the better key? ") + write_file! [:node_ssh_pub_key, node.name], best_key.to_s + end + else + log(3, "current host key does not need updating") + end + end + +end; end diff --git a/lib/leap_cli/commands/ssh.rb b/lib/leap_cli/commands/ssh.rb new file mode 100644 index 00000000..3887618e --- /dev/null +++ b/lib/leap_cli/commands/ssh.rb @@ -0,0 +1,225 @@ +module LeapCli; module Commands + + desc 'Log in to the specified node with an interactive shell.' + arg_name 'NAME' #, :optional => false, :multiple => false + command :ssh do |c| + c.flag 'ssh', :desc => "Pass through raw options to ssh (e.g. `--ssh '-F ~/sshconfig'`)." + c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`.' + c.action do |global_options,options,args| + exec_ssh(:ssh, options, args) + end + end + + desc 'Log in to the specified node with an interactive shell using mosh (requires node to have mosh.enabled set to true).' + arg_name 'NAME' + command :mosh do |c| + c.flag 'ssh', :desc => "Pass through raw options to ssh (e.g. `--ssh '-F ~/sshconfig'`)." + c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`.' + c.action do |global_options,options,args| + exec_ssh(:mosh, options, args) + end + end + + desc 'Creates an SSH port forward (tunnel) to the node NAME. REMOTE_PORT is the port on the remote node that the tunnel will connect to. LOCAL_PORT is the optional port on your local machine. For example: `leap tunnel couch1:5984`.' + arg_name '[LOCAL_PORT:]NAME:REMOTE_PORT' + command :tunnel do |c| + c.flag 'ssh', :desc => "Pass through raw options to ssh (e.g. --ssh '-F ~/sshconfig')." + c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`.' + c.action do |global_options,options,args| + local_port, node, remote_port = parse_tunnel_arg(args.first) + unless node.ssh.config.AllowTcpForwarding == "yes" + log :warning, "It looks like TCP forwarding is not enabled. "+ + "The tunnel command requires that the node property ssh.config.AllowTcpForwarding "+ + "be set to 'yes'. Add this property to #{node.name}.json, deploy, and then try tunnel again." + end + options[:ssh] = [options[:ssh], "-N -L 127.0.0.1:#{local_port}:0.0.0.0:#{remote_port}"].join(' ') + log("Forward port localhost:#{local_port} to #{node.name}:#{remote_port}") + if is_port_available?(local_port) + exec_ssh(:ssh, options, [node.name]) + end + end + end + + desc 'Secure copy from FILE1 to FILE2. Files are specified as NODE_NAME:FILE_PATH. For local paths, omit "NODE_NAME:".' + arg_name 'FILE1 FILE2' + command :scp do |c| + c.switch :r, :desc => 'Copy recursively' + c.action do |global_options, options, args| + if args.size != 2 + bail!('You must specificy both FILE1 and FILE2') + end + from, to = args + if (from !~ /:/ && to !~ /:/) || (from =~ /:/ && to =~ /:/) + bail!('One FILE must be remote and the other local.') + end + src_node_name = src_file_path = src_node = nil + dst_node_name = dst_file_path = dst_node = nil + if from =~ /:/ + src_node_name, src_file_path = from.split(':') + src_node = get_node_from_args([src_node_name], :include_disabled => true) + dst_file_path = to + else + dst_node_name, dst_file_path = to.split(':') + dst_node = get_node_from_args([dst_node_name], :include_disabled => true) + src_file_path = from + end + exec_scp(options, src_node, src_file_path, dst_node, dst_file_path) + end + end + + protected + + # + # allow for ssh overrides of all commands that use ssh_connect + # + def connect_options(options) + connect_options = {:ssh_options=>{}} + if options[:port] + connect_options[:ssh_options][:port] = options[:port] + end + if options[:ip] + connect_options[:ssh_options][:host_name] = options[:ip] + end + return connect_options + end + + def ssh_config_help_message + puts "" + puts "Are 'too many authentication failures' getting you down?" + puts "Then we have the solution for you! Add something like this to your ~/.ssh/config file:" + puts " Host *.#{manager.provider.domain}" + puts " IdentityFile ~/.ssh/id_rsa" + puts " IdentitiesOnly=yes" + puts "(replace `id_rsa` with the actual private key filename that you use for this provider)" + end + + require 'socket' + def is_port_available?(port) + TCPServer.open('127.0.0.1', port) {} + true + rescue Errno::EACCES + bail!("You don't have permission to bind to port #{port}.") + rescue Errno::EADDRINUSE + bail!("Local port #{port} is already in use. Specify LOCAL_PORT to pick another.") + rescue Exception => exc + bail!(exc.to_s) + end + + private + + def exec_ssh(cmd, cli_options, args) + node = get_node_from_args(args, :include_disabled => true) + port = node.ssh.port + options = ssh_config(node) + username = 'root' + if LeapCli.log_level >= 3 + options << "-vv" + elsif LeapCli.log_level >= 2 + options << "-v" + end + if cli_options[:port] + port = cli_options[:port] + end + if cli_options[:ssh] + options << cli_options[:ssh] + end + ssh = "ssh -l #{username} -p #{port} #{options.join(' ')}" + if cmd == :ssh + command = "#{ssh} #{node.domain.full}" + elsif cmd == :mosh + command = "MOSH_TITLE_NOPREFIX=1 mosh --ssh \"#{ssh}\" #{node.domain.full}" + end + log 2, command + + # exec the shell command in a subprocess + pid = fork { exec "#{command}" } + + Signal.trap("SIGINT") do + Process.kill("KILL", pid) + Process.wait(pid) + exit(0) + end + + # wait for shell to exit so we can grab the exit status + _, status = Process.waitpid2(pid) + + if status.exitstatus == 255 + ssh_config_help_message + elsif status.exitstatus != 0 + exit(status.exitstatus) + end + end + + def exec_scp(cli_options, src_node, src_file_path, dst_node, dst_file_path) + node = src_node || dst_node + options = ssh_config(node) + port = node.ssh.port + username = 'root' + options << "-r" if cli_options[:r] + scp = "scp -P #{port} #{options.join(' ')}" + if src_node + command = "#{scp} #{username}@#{src_node.domain.full}:#{src_file_path} #{dst_file_path}" + elsif dst_node + command = "#{scp} #{src_file_path} #{username}@#{dst_node.domain.full}:#{dst_file_path}" + end + log 2, command + + # exec the shell command in a subprocess + pid = fork { exec "#{command}" } + + Signal.trap("SIGINT") do + Process.kill("KILL", pid) + Process.wait(pid) + exit(0) + end + + # wait for shell to exit so we can grab the exit status + _, status = Process.waitpid2(pid) + exit(status.exitstatus) + end + + # + # SSH command line -o options. See `man ssh_config` + # + # NOTES: + # + # The option 'HostKeyAlias=#{node.name}' is oddly incompatible with ports in + # known_hosts file, so we must not use this or non-standard ports break. + # + def ssh_config(node) + options = [ + "-o 'HostName=#{node.ip_address}'", + "-o 'GlobalKnownHostsFile=#{path(:known_hosts)}'", + "-o 'UserKnownHostsFile=/dev/null'" + ] + if node.vagrant? + options << "-i #{vagrant_ssh_key_file}" # use the universal vagrant insecure key + options << "-o IdentitiesOnly=yes" # force the use of the insecure vagrant key + options << "-o 'StrictHostKeyChecking=no'" # blindly accept host key and don't save it + # (since userknownhostsfile is /dev/null) + else + options << "-o 'StrictHostKeyChecking=yes'" + end + if !node.supported_ssh_host_key_algorithms.empty? + options << "-o 'HostKeyAlgorithms=#{node.supported_ssh_host_key_algorithms}'" + end + return options + end + + def parse_tunnel_arg(arg) + if arg.count(':') == 1 + node_name, remote = arg.split(':') + local = nil + elsif arg.count(':') == 2 + local, node_name, remote = arg.split(':') + else + bail!('Argument NAME:REMOTE_PORT required.') + end + node = get_node_from_args([node_name], :include_disabled => true) + remote = remote.to_i + local = local || remote + local = local.to_i + return [local, node, remote] + end + +end; end
\ No newline at end of file diff --git a/lib/leap_cli/commands/test.rb b/lib/leap_cli/commands/test.rb new file mode 100644 index 00000000..73207b31 --- /dev/null +++ b/lib/leap_cli/commands/test.rb @@ -0,0 +1,74 @@ +module LeapCli; module Commands + + desc 'Run tests.' + command [:test, :t] do |test| + test.desc 'Run the test suit on FILTER nodes.' + test.arg_name 'FILTER', :optional => true + test.command :run do |run| + run.switch 'continue', :desc => 'Continue over errors and failures (default is --no-continue).', :negatable => true + run.action do |global_options,options,args| + test_order = File.join(Path.platform, 'tests/order.rb') + if File.exists?(test_order) + require test_order + end + manager.filter!(args).names_in_test_dependency_order.each do |node_name| + node = manager.nodes[node_name] + begin + ssh_connect(node) do |ssh| + ssh.run(test_cmd(options)) + end + rescue Capistrano::CommandError => exc + if options[:continue] + exit_status(1) + else + bail! + end + end + end + end + end + + test.desc 'Creates files needed to run tests.' + test.command :init do |init| + init.action do |global_options,options,args| + generate_test_client_openvpn_configs + end + end + + test.default_command :run + end + + private + + def test_cmd(options) + if options[:continue] + "#{Leap::Platform.leap_dir}/bin/run_tests --continue" + else + "#{Leap::Platform.leap_dir}/bin/run_tests" + end + end + + # + # generates a whole bunch of openvpn configs that can be used to connect to different openvpn gateways + # + def generate_test_client_openvpn_configs + assert_config! 'provider.ca.client_certificates.unlimited_prefix' + assert_config! 'provider.ca.client_certificates.limited_prefix' + template = read_file! Path.find_file(:test_client_openvpn_template) + manager.environment_names.each do |env| + vpn_nodes = manager.nodes[:environment => env][:services => 'openvpn']['openvpn.allow_limited' => true] + if vpn_nodes.any? + generate_test_client_cert(provider.ca.client_certificates.limited_prefix) do |key, cert| + write_file! [:test_openvpn_config, [env, 'limited'].compact.join('_')], Util.erb_eval(template, binding) + end + end + vpn_nodes = manager.nodes[:environment => env][:services => 'openvpn']['openvpn.allow_unlimited' => true] + if vpn_nodes.any? + generate_test_client_cert(provider.ca.client_certificates.unlimited_prefix) do |key, cert| + write_file! [:test_openvpn_config, [env, 'unlimited'].compact.join('_')], Util.erb_eval(template, binding) + end + end + end + end + +end; end diff --git a/lib/leap_cli/commands/user.rb b/lib/leap_cli/commands/user.rb new file mode 100644 index 00000000..b842e854 --- /dev/null +++ b/lib/leap_cli/commands/user.rb @@ -0,0 +1,136 @@ + +# +# perhaps we want to verify that the key files are actually the key files we expect. +# we could use 'file' for this: +# +# > file ~/.gnupg/00440025.asc +# ~/.gnupg/00440025.asc: PGP public key block +# +# > file ~/.ssh/id_rsa.pub +# ~/.ssh/id_rsa.pub: OpenSSH RSA public key +# + +module LeapCli + module Commands + + desc 'Adds a new trusted sysadmin by adding public keys to the "users" directory.' + arg_name 'USERNAME' #, :optional => false, :multiple => false + command :'add-user' do |c| + + c.switch 'self', :desc => 'Add yourself as a trusted sysadmin by choosing among the public keys available for the current user.', :negatable => false + c.flag 'ssh-pub-key', :desc => 'SSH public key file for this new user' + c.flag 'pgp-pub-key', :desc => 'OpenPGP public key file for this new user' + + c.action do |global_options,options,args| + username = args.first + if !username.any? + if options[:self] + username ||= `whoami`.strip + else + help! "Either USERNAME argument or --self flag is required." + end + end + if Leap::Platform.reserved_usernames.include? username + bail! %(The username "#{username}" is reserved. Sorry, pick another.) + end + + ssh_pub_key = nil + pgp_pub_key = nil + + if options['ssh-pub-key'] + ssh_pub_key = read_file!(options['ssh-pub-key']) + end + if options['pgp-pub-key'] + pgp_pub_key = read_file!(options['pgp-pub-key']) + end + + if options[:self] + ssh_pub_key ||= pick_ssh_key.to_s + pgp_pub_key ||= pick_pgp_key + end + + assert!(ssh_pub_key, 'Sorry, could not find SSH public key.') + + if ssh_pub_key + write_file!([:user_ssh, username], ssh_pub_key) + end + if pgp_pub_key + write_file!([:user_pgp, username], pgp_pub_key) + end + + update_authorized_keys + end + end + + # + # let the the user choose among the ssh public keys that we encounter, or just pick the key if there is only one. + # + def pick_ssh_key + ssh_keys = [] + Dir.glob("#{ENV['HOME']}/.ssh/*.pub").each do |keyfile| + ssh_keys << SshKey.load(keyfile) + end + + if `which ssh-add`.strip.any? + `ssh-add -L 2> /dev/null`.split("\n").compact.each do |line| + key = SshKey.load(line) + if key + key.comment = 'ssh-agent' + ssh_keys << key unless ssh_keys.include?(key) + end + end + end + ssh_keys.compact! + + assert! ssh_keys.any?, 'Sorry, could not find any SSH public key for you. Have you run ssh-keygen?' + + if ssh_keys.length > 1 + key_index = numbered_choice_menu('Choose your SSH public key', ssh_keys.collect(&:summary)) do |line, i| + say("#{i+1}. #{line}") + end + else + key_index = 0 + end + + return ssh_keys[key_index] + end + + # + # let the the user choose among the gpg public keys that we encounter, or just pick the key if there is only one. + # + def pick_pgp_key + begin + require 'gpgme' + rescue LoadError + log "Skipping OpenPGP setup because gpgme is not installed." + return + end + + secret_keys = GPGME::Key.find(:secret) + if secret_keys.empty? + log "Skipping OpenPGP setup because I could not find any OpenPGP keys for you" + return nil + end + + secret_keys.select!{|key| !key.expired} + + if secret_keys.length > 1 + key_index = numbered_choice_menu('Choose your OpenPGP public key', secret_keys) do |key, i| + key_info = key.to_s.split("\n")[0..1].map{|line| line.sub(/^\s*(sec|uid)\s*/,'')}.join(' -- ') + say("#{i+1}. #{key_info}") + end + else + key_index = 0 + end + + key_id = secret_keys[key_index].sha + + # can't use this, it includes signatures: + #puts GPGME::Key.export(key_id, :armor => true, :export_options => :export_minimal) + + # export with signatures removed: + return `gpg --armor --export-options export-minimal --export #{key_id}`.strip + end + + end +end diff --git a/lib/leap_cli/commands/util.rb b/lib/leap_cli/commands/util.rb new file mode 100644 index 00000000..c1da570e --- /dev/null +++ b/lib/leap_cli/commands/util.rb @@ -0,0 +1,50 @@ +module LeapCli; module Commands + + extend self + extend LeapCli::Util + extend LeapCli::Util::RemoteCommand + + def path(name) + Path.named_path(name) + end + + # + # keeps prompting the user for a numbered choice, until they pick a good one or bail out. + # + # block is yielded and is responsible for rendering the choices. + # + def numbered_choice_menu(msg, items, &block) + while true + say("\n" + msg + ':') + items.each_with_index &block + say("q. quit") + index = ask("number 1-#{items.length}> ") + if index.empty? + next + elsif index =~ /q/ + bail! + else + i = index.to_i - 1 + if i < 0 || i >= items.length + bail! + else + return i + end + end + end + end + + + def parse_node_list(nodes) + if nodes.is_a? Config::Object + Config::ObjectList.new(nodes) + elsif nodes.is_a? Config::ObjectList + nodes + elsif nodes.is_a? String + manager.filter!(nodes) + else + bail! "argument error" + end + end + +end; end diff --git a/lib/leap_cli/commands/vagrant.rb b/lib/leap_cli/commands/vagrant.rb new file mode 100644 index 00000000..9fdd48e3 --- /dev/null +++ b/lib/leap_cli/commands/vagrant.rb @@ -0,0 +1,180 @@ +autoload :IPAddr, 'ipaddr' +require 'fileutils' + +module LeapCli; module Commands + + desc "Manage local virtual machines." + long_desc "This command provides a convient way to manage Vagrant-based virtual machines. If FILTER argument is missing, the command runs on all local virtual machines. The Vagrantfile is automatically generated in 'test/Vagrantfile'. If you want to run vagrant commands manually, cd to 'test'." + command [:local, :l] do |local| + local.desc 'Starts up the virtual machine(s)' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :start do |start| + start.flag(:basebox, + :desc => "The basebox to use. This value is passed to vagrant as the "+ + "`config.vm.box` option. The value here should be the name of an installed box or a "+ + "shorthand name of a box in HashiCorp's Atlas.", + :arg_name => 'BASEBOX', + :default_value => 'LEAP/jessie' + ) + start.action do |global_options,options,args| + vagrant_command(["up", "sandbox on"], args, options) + end + end + + local.desc 'Shuts down the virtual machine(s)' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :stop do |stop| + stop.action do |global_options,options,args| + if global_options[:yes] + vagrant_command("halt --force", args) + else + vagrant_command("halt", args) + end + end + end + + local.desc 'Destroys the virtual machine(s), reclaiming the disk space' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :destroy do |destroy| + destroy.action do |global_options,options,args| + if global_options[:yes] + vagrant_command("destroy --force", args) + else + vagrant_command("destroy", args) + end + end + end + + local.desc 'Print the status of local virtual machine(s)' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :status do |status| + status.action do |global_options,options,args| + vagrant_command("status", args) + end + end + + local.desc 'Saves the current state of the virtual machine as a new snapshot' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :save do |status| + status.action do |global_options,options,args| + vagrant_command("sandbox commit", args) + end + end + + local.desc 'Resets virtual machine(s) to the last saved snapshot' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :reset do |reset| + reset.action do |global_options,options,args| + vagrant_command("sandbox rollback", args) + end + end + end + + public + + # + # returns the path to a vagrant ssh private key file. + # + # if the vagrant.key file is owned by root or ourselves, then + # we need to make sure that it owned by us and not world readable. + # + def vagrant_ssh_key_file + file_path = Path.vagrant_ssh_priv_key_file + Util.assert_files_exist! file_path + uid = File.new(file_path).stat.uid + if uid == 0 || uid == Process.euid + FileUtils.install file_path, '/tmp/vagrant.key', :mode => 0600 + file_path = '/tmp/vagrant.key' + end + return file_path + end + + protected + + def vagrant_command(cmds, args, options={}) + vagrant_setup(options) + cmds = cmds.to_a + if args.empty? + nodes = [""] + else + nodes = manager.filter(args)[:environment => "local"].field(:name) + end + if nodes.any? + vagrant_dir = File.dirname(Path.named_path(:vagrantfile)) + exec = ["cd #{vagrant_dir}"] + cmds.each do |cmd| + nodes.each do |node| + exec << "vagrant #{cmd} #{node}" + end + end + execute exec.join('; ') + else + bail! "No nodes found. This command only works on nodes with ip_address in the network #{LeapCli.leapfile.vagrant_network}" + end + end + + private + + def vagrant_setup(options) + assert_bin! 'vagrant', 'Vagrant is required for running local virtual machines. Run "sudo apt-get install vagrant".' + assert! (vagrant_version >= Gem::Version.new('1.1')), 'Vagrant version >= 1.1 is required for running local virtual machines. Please upgrade.' + + unless assert_run!('vagrant plugin list | grep sahara | cat').chars.any? + log :installing, "vagrant plugin 'sahara'" + assert_run! 'vagrant plugin install sahara' + end + create_vagrant_file(options) + end + + def vagrant_version + @vagrant_version ||= Gem::Version.new(assert_run!('vagrant --version').split(' ')[1]) + end + + def execute(cmd) + log 2, :run, cmd + exec cmd + end + + def create_vagrant_file(options) + lines = [] + + basebox = options[:basebox] || 'LEAP/jessie' + # override basebox with custom setting from Leapfile or ~/.leaprc + basebox = leapfile.vagrant_basebox || basebox + + lines << %[Vagrant.configure("2") do |config|] + manager.each_node do |node| + if node.vagrant? + lines << %[ config.vm.define :#{node.name} do |config|] + lines << %[ config.vm.box = "#{basebox}"] + lines << %[ config.vm.network :private_network, ip: "#{node.ip_address}"] + lines << %[ config.vm.provider "virtualbox" do |v|] + lines << %[ v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]] + lines << %[ v.name = "#{node.name}"] + lines << %[ v.memory = 1536] + lines << %[ end] + lines << %[ config.vm.provider "libvirt" do |v|] + lines << %[ v.memory = 1536] + lines << %[ end] + lines << %[ #{leapfile.custom_vagrant_vm_line}] if leapfile.custom_vagrant_vm_line + lines << %[ end] + end + end + + lines << %[end] + lines << "" + write_file! :vagrantfile, lines.join("\n") + end + + def pick_next_vagrant_ip_address + taken_ips = manager.nodes[:environment => "local"].field(:ip_address) + if taken_ips.any? + highest_ip = taken_ips.map{|ip| IPAddr.new(ip)}.max + new_ip = highest_ip.succ + else + new_ip = IPAddr.new(LeapCli.leapfile.vagrant_network).succ.succ + end + return new_ip.to_s + end + +end; end diff --git a/provider_base/lib/macros.rb b/lib/leap_cli/macros.rb index ecc3e6ba..fdb9a94e 100644 --- a/provider_base/lib/macros.rb +++ b/lib/leap_cli/macros.rb @@ -13,3 +13,4 @@ require_relative 'macros/keys' require_relative 'macros/nodes' require_relative 'macros/secrets' require_relative 'macros/stunnel' +require_relative 'macros/provider' diff --git a/provider_base/lib/macros/core.rb b/lib/leap_cli/macros/core.rb index 7de50f2f..873da358 100644 --- a/provider_base/lib/macros/core.rb +++ b/lib/leap_cli/macros/core.rb @@ -4,13 +4,6 @@ module LeapCli module Macro # - # return a fingerprint for a x509 certificate - # - def fingerprint(filename) - "SHA256: " + X509.fingerprint("SHA256", Path.named_path(filename)) - end - - # # Creates a hash from the ssh key info in users directory, for use in # updating authorized_keys file. Additionally, the 'monitor' public key is # included, which is used by the monitor nodes to run particular commands @@ -54,8 +47,13 @@ module LeapCli # applies a JSON partial to this node # def apply_partial(partial_path) - manager.partials(partial_path).each do |partial_data| - self.deep_merge!(partial_data) + if env.partials[partial_path] + self.deep_merge!(env.partials[partial_path]) + else + raise ArgumentError.new( + "No such partial `%s`. Available partials include:\n%s" % + [partial_path, env.partials.keys.join(", ")] + ) end end diff --git a/lib/leap_cli/macros/files.rb b/lib/leap_cli/macros/files.rb new file mode 100644 index 00000000..04c94edf --- /dev/null +++ b/lib/leap_cli/macros/files.rb @@ -0,0 +1,124 @@ +# encoding: utf-8 + +## +## FILES +## + +module LeapCli + module Macro + + # + # inserts the contents of a file + # + def file(filename, options={}) + if filename.is_a? Symbol + filename = [filename, @node.name] + end + filepath = Path.find_file(filename) + if filepath + if filepath =~ /\.erb$/ + return ERB.new(File.read(filepath, :encoding => 'UTF-8'), nil, '%<>').result(binding) + else + return File.read(filepath, :encoding => 'UTF-8') + end + else + raise FileMissing.new(Path.named_path(filename), options) + end + end + + # + # like #file, but allow missing files + # + def try_file(filename) + return file(filename) + rescue FileMissing + return nil + end + + # + # returns the location of a file that is stored on the local + # host, under PROVIDER_DIR/files. + # + def local_file_path(path, options={}) + if path.is_a? Symbol + path = [path, @node.name] + elsif path.is_a? String + # ensure it prefixed with files/ + unless path =~ /^files\// + path = "files/" + path + end + end + local_path = Path.find_file(path) + if local_path.nil? + if options[:missing] + raise FileMissing.new(Path.named_path(path), options) + elsif block_given? + yield + return local_file_path(path, options) # try again. + else + Util::log 2, :skipping, "local_file_path(\"#{path}\") because there is no such file." + return nil + end + else + return local_path + end + end + + # + # Returns the location of a file once it is deployed via rsync to the a + # remote server. An internal list of discovered file paths is saved, in + # order to rsync these files when needed. + # + # If the file does not exist, nil is returned. + # + # If there is a block given and the file does not actually exist, the + # block will be yielded to give an opportunity for some code to create the + # file. + # + # For example: + # + # file_path(:dkim_priv_key) {generate_dkim_key} + # + # notes: + # + # * argument 'path' is relative to Path.provider/files or + # Path.provider_base/files + # * the path returned by this method is absolute + # * the path stored for use later by rsync is relative to Path.provider + # * if the path does not exist locally, but exists in provider_base, + # then the default file from provider_base is copied locally. this + # is required for rsync to work correctly. + # + def remote_file_path(path, options={}, &block) + local_path = local_file_path(path, options, &block) + + return nil if local_path.nil? + + # if file is under Path.provider_base, we must copy the default file to + # to Path.provider in order for rsync to be able to sync the file. + if local_path =~ /^#{Regexp.escape(Path.provider_base)}/ + local_provider_path = local_path.sub(/^#{Regexp.escape(Path.provider_base)}/, Path.provider) + FileUtils.mkdir_p File.dirname(local_provider_path), :mode => 0700 + FileUtils.install local_path, local_provider_path, :mode => 0600 + Util.log :created, Path.relative_path(local_provider_path) + local_path = local_provider_path + end + + # ensure directories end with /, important for building rsync command + if File.directory?(local_path) && local_path !~ /\/$/ + local_path += '/' + end + + relative_path = Path.relative_path(local_path) + relative_path.sub!(/^files\//, '') # remove "files/" prefix + @node.file_paths << relative_path + return File.join(Leap::Platform.files_dir, relative_path) + end + + # deprecated + def file_path(path, options={}) + return remote_file_path(path, options) + end + + end +end
\ No newline at end of file diff --git a/provider_base/lib/macros/haproxy.rb b/lib/leap_cli/macros/haproxy.rb index 602ae726..602ae726 100644 --- a/provider_base/lib/macros/haproxy.rb +++ b/lib/leap_cli/macros/haproxy.rb diff --git a/provider_base/lib/macros/hosts.rb b/lib/leap_cli/macros/hosts.rb index 8281329f..963857ae 100644 --- a/provider_base/lib/macros/hosts.rb +++ b/lib/leap_cli/macros/hosts.rb @@ -4,6 +4,24 @@ module LeapCli module Macro ## + ## IPs + ## + + # + # returns a simple array of all the IPs for the specified node list + # + def host_ips(node_list) + if self.vagrant? + node_list = node_list['environment' => 'local'] + else + node_list = node_list['environment' => '!local'] + end + node_list.map {|name, n| + [n.ip_address, (manager.facts[name]||{})['ec2_public_ipv4']] + }.flatten.compact.uniq + end + + ## ## HOSTS ## @@ -48,6 +66,10 @@ module LeapCli 'domain_full' => node.domain.full, 'port' => node.ssh.port } + if node.dns['aliases'] && node.dns['aliases'].any? + # include aliases, but without domain.full + hosts[node.name]['aliases'] = node.dns['aliases'] - [node.domain.full] + end node_location = node['location'] ? node['location']['name'] : nil if my_location == node_location if facts = @node.manager.facts[node.name] diff --git a/provider_base/lib/macros/keys.rb b/lib/leap_cli/macros/keys.rb index 0ed7ccd0..e7a75cfb 100644 --- a/provider_base/lib/macros/keys.rb +++ b/lib/leap_cli/macros/keys.rb @@ -8,17 +8,28 @@ module LeapCli module Macro # + # return a fingerprint for a key or certificate + # + def fingerprint(filename, options={}) + options[:mode] ||= :x509 + if options[:mode] == :x509 + "SHA256: " + X509.fingerprint("SHA256", Path.named_path(filename)) + elsif options[:mode] == :rsa + key = OpenSSL::PKey::RSA.new(File.read(filename)) + Digest::SHA1.new.hexdigest(key.to_der) + end + end + + ## + ## TOR + ## + + # # return the path to the tor public key # generating key if it is missing # def tor_public_key_path(path_name, key_type) - path = file_path(path_name) - if path.nil? - generate_tor_key(key_type) - file_path(path_name) - else - path - end + file_path(path_name) { generate_tor_key(key_type) } end # @@ -26,13 +37,7 @@ module LeapCli # generating key if it is missing # def tor_private_key_path(path_name, key_type) - path = file_path(path_name) - if path.nil? - generate_tor_key(key_type) - file_path(path_name) - else - path - end + file_path(path_name) { generate_tor_key(key_type) } end # @@ -62,6 +67,15 @@ module LeapCli end end + def generate_dkim_key(bit_size=2048) + LeapCli.log :generating, "%s bit RSA DKIM key" % bit_size do + private_key = OpenSSL::PKey::RSA.new(bit_size) + public_key = private_key.public_key + LeapCli::Util.write_file! :dkim_priv_key, private_key.to_pem + LeapCli::Util.write_file! :dkim_pub_key, public_key.to_pem + end + end + private def generate_tor_key(key_type) diff --git a/provider_base/lib/macros/nodes.rb b/lib/leap_cli/macros/nodes.rb index 8b961cbc..0e23831d 100644 --- a/provider_base/lib/macros/nodes.rb +++ b/lib/leap_cli/macros/nodes.rb @@ -11,14 +11,14 @@ module LeapCli # the list of all the nodes # def nodes - global.nodes + env.nodes end # # simple alias for global.provider # def provider - global.provider + env.provider end # diff --git a/lib/leap_cli/macros/provider.rb b/lib/leap_cli/macros/provider.rb new file mode 100644 index 00000000..4e74da01 --- /dev/null +++ b/lib/leap_cli/macros/provider.rb @@ -0,0 +1,90 @@ +# +# These macros are intended only for use in provider.json, although they are +# currently loaded in all .json contexts. +# + +module LeapCli + module Macro + + # + # returns an array of the service names, including only those services that + # are enabled for this environment. + # + def enabled_services + manager.env(self.environment).services[:service_type => :user_service].field(:name).select { |service| + manager.nodes[:environment => self.environment][:services => service].any? + } + end + + # + # The webapp will not work unless the service level configuration is precisely defined. + # Here, we take what the sysadmin has specified in provider.json and clean it up to + # ensure it is OK. + # + # It would be better to add support for JSON schema. + # + def service_levels() + levels = {} + provider.service.levels.each do |name, level| + if name =~ /^[0-9]+$/ + name = name.to_i + end + levels[name] = level_cleanup(name, level.clone) + end + levels + end + + private + + def print_warning(name, msg) + if self.environment + provider_str = "provider.json or %s" % ['provider', self.environment, 'json'].join('.') + else + provider_str = "provider.json" + end + LeapCli::log :warning, "In #{provider_str}, you have an incorrect definition for service level '#{name}':" do + LeapCli::log msg + end + end + + def level_cleanup(name, level) + unless level['name'] + print_warning(name, 'required field "name" is missing') + end + unless level['description'] + print_warning(name, 'required field "description" is missing') + end + unless level['bandwidth'].nil? || level['bandwidth'] == 'limited' + print_warning(name, 'field "bandwidth" must be nil or "limited"') + end + unless level['rate'].nil? || level['rate'].is_a?(Hash) + print_warning(name, 'field "rate" must be nil or a hash (e.g. {"USD":10, "EUR":10})') + end + possible_services = enabled_services + if level['services'] + level['services'].each do |service| + unless possible_services.include? service + print_warning(name, "the service '#{service}' does not exist or there are no nodes that provide this service.") + LeapCli::Util::bail! + end + end + else + level['services'] = possible_services + end + level['services'] = remap_services(level['services']) + level + end + + # + # the service names that the webapp uses and that leap_platform uses are different. ugh. + # + SERVICE_MAP = { + "mx" => "email", + "openvpn" => "eip" + } + def remap_services(services) + services.map {|srv| SERVICE_MAP[srv]} + end + + end +end diff --git a/provider_base/lib/macros/secrets.rb b/lib/leap_cli/macros/secrets.rb index 8d1feb55..8d1feb55 100644 --- a/provider_base/lib/macros/secrets.rb +++ b/lib/leap_cli/macros/secrets.rb diff --git a/provider_base/lib/macros/stunnel.rb b/lib/leap_cli/macros/stunnel.rb index f16308c7..821bda38 100644 --- a/provider_base/lib/macros/stunnel.rb +++ b/lib/leap_cli/macros/stunnel.rb @@ -49,12 +49,14 @@ module LeapCli result = Config::ObjectList.new node_list.each_node do |node| if node.name != self.name || options[:include_self] + s_port = stunnel_port(port) result["#{node.name}_#{port}"] = Config::Object[ 'accept_port', @next_stunnel_port, 'connect', node.domain.internal, - 'connect_port', stunnel_port(port), + 'connect_port', s_port, 'original_port', port ] + manager.connections.add(:from => @node.ip_address, :to => node.ip_address, :port => s_port) @next_stunnel_port += 1 end end @@ -76,6 +78,15 @@ module LeapCli } end + # + # lists the ips that connect to this node, on particular ports. + # + def stunnel_firewall + manager.connections.select {|connection| + connection['to'] == @node.ip_address + } + end + private # diff --git a/platform.rb b/platform.rb index 0c3de2a0..1e19a2a9 100644 --- a/platform.rb +++ b/platform.rb @@ -4,8 +4,8 @@ # Leap::Platform.define do - self.version = "0.7.1" - self.compatible_cli = "1.7.0".."1.7.99" + self.version = "0.8" + self.compatible_cli = "1.8".."1.99" # # the facter facts that should be gathered @@ -15,6 +15,7 @@ Leap::Platform.define do # # absolute paths on the destination server # + self.hiera_dir = '/etc/leap' if self.respond_to?(:hiera_dir) self.hiera_path = '/etc/leap/hiera.yaml' self.leap_dir = '/srv/leap' self.files_dir = '/srv/leap/files' @@ -30,18 +31,21 @@ Leap::Platform.define do :files_dir => 'files', :nodes_dir => 'nodes', :services_dir => 'services', + :templates_dir => 'templates', :tags_dir => 'tags', :node_files_dir => 'files/nodes/#{arg}', # input config files :common_config => 'common.json', :provider_config => 'provider.json', - :secrets_config => 'secrets.json', - :node_config => 'nodes/#{arg}.json', :service_config => 'services/#{arg}.json', :tag_config => 'tags/#{arg}.json', + :template_config => 'templates/#{arg}.json', + :secrets_config => 'secrets.json', + :node_config => 'nodes/#{arg}.json', # input config files, environmentally scoped + :common_env_config => 'commmon.#{arg}.json', :provider_env_config => 'provider.#{arg}.json', :service_env_config => 'services/#{arg[0]}.#{arg[1]}.json', :tag_env_config => 'tags/#{arg[0]}.#{arg[1]}.json', @@ -75,6 +79,9 @@ Leap::Platform.define do :commercial_key => 'files/cert/#{arg}.key', :commercial_csr => 'files/cert/#{arg}.csr', :commercial_cert => 'files/cert/#{arg}.crt', + :dkim_priv_key => 'files/mx/dkim.key', + :dkim_pub_key => 'files/mx/dkim.pub', + :commercial_ca_cert => 'files/cert/commercial_ca.crt', :vagrantfile => 'test/Vagrantfile', :static_web_provider_json => 'files/web/bootstrap/#{arg}/provider.json', diff --git a/provider_base/common.json b/provider_base/common.json index 3d2965d7..5e689109 100644 --- a/provider_base/common.json +++ b/provider_base/common.json @@ -16,6 +16,9 @@ }, "ssh": { "authorized_keys": "= authorized_keys", + "config": { + "AllowTcpForwarding": "no" + }, "port": 22, "mosh": { "ports": "60000:61000", @@ -47,6 +50,14 @@ "clients": {}, "servers": {} }, + "firewall": { + "ssh": { + "from": "sysadmin", + "to": "= ip_address", + "port": "= ssh.port" + }, + "stunnel": "=> stunnel_firewall" + }, "platform": { "version": "= Leap::Platform.version.to_s", "major_version": "= Leap::Platform.major_version" @@ -65,22 +76,22 @@ "nickserver": { "type": "git", "source": "https://leap.se/git/nickserver", - "revision": "origin/master" + "revision": "origin/version/0.8" + }, + "platform": { + "apt": { + "basic": "= 'http://deb.leap.se/' + Leap::Platform.major_version" + } }, "soledad": { "type": "apt", "package": "soledad-server", "revision": "latest" }, - "tapicero": { - "type": "git", - "source": "https://leap.se/git/tapicero", - "revision": "origin/version/0.7" - }, "webapp": { "type": "git", "source": "https://leap.se/git/leap_web", - "revision": "origin/version/0.7.1" + "revision": "origin/version/0.8" } } } diff --git a/provider_base/files/service-definitions/provider.json.erb b/provider_base/files/service-definitions/provider.json.erb index be8ae484..a75bea61 100644 --- a/provider_base/files/service-definitions/provider.json.erb +++ b/provider_base/files/service-definitions/provider.json.erb @@ -6,15 +6,10 @@ ) hsh['domain'] = domain.full_suffix - # advertise services that are 'user services' and for which there are actually nodes - hsh['services'] ||= global.services[:service_type => :user_service].field(:name).select do |service| - nodes_like_me[:services => service].any? - end - hsh['api_version'] = "1" hsh['api_uri'] = ["https://", api.domain, ':', api.port].join - hsh['ca_cert_uri'] = 'https://' + webapp.domain + '/ca.crt' + hsh['ca_cert_uri'] = api.ca_cert_uri hsh['ca_cert_fingerprint'] = fingerprint(:ca_cert) hsh.dump_json diff --git a/provider_base/lib/macros/files.rb b/provider_base/lib/macros/files.rb deleted file mode 100644 index 958958bc..00000000 --- a/provider_base/lib/macros/files.rb +++ /dev/null @@ -1,89 +0,0 @@ -# encoding: utf-8 - -## -## FILES -## - -module LeapCli - module Macro - - # - # inserts the contents of a file - # - def file(filename, options={}) - if filename.is_a? Symbol - filename = [filename, @node.name] - end - filepath = Path.find_file(filename) - if filepath - if filepath =~ /\.erb$/ - ERB.new(File.read(filepath, :encoding => 'UTF-8'), nil, '%<>').result(binding) - else - File.read(filepath, :encoding => 'UTF-8') - end - else - raise FileMissing.new(Path.named_path(filename), options) - "" - end - end - - # - # like #file, but allow missing files - # - def try_file(filename) - return file(filename) - rescue FileMissing - return nil - end - - # - # returns what the file path will be, once the file is rsynced to the server. - # an internal list of discovered file paths is saved, in order to rsync these files when needed. - # - # notes: - # - # * argument 'path' is relative to Path.provider/files or Path.provider_base/files - # * the path returned by this method is absolute - # * the path stored for use later by rsync is relative to Path.provider - # * if the path does not exist locally, but exists in provider_base, then the default file from - # provider_base is copied locally. this is required for rsync to work correctly. - # - def file_path(path, options={}) - if path.is_a? Symbol - path = [path, @node.name] - elsif path.is_a? String - # ensure it prefixed with files/ - unless path =~ /^files\// - path = "files/" + path - end - end - actual_path = Path.find_file(path) - if actual_path.nil? - if options[:missing] - raise FileMissing.new(Path.named_path(path), options) - else - Util::log 2, :skipping, "file_path(\"#{path}\") because there is no such file." - end - nil - else - if actual_path =~ /^#{Regexp.escape(Path.provider_base)}/ - # if file is under Path.provider_base, we must copy the default file to - # to Path.provider in order for rsync to be able to sync the file. - local_provider_path = actual_path.sub(/^#{Regexp.escape(Path.provider_base)}/, Path.provider) - FileUtils.mkdir_p File.dirname(local_provider_path), :mode => 0700 - FileUtils.install actual_path, local_provider_path, :mode => 0600 - Util.log :created, Path.relative_path(local_provider_path) - actual_path = local_provider_path - end - if File.directory?(actual_path) && actual_path !~ /\/$/ - actual_path += '/' # ensure directories end with /, important for building rsync command - end - relative_path = Path.relative_path(actual_path) - relative_path.sub!(/^files\//, '') # remove "files/" prefix - @node.file_paths << relative_path - File.join(Leap::Platform.files_dir, relative_path) - end - end - - end -end
\ No newline at end of file diff --git a/provider_base/provider.json b/provider_base/provider.json index 60ad2a9e..81b2ea98 100644 --- a/provider_base/provider.json +++ b/provider_base/provider.json @@ -14,6 +14,7 @@ "languages": ["en"], "default_language": "en", "enrollment_policy": "open", + "services": "= enabled_services", "service": { // bandwidth limit is in Bytes, storage limit is in MB. // for example: @@ -31,8 +32,8 @@ "bandwidth_limit": 102400, "allow_free": "= provider.service.levels.select {|l| l['rate'].nil?}.any?", "allow_paid": "= provider.service.levels.select {|l| !l['rate'].nil?}.any?", - "allow_anonymous": "= provider.service.levels.select {|l| l['name'] == 'anonymous'}.any?", - "allow_registration": "= provider.service.levels.select {|l| l['name'] != 'anonymous'}.any?", + "allow_anonymous": "= provider.service.levels.select {|l| l['name'] == 'anonymous'}.any? && services.include?('openvpn')", + "allow_registration": "= provider.enrollment_policy != 'closed' && provider.service.levels.select {|l| l['name'] != 'anonymous'}.any?", "allow_limited_bandwidth": "= provider.service.levels.select {|l| l['bandwidth'] == 'limited'}.any?", "allow_unlimited_bandwidth": "= provider.service.levels.select {|l| l['bandwidth'].nil?}.any?" }, diff --git a/provider_base/services/_api_tester.json b/provider_base/services/_api_tester.json new file mode 100644 index 00000000..790aa7d8 --- /dev/null +++ b/provider_base/services/_api_tester.json @@ -0,0 +1,13 @@ +// +// This partial should be added to any service that runs tests that rely on +// accessing the bonafide webapp API. +// +{ + "testing": { + "monitor_auth_token": "= secret :api_monitor_auth_token", + "api_uri": "= global.services[:webapp].api.uri", + // api_hosts is not used directly, but calling hostnames() will ensure + // that the hostnames are added to /etc/hosts + "api_hosts": "= hostnames(nodes_like_me[:services => 'webapp'])" + } +}
\ No newline at end of file diff --git a/provider_base/services/_couchdb_master.json b/provider_base/services/_couchdb_master.json deleted file mode 100644 index 20c6f99b..00000000 --- a/provider_base/services/_couchdb_master.json +++ /dev/null @@ -1,8 +0,0 @@ -// -// Applied to master couchdb node when there is a single master -// -{ - "couch": { - "mode": "master" - } -}
\ No newline at end of file diff --git a/provider_base/services/_couchdb_mirror.json b/provider_base/services/_couchdb_mirror.json index 6a3402bd..da496bae 100644 --- a/provider_base/services/_couchdb_mirror.json +++ b/provider_base/services/_couchdb_mirror.json @@ -1,5 +1,6 @@ // // Applied to all non-master couchdb nodes +// NOT CURRENTLY SUPPORTED // { "stunnel": { diff --git a/provider_base/services/_couchdb_multimaster.json b/provider_base/services/_couchdb_multimaster.json index 0f340e00..803a9416 100644 --- a/provider_base/services/_couchdb_multimaster.json +++ b/provider_base/services/_couchdb_multimaster.json @@ -1,6 +1,6 @@ // // Only applied to master couchdb nodes when there are multiple masters -// +// NOT CURRENTLY USED. { "stunnel": { "servers": { diff --git a/provider_base/services/couchdb.json b/provider_base/services/couchdb.json index 8b1386f8..30cb53d1 100644 --- a/provider_base/services/couchdb.json +++ b/provider_base/services/couchdb.json @@ -8,8 +8,8 @@ } }, "couch": { - "master": false, "port": 5984, + "mode": "plain", "users": { "admin": { "username": "admin", @@ -31,11 +31,6 @@ "password": "= secret :couch_soledad_password", "salt": "= hex_secret :couch_soledad_password_salt, 128" }, - "tapicero": { - "username": "tapicero", - "password": "= secret :couch_tapicero_password", - "salt": "= hex_secret :couch_tapicero_password_salt, 128" - }, "webapp": { "username": "webapp", "password": "= secret :couch_webapp_password", diff --git a/provider_base/services/couchdb.rb b/provider_base/services/couchdb.rb index 3bee3a67..ba7e5ae5 100644 --- a/provider_base/services/couchdb.rb +++ b/provider_base/services/couchdb.rb @@ -1,60 +1,27 @@ -####################################################################### -### -### NOTE! -### -### Currently, mirrors do not work! The only thing that works is all -### nodes multimaster or a single master. -### -####################################################################### # # custom logic for couchdb json resolution # ============================================ # -# There are three modes for a node: -# -# Multimaster -# ----------- -# -# Multimaster uses bigcouch (soon to use couchdb in replication mode -# similar to bigcouch). -# -# Use "multimaster" mode when: -# -# * multiple nodes are marked couch.master -# * OR no nodes are marked couch.master -# -# Master -# ------ -# -# Master uses plain couchdb that is readable and writable. -# -# Use "master" mode when: -# -# * Exactly one node, this one, is marked as master. -# -# Mirror -# ------ -# -# Mirror creates a read-only copy of the database. It uses plain coucdhb -# with legacy couchdb replication (http based). -# -# This does not currently work, because http replication can't handle -# the number of user databases. -# -# Use "mirror" mode when: -# -# * some nodes are marked couch.master -# * AND this node is not a master +# bigcouch is no longer maintained, so now couchdb is required... +# no matter what! # -master_count = nodes_like_me['services' => 'couchdb']['couch.master' => true].size +if self.couch['master'] + LeapCli::log :warning, %("The node property {couch.master:true} is deprecated.\n) + + %( Only {couch.mode:plain} is supported. (node #{self.name})) +end -if master_count == 0 - apply_partial 'services/_couchdb_multimaster.json' -elsif couch.master && master_count > 1 - apply_partial 'services/_couchdb_multimaster.json' -elsif couch.master && master_count == 1 - apply_partial 'services/_couchdb_master.json' -else - apply_partial 'services/_couchdb_mirror.json' +couchdb_nodes = nodes_like_me['services' => 'couchdb'] + +if couchdb_nodes.size > 1 + LeapCli::log :error, "Having multiple nodes with {services:couchdb} is no longer supported (nodes #{couchdb_nodes.keys.join(', ')})." +elsif self.couch.mode == "multimaster" + LeapCli::log :error, "Nodes with {couch.mode:multimaster} are no longer supported (node #{self.name})." end + +# +# This is needed for the "test" that creates and removes the storage db +# for test_user_email. If that test is removed, then this is no longer +# necessary: +# +apply_partial('_api_tester')
\ No newline at end of file diff --git a/provider_base/services/dns.json b/provider_base/services/dns.json index 677d9b2c..67948ef8 100644 --- a/provider_base/services/dns.json +++ b/provider_base/services/dns.json @@ -3,5 +3,12 @@ "public": "= nodes['dns.public' => true].fields('domain.name', 'dns.aliases', 'ip_address')", "private": "= nodes['dns.public' => false].fields('domain.name', 'dns.aliases', 'ip_address')" }, - "service_type": "public_service" + "service_type": "public_service", + "firewall": { + "dns": { + "from": "*", + "to": "= ip_address", + "port": "53" + } + } }
\ No newline at end of file diff --git a/provider_base/services/monitor.json b/provider_base/services/monitor.json index 10d5ac81..9ddc0ec7 100644 --- a/provider_base/services/monitor.json +++ b/provider_base/services/monitor.json @@ -2,7 +2,7 @@ "nagios": { "nagiosadmin_pw": "= secret :nagios_admin_password", "domains_internal": "= nagios.hosts.values.map{|h|h['domain_internal_suffix']}.uniq", - "environments": "= Hash[ nagios.hosts.values.map{|h|h['environment']}.uniq.map{|e| [e||'default',{'contact_emails'=>global.env(e).provider.contacts.default}]} ]", + "environments": "= Hash[ nagios.hosts.values.map{|h|h['environment']}.uniq.map{|e| [e||'default',{'contact_emails'=>manager.env(e).provider.contacts.default}]} ]", "hosts": "= (self.environment == 'local' ? nodes_like_me : nodes[:environment => '!local']).pick_fields('environment', 'domain.internal', 'domain.internal_suffix', 'domain.full_suffix', 'ip_address', 'services', 'openvpn.gateway_address', 'ssh.port')" }, "hosts": "= self.environment == 'local' ? hosts_file(nodes_like_me) : hosts_file(nodes[:environment => '!local'])", @@ -18,5 +18,12 @@ "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'" + }, + "firewall": { + "monitor": { + "from": "sysadmin", + "to": "= ip_address", + "port": [443, 80] + } } } diff --git a/provider_base/services/monitor.rb b/provider_base/services/monitor.rb new file mode 100644 index 00000000..01590d5c --- /dev/null +++ b/provider_base/services/monitor.rb @@ -0,0 +1,3 @@ +unless self.services.include? "webapp" + LeapCli.log :error, "service `monitor` requires service `webapp` on the same node (node #{self.name})." +end diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json index 11293ae8..c7e99d85 100644 --- a/provider_base/services/mx.json +++ b/provider_base/services/mx.json @@ -1,4 +1,19 @@ { + "mx": { + // provider should define their own custom aliases. + // these are in *addition* to the standard reserved aliases for root and postmaster, etc. + "aliases": {}, + // this is the domain that is used for the OpenPGP header + "key_lookup_domain": "= global.services[:webapp].webapp.domain", + "dkim": { + // bit sizes larger than 2048 are not necessarily supported + "bit_size": 2048, + "public_key": "= remote_file_path(:dkim_pub_key) { generate_dkim_key(mx.dkim.bit_size) }", + "private_key": "= remote_file_path(:dkim_priv_key) { generate_dkim_key(mx.dkim.bit_size) }", + // generate selector based on first ten digits of pub key fingerprint: + "selector": "= fingerprint(local_file_path(:dkim_pub_key) { generate_dkim_key(mx.dkim.bit_size) }, :mode => :rsa).slice(0,10)" + } + }, "stunnel": { "clients": { "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" @@ -15,7 +30,11 @@ "password": "= secret :couch_leap_mx_password", "salt": "= hex_secret :couch_leap_mx_password_salt, 128" }, - "mynetworks": "= nodes['environment' => '!local'].map{|name, n| [n.ip_address, (global.facts[name]||{})['ec2_public_ipv4']]}.flatten.compact.uniq", + "mynetworks": "= host_ips(nodes)", + "rbls": ["zen.spamhaus.org"], + "clamav": { + "whitelisted_addresses": [] + }, "x509": { "use": true, "use_commercial": true, @@ -23,5 +42,12 @@ "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'" }, - "service_type": "user_service" + "service_type": "user_service", + "firewall": { + "mx": { + "from": "*", + "to": "= ip_address", + "port": [25, 465] + } + } } diff --git a/provider_base/services/mx.rb b/provider_base/services/mx.rb new file mode 100644 index 00000000..03ee561f --- /dev/null +++ b/provider_base/services/mx.rb @@ -0,0 +1 @@ +apply_partial('_api_tester') diff --git a/provider_base/services/openvpn.json b/provider_base/services/openvpn.json index 11cb0dc2..6f73e31c 100644 --- a/provider_base/services/openvpn.json +++ b/provider_base/services/openvpn.json @@ -34,5 +34,12 @@ "port" : "= rand_range('scramblesuit_port_'+name, 18000..32000)" }, "gateway_address": "= openvpn.gateway_address" + }, + "firewall": { + "vpn": { + "from": "*", + "to": "= openvpn.gateway_address", + "port": "= openvpn.ports + [obfsproxy.scramblesuit.port]" + } } } diff --git a/provider_base/services/soledad.json b/provider_base/services/soledad.json index ed6fbc9f..169588c8 100644 --- a/provider_base/services/soledad.json +++ b/provider_base/services/soledad.json @@ -1,12 +1,21 @@ { "soledad": { "port": 2323, - "require_couchdb": "=> assert %(services.include? 'couchdb')", "couchdb_soledad_user": { "username": "= global.services[:couchdb].couch.users[:soledad].username", "password": "= secret :couch_soledad_password", "salt": "= hex_secret :couch_soledad_password_salt, 128" + }, + "couchdb_leap_mx_user": { + "username": "= global.services[:couchdb].couch.users[:leap_mx].username" } }, - "service_type": "public_service" + "service_type": "public_service", + "firewall": { + "soledad": { + "from": "*", + "to": "= ip_address", + "port": "= soledad.port" + } + } } diff --git a/provider_base/services/soledad.rb b/provider_base/services/soledad.rb new file mode 100644 index 00000000..9b220c39 --- /dev/null +++ b/provider_base/services/soledad.rb @@ -0,0 +1,3 @@ +unless self.services.include? "couchdb" + LeapCli.log :error, "service `soledad` requires service `couchdb` on the same node (node #{self.name})." +end diff --git a/provider_base/services/static.json b/provider_base/services/static.json index d9f52b36..2f408ec1 100644 --- a/provider_base/services/static.json +++ b/provider_base/services/static.json @@ -9,5 +9,12 @@ "client_version": "= static.bootstrap_files.enabled ? provider.client_version : nil" } }, - "service_type": "public_service" + "service_type": "public_service", + "firewall": { + "static": { + "from": "*", + "to": "= ip_address", + "port": [80, 443] + } + } }
\ No newline at end of file diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index 941f4f61..b1d2ca59 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -9,7 +9,7 @@ "owner", "owners", "postmaster", "reply", "robot", "ssladmin", "staff", "support", "tech-support", "tech_support", "techsupport", "ticket", "tickets", "vmail", "www-data"], - "domain": "= domain.full_suffix", + "domain": "= provider.domain", "modules": ["user", "billing", "help"], "couchdb_webapp_user": "= global.services[:couchdb].couch.users[:webapp]", "couchdb_admin_user": "= global.services[:couchdb].couch.users[:admin]", @@ -20,7 +20,7 @@ "allow_anonymous_certs": "= provider.service.allow_anonymous", "allow_registration": "= provider.service.allow_registration", "default_service_level": "= provider.service.default_service_level", - "service_levels": "= provider.service.levels", + "service_levels": "= service_levels()", "secret_token": "= secret :webapp_secret_token", "api_version": 1, "secure": false, @@ -31,7 +31,13 @@ }, "engines": [ "support" - ] + ], + "locales": "= provider.languages", + "default_locale": "= provider.default_language", + "api_tokens": { + "monitor": "= secret :api_monitor_auth_token", + "allowed_ips": "= host_ips(nodes_like_me)" + } }, "stunnel": { "clients": { @@ -53,7 +59,10 @@ "service_type": "public_service", "api": { "domain": "= 'api.' + webapp.domain", - "port": 4430 + "version": 1, + "port": 4430, + "ca_cert_uri": "= 'https://' + webapp.domain + '/ca.crt'", + "uri": "= %(https://#{api.domain}:#{api.port}/#{api.version})" }, "nickserver": { "domain": "= 'nicknym.' + domain.full_suffix", @@ -73,5 +82,12 @@ "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`.'", "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`.'" + }, + "firewall": { + "webapp": { + "from": "*", + "to": "= ip_address", + "port": "= [api.port, 443, 80, nickserver.port]" + } } } diff --git a/provider_base/tags/development.json b/provider_base/tags/development.json index d9c2c007..caf18e9d 100644 --- a/provider_base/tags/development.json +++ b/provider_base/tags/development.json @@ -1,7 +1,3 @@ { - "environment": "development", - "domain": { - "full_suffix": "= 'dev.' + provider.domain", - "internal_suffix": "= 'dev.' + provider.domain_internal" - } + "environment": "development" }
\ No newline at end of file diff --git a/provider_base/templates/common.json b/provider_base/templates/common.json new file mode 100644 index 00000000..a7675b15 --- /dev/null +++ b/provider_base/templates/common.json @@ -0,0 +1,3 @@ +{ + "ip_address": "REQUIRED" +}
\ No newline at end of file diff --git a/provider_base/templates/couchdb.json b/provider_base/templates/couchdb.json new file mode 100644 index 00000000..34b60915 --- /dev/null +++ b/provider_base/templates/couchdb.json @@ -0,0 +1,5 @@ +{ + "couch": { + "mode": "plain" + } +} diff --git a/provider_base/templates/openvpn.json b/provider_base/templates/openvpn.json new file mode 100644 index 00000000..cbe183e8 --- /dev/null +++ b/provider_base/templates/openvpn.json @@ -0,0 +1,7 @@ +{ + "openvpn": { + "gateway_address": "REQUIRED", + "ports": ["443"], + "protocols": ["tcp"] + } +} diff --git a/puppet/lib/puppet/parser/functions/sorted_yaml.rb b/puppet/lib/puppet/parser/functions/sorted_yaml.rb new file mode 100644 index 00000000..46cd46ce --- /dev/null +++ b/puppet/lib/puppet/parser/functions/sorted_yaml.rb @@ -0,0 +1,400 @@ +# encoding: UTF-8 +# +# provides sorted_yaml() function, using Ya2YAML. +# see https://github.com/afunai/ya2yaml +# + +class Ya2YAML + # + # Author:: Akira FUNAI + # Copyright:: Copyright (c) 2006-2010 Akira FUNAI + # License:: MIT License + # + + def initialize(opts = {}) + options = opts.dup + options[:indent_size] = 2 if options[:indent_size].to_i <= 0 + options[:minimum_block_length] = 0 if options[:minimum_block_length].to_i <= 0 + options.update( + { + :printable_with_syck => true, + :escape_b_specific => true, + :escape_as_utf8 => true, + } + ) if options[:syck_compatible] + + @options = options + end + + def _ya2yaml(obj) + #raise 'set $KCODE to "UTF8".' if (RUBY_VERSION < '1.9.0') && ($KCODE != 'UTF8') + if (RUBY_VERSION < '1.9.0') + $KCODE = 'UTF8' + end + '--- ' + emit(obj, 1) + "\n" + rescue SystemStackError + raise ArgumentError, "ya2yaml can't handle circular references" + end + + private + + def emit(obj, level) + case obj + when Array + if (obj.length == 0) + '[]' + else + indent = "\n" + s_indent(level - 1) + ### + ### NOTE: a minor modification to normal Ya2YAML... + ### We want arrays to be output in sorted order, not just + ### Hashes. + ### + #obj.collect {|o| + # indent + '- ' + emit(o, level + 1) + #}.join('') + obj.sort {|a,b| a.to_s <=> b.to_s}.collect {|o| + indent + '- ' + emit(o, level + 1) + }.join('') + end + when Hash + if (obj.length == 0) + '{}' + else + indent = "\n" + s_indent(level - 1) + hash_order = @options[:hash_order] + if (hash_order && level == 1) + hash_keys = obj.keys.sort {|x, y| + x_order = hash_order.index(x) ? hash_order.index(x) : Float::MAX + y_order = hash_order.index(y) ? hash_order.index(y) : Float::MAX + o = (x_order <=> y_order) + (o != 0) ? o : (x.to_s <=> y.to_s) + } + elsif @options[:preserve_order] + hash_keys = obj.keys + else + hash_keys = obj.keys.sort {|x, y| x.to_s <=> y.to_s } + end + hash_keys.collect {|k| + key = emit(k, level + 1) + if ( + is_one_plain_line?(key) || + key =~ /\A(#{REX_BOOL}|#{REX_FLOAT}|#{REX_INT}|#{REX_NULL})\z/x + ) + indent + key + ': ' + emit(obj[k], level + 1) + else + indent + '? ' + key + + indent + ': ' + emit(obj[k], level + 1) + end + }.join('') + end + when NilClass + '~' + when String + emit_string(obj, level) + when TrueClass, FalseClass + obj.to_s + when Fixnum, Bignum, Float + obj.to_s + when Date + obj.to_s + when Time + offset = obj.gmtoff + off_hm = sprintf( + '%+.2d:%.2d', + (offset / 3600.0).to_i, + (offset % 3600.0) / 60 + ) + u_sec = (obj.usec != 0) ? sprintf(".%.6d", obj.usec) : '' + obj.strftime("%Y-%m-%d %H:%M:%S#{u_sec} #{off_hm}") + when Symbol + '!ruby/symbol ' + emit_string(obj.to_s, level) + when Range + '!ruby/range ' + obj.to_s + when Regexp + '!ruby/regexp ' + obj.inspect + else + case + when obj.is_a?(Struct) + struct_members = {} + obj.each_pair{|k, v| struct_members[k.to_s] = v } + '!ruby/struct:' + obj.class.to_s.sub(/^(Struct::(.+)|.*)$/, '\2') + ' ' + + emit(struct_members, level + 1) + else + # serialized as a generic object + object_members = {} + obj.instance_variables.each{|k, v| + object_members[k.to_s.sub(/^@/, '')] = obj.instance_variable_get(k) + } + '!ruby/object:' + obj.class.to_s + ' ' + + emit(object_members, level + 1) + end + end + end + + def emit_string(str, level) + (is_string, is_printable, is_one_line, is_one_plain_line) = string_type(str) + if is_string + if is_printable + if is_one_plain_line + emit_simple_string(str, level) + else + (is_one_line || str.length < @options[:minimum_block_length]) ? + emit_quoted_string(str, level) : + emit_block_string(str, level) + end + else + emit_quoted_string(str, level) + end + else + emit_base64_binary(str, level) + end + end + + def emit_simple_string(str, level) + str + end + + def emit_block_string(str, level) + str = normalize_line_break(str) + + indent = s_indent(level) + indentation_indicator = (str =~ /\A /) ? indent.size.to_s : '' + str =~ /(#{REX_NORMAL_LB}*)\z/ + chomping_indicator = case $1.length + when 0 + '-' + when 1 + '' + else + '+' + end + + str.chomp! + str.gsub!(/#{REX_NORMAL_LB}/) { + $1 + indent + } + '|' + indentation_indicator + chomping_indicator + "\n" + indent + str + end + + def emit_quoted_string(str, level) + str = yaml_escape(normalize_line_break(str)) + if (str.length < @options[:minimum_block_length]) + str.gsub!(/#{REX_NORMAL_LB}/) { ESCAPE_SEQ_LB[$1] } + else + str.gsub!(/#{REX_NORMAL_LB}$/) { ESCAPE_SEQ_LB[$1] } + str.gsub!(/(#{REX_NORMAL_LB}+)(.)/) { + trail_c = $3 + $1 + trail_c.sub(/([\t ])/) { ESCAPE_SEQ_WS[$1] } + } + indent = s_indent(level) + str.gsub!(/#{REX_NORMAL_LB}/) { + ESCAPE_SEQ_LB[$1] + "\\\n" + indent + } + end + '"' + str + '"' + end + + def emit_base64_binary(str, level) + indent = "\n" + s_indent(level) + base64 = [str].pack('m') + '!binary |' + indent + base64.gsub(/\n(?!\z)/, indent) + end + + def string_type(str) + if str.respond_to?(:encoding) && (!str.valid_encoding? || str.encoding == Encoding::ASCII_8BIT) + return false, false, false, false + end + (ucs_codes = str.unpack('U*')) rescue ( + # ArgumentError -> binary data + return false, false, false, false + ) + if ( + @options[:printable_with_syck] && + str =~ /\A#{REX_ANY_LB}* | #{REX_ANY_LB}*\z|#{REX_ANY_LB}{2}\z/ + ) + # detour Syck bug + return true, false, nil, false + end + ucs_codes.each {|ucs_code| + return true, false, nil, false unless is_printable?(ucs_code) + } + return true, true, is_one_line?(str), is_one_plain_line?(str) + end + + def is_printable?(ucs_code) + # YAML 1.1 / 4.1.1. + ( + [0x09, 0x0a, 0x0d, 0x85].include?(ucs_code) || + (ucs_code <= 0x7e && ucs_code >= 0x20) || + (ucs_code <= 0xd7ff && ucs_code >= 0xa0) || + (ucs_code <= 0xfffd && ucs_code >= 0xe000) || + (ucs_code <= 0x10ffff && ucs_code >= 0x10000) + ) && + !( + # treat LS/PS as non-printable characters + @options[:escape_b_specific] && + (ucs_code == 0x2028 || ucs_code == 0x2029) + ) + end + + def is_one_line?(str) + str !~ /#{REX_ANY_LB}(?!\z)/ + end + + def is_one_plain_line?(str) + # YAML 1.1 / 4.6.11. + str !~ /^([\-\?:,\[\]\{\}\#&\*!\|>'"%@`\s]|---|\.\.\.)/ && + str !~ /[:\#\s\[\]\{\},]/ && + str !~ /#{REX_ANY_LB}/ && + str !~ /^(#{REX_BOOL}|#{REX_FLOAT}|#{REX_INT}|#{REX_MERGE} + |#{REX_NULL}|#{REX_TIMESTAMP}|#{REX_VALUE})$/x + end + + def s_indent(level) + # YAML 1.1 / 4.2.2. + ' ' * (level * @options[:indent_size]) + end + + def normalize_line_break(str) + # YAML 1.1 / 4.1.4. + str.gsub(/(#{REX_CRLF}|#{REX_CR}|#{REX_NEL})/, "\n") + end + + def yaml_escape(str) + # YAML 1.1 / 4.1.6. + str.gsub(/[^a-zA-Z0-9]/u) {|c| + ucs_code, = (c.unpack('U') rescue [??]) + case + when ESCAPE_SEQ[c] + ESCAPE_SEQ[c] + when is_printable?(ucs_code) + c + when @options[:escape_as_utf8] + c.respond_to?(:bytes) ? + c.bytes.collect {|b| '\\x%.2x' % b }.join : + '\\x' + c.unpack('H2' * c.size).join('\\x') + when ucs_code == 0x2028 || ucs_code == 0x2029 + ESCAPE_SEQ_LB[c] + when ucs_code <= 0x7f + sprintf('\\x%.2x', ucs_code) + when ucs_code <= 0xffff + sprintf('\\u%.4x', ucs_code) + else + sprintf('\\U%.8x', ucs_code) + end + } + end + + module Constants + UCS_0X85 = [0x85].pack('U') # c285@UTF8 Unicode next line + UCS_0XA0 = [0xa0].pack('U') # c2a0@UTF8 Unicode non-breaking space + UCS_0X2028 = [0x2028].pack('U') # e280a8@UTF8 Unicode line separator + UCS_0X2029 = [0x2029].pack('U') # e280a9@UTF8 Unicode paragraph separator + + # non-break characters + ESCAPE_SEQ = { + "\x00" => '\\0', + "\x07" => '\\a', + "\x08" => '\\b', + "\x0b" => '\\v', + "\x0c" => '\\f', + "\x1b" => '\\e', + "\"" => '\\"', + "\\" => '\\\\', + } + + # non-breaking space + ESCAPE_SEQ_NS = { + UCS_0XA0 => '\\_', + } + + # white spaces + ESCAPE_SEQ_WS = { + "\x09" => '\\t', + " " => '\\x20', + } + + # line breaks + ESCAPE_SEQ_LB ={ + "\x0a" => '\\n', + "\x0d" => '\\r', + UCS_0X85 => '\\N', + UCS_0X2028 => '\\L', + UCS_0X2029 => '\\P', + } + + # regexps for line breaks + REX_LF = Regexp.escape("\x0a") + REX_CR = Regexp.escape("\x0d") + REX_CRLF = Regexp.escape("\x0d\x0a") + REX_NEL = Regexp.escape(UCS_0X85) + REX_LS = Regexp.escape(UCS_0X2028) + REX_PS = Regexp.escape(UCS_0X2029) + + REX_ANY_LB = /(#{REX_LF}|#{REX_CR}|#{REX_NEL}|#{REX_LS}|#{REX_PS})/ + REX_NORMAL_LB = /(#{REX_LF}|#{REX_LS}|#{REX_PS})/ + + # regexps for language-Independent types for YAML1.1 + REX_BOOL = / + y|Y|yes|Yes|YES|n|N|no|No|NO + |true|True|TRUE|false|False|FALSE + |on|On|ON|off|Off|OFF + /x + REX_FLOAT = / + [-+]?([0-9][0-9_]*)?\.[0-9.]*([eE][-+][0-9]+)? # (base 10) + |[-+]?[0-9][0-9_]*(:[0-5]?[0-9])+\.[0-9_]* # (base 60) + |[-+]?\.(inf|Inf|INF) # (infinity) + |\.(nan|NaN|NAN) # (not a number) + /x + REX_INT = / + [-+]?0b[0-1_]+ # (base 2) + |[-+]?0[0-7_]+ # (base 8) + |[-+]?(0|[1-9][0-9_]*) # (base 10) + |[-+]?0x[0-9a-fA-F_]+ # (base 16) + |[-+]?[1-9][0-9_]*(:[0-5]?[0-9])+ # (base 60) + /x + REX_MERGE = / + << + /x + REX_NULL = / + ~ # (canonical) + |null|Null|NULL # (English) + | # (Empty) + /x + REX_TIMESTAMP = / + [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] # (ymd) + |[0-9][0-9][0-9][0-9] # (year) + -[0-9][0-9]? # (month) + -[0-9][0-9]? # (day) + ([Tt]|[ \t]+)[0-9][0-9]? # (hour) + :[0-9][0-9] # (minute) + :[0-9][0-9] # (second) + (\.[0-9]*)? # (fraction) + (([ \t]*)Z|[-+][0-9][0-9]?(:[0-9][0-9])?)? # (time zone) + /x + REX_VALUE = / + = + /x + end + + include Constants +end + +module Puppet::Parser::Functions + newfunction(:sorted_yaml, + :type => :rvalue, + :doc => "This function outputs yaml, but ensures the keys are sorted." + ) do |arguments| + + if arguments.is_a?(Array) + if arguments.size != 1 + raise(Puppet::ParseError, "sorted_yaml(): Wrong number of arguments given (#{arguments.size} for 1)") + end + yaml = arguments.first + else + yaml = arguments + end + return Ya2YAML.new()._ya2yaml(yaml) + end +end diff --git a/puppet/manifests/site.pp b/puppet/manifests/site.pp index 912234ac..ecda4012 100644 --- a/puppet/manifests/site.pp +++ b/puppet/manifests/site.pp @@ -2,30 +2,37 @@ # the logoutput exec parameter defaults to "on_error" in puppet 3, # but to "false" in puppet 2.7, so we need to set this globally here Exec { - logoutput => on_failure, - path => '/usr/bin:/usr/sbin/:/bin:/sbin:/usr/local/bin:/usr/local/sbin' + logoutput => on_failure, + path => '/usr/bin:/usr/sbin/:/bin:/sbin:/usr/local/bin:/usr/local/sbin' } -include site_config::setup -include site_config::default +Package <| provider == 'apt' |> { + install_options => ['--no-install-recommends'], +} $services = hiera('services', []) $services_str = join($services, ', ') notice("Services for ${fqdn}: ${services_str}") +# In the default deployment case, we want to run an 'apt-get dist-upgrade' +# to ensure the latest packages are installed. This is done by including the +# class 'site_config::slow' here. However, you only changed a small bit of +# the platform and want to skip this slow part of deployment, you can do that +# by using 'leap deploy --fast' which will only apply those resources that are +# tagged with 'leap_base' or 'leap_service'. +# See https://leap.se/en/docs/platform/details/under-the-hood#tags +include site_config::slow + if member($services, 'openvpn') { include site_openvpn - include site_obfsproxy } if member($services, 'couchdb') { include site_couchdb - include tapicero } if member($services, 'webapp') { include site_webapp - include site_nickserver } if member($services, 'soledad') { @@ -51,5 +58,3 @@ if member($services, 'static') { if member($services, 'obfsproxy') { include site_obfsproxy } - -include site_config::packages::uninstall diff --git a/puppet/modules/apache b/puppet/modules/apache -Subproject c3e92a9b3cb02f1546b6b1570f10a968d380005 +Subproject 117bed9a9263c21d253d86b667eb165948efdc2 diff --git a/puppet/modules/apt b/puppet/modules/apt -Subproject fca103484ddc1f647a54135b6a902edabf45955 +Subproject 33c61e8df59db1abbed379a9e9790946060a8f1 diff --git a/puppet/modules/backupninja b/puppet/modules/backupninja -Subproject daeb1a1f112a4dbf6b39565f0dea461e46a6468 +Subproject 497513547be79f9d3c8e96f1650ec43ee634b27 diff --git a/puppet/modules/check_mk b/puppet/modules/check_mk -Subproject 205859d87884ac4ceee6d1365548e7dc55640bf +Subproject aa02571537af90ac73309e6e216c9417802548c diff --git a/puppet/modules/clamav/files/01-leap.conf b/puppet/modules/clamav/files/01-leap.conf new file mode 100644 index 00000000..a7e49d17 --- /dev/null +++ b/puppet/modules/clamav/files/01-leap.conf @@ -0,0 +1,58 @@ +# If running clamd in "LocalSocket" mode (*NOT* in TCP/IP mode), and +# either "SOcket Cat" (socat) or the "IO::Socket::UNIX" perl module +# are installed on the system, and you want to report whether clamd +# is running or not, uncomment the "clamd_socket" variable below (you +# will be warned if neither socat nor IO::Socket::UNIX are found, but +# the script will still run). You will also need to set the correct +# path to your clamd socket file (if unsure of the path, check the +# "LocalSocket" setting in your clamd.conf file for socket location). +clamd_socket="/run/clamav/clamd.ctl" + +# If you would like to attempt to restart ClamD if detected not running, +# uncomment the next 2 lines. Confirm the path to the "clamd_lock" file +# (usually can be found in the clamd init script) and also enter the clamd +# start command for your particular distro for the "start_clamd" variable +# (the sample start command shown below should work for most linux distros). +# NOTE: these 2 variables are dependant on the "clamd_socket" variable +# shown above - if not enabled, then the following 2 variables will be +# ignored, whether enabled or not. +clamd_lock="/run/clamav/clamd.pid" +start_clamd="clamdscan --reload" + +ss_dbs=" + junk.ndb + phish.ndb + rogue.hdb + sanesecurity.ftm + scam.ndb + sigwhitelist.ign2 + spamattach.hdb + spamimg.hdb + winnow.attachments.hdb + winnow_bad_cw.hdb + winnow_extended_malware.hdb + winnow_malware.hdb + winnow_malware_links.ndb + malwarehash.hsb + doppelstern.hdb + bofhland_cracked_URL.ndb + bofhland_malware_attach.hdb + bofhland_malware_URL.ndb + bofhland_phishing_URL.ndb + crdfam.clamav.hdb + phishtank.ndb + porcupine.ndb + spear.ndb + spearl.ndb +" + +# ======================== +# SecuriteInfo Database(s) +# ======================== +# Add or remove database file names between quote marks as needed. To +# disable any SecuriteInfo database downloads, remove the appropriate +# lines below. To disable all SecuriteInfo database file downloads, +# comment all of the following lines. +si_dbs="" + +mbl_dbs=""
\ No newline at end of file diff --git a/puppet/modules/clamav/files/clamav-daemon_default b/puppet/modules/clamav/files/clamav-daemon_default new file mode 100644 index 00000000..b4cd6a4f --- /dev/null +++ b/puppet/modules/clamav/files/clamav-daemon_default @@ -0,0 +1,8 @@ +# This is a file designed only t0 set special environment variables +# eg TMP or TMPDIR. It is sourced from a shell script, so anything +# put in here must be in variable=value format, suitable for sourcing +# from a shell script. +# Examples: +# export TMPDIR=/dev/shm +export TMP=/var/tmp +export TMPDIR=/var/tmp diff --git a/puppet/modules/clamav/files/clamav-milter_default b/puppet/modules/clamav/files/clamav-milter_default new file mode 100644 index 00000000..5e33e822 --- /dev/null +++ b/puppet/modules/clamav/files/clamav-milter_default @@ -0,0 +1,14 @@ +# +# clamav-milter init options +# + +## SOCKET_RWGROUP +# by default, the socket created by the milter has permissions +# clamav:clamav:755. SOCKET_RWGROUP changes the group and changes the +# permissions to 775 to give read-write access to that group. +# +# If you are using postfix to speak to the milter, you have to give permission +# to the postfix group to write +# +SOCKET_RWGROUP=postfix +export TMPDIR=/var/tmp diff --git a/puppet/modules/clamav/manifests/daemon.pp b/puppet/modules/clamav/manifests/daemon.pp new file mode 100644 index 00000000..2e13a8fb --- /dev/null +++ b/puppet/modules/clamav/manifests/daemon.pp @@ -0,0 +1,91 @@ +# deploy clamav daemon +class clamav::daemon { + + $domain_hash = hiera('domain') + $domain = $domain_hash['full_suffix'] + + package { [ 'clamav-daemon', 'arj' ]: + ensure => installed; + } + + service { + 'clamav-daemon': + ensure => running, + name => clamav-daemon, + pattern => '/usr/sbin/clamd', + enable => true, + hasrestart => true, + subscribe => File['/etc/default/clamav-daemon'], + require => Package['clamav-daemon']; + } + + file { + '/var/run/clamav': + ensure => directory, + mode => '0750', + owner => clamav, + group => postfix, + require => [Package['postfix'], Package['clamav-daemon']]; + + '/var/lib/clamav': + mode => '0755', + owner => clamav, + group => clamav, + require => Package['clamav-daemon']; + + '/etc/default/clamav-daemon': + source => 'puppet:///modules/clamav/clamav-daemon_default', + mode => '0644', + owner => root, + group => root; + + # this file contains additional domains that we want the clamav + # phishing process to look for (our domain) + '/var/lib/clamav/local.pdb': + content => template('clamav/local.pdb.erb'), + mode => '0644', + owner => clamav, + group => clamav, + require => Package['clamav-daemon']; + } + + file_line { + 'clamav_daemon_tmp': + path => '/etc/clamav/clamd.conf', + line => 'TemporaryDirectory /var/tmp', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + + 'enable_phishscanurls': + path => '/etc/clamav/clamd.conf', + match => 'PhishingScanURLs no', + line => 'PhishingScanURLs yes', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + + 'clamav_LogSyslog_true': + path => '/etc/clamav/clamd.conf', + match => '^LogSyslog false', + line => 'LogSyslog true', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + + 'clamav_MaxThreads': + path => '/etc/clamav/clamd.conf', + match => 'MaxThreads 20', + line => 'MaxThreads 100', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + } + + # remove LogFile line + file_line { + 'clamav_LogFile': + path => '/etc/clamav/clamd.conf', + match => '^LogFile .*', + line => '', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + } + +} diff --git a/puppet/modules/clamav/manifests/freshclam.pp b/puppet/modules/clamav/manifests/freshclam.pp new file mode 100644 index 00000000..80c822a4 --- /dev/null +++ b/puppet/modules/clamav/manifests/freshclam.pp @@ -0,0 +1,23 @@ +class clamav::freshclam { + + package { 'clamav-freshclam': ensure => installed } + + service { + 'freshclam': + ensure => running, + enable => true, + name => clamav-freshclam, + pattern => '/usr/bin/freshclam', + hasrestart => true, + require => Package['clamav-freshclam']; + } + + file_line { + 'freshclam_notify': + path => '/etc/clamav/freshclam.conf', + line => 'NotifyClamd /etc/clamav/clamd.conf', + require => Package['clamav-freshclam'], + notify => Service['freshclam']; + } + +} diff --git a/puppet/modules/clamav/manifests/init.pp b/puppet/modules/clamav/manifests/init.pp new file mode 100644 index 00000000..de8fb4dc --- /dev/null +++ b/puppet/modules/clamav/manifests/init.pp @@ -0,0 +1,8 @@ +class clamav { + + include clamav::daemon + include clamav::milter + include clamav::unofficial_sigs + include clamav::freshclam + +} diff --git a/puppet/modules/clamav/manifests/milter.pp b/puppet/modules/clamav/manifests/milter.pp new file mode 100644 index 00000000..e8a85e3f --- /dev/null +++ b/puppet/modules/clamav/manifests/milter.pp @@ -0,0 +1,50 @@ +class clamav::milter { + + $clamav = hiera('clamav') + $whitelisted_addresses = $clamav['whitelisted_addresses'] + $domain_hash = hiera('domain') + $domain = $domain_hash['full_suffix'] + + package { 'clamav-milter': ensure => installed } + + service { + 'clamav-milter': + ensure => running, + enable => true, + name => clamav-milter, + pattern => '/usr/sbin/clamav-milter', + hasrestart => true, + require => Package['clamav-milter'], + subscribe => File['/etc/default/clamav-milter']; + } + + file { + '/run/clamav/milter.ctl': + mode => '0666', + owner => clamav, + group => postfix, + require => Class['clamav::daemon']; + + '/etc/clamav/clamav-milter.conf': + content => template('clamav/clamav-milter.conf.erb'), + mode => '0644', + owner => root, + group => root, + require => Package['clamav-milter'], + subscribe => Service['clamav-milter']; + + '/etc/default/clamav-milter': + source => 'puppet:///modules/clamav/clamav-milter_default', + mode => '0644', + owner => root, + group => root; + + '/etc/clamav/whitelisted_addresses': + content => template('clamav/whitelisted_addresses.erb'), + mode => '0644', + owner => root, + group => root, + require => Package['clamav-milter']; + } + +} diff --git a/puppet/modules/clamav/manifests/unofficial_sigs.pp b/puppet/modules/clamav/manifests/unofficial_sigs.pp new file mode 100644 index 00000000..2d849585 --- /dev/null +++ b/puppet/modules/clamav/manifests/unofficial_sigs.pp @@ -0,0 +1,23 @@ +class clamav::unofficial_sigs { + + package { 'clamav-unofficial-sigs': + ensure => installed + } + + ensure_packages(['wget', 'gnupg', 'socat', 'rsync', 'curl']) + + file { + '/var/log/clamav-unofficial-sigs.log': + ensure => file, + owner => clamav, + group => clamav, + require => Package['clamav-unofficial-sigs']; + + '/etc/clamav-unofficial-sigs.conf.d/01-leap.conf': + source => 'puppet:///modules/clamav/01-leap.conf', + mode => '0755', + owner => root, + group => root, + require => Package['clamav-unofficial-sigs']; + } +} diff --git a/puppet/modules/clamav/templates/clamav-milter.conf.erb b/puppet/modules/clamav/templates/clamav-milter.conf.erb new file mode 100644 index 00000000..9bf7099e --- /dev/null +++ b/puppet/modules/clamav/templates/clamav-milter.conf.erb @@ -0,0 +1,28 @@ +# THIS FILE MANAGED BY PUPPET +MilterSocket /var/run/clamav/milter.ctl +FixStaleSocket true +User clamav +MilterSocketGroup clamav +MilterSocketMode 666 +AllowSupplementaryGroups true +ReadTimeout 120 +Foreground false +PidFile /var/run/clamav/clamav-milter.pid +ClamdSocket unix:/var/run/clamav/clamd.ctl +OnClean Accept +OnInfected Reject +OnFail Defer +AddHeader Replace +LogSyslog true +LogFacility LOG_LOCAL6 +LogVerbose yes +LogInfected Basic +LogTime true +LogFileUnlock false +LogClean Off +LogRotate true +SupportMultipleRecipients false +MaxFileSize 10M +TemporaryDirectory /var/tmp +RejectMsg "Message refused due to content violation: %v - contact https://<%= @domain %>/tickets/new if this is in error" +Whitelist /etc/clamav/whitelisted_addresses diff --git a/puppet/modules/clamav/templates/local.pdb.erb b/puppet/modules/clamav/templates/local.pdb.erb new file mode 100644 index 00000000..9ea0584a --- /dev/null +++ b/puppet/modules/clamav/templates/local.pdb.erb @@ -0,0 +1 @@ +H:<%= @domain %> diff --git a/puppet/modules/clamav/templates/whitelisted_addresses.erb b/puppet/modules/clamav/templates/whitelisted_addresses.erb new file mode 100644 index 00000000..9e068ec5 --- /dev/null +++ b/puppet/modules/clamav/templates/whitelisted_addresses.erb @@ -0,0 +1,5 @@ +<%- if @whitelisted_addresses then -%> +<% @whitelisted_addresses.each do |name| -%> +From::<%= name %> +<% end -%> +<% end -%> diff --git a/puppet/modules/couchdb b/puppet/modules/couchdb -Subproject 3c20a3169e77e5a5f9abc06788c3a7730d5530c +Subproject 40d2289f8e10625cd45fdccdf492b5fb6490e66 diff --git a/puppet/modules/haproxy b/puppet/modules/haproxy -Subproject b398f3cb0a67d1170d0564a3f03977f9a08c2b6 +Subproject af322a73c013f80a958ab7d5d31d0c75cf6d052 diff --git a/puppet/modules/journald/manifests/init.pp b/puppet/modules/journald/manifests/init.pp new file mode 100644 index 00000000..879baba4 --- /dev/null +++ b/puppet/modules/journald/manifests/init.pp @@ -0,0 +1,7 @@ +class journald { + + service { 'systemd-journald': + ensure => running, + enable => true, + } +} diff --git a/puppet/modules/leap/manifests/cli/install.pp b/puppet/modules/leap/manifests/cli/install.pp new file mode 100644 index 00000000..25e87033 --- /dev/null +++ b/puppet/modules/leap/manifests/cli/install.pp @@ -0,0 +1,46 @@ +# installs leap_cli on node +class leap::cli::install ( $source = false ) { + if $source { + # needed for building leap_cli from source + include ::git + include ::rubygems + + class { '::ruby': + install_dev => true + } + + class { 'bundler::install': install_method => 'package' } + + Class[Ruby] -> + Class[rubygems] -> + Class[bundler::install] + + + vcsrepo { '/srv/leap/cli': + ensure => present, + force => true, + revision => 'develop', + provider => 'git', + source => 'https://leap.se/git/leap_cli.git', + owner => 'root', + group => 'root', + notify => Exec['install_leap_cli'], + require => Package['git'] + } + + exec { 'install_leap_cli': + command => '/usr/bin/rake build && /usr/bin/rake install', + cwd => '/srv/leap/cli', + user => 'root', + environment => 'USER=root', + refreshonly => true, + require => [ Class[bundler::install] ] + } + } + else { + package { 'leap_cli': + ensure => installed, + provider => gem + } + } +} diff --git a/puppet/modules/leap/manifests/logfile.pp b/puppet/modules/leap/manifests/logfile.pp index 63dbd16b..adb3ca8a 100644 --- a/puppet/modules/leap/manifests/logfile.pp +++ b/puppet/modules/leap/manifests/logfile.pp @@ -1,9 +1,18 @@ # # make syslog log to a particular file for a particular process. # - -define leap::logfile($process=$name) { - $logfile = "/var/log/leap/${name}.log" +# arguments: +# +# * name: what config files are named as (eg. /etc/rsyslog.d/50-$name.conf) +# * log: the full path of the log file (defaults to /var/log/leap/$name.log +# * process: the syslog tag to filter on (defaults to name) +# +define leap::logfile($process = $name, $log = undef) { + if $log { + $logfile = $log + } else { + $logfile = "/var/log/leap/${name}.log" + } rsyslog::snippet { "50-${name}": content => template('leap/rsyslog.erb') diff --git a/puppet/modules/leap_mx/manifests/init.pp b/puppet/modules/leap_mx/manifests/init.pp index 284662d2..d758e3ab 100644 --- a/puppet/modules/leap_mx/manifests/init.pp +++ b/puppet/modules/leap_mx/manifests/init.pp @@ -1,3 +1,4 @@ +# deploy leap mx service class leap_mx { $leap_mx = hiera('couchdb_leap_mx_user') @@ -10,23 +11,66 @@ class leap_mx { $sources = hiera('sources') include soledad::common - include site_apt::preferences::twisted # # USER AND GROUP # + # Make the user for leap-mx. This user is where all legitimate, non-system + # mail is delivered so leap-mx can process it. Previously, we let the system + # pick a uid/gid, but we need to know what they are set to in order to set the + # virtual_uid_maps and virtual_gid_maps. Its a bit overkill write a fact just + # for this, so instead we pick arbitrary numbers that seem unlikely to be used + # and then use them in the postfix configuration group { 'leap-mx': ensure => present, + gid => 42424, allowdupe => false; } user { 'leap-mx': - ensure => present, - allowdupe => false, - gid => 'leap-mx', - home => '/etc/leap', - require => Group['leap-mx']; + ensure => present, + comment => 'Leap Mail', + allowdupe => false, + uid => 42424, + gid => 'leap-mx', + home => '/var/mail/leap-mx', + shell => '/bin/false', + managehome => true, + require => Group['leap-mx']; + } + + file { + '/var/mail/leap-mx': + ensure => directory, + owner => 'leap-mx', + group => 'leap-mx', + mode => '0755', + require => User['leap-mx']; + + '/var/mail/leap-mx/Maildir': + ensure => directory, + owner => 'leap-mx', + group => 'leap-mx', + mode => '0700'; + + '/var/mail/leap-mx/Maildir/new': + ensure => directory, + owner => 'leap-mx', + group => 'leap-mx', + mode => '0700'; + + '/var/mail/leap-mx/Maildir/cur': + ensure => directory, + owner => 'leap-mx', + group => 'leap-mx', + mode => '0700'; + + '/var/mail/leap-mx/Maildir/tmp': + ensure => directory, + owner => 'leap-mx', + group => 'leap-mx', + mode => '0700'; } # @@ -41,12 +85,9 @@ class leap_mx { notify => Service['leap-mx']; } - file { '/etc/default/leap_mx': - content => 'LOGFILE=/var/log/leap/mx.log', - owner => 'root', - group => 'root', - mode => '0644', - notify => Service['leap-mx']; + leap::logfile { 'leap-mx': + log => '/var/log/leap/mx.log', + process => 'leap-mx' } # @@ -57,8 +98,8 @@ class leap_mx { $sources['leap-mx']['package']: ensure => $sources['leap-mx']['revision'], require => [ - Class['site_apt::preferences::twisted'], - Class['site_apt::leap_repo'] ]; + Class['site_apt::leap_repo'], + User['leap-mx'] ]; 'leap-keymanager': ensure => latest; @@ -75,20 +116,4 @@ class leap_mx { hasrestart => true, require => [ Package['leap-mx'] ]; } - - augeas { - 'logrotate_mx': - context => '/files/etc/logrotate.d/leap-mx/rule', - changes => [ - 'set file /var/log/leap/mx.log', - 'set rotate 5', - 'set schedule daily', - 'clear nocreate', - 'rm create', - 'rm ifempty', - 'set compress compress', - 'set missingok missingok', - 'set copytruncate copytruncate' - ] - } } diff --git a/puppet/modules/leap_mx/templates/mx.conf.erb b/puppet/modules/leap_mx/templates/mx.conf.erb index e05bc150..b54b3a86 100644 --- a/puppet/modules/leap_mx/templates/mx.conf.erb +++ b/puppet/modules/leap_mx/templates/mx.conf.erb @@ -1,5 +1,5 @@ [mail1] -path=/var/mail/vmail/Maildir +path=/var/mail/leap-mx/Maildir recursive=True [couchdb] @@ -13,3 +13,6 @@ port=4242 [check recipient] port=2244 + +[fingerprint map] +port=2424 diff --git a/puppet/modules/nagios b/puppet/modules/nagios -Subproject b55f23d4d90c97cec08251544aa9700df86ad0b +Subproject 68dab01a85996e14efcccf856b623a2caf25782 diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp index 61714fdf..6a3d2c72 100644 --- a/puppet/modules/obfsproxy/manifests/init.pp +++ b/puppet/modules/obfsproxy/manifests/init.pp @@ -1,3 +1,4 @@ +# deploy obfsproxy service class obfsproxy ( $transport, $bind_address, @@ -23,18 +24,18 @@ class obfsproxy ( } file { '/etc/init.d/obfsproxy': - path => '/etc/init.d/obfsproxy', - ensure => present, - source => 'puppet:///modules/obfsproxy/obfsproxy_init', - owner => 'root', - group => 'root', - mode => '0750', - require => File[$conf], + ensure => present, + path => '/etc/init.d/obfsproxy', + source => 'puppet:///modules/obfsproxy/obfsproxy_init', + owner => 'root', + group => 'root', + mode => '0750', + require => File[$conf], } file { $conf : - path => $conf, ensure => present, + path => $conf, owner => 'root', group => 'root', mode => '0600', @@ -67,8 +68,7 @@ class obfsproxy ( } package { 'obfsproxy': - ensure => present, - require => Class['site_apt::preferences::obfsproxy'], + ensure => present } service { 'obfsproxy': diff --git a/puppet/modules/opendkim/manifests/init.pp b/puppet/modules/opendkim/manifests/init.pp new file mode 100644 index 00000000..4d4c5312 --- /dev/null +++ b/puppet/modules/opendkim/manifests/init.pp @@ -0,0 +1,67 @@ +# +# I am not sure about what issues might arise with DKIM key sizes +# larger than 2048. It might or might not be supported. See: +# http://dkim.org/specs/rfc4871-dkimbase.html#rfc.section.3.3.3 +# +class opendkim { + + $domain_hash = hiera('domain') + $domain = $domain_hash['full_suffix'] + $mx = hiera('mx') + $dkim = $mx['dkim'] + $selector = $dkim['selector'] + $dkim_cert = $dkim['public_key'] + $dkim_key = $dkim['private_key'] + + ensure_packages(['opendkim', 'libvbr2']) + + # postfix user needs to be in the opendkim group + # in order to access the opendkim socket located at: + # local:/var/run/opendkim/opendkim.sock + user { 'postfix': + groups => 'opendkim', + require => Package['opendkim']; + } + + service { 'opendkim': + ensure => running, + enable => true, + hasstatus => true, + hasrestart => true, + subscribe => File[$dkim_key]; + } + + file { + '/etc/opendkim.conf': + ensure => file, + content => template('opendkim/opendkim.conf'), + mode => '0644', + owner => root, + group => root, + notify => Service['opendkim'], + require => Package['opendkim']; + + '/etc/default/opendkim.conf': + ensure => file, + content => 'SOCKET="inet:8891@localhost" # listen on loopback on port 8891', + mode => '0644', + owner => root, + group => root, + notify => Service['opendkim'], + require => Package['opendkim']; + + $dkim_key: + ensure => file, + mode => '0600', + owner => 'opendkim', + group => 'opendkim', + require => Package['opendkim']; + + $dkim_cert: + ensure => file, + mode => '0600', + owner => 'opendkim', + group => 'opendkim', + require => Package['opendkim']; + } +} diff --git a/puppet/modules/opendkim/templates/opendkim.conf b/puppet/modules/opendkim/templates/opendkim.conf new file mode 100644 index 00000000..5a948229 --- /dev/null +++ b/puppet/modules/opendkim/templates/opendkim.conf @@ -0,0 +1,45 @@ +# This is a basic configuration that can easily be adapted to suit a standard +# installation. For more advanced options, see opendkim.conf(5) and/or +# /usr/share/doc/opendkim/examples/opendkim.conf.sample. + +# Log to syslog +Syslog yes +SyslogSuccess yes +LogWhy no +# Required to use local socket with MTAs that access the socket as a non- +# privileged user (e.g. Postfix) +UMask 002 + +Domain <%= @domain %> +SubDomains yes + +# set internal hosts to all the known hosts, like mydomains? + +# can we generate a larger key and get it in dns? +KeyFile <%= @dkim_key %> + +Selector <%= @selector %> + +# Commonly-used options; the commented-out versions show the defaults. +Canonicalization relaxed +#Mode sv +#ADSPDiscard no + +SignatureAlgorithm rsa-sha256 + +# Always oversign From (sign using actual From and a null From to prevent +# malicious signatures header fields (From and/or others) between the signer +# and the verifier. From is oversigned by default in the Debian pacakge +# because it is often the identity key used by reputation systems and thus +# somewhat security sensitive. +OversignHeaders From + +# List domains to use for RFC 6541 DKIM Authorized Third-Party Signatures +# (ATPS) (experimental) + +#ATPSDomains example.com + +RemoveOldSignatures yes + +Mode sv +BaseDirectory /var/tmp diff --git a/puppet/modules/postfix b/puppet/modules/postfix -Subproject f09cd0eff2bcab7e12c09ec67be3c918bc83fac +Subproject cce918f784ebf8a8875f43c79bc3a1f39ab9456 diff --git a/puppet/modules/postfwd/files/postfwd_default b/puppet/modules/postfwd/files/postfwd_default new file mode 100644 index 00000000..83742e40 --- /dev/null +++ b/puppet/modules/postfwd/files/postfwd_default @@ -0,0 +1,19 @@ +### This file managed by Puppet +# Global options for postfwd(8). + +# Set to '1' to enable startup (daemon mode) +STARTUP=1 + +# Config file +CONF=/etc/postfix/postfwd.cf +# IP where listen to +INET=127.0.0.1 +# Port where listen to +PORT=10040 +# run as user postfwd +RUNAS="postfw" +# Arguments passed on start (--daemon implied) +# disable summary and cache-no-size +#ARGS="--summary=600 --cache=600 --cache-rdomain-only --cache-no-size" +ARGS="--cache=600 --cache-rdomain-only --no-rulestats" + diff --git a/puppet/modules/postfwd/manifests/init.pp b/puppet/modules/postfwd/manifests/init.pp new file mode 100644 index 00000000..6db3fa52 --- /dev/null +++ b/puppet/modules/postfwd/manifests/init.pp @@ -0,0 +1,43 @@ +# This class provides rate-limiting for outgoing SMTP, using postfwd +# it is configured with some limits that seem reasonable for a generic +# use-case. Each of the following applies to sasl_authenticated users: +# +# . 150 recipients at a time +# . no more than 50 messages in 60 minutes +# . no more than 250 recipients in 60 minutes. +# +# This class could be easily extended to add overrides to these rules, +# maximum sizes per client, or additional rules +class postfwd { + + ensure_packages(['libnet-server-perl', 'libnet-dns-perl', 'postfwd']) + + file { + '/etc/default/postfwd': + source => 'puppet:///modules/postfwd/postfwd_default', + mode => '0644', + owner => root, + group => root, + before => Package['postfwd']; + + '/etc/postfix/postfwd.cf': + content => template('postfwd/postfwd.cf.erb'), + mode => '0644', + owner => root, + group => root, + require => Package['postfix'], + before => Package['postfwd']; + } + + service { + 'postfwd': + ensure => running, + name => postfwd, + pattern => '/usr/sbin/postfwd', + enable => true, + hasrestart => true, + hasstatus => false, + require => [ File['/etc/default/postfwd'], + File['/etc/postfix/postfwd.cf']]; + } +} diff --git a/puppet/modules/postfwd/templates/postfwd.cf.erb b/puppet/modules/postfwd/templates/postfwd.cf.erb new file mode 100644 index 00000000..1c45dd03 --- /dev/null +++ b/puppet/modules/postfwd/templates/postfwd.cf.erb @@ -0,0 +1,28 @@ +### This file managed by Puppet +# Before deploying a rule +# 1. test with an additional "sender==test@domain.org;" in the rule so it +# only applies to your test account +# 2. then when ready to test for all users, use WARN and watch the logs +# for a few days and make sure it working the way you like +# 3. Then when ready to deploy for real set a proper error code + +## Overrides - make like the following example +# id=exampleuser; sasl_username==exampleuser; action=dunno + +## Rules that apply to all senders +# Recipient Per Message Limit +# We only receive mail via smtp from sasl authenticated users +# directly. We want to limit to a lower amount to prevent phished accounts +# spamming +id=RCPTSENDER; recipient_count=150; action=REJECT Too many recipients, please try again. Contact http://<%= @domain %>/tickets/new if this is in error. ERROR:RCPTSENDER + +# Message Rate Limit +# This limits sasl authenticated users to no more than 50/60mins +# NOTE: sasl_username needs to be set to something or this check will fail +id=MSGRATE ; sasl_username=!!(^$); action==rate($$sasl_username/100/3600/450 4.7.1 exceeded message rate. Contact Contact http://<%= @domain %>/tickets/new if this is in error. ERROR:MSGRATE) + +# Total Recipient Rate Limit +# This adds up the recipients for all the sasl authenticated users messages +# and can't exceed more than 250/60min +# NOTE: sasl_username needs to be set to something or this check will fail +id=RCPTRATE ; sasl_username=!!(^$); action==rcpt($$sasl_username/500/3600/450 4.7.1 exceeded message rate. Contact http://<%= @domain %>/tickets/new if this is in error. ERROR:RCPTRATE) diff --git a/puppet/modules/ruby b/puppet/modules/ruby -Subproject e4de25d78eefc7df70a35dee22a3e0dc1b7e1d0 +Subproject 9ccd853c49af7d0b57ebd9c2ea7673b193fce24 diff --git a/puppet/modules/rubygems b/puppet/modules/rubygems -Subproject ef820cfec3321d17be99ef814318adb4e3cc1e9 +Subproject e704c9fe1c40fea5b10fe3ca2b4f5de825341cc diff --git a/puppet/modules/site_apache/manifests/common.pp b/puppet/modules/site_apache/manifests/common.pp index 2b83ffa5..8a11759a 100644 --- a/puppet/modules/site_apache/manifests/common.pp +++ b/puppet/modules/site_apache/manifests/common.pp @@ -1,27 +1,30 @@ +# install basic apache modules needed for all services (nagios, webapp) class site_apache::common { - # installs x509 cert + key and common config - # that both nagios + leap webapp use - $web_domain = hiera('domain') - $domain_name = $web_domain['name'] + include apache::module::rewrite + include apache::module::env - include x509::variables - include site_config::x509::commercial::cert - include site_config::x509::commercial::key - include site_config::x509::commercial::ca - - Class['Site_config::X509::Commercial::Key'] ~> Service[apache] - Class['Site_config::X509::Commercial::Cert'] ~> Service[apache] - Class['Site_config::X509::Commercial::Ca'] ~> Service[apache] - - include site_apache::module::rewrite + class { '::apache': + no_default_site => true, + ssl => true, + ssl_cipher_suite => 'HIGH:MEDIUM:!aNULL:!MD5' + } - class { '::apache': no_default_site => true, ssl => true } + # needed for the mod_ssl config + include apache::module::mime - apache::vhost::file { - 'common': - content => template('site_apache/vhosts.d/common.conf.erb') + # load mods depending on apache version + if ( $::lsbdistcodename == 'jessie' ) { + # apache >= 2.4, debian jessie + # needed for mod_ssl config + include apache::module::socache_shmcb + # generally needed + include apache::module::mpm_prefork + } else { + # apache < 2.4, debian wheezy + # for "Order" directive, i.e. main apache2.conf + include apache::module::authz_host } - apache::config::include{ 'ssl_common.inc': } + include site_apache::common::tls } diff --git a/puppet/modules/site_apache/manifests/common/tls.pp b/puppet/modules/site_apache/manifests/common/tls.pp new file mode 100644 index 00000000..040868bf --- /dev/null +++ b/puppet/modules/site_apache/manifests/common/tls.pp @@ -0,0 +1,6 @@ +class site_apache::common::tls { + # class to setup common SSL configurations + + apache::config::include{ 'ssl_common.inc': } + +} diff --git a/puppet/modules/site_apache/manifests/module/alias.pp b/puppet/modules/site_apache/manifests/module/alias.pp deleted file mode 100644 index c1f5e185..00000000 --- a/puppet/modules/site_apache/manifests/module/alias.pp +++ /dev/null @@ -1,5 +0,0 @@ -class site_apache::module::alias ( $ensure = present ) -{ - - apache::module { 'alias': ensure => $ensure } -} diff --git a/puppet/modules/site_apache/manifests/module/expires.pp b/puppet/modules/site_apache/manifests/module/expires.pp deleted file mode 100644 index f73a5607..00000000 --- a/puppet/modules/site_apache/manifests/module/expires.pp +++ /dev/null @@ -1,4 +0,0 @@ -class site_apache::module::expires ( $ensure = present ) -{ - apache::module { 'expires': ensure => $ensure } -} diff --git a/puppet/modules/site_apache/manifests/module/headers.pp b/puppet/modules/site_apache/manifests/module/headers.pp deleted file mode 100644 index f7caa28c..00000000 --- a/puppet/modules/site_apache/manifests/module/headers.pp +++ /dev/null @@ -1,5 +0,0 @@ -class site_apache::module::headers ( $ensure = present ) -{ - - apache::module {'headers': ensure => $ensure } -} diff --git a/puppet/modules/site_apache/manifests/module/removeip.pp b/puppet/modules/site_apache/manifests/module/removeip.pp deleted file mode 100644 index f106167a..00000000 --- a/puppet/modules/site_apache/manifests/module/removeip.pp +++ /dev/null @@ -1,5 +0,0 @@ -class site_apache::module::removeip ( $ensure = present ) -{ - package { 'libapache2-mod-removeip': ensure => $ensure } - apache::module { 'removeip': ensure => $ensure } -} diff --git a/puppet/modules/site_apache/manifests/module/rewrite.pp b/puppet/modules/site_apache/manifests/module/rewrite.pp deleted file mode 100644 index 7ad00a0c..00000000 --- a/puppet/modules/site_apache/manifests/module/rewrite.pp +++ /dev/null @@ -1,5 +0,0 @@ -class site_apache::module::rewrite ( $ensure = present ) -{ - - apache::module { 'rewrite': ensure => $ensure } -} diff --git a/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb index 0396f54b..bfa5d04d 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb @@ -1,18 +1,17 @@ <VirtualHost *:80> - ServerName <%= api_domain %> + ServerName <%= @api_domain %> RewriteEngine On - RewriteRule ^.*$ https://<%= api_domain -%>:<%= api_port -%>%{REQUEST_URI} [R=permanent,L] + RewriteRule ^.*$ https://<%= @api_domain -%>:<%= @api_port -%>%{REQUEST_URI} [R=permanent,L] CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common </VirtualHost> -Listen 0.0.0.0:<%= api_port %> +Listen 0.0.0.0:<%= @api_port %> -<VirtualHost *:<%= api_port -%>> - ServerName <%= api_domain %> +<VirtualHost *:<%= @api_port -%>> + ServerName <%= @api_domain %> CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common SSLCACertificatePath /etc/ssl/certs - SSLCertificateChainFile <%= scope.lookupvar('x509::variables::local_CAs') %>/<%= scope.lookupvar('site_config::params::ca_name') %>.crt SSLCertificateKeyFile <%= scope.lookupvar('x509::variables::keys') %>/<%= scope.lookupvar('site_config::params::cert_name') %>.key SSLCertificateFile <%= scope.lookupvar('x509::variables::certs') %>/<%= scope.lookupvar('site_config::params::cert_name') %>.crt @@ -27,6 +26,12 @@ Listen 0.0.0.0:<%= api_port %> </IfModule> DocumentRoot /srv/leap/webapp/public + <% if scope.function_guess_apache_version([]) == '2.4' %> + <Directory /srv/leap/webapp/public> + AllowOverride None + Require all granted + </Directory> + <% end %> # Check for maintenance file and redirect all requests RewriteEngine On diff --git a/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb index ee5cd707..bf60e794 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb @@ -1,22 +1,21 @@ <VirtualHost *:80> - ServerName <%= webapp_domain %> - ServerAlias <%= domain_name %> - ServerAlias <%= domain %> - ServerAlias www.<%= domain %> + ServerName <%= @webapp_domain %> + ServerAlias <%= @domain_name %> + ServerAlias <%= @domain %> + ServerAlias www.<%= @domain %> RewriteEngine On - RewriteRule ^.*$ https://<%= domain -%>%{REQUEST_URI} [R=permanent,L] + RewriteRule ^.*$ https://<%= @webapp_domain -%>%{REQUEST_URI} [R=permanent,L] CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common </VirtualHost> <VirtualHost *:443> - ServerName <%= webapp_domain %> - ServerAlias <%= domain_name %> - ServerAlias <%= domain %> - ServerAlias www.<%= domain %> + ServerName <%= @webapp_domain %> + ServerAlias <%= @domain_name %> + ServerAlias <%= @domain %> + ServerAlias www.<%= @domain %> CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common SSLCACertificatePath /etc/ssl/certs - SSLCertificateChainFile <%= scope.lookupvar('x509::variables::local_CAs') %>/<%= scope.lookupvar('site_config::params::commercial_ca_name') %>.crt SSLCertificateKeyFile <%= scope.lookupvar('x509::variables::keys') %>/<%= scope.lookupvar('site_config::params::commercial_cert_name') %>.key SSLCertificateFile <%= scope.lookupvar('x509::variables::certs') %>/<%= scope.lookupvar('site_config::params::commercial_cert_name') %>.crt @@ -32,6 +31,12 @@ <% if (defined? @services) and (@services.include? 'webapp') -%> DocumentRoot /srv/leap/webapp/public + <% if scope.function_guess_apache_version([]) == '2.4' %> + <Directory /srv/leap/webapp/public> + AllowOverride None + Require all granted + </Directory> + <% end %> RewriteEngine On # Check for maintenance file and redirect all requests @@ -69,4 +74,3 @@ </DirectoryMatch> <% end -%> </VirtualHost> - diff --git a/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb index 0c6f3b8e..232b1577 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb @@ -1,5 +1,5 @@ <VirtualHost 127.0.0.1:80> - ServerName <%= tor_domain %> + ServerName <%= @tor_domain %> <IfModule mod_headers.c> Header always unset X-Powered-By @@ -8,6 +8,12 @@ <% if (defined? @services) and (@services.include? 'webapp') -%> DocumentRoot /srv/leap/webapp/public + <% if scope.function_guess_apache_version([]) == '2.4' %> + <Directory /srv/leap/webapp/public> + AllowOverride None + Require all granted + </Directory> + <% end %> RewriteEngine On # Check for maintenance file and redirect all requests @@ -30,4 +36,20 @@ ExpiresDefault "access plus 1 year" </Location> <% end -%> + +<% if (defined? @services) and (@services.include? 'static') -%> + DocumentRoot "/srv/static/root/public" + <% if scope.function_guess_apache_version([]) == '2.4' %> + <Directory /srv/static/root/public> + AllowOverride None + Require all granted + </Directory> + <% end %> + AccessFileName .htaccess + + Alias /provider.json /srv/leap/provider.json + <Location /provider.json> + Header set X-Minimum-Client-Version 0.5 + </Location> +<% end -%> </VirtualHost> diff --git a/puppet/modules/site_apt/files/Debian/51unattended-upgrades-leap b/puppet/modules/site_apt/files/Debian/51unattended-upgrades-leap new file mode 100644 index 00000000..bbaac6a2 --- /dev/null +++ b/puppet/modules/site_apt/files/Debian/51unattended-upgrades-leap @@ -0,0 +1,6 @@ +// this file is managed by puppet ! + +Unattended-Upgrade::Allowed-Origins { + "leap.se:stable"; +} + diff --git a/puppet/modules/site_apt/files/keys/leap-archive.gpg b/puppet/modules/site_apt/files/keys/leap-archive.gpg Binary files differnew file mode 100644 index 00000000..dd7f3be6 --- /dev/null +++ b/puppet/modules/site_apt/files/keys/leap-archive.gpg diff --git a/puppet/modules/site_apt/files/keys/leap-experimental-archive.gpg b/puppet/modules/site_apt/files/keys/leap-experimental-archive.gpg Binary files differnew file mode 100644 index 00000000..5cc9064b --- /dev/null +++ b/puppet/modules/site_apt/files/keys/leap-experimental-archive.gpg diff --git a/puppet/modules/site_apt/files/keys/leap_key.asc b/puppet/modules/site_apt/files/keys/leap_key.asc deleted file mode 100644 index 5d9fb310..00000000 --- a/puppet/modules/site_apt/files/keys/leap_key.asc +++ /dev/null @@ -1,443 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- - -mQINBFESwt0BEAC2CR+XgW04DVwT427v2T4+qz+O/xGOwQcalVaSOUuguYgf29en -Apb6mUqROOTuJWN1nw1lvXiA6iFxg6DjDUhsp6j54X7GAAAjZ9QuavPgcsractsJ -LRz9WSWqDjOAYsb4B5pwmSPAKYtmRAxLVzdxUsuHs2HxRO4VWnaNJQEBj7j7zuGs -gvSJBSq9Vici6cGI9c1fsWyKsnp7R6M54mmQRbsCg2+G/N0hqOz0HE6ZlJKVKaZq -uTrPxGWFuU3mAUpzFLa6Wj8DSUYiWZ/xrqiFdbB4t1HM3vlKB9LEg93DEuG/8Q0T -g2KS0lEWxequBXyE6+jklDNqJeyHmfgkuAfFlkNYa5870XT87MzGE/hS40lbmhQV -HHlwxMkAiERMc0Ys+OfgUJMbIDQBNRFg3Q/bjajFoVBgBoKFp7C22zgoJkUNT+7H -Yv/t6zeDlIzNhgYms5d0gEiAeLauwju36BmwUsbQHwejWKP8pADRZL1bTj0E+rRU -M4FFNh9D2XTFFKaaNubub8tUmo+ZUIEEKfPhNHK9wS/bsFyPv9y3HLe2b3NYGFK5 -+Hznqg8N0H+29I7zLx7VpOh3iRN3Lbxv9dMmukVJtw8Rq/Udprd3Z5p8oCisFo+k -nY+J+IgNjC0eniN8rkkl/4rIN5fvvOR8YCts50hL1fAy3dd/MKExz+QTXQARAQAB -tClMRUFQIGFyY2hpdmUgc2lnbmluZyBrZXkgPHN5c2RldkBsZWFwLnNlPokCHAQQ -AQoABgUCURPzwAAKCRBIWxL6IY6B65FzEACn1Q+9dcLig6yCRPGF8d5qdnWYquts -fLc/W8P9uFCo4bLFhy+BlalZVhOSPt2KMBCApoW0fAc5aXOWjxEmtFOvziPtJ0N7 -uJj7y8XLk1//v7QXDJNYotiO82b9XTmF2G9URhxe/YU7mgx1cRW9X2h6LOG4VCIw -Bd00wM9vV984f50hpftdyjCcWTO9WoSus7dOL457DhcX7uX89AGUJLC9RTiaDtIL -/G/VEM8pIx5zW6Q2TwUXndVsNqyG5s0J0908KNyp5IPI66M07rR939JVAL8HXMxY -KdA9pxkKzPSThx8yWZknJoINsUhrd5ijfiA6kM7HJlJF1SnwyHSSs3KydKHj5zN2 -n3oGGT0bjZiXZHShsWa5mjEvCJ7oqwtcCdo8thW128LY2/0h3JkSsYdgdsJjGJbG -76nYjCIZYa6the4+QI8HM2WG5nrZL4B/EnYHK2lDdeVy/ynu96YhC4mdk566Vcqs -RrWJgRxImkSbxp3f6SAOsLwOdmrs52wCoEpAYPMbu79jb2G7JbR4uDB0i/pXCp+c -aleyKb4ve2EjHAY/VPF5BXKaQh3JIvGKVEZIv5ospoosr78UHBk60RMMzDSlOFso -BcB6Plpqoq4lI/4Zh8M1+eDjAOnOKwQanS4Hv7O2PqldGBUAXS3m6OI2Kvv3VqnM -X0GOB2sX4Ox8UYkCPQQTAQoAJwIbAwULCQgHAwUVCgkICwUWAgMBAAIeAQIXgAUC -UvT9ZgUJA8NuBQAKCRAeNKGCjiB5AXB+D/9k/BzZdAczQ3/v7hKrN9y3/D8kOEYK -rF8HdcBOH522sN6mqvm7wGkf3RmNSi731m6vzlbBSonrAT5KDMpj+THOmUcY29V5 -a1YOgFCCkToOfl+LmlLiuqfrGCJyE28MKMrsi2zMBKhsSxhvcI0EhJkQpPBu8gUs -XW1GSHuh5CYzwf/i8eNDpVrhHjRF0AVCOWIq52LTR62QchR+6ci/wVDHWd9Ase5X -8rxNnt2/pCbgATklQbmRcQS6efTVk3oXk1DZ8M46vayJ1g2BFuIi7pohiekLAAAt -MCwRKHTHvtPkGAUAEXExPGS78qHxLHIau2VCtSBxm+bQX+ZyCMANDpI+ZTFp1APJ -9SpbtGozuQOpWFjWY1rERunrbyWHIb2DuVVNKGiHlkMJB76zzysvbIPYWx1RqD6s -KFJBkjrM0xn8H+D6qzwzGfmX1Yaw12oYA6pcai4aK5sO7KHt+THAxYAcVF7qxGU7 -lnDifM56hrH/DbE5InlDC8OUqDysj0cHacRee+ZYtj7TiEykWfP5RrZCLQ7L6Jd/ -HtgQti/9TVUaFkIlQCfvF+l4BYZQYvnhx3MVK7ChKLmy6AVQLWnDrBrDvl07HLvW -6pslRzVHfWyIYng0pZ0HvK+MpQztCoUcDK470mjlpAtjNHuyKh6r6TtaiVK8MgbR -Sx/NMHb1/PXQJokCPQQTAQoAJwUCURLC3QIbAwUJAeEzgAULCQgHAwUVCgkICwUW -AgMBAAIeAQIXgAAKCRAeNKGCjiB5AZBiD/wJwUVZjfNeWdpKrYy9HtZExtTcU/94 -3lgRUNinUuLPFU4i2s+hR3h5fzXR15nUD+IBJlXlzLV2G/IjXYPTp4a0gqHpWULa -b5Stu7AzFiO42/RWUAzWD1Fyh6SuZ3FDERvheid8s4SXoe6y4cJ5ErfSlJS6qqku -8ss8mS8lM1Mp+lc9wYTWQ+8hmSUivAZb9WLEljFxhvEnvAKPwD18o7+S9GABFwYs -xflQvKZHguaOVqBEksry+vu8okWNrg3Ll3dDQEeahr7nrLrHe8gqONJgOE9jjxRv -bJmGtIUTyGqgWZZzBfQXL/6uXL23bWkYZDkQNhfsm+colAV8gpj+/E3q/uMXwqz1 -bv06K/LsK3NHzBNE57kJHEhg9K3Uw2Wx5qwFMU1GDxsB3P9p+TyqAboEZAB2irTR -y9k8peFB7wwf0sW3Eg78XFsfy4gyV619VnBR+PbfOpKqFFXAodF1mFiIrPeefaVp -F9fiQ5Owt0sJjDaJnYT83ksAO2Aj+VsY3UjnDrGFaiV8Neit9y/8W8DqmZ3EZEF/ -M3iS0yDjqqt9ACFD+jkGlKYsyHv7gbpTq0yi6u/kRXHUTIvVwFL9M6Z6AUcG8gzo -qbKhXGfWKEq0lN5HAjJ//V9ro3DekFd0A+NQOlFV6XtspZwphVdtW1WS078HmVlw -F5dbD8pcfT/RjYkCHAQQAQoABgUCU4eKwAAKCRCMv5oyKGGnkL6sD/4+jhdGUEO+ -n20VXPWZ7hV4GzVezVV2yLHd3kCfG+wKQPtpY9Qxc+3gyI5rOKxyIp2sYh0xpMkT -bEy7HjFJq3gFtAC/lQQfoncZUrGL6xsh6PmYx3MzXQteoJ/u94c6LXyZoYhYfSi6 -jv4/DASeIdxDZwaaVm7WRb/FcT54l1iazevjJp4h/+udPeGjsS2c9cUu2URJYv2D -lxO6JyKRdv6H4e/RgzUmf5LJ83LLQ3V2BUrpZSnpw1YZkdUESNyNAKXfdAkNOTZX -HM7BbmBSg7MSpaz2PZXgf51CfoKYkNtfyIppiTNxRlmImwXKl9+efQUOiWokuQGb -683bUMRbq2S08CSzrvTAXuDLTrro3emAvhO+NxFQN7Lf5s8eElcTH/dDG12HlMSZ -cQU2S9mEl/N+LwdOpOP84hHU7or6q5riUcALYIYPRcaU8O4QqHKPOId2G60f4S2g -7hHVTi3B7IXCgSzusciSVCiol8Q5jqngBSu3qySjuGtF2CHaz36Qr5cdRSJSAPgm -2mquVOdUtY87a4+662bHmf3w5Ma8cTjnoCntEmXL5YzxMmPWhCR7aNvrL8/TPUwI -PFQfbG+CDgg5hgjJ+gM3tlm4x2LyAcWqJJSua6u58TGEdNKe3c1jJuub32/D0xqI -JF6PA63sIIDHal1488NvPMI32HDITm8H9IkCHAQQAQgABgUCU44YbwAKCRC9aMeq -mX+nf6J8D/sFps0R7GCzzEKgIdDCA4lAWj5XXa5PkGAoZAcPj0kbn3x+lrLtScjA -oDXRFjoJpP+bqh1tBHvLPn849mIcX+jKgTh/HvSWSc4ShSt/4ejKagDUdxHzKipv -H9A73FmVJI4DVTthjLl9d3EK2+de02jzQsHh8mzTjKJPnUUUI61BmdGyQCA+COaV -G66tjZBMb8LuQoXQqgxpzYfDW3EC0IT5MdDu3wDFxOVCVLja0wPsOfp4K4XRsfkA -Ok3AaXVt2DQv4LCEa02FfrKDrsJECRgWfiu4kxVzxR8A5O2ZgPreOh8sWRnDIt2e -Mmzj651hB+W+ym3A9o6WoileJ3yQHfOqK+4gvreTzaQERtR3Bj2YgphlnIi/Mwbu -ofjBhVbM+Zg55D6HS+g12xadxAsFJJ9Z1lhLINgOP+W6pZTU3zgktEc93IagU5Dy -QN2d0nl0Mc9yQHQhYzpDBF2Ub4Czk7uFmTmRVGf8xE9VThv4WeTA60zuonh3hZGr -dkgdgp6xrZ5jJs8SJLxXsIX1umrRSJxCxOJ4FfmsTRDtUlJdOFEg7qmXpjKYe5MH -Gx8PnS6tRiCy+0JnDUXnA2hDBESSrvXcTUQ4iPTpWGluq3vicle7OmKU3Gzxu9VF -ETOGZbZKEbdU9btswvSNaQ+Yat+mr9hJKEbXS1XS6srpT2FwBBhK9IkBHAQQAQgA -BgUCU44c8gAKCRBC6GoqEfSNNttyB/9jiQeAfppuZVfobn4mdMiolTJemB2umaXx -qmP6Pa+UTALhJ+OJFM1j3LBGXChVcU/kox84ndRmjByiP/PlN2yQAZuOMTr0JZis -1Ht+s8o+zE9nFBppFdZa8AsY1ke+4FDz3zqBhAIEL+MNfGlvaBzdWK4MowQzOa/h -r+q1Ysm6gT9RUOaNedUAroHCbGtbbQLCP7fF3muIMD3ItgdiFtqFZp58mlq1OQGG -6qA1EBSy2uinFwaniu87uI4/GIIbRnzjev5q0AfOuPP9996CAbiMvVhs+JjHuAOq -L5G0UY0lysqGJcuZxtYVS/ZfGUQlAzKmjCTKCZrAiqRrTxJMdcBgiQIiBBABCgAM -BQJTjh+mBYMDwmcAAAoJEMzS7ZTSFznprd4QAI/TP9vUqTEm/4Rdzh6oEu7M15f9 -4ErZJHQ5vSR5Z5gzXy4TEnNrMcKLRcYPGLsNJYHxG/H0mCwckPLLSOHwiBjCMuwG -458p8HjfCB49BgLwLjobuXuaOpOmIWIzgUU97ylSn46MI436HLRdcryghEaATXgo -XkTdNX+2WDapG7nbMb7IF4rsvNSNMmgLsbXV2pTMk+wDDZZz3R3Dj/b1XA1aKFhV -3aJU2G8Q1VBwmGESB55OP91IAWQ0LzQpD+K99GPNE9TdA42VLPruQ2Rw8gdP3CU4 -W4+KjgGyX/aFjJTsgid/XPnCWKBLSPto6t/vRePy5FqlzzwcT9rjVxXSFvLnIhMT -C8tp6gfe9Bzx2VXjKENKhLoaYZZLlUkS816b20ebVCcBscjja4CGK1kwyMn6F6FP -abTk/RcakO+M9+JlY/YwCaIeQ5nf15IpMJuBvV1e0Ex4STeIf9VGgL3+mVQpavqA -wl4ks+knJssVF/VYx+doUpVKrj+h4nLM+OAEnEoBKPaOPTnnLPZNQupklSMapdsd -rmkVYuzZFbMjJXa3kFyfynVejgeJxLOPOPbSpeOjOa39CjeE7l0MtN+tDzCeOqrt -tK7h3sy/5KOfNcFp27xxlwRUWipJjQyRzxHKUyL1sSChdNTJozWnLiU3fOb7PUWY -7YogPiVak9tkOiAjiQEcBBABAgAGBQJTjlvwAAoJEAwyonG9PMeAAgUH/i8DN3Fk -74sCBb/eQmUQBeYfNcNvItj4bR17om2R/nvxtNp/RYrJywWwT95rL1gmOwbNtGpT -h8P/34bH5JbagPQZCq2TVVgOgPOa78ljhsx4iTd3jK/oYnJapkCm6JypZ88XudMu -3bciGV6khpZ01KtzJCjV7hsF1HAogU815ivdKv8uBGa8H4og3ooEbE0Yc1byOCk6 -m210ru5Oe3OoK0MjV3F0b2q8Bs8Dzy0b+5kPdL6QCDgErx6TqaniKX7lqPI+GoCp -2IIdacWkhCVjCwKGsDIplruJ108BCV79FJ08ZmSzN/Qyt6VuvWghchPtPWwmCTG3 -MqAdjtyizNX75uOJAhwEEAECAAYFAlOOH/4ACgkQobTFvX+/A7P2Dw//Qp6r2e+K -oxjYQaHJX3Bp0aH90NKqVD//hwF1vLLSQhbFj6HreULbweyjkRVzQtNDVr3xzDKu -Cs05DiTls1NT4p5ujMQllInIZDcFMjkzN6f50RXvI19UE0iBigu72TE4IWerFHNG -vTFM+XtmXe44+G9iD8G76NCnmSxFvszZO+JX+YUrP7/juoMvUsK2ayCo05UQ0PS9 -OmsIZWg0Y47WLAv3TyixeQxe/auX0XcL5fk4Ehv5N1jW7NSukP7MaqX+KZeeC+7B -PWzK321Ofq1/6S/eipmr6cN0t5O3CvP9spJ/4DLUbhsHYrm1qFqD1xDLK/lR0EVP -5qp0s4iSCVjUYUv1VAFAQzu+LtQ90SubzAvjCShQlMB7j4XHqikG3qugnMYuQNzd -oMokATXLsXEhKDkN6tiqFzGmsF7JwyC8cWIDyCZ/X+4O3HmUtZM9Vh86wgtsCVi6 -LEM3ya/l4YS9SwOGXh5oWiBSAatJGOOitObiTpZWgqTd/T9/Am/Nqs2i1R9WQrNc -31N6uisx+jN8r18VZVOl/IXtPsnCut6QHkAal8RH6NtoFqrnUnIwR2tzrDA6dYoJ -Imb8COoFhW5EXErjJPeFq785tlhceApRgeQOuWk7fZyK+vIqSb5jqwAON23dM2aS -h8ULT95Sc53sKPE0Pcqa8njzMZYFmA/uhwGJAhwEEAEIAAYFAlOO1jwACgkQxzKx -0cKPTi9E2g/7BTEDp7obR4wWFSY0MQAWsLTbKe1bVD/fIYH3IXNohnl7uSA7t34K -Mn++vYgBaucdSutsj9YDqBG5NXwp+UbWU+KUNA2fsKSsRC6ZMFMHPEef2kjmmMcZ -ek8lQ03hILjEJFmIg4FQt0mgwk4ejsZh6yBETp4goSPdqAfVV70+mzdsHsNrBMGO -5O2wk7ro7MsFv2s7HPKI3N1r8HXwxJa4mwpfHhQh2BiEIPmFnESL9D0tk8CaBKqY -ypC0igDTXgx8AhV+kDcG3SSVU3y9AaBCjVMShvi3OmO+PIeQ6QOJ3z+9992enqAM -mjngkA0nACclVttNzkdpljb7Oz1ootvoL6/Npj3tbZ3EoawOoHZ/Mhn9W5ncjTqg -6Zzs9l6QyoIMu7ZSJQViD879oPO7Gkv4I3Q/89dQ1yXRvHdJSctrMCpFg7VGW59O -DMNksJGdsXkrn6ynyTfNZCxDP8wLXw9pAk79yopTZ44y0TTKzNOqm8K7scOxvcet -ZC6yyZQmUxul0Z6pp9NzomigAp3qUGOt8k2cp4V8LVj+FcR5sU6Lyk+00mDfr0qJ -lUYwL3nJvxFc+B+tDoM80QK1p08drZrJ7bc6Eh7JgNezu0wNiAQUqxdgWOPjoBL0 -kIlJXbHxAv6j8JUj9NemdaA99AweW1ezwZtFIXjpedQdkodRPzeANbKJAhwEEAEK -AAYFAlOQULEACgkQ1FUjZ27WELeQfg/8C9Tosw7E7+n8fo2z/P/w6XcjvfGGoGAg -MaF02yeVkC5E16VllTBzY4aPMsPR+XfR0BK9DZHcglhEHaKHGjh/qaN9PI3gIodL -o2y5hgtCYxTMeICmHRsqh/1biMMeKHT85ZpoHDA74wB+8p7oY21941aUZM/tKkpN -uE8mriFxpkDB1A+vXe6G3JJi8z0S7QWT/2UleVpo7eXihiC5o3AZmQHXuTyF0UXn -8zLYLHstJpC/AFMi20Pv7jUcroxr3BLaJ23ai2ks15UueczgHLzjPyIg/4zNawpD -fX2r4whbWA/+KC7VyAkR5lQ4i438bKhPPmYkbAP3g2WHqow34dXNaQm/eSLz/+fi -1l3UtgjpNVvxYHj5Sg7q5bz0ZkTjS+jWsc/sIb2lPlwL0u7i7+Jb7njlbJqaiNPZ -sIoM284SKTj9oWrz9KOqerRA3kc4xFH2oh8Acqc+h67MiNCOnHUmKNgtZobe10qT -GCYhCV7CDp7XYZikDGeUyhri2sxASRVruHGlv9qKGbmZyNmoBPhjhyHtY6HZg4lv -SOaZSJ47fCSfTN712sJE6pArsMK2o5hI9Nh/4LgmYiDQYwYGKhB5KeRE82jyATvf -/om1EaEcVD0QyrF4oNcHioNcznQNFhcP49DNjVWTdqrgxQ6vr4+6avwAW2wyOsW+ -cpShmueTf6SJAhwEEAEKAAYFAlOh6LIACgkQTQVh6fnb7hNfYhAAv6dqpWvHDuF4 -QtNxk9vUUgV3RFPIxeVKf5mI9RSU+9Kk/fxSxmqnAXhghqN1SAZyw4jh1A0emQNQ -xldoXAq9G8I539UcA0qQMfoH5YzD21CRNlgHJ/f4fexqZM9aRiMmVGJiAkr7Ih6u -HgLohH0Qjfa02GL/qoxNpGiBcpJ6y6KLuNMi/E572UYugxsYdnRct12jzfkErGdx -rRZlwNsZHdYszxnTCdS6zT1jp3IY5yO8gRcrcazJUo0HK0CEs85nHM7JesVbwkC9 -Qyg1Tn0katx+7NGm22Bytk4sMXnZ1OUcRC1zkGFAu5KXaSPqrLEsdM1sXQuxaXjh -G0GeIflEzuXTNsSPS0UwAirvu9/hjsh4yEvvYhdmwaSaB9rZqIlJxCtO8VPcNXYo -Dz8UEiHxQvzVmm2d2PS/LOXnyod3TdFnM6McDVmb4qiJPGgS0ZBIeHH2wla1cqli -zWK1nCSL+aBPDJHAl+7U1WmokCVJWC4TDarjOnZauEMr6VRKaYE9tAQsP1h/EWtO -J9NSkZRZN6eBCnO5PREw6pznR0IR5ciwQcrYAecYKadwyFl7KTKFXCWV/7v1023U -Q8QunMsATVISQH4VWPDl8ANqy1vqdSgYv9g0/ho84dBWbXmlMmFGuySEwg2KGeYY -1hWgmIfRLbsoqxWdrpkpD2+qAZDYSm+IXQQQEQgABgUCU6MneAAKCRCcAqxiqPWk -3xK8AQCwKzsh41vFnJJ7zTNf4DHzHdBVJNj3WkJKd9aTW9QRogD46xmr0lT1wDS+ -9aKssk6p8qRF0C9bKvxjLI0FLw7oH4kCHAQQAQIABgUCU6PKNgAKCRCEb/rGNK7q -P75REACeQFrfO0xSXZysAboGdzv6I7ZE8YYW3/rd5+RKY/m8F374MXf3YN7KiIdh -g71h/sN4/lzgJl48so9B86Z84pmUOGPVaIGmqzDJBGBY1Z2ULU+sagy8ti94kdES -LLVencI/dQXaL/dxoOerMgULGjNvuftZNFebwMdf0VjUCvWkV6T7pubEgyN8h4Mo -J+rclg/SWf3O4QxFzDyNSdwDkNgTzew7n9DiNDqrTBEvV6ahMI8ayhhs1lzlTOSj -tG8c1edsBa2tQSpkV6BuAv5Fqbp8hAkP79QO4aGATdKuoLIYHD8eULwDArnrR8VU -aeLZTPzfobzg6KfHYTZNw8SXAl640DNbgvMQk9IPtfUXDP1b8EnGf76CjpGRhDNS -VAnPhoFFd5OIzmJikocfYZkuuIw5EveQ+l5GAroO4TytdfGwYl0KhNAqExgbkWT4 -UVQwin24B+1MHN0Iwqg9zupQI0+IjZeWyzkmuHek6sphXC33HarRYeytte2HP35w -MmALHeADTVrngSCsckzbVL5GCV5voiLaplqS4vqE6liqW1pEXWOYwfoPipyVDZxZ -d9EsmErW4FPotht7rWKeizZqAdyQQft07dAckwKF92bG4cez3fFvW1hQKYHM61a6 -91qG8TRybZvnBh8aemrS5o8QChTk0UFQUjpRBbAOb+ayf/dgjIkCHAQQAQgABgUC -U6Mo0wAKCRCl5rymKbpBJy+OEADI3UuK8K8IdayCpC9UOwuA/LDF01RW2EoogmwL -ywSS0ipf+D0rEvH9EtqXbz2ibOBhZIxtYzXXEx0Kx3fnYQvInm8wdnc0650tGMCy -U8sBTaV9B0mdYakMmQZc8Hbwd88fLnwEFn+qHJ0ac31geIxDN7vfgKeqea+4VGKn -NbG1h0bRn2CK2pccnp8sm1eHJ/u25X05Z0Ofrx4It53OhO5RTSHGhHM62+48FcdE -Ytm6jPcS6RY/mXWBWPJsjmvvtQ+67PfJVHVJMDVFJV98PSqLbD69doz+golpESEd -O5uWrLJEXMj8O154HQg5xnOXWurzNG4FdL6eeXk7+1UfXSXLUzGkaMgB6HjNTTrj -JG19B/+9F4nbItN2EOJ4lVEvDCL5uE0CBxvG9y04dxzI8jCg1/hPjYyd4Qb7N6CD -Doo7M0lhm5QSlDa0NDfaIpxEPBtcb0wOKaJUHfDMertl0mWvYbPhP5Lftq+h4syv -Q9iu0gitGCGW6O/GXH3/ZhnyjVJRJnFkcYIeckKXVK7aCEtyS8adffjrwYvH681P -5d+EwDxY2B8UJE9sPx+Bz5mpsETSVbm+aA0KsCwc9jLnJcDB84/7ovuRXdP0C06Q -+YC8fY8QQZmIl/bkzwAWWyZVSw4O9/3pWnjXvqSu/6xOSVfF0455bMvyv+MOiTlx -8NqfgYkCHAQQAQIABgUCU6/1AAAKCRABogUB6BpLumA4D/4hW/M4AcQ+uZHnWmA+ -N+JC3WbWnjOcHcrLplpxSY5paom8nHGOfdSivqpDNaY+5yQLz1i6DFp0bj4c7zL+ -g7s7hRDz1Qf4S0e+dy96NleRkf2RVijhtkNxeyISaHEU7wImFdrrsycP7OImvvZv -Jk6bpWljk9FsXamLEySRKtq0WSa/UdcrLkR/93Wo2PLPpXoMRX8+jV8nLpL9RYkR -ThH05bgaYzGPPGySH4pIkR8RlIvzd170glzNbuuPhhAV8kJUtXsj6W71AWLypnMk -rUKseVL1t4iN+nf3ZFo6y/+s2MIzgiEHN3Ju+aJ+uSNpJdSwD2S89vhsHT22dQOq -DHWdUo1DRYcL9kEug3navGxVUKZ2etjaeT1PFfweLacwl+Uu45RCbmQD6pFzda7W -ng8GbDERndUQ7qA/DCSBfnmJBNhXIMWJd8lA0puzd1wP189ZUzbge7AVLXJfFFuj -5hVqkTKeWW8kodSzYKVfS1lq/jT2rtMDLK6k7vyf0Gjor5XJhfla68BtBNxn0+dU -q25FOTAPy5ETf8BFL0Au8CboxnOMju02WBFb5Q5oNs7fk17hvzR7Ou/f/wMVQDZ1 -6lGRROPsvN6wi6z1j04V4QNqOd60JnBVq+O2qmbtXjou7qxHR0ZBwI/qPaktwFJI -AXmeOFdKQyl+I7TI6FmdpUB6x4kCHAQQAQIABgUCU6h3iQAKCRCDgslcKQI9+WQg -D/9kFhcS5qu+l0RsHNmKfhk6RMlnPs0jY6E2BxdXbJaNBFN1NW3kIVNYPFflvazC -owBP2ek+gtiJW5/uH3C2FAW//Dt2pVcjRN8tIqqrvMRJxLQEP2zJ1YFKnbYl30+e -hHWlUUyleKzIBA/6GbiAig/TbaA1cOgZNVDdtfJUxLphzNZ7FGatfjkKEYP8+xd/ -3Oj/Wba8F4cNhMUbsV2qOs3nXlnVt2hKVB9kOpUJQdjlJp3DKbim6uj/sLs9KRLd -Bfhrezd2KoIpvvGDo/V4xqdhW/J/vwjrISOQ05YsrCMOL2O9/Sdron+gA1zm/xQf -C9k/Nv6lcCaOLywdcaYd3XJqXZBuZy5KEljm5Nyo4Hxhd9ZJiXw+pMzX75NLZr6X -nZz7/t/8FbRKhlHfAl/6hoPK3f31os8KHyj6YrqGDGg8dwvhxMsHpPlvsx4jpIsb -cE7kTwidvhJwe8qet4XicTxnF43irlAk0xIbmxnSgH6XX4G6vejPsqr89p9pCWLj -URTT+tr+U1XiNfehomdhDUYXZv1Sg5pK2HNEqkGKQTsjzb29t6kiGA1JsmBCPP5S -PGdHDagMfa+VFlT/+bmwWsad5R+f3vhTQJ4GBcpBgMpQwEX5wHSGs7dpKxUaL2ki -nRR7wBBtPLO0P0GLJ6vfkBhrtfA4gQCGqT+cOEdHlH8gC4kCHAQQAQIABgUCU6V1 -nAAKCRB4YIWsINNFfeXiEACXy45MLSpjTfAF6xuxtlN2zBuo7gP3VNboIq0MZcD1 -yxRIhSwKS1QAeWso2a7YJ6Kzp/HG2hi60Zi/Hd1vXSnNYwLD50GszInc8iNA2OEZ -UuDiw7c65lWxOdjc9jbnd4tbEBiupfqZ/15R+R7gmeOVElW5d3owC+ENgbPOhmZ3 -AH5cx4QdXtmPy5oWpHyU13izxvV934aY5OXcvsCYwIt7DCISEgiuPJ2azkzX7ak8 -C8U+diHF5M5Ps+6nBmNQ9bpHHwZ7hZOUQGs/1UW5cEeWwJEMcTEPLwvs3qjtK1L/ -k8DS6t95hr7xiNiyojCpC4L1XwT05MxGHKwY5qXMw4bqInyGsnf28NuoaUk+V+fq -IS6uC+p7/D2l9RU206R93vGTOwIxxEaehy8i7GadhiHOKGADm7yatlmOJfMx+9D/ -X7q2wWB+t0Hu5fr34NwEaMfJJgjOHhLqtBiuPsrr6D7+sNKIXkZcJNKE9nf+Nef1 -jygPjDF2O6AcHl2P6vPoKUwEAYYUyYB2ku/pm5wJ3V1bbXkTsYse8bWCUf6jSSWI -acIoHAXtZjOe3BhKGKxG84h/7i/054SYI0+IIuec3zwiy+te+V5cyGbutaVg4Wnd -nH/j27B7oEWmwstsu7i0EIO1dw3WwjhjiXsflaseWoB5LrOHaHpRccEeiRdllka4 -uYkCHAQQAQgABgUCU6f59gAKCRDfoyRRwJ6+uqx7EADLCOXDrUmHbHQzOn5B+uVf -DQJrtVz3jNFdHb3p4qRD01tJ67/GhcoZ7NUVo79ELuWJalQaIM9agmYZa5CsJIUL -RI8JtalcLRXCxFX4ls7L/XRRO0atwzpt/K9hao6Taewp+J0Vy+TVLVeh2jcajRL6 -DfrlawMDpD2OxDw16KGMZagmNoWcuMZqK6PRXBnFGy6lEL7+gCy+vHe30nl9R64q -MhTxnpCFvK1UCk2Khzz/7nUNwXw4McU2KYnU7AiSMNdfPWv/AW0m0MyWRBLmEy/n -en+hIEmoxIomifroX3PuyChjZ2FaNWs3Kb7HKhXGSeCl1UkLBPeZa+obfP0D57qO -DeXMUxny8yOGOXOrHlRhdJz01qZ9v0UCfwnBJo1ewLaacJ1l5Va26GwDbCh03mhw -SE+AWXZbVTuemzUS5QBBYzC+SBxSqF+sE1hFy9okReGFY+AjsjI+uu3A9ht3+vmv -cgLWPyvkxIVksnvcmPp/k2RLkyk/tQWA31pNRMyqaReResg4mFszKEtbC+tFVquc -pCxd/Rvvvz918lkEFjIPN0yQoRenb/bKB4i3jbKIKMnJzCsNeaobdpsNcAdl0DOp -LMfo+RFzheQnMkwBQgI0XNxz3ZNMPN+BkEFWs7u7mfpKk+ZnU/9fKOyrOC1z2KZw -NhjOF9oHv+wfa3ZQ7I/nkokCHAQQAQoABgUCU6xtjAAKCRCqaOzI6YAJU6erEACj -8z2iMep2PzgcnpHz8csKlAmTrBRmNgErJNWfSPG5Uw1wvf6RzcD5iZ9D9Xu6SCcZ -0DVtLdho9Z1/Zb22ocOxTBd40DkryDdO9ANSHj/bw4tYzTtWmftoU5qMdD7nQgp+ -2fMTuptASAx/WK/n1tWam9XyM/i9fdD4iu2pzbBdX0F2RgmMS197fQgAAVMfDDHj -gquiyGBefCaMxv4FEM/Omn5ipkhrmPpsqTmZJNxAusiOiPJS+pjl18Xjf+VKtRgL -tr4pu0y+NVVXmI9IvncFlR1viyaKDoudB0K9UXlVKUURP7ambdS/yginiVbgaZ9T -x5STFC1FYKk69JdouvzYQc9vWW3C0cv68GWivpdFQMHrDAjAy9rFpeqs9UBbHpxC -Q38ReoYDyxL4QJulEv3HpIRRt+UV7NE+3+Ln3a3/nTnUKJZ3m+kX6qKh875Td1OB -ZAmgfy8zlDdd50n0bQjSt8/DZsRdpFq5LG5vXA80Y+Tpn3i0jTJ37JGV+7+Yxb3q -XM8XY0DFnn/cf/BFbbqLwEUxalooKp8eOO1G8yW8kqBdxGe6I1gJEaht5hkbrh7L -Ozjs22Tyz+DN6uo3Lm+cIu2YmC1j38x9LzVxUJHIxFA0XTurwsMDMB4RvSs+jdWC -CdkQM+1StiYeYDC7+dG2FquqpbsFl6fsoICKCksvVIkCHAQTAQIABgUCU7GS8AAK -CRAjY4v3LFk7wWflD/oDgQNEhpD6EFFCQ/y3g5PtOVjeF349vuTIXcTcQI657s7K -5ftCYgWu1NNYUyLFiQJ1siRVZCqq39k7SIG3+4X9mUvGubqXog3+YZRy4FwnLGiz -r6PwEHsLeHoFiGAlbyaBrojgSgwVi5sir7JsH2PugxEFCZj9l6F3RbslBpDW0QwS -sdenw1GQJrejZC3HkuE+D0U2cQ/YMebUaKiIi535H1N1oimcVpowRSX1NoyYI69w -1JSyvc0fau7KfQua0+iGhyicL1BecGPKF+YIns6fuunnZuwsH+K35cUu2tpi58bO -42bbfirpn57LF/a7+o2pjUSMzez06aUqS4WZjw47bnpIm92gUarM41ExXuXLXZlE -ofeiUnWbVkJ5+MWeBKEfTBeqHToESEYoFSVgfszCR/0hQNy7E5ADOPorbvJaqUlN -L6/cDURnQKe85QFi8y+oOzEkf7D6XvouMGQyxsYVjHEpEZdN63evTe99Ir1nFc+F -UCgch0wLPTKlYDpQBsYiZRYrZMG/pvKvu9+ig2sqRjI6nlARN6HocMoAGViK23JG -ikC5B15lblCGTL8bmiDMtd0+JccCSLv7dAJI1FSJEFqyI2CAGm0Rj5Qh2/Zfu6Xl -sJdlnaCIXsqMB+5N50uXY4VuZ0L2+cokpYm+Pn9zJBFyxX3C2F7r1Vsuysds4okC -HAQSAQgABgUCVAJs5gAKCRAIlNrgSWRisWpGD/4lp2TRzpJkSpn/MKcMDPjxu3eJ -bZLISDHeerVhVU7BpyaxUpAHaFyv5eTi2SoTXIWPFUGzZXj0kREmWBQLk/OM2VcL -pN4oSt2b05TPWNXatIve6Zlg1AMJXz+i7iTTLCgjYUhL6uvfIczX3Khx0VwfOZ1x -T+VNJqyXXlL/eONc/+a/VB/ZikvY789VfgS/Xq3JTNQpUnGFf87YBrThd9xLsaVg -pTNhmOQpAEHLaZIRw778roAcQWyZ+/apIRuMUcOyishyT9p68DDu5N91TYfXayy2 -ckgmAt7Cf6j70YT9Xc2u374F/AvtXHHlnX3/hTI3cVKF+Wio8L95aM4BDmoHkk/7 -x2S6TUiHDiBQnLgSuwLdYr9eQKSZIjek94y5nz9KabKwj09vqW1b6jbt1eEsoeBd -FGfL1GgmrSMO/erz7a+KrLgn+4Ld6GB/dLLhPCbf1ILe6oyThzvt+9vGsUqSNw6R -3O1J1VYZBBHaDMZwvLRZcYOTfdpXOtmV5mNRCvDHyVR1OpsAY9m7EpF7npsgJa3U -D+kaf9krVvua+aAR1Cj462M+UAcFFTOBQ5PciWtxKm6R1tFI23K9zqIGl+7Vnrgi -OiwQdEByFtdOnAOcbXf+iCyYZY4iv3ZzDt5ybREdPxgf+6LUVoKBh+Yy70ut/HmE -I0P1exJc3k8ZoZtQUYkCHAQQAQoABgUCU7TR8wAKCRB3w9S09qPTPwxAD/9Yt9gF -oQptDzAEEzbTICYmA8zj2fyvPmSFCYfMQfAEOH0oJerU7ux/7zr3XnzekFd1NPom -HhJuGBLyftHYJvird9KafLznBW8NB54kCHARHqdyqQYoEs1qUIuBNn3TGW+7Ao1q -jXkUr59PzFqILJH1aCYelgy/NN81NNTESw2in0xZ/OliVGZuXtTbYiS4js0p2bkP -fzT/f8FeMGwzAX5RrvP7of6lW2U9pslpUf0g+3m77/G3nwQkV70e8NvL7j9Q+zkH -33IMlnrbDPiIhwgjnrtokUbGbs4XHyhFZxoSMddOy75lRs3YhS+vGIEvfLo/P3ai -MIDLEupdYR6ZbYZxCF2y/+h/KKgypirzK3Z3zePA3KQeEIvlq+vo0nxJCyfnj9wz -M7AJ3IJKzMkvmvYSVqaX1OCBaTLeTUkpFUh4Nq2fv62RRtlRqdIzish3bqlmHwFB -ApQwvm0/uH2k3lucl9Kz7S+unh1h9NR4BVY2l+f8Oi4T4Irut3qJnsrZXunPBMJr -KkPYU8iKEVmSM6E+M4T1yGoVse3JAIFpDVNHlZfckOYmqYG3NAvuralw3mis/eaG -lnBrBI9zp66FzZpoqaATzz6/EIbuZ2vs47xIIgVUoW9NZwGo5igYhW9ixUURznFD -PRrg5+UfoBoCcQ4i2wVvciqbTaetJ6+QuqCadYkCHAQQAQoABgUCU7u12wAKCRDn -vXCXmESXmUfAEACrMuez//VqyvynY5ADBUMytq2jrK8GUqfbxDOzeRe9++16xIYJ -Z5qNsdFBL2ezryvuafcPsIqMBvwucEJ88jU7/BaUClJo2QPD+WLUit99J/O68DTi -2m9Jxn+EUi/L2d0e+Kcb17zbEfbcg42+TZfxQhf1kUWLH71UsDwAiiSnfy6dkIDg -wPnlTpuerRmsgI5dkFeCt8boYFRXatVrjzJsX6rBp+CEasAXHuhAQvIoqepi3CTA -Uo5gPHSF+aldAD2SRw3xegmRnJkKNOYu4RfCUi3iRLV7Lm898pL82659jSUQMo7a -ZevwhaKQU/gm0LBbXHGizV7JuKyYE5vzEpAEA+ChOntE4dcsN2B5m81lD0ra8Hbb -qTO2AVlp/BJxNueGm/WdYHXdpGwLvWWBP5ZMqaPhqBdQ/5DMzkeyZnSBjykMNIFQ -Ii2jRLe6/poReh9ieIuzO8QIMA4X3QTsRv3qaF/TkjTzUpB0x13fXWuyp4mrSEbF -msgMEdsHqOLkBAa2w1RLgvPQ21FKewWyasv2T8NdvuZT0sARuirvxLMFrb8fMC4L -y8xBhXVbpWpFz6eo8leygqSSNSsMzzCtm6yJ0sO9ufQWrQZMmcqV5ERGecnjYJ6Q -CPWEC+xCKGS2W4PmnkO96alEfDjEKsA6uGi4z1FYvx9macKX35Hx1cXsUIkCHAQT -AQIABgUCU/Dt2QAKCRBtnoZL9ilTXmhID/9PzhdXfJKYQLzR9/NBX05SlWVrQU8K -OA4KtaOE9AVCVgD4bcIuH5Sf437xtaLUChw/x112y37kT6i8+NzyedmLjXvZ59bm -aUEzrLw3aZUZ9vNdkLeqQgmk2t0tIfQjVssv3qFqbHmdRT3pXqMRz4JzyA8Q1ogM -3C6Ng7qv1bpQS0JerfSchGrINGSRWIrbsguhmAIiJ/Vt0X8yALzqmg6kt0Lnz5Vn -G7dMpIfDqyef761o/wbAvVx9WphEmlCqiMd/r9kThQT2rQnqjvi2pFnUEwUj1zQf -7ArpErcB50OfBFh9o4OPt8BPYdOkXprsIl+MI7Ax3C+Nu08+jtyKZxTL4gQ6lBvz -jQyseX2tmw3LMd4nekrJhifF406zUY4uDEVINLyDCOyaelKzSsOk92vHD17s6h33 -PxyS5oUZl8ieM7E7s2ibJAa6BAKWXBsQ9JTpKDb7csFz+TOWvqsUMQZVGiINe/YA -fcGQWLOK6NHbEnIOZ5dRSPXKxmWyXKe7KvZBRNExZfE494/ZhKC+Z9/wRRfg7B72 -fkFghwuwuxsmjuEv3DXXqfLeyGgTRQgF2LnFAoShjrJDJUE+ZCtZ2cmdfLDy9wdL -Z0cRHbW26FzP26MUxiQmeOUiJnh2F6uKuF5XbFHipKVmSz0BcP0rNui1+Rq9Uc+j -JunKkuGZ8QRTDIkCHAQTAQgABgUCU8F/NwAKCRDYolbJslup1C3WD/4kJ9NqhgjB -BdW1Qk1VimzcDnMu2HRJOSYNqp/UVV8DRoet+f+507VCYaNgrczhKUra8wHQjiXg -5blBAa/Y4rXxSFP+Yhow4e9DM+0CW3SLWByxXkNfGFtAuflaOFYqbH6fn+1T2meu -ktXqVZroK2gXG1SDKHarg/XAK8QN1ism+tuGHSaO4ry1WLy8Ok94xPG2ca+2ViRh -Zm9FBqX+b4++3sIwycWIoo4MV7QzmFAJR1YxD2neSFp5hul9BgwUM0RLwW2DfbKB -6fBCwVsZBBYRNs3gyrBAguZHi1hb2KGnHZAEgY9XEqfFlUon2A9VGs3MEq0eO67M -pRNvcuVWr79i5stQL2FSj2+U44YlpOjY9zoqQE3ISerOQ/cj7ryqogQqDzbC8C9M -PHCKI+2k6CvX2Bzk5HJWk7KNOM9n94VAiRD8a6NrpBTWrql5DSF46cVyQhUU56AK -EL10tYBbtvm+QkhFvUudcomCxY1WJiNqOZQ0wO18Lw8gyNsymEaija3z/ANIbGUQ -OQN4zqKOJ2TrhrTrZ50JUO6umHRbmObMZ3Y+G/R8WhgKs5kpJjZsvGnCNmi6cVeC -KbLWtlevs0S5lfLafVX4/rGSkrD4TJk9mgzjLygYWiKqygmI2NaIKWslYua93V91 -SMSeSCz5w3Qzc1SRPjLJaeyNEX+cnO1aYIkCHAQTAQgABgUCU8kmlgAKCRBDYhta -0kTdB4ruD/wKAy1lg71RR5uVOsoeM40WKCkaqF/G+9RDRtYxXaj6LKSv7CQ3dVDu -NlAbJXZOt6j3BPFtUSvcZaeSRtvuF3oQDJF78Zhc3UpMEpgcs4k86OOaspNAwb8B -1clh9Yf5qCingQp3nr63a75d7d+5O8hDmqgREqsVHI4XPFz+/rlEQrwxkWaK2ht7 -fP2ARLCcfWccr9m2k4zPyacgjc6u7u3mdLcSiADsVcraoqBIaw/VLP3VBi+cv0B+ -Z99RBbvGgzdnkY1kOESEwmXJAMwv5pwxBXzI0ooVzMJM1pbGTmec9pB4z3FzShsI -wISHV+Z9HPsO+mkJ4W/LhGUZhrc9Fas+zk8loqCuI4u9hDooapkIwwBgeR1O/Gea -NwDU2qLYRmmv3PLATv+bi4qmYXsLbOMgvEEM59WqJxJMGx8QmavFprVCRaO4yTsq -Ih482wFHbOHSZLOXGpzxggaLvveIVTncUhpFVOumCxLblOV0dyqjdCY0xQQC4YiB -N4IH+3u4uwEG34htyZo6US57ilXxCW7nBDVFsKGUpgm5gs4QUSIfK4bpFOVOfKF9 -tyTzwF8NM10HB4HP3AnglgNCsxxoWhtY9H9rqwVxdxBfSmtxureQFSVNUy6JxyfJ -rq+Io3c3oAIlCAJ76IMal7p9XvF2uMsVDnfkvSvcgF3Ba8U2AiJMYokCHAQTAQoA -BgUCU/yTLAAKCRCL3If7chXhiap8EACdzs4+K8bjmXcStt6ONKhWG086FXNDCAre -5PuQ5k4NDqwYvQUBbJ6T1yrSuSa/FNmM+uKwfp3AEAqwNCWrZQHxVagsfSJLcNhP -hc6uON3twtJuC8eIKnXOIob1ITQEwMSGdW4dqR2w3jcwfCLEx2wvlxjFYN7yHYj+ -QdxaidWONjep5VSAt/noovt8WPZgBBoX126QV0gvKF1AQNpqOtiRFRLRceXFfrnN -UuWCchZ6SzGxyer40tphmposQ4Ikehh2NQxTMx9ivbnKpfEYOhffDcSRcUMtYe0q -/XqxcPcas/+Q0BhJ7LD7Tzw5huLRne80d5pX0SSkuPjsZJVS3ZzxQdI+1UYjJrvD -CL0xIFg95JpGsEmeaWc3WsZVnQwVUjhd+yzgE+7jmEFLzlk/j4x1Ofg9Gn1TH72k -XBZb1gqQZa/nI1daydUZu6JuoXLbGrBn3tdalEIagz23SWvMpz6L/yNU56quv0kV -lCv8tEzCojQP6a6TkBIuvtCgjjKEIROqCQQbmGGxZpNVukXC6P8+3Tva305iT+SO -YvQ8VYOh0zPMAt3xjN1fU1/MdaSlZ7n3utF64yp4OkvAzugn13AjHXlYvy09dC9T -RGOyAK8j/YYJ0YBdApBOrLdz3Rja1Xfc2x3WWtcTXigDiScpT9afUXrKSHHAi5xk -ia94q8dGAYkCHAQQAQoABgUCVHNXLgAKCRA1Dr6IHnUkHiwrEADJGKIoU5qyXiSi -kVtmBdYmZilF9T9fzfcrL4TNmL/V/VCZgd9zRANBN+saSXkgm0CG+3jLIBlFhg/+ -zBlxceGrnDMB3M7C5n9kvymh5pAGzOVlYwfR5QICvJ+nEmk/WcaxiEUzGj04i5VO -uRUr0aEVbVqubznnFtc+qEntQ1oWNW/fSvMmfBHBxSLRbqDBUGciDizh1R5XuJBM -S+6freS5NeBnZki4Tl4zYS5Xu4qW2Y8PEHmWeLluNK5EnXmDqK+cZ+xQWn76Or0s -T9xx7hZUvef8y1T9RGoS4lpo96PylZtAMnCJPT3NffaupN/P0XIu9cI90fmRFWvW -Rpj4bF0MWCoA/iGz47rQuIIBo1EAU3oV0I9v7TqEwhkNRFHuKxcqA7dETDvRbsqy -D6/Rkp0696cRCWMOuaokNSEKtjZEeRep07NWckGgBPbLppuS9HC5b1cwhZg9DWz6 -Shy3eZ/UcbCNdLoTt0nJ3/FIInLxCSzKjeKeeaA+OJL+cVz2XiGAqZ1UzBIb4NbN -uTn9jGxc2MfyrpF1lkrYTuXa19ZF6bD8TkH93RGI8FUid7Y58w23OLF77z++glVu -kpNB2BXCPppxhzaAyQIvrCuLVBBABf24JOP5kbSY6yZJL7NPMPW9E/RDRTcX0Yxz -jW5LWdzdRe7NKSQ2Djx8ixT8B6uoV4kCHAQSAQoABgUCU/DRxQAKCRD5gmtHlHEU -hhpDEACbY5HkQMYaXjjsjCzTMsl6oxEXrPlVAFCBk02k47Sb4xLyKYdvJ0w1Kfsc -eo1EzUviwRbQuO/VAKfIc8ZtbikHLhKGU4/e/DVZHF8RtHyTgp9yWACbM8fDP2mi -Grig2ESTxskGLfWIF/nx5fkkpMjEdTs0CQvgT2sV+RUTYRY7ZKy2tmnqctrdIeTJ -Bw1ao1xKgGyqYp2LD4YmHyTW7pQPKIn5XzJr7BUgvRdpeZiC36QbuOJlS0UETk+X -1ncL7VdpBhIGUoN0b22H4euEv1oRvK7wmZHDDI/wIW6F85jFMpEJXRAR3xgMrd45 -zYwM7q8UW3mYpR5qomfMAVtAFPckIt3ptkOuUIHVsUExzJS7AfeqZQxh5RDxphB5 -D+qaHHfqNNBCc2t8Aba89mDIk+789CY/Oxv11BxhVvrK2ZoOLhgprBO5q/edjwXY -Bjtql7TRWrmQuhuFnBbL7Iq5GXovlnp7YSqtxBvA+1uneM2Fp419krP0Wd6WxkXn -K91Nul9jkjN7afnAc4mAjUSmJfeZH8rkQ3Wv8iHEFkVUNd60ELICuIAtQSwlxsYR -y0vpJP5RAzFgVCGukPnAKlnB653vBPGYb7PgCmYFrQ2A/q2TDlIkP3VlCNmvlSdm -A9zIIl75Zi1qgDeOqTKuygQK95BfGbIUMeiJmI7zTnwdAzaPg4kCPQQTAQoAJwIb -AwULCQgHAwUVCgkICwUWAgMBAAIeAQIXgAUCVO8/PgUJB57jXQAKCRAeNKGCjiB5 -Aa3dD/9zk64GNdJbN4sxdXQ/Np48wXwnOxCY9AYTRP7rJDjqnGqeJo6+5koHbPVw -bUZCzU4ZcI71MUm3fvit+qedmwJskp+8B9WvGkK2mPCAdkCxXjp9UCx9XSV55Jyc -yXTp2O3oVrm9n9YWmhA0FWssE8qsVaoWAXYIuobDlk3t8Tb978Nug7FYFPmRHB8v -x3pOta8UMmFQ0sREbjwFgrbM+TH9gM4zL5pX4rzWQaIO/0gDaNJWETNNbVbTcXxC -m2r78D/IflRLKSxKAp8e/le5M+WyawJnCydoBrNysXxyI8YDFVsCjXEB0RQ2SC/3 -pFpUGISGdV+iWDrfjv4rKNCjJNC8CKRZG4txNojCJy1HsS3jcslzdHX4uyXTdRN7 -wmeFRwVIYsDw3saqF2F2ajivU7k+pGcl509cOuvFy7aSy4o1QkIvOpQ12ljPC99V -0443qF9DDlhfi9+gesUJyiQbq7x3TTNzZce7mkWjABKGt+u1t8V0KO81DYVjy3td -QfBp7xAjqQvqJnWXhxNjOKVhiha9E5n+1n8ZmScmT3Udy7n0qbQbZvkofaJa9wNi -S3BBl+Lr+ScMdLUhMIus5RGn/5x9a7XL7a+W2vtpBlx5r90CWI1XnOadauzP0m0Y -mervzkNDZq1r8Nh3fGVNv/MM3DGISAAhmx7xCtflGbDDwv0aMYkCPQQTAQoAJwIb -AwULCQgHAwUVCgkICwUWAgMBAAIeAQIXgAUCVMvC/gUJBZoznQAKCRAeNKGCjiB5 -AbOiD/9uuAChZPgW6Sn96awak3mpeaKLE6MsOCRI++ahc6RGUEz+8ny3frzQSruW -IutcRtLSRGKuNAaITku7CErjdyrWcAcL7YW1YU8+911U4tEI9rC/Qcy3/T4ULH7M -SqXskVoMcd8PLG9F1Si/tW7rHjVeFNRh64eg9frYCArEFrm/5786IqS67yHQp+mF -LfWqKws8rGg2fkpSE38gWEgn47/Qk6HQXEiNvItJ7XWvOg8xMbR0X83kP6Ctn3sr -tzMEaPsoABxVvu8BmYO1SFfEbR2MGiVbVb8dR1373E6dY1QcoNZVXfYPE0QaJff9 -vLORfXmjkqbeefZxgNj5BA2+yIE04td6jGmuVOl79kJz6VFm8osror68aMl/tP77 -70cA/mg632uMNQqhPdkA+iV+GDWbvscXGA9vCUztptX8F6V4eDy9sJKfyPg/Zz33 -ZSoWDe5C+lDLF2zg0Vrv5bXj0qtIpk704qSIcGnehXXeLaWndCnKb88p+W8/jcfP -QeLjCWnCPjGgeAHyAqEIO73puNltOn+aPXyxjtOi/KQRaS81aJZTxq4ffR44FB2z -I082Oa33gmnZbP3sYsVuH44QQAYUEkHBIIosiqklqEoa9G6+Dtm5zUZtmyP6Cx1a -xiN+5XQGoz1fh6sk9/TMW3fpqV0KS1Gu5JsgXTSxxXkarGueFrkCDQRREsLdARAA -3Frw+j6H9McEIi/gjiGwvxnIdGc8McWchnFpOWvdhTW9056v+y22DoKbULjT8k+8 -GzuRQ0xp4VwCC1rX3UExwceczzGs+tSKuIGmg1ELygsaOZHdQBNLGPvn+TZNGlaY -XPlQo7m8YhXGHwgQrdKyjcFD5xnOHxe981LTq+IQ6jVYhho7/Qik9rVE1XHxoOfY -vnNZJD0cFdf9OcX47YoqmM4sZYPMoOmKoVQTsAAQ527wz742Bd6SpuhqBpdEw6Yi -CYxEoo5kBY3IhP3L5OTS4tzhOkdf1xlhWSnCFE7NkPcK6o+r6qCcUqRGV9jRwI97 -JlPKegEHYWvLD4Sk31pWi8NZ0toU/nqRvxbhhtHxuNf3jeAAzxQBhGVi0C/IBr4v -qyFqmEHr9JxIa3DTV8w/a0Y4hX2bczL9Y1cB6n8qOA68aAn+xerJcSOroTIJh83D -/7OguexGGYoZBDvX6dWguf8udFPeYpJvkT6TSYF9U0JpVTtlCNutjScUO2uaV9+u -DqACngwqbzBTjL8UucAleVcFfOi48yepnOd11YFYxbw+/BcqLNhi1eP2AaGxIgXb -R88tF9OC0SXaCH+1Z1bbalOmQNYstOv9BbsHvW7mPgX2xhyoDkVRWaNAQoDLbnJr -4gi9cD8/kQMzdlGOzt2ist/+xueblXJs5TOO80Rw+AEAEQEAAYkCUwQoAQoAPQUC -U4eKeDYdA1NpZ25pbmctb25seSBrZXksIHJldm9raW5nIGVuY3J5cHRpb24gY2Fw -YWJsZSBzdWJrZXkACgkQHjShgo4geQGeGw//fLw5CXRJ/aqz8qgEtI2+9O+Jxh6+ -Jiqyu7cYrRwcuTQLUXAjkE3ZrPRmWGHKL2xsshfO4D2R2KCU7eiy1J0WWvJrMQe6 -u/g8ryJX29tm4rt30L5Vn+iTOms6vHnaDzzK/KpZyrRNMlIhFaJhPYnx950uKVny -F7BSbKIqC3ngApZur4QN5oAv9W/09rMSJ2GIMXa9Mo+4fkDiZ8coHByyxBmbor8I -YsaLCz/ZKuT7RHBbh9mt3S2eIa79M7GV0+4lM2wi+hmPYoA8/Ngv+04hurNNrvgj -6vj1LH+qpEZ+zQdH4QcrQYXKO/1QCXFKKAk8CQw+lGGFcCcrwVgGfzm+ZTGS0OxJ -hXosHb+Bsdfh29fexCiAWMH9mSsNWmo554ZQSdmhOSe6CaYoDSDb+/+FDquzXwJW -ZjTYYez6lbHkJ7icPo5IGb3xxjsXB7vrDOWSIqNptsaRBSjQthsX8NUD8AC/qeSE -K7YL0J3tbuwBYUz3QN5Jk3nmZi+WLBHb9Gz6XruoUN5Cp0PZmYiLQjpU+ZNp2Nb2 -fVNZ/FurSHgu28VZN3fFdW7sQ5/moBLWP/8KK+yMWpJ6Aj5162dOL0+41y9w+u+c -xHuEZsOXFixgubFcljipTDNRfeHNZlsyuWubwYfiHMCFgArktdFRnmtDAnjDqTOk -8ENKxCfgWSoZzteJAiUEGAEKAA8CGwwFAlN2ZdEFCQRE1kEACgkQHjShgo4geQFc -nxAAmTf3BD3xxkU9l0j0uxkizVIWsh4jp+GNVhD80r1QZ6XUfQWlNZS0w8wQAymf -cD8u5EQEnkIegaBecEIVBTAcXxuSqWECaqD2L7S+A8J37gKK+wQ8TPQu+hQivGnR -5STjI/dWZ9iqyR5ZLrvK38obeoylsdRMOncIgJ1J5/ZOyH4IoAEbIwabrfJ/vZ/2 -R/lQXnoE9cu/095pECwHR5AUHl+FcG7B5KPYoY25cI8ZLMn2b9Nl3aK8/b4lVnJY -N9X6c7xcyKhcoUOW1KffVR9X7iiqWuLaZQVV4HX+flxsG6WrmQvGwzXcbcW0/AQA -4Rgd8Rz3X+5zvg2YbdjUMedC1C+nYJ4/nKQGm0uhA+p59jvQX95E2+Bsh4YTw5/v -GqHKRj2IOhROfXEPUtNLIoFbviR/uIEGaaXdhY/XIXn9AemJFXtwV4kBCmqbgou5 -DCKh5MVL8aq6cKfYUhDmlqk2fS7ONVUoMUdan4Ntz9COmCNT6Mdcv9jaZIjhkqzV -OW4cQnMGweTNFGnr9unHxztDA9pXrjyINcEZcqCEXNNEYoha0RQdZvEZzzed+65Y -IgwVvcx0n4doOiJmCI0WNPns1mhzKxDqLz8IfcpxWC1oCCFGP8is7Fgs0UPXoHem -1IsTue7PnRUH7PIWANg464yLlTCqr+hLhsLGjw7D0BR98zCZAg0EURLC3QEQALYJ -H5eBbTgNXBPjbu/ZPj6rP47/EY7BBxqVVpI5S6C5iB/b16cClvqZSpE45O4lY3Wf -DWW9eIDqIXGDoOMNSGynqPnhfsYAACNn1C5q8+Byytpy2wktHP1ZJaoOM4BixvgH -mnCZI8Api2ZEDEtXN3FSy4ezYfFE7hVado0lAQGPuPvO4ayC9IkFKr1WJyLpwYj1 -zV+xbIqyentHozniaZBFuwKDb4b83SGo7PQcTpmUkpUppmq5Os/EZYW5TeYBSnMU -trpaPwNJRiJZn/GuqIV1sHi3Ucze+UoH0sSD3cMS4b/xDRODYpLSURbF6q4FfITr -6OSUM2ol7IeZ+CS4B8WWQ1hrnzvRdPzszMYT+FLjSVuaFBUceXDEyQCIRExzRiz4 -5+BQkxsgNAE1EWDdD9uNqMWhUGAGgoWnsLbbOCgmRQ1P7sdi/+3rN4OUjM2GBiaz -l3SASIB4tq7CO7foGbBSxtAfB6NYo/ykANFkvVtOPQT6tFQzgUU2H0PZdMUUppo2 -5u5vy1Saj5lQgQQp8+E0cr3BL9uwXI+/3Lcct7Zvc1gYUrn4fOeqDw3Qf7b0jvMv -HtWk6HeJE3ctvG/10ya6RUm3DxGr9R2mt3dnmnygKKwWj6Sdj4n4iA2MLR6eI3yu -SSX/isg3l++85HxgK2znSEvV8DLd138woTHP5BNdABEBAAG0KUxFQVAgYXJjaGl2 -ZSBzaWduaW5nIGtleSA8c3lzZGV2QGxlYXAuc2U+iQI9BBMBCgAnBQJREsLdAhsD -BQkB4TOABQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEB40oYKOIHkBkGIP/AnB -RVmN815Z2kqtjL0e1kTG1NxT/3jeWBFQ2KdS4s8VTiLaz6FHeHl/NdHXmdQP4gEm -VeXMtXYb8iNdg9OnhrSCoelZQtpvlK27sDMWI7jb9FZQDNYPUXKHpK5ncUMRG+F6 -J3yzhJeh7rLhwnkSt9KUlLqqqS7yyzyZLyUzUyn6Vz3BhNZD7yGZJSK8Blv1YsSW -MXGG8Se8Ao/APXyjv5L0YAEXBizF+VC8pkeC5o5WoESSyvL6+7yiRY2uDcuXd0NA -R5qGvuesusd7yCo40mA4T2OPFG9smYa0hRPIaqBZlnMF9Bcv/q5cvbdtaRhkORA2 -F+yb5yiUBXyCmP78Ter+4xfCrPVu/Tor8uwrc0fME0TnuQkcSGD0rdTDZbHmrAUx -TUYPGwHc/2n5PKoBugRkAHaKtNHL2Tyl4UHvDB/SxbcSDvxcWx/LiDJXrX1WcFH4 -9t86kqoUVcCh0XWYWIis9559pWkX1+JDk7C3SwmMNomdhPzeSwA7YCP5WxjdSOcO -sYVqJXw16K33L/xbwOqZncRkQX8zeJLTIOOqq30AIUP6OQaUpizIe/uBulOrTKLq -7+RFcdRMi9XAUv0zpnoBRwbyDOipsqFcZ9YoSrSU3kcCMn/9X2ujcN6QV3QD41A6 -UVXpe2ylnCmFV21bVZLTvweZWXAXl1sPylx9P9GNuQINBFESwt0BEADcWvD6Pof0 -xwQiL+COIbC/Gch0ZzwxxZyGcWk5a92FNb3Tnq/7LbYOgptQuNPyT7wbO5FDTGnh -XAILWtfdQTHBx5zPMaz61Iq4gaaDUQvKCxo5kd1AE0sY++f5Nk0aVphc+VCjubxi -FcYfCBCt0rKNwUPnGc4fF73zUtOr4hDqNViGGjv9CKT2tUTVcfGg59i+c1kkPRwV -1/05xfjtiiqYzixlg8yg6YqhVBOwABDnbvDPvjYF3pKm6GoGl0TDpiIJjESijmQF -jciE/cvk5NLi3OE6R1/XGWFZKcIUTs2Q9wrqj6vqoJxSpEZX2NHAj3smU8p6AQdh -a8sPhKTfWlaLw1nS2hT+epG/FuGG0fG41/eN4ADPFAGEZWLQL8gGvi+rIWqYQev0 -nEhrcNNXzD9rRjiFfZtzMv1jVwHqfyo4DrxoCf7F6slxI6uhMgmHzcP/s6C57EYZ -ihkEO9fp1aC5/y50U95ikm+RPpNJgX1TQmlVO2UI262NJxQ7a5pX364OoAKeDCpv -MFOMvxS5wCV5VwV86LjzJ6mc53XVgVjFvD78Fyos2GLV4/YBobEiBdtHzy0X04LR -JdoIf7VnVttqU6ZA1iy06/0Fuwe9buY+BfbGHKgORVFZo0BCgMtucmviCL1wPz+R -AzN2UY7O3aKy3/7G55uVcmzlM47zRHD4AQARAQABiQIlBBgBCgAPBQJREsLdAhsM -BQkB4TOAAAoJEB40oYKOIHkB0yoP/1Je6UmrpDhTto3qjtMsxTYyCp5aq0GrBM0i -KwGhImNer053iI3ZLkCTCj9lA7TWdE089z3KPCJ2BvvAJQUM2CVQO4ZLaddpDRcA -7zeocw6w5E3ZL3d07rFVFfYKGHOX1tSvVYhBwz6Uiz8tkJfau8qWwViZlAdf+JuF -CztPTVFdqPAmrDHJfJU8v6Q816jC9rhlR/qvN2opkt8WDInIZ0cjXxsXqVqGAd03 -Y45rAR5TYr4yWW5HdlqGjI3i7UyqKN/qyeZ93T74QioMfBg+fvQXgk90yHK5WlgF -XJW2yAfL1bl9L7NKP69f6qPowFd0UGz5r2evwZkk7gFQ/aa3I2CYR9RVB59Ieer7 -Q6UY9kbitrFMSol+nYpNEIYHiEjNhY7WDVw4F5fEV9cl2Rfwugps/mzYrOf2wAro -HGZrM69I2m6KYIM28qNHTSIo1LgPGLHYUFdBT/ilWkYeNdNA/pGp5OsB51e+mAId -Y/VQiWzAhB5waZBwn+FFmN37S+VyN8JiFkevs33oNroLLy/Z9mJkdpIC3tPaghn1 -feASH+Q6UsNy4h28wdR+iOfQys8J7UCo4fJBCqGRXKW9mlLQ90TRVbc+Yn3y4p9f -YVUA5NQPhHDef0TDtt8zbj0e4/gk5eL9KSniwEH/GCu1XW74T4FbRJL8E6LB+sjj -FTL4yta7 -=DRTl ------END PGP PUBLIC KEY BLOCK----- diff --git a/puppet/modules/site_apt/manifests/dist_upgrade.pp b/puppet/modules/site_apt/manifests/dist_upgrade.pp index 08de31bb..0eb98cea 100644 --- a/puppet/modules/site_apt/manifests/dist_upgrade.pp +++ b/puppet/modules/site_apt/manifests/dist_upgrade.pp @@ -1,17 +1,17 @@ +# upgrade all packages class site_apt::dist_upgrade { + # facter returns 'true' as string + # lint:ignore:quoted_booleans if $::apt_running == 'true' { + # lint:endignore fail ('apt-get is running in background - Please wait until it finishes. Exiting.') } else { - exec{'initial_apt_update': - command => '/usr/bin/apt-get update', - refreshonly => false, - timeout => 360, - } exec{'initial_apt_dist_upgrade': command => "/usr/bin/apt-get -q -y -o 'DPkg::Options::=--force-confold' dist-upgrade", refreshonly => false, timeout => 1200, + require => Exec['apt_updated'] } } } diff --git a/puppet/modules/site_apt/manifests/init.pp b/puppet/modules/site_apt/manifests/init.pp index cf49f870..455425c1 100644 --- a/puppet/modules/site_apt/manifests/init.pp +++ b/puppet/modules/site_apt/manifests/init.pp @@ -3,15 +3,29 @@ class site_apt { $sources = hiera('sources') $apt_config = $sources['apt'] + + # debian repo urls $apt_url_basic = $apt_config['basic'] $apt_url_security = $apt_config['security'] $apt_url_backports = $apt_config['backports'] + # leap repo url + $platform_sources = $sources['platform'] + $apt_url_platform_basic = $platform_sources['apt']['basic'] + + # needed on jessie hosts for getting pnp4nagios from testing + if ( $::operatingsystemmajrelease == '8' ) { + $use_next_release = true + } else { + $use_next_release = false + } + class { 'apt': - custom_key_dir => 'puppet:///modules/site_apt/keys', - debian_url => $apt_url_basic, - security_url => $apt_url_security, - backports_url => $apt_url_backports + custom_key_dir => 'puppet:///modules/site_apt/keys', + debian_url => $apt_url_basic, + security_url => $apt_url_security, + backports_url => $apt_url_backports, + use_next_release => $use_next_release } # enable http://deb.leap.se debian package repository @@ -23,14 +37,10 @@ class site_apt { include ::site_apt::unattended_upgrades - apt::sources_list { 'secondary.list.disabled': - content => template('site_apt/secondary.list'); - } - - apt::preferences_snippet { 'facter': - release => "${::lsbdistcodename}-backports", - priority => 999 - } + # not currently used + #apt::sources_list { 'secondary.list': + # content => template('site_apt/secondary.list'); + #} apt::preferences_snippet { 'leap': priority => 999, @@ -38,13 +48,8 @@ class site_apt { pin => 'origin "deb.leap.se"' } - # All packages should be installed _after_ refresh_apt is called, - # which does an apt-get update. - # There is one exception: - # The creation of sources.list depends on the lsb package + # All packages should be installed after 'update_apt' is called, + # which does an 'apt-get update'. + Exec['update_apt'] -> Package <||> - File['/etc/apt/preferences'] -> - Apt::Preferences_snippet <| |> -> - Exec['refresh_apt'] -> - Package <| ( title != 'lsb' ) |> } diff --git a/puppet/modules/site_apt/manifests/leap_repo.pp b/puppet/modules/site_apt/manifests/leap_repo.pp index 2d4ba0e1..5eedce45 100644 --- a/puppet/modules/site_apt/manifests/leap_repo.pp +++ b/puppet/modules/site_apt/manifests/leap_repo.pp @@ -1,17 +1,16 @@ +# install leap deb repo together with leap-keyring package +# containing the apt signing key class site_apt::leap_repo { $platform = hiera_hash('platform') $major_version = $platform['major_version'] apt::sources_list { 'leap.list': - content => "deb http://deb.leap.se/${major_version} wheezy main\n", + content => "deb ${::site_apt::apt_url_platform_basic} ${::lsbdistcodename} main\n", before => Exec[refresh_apt] } - package { 'leap-keyring': + package { 'leap-archive-keyring': ensure => latest } - # We wont be able to install the leap-keyring package unless the leap apt - # source has been added and apt has been refreshed - Exec['refresh_apt'] -> Package['leap-keyring'] } diff --git a/puppet/modules/site_apt/manifests/preferences/obfsproxy.pp b/puppet/modules/site_apt/manifests/preferences/obfsproxy.pp deleted file mode 100644 index 75b01956..00000000 --- a/puppet/modules/site_apt/manifests/preferences/obfsproxy.pp +++ /dev/null @@ -1,9 +0,0 @@ -class site_apt::preferences::obfsproxy { - - apt::preferences_snippet { 'obfsproxy': - package => 'obfsproxy', - release => 'wheezy-backports', - priority => 999; - } - -} diff --git a/puppet/modules/site_apt/manifests/preferences/openvpn.pp b/puppet/modules/site_apt/manifests/preferences/openvpn.pp deleted file mode 100644 index c7ddae25..00000000 --- a/puppet/modules/site_apt/manifests/preferences/openvpn.pp +++ /dev/null @@ -1,9 +0,0 @@ -class site_apt::preferences::openvpn { - - apt::preferences_snippet { 'openvpn': - package => 'openvpn', - release => "${::lsbdistcodename}-backports", - priority => 999; - } - -} diff --git a/puppet/modules/site_apt/manifests/preferences/twisted.pp b/puppet/modules/site_apt/manifests/preferences/twisted.pp deleted file mode 100644 index abff6838..00000000 --- a/puppet/modules/site_apt/manifests/preferences/twisted.pp +++ /dev/null @@ -1,9 +0,0 @@ -class site_apt::preferences::twisted { - - apt::preferences_snippet { 'python-twisted': - package => 'python-twisted*', - release => "${::lsbdistcodename}-backports", - priority => 999; - } - -} diff --git a/puppet/modules/site_apt/manifests/preferences/unbound.pp b/puppet/modules/site_apt/manifests/preferences/unbound.pp deleted file mode 100644 index 6da964f9..00000000 --- a/puppet/modules/site_apt/manifests/preferences/unbound.pp +++ /dev/null @@ -1,10 +0,0 @@ -class site_apt::preferences::unbound { - - apt::preferences_snippet { 'unbound': - package => 'libunbound* unbound*', - release => "${::lsbdistcodename}-backports", - priority => 999, - before => Class['unbound::package']; - } - -} diff --git a/puppet/modules/site_apt/manifests/unattended_upgrades.pp b/puppet/modules/site_apt/manifests/unattended_upgrades.pp index 40111deb..42f1f4c6 100644 --- a/puppet/modules/site_apt/manifests/unattended_upgrades.pp +++ b/puppet/modules/site_apt/manifests/unattended_upgrades.pp @@ -1,9 +1,20 @@ +# configute unattended upgrades so packages from both Debian and LEAP +# repos get upgraded unattended class site_apt::unattended_upgrades { # override unattended-upgrades package resource to make sure # that it is upgraded on every deploy (#6245) + # configure upgrades for Debian class { 'apt::unattended_upgrades': - config_content => template('site_apt/50unattended-upgrades'), ensure_version => latest } + + # configure LEAP upgrades + apt::apt_conf { '51unattended-upgrades-leap': + source => [ + "puppet:///modules/site_apt/${::lsbdistid}/51unattended-upgrades-leap"], + require => Package['unattended-upgrades'], + refresh_apt => false, + } + } diff --git a/puppet/modules/site_apt/templates/50unattended-upgrades b/puppet/modules/site_apt/templates/50unattended-upgrades deleted file mode 100644 index 9ae3ab84..00000000 --- a/puppet/modules/site_apt/templates/50unattended-upgrades +++ /dev/null @@ -1,16 +0,0 @@ -// this file is managed by puppet ! - -Unattended-Upgrade::Allowed-Origins { - "${distro_id}:oldstable"; - "${distro_id}:${distro_codename}-security"; - "${distro_id}:${distro_codename}-updates"; - "${distro_id} Backports:${distro_codename}-backports"; - "leap.se:stable"; -}; - -APT::Periodic::Update-Package-Lists "1"; -APT::Periodic::Download-Upgradeable-Packages "1"; -APT::Periodic::Unattended-Upgrade "1"; - -Unattended-Upgrade::Mail "root"; -Unattended-Upgrade::MailOnlyOnError "true"; diff --git a/puppet/modules/site_apt/templates/jessie/postfix.seeds b/puppet/modules/site_apt/templates/jessie/postfix.seeds new file mode 100644 index 00000000..1a878ccc --- /dev/null +++ b/puppet/modules/site_apt/templates/jessie/postfix.seeds @@ -0,0 +1 @@ +postfix postfix/main_mailer_type select No configuration diff --git a/puppet/modules/site_apt/templates/secondary.list b/puppet/modules/site_apt/templates/secondary.list index 41334b0b..0c024549 100644 --- a/puppet/modules/site_apt/templates/secondary.list +++ b/puppet/modules/site_apt/templates/secondary.list @@ -1,3 +1,3 @@ # basic -deb http://ftp.debian.org/debian/ <%= lsbdistcodename %> main contrib non-free +deb http://ftp.debian.org/debian/ <%= @lsbdistcodename %> main contrib non-free diff --git a/puppet/modules/site_check_mk/files/agent/local_checks/couchdb/leap_couch_stats.sh b/puppet/modules/site_check_mk/files/agent/local_checks/couchdb/leap_couch_stats.sh index 83b407e0..c7477b18 100755 --- a/puppet/modules/site_check_mk/files/agent/local_checks/couchdb/leap_couch_stats.sh +++ b/puppet/modules/site_check_mk/files/agent/local_checks/couchdb/leap_couch_stats.sh @@ -11,7 +11,7 @@ start_time=$(date +%s.%N) CURL='curl -s --netrc-file /etc/couchdb/couchdb.netrc' URL='http://127.0.0.1:5984' TMPFILE=$(mktemp) -DBLIST_EXCLUDE='(user-|sessions_|tokens_)' +DBLIST_EXCLUDE='(user-|sessions_|tokens_|_replicator|_users)' PREFIX='Couchdb_' @@ -104,7 +104,7 @@ do done # special handling for rotated dbs -suffix=$(($(date +'%s') / (60*60*24*30) + 1)) +suffix=$(($(date +'%s') / (60*60*24*30))) db_stats "sessions_${suffix}" "sessions" db_stats "tokens_${suffix}" "tokens" diff --git a/puppet/modules/site_check_mk/files/agent/local_checks/mx/check_leap_mx.sh b/puppet/modules/site_check_mk/files/agent/local_checks/mx/check_leap_mx.sh index b8687c9a..4711e247 100755 --- a/puppet/modules/site_check_mk/files/agent/local_checks/mx/check_leap_mx.sh +++ b/puppet/modules/site_check_mk/files/agent/local_checks/mx/check_leap_mx.sh @@ -12,7 +12,7 @@ STATUS[1]='Warning' STATUS[2]='Critical' CHECKNAME='Leap_MX_Queue' -WATCHDIR='/var/mail/vmail/Maildir/new/' +WATCHDIR='/var/mail/leap-mx/Maildir/new/' total=`find $WATCHDIR -type f -mmin +$MAXAGE | wc -l` diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg index 95ddd2ca..0f378a5a 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg @@ -6,7 +6,7 @@ I 127.0.0.1 localhost:5984 .* ok # https://leap.se/code/issues/5246 I Shutting down group server - # ignore bigcouch conflict errors, mainly coming from tapicero creating new users + # ignore bigcouch conflict errors I Error in process.*{{nocatch,conflict} # ignore "Uncaught error in HTTP request: {exit, normal}" error # it's suppressed in later versions of bigcouch anhow diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/bigcouch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/bigcouch.cfg new file mode 100644 index 00000000..f53f0780 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/bigcouch.cfg @@ -0,0 +1,5 @@ +# on one-node bigcouch setups, we'll get this msg +# a lot, so we ignore it here until we fix +# https://leap.se/code/issues/5244 + I epmd: got partial packet only on file descriptor + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg index f546135a..5f8d5b95 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg @@ -1,7 +1,2 @@ C /usr/local/bin/couch-doc-update.*failed C /usr/local/bin/couch-doc-update.*ERROR -# on one-node bigcouch setups, we'll get this msg -# a lot, so we ignore it here until we fix -# https://leap.se/code/issues/5244 - I epmd: got partial packet only on file descriptor - diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/tapicero.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/tapicero.cfg deleted file mode 100644 index d98f5094..00000000 --- a/puppet/modules/site_check_mk/files/agent/logwatch/tapicero.cfg +++ /dev/null @@ -1,11 +0,0 @@ -/var/log/leap/tapicero.log -# Ignore transient Tapicero errors when creating a db (#6511) - I tapicero.*(Creating database|Checking security of|Writing security to|Uploading design doc to) user-.* failed (\(trying again soon\)|(twice )?due to): (RestClient::ResourceNotFound|RestClient::InternalServerError): (404 Resource Not Found|500 Internal Server Error) - C tapicero.*RestClient::InternalServerError: -# possible race condition between multiple tapicero -# instances, so we ignore it -# see https://leap.se/code/issues/5168 - I tapicero.*RestClient::PreconditionFailed: - C tapicero.*Creating database.*failed due to: - C tapicero.*failed - W tapicero.*Couch stream ended unexpectedly. diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/webapp.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/webapp.cfg index 008e9e09..337d9ec6 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/webapp.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/webapp.cfg @@ -1,5 +1,7 @@ /var/log/leap/webapp.log # check for webapp errors + C Completed 500 +# couch connection issues C webapp.*Could not connect to couch database messages due to 401 Unauthorized: {"error":"unauthorized","reason":"You are not a server admin."} # ignore RoutingErrors that rails throw when it can't handle a url # see https://leap.se/code/issues/5173 diff --git a/puppet/modules/site_check_mk/files/extra_host_conf.mk b/puppet/modules/site_check_mk/files/extra_host_conf.mk deleted file mode 100644 index 2c96f97a..00000000 --- a/puppet/modules/site_check_mk/files/extra_host_conf.mk +++ /dev/null @@ -1,6 +0,0 @@ -# retry 3 times before setting a host into a hard state -# and send out notification -extra_host_conf["max_check_attempts"] = [ - ("4", ALL_HOSTS ) -] - diff --git a/puppet/modules/site_check_mk/manifests/agent.pp b/puppet/modules/site_check_mk/manifests/agent.pp index 589041eb..b95d5d64 100644 --- a/puppet/modules/site_check_mk/manifests/agent.pp +++ b/puppet/modules/site_check_mk/manifests/agent.pp @@ -1,17 +1,24 @@ +# installs check-mk agent class site_check_mk::agent { $ssh_hash = hiera('ssh') $pubkey = $ssh_hash['authorized_keys']['monitor']['key'] $type = $ssh_hash['authorized_keys']['monitor']['type'] + + # /usr/bin/mk-job depends on /usr/bin/time + ensure_packages('time') + class { 'site_apt::preferences::check_mk': } -> class { 'check_mk::agent': agent_package_name => 'check-mk-agent', agent_logwatch_package_name => 'check-mk-agent-logwatch', method => 'ssh', - homedir => '/etc/nagios/check_mk', - register_agent => false + authdir => '/root/.ssh', + authfile => 'authorized_keys', + register_agent => false, + require => Package['time'] } -> class { 'site_check_mk::agent::mrpe': } -> diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp index abfc7ad0..1554fd3c 100644 --- a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp @@ -1,32 +1,18 @@ +# configure logwatch and nagios checks for couchdb (both bigcouch and plain +# couchdb installations) class site_check_mk::agent::couchdb { - # watch logs - file { '/etc/check_mk/logwatch.d/bigcouch.cfg': - source => 'puppet:///modules/site_check_mk/agent/logwatch/bigcouch.cfg', - } concat::fragment { 'syslog_couchdb': source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/couchdb.cfg', target => '/etc/check_mk/logwatch.d/syslog.cfg', order => '02'; } - - # check bigcouch processes - augeas { - 'Bigcouch_epmd_procs': - incl => '/etc/check_mk/mrpe.cfg', - lens => 'Spacevars.lns', - changes => [ - 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_epmd_procs', - 'set Bigcouch_epmd_procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/epmd\'' ], - require => File['/etc/check_mk/mrpe.cfg']; - 'Bigcouch_beam_procs': - incl => '/etc/check_mk/mrpe.cfg', - lens => 'Spacevars.lns', - changes => [ - 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_beam_procs', - 'set Bigcouch_beam_procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/beam\'' ], - require => File['/etc/check_mk/mrpe.cfg']; + # check different couchdb stats + file { '/usr/lib/check_mk_agent/local/leap_couch_stats.sh': + source => 'puppet:///modules/site_check_mk/agent/local_checks/couchdb/leap_couch_stats.sh', + mode => '0755', + require => Package['check_mk-agent'] } # check open files for bigcouch proc @@ -36,20 +22,13 @@ class site_check_mk::agent::couchdb { mode => '0755' } augeas { - 'Bigcouch_open_files': + 'Couchdb_open_files': incl => '/etc/check_mk/mrpe.cfg', lens => 'Spacevars.lns', changes => [ - 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_open_files', - 'set Bigcouch_open_files \'/srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 28672,28672 -c 30720,30720\'' ], + 'rm /files/etc/check_mk/mrpe.cfg/Couchdb_open_files', + 'set Couchdb_open_files \'/srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 28672,28672 -c 30720,30720\'' ], require => File['/etc/check_mk/mrpe.cfg']; } - - # check different couchdb stats - file { '/usr/lib/check_mk_agent/local/leap_couch_stats.sh': - source => 'puppet:///modules/site_check_mk/agent/local_checks/couchdb/leap_couch_stats.sh', - mode => '0755', - require => Package['check_mk-agent'] - } } diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp new file mode 100644 index 00000000..82c3ac72 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp @@ -0,0 +1,49 @@ +# configure logwatch and nagios checks for bigcouch +class site_check_mk::agent::couchdb::bigcouch { + + # watch bigcouch logs + # currently disabled because bigcouch is too noisy + # see https://leap.se/code/issues/7375 for more details + # and site_config::remove_files for removing leftovers + #file { '/etc/check_mk/logwatch.d/bigcouch.cfg': + # source => 'puppet:///modules/site_check_mk/agent/logwatch/bigcouch.cfg', + #} + + # check syslog msg from: + # - empd + # - /usr/local/bin/couch-doc-update + concat::fragment { 'syslog_bigcouch': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/bigcouch.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + + # check bigcouch processes + augeas { + 'Bigcouch_epmd_procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_epmd_procs', + 'set Bigcouch_epmd_procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/epmd\'' ], + require => File['/etc/check_mk/mrpe.cfg']; + 'Bigcouch_beam_procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_beam_procs', + 'set Bigcouch_beam_procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/beam\'' ], + require => File['/etc/check_mk/mrpe.cfg']; + } + + augeas { + 'Bigcouch_open_files': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_open_files', + 'set Bigcouch_open_files \'/srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 28672,28672 -c 30720,30720\'' ], + require => File['/etc/check_mk/mrpe.cfg']; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb/plain.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb/plain.pp new file mode 100644 index 00000000..3ec2267b --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb/plain.pp @@ -0,0 +1,23 @@ +# configure logwatch and nagios checks for plain single couchdb master +class site_check_mk::agent::couchdb::plain { + + # remove bigcouch leftovers + augeas { + 'Bigcouch_epmd_procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_epmd_procs', + require => File['/etc/check_mk/mrpe.cfg']; + 'Bigcouch_beam_procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_beam_procs', + require => File['/etc/check_mk/mrpe.cfg']; + 'Bigcouch_open_files': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_open_files', + require => File['/etc/check_mk/mrpe.cfg']; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/mx.pp b/puppet/modules/site_check_mk/manifests/agent/mx.pp index 98757b59..20cbcade 100644 --- a/puppet/modules/site_check_mk/manifests/agent/mx.pp +++ b/puppet/modules/site_check_mk/manifests/agent/mx.pp @@ -1,3 +1,4 @@ +# check check_mk agent checks for mx service class site_check_mk::agent::mx { # watch logs @@ -6,13 +7,13 @@ class site_check_mk::agent::mx { } # local nagios plugin checks via mrpe + # removed because leap_cli integrates a check for running mx procs already, + # which is also integrated into nagios (called "Mx/Are_MX_daemons_running") augeas { 'Leap_MX_Procs': incl => '/etc/check_mk/mrpe.cfg', lens => 'Spacevars.lns', - changes => [ - 'rm /files/etc/check_mk/mrpe.cfg/Leap_MX_Procs', - 'set Leap_MX_Procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a "/usr/bin/python /usr/bin/twistd --pidfile=/var/run/leap_mx.pid --rundir=/var/lib/leap_mx/ --python=/usr/share/app/leap_mx.tac --logfile=/var/log/leap/mx.log"\'' ], + changes => 'rm /files/etc/check_mk/mrpe.cfg/Leap_MX_Procs', require => File['/etc/check_mk/mrpe.cfg']; } diff --git a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp deleted file mode 100644 index 8505b34a..00000000 --- a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp +++ /dev/null @@ -1,26 +0,0 @@ -# sets up tapicero monitoring -class site_check_mk::agent::tapicero { - - include ::site_nagios::plugins - - # watch logs - file { '/etc/check_mk/logwatch.d/tapicero.cfg': - source => 'puppet:///modules/site_check_mk/agent/logwatch/tapicero.cfg', - } - - # local nagios plugin checks via mrpe - augeas { - 'Tapicero_Procs': - incl => '/etc/check_mk/mrpe.cfg', - lens => 'Spacevars.lns', - changes => [ - 'rm /files/etc/check_mk/mrpe.cfg/Tapicero_Procs', - "set Tapicero_Procs \"/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 --ereg-argument-array='^tapicero$'\"" ], - require => File['/etc/check_mk/mrpe.cfg']; - 'Tapicero_Heartbeat': - incl => '/etc/check_mk/mrpe.cfg', - lens => 'Spacevars.lns', - changes => 'set Tapicero_Heartbeat \'/usr/local/lib/nagios/plugins/check_last_regex_in_log -f /var/log/leap/tapicero.log -r "tapicero" -w 1200 -c 2400\'', - require => File['/etc/check_mk/mrpe.cfg']; - } -} diff --git a/puppet/modules/site_check_mk/manifests/server.pp b/puppet/modules/site_check_mk/manifests/server.pp index 67519513..7ff9eb4a 100644 --- a/puppet/modules/site_check_mk/manifests/server.pp +++ b/puppet/modules/site_check_mk/manifests/server.pp @@ -17,20 +17,35 @@ class site_check_mk::server { ensure => installed, } + # we don't use check-mk-multisite, and the jessie version + # of this config file breaks with apache 2.4 + # until https://gitlab.com/shared-puppet-modules-group/apache/issues/11 + # is not fixed, we need to use a generic file type here + #apache::config::global { 'check-mk-multisite.conf': + # ensure => absent + #} + + file { '/etc/apache2/conf-enabled/check-mk-multisite.conf': + ensure => absent, + require => Package['check-mk-server']; + } + # override paths to use the system check_mk rather than OMD class { 'check_mk::config': - site => '', - etc_dir => '/etc', - nagios_subdir => 'nagios3', - bin_dir => '/usr/bin', - host_groups => undef, - use_storedconfigs => false, - require => Package['check-mk-server'] + site => '', + etc_dir => '/etc', + nagios_subdir => 'nagios3', + bin_dir => '/usr/bin', + host_groups => undef, + use_storedconfigs => false, + inventory_only_on_changes => false, + require => Package['check-mk-server'] } - Exec['check_mk-reload'] -> + Exec['check_mk-refresh'] -> Exec['check_mk-refresh-inventory-daily'] -> - Service['nagios'] + Exec['check_mk-reload'] -> + Service['nagios'] file { '/etc/check_mk/conf.d/use_ssh.mk': @@ -54,7 +69,7 @@ class site_check_mk::server { notify => Exec['check_mk-refresh'], require => Package['check-mk-server']; '/etc/check_mk/conf.d/extra_host_conf.mk': - source => 'puppet:///modules/site_check_mk/extra_host_conf.mk', + content => template('site_check_mk/extra_host_conf.mk'), notify => Exec['check_mk-refresh'], require => Package['check-mk-server']; diff --git a/puppet/modules/site_check_mk/templates/extra_host_conf.mk b/puppet/modules/site_check_mk/templates/extra_host_conf.mk new file mode 100644 index 00000000..bc27b514 --- /dev/null +++ b/puppet/modules/site_check_mk/templates/extra_host_conf.mk @@ -0,0 +1,13 @@ +# retry 3 times before setting a host into a hard state +# and send out notification +extra_host_conf["max_check_attempts"] = [ + ("4", ALL_HOSTS ) +] + +# Use hostnames as alias so notification mail subjects +# are more readable and not so long. Alias defaults to +# the fqdn of a host is not changed. +extra_host_conf["alias"] = [ +<% @hosts.keys.sort.each do |key| -%> ( "<%= key.strip %>", ["<%= @hosts[key]['domain_internal']%>"]), +<% end -%> +] diff --git a/puppet/modules/site_check_mk/templates/use_ssh.mk b/puppet/modules/site_check_mk/templates/use_ssh.mk index 0bebebcf..55269536 100644 --- a/puppet/modules/site_check_mk/templates/use_ssh.mk +++ b/puppet/modules/site_check_mk/templates/use_ssh.mk @@ -1,6 +1,6 @@ # http://mathias-kettner.de/checkmk_datasource_programs.html datasource_programs = [ -<% nagios_hosts.sort.each do |name,config| %> +<% @nagios_hosts.sort.each do |name,config| %> ( "ssh -l root -i /etc/check_mk/.ssh/id_rsa -p <%=config['ssh_port']%> <%=config['domain_internal']%> check_mk_agent", [ "<%=config['domain_internal']%>" ], ),<%- end -%> ] diff --git a/puppet/modules/site_config/manifests/caching_resolver.pp b/puppet/modules/site_config/manifests/caching_resolver.pp index cdebbad0..8bf465c1 100644 --- a/puppet/modules/site_config/manifests/caching_resolver.pp +++ b/puppet/modules/site_config/manifests/caching_resolver.pp @@ -1,14 +1,13 @@ +# deploy local caching resolver class site_config::caching_resolver { tag 'leap_base' - include site_apt::preferences::unbound - class { 'unbound': root_hints => false, anchor => false, ssl => false, settings => { - server => { + server => { verbosity => '1', interface => [ '127.0.0.1', '::1' ], port => '53', diff --git a/puppet/modules/site_config/manifests/default.pp b/puppet/modules/site_config/manifests/default.pp index e69e4b7b..256de1a1 100644 --- a/puppet/modules/site_config/manifests/default.pp +++ b/puppet/modules/site_config/manifests/default.pp @@ -1,19 +1,11 @@ +# common things to set up on every node class site_config::default { tag 'leap_base' - # the logoutput exec parameter defaults to "on_error" in puppet 3, - # but to "false" in puppet 2.7, so we need to set this globally here - Exec<||> { logoutput => on_failure } - $services = hiera('services', []) $domain_hash = hiera('domain') include site_config::params - - # make sure apt is updated before any packages are installed - include apt::update - Package { require => Exec['apt_updated'] } - - include site_config::slow + include site_config::setup # default class, used by all hosts @@ -29,7 +21,10 @@ class site_config::default { # i.e. openstack/aws nodes, vagrant nodes # fix dhclient from changing resolver information - if $::dhcp_enabled == 'true' { + # facter returns 'true' as string + # lint:ignore:quoted_booleans + if $::dhcp_enabled == 'true' { + # lint:endignore include site_config::dhclient } @@ -46,7 +41,7 @@ class site_config::default { include haveged # install/remove base packages - include site_config::packages::base + include site_config::packages # include basic shorewall config include site_shorewall::defaults @@ -58,7 +53,9 @@ class site_config::default { # set up core leap files and directories include site_config::files - include site_config::remove_files + + # remove leftovers from previous deploys + include site_config::remove if ! member($services, 'mx') { include site_postfix::satellite diff --git a/puppet/modules/site_config/manifests/dhclient.pp b/puppet/modules/site_config/manifests/dhclient.pp index 7755413b..a1f87d41 100644 --- a/puppet/modules/site_config/manifests/dhclient.pp +++ b/puppet/modules/site_config/manifests/dhclient.pp @@ -1,10 +1,10 @@ +# Unfortunately, there does not seem to be a way to reload the dhclient.conf +# config file, or a convenient way to disable the modifications to +# /etc/resolv.conf. So the following makes the functions involved noops and +# ships a script to kill and restart dhclient. See the debian bugs: +# #681698, #712796 class site_config::dhclient { - # Unfortunately, there does not seem to be a way to reload the dhclient.conf - # config file, or a convenient way to disable the modifications to - # /etc/resolv.conf. So the following makes the functions involved noops and - # ships a script to kill and restart dhclient. See the debian bugs: - # #681698, #712796 include site_config::params @@ -23,10 +23,10 @@ class site_config::dhclient { } file { '/etc/dhcp/dhclient-enter-hooks.d': - ensure => directory, - mode => '0755', - owner => 'root', - group => 'root', + ensure => directory, + mode => '0755', + owner => 'root', + group => 'root', } file { '/etc/dhcp/dhclient-enter-hooks.d/disable_resolvconf': diff --git a/puppet/modules/site_config/manifests/files.pp b/puppet/modules/site_config/manifests/files.pp index 684d3ad0..d2ef8a98 100644 --- a/puppet/modules/site_config/manifests/files.pp +++ b/puppet/modules/site_config/manifests/files.pp @@ -1,3 +1,4 @@ +# set up core leap files and directories class site_config::files { file { @@ -7,15 +8,15 @@ class site_config::files { group => 'root', mode => '0711'; - '/var/lib/leap': + [ '/etc/leap', '/var/lib/leap']: ensure => directory, - owner => root, + owner => 'root', group => 'root', mode => '0755'; '/var/log/leap': ensure => directory, - owner => root, + owner => 'root', group => 'adm', mode => '0750'; } diff --git a/puppet/modules/site_config/manifests/packages.pp b/puppet/modules/site_config/manifests/packages.pp new file mode 100644 index 00000000..140189a4 --- /dev/null +++ b/puppet/modules/site_config/manifests/packages.pp @@ -0,0 +1,32 @@ +# install default packages and remove unwanted packages +class site_config::packages { + + + # base set of packages that we want to have installed everywhere + package { [ 'etckeeper', 'screen', 'less', 'ntp' ]: + ensure => installed, + } + + # base set of packages that we want to remove everywhere + package { [ + 'acpi', 'build-essential', + 'cpp', 'cpp-4.6', 'cpp-4.7', 'cpp-4.8', 'cpp-4.9', + 'eject', 'ftp', + 'g++', 'g++-4.6', 'g++-4.7', 'g++-4.8', 'g++-4.9', + 'gcc', 'gcc-4.6', 'gcc-4.7', 'gcc-4.8', 'gcc-4.9', + 'laptop-detect', 'libc6-dev', 'libssl-dev', 'lpr', 'make', + 'pppconfig', 'pppoe', 'pump', 'qstat', + 'samba-common', 'samba-common-bin', 'smbclient', + 'tcl8.5', 'tk8.5', 'os-prober', 'unzip', 'xauth', 'x11-common', + 'x11-utils', 'xterm' ]: + ensure => purged; + } + + # leave a few packages installed on local environments + # vagrant i.e. needs them for mounting shared folders + if $::site_config::params::environment != 'local' { + package { [ 'nfs-common', 'nfs-kernel-server', 'rpcbind', 'portmap' ]: + ensure => purged; + } + } +} diff --git a/puppet/modules/site_config/manifests/packages/base.pp b/puppet/modules/site_config/manifests/packages/base.pp deleted file mode 100644 index c23495fc..00000000 --- a/puppet/modules/site_config/manifests/packages/base.pp +++ /dev/null @@ -1,19 +0,0 @@ -# install default packages and remove unwanted packages -class site_config::packages::base { - - - # base set of packages that we want to have installed everywhere - package { [ 'etckeeper', 'screen', 'less', 'ntp' ]: - ensure => installed, - } - - # base set of packages that we want to remove everywhere - package { [ 'acpi', 'eject', 'ftp', - 'laptop-detect', 'lpr', 'nfs-common', 'nfs-kernel-server', - 'portmap', 'pppconfig', 'pppoe', 'pump', 'qstat', 'rpcbind', - 'samba-common', 'samba-common-bin', 'smbclient', 'tcl8.5', - 'tk8.5', 'os-prober', 'unzip', 'xauth', 'x11-common', - 'x11-utils', 'xterm' ]: - ensure => absent; - } -} diff --git a/puppet/modules/site_config/manifests/packages/build_essential.pp b/puppet/modules/site_config/manifests/packages/build_essential.pp index 7dfb8b03..2b3e13b9 100644 --- a/puppet/modules/site_config/manifests/packages/build_essential.pp +++ b/puppet/modules/site_config/manifests/packages/build_essential.pp @@ -1,11 +1,28 @@ # # include this whenever you want to ensure build-essential package and related compilers are installed. # -class site_config::packages::build_essential { - if !defined(Package['build-essential']) { - package { - ['build-essential', 'g++', 'g++-4.7', 'gcc', 'gcc-4.6', 'gcc-4.7', 'cpp', 'cpp-4.6', 'cpp-4.7', 'libc6-dev']: +class site_config::packages::build_essential inherits ::site_config::packages { + + # NICKSERVER CODE NOTE: in order to support TLS, libssl-dev must be installed + # before EventMachine gem is built/installed. + Package[ 'gcc', 'make', 'g++', 'cpp', 'libssl-dev', 'libc6-dev' ] { + ensure => present + } + + case $::operatingsystemrelease { + /^8.*/: { + Package[ 'gcc-4.9','g++-4.9', 'cpp-4.9' ] { + ensure => present + } + } + + /^7.*/: { + Package[ 'gcc-4.7','g++-4.7', 'cpp-4.7' ] { ensure => present + } } + + default: { } } -}
\ No newline at end of file + +} diff --git a/puppet/modules/site_config/manifests/packages/uninstall.pp b/puppet/modules/site_config/manifests/packages/uninstall.pp deleted file mode 100644 index 12f527d9..00000000 --- a/puppet/modules/site_config/manifests/packages/uninstall.pp +++ /dev/null @@ -1,16 +0,0 @@ -# -# Uninstall build-essential and compilers, unless they have been explicitly installed elsewhere. -# -class site_config::packages::uninstall { - tag 'leap_base' - - # generally, dev packages are needed for installing ruby gems with native extensions. - # (nickserver, webapp, etc) - - if !defined(Package['build-essential']) { - package { - ['build-essential', 'g++', 'g++-4.7', 'gcc', 'gcc-4.6', 'gcc-4.7', 'cpp', 'cpp-4.6', 'cpp-4.7', 'libc6-dev']: - ensure => purged - } - } -}
\ No newline at end of file diff --git a/puppet/modules/site_config/manifests/remove.pp b/puppet/modules/site_config/manifests/remove.pp new file mode 100644 index 00000000..443df9c2 --- /dev/null +++ b/puppet/modules/site_config/manifests/remove.pp @@ -0,0 +1,11 @@ +# remove leftovers from previous deploys +class site_config::remove { + include site_config::remove::files + + case $::operatingsystemrelease { + /^8.*/: { + include site_config::remove::jessie + } + default: { } + } +} diff --git a/puppet/modules/site_config/manifests/remove/bigcouch.pp b/puppet/modules/site_config/manifests/remove/bigcouch.pp new file mode 100644 index 00000000..3535c3c1 --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/bigcouch.pp @@ -0,0 +1,42 @@ +# remove bigcouch leftovers from previous installations +class site_config::remove::bigcouch { + + # Don't use check_mk logwatch to watch bigcouch logs anymore + # see https://leap.se/code/issues/7375 for more details + file { '/etc/check_mk/logwatch.d/bigcouch.cfg': + ensure => absent, + notify => [ + Exec['remove_bigcouch_logwatch_stateline'] + ] + } + + exec { 'remove_bigcouch_logwatch_stateline': + command => "sed -i '/bigcouch.log/d' /etc/check_mk/logwatch.state", + refreshonly => true, + } + + cron { 'compact_all_shards': + ensure => absent + } + + + exec { 'kill_bigcouch_stunnel_procs': + refreshonly => true, + command => '/usr/bin/pkill -f "/usr/bin/stunnel4 /etc/stunnel/(ednp|epmd)_server.conf"' + } + + # 'tidy' doesn't notify other resources, so we need to use file here instead + # see https://tickets.puppetlabs.com/browse/PUP-6021 + file { + [ '/etc/stunnel/ednp_server.conf', '/etc/stunnel/epmd_server.conf']: + ensure => absent, + # notifying Service[stunnel] doesn't work here because the config + # files contain the pid of the procs to stop/start. + # If we remove the config, and restart stunnel then it will only + # stop/start the procs for which config files are found and the stale + # service will continue to run. + # So we simply kill them. + notify => Exec['kill_bigcouch_stunnel_procs'] + } + +} diff --git a/puppet/modules/site_config/manifests/remove_files.pp b/puppet/modules/site_config/manifests/remove/files.pp index b339e6af..41d6462e 100644 --- a/puppet/modules/site_config/manifests/remove_files.pp +++ b/puppet/modules/site_config/manifests/remove/files.pp @@ -9,7 +9,16 @@ # release. # -class site_config::remove_files { +class site_config::remove::files { + + # Platform 0.8 removals + tidy { + '/etc/default/leap_mx':; + '/etc/logrotate.d/mx':; + '/etc/rsyslog.d/50-mx.conf':; + '/etc/apt/preferences.d/openvpn':; + '/etc/apt/sources.list.d/secondary.list.disabled.list':; + } # # Platform 0.7 removals @@ -17,24 +26,22 @@ class site_config::remove_files { tidy { '/etc/rsyslog.d/99-tapicero.conf':; - '/etc/rsyslog.d/99-leap-mx.conf':; '/etc/rsyslog.d/01-webapp.conf':; '/etc/rsyslog.d/50-stunnel.conf':; - '/etc/logrotate.d/mx':; '/etc/logrotate.d/stunnel':; '/var/log/stunnel4/stunnel.log':; 'leap_mx': path => '/var/log/', recurse => true, - matches => 'leap_mx*'; - 'leap_mx_rotate': - path => '/var/log/leap/', - recurse => true, - matches => [ 'mx.log.[0-9]', 'mx.log.[0-9]?', 'mx.log.[6-9]?gz']; + matches => ['leap_mx*', 'mx.log.[1-5]', 'mx.log.[6-9](.gz)?', + 'mx.log.[0-9][0-9](.gz)?']; '/srv/leap/webapp/public/provider.json':; '/srv/leap/couchdb/designs/tmp_users': recurse => true, rmdirs => true; + '/etc/leap/soledad-server.conf':; + '/var/log/leap/openvpn.log':; + '/etc/rsyslog.d/50-openvpn.conf':; } # leax-mx logged to /var/log/leap_mx.log in the past @@ -46,5 +53,4 @@ class site_config::remove_files { onlyif => "/bin/grep -qe 'leap_mx.log' /etc/check_mk/logwatch.state" } - } diff --git a/puppet/modules/site_config/manifests/remove/jessie.pp b/puppet/modules/site_config/manifests/remove/jessie.pp new file mode 100644 index 00000000..e9497baf --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/jessie.pp @@ -0,0 +1,14 @@ +# remove possible leftovers after upgrading from wheezy to jessie +class site_config::remove::jessie { + + tidy { + '/etc/apt/preferences.d/rsyslog_anon_depends': + notify => Exec['apt_updated']; + } + + apt::preferences_snippet { + [ 'facter', 'obfsproxy', 'python-twisted', 'unbound' ]: + ensure => absent; + } + +} diff --git a/puppet/modules/site_config/manifests/remove/monitoring.pp b/puppet/modules/site_config/manifests/remove/monitoring.pp new file mode 100644 index 00000000..18e2949b --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/monitoring.pp @@ -0,0 +1,13 @@ +# remove leftovers on monitoring nodes +class site_config::remove::monitoring { + + # Remove check_mk loggwatch spoolfiles for + # tapicero and bigcouch + tidy { + 'remove_logwatch_spoolfiles': + path => '/var/lib/check_mk/logwatch', + recurse => true, + matches => [ '*tapicero.log', '*bigcouch.log']; + } + +} diff --git a/puppet/modules/site_config/manifests/remove/tapicero.pp b/puppet/modules/site_config/manifests/remove/tapicero.pp new file mode 100644 index 00000000..07c3c6c6 --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/tapicero.pp @@ -0,0 +1,72 @@ +# remove tapicero leftovers from previous deploys on couchdb nodes +class site_config::remove::tapicero { + + ensure_packages('curl') + + # remove tapicero couchdb user + $couchdb_config = hiera('couch') + $couchdb_mode = $couchdb_config['mode'] + + if $couchdb_mode == 'multimaster' + { + $port = 5986 + } else { + $port = 5984 + } + + exec { 'remove_couchdb_user': + onlyif => "/usr/bin/curl -s 127.0.0.1:${port}/_users/org.couchdb.user:tapicero | grep -qv 'not_found'", + command => "/usr/local/bin/couch-doc-update --host 127.0.0.1:${port} --db _users --id org.couchdb.user:tapicero --delete", + require => Package['curl'] + } + + + exec { 'kill_tapicero': + onlyif => '/usr/bin/test -s /var/run/tapicero.pid', + command => '/usr/bin/pkill --pidfile /var/run/tapicero.pid' + } + + user { 'tapicero': + ensure => absent; + } + + group { 'tapicero': + ensure => absent, + require => User['tapicero']; + } + + tidy { + '/srv/leap/tapicero': + recurse => true, + require => [ Exec['kill_tapicero'] ]; + '/var/lib/leap/tapicero': + require => [ Exec['kill_tapicero'] ]; + '/var/run/tapicero': + require => [ Exec['kill_tapicero'] ]; + '/etc/leap/tapicero.yaml': + require => [ Exec['kill_tapicero'] ]; + '/etc/init.d/tapicero': + require => [ Exec['kill_tapicero'] ]; + 'tapicero_logs': + path => '/var/log/leap', + recurse => true, + matches => 'tapicero*', + require => [ Exec['kill_tapicero'] ]; + '/etc/check_mk/logwatch.d/tapicero.cfg':; + } + + # remove local nagios plugin checks via mrpe + augeas { + 'Tapicero_Procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Tapicero_Procs', + require => File['/etc/check_mk/mrpe.cfg']; + 'Tapicero_Heartbeat': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm Tapicero_Heartbeat', + require => File['/etc/check_mk/mrpe.cfg']; + } + +} diff --git a/puppet/modules/site_config/manifests/remove/webapp.pp b/puppet/modules/site_config/manifests/remove/webapp.pp new file mode 100644 index 00000000..58f59815 --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/webapp.pp @@ -0,0 +1,7 @@ +# remove leftovers on webapp nodes +class site_config::remove::webapp { + tidy { + '/etc/apache/sites-enabled/leap_webapp.conf': + notify => Service['apache']; + } +} diff --git a/puppet/modules/site_config/manifests/resolvconf.pp b/puppet/modules/site_config/manifests/resolvconf.pp index 05990c67..09f0b405 100644 --- a/puppet/modules/site_config/manifests/resolvconf.pp +++ b/puppet/modules/site_config/manifests/resolvconf.pp @@ -8,7 +8,7 @@ class site_config::resolvconf { nameservers => [ '127.0.0.1 # local caching-only, unbound', '85.214.20.141 # Digitalcourage, a german privacy organisation: (https://en.wikipedia.org/wiki/Digitalcourage)', - '77.109.138.45 # Swiss privacy Foundation (http://www.privacyfoundation.ch/de/service/server.html)' + '172.81.176.146 # OpenNIC (https://servers.opennicproject.org/edit.php?srv=ns1.tor.ca.dns.opennic.glue)' ] } } diff --git a/puppet/modules/site_config/manifests/ruby.pp b/puppet/modules/site_config/manifests/ruby.pp index 2a720114..5c13233d 100644 --- a/puppet/modules/site_config/manifests/ruby.pp +++ b/puppet/modules/site_config/manifests/ruby.pp @@ -1,14 +1,8 @@ +# install ruby, rubygems and bundler +# configure ruby settings common to all servers class site_config::ruby { Class[Ruby] -> Class[rubygems] -> Class[bundler::install] - class { '::ruby': ruby_version => '1.9.3' } + class { '::ruby': } class { 'bundler::install': install_method => 'package' } include rubygems } - - -# -# Ruby settings common to all servers -# -# Why this way? So that other classes can do 'include site_ruby' without creating redeclaration errors. -# See https://puppetlabs.com/blog/modeling-class-composition-with-parameterized-classes/ -# diff --git a/puppet/modules/site_config/manifests/ruby/dev.pp b/puppet/modules/site_config/manifests/ruby/dev.pp index 3ea6ca96..2b0b106d 100644 --- a/puppet/modules/site_config/manifests/ruby/dev.pp +++ b/puppet/modules/site_config/manifests/ruby/dev.pp @@ -1,8 +1,8 @@ -class site_config::ruby::dev inherits site_config::ruby { - Class['::ruby'] { - ruby_version => '1.9.3', - install_dev => true - } +# install ruby dev packages needed for building some gems +class site_config::ruby::dev { + include site_config::ruby + include ::ruby::devel + # building gems locally probably requires build-essential and gcc: include site_config::packages::build_essential } diff --git a/puppet/modules/site_config/manifests/setup.pp b/puppet/modules/site_config/manifests/setup.pp index b09d0413..82dfe76d 100644 --- a/puppet/modules/site_config/manifests/setup.pp +++ b/puppet/modules/site_config/manifests/setup.pp @@ -1,3 +1,7 @@ +# common things to set up on every node +# leftover from the past, where we did two puppetruns +# after another. We should consolidate this into site_config::default +# in the future. class site_config::setup { tag 'leap_base' @@ -13,17 +17,14 @@ class site_config::setup { include stdlib # configure /etc/hosts - class { 'site_config::hosts': - stage => setup, - } + class { 'site_config::hosts': } include site_config::initial_firewall include site_apt package { 'facter': - ensure => latest, - require => Exec['refresh_apt'] + ensure => latest } # if squid_deb_proxy_client is set to true, install and configure diff --git a/puppet/modules/site_config/manifests/slow.pp b/puppet/modules/site_config/manifests/slow.pp index 94bac88d..8e9b7035 100644 --- a/puppet/modules/site_config/manifests/slow.pp +++ b/puppet/modules/site_config/manifests/slow.pp @@ -1,6 +1,10 @@ +# this class is run by default, but can be excluded +# for testing purposes by calling "leap deploy" with +# the "--fast" parameter class site_config::slow { tag 'leap_slow' - class { 'site_apt::dist_upgrade': - stage => setup, - } + + include site_config::default + include apt::update + class { 'site_apt::dist_upgrade': } } diff --git a/puppet/modules/site_config/manifests/syslog.pp b/puppet/modules/site_config/manifests/syslog.pp index 83b49c8e..591e0601 100644 --- a/puppet/modules/site_config/manifests/syslog.pp +++ b/puppet/modules/site_config/manifests/syslog.pp @@ -1,10 +1,31 @@ +# configure rsyslog on all nodes class site_config::syslog { - include site_apt::preferences::rsyslog + # only pin rsyslog packages to backports on wheezy + case $::operatingsystemrelease { + /^7.*/: { + include ::site_apt::preferences::rsyslog + } + # on jessie+ systems, systemd and journald are enabled, + # and journald logs IP addresses, so we need to disable + # it until a solution is found, (#7863): + # https://github.com/systemd/systemd/issues/2447 + default: { + include ::journald + augeas { + 'disable_journald': + incl => '/etc/systemd/journald.conf', + lens => 'Puppet.lns', + changes => 'set /files/etc/systemd/journald.conf/Journal/Storage \'none\'', + notify => Service['systemd-journald']; + } + } + } - class { 'rsyslog::client': - log_remote => false, - log_local => true + class { '::rsyslog::client': + log_remote => false, + log_local => true, + custom_config => 'site_rsyslog/client.conf.erb' } rsyslog::snippet { '00-anonymize_logs': @@ -15,12 +36,13 @@ action(type="mmanon" ipv4.bits="32" mode="rewrite")' augeas { 'logrotate_leap_deploy': context => '/files/etc/logrotate.d/leap_deploy/rule', - changes => [ 'set file /var/log/leap/deploy.log', - 'set rotate 5', - 'set size 1M', - 'set compress compress', - 'set missingok missingok', - 'set copytruncate copytruncate' ]; + changes => [ + 'set file /var/log/leap/deploy.log', + 'set rotate 5', + 'set size 1M', + 'set compress compress', + 'set missingok missingok', + 'set copytruncate copytruncate' ]; # NOTE: # the puppet_command script requires the option delaycompress @@ -28,12 +50,13 @@ action(type="mmanon" ipv4.bits="32" mode="rewrite")' 'logrotate_leap_deploy_summary': context => '/files/etc/logrotate.d/leap_deploy_summary/rule', - changes => [ 'set file /var/log/leap/deploy-summary.log', - 'set rotate 5', - 'set size 100k', - 'set delaycompress delaycompress', - 'set compress compress', - 'set missingok missingok', - 'set copytruncate copytruncate' ] + changes => [ + 'set file /var/log/leap/deploy-summary.log', + 'set rotate 5', + 'set size 100k', + 'set delaycompress delaycompress', + 'set compress compress', + 'set missingok missingok', + 'set copytruncate copytruncate' ] } } diff --git a/puppet/modules/site_config/manifests/x509/ca.pp b/puppet/modules/site_config/manifests/x509/ca.pp index b16d0eeb..2880ecaf 100644 --- a/puppet/modules/site_config/manifests/x509/ca.pp +++ b/puppet/modules/site_config/manifests/x509/ca.pp @@ -1,5 +1,7 @@ class site_config::x509::ca { + include ::site_config::params + $x509 = hiera('x509') $ca = $x509['ca_cert'] diff --git a/puppet/modules/site_config/manifests/x509/ca_bundle.pp b/puppet/modules/site_config/manifests/x509/ca_bundle.pp index 4cbe574a..5808e29e 100644 --- a/puppet/modules/site_config/manifests/x509/ca_bundle.pp +++ b/puppet/modules/site_config/manifests/x509/ca_bundle.pp @@ -5,6 +5,7 @@ class site_config::x509::ca_bundle { # we will want to be able to smoothly phase out one CA and phase in another. # I tried "--capath" for this, but it did not work. + include ::site_config::params $x509 = hiera('x509') $ca = $x509['ca_cert'] diff --git a/puppet/modules/site_config/manifests/x509/cert.pp b/puppet/modules/site_config/manifests/x509/cert.pp index 7ed42959..7e5a36b9 100644 --- a/puppet/modules/site_config/manifests/x509/cert.pp +++ b/puppet/modules/site_config/manifests/x509/cert.pp @@ -1,5 +1,7 @@ class site_config::x509::cert { + include ::site_config::params + $x509 = hiera('x509') $cert = $x509['cert'] diff --git a/puppet/modules/site_config/manifests/x509/client_ca/ca.pp b/puppet/modules/site_config/manifests/x509/client_ca/ca.pp index 0f313898..3fbafa98 100644 --- a/puppet/modules/site_config/manifests/x509/client_ca/ca.pp +++ b/puppet/modules/site_config/manifests/x509/client_ca/ca.pp @@ -5,6 +5,8 @@ class site_config::x509::client_ca::ca { ## client certificates by the webapp. ## + include ::site_config::params + $x509 = hiera('x509') $cert = $x509['client_ca_cert'] diff --git a/puppet/modules/site_config/manifests/x509/client_ca/key.pp b/puppet/modules/site_config/manifests/x509/client_ca/key.pp index f9ef3f52..0b537e76 100644 --- a/puppet/modules/site_config/manifests/x509/client_ca/key.pp +++ b/puppet/modules/site_config/manifests/x509/client_ca/key.pp @@ -5,6 +5,8 @@ class site_config::x509::client_ca::key { ## client certificates by the webapp. ## + include ::site_config::params + $x509 = hiera('x509') $key = $x509['client_ca_key'] diff --git a/puppet/modules/site_config/manifests/x509/commercial/ca.pp b/puppet/modules/site_config/manifests/x509/commercial/ca.pp index 8f35759f..c76a9dbb 100644 --- a/puppet/modules/site_config/manifests/x509/commercial/ca.pp +++ b/puppet/modules/site_config/manifests/x509/commercial/ca.pp @@ -1,5 +1,7 @@ class site_config::x509::commercial::ca { + include ::site_config::params + $x509 = hiera('x509') $ca = $x509['commercial_ca_cert'] diff --git a/puppet/modules/site_config/manifests/x509/commercial/cert.pp b/puppet/modules/site_config/manifests/x509/commercial/cert.pp index 0c71a705..9dd6ffcd 100644 --- a/puppet/modules/site_config/manifests/x509/commercial/cert.pp +++ b/puppet/modules/site_config/manifests/x509/commercial/cert.pp @@ -1,10 +1,15 @@ class site_config::x509::commercial::cert { + include ::site_config::params + $x509 = hiera('x509') $cert = $x509['commercial_cert'] + $ca = $x509['commercial_ca_cert'] + + $cafile = "${cert}\n${ca}" x509::cert { $site_config::params::commercial_cert_name: - content => $cert + content => $cafile } } diff --git a/puppet/modules/site_config/manifests/x509/commercial/key.pp b/puppet/modules/site_config/manifests/x509/commercial/key.pp index d32e85ef..2be439fd 100644 --- a/puppet/modules/site_config/manifests/x509/commercial/key.pp +++ b/puppet/modules/site_config/manifests/x509/commercial/key.pp @@ -1,5 +1,7 @@ class site_config::x509::commercial::key { + include ::site_config::params + $x509 = hiera('x509') $key = $x509['commercial_key'] diff --git a/puppet/modules/site_config/manifests/x509/key.pp b/puppet/modules/site_config/manifests/x509/key.pp index 32b59726..448dc6a6 100644 --- a/puppet/modules/site_config/manifests/x509/key.pp +++ b/puppet/modules/site_config/manifests/x509/key.pp @@ -1,5 +1,7 @@ class site_config::x509::key { + include ::site_config::params + $x509 = hiera('x509') $key = $x509['key'] diff --git a/puppet/modules/site_config/templates/hosts b/puppet/modules/site_config/templates/hosts index d557f730..d62cbc3f 100644 --- a/puppet/modules/site_config/templates/hosts +++ b/puppet/modules/site_config/templates/hosts @@ -6,7 +6,8 @@ <%- if @hosts then -%> <% @hosts.keys.sort.each do |name| -%> <%- props = @hosts[name] -%> -<%= props["ip_address"] %> <%= props["domain_full"] %> <%= props["domain_internal"] %> <%= name %> +<%- aliases = props["aliases"] ? props["aliases"].join(' ') : nil -%> +<%= [props["ip_address"], props["domain_full"], props["domain_internal"], aliases, name].compact.uniq.join(' ') %> <% end -%> <% end -%> diff --git a/puppet/modules/site_couchdb/files/designs/identities/Identity.json b/puppet/modules/site_couchdb/files/designs/identities/Identity.json index 2ac092ab..b1c567c1 100644 --- a/puppet/modules/site_couchdb/files/designs/identities/Identity.json +++ b/puppet/modules/site_couchdb/files/designs/identities/Identity.json @@ -2,27 +2,33 @@ "_id": "_design/Identity", "language": "javascript", "views": { - "by_user_id": { - "map": " function(doc) {\n if ((doc['type'] == 'Identity') && (doc['user_id'] != null)) {\n emit(doc['user_id'], 1);\n }\n }\n", - "reduce": "_sum" - }, "by_address_and_destination": { "map": " function(doc) {\n if ((doc['type'] == 'Identity') && (doc['address'] != null) && (doc['destination'] != null)) {\n emit([doc['address'], doc['destination']], 1);\n }\n }\n", "reduce": "_sum" }, - "by_address": { - "map": " function(doc) {\n if ((doc['type'] == 'Identity') && (doc['address'] != null)) {\n emit(doc['address'], 1);\n }\n }\n", - "reduce": "_sum" + "all": { + "map": " function(doc) {\n if (doc['type'] == 'Identity') {\n emit(doc._id, null);\n }\n }\n" }, - "pgp_key_by_email": { - "map": " function(doc) {\n if (doc.type != 'Identity') {\n return;\n }\n if (typeof doc.keys === \"object\") {\n emit(doc.address, doc.keys[\"pgp\"]);\n }\n }\n" + "cert_fingerprints_by_expiry": { + "map": "function(doc) {\n if (doc.type != 'Identity') {\n return;\n }\n if (typeof doc.cert_fingerprints === \"object\") {\n for (fp in doc.cert_fingerprints) {\n if (doc.cert_fingerprints.hasOwnProperty(fp)) {\n emit(doc.cert_fingerprints[fp], fp);\n }\n }\n }\n}\n" + }, + "cert_expiry_by_fingerprint": { + "map": "function(doc) {\n if (doc.type != 'Identity') {\n return;\n }\n if (typeof doc.cert_fingerprints === \"object\") {\n for (fp in doc.cert_fingerprints) {\n if (doc.cert_fingerprints.hasOwnProperty(fp)) {\n emit(fp, doc.cert_fingerprints[fp]);\n }\n }\n }\n}\n" }, "disabled": { - "map": " function(doc) {\n if (doc.type != 'Identity') {\n return;\n }\n if (typeof doc.user_id === \"undefined\") {\n emit(doc._id, 1);\n }\n }\n" + "map": "function(doc) {\n if (doc.type != 'Identity') {\n return;\n }\n if (typeof doc.user_id === \"undefined\") {\n emit(doc._id, 1);\n }\n}\n" }, - "all": { - "map": " function(doc) {\n if (doc['type'] == 'Identity') {\n emit(doc._id, null);\n }\n }\n" + "pgp_key_by_email": { + "map": "function(doc) {\n if (doc.type != 'Identity') {\n return;\n }\n if (typeof doc.keys === \"object\") {\n emit(doc.address, doc.keys[\"pgp\"]);\n }\n}\n" + }, + "by_user_id": { + "map": " function(doc) {\n if ((doc['type'] == 'Identity') && (doc['user_id'] != null)) {\n emit(doc['user_id'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_address": { + "map": " function(doc) {\n if ((doc['type'] == 'Identity') && (doc['address'] != null)) {\n emit(doc['address'], 1);\n }\n }\n", + "reduce": "_sum" } }, - "couchrest-hash": "e9004d70e26770c621a9667536429a68" + "couchrest-hash": "4a774c3f56122b655a314670403b27e2" }
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json b/puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json new file mode 100644 index 00000000..006c1ea1 --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json @@ -0,0 +1,22 @@ +{ + "_id": "_design/InviteCode", + "language": "javascript", + "views": { + "by__id": { + "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['_id'] != null)) {\n emit(doc['_id'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_invite_code": { + "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['invite_code'] != null)) {\n emit(doc['invite_code'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_invite_count": { + "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['invite_count'] != null)) {\n emit(doc['invite_count'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "all": { + "map": " function(doc) {\n if (doc['type'] == 'InviteCode') {\n emit(doc._id, null);\n }\n }\n" + } + }, + "couchrest-hash": "83fb8f504520b4a9c7ddbb7928cd0ce3" +}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/messages/Message.json b/puppet/modules/site_couchdb/files/designs/messages/Message.json index 7bcd74c7..6a48fc4d 100644 --- a/puppet/modules/site_couchdb/files/designs/messages/Message.json +++ b/puppet/modules/site_couchdb/files/designs/messages/Message.json @@ -2,17 +2,17 @@ "_id": "_design/Message", "language": "javascript", "views": { - "by_user_ids_to_show_and_created_at": { - "map": "// not using at moment\n// call with something like Message.by_user_ids_to_show_and_created_at.startkey([user_id, start_date]).endkey([user_id,end_date])\nfunction (doc) {\n if (doc.type === 'Message' && doc.user_ids_to_show && Array.isArray(doc.user_ids_to_show)) {\n doc.user_ids_to_show.forEach(function (userId) {\n emit([userId, doc.created_at], 1);\n });\n }\n}\n", - "reduce": "function(key, values, rereduce) { return sum(values); }" - }, "by_user_ids_to_show": { "map": "function (doc) {\n if (doc.type === 'Message' && doc.user_ids_to_show && Array.isArray(doc.user_ids_to_show)) {\n doc.user_ids_to_show.forEach(function (userId) {\n emit(userId, 1);\n });\n }\n}\n", - "reduce": "function(key, values, rereduce) { return sum(values); }" + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" + }, + "by_user_ids_to_show_and_created_at": { + "map": "// not using at moment\n// call with something like Message.by_user_ids_to_show_and_created_at.startkey([user_id, start_date]).endkey([user_id,end_date])\nfunction (doc) {\n if (doc.type === 'Message' && doc.user_ids_to_show && Array.isArray(doc.user_ids_to_show)) {\n doc.user_ids_to_show.forEach(function (userId) {\n emit([userId, doc.created_at], 1);\n });\n }\n}\n", + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" }, "all": { "map": " function(doc) {\n if (doc['type'] == 'Message') {\n emit(doc._id, null);\n }\n }\n" } }, - "couchrest-hash": "0967e7cc5bb1e61edc1c085f6f0cecbf" + "couchrest-hash": "ba80168e51015d2678cad88fc6c5b986" }
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/tickets/Ticket.json b/puppet/modules/site_couchdb/files/designs/tickets/Ticket.json index 2c9408b8..578f632b 100644 --- a/puppet/modules/site_couchdb/files/designs/tickets/Ticket.json +++ b/puppet/modules/site_couchdb/files/designs/tickets/Ticket.json @@ -24,27 +24,27 @@ }, "by_includes_post_by_and_is_open_and_created_at": { "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.is_open, doc.created_at], 1);\n }\n });\n }\n}\n", - "reduce": "function(key, values, rereduce) { return sum(values); }" - }, - "by_includes_post_by_and_is_open_and_updated_at": { - "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.is_open, doc.updated_at], 1);\n }\n });\n }\n}\n", - "reduce": "function(key, values, rereduce) { return sum(values); }" - }, - "by_includes_post_by_and_updated_at": { - "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.updated_at], 1);\n }\n });\n }\n}\n", - "reduce": "function(key, values, rereduce) { return sum(values); }" + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" }, "by_includes_post_by": { "map": "// TODO: This view is only used in tests--should we keep it?\nfunction(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit(comment.posted_by, 1);\n }\n });\n }\n}\n", - "reduce": "function(key, values, rereduce) { return sum(values); }" + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" + }, + "by_includes_post_by_and_is_open_and_updated_at": { + "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.is_open, doc.updated_at], 1);\n }\n });\n }\n}\n", + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" }, "by_includes_post_by_and_created_at": { "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.created_at], 1);\n }\n });\n }\n}\n", - "reduce": "function(key, values, rereduce) { return sum(values); }" + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" + }, + "by_includes_post_by_and_updated_at": { + "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.updated_at], 1);\n }\n });\n }\n}\n", + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" }, "all": { "map": " function(doc) {\n if (doc['type'] == 'Ticket') {\n emit(doc._id, null);\n }\n }\n" } }, - "couchrest-hash": "9978e2cbeacbe8622c2a7f103bf8130f" + "couchrest-hash": "b21eaeea8ea66bfda65581b1b7ce06af" }
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/users/User.json b/puppet/modules/site_couchdb/files/designs/users/User.json index 4089ad97..8a82cf4a 100644 --- a/puppet/modules/site_couchdb/files/designs/users/User.json +++ b/puppet/modules/site_couchdb/files/designs/users/User.json @@ -11,12 +11,12 @@ }, "by_created_at_and_one_month_warning_not_sent": { "map": "function (doc) {\n if ((doc['type'] == 'User') && (doc['created_at'] != null) && (doc['one_month_warning_sent'] == null)) {\n emit(doc['created_at'], 1);\n } \n}\n", - "reduce": "function(key, values, rereduce) { return sum(values); }" + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" }, "by_created_at": { "map": " function(doc) {\n if ((doc['type'] == 'User') && (doc['created_at'] != null)) {\n emit(doc['created_at'], 1);\n }\n }\n", "reduce": "_sum" } }, - "couchrest-hash": "61840ab3ec0f94ef8bbd6dd208db3b70" + "couchrest-hash": "d854607d299887a347e554176cb79e20" }
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/manifests/add_users.pp b/puppet/modules/site_couchdb/manifests/add_users.pp index 2f734ed4..c905316b 100644 --- a/puppet/modules/site_couchdb/manifests/add_users.pp +++ b/puppet/modules/site_couchdb/manifests/add_users.pp @@ -1,3 +1,4 @@ +# add couchdb users for all services class site_couchdb::add_users { Class['site_couchdb::create_dbs'] @@ -35,16 +36,6 @@ class site_couchdb::add_users { require => Couchdb::Query::Setup['localhost'] } - ### tapicero couchdb user - ### admin: needs to be able to create user-<uuid> databases - ### read: users - couchdb::add_user { $site_couchdb::couchdb_tapicero_user: - roles => '["users"]', - pw => $site_couchdb::couchdb_tapicero_pw, - salt => $site_couchdb::couchdb_tapicero_salt, - require => Couchdb::Query::Setup['localhost'] - } - ## webapp couchdb user ## read/write: users, tokens, sessions, tickets, identities, customer couchdb::add_user { $site_couchdb::couchdb_webapp_user: diff --git a/puppet/modules/site_couchdb/manifests/bigcouch.pp b/puppet/modules/site_couchdb/manifests/bigcouch.pp index 469a2783..2de3d4d0 100644 --- a/puppet/modules/site_couchdb/manifests/bigcouch.pp +++ b/puppet/modules/site_couchdb/manifests/bigcouch.pp @@ -44,4 +44,7 @@ class site_couchdb::bigcouch { require => Package['couchdb'], notify => Service['couchdb'] } + + include site_check_mk::agent::couchdb::bigcouch + } diff --git a/puppet/modules/site_couchdb/manifests/create_dbs.pp b/puppet/modules/site_couchdb/manifests/create_dbs.pp index eea4bbf5..a2d1c655 100644 --- a/puppet/modules/site_couchdb/manifests/create_dbs.pp +++ b/puppet/modules/site_couchdb/manifests/create_dbs.pp @@ -90,4 +90,13 @@ class site_couchdb::create_dbs { members => "{ \"names\": [\"${site_couchdb::couchdb_webapp_user}\"], \"roles\": [\"replication\"] }", require => Couchdb::Query::Setup['localhost'] } + + ## invite_codes db + ## store invite codes for new signups + ## r/w: webapp + couchdb::create_db { 'invite_codes': + members => "{ \"names\": [\"${site_couchdb::couchdb_webapp_user}\"], \"roles\": [\"replication\"] }", + require => Couchdb::Query::Setup['localhost'] + } + } diff --git a/puppet/modules/site_couchdb/manifests/designs.pp b/puppet/modules/site_couchdb/manifests/designs.pp index 1ab1c6a1..e5fd94c6 100644 --- a/puppet/modules/site_couchdb/manifests/designs.pp +++ b/puppet/modules/site_couchdb/manifests/designs.pp @@ -12,12 +12,13 @@ class site_couchdb::designs { } site_couchdb::upload_design { - 'customers': design => 'customers/Customer.json'; - 'identities': design => 'identities/Identity.json'; - 'tickets': design => 'tickets/Ticket.json'; - 'messages': design => 'messages/Message.json'; - 'users': design => 'users/User.json'; - 'tmp_users': design => 'users/User.json'; + 'customers': design => 'customers/Customer.json'; + 'identities': design => 'identities/Identity.json'; + 'tickets': design => 'tickets/Ticket.json'; + 'messages': design => 'messages/Message.json'; + 'users': design => 'users/User.json'; + 'tmp_users': design => 'users/User.json'; + 'invite_codes': design => 'invite_codes/InviteCode.json'; 'shared_docs': db => 'shared', design => 'shared/docs.json'; diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 6b6ddd3a..c4fe6277 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -26,11 +26,6 @@ class site_couchdb { $couchdb_soledad_pw = $couchdb_soledad['password'] $couchdb_soledad_salt = $couchdb_soledad['salt'] - $couchdb_tapicero = $couchdb_users['tapicero'] - $couchdb_tapicero_user = $couchdb_tapicero['username'] - $couchdb_tapicero_pw = $couchdb_tapicero['password'] - $couchdb_tapicero_salt = $couchdb_tapicero['salt'] - $couchdb_webapp = $couchdb_users['webapp'] $couchdb_webapp_user = $couchdb_webapp['username'] $couchdb_webapp_pw = $couchdb_webapp['password'] @@ -43,11 +38,14 @@ class site_couchdb { $couchdb_backup = $couchdb_config['backup'] $couchdb_mode = $couchdb_config['mode'] - $couchdb_pwhash_alg = $couchdb_config['pwhash_alg'] - if $couchdb_mode == 'multimaster' { include site_couchdb::bigcouch } - if $couchdb_mode == 'master' { include site_couchdb::master } - if $couchdb_mode == 'mirror' { include site_couchdb::mirror } + # ensure bigcouch has been purged from the system: + # TODO: remove this check in 0.9 release + if file('/opt/bigcouch/bin/bigcouch', '/dev/null') != '' { + fail 'ERROR: BigCouch appears to be installed. Make sure you have migrated to CouchDB before proceeding. See https://leap.se/upgrade-0-8' + } + + include site_couchdb::plain Class['site_config::default'] -> Service['shorewall'] @@ -55,6 +53,7 @@ class site_couchdb { -> Class['couchdb'] -> Class['site_couchdb::setup'] + include ::site_config::default include site_stunnel include site_couchdb::setup @@ -66,6 +65,17 @@ class site_couchdb { if $couchdb_backup { include site_couchdb::backup } include site_check_mk::agent::couchdb - include site_check_mk::agent::tapicero + + # remove tapicero leftovers on couchdb nodes + include site_config::remove::tapicero + + # Destroy every per-user storage database + # where the corresponding user record does not exist. + cron { 'cleanup_stale_userdbs': + command => '(/bin/date; /srv/leap/couchdb/scripts/cleanup-user-dbs) >> /var/log/leap/couchdb-cleanup.log', + user => 'root', + hour => 4, + minute => 7; + } } diff --git a/puppet/modules/site_couchdb/manifests/logrotate.pp b/puppet/modules/site_couchdb/manifests/logrotate.pp index e1039d49..bb8843bb 100644 --- a/puppet/modules/site_couchdb/manifests/logrotate.pp +++ b/puppet/modules/site_couchdb/manifests/logrotate.pp @@ -1,12 +1,14 @@ +# configure couchdb logrotation class site_couchdb::logrotate { augeas { 'logrotate_bigcouch': context => '/files/etc/logrotate.d/bigcouch/rule', - changes => [ 'set file /opt/bigcouch/var/log/*.log', 'set rotate 7', - 'set schedule daily', 'set compress compress', - 'set missingok missingok', 'set ifempty notifempty', - 'set copytruncate copytruncate' ] + changes => [ + 'set file /opt/bigcouch/var/log/*.log', 'set rotate 7', + 'set schedule daily', 'set compress compress', + 'set missingok missingok', 'set ifempty notifempty', + 'set copytruncate copytruncate' ] } } diff --git a/puppet/modules/site_couchdb/manifests/master.pp b/puppet/modules/site_couchdb/manifests/master.pp deleted file mode 100644 index c28eee7d..00000000 --- a/puppet/modules/site_couchdb/manifests/master.pp +++ /dev/null @@ -1,9 +0,0 @@ -# this class sets up a single, plain couchdb node -class site_couchdb::master { - class { 'couchdb': - admin_pw => $site_couchdb::couchdb_admin_pw, - admin_salt => $site_couchdb::couchdb_admin_salt, - chttpd_bind_address => '127.0.0.1', - pwhash_alg => $site_couchdb::couchdb_pwhash_alg - } -} diff --git a/puppet/modules/site_couchdb/manifests/mirror.pp b/puppet/modules/site_couchdb/manifests/mirror.pp index abe35c4c..fb82b897 100644 --- a/puppet/modules/site_couchdb/manifests/mirror.pp +++ b/puppet/modules/site_couchdb/manifests/mirror.pp @@ -1,3 +1,4 @@ +# configure mirroring of couch nodes class site_couchdb::mirror { Class['site_couchdb::add_users'] @@ -22,55 +23,55 @@ class site_couchdb::mirror { ### customer database couchdb::mirror_db { 'customers': - from => $from, + from => $from, require => Couchdb::Query::Setup['localhost'] } ## identities database couchdb::mirror_db { 'identities': - from => $from, + from => $from, require => Couchdb::Query::Setup['localhost'] } ## keycache database couchdb::mirror_db { 'keycache': - from => $from, + from => $from, require => Couchdb::Query::Setup['localhost'] } ## sessions database couchdb::mirror_db { 'sessions': - from => $from, + from => $from, require => Couchdb::Query::Setup['localhost'] } ## shared database couchdb::mirror_db { 'shared': - from => $from, + from => $from, require => Couchdb::Query::Setup['localhost'] } ## tickets database couchdb::mirror_db { 'tickets': - from => $from, + from => $from, require => Couchdb::Query::Setup['localhost'] } ## tokens database couchdb::mirror_db { 'tokens': - from => $from, + from => $from, require => Couchdb::Query::Setup['localhost'] } ## users database couchdb::mirror_db { 'users': - from => $from, + from => $from, require => Couchdb::Query::Setup['localhost'] } ## messages db couchdb::mirror_db { 'messages': - from => $from, + from => $from, require => Couchdb::Query::Setup['localhost'] } diff --git a/puppet/modules/site_couchdb/manifests/plain.pp b/puppet/modules/site_couchdb/manifests/plain.pp new file mode 100644 index 00000000..b40fc100 --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/plain.pp @@ -0,0 +1,14 @@ +# this class sets up a single, plain couchdb node +class site_couchdb::plain { + class { 'couchdb': + admin_pw => $site_couchdb::couchdb_admin_pw, + admin_salt => $site_couchdb::couchdb_admin_salt, + chttpd_bind_address => '127.0.0.1' + } + + include site_check_mk::agent::couchdb::plain + + # remove bigcouch leftovers from previous installations + include ::site_config::remove::bigcouch + +} diff --git a/puppet/modules/site_couchdb/manifests/setup.pp b/puppet/modules/site_couchdb/manifests/setup.pp index 69bd1c6a..710d3c1c 100644 --- a/puppet/modules/site_couchdb/manifests/setup.pp +++ b/puppet/modules/site_couchdb/manifests/setup.pp @@ -12,27 +12,42 @@ class site_couchdb::setup { $user = $site_couchdb::couchdb_admin_user - # /etc/couchdb/couchdb-admin.netrc is deployed by couchdb::query::setup - # we symlink to couchdb.netrc for puppet commands. - # we symlink this to /root/.netrc for couchdb_scripts (eg. backup) - # and makes life easier for the admin (i.e. using curl/wget without - # passing credentials) + # setup /etc/couchdb/couchdb-admin.netrc for couchdb admin access + couchdb::query::setup { 'localhost': + user => $user, + pw => $site_couchdb::couchdb_admin_pw + } + + # We symlink /etc/couchdb/couchdb-admin.netrc to /etc/couchdb/couchdb.netrc + # for puppet commands, and to to /root/.netrc for couchdb_scripts + # (eg. backup) and to makes life easier for the admin on the command line + # (i.e. using curl/wget without passing credentials) file { '/etc/couchdb/couchdb.netrc': ensure => link, target => "/etc/couchdb/couchdb-${user}.netrc"; - '/root/.netrc': ensure => link, target => '/etc/couchdb/couchdb.netrc'; + } - '/srv/leap/couchdb': - ensure => directory + # setup /etc/couchdb/couchdb-soledad-admin.netrc file for couchdb admin + # access, accessible only for the soledad-admin user to create soledad + # userdbs + if member(hiera('services', []), 'soledad') { + file { '/etc/couchdb/couchdb-soledad-admin.netrc': + content => "machine localhost login ${user} password ${site_couchdb::couchdb_admin_pw}", + mode => '0400', + owner => 'soledad-admin', + group => 'root', + require => [ Package['couchdb'], User['soledad-admin'] ]; + } } - couchdb::query::setup { 'localhost': - user => $user, - pw => $site_couchdb::couchdb_admin_pw, + # Checkout couchdb_scripts repo + file { + '/srv/leap/couchdb': + ensure => directory } vcsrepo { '/srv/leap/couchdb/scripts': diff --git a/puppet/modules/site_couchdb/manifests/upload_design.pp b/puppet/modules/site_couchdb/manifests/upload_design.pp index 7b0cabd7..bd73ebf2 100644 --- a/puppet/modules/site_couchdb/manifests/upload_design.pp +++ b/puppet/modules/site_couchdb/manifests/upload_design.pp @@ -1,4 +1,5 @@ -define site_couchdb::upload_design($db = $title, $design) { +# upload a design doc to a db +define site_couchdb::upload_design($design, $db = $title) { $design_name = regsubst($design, '^.*\/(.*)\.json$', '\1') $id = "_design/${design_name}" $file = "/srv/leap/couchdb/designs/${design}" diff --git a/puppet/modules/site_mx/manifests/init.pp b/puppet/modules/site_mx/manifests/init.pp index 91014ed6..a9b0198b 100644 --- a/puppet/modules/site_mx/manifests/init.pp +++ b/puppet/modules/site_mx/manifests/init.pp @@ -2,6 +2,7 @@ class site_mx { tag 'leap_service' Class['site_config::default'] -> Class['site_mx'] + include site_config::default include site_config::x509::cert include site_config::x509::key include site_config::x509::ca diff --git a/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg b/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg index 0d729b8c..62f26f2c 100644 --- a/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg +++ b/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg @@ -22,18 +22,33 @@ log_file=/var/log/nagios3/nagios.log # if you wish (as shown below), or keep them all in a single config file. #cfg_file=/etc/nagios3/commands.cfg -# Puppet-managed configuration files -cfg_dir=/etc/nagios3/conf.d - -# check-mk managed configuration files +# Check_mk configuration files +cfg_dir=/etc/nagios3/conf.d/check_mk cfg_dir=/etc/nagios3/local +# Puppet-managed configuration files +cfg_file=/etc/nagios3/nagios_templates.cfg +cfg_file=/etc/nagios3/nagios_command.cfg +cfg_file=/etc/nagios3/nagios_contact.cfg +cfg_file=/etc/nagios3/nagios_contactgroup.cfg +cfg_file=/etc/nagios3/nagios_host.cfg +cfg_file=/etc/nagios3/nagios_hostdependency.cfg +cfg_file=/etc/nagios3/nagios_hostescalation.cfg +cfg_file=/etc/nagios3/nagios_hostextinfo.cfg +cfg_file=/etc/nagios3/nagios_hostgroup.cfg +cfg_file=/etc/nagios3/nagios_hostgroupescalation.cfg +cfg_file=/etc/nagios3/nagios_service.cfg +cfg_file=/etc/nagios3/nagios_servicedependency.cfg +cfg_file=/etc/nagios3/nagios_serviceescalation.cfg +cfg_file=/etc/nagios3/nagios_serviceextinfo.cfg +cfg_file=/etc/nagios3/nagios_servicegroup.cfg +cfg_file=/etc/nagios3/nagios_timeperiod.cfg + # Debian also defaults to using the check commands defined by the debian # nagios-plugins package cfg_dir=/etc/nagios-plugins/config - # OBJECT CACHE FILE # This option determines where object definitions are cached when # Nagios starts/restarts. The CGIs read object definitions from @@ -70,7 +85,7 @@ precached_object_file=/var/lib/nagios3/objects.precache # defined as macros in this file and restrictive permissions (600) # can be placed on this file. -resource_file=/etc/nagios3/private/resource.cfg +resource_file=/etc/nagios3/resource.cfg diff --git a/puppet/modules/site_nagios/manifests/init.pp b/puppet/modules/site_nagios/manifests/init.pp index eb08cdcb..f91bfc26 100644 --- a/puppet/modules/site_nagios/manifests/init.pp +++ b/puppet/modules/site_nagios/manifests/init.pp @@ -1,6 +1,13 @@ +# setup nagios on monitoring node class site_nagios { tag 'leap_service' + + include site_config::default + Class['site_config::default'] -> Class['site_nagios'] include site_nagios::server + + # remove leftovers on monitoring nodes + include site_config::remove::monitoring } diff --git a/puppet/modules/site_nagios/manifests/server.pp b/puppet/modules/site_nagios/manifests/server.pp index cb6c8d95..aa9b956e 100644 --- a/puppet/modules/site_nagios/manifests/server.pp +++ b/puppet/modules/site_nagios/manifests/server.pp @@ -1,8 +1,7 @@ # configures nagios on monitoring node +# lint:ignore:inherits_across_namespaces class site_nagios::server inherits nagios::base { - - # First, purge old nagios config (see #1467) - class { 'site_nagios::server::purge': } +# lint:endignore $nagios_hiera = hiera('nagios') $nagiosadmin_pw = htpasswd_sha1($nagios_hiera['nagiosadmin_pw']) @@ -22,17 +21,43 @@ class site_nagios::server inherits nagios::base { # it in site_apache::common httpd => 'absent', allow_external_cmd => true, - stored_config => false, + storeconfigs => false, + } + + # Delete nagios config files provided by packages + # These don't get parsed by nagios.conf, but are + # still irritating duplicates to the real config + # files deployed by puppet in /etc/nagios3/ + file { [ + '/etc/nagios3/conf.d/contacts_nagios2.cfg', + '/etc/nagios3/conf.d/extinfo_nagios2.cfg', + '/etc/nagios3/conf.d/generic-host_nagios2.cfg', + '/etc/nagios3/conf.d/generic-service_nagios2.cfg', + '/etc/nagios3/conf.d/hostgroups_nagios2.cfg', + '/etc/nagios3/conf.d/localhost_nagios2.cfg', + '/etc/nagios3/conf.d/pnp4nagios.cfg', + '/etc/nagios3/conf.d/services_nagios2.cfg', + '/etc/nagios3/conf.d/timeperiods_nagios2.cfg' ]: + ensure => absent; } - file { '/etc/apache2/conf.d/nagios3.conf': - ensure => link, - target => '/usr/share/doc/nagios3-common/examples/apache2.conf', - notify => Service['apache'] + # deploy apache nagios3 config + # until https://gitlab.com/shared-puppet-modules-group/apache/issues/11 + # is not fixed, we need to manually deploy the config file + file { + '/etc/apache2/conf-available/nagios3.conf': + ensure => present, + source => 'puppet:///modules/nagios/configs/apache2.conf', + require => [ Package['nagios3'], Package['apache2'] ]; + '/etc/apache2/conf-enabled/nagios3.conf': + ensure => link, + target => '/etc/apache2/conf-available/nagios3.conf', + require => [ Package['nagios3'], Package['apache2'] ]; } include site_apache::common - include site_apache::module::headers + include site_webapp::common_vhost + include apache::module::headers File ['nagios_htpasswd'] { source => undef, diff --git a/puppet/modules/site_nagios/manifests/server/add_contacts.pp b/puppet/modules/site_nagios/manifests/server/add_contacts.pp index db507abf..b5c6f0a5 100644 --- a/puppet/modules/site_nagios/manifests/server/add_contacts.pp +++ b/puppet/modules/site_nagios/manifests/server/add_contacts.pp @@ -1,3 +1,4 @@ +# configure a nagios_contact define site_nagios::server::add_contacts ($contact_emails) { $environment = $name @@ -11,6 +12,7 @@ define site_nagios::server::add_contacts ($contact_emails) { host_notification_options => 'd,r', service_notification_commands => 'notify-service-by-email', host_notification_commands => 'notify-host-by-email', - email => join($contact_emails, ', ') + email => join($contact_emails, ', '), + require => Package['nagios'] } } diff --git a/puppet/modules/site_nagios/manifests/server/apache.pp b/puppet/modules/site_nagios/manifests/server/apache.pp index 8dbc7e9b..82962e89 100644 --- a/puppet/modules/site_nagios/manifests/server/apache.pp +++ b/puppet/modules/site_nagios/manifests/server/apache.pp @@ -1,7 +1,25 @@ +# set up apache for nagios class site_nagios::server::apache { + include x509::variables + include site_config::x509::commercial::cert include site_config::x509::commercial::key include site_config::x509::commercial::ca + include apache::module::authn_file + # "AuthUserFile" + include apache::module::authz_user + # "AuthType Basic" + include apache::module::auth_basic + # "DirectoryIndex" + include apache::module::dir + include apache::module::php5 + include apache::module::cgi + + # apache >= 2.4, debian jessie + if ( $::lsbdistcodename == 'jessie' ) { + include apache::module::authn_core + } + } diff --git a/puppet/modules/site_nagios/manifests/server/contactgroup.pp b/puppet/modules/site_nagios/manifests/server/contactgroup.pp index 188c54f1..5e60dd06 100644 --- a/puppet/modules/site_nagios/manifests/server/contactgroup.pp +++ b/puppet/modules/site_nagios/manifests/server/contactgroup.pp @@ -1,6 +1,8 @@ +# configure a contactgroup define site_nagios::server::contactgroup ($contact_emails) { nagios_contactgroup { $name: - members => $name + members => $name, + require => Package['nagios'] } } diff --git a/puppet/modules/site_nagios/manifests/server/hostgroup.pp b/puppet/modules/site_nagios/manifests/server/hostgroup.pp index 6f85ca6d..0692fced 100644 --- a/puppet/modules/site_nagios/manifests/server/hostgroup.pp +++ b/puppet/modules/site_nagios/manifests/server/hostgroup.pp @@ -1,3 +1,7 @@ +# create a nagios hostsgroup define site_nagios::server::hostgroup ($contact_emails) { - nagios_hostgroup { $name: } + nagios_hostgroup { $name: + ensure => present, + require => Package['nagios'] + } } diff --git a/puppet/modules/site_nagios/manifests/server/purge.pp b/puppet/modules/site_nagios/manifests/server/purge.pp deleted file mode 100644 index 6815a703..00000000 --- a/puppet/modules/site_nagios/manifests/server/purge.pp +++ /dev/null @@ -1,19 +0,0 @@ -class site_nagios::server::purge inherits nagios::base { - # we don't want to get /etc/nagios3 and /etc/nagios3/conf.d - # purged, cause the check-mk-config-nagios3 package - # places its templates in /etc/nagios3/conf.d/check_mk, - # and check_mk -O updated it's nagios config in /etc/nagios3/conf.d/check_mk - File['nagios_cfgdir'] { - purge => false - } - File['nagios_confd'] { - purge => false - } - - # only purge files in the /etc/nagios3/conf.d/ dir, not in any subdir - exec {'purge_conf.d': - command => '/usr/bin/find /etc/nagios3/conf.d/ -maxdepth 1 -type f -exec rm {} \;', - onlyif => '/usr/bin/find /etc/nagios3/conf.d/ -maxdepth 1 -type f | grep -q "/etc/nagios3/conf.d"', - require => Package['nagios'] - } -} diff --git a/puppet/modules/site_nagios/templates/icli_aliases.erb b/puppet/modules/site_nagios/templates/icli_aliases.erb index f1428f9e..bcb2abb0 100644 --- a/puppet/modules/site_nagios/templates/icli_aliases.erb +++ b/puppet/modules/site_nagios/templates/icli_aliases.erb @@ -3,5 +3,5 @@ alias ncli_problems='ncli -z '!o,!A'' <% @environments.keys.sort.each do |env_name| %> alias ncli_<%= env_name %>='ncli -z '!o,!A' -g <%= env_name %>' -alias ncli_<%= env_name %>_recheck='ncli -s Check_MK -g <%= env_name %> -r' -<% end -%>
\ No newline at end of file +alias ncli_<%= env_name %>_recheck='ncli -s Check_MK -g <%= env_name %> -a R' +<% end -%> diff --git a/puppet/modules/site_nickserver/manifests/init.pp b/puppet/modules/site_nickserver/manifests/init.pp index c2deab0f..eb4415e7 100644 --- a/puppet/modules/site_nickserver/manifests/init.pp +++ b/puppet/modules/site_nickserver/manifests/init.pp @@ -61,13 +61,6 @@ class site_nickserver { require => Group['nickserver']; } - # - # NICKSERVER CODE NOTE: in order to support TLS, libssl-dev must be installed - # before EventMachine gem is built/installed. - # - - package { 'libssl-dev': ensure => installed } - vcsrepo { '/srv/leap/nickserver': ensure => present, revision => $sources['nickserver']['revision'], @@ -122,6 +115,20 @@ class site_nickserver { require => Vcsrepo['/srv/leap/nickserver']; } + # register initscript at systemd on nodes newer than wheezy + # see https://leap.se/code/issues/7614 + case $::operatingsystemrelease { + /^7.*/: { } + default: { + exec { 'register_systemd_nickserver': + refreshonly => true, + command => '/bin/systemctl enable nickserver', + subscribe => File['/etc/init.d/nickserver'], + before => Service['nickserver']; + } + } + } + service { 'nickserver': ensure => running, enable => true, @@ -129,6 +136,7 @@ class site_nickserver { hasstatus => true, require => [ File['/etc/init.d/nickserver'], + File['/usr/bin/nickserver'], Class['Site_config::X509::Key'], Class['Site_config::X509::Cert'], Class['Site_config::X509::Ca'] ]; diff --git a/puppet/modules/site_nickserver/templates/nickserver-proxy.conf.erb b/puppet/modules/site_nickserver/templates/nickserver-proxy.conf.erb index d4e734c3..8f59fe38 100644 --- a/puppet/modules/site_nickserver/templates/nickserver-proxy.conf.erb +++ b/puppet/modules/site_nickserver/templates/nickserver-proxy.conf.erb @@ -9,7 +9,6 @@ Listen 0.0.0.0:<%= @nickserver_port -%> ServerAlias <%= @address_domain %> SSLCACertificatePath /etc/ssl/certs - SSLCertificateChainFile <%= scope.lookupvar('x509::variables::local_CAs') %>/<%= scope.lookupvar('site_config::params::ca_name') %>.crt SSLCertificateKeyFile <%= scope.lookupvar('x509::variables::keys') %>/<%= scope.lookupvar('site_config::params::cert_name') %>.key SSLCertificateFile <%= scope.lookupvar('x509::variables::certs') %>/<%= scope.lookupvar('site_config::params::cert_name') %>.crt diff --git a/puppet/modules/site_obfsproxy/manifests/init.pp b/puppet/modules/site_obfsproxy/manifests/init.pp index 6275ebee..2ed5ec9e 100644 --- a/puppet/modules/site_obfsproxy/manifests/init.pp +++ b/puppet/modules/site_obfsproxy/manifests/init.pp @@ -19,8 +19,7 @@ class site_obfsproxy { $bind_address = hiera('ip_address') } - include site_apt::preferences::twisted - include site_apt::preferences::obfsproxy + include site_config::default class { 'obfsproxy': transport => $transport, diff --git a/puppet/modules/site_openvpn/manifests/init.pp b/puppet/modules/site_openvpn/manifests/init.pp index e2a3124e..f1ecefb9 100644 --- a/puppet/modules/site_openvpn/manifests/init.pp +++ b/puppet/modules/site_openvpn/manifests/init.pp @@ -24,9 +24,11 @@ class site_openvpn { include site_config::x509::key include site_config::x509::ca_bundle - + include site_config::default Class['site_config::default'] -> Class['site_openvpn'] + include ::site_obfsproxy + $openvpn = hiera('openvpn') $openvpn_ports = $openvpn['ports'] $openvpn_config = $openvpn['configuration'] @@ -67,7 +69,7 @@ class site_openvpn { # thx to https://blog.kumina.nl/tag/puppet-tips-and-tricks/ # we can do this using an inline_template: $factname_primary_netmask = "netmask_cidr_${::site_config::params::interface}" - $primary_netmask = inline_template('<%= scope.lookupvar(factname_primary_netmask) %>') + $primary_netmask = inline_template('<%= scope.lookupvar(@factname_primary_netmask) %>') # deploy dh keys include site_openvpn::dh_key @@ -85,24 +87,24 @@ class site_openvpn { if $openvpn_allow_unlimited { site_openvpn::server_config { 'tcp_config': - port => '1194', - proto => 'tcp', - local => $unlimited_gateway_address, - tls_remote => "\"${openvpn_unlimited_prefix}\"", - server => "${openvpn_unlimited_tcp_network_prefix}.0 ${openvpn_unlimited_tcp_netmask}", - push => "\"dhcp-option DNS ${openvpn_unlimited_tcp_network_prefix}.1\"", - management => '127.0.0.1 1000', - config => $openvpn_config + port => '1194', + proto => 'tcp', + local => $unlimited_gateway_address, + tls_remote => "\"${openvpn_unlimited_prefix}\"", + server => "${openvpn_unlimited_tcp_network_prefix}.0 ${openvpn_unlimited_tcp_netmask}", + push => "\"dhcp-option DNS ${openvpn_unlimited_tcp_network_prefix}.1\"", + management => '127.0.0.1 1000', + config => $openvpn_config } site_openvpn::server_config { 'udp_config': - port => '1194', - proto => 'udp', - local => $unlimited_gateway_address, - tls_remote => "\"${openvpn_unlimited_prefix}\"", - server => "${openvpn_unlimited_udp_network_prefix}.0 ${openvpn_unlimited_udp_netmask}", - push => "\"dhcp-option DNS ${openvpn_unlimited_udp_network_prefix}.1\"", - management => '127.0.0.1 1001', - config => $openvpn_config + port => '1194', + proto => 'udp', + local => $unlimited_gateway_address, + tls_remote => "\"${openvpn_unlimited_prefix}\"", + server => "${openvpn_unlimited_udp_network_prefix}.0 ${openvpn_unlimited_udp_netmask}", + push => "\"dhcp-option DNS ${openvpn_unlimited_udp_network_prefix}.1\"", + management => '127.0.0.1 1001', + config => $openvpn_config } } else { tidy { '/etc/openvpn/tcp_config.conf': } @@ -111,24 +113,24 @@ class site_openvpn { if $openvpn_allow_limited { site_openvpn::server_config { 'limited_tcp_config': - port => '1194', - proto => 'tcp', - local => $limited_gateway_address, - tls_remote => "\"${openvpn_limited_prefix}\"", - server => "${openvpn_limited_tcp_network_prefix}.0 ${openvpn_limited_tcp_netmask}", - push => "\"dhcp-option DNS ${openvpn_limited_tcp_network_prefix}.1\"", - management => '127.0.0.1 1002', - config => $openvpn_config + port => '1194', + proto => 'tcp', + local => $limited_gateway_address, + tls_remote => "\"${openvpn_limited_prefix}\"", + server => "${openvpn_limited_tcp_network_prefix}.0 ${openvpn_limited_tcp_netmask}", + push => "\"dhcp-option DNS ${openvpn_limited_tcp_network_prefix}.1\"", + management => '127.0.0.1 1002', + config => $openvpn_config } site_openvpn::server_config { 'limited_udp_config': - port => '1194', - proto => 'udp', - local => $limited_gateway_address, - tls_remote => "\"${openvpn_limited_prefix}\"", - server => "${openvpn_limited_udp_network_prefix}.0 ${openvpn_limited_udp_netmask}", - push => "\"dhcp-option DNS ${openvpn_limited_udp_network_prefix}.1\"", - management => '127.0.0.1 1003', - config => $openvpn_config + port => '1194', + proto => 'udp', + local => $limited_gateway_address, + tls_remote => "\"${openvpn_limited_prefix}\"", + server => "${openvpn_limited_udp_network_prefix}.0 ${openvpn_limited_udp_netmask}", + push => "\"dhcp-option DNS ${openvpn_limited_udp_network_prefix}.1\"", + management => '127.0.0.1 1003', + config => $openvpn_config } } else { tidy { '/etc/openvpn/limited_tcp_config.conf': } @@ -172,14 +174,8 @@ class site_openvpn { include site_shorewall::eip - # In wheezy, we need the openvpn backport to get the 2.3 version of - # openvpn which has proper ipv6 support - include site_apt::preferences::openvpn - package { - 'openvpn': - ensure => latest, - require => Class['site_apt::preferences::openvpn']; + 'openvpn': ensure => latest } service { @@ -228,7 +224,15 @@ class site_openvpn { order => 10; } - leap::logfile { 'openvpn': } + leap::logfile { 'openvpn_tcp': } + leap::logfile { 'openvpn_udp': } + + # Because we currently do not support ipv6 and instead block it (so no leaks + # happen), we get a large number of these messages, so we ignore them (#6540) + rsyslog::snippet { '01-ignore_icmpv6_send': + content => ':msg, contains, "icmpv6_send: no reply to icmp error" ~' + } + include site_check_mk::agent::openvpn } diff --git a/puppet/modules/site_openvpn/manifests/server_config.pp b/puppet/modules/site_openvpn/manifests/server_config.pp index 221c79a7..6decc665 100644 --- a/puppet/modules/site_openvpn/manifests/server_config.pp +++ b/puppet/modules/site_openvpn/manifests/server_config.pp @@ -109,7 +109,7 @@ define site_openvpn::server_config( "cert ${openvpn_configname}": key => 'cert', value => "${x509::variables::certs}/${site_config::params::cert_name}.crt", - server => $openvpn_configname; + server => $openvpn_configname; "key ${openvpn_configname}": key => 'key', value => "${x509::variables::keys}/${site_config::params::cert_name}.key", @@ -203,5 +203,26 @@ define site_openvpn::server_config( key => 'verb', value => '3', server => $openvpn_configname; + "log-append /var/log/leap/openvpn_${proto}.log": + key => 'log-append', + value => "/var/log/leap/openvpn_${proto}.log", + server => $openvpn_configname; + } + + # register openvpn services at systemd on nodes newer than wheezy + # see https://leap.se/code/issues/7798 + case $::operatingsystemrelease { + /^7.*/: { } + default: { + exec { "enable_systemd_${openvpn_configname}": + refreshonly => true, + command => "/bin/systemctl enable openvpn@${openvpn_configname}", + subscribe => File["/etc/openvpn/${openvpn_configname}.conf"], + notify => Service["openvpn@${openvpn_configname}"]; + } + service { "openvpn@${openvpn_configname}": + ensure => running + } + } } } diff --git a/puppet/modules/site_postfix/files/checks/received_anon b/puppet/modules/site_postfix/files/checks/received_anon index 2822973e..9de25e63 100644 --- a/puppet/modules/site_postfix/files/checks/received_anon +++ b/puppet/modules/site_postfix/files/checks/received_anon @@ -1,2 +1,2 @@ -/^Received: from (.* \([-._[:alnum:]]+ \[[.[:digit:]]{7,15}\]\))([[:space:]]+).*(\(using [.[:alnum:]]+ with cipher [-A-Z0-9]+ \([0-9]+\/[0-9]+ bits\)\))[[:space:]]+\(Client CN "([[:alnum:]]+)", Issuer "[[:print:]]+" \(verified OK\)\)[[:space:]]+by ([.[:alnum:]]+) \(([^)]+)\) with (E?SMTPS?A?) id ([A-F[:digit:]]+).*/ +/^Received: from (.* \([-._[:alnum:]]+ \[[.[:digit:]]{7,15}\]\))([[:space:]]+).*(\(using [.[:alnum:]]+ with cipher [-A-Z0-9]+ \([0-9]+\/[0-9]+ bits\)\))[[:space:]]+\(Client CN "([-._@[:alnum:]]+)", Issuer "[[:print:]]+" \(verified OK\)\)[[:space:]]+by ([.[:alnum:]]+) \(([^)]+)\) with (E?SMTPS?A?) id ([A-F[:digit:]]+).*/ REPLACE Received: from [127.0.0.1] (localhost [127.0.0.1])${2}${3}${2}(Authenticated sender: $4)${2}with $7 id $8 diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp index 49692d24..c269946b 100644 --- a/puppet/modules/site_postfix/manifests/mx.pp +++ b/puppet/modules/site_postfix/manifests/mx.pp @@ -7,10 +7,12 @@ class site_postfix::mx { $domain = $domain_hash['full_suffix'] $host_domain = $domain_hash['full'] $cert_name = hiera('name') - $mynetworks = join(hiera('mynetworks'), ' ') + $mynetworks = join(hiera('mynetworks', ''), ' ') + $rbls = suffix(prefix(hiera('rbls', []), 'reject_rbl_client '), ',') - $root_mail_recipient = hiera('contacts') - $postfix_smtp_listen = 'all' + $root_mail_recipient = hiera('contacts') + $postfix_smtp_listen = 'all' + $postfix_use_postscreen = 'yes' include site_config::x509::cert include site_config::x509::key @@ -20,20 +22,41 @@ class site_postfix::mx { postfix::config { 'mynetworks': value => "127.0.0.0/8 [::1]/128 [fe80::]/64 ${mynetworks}"; + # Note: mydestination should not include @domain, because this is + # used in virtual alias maps. 'mydestination': - value => "\$myorigin, localhost, localhost.\$mydomain, ${domain}"; + value => "\$myorigin, localhost, localhost.\$mydomain"; 'myhostname': value => $host_domain; 'mailbox_size_limit': value => '0'; 'home_mailbox': - value => 'Maildir/'; + value => ''; + 'virtual_mailbox_domains': + value => 'deliver.local'; + 'virtual_mailbox_base': + value => '/var/mail/leap-mx'; + 'virtual_mailbox_maps': + value => 'static:Maildir/'; + # Note: virtual-aliases map will take precedence over leap-mx + # lookup (tcp:localhost) 'virtual_alias_maps': - value => 'tcp:localhost:4242'; + value => 'hash:/etc/postfix/virtual-aliases tcp:localhost:4242'; 'luser_relay': - value => 'vmail'; - 'smtpd_tls_received_header': - value => 'yes'; + value => ''; + # uid and gid are set to an arbitrary hard-coded value here, this + # must match the 'leap-mx' user/group + 'virtual_uid_maps': + value => 'static:42424'; + 'virtual_gid_maps': + value => 'static:42424'; + # the two following configs are needed for matching user's client cert + # fingerprints to enable relaying (#3634). Satellites do not have + # these configured. + 'smtpd_tls_fingerprint_digest': + value => 'sha1'; + 'relay_clientcerts': + value => 'tcp:localhost:2424'; # Note: we are setting this here, instead of in site_postfix::mx::smtp_tls # because the satellites need to have a different value 'smtp_tls_security_level': @@ -44,40 +67,86 @@ class site_postfix::mx { # alias map 'local_recipient_maps': value => '$alias_maps'; + # setup clamav and opendkim on smtpd + 'smtpd_milters': + value => 'unix:/run/clamav/milter.ctl,inet:localhost:8891'; + # setup opendkim for smtp (non-smtpd) outgoing mail + 'non_smtpd_milters': + value => 'inet:localhost:8891'; + 'milter_default_action': + value => 'accept'; + # Make sure that the right values are set, these could be set to different + # things on install, depending on preseed or debconf options + # selected (see #7478) + 'relay_transport': + value => 'relay'; + 'default_transport': + value => 'smtp'; + 'mailbox_command': + value => ''; + 'header_checks': + value => ''; + 'postscreen_access_list': + value => 'permit_mynetworks'; + 'postscreen_greet_action': + value => 'enforce'; } - include site_postfix::mx::smtpd_checks - include site_postfix::mx::checks - include site_postfix::mx::smtp_tls - include site_postfix::mx::smtpd_tls - include site_postfix::mx::reserved_aliases + # Make sure that the cleanup serivce is not chrooted, otherwise it cannot + # access the opendkim milter socket (#8020) + exec { 'unset_cleanup_chroot': + command => '/usr/sbin/postconf -F "cleanup/unix/chroot=n"', + onlyif => '/usr/sbin/postconf -h -F "cleanup/unix/chroot" | egrep -q ^n', + notify => Service['postfix'], + require => File['/etc/postfix/master.cf'] + } + + include ::site_postfix::mx::smtpd_checks + include ::site_postfix::mx::checks + include ::site_postfix::mx::smtp_tls + include ::site_postfix::mx::smtpd_tls + include ::site_postfix::mx::static_aliases + include ::site_postfix::mx::rewrite_openpgp_header + include ::site_postfix::mx::received_anon + include ::clamav + include ::opendkim + include ::postfwd # greater verbosity for debugging, take out for production #include site_postfix::debug - user { 'vmail': - ensure => present, - comment => 'Leap Mailspool', - home => '/var/mail/vmail', - shell => '/bin/false', - managehome => true, + case $::operatingsystemrelease { + /^7.*/: { + $smtpd_relay_restrictions='' + } + default: { + $smtpd_relay_restrictions=" -o smtpd_relay_restrictions=\$smtps_relay_restrictions\n" + } } + $mastercf_tail = " +smtps inet n - - - - smtpd + -o smtpd_tls_wrappermode=yes + -o smtpd_tls_security_level=encrypt + -o tls_preempt_cipherlist=yes +${smtpd_relay_restrictions} -o smtpd_recipient_restrictions=\$smtps_recipient_restrictions + -o smtpd_helo_restrictions=\$smtps_helo_restrictions + -o smtpd_client_restrictions= + -o cleanup_service_name=clean_smtps +clean_smtps unix n - n - 0 cleanup + -o header_checks=pcre:/etc/postfix/checks/rewrite_openpgp_headers,pcre:/etc/postfix/checks/received_anon" + class { 'postfix': preseed => true, root_mail_recipient => $root_mail_recipient, smtp_listen => 'all', - mastercf_tail => - "smtps inet n - - - - smtpd - -o smtpd_tls_wrappermode=yes - -o smtpd_tls_security_level=encrypt - -o smtpd_recipient_restrictions=\$smtps_recipient_restrictions - -o smtpd_helo_restrictions=\$smtps_helo_restrictions", + mastercf_tail => $mastercf_tail, + use_postscreen => 'yes', require => [ Class['Site_config::X509::Key'], Class['Site_config::X509::Cert'], Class['Site_config::X509::Client_ca::Key'], Class['Site_config::X509::Client_ca::Ca'], - User['vmail'] ] + User['leap-mx'] ] } } diff --git a/puppet/modules/site_postfix/manifests/mx/checks.pp b/puppet/modules/site_postfix/manifests/mx/checks.pp index 5d75a5e5..f406ad34 100644 --- a/puppet/modules/site_postfix/manifests/mx/checks.pp +++ b/puppet/modules/site_postfix/manifests/mx/checks.pp @@ -20,22 +20,4 @@ class site_postfix::mx::checks { refreshonly => true, subscribe => File['/etc/postfix/checks/helo_checks']; } - - # Anonymize the user's home IP from the email headers (Feature #3866) - package { 'postfix-pcre': ensure => installed, require => Package['postfix'] } - - file { '/etc/postfix/checks/received_anon': - source => 'puppet:///modules/site_postfix/checks/received_anon', - mode => '0644', - owner => root, - group => root, - notify => Service['postfix'] - } - - postfix::config { - 'header_checks': - value => 'pcre:/etc/postfix/checks/received_anon', - require => File['/etc/postfix/checks/received_anon']; - } - } diff --git a/puppet/modules/site_postfix/manifests/mx/received_anon.pp b/puppet/modules/site_postfix/manifests/mx/received_anon.pp new file mode 100644 index 00000000..51ba3faa --- /dev/null +++ b/puppet/modules/site_postfix/manifests/mx/received_anon.pp @@ -0,0 +1,13 @@ +# Anonymize the user's home IP from the email headers (Feature #3866) +class site_postfix::mx::received_anon { + + package { 'postfix-pcre': ensure => installed, require => Package['postfix'] } + + file { '/etc/postfix/checks/received_anon': + source => 'puppet:///modules/site_postfix/checks/received_anon', + mode => '0644', + owner => root, + group => root, + notify => Service['postfix'] + } +} diff --git a/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp b/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp deleted file mode 100644 index 83e27376..00000000 --- a/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp +++ /dev/null @@ -1,15 +0,0 @@ -# Defines which mail addresses shouldn't be available and where they should fwd -class site_postfix::mx::reserved_aliases { - - postfix::mailalias { - [ 'abuse', 'admin', 'arin-admin', 'administrator', 'bin', 'cron', - 'certmaster', 'domainadmin', 'games', 'ftp', 'hostmaster', 'lp', - 'maildrop', 'mysql', 'news', 'nobody', 'noc', 'postmaster', 'postgresql', - 'security', 'ssladmin', 'sys', 'usenet', 'uucp', 'webmaster', 'www', - 'www-data', - ]: - ensure => present, - recipient => 'root' - } - -} diff --git a/puppet/modules/site_postfix/manifests/mx/rewrite_openpgp_header.pp b/puppet/modules/site_postfix/manifests/mx/rewrite_openpgp_header.pp new file mode 100644 index 00000000..71f945b8 --- /dev/null +++ b/puppet/modules/site_postfix/manifests/mx/rewrite_openpgp_header.pp @@ -0,0 +1,11 @@ +class site_postfix::mx::rewrite_openpgp_header { + $mx = hiera('mx') + $correct_domain = $mx['key_lookup_domain'] + + file { '/etc/postfix/checks/rewrite_openpgp_headers': + content => template('site_postfix/checks/rewrite_openpgp_headers.erb'), + mode => '0644', + owner => root, + group => root; + } +} diff --git a/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp b/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp index d56f6b54..c93c3ba2 100644 --- a/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp +++ b/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp @@ -1,11 +1,16 @@ +# configure smtp tls class site_postfix::mx::smtp_tls { include site_config::x509::ca include x509::variables + $cert_name = hiera('name') $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" + include site_config::x509::cert + include site_config::x509::key + # smtp TLS postfix::config { 'smtp_use_tls': value => 'yes'; @@ -20,9 +25,19 @@ class site_postfix::mx::smtp_tls { 'smtp_tls_fingerprint_digest': value => 'sha1'; 'smtp_tls_session_cache_database': - value => 'btree:${data_directory}/smtp_cache'; + value => "btree:\${data_directory}/smtp_cache"; # see issue #4011 'smtp_tls_protocols': value => '!SSLv2, !SSLv3'; + 'smtp_tls_mandatory_protocols': + value => '!SSLv2, !SSLv3'; + 'tls_ssl_options': + value => 'NO_COMPRESSION'; + # We can switch between the different postfix internal list of ciphers by + # using smtpd_tls_ciphers. For server-to-server connections we leave this + # at its default because of opportunistic encryption combined with many mail + # servers only support outdated protocols and ciphers and if we are too + # strict with required ciphers, then connections *will* fall-back to + # plain-text. Bad ciphers are still better than plain text transmission. } } diff --git a/puppet/modules/site_postfix/manifests/mx/smtpd_checks.pp b/puppet/modules/site_postfix/manifests/mx/smtpd_checks.pp index 0ec40277..291d7ee4 100644 --- a/puppet/modules/site_postfix/manifests/mx/smtpd_checks.pp +++ b/puppet/modules/site_postfix/manifests/mx/smtpd_checks.pp @@ -1,3 +1,5 @@ +# smtpd checks for incoming mail on smtp port 25 and +# mail sent via the bitmask client using smtps port 465 class site_postfix::mx::smtpd_checks { postfix::config { @@ -6,7 +8,7 @@ class site_postfix::mx::smtpd_checks { 'checks_dir': value => '$config_directory/checks'; 'smtpd_client_restrictions': - value => 'permit_mynetworks,permit'; + value => "permit_mynetworks,${site_postfix::mx::rbls},permit"; 'smtpd_data_restrictions': value => 'permit_mynetworks, reject_unauth_pipelining, permit'; 'smtpd_delay_reject': @@ -15,13 +17,16 @@ class site_postfix::mx::smtpd_checks { value => 'permit_mynetworks, reject_invalid_helo_hostname, reject_non_fqdn_helo_hostname, check_helo_access hash:$checks_dir/helo_checks, permit'; 'smtpd_recipient_restrictions': value => 'reject_unknown_recipient_domain, permit_mynetworks, check_recipient_access tcp:localhost:2244, reject_unauth_destination, permit'; - # We should change from permit_tls_all_clientcerts to permit_tls_clientcerts - # with a lookup on $relay_clientcerts! Right now we are listing the only - # valid CA that client certificates can use in the $smtp_tls_CAfile parameter - # but we cannot cut off a certificate that should no longer be used unless - # we use permit_tls_clientcerts with the $relay_clientcerts lookup + + # permit_tls_clientcerts will lookup client cert fingerprints from the tcp + # lookup on port 2424 (based on what is configured in relay_clientcerts + # paramter, see site_postfix::mx postfix::config resource) to determine + # if a client is allowed to relay mail through us. This enables us to + # disable a user by removing their valid client cert (#3634) 'smtps_recipient_restrictions': - value => 'permit_tls_all_clientcerts, check_recipient_access tcp:localhost:2244, reject_unauth_destination, permit'; + value => 'permit_tls_clientcerts, check_recipient_access tcp:localhost:2244, reject_unauth_destination, permit'; + 'smtps_relay_restrictions': + value => 'permit_mynetworks, permit_tls_clientcerts, defer_unauth_destination'; 'smtps_helo_restrictions': value => 'permit_mynetworks, check_helo_access hash:$checks_dir/helo_checks, permit'; 'smtpd_sender_restrictions': diff --git a/puppet/modules/site_postfix/manifests/mx/smtpd_tls.pp b/puppet/modules/site_postfix/manifests/mx/smtpd_tls.pp index 0809c75f..66297f55 100644 --- a/puppet/modules/site_postfix/manifests/mx/smtpd_tls.pp +++ b/puppet/modules/site_postfix/manifests/mx/smtpd_tls.pp @@ -1,3 +1,4 @@ +# configure smtpd tls class site_postfix::mx::smtpd_tls { include x509::variables @@ -12,12 +13,25 @@ class site_postfix::mx::smtpd_tls { 'smtpd_tls_cert_file': value => $cert_path; 'smtpd_tls_key_file': value => $key_path; 'smtpd_tls_ask_ccert': value => 'yes'; + 'smtpd_tls_received_header': + value => 'yes'; 'smtpd_tls_security_level': value => 'may'; 'smtpd_tls_eecdh_grade': value => 'ultra'; 'smtpd_tls_session_cache_database': - value => 'btree:${data_directory}/smtpd_scache'; + value => "btree:\${data_directory}/smtpd_scache"; + # see issue #4011 + 'smtpd_tls_mandatory_protocols': + value => '!SSLv2, !SSLv3'; + 'smtpd_tls_protocols': + value => '!SSLv2, !SSLv3'; + # For connections to MUAs, TLS is mandatory and the ciphersuite is modified. + # MX and SMTP client configuration + 'smtpd_tls_mandatory_ciphers': + value => 'high'; + 'tls_high_cipherlist': + value => 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!RC4:!MD5:!PSK!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA'; } # Setup DH parameters diff --git a/puppet/modules/site_postfix/manifests/mx/static_aliases.pp b/puppet/modules/site_postfix/manifests/mx/static_aliases.pp new file mode 100644 index 00000000..9cd7ca02 --- /dev/null +++ b/puppet/modules/site_postfix/manifests/mx/static_aliases.pp @@ -0,0 +1,88 @@ +# +# Defines static, hard coded aliases that are not in the database. +# These aliases take precedence over the database aliases. +# +# There are three classes of reserved names: +# +# (1) forbidden_usernames: +# Some usernames are forbidden and cannot be registered. +# this is defined in node property webapp.forbidden_usernames +# This is enforced by the webapp. +# +# (2) public aliases: +# Some aliases for root, and are publicly exposed so that anyone +# can deliver mail to them. For example, postmaster. +# These are implemented in the virtual alias map, which takes +# precedence over the local alias map. +# +# (3) local aliases: +# Some aliases are only available locally: mail can be delivered +# to the alias if the mail originates from the local host, or is +# hostname qualified, but otherwise it will be rejected. +# These are implemented in the local alias map. +# +# The alias for local 'root' is defined elsewhere. In this file, we +# define the virtual 'root@domain' (which can be overwritten by +# defining an entry for root in node property mx.aliases). +# + +class site_postfix::mx::static_aliases { + + $mx = hiera('mx') + $root_recipients = hiera('contacts') + + # + # LOCAL ALIASES + # + + # NOTE: if you remove one of these, they will still appear in the + # /etc/aliases file + $local_aliases = [ + 'admin', 'administrator', 'bin', 'cron', 'games', 'ftp', 'lp', 'maildrop', + 'mysql', 'news', 'nobody', 'noc', 'postgresql', 'ssladmin', 'sys', + 'usenet', 'uucp', 'www', 'www-data', 'leap-mx' + ] + + postfix::mailalias { + $local_aliases: + ensure => present, + recipient => 'root' + } + + # + # PUBLIC ALIASES + # + + $public_aliases = $mx['aliases'] + + $default_public_aliases = { + 'root' => $root_recipients, + 'abuse' => 'postmaster', + 'arin-admin' => 'root', + 'certmaster' => 'hostmaster', + 'domainadmin' => 'hostmaster', + 'hostmaster' => 'root', + 'mailer-daemon' => 'postmaster', + 'postmaster' => 'root', + 'security' => 'root', + 'webmaster' => 'hostmaster', + } + + $aliases = merge($default_public_aliases, $public_aliases) + + exec { 'postmap_virtual_aliases': + command => '/usr/sbin/postmap /etc/postfix/virtual-aliases', + refreshonly => true, + user => root, + group => root, + require => Package['postfix'], + subscribe => File['/etc/postfix/virtual-aliases'] + } + file { '/etc/postfix/virtual-aliases': + content => template('site_postfix/virtual-aliases.erb'), + owner => root, + group => root, + mode => '0600', + require => Package['postfix'] + } +} diff --git a/puppet/modules/site_postfix/templates/checks/helo_access.erb b/puppet/modules/site_postfix/templates/checks/helo_access.erb index bef3c11d..bac2c45a 100644 --- a/puppet/modules/site_postfix/templates/checks/helo_access.erb +++ b/puppet/modules/site_postfix/templates/checks/helo_access.erb @@ -18,4 +18,4 @@ # Reject anybody that HELO's as being in our own domain(s) # anyone who identifies themselves as us is a virus/spammer -<%= domain %> 554 You are not in domain <%= domain %> +<%= @domain %> 554 You are not in domain <%= @domain %> diff --git a/puppet/modules/site_postfix/templates/checks/rewrite_openpgp_headers.erb b/puppet/modules/site_postfix/templates/checks/rewrite_openpgp_headers.erb new file mode 100644 index 00000000..7af14f7d --- /dev/null +++ b/puppet/modules/site_postfix/templates/checks/rewrite_openpgp_headers.erb @@ -0,0 +1,13 @@ +# THIS FILE IS MANAGED BY PUPPET +# +# This will replace the OpenPGP header that the client adds, because it is +# sometimes incorrect (due to the client not always knowing what the proper URL +# is for the webapp). +# e.g. This will rewrite this header: +# OpenPGP: id=4C0E01CD50E2F653; url="https://leap.se/key/elijah"; preference="signencrypt +# with this replacement: +# OpenPGP: id=4C0E01CD50E2F653; url="https://user.leap.se/key/elijah"; preference="signencrypt +# +# Note: whitespace in the pattern is represented by [[:space:]] to avoid these warnings from postmap: +# "record is in "key: value" format; is this an alias file?" and "duplicate entry" +/^(OpenPGP:[[:space:]]id=[[:alnum:]]+;[[:space:]]url="https:\/\/)<%= @domain %>(\/key\/[[:alpha:]]+";.*)/i REPLACE ${1}<%= @correct_domain %>${2} diff --git a/puppet/modules/site_postfix/templates/virtual-aliases.erb b/puppet/modules/site_postfix/templates/virtual-aliases.erb new file mode 100644 index 00000000..8373de97 --- /dev/null +++ b/puppet/modules/site_postfix/templates/virtual-aliases.erb @@ -0,0 +1,21 @@ +# +# This file is managed by puppet. +# +# These virtual aliases take precedence over all other aliases. +# + +# +# enable these virtual domains: +# +<%= @domain %> enabled +<%- @aliases.keys.map {|addr| addr.split('@')[1] }.compact.sort.uniq.each do |virt_domain| -%> +<%= virt_domain %> enabled +<%- end %> + +# +# virtual aliases: +# +<%- @aliases.keys.sort.each do |from| -%> +<%- full_address = from =~ /@/ ? from : from + "@" + @domain -%> +<%= full_address %> <%= [@aliases[from]].flatten.map{|a| a =~ /@/ ? a : a + "@" + @domain}.join(', ') %> +<%- end -%> diff --git a/puppet/modules/site_rsyslog/templates/client.conf.erb b/puppet/modules/site_rsyslog/templates/client.conf.erb new file mode 100644 index 00000000..7f94759d --- /dev/null +++ b/puppet/modules/site_rsyslog/templates/client.conf.erb @@ -0,0 +1,134 @@ + +# An "In-Memory Queue" is created for remote logging. +$WorkDirectory <%= scope.lookupvar('rsyslog::spool_dir') -%> # where to place spool files +$ActionQueueFileName queue # unique name prefix for spool files +$ActionQueueMaxDiskSpace <%= scope.lookupvar('rsyslog::client::spool_size') -%> # spool space limit (use as much as possible) +$ActionQueueSaveOnShutdown on # save messages to disk on shutdown +$ActionQueueType LinkedList # run asynchronously +$ActionResumeRetryCount -1 # infinety retries if host is down +<% if scope.lookupvar('rsyslog::client::log_templates') and ! scope.lookupvar('rsyslog::client::log_templates').empty?-%> + +# Define custom logging templates +<% scope.lookupvar('rsyslog::client::log_templates').flatten.compact.each do |log_template| -%> +$template <%= log_template['name'] %>,"<%= log_template['template'] %>" +<% end -%> +<% end -%> +<% if scope.lookupvar('rsyslog::client::actionfiletemplate') -%> + +# Using specified format for default logging format: +$ActionFileDefaultTemplate <%= scope.lookupvar('rsyslog::client::actionfiletemplate') %> +<% else -%> + +#Using default format for default logging format: +$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat +<% end -%> +<% if scope.lookupvar('rsyslog::client::ssl') -%> + +# Setup SSL connection. +# CA/Cert +$DefaultNetStreamDriverCAFile <%= scope.lookupvar('rsyslog::client::ssl_ca') %> + +# Connection settings. +$DefaultNetstreamDriver gtls +$ActionSendStreamDriverMode 1 +$ActionSendStreamDriverAuthMode anon +<% end -%> +<% if scope.lookupvar('rsyslog::client::remote_servers') -%> + +<% scope.lookupvar('rsyslog::client::remote_servers').flatten.compact.each do |server| -%> +<% if server['pattern'] and server['pattern'] != ''-%> +<% pattern = server['pattern'] -%> +<% else -%> +<% pattern = '*.*' -%> +<% end -%> +<% if server['protocol'] == 'TCP' or server['protocol'] == 'tcp'-%> +<% protocol = '@@' -%> +<% protocol_type = 'TCP' -%> +<% else -%> +<% protocol = '@' -%> +<% protocol_type = 'UDP' -%> +<% end -%> +<% if server['host'] and server['host'] != ''-%> +<% host = server['host'] -%> +<% else -%> +<% host = 'localhost' -%> +<% end -%> +<% if server['port'] and server['port'] != ''-%> +<% port = server['port'] -%> +<% else -%> +<% port = '514' -%> +<% end -%> +<% if server['format'] -%> +<% format = ";#{server['format']}" -%> +<% format_type = server['format'] -%> +<% else -%> +<% format = '' -%> +<% format_type = 'the default' -%> +<% end -%> +# Sending logs that match <%= pattern %> to <%= host %> via <%= protocol_type %> on <%= port %> using <%=format_type %> format. +<%= pattern %> <%= protocol %><%= host %>:<%= port %><%= format %> +<% end -%> +<% elsif scope.lookupvar('rsyslog::client::log_remote') -%> + +# Log to remote syslog server using <%= scope.lookupvar('rsyslog::client::remote_type') %> +<% if scope.lookupvar('rsyslog::client::remote_type') == 'tcp' -%> +*.* @@<%= scope.lookupvar('rsyslog::client::server') -%>:<%= scope.lookupvar('rsyslog::client::port') -%>;<%= scope.lookupvar('remote_forward_format') -%> +<% else -%> +*.* @<%= scope.lookupvar('rsyslog::client::server') -%>:<%= scope.lookupvar('rsyslog::client::port') -%>;<%= scope.lookupvar('remote_forward_format') -%> +<% end -%> +<% end -%> +<% if scope.lookupvar('rsyslog::client::log_auth_local') or scope.lookupvar('rsyslog::client::log_local') -%> + +# Logging locally. + +<% if scope.lookupvar('rsyslog::log_style') == 'debian' -%> +# Log auth messages locally +.*;auth,authpriv.none;mail.none -/var/log/syslog +<% elsif scope.lookupvar('rsyslog::log_style') == 'redhat' -%> +# Log auth messages locally +auth,authpriv.* /var/log/secure +<% end -%> +<% end -%> +<% if scope.lookupvar('rsyslog::client::log_local') -%> +<% if scope.lookupvar('rsyslog::log_style') == 'debian' -%> +# First some standard log files. Log by facility. +# +*.*;auth,authpriv.none -/var/log/syslog +cron.* /var/log/cron.log +daemon.* -/var/log/daemon.log +kern.* -/var/log/kern.log +mail.* -/var/log/mail.log +user.* -/var/log/user.log + +# +# Some "catch-all" log files. +# +*.=debug;\ + auth,authpriv.none;\ + news.none;mail.none -/var/log/debug +*.=info;*.=notice;*.=warn;\ + auth,authpriv.none;\ + cron,daemon.none;\ + mail,news.none -/var/log/messages + +# Log anything (except mail) of level info or higher. +# Don't log private authentication messages! +*.info;mail.none;authpriv.none;cron.none /var/log/messages + +# Log cron stuff +cron.* /var/log/cron + +# Everybody gets emergency messages +<% if @rsyslog_version and @rsyslog_version.split('.')[0].to_i >= 8 -%> +*.emerg :omusrmsg:* +<% else -%> +*.emerg * +<% end -%> + +# Save boot messages also to boot.log +local7.* -/var/log/boot.log +<% end -%> +<% end -%> + + + diff --git a/puppet/modules/site_shorewall/files/Debian/shorewall.service b/puppet/modules/site_shorewall/files/Debian/shorewall.service new file mode 100644 index 00000000..ec250ef1 --- /dev/null +++ b/puppet/modules/site_shorewall/files/Debian/shorewall.service @@ -0,0 +1,23 @@ +# +# The Shoreline Firewall (Shorewall) Packet Filtering Firewall +# +# Copyright 2011 Jonathan Underwood <jonathan.underwood@gmail.com> +# Copyright 2015 Tom Eastep <teastep@shorewall.net> +# +[Unit] +Description=Shorewall IPv4 firewall +Wants=network-online.target +After=network-online.target +Conflicts=iptables.service firewalld.service + +[Service] +Type=oneshot +RemainAfterExit=yes +EnvironmentFile=-/etc/default/shorewall +StandardOutput=syslog +ExecStart=/sbin/shorewall $OPTIONS start $STARTOPTIONS +ExecStop=/sbin/shorewall $OPTIONS stop +ExecReload=/sbin/shorewall $OPTIONS reload $RELOADOPTIONS + +[Install] +WantedBy=basic.target diff --git a/puppet/modules/site_shorewall/manifests/defaults.pp b/puppet/modules/site_shorewall/manifests/defaults.pp index 8f56ac42..ceb17868 100644 --- a/puppet/modules/site_shorewall/manifests/defaults.pp +++ b/puppet/modules/site_shorewall/manifests/defaults.pp @@ -47,6 +47,18 @@ class site_shorewall::defaults { ensure => installed } + include ::systemd + file { '/etc/systemd/system/shorewall.service': + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/site_shorewall/Debian/shorewall.service', + require => Package['shorewall'], + notify => Service['shorewall'], + } ~> + Exec['systemctl-daemon-reload'] + augeas { # stop instead of clear firewall on shutdown 'shorewall_SAFESTOP': @@ -54,14 +66,14 @@ class site_shorewall::defaults { lens => 'Shellvars.lns', incl => '/etc/shorewall/shorewall.conf', require => Package['shorewall'], - notify => Service[shorewall]; + notify => Service['shorewall']; # require that the interface exist 'shorewall_REQUIRE_INTERFACE': changes => 'set /files/etc/shorewall/shorewall.conf/REQUIRE_INTERFACE Yes', lens => 'Shellvars.lns', incl => '/etc/shorewall/shorewall.conf', require => Package['shorewall'], - notify => Service[shorewall]; + notify => Service['shorewall']; # configure shorewall-init 'shorewall-init': changes => 'set /files/etc/default/shorewall-init/PRODUCTS shorewall', diff --git a/puppet/modules/site_shorewall/manifests/obfsproxy.pp b/puppet/modules/site_shorewall/manifests/obfsproxy.pp index 68fb9b9f..75846705 100644 --- a/puppet/modules/site_shorewall/manifests/obfsproxy.pp +++ b/puppet/modules/site_shorewall/manifests/obfsproxy.pp @@ -1,3 +1,4 @@ +# configure shorewell for obfsproxy class site_shorewall::obfsproxy { include site_shorewall::defaults @@ -8,7 +9,7 @@ class site_shorewall::obfsproxy { # define macro for incoming services file { '/etc/shorewall/macro.leap_obfsproxy': - content => "PARAM - - tcp $scram_port ", + content => "PARAM - - tcp ${scram_port} ", notify => Service['shorewall'], require => Package['shorewall'] } diff --git a/puppet/modules/site_shorewall/manifests/service/webapp_api.pp b/puppet/modules/site_shorewall/manifests/service/webapp_api.pp index 0c6c824d..d3a1aeed 100644 --- a/puppet/modules/site_shorewall/manifests/service/webapp_api.pp +++ b/puppet/modules/site_shorewall/manifests/service/webapp_api.pp @@ -1,3 +1,4 @@ +# configure shorewall for webapp api class site_shorewall::service::webapp_api { $api = hiera('api') @@ -5,7 +6,7 @@ class site_shorewall::service::webapp_api { # define macro for incoming services file { '/etc/shorewall/macro.leap_webapp_api': - content => "PARAM - - tcp $api_port ", + content => "PARAM - - tcp ${api_port} ", notify => Service['shorewall'], require => Package['shorewall'] } diff --git a/puppet/modules/site_shorewall/manifests/sshd.pp b/puppet/modules/site_shorewall/manifests/sshd.pp index 88b4102c..e2332592 100644 --- a/puppet/modules/site_shorewall/manifests/sshd.pp +++ b/puppet/modules/site_shorewall/manifests/sshd.pp @@ -1,3 +1,4 @@ +# configure shorewall for sshd class site_shorewall::sshd { $ssh_config = hiera('ssh') @@ -7,7 +8,7 @@ class site_shorewall::sshd { # define macro for incoming sshd file { '/etc/shorewall/macro.leap_sshd': - content => "PARAM - - tcp $ssh_port", + content => "PARAM - - tcp ${ssh_port}", notify => Service['shorewall'], require => Package['shorewall'] } diff --git a/puppet/modules/site_shorewall/manifests/tor.pp b/puppet/modules/site_shorewall/manifests/tor.pp index f35af985..324b4844 100644 --- a/puppet/modules/site_shorewall/manifests/tor.pp +++ b/puppet/modules/site_shorewall/manifests/tor.pp @@ -1,3 +1,4 @@ +# configure shorewall for tor class site_shorewall::tor { include site_shorewall::defaults @@ -7,7 +8,7 @@ class site_shorewall::tor { # define macro for incoming services file { '/etc/shorewall/macro.leap_tor': - content => "PARAM - - tcp $tor_port ", + content => "PARAM - - tcp ${tor_port} ", notify => Service['shorewall'], require => Package['shorewall'] } diff --git a/puppet/modules/site_sshd/manifests/authorized_keys.pp b/puppet/modules/site_sshd/manifests/authorized_keys.pp index 90a33d8d..a1fde3f6 100644 --- a/puppet/modules/site_sshd/manifests/authorized_keys.pp +++ b/puppet/modules/site_sshd/manifests/authorized_keys.pp @@ -1,20 +1,22 @@ +# We want to purge unmanaged keys from the authorized_keys file so that only +# keys added in the provider are valid. Any manually added keys will be +# overridden. +# +# In order to do this, we have to use a custom define to deploy the +# authorized_keys file because puppet's internal resource doesn't allow +# purging before populating this file. +# +# See the following for more information: +# https://tickets.puppetlabs.com/browse/PUP-1174 +# https://leap.se/code/issues/2990 +# https://leap.se/code/issues/3010 +# define site_sshd::authorized_keys ($keys, $ensure = 'present', $home = '') { - # We want to purge unmanaged keys from the authorized_keys file so that only - # keys added in the provider are valid. Any manually added keys will be - # overridden. - # - # In order to do this, we have to use a custom define to deploy the - # authorized_keys file because puppet's internal resource doesn't allow - # purging before populating this file. - # - # See the following for more information: - # https://tickets.puppetlabs.com/browse/PUP-1174 - # https://leap.se/code/issues/2990 - # https://leap.se/code/issues/3010 - # # This line allows default homedir based on $title variable. # If $home is empty, the default is used. $homedir = $home ? {'' => "/home/${title}", default => $home} + $owner = $ensure ? {'present' => $title, default => undef } + $group = $ensure ? {'present' => $title, default => undef } file { "${homedir}/.ssh": ensure => 'directory', @@ -23,8 +25,8 @@ define site_sshd::authorized_keys ($keys, $ensure = 'present', $home = '') { mode => '0700'; "${homedir}/.ssh/authorized_keys": ensure => $ensure, - owner => $ensure ? {'present' => $title, default => undef }, - group => $ensure ? {'present' => $title, default => undef }, + owner => $owner, + group => $group, mode => '0600', require => File["${homedir}/.ssh"], content => template('site_sshd/authorized_keys.erb'); diff --git a/puppet/modules/site_sshd/manifests/init.pp b/puppet/modules/site_sshd/manifests/init.pp index 1da2f1d5..a9202da4 100644 --- a/puppet/modules/site_sshd/manifests/init.pp +++ b/puppet/modules/site_sshd/manifests/init.pp @@ -1,6 +1,8 @@ +# configures sshd, mosh, authorized keys and known hosts class site_sshd { - $ssh = hiera_hash('ssh') - $hosts = hiera('hosts', '') + $ssh = hiera_hash('ssh') + $ssh_config = $ssh['config'] + $hosts = hiera('hosts', '') ## ## SETUP AUTHORIZED KEYS @@ -48,15 +50,33 @@ class site_sshd { } } + # we cannot use the 'hardened' parameter because leap_cli uses an + # old net-ssh gem that is incompatible with the included + # "KexAlgorithms curve25519-sha256@libssh.org", + # see https://leap.se/code/issues/7591 + # therefore we don't use it here, but include all other options + # that would be applied by the 'hardened' parameter + # not all options are available on wheezy + if ( $::lsbdistcodename == 'wheezy' ) { + $tail_additional_options = 'Ciphers aes256-ctr +MACs hmac-sha2-512,hmac-sha2-256,hmac-ripemd160' + } else { + $tail_additional_options = 'Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes256-ctr +MACs hmac-sha2-512,hmac-sha2-256,hmac-ripemd160' + } + ## ## SSHD SERVER CONFIGURATION ## class { '::sshd': - manage_nagios => false, - ports => [ $ssh['port'] ], - use_pam => 'yes', - hardened_ssl => 'yes', - print_motd => 'no', - manage_client => false + manage_nagios => false, + ports => [ $ssh['port'] ], + use_pam => 'yes', + print_motd => 'no', + tcp_forwarding => $ssh_config['AllowTcpForwarding'], + manage_client => false, + use_storedconfigs => false, + tail_additional_options => $tail_additional_options, + hostkey_type => [ 'rsa', 'dsa', 'ecdsa' ] } } diff --git a/puppet/modules/site_sshd/templates/authorized_keys.erb b/puppet/modules/site_sshd/templates/authorized_keys.erb index 69f4d8e6..51bdc5b3 100644 --- a/puppet/modules/site_sshd/templates/authorized_keys.erb +++ b/puppet/modules/site_sshd/templates/authorized_keys.erb @@ -1,7 +1,7 @@ # NOTICE: This file is autogenerated by Puppet # all manually added keys will be overridden -<% keys.sort.each do |user, hash| -%> +<% @keys.sort.each do |user, hash| -%> <% if user == 'monitor' -%> command="/usr/bin/check_mk_agent",no-port-forwarding,no-x11-forwarding,no-agent-forwarding,no-pty,no-user-rc, <%=hash['type']-%> <%=hash['key']%> <%=user%> <% else -%> diff --git a/puppet/modules/site_static/manifests/domain.pp b/puppet/modules/site_static/manifests/domain.pp index b9177f25..b26cc9e3 100644 --- a/puppet/modules/site_static/manifests/domain.pp +++ b/puppet/modules/site_static/manifests/domain.pp @@ -1,3 +1,4 @@ +# configure static service for domain define site_static::domain ( $ca_cert, $key, @@ -10,19 +11,19 @@ define site_static::domain ( $domain = $name $base_dir = '/srv/static' - create_resources(site_static::location, $locations) + $cafile = "${cert}\n${ca_cert}" + + if is_hash($locations) { + create_resources(site_static::location, $locations) + } x509::cert { $domain: - content => $cert, - notify => Service[apache] + content => $cafile, + notify => Service[apache] } x509::key { $domain: content => $key, - notify => Service[apache] - } - x509::ca { "${domain}_ca": - content => $ca_cert, - notify => Service[apache] + notify => Service[apache] } apache::vhost::file { $domain: diff --git a/puppet/modules/site_static/manifests/init.pp b/puppet/modules/site_static/manifests/init.pp index 1efc510b..4a722d62 100644 --- a/puppet/modules/site_static/manifests/init.pp +++ b/puppet/modules/site_static/manifests/init.pp @@ -1,6 +1,8 @@ +# deploy static service class site_static { tag 'leap_service' + include site_config::default include site_config::x509::cert include site_config::x509::key include site_config::x509::ca_bundle @@ -9,6 +11,7 @@ class site_static { $domains = $static['domains'] $formats = $static['formats'] $bootstrap = $static['bootstrap_files'] + $tor = hiera('tor', false) if $bootstrap['enabled'] { $bootstrap_domain = $bootstrap['domain'] @@ -27,14 +30,13 @@ class site_static { } } - class { '::apache': no_default_site => true, ssl => true } - include site_apache::module::headers - include site_apache::module::alias - include site_apache::module::expires - include site_apache::module::removeip - include site_apache::module::rewrite - apache::config::include{ 'ssl_common.inc': } - + include apache::module::headers + include apache::module::alias + include apache::module::expires + include apache::module::removeip + include apache::module::dir + include apache::module::negotiation + include site_apache::common include site_config::ruby::dev if (member($formats, 'rack')) { @@ -46,9 +48,9 @@ class site_static { } if (member($formats, 'amber')) { - rubygems::gem{'amber-0.3.7': - require => Package['zlib1g-dev'] - } + rubygems::gem{'amber-0.3.8': + require => Package['zlib1g-dev'] + } package { 'zlib1g-dev': ensure => installed @@ -57,6 +59,13 @@ class site_static { create_resources(site_static::domain, $domains) + if $tor { + $hidden_service = $tor['hidden_service'] + if $hidden_service['active'] { + include site_webapp::hidden_service + } + } + include site_shorewall::defaults include site_shorewall::service::http include site_shorewall::service::https diff --git a/puppet/modules/site_static/manifests/location.pp b/puppet/modules/site_static/manifests/location.pp index ce2af9af..d116de2f 100644 --- a/puppet/modules/site_static/manifests/location.pp +++ b/puppet/modules/site_static/manifests/location.pp @@ -1,3 +1,4 @@ +# configure static service for location define site_static::location($path, $format, $source) { $file_path = "/srv/static/${name}" @@ -14,10 +15,10 @@ define site_static::location($path, $format, $source) { if ($format == 'amber') { exec {"amber_build_${name}": - cwd => $file_path, - command => 'amber rebuild', - user => 'www-data', - timeout => 600, + cwd => $file_path, + command => 'amber rebuild', + user => 'www-data', + timeout => 600, subscribe => Vcsrepo[$file_path] } } diff --git a/puppet/modules/site_static/templates/amber.erb b/puppet/modules/site_static/templates/amber.erb index 17dc2ad6..694f1136 100644 --- a/puppet/modules/site_static/templates/amber.erb +++ b/puppet/modules/site_static/templates/amber.erb @@ -1,15 +1,13 @@ -<%- if @location_path == '' -%> - <Directory "<%= @directory %>/"> - AllowOverride FileInfo Indexes Options=All,MultiViews - Order deny,allow - Allow from all - </Directory> -<%- else -%> +<%- if @location_path != '' -%> AliasMatch ^/[a-z]{2}/<%=@location_path%>(/.+|/|)$ "<%=@directory%>/$1" Alias /<%=@location_path%> "<%=@directory%>/" +<%- end -%> <Directory "<%=@directory%>/"> AllowOverride FileInfo Indexes Options=All,MultiViews +<% if scope.function_guess_apache_version([]) == '2.4' %> + Require all granted +<% else %> Order deny,allow Allow from all +<% end %> </Directory> -<%- end -%> diff --git a/puppet/modules/site_static/templates/apache.conf.erb b/puppet/modules/site_static/templates/apache.conf.erb index 4d61cc08..6b969d1c 100644 --- a/puppet/modules/site_static/templates/apache.conf.erb +++ b/puppet/modules/site_static/templates/apache.conf.erb @@ -48,7 +48,7 @@ Include include.d/ssl_common.inc <%- if @tls_only -%> - Header add Strict-Transport-Security: "max-age=15768000;includeSubdomains" + Header always set Strict-Transport-Security: "max-age=15768000;includeSubdomains" <%- end -%> Header set X-Frame-Options "deny" Header always unset X-Powered-By @@ -56,7 +56,6 @@ SSLCertificateKeyFile /etc/x509/keys/<%= @domain %>.key SSLCertificateFile /etc/x509/certs/<%= @domain %>.crt - SSLCertificateChainFile /etc/ssl/certs/<%= @domain %>_ca.pem RequestHeader set X_FORWARDED_PROTO 'https' diff --git a/puppet/modules/site_static/templates/rack.erb b/puppet/modules/site_static/templates/rack.erb index aae91f1c..431778bb 100644 --- a/puppet/modules/site_static/templates/rack.erb +++ b/puppet/modules/site_static/templates/rack.erb @@ -1,21 +1,19 @@ #PassengerLogLevel 1 #PassengerAppEnv production #PassengerFriendlyErrorPages on -<%- if @location_path == '' -%> - <Directory "<%=@directory%>"> - Order deny,allow - Allow from all - Options -MultiViews - </Directory> -<%- else -%> +<%- if @location_path != '' -%> Alias /<%=@location_path%> "<%=@directory%>" <Location /<%=@location_path%>> PassengerBaseURI /<%=@location_path%> PassengerAppRoot "<%=File.dirname(@directory)%>" </Location> +<%- end -%> <Directory "<%=@directory%>"> + Options -MultiViews +<% if scope.function_guess_apache_version([]) == '2.4' %> + Require all granted +<% else %> Order deny,allow Allow from all - Options -MultiViews +<% end %> </Directory> -<%- end -%> diff --git a/puppet/modules/site_stunnel/manifests/init.pp b/puppet/modules/site_stunnel/manifests/init.pp index d919a072..a874721f 100644 --- a/puppet/modules/site_stunnel/manifests/init.pp +++ b/puppet/modules/site_stunnel/manifests/init.pp @@ -36,8 +36,8 @@ class site_stunnel { # the default is to keep 356 log files for each stunnel. # here we set a more reasonable number. augeas { - "logrotate_stunnel": - context => "/files/etc/logrotate.d/stunnel4/rule", + 'logrotate_stunnel': + context => '/files/etc/logrotate.d/stunnel4/rule', changes => [ 'set rotate 5', ] diff --git a/puppet/modules/site_stunnel/manifests/override_service.pp b/puppet/modules/site_stunnel/manifests/override_service.pp index 96187048..435b9aa0 100644 --- a/puppet/modules/site_stunnel/manifests/override_service.pp +++ b/puppet/modules/site_stunnel/manifests/override_service.pp @@ -1,4 +1,9 @@ +# override stunnel::debian defaults +# +# ignore puppet lint error about inheriting from different namespace +# lint:ignore:inherits_across_namespaces class site_stunnel::override_service inherits stunnel::debian { +# lint:endignore include site_config::x509::cert include site_config::x509::key diff --git a/puppet/modules/site_stunnel/manifests/servers.pp b/puppet/modules/site_stunnel/manifests/servers.pp index b6fac319..e76d1e9d 100644 --- a/puppet/modules/site_stunnel/manifests/servers.pp +++ b/puppet/modules/site_stunnel/manifests/servers.pp @@ -16,6 +16,8 @@ define site_stunnel::servers ( $rndfile = '/var/lib/stunnel4/.rnd', $debuglevel = '4' ) { + $logfile = "/var/log/stunnel4/${name}.log" + include site_config::x509::cert include site_config::x509::key include site_config::x509::ca @@ -35,7 +37,9 @@ define site_stunnel::servers ( pid => "/var/run/stunnel4/${pid}.pid", rndfile => '/var/lib/stunnel4/.rnd', debuglevel => $debuglevel, - sslversion => 'TLSv1'; + sslversion => 'TLSv1', + syslog => 'no', + output => $logfile; } # allow incoming connections on $accept_port diff --git a/puppet/modules/site_tor/manifests/disable_exit.pp b/puppet/modules/site_tor/manifests/disable_exit.pp index 73016646..078f80ae 100644 --- a/puppet/modules/site_tor/manifests/disable_exit.pp +++ b/puppet/modules/site_tor/manifests/disable_exit.pp @@ -1,7 +1,7 @@ class site_tor::disable_exit { tor::daemon::exit_policy { 'no_exit_at_all': - reject => '*:*'; + reject => [ '*:*' ]; } } diff --git a/puppet/modules/site_tor/manifests/init.pp b/puppet/modules/site_tor/manifests/init.pp index 80ccc5d3..2207a5a9 100644 --- a/puppet/modules/site_tor/manifests/init.pp +++ b/puppet/modules/site_tor/manifests/init.pp @@ -19,6 +19,7 @@ class site_tor { $openvpn_ports = [] } + include site_config::default include tor::daemon tor::daemon::relay { $nickname: port => 9001, diff --git a/puppet/modules/site_webapp/files/server-status.conf b/puppet/modules/site_webapp/files/server-status.conf new file mode 100644 index 00000000..10b2d4ed --- /dev/null +++ b/puppet/modules/site_webapp/files/server-status.conf @@ -0,0 +1,26 @@ +# Keep track of extended status information for each request +ExtendedStatus On + +# Determine if mod_status displays the first 63 characters of a request or +# the last 63, assuming the request itself is greater than 63 chars. +# Default: Off +#SeeRequestTail On + +Listen 127.0.0.1:8162 + +<VirtualHost 127.0.0.1:8162> + +<Location /server-status> + SetHandler server-status + Require all granted + Allow from 127.0.0.1 +</Location> + +</VirtualHost> + + +<IfModule mod_proxy.c> + # Show Proxy LoadBalancer status in mod_status + ProxyStatus On +</IfModule> + diff --git a/puppet/modules/site_webapp/manifests/apache.pp b/puppet/modules/site_webapp/manifests/apache.pp index 93e172a0..80c7b29b 100644 --- a/puppet/modules/site_webapp/manifests/apache.pp +++ b/puppet/modules/site_webapp/manifests/apache.pp @@ -1,3 +1,4 @@ +# configure apache and passenger to serve the webapp class site_webapp::apache { $web_api = hiera('api') @@ -11,16 +12,17 @@ class site_webapp::apache { $webapp_domain = $webapp['domain'] include site_apache::common - include site_apache::module::headers - include site_apache::module::alias - include site_apache::module::expires - include site_apache::module::removeip + include apache::module::headers + include apache::module::alias + include apache::module::expires + include apache::module::removeip + include site_webapp::common_vhost class { 'passenger': use_munin => false } apache::vhost::file { 'api': - content => template('site_apache/vhosts.d/api.conf.erb') + content => template('site_apache/vhosts.d/api.conf.erb'); } } diff --git a/puppet/modules/site_webapp/manifests/common_vhost.pp b/puppet/modules/site_webapp/manifests/common_vhost.pp new file mode 100644 index 00000000..c57aad57 --- /dev/null +++ b/puppet/modules/site_webapp/manifests/common_vhost.pp @@ -0,0 +1,18 @@ +class site_webapp::common_vhost { + # installs x509 cert + key and common config + # that both nagios + leap webapp use + + include x509::variables + include site_config::x509::commercial::cert + include site_config::x509::commercial::key + include site_config::x509::commercial::ca + + Class['Site_config::X509::Commercial::Key'] ~> Service[apache] + Class['Site_config::X509::Commercial::Cert'] ~> Service[apache] + Class['Site_config::X509::Commercial::Ca'] ~> Service[apache] + + apache::vhost::file { + 'common': + content => template('site_apache/vhosts.d/common.conf.erb') + } +} diff --git a/puppet/modules/site_webapp/manifests/couchdb.pp b/puppet/modules/site_webapp/manifests/couchdb.pp index 1dbc745d..71450370 100644 --- a/puppet/modules/site_webapp/manifests/couchdb.pp +++ b/puppet/modules/site_webapp/manifests/couchdb.pp @@ -14,29 +14,36 @@ class site_webapp::couchdb { file { '/srv/leap/webapp/config/couchdb.yml': content => template('site_webapp/couchdb.yml.erb'), - owner => leap-webapp, - group => leap-webapp, + owner => 'leap-webapp', + group => 'leap-webapp', mode => '0600', require => Vcsrepo['/srv/leap/webapp']; + # couchdb.admin.yml is a symlink to prevent the vcsrepo resource + # from changing its user permissions every time. '/srv/leap/webapp/config/couchdb.admin.yml': + ensure => 'link', + target => '/etc/leap/couchdb.admin.yml', + require => Vcsrepo['/srv/leap/webapp']; + + '/etc/leap/couchdb.admin.yml': content => template('site_webapp/couchdb.admin.yml.erb'), - owner => leap-webapp, - group => leap-webapp, + owner => 'root', + group => 'root', mode => '0600', - require => Vcsrepo['/srv/leap/webapp']; + require => File['/etc/leap']; '/srv/leap/webapp/log': ensure => directory, - owner => leap-webapp, - group => leap-webapp, + owner => 'leap-webapp', + group => 'leap-webapp', mode => '0755', require => Vcsrepo['/srv/leap/webapp']; '/srv/leap/webapp/log/production.log': ensure => present, - owner => leap-webapp, - group => leap-webapp, + owner => 'leap-webapp', + group => 'leap-webapp', mode => '0666', require => Vcsrepo['/srv/leap/webapp']; } diff --git a/puppet/modules/site_webapp/manifests/cron.pp b/puppet/modules/site_webapp/manifests/cron.pp index d26ee312..70b9da04 100644 --- a/puppet/modules/site_webapp/manifests/cron.pp +++ b/puppet/modules/site_webapp/manifests/cron.pp @@ -1,3 +1,4 @@ +# setup webapp cronjobs class site_webapp::cron { # cron tasks that need to be performed to cleanup the database @@ -5,27 +6,31 @@ class site_webapp::cron { 'rotate_databases': command => 'cd /srv/leap/webapp && bundle exec rake db:rotate', environment => 'RAILS_ENV=production', + user => 'root', hour => [0,6,12,18], minute => 0; 'delete_tmp_databases': command => 'cd /srv/leap/webapp && bundle exec rake db:deletetmp', environment => 'RAILS_ENV=production', + user => 'root', hour => 1, minute => 1; # there is no longer a need to remove expired sessions, since the database # will get destroyed. 'remove_expired_sessions': + ensure => absent, command => 'cd /srv/leap/webapp && bundle exec rake cleanup:sessions', environment => 'RAILS_ENV=production', + user => 'leap-webapp', hour => 2, - minute => 30, - ensure => absent; + minute => 30; 'remove_expired_tokens': command => 'cd /srv/leap/webapp && bundle exec rake cleanup:tokens', environment => 'RAILS_ENV=production', + user => 'leap-webapp', hour => 3, minute => 0; } diff --git a/puppet/modules/site_webapp/manifests/hidden_service.pp b/puppet/modules/site_webapp/manifests/hidden_service.pp index 16b6e2e7..72a2ce95 100644 --- a/puppet/modules/site_webapp/manifests/hidden_service.pp +++ b/puppet/modules/site_webapp/manifests/hidden_service.pp @@ -4,13 +4,13 @@ class site_webapp::hidden_service { $tor_domain = "${hidden_service['address']}.onion" include site_apache::common - include site_apache::module::headers - include site_apache::module::alias - include site_apache::module::expires - include site_apache::module::removeip - + include apache::module::headers + include apache::module::alias + include apache::module::expires + include apache::module::removeip + include tor::daemon - tor::daemon::hidden_service { 'webapp': ports => '80 127.0.0.1:80' } + tor::daemon::hidden_service { 'webapp': ports => [ '80 127.0.0.1:80'] } file { '/var/lib/tor/webapp/': @@ -34,10 +34,19 @@ class site_webapp::hidden_service { mode => '0600'; } + # it is necessary to zero out the config of the status module + # because we are configuring our own version that is unavailable + # over the hidden service (see: #7456 and #7776) + apache::module { 'status': ensure => present, conf_content => ' ' } + # the access_compat module is required to enable Allow directives + apache::module { 'access_compat': ensure => present } + apache::vhost::file { 'hidden_service': - content => template('site_apache/vhosts.d/hidden_service.conf.erb') + content => template('site_apache/vhosts.d/hidden_service.conf.erb'); + 'server_status': + vhost_source => 'modules/site_webapp/server-status.conf'; } include site_shorewall::tor -}
\ No newline at end of file +} diff --git a/puppet/modules/site_webapp/manifests/init.pp b/puppet/modules/site_webapp/manifests/init.pp index ec94c090..15925aba 100644 --- a/puppet/modules/site_webapp/manifests/init.pp +++ b/puppet/modules/site_webapp/manifests/init.pp @@ -1,3 +1,4 @@ +# configure webapp service class site_webapp { tag 'leap_service' $definition_files = hiera('definition_files') @@ -20,11 +21,16 @@ class site_webapp { include site_webapp::couchdb include site_haproxy include site_webapp::cron + include site_config::default include site_config::x509::cert include site_config::x509::key include site_config::x509::ca include site_config::x509::client_ca::ca include site_config::x509::client_ca::key + include site_nickserver + + # remove leftovers from previous installations on webapp nodes + include site_config::remove::webapp group { 'leap-webapp': ensure => present, @@ -54,7 +60,7 @@ class site_webapp { exec { 'bundler_update': cwd => '/srv/leap/webapp', - command => '/bin/bash -c "/usr/bin/bundle check --path vendor/bundle || /usr/bin/bundle install --path vendor/bundle --without test development"', + command => '/bin/bash -c "/usr/bin/bundle check --path vendor/bundle || /usr/bin/bundle install --path vendor/bundle --without test development debug"', unless => '/usr/bin/bundle check --path vendor/bundle', user => 'leap-webapp', timeout => 600, @@ -163,10 +169,8 @@ class site_webapp { # needed for the soledad-sync check which is run on the - # webapp node (#6520) - package { 'python-u1db': - ensure => latest, - } + # webapp node + include soledad::client leap::logfile { 'webapp': } diff --git a/puppet/modules/site_webapp/templates/config.yml.erb b/puppet/modules/site_webapp/templates/config.yml.erb index ccde2d2e..dd55d3e9 100644 --- a/puppet/modules/site_webapp/templates/config.yml.erb +++ b/puppet/modules/site_webapp/templates/config.yml.erb @@ -1,28 +1,36 @@ -<%- require 'json' -%> -<%- cert_options = @webapp['client_certificates'] -%> -production: - admins: <%= @webapp['admins'].inspect %> - domain: <%= @provider_domain %> - force_ssl: <%= @webapp['secure'] %> - client_ca_key: <%= scope.lookupvar('x509::variables::keys') %>/<%= scope.lookupvar('site_config::params::client_ca_name') %>.key - client_ca_cert: <%= scope.lookupvar('x509::variables::local_CAs') %>/<%= scope.lookupvar('site_config::params::client_ca_name') %>.crt - secret_token: "<%= @secret_token %>" - client_cert_lifespan: <%= cert_options['life_span'] %> - client_cert_bit_size: <%= cert_options['bit_size'].to_i %> - client_cert_hash: <%= cert_options['digest'] %> - allow_limited_certs: <%= @webapp['allow_limited_certs'].inspect %> - allow_unlimited_certs: <%= @webapp['allow_unlimited_certs'].inspect %> - allow_anonymous_certs: <%= @webapp['allow_anonymous_certs'].inspect %> - limited_cert_prefix: "<%= cert_options['limited_prefix'] %>" - unlimited_cert_prefix: "<%= cert_options['unlimited_prefix'] %>" - minimum_client_version: "<%= @webapp['client_version']['min'] %>" - default_service_level: "<%= @webapp['default_service_level'] %>" - service_levels: <%= scope.function_sorted_json([@webapp['service_levels']]) %> - allow_registration: <%= @webapp['allow_registration'].inspect %> - handle_blacklist: <%= @webapp['forbidden_usernames'].inspect %> -<%- if @webapp['engines'] && @webapp['engines'].any? -%> - engines: -<%- @webapp['engines'].each do |engine| -%> - - <%= engine %> -<%- end -%> -<%- end -%> +<% +cert_options = @webapp['client_certificates'] +production = { + "admins" => @webapp['admins'], + "default_locale" => @webapp['default_locale'], + "available_locales" => @webapp['locales'], + "domain" => @provider_domain, + "force_ssl" => @webapp['secure'], + "client_ca_key" => "%s/%s.key" % [scope.lookupvar('x509::variables::keys'), scope.lookupvar('site_config::params::client_ca_name')], + "client_ca_cert" => "%s/%s.crt" % [scope.lookupvar('x509::variables::local_CAs'), scope.lookupvar('site_config::params::client_ca_name')], + "secret_token" => @secret_token, + "client_cert_lifespan" => cert_options['life_span'], + "client_cert_bit_size" => cert_options['bit_size'].to_i, + "client_cert_hash" => cert_options['digest'], + "allow_limited_certs" => @webapp['allow_limited_certs'], + "allow_unlimited_certs" => @webapp['allow_unlimited_certs'], + "allow_anonymous_certs" => @webapp['allow_anonymous_certs'], + "limited_cert_prefix" => cert_options['limited_prefix'], + "unlimited_cert_prefix" => cert_options['unlimited_prefix'], + "minimum_client_version" => @webapp['client_version']['min'], + "default_service_level" => @webapp['default_service_level'], + "service_levels" => @webapp['service_levels'], + "allow_registration" => @webapp['allow_registration'], + "handle_blacklist" => @webapp['forbidden_usernames'], + "invite_required" => @webapp['invite_required'], + "api_tokens" => @webapp['api_tokens'] +} + +if @webapp['engines'] && @webapp['engines'].any? + production["engines"] = @webapp['engines'] +end +-%> +# +# This file is generated by puppet. This file inherits from defaults.yml. +# +<%= scope.function_sorted_yaml([{"production" => production}]) %> diff --git a/puppet/modules/soledad/manifests/client.pp b/puppet/modules/soledad/manifests/client.pp new file mode 100644 index 00000000..e470adeb --- /dev/null +++ b/puppet/modules/soledad/manifests/client.pp @@ -0,0 +1,16 @@ +# setup soledad-client +# currently needed on webapp node to run the soledad-sync test +class soledad::client { + + tag 'leap_service' + include soledad::common + + package { + 'soledad-client': + ensure => latest, + require => Class['site_apt::leap_repo']; + 'python-u1db': + ensure => latest; + } + +} diff --git a/puppet/modules/soledad/manifests/common.pp b/puppet/modules/soledad/manifests/common.pp index 8a1d664a..8d8339d4 100644 --- a/puppet/modules/soledad/manifests/common.pp +++ b/puppet/modules/soledad/manifests/common.pp @@ -1,10 +1,8 @@ +# install soledad-common, both needed both soledad-client and soledad-server class soledad::common { - include soledad - package { 'soledad-common': - ensure => latest, - require => User['soledad'] + ensure => latest; } } diff --git a/puppet/modules/soledad/manifests/init.pp b/puppet/modules/soledad/manifests/init.pp deleted file mode 100644 index 7cf0b729..00000000 --- a/puppet/modules/soledad/manifests/init.pp +++ /dev/null @@ -1,29 +0,0 @@ -class soledad { - - group { 'soledad': - ensure => present, - allowdupe => false; - } - - user { 'soledad': - ensure => present, - allowdupe => false, - gid => 'soledad', - home => '/srv/leap/soledad', - require => Group['soledad']; - } - - file { - '/srv/leap/soledad': - ensure => directory, - owner => 'soledad', - group => 'soledad', - require => User['soledad']; - - '/var/lib/soledad': - ensure => directory, - owner => 'soledad', - group => 'soledad', - require => User['soledad']; - } -} diff --git a/puppet/modules/soledad/manifests/server.pp b/puppet/modules/soledad/manifests/server.pp index b71fab69..8674f421 100644 --- a/puppet/modules/soledad/manifests/server.pp +++ b/puppet/modules/soledad/manifests/server.pp @@ -1,11 +1,14 @@ +# setup soledad-server class soledad::server { tag 'leap_service' - include soledad - include site_apt::preferences::twisted - $soledad = hiera('soledad') - $couchdb_user = $soledad['couchdb_soledad_user']['username'] - $couchdb_password = $soledad['couchdb_soledad_user']['password'] + include site_config::default + include soledad::common + + $soledad = hiera('soledad') + $couchdb_user = $soledad['couchdb_soledad_user']['username'] + $couchdb_password = $soledad['couchdb_soledad_user']['password'] + $couchdb_leap_mx_user = $soledad['couchdb_leap_mx_user']['username'] $couchdb_host = 'localhost' $couchdb_port = '5984' @@ -22,20 +25,34 @@ class soledad::server { # SOLEDAD CONFIG # - file { '/etc/leap/soledad-server.conf': - content => template('soledad/soledad-server.conf.erb'), - owner => 'soledad', - group => 'soledad', - mode => '0600', - notify => Service['soledad-server'], - require => Class['soledad']; + file { + '/etc/soledad': + ensure => directory, + owner => 'root', + group => 'root', + mode => '0755'; + '/etc/soledad/soledad-server.conf': + content => template('soledad/soledad-server.conf.erb'), + owner => 'soledad', + group => 'soledad', + mode => '0640', + notify => Service['soledad-server'], + require => [ User['soledad'], Group['soledad'] ]; + '/srv/leap/soledad': + ensure => directory, + owner => 'soledad', + group => 'soledad', + require => [ User['soledad'], Group['soledad'] ]; + '/var/lib/soledad': + ensure => directory, + owner => 'soledad', + group => 'soledad', + require => [ User['soledad'], Group['soledad'] ]; } package { $sources['soledad']['package']: ensure => $sources['soledad']['revision'], - require => [ - Class['site_apt::preferences::twisted'], - Class['site_apt::leap_repo'] ]; + require => Class['site_apt::leap_repo']; } file { '/etc/default/soledad': @@ -44,7 +61,7 @@ class soledad::server { group => 'soledad', mode => '0600', notify => Service['soledad-server'], - require => Class['soledad']; + require => [ User['soledad'], Group['soledad'] ]; } service { 'soledad-server': @@ -52,7 +69,7 @@ class soledad::server { enable => true, hasstatus => true, hasrestart => true, - require => Class['soledad'], + require => [ User['soledad'], Group['soledad'] ], subscribe => [ Package['soledad-server'], Class['Site_config::X509::Key'], @@ -62,4 +79,26 @@ class soledad::server { include site_shorewall::soledad include site_check_mk::agent::soledad + + # set up users, group and directories for soledad-server + # although the soledad users are already created by the + # soledad-server package + group { 'soledad': + ensure => present, + system => true, + } + user { + 'soledad': + ensure => present, + system => true, + gid => 'soledad', + home => '/srv/leap/soledad', + require => Group['soledad']; + 'soledad-admin': + ensure => present, + system => true, + gid => 'soledad', + home => '/srv/leap/soledad', + require => Group['soledad']; + } } diff --git a/puppet/modules/soledad/templates/soledad-server.conf.erb b/puppet/modules/soledad/templates/soledad-server.conf.erb index 47d1f6e4..1c6a0d19 100644 --- a/puppet/modules/soledad/templates/soledad-server.conf.erb +++ b/puppet/modules/soledad/templates/soledad-server.conf.erb @@ -1,3 +1,12 @@ [soledad-server] -couch_url = http://<%= @couchdb_user %>:<%= @couchdb_password %>@<%= @couchdb_host %>:<%= @couchdb_port %> +couch_url = http://<%= @couchdb_user %>:<%= @couchdb_password %>@<%= @couchdb_host %>:<%= @couchdb_port %> +create_cmd = sudo -u soledad-admin /usr/bin/create-user-db +admin_netrc = /etc/couchdb/couchdb-soledad-admin.netrc + +[database-security] +members = <%= @couchdb_user %>, <%= @couchdb_leap_mx_user %> +# not needed, but for documentation: +# members_roles = replication +# admins = admin +# admins_roles = replication diff --git a/puppet/modules/sshd b/puppet/modules/sshd -Subproject 750a497758d94c2f5a6cad23cecc3dbde2d2f92 +Subproject 76f4f872f81209a52df2205fd88b5619df58f00 diff --git a/puppet/modules/stunnel b/puppet/modules/stunnel -Subproject b0dc7c84b5f55aec12d7d65da812037913d9dbe +Subproject 79e874c1a86ad5c48c4e726a5d4c68bd879ce45 diff --git a/puppet/modules/systemd b/puppet/modules/systemd new file mode 160000 +Subproject 6d47fd4999fe03eba6fb11c4490dcbb90d93790 diff --git a/puppet/modules/tapicero/files/tapicero.init b/puppet/modules/tapicero/files/tapicero.init deleted file mode 100755 index 7a9af45f..00000000 --- a/puppet/modules/tapicero/files/tapicero.init +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh - -### BEGIN INIT INFO -# Provides: tapicero -# Required-Start: $remote_fs $syslog -# Required-Stop: $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: tapicero initscript -# Description: Controls tapicero daemon -### END INIT INFO - -PATH=/sbin:/usr/sbin:/bin:/usr/bin -BUNDLER=/usr/bin/bundle -NAME=tapicero -HOME="/srv/leap" -DAEMON="${HOME}/${NAME}/bin/${NAME}" -BUNDLE_GEMFILE="${HOME}/${NAME}/Gemfile" - -export BUNDLE_GEMFILE - -# exit if the daemon doesn't exist -[ -x "$DAEMON" ] || exit 0 - -. /lib/init/vars.sh -. /lib/lsb/init-functions - -if [ "$VERBOSE" != no ]; then - OPTIONS="--verbose" -else - OPTIONS="" -fi - -case "$1" in - start) - $BUNDLER exec $DAEMON start $OPTIONS - exit $? - ;; - stop) - $BUNDLER exec $DAEMON stop $OPTIONS - exit $? - ;; - restart) - $BUNDLER exec $DAEMON restart $OPTIONS - exit $? - ;; - reload) - $BUNDLER exec $DAEMON reload $OPTIONS - exit $? - ;; - status) - $BUNDLER exec $DAEMON status $OPTIONS - exit $? - ;; - *) - echo "Usage: /etc/init.d/$NAME {start|stop|reload|restart|status}" - exit 1 -esac - -exit 0 diff --git a/puppet/modules/tapicero/manifests/init.pp b/puppet/modules/tapicero/manifests/init.pp deleted file mode 100644 index ca8488c8..00000000 --- a/puppet/modules/tapicero/manifests/init.pp +++ /dev/null @@ -1,137 +0,0 @@ -class tapicero { - tag 'leap_service' - - $couchdb = hiera('couch') - $couchdb_port = $couchdb['port'] - - $couchdb_users = $couchdb['users'] - - $couchdb_admin_user = $couchdb_users['admin']['username'] - $couchdb_admin_password = $couchdb_users['admin']['password'] - - $couchdb_soledad_user = $couchdb_users['soledad']['username'] - $couchdb_leap_mx_user = $couchdb_users['leap_mx']['username'] - - $couchdb_mode = $couchdb['mode'] - $couchdb_replication = $couchdb['replication'] - - $sources = hiera('sources') - - Class['site_config::default'] -> Class['tapicero'] - - include site_config::ruby::dev - - # - # USER AND GROUP - # - - group { 'tapicero': - ensure => present, - allowdupe => false; - } - - user { 'tapicero': - ensure => present, - allowdupe => false, - gid => 'tapicero', - home => '/srv/leap/tapicero', - require => Group['tapicero']; - } - - # - # TAPICERO FILES - # - - file { - - # - # TAPICERO DIRECTORIES - # - - '/srv/leap/tapicero': - ensure => directory, - owner => 'tapicero', - group => 'tapicero', - require => User['tapicero']; - - '/var/lib/leap/tapicero': - ensure => directory, - owner => 'tapicero', - group => 'tapicero', - require => User['tapicero']; - - # for pid file - '/var/run/tapicero': - ensure => directory, - owner => 'tapicero', - group => 'tapicero', - require => User['tapicero']; - - # - # TAPICERO CONFIG - # - - '/etc/leap/tapicero.yaml': - content => template('tapicero/tapicero.yaml.erb'), - owner => 'tapicero', - group => 'tapicero', - mode => '0600', - notify => Service['tapicero']; - - # - # TAPICERO INIT - # - - '/etc/init.d/tapicero': - source => 'puppet:///modules/tapicero/tapicero.init', - owner => root, - group => 0, - mode => '0755', - require => Vcsrepo['/srv/leap/tapicero']; - } - - # - # TAPICERO CODE - # - - vcsrepo { '/srv/leap/tapicero': - ensure => present, - force => true, - revision => $sources['tapicero']['revision'], - provider => $sources['tapicero']['type'], - source => $sources['tapicero']['source'], - owner => 'tapicero', - group => 'tapicero', - require => [ User['tapicero'], Group['tapicero'] ], - notify => Exec['tapicero_bundler_update'] - } - - exec { 'tapicero_bundler_update': - cwd => '/srv/leap/tapicero', - command => '/bin/bash -c "/usr/bin/bundle check || /usr/bin/bundle install --path vendor/bundle --without test development"', - unless => '/usr/bin/bundle check', - user => 'tapicero', - timeout => 600, - require => [ - Class['bundler::install'], - Vcsrepo['/srv/leap/tapicero'], - Class['site_config::ruby::dev'] ], - notify => Service['tapicero']; - } - - # - # TAPICERO DAEMON - # - - service { 'tapicero': - ensure => running, - enable => true, - hasstatus => false, - hasrestart => true, - require => [ File['/etc/init.d/tapicero'], - File['/var/run/tapicero'], - Couchdb::Add_user[$::site_couchdb::couchdb_tapicero_user] ]; - } - - leap::logfile { 'tapicero': } -} diff --git a/puppet/modules/tapicero/templates/tapicero.yaml.erb b/puppet/modules/tapicero/templates/tapicero.yaml.erb deleted file mode 100644 index 8b08b49c..00000000 --- a/puppet/modules/tapicero/templates/tapicero.yaml.erb +++ /dev/null @@ -1,52 +0,0 @@ -<%- require 'json' -%> - -# -# Default configuration options for Tapicero -# - -# couch connection configuration -connection: - protocol: "http" - host: "localhost" - port: <%= @couchdb_port %> - username: <%= @couchdb_admin_user %> - password: <%= @couchdb_admin_password %> - prefix : "" - suffix : "" - netrc: "/etc/couchdb/couchdb.netrc" - -# file to store the last processed user record in so we can resume after -# a restart: -seq_dir: "/var/lib/leap/tapicero/" - -# Configure log_file like this if you want to log to a file instead of syslog: -#log_file: "/var/log/leap/tapicero.log" -#log_level: debug -log_level: info - -# tapicero specific options -options: - # prefix for per user databases: - db_prefix: "user-" - mode: <%= @couchdb_mode %> -<%- if @couchdb_replication %> - replication: <%= @couchdb_replication.to_json %> -<%- end -%> - - # security settings to be used for the per user databases - security: - admins: - names: - # We explicitly allow the admin user to access per user databases, even - # though admin access ignores per database security we just do this to be - # explicit about this - - <%= @couchdb_admin_user %> - roles: [] - members: - names: - - <%= @couchdb_soledad_user %> - - <%= @couchdb_leap_mx_user %> - roles: - - replication - - diff --git a/puppet/modules/tor b/puppet/modules/tor -Subproject dcb6e748864e7dfd3c14f4f2aba4c9120f12b78 +Subproject 8c936c166b6da1ebd0e8d95e56ceee5167357d6 diff --git a/puppet/modules/try/manifests/file.pp b/puppet/modules/try/manifests/file.pp index cd1bb035..2493d343 100644 --- a/puppet/modules/try/manifests/file.pp +++ b/puppet/modules/try/manifests/file.pp @@ -32,17 +32,17 @@ define try::file ( exec { "chmod_${name}": command => "/bin/chmod -R ${mode} '${name}'", - onlyif => "/usr/bin/test $mode", + onlyif => "/usr/bin/test ${mode}", refreshonly => true, loglevel => debug; "chown_${name}": command => "/bin/chown -R ${owner} '${name}'", - onlyif => "/usr/bin/test $owner", + onlyif => "/usr/bin/test ${owner}", refreshonly => true, loglevel => debug; "chgrp_${name}": command => "/bin/chgrp -R ${group} '${name}'", - onlyif => "/usr/bin/test $group", + onlyif => "/usr/bin/test ${group}", refreshonly => true, loglevel => debug; } @@ -50,31 +50,31 @@ define try::file ( if $target { exec { "symlink_${name}": command => "/bin/ln -s ${target} ${name}", - onlyif => "/usr/bin/test -d '${target}'", + onlyif => "/usr/bin/test -d '${target}'", } } elsif $source { if $ensure == 'directory' { if $purge { exec { "rsync_${name}": command => "/usr/bin/rsync -r --delete '${source}/' '${name}'", - onlyif => "/usr/bin/test -d '${source}'", - unless => "/usr/bin/diff -rq '${source}' '${name}'", - notify => [Exec["chmod_${name}"], Exec["chown_${name}"], Exec["chgrp_${name}"]] + onlyif => "/usr/bin/test -d '${source}'", + unless => "/usr/bin/diff -rq '${source}' '${name}'", + notify => [Exec["chmod_${name}"], Exec["chown_${name}"], Exec["chgrp_${name}"]] } } else { exec { "cp_r_${name}": command => "/bin/cp -r '${source}' '${name}'", - onlyif => "/usr/bin/test -d '${source}'", - unless => "/usr/bin/diff -rq '${source}' '${name}'", - notify => [Exec["chmod_${name}"], Exec["chown_${name}"], Exec["chgrp_${name}"]] + onlyif => "/usr/bin/test -d '${source}'", + unless => "/usr/bin/diff -rq '${source}' '${name}'", + notify => [Exec["chmod_${name}"], Exec["chown_${name}"], Exec["chgrp_${name}"]] } } } else { exec { "cp_${name}": command => "/bin/cp --remove-destination '${source}' '${name}'", - onlyif => "/usr/bin/test -e '${source}'", - unless => "/usr/bin/test ! -h '${name}' && /usr/bin/diff -q '${source}' '${name}'", - notify => [Exec["chmod_${name}"], Exec["chown_${name}"], Exec["chgrp_${name}"]] + onlyif => "/usr/bin/test -e '${source}'", + unless => "/usr/bin/test ! -h '${name}' && /usr/bin/diff -q '${source}' '${name}'", + notify => [Exec["chmod_${name}"], Exec["chown_${name}"], Exec["chgrp_${name}"]] } } } diff --git a/puppet/modules/unbound b/puppet/modules/unbound -Subproject 00646b0ffc71a86981b05f983c86ace0979d1b6 +Subproject a26b91dfea3189e6777629fa00d54f51dc41f4d diff --git a/puppet/modules/vcsrepo b/puppet/modules/vcsrepo -Subproject f92d09226cfddb0c7e5e342dd199d8ea05b497c +Subproject 4e23209eaccf1ab504d35158f4141b3053327c2 diff --git a/tests/helpers/bonafide_helper.rb b/tests/helpers/bonafide_helper.rb index 9b26eaaf..5b886228 100644 --- a/tests/helpers/bonafide_helper.rb +++ b/tests/helpers/bonafide_helper.rb @@ -17,45 +17,53 @@ class LeapTest raise exc end - def api_url(path) - api = property('api') - "https://%{domain}:%{port}#{path}" % { - :domain => api['domain'], - :port => api['port'] - } - end - # # attempts to create a user account via the API, # returning the user object if successful. # - def assert_create_user - user = SRP::User.new - url = api_url("/1/users.json") - assert_post(url, user.to_params) do |body| + def assert_create_user(username=nil, auth=nil) + user = SRP::User.new(username) + url = api_url("/users.json") + params = user.to_params + if auth + options = api_options(:auth => auth) + else + options = api_options + if property('webapp.invite_required') + @invite_code = generate_invite_code + params['user[invite_code]'] = @invite_code + end + end + + assert_post(url, params, options) do |body| assert response = JSON.parse(body), 'response should be JSON' assert response['ok'], "Creating a user should be successful, got #{response.inspect} instead." + user.ok = true + user.id = response['id'] end - user.ok = true return user end + # TODO: use the api for this instead. + def generate_invite_code + `cd /srv/leap/webapp/ && sudo -u leap-webapp RAILS_ENV=production bundle exec rake generate_invites[1]`.gsub(/\n/, "") + end + # # attempts to authenticate user. if successful, # user object is updated with id and session token. # def assert_authenticate_user(user) - url = api_url("/1/sessions.json") + url = api_url("/sessions.json") session = SRP::Session.new(user) params = {'login' => user.username, 'A' => session.aa} - assert_post(url, params) do |response, body| + assert_post(url, params, api_options) do |body, response| cookie = response['Set-Cookie'].split(';').first assert(response = JSON.parse(body), 'response should be JSON') assert(session.bb = response["B"], 'response should include "B"') - url = api_url("/1/sessions/login.json") + url = api_url("/sessions/login.json") params = {'client_auth' => session.m, 'A' => session.aa} - options = {:headers => {'Cookie' => cookie}} - assert_put(url, params, options) do |body| + assert_put(url, params, api_options('Cookie' => cookie)) do |body| assert(response = JSON.parse(body), 'response should be JSON') assert(response['M2'], 'response should include M2') user.session_token = response['token'] @@ -70,30 +78,158 @@ class LeapTest # attempts to destroy a user account via the API. # def assert_delete_user(user) - if user && user.ok && user.id && user.session_token && !user.deleted - url = api_url("/1/users/#{user.id}.json") - options = {:headers => { - "Authorization" => "Token token=\"#{user.session_token}\"" - }} - params = { - :identities => 'destroy' + if user.is_a? String + assert_delete_user_by_login(user) + elsif user.is_a? SRP::User + assert_delete_srp_user(user) + end + end + + # + # returns true if the identity exists, uses monitor token auth + # + def identity_exists?(address) + url = api_url("/identities/#{URI.encode(address)}.json") + options = {:ok_codes => [200, 404]}.merge( + api_options(:auth => :monitor) + ) + assert_get(url, nil, options) do |body, response| + return response.code == "200" + end + end + + def upload_public_key(user_id, public_key) + url = api_url("/users/#{user_id}.json") + params = {"user[public_key]" => public_key} + assert_put(url, params, api_options(:auth => :monitor)) + end + + # + # return user document as a Hash. uses monitor token auth + # + def find_user_by_id(user_id) + url = api_url("/users/#{user_id}.json") + assert_get(url, nil, api_options(:auth => :monitor)) do |body| + return JSON.parse(body) + end + end + + # + # return user document as a Hash. uses monitor token auth + # NOTE: this relies on deprecated behavior of the API + # and will not work when multi-domain support is added. + # + def find_user_by_login(login) + url = api_url("/users/0.json?login=#{login}") + options = {:ok_codes => [200, 404]}.merge( + api_options(:auth => :monitor) + ) + assert_get(url, nil, options) do |body, response| + if response.code == "200" + return JSON.parse(body) + else + return nil + end + end + end + + private + + def api_url(path) + unless path =~ /^\// + path = '/' + path + end + if property('testing.api_uri') + return property('testing.api_uri') + path + elsif property('api') + api = property('api') + return "https://%{domain}:%{port}/%{version}#{path}" % { + :domain => api['domain'], + :port => api['port'], + :version => api['version'] || 1 + } + else + fail 'This node needs to have either testing.api_url or api.{domain,port} configured.' + end + end + + # + # produces an options hash used for api http requests. + # + # argument options hash gets added to "headers" + # of the http request. + # + # special :auth key in argument will expand to + # add api_token_auth header. + # + # if you want to try manually: + # + # export API_URI=`grep api_uri /etc/leap/hiera.yaml | cut -d\" -f2` + # export TOKEN=`grep monitor_auth_token /etc/leap/hiera.yaml | awk '{print $2}'` + # curl -H "Accept: application/json" -H "Token: $TOKEN" $API_URI + # + def api_options(options={}) + # note: must be :headers, not "headers" + hsh = { + :headers => { + "Accept" => "application/json" } + } + if options[:auth] + hsh[:headers].merge!(api_token_auth(options.delete(:auth))) + end + hsh[:headers].merge!(options) + return hsh + end + + # + # add token authentication to a http request. + # + # returns a hash suitable for adding to the 'headers' option + # of an http function. + # + def api_token_auth(token) + if token.is_a?(Symbol) && property('testing') + if token == :monitor + token_str = property('testing.monitor_auth_token') + else + raise ArgumentError.new 'no such token' + end + else + token_str = token + end + {"Authorization" => "Token token=\"#{token_str}\""} + end + + # + # not actually used in any test, but useful when + # writing new tests. + # + def assert_delete_user_by_login(login_name) + user = find_user_by_login(login_name) + url = api_url("/users/#{user['id']}.json") + params = {:identities => 'destroy'} + delete(url, params, api_options(:auth => :monitor)) do |body, response, error| + assert error.nil?, "Error deleting user: #{error}" + assert response.code.to_i == 200, "Unable to delete user: HTTP response from API should have code 200, was #{response.code} #{error} #{body}" + assert(response = JSON.parse(body), 'Delete response should be JSON') + assert(response["success"], 'Deleting user should be a success') + end + end + + def assert_delete_srp_user(user) + if user && user.ok && user.id && user.session_token && !user.deleted + url = api_url("users/#{user.id}.json") + params = {:identities => 'destroy'} user.deleted = true - delete(url, params, options) do |body, response, error| + delete(url, params, api_options(:auth => user.session_token)) do |body, response, error| assert error.nil?, "Error deleting user: #{error}" assert response.code.to_i == 200, "Unable to delete user: HTTP response from API should have code 200, was #{response.code} #{error} #{body}" assert(response = JSON.parse(body), 'Delete response should be JSON') assert(response["success"], 'Deleting user should be a success') end - domain = property('domain.full_suffix') - identities_url = couchdb_url("/identities/_design/Identity/_view/by_address?key=%22#{user.username}@#{domain}%22") - get(identities_url) do |body, response, error| - assert error.nil?, "Error checking identities db: #{error}" - assert response.code.to_i == 200, "Unable to check that user identity was deleted: HTTP response from API should have code 200, was #{response.code} #{error} #{body}" - assert(response = JSON.parse(body), 'Couch response should be JSON') - assert response['rows'].empty?, "Identity should have been deleted for test user #{user.username} (id #{user.id}), but was not! Response was: #{body}." - end end end + end diff --git a/tests/helpers/client_side_db.py b/tests/helpers/client_side_db.py new file mode 100644 index 00000000..2f8c220f --- /dev/null +++ b/tests/helpers/client_side_db.py @@ -0,0 +1,167 @@ +import logging +import os +import tempfile +import getpass +import binascii +import json + +try: + import requests + import srp._pysrp as srp +except ImportError: + pass + +from twisted.internet.defer import inlineCallbacks + +from leap.soledad.client import Soledad + + +""" +Helper functions to give access to client-side Soledad database. +Copied over from soledad/scripts folder. +""" + +# create a logger +logger = logging.getLogger(__name__) + +# DEBUG: enable debug logs +# LOG_FORMAT = '%(asctime)s %(message)s' +# logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG) + + +safe_unhexlify = lambda x: binascii.unhexlify(x) if ( + len(x) % 2 == 0) else binascii.unhexlify('0' + x) + + +def _fail(reason): + logger.error('Fail: ' + reason) + exit(2) + + +def get_soledad_instance(uuid, passphrase, basedir, server_url, cert_file, + token): + # setup soledad info + logger.info('UUID is %s' % uuid) + logger.info('Server URL is %s' % server_url) + secrets_path = os.path.join( + basedir, '%s.secret' % uuid) + local_db_path = os.path.join( + basedir, '%s.db' % uuid) + # instantiate soledad + return Soledad( + uuid, + unicode(passphrase), + secrets_path=secrets_path, + local_db_path=local_db_path, + server_url=server_url, + cert_file=cert_file, + auth_token=token, + defer_encryption=True) + + +def _get_api_info(provider): + info = requests.get( + 'https://'+provider+'/provider.json', verify=False).json() + return info['api_uri'], info['api_version'] + + +def _login(username, passphrase, provider, api_uri, api_version): + usr = srp.User(username, passphrase, srp.SHA256, srp.NG_1024) + auth = None + try: + auth = _authenticate(api_uri, api_version, usr).json() + except requests.exceptions.ConnectionError: + _fail('Could not connect to server.') + if 'errors' in auth: + _fail(str(auth['errors'])) + return api_uri, api_version, auth + + +def _authenticate(api_uri, api_version, usr): + api_url = "%s/%s" % (api_uri, api_version) + session = requests.session() + uname, A = usr.start_authentication() + params = {'login': uname, 'A': binascii.hexlify(A)} + init = session.post( + api_url + '/sessions', data=params, verify=False).json() + if 'errors' in init: + _fail('test user not found') + M = usr.process_challenge( + safe_unhexlify(init['salt']), safe_unhexlify(init['B'])) + return session.put(api_url + '/sessions/' + uname, verify=False, + data={'client_auth': binascii.hexlify(M)}) + + +def _get_soledad_info(username, provider, passphrase, basedir): + api_uri, api_version = _get_api_info(provider) + auth = _login(username, passphrase, provider, api_uri, api_version) + # get soledad server url + service_url = '%s/%s/config/soledad-service.json' % \ + (api_uri, api_version) + soledad_hosts = requests.get(service_url, verify=False).json()['hosts'] + hostnames = soledad_hosts.keys() + # allow for choosing the host + host = hostnames[0] + if len(hostnames) > 1: + i = 1 + print "There are many available hosts:" + for h in hostnames: + print " (%d) %s.%s" % (i, h, provider) + i += 1 + choice = raw_input("Choose a host to use (default: 1): ") + if choice != '': + host = hostnames[int(choice) - 1] + server_url = 'https://%s:%d/user-%s' % \ + (soledad_hosts[host]['hostname'], soledad_hosts[host]['port'], + auth[2]['id']) + # get provider ca certificate + ca_cert = requests.get('https://%s/ca.crt' % provider, verify=False).text + cert_file = os.path.join(basedir, 'ca.crt') + with open(cert_file, 'w') as f: + f.write(ca_cert) + return auth[2]['id'], server_url, cert_file, auth[2]['token'] + + +def _get_passphrase(args): + passphrase = args.passphrase + if passphrase is None: + passphrase = getpass.getpass( + 'Password for %s@%s: ' % (args.username, args.provider)) + return passphrase + + +def _get_basedir(args): + basedir = args.basedir + if basedir is None: + basedir = tempfile.mkdtemp() + elif not os.path.isdir(basedir): + os.mkdir(basedir) + logger.info('Using %s as base directory.' % basedir) + return basedir + + +@inlineCallbacks +def _export_key(args, km, fname, private=False): + address = args.username + "@" + args.provider + pkey = yield km.get_key( + address, OpenPGPKey, private=private, fetch_remote=False) + with open(args.export_private_key, "w") as f: + f.write(pkey.key_data) + + +@inlineCallbacks +def _export_incoming_messages(soledad, directory): + yield soledad.create_index("by-incoming", "bool(incoming)") + docs = yield soledad.get_from_index("by-incoming", '1') + i = 1 + for doc in docs: + with open(os.path.join(directory, "message_%d.gpg" % i), "w") as f: + f.write(doc.content["_enc_json"]) + i += 1 + + +@inlineCallbacks +def _get_all_docs(soledad): + _, docs = yield soledad.get_all_docs() + for doc in docs: + print json.dumps(doc.content, indent=4) diff --git a/tests/helpers/couchdb_helper.rb b/tests/helpers/couchdb_helper.rb index d4d3c0e0..b9085c1e 100644 --- a/tests/helpers/couchdb_helper.rb +++ b/tests/helpers/couchdb_helper.rb @@ -15,6 +15,7 @@ class LeapTest # connect_port: 15984 # def couchdb_urls_via_stunnel(path="", options=nil) + path = path.gsub('"', '%22') if options && options[:username] && options[:password] userpart = "%{username}:%{password}@" % options else @@ -46,6 +47,7 @@ class LeapTest # writable: true # def couchdb_url_via_haproxy(path="", options=nil) + path = path.gsub('"', '%22') if options && options[:username] && options[:password] userpart = "%{username}:%{password}@" % options else @@ -66,6 +68,7 @@ class LeapTest # port: 5984 # def couchdb_url_via_localhost(path="", options=nil) + path = path.gsub('"', '%22') port = (options && options[:port]) || assert_property('couch.port') if options && options[:username] password = property("couch.users.%{username}.password" % options) @@ -100,4 +103,40 @@ class LeapTest end end + def assert_destroy_user_db(user_id, options=nil) + db_name = "user-#{user_id}" + url = couchdb_url("/#{db_name}", options) + http_options = {:ok_codes => [200, 404]} # ignore missing dbs + assert_delete(url, nil, http_options) + end + + def assert_create_user_db(user_id, options=nil) + db_name = "user-#{user_id}" + url = couchdb_url("/#{db_name}", options) + http_options = {:ok_codes => [200, 404]} # ignore missing dbs + assert_put(url, nil, :format => :json) do |body| + assert response = JSON.parse(body), "PUT response should be JSON" + assert response["ok"], "PUT response should be OK" + end + end + + # + # returns true if the per-user db created by soledad-server exists. + # + def user_db_exists?(user_id, options=nil) + db_name = "user-#{user_id}" + url = couchdb_url("/#{db_name}", options) + get(url) do |body, response, error| + if response.nil? + fail "could not query couchdb #{url}: #{error}\n#{body}" + elsif response.code.to_i == 200 + return true + elsif response.code.to_i == 404 + return false + else + fail ["could not query couchdb #{url}: expected response code 200 or 404, but got #{response.code}.", error, body].compact.join("\n") + end + end + end + end
\ No newline at end of file diff --git a/tests/helpers/http_helper.rb b/tests/helpers/http_helper.rb index 0b13b754..0d0bb7d5 100644 --- a/tests/helpers/http_helper.rb +++ b/tests/helpers/http_helper.rb @@ -81,19 +81,31 @@ class LeapTest # # calls http_send, yielding results if successful or failing with - # descriptive infor otherwise. + # descriptive info otherwise. + # + # options: + # - error_msg: custom error message to display. + # - ok_codes: in addition to 2xx, codes in this array will not produce an error. # def assert_http_send(method, url, params=nil, options=nil, &block) options ||= {} error_msg = options[:error_msg] || (url.respond_to?(:memo) ? url.memo : nil) http_send(method, url, params, options) do |body, response, error| - if body && response && response.code.to_i >= 200 && response.code.to_i < 300 - if block - yield(body) if block.arity == 1 - yield(response, body) if block.arity == 2 + if response + code = response.code.to_i + ok = code >= 200 && code < 300 + if options[:ok_codes] + ok ||= options[:ok_codes].include?(code) + end + if ok + if block + yield(body) if block.arity == 1 + yield(body, response) if block.arity == 2 + yield(body, response, error) if block.arity == 3 + end + else + fail ["Expected success code from #{method} #{url}, but got #{response.code} instead.", error_msg, body].compact.join("\n") end - elsif response - fail ["Expected a 200 status code from #{method} #{url}, but got #{response.code} instead.", error_msg, body].compact.join("\n") else fail ["Expected a response from #{method} #{url}, but got \"#{error}\" instead.", error_msg, body].compact.join("\n"), error end diff --git a/tests/helpers/os_helper.rb b/tests/helpers/os_helper.rb index aad67dda..da9ac843 100644 --- a/tests/helpers/os_helper.rb +++ b/tests/helpers/os_helper.rb @@ -9,7 +9,10 @@ class LeapTest output.each_line.map{|line| pid = line.split(' ')[0] process = line.gsub(/(#{pid} |\n)/, '') - if process =~ /pgrep --full --list-name/ + # filter out pgrep cmd itself + # on wheezy hosts, the "process" var contains the whole cmd including all parameters + # on jessie hosts, it only contains the first cmd (which is the default sheel invoked by 'sh') + if process =~ /^sh/ nil else {:pid => pid, :process => process} diff --git a/tests/helpers/smtp_helper.rb b/tests/helpers/smtp_helper.rb new file mode 100644 index 00000000..ea7fb9fa --- /dev/null +++ b/tests/helpers/smtp_helper.rb @@ -0,0 +1,45 @@ +require 'net/smtp' + +class LeapTest + + TEST_EMAIL_USER = "test_user_email" + TEST_BAD_USER = "test_user_bad" + + MSG_BODY = %(Since it seems that any heart which beats for freedom has the right only to a +lump of lead, I too claim my share. If you let me live, I shall never stop +crying for revenge and I shall avenge my brothers. I have finished. If you are +not cowards, kill me! + +--Louise Michel) + + def send_email(recipient, options={}) + sender = options[:sender] || recipient + helo_domain = property('domain.full_suffix') + headers = { + "Date" => Time.now.utc, + "From" => sender, + "To" => recipient, + "Subject" => "Test Message", + "X-LEAP-TEST" => "true" + }.merge(options[:headers]||{}) + message = [] + headers.each do |key, value| + message << "#{key}: #{value}" + end + message << "" + message << MSG_BODY + Net::SMTP.start('localhost', 25, helo_domain) do |smtp| + smtp.send_message message.join("\n"), recipient, sender + end + end + + def assert_send_email(recipient, options={}) + begin + send_email(recipient, options) + rescue IOError, Net::OpenTimeout, + Net::ReadTimeout, Net::SMTPError => e + fail "Could not send mail to #{recipient} (#{e})" + end + end + +end
\ No newline at end of file diff --git a/tests/helpers/soledad_sync.py b/tests/helpers/soledad_sync.py index 2fb865fc..f4fc81ae 100755 --- a/tests/helpers/soledad_sync.py +++ b/tests/helpers/soledad_sync.py @@ -1,78 +1,89 @@ #!/usr/bin/env python +""" +soledad_sync.py -# -# Test Soledad sync -# -# This script performs a slightly modified U1DB sync to the Soledad server and -# returns whether that sync was successful or not. -# -# It takes three arguments: -# -# uuid -- uuid of the user to sync -# token -- a valid session token -# server -- the url of the soledad server we should connect to -# -# For example: -# -# soledad_sync.py f6bef0586fcfdb8705e26a58f2d9e580 uYO-4ucEJFksJ6afjmcYwIyap2vW7bv6uLxk0w_RfCc https://199.119.112.9:2323/user-f6bef0586fcfdb8705e26a58f2d9e580 -# +This script exercises soledad synchronization. +Its exit code is 0 if the sync took place correctly, 1 otherwise. +It takes 5 arguments: + + uuid: uuid of the user to sync + token: a valid session token + server: the url of the soledad server we should connect to + cert_file: the file containing the certificate for the CA that signed the + cert for the soledad server. + password: the password for the user to sync + +__author__: kali@leap.se +""" import os +import shutil import sys -import traceback import tempfile -import shutil -import u1db -from u1db.remote.http_target import HTTPSyncTarget +# This is needed because the twisted shipped with wheezy is too old +# to do proper ssl verification. +os.environ['SKIP_TWISTED_SSL_CHECK'] = '1' + +from twisted.internet import defer, reactor +from twisted.python import log + +from client_side_db import get_soledad_instance +from leap.common.events import flags + +flags.set_events_enabled(False) + +NUMDOCS = 1 +USAGE = "Usage: %s uuid token server cert_file password" % sys.argv[0] -# -# monkey patch U1DB's HTTPSyncTarget to perform token based auth -# -def set_token_credentials(self, uuid, token): - self._creds = {'token': (uuid, token)} +def bail(msg, exitcode): + print "[!] %s" % msg + sys.exit(exitcode) -def _sign_request(self, method, url_query, params): - uuid, token = self._creds['token'] - auth = '%s:%s' % (uuid, token) - return [('Authorization', 'Token %s' % auth.encode('base64')[:-1])] -HTTPSyncTarget.set_token_credentials = set_token_credentials -HTTPSyncTarget._sign_request = _sign_request +def create_docs(soledad): + """ + Populates the soledad database with dummy messages, so we can exercise + sending payloads during the sync. + """ + deferreds = [] + for index in xrange(NUMDOCS): + deferreds.append(soledad.create_doc({'payload': 'dummy'})) + return defer.gatherResults(deferreds) -# -# Create a temporary local u1db replica and attempt to sync to it. -# Returns a failure message if something went wrong. -# +# main program + +if __name__ == '__main__': -def soledad_sync(uuid, token, server): tempdir = tempfile.mkdtemp() - try: - db = u1db.open(os.path.join(tempdir, '%s.db' % uuid), True) - creds = {'token': {'uuid': uuid, 'token': token}} - db.sync(server, creds=creds, autocreate=False) - finally: + + def rm_tempdir(): shutil.rmtree(tempdir) -# -# exit codes: -# -# 0 - OK -# 1 - WARNING -# 2 - ERROR -# + if len(sys.argv) < 6: + bail(USAGE, 2) -if __name__ == '__main__': - try: - uuid, token, server = sys.argv[1:] - result = soledad_sync(uuid, token, server) - if result is None: - exit(0) - else: - print(result) - exit(1) - except Exception as exc: - print(exc.message or str(exc)) - traceback.print_exc(file=sys.stdout) - exit(2) + uuid, token, server, cert_file, passphrase = sys.argv[1:] + s = get_soledad_instance( + uuid, passphrase, tempdir, server, cert_file, token) + + def onSyncDone(sync_result): + print "SYNC_RESULT:", sync_result + s.close() + rm_tempdir() + reactor.stop() + + def log_and_exit(f): + log.err(f) + rm_tempdir() + reactor.stop() + + def start_sync(): + d = create_docs(s) + d.addCallback(lambda _: s.sync()) + d.addCallback(onSyncDone) + d.addErrback(log_and_exit) + + reactor.callWhenRunning(start_sync) + reactor.run() diff --git a/tests/helpers/srp_helper.rb b/tests/helpers/srp_helper.rb index 5d30b459..b30fa768 100644 --- a/tests/helpers/srp_helper.rb +++ b/tests/helpers/srp_helper.rb @@ -138,8 +138,8 @@ d15dc7d7b46154d6b6ce8ef4ad69b15d4982559b297bcf1885c529f566660e5 attr_accessor :username, :password, :salt, :verifier, :id, :session_token, :ok, :deleted - def initialize - @username = "test_user_" + SecureRandom.urlsafe_base64(10).downcase.gsub(/[_-]/, '') + def initialize(username=nil) + @username = username || "tmp_user_" + SecureRandom.urlsafe_base64(10).downcase.gsub(/[_-]/, '') @password = "password_" + SecureRandom.urlsafe_base64(10) @salt = bigrand(4).hex @verifier = modpow(GENERATOR, private_key) diff --git a/tests/order.rb b/tests/order.rb index 4468686f..14aad9be 100644 --- a/tests/order.rb +++ b/tests/order.rb @@ -9,11 +9,14 @@ class LeapCli::Config::Node # def test_dependencies dependents = LeapCli::Config::ObjectList.new - unless services.include?('couchdb') - if services.include?('webapp') || services.include?('mx') || services.include?('soledad') + + # webapp, mx, and soledad depend on couchdb nodes + if services.include?('webapp') || services.include?('mx') || services.include?('soledad') + if !services.include?('couchdb') dependents.merge! nodes_like_me[:services => 'couchdb'] end end + dependents.keys.delete_if {|name| self.name == name} end end
\ No newline at end of file diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb index 5ee12ff3..85dc6840 100644 --- a/tests/white-box/couchdb.rb +++ b/tests/white-box/couchdb.rb @@ -9,9 +9,8 @@ class CouchDB < LeapTest end def test_00_Are_daemons_running? - assert_running '^tapicero', :single => true + assert_running 'bin/beam' if multimaster? - assert_running 'bin/beam' assert_running 'bin/epmd' end pass @@ -70,7 +69,7 @@ class CouchDB < LeapTest end def test_04_Do_ACL_users_exist? - acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'tapicero', 'webapp', 'replication'] + acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'webapp', 'replication'] url = couchdb_backend_url("/_users/_all_docs", :username => 'admin') assert_get(url) do |body| response = JSON.parse(body) @@ -128,6 +127,33 @@ class CouchDB < LeapTest pass end + # + # This is not really a "test", just an attempt to make sure that + # the mx tests that fire off dummy emails don't fill up the + # storage db. + # + # mx tests can't run this because they don't have access to + # the storage db. + # + # This "test" is responsible for both creating the db if it does not + # exist, and destroying if it does. + # + # Yes, this is super hacky. Properly, we should add something to + # the soledad api to support create/delete of user storage dbs. + # + def test_99_Delete_mail_storage_used_in_mx_tests + user = find_user_by_login(TEST_EMAIL_USER) + if user + if user_db_exists?(user["id"]) + # keep the test email db from filling up: + assert_destroy_user_db(user["id"], :username => 'admin') + end + # either way, make sure we leave a db for the mx tests: + assert_create_user_db(user["id"], :username => 'admin') + end + silent_pass + end + private def multimaster? diff --git a/tests/white-box/mx.rb b/tests/white-box/mx.rb index 794a9a41..6c0982ce 100644 --- a/tests/white-box/mx.rb +++ b/tests/white-box/mx.rb @@ -1,9 +1,11 @@ raise SkipTest unless service?(:mx) require 'json' +require 'net/smtp' class Mx < LeapTest depends_on "Network" + depends_on "Webapp" if service?(:webapp) def setup end @@ -11,7 +13,7 @@ class Mx < LeapTest def test_01_Can_contact_couchdb? dbs = ["identities"] dbs.each do |db_name| - couchdb_urls("/"+db_name, url_options).each do |url| + couchdb_urls("/"+db_name, couch_url_options).each do |url| assert_get(url) do |body| assert response = JSON.parse(body) assert_equal db_name, response['db_name'] @@ -23,7 +25,7 @@ class Mx < LeapTest def test_02_Can_contact_couchdb_via_haproxy? if property('haproxy.couch') - url = couchdb_url_via_haproxy("", url_options) + url = couchdb_url_via_haproxy("", couch_url_options) assert_get(url) do |body| assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message." end @@ -31,20 +33,154 @@ class Mx < LeapTest end end - def test_03_Are_MX_daemons_running? - assert_running 'leap_mx' - assert_running '/usr/lib/postfix/master' - assert_running '/usr/sbin/unbound' + # + # this test picks a random identity document, then queries + # using the by_address view for that same document again. + # + def test_03_Can_query_identities_db? + assert_get(couchdb_url("/identities", couch_url_options)) do |body| + assert response = JSON.parse(body) + doc_count = response['doc_count'].to_i + if doc_count <= 1 + # the design document counts as one document. + skip "There are no identity documents yet." + else + # try five times to get a valid doc + for i in 1..5 + offset = rand(doc_count) # pick a random document + count_url = couchdb_url("/identities/_all_docs?include_docs=true&limit=1&skip=#{offset}", couch_url_options) + assert_get(count_url) do |body| + assert response = JSON.parse(body) + record = response['rows'].first + if record['id'] =~ /_design/ + next + else + address = record['doc']['address'] + assert address, "Identity document #{record['id']} is missing an address field. #{record['doc'].inspect}" + url_base = %(/identities/_design/Identity/_view/by_address) + params = %(?include_docs=true&reduce=false&startkey="#{address}"&endkey="#{address}") + assert_get(couchdb_url(url_base+params, couch_url_options)) do |body| + assert response = JSON.parse(body) + assert record = response['rows'].first + assert_equal address, record['doc']['address'] + pass + end + break + end + end + end + end + end + end + + def test_04_Are_MX_daemons_running? + assert_running '.*/usr/bin/twistd.*mx.tac' + assert_running '^/usr/lib/postfix/master$' + assert_running '^/usr/sbin/postfwd' + assert_running 'postfwd2::cache$' + assert_running 'postfwd2::policy$' + assert_running '^/usr/sbin/unbound$' + assert_running '^/usr/bin/freshclam' + assert_running '^/usr/sbin/opendkim' + if Dir.glob("/var/lib/clamav/main.{c[vl]d,inc}").size > 0 and Dir.glob("/var/lib/clamav/daily.{c[vl]d,inc}").size > 0 + assert_running '^/usr/sbin/clamd' + assert_running '^/usr/sbin/clamav-milter' + else + skip "Downloading the clamav signature files (/var/lib/clamav/{daily,main}.{c[vl]d,inc}) is still in progress, so clamd is not running.\nDon't worry, mail delivery will work without clamav. The download should finish soon." + end + pass + end + + # + # The email sent by this test might get bounced back. + # In this case, the test will pass, but the bounce message will + # get sent to root, so the sysadmin will still figure out pretty + # quickly that something is wrong. + # + def test_05_Can_deliver_email? + addr = [TEST_EMAIL_USER, property('domain.full_suffix')].join('@') + bad_addr = [TEST_BAD_USER, property('domain.full_suffix')].join('@') + + assert !identity_exists?(bad_addr), "the address #{bad_addr} must not exist." + if !identity_exists?(addr) + user = assert_create_user(TEST_EMAIL_USER, :monitor) + upload_public_key(user.id, TEST_EMAIL_PUBLIC_KEY) + end + assert identity_exists?(addr), "The identity #{addr} should have been created, but it doesn't exist yet." + assert_send_email(addr) + assert_raises(Net::SMTPError) do + send_email(bad_addr) + end pass end private - def url_options + def couch_url_options { :username => property('couchdb_leap_mx_user.username'), :password => property('couchdb_leap_mx_user.password') } end + TEST_EMAIL_PUBLIC_KEY=<<HERE +-----BEGIN PGP PUBLIC KEY BLOCK----- +mI0EVvzIKQEEAN4f8FOGntJGTTD+fFUQS6y/ihn6tYLtyGZZbCOd0t/9kHt/raoR +xEUks8rCOPMqHX+yeHsvDBtDyZYTvyhtfuWrBUbYGW+QZ4Pdvo+7NyLHPW0dKsCB +Czrx7pxqpq1oq+LpUFqpSfjJTfYaGVDNXrPK144a7Rox2+MCbgq3twnFABEBAAG0 +EiA8dGVzdF91c2VyX2VtYWlsPoi4BBMBAgAiBQJW/MgpAhsvBgsJCAcDAgYVCAIJ +CgsEFgIDAQIeAQIXgAAKCRAqYf65XmeSk0orBADUXjEiGnjzyBpXqaiVmJr4MyfP +IfKTK4a+4qvR+2fseD7hteF98m26i1YRI5omLp4/MnxGSpgKFKIuWIdkEiLg7IJc +pFZVdoDVufEtzbj9gmOHlnteksbCtuESyB0Hytsba4uS9afcTJdGiPNMHeniI/SY +UKcCcIrQmpNIoOA5OLiNBFb8yCkBBAC+WMUQ+FC6GQ+pyaWlwTRsBAT4+Tp8w9jD +7PK4xeEmVZDirP0VkW18UeQEueWJ63ia7wIGf1WyVH1tbvgVyRLsjT2cpKo8c6Ok +NkhfGfjTnUJPeBNy8734UDIdqZLXJl0z6Z1R0CfOjBqvV25kWUvMkz/NEgZBhE+c +m3JuZy1k7QARAQABiQE9BBgBAgAJBQJW/MgpAhsuAKgJECph/rleZ5KTnSAEGQEC +AAYFAlb8yCkACgkQsJSYitQUOv4w1wQAn3atI5EsmRyw6iC6UVWWJv/lKi1Priyt +DsrdH5xUmHUgp6VU8Pw9Y6G+sv50KLfbVQ1l+8/3B71TjadsOxh+PBPsEyYpK6WX +TVGy44IDvFWGyOod8tmfcFN9IpU5DmSk/vny9G7RK/nbnta2VnfZOzwm5i3cNkPr +FGPL1z0K3qs0VwP+M7BXdqBRSFDDBpG1J0TrZioEjvKeOsT/Ul8mbVt7HQpcN93I +wTO4uky0Woy2nb7SbTQw6wOpU54u7+5dSQ03ltUHg1owy6Y3CMOeFL+e9ALpAZAU +aMwY7zMFhqlPVZZMfdMLRsdLin67RIM+OJ6A925AM52bEQT1YwkQlP4mvQY= +=qclE +-----END PGP PUBLIC KEY BLOCK----- +HERE + + TEST_EMAIL_PRIVATE_KEY = <<HERE +-----BEGIN PGP PRIVATE KEY BLOCK----- +lQHYBFb8yCkBBADeH/BThp7SRk0w/nxVEEusv4oZ+rWC7chmWWwjndLf/ZB7f62q +EcRFJLPKwjjzKh1/snh7LwwbQ8mWE78obX7lqwVG2BlvkGeD3b6Puzcixz1tHSrA +gQs68e6caqataKvi6VBaqUn4yU32GhlQzV6zyteOGu0aMdvjAm4Kt7cJxQARAQAB +AAP8DTFfcE6UG1AioJDU6KZ9oCaGONHLuxmNaArSofDrR/ODA9rLAUlp22N5LEdJ +46NyOhXrEwHx2aK2k+vbVDbgrP4ZTH7GxIK/2KzmH4zX0fWUNsaRy94Q12lJegXH +sH2Im8Jjxu16YwGgFNTX1fCPqLB6WdQpf1796s6+/3PnCDcCAOXTCul3N7V5Yl+9 +N2Anupn+qNDXKT/kiKIZLHsMbo7EriGWReG3lLj1cOJPC6Nf0uOEri4ErSjFEadR +F2TNITsCAPdsZjc5RGppUXyBfxhQkAnZ0r+UT2meCH3g3EVh3W9SBrXNhwipNpW3 +bPzRjUCDtmA8EOvd93oPCZv4/tb50P8B/jC+QIZ3GncP1CFPSVDoIZ7OUU5M1330 +DP77vG1GxeQvYO/hlxL5/KdtTR6m5zlIuooDxUaNJz1w5/oVjlG3NZKpl7QSIDx0 +ZXN0X3VzZXJfZW1haWw+iLgEEwECACIFAlb8yCkCGy8GCwkIBwMCBhUIAgkKCwQW +AgMBAh4BAheAAAoJECph/rleZ5KTSisEANReMSIaePPIGlepqJWYmvgzJ88h8pMr +hr7iq9H7Z+x4PuG14X3ybbqLVhEjmiYunj8yfEZKmAoUoi5Yh2QSIuDsglykVlV2 +gNW58S3NuP2CY4eWe16SxsK24RLIHQfK2xtri5L1p9xMl0aI80wd6eIj9JhQpwJw +itCak0ig4Dk4nQHYBFb8yCkBBAC+WMUQ+FC6GQ+pyaWlwTRsBAT4+Tp8w9jD7PK4 +xeEmVZDirP0VkW18UeQEueWJ63ia7wIGf1WyVH1tbvgVyRLsjT2cpKo8c6OkNkhf +GfjTnUJPeBNy8734UDIdqZLXJl0z6Z1R0CfOjBqvV25kWUvMkz/NEgZBhE+cm3Ju +Zy1k7QARAQABAAP9HrUaGvdpqTwVx3cHyXUhId6GzCuuKyaP4mZoGeBCcaQS2vQR +YtiykwBwX/AlfwSFJmmHKB6EErWIA+QyaEFR/fO56cHD2TY3Ql0BGcuHIx3+9pkp +biPBZdiiGz7oa6k6GWsbKSksqwV8poSXV7qbn+Bjm2xCM4VnjNZIrFtL7fkCAMOf +e9yHBFoXfc175bkNXEUXrNS34kv2ODAlx6KyY+PS77D+nprpHpGCnLn77G+xH1Xi +qvX1Dr/iSQU5Tzsd+tcCAPkYZulaC/9itwme7wIT3ur+mdqMHymsCzv9193iLgjJ +9t7fARo18yB845hI9Xv7TwRcoyuSpfvuM05rCMRzydsCAOI1MZeKtZSogXVa9QTX +sVGZeCkrujSVOgsA3w48OLc2OrwZskDfx5QHfeJnumjQLut5qsnZ+1onj9P2dGdn +JaChe4kBPQQYAQIACQUCVvzIKQIbLgCoCRAqYf65XmeSk50gBBkBAgAGBQJW/Mgp +AAoJELCUmIrUFDr+MNcEAJ92rSORLJkcsOogulFVlib/5SotT64srQ7K3R+cVJh1 +IKelVPD8PWOhvrL+dCi321UNZfvP9we9U42nbDsYfjwT7BMmKSull01RsuOCA7xV +hsjqHfLZn3BTfSKVOQ5kpP758vRu0Sv5257WtlZ32Ts8JuYt3DZD6xRjy9c9Ct6r +NFcD/jOwV3agUUhQwwaRtSdE62YqBI7ynjrE/1JfJm1bex0KXDfdyMEzuLpMtFqM +tp2+0m00MOsDqVOeLu/uXUkNN5bVB4NaMMumNwjDnhS/nvQC6QGQFGjMGO8zBYap +T1WWTH3TC0bHS4p+u0SDPjiegPduQDOdmxEE9WMJEJT+Jr0G +=hvJM +-----END PGP PRIVATE KEY BLOCK----- +HERE + end diff --git a/tests/white-box/network.rb b/tests/white-box/network.rb index acb5c5e6..436fc8a8 100644 --- a/tests/white-box/network.rb +++ b/tests/white-box/network.rb @@ -1,4 +1,5 @@ require 'socket' +require 'openssl' raise SkipTest if $node["dummy"] @@ -28,11 +29,18 @@ class Network < LeapTest def test_02_Is_stunnel_running? ignore unless $node['stunnel'] good_stunnel_pids = [] + release = `facter lsbmajdistrelease` + if release.to_i > 7 + # on jessie, there is only one stunnel proc running instead of 6 + expected = 1 + else + expected = 6 + end $node['stunnel']['clients'].each do |stunnel_type, stunnel_configs| stunnel_configs.each do |stunnel_name, stunnel_conf| config_file_name = "/etc/stunnel/#{stunnel_name}.conf" processes = pgrep(config_file_name) - assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`" + assert_equal expected, processes.length, "There should be #{expected} stunnel processes running for `#{config_file_name}`" good_stunnel_pids += processes.map{|ps| ps[:pid]} assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.' assert_tcp_socket('localhost', port) @@ -41,7 +49,7 @@ class Network < LeapTest $node['stunnel']['servers'].each do |stunnel_name, stunnel_conf| config_file_name = "/etc/stunnel/#{stunnel_name}.conf" processes = pgrep(config_file_name) - assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`" + assert_equal expected, processes.length, "There should be #{expected} stunnel processes running for `#{config_file_name}`" good_stunnel_pids += processes.map{|ps| ps[:pid]} assert accept_port = stunnel_conf['accept_port'], "Field `accept` must be present in property `stunnel.servers.#{stunnel_name}`" assert_tcp_socket('localhost', accept_port) @@ -62,4 +70,21 @@ class Network < LeapTest pass end + THIRTY_DAYS = 60*60*24*30 + + def test_04_Are_server_certificates_valid? + cert_paths = ["/etc/x509/certs/leap_commercial.crt", "/etc/x509/certs/leap.crt"] + cert_paths.each do |cert_path| + if File.exists?(cert_path) + cert = OpenSSL::X509::Certificate.new(File.read(cert_path)) + if Time.now > cert.not_after + fail "The certificate #{cert_path} expired on #{cert.not_after}" + elsif Time.now + THIRTY_DAYS > cert.not_after + fail "The certificate #{cert_path} will expire soon, on #{cert.not_after}" + end + end + end + pass + end + end diff --git a/tests/white-box/openvpn.rb b/tests/white-box/openvpn.rb index 23a40426..170d4503 100644 --- a/tests/white-box/openvpn.rb +++ b/tests/white-box/openvpn.rb @@ -7,9 +7,9 @@ class OpenVPN < LeapTest end def test_01_Are_daemons_running? - assert_running '/usr/sbin/openvpn .* /etc/openvpn/tcp_config.conf' - assert_running '/usr/sbin/openvpn .* /etc/openvpn/udp_config.conf' - assert_running '/usr/sbin/unbound' + assert_running '^/usr/sbin/openvpn .* /etc/openvpn/tcp_config.conf$' + assert_running '^/usr/sbin/openvpn .* /etc/openvpn/udp_config.conf$' + assert_running '^/usr/sbin/unbound$' pass end diff --git a/tests/white-box/soledad.rb b/tests/white-box/soledad.rb index 5a13e4a6..d41bee58 100644 --- a/tests/white-box/soledad.rb +++ b/tests/white-box/soledad.rb @@ -10,7 +10,7 @@ class Soledad < LeapTest end def test_00_Is_Soledad_running? - assert_running 'soledad' + assert_running '.*/usr/bin/twistd.*--wsgi=leap.soledad.server.application' pass end diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb index 9956eb35..68f3dcd2 100644 --- a/tests/white-box/webapp.rb +++ b/tests/white-box/webapp.rb @@ -27,8 +27,8 @@ class Webapp < LeapTest end def test_03_Are_daemons_running? - assert_running '/usr/sbin/apache2' - assert_running '/usr/bin/nickserver' + assert_running '^/usr/sbin/apache2' + assert_running '^/usr/bin/ruby /usr/bin/nickserver' pass end @@ -57,10 +57,11 @@ class Webapp < LeapTest soledad_server = pick_soledad_server(soledad_config) if soledad_server assert_tmp_user do |user| - assert_user_db_exists(user) command = File.expand_path "../../helpers/soledad_sync.py", __FILE__ soledad_url = "https://#{soledad_server}/user-#{user.id}" - assert_run "#{command} #{user.id} #{user.session_token} #{soledad_url}" + soledad_cert = "/usr/local/share/ca-certificates/leap_ca.crt" + assert_run "#{command} #{user.id} #{user.session_token} #{soledad_url} #{soledad_cert} #{user.password}" + assert_user_db_exists(user) pass end end @@ -73,8 +74,8 @@ class Webapp < LeapTest def url_options { - :username => property('couchdb_webapp_user.username'), - :password => property('couchdb_webapp_user.password') + :username => property('webapp.couchdb_webapp_user.username'), + :password => property('webapp.couchdb_webapp_user.password') } end @@ -95,7 +96,7 @@ class Webapp < LeapTest end # - # returns true if the per-user db created by tapicero exists. + # returns true if the per-user db created by soledad-server exists. # we try three times, and give up after that. # def assert_user_db_exists(user) @@ -118,7 +119,9 @@ class Webapp < LeapTest sleep 0.2 get(couchdb_url(url)) do |body, response, error| last_body, last_response, last_error = body, response, error - if response.code.to_i == 200 + # After moving to couchdb, webapp user is not allowed to Read user dbs, + # but the return code for non-existent databases is 404. See #7674 + if response.code.to_i == 401 return end end @@ -128,18 +131,4 @@ class Webapp < LeapTest return end - # - # I tried, but couldn't get this working: - # # - # # get an CSRF authenticity token - # # - # url = api_url("/") - # csrf_token = nil - # assert_get(url) do |body| - # lines = body.split("\n").grep(/csrf-token/) - # assert lines.any?, 'failed to find csrf-token' - # csrf_token = lines.first.split('"')[1] - # assert csrf_token, 'failed to find csrf-token' - # end - end diff --git a/vagrant/add-pixelated.sh b/vagrant/add-pixelated.sh new file mode 100755 index 00000000..f9908947 --- /dev/null +++ b/vagrant/add-pixelated.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# +# adds pixelated-server to the node + +. /vagrant/vagrant/vagrant.config + +cd "$PROVIDERDIR" + +if ! git submodule status files/puppet/modules/pixelated > /dev/null 2>&1; then + git submodule add https://github.com/pixelated/puppet-pixelated.git files/puppet/modules/pixelated +fi + +echo '{}' > services/pixelated.json +[ -d files/puppet/modules/custom/manifests ] || mkdir -p files/puppet/modules/custom/manifests +echo 'class custom { include ::pixelated}' > files/puppet/modules/custom/manifests/init.pp + +$LEAP $OPTS -v 2 deploy + +echo '===============================================' +echo 'testing the platform' +echo '===============================================' + +$LEAP $OPTS -v 2 test --continue + + +echo -e '\n===========================================================================================================\n\n' +echo -e 'You are now ready to use your vagrant Pixelated provider.\n' + +echo -e 'The LEAP webapp is available at https://localhost:4443. Use it to register an account before using the Pixelated Useragent.\n' +echo -e 'The Pixelated Useragent is available at https://localhost:8080\n' + +echo -e 'Please add an exception for both sites in your browser dialog to allow the self-signed certificate.\n' diff --git a/vagrant/configure-leap.sh b/vagrant/configure-leap.sh index 9541e194..9ddee039 100755 --- a/vagrant/configure-leap.sh +++ b/vagrant/configure-leap.sh @@ -1,13 +1,7 @@ #!/bin/bash -. /vagrant/vagrant/vagrant.config - -#OPTS='--no-color' -OPTS='' -PROVIDERDIR='/srv/leap/configuration' -NODE='node1' -LEAP='/usr/local/bin/leap' +. /vagrant/vagrant/vagrant.config echo '===============================================' echo 'configuring leap' @@ -15,48 +9,61 @@ echo '===============================================' # purge $PROVIDERDIR so this script can be run multiple times [ -e $PROVIDERDIR ] && rm -rf $PROVIDERDIR -mkdir $PROVIDERDIR + +mkdir -p $PROVIDERDIR +chown ${USER}:${USER} ${PROVIDERDIR} cd $PROVIDERDIR $LEAP $OPTS new --contacts "$contacts" --domain "$provider_domain" --name "$provider_name" --platform=/vagrant . -echo -e '\n@log = "/var/log/leap/deploy.log"' >> Leapfile +echo -e '\n@log = "./deploy.log"' >> Leapfile -if [ ! -e /root/.ssh/id_rsa ]; then - ssh-keygen -f /root/.ssh/id_rsa -P '' - cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys +if [ ! -e /home/${USER}/.ssh/id_rsa ]; then + $SUDO ssh-keygen -f /home/${USER}/.ssh/id_rsa -P '' + [ -d /root/.ssh ] || mkdir /root/.ssh + cat /home/${USER}/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys fi -mkdir -p $PROVIDERDIR/files/nodes/$NODE +$SUDO mkdir -p ${PROVIDERDIR}/files/nodes/${NODE} sh -c "cat /etc/ssh/ssh_host_rsa_key.pub | cut -d' ' -f1,2 >> $PROVIDERDIR/files/nodes/$NODE/${NODE}_ssh.pub" +chown ${USER}:${USER} ${PROVIDERDIR}/files/nodes/${NODE}/${NODE}_ssh.pub $LEAP $OPTS add-user --self $LEAP $OPTS cert ca $LEAP $OPTS cert csr -$LEAP $OPTS node add $NODE ip_address:"$(facter ipaddress)" services:"$services" tags:production +$LEAP $OPTS node add $NODE ip_address:"$(facter ipaddress)" couch.mode:plain services:"$services" tags:production echo '{ "webapp": { "admins": ["testadmin"] } }' > services/webapp.json $LEAP $OPTS compile -git init -git add . -git commit -m'configured provider' +$GIT init +$GIT add . +$GIT commit -m'configured provider' -$LEAP $OPTS node init $NODE +$LEAP $OPTS node init $NODE if [ $? -eq 1 ]; then echo 'node init failed' exit 1 fi +# couchrest gem does currently not install on jessie +# https://leap.se/code/issues/7754 +# workaround is to install rake as gem +gem install rake + $LEAP $OPTS -v 2 deploy -if [ $? -eq 1 ]; then - echo 'deploy failed' - exit 1 -fi -set +e -git add . -git commit -m'initialized and deployed provider' -set -e +$GIT add . +$GIT commit -m'initialized and deployed provider' + +# Vagrant: leap_mx fails to start on jessie +# https://leap.se/code/issues/7755 +# Workaround: we stop and start leap-mx after deploy and +# before testing + +service leap-mx stop +service leap-mx start + + echo '===============================================' echo 'testing the platform' @@ -69,15 +76,17 @@ echo 'setting node to demo-mode' echo '===============================================' postconf -e default_transport='error: in demo mode' -sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/g' /etc/ssh/sshd_config -/etc/init.d/ssh reload - # add users: testadmin and testuser with passwords "hallo123" curl -s -k https://localhost/1/users.json -d "user%5Blogin%5D=testuser&user%5Bpassword_salt%5D=7d4880237a038e0e&user%5Bpassword_verifier%5D=b98dc393afcd16e5a40fb57ce9cddfa6a978b84be326196627c111d426cada898cdaf3a6427e98b27daf4b0ed61d278bc856515aeceb2312e50c8f816659fcaa4460d839a1e2d7ffb867d32ac869962061368141c7571a53443d58dc84ca1fca34776894414c1090a93e296db6cef12c2cc3f7a991b05d49728ed358fd868286" curl -s -k https://localhost/1/users.json -d "user%5Blogin%5D=testadmin&user%5Bpassword_salt%5D=ece1c457014d8282&user%5Bpassword_verifier%5D=9654d93ab409edf4ff1543d07e08f321107c3fd00de05c646c637866a94f28b3eb263ea9129dacebb7291b3374cc6f0bf88eb3d231eb3a76eed330a0e8fd2a5c477ed2693694efc1cc23ae83c2ae351a21139701983dd595b6c3225a1bebd2a4e6122f83df87606f1a41152d9890e5a11ac3749b3bfcf4407fc83ef60b4ced68" -echo -e '\n\n\n' -echo 'You are now ready to use your provider. Please update your /etc/hosts with following dns overrides:' +echo -e '\n===========================================================================================================\n\n' +echo -e 'You are now ready to use your local LEAP provider.\n' +echo 'If you want to use the *Bitmask client* with your provider, please update your /etc/hosts with following dns overrides:' -$LEAP list --print ip_address,domain.full,dns.aliases | sed 's/,//g' | cut -d' ' -f 2- +$LEAP list --print ip_address,domain.full,dns.aliases | sed 's/^.* //' | sed 's/, null//g' | tr -d '\]\[",' +echo 'Please see https://leap.se/en/docs/platform/tutorials/vagrant#use-the-bitmask-client-to-do-an-initial-soledad-sync for more details how to use and test your LEAP provider.' +echo -e "\nIf you don't want to use the Bitmask client, please ignore the above instructions.\n" +echo -e 'The LEAP webapp is now available at https://localhost:4443\n' +echo -e 'Please add an exception in your browser dialog to allow the self-signed certificate.\n' diff --git a/vagrant/install-platform.pp b/vagrant/install-platform.pp index 465ca78a..223853c1 100755 --- a/vagrant/install-platform.pp +++ b/vagrant/install-platform.pp @@ -1,36 +1,15 @@ class {'apt': } -File['/etc/apt/preferences'] -> - Exec['refresh_apt'] -> - Package <| ( title != 'lsb' ) |> +Exec['update_apt'] -> Package <||> -package { [ 'rsync', 'ruby-hiera-puppet', 'git', 'ruby1.9.1-dev', 'rake', 'jq' ]: - ensure => installed -} - -file { '/etc/gemrc': - content => "---\n:sources:\n - https://rubygems.org/" -} - -vcsrepo { '/srv/leap/leap_cli': - ensure => present, - force => true, - revision => 'develop', - provider => 'git', - source => 'https://leap.se/git/leap_cli.git', - owner => 'root', - group => 'root', - notify => Exec['install_leap_cli'], - require => Package['git'] -} - -exec { 'install_leap_cli': - command => '/usr/bin/rake build && /usr/bin/rake install', - cwd => '/srv/leap/leap_cli', - refreshonly => true, - require => [ Package['ruby1.9.1-dev'], File['/etc/gemrc'], Package['rake'] ] +# install leap_cli from source, so it will work with the develop +# branch of leap_platform +class { '::leap::cli::install': + source => true, } file { [ '/srv/leap', '/srv/leap/configuration', '/var/log/leap' ]: ensure => directory } +# install prerequisites for configuring the provider +include ::git diff --git a/vagrant/vagrant.config b/vagrant/vagrant.config index ae124246..e601488d 100644 --- a/vagrant/vagrant.config +++ b/vagrant/vagrant.config @@ -1,4 +1,4 @@ -# config values used by configure-leap.sh +# provider config values used by vagrant provision scripts provider_domain='example.org' provider_name='Leap Example Provider' contacts="no-reply@$provider_domain" @@ -7,6 +7,16 @@ contacts="no-reply@$provider_domain" # note that the "openvpn" service does currently *not* work # in a vagrant setup, # see https://leap.se/en/docs/platform/troubleshooting/known-issues#Special.Environments -services='webapp,mx,couchdb,soledad,monitor' +# to speed up things, don't deploy monitor service by default +# services='webapp,mx,couchdb,soledad,monitor' +services='webapp,mx,couchdb,soledad' +# default vars used by vagrant provision scripts +OPTS='' +USER='vagrant' +NODE='node1' +SUDO="sudo -u ${USER}" +PROVIDERDIR="/home/${USER}/leap/configuration" +LEAP="$SUDO /usr/local/bin/leap" +GIT="$SUDO git" |