diff options
162 files changed, 5207 insertions, 764 deletions
@@ -1,6 +1,8 @@ -What is it? +Leap Platform ============================= +[](https://squirrel.leap.se:8080/job/platform_develop/) + The LEAP Platform is set of complementary packages and server recipes to automate the maintenance of LEAP services in a hardened Debian environment. Its goal is to make it as painless as possible for sysadmins to deploy and maintain a service provider's infrastructure for secure communication. These recipes define an abstract service provider. It is a set of Puppet modules designed to work together to provide to sysadmins everything they need to manage a service provider infrastructure that provides secure communication services. Getting started diff --git a/Vagrantfile b/Vagrantfile index c9c68284..cb9392e3 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,18 +1,17 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -Vagrant.configure("2") do |vagrant_config| - vagrant_config.vm.define :node1 do |config| - # Please verify the sha512 sum of the downloaded box before importing it into vagrant ! - # see https://leap.se/en/docs/platform/details/development#Verify.vagrantbox.download - # for details +Vagrant.configure("2") do |config| + # Please verify the sha512 sum of the downloaded box before importing it into vagrant ! + # see https://leap.se/en/docs/platform/details/development#Verify.vagrantbox.download + # for details + + config.vm.define :"wheezy", primary: true do |config| config.vm.box = "LEAP/wheezy" - #config.vm.network :private_network, ip: "10.5.5.102" config.vm.provider "virtualbox" do |v| - v.memory = 1024 v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] - v.name = "node1" + v.name = "wheezy" end config.vm.provision "puppet" do |puppet| @@ -22,8 +21,33 @@ Vagrant.configure("2") do |vagrant_config| puppet.options = "--verbose" end config.vm.provision "shell", path: "vagrant/configure-leap.sh" + config.ssh.username = "vagrant" + + # forward leap_web ports + config.vm.network "forwarded_port", guest: 80, host:8080 + config.vm.network "forwarded_port", guest: 443, host:4443 + end + + config.vm.define :"jessie", autostart: false do |config| + + config.vm.box = "LEAP/jessie" + config.vm.provider "virtualbox" do |v| + v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + v.name = "jessie" + end + config.vm.provision "puppet" do |puppet| + puppet.manifests_path = "./vagrant" + puppet.module_path = "./puppet/modules" + puppet.manifest_file = "install-platform.pp" + puppet.options = "--verbose" + end + config.vm.provision "shell", path: "vagrant/configure-leap.sh" config.ssh.username = "vagrant" + # forward leap_web ports + config.vm.network "forwarded_port", guest: 80, host:8080 + config.vm.network "forwarded_port", guest: 443, host:4443 end + end diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 00000000..e836bc7e --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,9 @@ +# Contributed Files + +## Commit Template + +to install this commit template, use following cmd (use --global to use it in your global .gitconfig): + + git config [--global] commit.template "~/path_to_leap_platform/contrib/commit-template.txt" + + diff --git a/contrib/commit-template.txt b/contrib/commit-template.txt new file mode 100644 index 00000000..9a1fa81b --- /dev/null +++ b/contrib/commit-template.txt @@ -0,0 +1,7 @@ +#[bug|feat|docs|style|refactor|test|pkg|i18n] + +#- Tested: [local singlenode|local multinode|citest|unstable.bitmask.net] +#- Resolves: #XYZ +#- Related: #XYZ +#- Documentation: #XYZ +#- Releases: XYZ diff --git a/vagrant/offlineimaprc.example.org b/contrib/offlineimaprc.example.org index 3d119634..3d119634 100644 --- a/vagrant/offlineimaprc.example.org +++ b/contrib/offlineimaprc.example.org diff --git a/lib/leap_cli/commands/README b/lib/leap_cli/commands/README new file mode 100644 index 00000000..bec78179 --- /dev/null +++ b/lib/leap_cli/commands/README @@ -0,0 +1,11 @@ +This directory contains ruby source files that define the available sub- +commands of the `leap` executable. + +For example, the command: + + leap compile + +Lives in lib/leap_cli/commands/init.rb + +These files use a DSL (called GLI) for defining command suites. +See https://github.com/davetron5000/gli for more information. diff --git a/lib/leap_cli/commands/ca.rb b/lib/leap_cli/commands/ca.rb new file mode 100644 index 00000000..3a0786e2 --- /dev/null +++ b/lib/leap_cli/commands/ca.rb @@ -0,0 +1,541 @@ +autoload :OpenSSL, 'openssl' +autoload :CertificateAuthority, 'certificate_authority' +autoload :Date, 'date' +require 'digest/md5' + +module LeapCli; module Commands + + desc "Manage X.509 certificates" + command :cert do |cert| + + cert.desc 'Creates two Certificate Authorities (one for validating servers and one for validating clients).' + cert.long_desc 'See see what values are used in the generation of the certificates (like name and key size), run `leap inspect provider` and look for the "ca" property. To see the details of the created certs, run `leap inspect <file>`.' + cert.command :ca do |ca| + ca.action do |global_options,options,args| + assert_config! 'provider.ca.name' + generate_new_certificate_authority(:ca_key, :ca_cert, provider.ca.name) + generate_new_certificate_authority(:client_ca_key, :client_ca_cert, provider.ca.name + ' (client certificates only!)') + end + end + + cert.desc 'Creates or renews a X.509 certificate/key pair for a single node or all nodes, but only if needed.' + cert.long_desc 'This command will a generate new certificate for a node if some value in the node has changed ' + + 'that is included in the certificate (like hostname or IP address), or if the old certificate will be expiring soon. ' + + 'Sometimes, you might want to force the generation of a new certificate, ' + + 'such as in the cases where you have changed a CA parameter for server certificates, like bit size or digest hash. ' + + 'In this case, use --force. If <node-filter> is empty, this command will apply to all nodes.' + cert.arg_name 'FILTER' + cert.command :update do |update| + update.switch 'force', :desc => 'Always generate new certificates', :negatable => false + update.action do |global_options,options,args| + update_certificates(manager.filter!(args), options) + end + end + + cert.desc 'Creates a Diffie-Hellman parameter file, needed for forward secret OpenVPN ciphers.' # (needed for server-side of some TLS connections) + cert.command :dh do |dh| + dh.action do |global_options,options,args| + long_running do + if cmd_exists?('certtool') + log 0, 'Generating DH parameters (takes a long time)...' + output = assert_run!('certtool --generate-dh-params --sec-param high') + output.sub! /.*(-----BEGIN DH PARAMETERS-----.*-----END DH PARAMETERS-----).*/m, '\1' + output << "\n" + write_file!(:dh_params, output) + else + log 0, 'Generating DH parameters (takes a REALLY long time)...' + output = OpenSSL::PKey::DH.generate(3248).to_pem + write_file!(:dh_params, output) + end + end + end + end + + # + # hints: + # + # inspect CSR: + # openssl req -noout -text -in files/cert/x.csr + # + # generate CSR with openssl to see how it compares: + # openssl req -sha256 -nodes -newkey rsa:2048 -keyout example.key -out example.csr + # + # validate a CSR: + # http://certlogik.com/decoder/ + # + # nice details about CSRs: + # http://www.redkestrel.co.uk/Articles/CSR.html + # + cert.desc "Creates a CSR for use in buying a commercial X.509 certificate." + cert.long_desc "Unless specified, the CSR is created for the provider's primary domain. "+ + "The properties used for this CSR come from `provider.ca.server_certificates`, "+ + "but may be overridden here." + cert.command :csr do |csr| + csr.flag 'domain', :arg_name => 'DOMAIN', :desc => 'Specify what domain to create the CSR for.' + csr.flag ['organization', 'O'], :arg_name => 'ORGANIZATION', :desc => "Override default O in distinguished name." + csr.flag ['unit', 'OU'], :arg_name => 'UNIT', :desc => "Set OU in distinguished name." + csr.flag 'email', :arg_name => 'EMAIL', :desc => "Set emailAddress in distinguished name." + csr.flag ['locality', 'L'], :arg_name => 'LOCALITY', :desc => "Set L in distinguished name." + csr.flag ['state', 'ST'], :arg_name => 'STATE', :desc => "Set ST in distinguished name." + csr.flag ['country', 'C'], :arg_name => 'COUNTRY', :desc => "Set C in distinguished name." + csr.flag :bits, :arg_name => 'BITS', :desc => "Override default certificate bit length" + csr.flag :digest, :arg_name => 'DIGEST', :desc => "Override default signature digest" + csr.action do |global_options,options,args| + assert_config! 'provider.domain' + assert_config! 'provider.name' + assert_config! 'provider.default_language' + assert_config! 'provider.ca.server_certificates.bit_size' + assert_config! 'provider.ca.server_certificates.digest' + domain = options[:domain] || provider.domain + + unless global_options[:force] + assert_files_missing! [:commercial_key, domain], [:commercial_csr, domain], + :msg => 'If you really want to create a new key and CSR, remove these files first or run with --force.' + end + + server_certificates = provider.ca.server_certificates + + # RSA key + keypair = CertificateAuthority::MemoryKeyMaterial.new + bit_size = (options[:bits] || server_certificates.bit_size).to_i + log :generating, "%s bit RSA key" % bit_size do + keypair.generate_key(bit_size) + write_file! [:commercial_key, domain], keypair.private_key.to_pem + end + + # CSR + dn = CertificateAuthority::DistinguishedName.new + dn.common_name = domain + dn.organization = options[:organization] || provider.name[provider.default_language] + dn.ou = options[:organizational_unit] # optional + dn.email_address = options[:email] # optional + dn.country = options[:country] || server_certificates['country'] # optional + dn.state = options[:state] || server_certificates['state'] # optional + dn.locality = options[:locality] || server_certificates['locality'] # optional + + digest = options[:digest] || server_certificates.digest + log :generating, "CSR with #{digest} digest and #{print_dn(dn)}" do + csr = create_csr(dn, keypair, digest) + request = csr.to_x509_csr + write_file! [:commercial_csr, domain], csr.to_pem + end + + # Sign using our own CA, for use in testing but hopefully not production. + # It is not that commerical CAs are so secure, it is just that signing your own certs is + # a total drag for the user because they must click through dire warnings. + #if options[:sign] + log :generating, "self-signed x509 server certificate for testing purposes" do + cert = csr.to_cert + cert.serial_number.number = cert_serial_number(domain) + cert.not_before = yesterday + cert.not_after = yesterday.advance(:years => 1) + cert.parent = ca_root + cert.sign! domain_test_signing_profile + write_file! [:commercial_cert, domain], cert.to_pem + log "please replace this file with the real certificate you get from a CA using #{Path.relative_path([:commercial_csr, domain])}" + end + #end + + # FAKE CA + unless file_exists? :commercial_ca_cert + log :using, "generated CA in place of commercial CA for testing purposes" do + write_file! :commercial_ca_cert, read_file!(:ca_cert) + log "please also replace this file with the CA cert from the commercial authority you use." + end + end + end + end + end + + protected + + # + # will generate new certificates for the specified nodes, if needed. + # + def update_certificates(nodes, options={}) + assert_files_exist! :ca_cert, :ca_key, :msg => 'Run `leap cert ca` to create them' + assert_config! 'provider.ca.server_certificates.bit_size' + assert_config! 'provider.ca.server_certificates.digest' + assert_config! 'provider.ca.server_certificates.life_span' + assert_config! 'common.x509.use' + + nodes.each_node do |node| + warn_if_commercial_cert_will_soon_expire(node) + if !node.x509.use + remove_file!([:node_x509_key, node.name]) + remove_file!([:node_x509_cert, node.name]) + elsif options[:force] || cert_needs_updating?(node) + generate_cert_for_node(node) + end + end + end + + private + + def generate_new_certificate_authority(key_file, cert_file, common_name) + assert_files_missing! key_file, cert_file + assert_config! 'provider.ca.name' + assert_config! 'provider.ca.bit_size' + assert_config! 'provider.ca.life_span' + + root = CertificateAuthority::Certificate.new + + # set subject + root.subject.common_name = common_name + possible = ['country', 'state', 'locality', 'organization', 'organizational_unit', 'email_address'] + provider.ca.keys.each do |key| + if possible.include?(key) + root.subject.send(key + '=', provider.ca[key]) + end + end + + # set expiration + root.not_before = yesterday + root.not_after = yesterday_advance(provider.ca.life_span) + + # generate private key + root.serial_number.number = 1 + root.key_material.generate_key(provider.ca.bit_size) + + # sign self + root.signing_entity = true + root.parent = root + root.sign!(ca_root_signing_profile) + + # save + write_file!(key_file, root.key_material.private_key.to_pem) + write_file!(cert_file, root.to_pem) + end + + # + # returns true if the certs associated with +node+ need to be regenerated. + # + def cert_needs_updating?(node) + if !file_exists?([:node_x509_cert, node.name], [:node_x509_key, node.name]) + return true + else + cert = load_certificate_file([:node_x509_cert, node.name]) + if !created_by_authority?(cert, ca_root) + log :updating, "cert for node '#{node.name}' because it was signed by an old CA root cert." + return true + end + if cert.not_after < Time.now.advance(:months => 2) + log :updating, "cert for node '#{node.name}' because it will expire soon" + return true + end + if cert.subject.common_name != node.domain.full + log :updating, "cert for node '#{node.name}' because domain.full has changed (was #{cert.subject.common_name}, now #{node.domain.full})" + return true + end + cert.openssl_body.extensions.each do |ext| + if ext.oid == "subjectAltName" + ips = [] + dns_names = [] + ext.value.split(",").each do |value| + value.strip! + ips << $1 if value =~ /^IP Address:(.*)$/ + dns_names << $1 if value =~ /^DNS:(.*)$/ + end + dns_names.sort! + if ips.first != node.ip_address + log :updating, "cert for node '#{node.name}' because ip_address has changed (from #{ips.first} to #{node.ip_address})" + return true + elsif dns_names != dns_names_for_node(node) + log :updating, "cert for node '#{node.name}' because domain name aliases have changed\n from: #{dns_names.inspect}\n to: #{dns_names_for_node(node).inspect})" + return true + end + end + end + end + return false + end + + def created_by_authority?(cert, ca) + authority_key_id = cert.extensions["authorityKeyIdentifier"].identifier.sub(/^keyid:/, '') + authority_key_id == public_key_id_for_ca(ca) + end + + # calculate the "key id" for a root CA, that matches the value + # Authority Key Identifier in the x509 extensions of a cert. + def public_key_id_for_ca(ca_cert) + @ca_key_ids ||= {} + @ca_key_ids[ca_cert.object_id] ||= begin + pubkey = ca_cert.key_material.public_key + seq = OpenSSL::ASN1::Sequence([ + OpenSSL::ASN1::Integer.new(pubkey.n), + OpenSSL::ASN1::Integer.new(pubkey.e) + ]) + Digest::SHA1.hexdigest(seq.to_der).upcase.scan(/../).join(':') + end + end + + def warn_if_commercial_cert_will_soon_expire(node) + dns_names_for_node(node).each do |domain| + if file_exists?([:commercial_cert, domain]) + cert = load_certificate_file([:commercial_cert, domain]) + path = Path.relative_path([:commercial_cert, domain]) + if cert.not_after < Time.now.utc + log :error, "the commercial certificate '#{path}' has EXPIRED! " + + "You should renew it with `leap cert csr --domain #{domain}`." + elsif cert.not_after < Time.now.advance(:months => 2) + log :warning, "the commercial certificate '#{path}' will expire soon. "+ + "You should renew it with `leap cert csr --domain #{domain}`." + end + end + end + end + + def generate_cert_for_node(node) + return if node.x509.use == false + + cert = CertificateAuthority::Certificate.new + + # set subject + cert.subject.common_name = node.domain.full + cert.serial_number.number = cert_serial_number(node.domain.full) + + # set expiration + cert.not_before = yesterday + cert.not_after = yesterday_advance(provider.ca.server_certificates.life_span) + + # generate key + cert.key_material.generate_key(provider.ca.server_certificates.bit_size) + + # sign + cert.parent = ca_root + cert.sign!(server_signing_profile(node)) + + # save + write_file!([:node_x509_key, node.name], cert.key_material.private_key.to_pem) + write_file!([:node_x509_cert, node.name], cert.to_pem) + end + + # + # yields client key and cert suitable for testing + # + def generate_test_client_cert(prefix=nil) + cert = CertificateAuthority::Certificate.new + cert.serial_number.number = cert_serial_number(provider.domain) + cert.subject.common_name = [prefix, random_common_name(provider.domain)].join + cert.not_before = yesterday + cert.not_after = yesterday.advance(:years => 1) + cert.key_material.generate_key(1024) # just for testing, remember! + cert.parent = client_ca_root + cert.sign! client_test_signing_profile + yield cert.key_material.private_key.to_pem, cert.to_pem + end + + # + # creates a CSR and returns it. + # with the correct extReq attribute so that the CA + # doens't generate certs with extensions we don't want. + # + def create_csr(dn, keypair, digest) + csr = CertificateAuthority::SigningRequest.new + csr.distinguished_name = dn + csr.key_material = keypair + csr.digest = digest + + # define extensions manually (library doesn't support setting these on CSRs) + extensions = [] + extensions << CertificateAuthority::Extensions::BasicConstraints.new.tap {|basic| + basic.ca = false + } + extensions << CertificateAuthority::Extensions::KeyUsage.new.tap {|keyusage| + keyusage.usage = ["digitalSignature", "keyEncipherment"] + } + extensions << CertificateAuthority::Extensions::ExtendedKeyUsage.new.tap {|extkeyusage| + extkeyusage.usage = [ "serverAuth"] + } + + # convert extensions to attribute 'extReq' + # aka "Requested Extensions" + factory = OpenSSL::X509::ExtensionFactory.new + attrval = OpenSSL::ASN1::Set([OpenSSL::ASN1::Sequence( + extensions.map{|e| factory.create_ext(e.openssl_identifier, e.to_s, e.critical)} + )]) + attrs = [ + OpenSSL::X509::Attribute.new("extReq", attrval), + ] + csr.attributes = attrs + + return csr + end + + def ca_root + @ca_root ||= begin + load_certificate_file(:ca_cert, :ca_key) + end + end + + def client_ca_root + @client_ca_root ||= begin + load_certificate_file(:client_ca_cert, :client_ca_key) + end + end + + def load_certificate_file(crt_file, key_file=nil, password=nil) + crt = read_file!(crt_file) + openssl_cert = OpenSSL::X509::Certificate.new(crt) + cert = CertificateAuthority::Certificate.from_openssl(openssl_cert) + if key_file + key = read_file!(key_file) + cert.key_material.private_key = OpenSSL::PKey::RSA.new(key, password) + end + return cert + end + + def ca_root_signing_profile + { + "extensions" => { + "basicConstraints" => {"ca" => true}, + "keyUsage" => { + "usage" => ["critical", "keyCertSign"] + }, + "extendedKeyUsage" => { + "usage" => [] + } + } + } + end + + # + # For keyusage, openvpn server certs can have keyEncipherment or keyAgreement. + # Web browsers seem to break without keyEncipherment. + # For now, I am using digitalSignature + keyEncipherment + # + # * digitalSignature -- for (EC)DHE cipher suites + # "The digitalSignature bit is asserted when the subject public key is used + # with a digital signature mechanism to support security services other + # than certificate signing (bit 5), or CRL signing (bit 6). Digital + # signature mechanisms are often used for entity authentication and data + # origin authentication with integrity." + # + # * keyEncipherment ==> for plain RSA cipher suites + # "The keyEncipherment bit is asserted when the subject public key is used for + # key transport. For example, when an RSA key is to be used for key management, + # then this bit is set." + # + # * keyAgreement ==> for used with DH, not RSA. + # "The keyAgreement bit is asserted when the subject public key is used for key + # agreement. For example, when a Diffie-Hellman key is to be used for key + # management, then this bit is set." + # + # digest options: SHA512, SHA256, SHA1 + # + def server_signing_profile(node) + { + "digest" => provider.ca.server_certificates.digest, + "extensions" => { + "keyUsage" => { + "usage" => ["digitalSignature", "keyEncipherment"] + }, + "extendedKeyUsage" => { + "usage" => ["serverAuth", "clientAuth"] + }, + "subjectAltName" => { + "ips" => [node.ip_address], + "dns_names" => dns_names_for_node(node) + } + } + } + end + + # + # This is used when signing the main cert for the provider's domain + # with our own CA (for testing purposes). Typically, this cert would + # be purchased from a commercial CA, and not signed this way. + # + def domain_test_signing_profile + { + "digest" => "SHA256", + "extensions" => { + "keyUsage" => { + "usage" => ["digitalSignature", "keyEncipherment"] + }, + "extendedKeyUsage" => { + "usage" => ["serverAuth"] + } + } + } + end + + # + # This is used when signing a dummy client certificate that is only to be + # used for testing. + # + def client_test_signing_profile + { + "digest" => "SHA256", + "extensions" => { + "keyUsage" => { + "usage" => ["digitalSignature"] + }, + "extendedKeyUsage" => { + "usage" => ["clientAuth"] + } + } + } + end + + def dns_names_for_node(node) + names = [node.domain.internal, node.domain.full] + if node['dns'] && node.dns['aliases'] && node.dns.aliases.any? + names += node.dns.aliases + end + names.compact! + names.sort! + names.uniq! + return names + end + + # + # For cert serial numbers, we need a non-colliding number less than 160 bits. + # md5 will do nicely, since there is no need for a secure hash, just a short one. + # (md5 is 128 bits) + # + def cert_serial_number(domain_name) + Digest::MD5.hexdigest("#{domain_name} -- #{Time.now}").to_i(16) + end + + # + # for the random common name, we need a text string that will be unique across all certs. + # ruby 1.8 doesn't have a built-in uuid generator, or we would use SecureRandom.uuid + # + def random_common_name(domain_name) + cert_serial_number(domain_name).to_s(36) + end + + # prints CertificateAuthority::DistinguishedName fields + def print_dn(dn) + fields = {} + [:common_name, :locality, :state, :country, :organization, :organizational_unit, :email_address].each do |attr| + fields[attr] = dn.send(attr) if dn.send(attr) + end + fields.inspect + end + + ## + ## TIME HELPERS + ## + ## note: we use 'yesterday' instead of 'today', because times are in UTC, and some people on the planet + ## are behind UTC. + ## + + def yesterday + t = Time.now - 24*24*60 + Time.utc t.year, t.month, t.day + end + + def yesterday_advance(string) + number, unit = string.split(' ') + unless ['years', 'months', 'days', 'hours', 'minutes'].include? unit + bail!("The time property '#{string}' is missing a unit (one of: years, months, days, hours, minutes).") + end + unless number.to_i.to_s == number + bail!("The time property '#{string}' is missing a number.") + end + yesterday.advance(unit.to_sym => number.to_i) + end + +end; end diff --git a/lib/leap_cli/commands/clean.rb b/lib/leap_cli/commands/clean.rb new file mode 100644 index 00000000..a9afff53 --- /dev/null +++ b/lib/leap_cli/commands/clean.rb @@ -0,0 +1,16 @@ +module LeapCli + module Commands + + desc 'Removes all files generated with the "compile" command.' + command :clean do |c| + c.action do |global_options,options,args| + Dir.glob(path([:hiera, '*'])).each do |file| + remove_file! file + end + remove_file! path(:authorized_keys) + remove_file! path(:known_hosts) + end + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/compile.rb b/lib/leap_cli/commands/compile.rb new file mode 100644 index 00000000..c388e5c3 --- /dev/null +++ b/lib/leap_cli/commands/compile.rb @@ -0,0 +1,415 @@ +require 'socket' + +module LeapCli + module Commands + + desc "Compile generated files." + command [:compile, :c] do |c| + c.desc 'Compiles node configuration files into hiera files used for deployment.' + c.arg_name 'ENVIRONMENT', :optional => true + c.command :all do |all| + all.action do |global_options,options,args| + environment = args.first + if !LeapCli.leapfile.environment.nil? && !environment.nil? && environment != LeapCli.leapfile.environment + bail! "You cannot specify an ENVIRONMENT argument while the environment is pinned." + end + if environment + if manager.environment_names.include?(environment) + compile_hiera_files(manager.filter([environment]), false) + else + bail! "There is no environment named `#{environment}`." + end + else + clean_export = LeapCli.leapfile.environment.nil? + compile_hiera_files(manager.filter, clean_export) + end + if file_exists?(:static_web_readme) + compile_provider_json(environment) + end + end + end + + c.desc "Compile a DNS zone file for your provider." + c.command :zone do |zone| + zone.action do |global_options, options, args| + compile_zone_file + end + end + + c.desc "Compile provider.json bootstrap files for your provider." + c.command 'provider.json' do |provider| + provider.action do |global_options, options, args| + compile_provider_json + end + end + + c.desc "Generate a list of firewall rules. These rules are already "+ + "implemented on each node, but you might want the list of all "+ + "rules in case you also have a restrictive network firewall." + c.command :firewall do |zone| + zone.action do |global_options, options, args| + compile_firewall + end + end + + c.default_command :all + end + + protected + + # + # a "clean" export of secrets will also remove keys that are no longer used, + # but this should not be done if we are not examining all possible nodes. + # + def compile_hiera_files(nodes, clean_export) + update_compiled_ssh_configs # must come first + sanity_check(nodes) + manager.export_nodes(nodes) + manager.export_secrets(clean_export) + end + + def update_compiled_ssh_configs + generate_monitor_ssh_keys + update_authorized_keys + update_known_hosts + end + + def sanity_check(nodes) + # confirm that every node has a unique ip address + ips = {} + nodes.pick_fields('ip_address').each do |name, ip_address| + if ips.key?(ip_address) + bail! { + log(:fatal_error, "Every node must have its own IP address.") { + log "Nodes `#{name}` and `#{ips[ip_address]}` are both configured with `#{ip_address}`." + } + } + else + ips[ip_address] = name + end + end + # confirm that the IP address of this machine is not also used for a node. + Socket.ip_address_list.each do |addrinfo| + if !addrinfo.ipv4_private? && ips.key?(addrinfo.ip_address) + ip = addrinfo.ip_address + name = ips[ip] + bail! { + log(:fatal_error, "Something is very wrong. The `leap` command must only be run on your sysadmin machine, not on a provider node.") { + log "This machine has the same IP address (#{ip}) as node `#{name}`." + } + } + end + end + end + + ## + ## SSH + ## + + # + # generates a ssh key pair that is used only by remote monitors + # to connect to nodes and run certain allowed commands. + # + # every node has the public monitor key added to their authorized + # keys, and every monitor node has a copy of the private monitor key. + # + def generate_monitor_ssh_keys + priv_key_file = path(:monitor_priv_key) + pub_key_file = path(:monitor_pub_key) + unless file_exists?(priv_key_file, pub_key_file) + ensure_dir(File.dirname(priv_key_file)) + ensure_dir(File.dirname(pub_key_file)) + cmd = %(ssh-keygen -N '' -C 'monitor' -t rsa -b 4096 -f '%s') % priv_key_file + assert_run! cmd + if file_exists?(priv_key_file, pub_key_file) + log :created, priv_key_file + log :created, pub_key_file + else + log :failed, 'to create monitor ssh keys' + end + end + end + + # + # Compiles the authorized keys file, which gets installed on every during init. + # Afterwards, puppet installs an authorized keys file that is generated differently + # (see authorized_keys() in macros.rb) + # + def update_authorized_keys + buffer = StringIO.new + keys = Dir.glob(path([:user_ssh, '*'])) + if keys.empty? + bail! "You must have at least one public SSH user key configured in order to proceed. See `leap help add-user`." + end + if file_exists?(path(:monitor_pub_key)) + keys << path(:monitor_pub_key) + end + keys.sort.each do |keyfile| + ssh_type, ssh_key = File.read(keyfile).strip.split(" ") + buffer << ssh_type + buffer << " " + buffer << ssh_key + buffer << " " + buffer << Path.relative_path(keyfile) + buffer << "\n" + end + write_file!(:authorized_keys, buffer.string) + end + + # + # generates the known_hosts file. + # + # we do a 'late' binding on the hostnames and ip part of the ssh pub key record in order to allow + # for the possibility that the hostnames or ip has changed in the node configuration. + # + def update_known_hosts + buffer = StringIO.new + buffer << "#\n" + buffer << "# This file is automatically generated by the command `leap`. You should NOT modify this file.\n" + buffer << "# Instead, rerun `leap node init` on whatever node is causing SSH problems.\n" + buffer << "#\n" + manager.nodes.keys.sort.each do |node_name| + node = manager.nodes[node_name] + hostnames = [node.name, node.domain.internal, node.domain.full, node.ip_address].join(',') + pub_key = read_file([:node_ssh_pub_key,node.name]) + if pub_key + buffer << [hostnames, pub_key].join(' ') + buffer << "\n" + end + end + write_file!(:known_hosts, buffer.string) + end + + ## + ## provider.json + ## + + # + # generates static provider.json files that can put into place + # (e.g. https://domain/provider.json) for the cases where the + # webapp domain does not match the provider's domain. + # + def compile_provider_json(environments=nil) + webapp_nodes = manager.nodes[:services => 'webapp'] + write_file!(:static_web_readme, STATIC_WEB_README) + environments ||= manager.environment_names + environments.each do |env| + node = webapp_nodes[:environment => env].values.first + if node + env ||= 'default' + write_file!( + [:static_web_provider_json, env], + node['definition_files']['provider'] + ) + write_file!( + [:static_web_htaccess, env], + HTACCESS_FILE % {:min_version => manager.env(env).provider.client_version['min']} + ) + end + end + end + + HTACCESS_FILE = %[ +<Files provider.json> + Header set X-Minimum-Client-Version %{min_version} +</Files> +] + + STATIC_WEB_README = %[ +This directory contains statically rendered copies of the `provider.json` file +used by the client to "bootstrap" configure itself for use with your service +provider. + +There is a separate provider.json file for each environment, although you +should only need 'production/provider.json' or, if you have no environments +configured, 'default/provider.json'. + +To clarify, this is the public `provider.json` file used by the client, not the +`provider.json` file that is used to configure the provider. + +The provider.json file must be available at `https://domain/provider.json` +(unless this provider is included in the list of providers which are pre- +seeded in client). + +This provider.json file can be served correctly in one of three ways: + +(1) If the property webapp.domain is not configured, then the web app will be + installed at https://domain/ and it will handle serving the provider.json file. + +(2) If one or more nodes have the 'static' service configured for the provider's + domain, then these 'static' nodes will correctly serve provider.json. + +(3) Otherwise, you must copy the provider.json file to your web + server and make it available at '/provider.json'. The example htaccess + file shows what header options should be sent by the web server + with the response. + +This directory is needed for method (3), but not for methods (1) or (2). + +This directory has been created by the command `leap compile provider.json`. +Once created, it will be kept up to date everytime you compile. You may safely +remove this directory if you don't use it. +] + + ## + ## + ## ZONE FILE + ## + + def relative_hostname(fqdn, provider) + @domain_regexp ||= /\.?#{Regexp.escape(provider.domain)}$/ + fqdn.sub(@domain_regexp, '') + end + + # + # serial is any number less than 2^32 (4294967296) + # + def compile_zone_file + provider = manager.env('default').provider + hosts_seen = {} + lines = [] + + # + # header + # + lines << ZONE_HEADER % {:domain => provider.domain, :ns => provider.domain, :contact => provider.contacts.default.first.sub('@','.')} + + # + # common records + # + lines << ORIGIN_HEADER + # 'A' records for primary domain + manager.nodes[:environment => '!local'].each_node do |node| + if node.dns['aliases'] && node.dns.aliases.include?(provider.domain) + lines << ["@", "IN A #{node.ip_address}"] + end + end + # NS records + if provider['dns'] && provider.dns['nameservers'] + provider.dns.nameservers.each do |ns| + lines << ["@", "IN NS #{ns}."] + end + end + + # environment records + manager.environment_names.each do |env| + next if env == 'local' + nodes = manager.nodes[:environment => env] + next unless nodes.any? + spf = nil + lines << ENV_HEADER % (env.nil? ? 'default' : env) + nodes.each_node do |node| + if node.dns.public + lines << [relative_hostname(node.domain.full, provider), "IN A #{node.ip_address}"] + end + if node.dns['aliases'] + node.dns.aliases.each do |host_alias| + if host_alias != node.domain.full && host_alias != provider.domain + lines << [relative_hostname(host_alias, provider), "IN A #{node.ip_address}"] + end + end + end + if node.services.include? 'mx' + mx_domain = relative_hostname(node.domain.full_suffix, provider) + lines << [mx_domain, "IN MX 10 #{relative_hostname(node.domain.full, provider)}"] + spf ||= [mx_domain, spf_record(node)] + end + end + lines << spf if spf + end + + # print the lines + max_width = lines.inject(0) {|max, line| line.is_a?(Array) ? [max, line[0].length].max : max} + lines.each do |host, line| + if line.nil? + puts(host) + else + host = '@' if host == '' + puts("%-#{max_width}s %s" % [host, line]) + end + end + end + + # + # allow mail from any mx node, plus the webapp nodes. + # + # TODO: ipv6 + # + def spf_record(node) + ips = node.nodes_like_me['services' => 'webapp'].values.collect {|n| + "ip4:" + n.ip_address + } + # TXT strings may not be longer than 255 characters, although + # you can chain multiple strings together. + strings = "v=spf1 MX #{ips.join(' ')} -all".scan(/.{1,255}/).join('" "') + %(IN TXT "#{strings}") + end + + ENV_HEADER = %[ +;; +;; ENVIRONMENT %s +;; + +] + + ZONE_HEADER = %[ +;; +;; BIND data file for %{domain} +;; + +$TTL 600 +$ORIGIN %{domain}. + +@ IN SOA %{ns}. %{contact}. ( + 0000 ; serial + 7200 ; refresh ( 24 hours) + 3600 ; retry ( 2 hours) + 1209600 ; expire (1000 hours) + 600 ) ; minimum ( 2 days) +; +] + + ORIGIN_HEADER = %[ +;; +;; ZONE ORIGIN +;; + +] + + ## + ## FIREWALL + ## + + def compile_firewall + manager.nodes.each_node(&:evaluate) + + rules = [["ALLOW TO", "PORTS", "ALLOW FROM"]] + manager.nodes[:environment => '!local'].values.each do |node| + next unless node['firewall'] + node.firewall.each do |name, rule| + if rule.is_a? Hash + rules << add_rule(rule) + elsif rule.is_a? Array + rule.each do |r| + rules << add_rule(r) + end + end + end + end + + max_to = rules.inject(0) {|max, r| [max, r[0].length].max} + max_port = rules.inject(0) {|max, r| [max, r[1].length].max} + max_from = rules.inject(0) {|max, r| [max, r[2].length].max} + rules.each do |rule| + puts "%-#{max_to}s %-#{max_port}s %-#{max_from}s" % rule + end + end + + private + + def add_rule(rule) + [rule["to"], [rule["port"]].compact.join(','), rule["from"]] + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/db.rb b/lib/leap_cli/commands/db.rb new file mode 100644 index 00000000..5307ac4d --- /dev/null +++ b/lib/leap_cli/commands/db.rb @@ -0,0 +1,86 @@ +module LeapCli; module Commands + + desc 'Database commands.' + command :db do |db| + db.desc 'Destroy one or more databases. If present, limit to FILTER nodes. For example `leap db destroy --db sessions,tokens testing`.' + db.arg_name 'FILTER', :optional => true + db.command :destroy do |destroy| + destroy.flag :db, :arg_name => "DATABASES", :desc => 'Comma separated list of databases to destroy (no space). Use "--db all" to destroy all databases.', :optional => true + destroy.flag :user, :arg_name => "USERS", :desc => 'Comma separated list of usernames. The storage databases for these user(s) will be destroyed.', :optional => true + destroy.action do |global_options,options,args| + dbs = (options[:db]||"").split(',') + users = (options[:user]||"").split(',') + if dbs.empty? && users.empty? + bail!('Either --db or --user is required.') + end + nodes = manager.filter(args) + if nodes.any? + nodes = nodes[:services => 'couchdb'] + end + unless nodes.any? + bail! 'No db nodes selected.' + end + if users.any? + unless global_options[:yes] + say 'You are about to permanently destroy user databases for [%s] for nodes [%s].' % [users.join(', '), nodes.keys.join(', ')] + bail! unless agree("Continue? ") + end + destroy_user_dbs(nodes, users) + elsif dbs.any? + unless global_options[:yes] + if dbs.include?('all') + say 'You are about to permanently destroy all database data for nodes [%s].' % nodes.keys.join(', ') + else + say 'You are about to permanently destroy databases [%s] for nodes [%s].' % [dbs.join(', '), nodes.keys.join(', ')] + end + bail! unless agree("Continue? ") + end + if dbs.include?('all') + destroy_all_dbs(nodes) + else + destroy_dbs(nodes, dbs) + end + say 'You must run `leap deploy` in order to create the databases again.' + end + end + end + end + + private + + def destroy_all_dbs(nodes) + ssh_connect(nodes) do |ssh| + ssh.run('/etc/init.d/bigcouch stop && test ! -z "$(ls /opt/bigcouch/var/lib/ 2> /dev/null)" && rm -r /opt/bigcouch/var/lib/* && echo "All DBs destroyed" || echo "DBs already destroyed"') + end + end + + def destroy_dbs(nodes, dbs) + nodes.each_node do |node| + ssh_connect(node) do |ssh| + dbs.each do |db| + ssh.run(DESTROY_DB_COMMAND % {:db => db}) + end + end + end + end + + def destroy_user_dbs(nodes, users) + nodes.each_node do |node| + ssh_connect(node) do |ssh| + users.each do |user| + ssh.run(DESTROY_USER_DB_COMMAND % {:user => user}) + end + end + end + end + + DESTROY_DB_COMMAND = %{ +if [ 200 = `curl -ns -w "%%{http_code}" -X GET "127.0.0.1:5984/%{db}" -o /dev/null` ]; then + echo "Result from DELETE /%{db}:" `curl -ns -X DELETE "127.0.0.1:5984/%{db}"`; +else + echo "Skipping db '%{db}': it does not exist or has already been deleted."; +fi +} + + DESTROY_USER_DB_COMMAND = %{/srv/leap/couchdb/scripts/destroy-user-db --username %{user}} +end; end diff --git a/lib/leap_cli/commands/deploy.rb b/lib/leap_cli/commands/deploy.rb new file mode 100644 index 00000000..c2a70afa --- /dev/null +++ b/lib/leap_cli/commands/deploy.rb @@ -0,0 +1,368 @@ +require 'etc' + +module LeapCli + module Commands + + desc 'Apply recipes to a node or set of nodes.' + long_desc 'The FILTER can be the name of a node, service, or tag.' + arg_name 'FILTER' + command [:deploy, :d] do |c| + + c.switch :fast, :desc => 'Makes the deploy command faster by skipping some slow steps. A "fast" deploy can be used safely if you recently completed a normal deploy.', + :negatable => false + + c.switch :sync, :desc => "Sync files, but don't actually apply recipes.", :negatable => false + + c.switch :force, :desc => 'Deploy even if there is a lockfile.', :negatable => false + + c.switch :downgrade, :desc => 'Allows deploy to run with an older platform version.', :negatable => false + + c.switch :dev, :desc => "Development mode: don't run 'git submodule update' before deploy.", :negatable => false + + c.flag :tags, :desc => 'Specify tags to pass through to puppet (overriding the default).', + :arg_name => 'TAG[,TAG]' + + c.flag :port, :desc => 'Override the default SSH port.', + :arg_name => 'PORT' + + c.flag :ip, :desc => 'Override the default SSH IP address.', + :arg_name => 'IPADDRESS' + + c.action do |global,options,args| + + if options[:dev] != true + init_submodules + end + + nodes = manager.filter!(args, :disabled => false) + if nodes.size > 1 + say "Deploying to these nodes: #{nodes.keys.join(', ')}" + if !global[:yes] && !agree("Continue? ") + quit! "OK. Bye." + end + end + + environments = nodes.field('environment').uniq + if environments.empty? + environments = [nil] + end + environments.each do |env| + check_platform_pinning(env, global) + end + # compile hiera files for all the nodes in every environment that is + # being deployed and only those environments. + compile_hiera_files(manager.filter(environments), false) + # update server certificates if needed + update_certificates(nodes) + + ssh_connect(nodes, connect_options(options)) do |ssh| + ssh.leap.log :checking, 'node' do + ssh.leap.check_for_no_deploy + ssh.leap.assert_initialized + end + ssh.leap.log :synching, "configuration files" do + sync_hiera_config(ssh) + sync_support_files(ssh) + end + ssh.leap.log :synching, "puppet manifests" do + sync_puppet_files(ssh) + end + unless options[:sync] + ssh.leap.log :applying, "puppet" do + ssh.puppet.apply(:verbosity => [LeapCli.log_level,5].min, + :tags => tags(options), + :force => options[:force], + :info => deploy_info, + :downgrade => options[:downgrade] + ) + end + end + end + if !Util.exit_status.nil? && Util.exit_status != 0 + log :warning, "puppet did not finish successfully." + end + end + end + + desc 'Display recent deployment history for a set of nodes.' + long_desc 'The FILTER can be the name of a node, service, or tag.' + arg_name 'FILTER' + command [:history, :h] do |c| + c.flag :port, :desc => 'Override the default SSH port.', + :arg_name => 'PORT' + c.flag :ip, :desc => 'Override the default SSH IP address.', + :arg_name => 'IPADDRESS' + c.action do |global,options,args| + nodes = manager.filter!(args) + ssh_connect(nodes, connect_options(options)) do |ssh| + ssh.leap.history + end + end + end + + private + + def forcible_prompt(forced, msg, prompt) + say(msg) + if forced + log :warning, "continuing anyway because of --force" + else + say "hint: use --force to skip this prompt." + quit!("OK. Bye.") unless agree(prompt) + end + end + + # + # The currently activated provider.json could have loaded some pinning + # information for the platform. If this is the case, refuse to deploy + # if there is a mismatch. + # + # For example: + # + # "platform": { + # "branch": "develop" + # "version": "1.0..99" + # "commit": "e1d6280e0a8c565b7fb1a4ed3969ea6fea31a5e2..HEAD" + # } + # + def check_platform_pinning(environment, global_options) + provider = manager.env(environment).provider + return unless provider['platform'] + + if environment.nil? || environment == 'default' + provider_json = 'provider.json' + else + provider_json = 'provider.' + environment + '.json' + end + + # can we have json schema verification already? + unless provider.platform.is_a? Hash + bail!('`platform` attribute in #{provider_json} must be a hash (was %s).' % provider.platform.inspect) + end + + # check version + if provider.platform['version'] + if !Leap::Platform.version_in_range?(provider.platform.version) + forcible_prompt( + global_options[:force], + "The platform is pinned to a version range of '#{provider.platform.version}' "+ + "by the `platform.version` property in #{provider_json}, but the platform "+ + "(#{Path.platform}) has version #{Leap::Platform.version}.", + "Do you really want to deploy from the wrong version? " + ) + end + end + + # check branch + if provider.platform['branch'] + if !is_git_directory?(Path.platform) + forcible_prompt( + global_options[:force], + "The platform is pinned to a particular branch by the `platform.branch` property "+ + "in #{provider_json}, but the platform directory (#{Path.platform}) is not a git repository.", + "Do you really want to deploy anyway? " + ) + end + unless provider.platform.branch == current_git_branch(Path.platform) + forcible_prompt( + global_options[:force], + "The platform is pinned to branch '#{provider.platform.branch}' by the `platform.branch` property "+ + "in #{provider_json}, but the current branch is '#{current_git_branch(Path.platform)}' " + + "(for directory '#{Path.platform}')", + "Do you really want to deploy from the wrong branch? " + ) + end + end + + # check commit + if provider.platform['commit'] + if !is_git_directory?(Path.platform) + forcible_prompt( + global_options[:force], + "The platform is pinned to a particular commit range by the `platform.commit` property "+ + "in #{provider_json}, but the platform directory (#{Path.platform}) is not a git repository.", + "Do you really want to deploy anyway? " + ) + end + current_commit = current_git_commit(Path.platform) + Dir.chdir(Path.platform) do + commit_range = assert_run!("git log --pretty='format:%H' '#{provider.platform.commit}'", + "The platform is pinned to a particular commit range by the `platform.commit` property "+ + "in #{provider_json}, but git was not able to find commits in the range specified "+ + "(#{provider.platform.commit}).") + commit_range = commit_range.split("\n") + if !commit_range.include?(current_commit) && + provider.platform.commit.split('..').first != current_commit + forcible_prompt( + global_options[:force], + "The platform is pinned via the `platform.commit` property in #{provider_json} " + + "to a commit in the range #{provider.platform.commit}, but the current HEAD " + + "(#{current_commit}) is not in that range.", + "Do you really want to deploy from the wrong commit? " + ) + end + end + end + end + + def sync_hiera_config(ssh) + ssh.rsync.update do |server| + node = manager.node(server.host) + hiera_file = Path.relative_path([:hiera, node.name]) + ssh.leap.log hiera_file + ' -> ' + node.name + ':' + Leap::Platform.hiera_path + { + :source => hiera_file, + :dest => Leap::Platform.hiera_path, + :flags => "-rltp --chmod=u+rX,go-rwx" + } + end + end + + # + # sync various support files. + # + def sync_support_files(ssh) + dest_dir = Leap::Platform.files_dir + custom_files = build_custom_file_list + ssh.rsync.update do |server| + node = manager.node(server.host) + files_to_sync = node.file_paths.collect {|path| Path.relative_path(path, Path.provider) } + files_to_sync += custom_files + if files_to_sync.any? + ssh.leap.log(files_to_sync.join(', ') + ' -> ' + node.name + ':' + dest_dir) + { + :chdir => Path.named_path(:files_dir), + :source => ".", + :dest => dest_dir, + :excludes => "*", + :includes => calculate_includes_from_files(files_to_sync, '/files'), + :flags => "-rltp --chmod=u+rX,go-rwx --relative --delete --delete-excluded --copy-links" + } + else + nil + end + end + end + + def sync_puppet_files(ssh) + ssh.rsync.update do |server| + ssh.leap.log(Path.platform + '/[bin,tests,puppet] -> ' + server.host + ':' + Leap::Platform.leap_dir) + { + :dest => Leap::Platform.leap_dir, + :source => '.', + :chdir => Path.platform, + :excludes => '*', + :includes => ['/bin', '/bin/**', '/puppet', '/puppet/**', '/tests', '/tests/**'], + :flags => "-rlt --relative --delete --copy-links" + } + end + end + + # + # ensure submodules are up to date, if the platform is a git + # repository. + # + def init_submodules + return unless is_git_directory?(Path.platform) + Dir.chdir Path.platform do + assert_run! "git submodule sync" + statuses = assert_run! "git submodule status" + statuses.strip.split("\n").each do |status_line| + if status_line =~ /^[\+-]/ + submodule = status_line.split(' ')[1] + log "Updating submodule #{submodule}" + assert_run! "git submodule update --init #{submodule}" + end + end + end + end + + # + # converts an array of file paths into an array + # suitable for --include of rsync + # + # if set, `prefix` is stripped off. + # + def calculate_includes_from_files(files, prefix=nil) + return nil unless files and files.any? + + # prepend '/' (kind of like ^ for rsync) + includes = files.collect {|file| file =~ /^\// ? file : '/' + file } + + # include all sub files of specified directories + includes.size.times do |i| + if includes[i] =~ /\/$/ + includes << includes[i] + '**' + end + end + + # include all parent directories (required because of --exclude '*') + includes.size.times do |i| + path = File.dirname(includes[i]) + while(path != '/') + includes << path unless includes.include?(path) + path = File.dirname(path) + end + end + + if prefix + includes.map! {|path| path.sub(/^#{Regexp.escape(prefix)}\//, '/')} + end + + return includes + end + + def tags(options) + if options[:tags] + tags = options[:tags].split(',') + else + tags = Leap::Platform.default_puppet_tags.dup + end + tags << 'leap_slow' unless options[:fast] + tags.join(',') + end + + # + # a provider might have various customization files that should be sync'ed to the server. + # this method builds that list of files to sync. + # + def build_custom_file_list + custom_files = [] + Leap::Platform.paths.keys.grep(/^custom_/).each do |path| + if file_exists?(path) + relative_path = Path.relative_path(path, Path.provider) + if dir_exists?(path) + custom_files << relative_path + '/' # rsync needs trailing slash + else + custom_files << relative_path + end + end + end + return custom_files + end + + def deploy_info + info = [] + info << "user: %s" % Etc.getpwuid(Process.euid).name + if is_git_directory?(Path.platform) && current_git_branch(Path.platform) != 'master' + info << "platform: %s (%s %s)" % [ + Leap::Platform.version, + current_git_branch(Path.platform), + current_git_commit(Path.platform)[0..4] + ] + else + info << "platform: %s" % Leap::Platform.version + end + if is_git_directory?(LEAP_CLI_BASE_DIR) + info << "leap_cli: %s (%s %s)" % [ + LeapCli::VERSION, + current_git_branch(LEAP_CLI_BASE_DIR), + current_git_commit(LEAP_CLI_BASE_DIR)[0..4] + ] + else + info << "leap_cli: %s" % LeapCli::VERSION + end + info.join(', ') + end + end +end diff --git a/lib/leap_cli/commands/env.rb b/lib/leap_cli/commands/env.rb new file mode 100644 index 00000000..80be2174 --- /dev/null +++ b/lib/leap_cli/commands/env.rb @@ -0,0 +1,76 @@ +module LeapCli + module Commands + + desc "Manipulate and query environment information." + long_desc "The 'environment' node property can be used to isolate sets of nodes into entirely separate environments. "+ + "A node in one environment will never interact with a node from another environment. "+ + "Environment pinning works by modifying your ~/.leaprc file and is dependent on the "+ + "absolute file path of your provider directory (pins don't apply if you move the directory)" + command [:env, :e] do |c| + c.desc "List the available environments. The pinned environment, if any, will be marked with '*'. Will also set the pin if run with an environment argument." + c.arg_name 'ENVIRONMENT', :optional => true + c.command :ls do |ls| + ls.action do |global_options, options, args| + environment = get_env_from_args(args) + if environment + pin(environment) + LeapCli.leapfile.load + end + print_envs + end + end + + c.desc 'Pin the environment to ENVIRONMENT. All subsequent commands will only apply to nodes in this environment.' + c.arg_name 'ENVIRONMENT' + c.command :pin do |pin| + pin.action do |global_options,options,args| + environment = get_env_from_args(args) + if environment + pin(environment) + else + bail! "There is no environment `#{environment}`" + end + end + end + + c.desc "Unpin the environment. All subsequent commands will apply to all nodes." + c.command :unpin do |unpin| + unpin.action do |global_options, options, args| + LeapCli.leapfile.unset('environment') + log 0, :saved, "~/.leaprc, removing environment property." + end + end + + c.default_command :ls + end + + protected + + def get_env_from_args(args) + environment = args.first + if environment == 'default' || (environment && manager.environment_names.include?(environment)) + return environment + else + return nil + end + end + + def pin(environment) + LeapCli.leapfile.set('environment', environment) + log 0, :saved, "~/.leaprc with environment set to #{environment}." + end + + def print_envs + envs = ["default"] + manager.environment_names.compact.sort + envs.each do |env| + if env + if LeapCli.leapfile.environment == env + puts "* #{env}" + else + puts " #{env}" + end + end + end + end + end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/facts.rb b/lib/leap_cli/commands/facts.rb new file mode 100644 index 00000000..11329ccc --- /dev/null +++ b/lib/leap_cli/commands/facts.rb @@ -0,0 +1,100 @@ +# +# Gather facter facts +# + +module LeapCli; module Commands + + desc 'Gather information on nodes.' + command :facts do |facts| + facts.desc 'Query servers to update facts.json.' + facts.long_desc "Queries every node included in FILTER and saves the important information to facts.json" + facts.arg_name 'FILTER' + facts.command :update do |update| + update.action do |global_options,options,args| + update_facts(global_options, options, args) + end + end + end + + protected + + def facter_cmd + 'facter --json ' + Leap::Platform.facts.join(' ') + end + + def remove_node_facts(name) + if file_exists?(:facts) + update_facts_file({name => nil}) + end + end + + def update_node_facts(name, facts) + update_facts_file({name => facts}) + end + + def rename_node_facts(old_name, new_name) + if file_exists?(:facts) + facts = JSON.parse(read_file(:facts) || {}) + facts[new_name] = facts[old_name] + facts[old_name] = nil + update_facts_file(facts, true) + end + end + + # + # if overwrite = true, then ignore existing facts.json. + # + def update_facts_file(new_facts, overwrite=false) + replace_file!(:facts) do |content| + if overwrite || content.nil? || content.empty? + old_facts = {} + else + old_facts = manager.facts + end + facts = old_facts.merge(new_facts) + facts.each do |name, value| + if value.is_a? String + if value == "" + value = nil + else + value = JSON.parse(value) rescue JSON::ParserError + end + end + if value.is_a? Hash + value.delete_if {|key,v| v.nil?} + end + facts[name] = value + end + facts.delete_if do |name, value| + value.nil? || value.empty? + end + if facts.empty? + "{}\n" + else + JSON.sorted_generate(facts) + "\n" + end + end + end + + private + + def update_facts(global_options, options, args) + nodes = manager.filter(args, :local => false, :disabled => false) + new_facts = {} + ssh_connect(nodes) do |ssh| + ssh.leap.run_with_progress(facter_cmd) do |response| + node = manager.node(response[:host]) + if node + new_facts[node.name] = response[:data].strip + else + log :warning, 'Could not find node for hostname %s' % response[:host] + end + end + end + # only overwrite the entire facts file if and only if we are gathering facts + # for all nodes in all environments. + overwrite_existing = args.empty? && LeapCli.leapfile.environment.nil? + update_facts_file(new_facts, overwrite_existing) + end + +end; end
\ No newline at end of file diff --git a/lib/leap_cli/commands/inspect.rb b/lib/leap_cli/commands/inspect.rb new file mode 100644 index 00000000..20654fa7 --- /dev/null +++ b/lib/leap_cli/commands/inspect.rb @@ -0,0 +1,144 @@ +module LeapCli; module Commands + + desc 'Prints details about a file. Alternately, the argument FILE can be the name of a node, service or tag.' + arg_name 'FILE' + command [:inspect, :i] do |c| + c.switch 'base', :desc => 'Inspect the FILE from the provider_base (i.e. without local inheritance).', :negatable => false + c.action do |global_options,options,args| + object = args.first + assert! object, 'A file path or node/service/tag name is required' + method = inspection_method(object) + if method && defined?(method) + self.send(method, object, options) + else + log "Sorry, I don't know how to inspect that." + end + end + end + + private + + FTYPE_MAP = { + "PEM certificate" => :inspect_x509_cert, + "PEM RSA private key" => :inspect_x509_key, + "OpenSSH RSA public key" => :inspect_ssh_pub_key, + "PEM certificate request" => :inspect_x509_csr + } + + def inspection_method(object) + if File.exists?(object) + ftype = `file #{object}`.split(':').last.strip + log 2, "file is of type '#{ftype}'" + if FTYPE_MAP[ftype] + FTYPE_MAP[ftype] + elsif File.extname(object) == ".json" + full_path = File.expand_path(object, Dir.pwd) + if path_match?(:node_config, full_path) + :inspect_node + elsif path_match?(:service_config, full_path) + :inspect_service + elsif path_match?(:tag_config, full_path) + :inspect_tag + elsif path_match?(:provider_config, full_path) || path_match?(:provider_env_config, full_path) + :inspect_provider + elsif path_match?(:common_config, full_path) + :inspect_common + else + nil + end + end + elsif manager.nodes[object] + :inspect_node + elsif manager.services[object] + :inspect_service + elsif manager.tags[object] + :inspect_tag + elsif object == "common" + :inspect_common + elsif object == "provider" + :inspect_provider + else + nil + end + end + + # + # inspectors + # + + def inspect_x509_key(file_path, options) + assert_bin! 'openssl' + puts assert_run! 'openssl rsa -in %s -text -check' % file_path + end + + def inspect_x509_cert(file_path, options) + assert_bin! 'openssl' + puts assert_run! 'openssl x509 -in %s -text -noout' % file_path + log 0, :"SHA256 fingerprint", X509.fingerprint("SHA256", file_path) + end + + def inspect_x509_csr(file_path, options) + assert_bin! 'openssl' + puts assert_run! 'openssl req -text -noout -verify -in %s' % file_path + end + + #def inspect_ssh_pub_key(file_path) + #end + + def inspect_node(arg, options) + inspect_json manager.nodes[name(arg)] + end + + def inspect_service(arg, options) + if options[:base] + inspect_json manager.base_services[name(arg)] + else + inspect_json manager.services[name(arg)] + end + end + + def inspect_tag(arg, options) + if options[:base] + inspect_json manager.base_tags[name(arg)] + else + inspect_json manager.tags[name(arg)] + end + end + + def inspect_provider(arg, options) + if options[:base] + inspect_json manager.base_provider + elsif arg =~ /provider\.(.*)\.json/ + inspect_json manager.env($1).provider + else + inspect_json manager.provider + end + end + + def inspect_common(arg, options) + if options[:base] + inspect_json manager.base_common + else + inspect_json manager.common + end + end + + # + # helpers + # + + def name(arg) + File.basename(arg).sub(/\.json$/, '') + end + + def inspect_json(config) + if config + puts JSON.sorted_generate(config) + end + end + + def path_match?(path_symbol, path) + Dir.glob(Path.named_path([path_symbol, '*'])).include?(path) + end + +end; end diff --git a/lib/leap_cli/commands/list.rb b/lib/leap_cli/commands/list.rb new file mode 100644 index 00000000..c562b59b --- /dev/null +++ b/lib/leap_cli/commands/list.rb @@ -0,0 +1,132 @@ +require 'command_line_reporter' + +module LeapCli; module Commands + + desc 'List nodes and their classifications' + long_desc 'Prints out a listing of nodes, services, or tags. ' + + 'If present, the FILTER can be a list of names of nodes, services, or tags. ' + + 'If the name is prefixed with +, this acts like an AND condition. ' + + "For example:\n\n" + + "`leap list node1 node2` matches all nodes named \"node1\" OR \"node2\"\n\n" + + "`leap list openvpn +local` matches all nodes with service \"openvpn\" AND tag \"local\"" + + arg_name 'FILTER', :optional => true + command [:list,:ls] do |c| + c.flag 'print', :desc => 'What attributes to print (optional)' + c.switch 'disabled', :desc => 'Include disabled nodes in the list.', :negatable => false + c.action do |global_options,options,args| + # don't rely on default manager(), because we want to pass custom options to load() + manager = LeapCli::Config::Manager.new + if global_options[:color] + colors = ['cyan', 'white'] + else + colors = [nil, nil] + end + puts + manager.load(:include_disabled => options['disabled'], :continue_on_error => true) + if options['print'] + print_node_properties(manager.filter(args), options['print']) + else + if args.any? + NodeTable.new(manager.filter(args), colors).run + else + environment = LeapCli.leapfile.environment || '_all_' + TagTable.new('SERVICES', manager.env(environment).services, colors).run + TagTable.new('TAGS', manager.env(environment).tags, colors).run + NodeTable.new(manager.filter(), colors).run + end + end + end + end + + private + + def self.print_node_properties(nodes, properties) + properties = properties.split(',') + max_width = nodes.keys.inject(0) {|max,i| [i.size,max].max} + nodes.each_node do |node| + value = properties.collect{|prop| + prop_value = node[prop] + if prop_value.nil? + "null" + elsif prop_value == "" + "empty" + elsif prop_value.is_a? LeapCli::Config::Object + node[prop].dump_json(:compact) # TODO: add option of getting pre-evaluation values. + else + prop_value.to_s + end + }.join(', ') + printf("%#{max_width}s %s\n", node.name, value) + end + puts + end + + class TagTable + include CommandLineReporter + def initialize(heading, tag_list, colors) + @heading = heading + @tag_list = tag_list + @colors = colors + end + def run + tags = @tag_list.keys.select{|tag| tag !~ /^_/}.sort # sorted list of tags, excluding _partials + max_width = [20, (tags+[@heading]).inject(0) {|max,i| [i.size,max].max}].max + table :border => false do + row :color => @colors[0] do + column @heading, :align => 'right', :width => max_width + column "NODES", :width => HighLine::SystemExtensions.terminal_size.first - max_width - 2, :padding => 2 + end + tags.each do |tag| + next if @tag_list[tag].node_list.empty? + row :color => @colors[1] do + column tag + column @tag_list[tag].node_list.keys.sort.join(', ') + end + end + end + vertical_spacing + end + end + + # + # might be handy: HighLine::SystemExtensions.terminal_size.first + # + class NodeTable + include CommandLineReporter + def initialize(node_list, colors) + @node_list = node_list + @colors = colors + end + def run + rows = @node_list.keys.sort.collect do |node_name| + [node_name, @node_list[node_name].services.sort.join(', '), @node_list[node_name].tags.sort.join(', ')] + end + unless rows.any? + puts Paint["no results", :red] + puts + return + end + padding = 2 + max_node_width = [20, (rows.map{|i|i[0]} + ["NODES"] ).inject(0) {|max,i| [i.size,max].max}].max + max_service_width = (rows.map{|i|i[1]} + ["SERVICES"]).inject(0) {|max,i| [i.size+padding+padding,max].max} + max_tag_width = (rows.map{|i|i[2]} + ["TAGS"] ).inject(0) {|max,i| [i.size,max].max} + table :border => false do + row :color => @colors[0] do + column "NODES", :align => 'right', :width => max_node_width + column "SERVICES", :width => max_service_width, :padding => 2 + column "TAGS", :width => max_tag_width + end + rows.each do |r| + row :color => @colors[1] do + column r[0] + column r[1] + column r[2] + end + end + end + vertical_spacing + end + end + +end; end diff --git a/lib/leap_cli/commands/node.rb b/lib/leap_cli/commands/node.rb new file mode 100644 index 00000000..12d6b49d --- /dev/null +++ b/lib/leap_cli/commands/node.rb @@ -0,0 +1,165 @@ +# +# fyi: the `node init` command lives in node_init.rb, +# but all other `node x` commands live here. +# + +autoload :IPAddr, 'ipaddr' + +module LeapCli; module Commands + + ## + ## COMMANDS + ## + + desc 'Node management' + command [:node, :n] do |node| + node.desc 'Create a new configuration file for a node named NAME.' + node.long_desc ["If specified, the optional argument SEED can be used to seed values in the node configuration file.", + "The format is property_name:value.", + "For example: `leap node add web1 ip_address:1.2.3.4 services:webapp`.", + "To set nested properties, property name can contain '.', like so: `leap node add web1 ssh.port:44`", + "Separeate multiple values for a single property with a comma, like so: `leap node add mynode services:webapp,dns`"].join("\n\n") + node.arg_name 'NAME [SEED]' # , :optional => false, :multiple => false + node.command :add do |add| + add.switch :local, :desc => 'Make a local testing node (by automatically assigning the next available local IP address). Local nodes are run as virtual machines on your computer.', :negatable => false + add.action do |global_options,options,args| + # argument sanity checks + name = args.first + assert_valid_node_name!(name, options[:local]) + assert_files_missing! [:node_config, name] + + # create and seed new node + node = Config::Node.new(manager) + if options[:local] + node['ip_address'] = pick_next_vagrant_ip_address + end + seed_node_data(node, args[1..-1]) + validate_ip_address(node) + begin + write_file! [:node_config, name], node.dump_json + "\n" + node['name'] = name + if file_exists? :ca_cert, :ca_key + generate_cert_for_node(manager.reload_node!(node)) + end + rescue LeapCli::ConfigError => exc + remove_node_files(name) + end + end + end + + node.desc 'Renames a node file, and all its related files.' + node.arg_name 'OLD_NAME NEW_NAME' + node.command :mv do |mv| + mv.action do |global_options,options,args| + node = get_node_from_args(args) + new_name = args.last + assert_valid_node_name!(new_name, node.vagrant?) + ensure_dir [:node_files_dir, new_name] + Leap::Platform.node_files.each do |path| + rename_file! [path, node.name], [path, new_name] + end + remove_directory! [:node_files_dir, node.name] + rename_node_facts(node.name, new_name) + end + end + + node.desc 'Removes all the files related to the node named NAME.' + node.arg_name 'NAME' #:optional => false #, :multiple => false + node.command :rm do |rm| + rm.action do |global_options,options,args| + node = get_node_from_args(args) + remove_node_files(node.name) + if node.vagrant? + vagrant_command("destroy --force", [node.name]) + end + remove_node_facts(node.name) + end + end + end + + ## + ## PUBLIC HELPERS + ## + + def get_node_from_args(args, options={}) + node_name = args.first + node = manager.node(node_name) + if node.nil? && options[:include_disabled] + node = manager.disabled_node(node_name) + end + assert!(node, "Node '#{node_name}' not found.") + node + end + + def seed_node_data(node, args) + args.each do |seed| + key, value = seed.split(':') + value = format_seed_value(value) + assert! key =~ /^[0-9a-z\._]+$/, "illegal characters used in property '#{key}'" + if key =~ /\./ + key_parts = key.split('.') + final_key = key_parts.pop + current_object = node + key_parts.each do |key_part| + current_object[key_part] ||= Config::Object.new + current_object = current_object[key_part] + end + current_object[final_key] = value + else + node[key] = value + end + end + end + + def remove_node_files(node_name) + (Leap::Platform.node_files + [:node_files_dir]).each do |path| + remove_file! [path, node_name] + end + end + + # + # conversions: + # + # "x,y,z" => ["x","y","z"] + # + # "22" => 22 + # + # "5.1" => 5.1 + # + def format_seed_value(v) + if v =~ /,/ + v = v.split(',') + v.map! do |i| + i = i.to_i if i.to_i.to_s == i + i = i.to_f if i.to_f.to_s == i + i + end + else + v = v.to_i if v.to_i.to_s == v + v = v.to_f if v.to_f.to_s == v + end + return v + end + + def validate_ip_address(node) + IPAddr.new(node['ip_address']) + rescue ArgumentError + bail! do + if node['ip_address'] + log :invalid, "ip_address #{node['ip_address'].inspect}" + else + log :missing, "ip_address" + end + end + end + + def assert_valid_node_name!(name, local=false) + assert! name, 'No <node-name> specified.' + if local + assert! name =~ /^[0-9a-z]+$/, "illegal characters used in node name '#{name}' (note: Vagrant does not allow hyphens or underscores)" + else + assert! name =~ /^[0-9a-z-]+$/, "illegal characters used in node name '#{name}' (note: Linux does not allow underscores)" + end + end + +end; end
\ No newline at end of file diff --git a/lib/leap_cli/commands/node_init.rb b/lib/leap_cli/commands/node_init.rb new file mode 100644 index 00000000..33f6288d --- /dev/null +++ b/lib/leap_cli/commands/node_init.rb @@ -0,0 +1,169 @@ +# +# Node initialization. +# Most of the fun stuff is in tasks.rb. +# + +module LeapCli; module Commands + + desc 'Node management' + command :node do |node| + node.desc 'Bootstraps a node or nodes, setting up SSH keys and installing prerequisite packages' + node.long_desc "This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, " + + "copying the authorized_keys file, installing packages that are required for deploying, and registering important facts. " + + "Node init must be run before deploying to a server, and the server must be running and available via the network. " + + "This command only needs to be run once, but there is no harm in running it multiple times." + node.arg_name 'FILTER' + node.command :init do |init| + init.switch 'echo', :desc => 'If set, passwords are visible as you type them (default is hidden)', :negatable => false + init.flag :port, :desc => 'Override the default SSH port.', :arg_name => 'PORT' + init.flag :ip, :desc => 'Override the default SSH IP address.', :arg_name => 'IPADDRESS' + + init.action do |global,options,args| + assert! args.any?, 'You must specify a FILTER' + finished = [] + manager.filter!(args).each_node do |node| + is_node_alive(node, options) + save_public_host_key(node, global, options) unless node.vagrant? + update_compiled_ssh_configs + ssh_connect_options = connect_options(options).merge({:bootstrap => true, :echo => options[:echo]}) + ssh_connect(node, ssh_connect_options) do |ssh| + if node.vagrant? + ssh.install_insecure_vagrant_key + end + ssh.install_authorized_keys + ssh.install_prerequisites + unless node.vagrant? + ssh.leap.log(:checking, "SSH host keys") do + ssh.leap.capture(get_ssh_keys_cmd) do |response| + update_local_ssh_host_keys(node, response[:data]) if response[:exitcode] == 0 + end + end + end + ssh.leap.log(:updating, "facts") do + ssh.leap.capture(facter_cmd) do |response| + if response[:exitcode] == 0 + update_node_facts(node.name, response[:data]) + else + log :failed, "to run facter on #{node.name}" + end + end + end + end + finished << node.name + end + log :completed, "initialization of nodes #{finished.join(', ')}" + end + end + end + + private + + ## + ## PRIVATE HELPERS + ## + + def is_node_alive(node, options) + address = options[:ip] || node.ip_address + port = options[:port] || node.ssh.port + log :connecting, "to node #{node.name}" + assert_run! "nc -zw3 #{address} #{port}", + "Failed to reach #{node.name} (address #{address}, port #{port}). You can override the configured IP address and port with --ip or --port." + end + + # + # saves the public ssh host key for node into the provider directory. + # + # see `man sshd` for the format of known_hosts + # + def save_public_host_key(node, global, options) + log :fetching, "public SSH host key for #{node.name}" + address = options[:ip] || node.ip_address + port = options[:port] || node.ssh.port + host_keys = get_public_keys_for_ip(address, port) + pub_key_path = Path.named_path([:node_ssh_pub_key, node.name]) + + if Path.exists?(pub_key_path) + if host_keys.include? SshKey.load(pub_key_path) + log :trusted, "- Public SSH host key for #{node.name} matches previously saved key", :indent => 1 + else + bail! do + log :error, "The public SSH host keys we just fetched for #{node.name} doesn't match what we have saved previously.", :indent => 1 + log "Delete the file #{pub_key_path} if you really want to remove the trusted SSH host key.", :indent => 2 + end + end + else + known_key = host_keys.detect{|k|k.in_known_hosts?(node.name, node.ip_address, node.domain.name)} + if known_key + log :trusted, "- Public SSH host key for #{node.name} is trusted (key found in your ~/.ssh/known_hosts)" + else + public_key = SshKey.pick_best_key(host_keys) + if public_key.nil? + bail!("We got back #{host_keys.size} host keys from #{node.name}, but we can't support any of them.") + else + say(" This is the SSH host key you got back from node \"#{node.name}\"") + say(" Type -- #{public_key.bits} bit #{public_key.type.upcase}") + say(" Fingerprint -- " + public_key.fingerprint) + say(" Public Key -- " + public_key.key) + if !global[:yes] && !agree(" Is this correct? ") + bail! + else + known_key = public_key + end + end + end + puts + write_file! [:node_ssh_pub_key, node.name], known_key.to_s + end + end + + # + # Get the public host keys for a host using ssh-keyscan. + # Return an array of SshKey objects, one for each key. + # + def get_public_keys_for_ip(address, port=22) + assert_bin!('ssh-keyscan') + output = assert_run! "ssh-keyscan -p #{port} #{address}", "Could not get the public host key from #{address}:#{port}. Maybe sshd is not running?" + if output.empty? + bail! :failed, "ssh-keyscan returned empty output." + end + + if output =~ /No route to host/ + bail! :failed, 'ssh-keyscan: no route to %s' % address + else + keys = SshKey.parse_keys(output) + if keys.empty? + bail! "ssh-keyscan got zero host keys back (that we understand)! Output was: #{output}" + else + return keys + end + end + end + + # run on the server to generate a string suitable for passing to SshKey.parse_keys() + def get_ssh_keys_cmd + "/bin/grep ^HostKey /etc/ssh/sshd_config | /usr/bin/awk '{print $2 \".pub\"}' | /usr/bin/xargs /bin/cat" + end + + # + # Sometimes the ssh host keys on the server will be better than what we have + # stored locally. In these cases, ask the user if they want to upgrade. + # + def update_local_ssh_host_keys(node, remote_keys_string) + remote_keys = SshKey.parse_keys(remote_keys_string) + return unless remote_keys.any? + current_key = SshKey.load(Path.named_path([:node_ssh_pub_key, node.name])) + best_key = SshKey.pick_best_key(remote_keys) + return unless best_key && current_key + if current_key != best_key + say(" One of the SSH host keys for node '#{node.name}' is better than what you currently have trusted.") + say(" Current key: #{current_key.summary}") + say(" Better key: #{best_key.summary}") + if agree(" Do you want to use the better key? ") + write_file! [:node_ssh_pub_key, node.name], best_key.to_s + end + else + log(3, "current host key does not need updating") + end + end + +end; end diff --git a/lib/leap_cli/commands/ssh.rb b/lib/leap_cli/commands/ssh.rb new file mode 100644 index 00000000..3887618e --- /dev/null +++ b/lib/leap_cli/commands/ssh.rb @@ -0,0 +1,225 @@ +module LeapCli; module Commands + + desc 'Log in to the specified node with an interactive shell.' + arg_name 'NAME' #, :optional => false, :multiple => false + command :ssh do |c| + c.flag 'ssh', :desc => "Pass through raw options to ssh (e.g. `--ssh '-F ~/sshconfig'`)." + c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`.' + c.action do |global_options,options,args| + exec_ssh(:ssh, options, args) + end + end + + desc 'Log in to the specified node with an interactive shell using mosh (requires node to have mosh.enabled set to true).' + arg_name 'NAME' + command :mosh do |c| + c.flag 'ssh', :desc => "Pass through raw options to ssh (e.g. `--ssh '-F ~/sshconfig'`)." + c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`.' + c.action do |global_options,options,args| + exec_ssh(:mosh, options, args) + end + end + + desc 'Creates an SSH port forward (tunnel) to the node NAME. REMOTE_PORT is the port on the remote node that the tunnel will connect to. LOCAL_PORT is the optional port on your local machine. For example: `leap tunnel couch1:5984`.' + arg_name '[LOCAL_PORT:]NAME:REMOTE_PORT' + command :tunnel do |c| + c.flag 'ssh', :desc => "Pass through raw options to ssh (e.g. --ssh '-F ~/sshconfig')." + c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`.' + c.action do |global_options,options,args| + local_port, node, remote_port = parse_tunnel_arg(args.first) + unless node.ssh.config.AllowTcpForwarding == "yes" + log :warning, "It looks like TCP forwarding is not enabled. "+ + "The tunnel command requires that the node property ssh.config.AllowTcpForwarding "+ + "be set to 'yes'. Add this property to #{node.name}.json, deploy, and then try tunnel again." + end + options[:ssh] = [options[:ssh], "-N -L 127.0.0.1:#{local_port}:0.0.0.0:#{remote_port}"].join(' ') + log("Forward port localhost:#{local_port} to #{node.name}:#{remote_port}") + if is_port_available?(local_port) + exec_ssh(:ssh, options, [node.name]) + end + end + end + + desc 'Secure copy from FILE1 to FILE2. Files are specified as NODE_NAME:FILE_PATH. For local paths, omit "NODE_NAME:".' + arg_name 'FILE1 FILE2' + command :scp do |c| + c.switch :r, :desc => 'Copy recursively' + c.action do |global_options, options, args| + if args.size != 2 + bail!('You must specificy both FILE1 and FILE2') + end + from, to = args + if (from !~ /:/ && to !~ /:/) || (from =~ /:/ && to =~ /:/) + bail!('One FILE must be remote and the other local.') + end + src_node_name = src_file_path = src_node = nil + dst_node_name = dst_file_path = dst_node = nil + if from =~ /:/ + src_node_name, src_file_path = from.split(':') + src_node = get_node_from_args([src_node_name], :include_disabled => true) + dst_file_path = to + else + dst_node_name, dst_file_path = to.split(':') + dst_node = get_node_from_args([dst_node_name], :include_disabled => true) + src_file_path = from + end + exec_scp(options, src_node, src_file_path, dst_node, dst_file_path) + end + end + + protected + + # + # allow for ssh overrides of all commands that use ssh_connect + # + def connect_options(options) + connect_options = {:ssh_options=>{}} + if options[:port] + connect_options[:ssh_options][:port] = options[:port] + end + if options[:ip] + connect_options[:ssh_options][:host_name] = options[:ip] + end + return connect_options + end + + def ssh_config_help_message + puts "" + puts "Are 'too many authentication failures' getting you down?" + puts "Then we have the solution for you! Add something like this to your ~/.ssh/config file:" + puts " Host *.#{manager.provider.domain}" + puts " IdentityFile ~/.ssh/id_rsa" + puts " IdentitiesOnly=yes" + puts "(replace `id_rsa` with the actual private key filename that you use for this provider)" + end + + require 'socket' + def is_port_available?(port) + TCPServer.open('127.0.0.1', port) {} + true + rescue Errno::EACCES + bail!("You don't have permission to bind to port #{port}.") + rescue Errno::EADDRINUSE + bail!("Local port #{port} is already in use. Specify LOCAL_PORT to pick another.") + rescue Exception => exc + bail!(exc.to_s) + end + + private + + def exec_ssh(cmd, cli_options, args) + node = get_node_from_args(args, :include_disabled => true) + port = node.ssh.port + options = ssh_config(node) + username = 'root' + if LeapCli.log_level >= 3 + options << "-vv" + elsif LeapCli.log_level >= 2 + options << "-v" + end + if cli_options[:port] + port = cli_options[:port] + end + if cli_options[:ssh] + options << cli_options[:ssh] + end + ssh = "ssh -l #{username} -p #{port} #{options.join(' ')}" + if cmd == :ssh + command = "#{ssh} #{node.domain.full}" + elsif cmd == :mosh + command = "MOSH_TITLE_NOPREFIX=1 mosh --ssh \"#{ssh}\" #{node.domain.full}" + end + log 2, command + + # exec the shell command in a subprocess + pid = fork { exec "#{command}" } + + Signal.trap("SIGINT") do + Process.kill("KILL", pid) + Process.wait(pid) + exit(0) + end + + # wait for shell to exit so we can grab the exit status + _, status = Process.waitpid2(pid) + + if status.exitstatus == 255 + ssh_config_help_message + elsif status.exitstatus != 0 + exit(status.exitstatus) + end + end + + def exec_scp(cli_options, src_node, src_file_path, dst_node, dst_file_path) + node = src_node || dst_node + options = ssh_config(node) + port = node.ssh.port + username = 'root' + options << "-r" if cli_options[:r] + scp = "scp -P #{port} #{options.join(' ')}" + if src_node + command = "#{scp} #{username}@#{src_node.domain.full}:#{src_file_path} #{dst_file_path}" + elsif dst_node + command = "#{scp} #{src_file_path} #{username}@#{dst_node.domain.full}:#{dst_file_path}" + end + log 2, command + + # exec the shell command in a subprocess + pid = fork { exec "#{command}" } + + Signal.trap("SIGINT") do + Process.kill("KILL", pid) + Process.wait(pid) + exit(0) + end + + # wait for shell to exit so we can grab the exit status + _, status = Process.waitpid2(pid) + exit(status.exitstatus) + end + + # + # SSH command line -o options. See `man ssh_config` + # + # NOTES: + # + # The option 'HostKeyAlias=#{node.name}' is oddly incompatible with ports in + # known_hosts file, so we must not use this or non-standard ports break. + # + def ssh_config(node) + options = [ + "-o 'HostName=#{node.ip_address}'", + "-o 'GlobalKnownHostsFile=#{path(:known_hosts)}'", + "-o 'UserKnownHostsFile=/dev/null'" + ] + if node.vagrant? + options << "-i #{vagrant_ssh_key_file}" # use the universal vagrant insecure key + options << "-o IdentitiesOnly=yes" # force the use of the insecure vagrant key + options << "-o 'StrictHostKeyChecking=no'" # blindly accept host key and don't save it + # (since userknownhostsfile is /dev/null) + else + options << "-o 'StrictHostKeyChecking=yes'" + end + if !node.supported_ssh_host_key_algorithms.empty? + options << "-o 'HostKeyAlgorithms=#{node.supported_ssh_host_key_algorithms}'" + end + return options + end + + def parse_tunnel_arg(arg) + if arg.count(':') == 1 + node_name, remote = arg.split(':') + local = nil + elsif arg.count(':') == 2 + local, node_name, remote = arg.split(':') + else + bail!('Argument NAME:REMOTE_PORT required.') + end + node = get_node_from_args([node_name], :include_disabled => true) + remote = remote.to_i + local = local || remote + local = local.to_i + return [local, node, remote] + end + +end; end
\ No newline at end of file diff --git a/lib/leap_cli/commands/test.rb b/lib/leap_cli/commands/test.rb new file mode 100644 index 00000000..73207b31 --- /dev/null +++ b/lib/leap_cli/commands/test.rb @@ -0,0 +1,74 @@ +module LeapCli; module Commands + + desc 'Run tests.' + command [:test, :t] do |test| + test.desc 'Run the test suit on FILTER nodes.' + test.arg_name 'FILTER', :optional => true + test.command :run do |run| + run.switch 'continue', :desc => 'Continue over errors and failures (default is --no-continue).', :negatable => true + run.action do |global_options,options,args| + test_order = File.join(Path.platform, 'tests/order.rb') + if File.exists?(test_order) + require test_order + end + manager.filter!(args).names_in_test_dependency_order.each do |node_name| + node = manager.nodes[node_name] + begin + ssh_connect(node) do |ssh| + ssh.run(test_cmd(options)) + end + rescue Capistrano::CommandError => exc + if options[:continue] + exit_status(1) + else + bail! + end + end + end + end + end + + test.desc 'Creates files needed to run tests.' + test.command :init do |init| + init.action do |global_options,options,args| + generate_test_client_openvpn_configs + end + end + + test.default_command :run + end + + private + + def test_cmd(options) + if options[:continue] + "#{Leap::Platform.leap_dir}/bin/run_tests --continue" + else + "#{Leap::Platform.leap_dir}/bin/run_tests" + end + end + + # + # generates a whole bunch of openvpn configs that can be used to connect to different openvpn gateways + # + def generate_test_client_openvpn_configs + assert_config! 'provider.ca.client_certificates.unlimited_prefix' + assert_config! 'provider.ca.client_certificates.limited_prefix' + template = read_file! Path.find_file(:test_client_openvpn_template) + manager.environment_names.each do |env| + vpn_nodes = manager.nodes[:environment => env][:services => 'openvpn']['openvpn.allow_limited' => true] + if vpn_nodes.any? + generate_test_client_cert(provider.ca.client_certificates.limited_prefix) do |key, cert| + write_file! [:test_openvpn_config, [env, 'limited'].compact.join('_')], Util.erb_eval(template, binding) + end + end + vpn_nodes = manager.nodes[:environment => env][:services => 'openvpn']['openvpn.allow_unlimited' => true] + if vpn_nodes.any? + generate_test_client_cert(provider.ca.client_certificates.unlimited_prefix) do |key, cert| + write_file! [:test_openvpn_config, [env, 'unlimited'].compact.join('_')], Util.erb_eval(template, binding) + end + end + end + end + +end; end diff --git a/lib/leap_cli/commands/user.rb b/lib/leap_cli/commands/user.rb new file mode 100644 index 00000000..480e9a9c --- /dev/null +++ b/lib/leap_cli/commands/user.rb @@ -0,0 +1,136 @@ + +# +# perhaps we want to verify that the key files are actually the key files we expect. +# we could use 'file' for this: +# +# > file ~/.gnupg/00440025.asc +# ~/.gnupg/00440025.asc: PGP public key block +# +# > file ~/.ssh/id_rsa.pub +# ~/.ssh/id_rsa.pub: OpenSSH RSA public key +# + +module LeapCli + module Commands + + desc 'Adds a new trusted sysadmin by adding public keys to the "users" directory.' + arg_name 'USERNAME' #, :optional => false, :multiple => false + command :'add-user' do |c| + + c.switch 'self', :desc => 'Add yourself as a trusted sysadin by choosing among the public keys available for the current user.', :negatable => false + c.flag 'ssh-pub-key', :desc => 'SSH public key file for this new user' + c.flag 'pgp-pub-key', :desc => 'OpenPGP public key file for this new user' + + c.action do |global_options,options,args| + username = args.first + if !username.any? + if options[:self] + username ||= `whoami`.strip + else + help! "Either USERNAME argument or --self flag is required." + end + end + if Leap::Platform.reserved_usernames.include? username + bail! %(The username "#{username}" is reserved. Sorry, pick another.) + end + + ssh_pub_key = nil + pgp_pub_key = nil + + if options['ssh-pub-key'] + ssh_pub_key = read_file!(options['ssh-pub-key']) + end + if options['pgp-pub-key'] + pgp_pub_key = read_file!(options['pgp-pub-key']) + end + + if options[:self] + ssh_pub_key ||= pick_ssh_key.to_s + pgp_pub_key ||= pick_pgp_key + end + + assert!(ssh_pub_key, 'Sorry, could not find SSH public key.') + + if ssh_pub_key + write_file!([:user_ssh, username], ssh_pub_key) + end + if pgp_pub_key + write_file!([:user_pgp, username], pgp_pub_key) + end + + update_authorized_keys + end + end + + # + # let the the user choose among the ssh public keys that we encounter, or just pick the key if there is only one. + # + def pick_ssh_key + ssh_keys = [] + Dir.glob("#{ENV['HOME']}/.ssh/*.pub").each do |keyfile| + ssh_keys << SshKey.load(keyfile) + end + + if `which ssh-add`.strip.any? + `ssh-add -L 2> /dev/null`.split("\n").compact.each do |line| + key = SshKey.load(line) + if key + key.comment = 'ssh-agent' + ssh_keys << key unless ssh_keys.include?(key) + end + end + end + ssh_keys.compact! + + assert! ssh_keys.any?, 'Sorry, could not find any SSH public key for you. Have you run ssh-keygen?' + + if ssh_keys.length > 1 + key_index = numbered_choice_menu('Choose your SSH public key', ssh_keys.collect(&:summary)) do |line, i| + say("#{i+1}. #{line}") + end + else + key_index = 0 + end + + return ssh_keys[key_index] + end + + # + # let the the user choose among the gpg public keys that we encounter, or just pick the key if there is only one. + # + def pick_pgp_key + begin + require 'gpgme' + rescue LoadError + log "Skipping OpenPGP setup because gpgme is not installed." + return + end + + secret_keys = GPGME::Key.find(:secret) + if secret_keys.empty? + log "Skipping OpenPGP setup because I could not find any OpenPGP keys for you" + return nil + end + + secret_keys.select!{|key| !key.expired} + + if secret_keys.length > 1 + key_index = numbered_choice_menu('Choose your OpenPGP public key', secret_keys) do |key, i| + key_info = key.to_s.split("\n")[0..1].map{|line| line.sub(/^\s*(sec|uid)\s*/,'')}.join(' -- ') + say("#{i+1}. #{key_info}") + end + else + key_index = 0 + end + + key_id = secret_keys[key_index].sha + + # can't use this, it includes signatures: + #puts GPGME::Key.export(key_id, :armor => true, :export_options => :export_minimal) + + # export with signatures removed: + return `gpg --armor --export-options export-minimal --export #{key_id}`.strip + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/util.rb b/lib/leap_cli/commands/util.rb new file mode 100644 index 00000000..c1da570e --- /dev/null +++ b/lib/leap_cli/commands/util.rb @@ -0,0 +1,50 @@ +module LeapCli; module Commands + + extend self + extend LeapCli::Util + extend LeapCli::Util::RemoteCommand + + def path(name) + Path.named_path(name) + end + + # + # keeps prompting the user for a numbered choice, until they pick a good one or bail out. + # + # block is yielded and is responsible for rendering the choices. + # + def numbered_choice_menu(msg, items, &block) + while true + say("\n" + msg + ':') + items.each_with_index &block + say("q. quit") + index = ask("number 1-#{items.length}> ") + if index.empty? + next + elsif index =~ /q/ + bail! + else + i = index.to_i - 1 + if i < 0 || i >= items.length + bail! + else + return i + end + end + end + end + + + def parse_node_list(nodes) + if nodes.is_a? Config::Object + Config::ObjectList.new(nodes) + elsif nodes.is_a? Config::ObjectList + nodes + elsif nodes.is_a? String + manager.filter!(nodes) + else + bail! "argument error" + end + end + +end; end diff --git a/lib/leap_cli/commands/vagrant.rb b/lib/leap_cli/commands/vagrant.rb new file mode 100644 index 00000000..bf683cb6 --- /dev/null +++ b/lib/leap_cli/commands/vagrant.rb @@ -0,0 +1,197 @@ +autoload :IPAddr, 'ipaddr' +require 'fileutils' + +module LeapCli; module Commands + + desc "Manage local virtual machines." + long_desc "This command provides a convient way to manage Vagrant-based virtual machines. If FILTER argument is missing, the command runs on all local virtual machines. The Vagrantfile is automatically generated in 'test/Vagrantfile'. If you want to run vagrant commands manually, cd to 'test'." + command [:local, :l] do |local| + local.desc 'Starts up the virtual machine(s)' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :start do |start| + start.flag(:basebox, + :desc => "The basebox to use. This value is passed to vagrant as the "+ + "`config.vm.box` option. The value here should be the name of an installed box or a "+ + "shorthand name of a box in HashiCorp's Atlas.", + :arg_name => 'BASEBOX', + :default_value => 'LEAP/jessie' + ) + start.action do |global_options,options,args| + vagrant_command(["up", "sandbox on"], args, options) + end + end + + local.desc 'Shuts down the virtual machine(s)' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :stop do |stop| + stop.action do |global_options,options,args| + if global_options[:yes] + vagrant_command("halt --force", args) + else + vagrant_command("halt", args) + end + end + end + + local.desc 'Destroys the virtual machine(s), reclaiming the disk space' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :destroy do |destroy| + destroy.action do |global_options,options,args| + if global_options[:yes] + vagrant_command("destroy --force", args) + else + vagrant_command("destroy", args) + end + end + end + + local.desc 'Print the status of local virtual machine(s)' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :status do |status| + status.action do |global_options,options,args| + vagrant_command("status", args) + end + end + + local.desc 'Saves the current state of the virtual machine as a new snapshot' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :save do |status| + status.action do |global_options,options,args| + vagrant_command("sandbox commit", args) + end + end + + local.desc 'Resets virtual machine(s) to the last saved snapshot' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :reset do |reset| + reset.action do |global_options,options,args| + vagrant_command("sandbox rollback", args) + end + end + end + + public + + # + # returns the path to a vagrant ssh private key file. + # + # if the vagrant.key file is owned by root or ourselves, then + # we need to make sure that it owned by us and not world readable. + # + def vagrant_ssh_key_file + file_path = Path.vagrant_ssh_priv_key_file + Util.assert_files_exist! file_path + uid = File.new(file_path).stat.uid + if uid == 0 || uid == Process.euid + FileUtils.install file_path, '/tmp/vagrant.key', :mode => 0600 + file_path = '/tmp/vagrant.key' + end + return file_path + end + + protected + + def vagrant_command(cmds, args, options={}) + vagrant_setup(options) + cmds = cmds.to_a + if args.empty? + nodes = [""] + else + nodes = manager.filter(args)[:environment => "local"].field(:name) + end + if nodes.any? + vagrant_dir = File.dirname(Path.named_path(:vagrantfile)) + exec = ["cd #{vagrant_dir}"] + cmds.each do |cmd| + nodes.each do |node| + exec << "vagrant #{cmd} #{node}" + end + end + execute exec.join('; ') + else + bail! "No nodes found. This command only works on nodes with ip_address in the network #{LeapCli.leapfile.vagrant_network}" + end + end + + private + + def vagrant_setup(options) + assert_bin! 'vagrant', 'Vagrant is required for running local virtual machines. Run "sudo apt-get install vagrant".' + + if vagrant_version <= Gem::Version.new('1.0.0') + gem_path = assert_run!('vagrant gem which sahara') + if gem_path.nil? || gem_path.empty? || gem_path =~ /^ERROR/ + log :installing, "vagrant plugin 'sahara'" + assert_run! 'vagrant gem install sahara -v 0.0.13' + end + else + unless assert_run!('vagrant plugin list | grep sahara | cat').chars.any? + log :installing, "vagrant plugin 'sahara'" + assert_run! 'vagrant plugin install sahara' + end + end + create_vagrant_file(options) + end + + def vagrant_version + @vagrant_version ||= Gem::Version.new(assert_run!('vagrant --version').split(' ')[1]) + end + + def execute(cmd) + log 2, :run, cmd + exec cmd + end + + def create_vagrant_file(options) + lines = [] + netmask = IPAddr.new('255.255.255.255').mask(LeapCli.leapfile.vagrant_network.split('/').last).to_s + + basebox = options[:basebox] || 'LEAP/jessie' + + if vagrant_version <= Gem::Version.new('1.1.0') + lines << %[Vagrant::Config.run do |config|] + manager.each_node do |node| + if node.vagrant? + lines << %[ config.vm.define :#{node.name} do |config|] + lines << %[ config.vm.box = "#{basebox}"] + lines << %[ config.vm.network :hostonly, "#{node.ip_address}", :netmask => "#{netmask}"] + lines << %[ config.vm.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]] + lines << %[ config.vm.customize ["modifyvm", :id, "--name", "#{node.name}"]] + lines << %[ #{leapfile.custom_vagrant_vm_line}] if leapfile.custom_vagrant_vm_line + lines << %[ end] + end + end + else + lines << %[Vagrant.configure("2") do |config|] + manager.each_node do |node| + if node.vagrant? + lines << %[ config.vm.define :#{node.name} do |config|] + lines << %[ config.vm.box = "#{basebox}"] + lines << %[ config.vm.network :private_network, ip: "#{node.ip_address}"] + lines << %[ config.vm.provider "virtualbox" do |v|] + lines << %[ v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]] + lines << %[ v.name = "#{node.name}"] + lines << %[ end] + lines << %[ #{leapfile.custom_vagrant_vm_line}] if leapfile.custom_vagrant_vm_line + lines << %[ end] + end + end + end + + lines << %[end] + lines << "" + write_file! :vagrantfile, lines.join("\n") + end + + def pick_next_vagrant_ip_address + taken_ips = manager.nodes[:environment => "local"].field(:ip_address) + if taken_ips.any? + highest_ip = taken_ips.map{|ip| IPAddr.new(ip)}.max + new_ip = highest_ip.succ + else + new_ip = IPAddr.new(LeapCli.leapfile.vagrant_network).succ.succ + end + return new_ip.to_s + end + +end; end diff --git a/provider_base/lib/macros.rb b/lib/leap_cli/macros.rb index ecc3e6ba..fdb9a94e 100644 --- a/provider_base/lib/macros.rb +++ b/lib/leap_cli/macros.rb @@ -13,3 +13,4 @@ require_relative 'macros/keys' require_relative 'macros/nodes' require_relative 'macros/secrets' require_relative 'macros/stunnel' +require_relative 'macros/provider' diff --git a/provider_base/lib/macros/core.rb b/lib/leap_cli/macros/core.rb index 7de50f2f..7de50f2f 100644 --- a/provider_base/lib/macros/core.rb +++ b/lib/leap_cli/macros/core.rb diff --git a/provider_base/lib/macros/files.rb b/lib/leap_cli/macros/files.rb index 958958bc..958958bc 100644 --- a/provider_base/lib/macros/files.rb +++ b/lib/leap_cli/macros/files.rb diff --git a/provider_base/lib/macros/haproxy.rb b/lib/leap_cli/macros/haproxy.rb index 602ae726..602ae726 100644 --- a/provider_base/lib/macros/haproxy.rb +++ b/lib/leap_cli/macros/haproxy.rb diff --git a/provider_base/lib/macros/hosts.rb b/lib/leap_cli/macros/hosts.rb index 8281329f..8281329f 100644 --- a/provider_base/lib/macros/hosts.rb +++ b/lib/leap_cli/macros/hosts.rb diff --git a/provider_base/lib/macros/keys.rb b/lib/leap_cli/macros/keys.rb index 0ed7ccd0..0ed7ccd0 100644 --- a/provider_base/lib/macros/keys.rb +++ b/lib/leap_cli/macros/keys.rb diff --git a/provider_base/lib/macros/nodes.rb b/lib/leap_cli/macros/nodes.rb index 8b961cbc..8b961cbc 100644 --- a/provider_base/lib/macros/nodes.rb +++ b/lib/leap_cli/macros/nodes.rb diff --git a/lib/leap_cli/macros/provider.rb b/lib/leap_cli/macros/provider.rb new file mode 100644 index 00000000..4e74da01 --- /dev/null +++ b/lib/leap_cli/macros/provider.rb @@ -0,0 +1,90 @@ +# +# These macros are intended only for use in provider.json, although they are +# currently loaded in all .json contexts. +# + +module LeapCli + module Macro + + # + # returns an array of the service names, including only those services that + # are enabled for this environment. + # + def enabled_services + manager.env(self.environment).services[:service_type => :user_service].field(:name).select { |service| + manager.nodes[:environment => self.environment][:services => service].any? + } + end + + # + # The webapp will not work unless the service level configuration is precisely defined. + # Here, we take what the sysadmin has specified in provider.json and clean it up to + # ensure it is OK. + # + # It would be better to add support for JSON schema. + # + def service_levels() + levels = {} + provider.service.levels.each do |name, level| + if name =~ /^[0-9]+$/ + name = name.to_i + end + levels[name] = level_cleanup(name, level.clone) + end + levels + end + + private + + def print_warning(name, msg) + if self.environment + provider_str = "provider.json or %s" % ['provider', self.environment, 'json'].join('.') + else + provider_str = "provider.json" + end + LeapCli::log :warning, "In #{provider_str}, you have an incorrect definition for service level '#{name}':" do + LeapCli::log msg + end + end + + def level_cleanup(name, level) + unless level['name'] + print_warning(name, 'required field "name" is missing') + end + unless level['description'] + print_warning(name, 'required field "description" is missing') + end + unless level['bandwidth'].nil? || level['bandwidth'] == 'limited' + print_warning(name, 'field "bandwidth" must be nil or "limited"') + end + unless level['rate'].nil? || level['rate'].is_a?(Hash) + print_warning(name, 'field "rate" must be nil or a hash (e.g. {"USD":10, "EUR":10})') + end + possible_services = enabled_services + if level['services'] + level['services'].each do |service| + unless possible_services.include? service + print_warning(name, "the service '#{service}' does not exist or there are no nodes that provide this service.") + LeapCli::Util::bail! + end + end + else + level['services'] = possible_services + end + level['services'] = remap_services(level['services']) + level + end + + # + # the service names that the webapp uses and that leap_platform uses are different. ugh. + # + SERVICE_MAP = { + "mx" => "email", + "openvpn" => "eip" + } + def remap_services(services) + services.map {|srv| SERVICE_MAP[srv]} + end + + end +end diff --git a/provider_base/lib/macros/secrets.rb b/lib/leap_cli/macros/secrets.rb index 8d1feb55..8d1feb55 100644 --- a/provider_base/lib/macros/secrets.rb +++ b/lib/leap_cli/macros/secrets.rb diff --git a/provider_base/lib/macros/stunnel.rb b/lib/leap_cli/macros/stunnel.rb index f16308c7..821bda38 100644 --- a/provider_base/lib/macros/stunnel.rb +++ b/lib/leap_cli/macros/stunnel.rb @@ -49,12 +49,14 @@ module LeapCli result = Config::ObjectList.new node_list.each_node do |node| if node.name != self.name || options[:include_self] + s_port = stunnel_port(port) result["#{node.name}_#{port}"] = Config::Object[ 'accept_port', @next_stunnel_port, 'connect', node.domain.internal, - 'connect_port', stunnel_port(port), + 'connect_port', s_port, 'original_port', port ] + manager.connections.add(:from => @node.ip_address, :to => node.ip_address, :port => s_port) @next_stunnel_port += 1 end end @@ -76,6 +78,15 @@ module LeapCli } end + # + # lists the ips that connect to this node, on particular ports. + # + def stunnel_firewall + manager.connections.select {|connection| + connection['to'] == @node.ip_address + } + end + private # diff --git a/platform.rb b/platform.rb index 0c3de2a0..61f0dd05 100644 --- a/platform.rb +++ b/platform.rb @@ -4,8 +4,8 @@ # Leap::Platform.define do - self.version = "0.7.1" - self.compatible_cli = "1.7.0".."1.7.99" + self.version = "0.8" + self.compatible_cli = "1.8".."1.99" # # the facter facts that should be gathered diff --git a/provider_base/common.json b/provider_base/common.json index 3d2965d7..5821789b 100644 --- a/provider_base/common.json +++ b/provider_base/common.json @@ -16,6 +16,9 @@ }, "ssh": { "authorized_keys": "= authorized_keys", + "config": { + "AllowTcpForwarding": "no" + }, "port": 22, "mosh": { "ports": "60000:61000", @@ -47,6 +50,14 @@ "clients": {}, "servers": {} }, + "firewall": { + "ssh": { + "from": "sysadmin", + "to": "= ip_address", + "port": "= ssh.port" + }, + "stunnel": "=> stunnel_firewall" + }, "platform": { "version": "= Leap::Platform.version.to_s", "major_version": "= Leap::Platform.major_version" @@ -72,11 +83,6 @@ "package": "soledad-server", "revision": "latest" }, - "tapicero": { - "type": "git", - "source": "https://leap.se/git/tapicero", - "revision": "origin/version/0.7" - }, "webapp": { "type": "git", "source": "https://leap.se/git/leap_web", diff --git a/provider_base/files/service-definitions/provider.json.erb b/provider_base/files/service-definitions/provider.json.erb index be8ae484..a75bea61 100644 --- a/provider_base/files/service-definitions/provider.json.erb +++ b/provider_base/files/service-definitions/provider.json.erb @@ -6,15 +6,10 @@ ) hsh['domain'] = domain.full_suffix - # advertise services that are 'user services' and for which there are actually nodes - hsh['services'] ||= global.services[:service_type => :user_service].field(:name).select do |service| - nodes_like_me[:services => service].any? - end - hsh['api_version'] = "1" hsh['api_uri'] = ["https://", api.domain, ':', api.port].join - hsh['ca_cert_uri'] = 'https://' + webapp.domain + '/ca.crt' + hsh['ca_cert_uri'] = api.ca_cert_uri hsh['ca_cert_fingerprint'] = fingerprint(:ca_cert) hsh.dump_json diff --git a/provider_base/provider.json b/provider_base/provider.json index 60ad2a9e..81b2ea98 100644 --- a/provider_base/provider.json +++ b/provider_base/provider.json @@ -14,6 +14,7 @@ "languages": ["en"], "default_language": "en", "enrollment_policy": "open", + "services": "= enabled_services", "service": { // bandwidth limit is in Bytes, storage limit is in MB. // for example: @@ -31,8 +32,8 @@ "bandwidth_limit": 102400, "allow_free": "= provider.service.levels.select {|l| l['rate'].nil?}.any?", "allow_paid": "= provider.service.levels.select {|l| !l['rate'].nil?}.any?", - "allow_anonymous": "= provider.service.levels.select {|l| l['name'] == 'anonymous'}.any?", - "allow_registration": "= provider.service.levels.select {|l| l['name'] != 'anonymous'}.any?", + "allow_anonymous": "= provider.service.levels.select {|l| l['name'] == 'anonymous'}.any? && services.include?('openvpn')", + "allow_registration": "= provider.enrollment_policy != 'closed' && provider.service.levels.select {|l| l['name'] != 'anonymous'}.any?", "allow_limited_bandwidth": "= provider.service.levels.select {|l| l['bandwidth'] == 'limited'}.any?", "allow_unlimited_bandwidth": "= provider.service.levels.select {|l| l['bandwidth'].nil?}.any?" }, diff --git a/provider_base/services/couchdb.json b/provider_base/services/couchdb.json index 8b1386f8..5e65b2ec 100644 --- a/provider_base/services/couchdb.json +++ b/provider_base/services/couchdb.json @@ -31,11 +31,6 @@ "password": "= secret :couch_soledad_password", "salt": "= hex_secret :couch_soledad_password_salt, 128" }, - "tapicero": { - "username": "tapicero", - "password": "= secret :couch_tapicero_password", - "salt": "= hex_secret :couch_tapicero_password_salt, 128" - }, "webapp": { "username": "webapp", "password": "= secret :couch_webapp_password", diff --git a/provider_base/services/dns.json b/provider_base/services/dns.json index 677d9b2c..67948ef8 100644 --- a/provider_base/services/dns.json +++ b/provider_base/services/dns.json @@ -3,5 +3,12 @@ "public": "= nodes['dns.public' => true].fields('domain.name', 'dns.aliases', 'ip_address')", "private": "= nodes['dns.public' => false].fields('domain.name', 'dns.aliases', 'ip_address')" }, - "service_type": "public_service" + "service_type": "public_service", + "firewall": { + "dns": { + "from": "*", + "to": "= ip_address", + "port": "53" + } + } }
\ No newline at end of file diff --git a/provider_base/services/monitor.json b/provider_base/services/monitor.json index 10d5ac81..28fb837c 100644 --- a/provider_base/services/monitor.json +++ b/provider_base/services/monitor.json @@ -18,5 +18,12 @@ "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'" + }, + "firewall": { + "monitor": { + "from": "sysadmin", + "to": "= ip_address", + "port": [443, 80] + } } } diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json index 11293ae8..70acf5cb 100644 --- a/provider_base/services/mx.json +++ b/provider_base/services/mx.json @@ -1,4 +1,11 @@ { + "mx": { + // provider should define their own custom aliases. + // these are in *addition* to the standard reserved aliases for root and postmaster, etc. + "aliases": {}, + // this is the domain that is used for the OpenPGP header + "key_lookup_domain": "= global.services[:webapp].webapp.domain" + }, "stunnel": { "clients": { "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" @@ -16,6 +23,10 @@ "salt": "= hex_secret :couch_leap_mx_password_salt, 128" }, "mynetworks": "= nodes['environment' => '!local'].map{|name, n| [n.ip_address, (global.facts[name]||{})['ec2_public_ipv4']]}.flatten.compact.uniq", + "rbls": ["zen.spamhaus.org"], + "clamav": { + "whitelisted_addresses": [] + }, "x509": { "use": true, "use_commercial": true, @@ -23,5 +34,12 @@ "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'" }, - "service_type": "user_service" + "service_type": "user_service", + "firewall": { + "mx": { + "from": "*", + "to": "= ip_address", + "port": [25, 465] + } + } } diff --git a/provider_base/services/openvpn.json b/provider_base/services/openvpn.json index 11cb0dc2..6f73e31c 100644 --- a/provider_base/services/openvpn.json +++ b/provider_base/services/openvpn.json @@ -34,5 +34,12 @@ "port" : "= rand_range('scramblesuit_port_'+name, 18000..32000)" }, "gateway_address": "= openvpn.gateway_address" + }, + "firewall": { + "vpn": { + "from": "*", + "to": "= openvpn.gateway_address", + "port": "= openvpn.ports + [obfsproxy.scramblesuit.port]" + } } } diff --git a/provider_base/services/soledad.json b/provider_base/services/soledad.json index ed6fbc9f..99390d17 100644 --- a/provider_base/services/soledad.json +++ b/provider_base/services/soledad.json @@ -6,7 +6,17 @@ "username": "= global.services[:couchdb].couch.users[:soledad].username", "password": "= secret :couch_soledad_password", "salt": "= hex_secret :couch_soledad_password_salt, 128" + }, + "couchdb_leap_mx_user": { + "username": "= global.services[:couchdb].couch.users[:leap_mx].username" } }, - "service_type": "public_service" + "service_type": "public_service", + "firewall": { + "soledad": { + "from": "*", + "to": "= ip_address", + "port": "= soledad.port" + } + } } diff --git a/provider_base/services/static.json b/provider_base/services/static.json index d9f52b36..2f408ec1 100644 --- a/provider_base/services/static.json +++ b/provider_base/services/static.json @@ -9,5 +9,12 @@ "client_version": "= static.bootstrap_files.enabled ? provider.client_version : nil" } }, - "service_type": "public_service" + "service_type": "public_service", + "firewall": { + "static": { + "from": "*", + "to": "= ip_address", + "port": [80, 443] + } + } }
\ No newline at end of file diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index 941f4f61..9e3d751b 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -9,7 +9,7 @@ "owner", "owners", "postmaster", "reply", "robot", "ssladmin", "staff", "support", "tech-support", "tech_support", "techsupport", "ticket", "tickets", "vmail", "www-data"], - "domain": "= domain.full_suffix", + "domain": "= provider.domain", "modules": ["user", "billing", "help"], "couchdb_webapp_user": "= global.services[:couchdb].couch.users[:webapp]", "couchdb_admin_user": "= global.services[:couchdb].couch.users[:admin]", @@ -20,7 +20,7 @@ "allow_anonymous_certs": "= provider.service.allow_anonymous", "allow_registration": "= provider.service.allow_registration", "default_service_level": "= provider.service.default_service_level", - "service_levels": "= provider.service.levels", + "service_levels": "= service_levels()", "secret_token": "= secret :webapp_secret_token", "api_version": 1, "secure": false, @@ -31,7 +31,9 @@ }, "engines": [ "support" - ] + ], + "locales": "= provider.languages", + "default_locale": "= provider.default_language" }, "stunnel": { "clients": { @@ -53,7 +55,8 @@ "service_type": "public_service", "api": { "domain": "= 'api.' + webapp.domain", - "port": 4430 + "port": 4430, + "ca_cert_uri": "= 'https://' + webapp.domain + '/ca.crt'" }, "nickserver": { "domain": "= 'nicknym.' + domain.full_suffix", @@ -73,5 +76,12 @@ "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`.'", "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`.'" + }, + "firewall": { + "webapp": { + "from": "*", + "to": "= ip_address", + "port": "= [api.port, 443, 80, nickserver.port]" + } } } diff --git a/provider_base/tags/development.json b/provider_base/tags/development.json index d9c2c007..caf18e9d 100644 --- a/provider_base/tags/development.json +++ b/provider_base/tags/development.json @@ -1,7 +1,3 @@ { - "environment": "development", - "domain": { - "full_suffix": "= 'dev.' + provider.domain", - "internal_suffix": "= 'dev.' + provider.domain_internal" - } + "environment": "development" }
\ No newline at end of file diff --git a/puppet/lib/puppet/parser/functions/sorted_yaml.rb b/puppet/lib/puppet/parser/functions/sorted_yaml.rb new file mode 100644 index 00000000..46cd46ce --- /dev/null +++ b/puppet/lib/puppet/parser/functions/sorted_yaml.rb @@ -0,0 +1,400 @@ +# encoding: UTF-8 +# +# provides sorted_yaml() function, using Ya2YAML. +# see https://github.com/afunai/ya2yaml +# + +class Ya2YAML + # + # Author:: Akira FUNAI + # Copyright:: Copyright (c) 2006-2010 Akira FUNAI + # License:: MIT License + # + + def initialize(opts = {}) + options = opts.dup + options[:indent_size] = 2 if options[:indent_size].to_i <= 0 + options[:minimum_block_length] = 0 if options[:minimum_block_length].to_i <= 0 + options.update( + { + :printable_with_syck => true, + :escape_b_specific => true, + :escape_as_utf8 => true, + } + ) if options[:syck_compatible] + + @options = options + end + + def _ya2yaml(obj) + #raise 'set $KCODE to "UTF8".' if (RUBY_VERSION < '1.9.0') && ($KCODE != 'UTF8') + if (RUBY_VERSION < '1.9.0') + $KCODE = 'UTF8' + end + '--- ' + emit(obj, 1) + "\n" + rescue SystemStackError + raise ArgumentError, "ya2yaml can't handle circular references" + end + + private + + def emit(obj, level) + case obj + when Array + if (obj.length == 0) + '[]' + else + indent = "\n" + s_indent(level - 1) + ### + ### NOTE: a minor modification to normal Ya2YAML... + ### We want arrays to be output in sorted order, not just + ### Hashes. + ### + #obj.collect {|o| + # indent + '- ' + emit(o, level + 1) + #}.join('') + obj.sort {|a,b| a.to_s <=> b.to_s}.collect {|o| + indent + '- ' + emit(o, level + 1) + }.join('') + end + when Hash + if (obj.length == 0) + '{}' + else + indent = "\n" + s_indent(level - 1) + hash_order = @options[:hash_order] + if (hash_order && level == 1) + hash_keys = obj.keys.sort {|x, y| + x_order = hash_order.index(x) ? hash_order.index(x) : Float::MAX + y_order = hash_order.index(y) ? hash_order.index(y) : Float::MAX + o = (x_order <=> y_order) + (o != 0) ? o : (x.to_s <=> y.to_s) + } + elsif @options[:preserve_order] + hash_keys = obj.keys + else + hash_keys = obj.keys.sort {|x, y| x.to_s <=> y.to_s } + end + hash_keys.collect {|k| + key = emit(k, level + 1) + if ( + is_one_plain_line?(key) || + key =~ /\A(#{REX_BOOL}|#{REX_FLOAT}|#{REX_INT}|#{REX_NULL})\z/x + ) + indent + key + ': ' + emit(obj[k], level + 1) + else + indent + '? ' + key + + indent + ': ' + emit(obj[k], level + 1) + end + }.join('') + end + when NilClass + '~' + when String + emit_string(obj, level) + when TrueClass, FalseClass + obj.to_s + when Fixnum, Bignum, Float + obj.to_s + when Date + obj.to_s + when Time + offset = obj.gmtoff + off_hm = sprintf( + '%+.2d:%.2d', + (offset / 3600.0).to_i, + (offset % 3600.0) / 60 + ) + u_sec = (obj.usec != 0) ? sprintf(".%.6d", obj.usec) : '' + obj.strftime("%Y-%m-%d %H:%M:%S#{u_sec} #{off_hm}") + when Symbol + '!ruby/symbol ' + emit_string(obj.to_s, level) + when Range + '!ruby/range ' + obj.to_s + when Regexp + '!ruby/regexp ' + obj.inspect + else + case + when obj.is_a?(Struct) + struct_members = {} + obj.each_pair{|k, v| struct_members[k.to_s] = v } + '!ruby/struct:' + obj.class.to_s.sub(/^(Struct::(.+)|.*)$/, '\2') + ' ' + + emit(struct_members, level + 1) + else + # serialized as a generic object + object_members = {} + obj.instance_variables.each{|k, v| + object_members[k.to_s.sub(/^@/, '')] = obj.instance_variable_get(k) + } + '!ruby/object:' + obj.class.to_s + ' ' + + emit(object_members, level + 1) + end + end + end + + def emit_string(str, level) + (is_string, is_printable, is_one_line, is_one_plain_line) = string_type(str) + if is_string + if is_printable + if is_one_plain_line + emit_simple_string(str, level) + else + (is_one_line || str.length < @options[:minimum_block_length]) ? + emit_quoted_string(str, level) : + emit_block_string(str, level) + end + else + emit_quoted_string(str, level) + end + else + emit_base64_binary(str, level) + end + end + + def emit_simple_string(str, level) + str + end + + def emit_block_string(str, level) + str = normalize_line_break(str) + + indent = s_indent(level) + indentation_indicator = (str =~ /\A /) ? indent.size.to_s : '' + str =~ /(#{REX_NORMAL_LB}*)\z/ + chomping_indicator = case $1.length + when 0 + '-' + when 1 + '' + else + '+' + end + + str.chomp! + str.gsub!(/#{REX_NORMAL_LB}/) { + $1 + indent + } + '|' + indentation_indicator + chomping_indicator + "\n" + indent + str + end + + def emit_quoted_string(str, level) + str = yaml_escape(normalize_line_break(str)) + if (str.length < @options[:minimum_block_length]) + str.gsub!(/#{REX_NORMAL_LB}/) { ESCAPE_SEQ_LB[$1] } + else + str.gsub!(/#{REX_NORMAL_LB}$/) { ESCAPE_SEQ_LB[$1] } + str.gsub!(/(#{REX_NORMAL_LB}+)(.)/) { + trail_c = $3 + $1 + trail_c.sub(/([\t ])/) { ESCAPE_SEQ_WS[$1] } + } + indent = s_indent(level) + str.gsub!(/#{REX_NORMAL_LB}/) { + ESCAPE_SEQ_LB[$1] + "\\\n" + indent + } + end + '"' + str + '"' + end + + def emit_base64_binary(str, level) + indent = "\n" + s_indent(level) + base64 = [str].pack('m') + '!binary |' + indent + base64.gsub(/\n(?!\z)/, indent) + end + + def string_type(str) + if str.respond_to?(:encoding) && (!str.valid_encoding? || str.encoding == Encoding::ASCII_8BIT) + return false, false, false, false + end + (ucs_codes = str.unpack('U*')) rescue ( + # ArgumentError -> binary data + return false, false, false, false + ) + if ( + @options[:printable_with_syck] && + str =~ /\A#{REX_ANY_LB}* | #{REX_ANY_LB}*\z|#{REX_ANY_LB}{2}\z/ + ) + # detour Syck bug + return true, false, nil, false + end + ucs_codes.each {|ucs_code| + return true, false, nil, false unless is_printable?(ucs_code) + } + return true, true, is_one_line?(str), is_one_plain_line?(str) + end + + def is_printable?(ucs_code) + # YAML 1.1 / 4.1.1. + ( + [0x09, 0x0a, 0x0d, 0x85].include?(ucs_code) || + (ucs_code <= 0x7e && ucs_code >= 0x20) || + (ucs_code <= 0xd7ff && ucs_code >= 0xa0) || + (ucs_code <= 0xfffd && ucs_code >= 0xe000) || + (ucs_code <= 0x10ffff && ucs_code >= 0x10000) + ) && + !( + # treat LS/PS as non-printable characters + @options[:escape_b_specific] && + (ucs_code == 0x2028 || ucs_code == 0x2029) + ) + end + + def is_one_line?(str) + str !~ /#{REX_ANY_LB}(?!\z)/ + end + + def is_one_plain_line?(str) + # YAML 1.1 / 4.6.11. + str !~ /^([\-\?:,\[\]\{\}\#&\*!\|>'"%@`\s]|---|\.\.\.)/ && + str !~ /[:\#\s\[\]\{\},]/ && + str !~ /#{REX_ANY_LB}/ && + str !~ /^(#{REX_BOOL}|#{REX_FLOAT}|#{REX_INT}|#{REX_MERGE} + |#{REX_NULL}|#{REX_TIMESTAMP}|#{REX_VALUE})$/x + end + + def s_indent(level) + # YAML 1.1 / 4.2.2. + ' ' * (level * @options[:indent_size]) + end + + def normalize_line_break(str) + # YAML 1.1 / 4.1.4. + str.gsub(/(#{REX_CRLF}|#{REX_CR}|#{REX_NEL})/, "\n") + end + + def yaml_escape(str) + # YAML 1.1 / 4.1.6. + str.gsub(/[^a-zA-Z0-9]/u) {|c| + ucs_code, = (c.unpack('U') rescue [??]) + case + when ESCAPE_SEQ[c] + ESCAPE_SEQ[c] + when is_printable?(ucs_code) + c + when @options[:escape_as_utf8] + c.respond_to?(:bytes) ? + c.bytes.collect {|b| '\\x%.2x' % b }.join : + '\\x' + c.unpack('H2' * c.size).join('\\x') + when ucs_code == 0x2028 || ucs_code == 0x2029 + ESCAPE_SEQ_LB[c] + when ucs_code <= 0x7f + sprintf('\\x%.2x', ucs_code) + when ucs_code <= 0xffff + sprintf('\\u%.4x', ucs_code) + else + sprintf('\\U%.8x', ucs_code) + end + } + end + + module Constants + UCS_0X85 = [0x85].pack('U') # c285@UTF8 Unicode next line + UCS_0XA0 = [0xa0].pack('U') # c2a0@UTF8 Unicode non-breaking space + UCS_0X2028 = [0x2028].pack('U') # e280a8@UTF8 Unicode line separator + UCS_0X2029 = [0x2029].pack('U') # e280a9@UTF8 Unicode paragraph separator + + # non-break characters + ESCAPE_SEQ = { + "\x00" => '\\0', + "\x07" => '\\a', + "\x08" => '\\b', + "\x0b" => '\\v', + "\x0c" => '\\f', + "\x1b" => '\\e', + "\"" => '\\"', + "\\" => '\\\\', + } + + # non-breaking space + ESCAPE_SEQ_NS = { + UCS_0XA0 => '\\_', + } + + # white spaces + ESCAPE_SEQ_WS = { + "\x09" => '\\t', + " " => '\\x20', + } + + # line breaks + ESCAPE_SEQ_LB ={ + "\x0a" => '\\n', + "\x0d" => '\\r', + UCS_0X85 => '\\N', + UCS_0X2028 => '\\L', + UCS_0X2029 => '\\P', + } + + # regexps for line breaks + REX_LF = Regexp.escape("\x0a") + REX_CR = Regexp.escape("\x0d") + REX_CRLF = Regexp.escape("\x0d\x0a") + REX_NEL = Regexp.escape(UCS_0X85) + REX_LS = Regexp.escape(UCS_0X2028) + REX_PS = Regexp.escape(UCS_0X2029) + + REX_ANY_LB = /(#{REX_LF}|#{REX_CR}|#{REX_NEL}|#{REX_LS}|#{REX_PS})/ + REX_NORMAL_LB = /(#{REX_LF}|#{REX_LS}|#{REX_PS})/ + + # regexps for language-Independent types for YAML1.1 + REX_BOOL = / + y|Y|yes|Yes|YES|n|N|no|No|NO + |true|True|TRUE|false|False|FALSE + |on|On|ON|off|Off|OFF + /x + REX_FLOAT = / + [-+]?([0-9][0-9_]*)?\.[0-9.]*([eE][-+][0-9]+)? # (base 10) + |[-+]?[0-9][0-9_]*(:[0-5]?[0-9])+\.[0-9_]* # (base 60) + |[-+]?\.(inf|Inf|INF) # (infinity) + |\.(nan|NaN|NAN) # (not a number) + /x + REX_INT = / + [-+]?0b[0-1_]+ # (base 2) + |[-+]?0[0-7_]+ # (base 8) + |[-+]?(0|[1-9][0-9_]*) # (base 10) + |[-+]?0x[0-9a-fA-F_]+ # (base 16) + |[-+]?[1-9][0-9_]*(:[0-5]?[0-9])+ # (base 60) + /x + REX_MERGE = / + << + /x + REX_NULL = / + ~ # (canonical) + |null|Null|NULL # (English) + | # (Empty) + /x + REX_TIMESTAMP = / + [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] # (ymd) + |[0-9][0-9][0-9][0-9] # (year) + -[0-9][0-9]? # (month) + -[0-9][0-9]? # (day) + ([Tt]|[ \t]+)[0-9][0-9]? # (hour) + :[0-9][0-9] # (minute) + :[0-9][0-9] # (second) + (\.[0-9]*)? # (fraction) + (([ \t]*)Z|[-+][0-9][0-9]?(:[0-9][0-9])?)? # (time zone) + /x + REX_VALUE = / + = + /x + end + + include Constants +end + +module Puppet::Parser::Functions + newfunction(:sorted_yaml, + :type => :rvalue, + :doc => "This function outputs yaml, but ensures the keys are sorted." + ) do |arguments| + + if arguments.is_a?(Array) + if arguments.size != 1 + raise(Puppet::ParseError, "sorted_yaml(): Wrong number of arguments given (#{arguments.size} for 1)") + end + yaml = arguments.first + else + yaml = arguments + end + return Ya2YAML.new()._ya2yaml(yaml) + end +end diff --git a/puppet/manifests/site.pp b/puppet/manifests/site.pp index 912234ac..91dd2d3c 100644 --- a/puppet/manifests/site.pp +++ b/puppet/manifests/site.pp @@ -20,7 +20,6 @@ if member($services, 'openvpn') { if member($services, 'couchdb') { include site_couchdb - include tapicero } if member($services, 'webapp') { diff --git a/puppet/modules/apache b/puppet/modules/apache -Subproject c3e92a9b3cb02f1546b6b1570f10a968d380005 +Subproject fcd2a84e535e5d280d5299a8ff489920e1ea230 diff --git a/puppet/modules/apt b/puppet/modules/apt -Subproject fca103484ddc1f647a54135b6a902edabf45955 +Subproject ab90d1d0fe9655d367c637e95dff59e4dbe2dd3 diff --git a/puppet/modules/clamav/files/01-leap.conf b/puppet/modules/clamav/files/01-leap.conf new file mode 100644 index 00000000..abeeb302 --- /dev/null +++ b/puppet/modules/clamav/files/01-leap.conf @@ -0,0 +1,58 @@ +# If running clamd in "LocalSocket" mode (*NOT* in TCP/IP mode), and +# either "SOcket Cat" (socat) or the "IO::Socket::UNIX" perl module +# are installed on the system, and you want to report whether clamd +# is running or not, uncomment the "clamd_socket" variable below (you +# will be warned if neither socat nor IO::Socket::UNIX are found, but +# the script will still run). You will also need to set the correct +# path to your clamd socket file (if unsure of the path, check the +# "LocalSocket" setting in your clamd.conf file for socket location). +clamd_socket="/run/clamav/clamd.ctl" + +# If you would like to attempt to restart ClamD if detected not running, +# uncomment the next 2 lines. Confirm the path to the "clamd_lock" file +# (usually can be found in the clamd init script) and also enter the clamd +# start command for your particular distro for the "start_clamd" variable +# (the sample start command shown below should work for most linux distros). +# NOTE: these 2 variables are dependant on the "clamd_socket" variable +# shown above - if not enabled, then the following 2 variables will be +# ignored, whether enabled or not. +clamd_lock="/run/clamav/clamd.pid" +start_clamd="service clamav-daemon start" + +ss_dbs=" + junk.ndb + phish.ndb + rogue.hdb + sanesecurity.ftm + scam.ndb + sigwhitelist.ign2 + spamattach.hdb + spamimg.hdb + winnow.attachments.hdb + winnow_bad_cw.hdb + winnow_extended_malware.hdb + winnow_malware.hdb + winnow_malware_links.ndb + malwarehash.hsb + doppelstern.hdb + bofhland_cracked_URL.ndb + bofhland_malware_attach.hdb + bofhland_malware_URL.ndb + bofhland_phishing_URL.ndb + crdfam.clamav.hdb + phishtank.ndb + porcupine.ndb + spear.ndb + spearl.ndb +" + +# ======================== +# SecuriteInfo Database(s) +# ======================== +# Add or remove database file names between quote marks as needed. To +# disable any SecuriteInfo database downloads, remove the appropriate +# lines below. To disable all SecuriteInfo database file downloads, +# comment all of the following lines. +si_dbs="" + +mbl_dbs=""
\ No newline at end of file diff --git a/puppet/modules/clamav/files/clamav-daemon_default b/puppet/modules/clamav/files/clamav-daemon_default new file mode 100644 index 00000000..b4cd6a4f --- /dev/null +++ b/puppet/modules/clamav/files/clamav-daemon_default @@ -0,0 +1,8 @@ +# This is a file designed only t0 set special environment variables +# eg TMP or TMPDIR. It is sourced from a shell script, so anything +# put in here must be in variable=value format, suitable for sourcing +# from a shell script. +# Examples: +# export TMPDIR=/dev/shm +export TMP=/var/tmp +export TMPDIR=/var/tmp diff --git a/puppet/modules/clamav/files/clamav-milter_default b/puppet/modules/clamav/files/clamav-milter_default new file mode 100644 index 00000000..5e33e822 --- /dev/null +++ b/puppet/modules/clamav/files/clamav-milter_default @@ -0,0 +1,14 @@ +# +# clamav-milter init options +# + +## SOCKET_RWGROUP +# by default, the socket created by the milter has permissions +# clamav:clamav:755. SOCKET_RWGROUP changes the group and changes the +# permissions to 775 to give read-write access to that group. +# +# If you are using postfix to speak to the milter, you have to give permission +# to the postfix group to write +# +SOCKET_RWGROUP=postfix +export TMPDIR=/var/tmp diff --git a/puppet/modules/clamav/manifests/daemon.pp b/puppet/modules/clamav/manifests/daemon.pp new file mode 100644 index 00000000..bf232e2c --- /dev/null +++ b/puppet/modules/clamav/manifests/daemon.pp @@ -0,0 +1,90 @@ +class clamav::daemon { + + $domain_hash = hiera('domain') + $domain = $domain_hash['full_suffix'] + + package { [ 'clamav-daemon', 'arj' ]: + ensure => installed; + } + + service { + 'clamav-daemon': + ensure => running, + name => clamav-daemon, + pattern => '/usr/sbin/clamd', + enable => true, + hasrestart => true, + subscribe => File['/etc/default/clamav-daemon'], + require => Package['clamav-daemon']; + } + + file { + '/var/run/clamav': + ensure => directory, + mode => '0750', + owner => clamav, + group => postfix, + require => [Package['postfix'], Package['clamav-daemon']]; + + '/var/lib/clamav': + mode => '0755', + owner => clamav, + group => clamav, + require => Package['clamav-daemon']; + + '/etc/default/clamav-daemon': + source => 'puppet:///modules/clamav/clamav-daemon_default', + mode => '0644', + owner => root, + group => root; + + # this file contains additional domains that we want the clamav + # phishing process to look for (our domain) + '/var/lib/clamav/local.pdb': + content => template('clamav/local.pdb.erb'), + mode => '0644', + owner => clamav, + group => clamav, + require => Package['clamav-daemon']; + } + + file_line { + 'clamav_daemon_tmp': + path => '/etc/clamav/clamd.conf', + line => 'TemporaryDirectory /var/tmp', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + + 'enable_phishscanurls': + path => '/etc/clamav/clamd.conf', + match => 'PhishingScanURLs no', + line => 'PhishingScanURLs yes', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + + 'clamav_LogSyslog_true': + path => '/etc/clamav/clamd.conf', + match => '^LogSyslog false', + line => 'LogSyslog true', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + + 'clamav_MaxThreads': + path => '/etc/clamav/clamd.conf', + match => 'MaxThreads 20', + line => 'MaxThreads 100', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + } + + # remove LogFile line + file_line { + 'clamav_LogFile': + path => '/etc/clamav/clamd.conf', + match => '^LogFile .*', + line => '', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + } + +} diff --git a/puppet/modules/clamav/manifests/freshclam.pp b/puppet/modules/clamav/manifests/freshclam.pp new file mode 100644 index 00000000..80c822a4 --- /dev/null +++ b/puppet/modules/clamav/manifests/freshclam.pp @@ -0,0 +1,23 @@ +class clamav::freshclam { + + package { 'clamav-freshclam': ensure => installed } + + service { + 'freshclam': + ensure => running, + enable => true, + name => clamav-freshclam, + pattern => '/usr/bin/freshclam', + hasrestart => true, + require => Package['clamav-freshclam']; + } + + file_line { + 'freshclam_notify': + path => '/etc/clamav/freshclam.conf', + line => 'NotifyClamd /etc/clamav/clamd.conf', + require => Package['clamav-freshclam'], + notify => Service['freshclam']; + } + +} diff --git a/puppet/modules/clamav/manifests/init.pp b/puppet/modules/clamav/manifests/init.pp new file mode 100644 index 00000000..de8fb4dc --- /dev/null +++ b/puppet/modules/clamav/manifests/init.pp @@ -0,0 +1,8 @@ +class clamav { + + include clamav::daemon + include clamav::milter + include clamav::unofficial_sigs + include clamav::freshclam + +} diff --git a/puppet/modules/clamav/manifests/milter.pp b/puppet/modules/clamav/manifests/milter.pp new file mode 100644 index 00000000..e8a85e3f --- /dev/null +++ b/puppet/modules/clamav/manifests/milter.pp @@ -0,0 +1,50 @@ +class clamav::milter { + + $clamav = hiera('clamav') + $whitelisted_addresses = $clamav['whitelisted_addresses'] + $domain_hash = hiera('domain') + $domain = $domain_hash['full_suffix'] + + package { 'clamav-milter': ensure => installed } + + service { + 'clamav-milter': + ensure => running, + enable => true, + name => clamav-milter, + pattern => '/usr/sbin/clamav-milter', + hasrestart => true, + require => Package['clamav-milter'], + subscribe => File['/etc/default/clamav-milter']; + } + + file { + '/run/clamav/milter.ctl': + mode => '0666', + owner => clamav, + group => postfix, + require => Class['clamav::daemon']; + + '/etc/clamav/clamav-milter.conf': + content => template('clamav/clamav-milter.conf.erb'), + mode => '0644', + owner => root, + group => root, + require => Package['clamav-milter'], + subscribe => Service['clamav-milter']; + + '/etc/default/clamav-milter': + source => 'puppet:///modules/clamav/clamav-milter_default', + mode => '0644', + owner => root, + group => root; + + '/etc/clamav/whitelisted_addresses': + content => template('clamav/whitelisted_addresses.erb'), + mode => '0644', + owner => root, + group => root, + require => Package['clamav-milter']; + } + +} diff --git a/puppet/modules/clamav/manifests/unofficial_sigs.pp b/puppet/modules/clamav/manifests/unofficial_sigs.pp new file mode 100644 index 00000000..2d849585 --- /dev/null +++ b/puppet/modules/clamav/manifests/unofficial_sigs.pp @@ -0,0 +1,23 @@ +class clamav::unofficial_sigs { + + package { 'clamav-unofficial-sigs': + ensure => installed + } + + ensure_packages(['wget', 'gnupg', 'socat', 'rsync', 'curl']) + + file { + '/var/log/clamav-unofficial-sigs.log': + ensure => file, + owner => clamav, + group => clamav, + require => Package['clamav-unofficial-sigs']; + + '/etc/clamav-unofficial-sigs.conf.d/01-leap.conf': + source => 'puppet:///modules/clamav/01-leap.conf', + mode => '0755', + owner => root, + group => root, + require => Package['clamav-unofficial-sigs']; + } +} diff --git a/puppet/modules/clamav/templates/clamav-milter.conf.erb b/puppet/modules/clamav/templates/clamav-milter.conf.erb new file mode 100644 index 00000000..9bf7099e --- /dev/null +++ b/puppet/modules/clamav/templates/clamav-milter.conf.erb @@ -0,0 +1,28 @@ +# THIS FILE MANAGED BY PUPPET +MilterSocket /var/run/clamav/milter.ctl +FixStaleSocket true +User clamav +MilterSocketGroup clamav +MilterSocketMode 666 +AllowSupplementaryGroups true +ReadTimeout 120 +Foreground false +PidFile /var/run/clamav/clamav-milter.pid +ClamdSocket unix:/var/run/clamav/clamd.ctl +OnClean Accept +OnInfected Reject +OnFail Defer +AddHeader Replace +LogSyslog true +LogFacility LOG_LOCAL6 +LogVerbose yes +LogInfected Basic +LogTime true +LogFileUnlock false +LogClean Off +LogRotate true +SupportMultipleRecipients false +MaxFileSize 10M +TemporaryDirectory /var/tmp +RejectMsg "Message refused due to content violation: %v - contact https://<%= @domain %>/tickets/new if this is in error" +Whitelist /etc/clamav/whitelisted_addresses diff --git a/puppet/modules/clamav/templates/local.pdb.erb b/puppet/modules/clamav/templates/local.pdb.erb new file mode 100644 index 00000000..9ea0584a --- /dev/null +++ b/puppet/modules/clamav/templates/local.pdb.erb @@ -0,0 +1 @@ +H:<%= @domain %> diff --git a/puppet/modules/clamav/templates/whitelisted_addresses.erb b/puppet/modules/clamav/templates/whitelisted_addresses.erb new file mode 100644 index 00000000..9e068ec5 --- /dev/null +++ b/puppet/modules/clamav/templates/whitelisted_addresses.erb @@ -0,0 +1,5 @@ +<%- if @whitelisted_addresses then -%> +<% @whitelisted_addresses.each do |name| -%> +From::<%= name %> +<% end -%> +<% end -%> diff --git a/puppet/modules/couchdb b/puppet/modules/couchdb -Subproject 3c20a3169e77e5a5f9abc06788c3a7730d5530c +Subproject 016ec71359f6b1b368624c6c94bac2b50979165 diff --git a/puppet/modules/haproxy b/puppet/modules/haproxy -Subproject b398f3cb0a67d1170d0564a3f03977f9a08c2b6 +Subproject af322a73c013f80a958ab7d5d31d0c75cf6d052 diff --git a/puppet/modules/leap/manifests/cli/install.pp b/puppet/modules/leap/manifests/cli/install.pp new file mode 100644 index 00000000..858bd7da --- /dev/null +++ b/puppet/modules/leap/manifests/cli/install.pp @@ -0,0 +1,33 @@ +# installs leap_cli on node +class leap::cli::install ( $source = false ) { + if $source { + # needed for building leap_cli from source + include ::git + include ::site_config::ruby::dev + + vcsrepo { '/srv/leap/cli': + ensure => present, + force => true, + revision => 'develop', + provider => 'git', + source => 'https://leap.se/git/leap_cli.git', + owner => 'root', + group => 'root', + notify => Exec['install_leap_cli'], + require => Package['git'] + } + + exec { 'install_leap_cli': + command => '/usr/bin/rake build && /usr/bin/rake install', + cwd => '/srv/leap/cli', + refreshonly => true, + require => [ Package['ruby-dev'], File['/etc/gemrc'], Package['rake'] ] + } + } + else { + package { 'leap_cli': + ensure => installed, + provider => gem + } + } +} diff --git a/puppet/modules/leap_mx/manifests/init.pp b/puppet/modules/leap_mx/manifests/init.pp index 284662d2..5561e326 100644 --- a/puppet/modules/leap_mx/manifests/init.pp +++ b/puppet/modules/leap_mx/manifests/init.pp @@ -41,13 +41,7 @@ class leap_mx { notify => Service['leap-mx']; } - file { '/etc/default/leap_mx': - content => 'LOGFILE=/var/log/leap/mx.log', - owner => 'root', - group => 'root', - mode => '0644', - notify => Service['leap-mx']; - } + leap::logfile { 'mx': } # # LEAP-MX CODE AND DEPENDENCIES @@ -75,20 +69,4 @@ class leap_mx { hasrestart => true, require => [ Package['leap-mx'] ]; } - - augeas { - 'logrotate_mx': - context => '/files/etc/logrotate.d/leap-mx/rule', - changes => [ - 'set file /var/log/leap/mx.log', - 'set rotate 5', - 'set schedule daily', - 'clear nocreate', - 'rm create', - 'rm ifempty', - 'set compress compress', - 'set missingok missingok', - 'set copytruncate copytruncate' - ] - } } diff --git a/puppet/modules/nagios b/puppet/modules/nagios -Subproject b55f23d4d90c97cec08251544aa9700df86ad0b +Subproject 6c3ca97f1524e2b6242c27a2c97dbfb78105889 diff --git a/puppet/modules/opendkim/manifests/init.pp b/puppet/modules/opendkim/manifests/init.pp new file mode 100644 index 00000000..9e67569e --- /dev/null +++ b/puppet/modules/opendkim/manifests/init.pp @@ -0,0 +1,38 @@ +# configure opendkim service (#5924) +class opendkim { + + $domain_hash = hiera('domain') + $domain = $domain_hash['full_suffix'] + $dkim = hiera('dkim') + $selector = $dkim['dkim_selector'] + + include site_config::x509::dkim::key + $dkim_key = "${x509::variables::keys}/dkim.key" + + ensure_packages(['opendkim', 'libopendkim7', 'libvbr2']) + + # postfix user needs to be in the opendkim group + # in order to access the opendkim socket located at: + # local:/var/run/opendkim/opendkim.sock + user { 'postfix': + groups => 'opendkim'; + } + + service { 'opendkim': + ensure => running, + enable => true, + hasstatus => true, + hasrestart => true, + require => Class['Site_config::X509::Dkim::Key'], + subscribe => File[$dkim_key]; + } + + file { '/etc/opendkim.conf': + ensure => present, + content => template('opendkim/opendkim.conf'), + mode => '0644', + owner => root, + group => root, + notify => Service['opendkim'], + require => Package['opendkim']; +} diff --git a/puppet/modules/opendkim/templates/opendkim.conf b/puppet/modules/opendkim/templates/opendkim.conf new file mode 100644 index 00000000..46ddb7a8 --- /dev/null +++ b/puppet/modules/opendkim/templates/opendkim.conf @@ -0,0 +1,44 @@ +# This is a basic configuration that can easily be adapted to suit a standard +# installation. For more advanced options, see opendkim.conf(5) and/or +# /usr/share/doc/opendkim/examples/opendkim.conf.sample. + +# Log to syslog +Syslog yes +SyslogSuccess yes +LogWhy no +# Required to use local socket with MTAs that access the socket as a non- +# privileged user (e.g. Postfix) +UMask 002 + +Domain <%= @domain %> +SubDomains yes + +# set internal hosts to all the known hosts, like mydomains? + +# can we generate a larger key and get it in dns? +KeyFile <%= @dkim_key %> + +# what selector do we use? +Selector <%= @selector %> + +# Commonly-used options; the commented-out versions show the defaults. +Canonicalization relaxed +#Mode sv +#ADSPDiscard no + +# Always oversign From (sign using actual From and a null From to prevent +# malicious signatures header fields (From and/or others) between the signer +# and the verifier. From is oversigned by default in the Debian pacakge +# because it is often the identity key used by reputation systems and thus +# somewhat security sensitive. +OversignHeaders From + +# List domains to use for RFC 6541 DKIM Authorized Third-Party Signatures +# (ATPS) (experimental) + +#ATPSDomains example.com + +RemoveOldSignatures yes + +Mode sv +BaseDirectory /var/tmp diff --git a/puppet/modules/postfix b/puppet/modules/postfix -Subproject f09cd0eff2bcab7e12c09ec67be3c918bc83fac +Subproject 53572a8934fe5b0a3a567cdec10664f28892373 diff --git a/puppet/modules/postfwd/files/postfwd_default b/puppet/modules/postfwd/files/postfwd_default new file mode 100644 index 00000000..79d0e3de --- /dev/null +++ b/puppet/modules/postfwd/files/postfwd_default @@ -0,0 +1,19 @@ +### This file managed by Puppet +# Global options for postfwd(8). + +# Set to '1' to enable startup (daemon mode) +STARTUP=1 + +# Config file +CONF=/etc/postfix/postfwd.cf +# IP where listen to +INET=127.0.0.1 +# Port where listen to +PORT=10040 +# run as user postfwd +RUNAS="postfw" +# Arguments passed on start (--daemon implied) +# RISEUP disable summary and cache-no-size +#ARGS="--summary=600 --cache=600 --cache-rdomain-only --cache-no-size" +ARGS="--cache=600 --cache-rdomain-only --no-rulestats" + diff --git a/puppet/modules/postfwd/manifests/init.pp b/puppet/modules/postfwd/manifests/init.pp new file mode 100644 index 00000000..1ebc1d53 --- /dev/null +++ b/puppet/modules/postfwd/manifests/init.pp @@ -0,0 +1,42 @@ +# This class provides rate-limiting for outgoing SMTP, using postfwd +# it is configured with some limits that seem reasonable for a generic +# use-case. Each of the following applies to sasl_authenticated users: +# +# . 150 recipients at a time +# . no more than 50 messages in 60 minutes +# . no more than 250 recipients in 60 minutes. +# +# This class could be easily extended to add overrides to these rules, +# maximum sizes per client, or additional rules +class postfwd { + + ensure_packages(['libnet-server-perl', 'libnet-dns-perl', 'postfwd']) + + file { + '/etc/default/postfwd': + source => 'puppet:///modules/postfwd/postfwd_default', + mode => '0644', + owner => root, + group => root, + require => Package['postfwd']; + + '/etc/postfix/postfwd.cf': + content => template('postfwd/postfwd.cf.erb'), + mode => '0644', + owner => root, + group => root, + require => Package['postfix']; + } + + service { + 'postfwd': + ensure => running, + name => postfwd, + pattern => '/usr/sbin/postfwd', + enable => true, + hasrestart => true, + hasstatus => false, + require => [ File['/etc/default/postfwd'], + File['/etc/postfix/postfwd.cf']]; + } +} diff --git a/puppet/modules/postfwd/templates/postfwd.cf.erb b/puppet/modules/postfwd/templates/postfwd.cf.erb new file mode 100644 index 00000000..1c45dd03 --- /dev/null +++ b/puppet/modules/postfwd/templates/postfwd.cf.erb @@ -0,0 +1,28 @@ +### This file managed by Puppet +# Before deploying a rule +# 1. test with an additional "sender==test@domain.org;" in the rule so it +# only applies to your test account +# 2. then when ready to test for all users, use WARN and watch the logs +# for a few days and make sure it working the way you like +# 3. Then when ready to deploy for real set a proper error code + +## Overrides - make like the following example +# id=exampleuser; sasl_username==exampleuser; action=dunno + +## Rules that apply to all senders +# Recipient Per Message Limit +# We only receive mail via smtp from sasl authenticated users +# directly. We want to limit to a lower amount to prevent phished accounts +# spamming +id=RCPTSENDER; recipient_count=150; action=REJECT Too many recipients, please try again. Contact http://<%= @domain %>/tickets/new if this is in error. ERROR:RCPTSENDER + +# Message Rate Limit +# This limits sasl authenticated users to no more than 50/60mins +# NOTE: sasl_username needs to be set to something or this check will fail +id=MSGRATE ; sasl_username=!!(^$); action==rate($$sasl_username/100/3600/450 4.7.1 exceeded message rate. Contact Contact http://<%= @domain %>/tickets/new if this is in error. ERROR:MSGRATE) + +# Total Recipient Rate Limit +# This adds up the recipients for all the sasl authenticated users messages +# and can't exceed more than 250/60min +# NOTE: sasl_username needs to be set to something or this check will fail +id=RCPTRATE ; sasl_username=!!(^$); action==rcpt($$sasl_username/500/3600/450 4.7.1 exceeded message rate. Contact http://<%= @domain %>/tickets/new if this is in error. ERROR:RCPTRATE) diff --git a/puppet/modules/ruby b/puppet/modules/ruby -Subproject e4de25d78eefc7df70a35dee22a3e0dc1b7e1d0 +Subproject 0fb2b398dbfce59c678d6f4044a55969e42c6d4 diff --git a/puppet/modules/site_apache/manifests/common.pp b/puppet/modules/site_apache/manifests/common.pp index 2b83ffa5..dadf7ea5 100644 --- a/puppet/modules/site_apache/manifests/common.pp +++ b/puppet/modules/site_apache/manifests/common.pp @@ -1,27 +1,26 @@ +# install basic apache modules needed for all services (nagios, webapp) class site_apache::common { - # installs x509 cert + key and common config - # that both nagios + leap webapp use - $web_domain = hiera('domain') - $domain_name = $web_domain['name'] - - include x509::variables - include site_config::x509::commercial::cert - include site_config::x509::commercial::key - include site_config::x509::commercial::ca - - Class['Site_config::X509::Commercial::Key'] ~> Service[apache] - Class['Site_config::X509::Commercial::Cert'] ~> Service[apache] - Class['Site_config::X509::Commercial::Ca'] ~> Service[apache] - - include site_apache::module::rewrite + include apache::module::rewrite + include apache::module::env class { '::apache': no_default_site => true, ssl => true } - apache::vhost::file { - 'common': - content => template('site_apache/vhosts.d/common.conf.erb') + # needed for the mod_ssl config + include apache::module::mime + + # load mods depending on apache version + if ( $::lsbdistcodename == 'jessie' ) { + # apache >= 2.4, debian jessie + # needed for mod_ssl config + include apache::module::socache_shmcb + # generally needed + include apache::module::mpm_prefork + } else { + # apache < 2.4, debian wheezy + # for "Order" directive, i.e. main apache2.conf + include apache::module::authz_host } - apache::config::include{ 'ssl_common.inc': } + include site_apache::common::tls } diff --git a/puppet/modules/site_apache/manifests/common/tls.pp b/puppet/modules/site_apache/manifests/common/tls.pp new file mode 100644 index 00000000..040868bf --- /dev/null +++ b/puppet/modules/site_apache/manifests/common/tls.pp @@ -0,0 +1,6 @@ +class site_apache::common::tls { + # class to setup common SSL configurations + + apache::config::include{ 'ssl_common.inc': } + +} diff --git a/puppet/modules/site_apache/manifests/module/alias.pp b/puppet/modules/site_apache/manifests/module/alias.pp deleted file mode 100644 index c1f5e185..00000000 --- a/puppet/modules/site_apache/manifests/module/alias.pp +++ /dev/null @@ -1,5 +0,0 @@ -class site_apache::module::alias ( $ensure = present ) -{ - - apache::module { 'alias': ensure => $ensure } -} diff --git a/puppet/modules/site_apache/manifests/module/expires.pp b/puppet/modules/site_apache/manifests/module/expires.pp deleted file mode 100644 index f73a5607..00000000 --- a/puppet/modules/site_apache/manifests/module/expires.pp +++ /dev/null @@ -1,4 +0,0 @@ -class site_apache::module::expires ( $ensure = present ) -{ - apache::module { 'expires': ensure => $ensure } -} diff --git a/puppet/modules/site_apache/manifests/module/headers.pp b/puppet/modules/site_apache/manifests/module/headers.pp deleted file mode 100644 index f7caa28c..00000000 --- a/puppet/modules/site_apache/manifests/module/headers.pp +++ /dev/null @@ -1,5 +0,0 @@ -class site_apache::module::headers ( $ensure = present ) -{ - - apache::module {'headers': ensure => $ensure } -} diff --git a/puppet/modules/site_apache/manifests/module/removeip.pp b/puppet/modules/site_apache/manifests/module/removeip.pp deleted file mode 100644 index f106167a..00000000 --- a/puppet/modules/site_apache/manifests/module/removeip.pp +++ /dev/null @@ -1,5 +0,0 @@ -class site_apache::module::removeip ( $ensure = present ) -{ - package { 'libapache2-mod-removeip': ensure => $ensure } - apache::module { 'removeip': ensure => $ensure } -} diff --git a/puppet/modules/site_apache/manifests/module/rewrite.pp b/puppet/modules/site_apache/manifests/module/rewrite.pp deleted file mode 100644 index 7ad00a0c..00000000 --- a/puppet/modules/site_apache/manifests/module/rewrite.pp +++ /dev/null @@ -1,5 +0,0 @@ -class site_apache::module::rewrite ( $ensure = present ) -{ - - apache::module { 'rewrite': ensure => $ensure } -} diff --git a/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb index 0396f54b..9efc6b41 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb @@ -1,14 +1,14 @@ <VirtualHost *:80> - ServerName <%= api_domain %> + ServerName <%= @api_domain %> RewriteEngine On - RewriteRule ^.*$ https://<%= api_domain -%>:<%= api_port -%>%{REQUEST_URI} [R=permanent,L] + RewriteRule ^.*$ https://<%= @api_domain -%>:<%= @api_port -%>%{REQUEST_URI} [R=permanent,L] CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common </VirtualHost> -Listen 0.0.0.0:<%= api_port %> +Listen 0.0.0.0:<%= @api_port %> -<VirtualHost *:<%= api_port -%>> - ServerName <%= api_domain %> +<VirtualHost *:<%= @api_port -%>> + ServerName <%= @api_domain %> CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common SSLCACertificatePath /etc/ssl/certs @@ -27,6 +27,12 @@ Listen 0.0.0.0:<%= api_port %> </IfModule> DocumentRoot /srv/leap/webapp/public + <% if Gem::Version.new(@apache_version) > Gem::Version.new('2.3') %> + <Directory /srv/leap/webapp/public> + AllowOverride None + Require all granted + </Directory> + <% end %> # Check for maintenance file and redirect all requests RewriteEngine On diff --git a/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb index ee5cd707..cbb08c30 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb @@ -1,18 +1,18 @@ <VirtualHost *:80> - ServerName <%= webapp_domain %> - ServerAlias <%= domain_name %> - ServerAlias <%= domain %> - ServerAlias www.<%= domain %> + ServerName <%= @webapp_domain %> + ServerAlias <%= @domain_name %> + ServerAlias <%= @domain %> + ServerAlias www.<%= @domain %> RewriteEngine On - RewriteRule ^.*$ https://<%= domain -%>%{REQUEST_URI} [R=permanent,L] + RewriteRule ^.*$ https://<%= @webapp_domain -%>%{REQUEST_URI} [R=permanent,L] CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common </VirtualHost> <VirtualHost *:443> - ServerName <%= webapp_domain %> - ServerAlias <%= domain_name %> - ServerAlias <%= domain %> - ServerAlias www.<%= domain %> + ServerName <%= @webapp_domain %> + ServerAlias <%= @domain_name %> + ServerAlias <%= @domain %> + ServerAlias www.<%= @domain %> CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common SSLCACertificatePath /etc/ssl/certs @@ -32,6 +32,12 @@ <% if (defined? @services) and (@services.include? 'webapp') -%> DocumentRoot /srv/leap/webapp/public + <% if Gem::Version.new(@apache_version) > Gem::Version.new('2.3') %> + <Directory /srv/leap/webapp/public> + AllowOverride None + Require all granted + </Directory> + <% end %> RewriteEngine On # Check for maintenance file and redirect all requests @@ -69,4 +75,3 @@ </DirectoryMatch> <% end -%> </VirtualHost> - diff --git a/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb index 0c6f3b8e..2c8d5eb5 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb @@ -30,4 +30,14 @@ ExpiresDefault "access plus 1 year" </Location> <% end -%> + +<% if (defined? @services) and (@services.include? 'static') -%> + DocumentRoot "/srv/static/root/public" + AccessFileName .htaccess + + Alias /provider.json /srv/leap/provider.json + <Location /provider.json> + Header set X-Minimum-Client-Version 0.5 + </Location> +<% end -%> </VirtualHost> diff --git a/puppet/modules/site_apt/manifests/init.pp b/puppet/modules/site_apt/manifests/init.pp index cf49f870..635ba975 100644 --- a/puppet/modules/site_apt/manifests/init.pp +++ b/puppet/modules/site_apt/manifests/init.pp @@ -7,11 +7,19 @@ class site_apt { $apt_url_security = $apt_config['security'] $apt_url_backports = $apt_config['backports'] + # needed on jessie hosts for getting pnp4nagios from testing + if ( $::operatingsystemmajrelease == '8' ) { + $use_next_release = true + } else { + $use_next_release = false + } + class { 'apt': - custom_key_dir => 'puppet:///modules/site_apt/keys', - debian_url => $apt_url_basic, - security_url => $apt_url_security, - backports_url => $apt_url_backports + custom_key_dir => 'puppet:///modules/site_apt/keys', + debian_url => $apt_url_basic, + security_url => $apt_url_security, + backports_url => $apt_url_backports, + use_next_release => $use_next_release } # enable http://deb.leap.se debian package repository diff --git a/puppet/modules/site_apt/manifests/leap_repo.pp b/puppet/modules/site_apt/manifests/leap_repo.pp index 2d4ba0e1..462b2686 100644 --- a/puppet/modules/site_apt/manifests/leap_repo.pp +++ b/puppet/modules/site_apt/manifests/leap_repo.pp @@ -1,9 +1,11 @@ +# install leap deb repo together with leap-keyring package +# containing the apt signing key class site_apt::leap_repo { $platform = hiera_hash('platform') $major_version = $platform['major_version'] apt::sources_list { 'leap.list': - content => "deb http://deb.leap.se/${major_version} wheezy main\n", + content => "deb http://deb.leap.se/${major_version} ${::lsbdistcodename} main\n", before => Exec[refresh_apt] } diff --git a/puppet/modules/site_apt/manifests/sid_repo.pp b/puppet/modules/site_apt/manifests/sid_repo.pp new file mode 100644 index 00000000..7c1d8783 --- /dev/null +++ b/puppet/modules/site_apt/manifests/sid_repo.pp @@ -0,0 +1,11 @@ +# configure debian unstable aka "sid" +# currently only used for installations that +# use plain couchdb instead of bigcouch +class site_apt::sid_repo { + + apt::sources_list { 'debian_sid.list': + content => "deb http://httpredir.debian.org/debian/ sid main\n", + before => Exec[refresh_apt] + } + +} diff --git a/puppet/modules/site_apt/templates/jessie/postfix.seeds b/puppet/modules/site_apt/templates/jessie/postfix.seeds new file mode 100644 index 00000000..1a878ccc --- /dev/null +++ b/puppet/modules/site_apt/templates/jessie/postfix.seeds @@ -0,0 +1 @@ +postfix postfix/main_mailer_type select No configuration diff --git a/puppet/modules/site_apt/templates/secondary.list b/puppet/modules/site_apt/templates/secondary.list index 41334b0b..0c024549 100644 --- a/puppet/modules/site_apt/templates/secondary.list +++ b/puppet/modules/site_apt/templates/secondary.list @@ -1,3 +1,3 @@ # basic -deb http://ftp.debian.org/debian/ <%= lsbdistcodename %> main contrib non-free +deb http://ftp.debian.org/debian/ <%= @lsbdistcodename %> main contrib non-free diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg index 95ddd2ca..0f378a5a 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg @@ -6,7 +6,7 @@ I 127.0.0.1 localhost:5984 .* ok # https://leap.se/code/issues/5246 I Shutting down group server - # ignore bigcouch conflict errors, mainly coming from tapicero creating new users + # ignore bigcouch conflict errors I Error in process.*{{nocatch,conflict} # ignore "Uncaught error in HTTP request: {exit, normal}" error # it's suppressed in later versions of bigcouch anhow diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/bigcouch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/bigcouch.cfg new file mode 100644 index 00000000..f53f0780 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/bigcouch.cfg @@ -0,0 +1,5 @@ +# on one-node bigcouch setups, we'll get this msg +# a lot, so we ignore it here until we fix +# https://leap.se/code/issues/5244 + I epmd: got partial packet only on file descriptor + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg index f546135a..5f8d5b95 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg @@ -1,7 +1,2 @@ C /usr/local/bin/couch-doc-update.*failed C /usr/local/bin/couch-doc-update.*ERROR -# on one-node bigcouch setups, we'll get this msg -# a lot, so we ignore it here until we fix -# https://leap.se/code/issues/5244 - I epmd: got partial packet only on file descriptor - diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/tapicero.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/tapicero.cfg deleted file mode 100644 index d98f5094..00000000 --- a/puppet/modules/site_check_mk/files/agent/logwatch/tapicero.cfg +++ /dev/null @@ -1,11 +0,0 @@ -/var/log/leap/tapicero.log -# Ignore transient Tapicero errors when creating a db (#6511) - I tapicero.*(Creating database|Checking security of|Writing security to|Uploading design doc to) user-.* failed (\(trying again soon\)|(twice )?due to): (RestClient::ResourceNotFound|RestClient::InternalServerError): (404 Resource Not Found|500 Internal Server Error) - C tapicero.*RestClient::InternalServerError: -# possible race condition between multiple tapicero -# instances, so we ignore it -# see https://leap.se/code/issues/5168 - I tapicero.*RestClient::PreconditionFailed: - C tapicero.*Creating database.*failed due to: - C tapicero.*failed - W tapicero.*Couch stream ended unexpectedly. diff --git a/puppet/modules/site_check_mk/files/extra_host_conf.mk b/puppet/modules/site_check_mk/files/extra_host_conf.mk deleted file mode 100644 index 2c96f97a..00000000 --- a/puppet/modules/site_check_mk/files/extra_host_conf.mk +++ /dev/null @@ -1,6 +0,0 @@ -# retry 3 times before setting a host into a hard state -# and send out notification -extra_host_conf["max_check_attempts"] = [ - ("4", ALL_HOSTS ) -] - diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp index abfc7ad0..1554fd3c 100644 --- a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp @@ -1,32 +1,18 @@ +# configure logwatch and nagios checks for couchdb (both bigcouch and plain +# couchdb installations) class site_check_mk::agent::couchdb { - # watch logs - file { '/etc/check_mk/logwatch.d/bigcouch.cfg': - source => 'puppet:///modules/site_check_mk/agent/logwatch/bigcouch.cfg', - } concat::fragment { 'syslog_couchdb': source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/couchdb.cfg', target => '/etc/check_mk/logwatch.d/syslog.cfg', order => '02'; } - - # check bigcouch processes - augeas { - 'Bigcouch_epmd_procs': - incl => '/etc/check_mk/mrpe.cfg', - lens => 'Spacevars.lns', - changes => [ - 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_epmd_procs', - 'set Bigcouch_epmd_procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/epmd\'' ], - require => File['/etc/check_mk/mrpe.cfg']; - 'Bigcouch_beam_procs': - incl => '/etc/check_mk/mrpe.cfg', - lens => 'Spacevars.lns', - changes => [ - 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_beam_procs', - 'set Bigcouch_beam_procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/beam\'' ], - require => File['/etc/check_mk/mrpe.cfg']; + # check different couchdb stats + file { '/usr/lib/check_mk_agent/local/leap_couch_stats.sh': + source => 'puppet:///modules/site_check_mk/agent/local_checks/couchdb/leap_couch_stats.sh', + mode => '0755', + require => Package['check_mk-agent'] } # check open files for bigcouch proc @@ -36,20 +22,13 @@ class site_check_mk::agent::couchdb { mode => '0755' } augeas { - 'Bigcouch_open_files': + 'Couchdb_open_files': incl => '/etc/check_mk/mrpe.cfg', lens => 'Spacevars.lns', changes => [ - 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_open_files', - 'set Bigcouch_open_files \'/srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 28672,28672 -c 30720,30720\'' ], + 'rm /files/etc/check_mk/mrpe.cfg/Couchdb_open_files', + 'set Couchdb_open_files \'/srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 28672,28672 -c 30720,30720\'' ], require => File['/etc/check_mk/mrpe.cfg']; } - - # check different couchdb stats - file { '/usr/lib/check_mk_agent/local/leap_couch_stats.sh': - source => 'puppet:///modules/site_check_mk/agent/local_checks/couchdb/leap_couch_stats.sh', - mode => '0755', - require => Package['check_mk-agent'] - } } diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp new file mode 100644 index 00000000..82c3ac72 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp @@ -0,0 +1,49 @@ +# configure logwatch and nagios checks for bigcouch +class site_check_mk::agent::couchdb::bigcouch { + + # watch bigcouch logs + # currently disabled because bigcouch is too noisy + # see https://leap.se/code/issues/7375 for more details + # and site_config::remove_files for removing leftovers + #file { '/etc/check_mk/logwatch.d/bigcouch.cfg': + # source => 'puppet:///modules/site_check_mk/agent/logwatch/bigcouch.cfg', + #} + + # check syslog msg from: + # - empd + # - /usr/local/bin/couch-doc-update + concat::fragment { 'syslog_bigcouch': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/bigcouch.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + + # check bigcouch processes + augeas { + 'Bigcouch_epmd_procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_epmd_procs', + 'set Bigcouch_epmd_procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/epmd\'' ], + require => File['/etc/check_mk/mrpe.cfg']; + 'Bigcouch_beam_procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_beam_procs', + 'set Bigcouch_beam_procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/beam\'' ], + require => File['/etc/check_mk/mrpe.cfg']; + } + + augeas { + 'Bigcouch_open_files': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_open_files', + 'set Bigcouch_open_files \'/srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 28672,28672 -c 30720,30720\'' ], + require => File['/etc/check_mk/mrpe.cfg']; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb/master.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb/master.pp new file mode 100644 index 00000000..291b87d1 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb/master.pp @@ -0,0 +1,23 @@ +# configure logwatch and nagios checks for plain single couchdb master +class site_check_mk::agent::couchdb::master { + + # remove bigcouch leftovers + augeas { + 'Bigcouch_epmd_procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_epmd_procs', + require => File['/etc/check_mk/mrpe.cfg']; + 'Bigcouch_beam_procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_beam_procs', + require => File['/etc/check_mk/mrpe.cfg']; + 'Bigcouch_open_files': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_open_files', + require => File['/etc/check_mk/mrpe.cfg']; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/mx.pp b/puppet/modules/site_check_mk/manifests/agent/mx.pp index 98757b59..20cbcade 100644 --- a/puppet/modules/site_check_mk/manifests/agent/mx.pp +++ b/puppet/modules/site_check_mk/manifests/agent/mx.pp @@ -1,3 +1,4 @@ +# check check_mk agent checks for mx service class site_check_mk::agent::mx { # watch logs @@ -6,13 +7,13 @@ class site_check_mk::agent::mx { } # local nagios plugin checks via mrpe + # removed because leap_cli integrates a check for running mx procs already, + # which is also integrated into nagios (called "Mx/Are_MX_daemons_running") augeas { 'Leap_MX_Procs': incl => '/etc/check_mk/mrpe.cfg', lens => 'Spacevars.lns', - changes => [ - 'rm /files/etc/check_mk/mrpe.cfg/Leap_MX_Procs', - 'set Leap_MX_Procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a "/usr/bin/python /usr/bin/twistd --pidfile=/var/run/leap_mx.pid --rundir=/var/lib/leap_mx/ --python=/usr/share/app/leap_mx.tac --logfile=/var/log/leap/mx.log"\'' ], + changes => 'rm /files/etc/check_mk/mrpe.cfg/Leap_MX_Procs', require => File['/etc/check_mk/mrpe.cfg']; } diff --git a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp deleted file mode 100644 index 8505b34a..00000000 --- a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp +++ /dev/null @@ -1,26 +0,0 @@ -# sets up tapicero monitoring -class site_check_mk::agent::tapicero { - - include ::site_nagios::plugins - - # watch logs - file { '/etc/check_mk/logwatch.d/tapicero.cfg': - source => 'puppet:///modules/site_check_mk/agent/logwatch/tapicero.cfg', - } - - # local nagios plugin checks via mrpe - augeas { - 'Tapicero_Procs': - incl => '/etc/check_mk/mrpe.cfg', - lens => 'Spacevars.lns', - changes => [ - 'rm /files/etc/check_mk/mrpe.cfg/Tapicero_Procs', - "set Tapicero_Procs \"/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 --ereg-argument-array='^tapicero$'\"" ], - require => File['/etc/check_mk/mrpe.cfg']; - 'Tapicero_Heartbeat': - incl => '/etc/check_mk/mrpe.cfg', - lens => 'Spacevars.lns', - changes => 'set Tapicero_Heartbeat \'/usr/local/lib/nagios/plugins/check_last_regex_in_log -f /var/log/leap/tapicero.log -r "tapicero" -w 1200 -c 2400\'', - require => File['/etc/check_mk/mrpe.cfg']; - } -} diff --git a/puppet/modules/site_check_mk/manifests/server.pp b/puppet/modules/site_check_mk/manifests/server.pp index 67519513..0159a050 100644 --- a/puppet/modules/site_check_mk/manifests/server.pp +++ b/puppet/modules/site_check_mk/manifests/server.pp @@ -17,6 +17,19 @@ class site_check_mk::server { ensure => installed, } + # we don't use check-mk-multisite, and the jessie version + # of this config file breaks with apache 2.4 + # until https://gitlab.com/shared-puppet-modules-group/apache/issues/11 + # is not fixed, we need to use a generic file type here + #apache::config::global { 'check-mk-multisite.conf': + # ensure => absent + #} + + file { '/etc/apache2/conf-enabled/check-mk-multisite.conf': + ensure => absent, + require => Package['check-mk-server']; + } + # override paths to use the system check_mk rather than OMD class { 'check_mk::config': site => '', @@ -54,7 +67,7 @@ class site_check_mk::server { notify => Exec['check_mk-refresh'], require => Package['check-mk-server']; '/etc/check_mk/conf.d/extra_host_conf.mk': - source => 'puppet:///modules/site_check_mk/extra_host_conf.mk', + content => template('site_check_mk/extra_host_conf.mk'), notify => Exec['check_mk-refresh'], require => Package['check-mk-server']; diff --git a/puppet/modules/site_check_mk/templates/extra_host_conf.mk b/puppet/modules/site_check_mk/templates/extra_host_conf.mk new file mode 100644 index 00000000..bc27b514 --- /dev/null +++ b/puppet/modules/site_check_mk/templates/extra_host_conf.mk @@ -0,0 +1,13 @@ +# retry 3 times before setting a host into a hard state +# and send out notification +extra_host_conf["max_check_attempts"] = [ + ("4", ALL_HOSTS ) +] + +# Use hostnames as alias so notification mail subjects +# are more readable and not so long. Alias defaults to +# the fqdn of a host is not changed. +extra_host_conf["alias"] = [ +<% @hosts.keys.sort.each do |key| -%> ( "<%= key.strip %>", ["<%= @hosts[key]['domain_internal']%>"]), +<% end -%> +] diff --git a/puppet/modules/site_check_mk/templates/use_ssh.mk b/puppet/modules/site_check_mk/templates/use_ssh.mk index 0bebebcf..55269536 100644 --- a/puppet/modules/site_check_mk/templates/use_ssh.mk +++ b/puppet/modules/site_check_mk/templates/use_ssh.mk @@ -1,6 +1,6 @@ # http://mathias-kettner.de/checkmk_datasource_programs.html datasource_programs = [ -<% nagios_hosts.sort.each do |name,config| %> +<% @nagios_hosts.sort.each do |name,config| %> ( "ssh -l root -i /etc/check_mk/.ssh/id_rsa -p <%=config['ssh_port']%> <%=config['domain_internal']%> check_mk_agent", [ "<%=config['domain_internal']%>" ], ),<%- end -%> ] diff --git a/puppet/modules/site_config/manifests/default.pp b/puppet/modules/site_config/manifests/default.pp index e69e4b7b..6b10dc19 100644 --- a/puppet/modules/site_config/manifests/default.pp +++ b/puppet/modules/site_config/manifests/default.pp @@ -1,3 +1,4 @@ +# common things to set up on every node class site_config::default { tag 'leap_base' @@ -29,7 +30,7 @@ class site_config::default { # i.e. openstack/aws nodes, vagrant nodes # fix dhclient from changing resolver information - if $::dhcp_enabled == 'true' { + if $::dhcp_enabled == 'true' { include site_config::dhclient } @@ -58,7 +59,9 @@ class site_config::default { # set up core leap files and directories include site_config::files - include site_config::remove_files + + # remove leftovers from previous deploys + include site_config::remove if ! member($services, 'mx') { include site_postfix::satellite diff --git a/puppet/modules/site_config/manifests/packages/build_essential.pp b/puppet/modules/site_config/manifests/packages/build_essential.pp index 7dfb8b03..8f3b2641 100644 --- a/puppet/modules/site_config/manifests/packages/build_essential.pp +++ b/puppet/modules/site_config/manifests/packages/build_essential.pp @@ -4,8 +4,8 @@ class site_config::packages::build_essential { if !defined(Package['build-essential']) { package { - ['build-essential', 'g++', 'g++-4.7', 'gcc', 'gcc-4.6', 'gcc-4.7', 'cpp', 'cpp-4.6', 'cpp-4.7', 'libc6-dev']: + ['build-essential', 'cpp']: ensure => present } } -}
\ No newline at end of file +} diff --git a/puppet/modules/site_config/manifests/remove.pp b/puppet/modules/site_config/manifests/remove.pp new file mode 100644 index 00000000..b1ad1a2b --- /dev/null +++ b/puppet/modules/site_config/manifests/remove.pp @@ -0,0 +1,4 @@ +# remove leftovers from previous deploys +class site_config::remove { + include site_config::remove::files +} diff --git a/puppet/modules/site_config/manifests/remove_files.pp b/puppet/modules/site_config/manifests/remove/files.pp index b339e6af..67171259 100644 --- a/puppet/modules/site_config/manifests/remove_files.pp +++ b/puppet/modules/site_config/manifests/remove/files.pp @@ -9,7 +9,13 @@ # release. # -class site_config::remove_files { +class site_config::remove::files { + + # Platform 0.8 removals + tidy { + '/etc/default/leap_mx':; + '/etc/logrotate.d/leap-mx':; + } # # Platform 0.7 removals @@ -20,21 +26,24 @@ class site_config::remove_files { '/etc/rsyslog.d/99-leap-mx.conf':; '/etc/rsyslog.d/01-webapp.conf':; '/etc/rsyslog.d/50-stunnel.conf':; - '/etc/logrotate.d/mx':; '/etc/logrotate.d/stunnel':; '/var/log/stunnel4/stunnel.log':; 'leap_mx': path => '/var/log/', recurse => true, matches => 'leap_mx*'; + # We rotate 5 logs, so we should only have mx.log, mx.log.[1-5], with an + # optional .gz suffix. The following will remove any logs that are out + # of this range 'leap_mx_rotate': path => '/var/log/leap/', recurse => true, - matches => [ 'mx.log.[0-9]', 'mx.log.[0-9]?', 'mx.log.[6-9]?gz']; + matches => [ 'mx.log.[6-9](.gz)?', 'mx.log.[0-9][0-9]']; '/srv/leap/webapp/public/provider.json':; '/srv/leap/couchdb/designs/tmp_users': recurse => true, rmdirs => true; + '/etc/leap/soledad-server.conf':; } # leax-mx logged to /var/log/leap_mx.log in the past @@ -46,5 +55,22 @@ class site_config::remove_files { onlyif => "/bin/grep -qe 'leap_mx.log' /etc/check_mk/logwatch.state" } - + # Don't use check_mk logwatch to watch bigcouch logs anymore + # see https://leap.se/code/issues/7375 for more details + file { '/etc/check_mk/logwatch.d/bigcouch.cfg': + ensure => absent, + notify => [ + Exec['remove_bigcouch_logwatch_spoolfiles'], + Exec['remove_bigcouch_logwatch_stateline'] + ] + } + # remove leftover bigcouch logwatch spool files + exec { 'remove_bigcouch_logwatch_spoolfiles': + command => 'find /var/lib/check_mk/logwatch -name \'\\opt\\bigcouch\\var\\log\\bigcouch.log\' -exec rm {} \;', + refreshonly => true, + } + exec { 'remove_bigcouch_logwatch_stateline': + command => "sed -i '/bigcouch.log/d' /etc/check_mk/logwatch.state", + refreshonly => true, + } } diff --git a/puppet/modules/site_config/manifests/remove/monitoring.pp b/puppet/modules/site_config/manifests/remove/monitoring.pp new file mode 100644 index 00000000..d7095597 --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/monitoring.pp @@ -0,0 +1,10 @@ +# remove leftovers on monitoring nodes +class site_config::remove::monitoring { + + tidy { + 'checkmk_logwatch_spool': + path => '/var/lib/check_mk/logwatch', + recurse => true, + matches => '*tapicero.log' + } +} diff --git a/puppet/modules/site_config/manifests/remove/tapicero.pp b/puppet/modules/site_config/manifests/remove/tapicero.pp new file mode 100644 index 00000000..4ce972d0 --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/tapicero.pp @@ -0,0 +1,69 @@ +# remove tapicero leftovers from previous deploys on couchdb nodes +class site_config::remove::tapicero { + + # remove tapicero couchdb user + $couchdb_config = hiera('couch') + $couchdb_mode = $couchdb_config['mode'] + + if $couchdb_mode == 'multimaster' + { + $port = 5986 + } else { + $port = 5984 + } + + exec { 'remove_couchdb_user': + onlyif => "/usr/bin/curl -s 127.0.0.1:${port}/_users/org.couchdb.user:tapicero | grep -qv 'not_found'", + command => "/usr/local/bin/couch-doc-update --host 127.0.0.1:${port} --db _users --id org.couchdb.user:tapicero --delete" + } + + + exec { 'kill_tapicero': + onlyif => '/usr/bin/test -s /var/run/tapicero.pid', + command => '/usr/bin/pkill --pidfile /var/run/tapicero.pid' + } + + user { 'tapicero': + ensure => absent; + } + + group { 'tapicero': + ensure => absent, + require => User['tapicero']; + } + + tidy { + '/srv/leap/tapicero': + recurse => true, + require => [ Exec['kill_tapicero'] ]; + '/var/lib/leap/tapicero': + require => [ Exec['kill_tapicero'] ]; + '/var/run/tapicero': + require => [ Exec['kill_tapicero'] ]; + '/etc/leap/tapicero.yaml': + require => [ Exec['kill_tapicero'] ]; + '/etc/init.d/tapicero': + require => [ Exec['kill_tapicero'] ]; + 'tapicero_logs': + path => '/var/log/leap', + recurse => true, + matches => 'tapicero*', + require => [ Exec['kill_tapicero'] ]; + '/etc/check_mk/logwatch.d/tapicero.cfg':; + } + + # remove local nagios plugin checks via mrpe + augeas { + 'Tapicero_Procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Tapicero_Procs', + require => File['/etc/check_mk/mrpe.cfg']; + 'Tapicero_Heartbeat': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm Tapicero_Heartbeat', + require => File['/etc/check_mk/mrpe.cfg']; + } + +} diff --git a/puppet/modules/site_config/manifests/remove/webapp.pp b/puppet/modules/site_config/manifests/remove/webapp.pp new file mode 100644 index 00000000..58f59815 --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/webapp.pp @@ -0,0 +1,7 @@ +# remove leftovers on webapp nodes +class site_config::remove::webapp { + tidy { + '/etc/apache/sites-enabled/leap_webapp.conf': + notify => Service['apache']; + } +} diff --git a/puppet/modules/site_config/manifests/ruby.pp b/puppet/modules/site_config/manifests/ruby.pp index 2a720114..5c13233d 100644 --- a/puppet/modules/site_config/manifests/ruby.pp +++ b/puppet/modules/site_config/manifests/ruby.pp @@ -1,14 +1,8 @@ +# install ruby, rubygems and bundler +# configure ruby settings common to all servers class site_config::ruby { Class[Ruby] -> Class[rubygems] -> Class[bundler::install] - class { '::ruby': ruby_version => '1.9.3' } + class { '::ruby': } class { 'bundler::install': install_method => 'package' } include rubygems } - - -# -# Ruby settings common to all servers -# -# Why this way? So that other classes can do 'include site_ruby' without creating redeclaration errors. -# See https://puppetlabs.com/blog/modeling-class-composition-with-parameterized-classes/ -# diff --git a/puppet/modules/site_config/manifests/ruby/dev.pp b/puppet/modules/site_config/manifests/ruby/dev.pp index 3ea6ca96..e6eb2f8a 100644 --- a/puppet/modules/site_config/manifests/ruby/dev.pp +++ b/puppet/modules/site_config/manifests/ruby/dev.pp @@ -1,6 +1,6 @@ +# install ruby dev packages needed for building some gems class site_config::ruby::dev inherits site_config::ruby { Class['::ruby'] { - ruby_version => '1.9.3', install_dev => true } # building gems locally probably requires build-essential and gcc: diff --git a/puppet/modules/site_config/manifests/x509/dkim/key.pp b/puppet/modules/site_config/manifests/x509/dkim/key.pp new file mode 100644 index 00000000..c63a7e94 --- /dev/null +++ b/puppet/modules/site_config/manifests/x509/dkim/key.pp @@ -0,0 +1,13 @@ +class site_config::x509::dkim::key { + + ## + ## This is for the DKIM key that is used exclusively for DKIM + ## signing + + $x509 = hiera('x509') + $key = $x509['dkim_key'] + + x509::key { 'dkim': + content => $key + } +} diff --git a/puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json b/puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json new file mode 100644 index 00000000..006c1ea1 --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json @@ -0,0 +1,22 @@ +{ + "_id": "_design/InviteCode", + "language": "javascript", + "views": { + "by__id": { + "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['_id'] != null)) {\n emit(doc['_id'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_invite_code": { + "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['invite_code'] != null)) {\n emit(doc['invite_code'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_invite_count": { + "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['invite_count'] != null)) {\n emit(doc['invite_count'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "all": { + "map": " function(doc) {\n if (doc['type'] == 'InviteCode') {\n emit(doc._id, null);\n }\n }\n" + } + }, + "couchrest-hash": "83fb8f504520b4a9c7ddbb7928cd0ce3" +}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/manifests/add_users.pp b/puppet/modules/site_couchdb/manifests/add_users.pp index 2f734ed4..c905316b 100644 --- a/puppet/modules/site_couchdb/manifests/add_users.pp +++ b/puppet/modules/site_couchdb/manifests/add_users.pp @@ -1,3 +1,4 @@ +# add couchdb users for all services class site_couchdb::add_users { Class['site_couchdb::create_dbs'] @@ -35,16 +36,6 @@ class site_couchdb::add_users { require => Couchdb::Query::Setup['localhost'] } - ### tapicero couchdb user - ### admin: needs to be able to create user-<uuid> databases - ### read: users - couchdb::add_user { $site_couchdb::couchdb_tapicero_user: - roles => '["users"]', - pw => $site_couchdb::couchdb_tapicero_pw, - salt => $site_couchdb::couchdb_tapicero_salt, - require => Couchdb::Query::Setup['localhost'] - } - ## webapp couchdb user ## read/write: users, tokens, sessions, tickets, identities, customer couchdb::add_user { $site_couchdb::couchdb_webapp_user: diff --git a/puppet/modules/site_couchdb/manifests/bigcouch.pp b/puppet/modules/site_couchdb/manifests/bigcouch.pp index 469a2783..2de3d4d0 100644 --- a/puppet/modules/site_couchdb/manifests/bigcouch.pp +++ b/puppet/modules/site_couchdb/manifests/bigcouch.pp @@ -44,4 +44,7 @@ class site_couchdb::bigcouch { require => Package['couchdb'], notify => Service['couchdb'] } + + include site_check_mk::agent::couchdb::bigcouch + } diff --git a/puppet/modules/site_couchdb/manifests/create_dbs.pp b/puppet/modules/site_couchdb/manifests/create_dbs.pp index eea4bbf5..a2d1c655 100644 --- a/puppet/modules/site_couchdb/manifests/create_dbs.pp +++ b/puppet/modules/site_couchdb/manifests/create_dbs.pp @@ -90,4 +90,13 @@ class site_couchdb::create_dbs { members => "{ \"names\": [\"${site_couchdb::couchdb_webapp_user}\"], \"roles\": [\"replication\"] }", require => Couchdb::Query::Setup['localhost'] } + + ## invite_codes db + ## store invite codes for new signups + ## r/w: webapp + couchdb::create_db { 'invite_codes': + members => "{ \"names\": [\"${site_couchdb::couchdb_webapp_user}\"], \"roles\": [\"replication\"] }", + require => Couchdb::Query::Setup['localhost'] + } + } diff --git a/puppet/modules/site_couchdb/manifests/designs.pp b/puppet/modules/site_couchdb/manifests/designs.pp index 1ab1c6a1..e5fd94c6 100644 --- a/puppet/modules/site_couchdb/manifests/designs.pp +++ b/puppet/modules/site_couchdb/manifests/designs.pp @@ -12,12 +12,13 @@ class site_couchdb::designs { } site_couchdb::upload_design { - 'customers': design => 'customers/Customer.json'; - 'identities': design => 'identities/Identity.json'; - 'tickets': design => 'tickets/Ticket.json'; - 'messages': design => 'messages/Message.json'; - 'users': design => 'users/User.json'; - 'tmp_users': design => 'users/User.json'; + 'customers': design => 'customers/Customer.json'; + 'identities': design => 'identities/Identity.json'; + 'tickets': design => 'tickets/Ticket.json'; + 'messages': design => 'messages/Message.json'; + 'users': design => 'users/User.json'; + 'tmp_users': design => 'users/User.json'; + 'invite_codes': design => 'invite_codes/InviteCode.json'; 'shared_docs': db => 'shared', design => 'shared/docs.json'; diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp index 6b6ddd3a..61aa887e 100644 --- a/puppet/modules/site_couchdb/manifests/init.pp +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -26,11 +26,6 @@ class site_couchdb { $couchdb_soledad_pw = $couchdb_soledad['password'] $couchdb_soledad_salt = $couchdb_soledad['salt'] - $couchdb_tapicero = $couchdb_users['tapicero'] - $couchdb_tapicero_user = $couchdb_tapicero['username'] - $couchdb_tapicero_pw = $couchdb_tapicero['password'] - $couchdb_tapicero_salt = $couchdb_tapicero['salt'] - $couchdb_webapp = $couchdb_users['webapp'] $couchdb_webapp_user = $couchdb_webapp['username'] $couchdb_webapp_pw = $couchdb_webapp['password'] @@ -66,6 +61,8 @@ class site_couchdb { if $couchdb_backup { include site_couchdb::backup } include site_check_mk::agent::couchdb - include site_check_mk::agent::tapicero + + # remove tapicero leftovers on couchdb nodes + include site_config::remove::tapicero } diff --git a/puppet/modules/site_couchdb/manifests/master.pp b/puppet/modules/site_couchdb/manifests/master.pp index c28eee7d..c50ed364 100644 --- a/puppet/modules/site_couchdb/manifests/master.pp +++ b/puppet/modules/site_couchdb/manifests/master.pp @@ -6,4 +6,11 @@ class site_couchdb::master { chttpd_bind_address => '127.0.0.1', pwhash_alg => $site_couchdb::couchdb_pwhash_alg } + + # couchdb is not available in jessie, and the + # leap deb repo only hosts a wheeyz version. + # we install it therefore from unstable + include site_apt::sid_repo + + include site_check_mk::agent::couchdb::master } diff --git a/puppet/modules/site_couchdb/manifests/setup.pp b/puppet/modules/site_couchdb/manifests/setup.pp index 69bd1c6a..fef48505 100644 --- a/puppet/modules/site_couchdb/manifests/setup.pp +++ b/puppet/modules/site_couchdb/manifests/setup.pp @@ -12,27 +12,40 @@ class site_couchdb::setup { $user = $site_couchdb::couchdb_admin_user - # /etc/couchdb/couchdb-admin.netrc is deployed by couchdb::query::setup - # we symlink to couchdb.netrc for puppet commands. - # we symlink this to /root/.netrc for couchdb_scripts (eg. backup) - # and makes life easier for the admin (i.e. using curl/wget without - # passing credentials) + # setup /etc/couchdb/couchdb-admin.netrc for couchdb admin access + couchdb::query::setup { 'localhost': + user => $user, + pw => $site_couchdb::couchdb_admin_pw + } + + # We symlink /etc/couchdb/couchdb-admin.netrc to /etc/couchdb/couchdb.netrc + # for puppet commands, and to to /root/.netrc for couchdb_scripts + # (eg. backup) and to makes life easier for the admin on the command line + # (i.e. using curl/wget without passing credentials) file { '/etc/couchdb/couchdb.netrc': ensure => link, target => "/etc/couchdb/couchdb-${user}.netrc"; - '/root/.netrc': ensure => link, target => '/etc/couchdb/couchdb.netrc'; + } - '/srv/leap/couchdb': - ensure => directory + # setup /etc/couchdb/couchdb-soledad-admin.netrc file for couchdb admin + # access, accessible only for the soledad-admin user to create soledad + # userdbs + file { '/etc/couchdb/couchdb-soledad-admin.netrc': + content => "machine localhost login ${user} password ${site_couchdb::couchdb_admin_pw}", + mode => '0400', + owner => 'soledad-admin', + group => 'root', + require => [ Package['couchdb'], User['soledad-admin'] ]; } - couchdb::query::setup { 'localhost': - user => $user, - pw => $site_couchdb::couchdb_admin_pw, + # Checkout couchdb_scripts repo + file { + '/srv/leap/couchdb': + ensure => directory } vcsrepo { '/srv/leap/couchdb/scripts': diff --git a/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg b/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg index 0d729b8c..981dc12a 100644 --- a/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg +++ b/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg @@ -70,7 +70,7 @@ precached_object_file=/var/lib/nagios3/objects.precache # defined as macros in this file and restrictive permissions (600) # can be placed on this file. -resource_file=/etc/nagios3/private/resource.cfg +resource_file=/etc/nagios3/resource.cfg diff --git a/puppet/modules/site_nagios/manifests/init.pp b/puppet/modules/site_nagios/manifests/init.pp index eb08cdcb..40ae4b86 100644 --- a/puppet/modules/site_nagios/manifests/init.pp +++ b/puppet/modules/site_nagios/manifests/init.pp @@ -1,6 +1,10 @@ +# setup nagios on monitoring node class site_nagios { tag 'leap_service' Class['site_config::default'] -> Class['site_nagios'] include site_nagios::server + + # remove leftovers on monitoring nodes + include site_config::remove::monitoring } diff --git a/puppet/modules/site_nagios/manifests/server.pp b/puppet/modules/site_nagios/manifests/server.pp index cb6c8d95..5c833508 100644 --- a/puppet/modules/site_nagios/manifests/server.pp +++ b/puppet/modules/site_nagios/manifests/server.pp @@ -32,7 +32,8 @@ class site_nagios::server inherits nagios::base { } include site_apache::common - include site_apache::module::headers + include site_webapp::common_vhost + include apache::module::headers File ['nagios_htpasswd'] { source => undef, diff --git a/puppet/modules/site_nagios/manifests/server/apache.pp b/puppet/modules/site_nagios/manifests/server/apache.pp index 8dbc7e9b..82962e89 100644 --- a/puppet/modules/site_nagios/manifests/server/apache.pp +++ b/puppet/modules/site_nagios/manifests/server/apache.pp @@ -1,7 +1,25 @@ +# set up apache for nagios class site_nagios::server::apache { + include x509::variables + include site_config::x509::commercial::cert include site_config::x509::commercial::key include site_config::x509::commercial::ca + include apache::module::authn_file + # "AuthUserFile" + include apache::module::authz_user + # "AuthType Basic" + include apache::module::auth_basic + # "DirectoryIndex" + include apache::module::dir + include apache::module::php5 + include apache::module::cgi + + # apache >= 2.4, debian jessie + if ( $::lsbdistcodename == 'jessie' ) { + include apache::module::authn_core + } + } diff --git a/puppet/modules/site_openvpn/manifests/init.pp b/puppet/modules/site_openvpn/manifests/init.pp index e2a3124e..ede35a9e 100644 --- a/puppet/modules/site_openvpn/manifests/init.pp +++ b/puppet/modules/site_openvpn/manifests/init.pp @@ -229,6 +229,13 @@ class site_openvpn { } leap::logfile { 'openvpn': } + + # Because we currently do not support ipv6 and instead block it (so no leaks + # happen), we get a large number of these messages, so we ignore them (#6540) + rsyslog::snippet { '01-ignore_icmpv6_send': + content => ':msg, contains, "icmpv6_send: no reply to icmp error" ~' + } + include site_check_mk::agent::openvpn } diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp index 49692d24..71d61621 100644 --- a/puppet/modules/site_postfix/manifests/mx.pp +++ b/puppet/modules/site_postfix/manifests/mx.pp @@ -7,7 +7,8 @@ class site_postfix::mx { $domain = $domain_hash['full_suffix'] $host_domain = $domain_hash['full'] $cert_name = hiera('name') - $mynetworks = join(hiera('mynetworks'), ' ') + $mynetworks = join(hiera('mynetworks', ''), ' ') + $rbls = suffix(prefix(hiera('rbls', []), 'reject_rbl_client '), ',') $root_mail_recipient = hiera('contacts') $postfix_smtp_listen = 'all' @@ -20,16 +21,20 @@ class site_postfix::mx { postfix::config { 'mynetworks': value => "127.0.0.0/8 [::1]/128 [fe80::]/64 ${mynetworks}"; + # Note: mydestination should not include @domain, because this is + # used in virtual alias maps. 'mydestination': - value => "\$myorigin, localhost, localhost.\$mydomain, ${domain}"; + value => "\$myorigin, localhost, localhost.\$mydomain"; 'myhostname': value => $host_domain; 'mailbox_size_limit': value => '0'; 'home_mailbox': value => 'Maildir/'; + # Note: virtual-aliases map will take precedence over leap_mx + # lookup (tcp:localhost) 'virtual_alias_maps': - value => 'tcp:localhost:4242'; + value => 'hash:/etc/postfix/virtual-aliases tcp:localhost:4242'; 'luser_relay': value => 'vmail'; 'smtpd_tls_received_header': @@ -44,13 +49,20 @@ class site_postfix::mx { # alias map 'local_recipient_maps': value => '$alias_maps'; + 'smtpd_milters': + value => 'unix:/run/clamav/milter.ctl,unix:/var/run/opendkim/opendkim.sock'; + 'milter_default_action': + value => 'accept'; } include site_postfix::mx::smtpd_checks include site_postfix::mx::checks include site_postfix::mx::smtp_tls include site_postfix::mx::smtpd_tls - include site_postfix::mx::reserved_aliases + include site_postfix::mx::static_aliases + include site_postfix::mx::rewrite_openpgp_header + include clamav + include postfwd # greater verbosity for debugging, take out for production #include site_postfix::debug @@ -72,7 +84,11 @@ class site_postfix::mx { -o smtpd_tls_wrappermode=yes -o smtpd_tls_security_level=encrypt -o smtpd_recipient_restrictions=\$smtps_recipient_restrictions - -o smtpd_helo_restrictions=\$smtps_helo_restrictions", + -o smtpd_helo_restrictions=\$smtps_helo_restrictions + -o smtpd_client_restrictions= + -o cleanup_service_name=clean_smtps +clean_smtps unix n - n - 0 cleanup + -o header_checks=pcre:/etc/postfix/checks/rewrite_openpgp_headers", require => [ Class['Site_config::X509::Key'], Class['Site_config::X509::Cert'], diff --git a/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp b/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp deleted file mode 100644 index 83e27376..00000000 --- a/puppet/modules/site_postfix/manifests/mx/reserved_aliases.pp +++ /dev/null @@ -1,15 +0,0 @@ -# Defines which mail addresses shouldn't be available and where they should fwd -class site_postfix::mx::reserved_aliases { - - postfix::mailalias { - [ 'abuse', 'admin', 'arin-admin', 'administrator', 'bin', 'cron', - 'certmaster', 'domainadmin', 'games', 'ftp', 'hostmaster', 'lp', - 'maildrop', 'mysql', 'news', 'nobody', 'noc', 'postmaster', 'postgresql', - 'security', 'ssladmin', 'sys', 'usenet', 'uucp', 'webmaster', 'www', - 'www-data', - ]: - ensure => present, - recipient => 'root' - } - -} diff --git a/puppet/modules/site_postfix/manifests/mx/rewrite_openpgp_header.pp b/puppet/modules/site_postfix/manifests/mx/rewrite_openpgp_header.pp new file mode 100644 index 00000000..71f945b8 --- /dev/null +++ b/puppet/modules/site_postfix/manifests/mx/rewrite_openpgp_header.pp @@ -0,0 +1,11 @@ +class site_postfix::mx::rewrite_openpgp_header { + $mx = hiera('mx') + $correct_domain = $mx['key_lookup_domain'] + + file { '/etc/postfix/checks/rewrite_openpgp_headers': + content => template('site_postfix/checks/rewrite_openpgp_headers.erb'), + mode => '0644', + owner => root, + group => root; + } +} diff --git a/puppet/modules/site_postfix/manifests/mx/smtpd_checks.pp b/puppet/modules/site_postfix/manifests/mx/smtpd_checks.pp index 0ec40277..1c3e5c92 100644 --- a/puppet/modules/site_postfix/manifests/mx/smtpd_checks.pp +++ b/puppet/modules/site_postfix/manifests/mx/smtpd_checks.pp @@ -6,7 +6,7 @@ class site_postfix::mx::smtpd_checks { 'checks_dir': value => '$config_directory/checks'; 'smtpd_client_restrictions': - value => 'permit_mynetworks,permit'; + value => "${site_postfix::mx::rbls}permit_mynetworks,permit"; 'smtpd_data_restrictions': value => 'permit_mynetworks, reject_unauth_pipelining, permit'; 'smtpd_delay_reject': diff --git a/puppet/modules/site_postfix/manifests/mx/static_aliases.pp b/puppet/modules/site_postfix/manifests/mx/static_aliases.pp new file mode 100644 index 00000000..71c0555a --- /dev/null +++ b/puppet/modules/site_postfix/manifests/mx/static_aliases.pp @@ -0,0 +1,88 @@ +# +# Defines static, hard coded aliases that are not in the database. +# These aliases take precedence over the database aliases. +# +# There are three classes of reserved names: +# +# (1) forbidden_usernames: +# Some usernames are forbidden and cannot be registered. +# this is defined in node property webapp.forbidden_usernames +# This is enforced by the webapp. +# +# (2) public aliases: +# Some aliases for root, and are publicly exposed so that anyone +# can deliver mail to them. For example, postmaster. +# These are implemented in the virtual alias map, which takes +# precedence over the local alias map. +# +# (3) local aliases: +# Some aliases are only available locally: mail can be delivered +# to the alias if the mail originates from the local host, or is +# hostname qualified, but otherwise it will be rejected. +# These are implemented in the local alias map. +# +# The alias for local 'root' is defined elsewhere. In this file, we +# define the virtual 'root@domain' (which can be overwritten by +# defining an entry for root in node property mx.aliases). +# + +class site_postfix::mx::static_aliases { + + $mx = hiera('mx') + $root_recipients = hiera('contacts') + + # + # LOCAL ALIASES + # + + # NOTE: if you remove one of these, they will still appear in the + # /etc/aliases file + $local_aliases = [ + 'admin', 'administrator', 'bin', 'cron', 'games', 'ftp', 'lp', 'maildrop', + 'mysql', 'news', 'nobody', 'noc', 'postgresql', 'ssladmin', 'sys', + 'usenet', 'uucp', 'www', 'www-data' + ] + + postfix::mailalias { + $local_aliases: + ensure => present, + recipient => 'root' + } + + # + # PUBLIC ALIASES + # + + $public_aliases = $mx['aliases'] + + $default_public_aliases = { + 'root' => $root_recipients, + 'abuse' => 'postmaster', + 'arin-admin' => 'root', + 'certmaster' => 'hostmaster', + 'domainadmin' => 'hostmaster', + 'hostmaster' => 'root', + 'mailer-daemon' => 'postmaster', + 'postmaster' => 'root', + 'security' => 'root', + 'webmaster' => 'hostmaster', + } + + $aliases = merge($default_public_aliases, $public_aliases) + + exec { 'postmap_virtual_aliases': + command => '/usr/sbin/postmap /etc/postfix/virtual-aliases', + refreshonly => true, + user => root, + group => root, + require => Package['postfix'], + subscribe => File['/etc/postfix/virtual-aliases'] + } + file { '/etc/postfix/virtual-aliases': + content => template('site_postfix/virtual-aliases.erb'), + owner => root, + group => root, + mode => '0600', + require => Package['postfix'] + } +} diff --git a/puppet/modules/site_postfix/templates/checks/helo_access.erb b/puppet/modules/site_postfix/templates/checks/helo_access.erb index bef3c11d..bac2c45a 100644 --- a/puppet/modules/site_postfix/templates/checks/helo_access.erb +++ b/puppet/modules/site_postfix/templates/checks/helo_access.erb @@ -18,4 +18,4 @@ # Reject anybody that HELO's as being in our own domain(s) # anyone who identifies themselves as us is a virus/spammer -<%= domain %> 554 You are not in domain <%= domain %> +<%= @domain %> 554 You are not in domain <%= @domain %> diff --git a/puppet/modules/site_postfix/templates/checks/rewrite_openpgp_headers.erb b/puppet/modules/site_postfix/templates/checks/rewrite_openpgp_headers.erb new file mode 100644 index 00000000..7af14f7d --- /dev/null +++ b/puppet/modules/site_postfix/templates/checks/rewrite_openpgp_headers.erb @@ -0,0 +1,13 @@ +# THIS FILE IS MANAGED BY PUPPET +# +# This will replace the OpenPGP header that the client adds, because it is +# sometimes incorrect (due to the client not always knowing what the proper URL +# is for the webapp). +# e.g. This will rewrite this header: +# OpenPGP: id=4C0E01CD50E2F653; url="https://leap.se/key/elijah"; preference="signencrypt +# with this replacement: +# OpenPGP: id=4C0E01CD50E2F653; url="https://user.leap.se/key/elijah"; preference="signencrypt +# +# Note: whitespace in the pattern is represented by [[:space:]] to avoid these warnings from postmap: +# "record is in "key: value" format; is this an alias file?" and "duplicate entry" +/^(OpenPGP:[[:space:]]id=[[:alnum:]]+;[[:space:]]url="https:\/\/)<%= @domain %>(\/key\/[[:alpha:]]+";.*)/i REPLACE ${1}<%= @correct_domain %>${2} diff --git a/puppet/modules/site_postfix/templates/virtual-aliases.erb b/puppet/modules/site_postfix/templates/virtual-aliases.erb new file mode 100644 index 00000000..8373de97 --- /dev/null +++ b/puppet/modules/site_postfix/templates/virtual-aliases.erb @@ -0,0 +1,21 @@ +# +# This file is managed by puppet. +# +# These virtual aliases take precedence over all other aliases. +# + +# +# enable these virtual domains: +# +<%= @domain %> enabled +<%- @aliases.keys.map {|addr| addr.split('@')[1] }.compact.sort.uniq.each do |virt_domain| -%> +<%= virt_domain %> enabled +<%- end %> + +# +# virtual aliases: +# +<%- @aliases.keys.sort.each do |from| -%> +<%- full_address = from =~ /@/ ? from : from + "@" + @domain -%> +<%= full_address %> <%= [@aliases[from]].flatten.map{|a| a =~ /@/ ? a : a + "@" + @domain}.join(', ') %> +<%- end -%> diff --git a/puppet/modules/site_sshd/manifests/init.pp b/puppet/modules/site_sshd/manifests/init.pp index 1da2f1d5..5efd459f 100644 --- a/puppet/modules/site_sshd/manifests/init.pp +++ b/puppet/modules/site_sshd/manifests/init.pp @@ -1,6 +1,8 @@ +# configures sshd, mosh, authorized keys and known hosts class site_sshd { - $ssh = hiera_hash('ssh') - $hosts = hiera('hosts', '') + $ssh = hiera_hash('ssh') + $ssh_config = $ssh['config'] + $hosts = hiera('hosts', '') ## ## SETUP AUTHORIZED KEYS @@ -48,15 +50,32 @@ class site_sshd { } } + # we cannot use the 'hardened' parameter because leap_cli uses an + # old net-ssh gem that is incompatible with the included + # "KexAlgorithms curve25519-sha256@libssh.org", + # see https://leap.se/code/issues/7591 + # therefore we don't use it here, but include all other options + # that would be applied by the 'hardened' parameter + # not all options are available on wheezy + if ( $::lsbdistcodename == 'wheezy' ) { + $tail_additional_options = 'Ciphers aes256-ctr +MACs hmac-sha2-512,hmac-sha2-256,hmac-ripemd160' + } else { + $tail_additional_options = 'Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes256-ctr +MACs hmac-sha2-512,hmac-sha2-256,hmac-ripemd160' + } + ## ## SSHD SERVER CONFIGURATION ## class { '::sshd': - manage_nagios => false, - ports => [ $ssh['port'] ], - use_pam => 'yes', - hardened_ssl => 'yes', - print_motd => 'no', - manage_client => false + manage_nagios => false, + ports => [ $ssh['port'] ], + use_pam => 'yes', + print_motd => 'no', + tcp_forwarding => $ssh_config['AllowTcpForwarding'], + manage_client => false, + use_storedconfigs => true, + tail_additional_options => $tail_additional_options } } diff --git a/puppet/modules/site_static/manifests/init.pp b/puppet/modules/site_static/manifests/init.pp index 1efc510b..8df53075 100644 --- a/puppet/modules/site_static/manifests/init.pp +++ b/puppet/modules/site_static/manifests/init.pp @@ -9,6 +9,7 @@ class site_static { $domains = $static['domains'] $formats = $static['formats'] $bootstrap = $static['bootstrap_files'] + $tor = hiera('tor', false) if $bootstrap['enabled'] { $bootstrap_domain = $bootstrap['domain'] @@ -27,14 +28,11 @@ class site_static { } } - class { '::apache': no_default_site => true, ssl => true } include site_apache::module::headers include site_apache::module::alias include site_apache::module::expires include site_apache::module::removeip - include site_apache::module::rewrite - apache::config::include{ 'ssl_common.inc': } - + include site_apache::common include site_config::ruby::dev if (member($formats, 'rack')) { @@ -46,7 +44,7 @@ class site_static { } if (member($formats, 'amber')) { - rubygems::gem{'amber-0.3.7': + rubygems::gem{'amber-0.3.8': require => Package['zlib1g-dev'] } @@ -57,6 +55,13 @@ class site_static { create_resources(site_static::domain, $domains) + if $tor { + $hidden_service = $tor['hidden_service'] + if $hidden_service['active'] { + include site_webapp::hidden_service + } + } + include site_shorewall::defaults include site_shorewall::service::http include site_shorewall::service::https diff --git a/puppet/modules/site_static/templates/apache.conf.erb b/puppet/modules/site_static/templates/apache.conf.erb index 4d61cc08..2853c5c7 100644 --- a/puppet/modules/site_static/templates/apache.conf.erb +++ b/puppet/modules/site_static/templates/apache.conf.erb @@ -48,7 +48,7 @@ Include include.d/ssl_common.inc <%- if @tls_only -%> - Header add Strict-Transport-Security: "max-age=15768000;includeSubdomains" + Header always set Strict-Transport-Security: "max-age=15768000;includeSubdomains" <%- end -%> Header set X-Frame-Options "deny" Header always unset X-Powered-By diff --git a/puppet/modules/site_webapp/files/server-status.conf b/puppet/modules/site_webapp/files/server-status.conf new file mode 100644 index 00000000..84cb9ae0 --- /dev/null +++ b/puppet/modules/site_webapp/files/server-status.conf @@ -0,0 +1,28 @@ +# Keep track of extended status information for each request +ExtendedStatus On + +# Determine if mod_status displays the first 63 characters of a request or +# the last 63, assuming the request itself is greater than 63 chars. +# Default: Off +#SeeRequestTail On + +Listen 127.0.0.1:8162 +NameVirtualHost 127.0.0.1:8162 + +<VirtualHost 127.0.0.1:8162> + +<Location /server-status> + SetHandler server-status + Order deny,allow + Deny from all + Allow from 127.0.0.1 +</Location> + +</VirtualHost> + + +<IfModule mod_proxy.c> + # Show Proxy LoadBalancer status in mod_status + ProxyStatus On +</IfModule> + diff --git a/puppet/modules/site_webapp/manifests/apache.pp b/puppet/modules/site_webapp/manifests/apache.pp index 93e172a0..80c7b29b 100644 --- a/puppet/modules/site_webapp/manifests/apache.pp +++ b/puppet/modules/site_webapp/manifests/apache.pp @@ -1,3 +1,4 @@ +# configure apache and passenger to serve the webapp class site_webapp::apache { $web_api = hiera('api') @@ -11,16 +12,17 @@ class site_webapp::apache { $webapp_domain = $webapp['domain'] include site_apache::common - include site_apache::module::headers - include site_apache::module::alias - include site_apache::module::expires - include site_apache::module::removeip + include apache::module::headers + include apache::module::alias + include apache::module::expires + include apache::module::removeip + include site_webapp::common_vhost class { 'passenger': use_munin => false } apache::vhost::file { 'api': - content => template('site_apache/vhosts.d/api.conf.erb') + content => template('site_apache/vhosts.d/api.conf.erb'); } } diff --git a/puppet/modules/site_webapp/manifests/common_vhost.pp b/puppet/modules/site_webapp/manifests/common_vhost.pp new file mode 100644 index 00000000..c57aad57 --- /dev/null +++ b/puppet/modules/site_webapp/manifests/common_vhost.pp @@ -0,0 +1,18 @@ +class site_webapp::common_vhost { + # installs x509 cert + key and common config + # that both nagios + leap webapp use + + include x509::variables + include site_config::x509::commercial::cert + include site_config::x509::commercial::key + include site_config::x509::commercial::ca + + Class['Site_config::X509::Commercial::Key'] ~> Service[apache] + Class['Site_config::X509::Commercial::Cert'] ~> Service[apache] + Class['Site_config::X509::Commercial::Ca'] ~> Service[apache] + + apache::vhost::file { + 'common': + content => template('site_apache/vhosts.d/common.conf.erb') + } +} diff --git a/puppet/modules/site_webapp/manifests/couchdb.pp b/puppet/modules/site_webapp/manifests/couchdb.pp index 1dbc745d..5cf7f953 100644 --- a/puppet/modules/site_webapp/manifests/couchdb.pp +++ b/puppet/modules/site_webapp/manifests/couchdb.pp @@ -14,29 +14,29 @@ class site_webapp::couchdb { file { '/srv/leap/webapp/config/couchdb.yml': content => template('site_webapp/couchdb.yml.erb'), - owner => leap-webapp, - group => leap-webapp, + owner => 'leap-webapp', + group => 'leap-webapp', mode => '0600', require => Vcsrepo['/srv/leap/webapp']; '/srv/leap/webapp/config/couchdb.admin.yml': content => template('site_webapp/couchdb.admin.yml.erb'), - owner => leap-webapp, - group => leap-webapp, + owner => 'root', + group => 'root', mode => '0600', require => Vcsrepo['/srv/leap/webapp']; '/srv/leap/webapp/log': ensure => directory, - owner => leap-webapp, - group => leap-webapp, + owner => 'leap-webapp', + group => 'leap-webapp', mode => '0755', require => Vcsrepo['/srv/leap/webapp']; '/srv/leap/webapp/log/production.log': ensure => present, - owner => leap-webapp, - group => leap-webapp, + owner => 'leap-webapp', + group => 'leap-webapp', mode => '0666', require => Vcsrepo['/srv/leap/webapp']; } diff --git a/puppet/modules/site_webapp/manifests/cron.pp b/puppet/modules/site_webapp/manifests/cron.pp index d26ee312..7147a0d2 100644 --- a/puppet/modules/site_webapp/manifests/cron.pp +++ b/puppet/modules/site_webapp/manifests/cron.pp @@ -5,12 +5,14 @@ class site_webapp::cron { 'rotate_databases': command => 'cd /srv/leap/webapp && bundle exec rake db:rotate', environment => 'RAILS_ENV=production', + user => 'root', hour => [0,6,12,18], minute => 0; 'delete_tmp_databases': command => 'cd /srv/leap/webapp && bundle exec rake db:deletetmp', environment => 'RAILS_ENV=production', + user => 'root', hour => 1, minute => 1; @@ -19,6 +21,7 @@ class site_webapp::cron { 'remove_expired_sessions': command => 'cd /srv/leap/webapp && bundle exec rake cleanup:sessions', environment => 'RAILS_ENV=production', + user => 'leap-webapp', hour => 2, minute => 30, ensure => absent; @@ -26,6 +29,7 @@ class site_webapp::cron { 'remove_expired_tokens': command => 'cd /srv/leap/webapp && bundle exec rake cleanup:tokens', environment => 'RAILS_ENV=production', + user => 'leap-webapp', hour => 3, minute => 0; } diff --git a/puppet/modules/site_webapp/manifests/hidden_service.pp b/puppet/modules/site_webapp/manifests/hidden_service.pp index 16b6e2e7..4cf7a8ca 100644 --- a/puppet/modules/site_webapp/manifests/hidden_service.pp +++ b/puppet/modules/site_webapp/manifests/hidden_service.pp @@ -4,10 +4,10 @@ class site_webapp::hidden_service { $tor_domain = "${hidden_service['address']}.onion" include site_apache::common - include site_apache::module::headers - include site_apache::module::alias - include site_apache::module::expires - include site_apache::module::removeip + include apache::module::headers + include apache::module::alias + include apache::module::expires + include apache::module::removeip include tor::daemon tor::daemon::hidden_service { 'webapp': ports => '80 127.0.0.1:80' } @@ -32,12 +32,18 @@ class site_webapp::hidden_service { owner => 'debian-tor', group => 'debian-tor', mode => '0600'; + + '/etc/apache2/mods-enabled/status.conf': + ensure => absent, + notify => Service['apache']; } apache::vhost::file { 'hidden_service': - content => template('site_apache/vhosts.d/hidden_service.conf.erb') + content => template('site_apache/vhosts.d/hidden_service.conf.erb'); + 'server_status': + vhost_source => 'modules/site_webapp/server-status.conf'; } include site_shorewall::tor -}
\ No newline at end of file +} diff --git a/puppet/modules/site_webapp/manifests/init.pp b/puppet/modules/site_webapp/manifests/init.pp index ec94c090..837950a8 100644 --- a/puppet/modules/site_webapp/manifests/init.pp +++ b/puppet/modules/site_webapp/manifests/init.pp @@ -1,3 +1,4 @@ +# configure webapp service class site_webapp { tag 'leap_service' $definition_files = hiera('definition_files') @@ -26,6 +27,9 @@ class site_webapp { include site_config::x509::client_ca::ca include site_config::x509::client_ca::key + # remove leftovers from previous installations on webapp nodes + include site_config::remove::webapp + group { 'leap-webapp': ensure => present, allowdupe => false; @@ -163,10 +167,8 @@ class site_webapp { # needed for the soledad-sync check which is run on the - # webapp node (#6520) - package { 'python-u1db': - ensure => latest, - } + # webapp node + include soledad::client leap::logfile { 'webapp': } diff --git a/puppet/modules/site_webapp/templates/config.yml.erb b/puppet/modules/site_webapp/templates/config.yml.erb index ccde2d2e..c2e9f3df 100644 --- a/puppet/modules/site_webapp/templates/config.yml.erb +++ b/puppet/modules/site_webapp/templates/config.yml.erb @@ -1,28 +1,35 @@ -<%- require 'json' -%> -<%- cert_options = @webapp['client_certificates'] -%> -production: - admins: <%= @webapp['admins'].inspect %> - domain: <%= @provider_domain %> - force_ssl: <%= @webapp['secure'] %> - client_ca_key: <%= scope.lookupvar('x509::variables::keys') %>/<%= scope.lookupvar('site_config::params::client_ca_name') %>.key - client_ca_cert: <%= scope.lookupvar('x509::variables::local_CAs') %>/<%= scope.lookupvar('site_config::params::client_ca_name') %>.crt - secret_token: "<%= @secret_token %>" - client_cert_lifespan: <%= cert_options['life_span'] %> - client_cert_bit_size: <%= cert_options['bit_size'].to_i %> - client_cert_hash: <%= cert_options['digest'] %> - allow_limited_certs: <%= @webapp['allow_limited_certs'].inspect %> - allow_unlimited_certs: <%= @webapp['allow_unlimited_certs'].inspect %> - allow_anonymous_certs: <%= @webapp['allow_anonymous_certs'].inspect %> - limited_cert_prefix: "<%= cert_options['limited_prefix'] %>" - unlimited_cert_prefix: "<%= cert_options['unlimited_prefix'] %>" - minimum_client_version: "<%= @webapp['client_version']['min'] %>" - default_service_level: "<%= @webapp['default_service_level'] %>" - service_levels: <%= scope.function_sorted_json([@webapp['service_levels']]) %> - allow_registration: <%= @webapp['allow_registration'].inspect %> - handle_blacklist: <%= @webapp['forbidden_usernames'].inspect %> -<%- if @webapp['engines'] && @webapp['engines'].any? -%> - engines: -<%- @webapp['engines'].each do |engine| -%> - - <%= engine %> -<%- end -%> -<%- end -%> +<% +cert_options = @webapp['client_certificates'] +production = { + "admins" => @webapp['admins'], + "default_locale" => @webapp['default_locale'], + "available_locales" => @webapp['locales'], + "domain" => @provider_domain, + "force_ssl" => @webapp['secure'], + "client_ca_key" => "%s/%s.key" % [scope.lookupvar('x509::variables::keys'), scope.lookupvar('site_config::params::client_ca_name')], + "client_ca_cert" => "%s/%s.crt" % [scope.lookupvar('x509::variables::local_CAs'), scope.lookupvar('site_config::params::client_ca_name')], + "secret_token" => @secret_token, + "client_cert_lifespan" => cert_options['life_span'], + "client_cert_bit_size" => cert_options['bit_size'].to_i, + "client_cert_hash" => cert_options['digest'], + "allow_limited_certs" => @webapp['allow_limited_certs'], + "allow_unlimited_certs" => @webapp['allow_unlimited_certs'], + "allow_anonymous_certs" => @webapp['allow_anonymous_certs'], + "limited_cert_prefix" => cert_options['limited_prefix'], + "unlimited_cert_prefix" => cert_options['unlimited_prefix'], + "minimum_client_version" => @webapp['client_version']['min'], + "default_service_level" => @webapp['default_service_level'], + "service_levels" => @webapp['service_levels'], + "allow_registration" => @webapp['allow_registration'], + "handle_blacklist" => @webapp['forbidden_usernames'], + "invite_required" => @webapp['invite_required'] +} + +if @webapp['engines'] && @webapp['engines'].any? + production["engines"] = @webapp['engines'] +end +-%> +# +# This file is generated by puppet. This file inherits from defaults.yml. +# +<%= scope.function_sorted_yaml([{"production" => production}]) %> diff --git a/puppet/modules/soledad/manifests/client.pp b/puppet/modules/soledad/manifests/client.pp new file mode 100644 index 00000000..5700cb09 --- /dev/null +++ b/puppet/modules/soledad/manifests/client.pp @@ -0,0 +1,18 @@ +# setup soledad-client +# currently needed on webapp node to run the soledad-sync test +class soledad::client { + + tag 'leap_service' + include soledad::common + + package { + 'soledad-client': + ensure => latest, + require => [ + Class['site_apt::preferences::twisted'], + Class['site_apt::leap_repo'] ]; + 'python-u1db': + ensure => latest; + } + +} diff --git a/puppet/modules/soledad/manifests/common.pp b/puppet/modules/soledad/manifests/common.pp index 8a1d664a..d66e943c 100644 --- a/puppet/modules/soledad/manifests/common.pp +++ b/puppet/modules/soledad/manifests/common.pp @@ -1,10 +1,10 @@ +# install soledad-common, both needed both soledad-client and soledad-server class soledad::common { - include soledad + include site_apt::preferences::twisted package { 'soledad-common': - ensure => latest, - require => User['soledad'] + ensure => latest; } } diff --git a/puppet/modules/soledad/manifests/init.pp b/puppet/modules/soledad/manifests/init.pp deleted file mode 100644 index 7cf0b729..00000000 --- a/puppet/modules/soledad/manifests/init.pp +++ /dev/null @@ -1,29 +0,0 @@ -class soledad { - - group { 'soledad': - ensure => present, - allowdupe => false; - } - - user { 'soledad': - ensure => present, - allowdupe => false, - gid => 'soledad', - home => '/srv/leap/soledad', - require => Group['soledad']; - } - - file { - '/srv/leap/soledad': - ensure => directory, - owner => 'soledad', - group => 'soledad', - require => User['soledad']; - - '/var/lib/soledad': - ensure => directory, - owner => 'soledad', - group => 'soledad', - require => User['soledad']; - } -} diff --git a/puppet/modules/soledad/manifests/server.pp b/puppet/modules/soledad/manifests/server.pp index b71fab69..5c5a1bb7 100644 --- a/puppet/modules/soledad/manifests/server.pp +++ b/puppet/modules/soledad/manifests/server.pp @@ -1,11 +1,12 @@ +# setup soledad-server class soledad::server { tag 'leap_service' - include soledad - include site_apt::preferences::twisted + include soledad::common - $soledad = hiera('soledad') - $couchdb_user = $soledad['couchdb_soledad_user']['username'] - $couchdb_password = $soledad['couchdb_soledad_user']['password'] + $soledad = hiera('soledad') + $couchdb_user = $soledad['couchdb_soledad_user']['username'] + $couchdb_password = $soledad['couchdb_soledad_user']['password'] + $couchdb_leap_mx_user = $soledad['couchdb_leap_mx_user']['username'] $couchdb_host = 'localhost' $couchdb_port = '5984' @@ -22,13 +23,29 @@ class soledad::server { # SOLEDAD CONFIG # - file { '/etc/leap/soledad-server.conf': - content => template('soledad/soledad-server.conf.erb'), - owner => 'soledad', - group => 'soledad', - mode => '0600', - notify => Service['soledad-server'], - require => Class['soledad']; + file { + '/etc/soledad': + ensure => directory, + owner => 'root', + group => 'root', + mode => '0755'; + '/etc/soledad/soledad-server.conf': + content => template('soledad/soledad-server.conf.erb'), + owner => 'soledad', + group => 'soledad', + mode => '0640', + notify => Service['soledad-server'], + require => [ User['soledad'], Group['soledad'] ]; + '/srv/leap/soledad': + ensure => directory, + owner => 'soledad', + group => 'soledad', + require => [ User['soledad'], Group['soledad'] ]; + '/var/lib/soledad': + ensure => directory, + owner => 'soledad', + group => 'soledad', + require => [ User['soledad'], Group['soledad'] ]; } package { $sources['soledad']['package']: @@ -44,7 +61,7 @@ class soledad::server { group => 'soledad', mode => '0600', notify => Service['soledad-server'], - require => Class['soledad']; + require => [ User['soledad'], Group['soledad'] ]; } service { 'soledad-server': @@ -52,7 +69,7 @@ class soledad::server { enable => true, hasstatus => true, hasrestart => true, - require => Class['soledad'], + require => [ User['soledad'], Group['soledad'] ], subscribe => [ Package['soledad-server'], Class['Site_config::X509::Key'], @@ -62,4 +79,26 @@ class soledad::server { include site_shorewall::soledad include site_check_mk::agent::soledad + + # set up users, group and directories for soledad-server + # although the soledad users are already created by the + # soledad-server package + group { 'soledad': + ensure => present, + system => true, + } + user { + 'soledad': + ensure => present, + system => true, + gid => 'soledad', + home => '/srv/leap/soledad', + require => Group['soledad']; + 'soledad-admin': + ensure => present, + system => true, + gid => 'soledad', + home => '/srv/leap/soledad', + require => Group['soledad']; + } } diff --git a/puppet/modules/soledad/templates/soledad-server.conf.erb b/puppet/modules/soledad/templates/soledad-server.conf.erb index 47d1f6e4..1c6a0d19 100644 --- a/puppet/modules/soledad/templates/soledad-server.conf.erb +++ b/puppet/modules/soledad/templates/soledad-server.conf.erb @@ -1,3 +1,12 @@ [soledad-server] -couch_url = http://<%= @couchdb_user %>:<%= @couchdb_password %>@<%= @couchdb_host %>:<%= @couchdb_port %> +couch_url = http://<%= @couchdb_user %>:<%= @couchdb_password %>@<%= @couchdb_host %>:<%= @couchdb_port %> +create_cmd = sudo -u soledad-admin /usr/bin/create-user-db +admin_netrc = /etc/couchdb/couchdb-soledad-admin.netrc + +[database-security] +members = <%= @couchdb_user %>, <%= @couchdb_leap_mx_user %> +# not needed, but for documentation: +# members_roles = replication +# admins = admin +# admins_roles = replication diff --git a/puppet/modules/sshd b/puppet/modules/sshd -Subproject 750a497758d94c2f5a6cad23cecc3dbde2d2f92 +Subproject 76f4f872f81209a52df2205fd88b5619df58f00 diff --git a/puppet/modules/tapicero/files/tapicero.init b/puppet/modules/tapicero/files/tapicero.init deleted file mode 100755 index 7a9af45f..00000000 --- a/puppet/modules/tapicero/files/tapicero.init +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh - -### BEGIN INIT INFO -# Provides: tapicero -# Required-Start: $remote_fs $syslog -# Required-Stop: $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: tapicero initscript -# Description: Controls tapicero daemon -### END INIT INFO - -PATH=/sbin:/usr/sbin:/bin:/usr/bin -BUNDLER=/usr/bin/bundle -NAME=tapicero -HOME="/srv/leap" -DAEMON="${HOME}/${NAME}/bin/${NAME}" -BUNDLE_GEMFILE="${HOME}/${NAME}/Gemfile" - -export BUNDLE_GEMFILE - -# exit if the daemon doesn't exist -[ -x "$DAEMON" ] || exit 0 - -. /lib/init/vars.sh -. /lib/lsb/init-functions - -if [ "$VERBOSE" != no ]; then - OPTIONS="--verbose" -else - OPTIONS="" -fi - -case "$1" in - start) - $BUNDLER exec $DAEMON start $OPTIONS - exit $? - ;; - stop) - $BUNDLER exec $DAEMON stop $OPTIONS - exit $? - ;; - restart) - $BUNDLER exec $DAEMON restart $OPTIONS - exit $? - ;; - reload) - $BUNDLER exec $DAEMON reload $OPTIONS - exit $? - ;; - status) - $BUNDLER exec $DAEMON status $OPTIONS - exit $? - ;; - *) - echo "Usage: /etc/init.d/$NAME {start|stop|reload|restart|status}" - exit 1 -esac - -exit 0 diff --git a/puppet/modules/tapicero/manifests/init.pp b/puppet/modules/tapicero/manifests/init.pp deleted file mode 100644 index ca8488c8..00000000 --- a/puppet/modules/tapicero/manifests/init.pp +++ /dev/null @@ -1,137 +0,0 @@ -class tapicero { - tag 'leap_service' - - $couchdb = hiera('couch') - $couchdb_port = $couchdb['port'] - - $couchdb_users = $couchdb['users'] - - $couchdb_admin_user = $couchdb_users['admin']['username'] - $couchdb_admin_password = $couchdb_users['admin']['password'] - - $couchdb_soledad_user = $couchdb_users['soledad']['username'] - $couchdb_leap_mx_user = $couchdb_users['leap_mx']['username'] - - $couchdb_mode = $couchdb['mode'] - $couchdb_replication = $couchdb['replication'] - - $sources = hiera('sources') - - Class['site_config::default'] -> Class['tapicero'] - - include site_config::ruby::dev - - # - # USER AND GROUP - # - - group { 'tapicero': - ensure => present, - allowdupe => false; - } - - user { 'tapicero': - ensure => present, - allowdupe => false, - gid => 'tapicero', - home => '/srv/leap/tapicero', - require => Group['tapicero']; - } - - # - # TAPICERO FILES - # - - file { - - # - # TAPICERO DIRECTORIES - # - - '/srv/leap/tapicero': - ensure => directory, - owner => 'tapicero', - group => 'tapicero', - require => User['tapicero']; - - '/var/lib/leap/tapicero': - ensure => directory, - owner => 'tapicero', - group => 'tapicero', - require => User['tapicero']; - - # for pid file - '/var/run/tapicero': - ensure => directory, - owner => 'tapicero', - group => 'tapicero', - require => User['tapicero']; - - # - # TAPICERO CONFIG - # - - '/etc/leap/tapicero.yaml': - content => template('tapicero/tapicero.yaml.erb'), - owner => 'tapicero', - group => 'tapicero', - mode => '0600', - notify => Service['tapicero']; - - # - # TAPICERO INIT - # - - '/etc/init.d/tapicero': - source => 'puppet:///modules/tapicero/tapicero.init', - owner => root, - group => 0, - mode => '0755', - require => Vcsrepo['/srv/leap/tapicero']; - } - - # - # TAPICERO CODE - # - - vcsrepo { '/srv/leap/tapicero': - ensure => present, - force => true, - revision => $sources['tapicero']['revision'], - provider => $sources['tapicero']['type'], - source => $sources['tapicero']['source'], - owner => 'tapicero', - group => 'tapicero', - require => [ User['tapicero'], Group['tapicero'] ], - notify => Exec['tapicero_bundler_update'] - } - - exec { 'tapicero_bundler_update': - cwd => '/srv/leap/tapicero', - command => '/bin/bash -c "/usr/bin/bundle check || /usr/bin/bundle install --path vendor/bundle --without test development"', - unless => '/usr/bin/bundle check', - user => 'tapicero', - timeout => 600, - require => [ - Class['bundler::install'], - Vcsrepo['/srv/leap/tapicero'], - Class['site_config::ruby::dev'] ], - notify => Service['tapicero']; - } - - # - # TAPICERO DAEMON - # - - service { 'tapicero': - ensure => running, - enable => true, - hasstatus => false, - hasrestart => true, - require => [ File['/etc/init.d/tapicero'], - File['/var/run/tapicero'], - Couchdb::Add_user[$::site_couchdb::couchdb_tapicero_user] ]; - } - - leap::logfile { 'tapicero': } -} diff --git a/puppet/modules/tapicero/templates/tapicero.yaml.erb b/puppet/modules/tapicero/templates/tapicero.yaml.erb deleted file mode 100644 index 8b08b49c..00000000 --- a/puppet/modules/tapicero/templates/tapicero.yaml.erb +++ /dev/null @@ -1,52 +0,0 @@ -<%- require 'json' -%> - -# -# Default configuration options for Tapicero -# - -# couch connection configuration -connection: - protocol: "http" - host: "localhost" - port: <%= @couchdb_port %> - username: <%= @couchdb_admin_user %> - password: <%= @couchdb_admin_password %> - prefix : "" - suffix : "" - netrc: "/etc/couchdb/couchdb.netrc" - -# file to store the last processed user record in so we can resume after -# a restart: -seq_dir: "/var/lib/leap/tapicero/" - -# Configure log_file like this if you want to log to a file instead of syslog: -#log_file: "/var/log/leap/tapicero.log" -#log_level: debug -log_level: info - -# tapicero specific options -options: - # prefix for per user databases: - db_prefix: "user-" - mode: <%= @couchdb_mode %> -<%- if @couchdb_replication %> - replication: <%= @couchdb_replication.to_json %> -<%- end -%> - - # security settings to be used for the per user databases - security: - admins: - names: - # We explicitly allow the admin user to access per user databases, even - # though admin access ignores per database security we just do this to be - # explicit about this - - <%= @couchdb_admin_user %> - roles: [] - members: - names: - - <%= @couchdb_soledad_user %> - - <%= @couchdb_leap_mx_user %> - roles: - - replication - - diff --git a/puppet/modules/unbound b/puppet/modules/unbound -Subproject 00646b0ffc71a86981b05f983c86ace0979d1b6 +Subproject a26b91dfea3189e6777629fa00d54f51dc41f4d diff --git a/tests/helpers/bonafide_helper.rb b/tests/helpers/bonafide_helper.rb index 9b26eaaf..82db3973 100644 --- a/tests/helpers/bonafide_helper.rb +++ b/tests/helpers/bonafide_helper.rb @@ -32,7 +32,15 @@ class LeapTest def assert_create_user user = SRP::User.new url = api_url("/1/users.json") - assert_post(url, user.to_params) do |body| + + params = user.to_params + + if property('webapp.invite_required') + @invite_code = generate_invite_code + params['user[invite_code]'] = @invite_code + end + + assert_post(url, params) do |body| assert response = JSON.parse(body), 'response should be JSON' assert response['ok'], "Creating a user should be successful, got #{response.inspect} instead." end @@ -40,6 +48,11 @@ class LeapTest return user end + def generate_invite_code + `cd /srv/leap/webapp/ && sudo -u leap-webapp RAILS_ENV=production bundle exec rake generate_invites[1]`.gsub(/\n/, "") + end + + # # attempts to authenticate user. if successful, # user object is updated with id and session token. diff --git a/tests/helpers/client_side_db.py b/tests/helpers/client_side_db.py new file mode 100644 index 00000000..2f8c220f --- /dev/null +++ b/tests/helpers/client_side_db.py @@ -0,0 +1,167 @@ +import logging +import os +import tempfile +import getpass +import binascii +import json + +try: + import requests + import srp._pysrp as srp +except ImportError: + pass + +from twisted.internet.defer import inlineCallbacks + +from leap.soledad.client import Soledad + + +""" +Helper functions to give access to client-side Soledad database. +Copied over from soledad/scripts folder. +""" + +# create a logger +logger = logging.getLogger(__name__) + +# DEBUG: enable debug logs +# LOG_FORMAT = '%(asctime)s %(message)s' +# logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG) + + +safe_unhexlify = lambda x: binascii.unhexlify(x) if ( + len(x) % 2 == 0) else binascii.unhexlify('0' + x) + + +def _fail(reason): + logger.error('Fail: ' + reason) + exit(2) + + +def get_soledad_instance(uuid, passphrase, basedir, server_url, cert_file, + token): + # setup soledad info + logger.info('UUID is %s' % uuid) + logger.info('Server URL is %s' % server_url) + secrets_path = os.path.join( + basedir, '%s.secret' % uuid) + local_db_path = os.path.join( + basedir, '%s.db' % uuid) + # instantiate soledad + return Soledad( + uuid, + unicode(passphrase), + secrets_path=secrets_path, + local_db_path=local_db_path, + server_url=server_url, + cert_file=cert_file, + auth_token=token, + defer_encryption=True) + + +def _get_api_info(provider): + info = requests.get( + 'https://'+provider+'/provider.json', verify=False).json() + return info['api_uri'], info['api_version'] + + +def _login(username, passphrase, provider, api_uri, api_version): + usr = srp.User(username, passphrase, srp.SHA256, srp.NG_1024) + auth = None + try: + auth = _authenticate(api_uri, api_version, usr).json() + except requests.exceptions.ConnectionError: + _fail('Could not connect to server.') + if 'errors' in auth: + _fail(str(auth['errors'])) + return api_uri, api_version, auth + + +def _authenticate(api_uri, api_version, usr): + api_url = "%s/%s" % (api_uri, api_version) + session = requests.session() + uname, A = usr.start_authentication() + params = {'login': uname, 'A': binascii.hexlify(A)} + init = session.post( + api_url + '/sessions', data=params, verify=False).json() + if 'errors' in init: + _fail('test user not found') + M = usr.process_challenge( + safe_unhexlify(init['salt']), safe_unhexlify(init['B'])) + return session.put(api_url + '/sessions/' + uname, verify=False, + data={'client_auth': binascii.hexlify(M)}) + + +def _get_soledad_info(username, provider, passphrase, basedir): + api_uri, api_version = _get_api_info(provider) + auth = _login(username, passphrase, provider, api_uri, api_version) + # get soledad server url + service_url = '%s/%s/config/soledad-service.json' % \ + (api_uri, api_version) + soledad_hosts = requests.get(service_url, verify=False).json()['hosts'] + hostnames = soledad_hosts.keys() + # allow for choosing the host + host = hostnames[0] + if len(hostnames) > 1: + i = 1 + print "There are many available hosts:" + for h in hostnames: + print " (%d) %s.%s" % (i, h, provider) + i += 1 + choice = raw_input("Choose a host to use (default: 1): ") + if choice != '': + host = hostnames[int(choice) - 1] + server_url = 'https://%s:%d/user-%s' % \ + (soledad_hosts[host]['hostname'], soledad_hosts[host]['port'], + auth[2]['id']) + # get provider ca certificate + ca_cert = requests.get('https://%s/ca.crt' % provider, verify=False).text + cert_file = os.path.join(basedir, 'ca.crt') + with open(cert_file, 'w') as f: + f.write(ca_cert) + return auth[2]['id'], server_url, cert_file, auth[2]['token'] + + +def _get_passphrase(args): + passphrase = args.passphrase + if passphrase is None: + passphrase = getpass.getpass( + 'Password for %s@%s: ' % (args.username, args.provider)) + return passphrase + + +def _get_basedir(args): + basedir = args.basedir + if basedir is None: + basedir = tempfile.mkdtemp() + elif not os.path.isdir(basedir): + os.mkdir(basedir) + logger.info('Using %s as base directory.' % basedir) + return basedir + + +@inlineCallbacks +def _export_key(args, km, fname, private=False): + address = args.username + "@" + args.provider + pkey = yield km.get_key( + address, OpenPGPKey, private=private, fetch_remote=False) + with open(args.export_private_key, "w") as f: + f.write(pkey.key_data) + + +@inlineCallbacks +def _export_incoming_messages(soledad, directory): + yield soledad.create_index("by-incoming", "bool(incoming)") + docs = yield soledad.get_from_index("by-incoming", '1') + i = 1 + for doc in docs: + with open(os.path.join(directory, "message_%d.gpg" % i), "w") as f: + f.write(doc.content["_enc_json"]) + i += 1 + + +@inlineCallbacks +def _get_all_docs(soledad): + _, docs = yield soledad.get_all_docs() + for doc in docs: + print json.dumps(doc.content, indent=4) diff --git a/tests/helpers/os_helper.rb b/tests/helpers/os_helper.rb index aad67dda..da9ac843 100644 --- a/tests/helpers/os_helper.rb +++ b/tests/helpers/os_helper.rb @@ -9,7 +9,10 @@ class LeapTest output.each_line.map{|line| pid = line.split(' ')[0] process = line.gsub(/(#{pid} |\n)/, '') - if process =~ /pgrep --full --list-name/ + # filter out pgrep cmd itself + # on wheezy hosts, the "process" var contains the whole cmd including all parameters + # on jessie hosts, it only contains the first cmd (which is the default sheel invoked by 'sh') + if process =~ /^sh/ nil else {:pid => pid, :process => process} diff --git a/tests/helpers/soledad_sync.py b/tests/helpers/soledad_sync.py index 2fb865fc..a1cea069 100755 --- a/tests/helpers/soledad_sync.py +++ b/tests/helpers/soledad_sync.py @@ -1,78 +1,75 @@ #!/usr/bin/env python +""" +soledad_sync.py -# -# Test Soledad sync -# -# This script performs a slightly modified U1DB sync to the Soledad server and -# returns whether that sync was successful or not. -# -# It takes three arguments: -# -# uuid -- uuid of the user to sync -# token -- a valid session token -# server -- the url of the soledad server we should connect to -# -# For example: -# -# soledad_sync.py f6bef0586fcfdb8705e26a58f2d9e580 uYO-4ucEJFksJ6afjmcYwIyap2vW7bv6uLxk0w_RfCc https://199.119.112.9:2323/user-f6bef0586fcfdb8705e26a58f2d9e580 -# +This script exercises soledad synchronization. +Its exit code is 0 if the sync took place correctly, 1 otherwise. +It takes 5 arguments: + + uuid: uuid of the user to sync + token: a valid session token + server: the url of the soledad server we should connect to + cert_file: the file containing the certificate for the CA that signed the + cert for the soledad server. + password: the password for the user to sync + +__author__: kali@leap.se +""" import os import sys -import traceback import tempfile -import shutil -import u1db -from u1db.remote.http_target import HTTPSyncTarget +# This is needed because the twisted shipped with wheezy is too old +# to do proper ssl verification. +os.environ['SKIP_TWISTED_SSL_CHECK'] = '1' -# -# monkey patch U1DB's HTTPSyncTarget to perform token based auth -# +from twisted.internet import defer, reactor -def set_token_credentials(self, uuid, token): - self._creds = {'token': (uuid, token)} +from client_side_db import get_soledad_instance +from leap.common.events import flags -def _sign_request(self, method, url_query, params): - uuid, token = self._creds['token'] - auth = '%s:%s' % (uuid, token) - return [('Authorization', 'Token %s' % auth.encode('base64')[:-1])] +flags.set_events_enabled(False) -HTTPSyncTarget.set_token_credentials = set_token_credentials -HTTPSyncTarget._sign_request = _sign_request +NUMDOCS = 1 +USAGE = "Usage: %s uuid token server cert_file password" % sys.argv[0] -# -# Create a temporary local u1db replica and attempt to sync to it. -# Returns a failure message if something went wrong. -# -def soledad_sync(uuid, token, server): - tempdir = tempfile.mkdtemp() - try: - db = u1db.open(os.path.join(tempdir, '%s.db' % uuid), True) - creds = {'token': {'uuid': uuid, 'token': token}} - db.sync(server, creds=creds, autocreate=False) - finally: - shutil.rmtree(tempdir) - -# -# exit codes: -# -# 0 - OK -# 1 - WARNING -# 2 - ERROR -# +def bail(msg, exitcode): + print "[!] %s" % msg + sys.exit(exitcode) + + +def create_docs(soledad): + """ + Populates the soledad database with dummy messages, so we can exercise + sending payloads during the sync. + """ + deferreds = [] + for index in xrange(NUMDOCS): + deferreds.append(soledad.create_doc({'payload': 'dummy'})) + return defer.gatherResults(deferreds) + +# main program if __name__ == '__main__': - try: - uuid, token, server = sys.argv[1:] - result = soledad_sync(uuid, token, server) - if result is None: - exit(0) - else: - print(result) - exit(1) - except Exception as exc: - print(exc.message or str(exc)) - traceback.print_exc(file=sys.stdout) - exit(2) + + tempdir = tempfile.mkdtemp() + if len(sys.argv) < 6: + bail(USAGE, 2) + uuid, token, server, cert_file, passphrase = sys.argv[1:] + s = get_soledad_instance( + uuid, passphrase, tempdir, server, cert_file, token) + + def onSyncDone(sync_result): + print "SYNC_RESULT:", sync_result + s.close() + reactor.stop() + + def start_sync(): + d = create_docs(s) + d.addCallback(lambda _: s.sync()) + d.addCallback(onSyncDone) + + reactor.callWhenRunning(start_sync) + reactor.run() diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb index 5ee12ff3..edb28eac 100644 --- a/tests/white-box/couchdb.rb +++ b/tests/white-box/couchdb.rb @@ -9,7 +9,6 @@ class CouchDB < LeapTest end def test_00_Are_daemons_running? - assert_running '^tapicero', :single => true if multimaster? assert_running 'bin/beam' assert_running 'bin/epmd' @@ -70,7 +69,7 @@ class CouchDB < LeapTest end def test_04_Do_ACL_users_exist? - acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'tapicero', 'webapp', 'replication'] + acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'webapp', 'replication'] url = couchdb_backend_url("/_users/_all_docs", :username => 'admin') assert_get(url) do |body| response = JSON.parse(body) diff --git a/tests/white-box/mx.rb b/tests/white-box/mx.rb index 794a9a41..f49d2ab4 100644 --- a/tests/white-box/mx.rb +++ b/tests/white-box/mx.rb @@ -34,6 +34,9 @@ class Mx < LeapTest def test_03_Are_MX_daemons_running? assert_running 'leap_mx' assert_running '/usr/lib/postfix/master' + assert_running '/usr/sbin/postfwd' + assert_running 'postfwd2::cache' + assert_running 'postfwd2::policy' assert_running '/usr/sbin/unbound' pass end diff --git a/tests/white-box/network.rb b/tests/white-box/network.rb index acb5c5e6..382f857b 100644 --- a/tests/white-box/network.rb +++ b/tests/white-box/network.rb @@ -28,11 +28,18 @@ class Network < LeapTest def test_02_Is_stunnel_running? ignore unless $node['stunnel'] good_stunnel_pids = [] + release = `facter lsbmajdistrelease` + if release.to_i > 7 + # on jessie, there is only one stunnel proc running instead of 6 + expected = 1 + else + expected = 6 + end $node['stunnel']['clients'].each do |stunnel_type, stunnel_configs| stunnel_configs.each do |stunnel_name, stunnel_conf| config_file_name = "/etc/stunnel/#{stunnel_name}.conf" processes = pgrep(config_file_name) - assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`" + assert_equal expected, processes.length, "There should be #{expected} stunnel processes running for `#{config_file_name}`" good_stunnel_pids += processes.map{|ps| ps[:pid]} assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.' assert_tcp_socket('localhost', port) @@ -41,7 +48,7 @@ class Network < LeapTest $node['stunnel']['servers'].each do |stunnel_name, stunnel_conf| config_file_name = "/etc/stunnel/#{stunnel_name}.conf" processes = pgrep(config_file_name) - assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`" + assert_equal expected, processes.length, "There should be #{expected} stunnel processes running for `#{config_file_name}`" good_stunnel_pids += processes.map{|ps| ps[:pid]} assert accept_port = stunnel_conf['accept_port'], "Field `accept` must be present in property `stunnel.servers.#{stunnel_name}`" assert_tcp_socket('localhost', accept_port) diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb index 9956eb35..e689c143 100644 --- a/tests/white-box/webapp.rb +++ b/tests/white-box/webapp.rb @@ -57,10 +57,11 @@ class Webapp < LeapTest soledad_server = pick_soledad_server(soledad_config) if soledad_server assert_tmp_user do |user| - assert_user_db_exists(user) command = File.expand_path "../../helpers/soledad_sync.py", __FILE__ soledad_url = "https://#{soledad_server}/user-#{user.id}" - assert_run "#{command} #{user.id} #{user.session_token} #{soledad_url}" + soledad_cert = "/usr/local/share/ca-certificates/leap_ca.crt" + assert_run "#{command} #{user.id} #{user.session_token} #{soledad_url} #{soledad_cert} #{user.password}" + assert_user_db_exists(user) pass end end @@ -95,7 +96,7 @@ class Webapp < LeapTest end # - # returns true if the per-user db created by tapicero exists. + # returns true if the per-user db created by soledad-server exists. # we try three times, and give up after that. # def assert_user_db_exists(user) diff --git a/vagrant/configure-leap.sh b/vagrant/configure-leap.sh index 9541e194..332bdddf 100755 --- a/vagrant/configure-leap.sh +++ b/vagrant/configure-leap.sh @@ -1,13 +1,15 @@ #!/bin/bash -. /vagrant/vagrant/vagrant.config +. /vagrant/vagrant/vagrant.config #OPTS='--no-color' OPTS='' -PROVIDERDIR='/srv/leap/configuration' +USER='vagrant' NODE='node1' -LEAP='/usr/local/bin/leap' +SUDO="sudo -u ${USER}" +PROVIDERDIR="/home/${USER}/leap/configuration" +LEAP="$SUDO /usr/local/bin/leap" echo '===============================================' echo 'configuring leap' @@ -15,19 +17,22 @@ echo '===============================================' # purge $PROVIDERDIR so this script can be run multiple times [ -e $PROVIDERDIR ] && rm -rf $PROVIDERDIR -mkdir $PROVIDERDIR + +mkdir -p $PROVIDERDIR +chown ${USER}:${USER} ${PROVIDERDIR} cd $PROVIDERDIR $LEAP $OPTS new --contacts "$contacts" --domain "$provider_domain" --name "$provider_name" --platform=/vagrant . -echo -e '\n@log = "/var/log/leap/deploy.log"' >> Leapfile +$SUDO echo -e '\n@log = "/var/log/leap/deploy.log"' >> Leapfile -if [ ! -e /root/.ssh/id_rsa ]; then - ssh-keygen -f /root/.ssh/id_rsa -P '' - cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys +if [ ! -e /home/${USER}/.ssh/id_rsa ]; then + $SUDO ssh-keygen -f /home/${USER}/.ssh/id_rsa -P '' + cat /home/${USER}/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys fi -mkdir -p $PROVIDERDIR/files/nodes/$NODE +$SUDO mkdir -p ${PROVIDERDIR}/files/nodes/${NODE} sh -c "cat /etc/ssh/ssh_host_rsa_key.pub | cut -d' ' -f1,2 >> $PROVIDERDIR/files/nodes/$NODE/${NODE}_ssh.pub" +chown ${USER}:${USER} ${PROVIDERDIR}/files/nodes/${NODE}/${NODE}_ssh.pub $LEAP $OPTS add-user --self $LEAP $OPTS cert ca @@ -41,17 +46,13 @@ git init git add . git commit -m'configured provider' -$LEAP $OPTS node init $NODE +$LEAP $OPTS node init $NODE if [ $? -eq 1 ]; then echo 'node init failed' exit 1 fi $LEAP $OPTS -v 2 deploy -if [ $? -eq 1 ]; then - echo 'deploy failed' - exit 1 -fi set +e git add . @@ -69,9 +70,6 @@ echo 'setting node to demo-mode' echo '===============================================' postconf -e default_transport='error: in demo mode' -sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/g' /etc/ssh/sshd_config -/etc/init.d/ssh reload - # add users: testadmin and testuser with passwords "hallo123" curl -s -k https://localhost/1/users.json -d "user%5Blogin%5D=testuser&user%5Bpassword_salt%5D=7d4880237a038e0e&user%5Bpassword_verifier%5D=b98dc393afcd16e5a40fb57ce9cddfa6a978b84be326196627c111d426cada898cdaf3a6427e98b27daf4b0ed61d278bc856515aeceb2312e50c8f816659fcaa4460d839a1e2d7ffb867d32ac869962061368141c7571a53443d58dc84ca1fca34776894414c1090a93e296db6cef12c2cc3f7a991b05d49728ed358fd868286" curl -s -k https://localhost/1/users.json -d "user%5Blogin%5D=testadmin&user%5Bpassword_salt%5D=ece1c457014d8282&user%5Bpassword_verifier%5D=9654d93ab409edf4ff1543d07e08f321107c3fd00de05c646c637866a94f28b3eb263ea9129dacebb7291b3374cc6f0bf88eb3d231eb3a76eed330a0e8fd2a5c477ed2693694efc1cc23ae83c2ae351a21139701983dd595b6c3225a1bebd2a4e6122f83df87606f1a41152d9890e5a11ac3749b3bfcf4407fc83ef60b4ced68" @@ -80,4 +78,3 @@ echo -e '\n\n\n' echo 'You are now ready to use your provider. Please update your /etc/hosts with following dns overrides:' $LEAP list --print ip_address,domain.full,dns.aliases | sed 's/,//g' | cut -d' ' -f 2- - diff --git a/vagrant/install-platform.pp b/vagrant/install-platform.pp index 465ca78a..5ea834b1 100755 --- a/vagrant/install-platform.pp +++ b/vagrant/install-platform.pp @@ -3,34 +3,22 @@ File['/etc/apt/preferences'] -> Exec['refresh_apt'] -> Package <| ( title != 'lsb' ) |> -package { [ 'rsync', 'ruby-hiera-puppet', 'git', 'ruby1.9.1-dev', 'rake', 'jq' ]: - ensure => installed -} - -file { '/etc/gemrc': - content => "---\n:sources:\n - https://rubygems.org/" -} -vcsrepo { '/srv/leap/leap_cli': - ensure => present, - force => true, - revision => 'develop', - provider => 'git', - source => 'https://leap.se/git/leap_cli.git', - owner => 'root', - group => 'root', - notify => Exec['install_leap_cli'], - require => Package['git'] +if $::lsbdistcodename == 'wheezy' { + package { 'ruby-hiera-puppet': + ensure => installed + } } -exec { 'install_leap_cli': - command => '/usr/bin/rake build && /usr/bin/rake install', - cwd => '/srv/leap/leap_cli', - refreshonly => true, - require => [ Package['ruby1.9.1-dev'], File['/etc/gemrc'], Package['rake'] ] +# install leap_cli from source, so it will work with the develop +# branch of leap_platform +class { '::leap::cli::install': + source => true, } file { [ '/srv/leap', '/srv/leap/configuration', '/var/log/leap' ]: ensure => directory } +# install prerequisites for configuring the provider +include ::git |