diff options
| author | elijah <elijah@riseup.net> | 2015-08-12 14:37:21 -0700 | 
|---|---|---|
| committer | elijah <elijah@riseup.net> | 2015-08-19 12:24:51 -0700 | 
| commit | 64787942086b6fbfdf432cd6250f0937c785de1a (patch) | |
| tree | 06634196af8066cb782032f731ee7cae939367ce /lib | |
| parent | e6f6324ebea9fc46d5d617a08c2eb239349d9f90 (diff) | |
mv commands and macros to lib/leap_cli
Diffstat (limited to 'lib')
27 files changed, 3490 insertions, 0 deletions
| diff --git a/lib/leap_cli/commands/README b/lib/leap_cli/commands/README new file mode 100644 index 00000000..bec78179 --- /dev/null +++ b/lib/leap_cli/commands/README @@ -0,0 +1,11 @@ +This directory contains ruby source files that define the available sub- +commands of the `leap` executable. + +For example, the command: + +  leap compile + +Lives in lib/leap_cli/commands/init.rb + +These files use a DSL (called GLI) for defining command suites. +See https://github.com/davetron5000/gli for more information. diff --git a/lib/leap_cli/commands/ca.rb b/lib/leap_cli/commands/ca.rb new file mode 100644 index 00000000..d5c6240d --- /dev/null +++ b/lib/leap_cli/commands/ca.rb @@ -0,0 +1,518 @@ +autoload :OpenSSL, 'openssl' +autoload :CertificateAuthority, 'certificate_authority' +autoload :Date, 'date' +require 'digest/md5' + +module LeapCli; module Commands + +  desc "Manage X.509 certificates" +  command :cert do |cert| + +    cert.desc 'Creates two Certificate Authorities (one for validating servers and one for validating clients).' +    cert.long_desc 'See see what values are used in the generation of the certificates (like name and key size), run `leap inspect provider` and look for the "ca" property. To see the details of the created certs, run `leap inspect <file>`.' +    cert.command :ca do |ca| +      ca.action do |global_options,options,args| +        assert_config! 'provider.ca.name' +        generate_new_certificate_authority(:ca_key, :ca_cert, provider.ca.name) +        generate_new_certificate_authority(:client_ca_key, :client_ca_cert, provider.ca.name + ' (client certificates only!)') +      end +    end + +    cert.desc 'Creates or renews a X.509 certificate/key pair for a single node or all nodes, but only if needed.' +    cert.long_desc 'This command will a generate new certificate for a node if some value in the node has changed ' + +                   'that is included in the certificate (like hostname or IP address), or if the old certificate will be expiring soon. ' + +                   'Sometimes, you might want to force the generation of a new certificate, ' + +                   'such as in the cases where you have changed a CA parameter for server certificates, like bit size or digest hash. ' + +                   'In this case, use --force. If <node-filter> is empty, this command will apply to all nodes.' +    cert.arg_name 'FILTER' +    cert.command :update do |update| +      update.switch 'force', :desc => 'Always generate new certificates', :negatable => false +      update.action do |global_options,options,args| +        update_certificates(manager.filter!(args), options) +      end +    end + +    cert.desc 'Creates a Diffie-Hellman parameter file, needed for forward secret OpenVPN ciphers.' # (needed for server-side of some TLS connections) +    cert.command :dh do |dh| +      dh.action do |global_options,options,args| +        long_running do +          if cmd_exists?('certtool') +            log 0, 'Generating DH parameters (takes a long time)...' +            output = assert_run!('certtool --generate-dh-params --sec-param high') +            output.sub! /.*(-----BEGIN DH PARAMETERS-----.*-----END DH PARAMETERS-----).*/m, '\1' +            output << "\n" +            write_file!(:dh_params, output) +          else +            log 0, 'Generating DH parameters (takes a REALLY long time)...' +            output = OpenSSL::PKey::DH.generate(3248).to_pem +            write_file!(:dh_params, output) +          end +        end +      end +    end + +    # +    # hints: +    # +    # inspect CSR: +    #   openssl req -noout -text -in files/cert/x.csr +    # +    # generate CSR with openssl to see how it compares: +    #   openssl req -sha256 -nodes -newkey rsa:2048 -keyout example.key -out example.csr +    # +    # validate a CSR: +    #   http://certlogik.com/decoder/ +    # +    # nice details about CSRs: +    #   http://www.redkestrel.co.uk/Articles/CSR.html +    # +    cert.desc "Creates a CSR for use in buying a commercial X.509 certificate." +    cert.long_desc "Unless specified, the CSR is created for the provider's primary domain. "+ +      "The properties used for this CSR come from `provider.ca.server_certificates`, "+ +      "but may be overridden here." +    cert.command :csr do |csr| +      csr.flag 'domain', :arg_name => 'DOMAIN', :desc => 'Specify what domain to create the CSR for.' +      csr.flag ['organization', 'O'], :arg_name => 'ORGANIZATION', :desc => "Override default O in distinguished name." +      csr.flag ['unit', 'OU'], :arg_name => 'UNIT', :desc => "Set OU in distinguished name." +      csr.flag 'email', :arg_name => 'EMAIL', :desc => "Set emailAddress in distinguished name." +      csr.flag ['locality', 'L'], :arg_name => 'LOCALITY', :desc => "Set L in distinguished name." +      csr.flag ['state', 'ST'], :arg_name => 'STATE', :desc => "Set ST in distinguished name." +      csr.flag ['country', 'C'], :arg_name => 'COUNTRY', :desc => "Set C in distinguished name." +      csr.flag :bits, :arg_name => 'BITS', :desc => "Override default certificate bit length" +      csr.flag :digest, :arg_name => 'DIGEST', :desc => "Override default signature digest" +      csr.action do |global_options,options,args| +        assert_config! 'provider.domain' +        assert_config! 'provider.name' +        assert_config! 'provider.default_language' +        assert_config! 'provider.ca.server_certificates.bit_size' +        assert_config! 'provider.ca.server_certificates.digest' +        domain = options[:domain] || provider.domain + +        unless global_options[:force] +          assert_files_missing! [:commercial_key, domain], [:commercial_csr, domain], +            :msg => 'If you really want to create a new key and CSR, remove these files first or run with --force.' +        end + +        server_certificates = provider.ca.server_certificates + +        # RSA key +        keypair = CertificateAuthority::MemoryKeyMaterial.new +        bit_size = (options[:bits] || server_certificates.bit_size).to_i +        log :generating, "%s bit RSA key" % bit_size do +          keypair.generate_key(bit_size) +          write_file! [:commercial_key, domain], keypair.private_key.to_pem +        end + +        # CSR +        dn  = CertificateAuthority::DistinguishedName.new +        dn.common_name   = domain +        dn.organization  = options[:organization] || provider.name[provider.default_language] +        dn.ou            = options[:organizational_unit] # optional +        dn.email_address = options[:email] # optional +        dn.country       = options[:country] || server_certificates['country']   # optional +        dn.state         = options[:state] || server_certificates['state']       # optional +        dn.locality      = options[:locality] || server_certificates['locality'] # optional + +        digest = options[:digest] || server_certificates.digest +        log :generating, "CSR with #{digest} digest and #{print_dn(dn)}" do +          csr = create_csr(dn, keypair, digest) +          request = csr.to_x509_csr +          write_file! [:commercial_csr, domain], csr.to_pem +        end + +        # Sign using our own CA, for use in testing but hopefully not production. +        # It is not that commerical CAs are so secure, it is just that signing your own certs is +        # a total drag for the user because they must click through dire warnings. +        #if options[:sign] +          log :generating, "self-signed x509 server certificate for testing purposes" do +            cert = csr.to_cert +            cert.serial_number.number = cert_serial_number(domain) +            cert.not_before = yesterday +            cert.not_after  = yesterday.advance(:years => 1) +            cert.parent = ca_root +            cert.sign! domain_test_signing_profile +            write_file! [:commercial_cert, domain], cert.to_pem +            log "please replace this file with the real certificate you get from a CA using #{Path.relative_path([:commercial_csr, domain])}" +          end +        #end + +        # FAKE CA +        unless file_exists? :commercial_ca_cert +          log :using, "generated CA in place of commercial CA for testing purposes" do +            write_file! :commercial_ca_cert, read_file!(:ca_cert) +            log "please also replace this file with the CA cert from the commercial authority you use." +          end +        end +      end +    end +  end + +  protected + +  # +  # will generate new certificates for the specified nodes, if needed. +  # +  def update_certificates(nodes, options={}) +    assert_files_exist! :ca_cert, :ca_key, :msg => 'Run `leap cert ca` to create them' +    assert_config! 'provider.ca.server_certificates.bit_size' +    assert_config! 'provider.ca.server_certificates.digest' +    assert_config! 'provider.ca.server_certificates.life_span' +    assert_config! 'common.x509.use' + +    nodes.each_node do |node| +      warn_if_commercial_cert_will_soon_expire(node) +      if !node.x509.use +        remove_file!([:node_x509_key, node.name]) +        remove_file!([:node_x509_cert, node.name]) +      elsif options[:force] || cert_needs_updating?(node) +        generate_cert_for_node(node) +      end +    end +  end + +  private + +  def generate_new_certificate_authority(key_file, cert_file, common_name) +    assert_files_missing! key_file, cert_file +    assert_config! 'provider.ca.name' +    assert_config! 'provider.ca.bit_size' +    assert_config! 'provider.ca.life_span' + +    root = CertificateAuthority::Certificate.new + +    # set subject +    root.subject.common_name = common_name +    possible = ['country', 'state', 'locality', 'organization', 'organizational_unit', 'email_address'] +    provider.ca.keys.each do |key| +      if possible.include?(key) +        root.subject.send(key + '=', provider.ca[key]) +      end +    end + +    # set expiration +    root.not_before = yesterday +    root.not_after = yesterday_advance(provider.ca.life_span) + +    # generate private key +    root.serial_number.number = 1 +    root.key_material.generate_key(provider.ca.bit_size) + +    # sign self +    root.signing_entity = true +    root.parent = root +    root.sign!(ca_root_signing_profile) + +    # save +    write_file!(key_file, root.key_material.private_key.to_pem) +    write_file!(cert_file, root.to_pem) +  end + +  # +  # returns true if the certs associated with +node+ need to be regenerated. +  # +  def cert_needs_updating?(node) +    if !file_exists?([:node_x509_cert, node.name], [:node_x509_key, node.name]) +      return true +    else +      cert = load_certificate_file([:node_x509_cert, node.name]) +      if cert.not_after < Time.now.advance(:months => 2) +        log :updating, "cert for node '#{node.name}' because it will expire soon" +        return true +      end +      if cert.subject.common_name != node.domain.full +        log :updating, "cert for node '#{node.name}' because domain.full has changed (was #{cert.subject.common_name}, now #{node.domain.full})" +        return true +      end +      cert.openssl_body.extensions.each do |ext| +        if ext.oid == "subjectAltName" +          ips = [] +          dns_names = [] +          ext.value.split(",").each do |value| +            value.strip! +            ips << $1          if value =~ /^IP Address:(.*)$/ +            dns_names << $1    if value =~ /^DNS:(.*)$/ +          end +          dns_names.sort! +          if ips.first != node.ip_address +            log :updating, "cert for node '#{node.name}' because ip_address has changed (from #{ips.first} to #{node.ip_address})" +            return true +          elsif dns_names != dns_names_for_node(node) +            log :updating, "cert for node '#{node.name}' because domain name aliases have changed\n    from: #{dns_names.inspect}\n    to: #{dns_names_for_node(node).inspect})" +            return true +          end +        end +      end +    end +    return false +  end + +  def warn_if_commercial_cert_will_soon_expire(node) +    dns_names_for_node(node).each do |domain| +      if file_exists?([:commercial_cert, domain]) +        cert = load_certificate_file([:commercial_cert, domain]) +        path = Path.relative_path([:commercial_cert, domain]) +        if cert.not_after < Time.now.utc +          log :error, "the commercial certificate '#{path}' has EXPIRED! " + +            "You should renew it with `leap cert csr --domain #{domain}`." +        elsif cert.not_after < Time.now.advance(:months => 2) +          log :warning, "the commercial certificate '#{path}' will expire soon. "+ +            "You should renew it with `leap cert csr --domain #{domain}`." +        end +      end +    end +  end + +  def generate_cert_for_node(node) +    return if node.x509.use == false + +    cert = CertificateAuthority::Certificate.new + +    # set subject +    cert.subject.common_name = node.domain.full +    cert.serial_number.number = cert_serial_number(node.domain.full) + +    # set expiration +    cert.not_before = yesterday +    cert.not_after = yesterday_advance(provider.ca.server_certificates.life_span) + +    # generate key +    cert.key_material.generate_key(provider.ca.server_certificates.bit_size) + +    # sign +    cert.parent = ca_root +    cert.sign!(server_signing_profile(node)) + +    # save +    write_file!([:node_x509_key, node.name], cert.key_material.private_key.to_pem) +    write_file!([:node_x509_cert, node.name], cert.to_pem) +  end + +  # +  # yields client key and cert suitable for testing +  # +  def generate_test_client_cert(prefix=nil) +    cert = CertificateAuthority::Certificate.new +    cert.serial_number.number = cert_serial_number(provider.domain) +    cert.subject.common_name = [prefix, random_common_name(provider.domain)].join +    cert.not_before = yesterday +    cert.not_after  = yesterday.advance(:years => 1) +    cert.key_material.generate_key(1024) # just for testing, remember! +    cert.parent = client_ca_root +    cert.sign! client_test_signing_profile +    yield cert.key_material.private_key.to_pem, cert.to_pem +  end + +  # +  # creates a CSR and returns it. +  # with the correct extReq attribute so that the CA +  # doens't generate certs with extensions we don't want. +  # +  def create_csr(dn, keypair, digest) +    csr = CertificateAuthority::SigningRequest.new +    csr.distinguished_name = dn +    csr.key_material = keypair +    csr.digest = digest + +    # define extensions manually (library doesn't support setting these on CSRs) +    extensions = [] +    extensions << CertificateAuthority::Extensions::BasicConstraints.new.tap {|basic| +      basic.ca = false +    } +    extensions << CertificateAuthority::Extensions::KeyUsage.new.tap {|keyusage| +      keyusage.usage = ["digitalSignature", "keyEncipherment"] +    } +    extensions << CertificateAuthority::Extensions::ExtendedKeyUsage.new.tap {|extkeyusage| +      extkeyusage.usage = [ "serverAuth"] +    } + +    # convert extensions to attribute 'extReq' +    # aka "Requested Extensions" +    factory = OpenSSL::X509::ExtensionFactory.new +    attrval = OpenSSL::ASN1::Set([OpenSSL::ASN1::Sequence( +      extensions.map{|e| factory.create_ext(e.openssl_identifier, e.to_s, e.critical)} +    )]) +    attrs = [ +      OpenSSL::X509::Attribute.new("extReq", attrval), +    ] +    csr.attributes = attrs + +    return csr +  end + +  def ca_root +    @ca_root ||= begin +      load_certificate_file(:ca_cert, :ca_key) +    end +  end + +  def client_ca_root +    @client_ca_root ||= begin +      load_certificate_file(:client_ca_cert, :client_ca_key) +    end +  end + +  def load_certificate_file(crt_file, key_file=nil, password=nil) +    crt = read_file!(crt_file) +    openssl_cert = OpenSSL::X509::Certificate.new(crt) +    cert = CertificateAuthority::Certificate.from_openssl(openssl_cert) +    if key_file +      key = read_file!(key_file) +      cert.key_material.private_key = OpenSSL::PKey::RSA.new(key, password) +    end +    return cert +  end + +  def ca_root_signing_profile +    { +      "extensions" => { +        "basicConstraints" => {"ca" => true}, +        "keyUsage" => { +          "usage" => ["critical", "keyCertSign"] +        }, +        "extendedKeyUsage" => { +          "usage" => [] +        } +      } +    } +  end + +  # +  # For keyusage, openvpn server certs can have keyEncipherment or keyAgreement. +  # Web browsers seem to break without keyEncipherment. +  # For now, I am using digitalSignature + keyEncipherment +  # +  # * digitalSignature -- for (EC)DHE cipher suites +  #   "The digitalSignature bit is asserted when the subject public key is used +  #    with a digital signature mechanism to support security services other +  #    than certificate signing (bit 5), or CRL signing (bit 6). Digital +  #    signature mechanisms are often used for entity authentication and data +  #    origin authentication with integrity." +  # +  # * keyEncipherment  ==> for plain RSA cipher suites +  #   "The keyEncipherment bit is asserted when the subject public key is used for +  #    key transport. For example, when an RSA key is to be used for key management, +  #    then this bit is set." +  # +  # * keyAgreement     ==> for used with DH, not RSA. +  #   "The keyAgreement bit is asserted when the subject public key is used for key +  #    agreement. For example, when a Diffie-Hellman key is to be used for key +  #    management, then this bit is set." +  # +  # digest options: SHA512, SHA256, SHA1 +  # +  def server_signing_profile(node) +    { +      "digest" => provider.ca.server_certificates.digest, +      "extensions" => { +        "keyUsage" => { +          "usage" => ["digitalSignature", "keyEncipherment"] +        }, +        "extendedKeyUsage" => { +          "usage" => ["serverAuth", "clientAuth"] +        }, +        "subjectAltName" => { +          "ips" => [node.ip_address], +          "dns_names" => dns_names_for_node(node) +        } +      } +    } +  end + +  # +  # This is used when signing the main cert for the provider's domain +  # with our own CA (for testing purposes). Typically, this cert would +  # be purchased from a commercial CA, and not signed this way. +  # +  def domain_test_signing_profile +    { +      "digest" => "SHA256", +      "extensions" => { +        "keyUsage" => { +          "usage" => ["digitalSignature", "keyEncipherment"] +        }, +        "extendedKeyUsage" => { +          "usage" => ["serverAuth"] +        } +      } +    } +  end + +  # +  # This is used when signing a dummy client certificate that is only to be +  # used for testing. +  # +  def client_test_signing_profile +    { +      "digest" => "SHA256", +      "extensions" => { +        "keyUsage" => { +          "usage" => ["digitalSignature"] +        }, +        "extendedKeyUsage" => { +          "usage" => ["clientAuth"] +        } +      } +    } +  end + +  def dns_names_for_node(node) +    names = [node.domain.internal, node.domain.full] +    if node['dns'] && node.dns['aliases'] && node.dns.aliases.any? +      names += node.dns.aliases +    end +    names.compact! +    names.sort! +    names.uniq! +    return names +  end + +  # +  # For cert serial numbers, we need a non-colliding number less than 160 bits. +  # md5 will do nicely, since there is no need for a secure hash, just a short one. +  # (md5 is 128 bits) +  # +  def cert_serial_number(domain_name) +    Digest::MD5.hexdigest("#{domain_name} -- #{Time.now}").to_i(16) +  end + +  # +  # for the random common name, we need a text string that will be unique across all certs. +  # ruby 1.8 doesn't have a built-in uuid generator, or we would use SecureRandom.uuid +  # +  def random_common_name(domain_name) +    cert_serial_number(domain_name).to_s(36) +  end + +  # prints CertificateAuthority::DistinguishedName fields +  def print_dn(dn) +    fields = {} +    [:common_name, :locality, :state, :country, :organization, :organizational_unit, :email_address].each do |attr| +      fields[attr] = dn.send(attr) if dn.send(attr) +    end +    fields.inspect +  end + +  ## +  ## TIME HELPERS +  ## +  ## note: we use 'yesterday' instead of 'today', because times are in UTC, and some people on the planet +  ## are behind UTC. +  ## + +  def yesterday +    t = Time.now - 24*24*60 +    Time.utc t.year, t.month, t.day +  end + +  def yesterday_advance(string) +    number, unit = string.split(' ') +    unless ['years', 'months', 'days', 'hours', 'minutes'].include? unit +      bail!("The time property '#{string}' is missing a unit (one of: years, months, days, hours, minutes).") +    end +    unless number.to_i.to_s == number +      bail!("The time property '#{string}' is missing a number.") +    end +    yesterday.advance(unit.to_sym => number.to_i) +  end + +end; end diff --git a/lib/leap_cli/commands/clean.rb b/lib/leap_cli/commands/clean.rb new file mode 100644 index 00000000..a9afff53 --- /dev/null +++ b/lib/leap_cli/commands/clean.rb @@ -0,0 +1,16 @@ +module LeapCli +  module Commands + +    desc 'Removes all files generated with the "compile" command.' +    command :clean do |c| +      c.action do |global_options,options,args| +        Dir.glob(path([:hiera, '*'])).each do |file| +          remove_file! file +        end +        remove_file! path(:authorized_keys) +        remove_file! path(:known_hosts) +      end +    end + +  end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/compile.rb b/lib/leap_cli/commands/compile.rb new file mode 100644 index 00000000..f5895b8b --- /dev/null +++ b/lib/leap_cli/commands/compile.rb @@ -0,0 +1,384 @@ +require 'socket' + +module LeapCli +  module Commands + +    desc "Compile generated files." +    command [:compile, :c] do |c| +      c.desc 'Compiles node configuration files into hiera files used for deployment.' +      c.arg_name 'ENVIRONMENT', :optional => true +      c.command :all do |all| +        all.action do |global_options,options,args| +          environment = args.first +          if !LeapCli.leapfile.environment.nil? && !environment.nil? && environment != LeapCli.leapfile.environment +            bail! "You cannot specify an ENVIRONMENT argument while the environment is pinned." +          end +          if environment +            if manager.environment_names.include?(environment) +              compile_hiera_files(manager.filter([environment]), false) +            else +              bail! "There is no environment named `#{environment}`." +            end +          else +            clean_export = LeapCli.leapfile.environment.nil? +            compile_hiera_files(manager.filter, clean_export) +          end +          if file_exists?(:static_web_readme) +            compile_provider_json(environment) +          end +        end +      end + +      c.desc "Compile a DNS zone file for your provider." +      c.command :zone do |zone| +        zone.action do |global_options, options, args| +          compile_zone_file +        end +      end + +      c.desc "Compile provider.json bootstrap files for your provider." +      c.command 'provider.json' do |provider| +        provider.action do |global_options, options, args| +          compile_provider_json +        end +      end + +      c.desc "Generate a list of firewall rules. These rules are already "+ +             "implemented on each node, but you might want the list of all "+ +             "rules in case you also have a restrictive network firewall." +      c.command :firewall do |zone| +        zone.action do |global_options, options, args| +          compile_firewall +        end +      end + +      c.default_command :all +    end + +    protected + +    # +    # a "clean" export of secrets will also remove keys that are no longer used, +    # but this should not be done if we are not examining all possible nodes. +    # +    def compile_hiera_files(nodes, clean_export) +      update_compiled_ssh_configs # must come first +      sanity_check(nodes) +      manager.export_nodes(nodes) +      manager.export_secrets(clean_export) +    end + +    def update_compiled_ssh_configs +      generate_monitor_ssh_keys +      update_authorized_keys +      update_known_hosts +    end + +    def sanity_check(nodes) +      # confirm that every node has a unique ip address +      ips = {} +      nodes.pick_fields('ip_address').each do |name, ip_address| +        if ips.key?(ip_address) +          bail! { +            log(:fatal_error, "Every node must have its own IP address.") { +              log "Nodes `#{name}` and `#{ips[ip_address]}` are both configured with `#{ip_address}`." +            } +          } +        else +          ips[ip_address] = name +        end +      end +      # confirm that the IP address of this machine is not also used for a node. +      Socket.ip_address_list.each do |addrinfo| +        if !addrinfo.ipv4_private? && ips.key?(addrinfo.ip_address) +          ip = addrinfo.ip_address +          name = ips[ip] +          bail! { +            log(:fatal_error, "Something is very wrong. The `leap` command must only be run on your sysadmin machine, not on a provider node.") { +              log "This machine has the same IP address (#{ip}) as node `#{name}`." +            } +          } +        end +      end +    end + +    ## +    ## SSH +    ## + +    # +    # generates a ssh key pair that is used only by remote monitors +    # to connect to nodes and run certain allowed commands. +    # +    # every node has the public monitor key added to their authorized +    # keys, and every monitor node has a copy of the private monitor key. +    # +    def generate_monitor_ssh_keys +      priv_key_file = path(:monitor_priv_key) +      pub_key_file  = path(:monitor_pub_key) +      unless file_exists?(priv_key_file, pub_key_file) +        ensure_dir(File.dirname(priv_key_file)) +        ensure_dir(File.dirname(pub_key_file)) +        cmd = %(ssh-keygen -N '' -C 'monitor' -t rsa -b 4096 -f '%s') % priv_key_file +        assert_run! cmd +        if file_exists?(priv_key_file, pub_key_file) +          log :created, priv_key_file +          log :created, pub_key_file +        else +          log :failed, 'to create monitor ssh keys' +        end +      end +    end + +    # +    # Compiles the authorized keys file, which gets installed on every during init. +    # Afterwards, puppet installs an authorized keys file that is generated differently +    # (see authorized_keys() in macros.rb) +    # +    def update_authorized_keys +      buffer = StringIO.new +      keys = Dir.glob(path([:user_ssh, '*'])) +      if keys.empty? +        bail! "You must have at least one public SSH user key configured in order to proceed. See `leap help add-user`." +      end +      if file_exists?(path(:monitor_pub_key)) +        keys << path(:monitor_pub_key) +      end +      keys.sort.each do |keyfile| +        ssh_type, ssh_key = File.read(keyfile).strip.split(" ") +        buffer << ssh_type +        buffer << " " +        buffer << ssh_key +        buffer << " " +        buffer << Path.relative_path(keyfile) +        buffer << "\n" +      end +      write_file!(:authorized_keys, buffer.string) +    end + +    # +    # generates the known_hosts file. +    # +    # we do a 'late' binding on the hostnames and ip part of the ssh pub key record in order to allow +    # for the possibility that the hostnames or ip has changed in the node configuration. +    # +    def update_known_hosts +      buffer = StringIO.new +      buffer << "#\n" +      buffer << "# This file is automatically generated by the command `leap`. You should NOT modify this file.\n" +      buffer << "# Instead, rerun `leap node init` on whatever node is causing SSH problems.\n" +      buffer << "#\n" +      manager.nodes.keys.sort.each do |node_name| +        node = manager.nodes[node_name] +        hostnames = [node.name, node.domain.internal, node.domain.full, node.ip_address].join(',') +        pub_key = read_file([:node_ssh_pub_key,node.name]) +        if pub_key +          buffer << [hostnames, pub_key].join(' ') +          buffer << "\n" +        end +      end +      write_file!(:known_hosts, buffer.string) +    end + +    ## +    ## provider.json +    ## + +    # +    # generates static provider.json files that can put into place +    # (e.g. https://domain/provider.json) for the cases where the +    # webapp domain does not match the provider's domain. +    # +    def compile_provider_json(environments=nil) +      webapp_nodes = manager.nodes[:services => 'webapp'] +      write_file!(:static_web_readme, STATIC_WEB_README) +      environments ||= manager.environment_names +      environments.each do |env| +        node = webapp_nodes[:environment => env].values.first +        if node +          env ||= 'default' +          write_file!( +            [:static_web_provider_json, env], +            node['definition_files']['provider'] +          ) +          write_file!( +            [:static_web_htaccess, env], +            HTACCESS_FILE % {:min_version => manager.env(env).provider.client_version['min']} +          ) +        end +      end +    end + +    HTACCESS_FILE = %[ +<Files provider.json> +  Header set X-Minimum-Client-Version %{min_version} +</Files> +] + +    STATIC_WEB_README = %[ +This directory contains statically rendered copies of the `provider.json` file +used by the client to "bootstrap" configure itself for use with your service +provider. + +There is a separate provider.json file for each environment, although you +should only need 'production/provider.json' or, if you have no environments +configured, 'default/provider.json'. + +To clarify, this is the public `provider.json` file used by the client, not the +`provider.json` file that is used to configure the provider. + +The provider.json file must be available at `https://domain/provider.json` +(unless this provider is included in the list of providers which are pre- +seeded in client). + +This provider.json file can be served correctly in one of three ways: + +(1) If the property webapp.domain is not configured, then the web app will be +    installed at https://domain/ and it will handle serving the provider.json file. + +(2) If one or more nodes have the 'static' service configured for the provider's +    domain, then these 'static' nodes will correctly serve provider.json. + +(3) Otherwise, you must copy the provider.json file to your web +    server and make it available at '/provider.json'. The example htaccess +    file shows what header options should be sent by the web server +    with the response. + +This directory is needed for method (3), but not for methods (1) or (2). + +This directory has been created by the command `leap compile provider.json`. +Once created, it will be kept up to date everytime you compile. You may safely +remove this directory if you don't use it. +] + +    ## +    ## +    ## ZONE FILE +    ## + +    def relative_hostname(fqdn) +      @domain_regexp ||= /\.?#{Regexp.escape(provider.domain)}$/ +      fqdn.sub(@domain_regexp, '') +    end + +    # +    # serial is any number less than 2^32 (4294967296) +    # +    def compile_zone_file +      hosts_seen = {} +      f = $stdout +      f.puts ZONE_HEADER % {:domain => provider.domain, :ns => provider.domain, :contact => provider.contacts.default.first.sub('@','.')} +      max_width = manager.nodes.values.inject(0) {|max, node| [max, relative_hostname(node.domain.full).length].max } +      put_line = lambda do |host, line| +        host = '@' if host == '' +        f.puts("%-#{max_width}s %s" % [host, line]) +      end + +      f.puts ORIGIN_HEADER +      # 'A' records for primary domain +      manager.nodes[:environment => '!local'].each_node do |node| +        if node.dns['aliases'] && node.dns.aliases.include?(provider.domain) +          put_line.call "", "IN A      #{node.ip_address}" +        end +      end + +      # NS records +      if provider['dns'] && provider.dns['nameservers'] +        provider.dns.nameservers.each do |ns| +          put_line.call "", "IN NS #{ns}." +        end +      end + +      # all other records +      manager.environment_names.each do |env| +        next if env == 'local' +        nodes = manager.nodes[:environment => env] +        next unless nodes.any? +        f.puts ENV_HEADER % (env.nil? ? 'default' : env) +        nodes.each_node do |node| +          if node.dns.public +            hostname = relative_hostname(node.domain.full) +            put_line.call relative_hostname(node.domain.full), "IN A      #{node.ip_address}" +          end +          if node.dns['aliases'] +            node.dns.aliases.each do |host_alias| +              if host_alias != node.domain.full && host_alias != provider.domain +                put_line.call relative_hostname(host_alias), "IN A      #{node.ip_address}" +              end +            end +          end +          if node.services.include? 'mx' +            put_line.call relative_hostname(node.domain.full_suffix), "IN MX 10  #{relative_hostname(node.domain.full)}" +          end +        end +      end +    end + +    ENV_HEADER = %[ +;; +;; ENVIRONMENT %s +;; + +] + +    ZONE_HEADER = %[ +;; +;; BIND data file for %{domain} +;; + +$TTL 600 +$ORIGIN %{domain}. + +@ IN SOA %{ns}. %{contact}. ( +  0000          ; serial +  7200          ; refresh (  24 hours) +  3600          ; retry   (   2 hours) +  1209600       ; expire  (1000 hours) +  600 )         ; minimum (   2 days) +; +] + +    ORIGIN_HEADER = %[ +;; +;; ZONE ORIGIN +;; + +] + +    ## +    ## FIREWALL +    ## + +    def compile_firewall +      manager.nodes.each_node(&:evaluate) + +      rules = [["ALLOW TO", "PORTS", "ALLOW FROM"]] +      manager.nodes[:environment => '!local'].values.each do |node| +        next unless node['firewall'] +        node.firewall.each do |name, rule| +          if rule.is_a? Hash +            rules << add_rule(rule) +          elsif rule.is_a? Array +            rule.each do |r| +              rules << add_rule(r) +            end +          end +        end +      end + +      max_to    = rules.inject(0) {|max, r| [max, r[0].length].max} +      max_port  = rules.inject(0) {|max, r| [max, r[1].length].max} +      max_from  = rules.inject(0) {|max, r| [max, r[2].length].max} +      rules.each do |rule| +        puts "%-#{max_to}s   %-#{max_port}s   %-#{max_from}s" % rule +      end +    end + +    private + +    def add_rule(rule) +      [rule["to"], [rule["port"]].compact.join(','), rule["from"]] +    end + +  end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/db.rb b/lib/leap_cli/commands/db.rb new file mode 100644 index 00000000..e4fd3858 --- /dev/null +++ b/lib/leap_cli/commands/db.rb @@ -0,0 +1,65 @@ +module LeapCli; module Commands + +  desc 'Database commands.' +  command :db do |db| +    db.desc 'Destroy one or more databases. If present, limit to FILTER nodes. For example `leap db destroy --db sessions,tokens testing`.' +    db.arg_name 'FILTER', :optional => true +    db.command :destroy do |destroy| +      destroy.flag :db, :arg_name => "DATABASES", :desc => 'Comma separated list of databases to destroy (no space). Use "--db all" to destroy all databases.', :optional => false +      destroy.action do |global_options,options,args| +        dbs = (options[:db]||"").split(',') +        bail!('No databases specified') if dbs.empty? +        nodes = manager.filter(args) +        if nodes.any? +          nodes = nodes[:services => 'couchdb'] +        end +        if nodes.any? +          unless global_options[:yes] +            if dbs.include?('all') +              say 'You are about to permanently destroy all database data for nodes [%s].' % nodes.keys.join(', ') +            else +              say 'You are about to permanently destroy databases [%s] for nodes [%s].' % [dbs.join(', '), nodes.keys.join(', ')] +            end +            bail! unless agree("Continue? ") +          end +          if dbs.include?('all') +            destroy_all_dbs(nodes) +          else +            destroy_dbs(nodes, dbs) +          end +          say 'You must run `leap deploy` in order to create the databases again.' +        else +          say 'No nodes' +        end +      end +    end +  end + +  private + +  def destroy_all_dbs(nodes) +    ssh_connect(nodes) do |ssh| +      ssh.run('/etc/init.d/bigcouch stop && test ! -z "$(ls /opt/bigcouch/var/lib/ 2> /dev/null)" && rm -r /opt/bigcouch/var/lib/* && echo "db destroyed" || echo "db already destroyed"') +      ssh.run('grep ^seq_dir /etc/leap/tapicero.yaml | cut -f2 -d\" | xargs rm -rv') +    end +  end + +  def destroy_dbs(nodes, dbs) +    nodes.each_node do |node| +      ssh_connect(node) do |ssh| +        dbs.each do |db| +          ssh.run(DESTROY_DB_COMMAND % {:db => db}) +        end +      end +    end +  end + +  DESTROY_DB_COMMAND = %{ +if [ 200 = `curl -ns -w "%%{http_code}" -X GET "127.0.0.1:5984/%{db}" -o /dev/null` ]; then +  echo "Result from DELETE /%{db}:" `curl -ns -X DELETE "127.0.0.1:5984/%{db}"`; +else +  echo "Skipping db '%{db}': it does not exist or has already been deleted."; +fi +} + +end; end diff --git a/lib/leap_cli/commands/deploy.rb b/lib/leap_cli/commands/deploy.rb new file mode 100644 index 00000000..c2a70afa --- /dev/null +++ b/lib/leap_cli/commands/deploy.rb @@ -0,0 +1,368 @@ +require 'etc' + +module LeapCli +  module Commands + +    desc 'Apply recipes to a node or set of nodes.' +    long_desc 'The FILTER can be the name of a node, service, or tag.' +    arg_name 'FILTER' +    command [:deploy, :d] do |c| + +      c.switch :fast, :desc => 'Makes the deploy command faster by skipping some slow steps. A "fast" deploy can be used safely if you recently completed a normal deploy.', +                      :negatable => false + +      c.switch :sync, :desc => "Sync files, but don't actually apply recipes.", :negatable => false + +      c.switch :force, :desc => 'Deploy even if there is a lockfile.', :negatable => false + +      c.switch :downgrade, :desc => 'Allows deploy to run with an older platform version.', :negatable => false + +      c.switch :dev, :desc => "Development mode: don't run 'git submodule update' before deploy.", :negatable => false + +      c.flag :tags, :desc => 'Specify tags to pass through to puppet (overriding the default).', +                    :arg_name => 'TAG[,TAG]' + +      c.flag :port, :desc => 'Override the default SSH port.', +                    :arg_name => 'PORT' + +      c.flag :ip,   :desc => 'Override the default SSH IP address.', +                    :arg_name => 'IPADDRESS' + +      c.action do |global,options,args| + +        if options[:dev] != true +          init_submodules +        end + +        nodes = manager.filter!(args, :disabled => false) +        if nodes.size > 1 +          say "Deploying to these nodes: #{nodes.keys.join(', ')}" +          if !global[:yes] && !agree("Continue? ") +            quit! "OK. Bye." +          end +        end + +        environments = nodes.field('environment').uniq +        if environments.empty? +          environments = [nil] +        end +        environments.each do |env| +          check_platform_pinning(env, global) +        end +        # compile hiera files for all the nodes in every environment that is +        # being deployed and only those environments. +        compile_hiera_files(manager.filter(environments), false) +        # update server certificates if needed +        update_certificates(nodes) + +        ssh_connect(nodes, connect_options(options)) do |ssh| +          ssh.leap.log :checking, 'node' do +            ssh.leap.check_for_no_deploy +            ssh.leap.assert_initialized +          end +          ssh.leap.log :synching, "configuration files" do +            sync_hiera_config(ssh) +            sync_support_files(ssh) +          end +          ssh.leap.log :synching, "puppet manifests" do +            sync_puppet_files(ssh) +          end +          unless options[:sync] +            ssh.leap.log :applying, "puppet" do +              ssh.puppet.apply(:verbosity => [LeapCli.log_level,5].min, +                :tags => tags(options), +                :force => options[:force], +                :info => deploy_info, +                :downgrade => options[:downgrade] +              ) +            end +          end +        end +        if !Util.exit_status.nil? && Util.exit_status != 0 +          log :warning, "puppet did not finish successfully." +        end +      end +    end + +    desc 'Display recent deployment history for a set of nodes.' +    long_desc 'The FILTER can be the name of a node, service, or tag.' +    arg_name 'FILTER' +    command [:history, :h] do |c| +      c.flag :port, :desc => 'Override the default SSH port.', +                    :arg_name => 'PORT' +      c.flag :ip,   :desc => 'Override the default SSH IP address.', +                    :arg_name => 'IPADDRESS' +      c.action do |global,options,args| +        nodes = manager.filter!(args) +        ssh_connect(nodes, connect_options(options)) do |ssh| +          ssh.leap.history +        end +      end +    end + +    private + +    def forcible_prompt(forced, msg, prompt) +      say(msg) +      if forced +        log :warning, "continuing anyway because of --force" +      else +        say "hint: use --force to skip this prompt." +        quit!("OK. Bye.") unless agree(prompt) +      end +    end + +    # +    # The currently activated provider.json could have loaded some pinning +    # information for the platform. If this is the case, refuse to deploy +    # if there is a mismatch. +    # +    # For example: +    # +    # "platform": { +    #   "branch": "develop" +    #   "version": "1.0..99" +    #   "commit": "e1d6280e0a8c565b7fb1a4ed3969ea6fea31a5e2..HEAD" +    # } +    # +    def check_platform_pinning(environment, global_options) +      provider = manager.env(environment).provider +      return unless provider['platform'] + +      if environment.nil? || environment == 'default' +        provider_json = 'provider.json' +      else +        provider_json = 'provider.' + environment + '.json' +      end + +      # can we have json schema verification already? +      unless provider.platform.is_a? Hash +        bail!('`platform` attribute in #{provider_json} must be a hash (was %s).' % provider.platform.inspect) +      end + +      # check version +      if provider.platform['version'] +        if !Leap::Platform.version_in_range?(provider.platform.version) +          forcible_prompt( +            global_options[:force], +            "The platform is pinned to a version range of '#{provider.platform.version}' "+ +              "by the `platform.version` property in #{provider_json}, but the platform "+ +              "(#{Path.platform}) has version #{Leap::Platform.version}.", +            "Do you really want to deploy from the wrong version? " +          ) +        end +      end + +      # check branch +      if provider.platform['branch'] +        if !is_git_directory?(Path.platform) +          forcible_prompt( +            global_options[:force], +            "The platform is pinned to a particular branch by the `platform.branch` property "+ +              "in #{provider_json}, but the platform directory (#{Path.platform}) is not a git repository.", +            "Do you really want to deploy anyway? " +          ) +        end +        unless provider.platform.branch == current_git_branch(Path.platform) +          forcible_prompt( +            global_options[:force], +            "The platform is pinned to branch '#{provider.platform.branch}' by the `platform.branch` property "+ +              "in #{provider_json}, but the current branch is '#{current_git_branch(Path.platform)}' " + +              "(for directory '#{Path.platform}')", +            "Do you really want to deploy from the wrong branch? " +          ) +        end +      end + +      # check commit +      if provider.platform['commit'] +        if !is_git_directory?(Path.platform) +          forcible_prompt( +            global_options[:force], +            "The platform is pinned to a particular commit range by the `platform.commit` property "+ +              "in #{provider_json}, but the platform directory (#{Path.platform}) is not a git repository.", +            "Do you really want to deploy anyway? " +          ) +        end +        current_commit = current_git_commit(Path.platform) +        Dir.chdir(Path.platform) do +          commit_range = assert_run!("git log --pretty='format:%H' '#{provider.platform.commit}'", +            "The platform is pinned to a particular commit range by the `platform.commit` property "+ +            "in #{provider_json}, but git was not able to find commits in the range specified "+ +            "(#{provider.platform.commit}).") +          commit_range = commit_range.split("\n") +          if !commit_range.include?(current_commit) && +              provider.platform.commit.split('..').first != current_commit +            forcible_prompt( +              global_options[:force], +              "The platform is pinned via the `platform.commit` property in #{provider_json} " + +                "to a commit in the range #{provider.platform.commit}, but the current HEAD " + +                "(#{current_commit}) is not in that range.", +              "Do you really want to deploy from the wrong commit? " +            ) +          end +        end +      end +    end + +    def sync_hiera_config(ssh) +      ssh.rsync.update do |server| +        node = manager.node(server.host) +        hiera_file = Path.relative_path([:hiera, node.name]) +        ssh.leap.log hiera_file + ' -> ' + node.name + ':' + Leap::Platform.hiera_path +        { +          :source => hiera_file, +          :dest => Leap::Platform.hiera_path, +          :flags => "-rltp --chmod=u+rX,go-rwx" +        } +      end +    end + +    # +    # sync various support files. +    # +    def sync_support_files(ssh) +      dest_dir = Leap::Platform.files_dir +      custom_files = build_custom_file_list +      ssh.rsync.update do |server| +        node = manager.node(server.host) +        files_to_sync = node.file_paths.collect {|path| Path.relative_path(path, Path.provider) } +        files_to_sync += custom_files +        if files_to_sync.any? +          ssh.leap.log(files_to_sync.join(', ') + ' -> ' + node.name + ':' + dest_dir) +          { +            :chdir => Path.named_path(:files_dir), +            :source => ".", +            :dest => dest_dir, +            :excludes => "*", +            :includes => calculate_includes_from_files(files_to_sync, '/files'), +            :flags => "-rltp --chmod=u+rX,go-rwx --relative --delete --delete-excluded --copy-links" +          } +        else +          nil +        end +      end +    end + +    def sync_puppet_files(ssh) +      ssh.rsync.update do |server| +        ssh.leap.log(Path.platform + '/[bin,tests,puppet] -> ' + server.host + ':' + Leap::Platform.leap_dir) +        { +          :dest => Leap::Platform.leap_dir, +          :source => '.', +          :chdir => Path.platform, +          :excludes => '*', +          :includes => ['/bin', '/bin/**', '/puppet', '/puppet/**', '/tests', '/tests/**'], +          :flags => "-rlt --relative --delete --copy-links" +        } +      end +    end + +    # +    # ensure submodules are up to date, if the platform is a git +    # repository. +    # +    def init_submodules +      return unless is_git_directory?(Path.platform) +      Dir.chdir Path.platform do +        assert_run! "git submodule sync" +        statuses = assert_run! "git submodule status" +        statuses.strip.split("\n").each do |status_line| +          if status_line =~ /^[\+-]/ +            submodule = status_line.split(' ')[1] +            log "Updating submodule #{submodule}" +            assert_run! "git submodule update --init #{submodule}" +          end +        end +      end +    end + +    # +    # converts an array of file paths into an array +    # suitable for --include of rsync +    # +    # if set, `prefix` is stripped off. +    # +    def calculate_includes_from_files(files, prefix=nil) +      return nil unless files and files.any? + +      # prepend '/' (kind of like ^ for rsync) +      includes = files.collect {|file| file =~ /^\// ? file : '/' + file } + +      # include all sub files of specified directories +      includes.size.times do |i| +        if includes[i] =~ /\/$/ +          includes << includes[i] + '**' +        end +      end + +      # include all parent directories (required because of --exclude '*') +      includes.size.times do |i| +        path = File.dirname(includes[i]) +        while(path != '/') +          includes << path unless includes.include?(path) +          path = File.dirname(path) +        end +      end + +      if prefix +        includes.map! {|path| path.sub(/^#{Regexp.escape(prefix)}\//, '/')} +      end + +      return includes +    end + +    def tags(options) +      if options[:tags] +        tags = options[:tags].split(',') +      else +        tags = Leap::Platform.default_puppet_tags.dup +      end +      tags << 'leap_slow' unless options[:fast] +      tags.join(',') +    end + +    # +    # a provider might have various customization files that should be sync'ed to the server. +    # this method builds that list of files to sync. +    # +    def build_custom_file_list +      custom_files = [] +      Leap::Platform.paths.keys.grep(/^custom_/).each do |path| +        if file_exists?(path) +          relative_path = Path.relative_path(path, Path.provider) +          if dir_exists?(path) +            custom_files << relative_path + '/' # rsync needs trailing slash +          else +            custom_files << relative_path +          end +        end +      end +      return custom_files +    end + +    def deploy_info +      info = [] +      info << "user: %s" % Etc.getpwuid(Process.euid).name +      if is_git_directory?(Path.platform) && current_git_branch(Path.platform) != 'master' +        info << "platform: %s (%s %s)" % [ +          Leap::Platform.version, +          current_git_branch(Path.platform), +          current_git_commit(Path.platform)[0..4] +        ] +      else +        info << "platform: %s" % Leap::Platform.version +      end +      if is_git_directory?(LEAP_CLI_BASE_DIR) +        info << "leap_cli: %s (%s %s)" % [ +          LeapCli::VERSION, +          current_git_branch(LEAP_CLI_BASE_DIR), +          current_git_commit(LEAP_CLI_BASE_DIR)[0..4] +        ] +      else +        info << "leap_cli: %s" % LeapCli::VERSION +      end +      info.join(', ') +    end +  end +end diff --git a/lib/leap_cli/commands/env.rb b/lib/leap_cli/commands/env.rb new file mode 100644 index 00000000..80be2174 --- /dev/null +++ b/lib/leap_cli/commands/env.rb @@ -0,0 +1,76 @@ +module LeapCli +  module Commands + +    desc "Manipulate and query environment information." +    long_desc "The 'environment' node property can be used to isolate sets of nodes into entirely separate environments. "+ +      "A node in one environment will never interact with a node from another environment. "+ +      "Environment pinning works by modifying your ~/.leaprc file and is dependent on the "+ +      "absolute file path of your provider directory (pins don't apply if you move the directory)" +    command [:env, :e] do |c| +      c.desc "List the available environments. The pinned environment, if any, will be marked with '*'. Will also set the pin if run with an environment argument." +      c.arg_name 'ENVIRONMENT', :optional => true +      c.command :ls do |ls| +        ls.action do |global_options, options, args| +          environment = get_env_from_args(args) +          if environment +            pin(environment) +            LeapCli.leapfile.load +          end +          print_envs +        end +      end + +      c.desc 'Pin the environment to ENVIRONMENT. All subsequent commands will only apply to nodes in this environment.' +      c.arg_name 'ENVIRONMENT' +      c.command :pin do |pin| +        pin.action do |global_options,options,args| +          environment = get_env_from_args(args) +          if environment +            pin(environment) +          else +            bail! "There is no environment `#{environment}`" +          end +        end +      end + +      c.desc "Unpin the environment. All subsequent commands will apply to all nodes." +      c.command :unpin do |unpin| +        unpin.action do |global_options, options, args| +          LeapCli.leapfile.unset('environment') +          log 0, :saved, "~/.leaprc, removing environment property." +        end +      end + +      c.default_command :ls +    end + +    protected + +    def get_env_from_args(args) +      environment = args.first +      if environment == 'default' || (environment && manager.environment_names.include?(environment)) +        return environment +      else +        return nil +      end +    end + +    def pin(environment) +      LeapCli.leapfile.set('environment', environment) +      log 0, :saved, "~/.leaprc with environment set to #{environment}." +    end + +    def print_envs +      envs = ["default"] + manager.environment_names.compact.sort +      envs.each do |env| +        if env +          if LeapCli.leapfile.environment == env +            puts "* #{env}" +          else +            puts "  #{env}" +          end +        end +      end +    end +  end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/facts.rb b/lib/leap_cli/commands/facts.rb new file mode 100644 index 00000000..11329ccc --- /dev/null +++ b/lib/leap_cli/commands/facts.rb @@ -0,0 +1,100 @@ +# +# Gather facter facts +# + +module LeapCli; module Commands + +  desc 'Gather information on nodes.' +  command :facts do |facts| +    facts.desc 'Query servers to update facts.json.' +    facts.long_desc "Queries every node included in FILTER and saves the important information to facts.json" +    facts.arg_name 'FILTER' +    facts.command :update do |update| +      update.action do |global_options,options,args| +        update_facts(global_options, options, args) +      end +    end +  end + +  protected + +  def facter_cmd +    'facter --json ' + Leap::Platform.facts.join(' ') +  end + +  def remove_node_facts(name) +    if file_exists?(:facts) +      update_facts_file({name => nil}) +    end +  end + +  def update_node_facts(name, facts) +    update_facts_file({name => facts}) +  end + +  def rename_node_facts(old_name, new_name) +    if file_exists?(:facts) +      facts = JSON.parse(read_file(:facts) || {}) +      facts[new_name] = facts[old_name] +      facts[old_name] = nil +      update_facts_file(facts, true) +    end +  end + +  # +  # if overwrite = true, then ignore existing facts.json. +  # +  def update_facts_file(new_facts, overwrite=false) +    replace_file!(:facts) do |content| +      if overwrite || content.nil? || content.empty? +        old_facts = {} +      else +        old_facts = manager.facts +      end +      facts = old_facts.merge(new_facts) +      facts.each do |name, value| +        if value.is_a? String +          if value == "" +            value = nil +          else +            value = JSON.parse(value) rescue JSON::ParserError +          end +        end +        if value.is_a? Hash +          value.delete_if {|key,v| v.nil?} +        end +        facts[name] = value +      end +      facts.delete_if do |name, value| +        value.nil? || value.empty? +      end +      if facts.empty? +        "{}\n" +      else +        JSON.sorted_generate(facts) + "\n" +      end +    end +  end + +  private + +  def update_facts(global_options, options, args) +    nodes = manager.filter(args, :local => false, :disabled => false) +    new_facts = {} +    ssh_connect(nodes) do |ssh| +      ssh.leap.run_with_progress(facter_cmd) do |response| +        node = manager.node(response[:host]) +        if node +          new_facts[node.name] = response[:data].strip +        else +          log :warning, 'Could not find node for hostname %s' % response[:host] +        end +      end +    end +    # only overwrite the entire facts file if and only if we are gathering facts +    # for all nodes in all environments. +    overwrite_existing = args.empty? && LeapCli.leapfile.environment.nil? +    update_facts_file(new_facts, overwrite_existing) +  end + +end; end
\ No newline at end of file diff --git a/lib/leap_cli/commands/inspect.rb b/lib/leap_cli/commands/inspect.rb new file mode 100644 index 00000000..20654fa7 --- /dev/null +++ b/lib/leap_cli/commands/inspect.rb @@ -0,0 +1,144 @@ +module LeapCli; module Commands + +  desc 'Prints details about a file. Alternately, the argument FILE can be the name of a node, service or tag.' +  arg_name 'FILE' +  command [:inspect, :i] do |c| +    c.switch 'base', :desc => 'Inspect the FILE from the provider_base (i.e. without local inheritance).', :negatable => false +    c.action do |global_options,options,args| +      object = args.first +      assert! object, 'A file path or node/service/tag name is required' +      method = inspection_method(object) +      if method && defined?(method) +        self.send(method, object, options) +      else +        log "Sorry, I don't know how to inspect that." +      end +    end +  end + +  private + +  FTYPE_MAP = { +    "PEM certificate"         => :inspect_x509_cert, +    "PEM RSA private key"     => :inspect_x509_key, +    "OpenSSH RSA public key"  => :inspect_ssh_pub_key, +    "PEM certificate request" => :inspect_x509_csr +  } + +  def inspection_method(object) +    if File.exists?(object) +      ftype = `file #{object}`.split(':').last.strip +      log 2, "file is of type '#{ftype}'" +      if FTYPE_MAP[ftype] +        FTYPE_MAP[ftype] +      elsif File.extname(object) == ".json" +        full_path = File.expand_path(object, Dir.pwd) +        if path_match?(:node_config, full_path) +          :inspect_node +        elsif path_match?(:service_config, full_path) +          :inspect_service +        elsif path_match?(:tag_config, full_path) +          :inspect_tag +        elsif path_match?(:provider_config, full_path) || path_match?(:provider_env_config, full_path) +          :inspect_provider +        elsif path_match?(:common_config, full_path) +          :inspect_common +        else +          nil +        end +      end +    elsif manager.nodes[object] +      :inspect_node +    elsif manager.services[object] +      :inspect_service +    elsif manager.tags[object] +      :inspect_tag +    elsif object == "common" +      :inspect_common +    elsif object == "provider" +      :inspect_provider +    else +      nil +    end +  end + +  # +  # inspectors +  # + +  def inspect_x509_key(file_path, options) +    assert_bin! 'openssl' +    puts assert_run! 'openssl rsa -in %s -text -check' % file_path +  end + +  def inspect_x509_cert(file_path, options) +    assert_bin! 'openssl' +    puts assert_run! 'openssl x509 -in %s -text -noout' % file_path +    log 0, :"SHA256 fingerprint", X509.fingerprint("SHA256", file_path) +  end + +  def inspect_x509_csr(file_path, options) +    assert_bin! 'openssl' +    puts assert_run! 'openssl req -text -noout -verify -in %s' % file_path +  end + +  #def inspect_ssh_pub_key(file_path) +  #end + +  def inspect_node(arg, options) +    inspect_json manager.nodes[name(arg)] +  end + +  def inspect_service(arg, options) +    if options[:base] +      inspect_json manager.base_services[name(arg)] +    else +      inspect_json manager.services[name(arg)] +    end +  end + +  def inspect_tag(arg, options) +    if options[:base] +      inspect_json manager.base_tags[name(arg)] +    else +      inspect_json manager.tags[name(arg)] +    end +  end + +  def inspect_provider(arg, options) +    if options[:base] +      inspect_json manager.base_provider +    elsif arg =~ /provider\.(.*)\.json/ +      inspect_json manager.env($1).provider +    else +      inspect_json manager.provider +    end +  end + +  def inspect_common(arg, options) +    if options[:base] +      inspect_json manager.base_common +    else +      inspect_json manager.common +    end +  end + +  # +  # helpers +  # + +  def name(arg) +    File.basename(arg).sub(/\.json$/, '') +  end + +  def inspect_json(config) +    if config +      puts JSON.sorted_generate(config) +    end +  end + +  def path_match?(path_symbol, path) +    Dir.glob(Path.named_path([path_symbol, '*'])).include?(path) +  end + +end; end diff --git a/lib/leap_cli/commands/list.rb b/lib/leap_cli/commands/list.rb new file mode 100644 index 00000000..c562b59b --- /dev/null +++ b/lib/leap_cli/commands/list.rb @@ -0,0 +1,132 @@ +require 'command_line_reporter' + +module LeapCli; module Commands + +  desc 'List nodes and their classifications' +  long_desc 'Prints out a listing of nodes, services, or tags. ' + +            'If present, the FILTER can be a list of names of nodes, services, or tags. ' + +            'If the name is prefixed with +, this acts like an AND condition. ' + +            "For example:\n\n" + +            "`leap list node1 node2` matches all nodes named \"node1\" OR \"node2\"\n\n" + +            "`leap list openvpn +local` matches all nodes with service \"openvpn\" AND tag \"local\"" + +  arg_name 'FILTER', :optional => true +  command [:list,:ls] do |c| +    c.flag 'print', :desc => 'What attributes to print (optional)' +    c.switch 'disabled', :desc => 'Include disabled nodes in the list.', :negatable => false +    c.action do |global_options,options,args| +      # don't rely on default manager(), because we want to pass custom options to load() +      manager = LeapCli::Config::Manager.new +      if global_options[:color] +        colors = ['cyan', 'white'] +      else +        colors = [nil, nil] +      end +      puts +      manager.load(:include_disabled => options['disabled'], :continue_on_error => true) +      if options['print'] +        print_node_properties(manager.filter(args), options['print']) +      else +        if args.any? +          NodeTable.new(manager.filter(args), colors).run +        else +          environment = LeapCli.leapfile.environment || '_all_' +          TagTable.new('SERVICES', manager.env(environment).services, colors).run +          TagTable.new('TAGS', manager.env(environment).tags, colors).run +          NodeTable.new(manager.filter(), colors).run +        end +      end +    end +  end + +  private + +  def self.print_node_properties(nodes, properties) +    properties = properties.split(',') +    max_width = nodes.keys.inject(0) {|max,i| [i.size,max].max} +    nodes.each_node do |node| +      value = properties.collect{|prop| +        prop_value = node[prop] +        if prop_value.nil? +          "null" +        elsif prop_value == "" +          "empty" +        elsif prop_value.is_a? LeapCli::Config::Object +          node[prop].dump_json(:compact) # TODO: add option of getting pre-evaluation values. +        else +          prop_value.to_s +        end +      }.join(', ') +      printf("%#{max_width}s  %s\n", node.name, value) +    end +    puts +  end + +  class TagTable +    include CommandLineReporter +    def initialize(heading, tag_list, colors) +      @heading = heading +      @tag_list = tag_list +      @colors = colors +    end +    def run +      tags = @tag_list.keys.select{|tag| tag !~ /^_/}.sort # sorted list of tags, excluding _partials +      max_width = [20, (tags+[@heading]).inject(0) {|max,i| [i.size,max].max}].max +      table :border => false do +        row :color => @colors[0]  do +          column @heading, :align => 'right', :width => max_width +          column "NODES", :width => HighLine::SystemExtensions.terminal_size.first - max_width - 2, :padding => 2 +        end +        tags.each do |tag| +          next if @tag_list[tag].node_list.empty? +          row :color => @colors[1] do +            column tag +            column @tag_list[tag].node_list.keys.sort.join(', ') +          end +        end +      end +      vertical_spacing +    end +  end + +  # +  # might be handy: HighLine::SystemExtensions.terminal_size.first +  # +  class NodeTable +    include CommandLineReporter +    def initialize(node_list, colors) +      @node_list = node_list +      @colors = colors +    end +    def run +      rows = @node_list.keys.sort.collect do |node_name| +        [node_name, @node_list[node_name].services.sort.join(', '), @node_list[node_name].tags.sort.join(', ')] +      end +      unless rows.any? +        puts Paint["no results", :red] +        puts +        return +      end +      padding = 2 +      max_node_width    = [20, (rows.map{|i|i[0]} + ["NODES"]   ).inject(0) {|max,i| [i.size,max].max}].max +      max_service_width = (rows.map{|i|i[1]} + ["SERVICES"]).inject(0) {|max,i| [i.size+padding+padding,max].max} +      max_tag_width     = (rows.map{|i|i[2]} + ["TAGS"]    ).inject(0) {|max,i| [i.size,max].max} +      table :border => false do +        row :color => @colors[0]  do +          column "NODES", :align => 'right', :width => max_node_width +          column "SERVICES", :width => max_service_width, :padding => 2 +          column "TAGS", :width => max_tag_width +        end +        rows.each do |r| +          row :color => @colors[1] do +            column r[0] +            column r[1] +            column r[2] +          end +        end +      end +      vertical_spacing +    end +  end + +end; end diff --git a/lib/leap_cli/commands/node.rb b/lib/leap_cli/commands/node.rb new file mode 100644 index 00000000..12d6b49d --- /dev/null +++ b/lib/leap_cli/commands/node.rb @@ -0,0 +1,165 @@ +# +# fyi: the `node init` command lives in node_init.rb, +#      but all other `node x` commands live here. +# + +autoload :IPAddr, 'ipaddr' + +module LeapCli; module Commands + +  ## +  ## COMMANDS +  ## + +  desc 'Node management' +  command [:node, :n] do |node| +    node.desc 'Create a new configuration file for a node named NAME.' +    node.long_desc ["If specified, the optional argument SEED can be used to seed values in the node configuration file.", +                    "The format is property_name:value.", +                    "For example: `leap node add web1 ip_address:1.2.3.4 services:webapp`.", +                    "To set nested properties, property name can contain '.', like so: `leap node add web1 ssh.port:44`", +                    "Separeate multiple values for a single property with a comma, like so: `leap node add mynode services:webapp,dns`"].join("\n\n") +    node.arg_name 'NAME [SEED]' # , :optional => false, :multiple => false +    node.command :add do |add| +      add.switch :local, :desc => 'Make a local testing node (by automatically assigning the next available local IP address). Local nodes are run as virtual machines on your computer.', :negatable => false +      add.action do |global_options,options,args| +        # argument sanity checks +        name = args.first +        assert_valid_node_name!(name, options[:local]) +        assert_files_missing! [:node_config, name] + +        # create and seed new node +        node = Config::Node.new(manager) +        if options[:local] +          node['ip_address'] = pick_next_vagrant_ip_address +        end +        seed_node_data(node, args[1..-1]) +        validate_ip_address(node) +        begin +          write_file! [:node_config, name], node.dump_json + "\n" +          node['name'] = name +          if file_exists? :ca_cert, :ca_key +            generate_cert_for_node(manager.reload_node!(node)) +          end +        rescue LeapCli::ConfigError => exc +          remove_node_files(name) +        end +      end +    end + +    node.desc 'Renames a node file, and all its related files.' +    node.arg_name 'OLD_NAME NEW_NAME' +    node.command :mv do |mv| +      mv.action do |global_options,options,args| +        node = get_node_from_args(args) +        new_name = args.last +        assert_valid_node_name!(new_name, node.vagrant?) +        ensure_dir [:node_files_dir, new_name] +        Leap::Platform.node_files.each do |path| +          rename_file! [path, node.name], [path, new_name] +        end +        remove_directory! [:node_files_dir, node.name] +        rename_node_facts(node.name, new_name) +      end +    end + +    node.desc 'Removes all the files related to the node named NAME.' +    node.arg_name 'NAME' #:optional => false #, :multiple => false +    node.command :rm do |rm| +      rm.action do |global_options,options,args| +        node = get_node_from_args(args) +        remove_node_files(node.name) +        if node.vagrant? +          vagrant_command("destroy --force", [node.name]) +        end +        remove_node_facts(node.name) +      end +    end +  end + +  ## +  ## PUBLIC HELPERS +  ## + +  def get_node_from_args(args, options={}) +    node_name = args.first +    node = manager.node(node_name) +    if node.nil? && options[:include_disabled] +      node = manager.disabled_node(node_name) +    end +    assert!(node, "Node '#{node_name}' not found.") +    node +  end + +  def seed_node_data(node, args) +    args.each do |seed| +      key, value = seed.split(':') +      value = format_seed_value(value) +      assert! key =~ /^[0-9a-z\._]+$/, "illegal characters used in property '#{key}'" +      if key =~ /\./ +        key_parts = key.split('.') +        final_key = key_parts.pop +        current_object = node +        key_parts.each do |key_part| +          current_object[key_part] ||= Config::Object.new +          current_object = current_object[key_part] +        end +        current_object[final_key] = value +      else +        node[key] = value +      end +    end +  end + +  def remove_node_files(node_name) +    (Leap::Platform.node_files + [:node_files_dir]).each do |path| +      remove_file! [path, node_name] +    end +  end + +  # +  # conversions: +  # +  #   "x,y,z" => ["x","y","z"] +  # +  #   "22" => 22 +  # +  #   "5.1" => 5.1 +  # +  def format_seed_value(v) +    if v =~ /,/ +      v = v.split(',') +      v.map! do |i| +        i = i.to_i if i.to_i.to_s == i +        i = i.to_f if i.to_f.to_s == i +        i +      end +    else +      v = v.to_i if v.to_i.to_s == v +      v = v.to_f if v.to_f.to_s == v +    end +    return v +  end + +  def validate_ip_address(node) +    IPAddr.new(node['ip_address']) +  rescue ArgumentError +    bail! do +      if node['ip_address'] +        log :invalid, "ip_address #{node['ip_address'].inspect}" +      else +        log :missing, "ip_address" +      end +    end +  end + +  def assert_valid_node_name!(name, local=false) +    assert! name, 'No <node-name> specified.' +    if local +      assert! name =~ /^[0-9a-z]+$/, "illegal characters used in node name '#{name}' (note: Vagrant does not allow hyphens or underscores)" +    else +      assert! name =~ /^[0-9a-z-]+$/, "illegal characters used in node name '#{name}' (note: Linux does not allow underscores)" +    end +  end + +end; end
\ No newline at end of file diff --git a/lib/leap_cli/commands/node_init.rb b/lib/leap_cli/commands/node_init.rb new file mode 100644 index 00000000..33f6288d --- /dev/null +++ b/lib/leap_cli/commands/node_init.rb @@ -0,0 +1,169 @@ +# +# Node initialization. +# Most of the fun stuff is in tasks.rb. +# + +module LeapCli; module Commands + +  desc 'Node management' +  command :node do |node| +    node.desc 'Bootstraps a node or nodes, setting up SSH keys and installing prerequisite packages' +    node.long_desc "This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, " + +                   "copying the authorized_keys file, installing packages that are required for deploying, and registering important facts. " + +                   "Node init must be run before deploying to a server, and the server must be running and available via the network. " + +                   "This command only needs to be run once, but there is no harm in running it multiple times." +    node.arg_name 'FILTER' +    node.command :init do |init| +      init.switch 'echo', :desc => 'If set, passwords are visible as you type them (default is hidden)', :negatable => false +      init.flag :port, :desc => 'Override the default SSH port.', :arg_name => 'PORT' +      init.flag :ip,   :desc => 'Override the default SSH IP address.', :arg_name => 'IPADDRESS' + +      init.action do |global,options,args| +        assert! args.any?, 'You must specify a FILTER' +        finished = [] +        manager.filter!(args).each_node do |node| +          is_node_alive(node, options) +          save_public_host_key(node, global, options) unless node.vagrant? +          update_compiled_ssh_configs +          ssh_connect_options = connect_options(options).merge({:bootstrap => true, :echo => options[:echo]}) +          ssh_connect(node, ssh_connect_options) do |ssh| +            if node.vagrant? +              ssh.install_insecure_vagrant_key +            end +            ssh.install_authorized_keys +            ssh.install_prerequisites +            unless node.vagrant? +              ssh.leap.log(:checking, "SSH host keys") do +                ssh.leap.capture(get_ssh_keys_cmd) do |response| +                  update_local_ssh_host_keys(node, response[:data]) if response[:exitcode] == 0 +                end +              end +            end +            ssh.leap.log(:updating, "facts") do +              ssh.leap.capture(facter_cmd) do |response| +                if response[:exitcode] == 0 +                  update_node_facts(node.name, response[:data]) +                else +                  log :failed, "to run facter on #{node.name}" +                end +              end +            end +          end +          finished << node.name +        end +        log :completed, "initialization of nodes #{finished.join(', ')}" +      end +    end +  end + +  private + +  ## +  ## PRIVATE HELPERS +  ## + +  def is_node_alive(node, options) +    address = options[:ip] || node.ip_address +    port = options[:port] || node.ssh.port +    log :connecting, "to node #{node.name}" +    assert_run! "nc -zw3 #{address} #{port}", +      "Failed to reach #{node.name} (address #{address}, port #{port}). You can override the configured IP address and port with --ip or --port." +  end + +  # +  # saves the public ssh host key for node into the provider directory. +  # +  # see `man sshd` for the format of known_hosts +  # +  def save_public_host_key(node, global, options) +    log :fetching, "public SSH host key for #{node.name}" +    address = options[:ip] || node.ip_address +    port = options[:port] || node.ssh.port +    host_keys = get_public_keys_for_ip(address, port) +    pub_key_path = Path.named_path([:node_ssh_pub_key, node.name]) + +    if Path.exists?(pub_key_path) +      if host_keys.include? SshKey.load(pub_key_path) +        log :trusted, "- Public SSH host key for #{node.name} matches previously saved key", :indent => 1 +      else +        bail! do +          log :error, "The public SSH host keys we just fetched for #{node.name} doesn't match what we have saved previously.", :indent => 1 +          log "Delete the file #{pub_key_path} if you really want to remove the trusted SSH host key.", :indent => 2 +        end +      end +    else +      known_key = host_keys.detect{|k|k.in_known_hosts?(node.name, node.ip_address, node.domain.name)} +      if known_key +        log :trusted, "- Public SSH host key for #{node.name} is trusted (key found in your ~/.ssh/known_hosts)" +      else +        public_key = SshKey.pick_best_key(host_keys) +        if public_key.nil? +          bail!("We got back #{host_keys.size} host keys from #{node.name}, but we can't support any of them.") +        else +          say("   This is the SSH host key you got back from node \"#{node.name}\"") +          say("   Type        -- #{public_key.bits} bit #{public_key.type.upcase}") +          say("   Fingerprint -- " + public_key.fingerprint) +          say("   Public Key  -- " + public_key.key) +          if !global[:yes] && !agree("   Is this correct? ") +            bail! +          else +            known_key = public_key +          end +        end +      end +      puts +      write_file! [:node_ssh_pub_key, node.name], known_key.to_s +    end +  end + +  # +  # Get the public host keys for a host using ssh-keyscan. +  # Return an array of SshKey objects, one for each key. +  # +  def get_public_keys_for_ip(address, port=22) +    assert_bin!('ssh-keyscan') +    output = assert_run! "ssh-keyscan -p #{port} #{address}", "Could not get the public host key from #{address}:#{port}. Maybe sshd is not running?" +    if output.empty? +      bail! :failed, "ssh-keyscan returned empty output." +    end + +    if output =~ /No route to host/ +      bail! :failed, 'ssh-keyscan: no route to %s' % address +    else +      keys = SshKey.parse_keys(output) +      if keys.empty? +        bail! "ssh-keyscan got zero host keys back (that we understand)! Output was: #{output}" +      else +        return keys +      end +    end +  end + +  # run on the server to generate a string suitable for passing to SshKey.parse_keys() +  def get_ssh_keys_cmd +    "/bin/grep ^HostKey /etc/ssh/sshd_config | /usr/bin/awk '{print $2 \".pub\"}' | /usr/bin/xargs /bin/cat" +  end + +  # +  # Sometimes the ssh host keys on the server will be better than what we have +  # stored locally. In these cases, ask the user if they want to upgrade. +  # +  def update_local_ssh_host_keys(node, remote_keys_string) +    remote_keys = SshKey.parse_keys(remote_keys_string) +    return unless remote_keys.any? +    current_key = SshKey.load(Path.named_path([:node_ssh_pub_key, node.name])) +    best_key = SshKey.pick_best_key(remote_keys) +    return unless best_key && current_key +    if current_key != best_key +      say("   One of the SSH host keys for node '#{node.name}' is better than what you currently have trusted.") +      say("     Current key: #{current_key.summary}") +      say("     Better key: #{best_key.summary}") +      if agree("   Do you want to use the better key? ") +        write_file! [:node_ssh_pub_key, node.name], best_key.to_s +      end +    else +      log(3, "current host key does not need updating") +    end +  end + +end; end diff --git a/lib/leap_cli/commands/ssh.rb b/lib/leap_cli/commands/ssh.rb new file mode 100644 index 00000000..1a81902c --- /dev/null +++ b/lib/leap_cli/commands/ssh.rb @@ -0,0 +1,220 @@ +module LeapCli; module Commands + +  desc 'Log in to the specified node with an interactive shell.' +  arg_name 'NAME' #, :optional => false, :multiple => false +  command :ssh do |c| +    c.flag 'ssh', :desc => "Pass through raw options to ssh (e.g. `--ssh '-F ~/sshconfig'`)." +    c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`.' +    c.action do |global_options,options,args| +      exec_ssh(:ssh, options, args) +    end +  end + +  desc 'Log in to the specified node with an interactive shell using mosh (requires node to have mosh.enabled set to true).' +  arg_name 'NAME' +  command :mosh do |c| +    c.flag 'ssh', :desc => "Pass through raw options to ssh (e.g. `--ssh '-F ~/sshconfig'`)." +    c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`.' +    c.action do |global_options,options,args| +      exec_ssh(:mosh, options, args) +    end +  end + +  desc 'Creates an SSH port forward (tunnel) to the node NAME. REMOTE_PORT is the port on the remote node that the tunnel will connect to. LOCAL_PORT is the optional port on your local machine. For example: `leap tunnel couch1:5984`.' +  arg_name '[LOCAL_PORT:]NAME:REMOTE_PORT' +  command :tunnel do |c| +    c.flag 'ssh', :desc => "Pass through raw options to ssh (e.g. --ssh '-F ~/sshconfig')." +    c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`.' +    c.action do |global_options,options,args| +      local_port, node, remote_port = parse_tunnel_arg(args.first) +      options[:ssh] = [options[:ssh], "-N -L 127.0.0.1:#{local_port}:0.0.0.0:#{remote_port}"].join(' ') +      log("Forward port localhost:#{local_port} to #{node.name}:#{remote_port}") +      if is_port_available?(local_port) +        exec_ssh(:ssh, options, [node.name]) +      end +    end +  end + +  desc 'Secure copy from FILE1 to FILE2. Files are specified as NODE_NAME:FILE_PATH. For local paths, omit "NODE_NAME:".' +  arg_name 'FILE1 FILE2' +  command :scp do |c| +    c.switch :r, :desc => 'Copy recursively' +    c.action do |global_options, options, args| +      if args.size != 2 +        bail!('You must specificy both FILE1 and FILE2') +      end +      from, to = args +      if (from !~ /:/ && to !~ /:/) || (from =~ /:/ && to =~ /:/) +        bail!('One FILE must be remote and the other local.') +      end +      src_node_name = src_file_path = src_node = nil +      dst_node_name = dst_file_path = dst_node = nil +      if from =~ /:/ +        src_node_name, src_file_path = from.split(':') +        src_node = get_node_from_args([src_node_name], :include_disabled => true) +        dst_file_path = to +      else +        dst_node_name, dst_file_path = to.split(':') +        dst_node = get_node_from_args([dst_node_name], :include_disabled => true) +        src_file_path = from +      end +      exec_scp(options, src_node, src_file_path, dst_node, dst_file_path) +    end +  end + +  protected + +  # +  # allow for ssh overrides of all commands that use ssh_connect +  # +  def connect_options(options) +    connect_options = {:ssh_options=>{}} +    if options[:port] +      connect_options[:ssh_options][:port] = options[:port] +    end +    if options[:ip] +      connect_options[:ssh_options][:host_name] = options[:ip] +    end +    return connect_options +  end + +  def ssh_config_help_message +    puts "" +    puts "Are 'too many authentication failures' getting you down?" +    puts "Then we have the solution for you! Add something like this to your ~/.ssh/config file:" +    puts "  Host *.#{manager.provider.domain}" +    puts "  IdentityFile ~/.ssh/id_rsa" +    puts "  IdentitiesOnly=yes" +    puts "(replace `id_rsa` with the actual private key filename that you use for this provider)" +  end + +  require 'socket' +  def is_port_available?(port) +    TCPServer.open('127.0.0.1', port) {} +    true +  rescue Errno::EACCES +    bail!("You don't have permission to bind to port #{port}.") +  rescue Errno::EADDRINUSE +    bail!("Local port #{port} is already in use. Specify LOCAL_PORT to pick another.") +  rescue Exception => exc +    bail!(exc.to_s) +  end + +  private + +  def exec_ssh(cmd, cli_options, args) +    node = get_node_from_args(args, :include_disabled => true) +    port = node.ssh.port +    options = ssh_config(node) +    username = 'root' +    if LeapCli.log_level >= 3 +      options << "-vv" +    elsif LeapCli.log_level >= 2 +      options << "-v" +    end +    if cli_options[:port] +      port = cli_options[:port] +    end +    if cli_options[:ssh] +      options << cli_options[:ssh] +    end +    ssh = "ssh -l #{username} -p #{port} #{options.join(' ')}" +    if cmd == :ssh +      command = "#{ssh} #{node.domain.full}" +    elsif cmd == :mosh +      command = "MOSH_TITLE_NOPREFIX=1 mosh --ssh \"#{ssh}\" #{node.domain.full}" +    end +    log 2, command + +    # exec the shell command in a subprocess +    pid = fork { exec "#{command}" } + +    Signal.trap("SIGINT") do +      Process.kill("KILL", pid) +      Process.wait(pid) +      exit(0) +    end + +    # wait for shell to exit so we can grab the exit status +    _, status = Process.waitpid2(pid) + +    if status.exitstatus == 255 +      ssh_config_help_message +    elsif status.exitstatus != 0 +      exit(status.exitstatus) +    end +  end + +  def exec_scp(cli_options, src_node, src_file_path, dst_node, dst_file_path) +    node = src_node || dst_node +    options = ssh_config(node) +    port = node.ssh.port +    username = 'root' +    options << "-r" if cli_options[:r] +    scp = "scp -P #{port} #{options.join(' ')}" +    if src_node +      command = "#{scp} #{username}@#{src_node.domain.full}:#{src_file_path} #{dst_file_path}" +    elsif dst_node +      command = "#{scp} #{src_file_path} #{username}@#{dst_node.domain.full}:#{dst_file_path}" +    end +    log 2, command + +    # exec the shell command in a subprocess +    pid = fork { exec "#{command}" } + +    Signal.trap("SIGINT") do +      Process.kill("KILL", pid) +      Process.wait(pid) +      exit(0) +    end + +    # wait for shell to exit so we can grab the exit status +    _, status = Process.waitpid2(pid) +    exit(status.exitstatus) +  end + +  # +  # SSH command line -o options. See `man ssh_config` +  # +  # NOTES: +  # +  # The option 'HostKeyAlias=#{node.name}' is oddly incompatible with ports in +  # known_hosts file, so we must not use this or non-standard ports break. +  # +  def ssh_config(node) +    options = [ +      "-o 'HostName=#{node.ip_address}'", +      "-o 'GlobalKnownHostsFile=#{path(:known_hosts)}'", +      "-o 'UserKnownHostsFile=/dev/null'" +    ] +    if node.vagrant? +      options << "-i #{vagrant_ssh_key_file}"    # use the universal vagrant insecure key +      options << "-o IdentitiesOnly=yes"         # force the use of the insecure vagrant key +      options << "-o 'StrictHostKeyChecking=no'" # blindly accept host key and don't save it +                                                 # (since userknownhostsfile is /dev/null) +    else +      options << "-o 'StrictHostKeyChecking=yes'" +    end +    if !node.supported_ssh_host_key_algorithms.empty? +      options << "-o 'HostKeyAlgorithms=#{node.supported_ssh_host_key_algorithms}'" +    end +    return options +  end + +  def parse_tunnel_arg(arg) +    if arg.count(':') == 1 +      node_name, remote = arg.split(':') +      local = nil +    elsif arg.count(':') == 2 +      local, node_name, remote = arg.split(':') +    else +      bail!('Argument NAME:REMOTE_PORT required.') +    end +    node = get_node_from_args([node_name], :include_disabled => true) +    remote = remote.to_i +    local = local || remote +    local = local.to_i +    return [local, node, remote] +  end + +end; end
\ No newline at end of file diff --git a/lib/leap_cli/commands/test.rb b/lib/leap_cli/commands/test.rb new file mode 100644 index 00000000..73207b31 --- /dev/null +++ b/lib/leap_cli/commands/test.rb @@ -0,0 +1,74 @@ +module LeapCli; module Commands + +  desc 'Run tests.' +  command [:test, :t] do |test| +    test.desc 'Run the test suit on FILTER nodes.' +    test.arg_name 'FILTER', :optional => true +    test.command :run do |run| +      run.switch 'continue', :desc => 'Continue over errors and failures (default is --no-continue).', :negatable => true +      run.action do |global_options,options,args| +        test_order = File.join(Path.platform, 'tests/order.rb') +        if File.exists?(test_order) +          require test_order +        end +        manager.filter!(args).names_in_test_dependency_order.each do |node_name| +          node = manager.nodes[node_name] +          begin +            ssh_connect(node) do |ssh| +              ssh.run(test_cmd(options)) +            end +          rescue Capistrano::CommandError => exc +            if options[:continue] +              exit_status(1) +            else +              bail! +            end +          end +        end +      end +    end + +    test.desc 'Creates files needed to run tests.' +    test.command :init do |init| +      init.action do |global_options,options,args| +        generate_test_client_openvpn_configs +      end +    end + +    test.default_command :run +  end + +  private + +  def test_cmd(options) +    if options[:continue] +      "#{Leap::Platform.leap_dir}/bin/run_tests --continue" +    else +      "#{Leap::Platform.leap_dir}/bin/run_tests" +    end +  end + +  # +  # generates a whole bunch of openvpn configs that can be used to connect to different openvpn gateways +  # +  def generate_test_client_openvpn_configs +    assert_config! 'provider.ca.client_certificates.unlimited_prefix' +    assert_config! 'provider.ca.client_certificates.limited_prefix' +    template = read_file! Path.find_file(:test_client_openvpn_template) +    manager.environment_names.each do |env| +      vpn_nodes = manager.nodes[:environment => env][:services => 'openvpn']['openvpn.allow_limited' => true] +      if vpn_nodes.any? +        generate_test_client_cert(provider.ca.client_certificates.limited_prefix) do |key, cert| +          write_file! [:test_openvpn_config, [env, 'limited'].compact.join('_')], Util.erb_eval(template, binding) +        end +      end +      vpn_nodes = manager.nodes[:environment => env][:services => 'openvpn']['openvpn.allow_unlimited' => true] +      if vpn_nodes.any? +        generate_test_client_cert(provider.ca.client_certificates.unlimited_prefix) do |key, cert| +          write_file! [:test_openvpn_config, [env, 'unlimited'].compact.join('_')], Util.erb_eval(template, binding) +        end +      end +    end +  end + +end; end diff --git a/lib/leap_cli/commands/user.rb b/lib/leap_cli/commands/user.rb new file mode 100644 index 00000000..480e9a9c --- /dev/null +++ b/lib/leap_cli/commands/user.rb @@ -0,0 +1,136 @@ + +# +# perhaps we want to verify that the key files are actually the key files we expect. +# we could use 'file' for this: +# +# > file ~/.gnupg/00440025.asc +# ~/.gnupg/00440025.asc: PGP public key block +# +# > file ~/.ssh/id_rsa.pub +# ~/.ssh/id_rsa.pub: OpenSSH RSA public key +# + +module LeapCli +  module Commands + +    desc 'Adds a new trusted sysadmin by adding public keys to the "users" directory.' +    arg_name 'USERNAME' #, :optional => false, :multiple => false +    command :'add-user' do |c| + +      c.switch 'self', :desc => 'Add yourself as a trusted sysadin by choosing among the public keys available for the current user.', :negatable => false +      c.flag 'ssh-pub-key', :desc => 'SSH public key file for this new user' +      c.flag 'pgp-pub-key', :desc => 'OpenPGP public key file for this new user' + +      c.action do |global_options,options,args| +        username = args.first +        if !username.any? +          if options[:self] +            username ||= `whoami`.strip +          else +            help! "Either USERNAME argument or --self flag is required." +          end +        end +        if Leap::Platform.reserved_usernames.include? username +          bail! %(The username "#{username}" is reserved. Sorry, pick another.) +        end + +        ssh_pub_key = nil +        pgp_pub_key = nil + +        if options['ssh-pub-key'] +          ssh_pub_key = read_file!(options['ssh-pub-key']) +        end +        if options['pgp-pub-key'] +          pgp_pub_key = read_file!(options['pgp-pub-key']) +        end + +        if options[:self] +          ssh_pub_key ||= pick_ssh_key.to_s +          pgp_pub_key ||= pick_pgp_key +        end + +        assert!(ssh_pub_key, 'Sorry, could not find SSH public key.') + +        if ssh_pub_key +          write_file!([:user_ssh, username], ssh_pub_key) +        end +        if pgp_pub_key +          write_file!([:user_pgp, username], pgp_pub_key) +        end + +        update_authorized_keys +      end +    end + +    # +    # let the the user choose among the ssh public keys that we encounter, or just pick the key if there is only one. +    # +    def pick_ssh_key +      ssh_keys = [] +      Dir.glob("#{ENV['HOME']}/.ssh/*.pub").each do |keyfile| +        ssh_keys << SshKey.load(keyfile) +      end + +      if `which ssh-add`.strip.any? +        `ssh-add -L 2> /dev/null`.split("\n").compact.each do |line| +          key = SshKey.load(line) +          if key +            key.comment = 'ssh-agent' +            ssh_keys << key unless ssh_keys.include?(key) +          end +        end +      end +      ssh_keys.compact! + +      assert! ssh_keys.any?, 'Sorry, could not find any SSH public key for you. Have you run ssh-keygen?' + +      if ssh_keys.length > 1 +        key_index = numbered_choice_menu('Choose your SSH public key', ssh_keys.collect(&:summary)) do |line, i| +          say("#{i+1}. #{line}") +        end +      else +        key_index = 0 +      end + +      return ssh_keys[key_index] +    end + +    # +    # let the the user choose among the gpg public keys that we encounter, or just pick the key if there is only one. +    # +    def pick_pgp_key +      begin +        require 'gpgme' +      rescue LoadError +        log "Skipping OpenPGP setup because gpgme is not installed." +        return +      end + +      secret_keys = GPGME::Key.find(:secret) +      if secret_keys.empty? +        log "Skipping OpenPGP setup because I could not find any OpenPGP keys for you" +        return nil +      end + +      secret_keys.select!{|key| !key.expired} + +      if secret_keys.length > 1 +        key_index = numbered_choice_menu('Choose your OpenPGP public key', secret_keys) do |key, i| +          key_info = key.to_s.split("\n")[0..1].map{|line| line.sub(/^\s*(sec|uid)\s*/,'')}.join(' -- ') +          say("#{i+1}. #{key_info}") +        end +      else +        key_index = 0 +      end + +      key_id = secret_keys[key_index].sha + +      # can't use this, it includes signatures: +      #puts GPGME::Key.export(key_id, :armor => true, :export_options => :export_minimal) + +      # export with signatures removed: +      return `gpg --armor --export-options export-minimal --export #{key_id}`.strip +    end + +  end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/util.rb b/lib/leap_cli/commands/util.rb new file mode 100644 index 00000000..c1da570e --- /dev/null +++ b/lib/leap_cli/commands/util.rb @@ -0,0 +1,50 @@ +module LeapCli; module Commands + +  extend self +  extend LeapCli::Util +  extend LeapCli::Util::RemoteCommand + +  def path(name) +    Path.named_path(name) +  end + +  # +  # keeps prompting the user for a numbered choice, until they pick a good one or bail out. +  # +  # block is yielded and is responsible for rendering the choices. +  # +  def numbered_choice_menu(msg, items, &block) +    while true +      say("\n" + msg + ':') +      items.each_with_index &block +      say("q. quit") +      index = ask("number 1-#{items.length}> ") +      if index.empty? +        next +      elsif index =~ /q/ +        bail! +      else +        i = index.to_i - 1 +        if i < 0 || i >= items.length +          bail! +        else +          return i +        end +      end +    end +  end + + +  def parse_node_list(nodes) +    if nodes.is_a? Config::Object +      Config::ObjectList.new(nodes) +    elsif nodes.is_a? Config::ObjectList +      nodes +    elsif nodes.is_a? String +      manager.filter!(nodes) +    else +      bail! "argument error" +    end +  end + +end; end diff --git a/lib/leap_cli/commands/vagrant.rb b/lib/leap_cli/commands/vagrant.rb new file mode 100644 index 00000000..27c739b1 --- /dev/null +++ b/lib/leap_cli/commands/vagrant.rb @@ -0,0 +1,197 @@ +autoload :IPAddr, 'ipaddr' +require 'fileutils' + +module LeapCli; module Commands + +  desc "Manage local virtual machines." +  long_desc "This command provides a convient way to manage Vagrant-based virtual machines. If FILTER argument is missing, the command runs on all local virtual machines. The Vagrantfile is automatically generated in 'test/Vagrantfile'. If you want to run vagrant commands manually, cd to 'test'." +  command [:local, :l] do |local| +    local.desc 'Starts up the virtual machine(s)' +    local.arg_name 'FILTER', :optional => true #, :multiple => false +    local.command :start do |start| +      start.flag(:basebox, +        :desc => "The basebox to use. This value is passed to vagrant as the "+ +          "`config.vm.box` option. The value here should be the name of an installed box or a "+ +          "shorthand name of a box in HashiCorp's Atlas.", +        :arg_name => 'BASEBOX', +        :default_value => 'LEAP/wheezy' +      ) +      start.action do |global_options,options,args| +        vagrant_command(["up", "sandbox on"], args, options) +      end +    end + +    local.desc 'Shuts down the virtual machine(s)' +    local.arg_name 'FILTER', :optional => true #, :multiple => false +    local.command :stop do |stop| +      stop.action do |global_options,options,args| +        if global_options[:yes] +          vagrant_command("halt --force", args) +        else +          vagrant_command("halt", args) +        end +      end +    end + +    local.desc 'Destroys the virtual machine(s), reclaiming the disk space' +    local.arg_name 'FILTER', :optional => true #, :multiple => false +    local.command :destroy do |destroy| +      destroy.action do |global_options,options,args| +        if global_options[:yes] +          vagrant_command("destroy --force", args) +        else +          vagrant_command("destroy", args) +        end +      end +    end + +    local.desc 'Print the status of local virtual machine(s)' +    local.arg_name 'FILTER', :optional => true #, :multiple => false +    local.command :status do |status| +      status.action do |global_options,options,args| +        vagrant_command("status", args) +      end +    end + +    local.desc 'Saves the current state of the virtual machine as a new snapshot' +    local.arg_name 'FILTER', :optional => true #, :multiple => false +    local.command :save do |status| +      status.action do |global_options,options,args| +        vagrant_command("sandbox commit", args) +      end +    end + +    local.desc 'Resets virtual machine(s) to the last saved snapshot' +    local.arg_name 'FILTER', :optional => true #, :multiple => false +    local.command :reset do |reset| +      reset.action do |global_options,options,args| +        vagrant_command("sandbox rollback", args) +      end +    end +  end + +  public + +  # +  # returns the path to a vagrant ssh key file. +  # +  # if the vagrant.key file is owned by root or ourselves, then +  # we need to make sure that it owned by us and not world readable. +  # +  def vagrant_ssh_key_file +    file_path = File.expand_path('../../../vendor/vagrant_ssh_keys/vagrant.key', File.dirname(__FILE__)) +    Util.assert_files_exist! file_path +    uid = File.new(file_path).stat.uid +    if uid == 0 || uid == Process.euid +      FileUtils.install file_path, '/tmp/vagrant.key', :mode => 0600 +      file_path = '/tmp/vagrant.key' +    end +    return file_path +  end + +  protected + +  def vagrant_command(cmds, args, options={}) +    vagrant_setup(options) +    cmds = cmds.to_a +    if args.empty? +      nodes = [""] +    else +      nodes = manager.filter(args)[:environment => "local"].field(:name) +    end +    if nodes.any? +      vagrant_dir = File.dirname(Path.named_path(:vagrantfile)) +      exec = ["cd #{vagrant_dir}"] +      cmds.each do |cmd| +        nodes.each do |node| +          exec << "vagrant #{cmd} #{node}" +        end +      end +      execute exec.join('; ') +    else +      bail! "No nodes found. This command only works on nodes with ip_address in the network #{LeapCli.leapfile.vagrant_network}" +    end +  end + +  private + +  def vagrant_setup(options) +    assert_bin! 'vagrant', 'Vagrant is required for running local virtual machines. Run "sudo apt-get install vagrant".' + +    if vagrant_version <= Gem::Version.new('1.0.0') +      gem_path = assert_run!('vagrant gem which sahara') +      if gem_path.nil? || gem_path.empty? || gem_path =~ /^ERROR/ +        log :installing, "vagrant plugin 'sahara'" +        assert_run! 'vagrant gem install sahara -v 0.0.13' +      end +    else +      unless assert_run!('vagrant plugin list | grep sahara | cat').chars.any? +        log :installing, "vagrant plugin 'sahara'" +        assert_run! 'vagrant plugin install sahara' +      end +    end +    create_vagrant_file(options) +  end + +  def vagrant_version +    @vagrant_version ||= Gem::Version.new(assert_run!('vagrant --version').split(' ')[1]) +  end + +  def execute(cmd) +    log 2, :run, cmd +    exec cmd +  end + +  def create_vagrant_file(options) +    lines = [] +    netmask = IPAddr.new('255.255.255.255').mask(LeapCli.leapfile.vagrant_network.split('/').last).to_s + +    basebox = options[:basebox] || 'LEAP/wheezy' + +    if vagrant_version <= Gem::Version.new('1.1.0') +      lines << %[Vagrant::Config.run do |config|] +      manager.each_node do |node| +        if node.vagrant? +          lines << %[  config.vm.define :#{node.name} do |config|] +          lines << %[    config.vm.box = "#{basebox}"] +          lines << %[    config.vm.network :hostonly, "#{node.ip_address}", :netmask => "#{netmask}"] +          lines << %[    config.vm.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]] +          lines << %[    config.vm.customize ["modifyvm", :id, "--name", "#{node.name}"]] +          lines << %[    #{leapfile.custom_vagrant_vm_line}] if leapfile.custom_vagrant_vm_line +          lines << %[  end] +        end +      end +    else +      lines << %[Vagrant.configure("2") do |config|] +      manager.each_node do |node| +        if node.vagrant? +          lines << %[  config.vm.define :#{node.name} do |config|] +          lines << %[    config.vm.box = "#{basebox}"] +          lines << %[    config.vm.network :private_network, ip: "#{node.ip_address}"] +          lines << %[    config.vm.provider "virtualbox" do |v|] +          lines << %[      v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]] +          lines << %[      v.name = "#{node.name}"] +          lines << %[    end] +          lines << %[    #{leapfile.custom_vagrant_vm_line}] if leapfile.custom_vagrant_vm_line +          lines << %[  end] +        end +      end +    end + +    lines << %[end] +    lines << "" +    write_file! :vagrantfile, lines.join("\n") +  end + +  def pick_next_vagrant_ip_address +    taken_ips = manager.nodes[:environment => "local"].field(:ip_address) +    if taken_ips.any? +      highest_ip = taken_ips.map{|ip| IPAddr.new(ip)}.max +      new_ip = highest_ip.succ +    else +      new_ip = IPAddr.new(LeapCli.leapfile.vagrant_network).succ.succ +    end +    return new_ip.to_s +  end + +end; end diff --git a/lib/leap_cli/macros.rb b/lib/leap_cli/macros.rb new file mode 100644 index 00000000..fdb9a94e --- /dev/null +++ b/lib/leap_cli/macros.rb @@ -0,0 +1,16 @@ +# +# MACROS +# +# The methods in these files are available in the context of a .json configuration file. +# (The module LeapCli::Macro is included in Config::Object) +# + +require_relative 'macros/core' +require_relative 'macros/files' +require_relative 'macros/haproxy' +require_relative 'macros/hosts' +require_relative 'macros/keys' +require_relative 'macros/nodes' +require_relative 'macros/secrets' +require_relative 'macros/stunnel' +require_relative 'macros/provider' diff --git a/lib/leap_cli/macros/core.rb b/lib/leap_cli/macros/core.rb new file mode 100644 index 00000000..7de50f2f --- /dev/null +++ b/lib/leap_cli/macros/core.rb @@ -0,0 +1,94 @@ +# encoding: utf-8 + +module LeapCli +  module Macro + +    # +    # return a fingerprint for a x509 certificate +    # +    def fingerprint(filename) +      "SHA256: " + X509.fingerprint("SHA256", Path.named_path(filename)) +    end + +    # +    # Creates a hash from the ssh key info in users directory, for use in +    # updating authorized_keys file. Additionally, the 'monitor' public key is +    # included, which is used by the monitor nodes to run particular commands +    # remotely. +    # +    def authorized_keys +      hash = {} +      keys = Dir.glob(Path.named_path([:user_ssh, '*'])) +      keys.sort.each do |keyfile| +        ssh_type, ssh_key = File.read(keyfile, :encoding => 'UTF-8').strip.split(" ") +        name = File.basename(File.dirname(keyfile)) +        until hash[name].nil? +          i ||= 1; name = "#{name}#{i+=1}" +        end +        hash[name] = { +          "type" => ssh_type, +          "key" => ssh_key +        } +      end +      ssh_type, ssh_key = File.read(Path.named_path(:monitor_pub_key), :encoding => 'UTF-8').strip.split(" ") +      hash[Leap::Platform.monitor_username] = { +        "type" => ssh_type, +        "key" => ssh_key +      } +      hash +    end + +    def assert(assertion) +      if instance_eval(assertion) +        true +      else +        raise AssertionFailed.new(assertion), assertion, caller +      end +    end + +    def error(msg) +      raise ConfigError.new(@node, msg), msg, caller +    end + +    # +    # applies a JSON partial to this node +    # +    def apply_partial(partial_path) +      manager.partials(partial_path).each do |partial_data| +        self.deep_merge!(partial_data) +      end +    end + +    # +    # If at first you don't succeed, then it is time to give up. +    # +    # try{} returns nil if anything in the block throws an exception. +    # +    # You can wrap something that might fail in `try`, like so. +    # +    #   "= try{ nodes[:services => 'tor'].first.ip_address } " +    # +    def try(&block) +      yield +    rescue NoMethodError +    rescue ArgumentError +      nil +    end + +    protected + +    # +    # returns a node list, if argument is not already one +    # +    def listify(node_list) +      if node_list.is_a? Config::ObjectList +        node_list +      elsif node_list.is_a? Config::Object +        Config::ObjectList.new(node_list) +      else +        raise ArgumentError, 'argument must be a node or node list, not a `%s`' % node_list.class, caller +      end +    end + +  end +end diff --git a/lib/leap_cli/macros/files.rb b/lib/leap_cli/macros/files.rb new file mode 100644 index 00000000..958958bc --- /dev/null +++ b/lib/leap_cli/macros/files.rb @@ -0,0 +1,89 @@ +# encoding: utf-8 + +## +## FILES +## + +module LeapCli +  module Macro + +    # +    # inserts the contents of a file +    # +    def file(filename, options={}) +      if filename.is_a? Symbol +        filename = [filename, @node.name] +      end +      filepath = Path.find_file(filename) +      if filepath +        if filepath =~ /\.erb$/ +          ERB.new(File.read(filepath, :encoding => 'UTF-8'), nil, '%<>').result(binding) +        else +          File.read(filepath, :encoding => 'UTF-8') +        end +      else +        raise FileMissing.new(Path.named_path(filename), options) +        "" +      end +    end + +    # +    # like #file, but allow missing files +    # +    def try_file(filename) +      return file(filename) +    rescue FileMissing +      return nil +    end + +    # +    # returns what the file path will be, once the file is rsynced to the server. +    # an internal list of discovered file paths is saved, in order to rsync these files when needed. +    # +    # notes: +    # +    # * argument 'path' is relative to Path.provider/files or Path.provider_base/files +    # * the path returned by this method is absolute +    # * the path stored for use later by rsync is relative to Path.provider +    # * if the path does not exist locally, but exists in provider_base, then the default file from +    #   provider_base is copied locally. this is required for rsync to work correctly. +    # +    def file_path(path, options={}) +      if path.is_a? Symbol +        path = [path, @node.name] +      elsif path.is_a? String +        # ensure it prefixed with files/ +        unless path =~ /^files\// +          path = "files/" + path +        end +      end +      actual_path = Path.find_file(path) +      if actual_path.nil? +        if options[:missing] +          raise FileMissing.new(Path.named_path(path), options) +        else +          Util::log 2, :skipping, "file_path(\"#{path}\") because there is no such file." +        end +        nil +      else +        if actual_path =~ /^#{Regexp.escape(Path.provider_base)}/ +          # if file is under Path.provider_base, we must copy the default file to +          # to Path.provider in order for rsync to be able to sync the file. +          local_provider_path = actual_path.sub(/^#{Regexp.escape(Path.provider_base)}/, Path.provider) +          FileUtils.mkdir_p File.dirname(local_provider_path), :mode => 0700 +          FileUtils.install actual_path, local_provider_path, :mode => 0600 +          Util.log :created, Path.relative_path(local_provider_path) +          actual_path = local_provider_path +        end +        if File.directory?(actual_path) && actual_path !~ /\/$/ +          actual_path += '/' # ensure directories end with /, important for building rsync command +        end +        relative_path = Path.relative_path(actual_path) +        relative_path.sub!(/^files\//, '') # remove "files/" prefix +        @node.file_paths << relative_path +        File.join(Leap::Platform.files_dir, relative_path) +      end +    end + +  end +end
\ No newline at end of file diff --git a/lib/leap_cli/macros/haproxy.rb b/lib/leap_cli/macros/haproxy.rb new file mode 100644 index 00000000..602ae726 --- /dev/null +++ b/lib/leap_cli/macros/haproxy.rb @@ -0,0 +1,73 @@ +# encoding: utf-8 + +## +## HAPROXY +## + +module LeapCli +  module Macro + +    # +    # creates a hash suitable for configuring haproxy. the key is the node name of the server we are proxying to. +    # +    # * node_list - a hash of nodes for the haproxy servers +    # * stunnel_client - contains the mappings to local ports for each server node. +    # * non_stunnel_port - in case self is included in node_list, the port to connect to. +    # +    # 1000 weight is used for nodes in the same location. +    # 100 otherwise. +    # +    def haproxy_servers(node_list, stunnel_clients, non_stunnel_port=nil) +      default_weight = 10 +      local_weight = 100 + +      # record the hosts_file +      hostnames(node_list) + +      # create a simple map for node name -> local stunnel accept port +      accept_ports = stunnel_clients.inject({}) do |hsh, stunnel_entry| +        name = stunnel_entry.first.sub /_[0-9]+$/, '' +        hsh[name] = stunnel_entry.last['accept_port'] +        hsh +      end + +      # if one the nodes in the node list is ourself, then there will not be a stunnel to it, +      # but we need to include it anyway in the haproxy config. +      if node_list[self.name] && non_stunnel_port +        accept_ports[self.name] = non_stunnel_port +      end + +      # create the first pass of the servers hash +      servers = node_list.values.inject(Config::ObjectList.new) do |hsh, node| +        # make sure we have a port to talk to +        unless accept_ports[node.name] +          error "haproxy needs a local port to talk to when connecting to #{node.name}" +        end +        weight = default_weight +        try { +          weight = local_weight if self.location.name == node.location.name +        } +        hsh[node.name] = Config::Object[ +          'backup', false, +          'host', 'localhost', +          'port', accept_ports[node.name], +          'weight', weight +        ] +        if node.services.include?('couchdb') +          hsh[node.name]['writable'] = node.couch.mode != 'mirror' +        end +        hsh +      end + +      # if there are some local servers, make the others backup +      if servers.detect{|k,v| v.weight == local_weight} +        servers.each do |k,server| +          server['backup'] = server['weight'] == default_weight +        end +      end + +      return servers +    end + +  end +end diff --git a/lib/leap_cli/macros/hosts.rb b/lib/leap_cli/macros/hosts.rb new file mode 100644 index 00000000..8281329f --- /dev/null +++ b/lib/leap_cli/macros/hosts.rb @@ -0,0 +1,68 @@ +# encoding: utf-8 + +module LeapCli +  module Macro + +    ## +    ## HOSTS +    ## + +    # +    # records the list of hosts that are encountered for this node +    # +    def hostnames(nodes) +      @referenced_nodes ||= Config::ObjectList.new +      nodes = listify(nodes) +      nodes.each_node do |node| +        @referenced_nodes[node.name] ||= node +      end +      return nodes.values.collect {|node| node.domain.name} +    end + +    # +    # Generates entries needed for updating /etc/hosts on a node (as a hash). +    # +    # Argument `nodes` can be nil or a list of nodes. If nil, only include the +    # IPs of the other nodes this @node as has encountered (plus all mx nodes). +    # +    # Also, for virtual machines, we use the local address if this @node is in +    # the same location as the node in question. +    # +    # We include the ssh public key for each host, so that the hash can also +    # be used to generate the /etc/ssh/known_hosts +    # +    def hosts_file(nodes=nil) +      if nodes.nil? +        if @referenced_nodes && @referenced_nodes.any? +          nodes = @referenced_nodes +          nodes = nodes.merge(nodes_like_me[:services => 'mx'])  # all nodes always need to communicate with mx nodes. +        end +      end +      return {} unless nodes +      hosts = {} +      my_location = @node['location'] ? @node['location']['name'] : nil +      nodes.each_node do |node| +        hosts[node.name] = { +          'ip_address' => node.ip_address, +          'domain_internal' => node.domain.internal, +          'domain_full' => node.domain.full, +          'port' => node.ssh.port +        } +        node_location = node['location'] ? node['location']['name'] : nil +        if my_location == node_location +          if facts = @node.manager.facts[node.name] +            if facts['ec2_public_ipv4'] +              hosts[node.name]['ip_address'] = facts['ec2_public_ipv4'] +            end +          end +        end +        host_pub_key   = Util::read_file([:node_ssh_pub_key,node.name]) +        if host_pub_key +          hosts[node.name]['host_pub_key'] = host_pub_key +        end +      end +      hosts +    end + +  end +end
\ No newline at end of file diff --git a/lib/leap_cli/macros/keys.rb b/lib/leap_cli/macros/keys.rb new file mode 100644 index 00000000..0ed7ccd0 --- /dev/null +++ b/lib/leap_cli/macros/keys.rb @@ -0,0 +1,83 @@ +# encoding: utf-8 + +# +# Macro for dealing with cryptographic keys +# + +module LeapCli +  module Macro + +    # +    # return the path to the tor public key +    # generating key if it is missing +    # +    def tor_public_key_path(path_name, key_type) +      path = file_path(path_name) +      if path.nil? +        generate_tor_key(key_type) +        file_path(path_name) +      else +        path +      end +    end + +    # +    # return the path to the tor private key +    # generating key if it is missing +    # +    def tor_private_key_path(path_name, key_type) +      path = file_path(path_name) +      if path.nil? +        generate_tor_key(key_type) +        file_path(path_name) +      else +        path +      end +    end + +    # +    # Generates a onion_address from a public RSA key file. +    # +    # path_name is the named path of the Tor public key. +    # +    # Basically, an onion address is nothing more than a base32 encoding +    # of the first 10 bytes of a sha1 digest of the public key. +    # +    # Additionally, Tor ignores the 22 byte header of the public key +    # before taking the sha1 digest. +    # +    def onion_address(path_name) +      require 'base32' +      require 'base64' +      require 'openssl' +      path = Path.find_file([path_name, self.name]) +      if path && File.exists?(path) +        public_key_str = File.readlines(path).grep(/^[^-]/).join +        public_key     = Base64.decode64(public_key_str) +        public_key     = public_key.slice(22..-1) # Tor ignores the 22 byte SPKI header +        sha1sum        = Digest::SHA1.new.digest(public_key) +        Base32.encode(sha1sum.slice(0,10)).downcase +      else +        LeapCli.log :warning, 'Tor public key file "%s" does not exist' % tor_public_key_path +      end +    end + +    private + +    def generate_tor_key(key_type) +      if key_type == 'RSA' +        require 'certificate_authority' +        keypair = CertificateAuthority::MemoryKeyMaterial.new +        bit_size = 1024 +        LeapCli.log :generating, "%s bit RSA Tor key" % bit_size do +          keypair.generate_key(bit_size) +          LeapCli::Util.write_file! [:node_tor_priv_key, self.name], keypair.private_key.to_pem +          LeapCli::Util.write_file! [:node_tor_pub_key, self.name], keypair.public_key.to_pem +        end +      else +        LeapCli.bail! 'tor.key.type of %s is not yet supported' % key_type +      end +    end + +  end +end diff --git a/lib/leap_cli/macros/nodes.rb b/lib/leap_cli/macros/nodes.rb new file mode 100644 index 00000000..8b961cbc --- /dev/null +++ b/lib/leap_cli/macros/nodes.rb @@ -0,0 +1,88 @@ +# encoding: utf-8 + +## +## node related macros +## + +module LeapCli +  module Macro + +    # +    # the list of all the nodes +    # +    def nodes +      global.nodes +    end + +    # +    # simple alias for global.provider +    # +    def provider +      global.provider +    end + +    # +    # returns a list of nodes that match the same environment +    # +    # if @node.environment is not set, we return other nodes +    # where environment is not set. +    # +    def nodes_like_me +      nodes[:environment => @node.environment] +    end + +    # +    # returns a list of nodes that match the location name +    # and environment of @node. +    # +    def nodes_near_me +      if @node['location'] && @node['location']['name'] +        nodes_like_me['location.name' => @node.location.name] +      else +        nodes_like_me['location' => nil] +      end +    end + +    # +    # +    # picks a node out from the node list in such a way that: +    # +    # (1) which nodes picked which nodes is saved in secrets.json +    # (2) when other nodes call this macro with the same node list, they are guaranteed to get a different node +    # (3) if all the nodes in the pick_node list have been picked, remaining nodes are distributed randomly. +    # +    # if the node_list is empty, an exception is raised. +    # if node_list size is 1, then that node is returned and nothing is +    # memorized via the secrets.json file. +    # +    # `label` is needed to distinguish between pools of nodes for different purposes. +    # +    # TODO: more evenly balance after all the nodes have been picked. +    # +    def pick_node(label, node_list) +      if node_list.any? +        if node_list.size == 1 +          return node_list.values.first +        else +          secrets_key = "pick_node(:#{label},#{node_list.keys.sort.join(',')})" +          secrets_value = @manager.secrets.retrieve(secrets_key, @node.environment) || {} +          secrets_value[@node.name] ||= begin +            node_to_pick = nil +            node_list.each_node do |node| +              next if secrets_value.values.include?(node.name) +              node_to_pick = node.name +            end +            node_to_pick ||= secrets_value.values.shuffle.first # all picked already, so pick a random one. +            node_to_pick +          end +          picked_node_name = secrets_value[@node.name] +          @manager.secrets.set(secrets_key, secrets_value, @node.environment) +          return node_list[picked_node_name] +        end +      else +        raise ArgumentError.new('pick_node(node_list): node_list cannot be empty') +      end +    end + +  end +end
\ No newline at end of file diff --git a/lib/leap_cli/macros/provider.rb b/lib/leap_cli/macros/provider.rb new file mode 100644 index 00000000..84c4e1b8 --- /dev/null +++ b/lib/leap_cli/macros/provider.rb @@ -0,0 +1,20 @@ +# +# These macros are intended only for use in provider.json, although they are +# currently loaded in all .json contexts. +# + +module LeapCli +  module Macro + +    # +    # returns an array of the service names, including only those services that +    # are enabled for this environment. +    # +    def enabled_services +      manager.env(self.environment).services[:service_type => :user_service].field(:name).select { |service| +        manager.nodes[:environment => self.environment][:services => service].any? +      } +    end + +  end +end diff --git a/lib/leap_cli/macros/secrets.rb b/lib/leap_cli/macros/secrets.rb new file mode 100644 index 00000000..8d1feb55 --- /dev/null +++ b/lib/leap_cli/macros/secrets.rb @@ -0,0 +1,39 @@ +# encoding: utf-8 + +require 'base32' + +module LeapCli +  module Macro + +    # +    # inserts a named secret, generating it if needed. +    # +    # manager.export_secrets should be called later to capture any newly generated secrets. +    # +    # +length+ is the character length of the generated password. +    # +    def secret(name, length=32) +      manager.secrets.set(name, @node.environment) { Util::Secret.generate(length) } +    end + +    # inserts a base32 encoded secret +    def base32_secret(name, length=20) +      manager.secrets.set(name, @node.environment) { Base32.encode(Util::Secret.generate(length)) } +    end + +    # Picks a random obfsproxy port from given range +    def rand_range(name, range) +      manager.secrets.set(name, @node.environment) { rand(range) } +    end + +    # +    # inserts an hexidecimal secret string, generating it if needed. +    # +    # +bit_length+ is the bits in the secret, (ie length of resulting hex string will be bit_length/4) +    # +    def hex_secret(name, bit_length=128) +      manager.secrets.set(name, @node.environment) { Util::Secret.generate_hex(bit_length) } +    end + +  end +end
\ No newline at end of file diff --git a/lib/leap_cli/macros/stunnel.rb b/lib/leap_cli/macros/stunnel.rb new file mode 100644 index 00000000..f16308c7 --- /dev/null +++ b/lib/leap_cli/macros/stunnel.rb @@ -0,0 +1,95 @@ +## +## STUNNEL +## + +# +# About stunnel +# -------------------------- +# +# The network looks like this: +# +#   From the client's perspective: +# +#   |------- stunnel client --------------|    |---------- stunnel server -----------------------| +#    consumer app -> localhost:accept_port  ->  connect:connect_port -> ?? +# +#   From the server's perspective: +# +#   |------- stunnel client --------------|    |---------- stunnel server -----------------------| +#                                       ??  ->  *:accept_port -> localhost:connect_port -> service +# + +module LeapCli +  module Macro + +    # +    # stunnel configuration for the client side. +    # +    # +node_list+ is a ObjectList of nodes running stunnel servers. +    # +    # +port+ is the real port of the ultimate service running on the servers +    # that the client wants to connect to. +    # +    # * accept_port is the port on localhost to which local clients +    #   can connect. it is auto generated serially. +    # +    # * connect_port is the port on the stunnel server to connect to. +    #   it is auto generated from the +port+ argument. +    # +    # generates an entry appropriate to be passed directly to +    # create_resources(stunnel::service, hiera('..'), defaults) +    # +    # local ports are automatically generated, starting at 4000 +    # and incrementing in sorted order (by node name). +    # +    def stunnel_client(node_list, port, options={}) +      @next_stunnel_port ||= 4000 +      node_list = listify(node_list) +      hostnames(node_list) # record the hosts +      result = Config::ObjectList.new +      node_list.each_node do |node| +        if node.name != self.name || options[:include_self] +          result["#{node.name}_#{port}"] = Config::Object[ +            'accept_port', @next_stunnel_port, +            'connect', node.domain.internal, +            'connect_port', stunnel_port(port), +            'original_port', port +          ] +          @next_stunnel_port += 1 +        end +      end +      result +    end + +    # +    # generates a stunnel server entry. +    # +    # +port+ is the real port targeted service. +    # +    # * `accept_port` is the publicly bound port +    # * `connect_port` is the port that the local service is running on. +    # +    def stunnel_server(port) +      { +        "accept_port" => stunnel_port(port), +        "connect_port" => port +      } +    end + +    private + +    # +    # maps a real port to a stunnel port (used as the connect_port in the client config +    # and the accept_port in the server config) +    # +    def stunnel_port(port) +      port = port.to_i +      if port < 50000 +        return port + 10000 +      else +        return port - 10000 +      end +    end + +  end +end
\ No newline at end of file | 
