diff options
Diffstat (limited to 'lib')
57 files changed, 5813 insertions, 917 deletions
diff --git a/lib/leap/platform.rb b/lib/leap/platform.rb new file mode 100644 index 00000000..9e6cadd5 --- /dev/null +++ b/lib/leap/platform.rb @@ -0,0 +1,99 @@ +module Leap + + class Platform + class << self + # + # configuration + # + + attr_reader :version + attr_reader :compatible_cli + attr_accessor :facts + attr_accessor :paths + attr_accessor :node_files + attr_accessor :monitor_username + attr_accessor :reserved_usernames + + attr_accessor :hiera_dir + attr_accessor :hiera_path + attr_accessor :files_dir + attr_accessor :leap_dir + attr_accessor :init_path + + attr_accessor :default_puppet_tags + + def define(&block) + # some defaults: + @reserved_usernames = [] + @hiera_dir = '/etc/leap' + @hiera_path = '/etc/leap/hiera.yaml' + @leap_dir = '/srv/leap' + @files_dir = '/srv/leap/files' + @init_path = '/srv/leap/initialized' + @default_puppet_tags = [] + + self.instance_eval(&block) + + @version ||= Gem::Version.new("0.0") + end + + def validate!(cli_version, compatible_platforms, leapfile) + if !compatible_with_cli?(cli_version) || !version_in_range?(compatible_platforms) + raise StandardError, "This leap command (v#{cli_version}) " + + "is not compatible with the platform #{leapfile.platform_directory_path} (v#{version}).\n " + + "You need either leap command #{compatible_cli.first} to #{compatible_cli.last} or " + + "platform version #{compatible_platforms.first} to #{compatible_platforms.last}" + end + end + + def version=(version) + @version = Gem::Version.new(version) + end + + def compatible_cli=(range) + @compatible_cli = range + @minimum_cli_version = Gem::Version.new(range.first) + @maximum_cli_version = Gem::Version.new(range.last) + end + + # + # return true if the cli_version is compatible with this platform. + # + def compatible_with_cli?(cli_version) + cli_version = Gem::Version.new(cli_version) + cli_version >= @minimum_cli_version && cli_version <= @maximum_cli_version + end + + # + # return true if the platform version is within the specified range. + # + def version_in_range?(range) + if range.is_a? String + range = range.split('..') + end + minimum_platform_version = Gem::Version.new(range.first) + maximum_platform_version = Gem::Version.new(range.last) + @version >= minimum_platform_version && @version <= maximum_platform_version + end + + def major_version + if @version.segments.first == 0 + @version.segments[0..1].join('.') + else + @version.segments.first + end + end + + def method_missing(method, *args) + puts + puts "WARNING:" + puts " leap_cli is out of date and does not understand `#{method}`." + puts " called from: #{caller.first}" + puts " please upgrade to a newer leap_cli" + end + + end + + end + +end
\ No newline at end of file diff --git a/lib/leap_cli/acme.rb b/lib/leap_cli/acme.rb new file mode 100644 index 00000000..6c7dbe98 --- /dev/null +++ b/lib/leap_cli/acme.rb @@ -0,0 +1,101 @@ +require 'openssl' +require 'acme-client' + +# +# A little bit of sugar around gem acme-client +# + +module LeapCli + class Acme + + if ENV['ACME_STAGING'] + ENDPOINT = 'https://acme-staging.api.letsencrypt.org/' + puts "using endpoint " + ENDPOINT + else + ENDPOINT = 'https://acme-v01.api.letsencrypt.org/' + end + + def initialize(domain: nil, key:) + @client = ::Acme::Client.new( + private_key: key, + endpoint: ENDPOINT, + connection_options: {request: {open_timeout: 5, timeout: 5}} + ) + @domain = domain + end + + # + # static methods + # + + def self.new_private_key + return OpenSSL::PKey::RSA.new(4096) + end + + def self.load_private_key(pem_encoded_key) + return OpenSSL::PKey::RSA.new(pem_encoded_key) + end + + def self.load_csr(pem_encoded_csr) + return OpenSSL::X509::Request.new(pem_encoded_csr) + end + + # + # instance methods + # + + # + # register a new account key with CA + # + def register(contact) + registration = @client.register(contact: 'mailto:' + contact) + if registration && registration.agree_terms + return registration + else + return false + end + end + + # + # authorize account key for domain + # + def authorize + authorization = @client.authorize(domain: @domain) + challenge = nil + begin + while true + if authorization.status == 'pending' + challenge = authorization.http01 + yield challenge + challenge.request_verification + sleep 1 + authorization.verify_status + if challenge.error + return 'error', challenge.error + end + elsif authorization.status == 'invalid' + challenge_msg = (challenge.nil? ? '' : challenge.error) + return 'error', 'Something bad happened. %s' % challenge_msg + elsif authorization.status == 'valid' + return 'valid', nil + else + challenge_msg = (challenge.nil? ? '' : challenge.error) + return 'error', 'status: %s, response message: %s' % [authorization.status, challenge_msg] + end + end + rescue Interrupt + return 'error', 'interrupted' + end + rescue ::Acme::Client::Error::Unauthorized => exc + return 'unauthorized', exc.to_s + end + + # + # get new certificate + # + def get_certificate(csr) + return @client.new_certificate(csr) + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/cloud.rb b/lib/leap_cli/cloud.rb new file mode 100644 index 00000000..268cea38 --- /dev/null +++ b/lib/leap_cli/cloud.rb @@ -0,0 +1,4 @@ + +require 'fog/aws' +require_relative 'cloud/cloud.rb' +require_relative 'cloud/image.rb' diff --git a/lib/leap_cli/cloud/cloud.rb b/lib/leap_cli/cloud/cloud.rb new file mode 100644 index 00000000..2c06e7ed --- /dev/null +++ b/lib/leap_cli/cloud/cloud.rb @@ -0,0 +1,380 @@ +# +# An abstractions in front of Fog, which is an abstraction in front of +# the AWS api. Oh my! +# +# A Cloud object binds a particular node with particular Fog +# authentication credentials. +# +# NOTE: Possible AWS options for creating instances: +# +# options = { +# 'BlockDeviceMapping' => block_device_mapping, +# 'NetworkInterfaces' => network_interfaces, +# 'ClientToken' => client_token, +# 'DisableApiTermination' => disable_api_termination, +# 'EbsOptimized' => ebs_optimized, +# 'IamInstanceProfile.Arn' => @iam_instance_profile_arn, +# 'IamInstanceProfile.Name' => @iam_instance_profile_name, +# 'InstanceInitiatedShutdownBehavior' => instance_initiated_shutdown_behavior, +# 'InstanceType' => flavor_id, +# 'KernelId' => kernel_id, +# 'KeyName' => key_name, +# 'Monitoring.Enabled' => monitoring, +# 'Placement.AvailabilityZone' => availability_zone, +# 'Placement.GroupName' => placement_group, +# 'Placement.Tenancy' => tenancy, +# 'PrivateIpAddress' => private_ip_address, +# 'RamdiskId' => ramdisk_id, +# 'SecurityGroup' => groups, +# 'SecurityGroupId' => security_group_ids, +# 'SubnetId' => subnet_id, +# 'UserData' => user_data, +# } +# + +module LeapCli + class Cloud + DEFAULT_INSTANCE_OPTIONS = { + "InstanceType" => "t2.nano" + } + LEAP_SG_NAME = 'leap_default' + LEAP_SG_DESC = 'Default security group for LEAP nodes' + + include LeapCli::LogCommand + + attr_reader :compute # Fog::Compute object + attr_reader :node # Config::Node, if any + attr_reader :options # options for the VMs, if any + attr_reader :image # which vm image to use, if any + attr_reader :name # name of which entry in cloud.json to use + + def initialize(name, conf, node=nil) + @node = node + @name = name + @conf = conf + @compute = nil + @options = DEFAULT_INSTANCE_OPTIONS + @image = nil + + raise ArgumentError, 'name missing' unless @name + raise ArgumentError, 'config missing' unless @conf + raise ArgumentError, 'config auth missing' unless @conf["auth"] + raise ArgumentError, 'config auth missing' unless @conf["vendor"] + + credentials = @conf["auth"].symbolize_keys + credentials[:provider] = @conf["vendor"] + @compute = Fog::Compute.new(credentials) + + if @conf['default_options'] + @options = @options.merge(@conf['default_options']) + end + @image = @conf['default_image'] || Cloud.aws_image(credentials[:region]) + if @node + @options = @options.merge(node.vm.options) if node['vm.options'] + @image = node.vm.image if node['vm.image'] + end + + unless @options['InstanceType'] + raise ArgumentError, 'VM instance type is required. See https://leap.se/virtual-machines for more information.' + end + unless @image + raise ArgumentError, 'VM image is required. See https://leap.se/virtual-machines for more information.' + end + end + + # + # fetches or creates a server for this cloud object. + # + def fetch_or_create_server(options) + fetch_server_for_node || create_new_vm_instance(choose_ssh_key: options[:choose_ssh_key]) + end + + # + # fetches the server for a particular node. + # + # return nil if this cloud object has no node, or there is no corresponding + # server. + # + def fetch_server_for_node(bail_on_failure=false) + server = nil + return nil unless @node + + # does an instance exist that matches the node's vm.id? + if @node.vm_id? + instance_id = @node.vm.id + server = @compute.servers.get(instance_id) + end + + # does an instance exist that is tagged with this node name? + if server.nil? + response = @compute.describe_instances({"tag:node_name" => @node.name}) + # puts JSON.pretty_generate(response.body) + if !response.body["reservationSet"].empty? + instances = response.body["reservationSet"].first["instancesSet"] + if instances.size > 1 + bail! "There are multiple VMs with the same node name tag! Manually remove one before continuing." + elsif instances.size == 1 + instance_id = instances.first["instanceId"] + server = @compute.servers.get(instance_id) + end + end + end + + if server.nil? && bail_on_failure + bail! :error, "A virtual machine could not be found for node `#{@node.name}`. Things to try:" do + log "check the output of `leap vm status`" + log "check the value of `vm.id` in #{@node.name}.json" + log "run `leap vm add #{@node.name}` to create a corresponding virtual machine" + end + end + + return server + end + + # + # associates a node with a vm + # + def bind_server_to_node(server) + unless @node + raise ArgumentError, 'no node' + end + if server.public_ip_address.nil? + bail! do + log 'The virtual machine `%s` must have an IP address in order to bind it to the configuration `%s`.' % [ + server.id, Path.relative_path(Path.named_path([:node_config, @node.name]))] + log 'To fix, run `leap vm start %s`' % server.id + end + end + + # assign tag + @compute.create_tags(server.id, {'node_name' => @node.name}) + log :created, "association between node '%s' and vm '%s'" % [@node.name, server.id] + + # update node json + @node.update_json({ + "ip_address" => server.public_ip_address, + "vm"=> {"id"=>server.id} + }) + end + + # + # disassociates a node from a vm + # + def unbind_server_from_node(server) + raise ArgumentError, 'no node' unless @node + + # assign tag + @compute.delete_tags(server.id, {'node_name' => @node.name}) + log :removed, "association between node '%s' and vm '%s'" % [@node.name, server.id] + + # update node json + @node.update_json({ + "ip_address" => '0.0.0.0', + "vm"=> {"id" => ""} + }) + end + + # + # return an AWS KeyPair object, potentially uploading it to the server + # if necessary. + # + # this is used when initially creating the vm. After the first `node init`, then + # all sysadmins should have access to the server. + # + # NOTE: ssh and aws use different types of fingerprint + # + def find_or_create_key_pair(pick_ssh_key_method) + require 'leap_cli/ssh' + key_pair, local_key = match_ssh_key(:user_only => true) + if key_pair + log :using, "user SSH key #{local_key.filename}" do + log 'AWS MD5 fingerprint: ' + local_key.fingerprint(:digest => :md5, :type => :der, :encoding => :hex) + log 'SSH MD5 fingerprint: ' + local_key.fingerprint(:digest => :md5, :type => :ssh, :encoding => :hex) + log 'SSH SHA256 fingerprint: ' + local_key.fingerprint(:digest => :sha256, :type => :ssh, :encoding => :base64) + end + elsif key_pair.nil? + username, key = pick_ssh_key_method.call(self) + key_pair = upload_ssh_key(username, key) + end + return key_pair + end + + # + # checks if there is a match between a local key and a registered key_pair + # + # options: + # :key_pair -- limit comparisons to this key_pair object. + # :user_only -- limit comparisons to the user's ~/.ssh directory only + # + # returns: + # + # key_pair -- an AWS KeyPair + # local_key -- a LeapCLi::SSH::Key + # + def match_ssh_key(options={}) + key_pair = options[:key_pair] + local_keys_to_check = LeapCli::SSH::Key.my_public_keys + unless options[:user_only] + local_keys_to_check += LeapCli::SSH::Key.provider_public_keys + end + fingerprints = Hash[local_keys_to_check.map {|k| + [k.fingerprint(:digest => :md5, :type => :der, :encoding => :hex), k] + }] + key_pair ||= @compute.key_pairs.select {|key_pair| + fingerprints.include?(key_pair.fingerprint) + }.first + if key_pair + local_key = fingerprints[key_pair.fingerprint] + return key_pair, local_key + else + return nil, nil + end + end + + def wait_for_ssh_host_key(server) + require 'leap_cli/ssh' + return nil if Fog.mock? + tries = 0 + host_key = nil + cloud = self + server.wait_for { + if tries > 0 + LeapCli.log :waiting, "for SSH host key..." + elsif tries > 20 + return nil + end + tries += 1 + ssh_host_keys = cloud.ssh_host_keys(server) + if ssh_host_keys.nil? + false + else + host_key = SSH::Key.pick_best_key(ssh_host_keys) + true + end + } + return host_key + end + + # + # checks the console of the server for the ssh host keys + # + # returns nil if they cannot be found. + # + def ssh_host_keys(server) + require 'leap_cli/ssh' + return nil if Fog.mock? + response = @compute.get_console_output(server.id) + output = response.body["output"] + if output.nil? + return nil + end + keys = output.match( + /-----BEGIN SSH HOST KEY KEYS-----(.*)-----END SSH HOST KEY KEYS-----/m + ) + if keys.nil? + return nil + else + ssh_key_list = keys[1].strip.split("\r\n").map {|key_str| + SSH::Key.load(key_str) + } + return ssh_key_list.compact + end + end + + private + + # + # Every AWS instance requires a security group, which is just a simple firewall. + # In the future, we could create a separate security group for each node, + # and set the rules to match what the rules should be for that node. + # + # However, for now, we just use a security group 'leap_default' that opens + # all the ports. + # + # The default behavior for AWS security groups is: + # all ingress traffic is blocked and all egress traffic is allowed. + # + def find_or_create_security_group + group = @compute.security_groups.get(LEAP_SG_NAME) + if group.nil? + group = @compute.security_groups.create( + :name => LEAP_SG_NAME, + :description => LEAP_SG_DESC + ) + group.authorize_port_range(0..65535, + :ip_protocol => 'tcp', + :cidr_ip => '0.0.0.0/0', + ) + group.authorize_port_range(0..65535, + :ip_protocol => 'udp', + :cidr_ip => '0.0.0.0/0', + ) + end + return group + end + + # + # key - a LeapCli::SSH::Key object + # returns -- AWS KeyPair + # + def upload_ssh_key(username, key) + key_name = 'leap_' + username + key_pair = @compute.key_pairs.create( + :name => key_name, + :public_key => key.public_key.to_s + ) + log :registered, "public key" do + log 'cloud provider: ' + @name + log 'name: ' + key_name + log 'AWS MD5 fingerprint: ' + key.fingerprint(:digest => :md5, :type => :der, :encoding => :hex) + log 'SSH MD5 fingerprint: ' + key.fingerprint(:digest => :md5, :type => :ssh, :encoding => :hex) + log 'SSH SHA256 fingerprint: ' + key.fingerprint(:digest => :sha256, :type => :ssh, :encoding => :base64) + end + return key_pair + end + + def create_new_vm_instance(choose_ssh_key: nil) + log :creating, "new vm instance..." + assert! @image, "No image found. Specify `default_image` in cloud.json or `vm.image` in node's config." + if Fog.mock? + options = @options + else + key_pair = find_or_create_key_pair(choose_ssh_key) + security_group = find_or_create_security_group + options = @options.merge({ + 'KeyName' => key_pair.name, + 'SecurityGroup' => security_group.name + }) + end + response = @compute.run_instances( + @image, + 1, # min count + 1, # max count + options + ) + instance_id = response.body["instancesSet"].first["instanceId"] + log :created, "vm with instance id #{instance_id}." + server = @compute.servers.get(instance_id) + if server.nil? + bail! :error, "could not query instance '#{instance_id}'." + end + unless Fog.mock? + tries = 0 + server.wait_for { + if tries > 0 + LeapCli.log :waiting, "for IP address to be assigned..." + end + tries += 1 + !public_ip_address.nil? + } + if options[:wait] + log :waiting, "for vm #{instance_id} to start..." + server.wait_for { ready? } + log :started, "#{instance_id} with #{server.public_ip_address}" + end + end + return server + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/cloud/dependencies.rb b/lib/leap_cli/cloud/dependencies.rb new file mode 100644 index 00000000..fd690e59 --- /dev/null +++ b/lib/leap_cli/cloud/dependencies.rb @@ -0,0 +1,40 @@ +# +# I am not sure this is a good idea, but it might be. Tricky, so disabled for now +# + +=begin +module LeapCli + class Cloud + + def self.check_required_gems + begin + require "fog" + rescue LoadError + bail! do + log :error, "The 'vm' command requires the gem 'fog-core'. Please run `gem install fog-core` and try again." + end + end + + fog_gems = @cloud.required_gems + if !options[:mock] && fog_gems.empty? + bail! do + log :warning, "no vm providers are configured in cloud.json." + log "You must have credentials for one of: #{@cloud.possible_apis.join(', ')}." + end + end + + fog_gems.each do |name, gem_name| + begin + require gem_name.sub('-','/') + rescue LoadError + bail! do + log :error, "The 'vm' command requires the gem '#{gem_name}' (because of what is configured in cloud.json)." + log "Please run `sudo gem install #{gem_name}` and try again." + end + end + end + end + + end +end +=end
\ No newline at end of file diff --git a/lib/leap_cli/cloud/image.rb b/lib/leap_cli/cloud/image.rb new file mode 100644 index 00000000..1f7f47b9 --- /dev/null +++ b/lib/leap_cli/cloud/image.rb @@ -0,0 +1,31 @@ +module LeapCli + class Cloud + + # + # returns the latest official debian image for + # a particular AWS region + # + # https://wiki.debian.org/Cloud/AmazonEC2Image/Jessie + # current list based on Debian 8.4 + # + # might return nil if no image is found. + # + def self.aws_image(region) + image_list = %q[ + ap-northeast-1 ami-d7d4c5b9 + ap-northeast-2 ami-9a03caf4 + ap-southeast-1 ami-73974210 + ap-southeast-2 ami-09daf96a + eu-central-1 ami-ccc021a3 + eu-west-1 ami-e079f893 + sa-east-1 ami-d3ae21bf + us-east-1 ami-c8bda8a2 + us-west-1 ami-45374b25 + us-west-2 ami-98e114f8 + ] + region_to_image = Hash[image_list.strip.split("\n").map{|i| i.split(" ")}] + return region_to_image[region] + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/ca.rb b/lib/leap_cli/commands/ca.rb index 1b311eee..3c5fc7d5 100644 --- a/lib/leap_cli/commands/ca.rb +++ b/lib/leap_cli/commands/ca.rb @@ -1,8 +1,3 @@ -autoload :OpenSSL, 'openssl' -autoload :CertificateAuthority, 'certificate_authority' -autoload :Date, 'date' -require 'digest/md5' - module LeapCli; module Commands desc "Manage X.509 certificates" @@ -35,41 +30,15 @@ module LeapCli; module Commands cert.desc 'Creates a Diffie-Hellman parameter file, needed for forward secret OpenVPN ciphers.' # (needed for server-side of some TLS connections) cert.command :dh do |dh| dh.action do |global_options,options,args| - long_running do - if cmd_exists?('certtool') - log 0, 'Generating DH parameters (takes a long time)...' - output = assert_run!('certtool --generate-dh-params --sec-param high') - output.sub! /.*(-----BEGIN DH PARAMETERS-----.*-----END DH PARAMETERS-----).*/m, '\1' - output << "\n" - write_file!(:dh_params, output) - else - log 0, 'Generating DH parameters (takes a REALLY long time)...' - output = OpenSSL::PKey::DH.generate(3248).to_pem - write_file!(:dh_params, output) - end - end + generate_dh end end - # - # hints: - # - # inspect CSR: - # openssl req -noout -text -in files/cert/x.csr - # - # generate CSR with openssl to see how it compares: - # openssl req -sha256 -nodes -newkey rsa:2048 -keyout example.key -out example.csr - # - # validate a CSR: - # http://certlogik.com/decoder/ - # - # nice details about CSRs: - # http://www.redkestrel.co.uk/Articles/CSR.html - # cert.desc "Creates a CSR for use in buying a commercial X.509 certificate." cert.long_desc "Unless specified, the CSR is created for the provider's primary domain. "+ "The properties used for this CSR come from `provider.ca.server_certificates`, "+ "but may be overridden here." + cert.arg_name "DOMAIN" cert.command :csr do |csr| csr.flag 'domain', :arg_name => 'DOMAIN', :desc => 'Specify what domain to create the CSR for.' csr.flag ['organization', 'O'], :arg_name => 'ORGANIZATION', :desc => "Override default O in distinguished name." @@ -81,70 +50,26 @@ module LeapCli; module Commands csr.flag :bits, :arg_name => 'BITS', :desc => "Override default certificate bit length" csr.flag :digest, :arg_name => 'DIGEST', :desc => "Override default signature digest" csr.action do |global_options,options,args| - assert_config! 'provider.domain' - assert_config! 'provider.name' - assert_config! 'provider.default_language' - assert_config! 'provider.ca.server_certificates.bit_size' - assert_config! 'provider.ca.server_certificates.digest' - domain = options[:domain] || provider.domain - - unless global_options[:force] - assert_files_missing! [:commercial_key, domain], [:commercial_csr, domain], - :msg => 'If you really want to create a new key and CSR, remove these files first or run with --force.' - end - - server_certificates = provider.ca.server_certificates - - # RSA key - keypair = CertificateAuthority::MemoryKeyMaterial.new - bit_size = (options[:bits] || server_certificates.bit_size).to_i - log :generating, "%s bit RSA key" % bit_size do - keypair.generate_key(bit_size) - write_file! [:commercial_key, domain], keypair.private_key.to_pem - end - - # CSR - dn = CertificateAuthority::DistinguishedName.new - dn.common_name = domain - dn.organization = options[:organization] || provider.name[provider.default_language] - dn.ou = options[:organizational_unit] # optional - dn.email_address = options[:email] # optional - dn.country = options[:country] || server_certificates['country'] # optional - dn.state = options[:state] || server_certificates['state'] # optional - dn.locality = options[:locality] || server_certificates['locality'] # optional - - digest = options[:digest] || server_certificates.digest - log :generating, "CSR with #{digest} digest and #{print_dn(dn)}" do - csr = create_csr(dn, keypair, digest) - request = csr.to_x509_csr - write_file! [:commercial_csr, domain], csr.to_pem - end + generate_csr(global_options, options, args) + end + end - # Sign using our own CA, for use in testing but hopefully not production. - # It is not that commerical CAs are so secure, it is just that signing your own certs is - # a total drag for the user because they must click through dire warnings. - #if options[:sign] - log :generating, "self-signed x509 server certificate for testing purposes" do - cert = csr.to_cert - cert.serial_number.number = cert_serial_number(domain) - cert.not_before = yesterday - cert.not_after = yesterday.advance(:years => 1) - cert.parent = ca_root - cert.sign! domain_test_signing_profile - write_file! [:commercial_cert, domain], cert.to_pem - log "please replace this file with the real certificate you get from a CA using #{Path.relative_path([:commercial_csr, domain])}" - end - #end + cert.desc "Register an authorization key with the CA letsencrypt.org" + cert.long_desc "This only needs to be done once." + cert.command :register do |register| + register.action do |global, options, args| + do_register_key(global, options, args) + end + end - # FAKE CA - unless file_exists? :commercial_ca_cert - log :using, "generated CA in place of commercial CA for testing purposes" do - write_file! :commercial_ca_cert, read_file!(:ca_cert) - log "please also replace this file with the CA cert from the commercial authority you use." - end - end + cert.desc "Renews a certificate using the CA letsencrypt.org" + cert.arg_name "DOMAIN" + cert.command :renew do |renew| + renew.action do |global, options, args| + do_renew_cert(global, options, args) end end + end protected @@ -153,6 +78,7 @@ module LeapCli; module Commands # will generate new certificates for the specified nodes, if needed. # def update_certificates(nodes, options={}) + require 'leap_cli/x509' assert_files_exist! :ca_cert, :ca_key, :msg => 'Run `leap cert ca` to create them' assert_config! 'provider.ca.server_certificates.bit_size' assert_config! 'provider.ca.server_certificates.digest' @@ -160,382 +86,281 @@ module LeapCli; module Commands assert_config! 'common.x509.use' nodes.each_node do |node| - warn_if_commercial_cert_will_soon_expire(node) + node.warn_if_commercial_cert_will_soon_expire if !node.x509.use remove_file!([:node_x509_key, node.name]) remove_file!([:node_x509_cert, node.name]) - elsif options[:force] || cert_needs_updating?(node) - generate_cert_for_node(node) + elsif options[:force] || node.cert_needs_updating? + node.generate_cert end end end + # + # yields client key and cert suitable for testing + # + def generate_test_client_cert(prefix=nil) + require 'leap_cli/x509' + cert = CertificateAuthority::Certificate.new + cert.serial_number.number = cert_serial_number(provider.domain) + cert.subject.common_name = [prefix, random_common_name(provider.domain)].join + cert.not_before = X509.yesterday + cert.not_after = X509.yesterday.advance(:years => 1) + cert.key_material.generate_key(1024) # just for testing, remember! + cert.parent = client_ca_root + cert.sign! client_test_signing_profile + yield cert.key_material.private_key.to_pem, cert.to_pem + end + private def generate_new_certificate_authority(key_file, cert_file, common_name) + require 'leap_cli/x509' assert_files_missing! key_file, cert_file assert_config! 'provider.ca.name' assert_config! 'provider.ca.bit_size' assert_config! 'provider.ca.life_span' - root = CertificateAuthority::Certificate.new + root = X509.new_ca(provider.ca, common_name) - # set subject - root.subject.common_name = common_name - possible = ['country', 'state', 'locality', 'organization', 'organizational_unit', 'email_address'] - provider.ca.keys.each do |key| - if possible.include?(key) - root.subject.send(key + '=', provider.ca[key]) + write_file!(key_file, root.key_material.private_key.to_pem) + write_file!(cert_file, root.to_pem) + end + + def generate_dh + require 'leap_cli/x509' + long_running do + if cmd_exists?('certtool') + log 0, 'Generating DH parameters (takes a long time)...' + output = assert_run!('certtool --generate-dh-params --sec-param high') + output.sub!(/.*(-----BEGIN DH PARAMETERS-----.*-----END DH PARAMETERS-----).*/m, '\1') + output << "\n" + write_file!(:dh_params, output) + else + log 0, 'Generating DH parameters (takes a REALLY long time)...' + output = OpenSSL::PKey::DH.generate(3248).to_pem + write_file!(:dh_params, output) end end + end - # set expiration - root.not_before = yesterday - root.not_after = yesterday_advance(provider.ca.life_span) - - # generate private key - root.serial_number.number = 1 - root.key_material.generate_key(provider.ca.bit_size) + # + # hints: + # + # inspect CSR: + # openssl req -noout -text -in files/cert/x.csr + # + # generate CSR with openssl to see how it compares: + # openssl req -sha256 -nodes -newkey rsa:2048 -keyout example.key -out example.csr + # + # validate a CSR: + # http://certlogik.com/decoder/ + # + # nice details about CSRs: + # http://www.redkestrel.co.uk/Articles/CSR.html + # + def generate_csr(global_options, options, args) + require 'leap_cli/x509' + assert_config! 'provider.domain' + assert_config! 'provider.name' + assert_config! 'provider.default_language' + assert_config! 'provider.ca.server_certificates.bit_size' + assert_config! 'provider.ca.server_certificates.digest' - # sign self - root.signing_entity = true - root.parent = root - root.sign!(ca_root_signing_profile) + server_certificates = provider.ca.server_certificates + options[:domain] ||= args.first || provider.domain + options[:organization] ||= provider.name[provider.default_language] + options[:country] ||= server_certificates['country'] + options[:state] ||= server_certificates['state'] + options[:locality] ||= server_certificates['locality'] + options[:bits] ||= server_certificates.bit_size + options[:digest] ||= server_certificates.digest + + unless global_options[:force] + assert_files_missing! [:commercial_key, options[:domain]], [:commercial_csr, options[:domain]], + :msg => 'If you really want to create a new key and CSR, remove these files first or run with --force.' + end - # save - write_file!(key_file, root.key_material.private_key.to_pem) - write_file!(cert_file, root.to_pem) + X509.create_csr_and_cert(options) end # - # returns true if the certs associated with +node+ need to be regenerated. + # letsencrypt.org # - def cert_needs_updating?(node) - if !file_exists?([:node_x509_cert, node.name], [:node_x509_key, node.name]) - return true - else - cert = load_certificate_file([:node_x509_cert, node.name]) - if !created_by_authority?(cert, ca_root) - log :updating, "cert for node '#{node.name}' because it was signed by an old CA root cert." - return true - end - if cert.not_after < Time.now.advance(:months => 2) - log :updating, "cert for node '#{node.name}' because it will expire soon" - return true - end - if cert.subject.common_name != node.domain.full - log :updating, "cert for node '#{node.name}' because domain.full has changed (was #{cert.subject.common_name}, now #{node.domain.full})" - return true + + def do_register_key(global, options, args) + require 'leap_cli/acme' + assert_config! 'provider.contacts.default' + contact = manager.provider.contacts.default.first + + if file_exists?(:acme_key) && !global[:force] + bail! do + log "the authorization key for letsencrypt.org already exists" + log "run with --force if you really want to register a new key." end - cert.openssl_body.extensions.each do |ext| - if ext.oid == "subjectAltName" - ips = [] - dns_names = [] - ext.value.split(",").each do |value| - value.strip! - ips << $1 if value =~ /^IP Address:(.*)$/ - dns_names << $1 if value =~ /^DNS:(.*)$/ - end - dns_names.sort! - if ips.first != node.ip_address - log :updating, "cert for node '#{node.name}' because ip_address has changed (from #{ips.first} to #{node.ip_address})" - return true - elsif dns_names != dns_names_for_node(node) - log :updating, "cert for node '#{node.name}' because domain name aliases have changed\n from: #{dns_names.inspect}\n to: #{dns_names_for_node(node).inspect})" - return true - end + else + private_key = Acme.new_private_key + registration = nil + + log(:registering, "letsencrypt.org authorization key using contact `%s`" % contact) do + acme = Acme.new(key: private_key) + registration = acme.register(contact) + if registration + log 'success!', :color => :green, :style => :bold + else + bail! "could not register authorization key." end end - end - return false - end - def created_by_authority?(cert, ca) - authority_key_id = cert.extensions["authorityKeyIdentifier"].identifier.sub(/^keyid:/, '') - authority_key_id == public_key_id_for_ca(ca) - end - - # calculate the "key id" for a root CA, that matches the value - # Authority Key Identifier in the x509 extensions of a cert. - def public_key_id_for_ca(ca_cert) - @ca_key_ids ||= {} - @ca_key_ids[ca_cert.object_id] ||= begin - pubkey = ca_cert.key_material.public_key - seq = OpenSSL::ASN1::Sequence([ - OpenSSL::ASN1::Integer.new(pubkey.n), - OpenSSL::ASN1::Integer.new(pubkey.e) - ]) - Digest::SHA1.hexdigest(seq.to_der).upcase.scan(/../).join(':') + log :saving, "authorization key for letsencrypt.org" do + write_file!(:acme_key, private_key.to_pem) + write_file!(:acme_info, JSON.sorted_generate({ + id: registration.id, + contact: registration.contact, + key: registration.key, + uri: registration.uri + })) + log :warning, "keep key file private!" + end end end - def warn_if_commercial_cert_will_soon_expire(node) - dns_names_for_node(node).each do |domain| - if file_exists?([:commercial_cert, domain]) - cert = load_certificate_file([:commercial_cert, domain]) - path = Path.relative_path([:commercial_cert, domain]) - if cert.not_after < Time.now.utc - log :error, "the commercial certificate '#{path}' has EXPIRED! " + - "You should renew it with `leap cert csr --domain #{domain}`." - elsif cert.not_after < Time.now.advance(:months => 2) - log :warning, "the commercial certificate '#{path}' will expire soon (#{cert.not_after}). "+ - "You should renew it with `leap cert csr --domain #{domain}`." - end - end + def assert_no_errors!(msg) + yield + rescue StandardError => exc + bail! :error, msg do + log exc.to_s end end - def generate_cert_for_node(node) - return if node.x509.use == false - - cert = CertificateAuthority::Certificate.new - - # set subject - cert.subject.common_name = node.domain.full - cert.serial_number.number = cert_serial_number(node.domain.full) + def do_renew_cert(global, options, args) + require 'leap_cli/acme' + require 'leap_cli/ssh' + require 'socket' + require 'net/http' - # set expiration - cert.not_before = yesterday - cert.not_after = yesterday_advance(provider.ca.server_certificates.life_span) + csr = nil + account_key = nil + cert = nil + acme = nil - # generate key - cert.key_material.generate_key(provider.ca.server_certificates.bit_size) - - # sign - cert.parent = ca_root - cert.sign!(server_signing_profile(node)) - - # save - write_file!([:node_x509_key, node.name], cert.key_material.private_key.to_pem) - write_file!([:node_x509_cert, node.name], cert.to_pem) - end - - # - # yields client key and cert suitable for testing - # - def generate_test_client_cert(prefix=nil) - cert = CertificateAuthority::Certificate.new - cert.serial_number.number = cert_serial_number(provider.domain) - cert.subject.common_name = [prefix, random_common_name(provider.domain)].join - cert.not_before = yesterday - cert.not_after = yesterday.advance(:years => 1) - cert.key_material.generate_key(1024) # just for testing, remember! - cert.parent = client_ca_root - cert.sign! client_test_signing_profile - yield cert.key_material.private_key.to_pem, cert.to_pem - end - - # - # creates a CSR and returns it. - # with the correct extReq attribute so that the CA - # doens't generate certs with extensions we don't want. - # - def create_csr(dn, keypair, digest) - csr = CertificateAuthority::SigningRequest.new - csr.distinguished_name = dn - csr.key_material = keypair - csr.digest = digest - - # define extensions manually (library doesn't support setting these on CSRs) - extensions = [] - extensions << CertificateAuthority::Extensions::BasicConstraints.new.tap {|basic| - basic.ca = false - } - extensions << CertificateAuthority::Extensions::KeyUsage.new.tap {|keyusage| - keyusage.usage = ["digitalSignature", "keyEncipherment"] - } - extensions << CertificateAuthority::Extensions::ExtendedKeyUsage.new.tap {|extkeyusage| - extkeyusage.usage = [ "serverAuth"] - } - - # convert extensions to attribute 'extReq' - # aka "Requested Extensions" - factory = OpenSSL::X509::ExtensionFactory.new - attrval = OpenSSL::ASN1::Set([OpenSSL::ASN1::Sequence( - extensions.map{|e| factory.create_ext(e.openssl_identifier, e.to_s, e.critical)} - )]) - attrs = [ - OpenSSL::X509::Attribute.new("extReq", attrval), - ] - csr.attributes = attrs - - return csr - end + # + # sanity check the domain + # + domain = args.first + nodes = nodes_for_domain(domain) + domain_ready_for_acme!(domain) - def ca_root - @ca_root ||= begin - load_certificate_file(:ca_cert, :ca_key) + # + # load key material + # + assert_files_exist!([:commercial_key, domain], [:commercial_csr, domain], + :msg => 'Please create the CSR first with `leap cert csr %s`' % domain) + assert_no_errors!("Could not load #{path([:commercial_csr, domain])}") do + csr = Acme.load_csr(read_file!([:commercial_csr, domain])) end - end - - def client_ca_root - @client_ca_root ||= begin - load_certificate_file(:client_ca_cert, :client_ca_key) + assert_files_exist!(:acme_key, + :msg => "Please run `leap cert register` first. This only needs to be done once.") + assert_no_errors!("Could not load #{path(:acme_key)}") do + account_key = Acme.load_private_key(read_file!(:acme_key)) end - end - def load_certificate_file(crt_file, key_file=nil, password=nil) - crt = read_file!(crt_file) - openssl_cert = OpenSSL::X509::Certificate.new(crt) - cert = CertificateAuthority::Certificate.from_openssl(openssl_cert) - if key_file - key = read_file!(key_file) - cert.key_material.private_key = OpenSSL::PKey::RSA.new(key, password) + # + # check authorization for this domain + # + log :checking, "authorization" + acme = Acme.new(domain: domain, key: account_key) + status, message = acme.authorize do |challenge| + log(:uploading, 'challenge to server %s' % domain) do + SSH.remote_command(nodes) do |ssh, host| + ssh.scripts.upload_acme_challenge(challenge.token, challenge.file_content) + end + end + log :waiting, "for letsencrypt.org to verify challenge" + end + if status == 'valid' + log 'authorized!', color: :green, style: :bold + elsif status == 'error' + bail! :error, message + elsif status == 'unauthorized' + bail!(:unauthorized, message, color: :yellow, style: :bold) do + log 'You must first run `leap cert register` to register the account key with letsencrypt.org' + end end - return cert - end - - def ca_root_signing_profile - { - "extensions" => { - "basicConstraints" => {"ca" => true}, - "keyUsage" => { - "usage" => ["critical", "keyCertSign"] - }, - "extendedKeyUsage" => { - "usage" => [] - } - } - } - end - # - # For keyusage, openvpn server certs can have keyEncipherment or keyAgreement. - # Web browsers seem to break without keyEncipherment. - # For now, I am using digitalSignature + keyEncipherment - # - # * digitalSignature -- for (EC)DHE cipher suites - # "The digitalSignature bit is asserted when the subject public key is used - # with a digital signature mechanism to support security services other - # than certificate signing (bit 5), or CRL signing (bit 6). Digital - # signature mechanisms are often used for entity authentication and data - # origin authentication with integrity." - # - # * keyEncipherment ==> for plain RSA cipher suites - # "The keyEncipherment bit is asserted when the subject public key is used for - # key transport. For example, when an RSA key is to be used for key management, - # then this bit is set." - # - # * keyAgreement ==> for used with DH, not RSA. - # "The keyAgreement bit is asserted when the subject public key is used for key - # agreement. For example, when a Diffie-Hellman key is to be used for key - # management, then this bit is set." - # - # digest options: SHA512, SHA256, SHA1 - # - def server_signing_profile(node) - { - "digest" => provider.ca.server_certificates.digest, - "extensions" => { - "keyUsage" => { - "usage" => ["digitalSignature", "keyEncipherment"] - }, - "extendedKeyUsage" => { - "usage" => ["serverAuth", "clientAuth"] - }, - "subjectAltName" => { - "ips" => [node.ip_address], - "dns_names" => dns_names_for_node(node) - } - } - } + log :fetching, "new certificate from letsencrypt.org" + assert_no_errors!("could not renew certificate") do + cert = acme.get_certificate(csr) + end + log 'success', color: :green, style: :bold + write_file!([:commercial_cert, domain], cert.fullchain_to_pem) + log 'You should now run `leap deploy` to deploy the new certificate.' end # - # This is used when signing the main cert for the provider's domain - # with our own CA (for testing purposes). Typically, this cert would - # be purchased from a commercial CA, and not signed this way. + # Returns a hash of nodes that match this domain. It also checks: # - def domain_test_signing_profile - { - "digest" => "SHA256", - "extensions" => { - "keyUsage" => { - "usage" => ["digitalSignature", "keyEncipherment"] - }, - "extendedKeyUsage" => { - "usage" => ["serverAuth"] - } - } - } - end - + # * a node configuration has this domain + # * the dns for the domain exists # - # This is used when signing a dummy client certificate that is only to be - # used for testing. + # This method will bail if any checks fail. # - def client_test_signing_profile - { - "digest" => "SHA256", - "extensions" => { - "keyUsage" => { - "usage" => ["digitalSignature"] - }, - "extendedKeyUsage" => { - "usage" => ["clientAuth"] - } - } - } - end - - def dns_names_for_node(node) - names = [node.domain.internal, node.domain.full] - if node['dns'] && node.dns['aliases'] && node.dns.aliases.any? - names += node.dns.aliases + def nodes_for_domain(domain) + bail! { log 'Argument DOMAIN is required' } if domain.nil? || domain.empty? + nodes = manager.nodes['dns.aliases' => domain] + if nodes.empty? + bail! :error, "There are no nodes configured for domain `%s`" % domain end - names.compact! - names.sort! - names.uniq! - return names + begin + ips = Socket.getaddrinfo(domain, 'http').map {|record| record[2]}.uniq + nodes = nodes['ip_address' => ips] + if nodes.empty? + bail! do + log :error, "The domain `%s` resolves to [%s]" % [domain, ips.join(', ')] + log :error, "But there no nodes configured for this domain with these adddresses." + end + end + rescue SocketError + bail! :error, "Could not resolve the DNS for `#{domain}`. Without a DNS " + + "entry for this domain, authorization will not work." + end + return nodes end # - # For cert serial numbers, we need a non-colliding number less than 160 bits. - # md5 will do nicely, since there is no need for a secure hash, just a short one. - # (md5 is 128 bits) + # runs the following checks on the domain: # - def cert_serial_number(domain_name) - Digest::MD5.hexdigest("#{domain_name} -- #{Time.now}").to_i(16) - end - + # * we are able to get /.well-known/acme-challenge/ok # - # for the random common name, we need a text string that will be unique across all certs. - # ruby 1.8 doesn't have a built-in uuid generator, or we would use SecureRandom.uuid + # This method will bail if any checks fail. # - def random_common_name(domain_name) - cert_serial_number(domain_name).to_s(36) - end - - # prints CertificateAuthority::DistinguishedName fields - def print_dn(dn) - fields = {} - [:common_name, :locality, :state, :country, :organization, :organizational_unit, :email_address].each do |attr| - fields[attr] = dn.send(attr) if dn.send(attr) - end - fields.inspect - end - - ## - ## TIME HELPERS - ## - ## note: we use 'yesterday' instead of 'today', because times are in UTC, and some people on the planet - ## are behind UTC. - ## - - def yesterday - t = Time.now - 24*24*60 - Time.utc t.year, t.month, t.day - end - - def yesterday_advance(string) - number, unit = string.split(' ') - unless ['years', 'months', 'days', 'hours', 'minutes'].include? unit - bail!("The time property '#{string}' is missing a unit (one of: years, months, days, hours, minutes).") - end - unless number.to_i.to_s == number - bail!("The time property '#{string}' is missing a number.") + def domain_ready_for_acme!(domain) + begin + uri = URI("https://#{domain}/.well-known/acme-challenge/ok") + options = { + use_ssl: true, + open_timeout: 5, + verify_mode: OpenSSL::SSL::VERIFY_NONE + } + Net::HTTP.start(uri.host, uri.port, options) do |http| + http.request(Net::HTTP::Get.new(uri)) do |response| + if !response.is_a?(Net::HTTPSuccess) + bail!(:error, "Could not GET %s" % uri) do + log "%s %s" % [response.code, response.message] + log "You may need to run `leap deploy`" + end + end + end + end + rescue Errno::ETIMEDOUT, Net::OpenTimeout + bail! :error, "Connection attempt timed out: %s" % uri + rescue Interrupt + bail! + rescue StandardError => exc + bail!(:error, "Could not GET %s" % uri) do + log exc.to_s + end end - yesterday.advance(unit.to_sym => number.to_i) end end; end diff --git a/lib/leap_cli/commands/compile.rb b/lib/leap_cli/commands/compile.rb index f9079279..92c879d7 100644 --- a/lib/leap_cli/commands/compile.rb +++ b/lib/leap_cli/commands/compile.rb @@ -284,7 +284,6 @@ remove this directory if you don't use it. # note: we use the default provider for all nodes, because we use it # to generate hostnames that are relative to the default domain. provider = manager.env('default').provider - hosts_seen = {} lines = [] # diff --git a/lib/leap_cli/commands/db.rb b/lib/leap_cli/commands/db.rb index 5307ac4d..227d429d 100644 --- a/lib/leap_cli/commands/db.rb +++ b/lib/leap_cli/commands/db.rb @@ -50,7 +50,7 @@ module LeapCli; module Commands def destroy_all_dbs(nodes) ssh_connect(nodes) do |ssh| - ssh.run('/etc/init.d/bigcouch stop && test ! -z "$(ls /opt/bigcouch/var/lib/ 2> /dev/null)" && rm -r /opt/bigcouch/var/lib/* && echo "All DBs destroyed" || echo "DBs already destroyed"') + ssh.run('/etc/init.d/couchdb stop && test ! -z "$(ls /var/lib/couchdb 2> /dev/null)" && rm -r /var/lib/couchdb/* && echo "All DBs destroyed" || echo "DBs already destroyed"') end end diff --git a/lib/leap_cli/commands/deploy.rb b/lib/leap_cli/commands/deploy.rb index 9dd190ab..91e25a96 100644 --- a/lib/leap_cli/commands/deploy.rb +++ b/lib/leap_cli/commands/deploy.rb @@ -29,57 +29,7 @@ module LeapCli :arg_name => 'IPADDRESS' c.action do |global,options,args| - - if options[:dev] != true - init_submodules - end - - nodes = manager.filter!(args, :disabled => false) - if nodes.size > 1 - say "Deploying to these nodes: #{nodes.keys.join(', ')}" - if !global[:yes] && !agree("Continue? ") - quit! "OK. Bye." - end - end - - environments = nodes.field('environment').uniq - if environments.empty? - environments = [nil] - end - environments.each do |env| - check_platform_pinning(env, global) - end - - # compile hiera files for all the nodes in every environment that is - # being deployed and only those environments. - compile_hiera_files(manager.filter(environments), false) - - ssh_connect(nodes, connect_options(options)) do |ssh| - ssh.leap.log :checking, 'node' do - ssh.leap.check_for_no_deploy - ssh.leap.assert_initialized - end - ssh.leap.log :synching, "configuration files" do - sync_hiera_config(ssh) - sync_support_files(ssh) - end - ssh.leap.log :synching, "puppet manifests" do - sync_puppet_files(ssh) - end - unless options[:sync] - ssh.leap.log :applying, "puppet" do - ssh.puppet.apply(:verbosity => [LeapCli.log_level,5].min, - :tags => tags(options), - :force => options[:force], - :info => deploy_info, - :downgrade => options[:downgrade] - ) - end - end - end - if !Util.exit_status.nil? && Util.exit_status != 0 - log :warning, "puppet did not finish successfully." - end + run_deploy(global, options, args) end end @@ -94,19 +44,91 @@ module LeapCli c.switch :last, :desc => 'Show last deploy only', :negatable => false c.action do |global,options,args| - if options[:last] == true - lines = 1 - else - lines = 10 + run_history(global, options, args) + end + end + + private + + def run_deploy(global, options, args) + require 'leap_cli/ssh' + + if options[:dev] != true + init_submodules + end + + nodes = manager.filter!(args, :disabled => false) + if nodes.size > 1 + say "Deploying to these nodes: #{nodes.keys.join(', ')}" + if !global[:yes] && !agree("Continue? ") + quit! "OK. Bye." end - nodes = manager.filter!(args) - ssh_connect(nodes, connect_options(options)) do |ssh| - ssh.leap.history(lines) + end + + environments = nodes.field('environment').uniq + if environments.empty? + environments = [nil] + end + environments.each do |env| + check_platform_pinning(env, global) + end + + # compile hiera files for all the nodes in every environment that is + # being deployed and only those environments. + compile_hiera_files(manager.filter(environments), false) + + log :checking, 'nodes' do + SSH.remote_command(nodes, options) do |ssh, host| + begin + ssh.scripts.check_for_no_deploy + ssh.scripts.assert_initialized + rescue SSH::ExecuteError + # skip nodes with errors, but run others + nodes.delete(host.hostname) + end + end + end + + if nodes.empty? + return + end + + log :synching, "configuration files" do + sync_hiera_config(nodes, options) + sync_support_files(nodes, options) + end + log :synching, "puppet manifests" do + sync_puppet_files(nodes, options) + end + + unless options[:sync] + log :applying, "puppet" do + SSH.remote_command(nodes, options) do |ssh, host| + ssh.scripts.puppet_apply( + :verbosity => [LeapCli.log_level,5].min, + :tags => tags(options), + :force => options[:force], + :info => deploy_info, + :downgrade => options[:downgrade] + ) + end end end end - private + def run_history(global, options, args) + require 'leap_cli/ssh' + + if options[:last] == true + lines = 1 + else + lines = 10 + end + nodes = manager.filter!(args) + SSH.remote_command(nodes, options) do |ssh, host| + ssh.scripts.history(lines) + end + end def forcible_prompt(forced, msg, prompt) say(msg) @@ -211,56 +233,51 @@ module LeapCli end end - def sync_hiera_config(ssh) - ssh.rsync.update do |server| - node = manager.node(server.host) + def sync_hiera_config(nodes, options) + SSH.remote_sync(nodes, options) do |sync, host| + node = manager.node(host.hostname) hiera_file = Path.relative_path([:hiera, node.name]) - ssh.leap.log hiera_file + ' -> ' + node.name + ':' + Leap::Platform.hiera_path - { - :source => hiera_file, - :dest => Leap::Platform.hiera_path, - :flags => "-rltp --chmod=u+rX,go-rwx" - } + sync.log hiera_file + ' -> ' + node.name + ':' + Leap::Platform.hiera_path + sync.source = hiera_file + sync.dest = Leap::Platform.hiera_path + sync.flags = "-rltp --chmod=u+rX,go-rwx" + sync.exec end end # # sync various support files. # - def sync_support_files(ssh) - dest_dir = Leap::Platform.files_dir + def sync_support_files(nodes, options) + dest_dir = Leap::Platform.files_dir custom_files = build_custom_file_list - ssh.rsync.update do |server| - node = manager.node(server.host) + SSH.remote_sync(nodes, options) do |sync, host| + node = manager.node(host.hostname) files_to_sync = node.file_paths.collect {|path| Path.relative_path(path, Path.provider) } files_to_sync += custom_files if files_to_sync.any? - ssh.leap.log(files_to_sync.join(', ') + ' -> ' + node.name + ':' + dest_dir) - { - :chdir => Path.named_path(:files_dir), - :source => ".", - :dest => dest_dir, - :excludes => "*", - :includes => calculate_includes_from_files(files_to_sync, '/files'), - :flags => "-rltp --chmod=u+rX,go-rwx --relative --delete --delete-excluded --copy-links" - } - else - nil + sync.log(files_to_sync.join(', ') + ' -> ' + node.name + ':' + dest_dir) + sync.chdir = Path.named_path(:files_dir) + sync.source = "." + sync.dest = dest_dir + sync.excludes = "*" + sync.includes = calculate_includes_from_files(files_to_sync, '/files') + sync.flags = "-rltp --chmod=u+rX,go-rwx --relative --delete --delete-excluded --copy-links" + sync.exec end end end - def sync_puppet_files(ssh) - ssh.rsync.update do |server| - ssh.leap.log(Path.platform + '/[bin,tests,puppet] -> ' + server.host + ':' + Leap::Platform.leap_dir) - { - :dest => Leap::Platform.leap_dir, - :source => '.', - :chdir => Path.platform, - :excludes => '*', - :includes => ['/bin', '/bin/**', '/puppet', '/puppet/**', '/tests', '/tests/**'], - :flags => "-rlt --relative --delete --copy-links" - } + def sync_puppet_files(nodes, options) + SSH.remote_sync(nodes, options) do |sync, host| + sync.log(Path.platform + '/[bin,tests,puppet] -> ' + host.hostname + ':' + Leap::Platform.leap_dir) + sync.dest = Leap::Platform.leap_dir + sync.source = '.' + sync.chdir = Path.platform + sync.excludes = '*' + sync.includes = ['/bin', '/bin/**', '/puppet', '/puppet/**', '/tests', '/tests/server-tests', '/tests/server-tests/**'] + sync.flags = "-rlt --relative --delete --copy-links" + sync.exec end end @@ -269,7 +286,7 @@ module LeapCli # repository. # def init_submodules - return unless is_git_directory?(Path.platform) + return unless is_git_directory?(Path.platform) && !is_git_subrepo?(Path.platform) Dir.chdir Path.platform do assert_run! "git submodule sync" statuses = assert_run! "git submodule status" diff --git a/lib/leap_cli/commands/facts.rb b/lib/leap_cli/commands/facts.rb index 11329ccc..74ef463d 100644 --- a/lib/leap_cli/commands/facts.rb +++ b/lib/leap_cli/commands/facts.rb @@ -79,15 +79,18 @@ module LeapCli; module Commands private def update_facts(global_options, options, args) + require 'leap_cli/ssh' nodes = manager.filter(args, :local => false, :disabled => false) new_facts = {} - ssh_connect(nodes) do |ssh| - ssh.leap.run_with_progress(facter_cmd) do |response| - node = manager.node(response[:host]) + SSH.remote_command(nodes) do |ssh, host| + response = ssh.capture(facter_cmd, :log_output => false) + if response + node = manager.node(host.hostname) if node - new_facts[node.name] = response[:data].strip + new_facts[node.name] = response.strip + log 'done', :host => host.to_s else - log :warning, 'Could not find node for hostname %s' % response[:host] + log :warning, 'Could not find node for hostname %s' % host end end end diff --git a/lib/leap_cli/commands/info.rb b/lib/leap_cli/commands/info.rb index 52225a94..a49c20c9 100644 --- a/lib/leap_cli/commands/info.rb +++ b/lib/leap_cli/commands/info.rb @@ -5,10 +5,17 @@ module LeapCli; module Commands arg_name 'FILTER' command [:info] do |c| c.action do |global,options,args| - nodes = manager.filter!(args) - ssh_connect(nodes, connect_options(options)) do |ssh| - ssh.leap.debug - end + run_info(global, options, args) + end + end + + private + + def run_info(global, options, args) + require 'leap_cli/ssh' + nodes = manager.filter!(args) + SSH.remote_command(nodes, options) do |ssh, host| + ssh.scripts.debug end end diff --git a/lib/leap_cli/commands/inspect.rb b/lib/leap_cli/commands/inspect.rb index 20654fa7..b71da80e 100644 --- a/lib/leap_cli/commands/inspect.rb +++ b/lib/leap_cli/commands/inspect.rb @@ -25,27 +25,22 @@ module LeapCli; module Commands "PEM certificate request" => :inspect_x509_csr } + SUFFIX_MAP = { + ".json" => :inspect_unknown_json, + ".key" => :inspect_x509_key + } + def inspection_method(object) - if File.exists?(object) + if File.exist?(object) ftype = `file #{object}`.split(':').last.strip + suffix = File.extname(object) log 2, "file is of type '#{ftype}'" if FTYPE_MAP[ftype] FTYPE_MAP[ftype] - elsif File.extname(object) == ".json" - full_path = File.expand_path(object, Dir.pwd) - if path_match?(:node_config, full_path) - :inspect_node - elsif path_match?(:service_config, full_path) - :inspect_service - elsif path_match?(:tag_config, full_path) - :inspect_tag - elsif path_match?(:provider_config, full_path) || path_match?(:provider_env_config, full_path) - :inspect_provider - elsif path_match?(:common_config, full_path) - :inspect_common - else - nil - end + elsif SUFFIX_MAP[suffix] + SUFFIX_MAP[suffix] + else + nil end elsif manager.nodes[object] :inspect_node @@ -72,8 +67,10 @@ module LeapCli; module Commands end def inspect_x509_cert(file_path, options) + require 'leap_cli/x509' assert_bin! 'openssl' puts assert_run! 'openssl x509 -in %s -text -noout' % file_path + log 0, :"SHA1 fingerprint", X509.fingerprint("SHA1", file_path) log 0, :"SHA256 fingerprint", X509.fingerprint("SHA256", file_path) end @@ -123,6 +120,23 @@ module LeapCli; module Commands end end + def inspect_unknown_json(arg, options) + full_path = File.expand_path(arg, Dir.pwd) + if path_match?(:node_config, full_path) + inspect_node(arg, options) + elsif path_match?(:service_config, full_path) + inspect_service(arg, options) + elsif path_match?(:tag_config, full_path) + inspect_tag(arg, options) + elsif path_match?(:provider_config, full_path) || path_match?(:provider_env_config, full_path) + inspect_provider(arg, options) + elsif path_match?(:common_config, full_path) + inspect_common(arg, options) + else + inspect_json(arg, options) + end + end + # # helpers # diff --git a/lib/leap_cli/commands/list.rb b/lib/leap_cli/commands/list.rb index aa425432..1b3efc27 100644 --- a/lib/leap_cli/commands/list.rb +++ b/lib/leap_cli/commands/list.rb @@ -1,5 +1,3 @@ -require 'command_line_reporter' - module LeapCli; module Commands desc 'List nodes and their classifications' @@ -15,33 +13,38 @@ module LeapCli; module Commands c.flag 'print', :desc => 'What attributes to print (optional)' c.switch 'disabled', :desc => 'Include disabled nodes in the list.', :negatable => false c.action do |global_options,options,args| - # don't rely on default manager(), because we want to pass custom options to load() - manager = LeapCli::Config::Manager.new - if global_options[:color] - colors = ['cyan', 'white'] - else - colors = [nil, nil] - end - puts - manager.load(:include_disabled => options['disabled'], :continue_on_error => true) - if options['print'] - print_node_properties(manager.filter(args), options['print']) - else - if args.any? - NodeTable.new(manager.filter(args), colors).run - else - environment = LeapCli.leapfile.environment || '_all_' - TagTable.new('SERVICES', manager.env(environment).services, colors).run - TagTable.new('TAGS', manager.env(environment).tags, colors).run - NodeTable.new(manager.filter(), colors).run - end - end + do_list(global_options, options, args) end end private - def self.print_node_properties(nodes, properties) + def do_list(global, options, args) + require 'leap_cli/util/console_table' + # don't rely on default manager(), because we want to pass custom options to load() + manager = LeapCli::Config::Manager.new + if global[:color] + colors = [:cyan, nil] + else + colors = [nil, nil] + end + puts + manager.load(:include_disabled => options['disabled'], :continue_on_error => true) + if options['print'] + print_node_properties(manager.filter(args), options['print']) + else + if args.any? + NodeTable.new(manager.filter(args), colors).run + else + environment = LeapCli.leapfile.environment || '_all_' + TagTable.new('SERVICES', manager.env(environment).services, colors).run + TagTable.new('TAGS', manager.env(environment).tags, colors).run + NodeTable.new(manager.filter(), colors).run + end + end + end + + def print_node_properties(nodes, properties) properties = properties.split(',') max_width = nodes.keys.inject(0) {|max,i| [i.size,max].max} nodes.each_node do |node| @@ -62,8 +65,7 @@ module LeapCli; module Commands puts end - class TagTable - include CommandLineReporter + class TagTable < LeapCli::Util::ConsoleTable def initialize(heading, tag_list, colors) @heading = heading @tag_list = tag_list @@ -71,29 +73,24 @@ module LeapCli; module Commands end def run tags = @tag_list.keys.select{|tag| tag !~ /^_/}.sort # sorted list of tags, excluding _partials - max_width = [20, (tags+[@heading]).inject(0) {|max,i| [i.size,max].max}].max - table :border => false do - row :color => @colors[0] do - column @heading, :align => 'right', :width => max_width - column "NODES", :width => HighLine::SystemExtensions.terminal_size.first - max_width - 2, :padding => 2 + table do + row(color: @colors[0]) do + column @heading, align: 'right', min_width: 20 + column "NODES" end tags.each do |tag| next if @tag_list[tag].node_list.empty? - row :color => @colors[1] do + row(color: @colors[1]) do column tag column @tag_list[tag].node_list.keys.sort.join(', ') end end end - vertical_spacing + draw_table end end - # - # might be handy: HighLine::SystemExtensions.terminal_size.first - # - class NodeTable - include CommandLineReporter + class NodeTable < LeapCli::Util::ConsoleTable def initialize(node_list, colors) @node_list = node_list @colors = colors @@ -103,29 +100,25 @@ module LeapCli; module Commands [node_name, @node_list[node_name].services.sort.join(', '), @node_list[node_name].tags.sort.join(', ')] end unless rows.any? - puts Paint["no results", :red] + puts " = " + LeapCli.logger.colorize("no results", :red) puts return end - padding = 2 - max_node_width = [20, (rows.map{|i|i[0]} + ["NODES"] ).inject(0) {|max,i| [i.size,max].max}].max - max_service_width = (rows.map{|i|i[1]} + ["SERVICES"]).inject(0) {|max,i| [i.size+padding+padding,max].max} - max_tag_width = (rows.map{|i|i[2]} + ["TAGS"] ).inject(0) {|max,i| [i.size,max].max} - table :border => false do - row :color => @colors[0] do - column "NODES", :align => 'right', :width => max_node_width - column "SERVICES", :width => max_service_width, :padding => 2 - column "TAGS", :width => max_tag_width + table do + row(color: @colors[0]) do + column "NODES", align: 'right', min_width: 20 + column "SERVICES" + column "TAGS" end rows.each do |r| - row :color => @colors[1] do + row(color: @colors[1]) do column r[0] column r[1] column r[2] end end end - vertical_spacing + draw_table end end diff --git a/lib/leap_cli/commands/node.rb b/lib/leap_cli/commands/node.rb index a23661b3..60540de9 100644 --- a/lib/leap_cli/commands/node.rb +++ b/lib/leap_cli/commands/node.rb @@ -3,8 +3,6 @@ # but all other `node x` commands live here. # -autoload :IPAddr, 'ipaddr' - module LeapCli; module Commands ## @@ -18,33 +16,16 @@ module LeapCli; module Commands "The format is property_name:value.", "For example: `leap node add web1 ip_address:1.2.3.4 services:webapp`.", "To set nested properties, property name can contain '.', like so: `leap node add web1 ssh.port:44`", - "Separeate multiple values for a single property with a comma, like so: `leap node add mynode services:webapp,dns`"].join("\n\n") + "Separate multiple values for a single property with a comma, like so: `leap node add mynode services:webapp,dns`"].join("\n\n") node.arg_name 'NAME [SEED]' # , :optional => false, :multiple => false node.command :add do |add| - add.switch :local, :desc => 'Make a local testing node (by automatically assigning the next available local IP address). Local nodes are run as virtual machines on your computer.', :negatable => false + add.switch :local, :desc => 'Make a local testing node (by assigning the next available local IP address). Local nodes are run as virtual machines on your computer.', :negatable => false + add.switch :vm, :desc => 'Make a remote virtual machine for this node. Requires a valid cloud.json configuration.', :negatable => false add.action do |global_options,options,args| - # argument sanity checks - name = args.first - assert_valid_node_name!(name, options[:local]) - assert_files_missing! [:node_config, name] - - # create and seed new node - node = Config::Node.new(manager.env) - if options[:local] - node['ip_address'] = pick_next_vagrant_ip_address - end - seed_node_data_from_cmd_line(node, args[1..-1]) - seed_node_data_from_template(node) - validate_ip_address(node) - begin - node['name'] = name - json = node.dump_json(:exclude => ['name']) - write_file!([:node_config, name], json + "\n") - if file_exists? :ca_cert, :ca_key - generate_cert_for_node(manager.reload_node!(node)) - end - rescue LeapCli::ConfigError => exc - remove_node_files(name) + if options[:vm] + do_vm_add(global_options, options, args) + else + do_node_add(global_options, options, args) end end end @@ -53,15 +34,7 @@ module LeapCli; module Commands node.arg_name 'OLD_NAME NEW_NAME' node.command :mv do |mv| mv.action do |global_options,options,args| - node = get_node_from_args(args, include_disabled: true) - new_name = args.last - assert_valid_node_name!(new_name, node.vagrant?) - ensure_dir [:node_files_dir, new_name] - Leap::Platform.node_files.each do |path| - rename_file! [path, node.name], [path, new_name] - end - remove_directory! [:node_files_dir, node.name] - rename_node_facts(node.name, new_name) + do_node_move(global_options, options, args) end end @@ -69,12 +42,7 @@ module LeapCli; module Commands node.arg_name 'NAME' #:optional => false #, :multiple => false node.command :rm do |rm| rm.action do |global_options,options,args| - node = get_node_from_args(args, include_disabled: true) - remove_node_files(node.name) - if node.vagrant? - vagrant_command("destroy --force", [node.name]) - end - remove_node_facts(node.name) + do_node_rm(global_options, options, args) end end end @@ -93,96 +61,69 @@ module LeapCli; module Commands node end - def seed_node_data_from_cmd_line(node, args) - args.each do |seed| - key, value = seed.split(':', 2) - value = format_seed_value(value) - assert! key =~ /^[0-9a-z\._]+$/, "illegal characters used in property '#{key}'" - if key =~ /\./ - key_parts = key.split('.') - final_key = key_parts.pop - current_object = node - key_parts.each do |key_part| - current_object[key_part] ||= Config::Object.new - current_object = current_object[key_part] - end - current_object[final_key] = value - else - node[key] = value - end - end - end + protected # - # load "new node template" information into the `node`, modifying `node`. - # values in the template will not override existing node values. + # additionally called by `leap vm add` # - def seed_node_data_from_template(node) - node.inherit_from!(manager.template('common')) - [node['services']].flatten.each do |service| - if service - template = manager.template(service) - if template - node.inherit_from!(template) - end - end + def do_node_add(global, options, args) + name = args.first + unless global[:force] + assert_files_missing! [:node_config, name] end - end - - def remove_node_files(node_name) - (Leap::Platform.node_files + [:node_files_dir]).each do |path| - remove_file! [path, node_name] + node = Config::Node.new(manager.env) + node['name'] = name + if options[:ip_address] + node['ip_address'] = options[:ip_address] + elsif options[:local] + node['ip_address'] = pick_next_vagrant_ip_address end + node.seed_from_args(args[1..-1]) + node.seed_from_template + node.validate! + node.write_configs + # reapply inheritance, since tags/services might have changed: + node = manager.reload_node!(node) + node.generate_cert end - # - # conversions: - # - # "x,y,z" => ["x","y","z"] - # - # "22" => 22 - # - # "5.1" => 5.1 - # - def format_seed_value(v) - if v =~ /,/ - v = v.split(',') - v.map! do |i| - i = i.to_i if i.to_i.to_s == i - i = i.to_f if i.to_f.to_s == i - i - end - else - v = v.to_i if v.to_i.to_s == v - v = v.to_f if v.to_f.to_s == v - end - return v - end + private - def validate_ip_address(node) - if node['ip_address'] == "REQUIRED" - bail! do - log :error, "ip_address is not set. Specify with `leap node add NAME ip_address:ADDRESS`." - end + def do_node_move(global, options, args) + node = get_node_from_args(args, include_disabled: true) + new_name = args.last + Config::Node.validate_name!(new_name, node.vagrant?) + ensure_dir [:node_files_dir, new_name] + Leap::Platform.node_files.each do |path| + rename_file! [path, node.name], [path, new_name] end - IPAddr.new(node['ip_address']) - rescue ArgumentError - bail! do - if node['ip_address'] - log :invalid, "ip_address #{node['ip_address'].inspect}" - else - log :missing, "ip_address" - end + remove_directory! [:node_files_dir, node.name] + rename_node_facts(node.name, new_name) + if node.vm_id? + node['name'] = new_name + bind_server_to_node(node.vm.id, node, options) end end - def assert_valid_node_name!(name, local=false) - assert! name, 'No <node-name> specified.' - if local - assert! name =~ /^[0-9a-z]+$/, "illegal characters used in node name '#{name}' (note: Vagrant does not allow hyphens or underscores)" - else - assert! name =~ /^[0-9a-z-]+$/, "illegal characters used in node name '#{name}' (note: Linux does not allow underscores)" + def do_node_rm(global, options, args) + node = get_node_from_args(args, include_disabled: true) + if node.vm? + if !node.vm_id? + log :warning, "The node #{node.name} is missing a 'vm.id' property. "+ + "You may have a virtual machine instance that is left "+ + "running. Check `leap vm status`" + else + msg = "The node #{node.name} appears to be associated with a virtual machine. " + + "Do you want to also destroy this virtual machine? " + if global[:yes] || agree(msg) + do_vm_rm(global, options, args) + end + end + elsif node.vagrant? + vagrant_command("destroy --force", [node.name]) end + node.remove_files + remove_node_facts(node.name) end end; end diff --git a/lib/leap_cli/commands/node_init.rb b/lib/leap_cli/commands/node_init.rb index 33f6288d..59661295 100644 --- a/lib/leap_cli/commands/node_init.rb +++ b/lib/leap_cli/commands/node_init.rb @@ -6,58 +6,70 @@ module LeapCli; module Commands desc 'Node management' - command :node do |node| - node.desc 'Bootstraps a node or nodes, setting up SSH keys and installing prerequisite packages' - node.long_desc "This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, " + + command :node do |cmd| + cmd.desc 'Bootstraps a node or nodes, setting up SSH keys and installing prerequisite packages' + cmd.long_desc "This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, " + "copying the authorized_keys file, installing packages that are required for deploying, and registering important facts. " + "Node init must be run before deploying to a server, and the server must be running and available via the network. " + "This command only needs to be run once, but there is no harm in running it multiple times." - node.arg_name 'FILTER' - node.command :init do |init| - init.switch 'echo', :desc => 'If set, passwords are visible as you type them (default is hidden)', :negatable => false + cmd.arg_name 'FILTER' + cmd.command :init do |init| + #init.switch 'echo', :desc => 'If set, passwords are visible as you type them (default is hidden)', :negatable => false + # ^^ i am not sure how to get this working with sshkit init.flag :port, :desc => 'Override the default SSH port.', :arg_name => 'PORT' init.flag :ip, :desc => 'Override the default SSH IP address.', :arg_name => 'IPADDRESS' init.action do |global,options,args| - assert! args.any?, 'You must specify a FILTER' - finished = [] - manager.filter!(args).each_node do |node| - is_node_alive(node, options) - save_public_host_key(node, global, options) unless node.vagrant? - update_compiled_ssh_configs - ssh_connect_options = connect_options(options).merge({:bootstrap => true, :echo => options[:echo]}) - ssh_connect(node, ssh_connect_options) do |ssh| - if node.vagrant? - ssh.install_insecure_vagrant_key - end - ssh.install_authorized_keys - ssh.install_prerequisites - unless node.vagrant? - ssh.leap.log(:checking, "SSH host keys") do - ssh.leap.capture(get_ssh_keys_cmd) do |response| - update_local_ssh_host_keys(node, response[:data]) if response[:exitcode] == 0 - end - end - end - ssh.leap.log(:updating, "facts") do - ssh.leap.capture(facter_cmd) do |response| - if response[:exitcode] == 0 - update_node_facts(node.name, response[:data]) - else - log :failed, "to run facter on #{node.name}" - end - end + run_node_init(global, options, args) + end + end + end + + private + + def run_node_init(global, options, args) + require 'leap_cli/ssh' + assert! args.any?, 'You must specify a FILTER' + finished = [] + manager.filter!(args).each_node do |node| + is_node_alive(node, options) + save_public_host_key(node, global, options) unless node.vagrant? + update_compiled_ssh_configs + # allow password auth for new nodes: + options[:auth_methods] = ["publickey", "password"] + if node.vm? + # new AWS virtual machines will only allow login as 'admin' + # before we continue, we must enable root access. + SSH.remote_command(node, options.merge(:user => 'admin')) do |ssh, host| + ssh.scripts.allow_root_ssh + end + end + SSH.remote_command(node, options) do |ssh, host| + if node.vagrant? + ssh.scripts.install_insecure_vagrant_key + end + ssh.scripts.install_authorized_keys + ssh.scripts.install_prerequisites + unless node.vagrant? + ssh.log(:checking, "SSH host keys") do + response = ssh.capture(get_ssh_keys_cmd, :log_output => false) + if response + update_local_ssh_host_keys(node, response) end end - finished << node.name end - log :completed, "initialization of nodes #{finished.join(', ')}" + ssh.log(:updating, "facts") do + response = ssh.capture(facter_cmd) + if response + update_node_facts(node.name, response) + end + end end + finished << node.name end + log :completed, "initialization of nodes #{finished.join(', ')}" end - private - ## ## PRIVATE HELPERS ## @@ -83,7 +95,7 @@ module LeapCli; module Commands pub_key_path = Path.named_path([:node_ssh_pub_key, node.name]) if Path.exists?(pub_key_path) - if host_keys.include? SshKey.load(pub_key_path) + if host_keys.include? SSH::Key.load(pub_key_path) log :trusted, "- Public SSH host key for #{node.name} matches previously saved key", :indent => 1 else bail! do @@ -96,7 +108,7 @@ module LeapCli; module Commands if known_key log :trusted, "- Public SSH host key for #{node.name} is trusted (key found in your ~/.ssh/known_hosts)" else - public_key = SshKey.pick_best_key(host_keys) + public_key = SSH::Key.pick_best_key(host_keys) if public_key.nil? bail!("We got back #{host_keys.size} host keys from #{node.name}, but we can't support any of them.") else @@ -118,7 +130,7 @@ module LeapCli; module Commands # # Get the public host keys for a host using ssh-keyscan. - # Return an array of SshKey objects, one for each key. + # Return an array of SSH::Key objects, one for each key. # def get_public_keys_for_ip(address, port=22) assert_bin!('ssh-keyscan') @@ -130,7 +142,7 @@ module LeapCli; module Commands if output =~ /No route to host/ bail! :failed, 'ssh-keyscan: no route to %s' % address else - keys = SshKey.parse_keys(output) + keys = SSH::Key.parse_keys(output) if keys.empty? bail! "ssh-keyscan got zero host keys back (that we understand)! Output was: #{output}" else @@ -139,7 +151,7 @@ module LeapCli; module Commands end end - # run on the server to generate a string suitable for passing to SshKey.parse_keys() + # run on the server to generate a string suitable for passing to SSH::Key.parse_keys() def get_ssh_keys_cmd "/bin/grep ^HostKey /etc/ssh/sshd_config | /usr/bin/awk '{print $2 \".pub\"}' | /usr/bin/xargs /bin/cat" end @@ -149,10 +161,10 @@ module LeapCli; module Commands # stored locally. In these cases, ask the user if they want to upgrade. # def update_local_ssh_host_keys(node, remote_keys_string) - remote_keys = SshKey.parse_keys(remote_keys_string) + remote_keys = SSH::Key.parse_keys(remote_keys_string) return unless remote_keys.any? - current_key = SshKey.load(Path.named_path([:node_ssh_pub_key, node.name])) - best_key = SshKey.pick_best_key(remote_keys) + current_key = SSH::Key.load(Path.named_path([:node_ssh_pub_key, node.name])) + best_key = SSH::Key.pick_best_key(remote_keys) return unless best_key && current_key if current_key != best_key say(" One of the SSH host keys for node '#{node.name}' is better than what you currently have trusted.") diff --git a/lib/leap_cli/commands/open.rb b/lib/leap_cli/commands/open.rb new file mode 100644 index 00000000..3de97298 --- /dev/null +++ b/lib/leap_cli/commands/open.rb @@ -0,0 +1,103 @@ +module LeapCli + module Commands + + desc 'Opens useful URLs in a web browser.' + long_desc "NAME can be one or more of: monitor, web, docs, bug" + arg_name 'NAME' + command :open do |c| + c.flag :env, :desc => 'Which environment to use (optional).', :arg_name => 'ENVIRONMENT' + c.switch :ip, :desc => 'To get around HSTS or DNS, open the URL using the IP address instead of the domain (optional).' + c.action do |global_options,options,args| + do_open_cmd(global_options, options, args) + end + end + + private + + def do_open_cmd(global, options, args) + env = options[:env] || LeapCli.leapfile.environment + args.each do |name| + if name == 'monitor' || name == 'nagios' + open_nagios(env, options[:ip]) + elsif name == 'web' || name == 'webapp' + open_webapp(env, options[:ip]) + elsif name == 'docs' || name == 'help' || name == 'doc' + open_url("https://leap.se/docs") + elsif name == 'bug' || name == 'feature' || name == 'bugreport' + open_url("https://leap.se/code") + else + bail! "'#{name}' is not a recognized URL." + end + end + end + + def find_node_with_service(service, environment) + nodes = manager.nodes[:services => service] + node = nil + if nodes.size == 0 + bail! "No nodes with '#{service}' service." + elsif nodes.size == 1 + node = nodes.values.first + elsif nodes.size > 1 + if environment + node = nodes[:environment => environment].values.first + if node.nil? + bail! "No nodes with '#{service}' service." + end + else + node_list = nodes.values + list = node_list.map {|i| "#{i.name} (#{i.environment})"} + index = numbered_choice_menu("Which #{service}?", list) do |line, i| + say("#{i+1}. #{line}") + end + node = node_list[index] + end + end + return node + end + + def pick_domain(node, ip) + bail! "monitor missing webapp service" unless node["webapp"] + if ip + domain = node["ip_address"] + else + domain = node["webapp"]["domain"] + bail! "webapp domain is missing" unless !domain.empty? + end + return domain + end + + def open_webapp(environment, ip) + node = find_node_with_service('webapp', environment) + domain = pick_domain(node, ip) + open_url("https://%s" % domain) + end + + def open_nagios(environment, ip) + node = find_node_with_service('monitor', environment) + domain = pick_domain(node, ip) + username = 'nagiosadmin' + password = manager.secrets.retrieve("nagios_admin_password", node.environment) + bail! "unable to find nagios_admin_password" unless !password.nil? && !password.empty? + open_url("https://%s:%s@%s/nagios3" % [username, password, domain]) + end + + def open_url(url) + log :opening, url + if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/ + system %(start "#{url}") + elsif RbConfig::CONFIG['host_os'] =~ /darwin/ + system %(open "#{url}") + elsif RbConfig::CONFIG['host_os'] =~ /linux|bsd/ + ['xdg-open', 'sensible-browser', 'gnome-open', 'kde-open'].each do |cmd| + if !`which #{cmd}`.strip.empty? + system %(#{cmd} "#{url}") + return + end + end + log :error, 'no command found to launch browser window.' + end + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/run.rb b/lib/leap_cli/commands/run.rb new file mode 100644 index 00000000..cad9b7a0 --- /dev/null +++ b/lib/leap_cli/commands/run.rb @@ -0,0 +1,50 @@ +module LeapCli; module Commands + + desc 'Run a shell command remotely' + long_desc "Runs the specified command COMMAND on each node in the FILTER set. " + + "For example, `leap run 'uname -a' webapp`" + arg_name 'COMMAND FILTER' + command :run do |c| + c.switch 'stream', :default => false, :desc => 'If set, stream the output as it arrives. (default: --stream for a single node, --no-stream for multiple nodes)' + c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server.' + c.action do |global, options, args| + run_shell_command(global, options, args) + end + end + + private + + def run_shell_command(global, options, args) + require 'leap_cli/ssh' + cmd = args[0] + filter = args[1..-1] + cmd = global[:force] ? cmd : LeapCli::SSH::Options.sanitize_command(cmd) + nodes = manager.filter!(filter) + if nodes.size == 1 || options[:stream] + stream_command(nodes, cmd, options) + else + capture_command(nodes, cmd, options) + end + end + + def capture_command(nodes, cmd, options) + SSH.remote_command(nodes, options) do |ssh, host| + output = ssh.capture(cmd, :log_output => false) + if output + logger = LeapCli.new_logger + logger.log(:ran, "`" + cmd + "`", host: host.hostname, color: :green) do + logger.log(output, wrap: true) + end + end + end + end + + def stream_command(nodes, cmd, options) + SSH.remote_command(nodes, options) do |ssh, host| + ssh.stream(cmd, :log_cmd => true, :log_finish => true, :fail_msg => 'oops') + end + end + +end; end + + diff --git a/lib/leap_cli/commands/ssh.rb b/lib/leap_cli/commands/ssh.rb index 3887618e..03192071 100644 --- a/lib/leap_cli/commands/ssh.rb +++ b/lib/leap_cli/commands/ssh.rb @@ -69,20 +69,6 @@ module LeapCli; module Commands protected - # - # allow for ssh overrides of all commands that use ssh_connect - # - def connect_options(options) - connect_options = {:ssh_options=>{}} - if options[:port] - connect_options[:ssh_options][:port] = options[:port] - end - if options[:ip] - connect_options[:ssh_options][:host_name] = options[:ip] - end - return connect_options - end - def ssh_config_help_message puts "" puts "Are 'too many authentication failures' getting you down?" @@ -193,7 +179,8 @@ module LeapCli; module Commands "-o 'UserKnownHostsFile=/dev/null'" ] if node.vagrant? - options << "-i #{vagrant_ssh_key_file}" # use the universal vagrant insecure key + # use the universal vagrant insecure key: + options << "-i #{LeapCli::Util::Vagrant.vagrant_ssh_key_file}" options << "-o IdentitiesOnly=yes" # force the use of the insecure vagrant key options << "-o 'StrictHostKeyChecking=no'" # blindly accept host key and don't save it # (since userknownhostsfile is /dev/null) diff --git a/lib/leap_cli/commands/test.rb b/lib/leap_cli/commands/test.rb index 73207b31..70eb00fd 100644 --- a/lib/leap_cli/commands/test.rb +++ b/lib/leap_cli/commands/test.rb @@ -7,24 +7,7 @@ module LeapCli; module Commands test.command :run do |run| run.switch 'continue', :desc => 'Continue over errors and failures (default is --no-continue).', :negatable => true run.action do |global_options,options,args| - test_order = File.join(Path.platform, 'tests/order.rb') - if File.exists?(test_order) - require test_order - end - manager.filter!(args).names_in_test_dependency_order.each do |node_name| - node = manager.nodes[node_name] - begin - ssh_connect(node) do |ssh| - ssh.run(test_cmd(options)) - end - rescue Capistrano::CommandError => exc - if options[:continue] - exit_status(1) - else - bail! - end - end - end + do_test_run(global_options, options, args) end end @@ -40,6 +23,28 @@ module LeapCli; module Commands private + def do_test_run(global_options, options, args) + require 'leap_cli/ssh' + test_order = File.join(Path.platform, 'tests/order.rb') + if File.exist?(test_order) + require test_order + end + manager.filter!(args).names_in_test_dependency_order.each do |node_name| + node = manager.nodes[node_name] + begin + SSH::remote_command(node, options) do |ssh, host| + ssh.stream(test_cmd(options), :raise_error => true, :log_wrap => true) + end + rescue LeapCli::SSH::ExecuteError + if options[:continue] + exit_status(1) + else + bail! + end + end + end + end + def test_cmd(options) if options[:continue] "#{Leap::Platform.leap_dir}/bin/run_tests --continue" diff --git a/lib/leap_cli/commands/user.rb b/lib/leap_cli/commands/user.rb index b842e854..1ca92719 100644 --- a/lib/leap_cli/commands/user.rb +++ b/lib/leap_cli/commands/user.rb @@ -13,67 +13,131 @@ module LeapCli module Commands - desc 'Adds a new trusted sysadmin by adding public keys to the "users" directory.' - arg_name 'USERNAME' #, :optional => false, :multiple => false + desc 'Manage trusted sysadmins (DEPRECATED)' + long_desc "Use `leap user add` instead" command :'add-user' do |c| - c.switch 'self', :desc => 'Add yourself as a trusted sysadmin by choosing among the public keys available for the current user.', :negatable => false c.flag 'ssh-pub-key', :desc => 'SSH public key file for this new user' c.flag 'pgp-pub-key', :desc => 'OpenPGP public key file for this new user' - c.action do |global_options,options,args| - username = args.first - if !username.any? - if options[:self] - username ||= `whoami`.strip - else - help! "Either USERNAME argument or --self flag is required." - end - end - if Leap::Platform.reserved_usernames.include? username - bail! %(The username "#{username}" is reserved. Sorry, pick another.) - end + do_add_user(global_options, options, args) + end + end - ssh_pub_key = nil - pgp_pub_key = nil + desc 'Manage trusted sysadmins' + long_desc "Manage the trusted sysadmins that are configured in the 'users' directory." + command :user do |user| + + user.desc 'Adds a new trusted sysadmin' + user.arg_name 'USERNAME' + user.command :add do |c| + c.switch 'self', :desc => 'Add yourself as a trusted sysadmin by choosing among the public keys available for the current user.', :negatable => false + c.flag 'ssh-pub-key', :desc => 'SSH public key file for this new user' + c.flag 'pgp-pub-key', :desc => 'OpenPGP public key file for this new user' + c.action do |global_options,options,args| + do_add_user(global_options, options, args) + end + end - if options['ssh-pub-key'] - ssh_pub_key = read_file!(options['ssh-pub-key']) + user.desc 'Removes a trusted sysadmin' + user.arg_name 'USERNAME' + user.command :rm do |c| + c.action do |global_options,options,args| + do_rm_user(global_options, options, args) end - if options['pgp-pub-key'] - pgp_pub_key = read_file!(options['pgp-pub-key']) + end + + user.desc 'Lists the configured sysadmins' + user.command :ls do |c| + c.action do |global_options,options,args| + do_list_users(global_options, options, args) end + end + + end + private + + def do_add_user(global, options, args) + require 'leap_cli/ssh' + + username = args.first + if !username.any? if options[:self] - ssh_pub_key ||= pick_ssh_key.to_s - pgp_pub_key ||= pick_pgp_key + username ||= `whoami`.strip + else + help! "Either USERNAME argument or --self flag is required." end + end + if Leap::Platform.reserved_usernames.include? username + bail! %(The username "#{username}" is reserved. Sorry, pick another.) + end - assert!(ssh_pub_key, 'Sorry, could not find SSH public key.') + ssh_pub_key = nil + pgp_pub_key = nil - if ssh_pub_key - write_file!([:user_ssh, username], ssh_pub_key) - end - if pgp_pub_key - write_file!([:user_pgp, username], pgp_pub_key) - end + if options['ssh-pub-key'] + ssh_pub_key = read_file!(options['ssh-pub-key']) + end + if options['pgp-pub-key'] + pgp_pub_key = read_file!(options['pgp-pub-key']) + end + if options[:self] + ssh_pub_key ||= pick_ssh_key.to_s + pgp_pub_key ||= pick_pgp_key + end + + assert!(ssh_pub_key, 'Sorry, could not find SSH public key.') + + if ssh_pub_key + write_file!([:user_ssh, username], ssh_pub_key) + end + if pgp_pub_key + write_file!([:user_pgp, username], pgp_pub_key) + end + + update_authorized_keys + end + + def do_rm_user(global, options, args) + dir = [:user_dir, args.first] + if Util.dir_exists?(dir) + Util.remove_file!(dir) update_authorized_keys + else + bail! :error, 'There is no directory `%s`' % Path.named_path(dir) + end + end + + def do_list_users(global, options, args) + require 'leap_cli/ssh' + + Dir.glob(path([:user_ssh, '*'])).each do |keyfile| + username = File.basename(File.dirname(keyfile)) + log username, :color => :cyan do + log Path.relative_path(keyfile) + key = SSH::Key.load(keyfile) + log 'SSH MD5 fingerprint: ' + key.fingerprint(:digest => :md5, :type => :ssh, :encoding => :hex) + log 'SSH SHA256 fingerprint: ' + key.fingerprint(:digest => :sha256, :type => :ssh, :encoding => :base64) + log 'DER MD5 fingerprint: ' + key.fingerprint(:digest => :md5, :type => :der, :encoding => :hex) + end end end # - # let the the user choose among the ssh public keys that we encounter, or just pick the key if there is only one. + # let the the user choose among the ssh public keys that we encounter, or + # just pick the key if there is only one. # def pick_ssh_key ssh_keys = [] Dir.glob("#{ENV['HOME']}/.ssh/*.pub").each do |keyfile| - ssh_keys << SshKey.load(keyfile) + ssh_keys << SSH::Key.load(keyfile) end if `which ssh-add`.strip.any? `ssh-add -L 2> /dev/null`.split("\n").compact.each do |line| - key = SshKey.load(line) + key = SSH::Key.load(line) if key key.comment = 'ssh-agent' ssh_keys << key unless ssh_keys.include?(key) diff --git a/lib/leap_cli/commands/util.rb b/lib/leap_cli/commands/util.rb deleted file mode 100644 index c1da570e..00000000 --- a/lib/leap_cli/commands/util.rb +++ /dev/null @@ -1,50 +0,0 @@ -module LeapCli; module Commands - - extend self - extend LeapCli::Util - extend LeapCli::Util::RemoteCommand - - def path(name) - Path.named_path(name) - end - - # - # keeps prompting the user for a numbered choice, until they pick a good one or bail out. - # - # block is yielded and is responsible for rendering the choices. - # - def numbered_choice_menu(msg, items, &block) - while true - say("\n" + msg + ':') - items.each_with_index &block - say("q. quit") - index = ask("number 1-#{items.length}> ") - if index.empty? - next - elsif index =~ /q/ - bail! - else - i = index.to_i - 1 - if i < 0 || i >= items.length - bail! - else - return i - end - end - end - end - - - def parse_node_list(nodes) - if nodes.is_a? Config::Object - Config::ObjectList.new(nodes) - elsif nodes.is_a? Config::ObjectList - nodes - elsif nodes.is_a? String - manager.filter!(nodes) - else - bail! "argument error" - end - end - -end; end diff --git a/lib/leap_cli/commands/vagrant.rb b/lib/leap_cli/commands/vagrant.rb index 9fdd48e3..f8a75b61 100644 --- a/lib/leap_cli/commands/vagrant.rb +++ b/lib/leap_cli/commands/vagrant.rb @@ -4,7 +4,7 @@ require 'fileutils' module LeapCli; module Commands desc "Manage local virtual machines." - long_desc "This command provides a convient way to manage Vagrant-based virtual machines. If FILTER argument is missing, the command runs on all local virtual machines. The Vagrantfile is automatically generated in 'test/Vagrantfile'. If you want to run vagrant commands manually, cd to 'test'." + long_desc "This command provides a convenient way to manage Vagrant-based virtual machines. If FILTER argument is missing, the command runs on all local virtual machines. The Vagrantfile is automatically generated in 'test/Vagrantfile'. If you want to run vagrant commands manually, cd to 'test'." command [:local, :l] do |local| local.desc 'Starts up the virtual machine(s)' local.arg_name 'FILTER', :optional => true #, :multiple => false @@ -35,7 +35,7 @@ module LeapCli; module Commands local.desc 'Destroys the virtual machine(s), reclaiming the disk space' local.arg_name 'FILTER', :optional => true #, :multiple => false - local.command :destroy do |destroy| + local.command [:rm, :destroy] do |destroy| destroy.action do |global_options,options,args| if global_options[:yes] vagrant_command("destroy --force", args) @@ -47,7 +47,7 @@ module LeapCli; module Commands local.desc 'Print the status of local virtual machine(s)' local.arg_name 'FILTER', :optional => true #, :multiple => false - local.command :status do |status| + local.command [:ls, :status] do |status| status.action do |global_options,options,args| vagrant_command("status", args) end @@ -70,25 +70,6 @@ module LeapCli; module Commands end end - public - - # - # returns the path to a vagrant ssh private key file. - # - # if the vagrant.key file is owned by root or ourselves, then - # we need to make sure that it owned by us and not world readable. - # - def vagrant_ssh_key_file - file_path = Path.vagrant_ssh_priv_key_file - Util.assert_files_exist! file_path - uid = File.new(file_path).stat.uid - if uid == 0 || uid == Process.euid - FileUtils.install file_path, '/tmp/vagrant.key', :mode => 0600 - file_path = '/tmp/vagrant.key' - end - return file_path - end - protected def vagrant_command(cmds, args, options={}) diff --git a/lib/leap_cli/commands/vm.rb b/lib/leap_cli/commands/vm.rb new file mode 100644 index 00000000..790774f1 --- /dev/null +++ b/lib/leap_cli/commands/vm.rb @@ -0,0 +1,467 @@ +module LeapCli; module Commands + + desc "Manage remote virtual machines (VMs)." + long_desc "This command provides a convenient way to manage virtual machines. " + + "FILTER may be a node filter or the ID of a virtual machine." + + command [:vm] do |vm| + vm.switch :mock, :desc => "Run as simulation, without actually connecting to a cloud provider. If set, --auth is ignored." + vm.switch :wait, :desc => "Wait for servers to start/stop before continuing." + vm.flag :auth, :arg_name => 'AUTH', + :desc => "Choose which authentication credentials to use from the file cloud.json. "+ + "If omitted, will default to the node's `vm.auth` property, or the first credentials in cloud.json" + + vm.desc "Allocates a new VM and/or associates it with node NAME." + vm.long_desc "If node configuration file does not yet exist, "+ + "it is created with the optional SEED values. "+ + "You can run this command when the virtual machine already exists "+ + "in order to update the node's `vm.id` property." + vm.arg_name 'NODE_NAME [SEED]' + vm.command :add do |cmd| + cmd.action do |global, options, args| + do_vm_add(global, options, args) + end + end + + vm.desc 'Starts one or more VMs' + vm.arg_name 'FILTER', :optional => true + vm.command :start do |start| + start.action do |global, options, args| + do_vm_start(global, options, args) + end + end + + vm.desc 'Shuts down one or more VMs' + vm.long_desc 'This keeps the storage allocated. To save resources, run `leap vm rm` instead.' + vm.arg_name 'FILTER', :optional => true + vm.command :stop do |stop| + stop.action do |global, options, args| + do_vm_stop(global, options, args) + end + end + + vm.desc 'Destroys one or more VMs' + vm.arg_name 'FILTER', :optional => true + vm.command :rm do |rm| + rm.action do |global, options, args| + do_vm_rm(global, options, args) + end + end + + vm.desc 'Print the status of all VMs' + vm.arg_name 'FILTER', :optional => true + vm.command [:status, :ls] do |status| + status.action do |global, options, args| + do_vm_status(global, options, args) + end + end + + vm.desc "Binds a running VM instance to a node configuration." + vm.long_desc "Afterwards, the VM will be assigned a label matching the node name, "+ + "and the node config will be updated with the instance ID." + vm.arg_name 'NODE_NAME INSTANCE_ID' + vm.command 'bind' do |cmd| + cmd.action do |global, options, args| + do_vm_bind(global, options, args) + end + end + + vm.desc "Registers a SSH public key for use when creating new VMs." + vm.long_desc "Note that only people who are creating new VM instances need to "+ + "have their key registered." + vm.command 'key-register' do |cmd| + cmd.action do |global, options, args| + do_vm_key_register(global, options, args) + end + end + + vm.desc "Lists the registered SSH public keys for a particular VM provider." + vm.command 'key-list' do |cmd| + cmd.action do |global, options, args| + do_vm_key_list(global, options, args) + end + end + + #vm.desc 'Saves the current state of the virtual machine as a new snapshot.' + #vm.arg_name 'FILTER', :optional => true + #vm.command :save do |save| + # save.action do |global, options, args| + # do_vm_save(global, options, args) + # end + #end + + #vm.desc 'Resets virtual machine(s) to the last saved snapshot' + #vm.arg_name 'FILTER', :optional => true + #vm.command :reset do |reset| + # reset.action do |global, options, args| + # do_vm_reset(global, options, args) + # end + #end + + #vm.desc 'Lists the available images.' + #vm.command 'image-list' do |cmd| + # cmd.action do |global, options, args| + # do_vm_image_list(global, options, args) + # end + #end + end + + ## + ## SHARED UTILITY METHODS + ## + + protected + + # + # a callback used if we need to upload a new ssh key + # + def choose_ssh_key_for_upload(cloud) + puts + bail! unless agree("The cloud provider `#{cloud.name}` does not have "+ + "your public key. Do you want to upload one? ") + key = pick_ssh_key + username = ask("username? ", :default => `whoami`.strip) + assert!(username && !username.empty? && username =~ /[0-9a-z_-]+/, "Username must consist of one or more letters or numbers") + puts + return username, key + end + + def bind_server_to_node(vm_id, node, options={}) + cloud = new_cloud_handle(node, options) + server = cloud.compute.servers.get(vm_id) + assert! server, "Could not find a VM instance with ID '#{vm_id}'" + cloud.bind_server_to_node(server) + end + + ## + ## COMMANDS + ## + + protected + + # + # entirely removes the vm, not just stopping it. + # + # This might be additionally called by the 'leap node rm' command. + # + def do_vm_rm(global, options, args) + servers_from_args(global, options, args) do |cloud, server| + cloud.unbind_server_from_node(server) if cloud.node + destroy_server(server, options[:wait]) + end + end + + private + + def do_vm_status(global, options, args) + cloud = new_cloud_handle(nil, options) + servers = cloud.compute.servers + + # + # PRETTY TABLE + # + t = LeapCli::Util::ConsoleTable.new + t.table do + t.row(color: :cyan) do + t.column "ID" + t.column "NODE" + t.column "STATE" + t.column "FLAVOR" + t.column "IP" + t.column "ZONE" + end + servers.each do |server| + t.row do + t.column server.id + t.column server.tags["node_name"] + t.column server.state, :color => state_color(server.state) + t.column server.flavor_id + t.column server.public_ip_address + t.column server.availability_zone + end + end + end + puts + t.draw_table + + # + # SANITY CHECKS + # + servers.each do |server| + name = server.tags["node_name"] + if name + node = manager.nodes[name] + if node.nil? + log :warning, 'A virtual machine has the name `%s`, but there is no corresponding node definition in `%s`.' % [ + name, relative_path(path([:node_config, name]))] + next + end + if node['vm'].nil? + log :warning, 'Node `%s` is not configured as a virtual machine' % name do + log 'You should fix this with `leap vm bind %s %s`' % [name, server.id] + end + next + end + if node['vm.id'] != server.id + message = 'Node `%s` is configured with virtual machine id `%s`' % [name, node['vm.id']] + log :warning, message do + log 'But the virtual machine with that name really has id `%s`' % server.id + log 'You should fix this with `leap vm bind %s %s`' % [name, server.id] + end + end + if server.state == 'running' + if node.ip_address != server.public_ip_address + message = 'The configuration file for node `%s` has IP address `%s`' % [name, node.ip_address] + log(:warning, message) do + log 'But the virtual machine actually has IP address `%s`' % server.public_ip_address + log 'You should fix this with `leap vm add %s`' % name + end + end + end + end + end + manager.filter(['vm']).each_node do |node| + if node['vm.id'].nil? + log :warning, 'The node `%s` is missing a server id' % node.name + next + end + if !servers.detect {|s| s.id == node.vm.id } + message = "The configuration file for node `%s` has virtual machine id of `%s`" % [node.name, node.vm.id] + log :warning, message do + log "But that does not match any actual virtual machines!" + end + end + if !servers.detect {|s| s.tags["node_name"] == node.name } + log :warning, "The node `%s` has no virtual machines with a matching name." % node.name do + server = servers.detect {|s| s.id == node.vm.id } + if server + log 'Run `leap bind %s %s` to fix this' % [node.name, server.id] + end + end + end + end + end + + def do_vm_add(global, options, args) + name = args.first + if manager.nodes[name].nil? + do_node_add(global, {:ip_address => '0.0.0.0'}.merge(options), args) + end + node = manager.nodes[name] + cloud = new_cloud_handle(node, options) + server = cloud.fetch_or_create_server(:choose_ssh_key => method(:choose_ssh_key_for_upload)) + + if server + cloud.bind_server_to_node(server) + ssh_host_key = cloud.wait_for_ssh_host_key(server) + if ssh_host_key.nil? + log :warning, "We could not get a SSH host key." do + log "Try running `leap vm add #{node.name}` again later." + end + else + log :saving, "SSH host key for #{node.name}" + write_file! [:node_ssh_pub_key, node.name], ssh_host_key.to_s + end + log "done", :color => :green, :style => :bold + end + end + + def do_vm_start(global, options, args) + servers_from_args(global, options, args) do |cloud, server| + start_server(server, options[:wait]) + end + end + + def do_vm_stop(global, options, args) + servers_from_args(global, options, args) do |cloud, server| + stop_server(server, options[:wait]) + end + end + + def do_vm_key_register(global, options, args) + cloud = new_cloud_handle(nil, options) + cloud.find_or_create_key_pair(method(:choose_ssh_key_for_upload)) + end + + def do_vm_key_list(global, options, args) + require 'leap_cli/ssh' + cloud = new_cloud_handle(nil, options) + cloud.compute.key_pairs.each do |key_pair| + log key_pair.name, :color => :cyan do + log "AWS fingerprint: " + key_pair.fingerprint + key_pair, local_key = cloud.match_ssh_key(:key_pair => key_pair) + if local_key + log "matches local key: " + local_key.filename + log 'SSH MD5 fingerprint: ' + local_key.fingerprint(:digest => :md5, :type => :ssh, :encoding => :hex) + log 'SSH SHA256 fingerprint: ' + local_key.fingerprint(:digest => :sha256, :type => :ssh, :encoding => :base64) + end + end + end + end + + # + # update association between node and virtual machine. + # + # This might additionally be called by the 'leap node mv' command. + # + def do_vm_bind(global, options, args) + node_name = args.first + vm_id = args.last + assert! node_name, "NODE_NAME is missing" + assert! vm_id, "INSTANCE_ID is missing" + node = manager.nodes[node_name] + assert! node, "No node with name '#{node_name}'" + bind_server_to_node(vm_id, node, options) + end + + #def do_vm_image_list(global, options, args) + # compute = fog_setup(nil, options) + # p compute.images.all + #end + + ## + ## PRIVATE UTILITY METHODS + ## + + def stop_server(server, wait=false) + if server.state == 'stopped' + log :skipping, "virtual machine `#{server.id}` (already stopped)." + elsif ['shutting-down', 'terminated'].include?(server.state) + log :skipping, "virtual machine `#{server.id}` (being destroyed)." + else + log :stopping, "virtual machine `#{server.id}` (#{server.flavor_id}, #{server.availability_zone})" + server.stop + if wait + log 'please wait...', :indent => 1 + server.wait_for { state == 'stopped' } + log 'done', :color => :green, :indent => 1 + end + end + end + + def start_server(server, wait=false) + if server.state == 'running' + log :skipping, "virtual machine `#{server.id}` (already running)." + elsif ['shutting-down', 'terminated'].include?(server.state) + log :skipping, "virtual machine `#{server.id}` (being destroyed)." + else + log :starting, "virtual machine `#{server.id}` (#{server.flavor_id}, #{server.availability_zone})" + server.start + if wait + log 'please wait...', :indent => 1 + server.wait_for { ready? } + log 'done', :color => :green, :indent => 1 + end + end + end + + def destroy_server(server, wait=false) + if ['shutting-down', 'terminated'].include?(server.state) + log :skipping, "virtual machine `#{server.id}` (already being removed)." + else + log :terminated, "virtual machine `#{server.id}` (#{server.flavor_id}, #{server.availability_zone})" + server.destroy + if wait + log 'please wait...', :indent => 1 + server.wait_for { state == 'terminated' } + log 'done', :color => :green, :indent => 1 + end + end + end + + # + # for each server it finds, yields cloud, server + # + def servers_from_args(global, options, args) + nodes = filter_vm_nodes(args) + if nodes.any? + nodes.each_node do |node| + cloud = new_cloud_handle(node, options) + server = cloud.fetch_server_for_node(true) + yield cloud, server + end + else + instance_id = args.first + cloud = new_cloud_handle(nil, options) + server = cloud.compute.servers.get(instance_id) + if server.nil? + bail! :error, "There is no virtual machine with ID `#{instance_id}`." + end + yield cloud, server + end + end + + # + # returns either: + # + # * the set of nodes specified by the filter, for this environment + # even if the result includes nodes that are not previously tagged with 'vm' + # + # * the list of all vm nodes for this environment, if filter is empty + # + def filter_vm_nodes(filter) + if filter.nil? || filter.empty? + return manager.filter(['vm'], :warning => false) + elsif filter.is_a? Array + return manager.filter(filter, :warning => false) + else + raise ArgumentError, 'could not understand filter' + end + end + + def new_cloud_handle(node, options) + require 'leap_cli/cloud' + + config = manager.env.cloud + name = nil + if options[:mock] + Fog.mock! + name = 'mock_aws' + config['mock_aws'] = { + "api" => "aws", + "vendor" => "aws", + "auth" => { + "aws_access_key_id" => "dummy", + "aws_secret_access_key" => "dummy", + "region" => "us-west-2" + }, + "instance_options" => { + "image" => "dummy" + } + } + elsif options[:auth] + name = options[:auth] + assert! config[name], "The value for --auth does not correspond to any value in cloud.json." + elsif node && node['vm.auth'] + name = node.vm.auth + assert! config[name], "The node '#{node.name}' has a value for property 'vm.auth' that does not correspond to any value in cloud.json." + elsif config.keys.length == 1 + name = config.keys.first + log :using, "cloud vendor credentials `#{name}`." + else + bail! "You must specify --mock, --auth, or a node filter." + end + + entry = config[name] # entry in cloud.json + assert! entry, "cloud.json: could not find cloud resource `#{name}`." + assert! entry['vendor'], "cloud.json: property `vendor` is missing from `#{name}` entry." + assert! entry['api'], "cloud.json: property `api` is missing from `#{name}` entry. It must be one of #{config.possible_apis.join(', ')}." + assert! entry['auth'], "cloud.json: property `auth` is missing from `#{name}` entry." + assert! entry['auth']['region'], "cloud.json: property `auth.region` is missing from `#{name}` entry." + assert! entry['api'] == 'aws', "cloud.json: currently, only 'aws' is supported for `api`." + assert! entry['vendor'] == 'aws', "cloud.json: currently, only 'aws' is supported for `vendor`." + + return LeapCli::Cloud.new(name, entry, node) + end + + def state_color(state) + case state + when 'running'; :green + when 'terminated'; :red + when 'stopped'; :magenta + when 'shutting-down'; :yellow + else; :white + end + end + +end; end diff --git a/lib/leap_cli/config/cloud.rb b/lib/leap_cli/config/cloud.rb new file mode 100644 index 00000000..e3e5c1f1 --- /dev/null +++ b/lib/leap_cli/config/cloud.rb @@ -0,0 +1,64 @@ +# encoding: utf-8 +# +# A class for the cloud.json file +# +# Example format: +# +# { +# "my_aws": { +# "api": "aws", +# "vendor": "aws", +# "auth": { +# "region": "us-west-2", +# "aws_access_key_id": "xxxxxxxxxxxxxxx", +# "aws_secret_access_key": "xxxxxxxxxxxxxxxxxxxxxxxxxx" +# }, +# "default_image": "ami-98e114f8", +# "default_options": { +# "InstanceType": "t2.nano" +# } +# } +# } +# + +module LeapCli; module Config + + # http://fog.io/about/supported_services.html + VM_APIS = { + 'aws' => 'fog-aws', + 'google' => 'fog-google', + 'libvirt' => 'fog-libvirt', + 'openstack' => 'fog-openstack', + 'rackspace' => 'fog-rackspace' + } + + class Cloud < Hash + def initialize(env=nil) + end + + # + # returns hash, each key is the name of an API that is + # needed and the value is the name of the gem. + # + # only provider APIs that are required because they are present + # in cloud.json are included. + # + def required_gems + required = {} + self.each do |name, conf| + api = conf["api"] + required_gems[api] = VM_APIS[api] + end + return required + end + + # + # returns an array of all possible providers + # + def possible_apis + VM_APIS.keys + end + + end + +end; end diff --git a/lib/leap_cli/config/environment.rb b/lib/leap_cli/config/environment.rb new file mode 100644 index 00000000..ce570839 --- /dev/null +++ b/lib/leap_cli/config/environment.rb @@ -0,0 +1,200 @@ +# +# All configurations files can be isolated into separate environments. +# +# Each config json in each environment inherits from the default environment, +# which in term inherits from the "_base_" environment: +# +# _base_ -- base provider in leap_platform +# '- default -- environment in provider dir when no env is set +# '- production -- example environment +# + +module LeapCli; module Config + + class Environment + # the String name of the environment + attr_accessor :name + + # the shared Manager object + attr_accessor :manager + + # hashes of {name => Config::Object} + attr_accessor :services, :tags, :partials + + # a Config::Provider + attr_accessor :provider + + # a Config::Object + attr_accessor :common + + # shared, non-inheritable + def nodes; @@nodes; end + def secrets; @@secrets; end + def cloud; @@cloud; end + + def initialize(manager, name, search_dir, parent, options={}) + @@nodes ||= nil + @@secrets ||= nil + @@cloud ||= nil + + @manager = manager + @name = name + + load_provider_files(search_dir, options) + + if parent + @services.inherit_from! parent.services, self + @tags.inherit_from! parent.tags , self + @partials.inherit_from! parent.partials, self + @common.inherit_from! parent.common + @provider.inherit_from! parent.provider + end + + if @provider + @provider.set_env(name) + @provider.validate! + end + end + + def load_provider_files(search_dir, options) + # + # load empty environment if search_dir doesn't exist + # + if search_dir.nil? || !Dir.exist?(search_dir) + @services = Config::ObjectList.new + @tags = Config::ObjectList.new + @partials = Config::ObjectList.new + @provider = Config::Provider.new + @common = Config::Object.new + @cloud = Config::Cloud.new + return + end + + # + # inheritable + # + if options[:scope] + scope = options[:scope] + @services = load_all_json(Path.named_path([:service_env_config, '*', scope], search_dir), Config::Tag, options) + @tags = load_all_json(Path.named_path([:tag_env_config, '*', scope], search_dir), Config::Tag, options) + @partials = load_all_json(Path.named_path([:service_env_config, '_*', scope], search_dir), Config::Tag, options) + @provider = load_json( Path.named_path([:provider_env_config, scope], search_dir), Config::Provider, options) + @common = load_json( Path.named_path([:common_env_config, scope], search_dir), Config::Object, options) + else + @services = load_all_json(Path.named_path([:service_config, '*'], search_dir), Config::Tag, options) + @tags = load_all_json(Path.named_path([:tag_config, '*'], search_dir), Config::Tag, options) + @partials = load_all_json(Path.named_path([:service_config, '_*'], search_dir), Config::Tag, options) + @provider = load_json( Path.named_path(:provider_config, search_dir), Config::Provider, options) + @common = load_json( Path.named_path(:common_config, search_dir), Config::Object, options) + end + + # remove 'name' from partials, since partials get merged with nodes + @partials.values.each {|partial| partial.delete('name'); } + + # + # shared + # + # shared configs are also non-inheritable + # load the first ones we find, and only those. + # + if @@nodes.nil? || @@nodes.empty? + @@nodes = load_all_json(Path.named_path([:node_config, '*'], search_dir), Config::Node, options) + end + if @@secrets.nil? || @@secrets.empty? + @@secrets = load_json(Path.named_path(:secrets_config, search_dir), Config::Secrets, options) + end + if @@cloud.nil? || @@cloud.empty? + @@cloud = load_json(Path.named_path(:cloud_config, search_dir), Config::Cloud) + end + end + + # + # Loads a json template file as a Hash (used only when creating a new node .json + # file for the first time). + # + def template(template) + path = Path.named_path([:template_config, template], Path.provider_base) + if File.exist?(path) + return load_json(path, Config::Object) + else + return nil + end + end + + # + # Alters the node's json config file. Unfortunately, doing this will + # strip out all the comments. + # + def update_node_json(node, new_values) + node_json_path = Path.named_path([:node_config, node.name]) + old_data = load_json(node_json_path, Config::Node) + new_data = old_data.merge(new_values) + new_contents = JSON.sorted_generate(new_data) + "\n" + Util::write_file! node_json_path, new_contents + end + + private + + def load_all_json(pattern, object_class, options={}) + results = Config::ObjectList.new + Dir.glob(pattern).each do |filename| + next if options[:no_dots] && File.basename(filename) !~ /^[^\.]*\.json$/ + obj = load_json(filename, object_class) + if obj + name = File.basename(filename).force_encoding('utf-8').sub(/^([^\.]+).*\.json$/,'\1') + obj['name'] ||= name + if options[:env] + obj.environment = options[:env] + end + results[name] = obj + end + end + results + end + + def load_json(filename, object_class, options={}) + if !File.exist?(filename) + return object_class.new(self) + end + + Util::log :loading, filename, 3 + + # + # Read a JSON file, strip out comments. + # + # UTF8 is the default encoding for JSON, but others are allowed: + # https://www.ietf.org/rfc/rfc4627.txt + # + buffer = StringIO.new + File.open(filename, "rb", :encoding => 'UTF-8') do |f| + while (line = f.gets) + next if line =~ /^\s*\/\// + buffer << line + end + end + + # + # force UTF-8 + # + if $ruby_version >= [1,9] + string = buffer.string.force_encoding('utf-8') + else + string = Iconv.conv("UTF-8//IGNORE", "UTF-8", buffer.string) + end + + # parse json + begin + hash = JSON.parse(string, :object_class => Hash, :array_class => Array) || {} + rescue SyntaxError, JSON::ParserError => exc + Util::log 0, :error, 'in file "%s":' % filename + Util::log 0, exc.to_s, :indent => 1 + return nil + end + object = object_class.new(self) + object.deep_merge!(hash) + return object + end + + end # end Environment + +end; end
\ No newline at end of file diff --git a/lib/leap_cli/config/filter.rb b/lib/leap_cli/config/filter.rb new file mode 100644 index 00000000..07424894 --- /dev/null +++ b/lib/leap_cli/config/filter.rb @@ -0,0 +1,181 @@ +# +# Many leap_cli commands accept a list of filters to select a subset of nodes for the command to +# be applied to. This class is a helper for manager to run these filters. +# +# Classes other than Manager should not use this class. +# +# Filter rules: +# +# * A filter consists of a list of tokens +# * A token may be a service name, tag name, environment name, or node name. +# * Each token may be optionally prefixed with a plus sign. +# * Multiple tokens with a plus are treated as an OR condition, +# but treated as an AND condition with the plus sign. +# +# For example +# +# * openvpn +development => all nodes with service 'openvpn' AND environment 'development' +# * openvpn seattle => all nodes with service 'openvpn' OR tag 'seattle'. +# +# There can only be one environment specified. Typically, there are also tags +# for each environment name. These name are treated as environments, not tags. +# +module LeapCli + module Config + class Filter + + # + # filter -- array of strings, each one a filter + # options -- hash, possible keys include + # :nopin -- disregard environment pinning + # :local -- if false, disallow local nodes + # :warning -- if false, don't print a warning when no nodes are found. + # + # A nil value in the filters array indicates + # the default environment. This is in order to support + # calls like `manager.filter(environments)` + # + def initialize(filters, options, manager) + @filters = filters.nil? ? [] : filters.dup + @environments = [] + @options = options + @manager = manager + + # split filters by pulling out items that happen + # to be environment names. + if LeapCli.leapfile.environment.nil? || @options[:nopin] + @environments = [] + else + @environments = [LeapCli.leapfile.environment] + end + @filters.select! do |filter| + if filter.nil? + @environments << nil unless @environments.include?(nil) + false + else + filter_text = filter.sub(/^\+/,'') + if is_environment?(filter_text) + if filter_text == LeapCli.leapfile.environment + # silently ignore already pinned environments + elsif (filter =~ /^\+/ || @filters.first == filter) && !@environments.empty? + LeapCli::Util.bail! do + LeapCli.log "Environments are exclusive: no node is in two environments." do + LeapCli.log "Tried to filter on '#{@environments.join('\' AND \'')}' AND '#{filter_text}'" + end + end + else + @environments << filter_text + end + false + else + true + end + end + end + + # don't let the first filter have a + prefix + if @filters[0] =~ /^\+/ + @filters[0] = @filters[0][1..-1] + end + end + + # actually run the filter, returns a filtered list of nodes + def nodes() + if @filters.empty? + return nodes_for_empty_filter + else + return nodes_for_filter + end + end + + private + + def nodes_for_empty_filter + node_list = @manager.nodes + if @environments.any? + node_list = node_list[ @environments.collect{|e|[:environment, env_to_filter(e)]} ] + end + if @options[:local] === false + node_list = node_list[:environment => '!local'] + end + if @options[:disabled] === false + node_list = node_list[:environment => '!disabled'] + end + node_list + end + + def nodes_for_filter + node_list = Config::ObjectList.new + @filters.each do |filter| + if filter =~ /^\+/ + keep_list = nodes_for_name(filter[1..-1]) + node_list.delete_if do |name, node| + if keep_list[name] + false + else + true + end + end + else + node_list.merge!(nodes_for_name(filter)) + end + end + node_list + end + + private + + # + # returns a set of nodes corresponding to a single name, + # where name could be a node name, service name, or tag name. + # + # For services and tags, we only include nodes for the + # environments that are active + # + def nodes_for_name(name) + if node = @manager.nodes[name] + return Config::ObjectList.new(node) + elsif @environments.empty? + if @manager.services[name] + return @manager.env('_all_').services[name].node_list + elsif @manager.tags[name] + return @manager.env('_all_').tags[name].node_list + elsif @options[:warning] != false + LeapCli.log :warning, "filter '#{name}' does not match any node names, tags, services, or environments." + return Config::ObjectList.new + else + return Config::ObjectList.new + end + else + node_list = Config::ObjectList.new + if @manager.services[name] + @environments.each do |env| + node_list.merge!(@manager.env(env).services[name].node_list) + end + elsif @manager.tags[name] + @environments.each do |env| + node_list.merge!(@manager.env(env).tags[name].node_list) + end + elsif @options[:warning] != false + LeapCli.log :warning, "filter '#{name}' does not match any node names, tags, services, or environments." + end + return node_list + end + end + + # + # when pinning, we use the name 'default' to specify nodes + # without an environment set, but when filtering, we need to filter + # on :environment => nil. + # + def env_to_filter(environment) + environment == 'default' ? nil : environment + end + + def is_environment?(text) + text == 'default' || @manager.environment_names.include?(text) + end + + end + end +end diff --git a/lib/leap_cli/config/manager.rb b/lib/leap_cli/config/manager.rb new file mode 100644 index 00000000..d69a5808 --- /dev/null +++ b/lib/leap_cli/config/manager.rb @@ -0,0 +1,475 @@ +# encoding: utf-8 + +require 'json/pure' + +if $ruby_version < [1,9] + require 'iconv' +end + +module LeapCli + module Config + + # + # A class to manage all the objects in all the configuration files. + # + class Manager + + def initialize + @environments = {} # hash of `Environment` objects, keyed by name. + Config::Object.send(:include, LeapCli::Macro) + end + + ## + ## ATTRIBUTES + ## + + # + # returns the Hash of the contents of facts.json + # + def facts + @facts ||= begin + content = Util.read_file(:facts) + if !content || content.empty? + content = "{}" + end + JSON.parse(content) + rescue SyntaxError, JSON::ParserError => exc + Util::bail! "Could not parse facts.json -- #{exc}" + end + end + + # + # returns an Array of all the environments defined for this provider. + # the returned array includes nil (for the default environment) + # + def environment_names + @environment_names ||= begin + [nil] + (env.tags.field('environment') + env.nodes.field('environment')).compact.uniq + end + end + + # + # Returns the appropriate environment variable + # + def env(env=nil) + @environments[env || 'default'] + end + + # + # The default accessors + # + # For these defaults, use 'default' environment, or whatever + # environment is pinned. + # + # I think it might be an error that these are ever used + # and I would like to get rid of them. + # + def services; env(default_environment).services; end + def tags; env(default_environment).tags; end + def partials; env(default_environment).partials; end + def provider; env(default_environment).provider; end + def common; env(default_environment).common; end + def secrets; env(default_environment).secrets; end + def nodes; env(default_environment).nodes; end + def template(*args) + self.env.template(*args) + end + + def default_environment + LeapCli.leapfile.environment + end + + ## + ## IMPORT EXPORT + ## + + def add_environment(args) + if args[:inherit] + parent = @environments[args.delete(:inherit)] + else + parent = nil + end + env = Environment.new( + self, + args.delete(:name), + args.delete(:dir), + parent, + args + ) + @environments[env.name] = env + end + + # + # load .json configuration files + # + def load(options = {}) + # load base + add_environment(name: '_base_', dir: Path.provider_base) + + # load provider + Util::assert_files_exist!(Path.named_path(:provider_config, Path.provider)) + add_environment(name: 'default', dir: Path.provider, + inherit: '_base_', no_dots: true) + + # create a special '_all_' environment, used for tracking + # the union of all the environments + add_environment(name: '_all_', inherit: 'default') + + # load environments + environment_names.each do |ename| + if ename + LeapCli.log 3, :loading, '%s environment...' % ename + add_environment(name: ename, dir: Path.provider, + inherit: 'default', scope: ename) + end + end + + # apply inheritance + env.nodes.each do |name, node| + Util::assert! name =~ /^[0-9a-z-]+$/, "Illegal character(s) used in node name '#{name}'" + env.nodes[name] = apply_inheritance(node) + end + + # do some node-list post-processing + cleanup_node_lists(options) + + # apply service.rb, common.rb, and provider.rb control files + apply_control_files + end + + # + # save compiled hiera .yaml files + # + # if a node_list is specified, only update those .yaml files. + # otherwise, update all files, destroying files that are no longer used. + # + def export_nodes(node_list=nil) + updated_hiera = [] + updated_files = [] + existing_hiera = nil + existing_files = nil + + unless node_list + node_list = env.nodes + existing_hiera = Dir.glob(Path.named_path([:hiera, '*'], Path.provider)) + existing_files = Dir.glob(Path.named_path([:node_files_dir, '*'], Path.provider)) + end + + node_list.each_node do |node| + filepath = Path.named_path([:node_files_dir, node.name], Path.provider) + hierapath = Path.named_path([:hiera, node.name], Path.provider) + Util::write_file!(hierapath, node.dump_yaml) + updated_files << filepath + updated_hiera << hierapath + end + + if @disabled_nodes + # make disabled nodes appear as if they are still active + @disabled_nodes.each_node do |node| + updated_files << Path.named_path([:node_files_dir, node.name], Path.provider) + updated_hiera << Path.named_path([:hiera, node.name], Path.provider) + end + end + + # remove files that are no longer needed + if existing_hiera + (existing_hiera - updated_hiera).each do |filepath| + Util::remove_file!(filepath) + end + end + if existing_files + (existing_files - updated_files).each do |filepath| + Util::remove_directory!(filepath) + end + end + end + + def export_secrets(clean_unused_secrets = false) + if env.secrets.any? + Util.write_file!([:secrets_config, Path.provider], env.secrets.dump_json(clean_unused_secrets) + "\n") + end + end + + ## + ## FILTERING + ## + + # + # returns a node list consisting only of nodes that satisfy the filter criteria. + # + # filter: condition [condition] [condition] [+condition] + # condition: [node_name | service_name | tag_name | environment_name] + # + # if conditions is prefixed with +, then it works like an AND. Otherwise, it works like an OR. + # + # args: + # filter -- array of filter terms, one per item + # + # options: + # :local -- if :local is false and the filter is empty, then local nodes are excluded. + # :nopin -- if true, ignore environment pinning + # + def filter(filters=nil, options={}) + Filter.new(filters, options, self).nodes() + end + + # + # same as filter(), but exits if there is no matching nodes + # + def filter!(filters, options={}) + node_list = filter(filters, options) + Util::assert! node_list.any?, "Could not match any nodes from '#{filters.join ' '}'" + return node_list + end + + # + # returns a single Config::Object that corresponds to a Node. + # + def node(name) + if name =~ /\./ + # probably got a fqdn, since periods are not allowed in node names. + # so, take the part before the first period as the node name + name = name.split('.').first + end + env.nodes[name] + end + + # + # returns a single node that is disabled + # + def disabled_node(name) + @disabled_nodes[name] + end + + # + # yields each node, in sorted order + # + def each_node(&block) + env.nodes.each_node(&block) + end + + def reload_node!(node) + env.nodes[node.name] = apply_inheritance!(node) + end + + ## + ## CONNECTIONS + ## + + class ConnectionList < Array + def add(data={}) + self << { + "from" => data[:from], + "to" => data[:to], + "port" => data[:port] + } + end + end + + def connections + @connections ||= ConnectionList.new + end + + ## + ## PRIVATE + ## + + private + + # + # makes a node inherit options from appropriate the common, service, and tag json files. + # + def apply_inheritance(node, throw_exceptions=false) + new_node = Config::Node.new(nil) + node_env = guess_node_env(node) + new_node.set_environment(node_env, new_node) + + # inherit from common + new_node.deep_merge!(node_env.common) + + # inherit from services + if node['services'] + node['services'].to_a.each do |node_service| + service = node_env.services[node_service] + if service.nil? + msg = 'in node "%s": the service "%s" does not exist.' % [node['name'], node_service] + LeapCli.log 0, :error, msg + raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions + else + new_node.deep_merge!(service) + end + end + end + + # inherit from tags + node['tags'] = (node['tags'] || []).to_a + if node.vagrant? + node['tags'] << 'local' + elsif node['vm'] + node['tags'] << 'vm' + end + node['tags'].each do |node_tag| + tag = node_env.tags[node_tag] + if tag.nil? + msg = 'in node `%s`: the tag "%s" does not exist!' % [node['name'], node_tag] + LeapCli.log 0, :error, msg + raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions + else + new_node.deep_merge!(tag) + end + end + + # inherit from node + new_node.deep_merge!(node) + return new_node + end + + def apply_inheritance!(node) + apply_inheritance(node, true) + end + + # + # Guess the environment of the node from the tag names. + # + # Technically, this is wrong: a tag that sets the environment might not be + # named the same as the environment. This code assumes that it is. + # + # Unfortunately, it is a chicken and egg problem. We need to know the nodes + # likely environment in order to apply the inheritance that will actually + # determine the node's properties. + # + def guess_node_env(node) + if node.vagrant? + return self.env("local") + else + environment = self.env(default_environment) + if node['tags'] + node['tags'].to_a.each do |tag| + if self.environment_names.include?(tag) + environment = self.env(tag) + end + end + end + return environment + end + end + + # + # does some final clean at the end of loading nodes. + # this includes removing disabled nodes, and populating + # the services[x].node_list and tags[x].node_list + # + def cleanup_node_lists(options) + @disabled_nodes = Config::ObjectList.new + env.nodes.each do |name, node| + if node.enabled || options[:include_disabled] + if node['services'] + node['services'].to_a.each do |node_service| + env(node.environment).services[node_service].node_list.add(node.name, node) + env('_all_').services[node_service].node_list.add(node.name, node) + end + end + if node['tags'] + node['tags'].to_a.each do |node_tag| + if env(node.environment).tags[node_tag] + # if tag exists + env(node.environment).tags[node_tag].node_list.add(node.name, node) + env('_all_').tags[node_tag].node_list.add(node.name, node) + end + end + end + if node.name == 'default' || environment_names.include?(node.name) + LeapCli::Util.bail! do + LeapCli.log :error, "The node name '#{node.name}' is invalid, because there is an environment with that same name." + end + end + elsif !options[:include_disabled] + LeapCli.log 2, :skipping, "disabled node #{name}." + env.nodes.delete(name) + @disabled_nodes[name] = node + end + end + end + + # + # Applies 'control' files for node .json files and provider.json. + # + # A control file is like a service or a tag JSON file, but it contains + # raw ruby code that gets evaluated in the context of the node. + # + # Yes, this entirely breaks our functional programming model for JSON + # generation. + # + # Control files are evaluated last, after everything else has run. + # + def apply_control_files + @environments.values.each do |e| + provider_control_files(e.name).each do |provider_rb| + begin + e.provider.eval_file provider_rb + rescue ConfigError => exc + if options[:continue_on_error] + exc.log + else + raise exc + end + end + end + end + env.nodes.each do |name, node| + node_control_files(node).each do |file| + begin + node.eval_file file + rescue ConfigError => exc + if options[:continue_on_error] + exc.log + else + raise exc + end + end + end + end + end + + def node_control_files(node) + files = [] + [Path.provider_base, Path.provider].each do |provider_dir| + # add common.rb + common = File.join(provider_dir, 'common.rb') + files << common if File.exist?(common) + + # add services/*.rb and tags/*.rb, as appropriate for this node + [['services', :service_config], ['tags', :tag_config]].each do |attribute, path_sym| + node[attribute].each do |attr_value| + path = Path.named_path([path_sym, "#{attr_value}.rb"], provider_dir).sub(/\.json$/,'') + if File.exist?(path) + files << path + end + end + end + end + return files + end + + def provider_control_files(env) + # skip envs that start with underscore + if env =~ /^_/ + return [] + end + files = [] + environments = [nil] + environments << env unless env == 'default' + environments.each do |environment| + [Path.provider_base, Path.provider].each do |provider_dir| + provider_rb = File.join( + provider_dir, ['provider', environment, 'rb'].compact.join('.') + ) + files << provider_rb if File.exist?(provider_rb) + end + end + return files + end + + end + end +end diff --git a/lib/leap_cli/config/node.rb b/lib/leap_cli/config/node.rb new file mode 100644 index 00000000..23abdee3 --- /dev/null +++ b/lib/leap_cli/config/node.rb @@ -0,0 +1,245 @@ +# +# Configuration for a 'node' (a server in the provider's infrastructure) +# + +require 'ipaddr' + +module LeapCli; module Config + + class Node < Object + attr_accessor :file_paths + + def initialize(environment=nil) + super(environment) + @node = self + @file_paths = [] + end + + # + # returns true if this node has an ip address in the range of the vagrant network + # + def vagrant? + ip = self['ip_address'] + return false unless ip + begin + vagrant_range = IPAddr.new LeapCli.leapfile.vagrant_network + rescue ArgumentError + Util::bail! { Util::log :invalid, "vagrant_network in Leapfile or .leaprc" } + end + + begin + ip_addr = IPAddr.new(ip) + rescue ArgumentError + Util::log :warning, "invalid ip address '#{ip}' for node '#{@node.name}'" + end + return vagrant_range.include?(ip_addr) + end + + def vm? + self['vm'] + end + + def vm_id? + self['vm.id'] && !self['vm.id'].empty? + end + + # + # Return a hash table representation of ourselves, with the key equal to the @node.name, + # and the value equal to the fields specified in *keys. + # + # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b' + # + # compare to Object#pick(*keys). This method is the sames as Config::ObjectList#pick_fields, + # but works on a single node. + # + # Example: + # + # node.pick('domain.internal') => + # + # { + # 'node1': { + # 'domain_internal': 'node1.example.i' + # } + # } + # + def pick_fields(*keys) + {@node.name => self.pick(*keys)} + end + + # + # can be overridden by the platform. + # returns a list of node names that should be tested before this node + # + def test_dependencies + [] + end + + # returns a string list of supported ssh host key algorithms for this node. + # or an empty string if it could not be determined + def supported_ssh_host_key_algorithms + require 'leap_cli/ssh' + @host_key_algo ||= LeapCli::SSH::Key.supported_host_key_algorithms( + Util.read_file([:node_ssh_pub_key, @node.name]) + ) + end + + # + # Takes strings such as "openvpn.gateway_address:1.1.1.1" + # and converts this to data stored in this node. + # + def seed_from_args(args) + args.each do |seed| + key, value = seed.split(':', 2) + value = format_seed_value(value) + Util.assert! key =~ /^[0-9a-z\._]+$/, "illegal characters used in property '#{key}'" + if key =~ /\./ + key_parts = key.split('.') + final_key = key_parts.pop + current_object = self + key_parts.each do |key_part| + current_object[key_part] ||= Config::Object.new + current_object = current_object[key_part] + end + current_object[final_key] = value + else + self[key] = value + end + end + end + + # + # Seeds values for this node from a template, based on the services. + # Values in the template will not override existing node values. + # + def seed_from_template + inherit_from!(manager.template('common')) + [self['services']].flatten.each do |service| + if service + template = manager.template(service) + if template + inherit_from!(template) + end + end + end + end + + # + # bails if the node is not valid. + # + def validate! + # + # validate ip_address + # + if self['ip_address'] == "REQUIRED" + Util.bail! do + Util.log :error, "ip_address is not set. " + + "Specify with `leap node add NAME ip_address:ADDRESS`." + end + elsif self['ip_address'] + begin + IPAddr.new(self['ip_address']) + rescue ArgumentError + Util.bail! do + Util.log :invalid, "ip_address #{self['ip_address'].inspect}" + end + end + end + + # + # validate name + # + self.class.validate_name!(self.name, self.vagrant?) + end + + # + # create or update all the configs needed for this node, + # including x.509 certs as needed. + # + # note: this method will write to disk EVERYTHING + # in the node, which is not what you want + # if the node has inheritance applied. + # + def write_configs + json = self.dump_json(:exclude => ['name']) + Util.write_file!([:node_config, name], json + "\n") + rescue LeapCli::ConfigError + Config::Node.remove_node_files(self.name) + end + + # + # modifies the config file nodes/NAME.json for this node. + # + def update_json(new_values) + self.env.update_node_json(node, new_values) + end + + # + # returns an array of all possible dns names for this node + # + def all_dns_names + names = [@node.domain.internal, @node.domain.full] + if @node['dns'] && @node.dns['aliases'] && @node.dns.aliases.any? + names += @node.dns.aliases + end + names.compact! + names.sort! + names.uniq! + return names + end + + def remove_files + self.class.remove_node_files(self.name) + end + + ## + ## Class Methods + ## + + def self.remove_node_files(node_name) + (Leap::Platform.node_files + [:node_files_dir]).each do |path| + Util.remove_file! [path, node_name] + end + end + + def self.validate_name!(name, local=false) + Util.assert! name, 'Node is missing a name.' + if local + Util.assert! name =~ /^[0-9a-z]+$/, + "illegal characters used in node name '#{name}' " + + "(note: Vagrant does not allow hyphens or underscores)" + else + Util.assert! name =~ /^[0-9a-z-]+$/, + "illegal characters used in node name '#{name}' " + + "(note: Linux does not allow underscores)" + end + end + + private + + # + # conversions: + # + # "x,y,z" => ["x","y","z"] + # + # "22" => 22 + # + # "5.1" => 5.1 + # + def format_seed_value(v) + if v =~ /,/ + v = v.split(',') + v.map! do |i| + i = i.to_i if i.to_i.to_s == i + i = i.to_f if i.to_f.to_s == i + i + end + else + v = v.to_i if v.to_i.to_s == v + v = v.to_f if v.to_f.to_s == v + end + return v + end + + end + +end; end diff --git a/lib/leap_cli/config/node_cert.rb b/lib/leap_cli/config/node_cert.rb new file mode 100644 index 00000000..da63d621 --- /dev/null +++ b/lib/leap_cli/config/node_cert.rb @@ -0,0 +1,124 @@ +# +# x509 related methods for Config::Node +# +module LeapCli; module Config + + class Node < Object + + # + # creates a new server certificate file for this node + # + def generate_cert + require 'leap_cli/x509' + + if self['x509.use'] == false || + !Util.file_exists?(:ca_cert, :ca_key) || + !self.cert_needs_updating? + return false + end + + cert = CertificateAuthority::Certificate.new + provider = env.provider + + # set subject + cert.subject.common_name = self.domain.full + cert.serial_number.number = X509.cert_serial_number(self.domain.full) + + # set expiration + cert.not_before = X509.yesterday + cert.not_after = X509.yesterday_advance(provider.ca.server_certificates.life_span) + + # generate key + cert.key_material.generate_key(provider.ca.server_certificates.bit_size) + + # sign + cert.parent = X509.ca_root + cert.sign!(X509.server_signing_profile(self)) + + # save + Util.write_file!([:node_x509_key, self.name], cert.key_material.private_key.to_pem) + Util.write_file!([:node_x509_cert, self.name], cert.to_pem) + end + + # + # returns true if the certs associated with +node+ need to be regenerated. + # + def cert_needs_updating?(log_comments=true) + require 'leap_cli/x509' + + if log_comments + def log(*args, &block) + Util.log(*args, &block) + end + else + def log(*args); end + end + + node = self + if !Util.file_exists?([:node_x509_cert, node.name], [:node_x509_key, node.name]) + return true + else + cert = X509.load_certificate_file([:node_x509_cert, node.name]) + if !X509.created_by_authority?(cert) + log :updating, "cert for node '#{node.name}' because it was signed by an old CA root cert." + return true + end + if cert.not_after < Time.now.advance(:months => 2) + log :updating, "cert for node '#{node.name}' because it will expire soon" + return true + end + if cert.subject.common_name != node.domain.full + log :updating, "cert for node '#{node.name}' because domain.full has changed (was #{cert.subject.common_name}, now #{node.domain.full})" + return true + end + cert.openssl_body.extensions.each do |ext| + if ext.oid == "subjectAltName" + ips = [] + dns_names = [] + ext.value.split(",").each do |value| + value.strip! + ips << $1 if value =~ /^IP Address:(.*)$/ + dns_names << $1 if value =~ /^DNS:(.*)$/ + end + dns_names.sort! + if ips.first != node.ip_address + log :updating, "cert for node '#{node.name}' because ip_address has changed (from #{ips.first} to #{node.ip_address})" + return true + elsif dns_names != node.all_dns_names + log :updating, "cert for node '#{node.name}' because domain name aliases have changed" do + log "from: #{dns_names.inspect}" + log "to: #{node.all_dns_names.inspect})" + end + return true + end + end + end + end + return false + end + + # + # check the expiration of commercial certs, if any. + # + def warn_if_commercial_cert_will_soon_expire + require 'leap_cli/x509' + + self.all_dns_names.each do |domain| + if Util.file_exists?([:commercial_cert, domain]) + cert = X509.load_certificate_file([:commercial_cert, domain]) + path = Path.relative_path([:commercial_cert, domain]) + if cert.not_after < Time.now.utc + Util.log :error, "the commercial certificate '#{path}' has EXPIRED! " + + "You should renew it with `leap cert renew #{domain}`." + elsif cert.not_after < Time.now.advance(:months => 2) + Util.log :warning, "the commercial certificate '#{path}' will expire soon (#{cert.not_after}). "+ + "You should renew it with `leap cert renew #{domain}`." + end + end + end + end + + end + +end; end + diff --git a/lib/leap_cli/config/object.rb b/lib/leap_cli/config/object.rb new file mode 100644 index 00000000..16c41999 --- /dev/null +++ b/lib/leap_cli/config/object.rb @@ -0,0 +1,454 @@ +# encoding: utf-8 + +require 'erb' +require 'json/pure' # pure ruby implementation is required for our sorted trick to work. + +if $ruby_version < [1,9] + $KCODE = 'UTF8' +end +require 'ya2yaml' # pure ruby yaml + +module LeapCli + module Config + + # + # This class represents the configuration for a single node, service, or tag. + # Also, all the nested hashes are also of this type. + # + # It is called 'object' because it corresponds to an Object in JSON. + # + class Object < Hash + + attr_reader :env + attr_reader :node + + def initialize(environment=nil, node=nil) + raise ArgumentError unless environment.nil? || environment.is_a?(Config::Environment) + @env = environment + # an object that is a node as @node equal to self, otherwise all the + # child objects point back to the top level node. + @node = node || self + end + + def manager + @env.manager + end + + # + # TODO: deprecate node.global() + # + def global + @env + end + + def environment=(e) + self.store('environment', e) + end + + def environment + self['environment'] + end + + def duplicate(env) + new_object = self.deep_dup + new_object.set_environment(env, new_object) + end + + # + # export YAML + # + # We use pure ruby yaml exporter ya2yaml instead of SYCK or PSYCH because it + # allows us greater compatibility regardless of installed ruby version and + # greater control over how the yaml is exported (sorted keys, in particular). + # + def dump_yaml + evaluate(@node) + sorted_ya2yaml(:syck_compatible => true) + end + + # + # export JSON + # + def dump_json(options={}) + evaluate(@node) + if options[:format] == :compact + return self.to_json + else + excluded = {} + if options[:exclude] + options[:exclude].each do |key| + excluded[key] = self[key] + self.delete(key) + end + end + json_str = JSON.sorted_generate(self) + if excluded.any? + self.merge!(excluded) + end + return json_str + end + end + + def evaluate(context=@node) + evaluate_everything(context) + late_evaluate_everything(context) + end + + ## + ## FETCHING VALUES + ## + + def [](key) + get(key) + end + + # Overrride some default methods in Hash that are likely to + # be used as attributes. + alias_method :hkey, :key + def key; get('key'); end + + # + # make hash addressable like an object (e.g. obj['name'] available as obj.name) + # + def method_missing(method, *args, &block) + get!(method) + end + + def get(key) + begin + get!(key) + rescue NoMethodError + nil + end + end + + # override behavior of #default() from Hash + def default + get!('default') + end + + # + # Like a normal Hash#[], except: + # + # (1) lazily eval dynamic values when we encounter them. (i.e. strings that start with "= ") + # + # (2) support for nested references in a single string (e.g. ['a.b'] is the same as ['a']['b']) + # the dot path is always absolute, starting at the top-most object. + # + def get!(key) + key = key.to_s + if self.has_key?(key) + fetch_value(key) + elsif key =~ /\./ + # for keys with with '.' in them, we start from the root object (@node). + keys = key.split('.') + value = self.get!(keys.first) + if value.is_a? Config::Object + value.get!(keys[1..-1].join('.')) + else + value + end + else + raise NoMethodError.new(key, "No method '#{key}' for #{self.class}") + end + end + + # + # works like Hash#store(key, value), but supports our nested dot notation, + # just like get() does. + # + def set(key, value) + key = key.to_s + # for keys with with '.' in them, we pop off the first part + # and recursively call ourselves. + if key =~ /\./ + keys = key.split('.') + parent_value = self.get!(keys.first) + if parent_value.is_a?(Config::Object) + parent_value.set(keys[1..-1].join('.'), value) + else + parent_value.store(keys[1..-1].join('.'), value) + end + else + self.store(key, value) + end + return nil + end + + ## + ## COPYING + ## + + # + # A deep (recursive) merge with another Config::Object. + # + # If prefer_self is set to true, the value from self will be picked when there is a conflict + # that cannot be merged. + # + # Merging rules: + # + # - If a value is a hash, we recursively merge it. + # - If the value is simple, like a string, the new one overwrites the value. + # - If the value is an array: + # - If both old and new values are arrays, the new one replaces the old. + # - If one of the values is simple but the other is an array, the simple is added to the array. + # + def deep_merge!(object, prefer_self=false) + object.each do |key,new_value| + if self.has_key?('+'+key) + mode = :add + old_value = self.fetch '+'+key, nil + self.delete('+'+key) + elsif self.has_key?('-'+key) + mode = :subtract + old_value = self.fetch '-'+key, nil + self.delete('-'+key) + elsif self.has_key?('!'+key) + mode = :replace + old_value = self.fetch '!'+key, nil + self.delete('!'+key) + else + mode = :normal + old_value = self.fetch key, nil + end + + # clean up boolean + new_value = true if new_value == "true" + new_value = false if new_value == "false" + old_value = true if old_value == "true" + old_value = false if old_value == "false" + + # force replace? + if mode == :replace && prefer_self + value = old_value + + # merge hashes + elsif old_value.is_a?(Hash) || new_value.is_a?(Hash) + value = Config::Object.new(@env, @node) + old_value.is_a?(Hash) ? value.deep_merge!(old_value) : (value[key] = old_value if !old_value.nil?) + new_value.is_a?(Hash) ? value.deep_merge!(new_value, prefer_self) : (value[key] = new_value if !new_value.nil?) + + # merge nil + elsif new_value.nil? + value = old_value + elsif old_value.nil? + value = new_value + + # merge arrays when one value is not an array + elsif old_value.is_a?(Array) && !new_value.is_a?(Array) + (value = (old_value.dup << new_value).compact.uniq).delete('REQUIRED') + elsif new_value.is_a?(Array) && !old_value.is_a?(Array) + (value = (new_value.dup << old_value).compact.uniq).delete('REQUIRED') + + # merge two arrays + elsif old_value.is_a?(Array) && new_value.is_a?(Array) + if mode == :add + value = (old_value + new_value).sort.uniq + elsif mode == :subtract + value = new_value - old_value + elsif prefer_self + value = old_value + else + value = new_value + end + + # catch errors + elsif type_mismatch?(old_value, new_value) + raise 'Type mismatch. Cannot merge %s (%s) with %s (%s). Key is "%s", name is "%s".' % [ + old_value.inspect, old_value.class, + new_value.inspect, new_value.class, + key, self.class + ] + + # merge simple strings & numbers + else + if prefer_self + value = old_value + else + value = new_value + end + end + + # save value + self[key] = value + end + self + end + + def set_environment(env, node) + @env = env + @node = node + self.each do |key, value| + if value.is_a?(Config::Object) + value.set_environment(env, node) + end + end + end + + # + # like a reverse deep merge + # (self takes precedence) + # + def inherit_from!(object) + self.deep_merge!(object, true) + end + + # + # Make a copy of ourselves, except only including the specified keys. + # + # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b' + # + def pick(*keys) + keys.map(&:to_s).inject(self.class.new(@manager)) do |hsh, key| + value = self.get(key) + if !value.nil? + hsh[key.gsub('.','_')] = value + end + hsh + end + end + + def eval_file(filename) + evaluate_ruby(filename, File.read(filename)) + end + + protected + + # + # walks the object tree, eval'ing all the attributes that are dynamic ruby (e.g. value starts with '= ') + # + def evaluate_everything(context) + keys.each do |key| + obj = fetch_value(key, context) + if is_required_value_not_set?(obj) + Util::log 0, :warning, "required property \"#{key}\" is not set in node \"#{node.name}\"." + elsif obj.is_a? Config::Object + obj.evaluate_everything(context) + end + end + end + + # + # some keys need to be evaluated 'late', after all the other keys have been evaluated. + # + def late_evaluate_everything(context) + if @late_eval_list + @late_eval_list.each do |key, value| + self[key] = context.evaluate_ruby(key, value) + if is_required_value_not_set?(self[key]) + Util::log 0, :warning, "required property \"#{key}\" is not set in node \"#{node.name}\"." + end + end + end + values.each do |obj| + if obj.is_a? Config::Object + obj.late_evaluate_everything(context) + end + end + end + + # + # evaluates the string `value` as ruby in the context of self. + # (`key` is just passed for debugging purposes) + # + def evaluate_ruby(key, value) + self.instance_eval(value, key, 1) + rescue ConfigError => exc + raise exc # pass through + rescue SystemStackError => exc + Util::log 0, :error, "while evaluating node '#{self.name}'" + Util::log 0, "offending key: #{key}", :indent => 1 + Util::log 0, "offending string: #{value}", :indent => 1 + Util::log 0, "STACK OVERFLOW, BAILING OUT. There must be an eval loop of death (variables with circular dependencies).", :indent => 1 + raise SystemExit.new(1) + rescue FileMissing => exc + Util::bail! do + if exc.options[:missing] + Util::log :missing, exc.options[:missing].gsub('$node', self.name).gsub('$file', exc.path) + else + Util::log :error, "while evaluating node '#{self.name}'" + Util::log "offending key: #{key}", :indent => 1 + Util::log "offending string: #{value}", :indent => 1 + Util::log "error message: no file '#{exc}'", :indent => 1 + end + raise exc if DEBUG + end + rescue AssertionFailed => exc + Util.bail! do + Util::log :failed, "assertion while evaluating node '#{self.name}'" + Util::log 'assertion: %s' % exc.assertion, :indent => 1 + Util::log "offending key: #{key}", :indent => 1 + raise exc if DEBUG + end + rescue SyntaxError, StandardError => exc + Util::bail! do + Util::log :error, "while evaluating node '#{self.name}'" + Util::log "offending key: #{key}", :indent => 1 + Util::log "offending string: #{value}", :indent => 1 + Util::log "error message: #{exc.inspect}", :indent => 1 + raise exc if DEBUG + end + end + + private + + # + # fetches the value for the key, evaluating the value as ruby if it begins with '=' + # + def fetch_value(key, context=@node) + value = fetch(key, nil) + if value.is_a?(String) && value =~ /^=/ + # strings prefix with '=' are evaluated as ruby code. + if value =~ /^=> (.*)$/ + value = evaluate_later(key, $1) + elsif value =~ /^= (.*)$/ + value = context.evaluate_ruby(key, $1) + end + self[key] = value + elsif value.is_a?(Proc) + # the value might be a proc, set by a 'control' file + self[key] = value.call + end + return value + end + + def evaluate_later(key, value) + @late_eval_list ||= [] + @late_eval_list << [key, value] + '<evaluate later>' + end + + # + # when merging, we raise an error if this method returns true for the two values. + # + def type_mismatch?(old_value, new_value) + if old_value.is_a?(Boolean) && new_value.is_a?(Boolean) + # note: FalseClass and TrueClass are different classes + # so we can't do old_value.class == new_value.class + return false + elsif old_value.is_a?(String) && old_value =~ /^=/ + # pass through macros, since we don't know what the type will eventually be. + return false + elsif new_value.is_a?(String) && new_value =~ /^=/ + return false + elsif old_value.class == new_value.class + return false + else + return true + end + end + + # + # returns true if the value has not been changed and the default is "REQUIRED" + # + def is_required_value_not_set?(value) + if value.is_a? Array + value == ["REQUIRED"] + else + value == "REQUIRED" + end + end + + end # class + end # module +end # module
\ No newline at end of file diff --git a/lib/leap_cli/config/object_list.rb b/lib/leap_cli/config/object_list.rb new file mode 100644 index 00000000..80f89d92 --- /dev/null +++ b/lib/leap_cli/config/object_list.rb @@ -0,0 +1,215 @@ +require 'tsort' + +module LeapCli + module Config + # + # A list of Config::Object instances (internally stored as a hash) + # + class ObjectList < Hash + include TSort + + def initialize(config=nil) + if config + self.add(config['name'], config) + end + end + + # + # If the key is a string, the Config::Object it references is returned. + # + # If the key is a hash, we treat it as a condition and filter all the Config::Objects using the condition. + # A new ObjectList is returned. + # + # Examples: + # + # nodes['vpn1'] + # node named 'vpn1' + # + # nodes[:public_dns => true] + # all nodes with public dns + # + # nodes[:services => 'openvpn', 'location.country_code' => 'US'] + # all nodes with services containing 'openvpn' OR country code of US + # + # Sometimes, you want to do an OR condition with multiple conditions + # for the same field. Since hash keys must be unique, you can use + # an array representation instead: + # + # nodes[[:services, 'openvpn'], [:services, 'tor']] + # nodes with openvpn OR tor service + # + # nodes[:services => 'openvpn'][:tags => 'production'] + # nodes with openvpn AND are production + # + def [](key) + if key.is_a?(Hash) || key.is_a?(Array) + filter(key) + else + super key.to_s + end + end + + def exclude(node) + list = self.dup + list.delete(node.name) + return list + end + + def each_node(&block) + self.keys.sort.each do |node_name| + yield self[node_name] + end + end + + # + # filters this object list, producing a new list. + # filter is an array or a hash. see [] + # + def filter(filter) + results = Config::ObjectList.new + filter.each do |field, match_value| + field = field.is_a?(Symbol) ? field.to_s : field + match_value = match_value.is_a?(Symbol) ? match_value.to_s : match_value + if match_value.is_a?(String) && match_value =~ /^!/ + operator = :not_equal + match_value = match_value.sub(/^!/, '') + else + operator = :equal + end + each do |name, config| + value = config[field] + if value.is_a? Array + if operator == :equal && value.include?(match_value) + results[name] = config + elsif operator == :not_equal && !value.include?(match_value) + results[name] = config + end + elsif match_value.is_a? Array + if operator == :equal && match_value.include?(value) + results[name] = config + elsif operator == :not_equal && !match_value.include?(value) + results[name] = config + end + else + if operator == :equal && value == match_value + results[name] = config + elsif operator == :not_equal && value != match_value + results[name] = config + end + end + end + end + results + end + + def add(name, object) + self[name] = object + end + + # + # converts the hash of configs into an array of hashes, with ONLY the specified fields + # + def fields(*fields) + result = [] + keys.sort.each do |name| + result << self[name].pick(*fields) + end + result + end + + # + # like fields(), but returns an array of values instead of an array of hashes. + # + def field(field) + field = field.to_s + result = [] + keys.sort.each do |name| + result << self[name].get(field) + end + result + end + + # + # pick_fields(field1, field2, ...) + # + # generates a Hash from the object list, but with only the fields that are picked. + # + # If there are more than one field, then the result is a Hash of Hashes. + # If there is just one field, it is a simple map to the value. + # + # For example: + # + # "neighbors" = "= nodes_like_me[:services => :couchdb].pick_fields('domain.full', 'ip_address')" + # + # generates this: + # + # neighbors: + # couch1: + # domain_full: couch1.bitmask.net + # ip_address: "10.5.5.44" + # couch2: + # domain_full: couch2.bitmask.net + # ip_address: "10.5.5.52" + # + # But this: + # + # "neighbors": "= nodes_like_me[:services => :couchdb].pick_fields('domain.full')" + # + # will generate this: + # + # neighbors: + # couch1: couch1.bitmask.net + # couch2: couch2.bitmask.net + # + def pick_fields(*fields) + self.values.inject({}) do |hsh, node| + value = self[node.name].pick(*fields) + if fields.size == 1 + value = value.values.first + end + hsh[node.name] = value + hsh + end + end + + # + # Applies inherit_from! to all objects. + # + # 'env' specifies what environment should be for + # each object in the list. + # + def inherit_from!(object_list, env) + object_list.each do |name, object| + if self[name] + self[name].inherit_from!(object) + else + self[name] = object.duplicate(env) + end + end + end + + # + # topographical sort based on test dependency + # + def tsort_each_node(&block) + self.each_key(&block) + end + + def tsort_each_child(node_name, &block) + if self[node_name] + self[node_name].test_dependencies.each do |test_me_first| + if self[test_me_first] # TODO: in the future, allow for ability to optionally pull in all dependencies. + # not just the ones that pass the node filter. + yield(test_me_first) + end + end + end + end + + def names_in_test_dependency_order + self.tsort + end + + end + end +end diff --git a/lib/leap_cli/config/provider.rb b/lib/leap_cli/config/provider.rb new file mode 100644 index 00000000..0d8bc1f3 --- /dev/null +++ b/lib/leap_cli/config/provider.rb @@ -0,0 +1,22 @@ +# +# Configuration class for provider.json +# + +module LeapCli; module Config + class Provider < Object + attr_reader :environment + def set_env(e) + if e == 'default' + @environment = nil + else + @environment = e + end + end + def provider + self + end + def validate! + # nothing here yet :( + end + end +end; end diff --git a/lib/leap_cli/config/secrets.rb b/lib/leap_cli/config/secrets.rb new file mode 100644 index 00000000..ca851c74 --- /dev/null +++ b/lib/leap_cli/config/secrets.rb @@ -0,0 +1,87 @@ +# encoding: utf-8 +# +# A class for the secrets.json file +# + +module LeapCli; module Config + + class Secrets < Object + attr_reader :node_list + + def initialize(manager=nil) + super(manager) + @discovered_keys = {} + end + + # we can't use fetch() or get(), since those already have special meanings + def retrieve(key, environment) + environment ||= 'default' + self.fetch(environment, {})[key.to_s] + end + + def set(*args, &block) + if block_given? + set_with_block(*args, &block) + else + set_without_block(*args) + end + end + + # searches over all keys matching the regexp, checking to see if the value + # has been already used by any of them. + def taken?(regexp, value, environment) + self.keys.grep(regexp).each do |key| + return true if self.retrieve(key, environment) == value + end + return false + end + + def set_without_block(key, value, environment) + set_with_block(key, environment) {value} + end + + def set_with_block(key, environment, &block) + environment ||= 'default' + key = key.to_s + @discovered_keys[environment] ||= {} + @discovered_keys[environment][key] = true + self[environment] ||= {} + self[environment][key] ||= yield + end + + # + # if clean is true, then only secrets that have been discovered + # during this run will be exported. + # + # if environment is also pinned, then we will clean those secrets + # just for that environment. + # + # the clean argument should only be used when all nodes have + # been processed, otherwise secrets that are actually in use will + # get mistakenly removed. + # + def dump_json(clean=false) + pinned_env = LeapCli.leapfile.environment + if clean + self.each_key do |environment| + if pinned_env.nil? || pinned_env == environment + env = self[environment] + if env.nil? + raise StandardError.new("secrets.json file seems corrupted. No such environment '#{environment}'") + end + env.each_key do |key| + unless @discovered_keys[environment] && @discovered_keys[environment][key] + self[environment].delete(key) + end + end + if self[environment].empty? + self.delete(environment) + end + end + end + end + super() + end + end + +end; end diff --git a/lib/leap_cli/config/sources.rb b/lib/leap_cli/config/sources.rb new file mode 100644 index 00000000..aee860de --- /dev/null +++ b/lib/leap_cli/config/sources.rb @@ -0,0 +1,11 @@ +# encoding: utf-8 +# +# A class for the sources.json file +# + +module LeapCli + module Config + class Sources < Object + end + end +end diff --git a/lib/leap_cli/config/tag.rb b/lib/leap_cli/config/tag.rb new file mode 100644 index 00000000..6bd8d1e9 --- /dev/null +++ b/lib/leap_cli/config/tag.rb @@ -0,0 +1,25 @@ +# +# +# A class for node services or node tags. +# +# + +module LeapCli; module Config + + class Tag < Object + attr_reader :node_list + + def initialize(environment=nil) + super(environment) + @node_list = Config::ObjectList.new + end + + # don't copy the node list pointer when this object is dup'ed. + def initialize_copy(orig) + super + @node_list = Config::ObjectList.new + end + + end + +end; end diff --git a/lib/leap_cli/leapfile_extensions.rb b/lib/leap_cli/leapfile_extensions.rb new file mode 100644 index 00000000..cba321f4 --- /dev/null +++ b/lib/leap_cli/leapfile_extensions.rb @@ -0,0 +1,24 @@ +module LeapCli + class Leapfile + attr_reader :custom_vagrant_vm_line + attr_reader :leap_version + attr_reader :log + attr_reader :vagrant_basebox + + def vagrant_network + @vagrant_network ||= '10.5.5.0/24' + end + + private + + PRIVATE_IP_RANGES = /(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)/ + + def validate + Util::assert! vagrant_network =~ PRIVATE_IP_RANGES do + Util::log 0, :error, "in #{file}: vagrant_network is not a local private network" + end + return true + end + + end +end diff --git a/lib/leap_cli/load_libraries.rb b/lib/leap_cli/load_libraries.rb new file mode 100644 index 00000000..01384f78 --- /dev/null +++ b/lib/leap_cli/load_libraries.rb @@ -0,0 +1,23 @@ +# +# load the commonly needed leap_cli libraries that live in the platform. +# +# loaded by leap_cli's bootstrap.rb +# + +require 'leap_cli/log_filter' + +require 'leap_cli/config/object' +require 'leap_cli/config/node' +require 'leap_cli/config/node_cert' +require 'leap_cli/config/tag' +require 'leap_cli/config/provider' +require 'leap_cli/config/secrets' +require 'leap_cli/config/cloud' +require 'leap_cli/config/object_list' +require 'leap_cli/config/filter' +require 'leap_cli/config/environment' +require 'leap_cli/config/manager' + +require 'leap_cli/util/secret' +require 'leap_cli/util/vagrant' +require 'leap_cli/util/console_table' diff --git a/lib/leap_cli/log_filter.rb b/lib/leap_cli/log_filter.rb new file mode 100644 index 00000000..c73f3a91 --- /dev/null +++ b/lib/leap_cli/log_filter.rb @@ -0,0 +1,176 @@ +# +# A module to hide, modify, and colorize log entries. +# + +module LeapCli + module LogFilter + # + # options for formatters: + # + # :match => regexp for matching a log line + # :color => what color the line should be + # :style => what style the line should be + # :priority => what order the formatters are applied in. higher numbers first. + # :match_level => only apply filter at the specified log level + # :level => make this line visible at this log level or higher + # :replace => replace the matched text + # :prepend => insert text at start of message + # :append => append text to end of message + # :exit => force the exit code to be this (does not interrupt program, just + # ensures a specific exit code when the program eventually exits) + # + FORMATTERS = [ + # TRACE + { :match => /command finished/, :color => :white, :style => :dim, :match_level => 3, :priority => -10 }, + { :match => /executing locally/, :color => :yellow, :match_level => 3, :priority => -20 }, + + # DEBUG + #{ :match => /executing .*/, :color => :green, :match_level => 2, :priority => -10, :timestamp => true }, + #{ :match => /.*/, :color => :yellow, :match_level => 2, :priority => -30 }, + { :match => /^transaction:/, :level => 3 }, + + # INFO + { :match => /.*out\] (fatal:|ERROR:).*/, :color => :red, :match_level => 1, :priority => -10 }, + { :match => /Permission denied/, :color => :red, :match_level => 1, :priority => -20 }, + { :match => /sh: .+: command not found/, :color => :magenta, :match_level => 1, :priority => -30 }, + + # IMPORTANT + { :match => /^(E|e)rr ::/, :color => :red, :match_level => 0, :priority => -10, :exit => 1}, + { :match => /^ERROR:/, :color => :red, :priority => -10, :exit => 1}, + #{ :match => /.*/, :color => :blue, :match_level => 0, :priority => -20 }, + + # CLEANUP + #{ :match => /\s+$/, :replace => '', :priority => 0}, + + # DEBIAN PACKAGES + { :match => /^(Hit|Ign) /, :color => :green, :priority => -20}, + { :match => /^Err /, :color => :red, :priority => -20}, + { :match => /^W(ARNING)?: /, :color => :yellow, :priority => -20}, + { :match => /^E: /, :color => :red, :priority => -20}, + { :match => /already the newest version/, :color => :green, :priority => -20}, + { :match => /WARNING: The following packages cannot be authenticated!/, :color => :red, :level => 0, :priority => -10}, + + # PUPPET + { :match => /^(W|w)arning: Not collecting exported resources without storeconfigs/, :level => 2, :color => :yellow, :priority => -10}, + { :match => /^(W|w)arning: Found multiple default providers for vcsrepo:/, :level => 2, :color => :yellow, :priority => -10}, + { :match => /^(W|w)arning: .*is deprecated.*$/, :level => 2, :color => :yellow, :priority => -10}, + { :match => /^(W|w)arning: Scope.*$/, :level => 2, :color => :yellow, :priority => -10}, + #{ :match => /^(N|n)otice:/, :level => 1, :color => :cyan, :priority => -20}, + #{ :match => /^(N|n)otice:.*executed successfully$/, :level => 2, :color => :cyan, :priority => -15}, + { :match => /^(W|w)arning:/, :level => 0, :color => :yellow, :priority => -20}, + { :match => /^Duplicate declaration:/, :level => 0, :color => :red, :priority => -20}, + #{ :match => /Finished catalog run/, :level => 0, :color => :green, :priority => -10}, + { :match => /^APPLY COMPLETE \(changes made\)/, :level => 0, :color => :green, :style => :bold, :priority => -10}, + { :match => /^APPLY COMPLETE \(no changes\)/, :level => 0, :color => :green, :style => :bold, :priority => -10}, + + # PUPPET FATAL ERRORS + { :match => /^(E|e)rr(or|):/, :level => 0, :color => :red, :priority => -1, :exit => 1}, + { :match => /^Wrapped exception:/, :level => 0, :color => :red, :priority => -1, :exit => 1}, + { :match => /^Failed to parse template/, :level => 0, :color => :red, :priority => -1, :exit => 1}, + { :match => /^Execution of.*returned/, :level => 0, :color => :red, :priority => -1, :exit => 1}, + { :match => /^Parameter matches failed:/, :level => 0, :color => :red, :priority => -1, :exit => 1}, + { :match => /^Syntax error/, :level => 0, :color => :red, :priority => -1, :exit => 1}, + { :match => /^Cannot reassign variable/, :level => 0, :color => :red, :priority => -1, :exit => 1}, + { :match => /^Could not find template/, :level => 0, :color => :red, :priority => -1, :exit => 1}, + { :match => /^APPLY COMPLETE.*fail/, :level => 0, :color => :red, :style => :bold, :priority => -1, :exit => 1}, + + # TESTS + { :match => /^PASS: /, :color => :green, :priority => -20}, + { :match => /^(FAIL|ERROR): /, :color => :red, :priority => -20}, + { :match => /^(SKIP|WARN): /, :color => :yellow, :priority => -20}, + { :match => /\d+ tests: \d+ passes, \d+ skips, 0 warnings, 0 failures, 0 errors/, + :color => :green, :style => :bold, :priority => -20 }, + { :match => /\d+ tests: \d+ passes, \d+ skips, [1-9][0-9]* warnings, 0 failures, 0 errors/, + :color => :yellow, :style => :bold, :priority => -20 }, + { :match => /\d+ tests: \d+ passes, \d+ skips, \d+ warnings, \d+ failures, [1-9][0-9]* errors/, + :color => :red, :style => :bold, :priority => -20 }, + { :match => /\d+ tests: \d+ passes, \d+ skips, \d+ warnings, [1-9][0-9]* failures, \d+ errors/, + :color => :red, :style => :bold, :priority => -20 }, + + # LOG SUPPRESSION + { :match => /^(W|w)arning: You cannot collect without storeconfigs being set/, :level => 2, :priority => 10}, + { :match => /^(W|w)arning: You cannot collect exported resources without storeconfigs being set/, :level => 2, :priority => 10} + ] + + SORTED_FORMATTERS = FORMATTERS.sort_by { |i| -(i[:priority] || i[:prio] || 0) } + + # + # same as normal formatters, but only applies to the title, not the message. + # + TITLE_FORMATTERS = [ + # red + { :match => /fatal_error/, :replace => 'fatal error:', :color => :red, :style => :bold }, + { :match => /error/, :color => :red, :style => :bold }, + { :match => /removed/, :color => :red, :style => :bold }, + { :match => /removing/, :color => :red, :style => :bold }, + { :match => /destroyed/, :color => :red, :style => :bold }, + { :match => /destroying/, :color => :red, :style => :bold }, + { :match => /terminated/, :color => :red, :style => :bold }, + { :match => /failed/, :replace => 'FAILED', :color => :red, :style => :bold }, + { :match => /bailing/, :replace => 'bailing', :color => :red, :style => :bold }, + { :match => /invalid/, :color => :red, :style => :bold }, + + # yellow + { :match => /warning/, :replace => 'warning:', :color => :yellow, :style => :bold }, + { :match => /missing/, :color => :yellow, :style => :bold }, + { :match => /skipping/, :color => :yellow, :style => :bold }, + + # green + { :match => /created/, :color => :green, :style => :bold }, + { :match => /completed/, :color => :green, :style => :bold }, + { :match => /ran/, :color => :green, :style => :bold }, + { :match => /^registered/, :color => :green, :style => :bold }, + + # cyan + { :match => /note/, :replace => 'NOTE:', :color => :cyan, :style => :bold }, + + # magenta + { :match => /nochange/, :replace => 'no change', :color => :magenta }, + { :match => /^loading/, :color => :magenta }, + ] + + def self.apply_message_filters(message) + return self.apply_filters(SORTED_FORMATTERS, message) + end + + def self.apply_title_filters(title) + return self.apply_filters(TITLE_FORMATTERS, title) + end + + private + + def self.apply_filters(formatters, message) + level = LeapCli.logger.log_level + result = {} + formatters.each do |formatter| + if (formatter[:match_level] == level || formatter[:match_level].nil?) + if message =~ formatter[:match] + # puts "applying formatter #{formatter.inspect}" + result[:level] = formatter[:level] if formatter[:level] + result[:color] = formatter[:color] if formatter[:color] + result[:style] = formatter[:style] || formatter[:attribute] # (support original cap colors) + + message.gsub!(formatter[:match], formatter[:replace]) if formatter[:replace] + message.replace(formatter[:prepend] + message) unless formatter[:prepend].nil? + message.replace(message + formatter[:append]) unless formatter[:append].nil? + message.replace(Time.now.strftime('%Y-%m-%d %T') + ' ' + message) if formatter[:timestamp] + + if formatter[:exit] + LeapCli::Util.exit_status(formatter[:exit]) + end + + # stop formatting, unless formatter was just for string replacement + break unless formatter[:replace] + end + end + end + + if result[:color] == :hide + return [nil, {}] + else + return [message, result] + end + end + + end +end diff --git a/lib/leap_cli/macros.rb b/lib/leap_cli/macros.rb deleted file mode 100644 index fdb9a94e..00000000 --- a/lib/leap_cli/macros.rb +++ /dev/null @@ -1,16 +0,0 @@ -# -# MACROS -# -# The methods in these files are available in the context of a .json configuration file. -# (The module LeapCli::Macro is included in Config::Object) -# - -require_relative 'macros/core' -require_relative 'macros/files' -require_relative 'macros/haproxy' -require_relative 'macros/hosts' -require_relative 'macros/keys' -require_relative 'macros/nodes' -require_relative 'macros/secrets' -require_relative 'macros/stunnel' -require_relative 'macros/provider' diff --git a/lib/leap_cli/macros/files.rb b/lib/leap_cli/macros/files.rb index 04c94edf..602fdddb 100644 --- a/lib/leap_cli/macros/files.rb +++ b/lib/leap_cli/macros/files.rb @@ -79,19 +79,26 @@ module LeapCli # # file_path(:dkim_priv_key) {generate_dkim_key} # - # notes: + # Notes: # - # * argument 'path' is relative to Path.provider/files or - # Path.provider_base/files - # * the path returned by this method is absolute - # * the path stored for use later by rsync is relative to Path.provider - # * if the path does not exist locally, but exists in provider_base, + # * Argument 'path' must be relative to Path.provider/files or + # Path.provider_base/files. It is OK for the path to be prefixed with + # with 'files/', this prefix will be ignored. + # + # * The path returned by this method is an absolute path on the server. + # + # * The path stored for use later by rsync is relative to the local + # Path.provider. It should always be prefixed with 'files/' + # + # * If the path does not exist locally, but exists in provider_base, # then the default file from provider_base is copied locally. this # is required for rsync to work correctly. # + # NOTE: this is an aweful way to do this. It would be better + # to rsync twice. + # def remote_file_path(path, options={}, &block) local_path = local_file_path(path, options, &block) - return nil if local_path.nil? # if file is under Path.provider_base, we must copy the default file to @@ -110,9 +117,11 @@ module LeapCli end relative_path = Path.relative_path(local_path) - relative_path.sub!(/^files\//, '') # remove "files/" prefix @node.file_paths << relative_path - return File.join(Leap::Platform.files_dir, relative_path) + return File.join( + Leap::Platform.files_dir, + relative_path.sub(/^files\//, '') + ) end # deprecated diff --git a/lib/leap_cli/macros/haproxy.rb b/lib/leap_cli/macros/haproxy.rb index 602ae726..3fef24c4 100644 --- a/lib/leap_cli/macros/haproxy.rb +++ b/lib/leap_cli/macros/haproxy.rb @@ -26,7 +26,7 @@ module LeapCli # create a simple map for node name -> local stunnel accept port accept_ports = stunnel_clients.inject({}) do |hsh, stunnel_entry| - name = stunnel_entry.first.sub /_[0-9]+$/, '' + name = stunnel_entry.first.sub(/_[0-9]+$/, '') hsh[name] = stunnel_entry.last['accept_port'] hsh end diff --git a/lib/leap_cli/macros/keys.rb b/lib/leap_cli/macros/keys.rb index e7a75cfb..9cc01fe7 100644 --- a/lib/leap_cli/macros/keys.rb +++ b/lib/leap_cli/macros/keys.rb @@ -29,7 +29,7 @@ module LeapCli # generating key if it is missing # def tor_public_key_path(path_name, key_type) - file_path(path_name) { generate_tor_key(key_type) } + remote_file_path(path_name) { generate_tor_key(key_type) } end # @@ -37,7 +37,7 @@ module LeapCli # generating key if it is missing # def tor_private_key_path(path_name, key_type) - file_path(path_name) { generate_tor_key(key_type) } + remote_file_path(path_name) { generate_tor_key(key_type) } end # @@ -55,15 +55,15 @@ module LeapCli require 'base32' require 'base64' require 'openssl' - path = Path.find_file([path_name, self.name]) - if path && File.exists?(path) + path = Path.named_path([path_name, self.name]) + if path && File.exist?(path) public_key_str = File.readlines(path).grep(/^[^-]/).join public_key = Base64.decode64(public_key_str) public_key = public_key.slice(22..-1) # Tor ignores the 22 byte SPKI header sha1sum = Digest::SHA1.new.digest(public_key) Base32.encode(sha1sum.slice(0,10)).downcase else - LeapCli.log :warning, 'Tor public key file "%s" does not exist' % tor_public_key_path + LeapCli.log :warning, 'Tor public key file "%s" does not exist' % path end end diff --git a/lib/leap_cli/ssh.rb b/lib/leap_cli/ssh.rb new file mode 100644 index 00000000..8b604d1d --- /dev/null +++ b/lib/leap_cli/ssh.rb @@ -0,0 +1,7 @@ +require 'sshkit' +require_relative 'ssh/options' +require_relative 'ssh/backend' +require_relative 'ssh/formatter' +require_relative 'ssh/scripts' +require_relative 'ssh/remote_command' +require_relative 'ssh/key' diff --git a/lib/leap_cli/ssh/backend.rb b/lib/leap_cli/ssh/backend.rb new file mode 100644 index 00000000..3894d815 --- /dev/null +++ b/lib/leap_cli/ssh/backend.rb @@ -0,0 +1,209 @@ +# +# A custome SSHKit backend, derived from the default netssh backend. +# Our custom backend modifies the logging behavior and gracefully captures +# common exceptions. +# + +require 'stringio' +require 'timeout' +require 'sshkit' +require 'leap_cli/ssh/formatter' +require 'leap_cli/ssh/scripts' + +module SSHKit + class Command + # + # override exit_status in order to be less verbose + # + def exit_status=(new_exit_status) + @finished_at = Time.now + @exit_status = new_exit_status + if options[:raise_on_non_zero_exit] && exit_status > 0 + message = "" + message += "exit status: " + exit_status.to_s + "\n" + message += "stdout: " + (full_stdout.strip.empty? ? "Nothing written" : full_stdout.strip) + "\n" + message += "stderr: " + (full_stderr.strip.empty? ? 'Nothing written' : full_stderr.strip) + "\n" + raise Failed, message + end + end + end +end + +module LeapCli + module SSH + class Backend < SSHKit::Backend::Netssh + + # since the @pool is a class instance variable, we need to copy + # the code from the superclass that initializes it. boo + @pool = SSHKit::Backend::ConnectionPool.new + + # modify to pass itself to the block, instead of relying on instance_exec. + def run + Thread.current["sshkit_backend"] = self + # was: instance_exec(@host, &@block) + @block.call(self, @host) + ensure + Thread.current["sshkit_backend"] = nil + end + + # if set, all the commands will begin with: + # sudo -u #{@user} -- sh -c '<command>' + def set_user(user='root') + @user = user + end + + # + # like default capture, but gracefully logs failures for us + # last argument can be an options hash. + # + # available options: + # + # :fail_msg - [nil] if set, log this instead of the default + # fail message. + # + # :raise_error - [nil] if true, then reraise failed command exception. + # + # :log_cmd - [false] if true, log what the command is that gets run. + # + # :log_output - [true] if true, log each output from the command as + # it is received. + # + # :log_finish - [false] if true, log the exit status and time + # to completion + # + # :log_wrap - [nil] passed to log method as :wrap option. + # + def capture(*args) + extract_options(args) + initialize_logger(:log_output => false) + rescue_ssh_errors(*args) do + return super(*args) + end + end + + # + # like default execute, but log the results as they come in. + # + # see capture() for available options + # + def stream(*args) + extract_options(args) + initialize_logger + rescue_ssh_errors(*args) do + execute(*args) + end + end + + def log(*args, &block) + @logger ||= LeapCli.new_logger + @logger.log(*args, &block) + end + + # some prewritten servers-side scripts + def scripts + @scripts ||= LeapCli::SSH::Scripts.new(self, @host.hostname) + end + + # + # sshkit just passes upload! and download! to Net::SCP, but Net::SCP + # make it impossible to set the file permissions. Here is how the mode + # is determined, from upload.rb: + # + # mode = channel[:stat] ? channel[:stat].mode & 07777 : channel[:options][:mode] + # + # The stat info from the file always overrides the mode you pass in options. + # However, the channel[:options][:mode] will be applied for pure in-memory + # uploads. So, if the mode is set, we convert the upload to be a memory + # upload instead of a file upload. + # + # Stupid, but blame Net::SCP. + # + def upload!(src, dest, options={}) + if options[:mode] + if src.is_a?(StringIO) + content = src + else + content = StringIO.new(File.read(src)) + end + super(content, dest, options) + else + super(src, dest, options) + end + end + + private + + # + # creates a new logger instance for this specific ssh command. + # by doing this, each ssh session has its own logger and its own + # indentation. + # + # potentially modifies 'args' array argument. + # + def initialize_logger(default_options={}) + @logger ||= LeapCli.new_logger + @output = LeapCli::SSH::Formatter.new(@logger, @host, default_options.merge(@options)) + end + + def extract_options(args) + if args.last.is_a? Hash + @options = args.pop + else + @options = {} + end + end + + # + # capture common exceptions + # + def rescue_ssh_errors(*args, &block) + yield + rescue Net::SSH::HostKeyMismatch => exc + @logger.log(:fatal_error, "Host key mismatch!") do + @logger.log(exc.to_s) + @logger.log("The ssh host key for the server does not match what is on "+ + " file in `%s`." % Path.named_path(:known_hosts)) + @logger.log("One of these is happening:") do + @logger.log("There is an active Man in The Middle attack against you.") + @logger.log("Or, someone has generated new host keys for the server " + + "and your provider files are out of date.") + @logger.log("Or, a new server is using this IP address " + + "and your provider files are out of date.") + @logger.log("Or, the server configuration has changed to use a different host key.") + end + @logger.log("You can pin a different host key using `leap node init NODE`, " + + "but you must verify the fingerprint of the new host key!") + end + exit(1) + rescue StandardError => exc + if exc.is_a?(SSHKit::Command::Failed) || exc.is_a?(SSHKit::Runner::ExecuteError) + if @options[:raise_error] + raise LeapCli::SSH::ExecuteError, exc.to_s + elsif @options[:fail_msg] + @logger.log(@options[:fail_msg], host: @host.hostname, :color => :red) + else + @logger.log(:failed, args.join(' '), host: @host.hostname) do + @logger.log(exc.to_s.strip, wrap: true) + end + end + elsif exc.is_a?(Timeout::Error) || exc.is_a?(Net::SSH::ConnectionTimeout) + @logger.log(:failed, args.join(' '), host: @host.hostname) do + @logger.log("Connection timed out") + end + if @options[:raise_error] + raise LeapCli::SSH::TimeoutError, exc.to_s + end + else + raise + end + return nil + end + + def output + @output ||= LeapCli::SSH::Formatter.new(@logger, @host) + end + + end + end +end + diff --git a/lib/leap_cli/ssh/formatter.rb b/lib/leap_cli/ssh/formatter.rb new file mode 100644 index 00000000..c2e386dc --- /dev/null +++ b/lib/leap_cli/ssh/formatter.rb @@ -0,0 +1,70 @@ +# +# A custom SSHKit formatter that uses LeapLogger. +# + +require 'sshkit' + +module LeapCli + module SSH + + class Formatter < SSHKit::Formatter::Abstract + + DEFAULT_OPTIONS = { + :log_cmd => false, # log what the command is that gets run. + :log_output => true, # log each output from the command as it is received. + :log_finish => false # log the exit status and time to completion. + } + + def initialize(logger, host, options={}) + @logger = logger || LeapCli.new_logger + @host = host + @options = DEFAULT_OPTIONS.merge(options) + end + + def write(obj) + @logger.log(obj.to_s, :host => @host.hostname) + end + + def log_command_start(command) + if @options[:log_cmd] + @logger.log(:running, "`" + command.to_s + "`", :host => @host.hostname) + end + end + + def log_command_data(command, stream_type, stream_data) + if @options[:log_output] + color = stream_type == :stderr ? :red : nil + @logger.log(stream_data.to_s.chomp, + :color => color, :host => @host.hostname, :wrap => options[:log_wrap]) + end + end + + def log_command_exit(command) + if @options[:log_finish] + runtime = sprintf('%5.3fs', command.runtime) + if command.failure? + message = "in #{runtime} with status #{command.exit_status}." + @logger.log(:failed, message, :host => @host.hostname) + else + message = "in #{runtime}." + @logger.log(:completed, message, :host => @host.hostname) + end + end + end + end + + end +end + + # + # A custom InteractionHandler that will output the results as they come in. + # + #class LoggingInteractionHandler + # def initialize(hostname, logger=nil) + # @hostname = hostname + # @logger = logger || LeapCli.new_logger + # end + # def on_data(command, stream_name, data, channel) + # @logger.log(data, host: @hostname, wrap: true) + # end + #end diff --git a/lib/leap_cli/ssh/key.rb b/lib/leap_cli/ssh/key.rb new file mode 100644 index 00000000..76223b7e --- /dev/null +++ b/lib/leap_cli/ssh/key.rb @@ -0,0 +1,310 @@ +# +# A wrapper around OpenSSL::PKey::RSA instances to provide a better api for +# dealing with SSH keys. +# +# NOTES: +# +# cipher 'ssh-ed25519' not supported yet because we are waiting +# for support in Net::SSH +# +# there are many ways to represent an SSH key, since SSH keys can be of +# a variety of types. +# +# To confuse matters more, there are multiple binary representations. +# So, for example, an RSA key has a native SSH representation +# (two bignums, e followed by n), and a DER representation. +# +# AWS uses fingerprints of the DER representation, but SSH typically reports +# fingerprints of the SSH representation. +# +# Also, SSH public key files are base64 encoded, but with whitespace removed +# so it all goes on one line. +# +# Some useful links: +# +# https://stackoverflow.com/questions/3162155/convert-rsa-public-key-to-rsa-der +# https://net-ssh.github.io/ssh/v2/api/classes/Net/SSH/Buffer.html +# https://serverfault.com/questions/603982/why-does-my-openssh-key-fingerprint-not-match-the-aws-ec2-console-keypair-finger +# + +require 'net/ssh' +require 'forwardable' +require 'base64' + +module LeapCli + module SSH + class Key + extend Forwardable + + attr_accessor :filename + attr_accessor :comment + + # supported ssh key types, in order of preference + SUPPORTED_TYPES = ['ssh-rsa', 'ecdsa-sha2-nistp256'] + SUPPORTED_TYPES_RE = /(#{SUPPORTED_TYPES.join('|')})/ + + ## + ## CLASS METHODS + ## + + def self.load(arg1, arg2=nil) + key = nil + if arg1.is_a? OpenSSL::PKey::RSA + key = Key.new arg1 + elsif arg1.is_a? String + if arg1 =~ /^ssh-/ + type, data = arg1.split(' ') + key = Key.new load_from_data(data, type) + elsif File.exist? arg1 + key = Key.new load_from_file(arg1) + key.filename = arg1 + else + key = Key.new load_from_data(arg1, arg2) + end + end + return key + rescue StandardError + end + + def self.load_from_file(filename) + public_key = nil + private_key = nil + begin + public_key = Net::SSH::KeyFactory.load_public_key(filename) + rescue NotImplementedError, Net::SSH::Exception, OpenSSL::PKey::PKeyError + begin + private_key = Net::SSH::KeyFactory.load_private_key(filename) + rescue NotImplementedError, Net::SSH::Exception, OpenSSL::PKey::PKeyError + end + end + public_key || private_key + end + + def self.load_from_data(data, type='ssh-rsa') + public_key = nil + private_key = nil + begin + public_key = Net::SSH::KeyFactory.load_data_public_key("#{type} #{data}") + rescue NotImplementedError, Net::SSH::Exception, OpenSSL::PKey::PKeyError + begin + private_key = Net::SSH::KeyFactory.load_data_private_key("#{type} #{data}") + rescue NotImplementedError, Net::SSH::Exception, OpenSSL::PKey::PKeyError + end + end + public_key || private_key + end + + def self.my_public_keys + load_keys_from_paths File.join(ENV['HOME'], '.ssh', '*.pub') + end + + def self.provider_public_keys + load_keys_from_paths Path.named_path([:user_ssh, '*']) + end + + # + # Picks one key out of an array of keys that we think is the "best", + # based on the order of preference in SUPPORTED_TYPES + # + # Currently, this does not take bitsize into account. + # + def self.pick_best_key(keys) + keys.select {|k| + SUPPORTED_TYPES.include?(k.type) + }.sort {|a,b| + SUPPORTED_TYPES.index(a.type) <=> SUPPORTED_TYPES.index(b.type) + }.first + end + + # + # takes a string with one or more ssh keys, one key per line, + # and returns an array of Key objects. + # + # the lines should be in one of these formats: + # + # 1. <hostname> <key-type> <key> + # 2. <key-type> <key> + # + def self.parse_keys(string) + keys = [] + lines = string.split("\n").grep(/^[^#]/) + lines.each do |line| + if line =~ / #{Key::SUPPORTED_TYPES_RE} / + # <hostname> <key-type> <key> + keys << line.split(' ')[1..2] + elsif line =~ /^#{Key::SUPPORTED_TYPES_RE} / + # <key-type> <key> + keys << line.split(' ') + end + end + return keys.map{|k| Key.load(k[1], k[0])} + end + + # + # takes a string with one or more ssh keys, one key per line, + # and returns a string that specified the ssh key algorithms + # that are supported by the keys, in order of preference. + # + # eg: ecdsa-sha2-nistp256,ssh-rsa,ssh-ed25519 + # + def self.supported_host_key_algorithms(string) + if string + self.parse_keys(string).map {|key| + key.type + }.join(',') + else + "" + end + end + + private + + def self.load_keys_from_paths(key_glob) + keys = [] + Dir.glob(key_glob).each do |file| + key = Key.load(file) + if key && key.public? + keys << key + end + end + return keys + end + + ## + ## INSTANCE METHODS + ## + + public + + def initialize(p_key) + @key = p_key + end + + def_delegator :@key, :ssh_type, :type + def_delegator :@key, :public_encrypt, :public_encrypt + def_delegator :@key, :public_decrypt, :public_decrypt + def_delegator :@key, :private_encrypt, :private_encrypt + def_delegator :@key, :private_decrypt, :private_decrypt + def_delegator :@key, :params, :params + def_delegator :@key, :to_text, :to_text + + def public_key + Key.new(@key.public_key) + end + + def private_key + Key.new(@key.private_key) + end + + def private? + @key.respond_to?(:private?) ? @key.private? : @key.private_key? + end + + def public? + @key.respond_to?(:public?) ? @key.public? : @key.public_key? + end + + # + # three arguments: + # + # - digest: one of md5, sha1, sha256, etc. (default sha256) + # - encoding: either :hex (default) or :base64 + # - type: fingerprint type, either :ssh (default) or :der + # + # NOTE: + # + # * I am not sure how to make a fingerprint for OpenSSL::PKey::EC::Point + # + # * AWS reports fingerprints using MD5 digest for uploaded ssh keys, + # but SHA1 for keys it created itself. + # + # * Also, AWS fingerprints are digests on the DER encoding of the key. + # But standard SSH fingerprints are digests of SSH encoding of the key. + # + # * Other tools will sometimes display fingerprints in hex and sometimes + # in base64. Arrrgh. + # + def fingerprint(type: :ssh, digest: :sha256, encoding: :hex) + require 'digest' + + digest = digest.to_s.upcase + digester = case digest + when "MD5" then Digest::MD5.new + when "SHA1" then Digest::SHA1.new + when "SHA256" then Digest::SHA256.new + when "SHA384" then Digest::SHA384.new + when "SHA512" then Digest::SHA512.new + else raise ArgumentError, "digest #{digest} is unknown" + end + + keymatter = nil + if type == :der && @key.respond_to?(:to_der) + keymatter = @key.to_der + else + keymatter = self.raw_key.to_s + end + + fp = nil + if encoding == :hex + fp = digester.hexdigest(keymatter) + elsif encoding == :base64 + fp = Base64.encode64(digester.digest(keymatter)).sub(/=$/, '') + else + raise ArgumentError, "encoding #{encoding} not understood" + end + + if digest == "MD5" && encoding == :hex + return fp.scan(/../).join(':') + else + return fp + end + end + + # + # not sure if this will always work, but is seems to for now. + # + def bits + Net::SSH::Buffer.from(:key, @key).to_s.split("\001\000").last.size * 8 + end + + def summary + if self.filename + "%s %s %s (%s)" % [self.type, self.bits, self.fingerprint, File.basename(self.filename)] + else + "%s %s %s" % [self.type, self.bits, self.fingerprint] + end + end + + def to_s + self.type + " " + self.key + end + + # + # base64 encoding of the key, with spaces removed. + # + def key + [Net::SSH::Buffer.from(:key, @key).to_s].pack("m*").gsub(/\s/, "") + end + + def raw_key + Net::SSH::Buffer.from(:key, @key) + end + + def ==(other_key) + return false if other_key.nil? + return false if self.class != other_key.class + return self.to_text == other_key.to_text + end + + def in_known_hosts?(*identifiers) + identifiers.each do |identifier| + Net::SSH::KnownHosts.search_for(identifier).each do |key| + return true if self == key + end + end + return false + end + + end + end +end diff --git a/lib/leap_cli/ssh/options.rb b/lib/leap_cli/ssh/options.rb new file mode 100644 index 00000000..7bc06564 --- /dev/null +++ b/lib/leap_cli/ssh/options.rb @@ -0,0 +1,100 @@ +# +# Options for passing to the ruby gem ssh-net +# + +module LeapCli + module SSH + module Options + + # + # options passed to net-ssh. See + # https://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start + # for the available options. + # + def self.global_options + { + #:keys_only => true, + :global_known_hosts_file => Path.named_path(:known_hosts), + :user_known_hosts_file => '/dev/null', + :paranoid => true, + :verbose => net_ssh_log_level, + :auth_methods => ["publickey"], + :timeout => 5 + } + end + + def self.node_options(node, ssh_options_override=nil) + { + # :host_key_alias => node.name, << incompatible with ports in known_hosts + :host_name => node.ip_address, + :port => node.ssh.port + }.merge( + contingent_ssh_options_for_node(node) + ).merge( + ssh_options_override||{} + ) + end + + def self.options_from_args(args) + ssh_options = {} + if args[:port] + ssh_options[:port] = args[:port] + end + if args[:ip] + ssh_options[:host_name] = args[:ip] + end + if args[:auth_methods] + ssh_options[:auth_methods] = args[:auth_methods] + end + if args[:user] + ssh_options[:user] = args[:user] + end + return ssh_options + end + + def self.sanitize_command(cmd) + if cmd =~ /(^|\/| )rm / || cmd =~ /(^|\/| )unlink / + LeapCli.log :warning, "You probably don't want to do that. Run with --force if you are really sure." + exit(1) + else + cmd + end + end + + private + + def self.contingent_ssh_options_for_node(node) + opts = {} + if node.vagrant? + opts[:keys] = [LeapCli::Util::Vagrant.vagrant_ssh_key_file] + opts[:keys_only] = true # only use the keys specified above, and + # ignore whatever keys the ssh-agent is aware of. + opts[:paranoid] = false # we skip host checking for vagrant nodes, + # because fingerprint is different for everyone. + if LeapCli.logger.log_level <= 1 + opts[:verbose] = :error # suppress all the warnings about adding + # host keys to known_hosts, since it is + # not actually doing that. + end + end + if !node.supported_ssh_host_key_algorithms.empty? + opts[:host_key] = node.supported_ssh_host_key_algorithms + end + return opts + end + + def self.net_ssh_log_level + if DEBUG + case LeapCli.logger.log_level + when 1 then :error + when 2 then :info + else :debug + end + else + :fatal + end + end + + end + end +end
\ No newline at end of file diff --git a/lib/leap_cli/ssh/remote_command.rb b/lib/leap_cli/ssh/remote_command.rb new file mode 100644 index 00000000..0e9f2d55 --- /dev/null +++ b/lib/leap_cli/ssh/remote_command.rb @@ -0,0 +1,124 @@ +# +# Provides SSH.remote_command for running commands in parallel or in sequence +# on remote servers. +# +# The gem sshkit is used for this. +# + +require 'sshkit' +require 'leap_cli/ssh/options' +require 'leap_cli/ssh/backend' + +SSHKit.config.backend = LeapCli::SSH::Backend +LeapCli::SSH::Backend.config.ssh_options = LeapCli::SSH::Options.global_options + +# +# define remote_command +# +module LeapCli + module SSH + + class ExecuteError < StandardError + end + + class TimeoutError < ExecuteError + end + + # override default runner mode + class CustomCoordinator < SSHKit::Coordinator + private + def default_options + { in: :groups, limit: 10, wait: 0 } + end + end + + # + # Available options: + # + # :port -- ssh port + # :ip -- ssh ip + # :auth_methods -- e.g. ["pubkey", "password"] + # :user -- default 'root' + # + def self.remote_command(nodes, options={}, &block) + CustomCoordinator.new( + host_list( + nodes, + SSH::Options.options_from_args(options) + ) + ).each do |ssh, host| + LeapCli.log 2, "ssh options for #{host.hostname}: #{host.ssh_options.inspect}" + if host.user != 'root' + # if the ssh user is not root, we want to make the ssh commands + # switch to root before they are run: + ssh.set_user('root') + end + yield ssh, host + end + end + + # + # For example: + # + # SSH.remote_sync(nodes) do |sync, host| + # sync.source = '/from' + # sync.dest = '/to' + # sync.flags = '' + # sync.includes = [] + # sync.excludes = [] + # sync.exec + # end + # + def self.remote_sync(nodes, options={}, &block) + require 'rsync_command' + hosts = host_list( + nodes, + SSH::Options.options_from_args(options) + ) + rsync = RsyncCommand.new(:logger => LeapCli::logger) + rsync.asynchronously(hosts) do |sync, host| + sync.logger = LeapCli.new_logger + sync.user = host.user || fetch(:user, ENV['USER']) + sync.host = host.hostname + sync.ssh = SSH::Options.global_options.merge(host.ssh_options) + sync.chdir = Path.provider + yield(sync, host) + end + if rsync.failed? + LeapCli::Util.bail! do + LeapCli.log :failed, "to rsync to #{rsync.failures.map{|f|f[:dest][:host]}.join(' ')}" + end + end + end + + private + + def self.host_list(nodes, ssh_options_override={}) + if nodes.is_a?(Config::ObjectList) + list = nodes.values + elsif nodes.is_a?(Config::Node) + list = [nodes] + else + raise ArgumentError, "I don't understand the type of argument `nodes`" + end + list.collect do |node| + options = SSH::Options.node_options(node, ssh_options_override) + user = options.delete(:user) || 'root' + # + # note: whatever hostname is specified here will be what is used + # when loading options from .ssh/config. However, this value + # has no impact on the actual ip address that is connected to, + # which is determined by the :host_name value in ssh_options. + # + SSHKit::Host.new( + :hostname => node.domain.full, + :user => user, + :ssh_options => options + ) + end + end + + end +end + + diff --git a/lib/leap_cli/ssh/scripts.rb b/lib/leap_cli/ssh/scripts.rb new file mode 100644 index 00000000..3dd6b604 --- /dev/null +++ b/lib/leap_cli/ssh/scripts.rb @@ -0,0 +1,163 @@ +# +# Common commands that we would like to run on remote servers. +# +# These scripts are available via: +# +# SSH.remote_command(nodes) do |ssh, host| +# ssh.script.custom_script_name +# end +# + +module LeapCli + module SSH + class Scripts + + REQUIRED_PACKAGES = "puppet rsync lsb-release locales" + + attr_reader :ssh, :host + def initialize(backend, hostname) + @ssh = backend + @host = hostname + end + + # + # creates directories that are owned by root and 700 permissions + # + def mkdirs(*dirs) + raise ArgumentError.new('illegal dir name') if dirs.grep(/[\' ]/).any? + ssh.stream dirs.collect{|dir| "mkdir -m 700 -p #{dir}; "}.join + end + + # + # echos "ok" if the node has been initialized and the required packages are installed, bails out otherwise. + # + def assert_initialized + begin + test_initialized_file = "test -f #{Leap::Platform.init_path}" + check_required_packages = "! dpkg-query -W --showformat='${Status}\n' #{REQUIRED_PACKAGES} 2>&1 | grep -q -E '(deinstall|no packages)'" + ssh.stream "#{test_initialized_file} && #{check_required_packages} && echo ok", :raise_error => true + rescue SSH::ExecuteError + ssh.log :error, "running deploy: node not initialized. Run `leap node init #{host}`.", :host => host + raise # will skip further action on this node + end + end + + # + # bails out the deploy if the file /etc/leap/no-deploy exists. + # + def check_for_no_deploy + begin + ssh.stream "test ! -f /etc/leap/no-deploy", :raise_error => true, :log_output => false + rescue SSH::TimeoutError + raise + rescue SSH::ExecuteError + ssh.log :warning, "can't continue because file /etc/leap/no-deploy exists", :host => host + raise # will skip further action on this node + end + end + + # + # dumps debugging information + # + def debug + output = ssh.capture "#{Leap::Platform.leap_dir}/bin/debug.sh" + ssh.log(output, :wrap => true, :host => host, :color => :cyan) + end + + # + # dumps the recent deploy history to the console + # + def history(lines) + cmd = "(test -s /var/log/leap/deploy-summary.log && tail -n #{lines} /var/log/leap/deploy-summary.log) || (test -s /var/log/leap/deploy-summary.log.1 && tail -n #{lines} /var/log/leap/deploy-summary.log.1) || (echo 'no history')" + history = ssh.capture(cmd, :log_output => false) + if history + ssh.log host, :color => :cyan, :style => :bold do + ssh.log history, :wrap => true + end + end + end + + # + # apply puppet! weeeeeee + # + def puppet_apply(options) + cmd = "#{Leap::Platform.leap_dir}/bin/puppet_command set_hostname apply #{flagize(options)}" + ssh.stream cmd, :log_finish => true + end + + def install_authorized_keys + ssh.log :updating, "authorized_keys" do + mkdirs '/root/.ssh' + ssh.upload! LeapCli::Path.named_path(:authorized_keys), '/root/.ssh/authorized_keys', :mode => 0600 + end + end + + # + # for vagrant nodes, we install insecure vagrant key to authorized_keys2, since deploy + # will overwrite authorized_keys. + # + # why force the insecure vagrant key? + # if we don't do this, then first time initialization might fail if the user has many keys + # (ssh will bomb out before it gets to the vagrant key). + # and it really doesn't make sense to ask users to pin the insecure vagrant key in their + # .ssh/config files. + # + def install_insecure_vagrant_key + ssh.log :installing, "insecure vagrant key" do + mkdirs '/root/.ssh' + ssh.upload! LeapCli::Path.vagrant_ssh_pub_key_file, '/root/.ssh/authorized_keys2', :mode => 0600 + end + end + + def install_prerequisites + bin_dir = File.join(Leap::Platform.leap_dir, 'bin') + node_init_path = File.join(bin_dir, 'node_init') + ssh.log :running, "node_init script" do + mkdirs bin_dir + ssh.upload! LeapCli::Path.node_init_script, node_init_path, :mode => 0700 + ssh.stream node_init_path, :log_wrap => true + end + end + + # + # AWS debian images only allow you to login as admin. This is done with a + # custom command in /root/.ssh/authorized_keys, instead of by modifying + # /etc/ssh/sshd_config. + # + # We need to be able to ssh as root for scp and rsync to work. + # + # This command is run as 'admin', with a sudo wrapper. In order for the + # sudo to work, the command must be specified as separate arguments with + # no spaces (that is how ssh-kit works). + # + def allow_root_ssh + ssh.execute 'cp', '/home/admin/.ssh/authorized_keys', '/root/.ssh/authorized_keys' + end + + # + # uploads an acme challenge for renewing certificates using Let's Encrypt CA. + # + # Filename is returned from acme api, so it must not be trusted. + # + def upload_acme_challenge(filename, content) + path = '/srv/acme/' + filename.gsub(/[^a-zA-Z0-9_-]/, '') + ssh.upload! StringIO.new(content), path, :mode => 0444 + end + + private + + def flagize(hsh) + hsh.inject([]) {|str, item| + if item[1] === false + str + elsif item[1] === true + str << "--" + item[0].to_s + else + str << "--" + item[0].to_s + " " + item[1].inspect + end + }.join(' ') + end + + end + end +end
\ No newline at end of file diff --git a/lib/leap_cli/util/console_table.rb b/lib/leap_cli/util/console_table.rb new file mode 100644 index 00000000..ccdcc2ab --- /dev/null +++ b/lib/leap_cli/util/console_table.rb @@ -0,0 +1,62 @@ +module LeapCli; module Util + + class ConsoleTable + def table + @rows = [] + @cell_options = [] + + @row_options = [] + @column_widths = [] + @column_options = [] + + @current_row = 0 + @current_column = 0 + yield + end + + def row(options=nil) + @current_column = 0 + @rows[@current_row] = [] + @cell_options[@current_row] = [] + @row_options[@current_row] ||= options + yield + @current_row += 1 + end + + def column(str, options={}) + str ||= "" + @rows[@current_row][@current_column] = str + @cell_options[@current_row][@current_column] = options + @column_widths[@current_column] = [str.length, options[:min_width]||0, @column_widths[@current_column]||0].max + @column_options[@current_column] ||= options + @current_column += 1 + end + + def draw_table + @rows.each_with_index do |row, i| + color = (@row_options[i]||{})[:color] + row.each_with_index do |column, j| + align = (@column_options[j]||{})[:align] || "left" + width = @column_widths[j] + cell_color = @cell_options[i][j] && @cell_options[i][j][:color] + cell_color ||= color + if cell_color + str = LeapCli.logger.colorize(column, cell_color) + extra_width = str.length - column.length + else + str = column + extra_width = 0 + end + if align == "right" + printf " %#{width+extra_width}s" % str + else + printf " %-#{width+extra_width}s" % str + end + end + puts + end + puts + end + end + +end; end
\ No newline at end of file diff --git a/lib/leap_cli/util/secret.rb b/lib/leap_cli/util/secret.rb new file mode 100644 index 00000000..749b9595 --- /dev/null +++ b/lib/leap_cli/util/secret.rb @@ -0,0 +1,55 @@ +# encoding: utf-8 +# +# A simple secret generator +# +# Uses OpenSSL random number generator instead of Ruby's rand function +# +autoload :OpenSSL, 'openssl' + +module LeapCli; module Util + class Secret + CHARS = (('A'..'Z').to_a + ('a'..'z').to_a + ('0'..'9').to_a) - "i1loO06G".split(//u) + HEX = (0..9).to_a + ('a'..'f').to_a + + # + # generate a secret with with no ambiguous characters. + # + # +length+ is in chars + # + # Only alphanumerics are allowed, in order to make these passwords work + # for REST url calls and to allow you to easily copy and paste them. + # + def self.generate(length = 16) + seed + OpenSSL::Random.random_bytes(length).bytes.to_a.collect { |byte| + CHARS[ byte % CHARS.length ] + }.join + end + + # + # generates a hex secret, instead of an alphanumeric on. + # + # length is in bits + # + def self.generate_hex(length = 128) + seed + OpenSSL::Random.random_bytes(length/4).bytes.to_a.collect { |byte| + HEX[ byte % HEX.length ] + }.join + end + + private + + def self.seed + @pid ||= 0 + pid = $$ + if @pid != pid + now = Time.now + ary = [now.to_i, now.nsec, @pid, pid] + OpenSSL::Random.seed(ary.to_s) + @pid = pid + end + end + + end +end; end diff --git a/lib/leap_cli/util/vagrant.rb b/lib/leap_cli/util/vagrant.rb new file mode 100644 index 00000000..c67ea4f1 --- /dev/null +++ b/lib/leap_cli/util/vagrant.rb @@ -0,0 +1,26 @@ +require 'fileutils' + +module LeapCli + module Util + module Vagrant + + # + # returns the path to a vagrant ssh private key file. + # + # if the vagrant.key file is owned by root or ourselves, then + # we need to make sure that it owned by us and not world readable. + # + def self.vagrant_ssh_key_file + file_path = Path.vagrant_ssh_priv_key_file + Util.assert_files_exist! file_path + uid = File.new(file_path).stat.uid + if uid == 0 || uid == Process.euid + FileUtils.install file_path, '/tmp/vagrant.key', :mode => 0600 + file_path = '/tmp/vagrant.key' + end + return file_path + end + + end + end +end
\ No newline at end of file diff --git a/lib/leap_cli/x509.rb b/lib/leap_cli/x509.rb new file mode 100644 index 00000000..68d13ddf --- /dev/null +++ b/lib/leap_cli/x509.rb @@ -0,0 +1,16 @@ +# +# optional. load if you want access to any methods in the module X509 +# + +require 'date' +require 'securerandom' +require 'openssl' +require 'digest' +require 'digest/md5' +require 'digest/sha1' + +require 'certificate_authority' + +require 'leap_cli/x509/certs' +require 'leap_cli/x509/signing_profiles' +require 'leap_cli/x509/utils' diff --git a/lib/leap_cli/x509/certs.rb b/lib/leap_cli/x509/certs.rb new file mode 100644 index 00000000..3b74d2fb --- /dev/null +++ b/lib/leap_cli/x509/certs.rb @@ -0,0 +1,232 @@ + +module LeapCli; module X509 + + # + # returns a fingerprint of a x509 certificate + # + # Note: there are different ways of computing a digest of a certificate. + # You can either take a digest of the entire cert in DER format, or you + # can take a digest of the public key. + # + # For now, we only support the DER method. + # + def self.fingerprint(digest, cert_file) + if cert_file.is_a? String + cert = OpenSSL::X509::Certificate.new(Util.read_file!(cert_file)) + elsif cert_file.is_a? OpenSSL::X509::Certificate + cert = cert_file + elsif cert_file.is_a? CertificateAuthority::Certificate + cert = cert_file.openssl_body + end + digester = case digest + when "MD5" then Digest::MD5.new + when "SHA1" then Digest::SHA1.new + when "SHA256" then Digest::SHA256.new + when "SHA384" then Digest::SHA384.new + when "SHA512" then Digest::SHA512.new + end + digester.hexdigest(cert.to_der) + end + + def self.ca_root + @ca_root ||= begin + load_certificate_file(:ca_cert, :ca_key) + end + end + + def self.client_ca_root + @client_ca_root ||= begin + load_certificate_file(:client_ca_cert, :client_ca_key) + end + end + + def self.load_certificate_file(crt_file, key_file=nil, password=nil) + crt = Util.read_file!(crt_file) + openssl_cert = OpenSSL::X509::Certificate.new(crt) + cert = CertificateAuthority::Certificate.from_openssl(openssl_cert) + if key_file + key = Util.read_file!(key_file) + cert.key_material.private_key = OpenSSL::PKey::RSA.new(key, password) + end + return cert + end + + # + # creates a new certificate authority. + # + def self.new_ca(options, common_name) + root = CertificateAuthority::Certificate.new + + # set subject + root.subject.common_name = common_name + possible = ['country', 'state', 'locality', 'organization', 'organizational_unit', 'email_address'] + options.keys.each do |key| + if possible.include?(key) + root.subject.send(key + '=', options[key]) + end + end + + # set expiration + root.not_before = X509.yesterday + root.not_after = X509.yesterday_advance(options['life_span']) + + # generate private key + root.serial_number.number = 1 + root.key_material.generate_key(options['bit_size']) + + # sign self + root.signing_entity = true + root.parent = root + root.sign!(ca_root_signing_profile) + return root + end + + # + # creates a CSR in memory and returns it. + # with the correct extReq attribute so that the CA + # doens't generate certs with extensions we don't want. + # + def self.new_csr(dn, keypair, digest) + csr = CertificateAuthority::SigningRequest.new + csr.distinguished_name = dn + csr.key_material = keypair + csr.digest = digest + + # define extensions manually (library doesn't support setting these on CSRs) + extensions = [] + extensions << CertificateAuthority::Extensions::BasicConstraints.new.tap {|basic| + basic.ca = false + } + extensions << CertificateAuthority::Extensions::KeyUsage.new.tap {|keyusage| + keyusage.usage = ["digitalSignature", "keyEncipherment"] + } + extensions << CertificateAuthority::Extensions::ExtendedKeyUsage.new.tap {|extkeyusage| + extkeyusage.usage = [ "serverAuth"] + } + + # convert extensions to attribute 'extReq' + # aka "Requested Extensions" + factory = OpenSSL::X509::ExtensionFactory.new + attrval = OpenSSL::ASN1::Set([OpenSSL::ASN1::Sequence( + extensions.map{|e| factory.create_ext(e.openssl_identifier, e.to_s, e.critical)} + )]) + attrs = [ + OpenSSL::X509::Attribute.new("extReq", attrval), + ] + csr.attributes = attrs + + return csr + end + + # + # creates new csr and cert files for a particular domain. + # + # The cert is signed with the ca_root, but should be replaced + # later with a real cert signed by a better ca + # + def self.create_csr_and_cert(options) + bit_size = options[:bits].to_i + digest = options[:digest] + + # RSA key + keypair = CertificateAuthority::MemoryKeyMaterial.new + Util.log :generating, "%s bit RSA key" % bit_size do + keypair.generate_key(bit_size) + Util.write_file! [:commercial_key, options[:domain]], keypair.private_key.to_pem + end + + # CSR + csr = nil + dn = CertificateAuthority::DistinguishedName.new + dn.common_name = options[:domain] + dn.organization = options[:organization] + dn.ou = options[:organizational_unit] + dn.email_address = options[:email] + dn.country = options[:country] + dn.state = options[:state] + dn.locality = options[:locality] + Util.log :generating, "CSR with #{digest} digest and #{print_dn(dn)}" do + csr = new_csr(dn, keypair, options[:digest]) + Util.write_file! [:commercial_csr, options[:domain]], csr.to_pem + end + + # Sign using our own CA, for use in testing but hopefully not production. + # It is not that commerical CAs are so secure, it is just that signing your own certs is + # a total drag for the user because they must click through dire warnings. + Util.log :generating, "self-signed x509 server certificate for testing purposes" do + cert = csr.to_cert + cert.serial_number.number = cert_serial_number(options[:domain]) + cert.not_before = yesterday + cert.not_after = yesterday.advance(:years => 1) + cert.parent = ca_root + cert.sign! domain_test_signing_profile + Util.write_file! [:commercial_cert, options[:domain]], cert.to_pem + Util.log "please replace this file with the real certificate you get from a CA using #{Path.relative_path([:commercial_csr, options[:domain]])}" + end + + # Fake CA + unless Util.file_exists? :commercial_ca_cert + Util.log :using, "generated CA in place of commercial CA for testing purposes" do + Util.write_file! :commercial_ca_cert, Util.read_file!(:ca_cert) + Util.log "please also replace this file with the CA cert from the commercial authority you use." + end + end + end + + # + # Return true if the given server cert has been signed by the given CA cert + # + # This does not actually validate the signature, it just checks the cert + # extensions. + # + def self.created_by_authority?(cert, ca=X509.ca_root) + authority_key_id = cert.extensions["authorityKeyIdentifier"].identifier.sub(/^keyid:/, '') + return authority_key_id == self.public_key_id_for_ca(ca) + end + + # + # For cert serial numbers, we need a non-colliding number less than 160 bits. + # md5 will do nicely, since there is no need for a secure hash, just a short one. + # (md5 is 128 bits) + # + def self.cert_serial_number(domain_name) + Digest::MD5.hexdigest("#{domain_name} -- #{Time.now}").to_i(16) + end + + # + # for the random common name, we need a text string that will be + # unique across all certs. + # + def self.random_common_name(domain_name) + #cert_serial_number(domain_name).to_s(36) + SecureRandom.uuid + end + + private + + # + # calculate the "key id" for a root CA, that matches the value + # Authority Key Identifier in the x509 extensions of a cert. + # + def self.public_key_id_for_ca(ca_cert) + @ca_key_ids ||= {} + @ca_key_ids[ca_cert.object_id] ||= begin + pubkey = ca_cert.key_material.public_key + seq = OpenSSL::ASN1::Sequence([ + OpenSSL::ASN1::Integer.new(pubkey.n), + OpenSSL::ASN1::Integer.new(pubkey.e) + ]) + Digest::SHA1.hexdigest(seq.to_der).upcase.scan(/../).join(':') + end + end + + # prints CertificateAuthority::DistinguishedName fields + def self.print_dn(dn) + fields = {} + [:common_name, :locality, :state, :country, :organization, :organizational_unit, :email_address].each do |attr| + fields[attr] = dn.send(attr) if dn.send(attr) + end + fields.inspect + end + +end; end diff --git a/lib/leap_cli/x509/signing_profiles.rb b/lib/leap_cli/x509/signing_profiles.rb new file mode 100644 index 00000000..56cd29c7 --- /dev/null +++ b/lib/leap_cli/x509/signing_profiles.rb @@ -0,0 +1,104 @@ +# +# Signing profiles are used by CertificateAuthority in order to +# set the correct flags when signing certificates. +# + +module LeapCli; module X509 + + # + # For CA self-signing + # + def self.ca_root_signing_profile + { + "extensions" => { + "basicConstraints" => {"ca" => true}, + "keyUsage" => { + "usage" => ["critical", "keyCertSign"] + }, + "extendedKeyUsage" => { + "usage" => [] + } + } + } + end + + # + # For keyusage, openvpn server certs can have keyEncipherment or keyAgreement. + # Web browsers seem to break without keyEncipherment. + # For now, I am using digitalSignature + keyEncipherment + # + # * digitalSignature -- for (EC)DHE cipher suites + # "The digitalSignature bit is asserted when the subject public key is used + # with a digital signature mechanism to support security services other + # than certificate signing (bit 5), or CRL signing (bit 6). Digital + # signature mechanisms are often used for entity authentication and data + # origin authentication with integrity." + # + # * keyEncipherment ==> for plain RSA cipher suites + # "The keyEncipherment bit is asserted when the subject public key is used for + # key transport. For example, when an RSA key is to be used for key management, + # then this bit is set." + # + # * keyAgreement ==> for used with DH, not RSA. + # "The keyAgreement bit is asserted when the subject public key is used for key + # agreement. For example, when a Diffie-Hellman key is to be used for key + # management, then this bit is set." + # + # digest options: SHA512, SHA256, SHA1 + # + def self.server_signing_profile(node) + { + "digest" => node.env.provider.ca.server_certificates.digest, + "extensions" => { + "keyUsage" => { + "usage" => ["digitalSignature", "keyEncipherment"] + }, + "extendedKeyUsage" => { + "usage" => ["serverAuth", "clientAuth"] + }, + "subjectAltName" => { + "ips" => [node.ip_address], + "dns_names" => node.all_dns_names + } + } + } + end + + # + # This is used when signing the main cert for the provider's domain + # with our own CA (for testing purposes). Typically, this cert would + # be purchased from a commercial CA, and not signed this way. + # + def self.domain_test_signing_profile + { + "digest" => "SHA256", + "extensions" => { + "keyUsage" => { + "usage" => ["digitalSignature", "keyEncipherment"] + }, + "extendedKeyUsage" => { + "usage" => ["serverAuth"] + } + } + } + end + + # + # This is used when signing a dummy client certificate that is only to be + # used for testing. + # + def self.client_test_signing_profile + { + "digest" => "SHA256", + "extensions" => { + "keyUsage" => { + "usage" => ["digitalSignature"] + }, + "extendedKeyUsage" => { + "usage" => ["clientAuth"] + } + } + } + end + +end; end
\ No newline at end of file diff --git a/lib/leap_cli/x509/utils.rb b/lib/leap_cli/x509/utils.rb new file mode 100644 index 00000000..98ff9c0b --- /dev/null +++ b/lib/leap_cli/x509/utils.rb @@ -0,0 +1,26 @@ +module LeapCli; module X509 + + # + # TIME HELPERS + # + # note: we use 'yesterday' instead of 'today', because times are in UTC, and + # some people on the planet are behind UTC! + # + + def self.yesterday + t = Time.now - 24*24*60 + Time.utc t.year, t.month, t.day + end + + def self.yesterday_advance(string) + number, unit = string.split(' ') + unless ['years', 'months', 'days', 'hours', 'minutes'].include? unit + bail!("The time property '#{string}' is missing a unit (one of: years, months, days, hours, minutes).") + end + unless number.to_i.to_s == number + bail!("The time property '#{string}' is missing a number.") + end + yesterday.advance(unit.to_sym => number.to_i) + end + +end; end
\ No newline at end of file |