diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/leap_cli/cloud.rb | 4 | ||||
-rw-r--r-- | lib/leap_cli/cloud/cloud.rb | 310 | ||||
-rw-r--r-- | lib/leap_cli/cloud/dependencies.rb | 40 | ||||
-rw-r--r-- | lib/leap_cli/cloud/image.rb | 31 | ||||
-rw-r--r-- | lib/leap_cli/commands/node.rb | 42 | ||||
-rw-r--r-- | lib/leap_cli/commands/node_init.rb | 7 | ||||
-rw-r--r-- | lib/leap_cli/commands/vm.rb | 399 | ||||
-rw-r--r-- | lib/leap_cli/config/cloud.rb | 64 | ||||
-rw-r--r-- | lib/leap_cli/config/environment.rb | 10 | ||||
-rw-r--r-- | lib/leap_cli/config/filter.rb | 7 | ||||
-rw-r--r-- | lib/leap_cli/config/manager.rb | 23 | ||||
-rw-r--r-- | lib/leap_cli/config/node.rb | 13 | ||||
-rw-r--r-- | lib/leap_cli/load_libraries.rb | 4 | ||||
-rw-r--r-- | lib/leap_cli/log_filter.rb | 9 | ||||
-rw-r--r-- | lib/leap_cli/ssh/backend.rb | 6 | ||||
-rw-r--r-- | lib/leap_cli/ssh/key.rb | 123 | ||||
-rw-r--r-- | lib/leap_cli/ssh/options.rb | 3 | ||||
-rw-r--r-- | lib/leap_cli/ssh/remote_command.rb | 6 | ||||
-rw-r--r-- | lib/leap_cli/ssh/scripts.rb | 15 | ||||
-rw-r--r-- | lib/leap_cli/util/console_table.rb | 13 |
20 files changed, 1090 insertions, 39 deletions
diff --git a/lib/leap_cli/cloud.rb b/lib/leap_cli/cloud.rb new file mode 100644 index 00000000..481dd6e7 --- /dev/null +++ b/lib/leap_cli/cloud.rb @@ -0,0 +1,4 @@ + +require 'fog' +require_relative 'cloud/cloud.rb' +require_relative 'cloud/image.rb' diff --git a/lib/leap_cli/cloud/cloud.rb b/lib/leap_cli/cloud/cloud.rb new file mode 100644 index 00000000..f0bb45b8 --- /dev/null +++ b/lib/leap_cli/cloud/cloud.rb @@ -0,0 +1,310 @@ +# +# An abstractions in front of Fog, which is an abstraction in front of +# the AWS api. Oh my! +# +# A Cloud object binds a particular node with particular Fog +# authentication credentials. +# +# NOTE: Possible AWS options for creating instances: +# +# options = { +# 'BlockDeviceMapping' => block_device_mapping, +# 'NetworkInterfaces' => network_interfaces, +# 'ClientToken' => client_token, +# 'DisableApiTermination' => disable_api_termination, +# 'EbsOptimized' => ebs_optimized, +# 'IamInstanceProfile.Arn' => @iam_instance_profile_arn, +# 'IamInstanceProfile.Name' => @iam_instance_profile_name, +# 'InstanceInitiatedShutdownBehavior' => instance_initiated_shutdown_behavior, +# 'InstanceType' => flavor_id, +# 'KernelId' => kernel_id, +# 'KeyName' => key_name, +# 'Monitoring.Enabled' => monitoring, +# 'Placement.AvailabilityZone' => availability_zone, +# 'Placement.GroupName' => placement_group, +# 'Placement.Tenancy' => tenancy, +# 'PrivateIpAddress' => private_ip_address, +# 'RamdiskId' => ramdisk_id, +# 'SecurityGroup' => groups, +# 'SecurityGroupId' => security_group_ids, +# 'SubnetId' => subnet_id, +# 'UserData' => user_data, +# } +# + +module LeapCli + class Cloud + LEAP_SG_NAME = 'leap_default' + LEAP_SG_DESC = 'Default security group for LEAP nodes' + + include LeapCli::LogCommand + + attr_reader :compute # Fog::Compute object + attr_reader :node # Config::Node, if any + attr_reader :options # options for the VMs, if any + attr_reader :image # which vm image to use, if any + attr_reader :name # name of which entry in cloud.json to use + + def initialize(name, conf, node=nil) + @node = node + @name = name + @conf = conf + @compute = nil + @options = nil + @image = nil + + raise ArgumentError, 'name missing' unless @name + raise ArgumentError, 'config missing' unless @conf + raise ArgumentError, 'config auth missing' unless @conf["auth"] + raise ArgumentError, 'config auth missing' unless @conf["vendor"] + + credentials = @conf["auth"].symbolize_keys + credentials[:provider] = @conf["vendor"] + @compute = Fog::Compute.new(credentials) + + @options = @conf['default_options'] || {} + @image = @conf['default_image'] || aws_image(credentials[:region]) + if @node + @options = node.vm.options if node['vm.options'] + @image = node.vm.image if node['vm.image'] + end + end + + # + # fetches or creates a server for this cloud object. + # + def fetch_or_create_server(options) + fetch_server_for_node || create_new_vm_instance(choose_ssh_key: options[:choose_ssh_key]) + end + + # + # fetches the server for a particular node. + # + # return nil if this cloud object has no node, or there is no corresponding + # server. + # + def fetch_server_for_node(bail_on_failure=false) + server = nil + return nil unless @node + + # does an instance exist that matches the node's vm.id? + if @node.vm_id? + instance_id = @node.vm.id + server = @compute.servers.get(instance_id) + end + + # does an instance exist that is tagged with this node name? + if server.nil? + response = @compute.describe_instances({"tag:node_name" => @node.name}) + # puts JSON.pretty_generate(response.body) + if !response.body["reservationSet"].empty? + instances = response.body["reservationSet"].first["instancesSet"] + if instances.size > 1 + bail! "There are multiple VMs with the same node name tag! Manually remove one before continuing." + elsif instances.size == 1 + instance_id = instances.first["instanceId"] + server = cloud.compute.servers.get(instance_id) + end + end + end + + if server.nil? && bail_on_failure + bail! :error, "A virtual machine could not be found for node `#{@node.name}`. Things to try:" do + log "check the output of `leap vm status`" + log "check the value of `vm.id` in #{@node.name}.json" + log "run `leap vm add #{@node.name}` to create a corresponding virtual machine" + end + end + + return server + end + + # + # associates a node with a vm + # + def bind_server_to_node(server) + raise ArgumentError, 'no node' unless @node + + # assign tag + @compute.create_tags(server.id, {'node_name' => @node.name}) + log :created, "association between node '%s' and vm '%s'" % [@node.name, server.id] + + # update node json + @node.update_json({ + "ip_address" => server.public_ip_address, + "vm"=> {"id"=>server.id} + }) + log "done", :color => :green, :style => :bold + end + + # + # disassociates a node from a vm + # + def unbind_server_from_node(server) + raise ArgumentError, 'no node' unless @node + + # assign tag + @compute.delete_tags(server.id, {'node_name' => @node.name}) + log :removed, "association between node '%s' and vm '%s'" % [@node.name, server.id] + + # update node json + @node.update_json({ + "ip_address" => '0.0.0.0', + "vm"=> {"id" => ""} + }) + end + + # + # return an AWS KeyPair object, potentially uploading it to the server + # if necessary. + # + # this is used when initially creating the vm. After the first `node init`, then + # all sysadmins should have access to the server. + # + # NOTE: ssh and aws use different types of fingerprint + # + def find_or_create_key_pair(pick_ssh_key_method) + require 'leap_cli/ssh' + key_pair, local_key = match_ssh_key(:user_only => true) + if key_pair + log :using, "SSH key #{local_key.filename}" do + log 'AWS MD5 fingerprint: ' + local_key.fingerprint(:digest => :md5, :type => :der, :encoding => :hex) + log 'SSH MD5 fingerprint: ' + local_key.fingerprint(:digest => :md5, :type => :ssh, :encoding => :hex) + log 'SSH SHA256 fingerprint: ' + local_key.fingerprint(:digest => :sha256, :type => :ssh, :encoding => :base64) + end + elsif key_pair.nil? + username, key = pick_ssh_key_method.call(self) + key_pair = upload_ssh_key(username, key) + end + return key_pair + end + + # + # checks if there is a match between a local key and a registered key_pair + # + # options: + # :key_pair -- limit comparisons to this key_pair object. + # :user_only -- limit comparisons to the user's ~/.ssh directory only + # + # returns: + # + # key_pair -- an AWS KeyPair + # local_key -- a LeapCLi::SSH::Key + # + def match_ssh_key(options={}) + key_pair = options[:key_pair] + local_keys_to_check = LeapCli::SSH::Key.my_public_keys + unless options[:user_only] + local_keys_to_check += LeapCli::SSH::Key.provider_public_keys + end + fingerprints = Hash[local_keys_to_check.map {|k| + [k.fingerprint(:digest => :md5, :type => :der, :encoding => :hex), k] + }] + key_pair ||= @compute.key_pairs.select {|key_pair| + fingerprints.include?(key_pair.fingerprint) + }.first + if key_pair + local_key = fingerprints[key_pair.fingerprint] + return key_pair, local_key + else + return nil, nil + end + end + + private + + # + # Every AWS instance requires a security group, which is just a simple firewall. + # In the future, we could create a separate security group for each node, + # and set the rules to match what the rules should be for that node. + # + # However, for now, we just use a security group 'leap_default' that opens + # all the ports. + # + # The default behavior for AWS security groups is: + # all ingress traffic is blocked and all egress traffic is allowed. + # + def find_or_create_security_group + group = @compute.security_groups.get(LEAP_SG_NAME) + if group.nil? + group = @compute.security_groups.create( + :name => LEAP_SG_NAME, + :description => LEAP_SG_DESC + ) + group.authorize_port_range(0..65535, + :ip_protocol => 'tcp', + :cidr_ip => '0.0.0.0/0', + ) + group.authorize_port_range(0..65535, + :ip_protocol => 'udp', + :cidr_ip => '0.0.0.0/0', + ) + end + return group + end + + # + # key - a LeapCli::SSH::Key object + # returns -- AWS KeyPair + # + def upload_ssh_key(username, key) + key_name = 'leap_' + username + key_pair = @compute.key_pairs.create( + :name => key_name, + :public_key => key.public_key.to_s + ) + log :registered, "public key" do + log 'cloud provider: ' + @name + log 'name: ' + key_name + log 'AWS MD5 fingerprint: ' + key.fingerprint(:digest => :md5, :type => :der, :encoding => :hex) + log 'SSH MD5 fingerprint: ' + key.fingerprint(:digest => :md5, :type => :ssh, :encoding => :hex) + log 'SSH SHA256 fingerprint: ' + key.fingerprint(:digest => :sha256, :type => :ssh, :encoding => :base64) + end + return key_pair + end + + def create_new_vm_instance(choose_ssh_key: nil) + log :creating, "new vm instance..." + assert! @image, "No image found. Specify `default_image` in cloud.json or `vm.image` in node's config." + if Fog.mock? + options = @options + else + key_pair = find_or_create_key_pair(choose_ssh_key) + security_group = find_or_create_security_group + options = @options.merge({ + 'KeyName' => key_pair.name, + 'SecurityGroup' => security_group.name + }) + end + response = @compute.run_instances( + @image, + 1, # min count + 1, # max count + options + ) + instance_id = response.body["instancesSet"].first["instanceId"] + log :created, "vm with instance id #{instance_id}." + server = @compute.servers.get(instance_id) + if server.nil? + bail! :error, "could not query instance '#{instance_id}'." + end + unless Fog.mock? + tries = 0 + server.wait_for { + if tries > 0 + LeapCli.log :waiting, "for IP address to be assigned..." + end + tries += 1 + !public_ip_address.nil? + } + if options[:wait] + log :waiting, "for vm #{instance_id} to start..." + server.wait_for { ready? } + log :started, "#{instance_id} with #{server.public_ip_address}" + end + end + return server + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/cloud/dependencies.rb b/lib/leap_cli/cloud/dependencies.rb new file mode 100644 index 00000000..fd690e59 --- /dev/null +++ b/lib/leap_cli/cloud/dependencies.rb @@ -0,0 +1,40 @@ +# +# I am not sure this is a good idea, but it might be. Tricky, so disabled for now +# + +=begin +module LeapCli + class Cloud + + def self.check_required_gems + begin + require "fog" + rescue LoadError + bail! do + log :error, "The 'vm' command requires the gem 'fog-core'. Please run `gem install fog-core` and try again." + end + end + + fog_gems = @cloud.required_gems + if !options[:mock] && fog_gems.empty? + bail! do + log :warning, "no vm providers are configured in cloud.json." + log "You must have credentials for one of: #{@cloud.possible_apis.join(', ')}." + end + end + + fog_gems.each do |name, gem_name| + begin + require gem_name.sub('-','/') + rescue LoadError + bail! do + log :error, "The 'vm' command requires the gem '#{gem_name}' (because of what is configured in cloud.json)." + log "Please run `sudo gem install #{gem_name}` and try again." + end + end + end + end + + end +end +=end
\ No newline at end of file diff --git a/lib/leap_cli/cloud/image.rb b/lib/leap_cli/cloud/image.rb new file mode 100644 index 00000000..1f7f47b9 --- /dev/null +++ b/lib/leap_cli/cloud/image.rb @@ -0,0 +1,31 @@ +module LeapCli + class Cloud + + # + # returns the latest official debian image for + # a particular AWS region + # + # https://wiki.debian.org/Cloud/AmazonEC2Image/Jessie + # current list based on Debian 8.4 + # + # might return nil if no image is found. + # + def self.aws_image(region) + image_list = %q[ + ap-northeast-1 ami-d7d4c5b9 + ap-northeast-2 ami-9a03caf4 + ap-southeast-1 ami-73974210 + ap-southeast-2 ami-09daf96a + eu-central-1 ami-ccc021a3 + eu-west-1 ami-e079f893 + sa-east-1 ami-d3ae21bf + us-east-1 ami-c8bda8a2 + us-west-1 ami-45374b25 + us-west-2 ami-98e114f8 + ] + region_to_image = Hash[image_list.strip.split("\n").map{|i| i.split(" ")}] + return region_to_image[region] + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/node.rb b/lib/leap_cli/commands/node.rb index 15575d4f..60540de9 100644 --- a/lib/leap_cli/commands/node.rb +++ b/lib/leap_cli/commands/node.rb @@ -19,9 +19,14 @@ module LeapCli; module Commands "Separate multiple values for a single property with a comma, like so: `leap node add mynode services:webapp,dns`"].join("\n\n") node.arg_name 'NAME [SEED]' # , :optional => false, :multiple => false node.command :add do |add| - add.switch :local, :desc => 'Make a local testing node (by automatically assigning the next available local IP address). Local nodes are run as virtual machines on your computer.', :negatable => false + add.switch :local, :desc => 'Make a local testing node (by assigning the next available local IP address). Local nodes are run as virtual machines on your computer.', :negatable => false + add.switch :vm, :desc => 'Make a remote virtual machine for this node. Requires a valid cloud.json configuration.', :negatable => false add.action do |global_options,options,args| - add_node(global_options, options, args) + if options[:vm] + do_vm_add(global_options, options, args) + else + do_node_add(global_options, options, args) + end end end @@ -29,7 +34,7 @@ module LeapCli; module Commands node.arg_name 'OLD_NAME NEW_NAME' node.command :mv do |mv| mv.action do |global_options,options,args| - move_node(global_options, options, args) + do_node_move(global_options, options, args) end end @@ -37,7 +42,7 @@ module LeapCli; module Commands node.arg_name 'NAME' #:optional => false #, :multiple => false node.command :rm do |rm| rm.action do |global_options,options,args| - rm_node(global_options, options, args) + do_node_rm(global_options, options, args) end end end @@ -58,7 +63,10 @@ module LeapCli; module Commands protected - def add_node(global, options, args) + # + # additionally called by `leap vm add` + # + def do_node_add(global, options, args) name = args.first unless global[:force] assert_files_missing! [:node_config, name] @@ -81,7 +89,7 @@ module LeapCli; module Commands private - def move_node(global, options, args) + def do_node_move(global, options, args) node = get_node_from_args(args, include_disabled: true) new_name = args.last Config::Node.validate_name!(new_name, node.vagrant?) @@ -91,14 +99,30 @@ module LeapCli; module Commands end remove_directory! [:node_files_dir, node.name] rename_node_facts(node.name, new_name) + if node.vm_id? + node['name'] = new_name + bind_server_to_node(node.vm.id, node, options) + end end - def rm_node(global, options, args) + def do_node_rm(global, options, args) node = get_node_from_args(args, include_disabled: true) - node.remove_files - if node.vagrant? + if node.vm? + if !node.vm_id? + log :warning, "The node #{node.name} is missing a 'vm.id' property. "+ + "You may have a virtual machine instance that is left "+ + "running. Check `leap vm status`" + else + msg = "The node #{node.name} appears to be associated with a virtual machine. " + + "Do you want to also destroy this virtual machine? " + if global[:yes] || agree(msg) + do_vm_rm(global, options, args) + end + end + elsif node.vagrant? vagrant_command("destroy --force", [node.name]) end + node.remove_files remove_node_facts(node.name) end diff --git a/lib/leap_cli/commands/node_init.rb b/lib/leap_cli/commands/node_init.rb index 62a57496..59661295 100644 --- a/lib/leap_cli/commands/node_init.rb +++ b/lib/leap_cli/commands/node_init.rb @@ -37,6 +37,13 @@ module LeapCli; module Commands update_compiled_ssh_configs # allow password auth for new nodes: options[:auth_methods] = ["publickey", "password"] + if node.vm? + # new AWS virtual machines will only allow login as 'admin' + # before we continue, we must enable root access. + SSH.remote_command(node, options.merge(:user => 'admin')) do |ssh, host| + ssh.scripts.allow_root_ssh + end + end SSH.remote_command(node, options) do |ssh, host| if node.vagrant? ssh.scripts.install_insecure_vagrant_key diff --git a/lib/leap_cli/commands/vm.rb b/lib/leap_cli/commands/vm.rb new file mode 100644 index 00000000..ec2b6993 --- /dev/null +++ b/lib/leap_cli/commands/vm.rb @@ -0,0 +1,399 @@ +module LeapCli; module Commands + + desc "Manage virtual machines." + long_desc "This command provides a convenient way to manage virtual machines. " + + "FILTER may be a node filter or the ID of a virtual machine." + + command [:vm] do |vm| + vm.switch :mock, :desc => "Run as simulation, without actually connecting to a cloud provider. If set, --auth is ignored." + vm.switch :wait, :desc => "Wait for servers to start/stop before continuing." + vm.flag :auth, :arg_name => 'AUTH', + :desc => "Choose which authentication credentials to use from the file cloud.json. "+ + "If omitted, will default to the node's `vm.auth` property, or the first credentials in cloud.json" + + vm.desc "Allocates a new virtual machine and/or associates it with node NAME. "+ + "If node configuration file does not yet exist, "+ + "it is created with the optional SEED values. "+ + "You can run this command when the virtual machine already exists "+ + "in order to update the node's `vm.id` property." + vm.arg_name 'NODE_NAME [SEED]' + vm.command :add do |cmd| + #cmd.flag(:image, + # :desc => "The image to use to create this virtual machine.", + # :arg_name => 'IMAGE' + #) + cmd.action do |global, options, args| + do_vm_add(global, options, args) + end + end + + vm.desc 'Starts the virtual machine(s)' + vm.arg_name 'FILTER', :optional => true + vm.command :start do |start| + start.action do |global, options, args| + do_vm_start(global, options, args) + end + end + + vm.desc 'Shuts down the virtual machine(s), but keeps the storage allocated (to save resources, run `leap vm rm` instead).' + vm.arg_name 'FILTER', :optional => true + vm.command :stop do |stop| + stop.action do |global, options, args| + do_vm_stop(global, options, args) + end + end + + vm.desc 'Destroys the virtual machine(s)' + vm.arg_name 'FILTER', :optional => true + vm.command :rm do |rm| + rm.action do |global, options, args| + do_vm_rm(global, options, args) + end + end + + vm.desc 'Print the status of virtual machine(s)' + vm.arg_name 'FILTER', :optional => true + vm.command :status do |status| + status.action do |global, options, args| + do_vm_status(global, options, args) + end + end + + vm.desc "Binds a running virtual machine instance to a node configuration. "+ + "Afterwards, the VM will be assigned a label matching the node name, "+ + "and the node config will be updated with the instance ID." + vm.arg_name 'NODE_NAME INSTANCE_ID' + vm.command 'bind' do |cmd| + cmd.action do |global, options, args| + do_vm_bind(global, options, args) + end + end + + vm.desc "Registers a SSH public key for use when creating new virtual machines. "+ + "Note that only people who are creating new VM instances need to "+ + "have their key registered." + vm.command 'key-register' do |cmd| + cmd.action do |global, options, args| + do_vm_key_register(global, options, args) + end + end + + vm.desc "Lists the registered SSH public keys for a particular virtual "+ + "machine provider." + vm.command 'key-list' do |cmd| + cmd.action do |global, options, args| + do_vm_key_list(global, options, args) + end + end + + #vm.desc 'Saves the current state of the virtual machine as a new snapshot.' + #vm.arg_name 'FILTER', :optional => true + #vm.command :save do |save| + # save.action do |global, options, args| + # do_vm_save(global, options, args) + # end + #end + + #vm.desc 'Resets virtual machine(s) to the last saved snapshot' + #vm.arg_name 'FILTER', :optional => true + #vm.command :reset do |reset| + # reset.action do |global, options, args| + # do_vm_reset(global, options, args) + # end + #end + + #vm.desc 'Lists the available images.' + #vm.command 'image-list' do |cmd| + # cmd.action do |global, options, args| + # do_vm_image_list(global, options, args) + # end + #end + end + + ## + ## SHARED UTILITY METHODS + ## + + protected + + # + # a callback used if we need to upload a new ssh key + # + def choose_ssh_key_for_upload(cloud) + puts + bail! unless agree("The cloud provider `#{cloud.name}` does not have "+ + "your public key. Do you want to upload one? ") + key = pick_ssh_key + username = ask("username? ", :default => `whoami`.strip) + assert!(username && !username.empty? && username =~ /[0-9a-z_-]+/, "Username must consist of one or more letters or numbers") + puts + return username, key + end + + def bind_server_to_node(vm_id, node, options={}) + cloud = new_cloud_handle(node, options) + server = cloud.compute.servers.get(vm_id) + assert! server, "Could not find a VM instance with ID '#{vm_id}'" + cloud.bind_server_to_node(server) + end + + ## + ## COMMANDS + ## + + protected + + # + # entirely removes the vm, not just stopping it. + # + # This might be additionally called by the 'leap node rm' command. + # + def do_vm_rm(global, options, args) + servers_from_args(global, options, args) do |cloud, server| + cloud.unbind_server_from_node(server) if cloud.node + destroy_server(server, options[:wait]) + end + end + + private + + def do_vm_status(global, options, args) + cloud = new_cloud_handle(nil, options) + servers = cloud.compute.servers + t = LeapCli::Util::ConsoleTable.new + t.table do + t.row(color: :cyan) do + t.column "ID" + t.column "NODE" + t.column "STATE" + t.column "FLAVOR" + t.column "IP" + t.column "ZONE" + end + servers.each do |server| + t.row do + t.column server.id + t.column server.tags["node_name"] + t.column server.state, :color => state_color(server.state) + t.column server.flavor_id + t.column server.public_ip_address + t.column server.availability_zone + end + end + end + puts + t.draw_table + end + + def do_vm_add(global, options, args) + name = args.first + if manager.nodes[name].nil? + do_node_add(global, {:ip_address => '0.0.0.0'}.merge(options), args) + end + node = manager.nodes[name] + cloud = new_cloud_handle(node, options) + server = cloud.fetch_or_create_server(:choose_ssh_key => method(:choose_ssh_key_for_upload)) + + if server + cloud.bind_server_to_node(server) + end + end + + def do_vm_start(global, options, args) + servers_from_args(global, options, args) do |cloud, server| + start_server(server, options[:wait]) + end + end + + def do_vm_stop(global, options, args) + servers_from_args(global, options, args) do |cloud, server| + stop_server(server, options[:wait]) + end + end + + def do_vm_key_register(global, options, args) + cloud = new_cloud_handle(nil, options) + cloud.find_or_create_key_pair(method(:choose_ssh_key_for_upload)) + end + + def do_vm_key_list(global, options, args) + require 'leap_cli/ssh' + cloud = new_cloud_handle(nil, options) + cloud.compute.key_pairs.each do |key_pair| + log key_pair.name, :color => :cyan do + log "AWS fingerprint: " + key_pair.fingerprint + key_pair, local_key = cloud.match_ssh_key(:key_pair => key_pair) + if local_key + log "matches local key: " + local_key.filename + log 'SSH MD5 fingerprint: ' + local_key.fingerprint(:digest => :md5, :type => :ssh, :encoding => :hex) + log 'SSH SHA256 fingerprint: ' + local_key.fingerprint(:digest => :sha256, :type => :ssh, :encoding => :base64) + end + end + end + end + + # + # update association between node and virtual machine. + # + # This might additionally be called by the 'leap node mv' command. + # + def do_vm_bind(global, options, args) + node_name = args.first + vm_id = args.last + assert! node_name, "NODE_NAME is missing" + assert! vm_id, "INSTANCE_ID is missing" + node = manager.nodes[node_name] + assert! node, "No node with name '#{node_name}'" + bind_server_to_node(vm_id, node, options) + end + + #def do_vm_image_list(global, options, args) + # compute = fog_setup(nil, options) + # p compute.images.all + #end + + ## + ## PRIVATE UTILITY METHODS + ## + + def stop_server(server, wait=false) + if server.state == 'stopped' + log :skipping, "virtual machine `#{server.id}` (already stopped)." + elsif ['shutting-down', 'terminated'].include?(server.state) + log :skipping, "virtual machine `#{server.id}` (being destroyed)." + else + log :stopping, "virtual machine `#{server.id}` (#{server.flavor_id}, #{server.availability_zone})" + server.stop + if wait + log 'please wait...', :indent => 1 + server.wait_for { state == 'stopped' } + log 'done', :color => :green, :indent => 1 + end + end + end + + def start_server(server, wait=false) + if server.state == 'running' + log :skipping, "virtual machine `#{server.id}` (already running)." + elsif ['shutting-down', 'terminated'].include?(server.state) + log :skipping, "virtual machine `#{server.id}` (being destroyed)." + else + log :starting, "virtual machine `#{server.id}` (#{server.flavor_id}, #{server.availability_zone})" + server.start + if wait + log 'please wait...', :indent => 1 + server.wait_for { ready? } + log 'done', :color => :green, :indent => 1 + end + end + end + + def destroy_server(server, wait=false) + if ['shutting-down', 'terminated'].include?(server.state) + log :skipping, "virtual machine `#{server.id}` (already being removed)." + else + log :terminated, "virtual machine `#{server.id}` (#{server.flavor_id}, #{server.availability_zone})" + server.destroy + if wait + log 'please wait...', :indent => 1 + server.wait_for { state == 'terminated' } + log 'done', :color => :green, :indent => 1 + end + end + end + + # + # for each server it finds, yields cloud, server + # + def servers_from_args(global, options, args) + nodes = filter_vm_nodes(args) + if nodes.any? + nodes.each_node do |node| + cloud = new_cloud_handle(node, options) + server = cloud.fetch_server_for_node(true) + yield cloud, server + end + else + instance_id = args.first + cloud = new_cloud_handle(nil, options) + server = cloud.compute.servers.get(instance_id) + if server.nil? + bail! :error, "There is no virtual machine with ID `#{instance_id}`." + end + yield cloud, server + end + end + + # + # returns either: + # + # * the set of nodes specified by the filter, for this environment + # even if the result includes nodes that are not previously tagged with 'vm' + # * the list of all vm nodes for this environment, if filter is empty + # + def filter_vm_nodes(filter) + if filter.nil? || filter.empty? + return manager.filter(['vm'], :warning => false) + elsif filter.is_a? Array + return manager.filter(filter + ['+vm'], :warning => false) + else + raise ArgumentError, 'could not understand filter' + end + end + + def new_cloud_handle(node, options) + require 'leap_cli/cloud' + + config = manager.env.cloud + name = nil + if options[:mock] + Fog.mock! + name = 'mock_aws' + config['mock_aws'] = { + "api" => "aws", + "vendor" => "aws", + "auth" => { + "aws_access_key_id" => "dummy", + "aws_secret_access_key" => "dummy", + "region" => "us-west-2" + }, + "instance_options" => { + "image" => "dummy" + } + } + elsif options[:auth] + name = options[:auth] + assert! config[name], "The value for --auth does not correspond to any value in cloud.json." + elsif node && node['vm.auth'] + name = node.vm.auth + assert! config[name], "The node '#{node.name}' has a value for property 'vm.auth' that does not correspond to any value in cloud.json." + elsif config.keys.length == 1 + name = config.keys.first + log :using, "cloud vendor credentials `#{name}`." + else + bail! "You must specify --mock, --auth, or a node filter." + end + + entry = config[name] # entry in cloud.json + assert! entry, "cloud.json: could not find cloud resource `#{name}`." + assert! entry['vendor'], "cloud.json: property `vendor` is missing from `#{name}` entry." + assert! entry['api'], "cloud.json: property `api` is missing from `#{name}` entry. It must be one of #{config.possible_apis.join(', ')}." + assert! entry['auth'], "cloud.json: property `auth` is missing from `#{name}` entry." + assert! entry['auth']['region'], "cloud.json: property `auth.region` is missing from `#{name}` entry." + assert! entry['api'] == 'aws', "cloud.json: currently, only 'aws' is supported for `api`." + assert! entry['vendor'] == 'aws', "cloud.json: currently, only 'aws' is supported for `vendor`." + + return LeapCli::Cloud.new(name, entry, node) + end + + def state_color(state) + case state + when 'running'; :green + when 'terminated'; :red + when 'stopped'; :magenta + when 'shutting-down'; :yellow + else; :white + end + end + +end; end diff --git a/lib/leap_cli/config/cloud.rb b/lib/leap_cli/config/cloud.rb new file mode 100644 index 00000000..e3e5c1f1 --- /dev/null +++ b/lib/leap_cli/config/cloud.rb @@ -0,0 +1,64 @@ +# encoding: utf-8 +# +# A class for the cloud.json file +# +# Example format: +# +# { +# "my_aws": { +# "api": "aws", +# "vendor": "aws", +# "auth": { +# "region": "us-west-2", +# "aws_access_key_id": "xxxxxxxxxxxxxxx", +# "aws_secret_access_key": "xxxxxxxxxxxxxxxxxxxxxxxxxx" +# }, +# "default_image": "ami-98e114f8", +# "default_options": { +# "InstanceType": "t2.nano" +# } +# } +# } +# + +module LeapCli; module Config + + # http://fog.io/about/supported_services.html + VM_APIS = { + 'aws' => 'fog-aws', + 'google' => 'fog-google', + 'libvirt' => 'fog-libvirt', + 'openstack' => 'fog-openstack', + 'rackspace' => 'fog-rackspace' + } + + class Cloud < Hash + def initialize(env=nil) + end + + # + # returns hash, each key is the name of an API that is + # needed and the value is the name of the gem. + # + # only provider APIs that are required because they are present + # in cloud.json are included. + # + def required_gems + required = {} + self.each do |name, conf| + api = conf["api"] + required_gems[api] = VM_APIS[api] + end + return required + end + + # + # returns an array of all possible providers + # + def possible_apis + VM_APIS.keys + end + + end + +end; end diff --git a/lib/leap_cli/config/environment.rb b/lib/leap_cli/config/environment.rb index 398fd023..dadd9eaf 100644 --- a/lib/leap_cli/config/environment.rb +++ b/lib/leap_cli/config/environment.rb @@ -30,10 +30,12 @@ module LeapCli; module Config # shared, non-inheritable def nodes; @@nodes; end def secrets; @@secrets; end + def cloud; @@cloud; end def initialize(manager, name, search_dir, parent, options={}) @@nodes ||= nil @@secrets ||= nil + @@cloud ||= nil @manager = manager @name = name @@ -64,6 +66,7 @@ module LeapCli; module Config @partials = Config::ObjectList.new @provider = Config::Provider.new @common = Config::Object.new + @cloud = Config::Cloud.new return end @@ -89,7 +92,9 @@ module LeapCli; module Config @partials.values.each {|partial| partial.delete('name'); } # - # shared: currently non-inheritable + # shared + # + # shared configs are also non-inheritable # load the first ones we find, and only those. # if @@nodes.nil? || @@nodes.empty? @@ -98,6 +103,9 @@ module LeapCli; module Config if @@secrets.nil? || @@secrets.empty? @@secrets = load_json(Path.named_path(:secrets_config, search_dir), Config::Secrets, options) end + if @@cloud.nil? || @@cloud.empty? + @@cloud = load_json(Path.named_path(:cloud_config, search_dir), Config::Cloud) + end end # diff --git a/lib/leap_cli/config/filter.rb b/lib/leap_cli/config/filter.rb index 27502577..07424894 100644 --- a/lib/leap_cli/config/filter.rb +++ b/lib/leap_cli/config/filter.rb @@ -29,6 +29,7 @@ module LeapCli # options -- hash, possible keys include # :nopin -- disregard environment pinning # :local -- if false, disallow local nodes + # :warning -- if false, don't print a warning when no nodes are found. # # A nil value in the filters array indicates # the default environment. This is in order to support @@ -139,9 +140,11 @@ module LeapCli return @manager.env('_all_').services[name].node_list elsif @manager.tags[name] return @manager.env('_all_').tags[name].node_list - else + elsif @options[:warning] != false LeapCli.log :warning, "filter '#{name}' does not match any node names, tags, services, or environments." return Config::ObjectList.new + else + return Config::ObjectList.new end else node_list = Config::ObjectList.new @@ -153,7 +156,7 @@ module LeapCli @environments.each do |env| node_list.merge!(@manager.env(env).tags[name].node_list) end - else + elsif @options[:warning] != false LeapCli.log :warning, "filter '#{name}' does not match any node names, tags, services, or environments." end return node_list diff --git a/lib/leap_cli/config/manager.rb b/lib/leap_cli/config/manager.rb index 62eaa894..b42f1ae0 100644 --- a/lib/leap_cli/config/manager.rb +++ b/lib/leap_cli/config/manager.rb @@ -315,19 +315,20 @@ module LeapCli end # inherit from tags + node['tags'] = (node['tags'] || []).to_a if node.vagrant? - node['tags'] = (node['tags'] || []).to_a + ['local'] + node['tags'] << 'local' + elsif node['vm'] + node['tags'] << 'vm' end - if node['tags'] - node['tags'].to_a.each do |node_tag| - tag = node_env.tags[node_tag] - if tag.nil? - msg = 'in node "%s": the tag "%s" does not exist.' % [node['name'], node_tag] - log 0, :error, msg - raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions - else - new_node.deep_merge!(tag) - end + node['tags'].each do |node_tag| + tag = node_env.tags[node_tag] + if tag.nil? + msg = 'in node "%s": the tag "%s" does not exist.' % [node['name'], node_tag] + log 0, :error, msg + raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions + else + new_node.deep_merge!(tag) end end diff --git a/lib/leap_cli/config/node.rb b/lib/leap_cli/config/node.rb index 2d76b814..23abdee3 100644 --- a/lib/leap_cli/config/node.rb +++ b/lib/leap_cli/config/node.rb @@ -9,11 +9,8 @@ module LeapCli; module Config class Node < Object attr_accessor :file_paths - def initialize(environment=nil) #, name=nil) + def initialize(environment=nil) super(environment) - #if name - # self['name'] = name - #end @node = self @file_paths = [] end @@ -38,6 +35,14 @@ module LeapCli; module Config return vagrant_range.include?(ip_addr) end + def vm? + self['vm'] + end + + def vm_id? + self['vm.id'] && !self['vm.id'].empty? + end + # # Return a hash table representation of ourselves, with the key equal to the @node.name, # and the value equal to the fields specified in *keys. diff --git a/lib/leap_cli/load_libraries.rb b/lib/leap_cli/load_libraries.rb index cec3812d..01384f78 100644 --- a/lib/leap_cli/load_libraries.rb +++ b/lib/leap_cli/load_libraries.rb @@ -8,14 +8,16 @@ require 'leap_cli/log_filter' require 'leap_cli/config/object' require 'leap_cli/config/node' +require 'leap_cli/config/node_cert' require 'leap_cli/config/tag' require 'leap_cli/config/provider' require 'leap_cli/config/secrets' +require 'leap_cli/config/cloud' require 'leap_cli/config/object_list' require 'leap_cli/config/filter' require 'leap_cli/config/environment' require 'leap_cli/config/manager' require 'leap_cli/util/secret' -require 'leap_cli/util/x509' require 'leap_cli/util/vagrant' +require 'leap_cli/util/console_table' diff --git a/lib/leap_cli/log_filter.rb b/lib/leap_cli/log_filter.rb index 0d745cc2..28504e1a 100644 --- a/lib/leap_cli/log_filter.rb +++ b/lib/leap_cli/log_filter.rb @@ -99,11 +99,15 @@ module LeapCli # TITLE_FORMATTERS = [ # red - { :match => /error/, :color => :red, :style => :bold }, { :match => /fatal_error/, :replace => 'fatal error:', :color => :red, :style => :bold }, + { :match => /error/, :color => :red, :style => :bold }, { :match => /removed/, :color => :red, :style => :bold }, + { :match => /removing/, :color => :red, :style => :bold }, + { :match => /destroyed/, :color => :red, :style => :bold }, + { :match => /destroying/, :color => :red, :style => :bold }, + { :match => /terminated/, :color => :red, :style => :bold }, { :match => /failed/, :replace => 'FAILED', :color => :red, :style => :bold }, - { :match => /bail/, :replace => 'bailing out', :color => :red, :style => :bold }, + { :match => /bailing/, :replace => 'bailing', :color => :red, :style => :bold }, { :match => /invalid/, :color => :red, :style => :bold }, # yellow @@ -115,6 +119,7 @@ module LeapCli { :match => /created/, :color => :green, :style => :bold }, { :match => /completed/, :color => :green, :style => :bold }, { :match => /ran/, :color => :green, :style => :bold }, + { :match => /registered/, :color => :green, :style => :bold }, # cyan { :match => /note/, :replace => 'NOTE:', :color => :cyan, :style => :bold }, diff --git a/lib/leap_cli/ssh/backend.rb b/lib/leap_cli/ssh/backend.rb index 80203b61..42e58c15 100644 --- a/lib/leap_cli/ssh/backend.rb +++ b/lib/leap_cli/ssh/backend.rb @@ -46,6 +46,12 @@ module LeapCli Thread.current["sshkit_backend"] = nil end + # if set, all the commands will begin with: + # sudo -u #{@user} -- sh -c '<command>' + def set_user(user='root') + @user = user + end + # # like default capture, but gracefully logs failures for us # last argument can be an options hash. diff --git a/lib/leap_cli/ssh/key.rb b/lib/leap_cli/ssh/key.rb index ad1ecf15..76223b7e 100644 --- a/lib/leap_cli/ssh/key.rb +++ b/lib/leap_cli/ssh/key.rb @@ -2,12 +2,34 @@ # A wrapper around OpenSSL::PKey::RSA instances to provide a better api for # dealing with SSH keys. # -# NOTE: cipher 'ssh-ed25519' not supported yet because we are waiting +# NOTES: +# +# cipher 'ssh-ed25519' not supported yet because we are waiting # for support in Net::SSH # +# there are many ways to represent an SSH key, since SSH keys can be of +# a variety of types. +# +# To confuse matters more, there are multiple binary representations. +# So, for example, an RSA key has a native SSH representation +# (two bignums, e followed by n), and a DER representation. +# +# AWS uses fingerprints of the DER representation, but SSH typically reports +# fingerprints of the SSH representation. +# +# Also, SSH public key files are base64 encoded, but with whitespace removed +# so it all goes on one line. +# +# Some useful links: +# +# https://stackoverflow.com/questions/3162155/convert-rsa-public-key-to-rsa-der +# https://net-ssh.github.io/ssh/v2/api/classes/Net/SSH/Buffer.html +# https://serverfault.com/questions/603982/why-does-my-openssh-key-fingerprint-not-match-the-aws-ec2-console-keypair-finger +# require 'net/ssh' require 'forwardable' +require 'base64' module LeapCli module SSH @@ -72,6 +94,14 @@ module LeapCli public_key || private_key end + def self.my_public_keys + load_keys_from_paths File.join(ENV['HOME'], '.ssh', '*.pub') + end + + def self.provider_public_keys + load_keys_from_paths Path.named_path([:user_ssh, '*']) + end + # # Picks one key out of an array of keys that we think is the "best", # based on the order of preference in SUPPORTED_TYPES @@ -127,19 +157,29 @@ module LeapCli end end + private + + def self.load_keys_from_paths(key_glob) + keys = [] + Dir.glob(key_glob).each do |file| + key = Key.load(file) + if key && key.public? + keys << key + end + end + return keys + end + ## ## INSTANCE METHODS ## public - def initialize(rsa_key) - @key = rsa_key + def initialize(p_key) + @key = p_key end - def_delegator :@key, :fingerprint, :fingerprint - def_delegator :@key, :public?, :public? - def_delegator :@key, :private?, :private? def_delegator :@key, :ssh_type, :type def_delegator :@key, :public_encrypt, :public_encrypt def_delegator :@key, :public_decrypt, :public_decrypt @@ -156,6 +196,70 @@ module LeapCli Key.new(@key.private_key) end + def private? + @key.respond_to?(:private?) ? @key.private? : @key.private_key? + end + + def public? + @key.respond_to?(:public?) ? @key.public? : @key.public_key? + end + + # + # three arguments: + # + # - digest: one of md5, sha1, sha256, etc. (default sha256) + # - encoding: either :hex (default) or :base64 + # - type: fingerprint type, either :ssh (default) or :der + # + # NOTE: + # + # * I am not sure how to make a fingerprint for OpenSSL::PKey::EC::Point + # + # * AWS reports fingerprints using MD5 digest for uploaded ssh keys, + # but SHA1 for keys it created itself. + # + # * Also, AWS fingerprints are digests on the DER encoding of the key. + # But standard SSH fingerprints are digests of SSH encoding of the key. + # + # * Other tools will sometimes display fingerprints in hex and sometimes + # in base64. Arrrgh. + # + def fingerprint(type: :ssh, digest: :sha256, encoding: :hex) + require 'digest' + + digest = digest.to_s.upcase + digester = case digest + when "MD5" then Digest::MD5.new + when "SHA1" then Digest::SHA1.new + when "SHA256" then Digest::SHA256.new + when "SHA384" then Digest::SHA384.new + when "SHA512" then Digest::SHA512.new + else raise ArgumentError, "digest #{digest} is unknown" + end + + keymatter = nil + if type == :der && @key.respond_to?(:to_der) + keymatter = @key.to_der + else + keymatter = self.raw_key.to_s + end + + fp = nil + if encoding == :hex + fp = digester.hexdigest(keymatter) + elsif encoding == :base64 + fp = Base64.encode64(digester.digest(keymatter)).sub(/=$/, '') + else + raise ArgumentError, "encoding #{encoding} not understood" + end + + if digest == "MD5" && encoding == :hex + return fp.scan(/../).join(':') + else + return fp + end + end + # # not sure if this will always work, but is seems to for now. # @@ -175,10 +279,17 @@ module LeapCli self.type + " " + self.key end + # + # base64 encoding of the key, with spaces removed. + # def key [Net::SSH::Buffer.from(:key, @key).to_s].pack("m*").gsub(/\s/, "") end + def raw_key + Net::SSH::Buffer.from(:key, @key) + end + def ==(other_key) return false if other_key.nil? return false if self.class != other_key.class diff --git a/lib/leap_cli/ssh/options.rb b/lib/leap_cli/ssh/options.rb index b8266d11..7bc06564 100644 --- a/lib/leap_cli/ssh/options.rb +++ b/lib/leap_cli/ssh/options.rb @@ -46,6 +46,9 @@ module LeapCli if args[:auth_methods] ssh_options[:auth_methods] = args[:auth_methods] end + if args[:user] + ssh_options[:user] = args[:user] + end return ssh_options end diff --git a/lib/leap_cli/ssh/remote_command.rb b/lib/leap_cli/ssh/remote_command.rb index 7195405e..0e9f2d55 100644 --- a/lib/leap_cli/ssh/remote_command.rb +++ b/lib/leap_cli/ssh/remote_command.rb @@ -38,6 +38,7 @@ module LeapCli # :port -- ssh port # :ip -- ssh ip # :auth_methods -- e.g. ["pubkey", "password"] + # :user -- default 'root' # def self.remote_command(nodes, options={}, &block) CustomCoordinator.new( @@ -47,6 +48,11 @@ module LeapCli ) ).each do |ssh, host| LeapCli.log 2, "ssh options for #{host.hostname}: #{host.ssh_options.inspect}" + if host.user != 'root' + # if the ssh user is not root, we want to make the ssh commands + # switch to root before they are run: + ssh.set_user('root') + end yield ssh, host end end diff --git a/lib/leap_cli/ssh/scripts.rb b/lib/leap_cli/ssh/scripts.rb index a15a9edd..9fef6240 100644 --- a/lib/leap_cli/ssh/scripts.rb +++ b/lib/leap_cli/ssh/scripts.rb @@ -119,6 +119,21 @@ module LeapCli end end + # + # AWS debian images only allow you to login as admin. This is done with a + # custom command in /root/.ssh/authorized_keys, instead of by modifying + # /etc/ssh/sshd_config. + # + # We need to be able to ssh as root for scp and rsync to work. + # + # This command is run as 'admin', with a sudo wrapper. In order for the + # sudo to work, the command must be specified as separate arguments with + # no spaces (that is how ssh-kit works). + # + def allow_root_ssh + ssh.execute 'cp', '/home/admin/.ssh/authorized_keys', '/root/.ssh/authorized_keys' + end + private def flagize(hsh) diff --git a/lib/leap_cli/util/console_table.rb b/lib/leap_cli/util/console_table.rb index 53c5e18a..ccdcc2ab 100644 --- a/lib/leap_cli/util/console_table.rb +++ b/lib/leap_cli/util/console_table.rb @@ -3,9 +3,12 @@ module LeapCli; module Util class ConsoleTable def table @rows = [] + @cell_options = [] + @row_options = [] @column_widths = [] @column_options = [] + @current_row = 0 @current_column = 0 yield @@ -13,6 +16,8 @@ module LeapCli; module Util def row(options=nil) @current_column = 0 + @rows[@current_row] = [] + @cell_options[@current_row] = [] @row_options[@current_row] ||= options yield @current_row += 1 @@ -20,8 +25,8 @@ module LeapCli; module Util def column(str, options={}) str ||= "" - @rows[@current_row] ||= [] @rows[@current_row][@current_column] = str + @cell_options[@current_row][@current_column] = options @column_widths[@current_column] = [str.length, options[:min_width]||0, @column_widths[@current_column]||0].max @column_options[@current_column] ||= options @current_column += 1 @@ -33,8 +38,10 @@ module LeapCli; module Util row.each_with_index do |column, j| align = (@column_options[j]||{})[:align] || "left" width = @column_widths[j] - if color - str = LeapCli.logger.colorize(column, color) + cell_color = @cell_options[i][j] && @cell_options[i][j][:color] + cell_color ||= color + if cell_color + str = LeapCli.logger.colorize(column, cell_color) extra_width = str.length - column.length else str = column |