From 205b61dfe721e6d88fc06b050a0497eeb35f4e02 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 21 Jul 2016 00:55:12 -0700 Subject: added 'leap vm' command --- lib/leap_cli/cloud/cloud.rb | 310 +++++++++++++++++++++++++++++++++++++ lib/leap_cli/cloud/dependencies.rb | 40 +++++ lib/leap_cli/cloud/image.rb | 31 ++++ 3 files changed, 381 insertions(+) create mode 100644 lib/leap_cli/cloud/cloud.rb create mode 100644 lib/leap_cli/cloud/dependencies.rb create mode 100644 lib/leap_cli/cloud/image.rb (limited to 'lib/leap_cli/cloud') diff --git a/lib/leap_cli/cloud/cloud.rb b/lib/leap_cli/cloud/cloud.rb new file mode 100644 index 00000000..f0bb45b8 --- /dev/null +++ b/lib/leap_cli/cloud/cloud.rb @@ -0,0 +1,310 @@ +# +# An abstractions in front of Fog, which is an abstraction in front of +# the AWS api. Oh my! +# +# A Cloud object binds a particular node with particular Fog +# authentication credentials. +# +# NOTE: Possible AWS options for creating instances: +# +# options = { +# 'BlockDeviceMapping' => block_device_mapping, +# 'NetworkInterfaces' => network_interfaces, +# 'ClientToken' => client_token, +# 'DisableApiTermination' => disable_api_termination, +# 'EbsOptimized' => ebs_optimized, +# 'IamInstanceProfile.Arn' => @iam_instance_profile_arn, +# 'IamInstanceProfile.Name' => @iam_instance_profile_name, +# 'InstanceInitiatedShutdownBehavior' => instance_initiated_shutdown_behavior, +# 'InstanceType' => flavor_id, +# 'KernelId' => kernel_id, +# 'KeyName' => key_name, +# 'Monitoring.Enabled' => monitoring, +# 'Placement.AvailabilityZone' => availability_zone, +# 'Placement.GroupName' => placement_group, +# 'Placement.Tenancy' => tenancy, +# 'PrivateIpAddress' => private_ip_address, +# 'RamdiskId' => ramdisk_id, +# 'SecurityGroup' => groups, +# 'SecurityGroupId' => security_group_ids, +# 'SubnetId' => subnet_id, +# 'UserData' => user_data, +# } +# + +module LeapCli + class Cloud + LEAP_SG_NAME = 'leap_default' + LEAP_SG_DESC = 'Default security group for LEAP nodes' + + include LeapCli::LogCommand + + attr_reader :compute # Fog::Compute object + attr_reader :node # Config::Node, if any + attr_reader :options # options for the VMs, if any + attr_reader :image # which vm image to use, if any + attr_reader :name # name of which entry in cloud.json to use + + def initialize(name, conf, node=nil) + @node = node + @name = name + @conf = conf + @compute = nil + @options = nil + @image = nil + + raise ArgumentError, 'name missing' unless @name + raise ArgumentError, 'config missing' unless @conf + raise ArgumentError, 'config auth missing' unless @conf["auth"] + raise ArgumentError, 'config auth missing' unless @conf["vendor"] + + credentials = @conf["auth"].symbolize_keys + credentials[:provider] = @conf["vendor"] + @compute = Fog::Compute.new(credentials) + + @options = @conf['default_options'] || {} + @image = @conf['default_image'] || aws_image(credentials[:region]) + if @node + @options = node.vm.options if node['vm.options'] + @image = node.vm.image if node['vm.image'] + end + end + + # + # fetches or creates a server for this cloud object. + # + def fetch_or_create_server(options) + fetch_server_for_node || create_new_vm_instance(choose_ssh_key: options[:choose_ssh_key]) + end + + # + # fetches the server for a particular node. + # + # return nil if this cloud object has no node, or there is no corresponding + # server. + # + def fetch_server_for_node(bail_on_failure=false) + server = nil + return nil unless @node + + # does an instance exist that matches the node's vm.id? + if @node.vm_id? + instance_id = @node.vm.id + server = @compute.servers.get(instance_id) + end + + # does an instance exist that is tagged with this node name? + if server.nil? + response = @compute.describe_instances({"tag:node_name" => @node.name}) + # puts JSON.pretty_generate(response.body) + if !response.body["reservationSet"].empty? + instances = response.body["reservationSet"].first["instancesSet"] + if instances.size > 1 + bail! "There are multiple VMs with the same node name tag! Manually remove one before continuing." + elsif instances.size == 1 + instance_id = instances.first["instanceId"] + server = cloud.compute.servers.get(instance_id) + end + end + end + + if server.nil? && bail_on_failure + bail! :error, "A virtual machine could not be found for node `#{@node.name}`. Things to try:" do + log "check the output of `leap vm status`" + log "check the value of `vm.id` in #{@node.name}.json" + log "run `leap vm add #{@node.name}` to create a corresponding virtual machine" + end + end + + return server + end + + # + # associates a node with a vm + # + def bind_server_to_node(server) + raise ArgumentError, 'no node' unless @node + + # assign tag + @compute.create_tags(server.id, {'node_name' => @node.name}) + log :created, "association between node '%s' and vm '%s'" % [@node.name, server.id] + + # update node json + @node.update_json({ + "ip_address" => server.public_ip_address, + "vm"=> {"id"=>server.id} + }) + log "done", :color => :green, :style => :bold + end + + # + # disassociates a node from a vm + # + def unbind_server_from_node(server) + raise ArgumentError, 'no node' unless @node + + # assign tag + @compute.delete_tags(server.id, {'node_name' => @node.name}) + log :removed, "association between node '%s' and vm '%s'" % [@node.name, server.id] + + # update node json + @node.update_json({ + "ip_address" => '0.0.0.0', + "vm"=> {"id" => ""} + }) + end + + # + # return an AWS KeyPair object, potentially uploading it to the server + # if necessary. + # + # this is used when initially creating the vm. After the first `node init`, then + # all sysadmins should have access to the server. + # + # NOTE: ssh and aws use different types of fingerprint + # + def find_or_create_key_pair(pick_ssh_key_method) + require 'leap_cli/ssh' + key_pair, local_key = match_ssh_key(:user_only => true) + if key_pair + log :using, "SSH key #{local_key.filename}" do + log 'AWS MD5 fingerprint: ' + local_key.fingerprint(:digest => :md5, :type => :der, :encoding => :hex) + log 'SSH MD5 fingerprint: ' + local_key.fingerprint(:digest => :md5, :type => :ssh, :encoding => :hex) + log 'SSH SHA256 fingerprint: ' + local_key.fingerprint(:digest => :sha256, :type => :ssh, :encoding => :base64) + end + elsif key_pair.nil? + username, key = pick_ssh_key_method.call(self) + key_pair = upload_ssh_key(username, key) + end + return key_pair + end + + # + # checks if there is a match between a local key and a registered key_pair + # + # options: + # :key_pair -- limit comparisons to this key_pair object. + # :user_only -- limit comparisons to the user's ~/.ssh directory only + # + # returns: + # + # key_pair -- an AWS KeyPair + # local_key -- a LeapCLi::SSH::Key + # + def match_ssh_key(options={}) + key_pair = options[:key_pair] + local_keys_to_check = LeapCli::SSH::Key.my_public_keys + unless options[:user_only] + local_keys_to_check += LeapCli::SSH::Key.provider_public_keys + end + fingerprints = Hash[local_keys_to_check.map {|k| + [k.fingerprint(:digest => :md5, :type => :der, :encoding => :hex), k] + }] + key_pair ||= @compute.key_pairs.select {|key_pair| + fingerprints.include?(key_pair.fingerprint) + }.first + if key_pair + local_key = fingerprints[key_pair.fingerprint] + return key_pair, local_key + else + return nil, nil + end + end + + private + + # + # Every AWS instance requires a security group, which is just a simple firewall. + # In the future, we could create a separate security group for each node, + # and set the rules to match what the rules should be for that node. + # + # However, for now, we just use a security group 'leap_default' that opens + # all the ports. + # + # The default behavior for AWS security groups is: + # all ingress traffic is blocked and all egress traffic is allowed. + # + def find_or_create_security_group + group = @compute.security_groups.get(LEAP_SG_NAME) + if group.nil? + group = @compute.security_groups.create( + :name => LEAP_SG_NAME, + :description => LEAP_SG_DESC + ) + group.authorize_port_range(0..65535, + :ip_protocol => 'tcp', + :cidr_ip => '0.0.0.0/0', + ) + group.authorize_port_range(0..65535, + :ip_protocol => 'udp', + :cidr_ip => '0.0.0.0/0', + ) + end + return group + end + + # + # key - a LeapCli::SSH::Key object + # returns -- AWS KeyPair + # + def upload_ssh_key(username, key) + key_name = 'leap_' + username + key_pair = @compute.key_pairs.create( + :name => key_name, + :public_key => key.public_key.to_s + ) + log :registered, "public key" do + log 'cloud provider: ' + @name + log 'name: ' + key_name + log 'AWS MD5 fingerprint: ' + key.fingerprint(:digest => :md5, :type => :der, :encoding => :hex) + log 'SSH MD5 fingerprint: ' + key.fingerprint(:digest => :md5, :type => :ssh, :encoding => :hex) + log 'SSH SHA256 fingerprint: ' + key.fingerprint(:digest => :sha256, :type => :ssh, :encoding => :base64) + end + return key_pair + end + + def create_new_vm_instance(choose_ssh_key: nil) + log :creating, "new vm instance..." + assert! @image, "No image found. Specify `default_image` in cloud.json or `vm.image` in node's config." + if Fog.mock? + options = @options + else + key_pair = find_or_create_key_pair(choose_ssh_key) + security_group = find_or_create_security_group + options = @options.merge({ + 'KeyName' => key_pair.name, + 'SecurityGroup' => security_group.name + }) + end + response = @compute.run_instances( + @image, + 1, # min count + 1, # max count + options + ) + instance_id = response.body["instancesSet"].first["instanceId"] + log :created, "vm with instance id #{instance_id}." + server = @compute.servers.get(instance_id) + if server.nil? + bail! :error, "could not query instance '#{instance_id}'." + end + unless Fog.mock? + tries = 0 + server.wait_for { + if tries > 0 + LeapCli.log :waiting, "for IP address to be assigned..." + end + tries += 1 + !public_ip_address.nil? + } + if options[:wait] + log :waiting, "for vm #{instance_id} to start..." + server.wait_for { ready? } + log :started, "#{instance_id} with #{server.public_ip_address}" + end + end + return server + end + + end +end \ No newline at end of file diff --git a/lib/leap_cli/cloud/dependencies.rb b/lib/leap_cli/cloud/dependencies.rb new file mode 100644 index 00000000..fd690e59 --- /dev/null +++ b/lib/leap_cli/cloud/dependencies.rb @@ -0,0 +1,40 @@ +# +# I am not sure this is a good idea, but it might be. Tricky, so disabled for now +# + +=begin +module LeapCli + class Cloud + + def self.check_required_gems + begin + require "fog" + rescue LoadError + bail! do + log :error, "The 'vm' command requires the gem 'fog-core'. Please run `gem install fog-core` and try again." + end + end + + fog_gems = @cloud.required_gems + if !options[:mock] && fog_gems.empty? + bail! do + log :warning, "no vm providers are configured in cloud.json." + log "You must have credentials for one of: #{@cloud.possible_apis.join(', ')}." + end + end + + fog_gems.each do |name, gem_name| + begin + require gem_name.sub('-','/') + rescue LoadError + bail! do + log :error, "The 'vm' command requires the gem '#{gem_name}' (because of what is configured in cloud.json)." + log "Please run `sudo gem install #{gem_name}` and try again." + end + end + end + end + + end +end +=end \ No newline at end of file diff --git a/lib/leap_cli/cloud/image.rb b/lib/leap_cli/cloud/image.rb new file mode 100644 index 00000000..1f7f47b9 --- /dev/null +++ b/lib/leap_cli/cloud/image.rb @@ -0,0 +1,31 @@ +module LeapCli + class Cloud + + # + # returns the latest official debian image for + # a particular AWS region + # + # https://wiki.debian.org/Cloud/AmazonEC2Image/Jessie + # current list based on Debian 8.4 + # + # might return nil if no image is found. + # + def self.aws_image(region) + image_list = %q[ + ap-northeast-1 ami-d7d4c5b9 + ap-northeast-2 ami-9a03caf4 + ap-southeast-1 ami-73974210 + ap-southeast-2 ami-09daf96a + eu-central-1 ami-ccc021a3 + eu-west-1 ami-e079f893 + sa-east-1 ami-d3ae21bf + us-east-1 ami-c8bda8a2 + us-west-1 ami-45374b25 + us-west-2 ami-98e114f8 + ] + region_to_image = Hash[image_list.strip.split("\n").map{|i| i.split(" ")}] + return region_to_image[region] + end + + end +end \ No newline at end of file -- cgit v1.2.3 From b928a8f29b40df545ac6a72ead6cb9ed6a36fbda Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 24 Aug 2016 17:37:00 -0700 Subject: leap vm: fixed bug, added more sanity checking. --- lib/leap_cli/cloud/cloud.rb | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'lib/leap_cli/cloud') diff --git a/lib/leap_cli/cloud/cloud.rb b/lib/leap_cli/cloud/cloud.rb index f0bb45b8..86a84e69 100644 --- a/lib/leap_cli/cloud/cloud.rb +++ b/lib/leap_cli/cloud/cloud.rb @@ -103,7 +103,7 @@ module LeapCli bail! "There are multiple VMs with the same node name tag! Manually remove one before continuing." elsif instances.size == 1 instance_id = instances.first["instanceId"] - server = cloud.compute.servers.get(instance_id) + server = @compute.servers.get(instance_id) end end end @@ -123,7 +123,16 @@ module LeapCli # associates a node with a vm # def bind_server_to_node(server) - raise ArgumentError, 'no node' unless @node + unless @node + raise ArgumentError, 'no node' + end + unless server.state == 'running' + bail! do + log 'The virtual machine `%s` must be running in order to bind it to the configuration `%s`.' % [ + server.id, Path.relative_path(Path.named_path([:node_config, @node.name]))] + log 'To fix, run `leap vm start %s`' % server.id + end + end # assign tag @compute.create_tags(server.id, {'node_name' => @node.name}) -- cgit v1.2.3 From ad5acc562718df0505a318907e26185c8b91a284 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 25 Aug 2016 10:22:34 -0700 Subject: leap vm: require ip address instead of status running for leap vm bind. --- lib/leap_cli/cloud/cloud.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/leap_cli/cloud') diff --git a/lib/leap_cli/cloud/cloud.rb b/lib/leap_cli/cloud/cloud.rb index 86a84e69..e4b00107 100644 --- a/lib/leap_cli/cloud/cloud.rb +++ b/lib/leap_cli/cloud/cloud.rb @@ -126,9 +126,9 @@ module LeapCli unless @node raise ArgumentError, 'no node' end - unless server.state == 'running' + if server.public_ip_address.nil? bail! do - log 'The virtual machine `%s` must be running in order to bind it to the configuration `%s`.' % [ + log 'The virtual machine `%s` must have an IP address in order to bind it to the configuration `%s`.' % [ server.id, Path.relative_path(Path.named_path([:node_config, @node.name]))] log 'To fix, run `leap vm start %s`' % server.id end -- cgit v1.2.3 From 3262fcfa9188bb45d40ea2234af9ce8119100570 Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 14 Sep 2016 12:09:02 -0700 Subject: leap vm: fix typo (closes #8468) --- lib/leap_cli/cloud/cloud.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/leap_cli/cloud') diff --git a/lib/leap_cli/cloud/cloud.rb b/lib/leap_cli/cloud/cloud.rb index e4b00107..231274a5 100644 --- a/lib/leap_cli/cloud/cloud.rb +++ b/lib/leap_cli/cloud/cloud.rb @@ -63,7 +63,7 @@ module LeapCli @compute = Fog::Compute.new(credentials) @options = @conf['default_options'] || {} - @image = @conf['default_image'] || aws_image(credentials[:region]) + @image = @conf['default_image'] || Cloud.aws_image(credentials[:region]) if @node @options = node.vm.options if node['vm.options'] @image = node.vm.image if node['vm.image'] -- cgit v1.2.3 From 836876b4b0763e01526c9aa9c1fc34e1d82416aa Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 14 Sep 2016 13:13:29 -0700 Subject: [bugfix] leap vm: make the default instance type 't2.nano' --- lib/leap_cli/cloud/cloud.rb | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'lib/leap_cli/cloud') diff --git a/lib/leap_cli/cloud/cloud.rb b/lib/leap_cli/cloud/cloud.rb index 231274a5..753041f6 100644 --- a/lib/leap_cli/cloud/cloud.rb +++ b/lib/leap_cli/cloud/cloud.rb @@ -34,6 +34,9 @@ module LeapCli class Cloud + DEFAULT_INSTANCE_OPTIONS = { + "InstanceType" => "t2.nano" + } LEAP_SG_NAME = 'leap_default' LEAP_SG_DESC = 'Default security group for LEAP nodes' @@ -50,7 +53,7 @@ module LeapCli @name = name @conf = conf @compute = nil - @options = nil + @options = DEFAULT_INSTANCE_OPTIONS @image = nil raise ArgumentError, 'name missing' unless @name @@ -62,12 +65,21 @@ module LeapCli credentials[:provider] = @conf["vendor"] @compute = Fog::Compute.new(credentials) - @options = @conf['default_options'] || {} - @image = @conf['default_image'] || Cloud.aws_image(credentials[:region]) + if @conf['default_options'] + @options = @options.merge(@conf['default_options']) + end + @image = @conf['default_image'] || Cloud.aws_image(credentials[:region]) if @node - @options = node.vm.options if node['vm.options'] + @options = @options.merge(node.vm.options) if node['vm.options'] @image = node.vm.image if node['vm.image'] end + + unless @options['InstanceType'] + raise ArgumentError, 'VM instance type is required. See https://leap.se/virtual-machines for more information.' + end + unless @image + raise ArgumentError, 'VM image is required. See https://leap.se/virtual-machines for more information.' + end end # -- cgit v1.2.3 From b13cbe4730a986a3b60c4c70ce2b5f16da8a4feb Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 15 Sep 2016 22:21:38 -0700 Subject: leap vm: grab ssh host key when adding a new vm --- lib/leap_cli/cloud/cloud.rb | 53 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) (limited to 'lib/leap_cli/cloud') diff --git a/lib/leap_cli/cloud/cloud.rb b/lib/leap_cli/cloud/cloud.rb index 753041f6..2c06e7ed 100644 --- a/lib/leap_cli/cloud/cloud.rb +++ b/lib/leap_cli/cloud/cloud.rb @@ -155,7 +155,6 @@ module LeapCli "ip_address" => server.public_ip_address, "vm"=> {"id"=>server.id} }) - log "done", :color => :green, :style => :bold end # @@ -188,7 +187,7 @@ module LeapCli require 'leap_cli/ssh' key_pair, local_key = match_ssh_key(:user_only => true) if key_pair - log :using, "SSH key #{local_key.filename}" do + log :using, "user SSH key #{local_key.filename}" do log 'AWS MD5 fingerprint: ' + local_key.fingerprint(:digest => :md5, :type => :der, :encoding => :hex) log 'SSH MD5 fingerprint: ' + local_key.fingerprint(:digest => :md5, :type => :ssh, :encoding => :hex) log 'SSH SHA256 fingerprint: ' + local_key.fingerprint(:digest => :sha256, :type => :ssh, :encoding => :base64) @@ -232,6 +231,56 @@ module LeapCli end end + def wait_for_ssh_host_key(server) + require 'leap_cli/ssh' + return nil if Fog.mock? + tries = 0 + host_key = nil + cloud = self + server.wait_for { + if tries > 0 + LeapCli.log :waiting, "for SSH host key..." + elsif tries > 20 + return nil + end + tries += 1 + ssh_host_keys = cloud.ssh_host_keys(server) + if ssh_host_keys.nil? + false + else + host_key = SSH::Key.pick_best_key(ssh_host_keys) + true + end + } + return host_key + end + + # + # checks the console of the server for the ssh host keys + # + # returns nil if they cannot be found. + # + def ssh_host_keys(server) + require 'leap_cli/ssh' + return nil if Fog.mock? + response = @compute.get_console_output(server.id) + output = response.body["output"] + if output.nil? + return nil + end + keys = output.match( + /-----BEGIN SSH HOST KEY KEYS-----(.*)-----END SSH HOST KEY KEYS-----/m + ) + if keys.nil? + return nil + else + ssh_key_list = keys[1].strip.split("\r\n").map {|key_str| + SSH::Key.load(key_str) + } + return ssh_key_list.compact + end + end + private # -- cgit v1.2.3