Merge branch 'develop'

6 files changed, 976 insertions, 0 deletions

diff --git a/lib/leap_cli/ssh/backend.rb b/lib/leap_cli/ssh/backend.rb
new file mode 100644
index 00000000..3894d815
--- /dev/null
+++ b/lib/leap_cli/ssh/backend.rb
@@ -0,0 +1,209 @@
+#
+# A custome SSHKit backend, derived from the default netssh backend.
+# Our custom backend modifies the logging behavior and gracefully captures
+# common exceptions.
+#
+
+require 'stringio'
+require 'timeout'
+require 'sshkit'
+require 'leap_cli/ssh/formatter'
+require 'leap_cli/ssh/scripts'
+
+module SSHKit
+  class Command
+    #
+    # override exit_status in order to be less verbose
+    #
+    def exit_status=(new_exit_status)
+      @finished_at = Time.now
+      @exit_status = new_exit_status
+      if options[:raise_on_non_zero_exit] && exit_status > 0
+        message = ""
+        message += "exit status: " + exit_status.to_s + "\n"
+        message += "stdout: " + (full_stdout.strip.empty? ? "Nothing written" : full_stdout.strip) + "\n"
+        message += "stderr: " + (full_stderr.strip.empty? ? 'Nothing written' : full_stderr.strip) + "\n"
+        raise Failed, message
+      end
+    end
+  end
+end
+
+module LeapCli
+  module SSH
+    class Backend < SSHKit::Backend::Netssh
+
+      # since the @pool is a class instance variable, we need to copy
+      # the code from the superclass that initializes it. boo
+      @pool = SSHKit::Backend::ConnectionPool.new
+
+      # modify to pass itself to the block, instead of relying on instance_exec.
+      def run
+        Thread.current["sshkit_backend"] = self
+        # was: instance_exec(@host, &@block)
+        @block.call(self, @host)
+      ensure
+        Thread.current["sshkit_backend"] = nil
+      end
+
+      # if set, all the commands will begin with:
+      # sudo -u #{@user} -- sh -c '<command>'
+      def set_user(user='root')
+        @user = user
+      end
+
+      #
+      # like default capture, but gracefully logs failures for us
+      # last argument can be an options hash.
+      #
+      # available options:
+      #
+      #   :fail_msg    - [nil] if set, log this instead of the default
+      #                  fail message.
+      #
+      #   :raise_error - [nil] if true, then reraise failed command exception.
+      #
+      #   :log_cmd     - [false] if true, log what the command is that gets run.
+      #
+      #   :log_output  - [true] if true, log each output from the command as
+      #                  it is received.
+      #
+      #   :log_finish  - [false] if true, log the exit status and time
+      #                  to completion
+      #
+      #   :log_wrap    - [nil] passed to log method as :wrap option.
+      #
+      def capture(*args)
+        extract_options(args)
+        initialize_logger(:log_output => false)
+        rescue_ssh_errors(*args) do
+          return super(*args)
+        end
+      end
+
+      #
+      # like default execute, but log the results as they come in.
+      #
+      # see capture() for available options
+      #
+      def stream(*args)
+        extract_options(args)
+        initialize_logger
+        rescue_ssh_errors(*args) do
+          execute(*args)
+        end
+      end
+
+      def log(*args, &block)
+        @logger ||= LeapCli.new_logger
+        @logger.log(*args, &block)
+      end
+
+      # some prewritten servers-side scripts
+      def scripts
+        @scripts ||= LeapCli::SSH::Scripts.new(self, @host.hostname)
+      end
+
+      #
+      # sshkit just passes upload! and download! to Net::SCP, but Net::SCP
+      # make it impossible to set the file permissions. Here is how the mode
+      # is determined, from upload.rb:
+      #
+      #    mode = channel[:stat] ? channel[:stat].mode & 07777 : channel[:options][:mode]
+      #
+      # The stat info from the file always overrides the mode you pass in options.
+      # However, the channel[:options][:mode] will be applied for pure in-memory
+      # uploads. So, if the mode is set, we convert the upload to be a memory
+      # upload instead of a file upload.
+      #
+      # Stupid, but blame Net::SCP.
+      #
+      def upload!(src, dest, options={})
+        if options[:mode]
+          if src.is_a?(StringIO)
+            content = src
+          else
+            content = StringIO.new(File.read(src))
+          end
+          super(content, dest, options)
+        else
+          super(src, dest, options)
+        end
+      end
+
+      private
+
+      #
+      # creates a new logger instance for this specific ssh command.
+      # by doing this, each ssh session has its own logger and its own
+      # indentation.
+      #
+      # potentially modifies 'args' array argument.
+      #
+      def initialize_logger(default_options={})
+        @logger ||= LeapCli.new_logger
+        @output = LeapCli::SSH::Formatter.new(@logger, @host, default_options.merge(@options))
+      end
+
+      def extract_options(args)
+        if args.last.is_a? Hash
+          @options = args.pop
+        else
+          @options = {}
+        end
+      end
+
+      #
+      # capture common exceptions
+      #
+      def rescue_ssh_errors(*args, &block)
+        yield
+      rescue Net::SSH::HostKeyMismatch => exc
+        @logger.log(:fatal_error, "Host key mismatch!") do
+          @logger.log(exc.to_s)
+          @logger.log("The ssh host key for the server does not match what is on "+
+            " file in `%s`." % Path.named_path(:known_hosts))
+          @logger.log("One of these is happening:") do
+            @logger.log("There is an active Man in The Middle attack against you.")
+            @logger.log("Or, someone has generated new host keys for the server " +
+               "and your provider files are out of date.")
+            @logger.log("Or, a new server is using this IP address " +
+               "and your provider files are out of date.")
+            @logger.log("Or, the server configuration has changed to use a different host key.")
+          end
+          @logger.log("You can pin a different host key using `leap node init NODE`, " +
+            "but you must verify the fingerprint of the new host key!")
+        end
+        exit(1)
+      rescue StandardError => exc
+        if exc.is_a?(SSHKit::Command::Failed) || exc.is_a?(SSHKit::Runner::ExecuteError)
+          if @options[:raise_error]
+            raise LeapCli::SSH::ExecuteError, exc.to_s
+          elsif @options[:fail_msg]
+            @logger.log(@options[:fail_msg], host: @host.hostname, :color => :red)
+          else
+            @logger.log(:failed, args.join(' '), host: @host.hostname) do
+              @logger.log(exc.to_s.strip, wrap: true)
+            end
+          end
+        elsif exc.is_a?(Timeout::Error) || exc.is_a?(Net::SSH::ConnectionTimeout)
+          @logger.log(:failed, args.join(' '), host: @host.hostname) do
+            @logger.log("Connection timed out")
+          end
+          if @options[:raise_error]
+            raise LeapCli::SSH::TimeoutError, exc.to_s
+          end
+        else
+          raise
+        end
+        return nil
+      end
+
+      def output
+        @output ||= LeapCli::SSH::Formatter.new(@logger, @host)
+      end
+
+    end
+  end
+end
+
diff --git a/lib/leap_cli/ssh/formatter.rb b/lib/leap_cli/ssh/formatter.rb
new file mode 100644
index 00000000..c2e386dc
--- /dev/null
+++ b/lib/leap_cli/ssh/formatter.rb
@@ -0,0 +1,70 @@
+#
+# A custom SSHKit formatter that uses LeapLogger.
+#
+
+require 'sshkit'
+
+module LeapCli
+  module SSH
+
+    class Formatter < SSHKit::Formatter::Abstract
+
+      DEFAULT_OPTIONS = {
+        :log_cmd => false,    # log what the command is that gets run.
+        :log_output => true,  # log each output from the command as it is received.
+        :log_finish => false  # log the exit status and time to completion.
+      }
+
+      def initialize(logger, host, options={})
+        @logger = logger || LeapCli.new_logger
+        @host = host
+        @options = DEFAULT_OPTIONS.merge(options)
+      end
+
+      def write(obj)
+        @logger.log(obj.to_s, :host => @host.hostname)
+      end
+
+      def log_command_start(command)
+        if @options[:log_cmd]
+          @logger.log(:running, "`" + command.to_s + "`", :host => @host.hostname)
+        end
+      end
+
+      def log_command_data(command, stream_type, stream_data)
+        if @options[:log_output]
+          color = stream_type == :stderr ? :red : nil
+          @logger.log(stream_data.to_s.chomp,
+            :color => color, :host => @host.hostname, :wrap => options[:log_wrap])
+        end
+      end
+
+      def log_command_exit(command)
+        if @options[:log_finish]
+          runtime = sprintf('%5.3fs', command.runtime)
+          if command.failure?
+            message = "in #{runtime} with status #{command.exit_status}."
+            @logger.log(:failed, message, :host => @host.hostname)
+          else
+            message = "in #{runtime}."
+            @logger.log(:completed, message, :host => @host.hostname)
+          end
+        end
+      end
+    end
+
+  end
+end
+
+  #
+  # A custom InteractionHandler that will output the results as they come in.
+  #
+  #class LoggingInteractionHandler
+  #  def initialize(hostname, logger=nil)
+  #    @hostname = hostname
+  #    @logger = logger || LeapCli.new_logger
+  #  end
+  #  def on_data(command, stream_name, data, channel)
+  #    @logger.log(data, host: @hostname, wrap: true)
+  #  end
+  #end
diff --git a/lib/leap_cli/ssh/key.rb b/lib/leap_cli/ssh/key.rb
new file mode 100644
index 00000000..76223b7e
--- /dev/null
+++ b/lib/leap_cli/ssh/key.rb
@@ -0,0 +1,310 @@
+#
+# A wrapper around OpenSSL::PKey::RSA instances to provide a better api for
+# dealing with SSH keys.
+#
+# NOTES:
+#
+# cipher 'ssh-ed25519' not supported yet because we are waiting
+# for support in Net::SSH
+#
+# there are many ways to represent an SSH key, since SSH keys can be of
+# a variety of types.
+#
+# To confuse matters more, there are multiple binary representations.
+# So, for example, an RSA key has a native SSH representation
+# (two bignums, e followed by n), and a DER representation.
+#
+# AWS uses fingerprints of the DER representation, but SSH typically reports
+# fingerprints of the SSH representation.
+#
+# Also, SSH public key files are base64 encoded, but with whitespace removed
+# so it all goes on one line.
+#
+# Some useful links:
+#
+# https://stackoverflow.com/questions/3162155/convert-rsa-public-key-to-rsa-der
+# https://net-ssh.github.io/ssh/v2/api/classes/Net/SSH/Buffer.html
+# https://serverfault.com/questions/603982/why-does-my-openssh-key-fingerprint-not-match-the-aws-ec2-console-keypair-finger
+#
+
+require 'net/ssh'
+require 'forwardable'
+require 'base64'
+
+module LeapCli
+  module SSH
+    class Key
+      extend Forwardable
+
+      attr_accessor :filename
+      attr_accessor :comment
+
+      # supported ssh key types, in order of preference
+      SUPPORTED_TYPES = ['ssh-rsa', 'ecdsa-sha2-nistp256']
+      SUPPORTED_TYPES_RE = /(#{SUPPORTED_TYPES.join('|')})/
+
+      ##
+      ## CLASS METHODS
+      ##
+
+      def self.load(arg1, arg2=nil)
+        key = nil
+        if arg1.is_a? OpenSSL::PKey::RSA
+          key = Key.new arg1
+        elsif arg1.is_a? String
+          if arg1 =~ /^ssh-/
+            type, data = arg1.split(' ')
+            key = Key.new load_from_data(data, type)
+          elsif File.exist? arg1
+            key = Key.new load_from_file(arg1)
+            key.filename = arg1
+          else
+            key = Key.new load_from_data(arg1, arg2)
+          end
+        end
+        return key
+      rescue StandardError
+      end
+
+      def self.load_from_file(filename)
+        public_key = nil
+        private_key = nil
+        begin
+          public_key = Net::SSH::KeyFactory.load_public_key(filename)
+        rescue NotImplementedError, Net::SSH::Exception, OpenSSL::PKey::PKeyError
+          begin
+            private_key = Net::SSH::KeyFactory.load_private_key(filename)
+          rescue NotImplementedError, Net::SSH::Exception, OpenSSL::PKey::PKeyError
+          end
+        end
+        public_key || private_key
+      end
+
+      def self.load_from_data(data, type='ssh-rsa')
+        public_key = nil
+        private_key = nil
+        begin
+          public_key = Net::SSH::KeyFactory.load_data_public_key("#{type} #{data}")
+        rescue NotImplementedError, Net::SSH::Exception, OpenSSL::PKey::PKeyError
+          begin
+            private_key = Net::SSH::KeyFactory.load_data_private_key("#{type} #{data}")
+          rescue NotImplementedError, Net::SSH::Exception, OpenSSL::PKey::PKeyError
+          end
+        end
+        public_key || private_key
+      end
+
+      def self.my_public_keys
+        load_keys_from_paths File.join(ENV['HOME'], '.ssh', '*.pub')
+      end
+
+      def self.provider_public_keys
+        load_keys_from_paths Path.named_path([:user_ssh, '*'])
+      end
+
+      #
+      # Picks one key out of an array of keys that we think is the "best",
+      # based on the order of preference in SUPPORTED_TYPES
+      #
+      # Currently, this does not take bitsize into account.
+      #
+      def self.pick_best_key(keys)
+        keys.select {|k|
+          SUPPORTED_TYPES.include?(k.type)
+        }.sort {|a,b|
+          SUPPORTED_TYPES.index(a.type) <=> SUPPORTED_TYPES.index(b.type)
+        }.first
+      end
+
+      #
+      # takes a string with one or more ssh keys, one key per line,
+      # and returns an array of Key objects.
+      #
+      # the lines should be in one of these formats:
+      #
+      # 1. <hostname> <key-type> <key>
+      # 2. <key-type> <key>
+      #
+      def self.parse_keys(string)
+        keys = []
+        lines = string.split("\n").grep(/^[^#]/)
+        lines.each do |line|
+          if line =~ / #{Key::SUPPORTED_TYPES_RE} /
+            # <hostname> <key-type> <key>
+            keys << line.split(' ')[1..2]
+          elsif line =~ /^#{Key::SUPPORTED_TYPES_RE} /
+            # <key-type> <key>
+            keys << line.split(' ')
+          end
+        end
+        return keys.map{|k| Key.load(k[1], k[0])}
+      end
+
+      #
+      # takes a string with one or more ssh keys, one key per line,
+      # and returns a string that specified the ssh key algorithms
+      # that are supported by the keys, in order of preference.
+      #
+      # eg: ecdsa-sha2-nistp256,ssh-rsa,ssh-ed25519
+      #
+      def self.supported_host_key_algorithms(string)
+        if string
+          self.parse_keys(string).map {|key|
+            key.type
+          }.join(',')
+        else
+          ""
+        end
+      end
+
+      private
+
+      def self.load_keys_from_paths(key_glob)
+        keys = []
+        Dir.glob(key_glob).each do |file|
+          key = Key.load(file)
+          if key && key.public?
+            keys << key
+          end
+        end
+        return keys
+      end
+
+      ##
+      ## INSTANCE METHODS
+      ##
+
+      public
+
+      def initialize(p_key)
+        @key = p_key
+      end
+
+      def_delegator :@key, :ssh_type, :type
+      def_delegator :@key, :public_encrypt, :public_encrypt
+      def_delegator :@key, :public_decrypt, :public_decrypt
+      def_delegator :@key, :private_encrypt, :private_encrypt
+      def_delegator :@key, :private_decrypt, :private_decrypt
+      def_delegator :@key, :params, :params
+      def_delegator :@key, :to_text, :to_text
+
+      def public_key
+        Key.new(@key.public_key)
+      end
+
+      def private_key
+        Key.new(@key.private_key)
+      end
+
+      def private?
+        @key.respond_to?(:private?) ? @key.private? : @key.private_key?
+      end
+
+      def public?
+        @key.respond_to?(:public?) ? @key.public? : @key.public_key?
+      end
+
+      #
+      # three arguments:
+      #
+      # - digest: one of md5, sha1, sha256, etc. (default sha256)
+      # - encoding: either :hex (default) or :base64
+      # - type: fingerprint type, either :ssh (default) or :der
+      #
+      # NOTE:
+      #
+      # * I am not sure how to make a fingerprint for OpenSSL::PKey::EC::Point
+      #
+      # * AWS reports fingerprints using MD5 digest for uploaded ssh keys,
+      #   but SHA1 for keys it created itself.
+      #
+      # * Also, AWS fingerprints are digests on the DER encoding of the key.
+      #   But standard SSH fingerprints are digests of SSH encoding of the key.
+      #
+      # * Other tools will sometimes display fingerprints in hex and sometimes
+      #   in base64. Arrrgh.
+      #
+      def fingerprint(type: :ssh, digest: :sha256, encoding: :hex)
+        require 'digest'
+
+        digest = digest.to_s.upcase
+        digester = case digest
+          when "MD5" then Digest::MD5.new
+          when "SHA1" then Digest::SHA1.new
+          when "SHA256" then Digest::SHA256.new
+          when "SHA384" then Digest::SHA384.new
+          when "SHA512" then Digest::SHA512.new
+          else raise ArgumentError, "digest #{digest} is unknown"
+        end
+
+        keymatter = nil
+        if type == :der && @key.respond_to?(:to_der)
+          keymatter = @key.to_der
+        else
+          keymatter = self.raw_key.to_s
+        end
+
+        fp = nil
+        if encoding == :hex
+          fp = digester.hexdigest(keymatter)
+        elsif encoding == :base64
+          fp = Base64.encode64(digester.digest(keymatter)).sub(/=$/, '')
+        else
+          raise ArgumentError, "encoding #{encoding} not understood"
+        end
+
+        if digest == "MD5" && encoding == :hex
+          return fp.scan(/../).join(':')
+        else
+          return fp
+        end
+      end
+
+      #
+      # not sure if this will always work, but is seems to for now.
+      #
+      def bits
+        Net::SSH::Buffer.from(:key, @key).to_s.split("\001\000").last.size * 8
+      end
+
+      def summary
+        if self.filename
+          "%s %s %s (%s)" % [self.type, self.bits, self.fingerprint, File.basename(self.filename)]
+        else
+          "%s %s %s" % [self.type, self.bits, self.fingerprint]
+        end
+      end
+
+      def to_s
+        self.type + " " + self.key
+      end
+
+      #
+      # base64 encoding of the key, with spaces removed.
+      #
+      def key
+        [Net::SSH::Buffer.from(:key, @key).to_s].pack("m*").gsub(/\s/, "")
+      end
+
+      def raw_key
+        Net::SSH::Buffer.from(:key, @key)
+      end
+
+      def ==(other_key)
+        return false if other_key.nil?
+        return false if self.class != other_key.class
+        return self.to_text == other_key.to_text
+      end
+
+      def in_known_hosts?(*identifiers)
+        identifiers.each do |identifier|
+          Net::SSH::KnownHosts.search_for(identifier).each do |key|
+            return true if self == key
+          end
+        end
+        return false
+      end
+
+    end
+  end
+end
diff --git a/lib/leap_cli/ssh/options.rb b/lib/leap_cli/ssh/options.rb
new file mode 100644
index 00000000..7bc06564
--- /dev/null
+++ b/lib/leap_cli/ssh/options.rb
@@ -0,0 +1,100 @@
+#
+# Options for passing to the ruby gem ssh-net
+#
+
+module LeapCli
+  module SSH
+    module Options
+
+      #
+      # options passed to net-ssh. See
+      # https://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start
+      # for the available options.
+      #
+      def self.global_options
+        {
+          #:keys_only => true,
+          :global_known_hosts_file => Path.named_path(:known_hosts),
+          :user_known_hosts_file => '/dev/null',
+          :paranoid => true,
+          :verbose => net_ssh_log_level,
+          :auth_methods => ["publickey"],
+          :timeout => 5
+        }
+      end
+
+      def self.node_options(node, ssh_options_override=nil)
+        {
+          # :host_key_alias => node.name, << incompatible with ports in known_hosts
+          :host_name => node.ip_address,
+          :port => node.ssh.port
+        }.merge(
+          contingent_ssh_options_for_node(node)
+        ).merge(
+          ssh_options_override||{}
+        )
+      end
+
+      def self.options_from_args(args)
+        ssh_options = {}
+        if args[:port]
+          ssh_options[:port] = args[:port]
+        end
+        if args[:ip]
+          ssh_options[:host_name] = args[:ip]
+        end
+        if args[:auth_methods]
+          ssh_options[:auth_methods] = args[:auth_methods]
+        end
+        if args[:user]
+          ssh_options[:user] = args[:user]
+        end
+        return ssh_options
+      end
+
+      def self.sanitize_command(cmd)
+        if cmd =~ /(^|\/| )rm / || cmd =~ /(^|\/| )unlink /
+          LeapCli.log :warning, "You probably don't want to do that. Run with --force if you are really sure."
+          exit(1)
+        else
+          cmd
+        end
+      end
+
+      private
+
+      def self.contingent_ssh_options_for_node(node)
+        opts = {}
+        if node.vagrant?
+          opts[:keys] = [LeapCli::Util::Vagrant.vagrant_ssh_key_file]
+          opts[:keys_only] = true # only use the keys specified above, and
+                                  # ignore whatever keys the ssh-agent is aware of.
+          opts[:paranoid] = false # we skip host checking for vagrant nodes,
+                                  # because fingerprint is different for everyone.
+          if LeapCli.logger.log_level <= 1
+            opts[:verbose] = :error # suppress all the warnings about adding
+                                    # host keys to known_hosts, since it is
+                                    # not actually doing that.
+          end
+        end
+        if !node.supported_ssh_host_key_algorithms.empty?
+          opts[:host_key] = node.supported_ssh_host_key_algorithms
+        end
+        return opts
+      end
+
+      def self.net_ssh_log_level
+        if DEBUG
+          case LeapCli.logger.log_level
+            when 1 then :error
+            when 2 then :info
+            else :debug
+          end
+        else
+          :fatal
+        end
+      end
+
+    end
+  end
+end
\ No newline at end of file
diff --git a/lib/leap_cli/ssh/remote_command.rb b/lib/leap_cli/ssh/remote_command.rb
new file mode 100644
index 00000000..0e9f2d55
--- /dev/null
+++ b/lib/leap_cli/ssh/remote_command.rb
@@ -0,0 +1,124 @@
+#
+# Provides SSH.remote_command for running commands in parallel or in sequence
+# on remote servers.
+#
+# The gem sshkit is used for this.
+#
+
+require 'sshkit'
+require 'leap_cli/ssh/options'
+require 'leap_cli/ssh/backend'
+
+SSHKit.config.backend = LeapCli::SSH::Backend
+LeapCli::SSH::Backend.config.ssh_options = LeapCli::SSH::Options.global_options
+
+#
+# define remote_command
+#
+module LeapCli
+  module SSH
+
+    class ExecuteError < StandardError
+    end
+
+    class TimeoutError < ExecuteError
+    end
+
+    # override default runner mode
+    class CustomCoordinator < SSHKit::Coordinator
+      private
+      def default_options
+        { in: :groups, limit: 10, wait: 0 }
+      end
+    end
+
+    #
+    # Available options:
+    #
+    #  :port -- ssh port
+    #  :ip   -- ssh ip
+    #  :auth_methods -- e.g. ["pubkey", "password"]
+    #  :user -- default 'root'
+    #
+    def self.remote_command(nodes, options={}, &block)
+      CustomCoordinator.new(
+        host_list(
+          nodes,
+          SSH::Options.options_from_args(options)
+        )
+      ).each do |ssh, host|
+        LeapCli.log 2, "ssh options for #{host.hostname}: #{host.ssh_options.inspect}"
+        if host.user != 'root'
+          # if the ssh user is not root, we want to make the ssh commands
+          # switch to root before they are run:
+          ssh.set_user('root')
+        end
+        yield ssh, host
+      end
+    end
+
+    #
+    # For example:
+    #
+    # SSH.remote_sync(nodes) do |sync, host|
+    #   sync.source = '/from'
+    #   sync.dest   = '/to'
+    #   sync.flags  = ''
+    #   sync.includes = []
+    #   sync.excludes = []
+    #   sync.exec
+    # end
+    #
+    def self.remote_sync(nodes, options={}, &block)
+      require 'rsync_command'
+      hosts = host_list(
+        nodes,
+        SSH::Options.options_from_args(options)
+      )
+      rsync = RsyncCommand.new(:logger => LeapCli::logger)
+      rsync.asynchronously(hosts) do |sync, host|
+        sync.logger = LeapCli.new_logger
+        sync.user   = host.user || fetch(:user, ENV['USER'])
+        sync.host   = host.hostname
+        sync.ssh    = SSH::Options.global_options.merge(host.ssh_options)
+        sync.chdir  = Path.provider
+        yield(sync, host)
+      end
+      if rsync.failed?
+        LeapCli::Util.bail! do
+          LeapCli.log :failed, "to rsync to #{rsync.failures.map{|f|f[:dest][:host]}.join(' ')}"
+        end
+      end
+    end
+
+    private
+
+    def self.host_list(nodes, ssh_options_override={})
+      if nodes.is_a?(Config::ObjectList)
+        list = nodes.values
+      elsif nodes.is_a?(Config::Node)
+        list = [nodes]
+      else
+        raise ArgumentError, "I don't understand the type of argument `nodes`"
+      end
+      list.collect do |node|
+        options = SSH::Options.node_options(node, ssh_options_override)
+        user    = options.delete(:user) || 'root'
+        #
+        # note: whatever hostname is specified here will be what is used
+        # when loading options from .ssh/config. However, this value
+        # has no impact on the actual ip address that is connected to,
+        # which is determined by the :host_name value in ssh_options.
+        #
+        SSHKit::Host.new(
+          :hostname => node.domain.full,
+          :user => user,
+          :ssh_options => options
+        )
+      end
+    end
+
+  end
+end
+
+
diff --git a/lib/leap_cli/ssh/scripts.rb b/lib/leap_cli/ssh/scripts.rb
new file mode 100644
index 00000000..3dd6b604
--- /dev/null
+++ b/lib/leap_cli/ssh/scripts.rb
@@ -0,0 +1,163 @@
+#
+# Common commands that we would like to run on remote servers.
+#
+# These scripts are available via:
+#
+# SSH.remote_command(nodes) do |ssh, host|
+#   ssh.script.custom_script_name
+# end
+#
+
+module LeapCli
+  module SSH
+    class Scripts
+
+      REQUIRED_PACKAGES = "puppet rsync lsb-release locales"
+
+      attr_reader :ssh, :host
+      def initialize(backend, hostname)
+        @ssh = backend
+        @host = hostname
+      end
+
+      #
+      # creates directories that are owned by root and 700 permissions
+      #
+      def mkdirs(*dirs)
+        raise ArgumentError.new('illegal dir name') if dirs.grep(/[\' ]/).any?
+        ssh.stream dirs.collect{|dir| "mkdir -m 700 -p #{dir}; "}.join
+      end
+
+      #
+      # echos "ok" if the node has been initialized and the required packages are installed, bails out otherwise.
+      #
+      def assert_initialized
+        begin
+          test_initialized_file = "test -f #{Leap::Platform.init_path}"
+          check_required_packages = "! dpkg-query -W --showformat='${Status}\n' #{REQUIRED_PACKAGES} 2>&1 | grep -q -E '(deinstall|no packages)'"
+          ssh.stream "#{test_initialized_file} && #{check_required_packages} && echo ok", :raise_error => true
+        rescue SSH::ExecuteError
+          ssh.log :error, "running deploy: node not initialized. Run `leap node init #{host}`.", :host => host
+          raise # will skip further action on this node
+        end
+      end
+
+      #
+      # bails out the deploy if the file /etc/leap/no-deploy exists.
+      #
+      def check_for_no_deploy
+        begin
+          ssh.stream "test ! -f /etc/leap/no-deploy", :raise_error => true, :log_output => false
+        rescue SSH::TimeoutError
+          raise
+        rescue SSH::ExecuteError
+          ssh.log :warning, "can't continue because file /etc/leap/no-deploy exists", :host => host
+          raise # will skip further action on this node
+        end
+      end
+
+      #
+      # dumps debugging information
+      #
+      def debug
+        output = ssh.capture "#{Leap::Platform.leap_dir}/bin/debug.sh"
+        ssh.log(output, :wrap => true, :host => host, :color => :cyan)
+      end
+
+      #
+      # dumps the recent deploy history to the console
+      #
+      def history(lines)
+        cmd = "(test -s /var/log/leap/deploy-summary.log && tail -n #{lines} /var/log/leap/deploy-summary.log) || (test -s /var/log/leap/deploy-summary.log.1 && tail -n #{lines} /var/log/leap/deploy-summary.log.1) || (echo 'no history')"
+        history = ssh.capture(cmd, :log_output => false)
+        if history
+          ssh.log host, :color => :cyan, :style => :bold do
+            ssh.log history, :wrap => true
+          end
+        end
+      end
+
+      #
+      # apply puppet! weeeeeee
+      #
+      def puppet_apply(options)
+        cmd = "#{Leap::Platform.leap_dir}/bin/puppet_command set_hostname apply #{flagize(options)}"
+        ssh.stream cmd, :log_finish => true
+      end
+
+      def install_authorized_keys
+        ssh.log :updating, "authorized_keys" do
+          mkdirs '/root/.ssh'
+          ssh.upload! LeapCli::Path.named_path(:authorized_keys), '/root/.ssh/authorized_keys', :mode => 0600
+        end
+      end
+
+      #
+      # for vagrant nodes, we install insecure vagrant key to authorized_keys2, since deploy
+      # will overwrite authorized_keys.
+      #
+      # why force the insecure vagrant key?
+      #   if we don't do this, then first time initialization might fail if the user has many keys
+      #   (ssh will bomb out before it gets to the vagrant key).
+      #   and it really doesn't make sense to ask users to pin the insecure vagrant key in their
+      #   .ssh/config files.
+      #
+      def install_insecure_vagrant_key
+        ssh.log :installing, "insecure vagrant key" do
+          mkdirs '/root/.ssh'
+          ssh.upload! LeapCli::Path.vagrant_ssh_pub_key_file, '/root/.ssh/authorized_keys2', :mode => 0600
+        end
+      end
+
+      def install_prerequisites
+        bin_dir = File.join(Leap::Platform.leap_dir, 'bin')
+        node_init_path = File.join(bin_dir, 'node_init')
+        ssh.log :running, "node_init script" do
+          mkdirs bin_dir
+          ssh.upload! LeapCli::Path.node_init_script, node_init_path, :mode => 0700
+          ssh.stream node_init_path, :log_wrap => true
+        end
+      end
+
+      #
+      # AWS debian images only allow you to login as admin. This is done with a
+      # custom command in /root/.ssh/authorized_keys, instead of by modifying
+      # /etc/ssh/sshd_config.
+      #
+      # We need to be able to ssh as root for scp and rsync to work.
+      #
+      # This command is run as 'admin', with a sudo wrapper. In order for the
+      # sudo to work, the command must be specified as separate arguments with
+      # no spaces (that is how ssh-kit works).
+      #
+      def allow_root_ssh
+        ssh.execute 'cp', '/home/admin/.ssh/authorized_keys', '/root/.ssh/authorized_keys'
+      end
+
+      #
+      # uploads an acme challenge for renewing certificates using Let's Encrypt CA.
+      #
+      # Filename is returned from acme api, so it must not be trusted.
+      #
+      def upload_acme_challenge(filename, content)
+        path = '/srv/acme/' + filename.gsub(/[^a-zA-Z0-9_-]/, '')
+        ssh.upload! StringIO.new(content), path, :mode => 0444
+      end
+
+      private
+
+      def flagize(hsh)
+        hsh.inject([]) {|str, item|
+          if item[1] === false
+            str
+          elsif item[1] === true
+            str << "--" + item[0].to_s
+          else
+            str << "--" + item[0].to_s + " " + item[1].inspect
+          end
+        }.join(' ')
+      end
+
+    end
+  end
+end
\ No newline at end of file