diff options
| -rw-r--r-- | lib/leap/platform.rb | 99 | ||||
| -rw-r--r-- | lib/leap_cli/commands/deploy.rb | 4 | ||||
| -rw-r--r-- | lib/leap_cli/config/environment.rb | 180 | ||||
| -rw-r--r-- | lib/leap_cli/config/filter.rb | 178 | ||||
| -rw-r--r-- | lib/leap_cli/config/manager.rb | 422 | ||||
| -rw-r--r-- | lib/leap_cli/config/node.rb | 78 | ||||
| -rw-r--r-- | lib/leap_cli/config/object.rb | 428 | ||||
| -rw-r--r-- | lib/leap_cli/config/object_list.rb | 209 | ||||
| -rw-r--r-- | lib/leap_cli/config/provider.rb | 22 | ||||
| -rw-r--r-- | lib/leap_cli/config/secrets.rb | 87 | ||||
| -rw-r--r-- | lib/leap_cli/config/sources.rb | 11 | ||||
| -rw-r--r-- | lib/leap_cli/config/tag.rb | 25 | ||||
| -rw-r--r-- | lib/leap_cli/leapfile_extensions.rb | 24 | ||||
| -rw-r--r-- | lib/leap_cli/load_libraries.rb | 20 | ||||
| -rw-r--r-- | lib/leap_cli/log_filter.rb | 171 | ||||
| -rw-r--r-- | lib/leap_cli/macros.rb | 16 | ||||
| -rw-r--r-- | lib/leap_cli/ssh/backend.rb | 5 | ||||
| -rw-r--r-- | lib/leap_cli/ssh/remote_command.rb | 3 | ||||
| -rw-r--r-- | lib/leap_cli/ssh/scripts.rb | 8 | ||||
| -rw-r--r-- | lib/leap_cli/util/secret.rb | 55 | ||||
| -rw-r--r-- | lib/leap_cli/util/x509.rb | 33 |
21 files changed, 2058 insertions, 20 deletions
diff --git a/lib/leap/platform.rb b/lib/leap/platform.rb new file mode 100644 index 00000000..9e6cadd5 --- /dev/null +++ b/lib/leap/platform.rb @@ -0,0 +1,99 @@ +module Leap + + class Platform + class << self + # + # configuration + # + + attr_reader :version + attr_reader :compatible_cli + attr_accessor :facts + attr_accessor :paths + attr_accessor :node_files + attr_accessor :monitor_username + attr_accessor :reserved_usernames + + attr_accessor :hiera_dir + attr_accessor :hiera_path + attr_accessor :files_dir + attr_accessor :leap_dir + attr_accessor :init_path + + attr_accessor :default_puppet_tags + + def define(&block) + # some defaults: + @reserved_usernames = [] + @hiera_dir = '/etc/leap' + @hiera_path = '/etc/leap/hiera.yaml' + @leap_dir = '/srv/leap' + @files_dir = '/srv/leap/files' + @init_path = '/srv/leap/initialized' + @default_puppet_tags = [] + + self.instance_eval(&block) + + @version ||= Gem::Version.new("0.0") + end + + def validate!(cli_version, compatible_platforms, leapfile) + if !compatible_with_cli?(cli_version) || !version_in_range?(compatible_platforms) + raise StandardError, "This leap command (v#{cli_version}) " + + "is not compatible with the platform #{leapfile.platform_directory_path} (v#{version}).\n " + + "You need either leap command #{compatible_cli.first} to #{compatible_cli.last} or " + + "platform version #{compatible_platforms.first} to #{compatible_platforms.last}" + end + end + + def version=(version) + @version = Gem::Version.new(version) + end + + def compatible_cli=(range) + @compatible_cli = range + @minimum_cli_version = Gem::Version.new(range.first) + @maximum_cli_version = Gem::Version.new(range.last) + end + + # + # return true if the cli_version is compatible with this platform. + # + def compatible_with_cli?(cli_version) + cli_version = Gem::Version.new(cli_version) + cli_version >= @minimum_cli_version && cli_version <= @maximum_cli_version + end + + # + # return true if the platform version is within the specified range. + # + def version_in_range?(range) + if range.is_a? String + range = range.split('..') + end + minimum_platform_version = Gem::Version.new(range.first) + maximum_platform_version = Gem::Version.new(range.last) + @version >= minimum_platform_version && @version <= maximum_platform_version + end + + def major_version + if @version.segments.first == 0 + @version.segments[0..1].join('.') + else + @version.segments.first + end + end + + def method_missing(method, *args) + puts + puts "WARNING:" + puts " leap_cli is out of date and does not understand `#{method}`." + puts " called from: #{caller.first}" + puts " please upgrade to a newer leap_cli" + end + + end + + end + +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/deploy.rb b/lib/leap_cli/commands/deploy.rb index 165ce588..d26b9905 100644 --- a/lib/leap_cli/commands/deploy.rb +++ b/lib/leap_cli/commands/deploy.rb @@ -89,6 +89,10 @@ module LeapCli end end + if nodes.empty? + return + end + log :synching, "configuration files" do sync_hiera_config(nodes, options) sync_support_files(nodes, options) diff --git a/lib/leap_cli/config/environment.rb b/lib/leap_cli/config/environment.rb new file mode 100644 index 00000000..398fd023 --- /dev/null +++ b/lib/leap_cli/config/environment.rb @@ -0,0 +1,180 @@ +# +# All configurations files can be isolated into separate environments. +# +# Each config json in each environment inherits from the default environment, +# which in term inherits from the "_base_" environment: +# +# _base_ -- base provider in leap_platform +# '- default -- environment in provider dir when no env is set +# '- production -- example environment +# + +module LeapCli; module Config + + class Environment + # the String name of the environment + attr_accessor :name + + # the shared Manager object + attr_accessor :manager + + # hashes of {name => Config::Object} + attr_accessor :services, :tags, :partials + + # a Config::Provider + attr_accessor :provider + + # a Config::Object + attr_accessor :common + + # shared, non-inheritable + def nodes; @@nodes; end + def secrets; @@secrets; end + + def initialize(manager, name, search_dir, parent, options={}) + @@nodes ||= nil + @@secrets ||= nil + + @manager = manager + @name = name + + load_provider_files(search_dir, options) + + if parent + @services.inherit_from! parent.services, self + @tags.inherit_from! parent.tags , self + @partials.inherit_from! parent.partials, self + @common.inherit_from! parent.common + @provider.inherit_from! parent.provider + end + + if @provider + @provider.set_env(name) + @provider.validate! + end + end + + def load_provider_files(search_dir, options) + # + # load empty environment if search_dir doesn't exist + # + if search_dir.nil? || !Dir.exist?(search_dir) + @services = Config::ObjectList.new + @tags = Config::ObjectList.new + @partials = Config::ObjectList.new + @provider = Config::Provider.new + @common = Config::Object.new + return + end + + # + # inheritable + # + if options[:scope] + scope = options[:scope] + @services = load_all_json(Path.named_path([:service_env_config, '*', scope], search_dir), Config::Tag, options) + @tags = load_all_json(Path.named_path([:tag_env_config, '*', scope], search_dir), Config::Tag, options) + @partials = load_all_json(Path.named_path([:service_env_config, '_*', scope], search_dir), Config::Tag, options) + @provider = load_json( Path.named_path([:provider_env_config, scope], search_dir), Config::Provider, options) + @common = load_json( Path.named_path([:common_env_config, scope], search_dir), Config::Object, options) + else + @services = load_all_json(Path.named_path([:service_config, '*'], search_dir), Config::Tag, options) + @tags = load_all_json(Path.named_path([:tag_config, '*'], search_dir), Config::Tag, options) + @partials = load_all_json(Path.named_path([:service_config, '_*'], search_dir), Config::Tag, options) + @provider = load_json( Path.named_path(:provider_config, search_dir), Config::Provider, options) + @common = load_json( Path.named_path(:common_config, search_dir), Config::Object, options) + end + + # remove 'name' from partials, since partials get merged with nodes + @partials.values.each {|partial| partial.delete('name'); } + + # + # shared: currently non-inheritable + # load the first ones we find, and only those. + # + if @@nodes.nil? || @@nodes.empty? + @@nodes = load_all_json(Path.named_path([:node_config, '*'], search_dir), Config::Node, options) + end + if @@secrets.nil? || @@secrets.empty? + @@secrets = load_json(Path.named_path(:secrets_config, search_dir), Config::Secrets, options) + end + end + + # + # Loads a json template file as a Hash (used only when creating a new node .json + # file for the first time). + # + def template(template) + path = Path.named_path([:template_config, template], Path.provider_base) + if File.exist?(path) + return load_json(path, Config::Object) + else + return nil + end + end + + private + + def load_all_json(pattern, object_class, options={}) + results = Config::ObjectList.new + Dir.glob(pattern).each do |filename| + next if options[:no_dots] && File.basename(filename) !~ /^[^\.]*\.json$/ + obj = load_json(filename, object_class) + if obj + name = File.basename(filename).force_encoding('utf-8').sub(/^([^\.]+).*\.json$/,'\1') + obj['name'] ||= name + if options[:env] + obj.environment = options[:env] + end + results[name] = obj + end + end + results + end + + def load_json(filename, object_class, options={}) + if !File.exist?(filename) + return object_class.new(self) + end + + Util::log :loading, filename, 3 + + # + # Read a JSON file, strip out comments. + # + # UTF8 is the default encoding for JSON, but others are allowed: + # https://www.ietf.org/rfc/rfc4627.txt + # + buffer = StringIO.new + File.open(filename, "rb", :encoding => 'UTF-8') do |f| + while (line = f.gets) + next if line =~ /^\s*\/\// + buffer << line + end + end + + # + # force UTF-8 + # + if $ruby_version >= [1,9] + string = buffer.string.force_encoding('utf-8') + else + string = Iconv.conv("UTF-8//IGNORE", "UTF-8", buffer.string) + end + + # parse json + begin + hash = JSON.parse(string, :object_class => Hash, :array_class => Array) || {} + rescue SyntaxError, JSON::ParserError => exc + Util::log 0, :error, 'in file "%s":' % filename + Util::log 0, exc.to_s, :indent => 1 + return nil + end + object = object_class.new(self) + object.deep_merge!(hash) + return object + end + + end # end Environment + +end; end
\ No newline at end of file diff --git a/lib/leap_cli/config/filter.rb b/lib/leap_cli/config/filter.rb new file mode 100644 index 00000000..27502577 --- /dev/null +++ b/lib/leap_cli/config/filter.rb @@ -0,0 +1,178 @@ +# +# Many leap_cli commands accept a list of filters to select a subset of nodes for the command to +# be applied to. This class is a helper for manager to run these filters. +# +# Classes other than Manager should not use this class. +# +# Filter rules: +# +# * A filter consists of a list of tokens +# * A token may be a service name, tag name, environment name, or node name. +# * Each token may be optionally prefixed with a plus sign. +# * Multiple tokens with a plus are treated as an OR condition, +# but treated as an AND condition with the plus sign. +# +# For example +# +# * openvpn +development => all nodes with service 'openvpn' AND environment 'development' +# * openvpn seattle => all nodes with service 'openvpn' OR tag 'seattle'. +# +# There can only be one environment specified. Typically, there are also tags +# for each environment name. These name are treated as environments, not tags. +# +module LeapCli + module Config + class Filter + + # + # filter -- array of strings, each one a filter + # options -- hash, possible keys include + # :nopin -- disregard environment pinning + # :local -- if false, disallow local nodes + # + # A nil value in the filters array indicates + # the default environment. This is in order to support + # calls like `manager.filter(environments)` + # + def initialize(filters, options, manager) + @filters = filters.nil? ? [] : filters.dup + @environments = [] + @options = options + @manager = manager + + # split filters by pulling out items that happen + # to be environment names. + if LeapCli.leapfile.environment.nil? || @options[:nopin] + @environments = [] + else + @environments = [LeapCli.leapfile.environment] + end + @filters.select! do |filter| + if filter.nil? + @environments << nil unless @environments.include?(nil) + false + else + filter_text = filter.sub(/^\+/,'') + if is_environment?(filter_text) + if filter_text == LeapCli.leapfile.environment + # silently ignore already pinned environments + elsif (filter =~ /^\+/ || @filters.first == filter) && !@environments.empty? + LeapCli::Util.bail! do + LeapCli.log "Environments are exclusive: no node is in two environments." do + LeapCli.log "Tried to filter on '#{@environments.join('\' AND \'')}' AND '#{filter_text}'" + end + end + else + @environments << filter_text + end + false + else + true + end + end + end + + # don't let the first filter have a + prefix + if @filters[0] =~ /^\+/ + @filters[0] = @filters[0][1..-1] + end + end + + # actually run the filter, returns a filtered list of nodes + def nodes() + if @filters.empty? + return nodes_for_empty_filter + else + return nodes_for_filter + end + end + + private + + def nodes_for_empty_filter + node_list = @manager.nodes + if @environments.any? + node_list = node_list[ @environments.collect{|e|[:environment, env_to_filter(e)]} ] + end + if @options[:local] === false + node_list = node_list[:environment => '!local'] + end + if @options[:disabled] === false + node_list = node_list[:environment => '!disabled'] + end + node_list + end + + def nodes_for_filter + node_list = Config::ObjectList.new + @filters.each do |filter| + if filter =~ /^\+/ + keep_list = nodes_for_name(filter[1..-1]) + node_list.delete_if do |name, node| + if keep_list[name] + false + else + true + end + end + else + node_list.merge!(nodes_for_name(filter)) + end + end + node_list + end + + private + + # + # returns a set of nodes corresponding to a single name, + # where name could be a node name, service name, or tag name. + # + # For services and tags, we only include nodes for the + # environments that are active + # + def nodes_for_name(name) + if node = @manager.nodes[name] + return Config::ObjectList.new(node) + elsif @environments.empty? + if @manager.services[name] + return @manager.env('_all_').services[name].node_list + elsif @manager.tags[name] + return @manager.env('_all_').tags[name].node_list + else + LeapCli.log :warning, "filter '#{name}' does not match any node names, tags, services, or environments." + return Config::ObjectList.new + end + else + node_list = Config::ObjectList.new + if @manager.services[name] + @environments.each do |env| + node_list.merge!(@manager.env(env).services[name].node_list) + end + elsif @manager.tags[name] + @environments.each do |env| + node_list.merge!(@manager.env(env).tags[name].node_list) + end + else + LeapCli.log :warning, "filter '#{name}' does not match any node names, tags, services, or environments." + end + return node_list + end + end + + # + # when pinning, we use the name 'default' to specify nodes + # without an environment set, but when filtering, we need to filter + # on :environment => nil. + # + def env_to_filter(environment) + environment == 'default' ? nil : environment + end + + def is_environment?(text) + text == 'default' || @manager.environment_names.include?(text) + end + + end + end +end diff --git a/lib/leap_cli/config/manager.rb b/lib/leap_cli/config/manager.rb new file mode 100644 index 00000000..aea1d322 --- /dev/null +++ b/lib/leap_cli/config/manager.rb @@ -0,0 +1,422 @@ +# encoding: utf-8 + +require 'json/pure' + +if $ruby_version < [1,9] + require 'iconv' +end + +module LeapCli + module Config + + # + # A class to manage all the objects in all the configuration files. + # + class Manager + + def initialize + @environments = {} # hash of `Environment` objects, keyed by name. + Config::Object.send(:include, LeapCli::Macro) + end + + ## + ## ATTRIBUTES + ## + + # + # returns the Hash of the contents of facts.json + # + def facts + @facts ||= begin + content = Util.read_file(:facts) + if !content || content.empty? + content = "{}" + end + JSON.parse(content) + rescue SyntaxError, JSON::ParserError => exc + Util::bail! "Could not parse facts.json -- #{exc}" + end + end + + # + # returns an Array of all the environments defined for this provider. + # the returned array includes nil (for the default environment) + # + def environment_names + @environment_names ||= begin + [nil] + (env.tags.field('environment') + env.nodes.field('environment')).compact.uniq + end + end + + # + # Returns the appropriate environment variable + # + def env(env=nil) + @environments[env || 'default'] + end + + # + # The default accessors + # + # For these defaults, use 'default' environment, or whatever + # environment is pinned. + # + # I think it might be an error that these are ever used + # and I would like to get rid of them. + # + def services; env(default_environment).services; end + def tags; env(default_environment).tags; end + def partials; env(default_environment).partials; end + def provider; env(default_environment).provider; end + def common; env(default_environment).common; end + def secrets; env(default_environment).secrets; end + def nodes; env(default_environment).nodes; end + def template(*args) + self.env.template(*args) + end + + def default_environment + LeapCli.leapfile.environment + end + + ## + ## IMPORT EXPORT + ## + + def add_environment(args) + if args[:inherit] + parent = @environments[args.delete(:inherit)] + else + parent = nil + end + @environments[args[:name]] = Environment.new( + self, + args.delete(:name), + args.delete(:dir), + parent, + args + ) + end + + # + # load .json configuration files + # + def load(options = {}) + @provider_dir = Path.provider + + # load base + add_environment(name: '_base_', dir: Path.provider_base) + + # load provider + Util::assert_files_exist!(Path.named_path(:provider_config, @provider_dir)) + add_environment(name: 'default', dir: @provider_dir, + inherit: '_base_', no_dots: true) + + # create a special '_all_' environment, used for tracking + # the union of all the environments + add_environment(name: '_all_', inherit: 'default') + + # load environments + environment_names.each do |ename| + if ename + LeapCli.log 3, :loading, '%s environment...' % ename + add_environment(name: ename, dir: @provider_dir, + inherit: 'default', scope: ename) + end + end + + # apply inheritance + env.nodes.each do |name, node| + Util::assert! name =~ /^[0-9a-z-]+$/, "Illegal character(s) used in node name '#{name}'" + env.nodes[name] = apply_inheritance(node) + end + + # do some node-list post-processing + cleanup_node_lists(options) + + # apply control files + env.nodes.each do |name, node| + control_files(node).each do |file| + begin + node.eval_file file + rescue ConfigError => exc + if options[:continue_on_error] + exc.log + else + raise exc + end + end + end + end + end + + # + # save compiled hiera .yaml files + # + # if a node_list is specified, only update those .yaml files. + # otherwise, update all files, destroying files that are no longer used. + # + def export_nodes(node_list=nil) + updated_hiera = [] + updated_files = [] + existing_hiera = nil + existing_files = nil + + unless node_list + node_list = env.nodes + existing_hiera = Dir.glob(Path.named_path([:hiera, '*'], @provider_dir)) + existing_files = Dir.glob(Path.named_path([:node_files_dir, '*'], @provider_dir)) + end + + node_list.each_node do |node| + filepath = Path.named_path([:node_files_dir, node.name], @provider_dir) + hierapath = Path.named_path([:hiera, node.name], @provider_dir) + Util::write_file!(hierapath, node.dump_yaml) + updated_files << filepath + updated_hiera << hierapath + end + + if @disabled_nodes + # make disabled nodes appear as if they are still active + @disabled_nodes.each_node do |node| + updated_files << Path.named_path([:node_files_dir, node.name], @provider_dir) + updated_hiera << Path.named_path([:hiera, node.name], @provider_dir) + end + end + + # remove files that are no longer needed + if existing_hiera + (existing_hiera - updated_hiera).each do |filepath| + Util::remove_file!(filepath) + end + end + if existing_files + (existing_files - updated_files).each do |filepath| + Util::remove_directory!(filepath) + end + end + end + + def export_secrets(clean_unused_secrets = false) + if env.secrets.any? + Util.write_file!([:secrets_config, @provider_dir], env.secrets.dump_json(clean_unused_secrets) + "\n") + end + end + + ## + ## FILTERING + ## + + # + # returns a node list consisting only of nodes that satisfy the filter criteria. + # + # filter: condition [condition] [condition] [+condition] + # condition: [node_name | service_name | tag_name | environment_name] + # + # if conditions is prefixed with +, then it works like an AND. Otherwise, it works like an OR. + # + # args: + # filter -- array of filter terms, one per item + # + # options: + # :local -- if :local is false and the filter is empty, then local nodes are excluded. + # :nopin -- if true, ignore environment pinning + # + def filter(filters=nil, options={}) + Filter.new(filters, options, self).nodes() + end + + # + # same as filter(), but exits if there is no matching nodes + # + def filter!(filters, options={}) + node_list = filter(filters, options) + Util::assert! node_list.any?, "Could not match any nodes from '#{filters.join ' '}'" + return node_list + end + + # + # returns a single Config::Object that corresponds to a Node. + # + def node(name) + if name =~ /\./ + # probably got a fqdn, since periods are not allowed in node names. + # so, take the part before the first period as the node name + name = name.split('.').first + end + env.nodes[name] + end + + # + # returns a single node that is disabled + # + def disabled_node(name) + @disabled_nodes[name] + end + + # + # yields each node, in sorted order + # + def each_node(&block) + env.nodes.each_node(&block) + end + + def reload_node!(node) + env.nodes[node.name] = apply_inheritance!(node) + end + + ## + ## CONNECTIONS + ## + + class ConnectionList < Array + def add(data={}) + self << { + "from" => data[:from], + "to" => data[:to], + "port" => data[:port] + } + end + end + + def connections + @connections ||= ConnectionList.new + end + + ## + ## PRIVATE + ## + + private + + # + # makes a node inherit options from appropriate the common, service, and tag json files. + # + def apply_inheritance(node, throw_exceptions=false) + new_node = Config::Node.new(nil) + node_env = guess_node_env(node) + new_node.set_environment(node_env, new_node) + + # inherit from common + new_node.deep_merge!(node_env.common) + + # inherit from services + if node['services'] + node['services'].to_a.each do |node_service| + service = node_env.services[node_service] + if service.nil? + msg = 'in node "%s": the service "%s" does not exist.' % [node['name'], node_service] + LeapCli.log 0, :error, msg + raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions + else + new_node.deep_merge!(service) + end + end + end + + # inherit from tags + if node.vagrant? + node['tags'] = (node['tags'] || []).to_a + ['local'] + end + if node['tags'] + node['tags'].to_a.each do |node_tag| + tag = node_env.tags[node_tag] + if tag.nil? + msg = 'in node "%s": the tag "%s" does not exist.' % [node['name'], node_tag] + log 0, :error, msg + raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions + else + new_node.deep_merge!(tag) + end + end + end + + # inherit from node + new_node.deep_merge!(node) + return new_node + end + + def apply_inheritance!(node) + apply_inheritance(node, true) + end + + # + # Guess the environment of the node from the tag names. + # + # Technically, this is wrong: a tag that sets the environment might not be + # named the same as the environment. This code assumes that it is. + # + # Unfortunately, it is a chicken and egg problem. We need to know the nodes + # likely environment in order to apply the inheritance that will actually + # determine the node's properties. + # + def guess_node_env(node) + if node.vagrant? + return self.env("local") + else + environment = self.env(default_environment) + if node['tags'] + node['tags'].to_a.each do |tag| + if self.environment_names.include?(tag) + environment = self.env(tag) + end + end + end + return environment + end + end + + # + # does some final clean at the end of loading nodes. + # this includes removing disabled nodes, and populating + # the services[x].node_list and tags[x].node_list + # + def cleanup_node_lists(options) + @disabled_nodes = Config::ObjectList.new + env.nodes.each do |name, node| + if node.enabled || options[:include_disabled] + if node['services'] + node['services'].to_a.each do |node_service| + env(node.environment).services[node_service].node_list.add(node.name, node) + env('_all_').services[node_service].node_list.add(node.name, node) + end + end + if node['tags'] + node['tags'].to_a.each do |node_tag| + env(node.environment).tags[node_tag].node_list.add(node.name, node) + env('_all_').tags[node_tag].node_list.add(node.name, node) + end + end + elsif !options[:include_disabled] + LeapCli.log 2, :skipping, "disabled node #{name}." + env.nodes.delete(name) + @disabled_nodes[name] = node + end + end + end + + # + # returns a list of 'control' files for this node. + # a control file is like a service or a tag JSON file, but it contains + # raw ruby code that gets evaluated in the context of the node. + # Yes, this entirely breaks our functional programming model + # for JSON generation. + # + def control_files(node) + files = [] + [Path.provider_base, @provider_dir].each do |provider_dir| + [['services', :service_config], ['tags', :tag_config]].each do |attribute, path_sym| + node[attribute].each do |attr_value| + path = Path.named_path([path_sym, "#{attr_value}.rb"], provider_dir).sub(/\.json$/,'') + if File.exist?(path) + files << path + end + end + end + end + return files + end + + end + end +end diff --git a/lib/leap_cli/config/node.rb b/lib/leap_cli/config/node.rb new file mode 100644 index 00000000..f8ec0527 --- /dev/null +++ b/lib/leap_cli/config/node.rb @@ -0,0 +1,78 @@ +# +# Configuration for a 'node' (a server in the provider's infrastructure) +# + +require 'ipaddr' + +module LeapCli; module Config + + class Node < Object + attr_accessor :file_paths + + def initialize(environment=nil) + super(environment) + @node = self + @file_paths = [] + end + + # + # returns true if this node has an ip address in the range of the vagrant network + # + def vagrant? + begin + vagrant_range = IPAddr.new LeapCli.leapfile.vagrant_network + rescue ArgumentError => exc + Util::bail! { Util::log :invalid, "ip address '#{@node.ip_address}' vagrant.network" } + end + + begin + ip_address = IPAddr.new @node.get('ip_address') + rescue ArgumentError => exc + Util::log :warning, "invalid ip address '#{@node.get('ip_address')}' for node '#{@node.name}'" + end + return vagrant_range.include?(ip_address) + end + + # + # Return a hash table representation of ourselves, with the key equal to the @node.name, + # and the value equal to the fields specified in *keys. + # + # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b' + # + # compare to Object#pick(*keys). This method is the sames as Config::ObjectList#pick_fields, + # but works on a single node. + # + # Example: + # + # node.pick('domain.internal') => + # + # { + # 'node1': { + # 'domain_internal': 'node1.example.i' + # } + # } + # + def pick_fields(*keys) + {@node.name => self.pick(*keys)} + end + + # + # can be overridden by the platform. + # returns a list of node names that should be tested before this node + # + def test_dependencies + [] + end + + # returns a string list of supported ssh host key algorithms for this node. + # or an empty string if it could not be determined + def supported_ssh_host_key_algorithms + require 'leap_cli/ssh' + @host_key_algo ||= LeapCli::SSH::Key.supported_host_key_algorithms( + Util.read_file([:node_ssh_pub_key, @node.name]) + ) + end + + end + +end; end diff --git a/lib/leap_cli/config/object.rb b/lib/leap_cli/config/object.rb new file mode 100644 index 00000000..b117c2f0 --- /dev/null +++ b/lib/leap_cli/config/object.rb @@ -0,0 +1,428 @@ +# encoding: utf-8 + +require 'erb' +require 'json/pure' # pure ruby implementation is required for our sorted trick to work. + +if $ruby_version < [1,9] + $KCODE = 'UTF8' +end +require 'ya2yaml' # pure ruby yaml + +module LeapCli + module Config + + # + # This class represents the configuration for a single node, service, or tag. + # Also, all the nested hashes are also of this type. + # + # It is called 'object' because it corresponds to an Object in JSON. + # + class Object < Hash + + attr_reader :env + attr_reader :node + + def initialize(environment=nil, node=nil) + raise ArgumentError unless environment.nil? || environment.is_a?(Config::Environment) + @env = environment + # an object that is a node as @node equal to self, otherwise all the + # child objects point back to the top level node. + @node = node || self + end + + def manager + @env.manager + end + + # + # TODO: deprecate node.global() + # + def global + @env + end + + def environment=(e) + self.store('environment', e) + end + + def environment + self['environment'] + end + + def duplicate(env) + new_object = self.deep_dup + new_object.set_environment(env, new_object) + end + + # + # export YAML + # + # We use pure ruby yaml exporter ya2yaml instead of SYCK or PSYCH because it + # allows us greater compatibility regardless of installed ruby version and + # greater control over how the yaml is exported (sorted keys, in particular). + # + def dump_yaml + evaluate(@node) + sorted_ya2yaml(:syck_compatible => true) + end + + # + # export JSON + # + def dump_json(options={}) + evaluate(@node) + if options[:format] == :compact + return self.to_json + else + excluded = {} + if options[:exclude] + options[:exclude].each do |key| + excluded[key] = self[key] + self.delete(key) + end + end + json_str = JSON.sorted_generate(self) + if excluded.any? + self.merge!(excluded) + end + return json_str + end + end + + def evaluate(context=@node) + evaluate_everything(context) + late_evaluate_everything(context) + end + + ## + ## FETCHING VALUES + ## + + def [](key) + get(key) + end + + # Overrride some default methods in Hash that are likely to + # be used as attributes. + alias_method :hkey, :key + def key; get('key'); end + + # + # make hash addressable like an object (e.g. obj['name'] available as obj.name) + # + def method_missing(method, *args, &block) + get!(method) + end + + def get(key) + begin + get!(key) + rescue NoMethodError + nil + end + end + + # override behavior of #default() from Hash + def default + get!('default') + end + + # + # Like a normal Hash#[], except: + # + # (1) lazily eval dynamic values when we encounter them. (i.e. strings that start with "= ") + # + # (2) support for nested references in a single string (e.g. ['a.b'] is the same as ['a']['b']) + # the dot path is always absolute, starting at the top-most object. + # + def get!(key) + key = key.to_s + if self.has_key?(key) + fetch_value(key) + elsif key =~ /\./ + # for keys with with '.' in them, we start from the root object (@node). + keys = key.split('.') + value = self.get!(keys.first) + if value.is_a? Config::Object + value.get!(keys[1..-1].join('.')) + else + value + end + else + raise NoMethodError.new(key, "No method '#{key}' for #{self.class}") + end + end + + ## + ## COPYING + ## + + # + # A deep (recursive) merge with another Config::Object. + # + # If prefer_self is set to true, the value from self will be picked when there is a conflict + # that cannot be merged. + # + # Merging rules: + # + # - If a value is a hash, we recursively merge it. + # - If the value is simple, like a string, the new one overwrites the value. + # - If the value is an array: + # - If both old and new values are arrays, the new one replaces the old. + # - If one of the values is simple but the other is an array, the simple is added to the array. + # + def deep_merge!(object, prefer_self=false) + object.each do |key,new_value| + if self.has_key?('+'+key) + mode = :add + old_value = self.fetch '+'+key, nil + self.delete('+'+key) + elsif self.has_key?('-'+key) + mode = :subtract + old_value = self.fetch '-'+key, nil + self.delete('-'+key) + elsif self.has_key?('!'+key) + mode = :replace + old_value = self.fetch '!'+key, nil + self.delete('!'+key) + else + mode = :normal + old_value = self.fetch key, nil + end + + # clean up boolean + new_value = true if new_value == "true" + new_value = false if new_value == "false" + old_value = true if old_value == "true" + old_value = false if old_value == "false" + + # force replace? + if mode == :replace && prefer_self + value = old_value + + # merge hashes + elsif old_value.is_a?(Hash) || new_value.is_a?(Hash) + value = Config::Object.new(@env, @node) + old_value.is_a?(Hash) ? value.deep_merge!(old_value) : (value[key] = old_value if !old_value.nil?) + new_value.is_a?(Hash) ? value.deep_merge!(new_value, prefer_self) : (value[key] = new_value if !new_value.nil?) + + # merge nil + elsif new_value.nil? + value = old_value + elsif old_value.nil? + value = new_value + + # merge arrays when one value is not an array + elsif old_value.is_a?(Array) && !new_value.is_a?(Array) + (value = (old_value.dup << new_value).compact.uniq).delete('REQUIRED') + elsif new_value.is_a?(Array) && !old_value.is_a?(Array) + (value = (new_value.dup << old_value).compact.uniq).delete('REQUIRED') + + # merge two arrays + elsif old_value.is_a?(Array) && new_value.is_a?(Array) + if mode == :add + value = (old_value + new_value).sort.uniq + elsif mode == :subtract + value = new_value - old_value + elsif prefer_self + value = old_value + else + value = new_value + end + + # catch errors + elsif type_mismatch?(old_value, new_value) + raise 'Type mismatch. Cannot merge %s (%s) with %s (%s). Key is "%s", name is "%s".' % [ + old_value.inspect, old_value.class, + new_value.inspect, new_value.class, + key, self.class + ] + + # merge simple strings & numbers + else + if prefer_self + value = old_value + else + value = new_value + end + end + + # save value + self[key] = value + end + self + end + + def set_environment(env, node) + @env = env + @node = node + self.each do |key, value| + if value.is_a?(Config::Object) + value.set_environment(env, node) + end + end + end + + # + # like a reverse deep merge + # (self takes precedence) + # + def inherit_from!(object) + self.deep_merge!(object, true) + end + + # + # Make a copy of ourselves, except only including the specified keys. + # + # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b' + # + def pick(*keys) + keys.map(&:to_s).inject(self.class.new(@manager)) do |hsh, key| + value = self.get(key) + if !value.nil? + hsh[key.gsub('.','_')] = value + end + hsh + end + end + + def eval_file(filename) + evaluate_ruby(filename, File.read(filename)) + end + + protected + + # + # walks the object tree, eval'ing all the attributes that are dynamic ruby (e.g. value starts with '= ') + # + def evaluate_everything(context) + keys.each do |key| + obj = fetch_value(key, context) + if is_required_value_not_set?(obj) + Util::log 0, :warning, "required property \"#{key}\" is not set in node \"#{node.name}\"." + elsif obj.is_a? Config::Object + obj.evaluate_everything(context) + end + end + end + + # + # some keys need to be evaluated 'late', after all the other keys have been evaluated. + # + def late_evaluate_everything(context) + if @late_eval_list + @late_eval_list.each do |key, value| + self[key] = context.evaluate_ruby(key, value) + if is_required_value_not_set?(self[key]) + Util::log 0, :warning, "required property \"#{key}\" is not set in node \"#{node.name}\"." + end + end + end + values.each do |obj| + if obj.is_a? Config::Object + obj.late_evaluate_everything(context) + end + end + end + + # + # evaluates the string `value` as ruby in the context of self. + # (`key` is just passed for debugging purposes) + # + def evaluate_ruby(key, value) + self.instance_eval(value, key, 1) + rescue ConfigError => exc + raise exc # pass through + rescue SystemStackError => exc + Util::log 0, :error, "while evaluating node '#{self.name}'" + Util::log 0, "offending key: #{key}", :indent => 1 + Util::log 0, "offending string: #{value}", :indent => 1 + Util::log 0, "STACK OVERFLOW, BAILING OUT. There must be an eval loop of death (variables with circular dependencies).", :indent => 1 + raise SystemExit.new(1) + rescue FileMissing => exc + Util::bail! do + if exc.options[:missing] + Util::log :missing, exc.options[:missing].gsub('$node', self.name).gsub('$file', exc.path) + else + Util::log :error, "while evaluating node '#{self.name}'" + Util::log "offending key: #{key}", :indent => 1 + Util::log "offending string: #{value}", :indent => 1 + Util::log "error message: no file '#{exc}'", :indent => 1 + end + raise exc if DEBUG + end + rescue AssertionFailed => exc + Util.bail! do + Util::log :failed, "assertion while evaluating node '#{self.name}'" + Util::log 'assertion: %s' % exc.assertion, :indent => 1 + Util::log "offending key: #{key}", :indent => 1 + raise exc if DEBUG + end + rescue SyntaxError, StandardError => exc + Util::bail! do + Util::log :error, "while evaluating node '#{self.name}'" + Util::log "offending key: #{key}", :indent => 1 + Util::log "offending string: #{value}", :indent => 1 + Util::log "error message: #{exc.inspect}", :indent => 1 + raise exc if DEBUG + end + end + + private + + # + # fetches the value for the key, evaluating the value as ruby if it begins with '=' + # + def fetch_value(key, context=@node) + value = fetch(key, nil) + if value.is_a?(String) && value =~ /^=/ + if value =~ /^=> (.*)$/ + value = evaluate_later(key, $1) + elsif value =~ /^= (.*)$/ + value = context.evaluate_ruby(key, $1) + end + self[key] = value + end + return value + end + + def evaluate_later(key, value) + @late_eval_list ||= [] + @late_eval_list << [key, value] + '<evaluate later>' + end + + # + # when merging, we raise an error if this method returns true for the two values. + # + def type_mismatch?(old_value, new_value) + if old_value.is_a?(Boolean) && new_value.is_a?(Boolean) + # note: FalseClass and TrueClass are different classes + # so we can't do old_value.class == new_value.class + return false + elsif old_value.is_a?(String) && old_value =~ /^=/ + # pass through macros, since we don't know what the type will eventually be. + return false + elsif new_value.is_a?(String) && new_value =~ /^=/ + return false + elsif old_value.class == new_value.class + return false + else + return true + end + end + + # + # returns true if the value has not been changed and the default is "REQUIRED" + # + def is_required_value_not_set?(value) + if value.is_a? Array + value == ["REQUIRED"] + else + value == "REQUIRED" + end + end + + end # class + end # module +end # module
\ No newline at end of file diff --git a/lib/leap_cli/config/object_list.rb b/lib/leap_cli/config/object_list.rb new file mode 100644 index 00000000..f9299a61 --- /dev/null +++ b/lib/leap_cli/config/object_list.rb @@ -0,0 +1,209 @@ +require 'tsort' + +module LeapCli + module Config + # + # A list of Config::Object instances (internally stored as a hash) + # + class ObjectList < Hash + include TSort + + def initialize(config=nil) + if config + self.add(config['name'], config) + end + end + + # + # If the key is a string, the Config::Object it references is returned. + # + # If the key is a hash, we treat it as a condition and filter all the Config::Objects using the condition. + # A new ObjectList is returned. + # + # Examples: + # + # nodes['vpn1'] + # node named 'vpn1' + # + # nodes[:public_dns => true] + # all nodes with public dns + # + # nodes[:services => 'openvpn', 'location.country_code' => 'US'] + # all nodes with services containing 'openvpn' OR country code of US + # + # Sometimes, you want to do an OR condition with multiple conditions + # for the same field. Since hash keys must be unique, you can use + # an array representation instead: + # + # nodes[[:services, 'openvpn'], [:services, 'tor']] + # nodes with openvpn OR tor service + # + # nodes[:services => 'openvpn'][:tags => 'production'] + # nodes with openvpn AND are production + # + def [](key) + if key.is_a?(Hash) || key.is_a?(Array) + filter(key) + else + super key.to_s + end + end + + def exclude(node) + list = self.dup + list.delete(node.name) + return list + end + + def each_node(&block) + self.keys.sort.each do |node_name| + yield self[node_name] + end + end + + # + # filters this object list, producing a new list. + # filter is an array or a hash. see [] + # + def filter(filter) + results = Config::ObjectList.new + filter.each do |field, match_value| + field = field.is_a?(Symbol) ? field.to_s : field + match_value = match_value.is_a?(Symbol) ? match_value.to_s : match_value + if match_value.is_a?(String) && match_value =~ /^!/ + operator = :not_equal + match_value = match_value.sub(/^!/, '') + else + operator = :equal + end + each do |name, config| + value = config[field] + if value.is_a? Array + if operator == :equal && value.include?(match_value) + results[name] = config + elsif operator == :not_equal && !value.include?(match_value) + results[name] = config + end + else + if operator == :equal && value == match_value + results[name] = config + elsif operator == :not_equal && value != match_value + results[name] = config + end + end + end + end + results + end + + def add(name, object) + self[name] = object + end + + # + # converts the hash of configs into an array of hashes, with ONLY the specified fields + # + def fields(*fields) + result = [] + keys.sort.each do |name| + result << self[name].pick(*fields) + end + result + end + + # + # like fields(), but returns an array of values instead of an array of hashes. + # + def field(field) + field = field.to_s + result = [] + keys.sort.each do |name| + result << self[name].get(field) + end + result + end + + # + # pick_fields(field1, field2, ...) + # + # generates a Hash from the object list, but with only the fields that are picked. + # + # If there are more than one field, then the result is a Hash of Hashes. + # If there is just one field, it is a simple map to the value. + # + # For example: + # + # "neighbors" = "= nodes_like_me[:services => :couchdb].pick_fields('domain.full', 'ip_address')" + # + # generates this: + # + # neighbors: + # couch1: + # domain_full: couch1.bitmask.net + # ip_address: "10.5.5.44" + # couch2: + # domain_full: couch2.bitmask.net + # ip_address: "10.5.5.52" + # + # But this: + # + # "neighbors": "= nodes_like_me[:services => :couchdb].pick_fields('domain.full')" + # + # will generate this: + # + # neighbors: + # couch1: couch1.bitmask.net + # couch2: couch2.bitmask.net + # + def pick_fields(*fields) + self.values.inject({}) do |hsh, node| + value = self[node.name].pick(*fields) + if fields.size == 1 + value = value.values.first + end + hsh[node.name] = value + hsh + end + end + + # + # Applies inherit_from! to all objects. + # + # 'env' specifies what environment should be for + # each object in the list. + # + def inherit_from!(object_list, env) + object_list.each do |name, object| + if self[name] + self[name].inherit_from!(object) + else + self[name] = object.duplicate(env) + end + end + end + + # + # topographical sort based on test dependency + # + def tsort_each_node(&block) + self.each_key(&block) + end + + def tsort_each_child(node_name, &block) + if self[node_name] + self[node_name].test_dependencies.each do |test_me_first| + if self[test_me_first] # TODO: in the future, allow for ability to optionally pull in all dependencies. + # not just the ones that pass the node filter. + yield(test_me_first) + end + end + end + end + + def names_in_test_dependency_order + self.tsort + end + + end + end +end diff --git a/lib/leap_cli/config/provider.rb b/lib/leap_cli/config/provider.rb new file mode 100644 index 00000000..0d8bc1f3 --- /dev/null +++ b/lib/leap_cli/config/provider.rb @@ -0,0 +1,22 @@ +# +# Configuration class for provider.json +# + +module LeapCli; module Config + class Provider < Object + attr_reader :environment + def set_env(e) + if e == 'default' + @environment = nil + else + @environment = e + end + end + def provider + self + end + def validate! + # nothing here yet :( + end + end +end; end diff --git a/lib/leap_cli/config/secrets.rb b/lib/leap_cli/config/secrets.rb new file mode 100644 index 00000000..ca851c74 --- /dev/null +++ b/lib/leap_cli/config/secrets.rb @@ -0,0 +1,87 @@ +# encoding: utf-8 +# +# A class for the secrets.json file +# + +module LeapCli; module Config + + class Secrets < Object + attr_reader :node_list + + def initialize(manager=nil) + super(manager) + @discovered_keys = {} + end + + # we can't use fetch() or get(), since those already have special meanings + def retrieve(key, environment) + environment ||= 'default' + self.fetch(environment, {})[key.to_s] + end + + def set(*args, &block) + if block_given? + set_with_block(*args, &block) + else + set_without_block(*args) + end + end + + # searches over all keys matching the regexp, checking to see if the value + # has been already used by any of them. + def taken?(regexp, value, environment) + self.keys.grep(regexp).each do |key| + return true if self.retrieve(key, environment) == value + end + return false + end + + def set_without_block(key, value, environment) + set_with_block(key, environment) {value} + end + + def set_with_block(key, environment, &block) + environment ||= 'default' + key = key.to_s + @discovered_keys[environment] ||= {} + @discovered_keys[environment][key] = true + self[environment] ||= {} + self[environment][key] ||= yield + end + + # + # if clean is true, then only secrets that have been discovered + # during this run will be exported. + # + # if environment is also pinned, then we will clean those secrets + # just for that environment. + # + # the clean argument should only be used when all nodes have + # been processed, otherwise secrets that are actually in use will + # get mistakenly removed. + # + def dump_json(clean=false) + pinned_env = LeapCli.leapfile.environment + if clean + self.each_key do |environment| + if pinned_env.nil? || pinned_env == environment + env = self[environment] + if env.nil? + raise StandardError.new("secrets.json file seems corrupted. No such environment '#{environment}'") + end + env.each_key do |key| + unless @discovered_keys[environment] && @discovered_keys[environment][key] + self[environment].delete(key) + end + end + if self[environment].empty? + self.delete(environment) + end + end + end + end + super() + end + end + +end; end diff --git a/lib/leap_cli/config/sources.rb b/lib/leap_cli/config/sources.rb new file mode 100644 index 00000000..aee860de --- /dev/null +++ b/lib/leap_cli/config/sources.rb @@ -0,0 +1,11 @@ +# encoding: utf-8 +# +# A class for the sources.json file +# + +module LeapCli + module Config + class Sources < Object + end + end +end diff --git a/lib/leap_cli/config/tag.rb b/lib/leap_cli/config/tag.rb new file mode 100644 index 00000000..6bd8d1e9 --- /dev/null +++ b/lib/leap_cli/config/tag.rb @@ -0,0 +1,25 @@ +# +# +# A class for node services or node tags. +# +# + +module LeapCli; module Config + + class Tag < Object + attr_reader :node_list + + def initialize(environment=nil) + super(environment) + @node_list = Config::ObjectList.new + end + + # don't copy the node list pointer when this object is dup'ed. + def initialize_copy(orig) + super + @node_list = Config::ObjectList.new + end + + end + +end; end diff --git a/lib/leap_cli/leapfile_extensions.rb b/lib/leap_cli/leapfile_extensions.rb new file mode 100644 index 00000000..cba321f4 --- /dev/null +++ b/lib/leap_cli/leapfile_extensions.rb @@ -0,0 +1,24 @@ +module LeapCli + class Leapfile + attr_reader :custom_vagrant_vm_line + attr_reader :leap_version + attr_reader :log + attr_reader :vagrant_basebox + + def vagrant_network + @vagrant_network ||= '10.5.5.0/24' + end + + private + + PRIVATE_IP_RANGES = /(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)/ + + def validate + Util::assert! vagrant_network =~ PRIVATE_IP_RANGES do + Util::log 0, :error, "in #{file}: vagrant_network is not a local private network" + end + return true + end + + end +end diff --git a/lib/leap_cli/load_libraries.rb b/lib/leap_cli/load_libraries.rb new file mode 100644 index 00000000..19f4edb5 --- /dev/null +++ b/lib/leap_cli/load_libraries.rb @@ -0,0 +1,20 @@ +# +# load the commonly needed leap_cli libraries that live in the platform. +# +# loaded by leap_cli's bootstrap.rb +# + +require 'leap_cli/log_filter' + +require 'leap_cli/config/object' +require 'leap_cli/config/node' +require 'leap_cli/config/tag' +require 'leap_cli/config/provider' +require 'leap_cli/config/secrets' +require 'leap_cli/config/object_list' +require 'leap_cli/config/filter' +require 'leap_cli/config/environment' +require 'leap_cli/config/manager' + +require 'leap_cli/util/secret' +require 'leap_cli/util/x509' diff --git a/lib/leap_cli/log_filter.rb b/lib/leap_cli/log_filter.rb new file mode 100644 index 00000000..0d745cc2 --- /dev/null +++ b/lib/leap_cli/log_filter.rb @@ -0,0 +1,171 @@ +# +# A module to hide, modify, and colorize log entries. +# + +module LeapCli + module LogFilter + # + # options for formatters: + # + # :match => regexp for matching a log line + # :color => what color the line should be + # :style => what style the line should be + # :priority => what order the formatters are applied in. higher numbers first. + # :match_level => only apply filter at the specified log level + # :level => make this line visible at this log level or higher + # :replace => replace the matched text + # :prepend => insert text at start of message + # :append => append text to end of message + # :exit => force the exit code to be this (does not interrupt program, just + # ensures a specific exit code when the program eventually exits) + # + FORMATTERS = [ + # TRACE + { :match => /command finished/, :color => :white, :style => :dim, :match_level => 3, :priority => -10 }, + { :match => /executing locally/, :color => :yellow, :match_level => 3, :priority => -20 }, + + # DEBUG + #{ :match => /executing .*/, :color => :green, :match_level => 2, :priority => -10, :timestamp => true }, + #{ :match => /.*/, :color => :yellow, :match_level => 2, :priority => -30 }, + { :match => /^transaction:/, :level => 3 }, + + # INFO + { :match => /.*out\] (fatal:|ERROR:).*/, :color => :red, :match_level => 1, :priority => -10 }, + { :match => /Permission denied/, :color => :red, :match_level => 1, :priority => -20 }, + { :match => /sh: .+: command not found/, :color => :magenta, :match_level => 1, :priority => -30 }, + + # IMPORTANT + { :match => /^(E|e)rr ::/, :color => :red, :match_level => 0, :priority => -10, :exit => 1}, + { :match => /^ERROR:/, :color => :red, :priority => -10, :exit => 1}, + #{ :match => /.*/, :color => :blue, :match_level => 0, :priority => -20 }, + + # CLEANUP + #{ :match => /\s+$/, :replace => '', :priority => 0}, + + # DEBIAN PACKAGES + { :match => /^(Hit|Ign) /, :color => :green, :priority => -20}, + { :match => /^Err /, :color => :red, :priority => -20}, + { :match => /^W(ARNING)?: /, :color => :yellow, :priority => -20}, + { :match => /^E: /, :color => :red, :priority => -20}, + { :match => /already the newest version/, :color => :green, :priority => -20}, + { :match => /WARNING: The following packages cannot be authenticated!/, :color => :red, :level => 0, :priority => -10}, + + # PUPPET + { :match => /^(W|w)arning: Not collecting exported resources without storeconfigs/, :level => 2, :color => :yellow, :priority => -10}, + { :match => /^(W|w)arning: Found multiple default providers for vcsrepo:/, :level => 2, :color => :yellow, :priority => -10}, + { :match => /^(W|w)arning: .*is deprecated.*$/, :level => 2, :color => :yellow, :priority => -10}, + { :match => /^(W|w)arning: Scope.*$/, :level => 2, :color => :yellow, :priority => -10}, + #{ :match => /^(N|n)otice:/, :level => 1, :color => :cyan, :priority => -20}, + #{ :match => /^(N|n)otice:.*executed successfully$/, :level => 2, :color => :cyan, :priority => -15}, + { :match => /^(W|w)arning:/, :level => 0, :color => :yellow, :priority => -20}, + { :match => /^Duplicate declaration:/, :level => 0, :color => :red, :priority => -20}, + #{ :match => /Finished catalog run/, :level => 0, :color => :green, :priority => -10}, + { :match => /^APPLY COMPLETE \(changes made\)/, :level => 0, :color => :green, :style => :bold, :priority => -10}, + { :match => /^APPLY COMPLETE \(no changes\)/, :level => 0, :color => :green, :style => :bold, :priority => -10}, + + # PUPPET FATAL ERRORS + { :match => /^(E|e)rr(or|):/, :level => 0, :color => :red, :priority => -1, :exit => 1}, + { :match => /^Wrapped exception:/, :level => 0, :color => :red, :priority => -1, :exit => 1}, + { :match => /^Failed to parse template/, :level => 0, :color => :red, :priority => -1, :exit => 1}, + { :match => /^Execution of.*returned/, :level => 0, :color => :red, :priority => -1, :exit => 1}, + { :match => /^Parameter matches failed:/, :level => 0, :color => :red, :priority => -1, :exit => 1}, + { :match => /^Syntax error/, :level => 0, :color => :red, :priority => -1, :exit => 1}, + { :match => /^Cannot reassign variable/, :level => 0, :color => :red, :priority => -1, :exit => 1}, + { :match => /^Could not find template/, :level => 0, :color => :red, :priority => -1, :exit => 1}, + { :match => /^APPLY COMPLETE.*fail/, :level => 0, :color => :red, :style => :bold, :priority => -1, :exit => 1}, + + # TESTS + { :match => /^PASS: /, :color => :green, :priority => -20}, + { :match => /^(FAIL|ERROR): /, :color => :red, :priority => -20}, + { :match => /^(SKIP|WARN): /, :color => :yellow, :priority => -20}, + { :match => /\d+ tests: \d+ passes, \d+ skips, 0 warnings, 0 failures, 0 errors/, + :color => :green, :style => :bold, :priority => -20 }, + { :match => /\d+ tests: \d+ passes, \d+ skips, [1-9][0-9]* warnings, 0 failures, 0 errors/, + :color => :yellow, :style => :bold, :priority => -20 }, + { :match => /\d+ tests: \d+ passes, \d+ skips, \d+ warnings, \d+ failures, [1-9][0-9]* errors/, + :color => :red, :style => :bold, :priority => -20 }, + { :match => /\d+ tests: \d+ passes, \d+ skips, \d+ warnings, [1-9][0-9]* failures, \d+ errors/, + :color => :red, :style => :bold, :priority => -20 }, + + # LOG SUPPRESSION + { :match => /^(W|w)arning: You cannot collect without storeconfigs being set/, :level => 2, :priority => 10}, + { :match => /^(W|w)arning: You cannot collect exported resources without storeconfigs being set/, :level => 2, :priority => 10} + ] + + SORTED_FORMATTERS = FORMATTERS.sort_by { |i| -(i[:priority] || i[:prio] || 0) } + + # + # same as normal formatters, but only applies to the title, not the message. + # + TITLE_FORMATTERS = [ + # red + { :match => /error/, :color => :red, :style => :bold }, + { :match => /fatal_error/, :replace => 'fatal error:', :color => :red, :style => :bold }, + { :match => /removed/, :color => :red, :style => :bold }, + { :match => /failed/, :replace => 'FAILED', :color => :red, :style => :bold }, + { :match => /bail/, :replace => 'bailing out', :color => :red, :style => :bold }, + { :match => /invalid/, :color => :red, :style => :bold }, + + # yellow + { :match => /warning/, :replace => 'warning:', :color => :yellow, :style => :bold }, + { :match => /missing/, :color => :yellow, :style => :bold }, + { :match => /skipping/, :color => :yellow, :style => :bold }, + + # green + { :match => /created/, :color => :green, :style => :bold }, + { :match => /completed/, :color => :green, :style => :bold }, + { :match => /ran/, :color => :green, :style => :bold }, + + # cyan + { :match => /note/, :replace => 'NOTE:', :color => :cyan, :style => :bold }, + + # magenta + { :match => /nochange/, :replace => 'no change', :color => :magenta }, + { :match => /loading/, :color => :magenta }, + ] + + def self.apply_message_filters(message) + return self.apply_filters(SORTED_FORMATTERS, message) + end + + def self.apply_title_filters(title) + return self.apply_filters(TITLE_FORMATTERS, title) + end + + private + + def self.apply_filters(formatters, message) + level = LeapCli.logger.log_level + result = {} + formatters.each do |formatter| + if (formatter[:match_level] == level || formatter[:match_level].nil?) + if message =~ formatter[:match] + # puts "applying formatter #{formatter.inspect}" + result[:level] = formatter[:level] if formatter[:level] + result[:color] = formatter[:color] if formatter[:color] + result[:style] = formatter[:style] || formatter[:attribute] # (support original cap colors) + + message.gsub!(formatter[:match], formatter[:replace]) if formatter[:replace] + message.replace(formatter[:prepend] + message) unless formatter[:prepend].nil? + message.replace(message + formatter[:append]) unless formatter[:append].nil? + message.replace(Time.now.strftime('%Y-%m-%d %T') + ' ' + message) if formatter[:timestamp] + + if formatter[:exit] + LeapCli::Util.exit_status(formatter[:exit]) + end + + # stop formatting, unless formatter was just for string replacement + break unless formatter[:replace] + end + end + end + + if result[:color] == :hide + return [nil, {}] + else + return [message, result] + end + end + + end +end diff --git a/lib/leap_cli/macros.rb b/lib/leap_cli/macros.rb deleted file mode 100644 index fdb9a94e..00000000 --- a/lib/leap_cli/macros.rb +++ /dev/null @@ -1,16 +0,0 @@ -# -# MACROS -# -# The methods in these files are available in the context of a .json configuration file. -# (The module LeapCli::Macro is included in Config::Object) -# - -require_relative 'macros/core' -require_relative 'macros/files' -require_relative 'macros/haproxy' -require_relative 'macros/hosts' -require_relative 'macros/keys' -require_relative 'macros/nodes' -require_relative 'macros/secrets' -require_relative 'macros/stunnel' -require_relative 'macros/provider' diff --git a/lib/leap_cli/ssh/backend.rb b/lib/leap_cli/ssh/backend.rb index f42379cc..67c6ec9a 100644 --- a/lib/leap_cli/ssh/backend.rb +++ b/lib/leap_cli/ssh/backend.rb @@ -94,7 +94,7 @@ module LeapCli # some prewritten servers-side scripts def scripts - @scripts ||= LeapCli::SSH::Scripts.new(self, @host) + @scripts ||= LeapCli::SSH::Scripts.new(self, @host.hostname) end private @@ -139,6 +139,9 @@ module LeapCli @logger.log(:failed, args.join(' '), host: @host.hostname) do @logger.log("Connection timed out") end + if @options[:raise_error] + raise LeapCli::SSH::TimeoutError, exc.to_s + end else raise end diff --git a/lib/leap_cli/ssh/remote_command.rb b/lib/leap_cli/ssh/remote_command.rb index fe9a344a..3ba86740 100644 --- a/lib/leap_cli/ssh/remote_command.rb +++ b/lib/leap_cli/ssh/remote_command.rb @@ -21,6 +21,9 @@ module LeapCli class ExecuteError < StandardError end + class TimeoutError < ExecuteError + end + # override default runner mode class CustomCoordinator < SSHKit::Coordinator private diff --git a/lib/leap_cli/ssh/scripts.rb b/lib/leap_cli/ssh/scripts.rb index 3d8b6570..feefdd46 100644 --- a/lib/leap_cli/ssh/scripts.rb +++ b/lib/leap_cli/ssh/scripts.rb @@ -15,7 +15,7 @@ module LeapCli REQUIRED_PACKAGES = "puppet rsync lsb-release locales" attr_reader :ssh, :host - def initialize(backend, host) + def initialize(backend, hostname) @ssh = backend @host = host end @@ -48,6 +48,8 @@ module LeapCli def check_for_no_deploy begin ssh.stream "test ! -f /etc/leap/no-deploy", :raise_error => true, :log_output => false + rescue SSH::TimeoutError + raise rescue SSH::ExecuteError ssh.log :warning, "can't continue because file /etc/leap/no-deploy exists", :host => host raise # will skip further action on this node @@ -59,7 +61,7 @@ module LeapCli # def debug output = ssh.capture "#{Leap::Platform.leap_dir}/bin/debug.sh" - ssh.log(output, :wrap => true, :host => host.hostname, :color => :cyan) + ssh.log(output, :wrap => true, :host => host, :color => :cyan) end # @@ -69,7 +71,7 @@ module LeapCli cmd = "(test -s /var/log/leap/deploy-summary.log && tail -n #{lines} /var/log/leap/deploy-summary.log) || (test -s /var/log/leap/deploy-summary.log.1 && tail -n #{lines} /var/log/leap/deploy-summary.log.1) || (echo 'no history')" history = ssh.capture(cmd, :log_output => false) if history - ssh.log host.hostname, :color => :cyan, :style => :bold do + ssh.log host, :color => :cyan, :style => :bold do ssh.log history, :wrap => true end end diff --git a/lib/leap_cli/util/secret.rb b/lib/leap_cli/util/secret.rb new file mode 100644 index 00000000..749b9595 --- /dev/null +++ b/lib/leap_cli/util/secret.rb @@ -0,0 +1,55 @@ +# encoding: utf-8 +# +# A simple secret generator +# +# Uses OpenSSL random number generator instead of Ruby's rand function +# +autoload :OpenSSL, 'openssl' + +module LeapCli; module Util + class Secret + CHARS = (('A'..'Z').to_a + ('a'..'z').to_a + ('0'..'9').to_a) - "i1loO06G".split(//u) + HEX = (0..9).to_a + ('a'..'f').to_a + + # + # generate a secret with with no ambiguous characters. + # + # +length+ is in chars + # + # Only alphanumerics are allowed, in order to make these passwords work + # for REST url calls and to allow you to easily copy and paste them. + # + def self.generate(length = 16) + seed + OpenSSL::Random.random_bytes(length).bytes.to_a.collect { |byte| + CHARS[ byte % CHARS.length ] + }.join + end + + # + # generates a hex secret, instead of an alphanumeric on. + # + # length is in bits + # + def self.generate_hex(length = 128) + seed + OpenSSL::Random.random_bytes(length/4).bytes.to_a.collect { |byte| + HEX[ byte % HEX.length ] + }.join + end + + private + + def self.seed + @pid ||= 0 + pid = $$ + if @pid != pid + now = Time.now + ary = [now.to_i, now.nsec, @pid, pid] + OpenSSL::Random.seed(ary.to_s) + @pid = pid + end + end + + end +end; end diff --git a/lib/leap_cli/util/x509.rb b/lib/leap_cli/util/x509.rb new file mode 100644 index 00000000..787fdfac --- /dev/null +++ b/lib/leap_cli/util/x509.rb @@ -0,0 +1,33 @@ +autoload :OpenSSL, 'openssl' +autoload :CertificateAuthority, 'certificate_authority' + +require 'digest' +require 'digest/md5' +require 'digest/sha1' + +module LeapCli; module X509 + extend self + + # + # returns a fingerprint of a x509 certificate + # + def fingerprint(digest, cert_file) + if cert_file.is_a? String + cert = OpenSSL::X509::Certificate.new(Util.read_file!(cert_file)) + elsif cert_file.is_a? OpenSSL::X509::Certificate + cert = cert_file + elsif cert_file.is_a? CertificateAuthority::Certificate + cert = cert_file.openssl_body + end + digester = case digest + when "MD5" then Digest::MD5.new + when "SHA1" then Digest::SHA1.new + when "SHA256" then Digest::SHA256.new + when "SHA384" then Digest::SHA384.new + when "SHA512" then Digest::SHA512.new + end + digester.hexdigest(cert.to_der) + end + + +end; end |