diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/leap_cli.rb | 1 | ||||
| -rw-r--r-- | lib/leap_cli/config/environment.rb | 166 | ||||
| -rw-r--r-- | lib/leap_cli/config/manager.rb | 284 | ||||
| -rw-r--r-- | lib/leap_cli/config/node.rb | 4 | ||||
| -rw-r--r-- | lib/leap_cli/config/object.rb | 66 | ||||
| -rw-r--r-- | lib/leap_cli/config/object_list.rb | 9 | ||||
| -rw-r--r-- | lib/leap_cli/config/provider.rb | 3 | ||||
| -rw-r--r-- | lib/leap_cli/config/tag.rb | 4 |
8 files changed, 293 insertions, 244 deletions
diff --git a/lib/leap_cli.rb b/lib/leap_cli.rb index 9052f8c..0563327 100644 --- a/lib/leap_cli.rb +++ b/lib/leap_cli.rb @@ -46,6 +46,7 @@ require 'leap_cli/config/provider' require 'leap_cli/config/secrets' require 'leap_cli/config/object_list' require 'leap_cli/config/filter' +require 'leap_cli/config/environment' require 'leap_cli/config/manager' require 'leap_cli/markdown_document_listener' diff --git a/lib/leap_cli/config/environment.rb b/lib/leap_cli/config/environment.rb new file mode 100644 index 0000000..d8f34ea --- /dev/null +++ b/lib/leap_cli/config/environment.rb @@ -0,0 +1,166 @@ +# +# All configurations files can be isolated into separate environments. +# +# Each config json in each environment inherits from the default environment, +# which in term inherits from the "_base_" environment: +# +# _base_ -- base provider in leap_platform +# '- default -- environment in provider dir when no env is set +# '- production -- example environment +# + +module LeapCli; module Config + + class Environment + # the String name of the environment + attr_accessor :name + + # the shared Manager object + attr_accessor :manager + + # hashes of {name => Config::Object} + attr_accessor :services, :tags, :partials + + # a Config::Provider + attr_accessor :provider + + # a Config::Object + attr_accessor :common + + # shared, non-inheritable + def nodes; @@nodes; end + def secrets; @@secrets; end + + def initialize(manager, name, search_dir, parent, options={}) + @manager = manager + @name = name + + load_provider_files(search_dir, options) + + if parent + @services.inherit_from! parent.services, self + @tags.inherit_from! parent.tags , self + @partials.inherit_from! parent.partials, self + @common.inherit_from! parent.common + @provider.inherit_from! parent.provider + end + + if @provider + @provider.set_env(name) + @provider.validate! + end + end + + def load_provider_files(search_dir, options) + # + # load empty environment if search_dir doesn't exist + # + if search_dir.nil? || !Dir.exist?(search_dir) + @services = Config::ObjectList.new + @tags = Config::ObjectList.new + @partials = Config::ObjectList.new + @provider = Config::Provider.new + @common = Config::Object.new + return + end + + # + # inheritable + # + if options[:scope] + scope = options[:scope] + @services = load_all_json(Path.named_path([:service_env_config, '*', scope], search_dir), Config::Tag, options) + @tags = load_all_json(Path.named_path([:tag_env_config, '*', scope], search_dir), Config::Tag, options) + @partials = load_all_json(Path.named_path([:service_env_config, '_*', scope], search_dir), Config::Tag, options) + @provider = load_json( Path.named_path([:provider_env_config, scope], search_dir), Config::Provider, options) + @common = load_json( Path.named_path([:common_env_config, scope], search_dir), Config::Object, options) + else + @services = load_all_json(Path.named_path([:service_config, '*'], search_dir), Config::Tag, options) + @tags = load_all_json(Path.named_path([:tag_config, '*'], search_dir), Config::Tag, options) + @partials = load_all_json(Path.named_path([:service_config, '_*'], search_dir), Config::Tag, options) + @provider = load_json( Path.named_path(:provider_config, search_dir), Config::Provider, options) + @common = load_json( Path.named_path(:common_config, search_dir), Config::Object, options) + end + + # remove 'name' from partials, since partials get merged with nodes + @partials.values.each {|partial| partial.delete('name'); } + + # + # shared: currently non-inheritable + # load the first ones we find, and only those. + # + @@nodes ||= begin + nodes = load_all_json(Path.named_path([:node_config, '*'], search_dir), Config::Node, options) + nodes.any? ? nodes : nil + end + @@secrets ||= begin + secrets = load_json(Path.named_path(:secrets_config, search_dir), Config::Secrets, options) + secrets.any? ? secrets : nil + end + end + + private + + def load_all_json(pattern, object_class, options={}) + results = Config::ObjectList.new + Dir.glob(pattern).each do |filename| + next if options[:no_dots] && File.basename(filename) !~ /^[^\.]*\.json$/ + obj = load_json(filename, object_class) + if obj + name = File.basename(filename).force_encoding('utf-8').sub(/^([^\.]+).*\.json$/,'\1') + obj['name'] ||= name + if options[:env] + obj.environment = options[:env] + end + results[name] = obj + end + end + results + end + + def load_json(filename, object_class, options={}) + if !File.exists?(filename) + return object_class.new(self) + end + + Util::log :loading, filename, 3 + + # + # Read a JSON file, strip out comments. + # + # UTF8 is the default encoding for JSON, but others are allowed: + # https://www.ietf.org/rfc/rfc4627.txt + # + buffer = StringIO.new + File.open(filename, "rb", :encoding => 'UTF-8') do |f| + while (line = f.gets) + next if line =~ /^\s*\/\// + buffer << line + end + end + + # + # force UTF-8 + # + if $ruby_version >= [1,9] + string = buffer.string.force_encoding('utf-8') + else + string = Iconv.conv("UTF-8//IGNORE", "UTF-8", buffer.string) + end + + # parse json + begin + hash = JSON.parse(string, :object_class => Hash, :array_class => Array) || {} + rescue SyntaxError, JSON::ParserError => exc + Util::log 0, :error, 'in file "%s":' % filename + Util::log 0, exc.to_s, :indent => 1 + return nil + end + object = object_class.new(self) + object.deep_merge!(hash) + return object + end + + end # end Environment + +end; end
\ No newline at end of file diff --git a/lib/leap_cli/config/manager.rb b/lib/leap_cli/config/manager.rb index ca739c3..4ad8b71 100644 --- a/lib/leap_cli/config/manager.rb +++ b/lib/leap_cli/config/manager.rb @@ -9,10 +9,6 @@ end module LeapCli module Config - class Environment - attr_accessor :services, :tags, :provider - end - # # A class to manage all the objects in all the configuration files. # @@ -27,9 +23,6 @@ module LeapCli ## ATTRIBUTES ## - attr_reader :nodes, :common, :secrets - attr_reader :base_services, :base_tags, :base_provider, :base_common - # # returns the Hash of the contents of facts.json # @@ -51,7 +44,7 @@ module LeapCli # def environment_names @environment_names ||= begin - [nil] + (env.tags.field('environment') + nodes.field('environment')).compact.uniq + [nil] + (env.tags.field('environment') + env.nodes.field('environment')).compact.uniq end end @@ -59,26 +52,25 @@ module LeapCli # Returns the appropriate environment variable # def env(env=nil) - env ||= 'default' - e = @environments[env] ||= Environment.new - yield e if block_given? - e + @environments[env || 'default'] end # - # The default accessors for services, tags, and provider. + # The default accessors + # # For these defaults, use 'default' environment, or whatever # environment is pinned. # - def services - env(default_environment).services - end - def tags - env(default_environment).tags - end - def provider - env(default_environment).provider - end + # I think it might be an error that these are ever used + # and I would like to get rid of them. + # + def services; env(default_environment).services; end + def tags; env(default_environment).tags; end + def partials; env(default_environment).partials; end + def provider; env(default_environment).provider; end + def common; env(default_environment).common; end + def secrets; env(default_environment).secrets; end + def nodes; env(default_environment).nodes; end def default_environment LeapCli.leapfile.environment @@ -88,6 +80,21 @@ module LeapCli ## IMPORT EXPORT ## + def add_environment(args) + if args[:inherit] + parent = @environments[args.delete(:inherit)] + else + parent = nil + end + @environments[args[:name]] = Environment.new( + self, + args.delete(:name), + args.delete(:dir), + parent, + args + ) + end + # # load .json configuration files # @@ -95,69 +102,37 @@ module LeapCli @provider_dir = Path.provider # load base - @base_services = load_all_json(Path.named_path([:service_config, '*'], Path.provider_base), Config::Tag) - @base_tags = load_all_json(Path.named_path([:tag_config, '*'], Path.provider_base), Config::Tag) - @base_common = load_json( Path.named_path(:common_config, Path.provider_base), Config::Object) - @base_provider = load_json( Path.named_path(:provider_config, Path.provider_base), Config::Provider) + add_environment(name: '_base_', dir: Path.provider_base) # load provider - @nodes = load_all_json(Path.named_path([:node_config, '*'], @provider_dir), Config::Node) - @common = load_json( Path.named_path(:common_config, @provider_dir), Config::Object) - @secrets = load_json( Path.named_path(:secrets_config, @provider_dir), Config::Secrets) - @common.inherit_from! @base_common - - # For the default environment, load provider services, tags, and provider.json - log 3, :loading, 'default environment...' - env('default') do |e| - e.services = load_all_json(Path.named_path([:service_config, '*'], @provider_dir), Config::Tag, :no_dots => true) - e.tags = load_all_json(Path.named_path([:tag_config, '*'], @provider_dir), Config::Tag, :no_dots => true) - e.provider = load_json( Path.named_path(:provider_config, @provider_dir), Config::Provider, :assert => true) - e.provider.set_env('default') - e.services.inherit_from! @base_services - e.tags.inherit_from! @base_tags - e.provider.inherit_from! @base_provider - validate_provider(e.provider) - end + Util::assert_files_exist!(Path.named_path(:provider_config, @provider_dir)) + add_environment(name: 'default', dir: @provider_dir, + inherit: '_base_', no_dots: true) - # create a special '_all_' environment, used for tracking the union - # of all the environments - env('_all_') do |e| - e.services = Config::ObjectList.new - e.tags = Config::ObjectList.new - e.provider = Config::Provider.new - e.services.inherit_from! env('default').services - e.tags.inherit_from! env('default').tags - e.provider.inherit_from! env('default').provider - e.provider.set_env('_all_') - end + # create a special '_all_' environment, used for tracking + # the union of all the environments + add_environment(name: '_all_', inherit: 'default') - # For each defined environment, load provider services, tags, and provider.json. + # load environments environment_names.each do |ename| - next unless ename - log 3, :loading, '%s environment...' % ename - env(ename) do |e| - e.services = load_all_json(Path.named_path([:service_env_config, '*', ename], @provider_dir), Config::Tag, :env => ename) - e.tags = load_all_json(Path.named_path([:tag_env_config, '*', ename], @provider_dir), Config::Tag, :env => ename) - e.provider = load_json( Path.named_path([:provider_env_config, ename], @provider_dir), Config::Provider, :env => ename) - e.services.inherit_from! env('default').services - e.tags.inherit_from! env('default').tags - e.provider.inherit_from! env('default').provider - e.provider.set_env(ename) - validate_provider(e.provider) + if ename + log 3, :loading, '%s environment...' % ename + add_environment(name: ename, dir: @provider_dir, + inherit: 'default', scope: ename) end end # apply inheritance - @nodes.each do |name, node| + env.nodes.each do |name, node| Util::assert! name =~ /^[0-9a-z-]+$/, "Illegal character(s) used in node name '#{name}'" - @nodes[name] = apply_inheritance(node) + env.nodes[name] = apply_inheritance(node) end # do some node-list post-processing cleanup_node_lists(options) # apply control files - @nodes.each do |name, node| + env.nodes.each do |name, node| control_files(node).each do |file| begin node.eval_file file @@ -185,7 +160,7 @@ module LeapCli existing_files = nil unless node_list - node_list = self.nodes + node_list = env.nodes existing_hiera = Dir.glob(Path.named_path([:hiera, '*'], @provider_dir)) existing_files = Dir.glob(Path.named_path([:node_files_dir, '*'], @provider_dir)) end @@ -220,8 +195,8 @@ module LeapCli end def export_secrets(clean_unused_secrets = false) - if @secrets.any? - Util.write_file!([:secrets_config, @provider_dir], @secrets.dump_json(clean_unused_secrets) + "\n") + if env.secrets.any? + Util.write_file!([:secrets_config, @provider_dir], env.secrets.dump_json(clean_unused_secrets) + "\n") end end @@ -266,7 +241,7 @@ module LeapCli # so, take the part before the first period as the node name name = name.split('.').first end - @nodes[name] + env.nodes[name] end # @@ -280,11 +255,11 @@ module LeapCli # yields each node, in sorted order # def each_node(&block) - nodes.each_node &block + env.nodes.each_node &block end def reload_node!(node) - @nodes[node.name] = apply_inheritance!(node) + env.nodes[node.name] = apply_inheritance!(node) end ## @@ -305,32 +280,6 @@ module LeapCli @connections ||= ConnectionList.new end - ## - ## PARTIALS - ## - - # - # returns all the partial data for the specified partial path. - # partial path is always relative to provider root, but there must be multiple files - # that match because provider root might be the base provider or the local provider. - # - def partials(partial_path) - @partials ||= {} - if @partials[partial_path].nil? - [Path.provider_base, Path.provider].each do |provider_dir| - path = File.join(provider_dir, partial_path) - if File.exists?(path) - @partials[partial_path] ||= [] - @partials[partial_path] << load_json(path, Config::Object) - end - end - if @partials[partial_path].nil? - raise RuntimeError, 'no such partial path `%s`' % partial_path, caller - end - end - @partials[partial_path] - end - # # Loads a json template file as a Hash (used only when creating a new node .json # file for the first time). @@ -344,108 +293,23 @@ module LeapCli end end - private - - def load_all_json(pattern, object_class, options={}) - results = Config::ObjectList.new - Dir.glob(pattern).each do |filename| - next if options[:no_dots] && File.basename(filename) !~ /^[^\.]*\.json$/ - obj = load_json(filename, object_class) - if obj - name = File.basename(filename).force_encoding('utf-8').sub(/^([^\.]+).*\.json$/,'\1') - obj['name'] ||= name - if options[:env] - obj.environment = options[:env] - end - results[name] = obj - end - end - results - end - - def load_json(filename, object_class, options={}) - if options[:assert] - Util::assert_files_exist!(filename) - end - if !File.exists?(filename) - return object_class.new(self) - end - - log :loading, filename, 3 - - # - # Read a JSON file, strip out comments. - # - # UTF8 is the default encoding for JSON, but others are allowed: - # https://www.ietf.org/rfc/rfc4627.txt - # - buffer = StringIO.new - File.open(filename, "rb", :encoding => 'UTF-8') do |f| - while (line = f.gets) - next if line =~ /^\s*\/\// - buffer << line - end - end - - # - # force UTF-8 - # - if $ruby_version >= [1,9] - string = buffer.string.force_encoding('utf-8') - else - string = Iconv.conv("UTF-8//IGNORE", "UTF-8", buffer.string) - end - - # parse json - begin - hash = JSON.parse(string, :object_class => Hash, :array_class => Array) || {} - rescue SyntaxError, JSON::ParserError => exc - log 0, :error, 'in file "%s":' % filename - log 0, exc.to_s, :indent => 1 - return nil - end - object = object_class.new(self) - object.deep_merge!(hash) - return object - end + ## + ## PRIVATE + ## - # - # remove all the nesting from a hash. - # - # def flatten_hash(input = {}, output = {}, options = {}) - # input.each do |key, value| - # key = options[:prefix].nil? ? "#{key}" : "#{options[:prefix]}#{options[:delimiter]||"_"}#{key}" - # if value.is_a? Hash - # flatten_hash(value, output, :prefix => key, :delimiter => options[:delimiter]) - # else - # output[key] = value - # end - # end - # output.replace(input) - # output - # end + private # # makes a node inherit options from appropriate the common, service, and tag json files. # def apply_inheritance(node, throw_exceptions=false) - new_node = Config::Node.new(self) - name = node.name - - # Guess the environment of the node from the tag names. - # (Technically, this is wrong: a tag that sets the environment might not be - # named the same as the environment. This code assumes that it is). - node_env = self.env - if node['tags'] - node['tags'].to_a.each do |tag| - if self.environment_names.include?(tag) - node_env = self.env(tag) - end - end - end + new_node = Config::Node.new(nil) + name = node.name + node_env = guess_node_env(node) + new_node.set_environment(node_env, new_node) # inherit from common - new_node.deep_merge!(@common) + new_node.deep_merge!(node_env.common) # inherit from services if node['services'] @@ -488,13 +352,35 @@ module LeapCli end # + # Guess the environment of the node from the tag names. + # + # Technically, this is wrong: a tag that sets the environment might not be + # named the same as the environment. This code assumes that it is. + # + # Unfortunately, it is a chicken and egg problem. We need to know the nodes + # likely environment in order to apply the inheritance that will actually + # determine the node's properties. + # + def guess_node_env(node) + environment = self.env(default_environment) + if node['tags'] + node['tags'].to_a.each do |tag| + if self.environment_names.include?(tag) + environment = self.env(tag) + end + end + end + return environment + end + + # # does some final clean at the end of loading nodes. # this includes removing disabled nodes, and populating # the services[x].node_list and tags[x].node_list # def cleanup_node_lists(options) @disabled_nodes = Config::ObjectList.new - @nodes.each do |name, node| + env.nodes.each do |name, node| if node.enabled || options[:include_disabled] if node['services'] node['services'].to_a.each do |node_service| @@ -510,16 +396,12 @@ module LeapCli end elsif !options[:include_disabled] log 2, :skipping, "disabled node #{name}." - @nodes.delete(name) + env.nodes.delete(name) @disabled_nodes[name] = node end end end - def validate_provider(provider) - # nothing yet. - end - # # returns a list of 'control' files for this node. # a control file is like a service or a tag JSON file, but it contains diff --git a/lib/leap_cli/config/node.rb b/lib/leap_cli/config/node.rb index fe685cf..65735d5 100644 --- a/lib/leap_cli/config/node.rb +++ b/lib/leap_cli/config/node.rb @@ -9,8 +9,8 @@ module LeapCli; module Config class Node < Object attr_accessor :file_paths - def initialize(manager=nil) - super(manager) + def initialize(environment=nil) + super(environment) @node = self @file_paths = [] end diff --git a/lib/leap_cli/config/object.rb b/lib/leap_cli/config/object.rb index bde5fe7..b117c2f 100644 --- a/lib/leap_cli/config/object.rb +++ b/lib/leap_cli/config/object.rb @@ -12,33 +12,6 @@ module LeapCli module Config # - # A proxy for Manager that binds to a particular object - # (so that we can bind to a particular environment) - # - class ManagerBinding - def initialize(manager, object) - @manager = manager - @object = object - end - - def services - @manager.env(@object.environment).services - end - - def tags - @manager.env(@object.environment).tags - end - - def provider - @manager.env(@object.environment).provider - end - - def method_missing(*args) - @manager.send(*args) - end - end - - # # This class represents the configuration for a single node, service, or tag. # Also, all the nested hashes are also of this type. # @@ -46,21 +19,27 @@ module LeapCli # class Object < Hash + attr_reader :env attr_reader :node - def initialize(manager=nil, node=nil) - # keep a global pointer around to the config manager. used a lot in the eval strings and templates - # (which are evaluated in the context of Config::Object) - @manager = manager - - # an object that is a node as @node equal to self, otherwise all the child objects point back to the top level node. + def initialize(environment=nil, node=nil) + raise ArgumentError unless environment.nil? || environment.is_a?(Config::Environment) + @env = environment + # an object that is a node as @node equal to self, otherwise all the + # child objects point back to the top level node. @node = node || self end def manager - ManagerBinding.new(@manager, self) + @env.manager + end + + # + # TODO: deprecate node.global() + # + def global + @env end - alias :global :manager def environment=(e) self.store('environment', e) @@ -70,6 +49,11 @@ module LeapCli self['environment'] end + def duplicate(env) + new_object = self.deep_dup + new_object.set_environment(env, new_object) + end + # # export YAML # @@ -218,7 +202,7 @@ module LeapCli # merge hashes elsif old_value.is_a?(Hash) || new_value.is_a?(Hash) - value = Config::Object.new(@manager, @node) + value = Config::Object.new(@env, @node) old_value.is_a?(Hash) ? value.deep_merge!(old_value) : (value[key] = old_value if !old_value.nil?) new_value.is_a?(Hash) ? value.deep_merge!(new_value, prefer_self) : (value[key] = new_value if !new_value.nil?) @@ -269,6 +253,16 @@ module LeapCli self end + def set_environment(env, node) + @env = env + @node = node + self.each do |key, value| + if value.is_a?(Config::Object) + value.set_environment(env, node) + end + end + end + # # like a reverse deep merge # (self takes precedence) diff --git a/lib/leap_cli/config/object_list.rb b/lib/leap_cli/config/object_list.rb index afcc6a6..f9299a6 100644 --- a/lib/leap_cli/config/object_list.rb +++ b/lib/leap_cli/config/object_list.rb @@ -167,14 +167,17 @@ module LeapCli end # - # applies inherit_from! to all objects. + # Applies inherit_from! to all objects. # - def inherit_from!(object_list) + # 'env' specifies what environment should be for + # each object in the list. + # + def inherit_from!(object_list, env) object_list.each do |name, object| if self[name] self[name].inherit_from!(object) else - self[name] = object.deep_dup + self[name] = object.duplicate(env) end end end diff --git a/lib/leap_cli/config/provider.rb b/lib/leap_cli/config/provider.rb index 3c03c5c..0d8bc1f 100644 --- a/lib/leap_cli/config/provider.rb +++ b/lib/leap_cli/config/provider.rb @@ -15,5 +15,8 @@ module LeapCli; module Config def provider self end + def validate! + # nothing here yet :( + end end end; end diff --git a/lib/leap_cli/config/tag.rb b/lib/leap_cli/config/tag.rb index 31f4f76..6bd8d1e 100644 --- a/lib/leap_cli/config/tag.rb +++ b/lib/leap_cli/config/tag.rb @@ -9,8 +9,8 @@ module LeapCli; module Config class Tag < Object attr_reader :node_list - def initialize(manager=nil) - super(manager) + def initialize(environment=nil) + super(environment) @node_list = Config::ObjectList.new end |