Merge branch 'develop'

12 files changed, 2103 insertions, 0 deletions

diff --git a/lib/leap_cli/config/cloud.rb b/lib/leap_cli/config/cloud.rb
new file mode 100644
index 00000000..e3e5c1f1
--- /dev/null
+++ b/lib/leap_cli/config/cloud.rb
@@ -0,0 +1,64 @@
+# encoding: utf-8
+#
+# A class for the cloud.json file
+#
+# Example format:
+#
+# {
+#   "my_aws": {
+#     "api": "aws",
+#     "vendor": "aws",
+#     "auth": {
+#       "region": "us-west-2",
+#       "aws_access_key_id": "xxxxxxxxxxxxxxx",
+#       "aws_secret_access_key": "xxxxxxxxxxxxxxxxxxxxxxxxxx"
+#     },
+#     "default_image": "ami-98e114f8",
+#     "default_options": {
+#       "InstanceType": "t2.nano"
+#     }
+#   }
+# }
+#
+
+module LeapCli; module Config
+
+  # http://fog.io/about/supported_services.html
+  VM_APIS = {
+    'aws' => 'fog-aws',
+    'google' => 'fog-google',
+    'libvirt' => 'fog-libvirt',
+    'openstack' => 'fog-openstack',
+    'rackspace' => 'fog-rackspace'
+  }
+
+  class Cloud < Hash
+    def initialize(env=nil)
+    end
+
+    #
+    # returns hash, each key is the name of an API that is
+    # needed and the value is the name of the gem.
+    #
+    # only provider APIs that are required because they are present
+    # in cloud.json are included.
+    #
+    def required_gems
+      required = {}
+      self.each do |name, conf|
+        api = conf["api"]
+        required_gems[api] = VM_APIS[api]
+      end
+      return required
+    end
+
+    #
+    # returns an array of all possible providers
+    #
+    def possible_apis
+      VM_APIS.keys
+    end
+
+  end
+
+end; end
diff --git a/lib/leap_cli/config/environment.rb b/lib/leap_cli/config/environment.rb
new file mode 100644
index 00000000..ce570839
--- /dev/null
+++ b/lib/leap_cli/config/environment.rb
@@ -0,0 +1,200 @@
+#
+# All configurations files can be isolated into separate environments.
+#
+# Each config json in each environment inherits from the default environment,
+# which in term inherits from the "_base_" environment:
+#
+# _base_             -- base provider in leap_platform
+# '- default         -- environment in provider dir when no env is set
+#    '- production   -- example environment
+#
+
+module LeapCli; module Config
+
+  class Environment
+    # the String name of the environment
+    attr_accessor :name
+
+    # the shared Manager object
+    attr_accessor :manager
+
+    # hashes of {name => Config::Object}
+    attr_accessor :services, :tags, :partials
+
+    # a Config::Provider
+    attr_accessor :provider
+
+    # a Config::Object
+    attr_accessor :common
+
+    # shared, non-inheritable
+    def nodes; @@nodes; end
+    def secrets; @@secrets; end
+    def cloud; @@cloud; end
+
+    def initialize(manager, name, search_dir, parent, options={})
+      @@nodes ||= nil
+      @@secrets ||= nil
+      @@cloud ||= nil
+
+      @manager = manager
+      @name    = name
+
+      load_provider_files(search_dir, options)
+
+      if parent
+        @services.inherit_from! parent.services, self
+            @tags.inherit_from! parent.tags    , self
+        @partials.inherit_from! parent.partials, self
+          @common.inherit_from! parent.common
+        @provider.inherit_from! parent.provider
+      end
+
+      if @provider
+        @provider.set_env(name)
+        @provider.validate!
+      end
+    end
+
+    def load_provider_files(search_dir, options)
+      #
+      # load empty environment if search_dir doesn't exist
+      #
+      if search_dir.nil? || !Dir.exist?(search_dir)
+        @services = Config::ObjectList.new
+        @tags     = Config::ObjectList.new
+        @partials = Config::ObjectList.new
+        @provider = Config::Provider.new
+        @common   = Config::Object.new
+        @cloud    = Config::Cloud.new
+        return
+      end
+
+      #
+      # inheritable
+      #
+      if options[:scope]
+        scope = options[:scope]
+        @services = load_all_json(Path.named_path([:service_env_config, '*', scope],  search_dir), Config::Tag, options)
+        @tags     = load_all_json(Path.named_path([:tag_env_config, '*', scope],      search_dir), Config::Tag, options)
+        @partials = load_all_json(Path.named_path([:service_env_config, '_*', scope], search_dir), Config::Tag, options)
+        @provider = load_json(    Path.named_path([:provider_env_config, scope],      search_dir), Config::Provider, options)
+        @common   = load_json(    Path.named_path([:common_env_config, scope],        search_dir), Config::Object, options)
+      else
+        @services = load_all_json(Path.named_path([:service_config, '*'],  search_dir), Config::Tag, options)
+        @tags     = load_all_json(Path.named_path([:tag_config, '*'],      search_dir), Config::Tag, options)
+        @partials = load_all_json(Path.named_path([:service_config, '_*'], search_dir), Config::Tag, options)
+        @provider = load_json(    Path.named_path(:provider_config,        search_dir), Config::Provider, options)
+        @common   = load_json(    Path.named_path(:common_config,          search_dir), Config::Object, options)
+      end
+
+      # remove 'name' from partials, since partials get merged with nodes
+      @partials.values.each {|partial| partial.delete('name'); }
+
+      #
+      # shared
+      #
+      # shared configs are also non-inheritable
+      # load the first ones we find, and only those.
+      #
+      if @@nodes.nil? || @@nodes.empty?
+        @@nodes = load_all_json(Path.named_path([:node_config, '*'], search_dir), Config::Node, options)
+      end
+      if @@secrets.nil? || @@secrets.empty?
+        @@secrets = load_json(Path.named_path(:secrets_config, search_dir), Config::Secrets, options)
+      end
+      if @@cloud.nil? || @@cloud.empty?
+        @@cloud = load_json(Path.named_path(:cloud_config, search_dir), Config::Cloud)
+      end
+    end
+
+    #
+    # Loads a json template file as a Hash (used only when creating a new node .json
+    # file for the first time).
+    #
+    def template(template)
+      path = Path.named_path([:template_config, template], Path.provider_base)
+      if File.exist?(path)
+        return load_json(path, Config::Object)
+      else
+        return nil
+      end
+    end
+
+    #
+    # Alters the node's json config file. Unfortunately, doing this will
+    # strip out all the comments.
+    #
+    def update_node_json(node, new_values)
+      node_json_path = Path.named_path([:node_config, node.name])
+      old_data       = load_json(node_json_path, Config::Node)
+      new_data       = old_data.merge(new_values)
+      new_contents   = JSON.sorted_generate(new_data) + "\n"
+      Util::write_file! node_json_path, new_contents
+    end
+
+    private
+
+    def load_all_json(pattern, object_class, options={})
+      results = Config::ObjectList.new
+      Dir.glob(pattern).each do |filename|
+        next if options[:no_dots] && File.basename(filename) !~ /^[^\.]*\.json$/
+        obj = load_json(filename, object_class)
+        if obj
+          name = File.basename(filename).force_encoding('utf-8').sub(/^([^\.]+).*\.json$/,'\1')
+          obj['name'] ||= name
+          if options[:env]
+            obj.environment = options[:env]
+          end
+          results[name] = obj
+        end
+      end
+      results
+    end
+
+    def load_json(filename, object_class, options={})
+      if !File.exist?(filename)
+        return object_class.new(self)
+      end
+
+      Util::log :loading, filename, 3
+
+      #
+      # Read a JSON file, strip out comments.
+      #
+      # UTF8 is the default encoding for JSON, but others are allowed:
+      # https://www.ietf.org/rfc/rfc4627.txt
+      #
+      buffer = StringIO.new
+      File.open(filename, "rb", :encoding => 'UTF-8') do |f|
+        while (line = f.gets)
+          next if line =~ /^\s*\/\//
+          buffer << line
+        end
+      end
+
+      #
+      # force UTF-8
+      #
+      if $ruby_version >= [1,9]
+        string = buffer.string.force_encoding('utf-8')
+      else
+        string = Iconv.conv("UTF-8//IGNORE", "UTF-8", buffer.string)
+      end
+
+      # parse json
+      begin
+        hash = JSON.parse(string, :object_class => Hash, :array_class => Array) || {}
+      rescue SyntaxError, JSON::ParserError => exc
+        Util::log 0, :error, 'in file "%s":' % filename
+        Util::log 0, exc.to_s, :indent => 1
+        return nil
+      end
+      object = object_class.new(self)
+      object.deep_merge!(hash)
+      return object
+    end
+
+  end # end Environment
+
+end; end
\ No newline at end of file
diff --git a/lib/leap_cli/config/filter.rb b/lib/leap_cli/config/filter.rb
new file mode 100644
index 00000000..07424894
--- /dev/null
+++ b/lib/leap_cli/config/filter.rb
@@ -0,0 +1,181 @@
+#
+# Many leap_cli commands accept a list of filters to select a subset of nodes for the command to
+# be applied to. This class is a helper for manager to run these filters.
+#
+# Classes other than Manager should not use this class.
+#
+# Filter rules:
+#
+# * A filter consists of a list of tokens
+# * A token may be a service name, tag name, environment name, or node name.
+# * Each token may be optionally prefixed with a plus sign.
+# * Multiple tokens with a plus are treated as an OR condition,
+#   but treated as an AND condition with the plus sign.
+#
+# For example
+#
+# * openvpn +development => all nodes with service 'openvpn' AND environment 'development'
+# * openvpn seattle => all nodes with service 'openvpn' OR tag 'seattle'.
+#
+# There can only be one environment specified. Typically, there are also tags
+# for each environment name. These name are treated as environments, not tags.
+#
+module LeapCli
+  module Config
+    class Filter
+
+      #
+      # filter -- array of strings, each one a filter
+      # options -- hash, possible keys include
+      #   :nopin -- disregard environment pinning
+      #   :local -- if false, disallow local nodes
+      #   :warning -- if false, don't print a warning when no nodes are found.
+      #
+      # A nil value in the filters array indicates
+      # the default environment. This is in order to support
+      # calls like `manager.filter(environments)`
+      #
+      def initialize(filters, options, manager)
+        @filters = filters.nil? ? [] : filters.dup
+        @environments = []
+        @options = options
+        @manager = manager
+
+        # split filters by pulling out items that happen
+        # to be environment names.
+        if LeapCli.leapfile.environment.nil? || @options[:nopin]
+          @environments = []
+        else
+          @environments = [LeapCli.leapfile.environment]
+        end
+        @filters.select! do |filter|
+          if filter.nil?
+            @environments << nil unless @environments.include?(nil)
+            false
+          else
+            filter_text = filter.sub(/^\+/,'')
+            if is_environment?(filter_text)
+              if filter_text == LeapCli.leapfile.environment
+                # silently ignore already pinned environments
+              elsif (filter =~ /^\+/ || @filters.first == filter) && !@environments.empty?
+                LeapCli::Util.bail! do
+                  LeapCli.log "Environments are exclusive: no node is in two environments." do
+                    LeapCli.log "Tried to filter on '#{@environments.join('\' AND \'')}' AND '#{filter_text}'"
+                  end
+                end
+              else
+                @environments << filter_text
+              end
+              false
+            else
+              true
+            end
+          end
+        end
+
+        # don't let the first filter have a + prefix
+        if @filters[0] =~ /^\+/
+          @filters[0] = @filters[0][1..-1]
+        end
+      end
+
+      # actually run the filter, returns a filtered list of nodes
+      def nodes()
+        if @filters.empty?
+          return nodes_for_empty_filter
+        else
+          return nodes_for_filter
+        end
+      end
+
+      private
+
+      def nodes_for_empty_filter
+        node_list = @manager.nodes
+        if @environments.any?
+          node_list = node_list[ @environments.collect{|e|[:environment, env_to_filter(e)]} ]
+        end
+        if @options[:local] === false
+          node_list = node_list[:environment => '!local']
+        end
+        if @options[:disabled] === false
+          node_list = node_list[:environment => '!disabled']
+        end
+        node_list
+      end
+
+      def nodes_for_filter
+        node_list = Config::ObjectList.new
+        @filters.each do |filter|
+          if filter =~ /^\+/
+            keep_list = nodes_for_name(filter[1..-1])
+            node_list.delete_if do |name, node|
+              if keep_list[name]
+                false
+              else
+                true
+              end
+            end
+          else
+            node_list.merge!(nodes_for_name(filter))
+          end
+        end
+        node_list
+      end
+
+      private
+
+      #
+      # returns a set of nodes corresponding to a single name,
+      # where name could be a node name, service name, or tag name.
+      #
+      # For services and tags, we only include nodes for the
+      # environments that are active
+      #
+      def nodes_for_name(name)
+        if node = @manager.nodes[name]
+          return Config::ObjectList.new(node)
+        elsif @environments.empty?
+          if @manager.services[name]
+            return @manager.env('_all_').services[name].node_list
+          elsif @manager.tags[name]
+            return @manager.env('_all_').tags[name].node_list
+          elsif @options[:warning] != false
+            LeapCli.log :warning, "filter '#{name}' does not match any node names, tags, services, or environments."
+            return Config::ObjectList.new
+          else
+            return Config::ObjectList.new
+          end
+        else
+          node_list = Config::ObjectList.new
+          if @manager.services[name]
+            @environments.each do |env|
+              node_list.merge!(@manager.env(env).services[name].node_list)
+            end
+          elsif @manager.tags[name]
+            @environments.each do |env|
+              node_list.merge!(@manager.env(env).tags[name].node_list)
+            end
+          elsif @options[:warning] != false
+            LeapCli.log :warning, "filter '#{name}' does not match any node names, tags, services, or environments."
+          end
+          return node_list
+        end
+      end
+
+      #
+      # when pinning, we use the name 'default' to specify nodes
+      # without an environment set, but when filtering, we need to filter
+      # on :environment => nil.
+      #
+      def env_to_filter(environment)
+        environment == 'default' ? nil : environment
+      end
+
+      def is_environment?(text)
+        text == 'default' || @manager.environment_names.include?(text)
+      end
+
+    end
+  end
+end
diff --git a/lib/leap_cli/config/manager.rb b/lib/leap_cli/config/manager.rb
new file mode 100644
index 00000000..d69a5808
--- /dev/null
+++ b/lib/leap_cli/config/manager.rb
@@ -0,0 +1,475 @@
+# encoding: utf-8
+
+require 'json/pure'
+
+if $ruby_version < [1,9]
+  require 'iconv'
+end
+
+module LeapCli
+  module Config
+
+    #
+    # A class to manage all the objects in all the configuration files.
+    #
+    class Manager
+
+      def initialize
+        @environments = {} # hash of `Environment` objects, keyed by name.
+        Config::Object.send(:include, LeapCli::Macro)
+      end
+
+      ##
+      ## ATTRIBUTES
+      ##
+
+      #
+      # returns the Hash of the contents of facts.json
+      #
+      def facts
+        @facts ||= begin
+          content = Util.read_file(:facts)
+          if !content || content.empty?
+            content = "{}"
+          end
+          JSON.parse(content)
+        rescue SyntaxError, JSON::ParserError => exc
+          Util::bail! "Could not parse facts.json -- #{exc}"
+        end
+      end
+
+      #
+      # returns an Array of all the environments defined for this provider.
+      # the returned array includes nil (for the default environment)
+      #
+      def environment_names
+        @environment_names ||= begin
+          [nil] + (env.tags.field('environment') + env.nodes.field('environment')).compact.uniq
+        end
+      end
+
+      #
+      # Returns the appropriate environment variable
+      #
+      def env(env=nil)
+        @environments[env || 'default']
+      end
+
+      #
+      # The default accessors
+      #
+      # For these defaults, use 'default' environment, or whatever
+      # environment is pinned.
+      #
+      # I think it might be an error that these are ever used
+      # and I would like to get rid of them.
+      #
+      def services; env(default_environment).services; end
+      def tags;     env(default_environment).tags;     end
+      def partials; env(default_environment).partials; end
+      def provider; env(default_environment).provider; end
+      def common;   env(default_environment).common;   end
+      def secrets;  env(default_environment).secrets;  end
+      def nodes;    env(default_environment).nodes;    end
+      def template(*args)
+        self.env.template(*args)
+      end
+
+      def default_environment
+        LeapCli.leapfile.environment
+      end
+
+      ##
+      ## IMPORT EXPORT
+      ##
+
+      def add_environment(args)
+        if args[:inherit]
+          parent = @environments[args.delete(:inherit)]
+        else
+          parent = nil
+        end
+        env = Environment.new(
+          self,
+          args.delete(:name),
+          args.delete(:dir),
+          parent,
+          args
+        )
+        @environments[env.name] = env
+      end
+
+      #
+      # load .json configuration files
+      #
+      def load(options = {})
+        # load base
+        add_environment(name: '_base_', dir: Path.provider_base)
+
+        # load provider
+        Util::assert_files_exist!(Path.named_path(:provider_config, Path.provider))
+        add_environment(name: 'default', dir: Path.provider,
+          inherit: '_base_', no_dots: true)
+
+        # create a special '_all_' environment, used for tracking
+        # the union of all the environments
+        add_environment(name: '_all_', inherit: 'default')
+
+        # load environments
+        environment_names.each do |ename|
+          if ename
+            LeapCli.log 3, :loading, '%s environment...' % ename
+            add_environment(name: ename, dir: Path.provider,
+              inherit: 'default', scope: ename)
+          end
+        end
+
+        # apply inheritance
+        env.nodes.each do |name, node|
+          Util::assert! name =~ /^[0-9a-z-]+$/, "Illegal character(s) used in node name '#{name}'"
+          env.nodes[name] = apply_inheritance(node)
+        end
+
+        # do some node-list post-processing
+        cleanup_node_lists(options)
+
+        # apply service.rb, common.rb, and provider.rb control files
+        apply_control_files
+      end
+
+      #
+      # save compiled hiera .yaml files
+      #
+      # if a node_list is specified, only update those .yaml files.
+      # otherwise, update all files, destroying files that are no longer used.
+      #
+      def export_nodes(node_list=nil)
+        updated_hiera = []
+        updated_files = []
+        existing_hiera = nil
+        existing_files = nil
+
+        unless node_list
+          node_list = env.nodes
+          existing_hiera = Dir.glob(Path.named_path([:hiera, '*'], Path.provider))
+          existing_files = Dir.glob(Path.named_path([:node_files_dir, '*'], Path.provider))
+        end
+
+        node_list.each_node do |node|
+          filepath = Path.named_path([:node_files_dir, node.name], Path.provider)
+          hierapath = Path.named_path([:hiera, node.name], Path.provider)
+          Util::write_file!(hierapath, node.dump_yaml)
+          updated_files << filepath
+          updated_hiera << hierapath
+        end
+
+        if @disabled_nodes
+          # make disabled nodes appear as if they are still active
+          @disabled_nodes.each_node do |node|
+            updated_files << Path.named_path([:node_files_dir, node.name], Path.provider)
+            updated_hiera << Path.named_path([:hiera, node.name], Path.provider)
+          end
+        end
+
+        # remove files that are no longer needed
+        if existing_hiera
+          (existing_hiera - updated_hiera).each do |filepath|
+            Util::remove_file!(filepath)
+          end
+        end
+        if existing_files
+          (existing_files - updated_files).each do |filepath|
+            Util::remove_directory!(filepath)
+          end
+        end
+      end
+
+      def export_secrets(clean_unused_secrets = false)
+        if env.secrets.any?
+          Util.write_file!([:secrets_config, Path.provider], env.secrets.dump_json(clean_unused_secrets) + "\n")
+        end
+      end
+
+      ##
+      ## FILTERING
+      ##
+
+      #
+      # returns a node list consisting only of nodes that satisfy the filter criteria.
+      #
+      # filter: condition [condition] [condition] [+condition]
+      # condition: [node_name | service_name | tag_name | environment_name]
+      #
+      # if conditions is prefixed with +, then it works like an AND. Otherwise, it works like an OR.
+      #
+      # args:
+      # filter -- array of filter terms, one per item
+      #
+      # options:
+      # :local -- if :local is false and the filter is empty, then local nodes are excluded.
+      # :nopin -- if true, ignore environment pinning
+      #
+      def filter(filters=nil, options={})
+        Filter.new(filters, options, self).nodes()
+      end
+
+      #
+      # same as filter(), but exits if there is no matching nodes
+      #
+      def filter!(filters, options={})
+        node_list = filter(filters, options)
+        Util::assert! node_list.any?, "Could not match any nodes from '#{filters.join ' '}'"
+        return node_list
+      end
+
+      #
+      # returns a single Config::Object that corresponds to a Node.
+      #
+      def node(name)
+        if name =~ /\./
+          # probably got a fqdn, since periods are not allowed in node names.
+          # so, take the part before the first period as the node name
+          name = name.split('.').first
+        end
+        env.nodes[name]
+      end
+
+      #
+      # returns a single node that is disabled
+      #
+      def disabled_node(name)
+        @disabled_nodes[name]
+      end
+
+      #
+      # yields each node, in sorted order
+      #
+      def each_node(&block)
+        env.nodes.each_node(&block)
+      end
+
+      def reload_node!(node)
+        env.nodes[node.name] = apply_inheritance!(node)
+      end
+
+      ##
+      ## CONNECTIONS
+      ##
+
+      class ConnectionList < Array
+        def add(data={})
+          self << {
+            "from" => data[:from],
+            "to" => data[:to],
+            "port" => data[:port]
+          }
+        end
+      end
+
+      def connections
+        @connections ||= ConnectionList.new
+      end
+
+      ##
+      ## PRIVATE
+      ##
+
+      private
+
+      #
+      # makes a node inherit options from appropriate the common, service, and tag json files.
+      #
+      def apply_inheritance(node, throw_exceptions=false)
+        new_node = Config::Node.new(nil)
+        node_env = guess_node_env(node)
+        new_node.set_environment(node_env, new_node)
+
+        # inherit from common
+        new_node.deep_merge!(node_env.common)
+
+        # inherit from services
+        if node['services']
+          node['services'].to_a.each do |node_service|
+            service = node_env.services[node_service]
+            if service.nil?
+              msg = 'in node "%s": the service "%s" does not exist.' % [node['name'], node_service]
+              LeapCli.log 0, :error, msg
+              raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions
+            else
+              new_node.deep_merge!(service)
+            end
+          end
+        end
+
+        # inherit from tags
+        node['tags'] = (node['tags'] || []).to_a
+        if node.vagrant?
+          node['tags'] << 'local'
+        elsif node['vm']
+          node['tags'] << 'vm'
+        end
+        node['tags'].each do |node_tag|
+          tag = node_env.tags[node_tag]
+          if tag.nil?
+            msg = 'in node `%s`: the tag "%s" does not exist!' % [node['name'], node_tag]
+            LeapCli.log 0, :error, msg
+            raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions
+          else
+            new_node.deep_merge!(tag)
+          end
+        end
+
+        # inherit from node
+        new_node.deep_merge!(node)
+        return new_node
+      end
+
+      def apply_inheritance!(node)
+        apply_inheritance(node, true)
+      end
+
+      #
+      # Guess the environment of the node from the tag names.
+      #
+      # Technically, this is wrong: a tag that sets the environment might not be
+      # named the same as the environment. This code assumes that it is.
+      #
+      # Unfortunately, it is a chicken and egg problem. We need to know the nodes
+      # likely environment in order to apply the inheritance that will actually
+      # determine the node's properties.
+      #
+      def guess_node_env(node)
+        if node.vagrant?
+          return self.env("local")
+        else
+          environment = self.env(default_environment)
+          if node['tags']
+            node['tags'].to_a.each do |tag|
+              if self.environment_names.include?(tag)
+                environment = self.env(tag)
+              end
+            end
+          end
+          return environment
+        end
+      end
+
+      #
+      # does some final clean at the end of loading nodes.
+      # this includes removing disabled nodes, and populating
+      # the services[x].node_list and tags[x].node_list
+      #
+      def cleanup_node_lists(options)
+        @disabled_nodes = Config::ObjectList.new
+        env.nodes.each do |name, node|
+          if node.enabled || options[:include_disabled]
+            if node['services']
+              node['services'].to_a.each do |node_service|
+                env(node.environment).services[node_service].node_list.add(node.name, node)
+                env('_all_').services[node_service].node_list.add(node.name, node)
+              end
+            end
+            if node['tags']
+              node['tags'].to_a.each do |node_tag|
+                if env(node.environment).tags[node_tag]
+                  # if tag exists
+                  env(node.environment).tags[node_tag].node_list.add(node.name, node)
+                  env('_all_').tags[node_tag].node_list.add(node.name, node)
+                end
+              end
+            end
+            if node.name == 'default' || environment_names.include?(node.name)
+              LeapCli::Util.bail! do
+                LeapCli.log :error, "The node name '#{node.name}' is invalid, because there is an environment with that same name."
+              end
+            end
+          elsif !options[:include_disabled]
+            LeapCli.log 2, :skipping, "disabled node #{name}."
+            env.nodes.delete(name)
+            @disabled_nodes[name] = node
+          end
+        end
+      end
+
+      #
+      # Applies 'control' files for node .json files and provider.json.
+      #
+      # A control file is like a service or a tag JSON file, but it contains
+      # raw ruby code that gets evaluated in the context of the node.
+      #
+      # Yes, this entirely breaks our functional programming model for JSON
+      # generation.
+      #
+      # Control files are evaluated last, after everything else has run.
+      #
+      def apply_control_files
+        @environments.values.each do |e|
+          provider_control_files(e.name).each do |provider_rb|
+            begin
+              e.provider.eval_file provider_rb
+            rescue ConfigError => exc
+              if options[:continue_on_error]
+                exc.log
+              else
+                raise exc
+              end
+            end
+          end
+        end
+        env.nodes.each do |name, node|
+          node_control_files(node).each do |file|
+            begin
+              node.eval_file file
+            rescue ConfigError => exc
+              if options[:continue_on_error]
+                exc.log
+              else
+                raise exc
+              end
+            end
+          end
+        end
+      end
+
+      def node_control_files(node)
+        files = []
+        [Path.provider_base, Path.provider].each do |provider_dir|
+          # add common.rb
+          common = File.join(provider_dir, 'common.rb')
+          files << common if File.exist?(common)
+
+          # add services/*.rb and tags/*.rb, as appropriate for this node
+          [['services', :service_config], ['tags', :tag_config]].each do |attribute, path_sym|
+            node[attribute].each do |attr_value|
+              path = Path.named_path([path_sym, "#{attr_value}.rb"], provider_dir).sub(/\.json$/,'')
+              if File.exist?(path)
+                files << path
+              end
+            end
+          end
+        end
+        return files
+      end
+
+      def provider_control_files(env)
+        # skip envs that start with underscore
+        if env =~ /^_/
+          return []
+        end
+        files = []
+        environments = [nil]
+        environments << env unless env == 'default'
+        environments.each do |environment|
+          [Path.provider_base, Path.provider].each do |provider_dir|
+            provider_rb = File.join(
+              provider_dir, ['provider', environment, 'rb'].compact.join('.')
+            )
+            files << provider_rb if File.exist?(provider_rb)
+          end
+        end
+        return files
+      end
+
+    end
+  end
+end
diff --git a/lib/leap_cli/config/node.rb b/lib/leap_cli/config/node.rb
new file mode 100644
index 00000000..23abdee3
--- /dev/null
+++ b/lib/leap_cli/config/node.rb
@@ -0,0 +1,245 @@
+#
+# Configuration for a 'node' (a server in the provider's infrastructure)
+#
+
+require 'ipaddr'
+
+module LeapCli; module Config
+
+  class Node < Object
+    attr_accessor :file_paths
+
+    def initialize(environment=nil)
+      super(environment)
+      @node = self
+      @file_paths = []
+    end
+
+    #
+    # returns true if this node has an ip address in the range of the vagrant network
+    #
+    def vagrant?
+      ip = self['ip_address']
+      return false unless ip
+      begin
+        vagrant_range = IPAddr.new LeapCli.leapfile.vagrant_network
+      rescue ArgumentError
+        Util::bail! { Util::log :invalid, "vagrant_network in Leapfile or .leaprc" }
+      end
+
+      begin
+        ip_addr = IPAddr.new(ip)
+      rescue ArgumentError
+        Util::log :warning, "invalid ip address '#{ip}' for node '#{@node.name}'"
+      end
+      return vagrant_range.include?(ip_addr)
+    end
+
+    def vm?
+      self['vm']
+    end
+
+    def vm_id?
+      self['vm.id'] && !self['vm.id'].empty?
+    end
+
+    #
+    # Return a hash table representation of ourselves, with the key equal to the @node.name,
+    # and the value equal to the fields specified in *keys.
+    #
+    # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b'
+    #
+    # compare to Object#pick(*keys). This method is the sames as Config::ObjectList#pick_fields,
+    # but works on a single node.
+    #
+    # Example:
+    #
+    #  node.pick('domain.internal') =>
+    #
+    #    {
+    #      'node1': {
+    #        'domain_internal': 'node1.example.i'
+    #      }
+    #    }
+    #
+    def pick_fields(*keys)
+      {@node.name => self.pick(*keys)}
+    end
+
+    #
+    # can be overridden by the platform.
+    # returns a list of node names that should be tested before this node
+    #
+    def test_dependencies
+      []
+    end
+
+    # returns a string list of supported ssh host key algorithms for this node.
+    # or an empty string if it could not be determined
+    def supported_ssh_host_key_algorithms
+      require 'leap_cli/ssh'
+      @host_key_algo ||= LeapCli::SSH::Key.supported_host_key_algorithms(
+        Util.read_file([:node_ssh_pub_key, @node.name])
+      )
+    end
+
+    #
+    # Takes strings such as "openvpn.gateway_address:1.1.1.1"
+    # and converts this to data stored in this node.
+    #
+    def seed_from_args(args)
+      args.each do |seed|
+        key, value = seed.split(':', 2)
+        value = format_seed_value(value)
+        Util.assert! key =~ /^[0-9a-z\._]+$/, "illegal characters used in property '#{key}'"
+        if key =~ /\./
+          key_parts = key.split('.')
+          final_key = key_parts.pop
+          current_object = self
+          key_parts.each do |key_part|
+            current_object[key_part] ||= Config::Object.new
+            current_object = current_object[key_part]
+          end
+          current_object[final_key] = value
+        else
+          self[key] = value
+        end
+      end
+    end
+
+    #
+    # Seeds values for this node from a template, based on the services.
+    # Values in the template will not override existing node values.
+    #
+    def seed_from_template
+      inherit_from!(manager.template('common'))
+      [self['services']].flatten.each do |service|
+        if service
+          template = manager.template(service)
+          if template
+            inherit_from!(template)
+          end
+        end
+      end
+    end
+
+    #
+    # bails if the node is not valid.
+    #
+    def validate!
+      #
+      # validate ip_address
+      #
+      if self['ip_address'] == "REQUIRED"
+        Util.bail! do
+          Util.log :error, "ip_address is not set. " +
+            "Specify with `leap node add NAME ip_address:ADDRESS`."
+        end
+      elsif self['ip_address']
+        begin
+          IPAddr.new(self['ip_address'])
+        rescue ArgumentError
+          Util.bail! do
+            Util.log :invalid, "ip_address #{self['ip_address'].inspect}"
+          end
+        end
+      end
+
+      #
+      # validate name
+      #
+      self.class.validate_name!(self.name, self.vagrant?)
+    end
+
+    #
+    # create or update all the configs needed for this node,
+    # including x.509 certs as needed.
+    #
+    # note: this method will write to disk EVERYTHING
+    # in the node, which is not what you want
+    # if the node has inheritance applied.
+    #
+    def write_configs
+      json = self.dump_json(:exclude => ['name'])
+      Util.write_file!([:node_config, name], json + "\n")
+    rescue LeapCli::ConfigError
+      Config::Node.remove_node_files(self.name)
+    end
+
+    #
+    # modifies the config file nodes/NAME.json for this node.
+    #
+    def update_json(new_values)
+      self.env.update_node_json(node, new_values)
+    end
+
+    #
+    # returns an array of all possible dns names for this node
+    #
+    def all_dns_names
+      names = [@node.domain.internal, @node.domain.full]
+      if @node['dns'] && @node.dns['aliases'] && @node.dns.aliases.any?
+        names += @node.dns.aliases
+      end
+      names.compact!
+      names.sort!
+      names.uniq!
+      return names
+    end
+
+    def remove_files
+      self.class.remove_node_files(self.name)
+    end
+
+    ##
+    ## Class Methods
+    ##
+
+    def self.remove_node_files(node_name)
+      (Leap::Platform.node_files + [:node_files_dir]).each do |path|
+        Util.remove_file! [path, node_name]
+      end
+    end
+
+    def self.validate_name!(name, local=false)
+      Util.assert! name, 'Node is missing a name.'
+      if local
+        Util.assert! name =~ /^[0-9a-z]+$/,
+          "illegal characters used in node name '#{name}' " +
+          "(note: Vagrant does not allow hyphens or underscores)"
+      else
+        Util.assert! name =~ /^[0-9a-z-]+$/,
+          "illegal characters used in node name '#{name}' " +
+          "(note: Linux does not allow underscores)"
+      end
+    end
+
+    private
+
+    #
+    # conversions:
+    #
+    #   "x,y,z" => ["x","y","z"]
+    #
+    #   "22" => 22
+    #
+    #   "5.1" => 5.1
+    #
+    def format_seed_value(v)
+      if v =~ /,/
+        v = v.split(',')
+        v.map! do |i|
+          i = i.to_i if i.to_i.to_s == i
+          i = i.to_f if i.to_f.to_s == i
+          i
+        end
+      else
+        v = v.to_i if v.to_i.to_s == v
+        v = v.to_f if v.to_f.to_s == v
+      end
+      return v
+    end
+
+  end
+
+end; end
diff --git a/lib/leap_cli/config/node_cert.rb b/lib/leap_cli/config/node_cert.rb
new file mode 100644
index 00000000..da63d621
--- /dev/null
+++ b/lib/leap_cli/config/node_cert.rb
@@ -0,0 +1,124 @@
+#
+# x509 related methods for Config::Node
+#
+module LeapCli; module Config
+
+  class Node < Object
+
+    #
+    # creates a new server certificate file for this node
+    #
+    def generate_cert
+      require 'leap_cli/x509'
+
+      if self['x509.use'] == false ||
+         !Util.file_exists?(:ca_cert, :ca_key) ||
+         !self.cert_needs_updating?
+        return false
+      end
+
+      cert = CertificateAuthority::Certificate.new
+      provider = env.provider
+
+      # set subject
+      cert.subject.common_name = self.domain.full
+      cert.serial_number.number = X509.cert_serial_number(self.domain.full)
+
+      # set expiration
+      cert.not_before = X509.yesterday
+      cert.not_after  = X509.yesterday_advance(provider.ca.server_certificates.life_span)
+
+      # generate key
+      cert.key_material.generate_key(provider.ca.server_certificates.bit_size)
+
+      # sign
+      cert.parent = X509.ca_root
+      cert.sign!(X509.server_signing_profile(self))
+
+      # save
+      Util.write_file!([:node_x509_key, self.name], cert.key_material.private_key.to_pem)
+      Util.write_file!([:node_x509_cert, self.name], cert.to_pem)
+    end
+
+    #
+    # returns true if the certs associated with +node+ need to be regenerated.
+    #
+    def cert_needs_updating?(log_comments=true)
+      require 'leap_cli/x509'
+
+      if log_comments
+        def log(*args, &block)
+          Util.log(*args, &block)
+        end
+      else
+        def log(*args); end
+      end
+
+      node = self
+      if !Util.file_exists?([:node_x509_cert, node.name], [:node_x509_key, node.name])
+        return true
+      else
+        cert = X509.load_certificate_file([:node_x509_cert, node.name])
+        if !X509.created_by_authority?(cert)
+          log :updating, "cert for node '#{node.name}' because it was signed by an old CA root cert."
+          return true
+        end
+        if cert.not_after < Time.now.advance(:months => 2)
+          log :updating, "cert for node '#{node.name}' because it will expire soon"
+          return true
+        end
+        if cert.subject.common_name != node.domain.full
+          log :updating, "cert for node '#{node.name}' because domain.full has changed (was #{cert.subject.common_name}, now #{node.domain.full})"
+          return true
+        end
+        cert.openssl_body.extensions.each do |ext|
+          if ext.oid == "subjectAltName"
+            ips = []
+            dns_names = []
+            ext.value.split(",").each do |value|
+              value.strip!
+              ips << $1          if value =~ /^IP Address:(.*)$/
+              dns_names << $1    if value =~ /^DNS:(.*)$/
+            end
+            dns_names.sort!
+            if ips.first != node.ip_address
+              log :updating, "cert for node '#{node.name}' because ip_address has changed (from #{ips.first} to #{node.ip_address})"
+              return true
+            elsif dns_names != node.all_dns_names
+              log :updating, "cert for node '#{node.name}' because domain name aliases have changed" do
+                log "from: #{dns_names.inspect}"
+                log "to: #{node.all_dns_names.inspect})"
+              end
+              return true
+            end
+          end
+        end
+      end
+      return false
+    end
+
+    #
+    # check the expiration of commercial certs, if any.
+    #
+    def warn_if_commercial_cert_will_soon_expire
+      require 'leap_cli/x509'
+
+      self.all_dns_names.each do |domain|
+        if Util.file_exists?([:commercial_cert, domain])
+          cert = X509.load_certificate_file([:commercial_cert, domain])
+          path = Path.relative_path([:commercial_cert, domain])
+          if cert.not_after < Time.now.utc
+            Util.log :error, "the commercial certificate '#{path}' has EXPIRED! " +
+              "You should renew it with `leap cert renew #{domain}`."
+          elsif cert.not_after < Time.now.advance(:months => 2)
+            Util.log :warning, "the commercial certificate '#{path}' will expire soon (#{cert.not_after}). "+
+              "You should renew it with `leap cert renew #{domain}`."
+          end
+        end
+      end
+    end
+
+  end
+
+end; end
+
diff --git a/lib/leap_cli/config/object.rb b/lib/leap_cli/config/object.rb
new file mode 100644
index 00000000..16c41999
--- /dev/null
+++ b/lib/leap_cli/config/object.rb
@@ -0,0 +1,454 @@
+# encoding: utf-8
+
+require 'erb'
+require 'json/pure'  # pure ruby implementation is required for our sorted trick to work.
+
+if $ruby_version < [1,9]
+  $KCODE = 'UTF8'
+end
+require 'ya2yaml' # pure ruby yaml
+
+module LeapCli
+  module Config
+
+    #
+    # This class represents the configuration for a single node, service, or tag.
+    # Also, all the nested hashes are also of this type.
+    #
+    # It is called 'object' because it corresponds to an Object in JSON.
+    #
+    class Object < Hash
+
+      attr_reader :env
+      attr_reader :node
+
+      def initialize(environment=nil, node=nil)
+        raise ArgumentError unless environment.nil? || environment.is_a?(Config::Environment)
+        @env = environment
+        # an object that is a node as @node equal to self, otherwise all the
+        # child objects point back to the top level node.
+        @node = node || self
+      end
+
+      def manager
+        @env.manager
+      end
+
+      #
+      # TODO: deprecate node.global()
+      #
+      def global
+        @env
+      end
+
+      def environment=(e)
+        self.store('environment', e)
+      end
+
+      def environment
+        self['environment']
+      end
+
+      def duplicate(env)
+        new_object = self.deep_dup
+        new_object.set_environment(env, new_object)
+      end
+
+      #
+      # export YAML
+      #
+      # We use pure ruby yaml exporter ya2yaml instead of SYCK or PSYCH because it
+      # allows us greater compatibility regardless of installed ruby version and
+      # greater control over how the yaml is exported (sorted keys, in particular).
+      #
+      def dump_yaml
+        evaluate(@node)
+        sorted_ya2yaml(:syck_compatible => true)
+      end
+
+      #
+      # export JSON
+      #
+      def dump_json(options={})
+        evaluate(@node)
+        if options[:format] == :compact
+          return self.to_json
+        else
+          excluded = {}
+          if options[:exclude]
+            options[:exclude].each do |key|
+              excluded[key] = self[key]
+              self.delete(key)
+            end
+          end
+          json_str = JSON.sorted_generate(self)
+          if excluded.any?
+            self.merge!(excluded)
+          end
+          return json_str
+        end
+      end
+
+      def evaluate(context=@node)
+        evaluate_everything(context)
+        late_evaluate_everything(context)
+      end
+
+      ##
+      ## FETCHING VALUES
+      ##
+
+      def [](key)
+        get(key)
+      end
+
+      # Overrride some default methods in Hash that are likely to
+      # be used as attributes.
+      alias_method :hkey, :key
+      def key; get('key'); end
+
+      #
+      # make hash addressable like an object (e.g. obj['name'] available as obj.name)
+      #
+      def method_missing(method, *args, &block)
+        get!(method)
+      end
+
+      def get(key)
+        begin
+          get!(key)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      # override behavior of #default() from Hash
+      def default
+        get!('default')
+      end
+
+      #
+      # Like a normal Hash#[], except:
+      #
+      # (1) lazily eval dynamic values when we encounter them. (i.e. strings that start with "= ")
+      #
+      # (2) support for nested references in a single string (e.g. ['a.b'] is the same as ['a']['b'])
+      #     the dot path is always absolute, starting at the top-most object.
+      #
+      def get!(key)
+        key = key.to_s
+        if self.has_key?(key)
+          fetch_value(key)
+        elsif key =~ /\./
+          # for keys with with '.' in them, we start from the root object (@node).
+          keys = key.split('.')
+          value = self.get!(keys.first)
+          if value.is_a? Config::Object
+            value.get!(keys[1..-1].join('.'))
+          else
+            value
+          end
+        else
+          raise NoMethodError.new(key, "No method '#{key}' for #{self.class}")
+        end
+      end
+
+      #
+      # works like Hash#store(key, value), but supports our nested dot notation,
+      # just like get() does.
+      #
+      def set(key, value)
+        key = key.to_s
+        # for keys with with '.' in them, we pop off the first part
+        # and recursively call ourselves.
+        if key =~ /\./
+          keys = key.split('.')
+          parent_value = self.get!(keys.first)
+          if parent_value.is_a?(Config::Object)
+            parent_value.set(keys[1..-1].join('.'), value)
+          else
+            parent_value.store(keys[1..-1].join('.'), value)
+          end
+        else
+          self.store(key, value)
+        end
+        return nil
+      end
+
+      ##
+      ## COPYING
+      ##
+
+      #
+      # A deep (recursive) merge with another Config::Object.
+      #
+      # If prefer_self is set to true, the value from self will be picked when there is a conflict
+      # that cannot be merged.
+      #
+      # Merging rules:
+      #
+      # - If a value is a hash, we recursively merge it.
+      # - If the value is simple, like a string, the new one overwrites the value.
+      # - If the value is an array:
+      #   - If both old and new values are arrays, the new one replaces the old.
+      #   - If one of the values is simple but the other is an array, the simple is added to the array.
+      #
+      def deep_merge!(object, prefer_self=false)
+        object.each do |key,new_value|
+          if self.has_key?('+'+key)
+            mode = :add
+            old_value = self.fetch '+'+key, nil
+            self.delete('+'+key)
+          elsif self.has_key?('-'+key)
+            mode = :subtract
+            old_value = self.fetch '-'+key, nil
+            self.delete('-'+key)
+          elsif self.has_key?('!'+key)
+            mode = :replace
+            old_value = self.fetch '!'+key, nil
+            self.delete('!'+key)
+          else
+            mode = :normal
+            old_value = self.fetch key, nil
+          end
+
+          # clean up boolean
+          new_value = true  if new_value == "true"
+          new_value = false if new_value == "false"
+          old_value = true  if old_value == "true"
+          old_value = false if old_value == "false"
+
+          # force replace?
+          if mode == :replace && prefer_self
+            value = old_value
+
+          # merge hashes
+          elsif old_value.is_a?(Hash) || new_value.is_a?(Hash)
+            value = Config::Object.new(@env, @node)
+            old_value.is_a?(Hash) ? value.deep_merge!(old_value) : (value[key] = old_value if !old_value.nil?)
+            new_value.is_a?(Hash) ? value.deep_merge!(new_value, prefer_self) : (value[key] = new_value if !new_value.nil?)
+
+          # merge nil
+          elsif new_value.nil?
+            value = old_value
+          elsif old_value.nil?
+            value = new_value
+
+          # merge arrays when one value is not an array
+          elsif old_value.is_a?(Array) && !new_value.is_a?(Array)
+            (value = (old_value.dup << new_value).compact.uniq).delete('REQUIRED')
+          elsif new_value.is_a?(Array) && !old_value.is_a?(Array)
+            (value = (new_value.dup << old_value).compact.uniq).delete('REQUIRED')
+
+          # merge two arrays
+          elsif old_value.is_a?(Array) && new_value.is_a?(Array)
+            if mode == :add
+              value = (old_value + new_value).sort.uniq
+            elsif mode == :subtract
+              value = new_value - old_value
+            elsif prefer_self
+              value = old_value
+            else
+              value = new_value
+            end
+
+          # catch errors
+          elsif type_mismatch?(old_value, new_value)
+            raise 'Type mismatch. Cannot merge %s (%s) with %s (%s). Key is "%s", name is "%s".' % [
+              old_value.inspect, old_value.class,
+              new_value.inspect, new_value.class,
+              key, self.class
+            ]
+
+          # merge simple strings & numbers
+          else
+            if prefer_self
+              value = old_value
+            else
+              value = new_value
+            end
+          end
+
+          # save value
+          self[key] = value
+        end
+        self
+      end
+
+      def set_environment(env, node)
+        @env = env
+        @node = node
+        self.each do |key, value|
+          if value.is_a?(Config::Object)
+            value.set_environment(env, node)
+          end
+        end
+      end
+
+      #
+      # like a reverse deep merge
+      # (self takes precedence)
+      #
+      def inherit_from!(object)
+        self.deep_merge!(object, true)
+      end
+
+      #
+      # Make a copy of ourselves, except only including the specified keys.
+      #
+      # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b'
+      #
+      def pick(*keys)
+        keys.map(&:to_s).inject(self.class.new(@manager)) do |hsh, key|
+          value = self.get(key)
+          if !value.nil?
+            hsh[key.gsub('.','_')] = value
+          end
+          hsh
+        end
+      end
+
+      def eval_file(filename)
+        evaluate_ruby(filename, File.read(filename))
+      end
+
+      protected
+
+      #
+      # walks the object tree, eval'ing all the attributes that are dynamic ruby (e.g. value starts with '= ')
+      #
+      def evaluate_everything(context)
+        keys.each do |key|
+          obj = fetch_value(key, context)
+          if is_required_value_not_set?(obj)
+            Util::log 0, :warning, "required property \"#{key}\" is not set in node \"#{node.name}\"."
+          elsif obj.is_a? Config::Object
+            obj.evaluate_everything(context)
+          end
+        end
+      end
+
+      #
+      # some keys need to be evaluated 'late', after all the other keys have been evaluated.
+      #
+      def late_evaluate_everything(context)
+        if @late_eval_list
+          @late_eval_list.each do |key, value|
+            self[key] = context.evaluate_ruby(key, value)
+            if is_required_value_not_set?(self[key])
+              Util::log 0, :warning, "required property \"#{key}\" is not set in node \"#{node.name}\"."
+            end
+          end
+        end
+        values.each do |obj|
+          if obj.is_a? Config::Object
+            obj.late_evaluate_everything(context)
+          end
+        end
+      end
+
+      #
+      # evaluates the string `value` as ruby in the context of self.
+      # (`key` is just passed for debugging purposes)
+      #
+      def evaluate_ruby(key, value)
+        self.instance_eval(value, key, 1)
+      rescue ConfigError => exc
+        raise exc # pass through
+      rescue SystemStackError => exc
+        Util::log 0, :error, "while evaluating node '#{self.name}'"
+        Util::log 0, "offending key: #{key}", :indent => 1
+        Util::log 0, "offending string: #{value}", :indent => 1
+        Util::log 0, "STACK OVERFLOW, BAILING OUT. There must be an eval loop of death (variables with circular dependencies).", :indent => 1
+        raise SystemExit.new(1)
+      rescue FileMissing => exc
+        Util::bail! do
+          if exc.options[:missing]
+            Util::log :missing, exc.options[:missing].gsub('$node', self.name).gsub('$file', exc.path)
+          else
+            Util::log :error, "while evaluating node '#{self.name}'"
+            Util::log "offending key: #{key}", :indent => 1
+            Util::log "offending string: #{value}", :indent => 1
+            Util::log "error message: no file '#{exc}'", :indent => 1
+          end
+          raise exc if DEBUG
+        end
+      rescue AssertionFailed => exc
+        Util.bail! do
+          Util::log :failed, "assertion while evaluating node '#{self.name}'"
+          Util::log 'assertion: %s' % exc.assertion, :indent => 1
+          Util::log "offending key: #{key}", :indent => 1
+          raise exc if DEBUG
+        end
+      rescue SyntaxError, StandardError => exc
+        Util::bail! do
+          Util::log :error, "while evaluating node '#{self.name}'"
+          Util::log "offending key: #{key}", :indent => 1
+          Util::log "offending string: #{value}", :indent => 1
+          Util::log "error message: #{exc.inspect}", :indent => 1
+          raise exc if DEBUG
+        end
+      end
+
+      private
+
+      #
+      # fetches the value for the key, evaluating the value as ruby if it begins with '='
+      #
+      def fetch_value(key, context=@node)
+        value = fetch(key, nil)
+        if value.is_a?(String) && value =~ /^=/
+          # strings prefix with '=' are evaluated as ruby code.
+          if value =~ /^=> (.*)$/
+            value = evaluate_later(key, $1)
+          elsif value =~ /^= (.*)$/
+            value = context.evaluate_ruby(key, $1)
+          end
+          self[key] = value
+        elsif value.is_a?(Proc)
+          # the value might be a proc, set by a 'control' file
+          self[key] = value.call
+        end
+        return value
+      end
+
+      def evaluate_later(key, value)
+        @late_eval_list ||= []
+        @late_eval_list << [key, value]
+        '<evaluate later>'
+      end
+
+      #
+      # when merging, we raise an error if this method returns true for the two values.
+      #
+      def type_mismatch?(old_value, new_value)
+        if old_value.is_a?(Boolean) && new_value.is_a?(Boolean)
+          # note: FalseClass and TrueClass are different classes
+          # so we can't do old_value.class == new_value.class
+          return false
+        elsif old_value.is_a?(String) && old_value =~ /^=/
+          # pass through macros, since we don't know what the type will eventually be.
+          return false
+        elsif new_value.is_a?(String) && new_value =~ /^=/
+          return false
+        elsif old_value.class == new_value.class
+          return false
+        else
+          return true
+        end
+      end
+
+      #
+      # returns true if the value has not been changed and the default is "REQUIRED"
+      #
+      def is_required_value_not_set?(value)
+        if value.is_a? Array
+          value == ["REQUIRED"]
+        else
+          value == "REQUIRED"
+        end
+      end
+
+    end # class
+  end # module
+end # module
\ No newline at end of file
diff --git a/lib/leap_cli/config/object_list.rb b/lib/leap_cli/config/object_list.rb
new file mode 100644
index 00000000..80f89d92
--- /dev/null
+++ b/lib/leap_cli/config/object_list.rb
@@ -0,0 +1,215 @@
+require 'tsort'
+
+module LeapCli
+  module Config
+    #
+    # A list of Config::Object instances (internally stored as a hash)
+    #
+    class ObjectList < Hash
+      include TSort
+
+      def initialize(config=nil)
+        if config
+          self.add(config['name'], config)
+        end
+      end
+
+      #
+      # If the key is a string, the Config::Object it references is returned.
+      #
+      # If the key is a hash, we treat it as a condition and filter all the Config::Objects using the condition.
+      # A new ObjectList is returned.
+      #
+      # Examples:
+      #
+      # nodes['vpn1']
+      #   node named 'vpn1'
+      #
+      # nodes[:public_dns => true]
+      #   all nodes with public dns
+      #
+      # nodes[:services => 'openvpn', 'location.country_code' => 'US']
+      #   all nodes with services containing 'openvpn' OR country code of US
+      #
+      # Sometimes, you want to do an OR condition with multiple conditions
+      # for the same field. Since hash keys must be unique, you can use
+      # an array representation instead:
+      #
+      # nodes[[:services, 'openvpn'], [:services, 'tor']]
+      #   nodes with openvpn OR tor service
+      #
+      # nodes[:services => 'openvpn'][:tags => 'production']
+      #   nodes with openvpn AND are production
+      #
+      def [](key)
+        if key.is_a?(Hash) || key.is_a?(Array)
+          filter(key)
+        else
+          super key.to_s
+        end
+      end
+
+      def exclude(node)
+        list = self.dup
+        list.delete(node.name)
+        return list
+      end
+
+      def each_node(&block)
+        self.keys.sort.each do |node_name|
+          yield self[node_name]
+        end
+      end
+
+      #
+      # filters this object list, producing a new list.
+      # filter is an array or a hash. see []
+      #
+      def filter(filter)
+        results = Config::ObjectList.new
+        filter.each do |field, match_value|
+          field = field.is_a?(Symbol) ? field.to_s : field
+          match_value = match_value.is_a?(Symbol) ? match_value.to_s : match_value
+          if match_value.is_a?(String) && match_value =~ /^!/
+            operator = :not_equal
+            match_value = match_value.sub(/^!/, '')
+          else
+            operator = :equal
+          end
+          each do |name, config|
+            value = config[field]
+            if value.is_a? Array
+              if operator == :equal && value.include?(match_value)
+                results[name] = config
+              elsif operator == :not_equal && !value.include?(match_value)
+                results[name] = config
+              end
+            elsif match_value.is_a? Array
+              if operator == :equal && match_value.include?(value)
+                results[name] = config
+              elsif operator == :not_equal && !match_value.include?(value)
+                results[name] = config
+              end
+            else
+              if operator == :equal && value == match_value
+                results[name] = config
+              elsif operator == :not_equal && value != match_value
+                results[name] = config
+              end
+            end
+          end
+        end
+        results
+      end
+
+      def add(name, object)
+        self[name] = object
+      end
+
+      #
+      # converts the hash of configs into an array of hashes, with ONLY the specified fields
+      #
+      def fields(*fields)
+        result = []
+        keys.sort.each do |name|
+          result << self[name].pick(*fields)
+        end
+        result
+      end
+
+      #
+      # like fields(), but returns an array of values instead of an array of hashes.
+      #
+      def field(field)
+        field = field.to_s
+        result = []
+        keys.sort.each do |name|
+          result << self[name].get(field)
+        end
+        result
+      end
+
+      #
+      # pick_fields(field1, field2, ...)
+      #
+      # generates a Hash from the object list, but with only the fields that are picked.
+      #
+      # If there are more than one field, then the result is a Hash of Hashes.
+      # If there is just one field, it is a simple map to the value.
+      #
+      # For example:
+      #
+      #   "neighbors" = "= nodes_like_me[:services => :couchdb].pick_fields('domain.full', 'ip_address')"
+      #
+      # generates this:
+      #
+      #   neighbors:
+      #     couch1:
+      #       domain_full: couch1.bitmask.net
+      #       ip_address: "10.5.5.44"
+      #     couch2:
+      #       domain_full: couch2.bitmask.net
+      #       ip_address: "10.5.5.52"
+      #
+      # But this:
+      #
+      #   "neighbors": "= nodes_like_me[:services => :couchdb].pick_fields('domain.full')"
+      #
+      # will generate this:
+      #
+      #   neighbors:
+      #     couch1: couch1.bitmask.net
+      #     couch2: couch2.bitmask.net
+      #
+      def pick_fields(*fields)
+        self.values.inject({}) do |hsh, node|
+          value = self[node.name].pick(*fields)
+          if fields.size == 1
+            value = value.values.first
+          end
+          hsh[node.name] = value
+          hsh
+        end
+      end
+
+      #
+      # Applies inherit_from! to all objects.
+      #
+      # 'env' specifies what environment should be for
+      # each object in the list.
+      #
+      def inherit_from!(object_list, env)
+        object_list.each do |name, object|
+          if self[name]
+            self[name].inherit_from!(object)
+          else
+            self[name] = object.duplicate(env)
+          end
+        end
+      end
+
+      #
+      # topographical sort based on test dependency
+      #
+      def tsort_each_node(&block)
+        self.each_key(&block)
+      end
+
+      def tsort_each_child(node_name, &block)
+        if self[node_name]
+          self[node_name].test_dependencies.each do |test_me_first|
+            if self[test_me_first] # TODO: in the future, allow for ability to optionally pull in all dependencies.
+                                   # not just the ones that pass the node filter.
+              yield(test_me_first)
+            end
+          end
+        end
+      end
+
+      def names_in_test_dependency_order
+        self.tsort
+      end
+
+    end
+  end
+end
diff --git a/lib/leap_cli/config/provider.rb b/lib/leap_cli/config/provider.rb
new file mode 100644
index 00000000..0d8bc1f3
--- /dev/null
+++ b/lib/leap_cli/config/provider.rb
@@ -0,0 +1,22 @@
+#
+# Configuration class for provider.json
+#
+
+module LeapCli; module Config
+  class Provider < Object
+    attr_reader :environment
+    def set_env(e)
+      if e == 'default'
+        @environment = nil
+      else
+        @environment = e
+      end
+    end
+    def provider
+      self
+    end
+    def validate!
+      # nothing here yet :(
+    end
+  end
+end; end
diff --git a/lib/leap_cli/config/secrets.rb b/lib/leap_cli/config/secrets.rb
new file mode 100644
index 00000000..ca851c74
--- /dev/null
+++ b/lib/leap_cli/config/secrets.rb
@@ -0,0 +1,87 @@
+# encoding: utf-8
+#
+# A class for the secrets.json file
+#
+
+module LeapCli; module Config
+
+  class Secrets < Object
+    attr_reader :node_list
+
+    def initialize(manager=nil)
+      super(manager)
+      @discovered_keys = {}
+    end
+
+    # we can't use fetch() or get(), since those already have special meanings
+    def retrieve(key, environment)
+      environment ||= 'default'
+      self.fetch(environment, {})[key.to_s]
+    end
+
+    def set(*args, &block)
+      if block_given?
+        set_with_block(*args, &block)
+      else
+        set_without_block(*args)
+      end
+    end
+
+    # searches over all keys matching the regexp, checking to see if the value
+    # has been already used by any of them.
+    def taken?(regexp, value, environment)
+      self.keys.grep(regexp).each do |key|
+        return true if self.retrieve(key, environment) == value
+      end
+      return false
+    end
+
+    def set_without_block(key, value, environment)
+      set_with_block(key, environment) {value}
+    end
+
+    def set_with_block(key, environment, &block)
+      environment ||= 'default'
+      key = key.to_s
+      @discovered_keys[environment] ||= {}
+      @discovered_keys[environment][key] = true
+      self[environment] ||= {}
+      self[environment][key] ||= yield
+    end
+
+    #
+    # if clean is true, then only secrets that have been discovered
+    # during this run will be exported.
+    #
+    # if environment is also pinned, then we will clean those secrets
+    # just for that environment.
+    #
+    # the clean argument should only be used when all nodes have
+    # been processed, otherwise secrets that are actually in use will
+    # get mistakenly removed.
+    #
+    def dump_json(clean=false)
+      pinned_env = LeapCli.leapfile.environment
+      if clean
+        self.each_key do |environment|
+          if pinned_env.nil? || pinned_env == environment
+            env = self[environment]
+            if env.nil?
+              raise StandardError.new("secrets.json file seems corrupted. No such environment '#{environment}'")
+            end
+            env.each_key do |key|
+              unless @discovered_keys[environment] && @discovered_keys[environment][key]
+                self[environment].delete(key)
+              end
+            end
+            if self[environment].empty?
+              self.delete(environment)
+            end
+          end
+        end
+      end
+      super()
+    end
+  end
+
+end; end
diff --git a/lib/leap_cli/config/sources.rb b/lib/leap_cli/config/sources.rb
new file mode 100644
index 00000000..aee860de
--- /dev/null
+++ b/lib/leap_cli/config/sources.rb
@@ -0,0 +1,11 @@
+# encoding: utf-8
+#
+# A class for the sources.json file
+#
+
+module LeapCli
+  module Config
+    class Sources < Object
+    end
+  end
+end
diff --git a/lib/leap_cli/config/tag.rb b/lib/leap_cli/config/tag.rb
new file mode 100644
index 00000000..6bd8d1e9
--- /dev/null
+++ b/lib/leap_cli/config/tag.rb
@@ -0,0 +1,25 @@
+#
+#
+# A class for node services or node tags.
+#
+#
+
+module LeapCli; module Config
+
+  class Tag < Object
+    attr_reader :node_list
+
+    def initialize(environment=nil)
+      super(environment)
+      @node_list = Config::ObjectList.new
+    end
+
+    # don't copy the node list pointer when this object is dup'ed.
+    def initialize_copy(orig)
+      super
+      @node_list = Config::ObjectList.new
+    end
+
+  end
+
+end; end