leap cli: move everything we can from leap_cli to leap_platform

21 files changed, 2058 insertions, 20 deletions

diff --git a/lib/leap/platform.rb b/lib/leap/platform.rb
new file mode 100644
index 00000000..9e6cadd5
--- /dev/null
+++ b/lib/leap/platform.rb
@@ -0,0 +1,99 @@
+module Leap
+
+  class Platform
+    class << self
+      #
+      # configuration
+      #
+
+      attr_reader :version
+      attr_reader :compatible_cli
+      attr_accessor :facts
+      attr_accessor :paths
+      attr_accessor :node_files
+      attr_accessor :monitor_username
+      attr_accessor :reserved_usernames
+
+      attr_accessor :hiera_dir
+      attr_accessor :hiera_path
+      attr_accessor :files_dir
+      attr_accessor :leap_dir
+      attr_accessor :init_path
+
+      attr_accessor :default_puppet_tags
+
+      def define(&block)
+        # some defaults:
+        @reserved_usernames = []
+        @hiera_dir  = '/etc/leap'
+        @hiera_path = '/etc/leap/hiera.yaml'
+        @leap_dir   = '/srv/leap'
+        @files_dir  = '/srv/leap/files'
+        @init_path  = '/srv/leap/initialized'
+        @default_puppet_tags = []
+
+        self.instance_eval(&block)
+
+        @version ||= Gem::Version.new("0.0")
+      end
+
+      def validate!(cli_version, compatible_platforms, leapfile)
+        if !compatible_with_cli?(cli_version) || !version_in_range?(compatible_platforms)
+          raise StandardError, "This leap command (v#{cli_version}) " +
+            "is not compatible with the platform #{leapfile.platform_directory_path} (v#{version}).\n   " +
+            "You need either leap command #{compatible_cli.first} to #{compatible_cli.last} or " +
+            "platform version #{compatible_platforms.first} to #{compatible_platforms.last}"
+        end
+      end
+
+      def version=(version)
+        @version = Gem::Version.new(version)
+      end
+
+      def compatible_cli=(range)
+        @compatible_cli = range
+        @minimum_cli_version = Gem::Version.new(range.first)
+        @maximum_cli_version = Gem::Version.new(range.last)
+      end
+
+      #
+      # return true if the cli_version is compatible with this platform.
+      #
+      def compatible_with_cli?(cli_version)
+        cli_version = Gem::Version.new(cli_version)
+        cli_version >= @minimum_cli_version && cli_version <= @maximum_cli_version
+      end
+
+      #
+      # return true if the platform version is within the specified range.
+      #
+      def version_in_range?(range)
+        if range.is_a? String
+          range = range.split('..')
+        end
+        minimum_platform_version = Gem::Version.new(range.first)
+        maximum_platform_version = Gem::Version.new(range.last)
+        @version >= minimum_platform_version && @version <= maximum_platform_version
+      end
+
+      def major_version
+        if @version.segments.first == 0
+          @version.segments[0..1].join('.')
+        else
+          @version.segments.first
+        end
+      end
+
+      def method_missing(method, *args)
+        puts
+        puts "WARNING:"
+        puts "  leap_cli is out of date and does not understand `#{method}`."
+        puts "  called from: #{caller.first}"
+        puts "  please upgrade to a newer leap_cli"
+      end
+
+    end
+
+  end
+
+end
\ No newline at end of file
diff --git a/lib/leap_cli/commands/deploy.rb b/lib/leap_cli/commands/deploy.rb
index 165ce588..d26b9905 100644
--- a/lib/leap_cli/commands/deploy.rb
+++ b/lib/leap_cli/commands/deploy.rb
@@ -89,6 +89,10 @@ module LeapCli
         end
       end
 
+      if nodes.empty?
+        return
+      end
+
       log :synching, "configuration files" do
         sync_hiera_config(nodes, options)
         sync_support_files(nodes, options)
diff --git a/lib/leap_cli/config/environment.rb b/lib/leap_cli/config/environment.rb
new file mode 100644
index 00000000..398fd023
--- /dev/null
+++ b/lib/leap_cli/config/environment.rb
@@ -0,0 +1,180 @@
+#
+# All configurations files can be isolated into separate environments.
+#
+# Each config json in each environment inherits from the default environment,
+# which in term inherits from the "_base_" environment:
+#
+# _base_             -- base provider in leap_platform
+# '- default         -- environment in provider dir when no env is set
+#    '- production   -- example environment
+#
+
+module LeapCli; module Config
+
+  class Environment
+    # the String name of the environment
+    attr_accessor :name
+
+    # the shared Manager object
+    attr_accessor :manager
+
+    # hashes of {name => Config::Object}
+    attr_accessor :services, :tags, :partials
+
+    # a Config::Provider
+    attr_accessor :provider
+
+    # a Config::Object
+    attr_accessor :common
+
+    # shared, non-inheritable
+    def nodes; @@nodes; end
+    def secrets; @@secrets; end
+
+    def initialize(manager, name, search_dir, parent, options={})
+      @@nodes ||= nil
+      @@secrets ||= nil
+
+      @manager = manager
+      @name    = name
+
+      load_provider_files(search_dir, options)
+
+      if parent
+        @services.inherit_from! parent.services, self
+            @tags.inherit_from! parent.tags    , self
+        @partials.inherit_from! parent.partials, self
+          @common.inherit_from! parent.common
+        @provider.inherit_from! parent.provider
+      end
+
+      if @provider
+        @provider.set_env(name)
+        @provider.validate!
+      end
+    end
+
+    def load_provider_files(search_dir, options)
+      #
+      # load empty environment if search_dir doesn't exist
+      #
+      if search_dir.nil? || !Dir.exist?(search_dir)
+        @services = Config::ObjectList.new
+        @tags     = Config::ObjectList.new
+        @partials = Config::ObjectList.new
+        @provider = Config::Provider.new
+        @common   = Config::Object.new
+        return
+      end
+
+      #
+      # inheritable
+      #
+      if options[:scope]
+        scope = options[:scope]
+        @services = load_all_json(Path.named_path([:service_env_config, '*', scope],  search_dir), Config::Tag, options)
+        @tags     = load_all_json(Path.named_path([:tag_env_config, '*', scope],      search_dir), Config::Tag, options)
+        @partials = load_all_json(Path.named_path([:service_env_config, '_*', scope], search_dir), Config::Tag, options)
+        @provider = load_json(    Path.named_path([:provider_env_config, scope],      search_dir), Config::Provider, options)
+        @common   = load_json(    Path.named_path([:common_env_config, scope],        search_dir), Config::Object, options)
+      else
+        @services = load_all_json(Path.named_path([:service_config, '*'],  search_dir), Config::Tag, options)
+        @tags     = load_all_json(Path.named_path([:tag_config, '*'],      search_dir), Config::Tag, options)
+        @partials = load_all_json(Path.named_path([:service_config, '_*'], search_dir), Config::Tag, options)
+        @provider = load_json(    Path.named_path(:provider_config,        search_dir), Config::Provider, options)
+        @common   = load_json(    Path.named_path(:common_config,          search_dir), Config::Object, options)
+      end
+
+      # remove 'name' from partials, since partials get merged with nodes
+      @partials.values.each {|partial| partial.delete('name'); }
+
+      #
+      # shared: currently non-inheritable
+      # load the first ones we find, and only those.
+      #
+      if @@nodes.nil? || @@nodes.empty?
+        @@nodes = load_all_json(Path.named_path([:node_config, '*'], search_dir), Config::Node, options)
+      end
+      if @@secrets.nil? || @@secrets.empty?
+        @@secrets = load_json(Path.named_path(:secrets_config, search_dir), Config::Secrets, options)
+      end
+    end
+
+    #
+    # Loads a json template file as a Hash (used only when creating a new node .json
+    # file for the first time).
+    #
+    def template(template)
+      path = Path.named_path([:template_config, template], Path.provider_base)
+      if File.exist?(path)
+        return load_json(path, Config::Object)
+      else
+        return nil
+      end
+    end
+
+    private
+
+    def load_all_json(pattern, object_class, options={})
+      results = Config::ObjectList.new
+      Dir.glob(pattern).each do |filename|
+        next if options[:no_dots] && File.basename(filename) !~ /^[^\.]*\.json$/
+        obj = load_json(filename, object_class)
+        if obj
+          name = File.basename(filename).force_encoding('utf-8').sub(/^([^\.]+).*\.json$/,'\1')
+          obj['name'] ||= name
+          if options[:env]
+            obj.environment = options[:env]
+          end
+          results[name] = obj
+        end
+      end
+      results
+    end
+
+    def load_json(filename, object_class, options={})
+      if !File.exist?(filename)
+        return object_class.new(self)
+      end
+
+      Util::log :loading, filename, 3
+
+      #
+      # Read a JSON file, strip out comments.
+      #
+      # UTF8 is the default encoding for JSON, but others are allowed:
+      # https://www.ietf.org/rfc/rfc4627.txt
+      #
+      buffer = StringIO.new
+      File.open(filename, "rb", :encoding => 'UTF-8') do |f|
+        while (line = f.gets)
+          next if line =~ /^\s*\/\//
+          buffer << line
+        end
+      end
+
+      #
+      # force UTF-8
+      #
+      if $ruby_version >= [1,9]
+        string = buffer.string.force_encoding('utf-8')
+      else
+        string = Iconv.conv("UTF-8//IGNORE", "UTF-8", buffer.string)
+      end
+
+      # parse json
+      begin
+        hash = JSON.parse(string, :object_class => Hash, :array_class => Array) || {}
+      rescue SyntaxError, JSON::ParserError => exc
+        Util::log 0, :error, 'in file "%s":' % filename
+        Util::log 0, exc.to_s, :indent => 1
+        return nil
+      end
+      object = object_class.new(self)
+      object.deep_merge!(hash)
+      return object
+    end
+
+  end # end Environment
+
+end; end
\ No newline at end of file
diff --git a/lib/leap_cli/config/filter.rb b/lib/leap_cli/config/filter.rb
new file mode 100644
index 00000000..27502577
--- /dev/null
+++ b/lib/leap_cli/config/filter.rb
@@ -0,0 +1,178 @@
+#
+# Many leap_cli commands accept a list of filters to select a subset of nodes for the command to
+# be applied to. This class is a helper for manager to run these filters.
+#
+# Classes other than Manager should not use this class.
+#
+# Filter rules:
+#
+# * A filter consists of a list of tokens
+# * A token may be a service name, tag name, environment name, or node name.
+# * Each token may be optionally prefixed with a plus sign.
+# * Multiple tokens with a plus are treated as an OR condition,
+#   but treated as an AND condition with the plus sign.
+#
+# For example
+#
+# * openvpn +development => all nodes with service 'openvpn' AND environment 'development'
+# * openvpn seattle => all nodes with service 'openvpn' OR tag 'seattle'.
+#
+# There can only be one environment specified. Typically, there are also tags
+# for each environment name. These name are treated as environments, not tags.
+#
+module LeapCli
+  module Config
+    class Filter
+
+      #
+      # filter -- array of strings, each one a filter
+      # options -- hash, possible keys include
+      #   :nopin -- disregard environment pinning
+      #   :local -- if false, disallow local nodes
+      #
+      # A nil value in the filters array indicates
+      # the default environment. This is in order to support
+      # calls like `manager.filter(environments)`
+      #
+      def initialize(filters, options, manager)
+        @filters = filters.nil? ? [] : filters.dup
+        @environments = []
+        @options = options
+        @manager = manager
+
+        # split filters by pulling out items that happen
+        # to be environment names.
+        if LeapCli.leapfile.environment.nil? || @options[:nopin]
+          @environments = []
+        else
+          @environments = [LeapCli.leapfile.environment]
+        end
+        @filters.select! do |filter|
+          if filter.nil?
+            @environments << nil unless @environments.include?(nil)
+            false
+          else
+            filter_text = filter.sub(/^\+/,'')
+            if is_environment?(filter_text)
+              if filter_text == LeapCli.leapfile.environment
+                # silently ignore already pinned environments
+              elsif (filter =~ /^\+/ || @filters.first == filter) && !@environments.empty?
+                LeapCli::Util.bail! do
+                  LeapCli.log "Environments are exclusive: no node is in two environments." do
+                    LeapCli.log "Tried to filter on '#{@environments.join('\' AND \'')}' AND '#{filter_text}'"
+                  end
+                end
+              else
+                @environments << filter_text
+              end
+              false
+            else
+              true
+            end
+          end
+        end
+
+        # don't let the first filter have a + prefix
+        if @filters[0] =~ /^\+/
+          @filters[0] = @filters[0][1..-1]
+        end
+      end
+
+      # actually run the filter, returns a filtered list of nodes
+      def nodes()
+        if @filters.empty?
+          return nodes_for_empty_filter
+        else
+          return nodes_for_filter
+        end
+      end
+
+      private
+
+      def nodes_for_empty_filter
+        node_list = @manager.nodes
+        if @environments.any?
+          node_list = node_list[ @environments.collect{|e|[:environment, env_to_filter(e)]} ]
+        end
+        if @options[:local] === false
+          node_list = node_list[:environment => '!local']
+        end
+        if @options[:disabled] === false
+          node_list = node_list[:environment => '!disabled']
+        end
+        node_list
+      end
+
+      def nodes_for_filter
+        node_list = Config::ObjectList.new
+        @filters.each do |filter|
+          if filter =~ /^\+/
+            keep_list = nodes_for_name(filter[1..-1])
+            node_list.delete_if do |name, node|
+              if keep_list[name]
+                false
+              else
+                true
+              end
+            end
+          else
+            node_list.merge!(nodes_for_name(filter))
+          end
+        end
+        node_list
+      end
+
+      private
+
+      #
+      # returns a set of nodes corresponding to a single name,
+      # where name could be a node name, service name, or tag name.
+      #
+      # For services and tags, we only include nodes for the
+      # environments that are active
+      #
+      def nodes_for_name(name)
+        if node = @manager.nodes[name]
+          return Config::ObjectList.new(node)
+        elsif @environments.empty?
+          if @manager.services[name]
+            return @manager.env('_all_').services[name].node_list
+          elsif @manager.tags[name]
+            return @manager.env('_all_').tags[name].node_list
+          else
+            LeapCli.log :warning, "filter '#{name}' does not match any node names, tags, services, or environments."
+            return Config::ObjectList.new
+          end
+        else
+          node_list = Config::ObjectList.new
+          if @manager.services[name]
+            @environments.each do |env|
+              node_list.merge!(@manager.env(env).services[name].node_list)
+            end
+          elsif @manager.tags[name]
+            @environments.each do |env|
+              node_list.merge!(@manager.env(env).tags[name].node_list)
+            end
+          else
+            LeapCli.log :warning, "filter '#{name}' does not match any node names, tags, services, or environments."
+          end
+          return node_list
+        end
+      end
+
+      #
+      # when pinning, we use the name 'default' to specify nodes
+      # without an environment set, but when filtering, we need to filter
+      # on :environment => nil.
+      #
+      def env_to_filter(environment)
+        environment == 'default' ? nil : environment
+      end
+
+      def is_environment?(text)
+        text == 'default' || @manager.environment_names.include?(text)
+      end
+
+    end
+  end
+end
diff --git a/lib/leap_cli/config/manager.rb b/lib/leap_cli/config/manager.rb
new file mode 100644
index 00000000..aea1d322
--- /dev/null
+++ b/lib/leap_cli/config/manager.rb
@@ -0,0 +1,422 @@
+# encoding: utf-8
+
+require 'json/pure'
+
+if $ruby_version < [1,9]
+  require 'iconv'
+end
+
+module LeapCli
+  module Config
+
+    #
+    # A class to manage all the objects in all the configuration files.
+    #
+    class Manager
+
+      def initialize
+        @environments = {} # hash of `Environment` objects, keyed by name.
+        Config::Object.send(:include, LeapCli::Macro)
+      end
+
+      ##
+      ## ATTRIBUTES
+      ##
+
+      #
+      # returns the Hash of the contents of facts.json
+      #
+      def facts
+        @facts ||= begin
+          content = Util.read_file(:facts)
+          if !content || content.empty?
+            content = "{}"
+          end
+          JSON.parse(content)
+        rescue SyntaxError, JSON::ParserError => exc
+          Util::bail! "Could not parse facts.json -- #{exc}"
+        end
+      end
+
+      #
+      # returns an Array of all the environments defined for this provider.
+      # the returned array includes nil (for the default environment)
+      #
+      def environment_names
+        @environment_names ||= begin
+          [nil] + (env.tags.field('environment') + env.nodes.field('environment')).compact.uniq
+        end
+      end
+
+      #
+      # Returns the appropriate environment variable
+      #
+      def env(env=nil)
+        @environments[env || 'default']
+      end
+
+      #
+      # The default accessors
+      #
+      # For these defaults, use 'default' environment, or whatever
+      # environment is pinned.
+      #
+      # I think it might be an error that these are ever used
+      # and I would like to get rid of them.
+      #
+      def services; env(default_environment).services; end
+      def tags;     env(default_environment).tags;     end
+      def partials; env(default_environment).partials; end
+      def provider; env(default_environment).provider; end
+      def common;   env(default_environment).common;   end
+      def secrets;  env(default_environment).secrets;  end
+      def nodes;    env(default_environment).nodes;    end
+      def template(*args)
+        self.env.template(*args)
+      end
+
+      def default_environment
+        LeapCli.leapfile.environment
+      end
+
+      ##
+      ## IMPORT EXPORT
+      ##
+
+      def add_environment(args)
+        if args[:inherit]
+          parent = @environments[args.delete(:inherit)]
+        else
+          parent = nil
+        end
+        @environments[args[:name]] = Environment.new(
+          self,
+          args.delete(:name),
+          args.delete(:dir),
+          parent,
+          args
+        )
+      end
+
+      #
+      # load .json configuration files
+      #
+      def load(options = {})
+        @provider_dir = Path.provider
+
+        # load base
+        add_environment(name: '_base_', dir: Path.provider_base)
+
+        # load provider
+        Util::assert_files_exist!(Path.named_path(:provider_config, @provider_dir))
+        add_environment(name: 'default', dir: @provider_dir,
+          inherit: '_base_', no_dots: true)
+
+        # create a special '_all_' environment, used for tracking
+        # the union of all the environments
+        add_environment(name: '_all_', inherit: 'default')
+
+        # load environments
+        environment_names.each do |ename|
+          if ename
+            LeapCli.log 3, :loading, '%s environment...' % ename
+            add_environment(name: ename, dir: @provider_dir,
+              inherit: 'default', scope: ename)
+          end
+        end
+
+        # apply inheritance
+        env.nodes.each do |name, node|
+          Util::assert! name =~ /^[0-9a-z-]+$/, "Illegal character(s) used in node name '#{name}'"
+          env.nodes[name] = apply_inheritance(node)
+        end
+
+        # do some node-list post-processing
+        cleanup_node_lists(options)
+
+        # apply control files
+        env.nodes.each do |name, node|
+          control_files(node).each do |file|
+            begin
+              node.eval_file file
+            rescue ConfigError => exc
+              if options[:continue_on_error]
+                exc.log
+              else
+                raise exc
+              end
+            end
+          end
+        end
+      end
+
+      #
+      # save compiled hiera .yaml files
+      #
+      # if a node_list is specified, only update those .yaml files.
+      # otherwise, update all files, destroying files that are no longer used.
+      #
+      def export_nodes(node_list=nil)
+        updated_hiera = []
+        updated_files = []
+        existing_hiera = nil
+        existing_files = nil
+
+        unless node_list
+          node_list = env.nodes
+          existing_hiera = Dir.glob(Path.named_path([:hiera, '*'], @provider_dir))
+          existing_files = Dir.glob(Path.named_path([:node_files_dir, '*'], @provider_dir))
+        end
+
+        node_list.each_node do |node|
+          filepath = Path.named_path([:node_files_dir, node.name], @provider_dir)
+          hierapath = Path.named_path([:hiera, node.name], @provider_dir)
+          Util::write_file!(hierapath, node.dump_yaml)
+          updated_files << filepath
+          updated_hiera << hierapath
+        end
+
+        if @disabled_nodes
+          # make disabled nodes appear as if they are still active
+          @disabled_nodes.each_node do |node|
+            updated_files << Path.named_path([:node_files_dir, node.name], @provider_dir)
+            updated_hiera << Path.named_path([:hiera, node.name], @provider_dir)
+          end
+        end
+
+        # remove files that are no longer needed
+        if existing_hiera
+          (existing_hiera - updated_hiera).each do |filepath|
+            Util::remove_file!(filepath)
+          end
+        end
+        if existing_files
+          (existing_files - updated_files).each do |filepath|
+            Util::remove_directory!(filepath)
+          end
+        end
+      end
+
+      def export_secrets(clean_unused_secrets = false)
+        if env.secrets.any?
+          Util.write_file!([:secrets_config, @provider_dir], env.secrets.dump_json(clean_unused_secrets) + "\n")
+        end
+      end
+
+      ##
+      ## FILTERING
+      ##
+
+      #
+      # returns a node list consisting only of nodes that satisfy the filter criteria.
+      #
+      # filter: condition [condition] [condition] [+condition]
+      # condition: [node_name | service_name | tag_name | environment_name]
+      #
+      # if conditions is prefixed with +, then it works like an AND. Otherwise, it works like an OR.
+      #
+      # args:
+      # filter -- array of filter terms, one per item
+      #
+      # options:
+      # :local -- if :local is false and the filter is empty, then local nodes are excluded.
+      # :nopin -- if true, ignore environment pinning
+      #
+      def filter(filters=nil, options={})
+        Filter.new(filters, options, self).nodes()
+      end
+
+      #
+      # same as filter(), but exits if there is no matching nodes
+      #
+      def filter!(filters, options={})
+        node_list = filter(filters, options)
+        Util::assert! node_list.any?, "Could not match any nodes from '#{filters.join ' '}'"
+        return node_list
+      end
+
+      #
+      # returns a single Config::Object that corresponds to a Node.
+      #
+      def node(name)
+        if name =~ /\./
+          # probably got a fqdn, since periods are not allowed in node names.
+          # so, take the part before the first period as the node name
+          name = name.split('.').first
+        end
+        env.nodes[name]
+      end
+
+      #
+      # returns a single node that is disabled
+      #
+      def disabled_node(name)
+        @disabled_nodes[name]
+      end
+
+      #
+      # yields each node, in sorted order
+      #
+      def each_node(&block)
+        env.nodes.each_node(&block)
+      end
+
+      def reload_node!(node)
+        env.nodes[node.name] = apply_inheritance!(node)
+      end
+
+      ##
+      ## CONNECTIONS
+      ##
+
+      class ConnectionList < Array
+        def add(data={})
+          self << {
+            "from" => data[:from],
+            "to" => data[:to],
+            "port" => data[:port]
+          }
+        end
+      end
+
+      def connections
+        @connections ||= ConnectionList.new
+      end
+
+      ##
+      ## PRIVATE
+      ##
+
+      private
+
+      #
+      # makes a node inherit options from appropriate the common, service, and tag json files.
+      #
+      def apply_inheritance(node, throw_exceptions=false)
+        new_node = Config::Node.new(nil)
+        node_env = guess_node_env(node)
+        new_node.set_environment(node_env, new_node)
+
+        # inherit from common
+        new_node.deep_merge!(node_env.common)
+
+        # inherit from services
+        if node['services']
+          node['services'].to_a.each do |node_service|
+            service = node_env.services[node_service]
+            if service.nil?
+              msg = 'in node "%s": the service "%s" does not exist.' % [node['name'], node_service]
+              LeapCli.log 0, :error, msg
+              raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions
+            else
+              new_node.deep_merge!(service)
+            end
+          end
+        end
+
+        # inherit from tags
+        if node.vagrant?
+          node['tags'] = (node['tags'] || []).to_a + ['local']
+        end
+        if node['tags']
+          node['tags'].to_a.each do |node_tag|
+            tag = node_env.tags[node_tag]
+            if tag.nil?
+              msg = 'in node "%s": the tag "%s" does not exist.' % [node['name'], node_tag]
+              log 0, :error, msg
+              raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions
+            else
+              new_node.deep_merge!(tag)
+            end
+          end
+        end
+
+        # inherit from node
+        new_node.deep_merge!(node)
+        return new_node
+      end
+
+      def apply_inheritance!(node)
+        apply_inheritance(node, true)
+      end
+
+      #
+      # Guess the environment of the node from the tag names.
+      #
+      # Technically, this is wrong: a tag that sets the environment might not be
+      # named the same as the environment. This code assumes that it is.
+      #
+      # Unfortunately, it is a chicken and egg problem. We need to know the nodes
+      # likely environment in order to apply the inheritance that will actually
+      # determine the node's properties.
+      #
+      def guess_node_env(node)
+        if node.vagrant?
+          return self.env("local")
+        else
+          environment = self.env(default_environment)
+          if node['tags']
+            node['tags'].to_a.each do |tag|
+              if self.environment_names.include?(tag)
+                environment = self.env(tag)
+              end
+            end
+          end
+          return environment
+        end
+      end
+
+      #
+      # does some final clean at the end of loading nodes.
+      # this includes removing disabled nodes, and populating
+      # the services[x].node_list and tags[x].node_list
+      #
+      def cleanup_node_lists(options)
+        @disabled_nodes = Config::ObjectList.new
+        env.nodes.each do |name, node|
+          if node.enabled || options[:include_disabled]
+            if node['services']
+              node['services'].to_a.each do |node_service|
+                env(node.environment).services[node_service].node_list.add(node.name, node)
+                env('_all_').services[node_service].node_list.add(node.name, node)
+              end
+            end
+            if node['tags']
+              node['tags'].to_a.each do |node_tag|
+                env(node.environment).tags[node_tag].node_list.add(node.name, node)
+                env('_all_').tags[node_tag].node_list.add(node.name, node)
+              end
+            end
+          elsif !options[:include_disabled]
+            LeapCli.log 2, :skipping, "disabled node #{name}."
+            env.nodes.delete(name)
+            @disabled_nodes[name] = node
+          end
+        end
+      end
+
+      #
+      # returns a list of 'control' files for this node.
+      # a control file is like a service or a tag JSON file, but it contains
+      # raw ruby code that gets evaluated in the context of the node.
+      # Yes, this entirely breaks our functional programming model
+      # for JSON generation.
+      #
+      def control_files(node)
+        files = []
+        [Path.provider_base, @provider_dir].each do |provider_dir|
+          [['services', :service_config], ['tags', :tag_config]].each do |attribute, path_sym|
+            node[attribute].each do |attr_value|
+              path = Path.named_path([path_sym, "#{attr_value}.rb"], provider_dir).sub(/\.json$/,'')
+              if File.exist?(path)
+                files << path
+              end
+            end
+          end
+        end
+        return files
+      end
+
+    end
+  end
+end
diff --git a/lib/leap_cli/config/node.rb b/lib/leap_cli/config/node.rb
new file mode 100644
index 00000000..f8ec0527
--- /dev/null
+++ b/lib/leap_cli/config/node.rb
@@ -0,0 +1,78 @@
+#
+# Configuration for a 'node' (a server in the provider's infrastructure)
+#
+
+require 'ipaddr'
+
+module LeapCli; module Config
+
+  class Node < Object
+    attr_accessor :file_paths
+
+    def initialize(environment=nil)
+      super(environment)
+      @node = self
+      @file_paths = []
+    end
+
+    #
+    # returns true if this node has an ip address in the range of the vagrant network
+    #
+    def vagrant?
+      begin
+        vagrant_range = IPAddr.new LeapCli.leapfile.vagrant_network
+      rescue ArgumentError => exc
+        Util::bail! { Util::log :invalid, "ip address '#{@node.ip_address}' vagrant.network" }
+      end
+
+      begin
+        ip_address = IPAddr.new @node.get('ip_address')
+      rescue ArgumentError => exc
+        Util::log :warning, "invalid ip address '#{@node.get('ip_address')}' for node '#{@node.name}'"
+      end
+      return vagrant_range.include?(ip_address)
+    end
+
+    #
+    # Return a hash table representation of ourselves, with the key equal to the @node.name,
+    # and the value equal to the fields specified in *keys.
+    #
+    # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b'
+    #
+    # compare to Object#pick(*keys). This method is the sames as Config::ObjectList#pick_fields,
+    # but works on a single node.
+    #
+    # Example:
+    #
+    #  node.pick('domain.internal') =>
+    #
+    #    {
+    #      'node1': {
+    #        'domain_internal': 'node1.example.i'
+    #      }
+    #    }
+    #
+    def pick_fields(*keys)
+      {@node.name => self.pick(*keys)}
+    end
+
+    #
+    # can be overridden by the platform.
+    # returns a list of node names that should be tested before this node
+    #
+    def test_dependencies
+      []
+    end
+
+    # returns a string list of supported ssh host key algorithms for this node.
+    # or an empty string if it could not be determined
+    def supported_ssh_host_key_algorithms
+      require 'leap_cli/ssh'
+      @host_key_algo ||= LeapCli::SSH::Key.supported_host_key_algorithms(
+        Util.read_file([:node_ssh_pub_key, @node.name])
+      )
+    end
+
+  end
+
+end; end
diff --git a/lib/leap_cli/config/object.rb b/lib/leap_cli/config/object.rb
new file mode 100644
index 00000000..b117c2f0
--- /dev/null
+++ b/lib/leap_cli/config/object.rb
@@ -0,0 +1,428 @@
+# encoding: utf-8
+
+require 'erb'
+require 'json/pure'  # pure ruby implementation is required for our sorted trick to work.
+
+if $ruby_version < [1,9]
+  $KCODE = 'UTF8'
+end
+require 'ya2yaml' # pure ruby yaml
+
+module LeapCli
+  module Config
+
+    #
+    # This class represents the configuration for a single node, service, or tag.
+    # Also, all the nested hashes are also of this type.
+    #
+    # It is called 'object' because it corresponds to an Object in JSON.
+    #
+    class Object < Hash
+
+      attr_reader :env
+      attr_reader :node
+
+      def initialize(environment=nil, node=nil)
+        raise ArgumentError unless environment.nil? || environment.is_a?(Config::Environment)
+        @env = environment
+        # an object that is a node as @node equal to self, otherwise all the
+        # child objects point back to the top level node.
+        @node = node || self
+      end
+
+      def manager
+        @env.manager
+      end
+
+      #
+      # TODO: deprecate node.global()
+      #
+      def global
+        @env
+      end
+
+      def environment=(e)
+        self.store('environment', e)
+      end
+
+      def environment
+        self['environment']
+      end
+
+      def duplicate(env)
+        new_object = self.deep_dup
+        new_object.set_environment(env, new_object)
+      end
+
+      #
+      # export YAML
+      #
+      # We use pure ruby yaml exporter ya2yaml instead of SYCK or PSYCH because it
+      # allows us greater compatibility regardless of installed ruby version and
+      # greater control over how the yaml is exported (sorted keys, in particular).
+      #
+      def dump_yaml
+        evaluate(@node)
+        sorted_ya2yaml(:syck_compatible => true)
+      end
+
+      #
+      # export JSON
+      #
+      def dump_json(options={})
+        evaluate(@node)
+        if options[:format] == :compact
+          return self.to_json
+        else
+          excluded = {}
+          if options[:exclude]
+            options[:exclude].each do |key|
+              excluded[key] = self[key]
+              self.delete(key)
+            end
+          end
+          json_str = JSON.sorted_generate(self)
+          if excluded.any?
+            self.merge!(excluded)
+          end
+          return json_str
+        end
+      end
+
+      def evaluate(context=@node)
+        evaluate_everything(context)
+        late_evaluate_everything(context)
+      end
+
+      ##
+      ## FETCHING VALUES
+      ##
+
+      def [](key)
+        get(key)
+      end
+
+      # Overrride some default methods in Hash that are likely to
+      # be used as attributes.
+      alias_method :hkey, :key
+      def key; get('key'); end
+
+      #
+      # make hash addressable like an object (e.g. obj['name'] available as obj.name)
+      #
+      def method_missing(method, *args, &block)
+        get!(method)
+      end
+
+      def get(key)
+        begin
+          get!(key)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      # override behavior of #default() from Hash
+      def default
+        get!('default')
+      end
+
+      #
+      # Like a normal Hash#[], except:
+      #
+      # (1) lazily eval dynamic values when we encounter them. (i.e. strings that start with "= ")
+      #
+      # (2) support for nested references in a single string (e.g. ['a.b'] is the same as ['a']['b'])
+      #     the dot path is always absolute, starting at the top-most object.
+      #
+      def get!(key)
+        key = key.to_s
+        if self.has_key?(key)
+          fetch_value(key)
+        elsif key =~ /\./
+          # for keys with with '.' in them, we start from the root object (@node).
+          keys = key.split('.')
+          value = self.get!(keys.first)
+          if value.is_a? Config::Object
+            value.get!(keys[1..-1].join('.'))
+          else
+            value
+          end
+        else
+          raise NoMethodError.new(key, "No method '#{key}' for #{self.class}")
+        end
+      end
+
+      ##
+      ## COPYING
+      ##
+
+      #
+      # A deep (recursive) merge with another Config::Object.
+      #
+      # If prefer_self is set to true, the value from self will be picked when there is a conflict
+      # that cannot be merged.
+      #
+      # Merging rules:
+      #
+      # - If a value is a hash, we recursively merge it.
+      # - If the value is simple, like a string, the new one overwrites the value.
+      # - If the value is an array:
+      #   - If both old and new values are arrays, the new one replaces the old.
+      #   - If one of the values is simple but the other is an array, the simple is added to the array.
+      #
+      def deep_merge!(object, prefer_self=false)
+        object.each do |key,new_value|
+          if self.has_key?('+'+key)
+            mode = :add
+            old_value = self.fetch '+'+key, nil
+            self.delete('+'+key)
+          elsif self.has_key?('-'+key)
+            mode = :subtract
+            old_value = self.fetch '-'+key, nil
+            self.delete('-'+key)
+          elsif self.has_key?('!'+key)
+            mode = :replace
+            old_value = self.fetch '!'+key, nil
+            self.delete('!'+key)
+          else
+            mode = :normal
+            old_value = self.fetch key, nil
+          end
+
+          # clean up boolean
+          new_value = true  if new_value == "true"
+          new_value = false if new_value == "false"
+          old_value = true  if old_value == "true"
+          old_value = false if old_value == "false"
+
+          # force replace?
+          if mode == :replace && prefer_self
+            value = old_value
+
+          # merge hashes
+          elsif old_value.is_a?(Hash) || new_value.is_a?(Hash)
+            value = Config::Object.new(@env, @node)
+            old_value.is_a?(Hash) ? value.deep_merge!(old_value) : (value[key] = old_value if !old_value.nil?)
+            new_value.is_a?(Hash) ? value.deep_merge!(new_value, prefer_self) : (value[key] = new_value if !new_value.nil?)
+
+          # merge nil
+          elsif new_value.nil?
+            value = old_value
+          elsif old_value.nil?
+            value = new_value
+
+          # merge arrays when one value is not an array
+          elsif old_value.is_a?(Array) && !new_value.is_a?(Array)
+            (value = (old_value.dup << new_value).compact.uniq).delete('REQUIRED')
+          elsif new_value.is_a?(Array) && !old_value.is_a?(Array)
+            (value = (new_value.dup << old_value).compact.uniq).delete('REQUIRED')
+
+          # merge two arrays
+          elsif old_value.is_a?(Array) && new_value.is_a?(Array)
+            if mode == :add
+              value = (old_value + new_value).sort.uniq
+            elsif mode == :subtract
+              value = new_value - old_value
+            elsif prefer_self
+              value = old_value
+            else
+              value = new_value
+            end
+
+          # catch errors
+          elsif type_mismatch?(old_value, new_value)
+            raise 'Type mismatch. Cannot merge %s (%s) with %s (%s). Key is "%s", name is "%s".' % [
+              old_value.inspect, old_value.class,
+              new_value.inspect, new_value.class,
+              key, self.class
+            ]
+
+          # merge simple strings & numbers
+          else
+            if prefer_self
+              value = old_value
+            else
+              value = new_value
+            end
+          end
+
+          # save value
+          self[key] = value
+        end
+        self
+      end
+
+      def set_environment(env, node)
+        @env = env
+        @node = node
+        self.each do |key, value|
+          if value.is_a?(Config::Object)
+            value.set_environment(env, node)
+          end
+        end
+      end
+
+      #
+      # like a reverse deep merge
+      # (self takes precedence)
+      #
+      def inherit_from!(object)
+        self.deep_merge!(object, true)
+      end
+
+      #
+      # Make a copy of ourselves, except only including the specified keys.
+      #
+      # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b'
+      #
+      def pick(*keys)
+        keys.map(&:to_s).inject(self.class.new(@manager)) do |hsh, key|
+          value = self.get(key)
+          if !value.nil?
+            hsh[key.gsub('.','_')] = value
+          end
+          hsh
+        end
+      end
+
+      def eval_file(filename)
+        evaluate_ruby(filename, File.read(filename))
+      end
+
+      protected
+
+      #
+      # walks the object tree, eval'ing all the attributes that are dynamic ruby (e.g. value starts with '= ')
+      #
+      def evaluate_everything(context)
+        keys.each do |key|
+          obj = fetch_value(key, context)
+          if is_required_value_not_set?(obj)
+            Util::log 0, :warning, "required property \"#{key}\" is not set in node \"#{node.name}\"."
+          elsif obj.is_a? Config::Object
+            obj.evaluate_everything(context)
+          end
+        end
+      end
+
+      #
+      # some keys need to be evaluated 'late', after all the other keys have been evaluated.
+      #
+      def late_evaluate_everything(context)
+        if @late_eval_list
+          @late_eval_list.each do |key, value|
+            self[key] = context.evaluate_ruby(key, value)
+            if is_required_value_not_set?(self[key])
+              Util::log 0, :warning, "required property \"#{key}\" is not set in node \"#{node.name}\"."
+            end
+          end
+        end
+        values.each do |obj|
+          if obj.is_a? Config::Object
+            obj.late_evaluate_everything(context)
+          end
+        end
+      end
+
+      #
+      # evaluates the string `value` as ruby in the context of self.
+      # (`key` is just passed for debugging purposes)
+      #
+      def evaluate_ruby(key, value)
+        self.instance_eval(value, key, 1)
+      rescue ConfigError => exc
+        raise exc # pass through
+      rescue SystemStackError => exc
+        Util::log 0, :error, "while evaluating node '#{self.name}'"
+        Util::log 0, "offending key: #{key}", :indent => 1
+        Util::log 0, "offending string: #{value}", :indent => 1
+        Util::log 0, "STACK OVERFLOW, BAILING OUT. There must be an eval loop of death (variables with circular dependencies).", :indent => 1
+        raise SystemExit.new(1)
+      rescue FileMissing => exc
+        Util::bail! do
+          if exc.options[:missing]
+            Util::log :missing, exc.options[:missing].gsub('$node', self.name).gsub('$file', exc.path)
+          else
+            Util::log :error, "while evaluating node '#{self.name}'"
+            Util::log "offending key: #{key}", :indent => 1
+            Util::log "offending string: #{value}", :indent => 1
+            Util::log "error message: no file '#{exc}'", :indent => 1
+          end
+          raise exc if DEBUG
+        end
+      rescue AssertionFailed => exc
+        Util.bail! do
+          Util::log :failed, "assertion while evaluating node '#{self.name}'"
+          Util::log 'assertion: %s' % exc.assertion, :indent => 1
+          Util::log "offending key: #{key}", :indent => 1
+          raise exc if DEBUG
+        end
+      rescue SyntaxError, StandardError => exc
+        Util::bail! do
+          Util::log :error, "while evaluating node '#{self.name}'"
+          Util::log "offending key: #{key}", :indent => 1
+          Util::log "offending string: #{value}", :indent => 1
+          Util::log "error message: #{exc.inspect}", :indent => 1
+          raise exc if DEBUG
+        end
+      end
+
+      private
+
+      #
+      # fetches the value for the key, evaluating the value as ruby if it begins with '='
+      #
+      def fetch_value(key, context=@node)
+        value = fetch(key, nil)
+        if value.is_a?(String) && value =~ /^=/
+          if value =~ /^=> (.*)$/
+            value = evaluate_later(key, $1)
+          elsif value =~ /^= (.*)$/
+            value = context.evaluate_ruby(key, $1)
+          end
+          self[key] = value
+        end
+        return value
+      end
+
+      def evaluate_later(key, value)
+        @late_eval_list ||= []
+        @late_eval_list << [key, value]
+        '<evaluate later>'
+      end
+
+      #
+      # when merging, we raise an error if this method returns true for the two values.
+      #
+      def type_mismatch?(old_value, new_value)
+        if old_value.is_a?(Boolean) && new_value.is_a?(Boolean)
+          # note: FalseClass and TrueClass are different classes
+          # so we can't do old_value.class == new_value.class
+          return false
+        elsif old_value.is_a?(String) && old_value =~ /^=/
+          # pass through macros, since we don't know what the type will eventually be.
+          return false
+        elsif new_value.is_a?(String) && new_value =~ /^=/
+          return false
+        elsif old_value.class == new_value.class
+          return false
+        else
+          return true
+        end
+      end
+
+      #
+      # returns true if the value has not been changed and the default is "REQUIRED"
+      #
+      def is_required_value_not_set?(value)
+        if value.is_a? Array
+          value == ["REQUIRED"]
+        else
+          value == "REQUIRED"
+        end
+      end
+
+    end # class
+  end # module
+end # module
\ No newline at end of file
diff --git a/lib/leap_cli/config/object_list.rb b/lib/leap_cli/config/object_list.rb
new file mode 100644
index 00000000..f9299a61
--- /dev/null
+++ b/lib/leap_cli/config/object_list.rb
@@ -0,0 +1,209 @@
+require 'tsort'
+
+module LeapCli
+  module Config
+    #
+    # A list of Config::Object instances (internally stored as a hash)
+    #
+    class ObjectList < Hash
+      include TSort
+
+      def initialize(config=nil)
+        if config
+          self.add(config['name'], config)
+        end
+      end
+
+      #
+      # If the key is a string, the Config::Object it references is returned.
+      #
+      # If the key is a hash, we treat it as a condition and filter all the Config::Objects using the condition.
+      # A new ObjectList is returned.
+      #
+      # Examples:
+      #
+      # nodes['vpn1']
+      #   node named 'vpn1'
+      #
+      # nodes[:public_dns => true]
+      #   all nodes with public dns
+      #
+      # nodes[:services => 'openvpn', 'location.country_code' => 'US']
+      #   all nodes with services containing 'openvpn' OR country code of US
+      #
+      # Sometimes, you want to do an OR condition with multiple conditions
+      # for the same field. Since hash keys must be unique, you can use
+      # an array representation instead:
+      #
+      # nodes[[:services, 'openvpn'], [:services, 'tor']]
+      #   nodes with openvpn OR tor service
+      #
+      # nodes[:services => 'openvpn'][:tags => 'production']
+      #   nodes with openvpn AND are production
+      #
+      def [](key)
+        if key.is_a?(Hash) || key.is_a?(Array)
+          filter(key)
+        else
+          super key.to_s
+        end
+      end
+
+      def exclude(node)
+        list = self.dup
+        list.delete(node.name)
+        return list
+      end
+
+      def each_node(&block)
+        self.keys.sort.each do |node_name|
+          yield self[node_name]
+        end
+      end
+
+      #
+      # filters this object list, producing a new list.
+      # filter is an array or a hash. see []
+      #
+      def filter(filter)
+        results = Config::ObjectList.new
+        filter.each do |field, match_value|
+          field = field.is_a?(Symbol) ? field.to_s : field
+          match_value = match_value.is_a?(Symbol) ? match_value.to_s : match_value
+          if match_value.is_a?(String) && match_value =~ /^!/
+            operator = :not_equal
+            match_value = match_value.sub(/^!/, '')
+          else
+            operator = :equal
+          end
+          each do |name, config|
+            value = config[field]
+            if value.is_a? Array
+              if operator == :equal && value.include?(match_value)
+                results[name] = config
+              elsif operator == :not_equal && !value.include?(match_value)
+                results[name] = config
+              end
+            else
+              if operator == :equal && value == match_value
+                results[name] = config
+              elsif operator == :not_equal && value != match_value
+                results[name] = config
+              end
+            end
+          end
+        end
+        results
+      end
+
+      def add(name, object)
+        self[name] = object
+      end
+
+      #
+      # converts the hash of configs into an array of hashes, with ONLY the specified fields
+      #
+      def fields(*fields)
+        result = []
+        keys.sort.each do |name|
+          result << self[name].pick(*fields)
+        end
+        result
+      end
+
+      #
+      # like fields(), but returns an array of values instead of an array of hashes.
+      #
+      def field(field)
+        field = field.to_s
+        result = []
+        keys.sort.each do |name|
+          result << self[name].get(field)
+        end
+        result
+      end
+
+      #
+      # pick_fields(field1, field2, ...)
+      #
+      # generates a Hash from the object list, but with only the fields that are picked.
+      #
+      # If there are more than one field, then the result is a Hash of Hashes.
+      # If there is just one field, it is a simple map to the value.
+      #
+      # For example:
+      #
+      #   "neighbors" = "= nodes_like_me[:services => :couchdb].pick_fields('domain.full', 'ip_address')"
+      #
+      # generates this:
+      #
+      #   neighbors:
+      #     couch1:
+      #       domain_full: couch1.bitmask.net
+      #       ip_address: "10.5.5.44"
+      #     couch2:
+      #       domain_full: couch2.bitmask.net
+      #       ip_address: "10.5.5.52"
+      #
+      # But this:
+      #
+      #   "neighbors": "= nodes_like_me[:services => :couchdb].pick_fields('domain.full')"
+      #
+      # will generate this:
+      #
+      #   neighbors:
+      #     couch1: couch1.bitmask.net
+      #     couch2: couch2.bitmask.net
+      #
+      def pick_fields(*fields)
+        self.values.inject({}) do |hsh, node|
+          value = self[node.name].pick(*fields)
+          if fields.size == 1
+            value = value.values.first
+          end
+          hsh[node.name] = value
+          hsh
+        end
+      end
+
+      #
+      # Applies inherit_from! to all objects.
+      #
+      # 'env' specifies what environment should be for
+      # each object in the list.
+      #
+      def inherit_from!(object_list, env)
+        object_list.each do |name, object|
+          if self[name]
+            self[name].inherit_from!(object)
+          else
+            self[name] = object.duplicate(env)
+          end
+        end
+      end
+
+      #
+      # topographical sort based on test dependency
+      #
+      def tsort_each_node(&block)
+        self.each_key(&block)
+      end
+
+      def tsort_each_child(node_name, &block)
+        if self[node_name]
+          self[node_name].test_dependencies.each do |test_me_first|
+            if self[test_me_first] # TODO: in the future, allow for ability to optionally pull in all dependencies.
+                                   # not just the ones that pass the node filter.
+              yield(test_me_first)
+            end
+          end
+        end
+      end
+
+      def names_in_test_dependency_order
+        self.tsort
+      end
+
+    end
+  end
+end
diff --git a/lib/leap_cli/config/provider.rb b/lib/leap_cli/config/provider.rb
new file mode 100644
index 00000000..0d8bc1f3
--- /dev/null
+++ b/lib/leap_cli/config/provider.rb
@@ -0,0 +1,22 @@
+#
+# Configuration class for provider.json
+#
+
+module LeapCli; module Config
+  class Provider < Object
+    attr_reader :environment
+    def set_env(e)
+      if e == 'default'
+        @environment = nil
+      else
+        @environment = e
+      end
+    end
+    def provider
+      self
+    end
+    def validate!
+      # nothing here yet :(
+    end
+  end
+end; end
diff --git a/lib/leap_cli/config/secrets.rb b/lib/leap_cli/config/secrets.rb
new file mode 100644
index 00000000..ca851c74
--- /dev/null
+++ b/lib/leap_cli/config/secrets.rb
@@ -0,0 +1,87 @@
+# encoding: utf-8
+#
+# A class for the secrets.json file
+#
+
+module LeapCli; module Config
+
+  class Secrets < Object
+    attr_reader :node_list
+
+    def initialize(manager=nil)
+      super(manager)
+      @discovered_keys = {}
+    end
+
+    # we can't use fetch() or get(), since those already have special meanings
+    def retrieve(key, environment)
+      environment ||= 'default'
+      self.fetch(environment, {})[key.to_s]
+    end
+
+    def set(*args, &block)
+      if block_given?
+        set_with_block(*args, &block)
+      else
+        set_without_block(*args)
+      end
+    end
+
+    # searches over all keys matching the regexp, checking to see if the value
+    # has been already used by any of them.
+    def taken?(regexp, value, environment)
+      self.keys.grep(regexp).each do |key|
+        return true if self.retrieve(key, environment) == value
+      end
+      return false
+    end
+
+    def set_without_block(key, value, environment)
+      set_with_block(key, environment) {value}
+    end
+
+    def set_with_block(key, environment, &block)
+      environment ||= 'default'
+      key = key.to_s
+      @discovered_keys[environment] ||= {}
+      @discovered_keys[environment][key] = true
+      self[environment] ||= {}
+      self[environment][key] ||= yield
+    end
+
+    #
+    # if clean is true, then only secrets that have been discovered
+    # during this run will be exported.
+    #
+    # if environment is also pinned, then we will clean those secrets
+    # just for that environment.
+    #
+    # the clean argument should only be used when all nodes have
+    # been processed, otherwise secrets that are actually in use will
+    # get mistakenly removed.
+    #
+    def dump_json(clean=false)
+      pinned_env = LeapCli.leapfile.environment
+      if clean
+        self.each_key do |environment|
+          if pinned_env.nil? || pinned_env == environment
+            env = self[environment]
+            if env.nil?
+              raise StandardError.new("secrets.json file seems corrupted. No such environment '#{environment}'")
+            end
+            env.each_key do |key|
+              unless @discovered_keys[environment] && @discovered_keys[environment][key]
+                self[environment].delete(key)
+              end
+            end
+            if self[environment].empty?
+              self.delete(environment)
+            end
+          end
+        end
+      end
+      super()
+    end
+  end
+
+end; end
diff --git a/lib/leap_cli/config/sources.rb b/lib/leap_cli/config/sources.rb
new file mode 100644
index 00000000..aee860de
--- /dev/null
+++ b/lib/leap_cli/config/sources.rb
@@ -0,0 +1,11 @@
+# encoding: utf-8
+#
+# A class for the sources.json file
+#
+
+module LeapCli
+  module Config
+    class Sources < Object
+    end
+  end
+end
diff --git a/lib/leap_cli/config/tag.rb b/lib/leap_cli/config/tag.rb
new file mode 100644
index 00000000..6bd8d1e9
--- /dev/null
+++ b/lib/leap_cli/config/tag.rb
@@ -0,0 +1,25 @@
+#
+#
+# A class for node services or node tags.
+#
+#
+
+module LeapCli; module Config
+
+  class Tag < Object
+    attr_reader :node_list
+
+    def initialize(environment=nil)
+      super(environment)
+      @node_list = Config::ObjectList.new
+    end
+
+    # don't copy the node list pointer when this object is dup'ed.
+    def initialize_copy(orig)
+      super
+      @node_list = Config::ObjectList.new
+    end
+
+  end
+
+end; end
diff --git a/lib/leap_cli/leapfile_extensions.rb b/lib/leap_cli/leapfile_extensions.rb
new file mode 100644
index 00000000..cba321f4
--- /dev/null
+++ b/lib/leap_cli/leapfile_extensions.rb
@@ -0,0 +1,24 @@
+module LeapCli
+  class Leapfile
+    attr_reader :custom_vagrant_vm_line
+    attr_reader :leap_version
+    attr_reader :log
+    attr_reader :vagrant_basebox
+
+    def vagrant_network
+      @vagrant_network ||= '10.5.5.0/24'
+    end
+
+    private
+
+    PRIVATE_IP_RANGES = /(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)/
+
+    def validate
+      Util::assert! vagrant_network =~ PRIVATE_IP_RANGES do
+        Util::log 0, :error, "in #{file}: vagrant_network is not a local private network"
+      end
+      return true
+    end
+
+  end
+end
diff --git a/lib/leap_cli/load_libraries.rb b/lib/leap_cli/load_libraries.rb
new file mode 100644
index 00000000..19f4edb5
--- /dev/null
+++ b/lib/leap_cli/load_libraries.rb
@@ -0,0 +1,20 @@
+#
+# load the commonly needed leap_cli libraries that live in the platform.
+#
+# loaded by leap_cli's bootstrap.rb
+#
+
+require 'leap_cli/log_filter'
+
+require 'leap_cli/config/object'
+require 'leap_cli/config/node'
+require 'leap_cli/config/tag'
+require 'leap_cli/config/provider'
+require 'leap_cli/config/secrets'
+require 'leap_cli/config/object_list'
+require 'leap_cli/config/filter'
+require 'leap_cli/config/environment'
+require 'leap_cli/config/manager'
+
+require 'leap_cli/util/secret'
+require 'leap_cli/util/x509'
diff --git a/lib/leap_cli/log_filter.rb b/lib/leap_cli/log_filter.rb
new file mode 100644
index 00000000..0d745cc2
--- /dev/null
+++ b/lib/leap_cli/log_filter.rb
@@ -0,0 +1,171 @@
+#
+# A module to hide, modify, and colorize log entries.
+#
+
+module LeapCli
+  module LogFilter
+    #
+    # options for formatters:
+    #
+    # :match       => regexp for matching a log line
+    # :color       => what color the line should be
+    # :style       => what style the line should be
+    # :priority    => what order the formatters are applied in. higher numbers first.
+    # :match_level => only apply filter at the specified log level
+    # :level       => make this line visible at this log level or higher
+    # :replace     => replace the matched text
+    # :prepend     => insert text at start of message
+    # :append      => append text to end of message
+    # :exit        => force the exit code to be this (does not interrupt program, just
+    #                 ensures a specific exit code when the program eventually exits)
+    #
+    FORMATTERS = [
+      # TRACE
+      { :match => /command finished/,          :color => :white,   :style => :dim, :match_level => 3, :priority => -10 },
+      { :match => /executing locally/,         :color => :yellow,  :match_level => 3, :priority => -20 },
+
+      # DEBUG
+      #{ :match => /executing .*/,             :color => :green,   :match_level => 2, :priority => -10, :timestamp => true },
+      #{ :match => /.*/,                       :color => :yellow,  :match_level => 2, :priority => -30 },
+      { :match => /^transaction:/,             :level => 3 },
+
+      # INFO
+      { :match => /.*out\] (fatal:|ERROR:).*/, :color => :red,     :match_level => 1, :priority => -10 },
+      { :match => /Permission denied/,         :color => :red,     :match_level => 1, :priority => -20 },
+      { :match => /sh: .+: command not found/, :color => :magenta, :match_level => 1, :priority => -30 },
+
+      # IMPORTANT
+      { :match => /^(E|e)rr ::/,               :color => :red,     :match_level => 0, :priority => -10, :exit => 1},
+      { :match => /^ERROR:/,                   :color => :red,                        :priority => -10, :exit => 1},
+      #{ :match => /.*/,                        :color => :blue,    :match_level => 0, :priority => -20 },
+
+      # CLEANUP
+      #{ :match => /\s+$/,                      :replace => '', :priority => 0},
+
+      # DEBIAN PACKAGES
+      { :match => /^(Hit|Ign) /,                :color => :green,   :priority => -20},
+      { :match => /^Err /,                      :color => :red,     :priority => -20},
+      { :match => /^W(ARNING)?: /,              :color => :yellow,  :priority => -20},
+      { :match => /^E: /,                       :color => :red,     :priority => -20},
+      { :match => /already the newest version/, :color => :green,   :priority => -20},
+      { :match => /WARNING: The following packages cannot be authenticated!/, :color => :red, :level => 0, :priority => -10},
+
+      # PUPPET
+      { :match => /^(W|w)arning: Not collecting exported resources without storeconfigs/, :level => 2, :color => :yellow, :priority => -10},
+      { :match => /^(W|w)arning: Found multiple default providers for vcsrepo:/,          :level => 2, :color => :yellow, :priority => -10},
+      { :match => /^(W|w)arning: .*is deprecated.*$/, :level => 2, :color => :yellow, :priority => -10},
+      { :match => /^(W|w)arning: Scope.*$/,           :level => 2, :color => :yellow, :priority => -10},
+      #{ :match => /^(N|n)otice:/,                     :level => 1, :color => :cyan,   :priority => -20},
+      #{ :match => /^(N|n)otice:.*executed successfully$/, :level => 2, :color => :cyan, :priority => -15},
+      { :match => /^(W|w)arning:/,                    :level => 0, :color => :yellow, :priority => -20},
+      { :match => /^Duplicate declaration:/,          :level => 0, :color => :red,    :priority => -20},
+      #{ :match => /Finished catalog run/,             :level => 0, :color => :green,  :priority => -10},
+      { :match => /^APPLY COMPLETE \(changes made\)/, :level => 0, :color => :green, :style => :bold, :priority => -10},
+      { :match => /^APPLY COMPLETE \(no changes\)/,   :level => 0, :color => :green, :style => :bold, :priority => -10},
+
+      # PUPPET FATAL ERRORS
+      { :match => /^(E|e)rr(or|):/,                :level => 0, :color => :red, :priority => -1, :exit => 1},
+      { :match => /^Wrapped exception:/,           :level => 0, :color => :red, :priority => -1, :exit => 1},
+      { :match => /^Failed to parse template/,     :level => 0, :color => :red, :priority => -1, :exit => 1},
+      { :match => /^Execution of.*returned/,       :level => 0, :color => :red, :priority => -1, :exit => 1},
+      { :match => /^Parameter matches failed:/,    :level => 0, :color => :red, :priority => -1, :exit => 1},
+      { :match => /^Syntax error/,                 :level => 0, :color => :red, :priority => -1, :exit => 1},
+      { :match => /^Cannot reassign variable/,     :level => 0, :color => :red, :priority => -1, :exit => 1},
+      { :match => /^Could not find template/,      :level => 0, :color => :red, :priority => -1, :exit => 1},
+      { :match => /^APPLY COMPLETE.*fail/,         :level => 0, :color => :red, :style => :bold, :priority => -1, :exit => 1},
+
+      # TESTS
+      { :match => /^PASS: /,                :color => :green,   :priority => -20},
+      { :match => /^(FAIL|ERROR): /,        :color => :red,     :priority => -20},
+      { :match => /^(SKIP|WARN): /,         :color => :yellow,  :priority => -20},
+      { :match => /\d+ tests: \d+ passes, \d+ skips, 0 warnings, 0 failures, 0 errors/,
+        :color => :green, :style => :bold, :priority => -20 },
+      { :match => /\d+ tests: \d+ passes, \d+ skips, [1-9][0-9]* warnings, 0 failures, 0 errors/,
+        :color => :yellow, :style => :bold,  :priority => -20 },
+      { :match => /\d+ tests: \d+ passes, \d+ skips, \d+ warnings, \d+ failures, [1-9][0-9]* errors/,
+        :color => :red, :style => :bold, :priority => -20 },
+      { :match => /\d+ tests: \d+ passes, \d+ skips, \d+ warnings, [1-9][0-9]* failures, \d+ errors/,
+        :color => :red, :style => :bold, :priority => -20 },
+
+      # LOG SUPPRESSION
+      { :match => /^(W|w)arning: You cannot collect without storeconfigs being set/, :level => 2, :priority => 10},
+      { :match => /^(W|w)arning: You cannot collect exported resources without storeconfigs being set/, :level => 2, :priority => 10}
+    ]
+
+    SORTED_FORMATTERS = FORMATTERS.sort_by { |i| -(i[:priority] || i[:prio] || 0) }
+
+    #
+    # same as normal formatters, but only applies to the title, not the message.
+    #
+    TITLE_FORMATTERS = [
+      # red
+      { :match => /error/, :color => :red, :style => :bold },
+      { :match => /fatal_error/, :replace => 'fatal error:', :color => :red, :style => :bold },
+      { :match => /removed/, :color => :red, :style => :bold },
+      { :match => /failed/, :replace => 'FAILED', :color => :red, :style => :bold },
+      { :match => /bail/, :replace => 'bailing out', :color => :red, :style => :bold },
+      { :match => /invalid/, :color => :red, :style => :bold },
+
+      # yellow
+      { :match => /warning/, :replace => 'warning:', :color => :yellow, :style => :bold },
+      { :match => /missing/, :color => :yellow, :style => :bold },
+      { :match => /skipping/, :color => :yellow, :style => :bold },
+
+      # green
+      { :match => /created/, :color => :green, :style => :bold },
+      { :match => /completed/, :color => :green, :style => :bold },
+      { :match => /ran/, :color => :green, :style => :bold },
+
+      # cyan
+      { :match => /note/, :replace => 'NOTE:', :color => :cyan, :style => :bold },
+
+      # magenta
+      { :match => /nochange/, :replace => 'no change', :color => :magenta },
+      { :match => /loading/, :color => :magenta },
+    ]
+
+    def self.apply_message_filters(message)
+      return self.apply_filters(SORTED_FORMATTERS, message)
+    end
+
+    def self.apply_title_filters(title)
+      return self.apply_filters(TITLE_FORMATTERS, title)
+    end
+
+    private
+
+    def self.apply_filters(formatters, message)
+      level = LeapCli.logger.log_level
+      result = {}
+      formatters.each do |formatter|
+        if (formatter[:match_level] == level || formatter[:match_level].nil?)
+          if message =~ formatter[:match]
+            # puts "applying formatter #{formatter.inspect}"
+            result[:level] = formatter[:level] if formatter[:level]
+            result[:color] = formatter[:color] if formatter[:color]
+            result[:style] = formatter[:style] || formatter[:attribute] # (support original cap colors)
+
+            message.gsub!(formatter[:match], formatter[:replace]) if formatter[:replace]
+            message.replace(formatter[:prepend] + message) unless formatter[:prepend].nil?
+            message.replace(message + formatter[:append])  unless formatter[:append].nil?
+            message.replace(Time.now.strftime('%Y-%m-%d %T') + ' ' + message) if formatter[:timestamp]
+
+            if formatter[:exit]
+              LeapCli::Util.exit_status(formatter[:exit])
+            end
+
+            # stop formatting, unless formatter was just for string replacement
+            break unless formatter[:replace]
+          end
+        end
+      end
+
+      if result[:color] == :hide
+        return [nil, {}]
+      else
+        return [message, result]
+      end
+    end
+
+  end
+end
diff --git a/lib/leap_cli/macros.rb b/lib/leap_cli/macros.rb
deleted file mode 100644
index fdb9a94e..00000000
--- a/lib/leap_cli/macros.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-#
-# MACROS
-#
-# The methods in these files are available in the context of a .json configuration file.
-# (The module LeapCli::Macro is included in Config::Object)
-#
-
-require_relative 'macros/core'
-require_relative 'macros/files'
-require_relative 'macros/haproxy'
-require_relative 'macros/hosts'
-require_relative 'macros/keys'
-require_relative 'macros/nodes'
-require_relative 'macros/secrets'
-require_relative 'macros/stunnel'
-require_relative 'macros/provider'
diff --git a/lib/leap_cli/ssh/backend.rb b/lib/leap_cli/ssh/backend.rb
index f42379cc..67c6ec9a 100644
--- a/lib/leap_cli/ssh/backend.rb
+++ b/lib/leap_cli/ssh/backend.rb
@@ -94,7 +94,7 @@ module LeapCli
 
       # some prewritten servers-side scripts
       def scripts
-        @scripts ||= LeapCli::SSH::Scripts.new(self, @host)
+        @scripts ||= LeapCli::SSH::Scripts.new(self, @host.hostname)
       end
 
       private
@@ -139,6 +139,9 @@ module LeapCli
           @logger.log(:failed, args.join(' '), host: @host.hostname) do
             @logger.log("Connection timed out")
           end
+          if @options[:raise_error]
+            raise LeapCli::SSH::TimeoutError, exc.to_s
+          end
         else
           raise
         end
diff --git a/lib/leap_cli/ssh/remote_command.rb b/lib/leap_cli/ssh/remote_command.rb
index fe9a344a..3ba86740 100644
--- a/lib/leap_cli/ssh/remote_command.rb
+++ b/lib/leap_cli/ssh/remote_command.rb
@@ -21,6 +21,9 @@ module LeapCli
     class ExecuteError < StandardError
     end
 
+    class TimeoutError < ExecuteError
+    end
+
     # override default runner mode
     class CustomCoordinator < SSHKit::Coordinator
       private
diff --git a/lib/leap_cli/ssh/scripts.rb b/lib/leap_cli/ssh/scripts.rb
index 3d8b6570..feefdd46 100644
--- a/lib/leap_cli/ssh/scripts.rb
+++ b/lib/leap_cli/ssh/scripts.rb
@@ -15,7 +15,7 @@ module LeapCli
       REQUIRED_PACKAGES = "puppet rsync lsb-release locales"
 
       attr_reader :ssh, :host
-      def initialize(backend, host)
+      def initialize(backend, hostname)
         @ssh = backend
         @host = host
       end
@@ -48,6 +48,8 @@ module LeapCli
       def check_for_no_deploy
         begin
           ssh.stream "test ! -f /etc/leap/no-deploy", :raise_error => true, :log_output => false
+        rescue SSH::TimeoutError
+          raise
         rescue SSH::ExecuteError
           ssh.log :warning, "can't continue because file /etc/leap/no-deploy exists", :host => host
           raise # will skip further action on this node
@@ -59,7 +61,7 @@ module LeapCli
       #
       def debug
         output = ssh.capture "#{Leap::Platform.leap_dir}/bin/debug.sh"
-        ssh.log(output, :wrap => true, :host => host.hostname, :color => :cyan)
+        ssh.log(output, :wrap => true, :host => host, :color => :cyan)
       end
 
       #
@@ -69,7 +71,7 @@ module LeapCli
         cmd = "(test -s /var/log/leap/deploy-summary.log && tail -n #{lines} /var/log/leap/deploy-summary.log) || (test -s /var/log/leap/deploy-summary.log.1 && tail -n #{lines} /var/log/leap/deploy-summary.log.1) || (echo 'no history')"
         history = ssh.capture(cmd, :log_output => false)
         if history
-          ssh.log host.hostname, :color => :cyan, :style => :bold do
+          ssh.log host, :color => :cyan, :style => :bold do
             ssh.log history, :wrap => true
           end
         end
diff --git a/lib/leap_cli/util/secret.rb b/lib/leap_cli/util/secret.rb
new file mode 100644
index 00000000..749b9595
--- /dev/null
+++ b/lib/leap_cli/util/secret.rb
@@ -0,0 +1,55 @@
+# encoding: utf-8
+#
+# A simple secret generator
+#
+# Uses OpenSSL random number generator instead of Ruby's rand function
+#
+autoload :OpenSSL, 'openssl'
+
+module LeapCli; module Util
+  class Secret
+    CHARS = (('A'..'Z').to_a + ('a'..'z').to_a + ('0'..'9').to_a) - "i1loO06G".split(//u)
+    HEX = (0..9).to_a + ('a'..'f').to_a
+
+    #
+    # generate a secret with with no ambiguous characters.
+    #
+    # +length+ is in chars
+    #
+    # Only alphanumerics are allowed, in order to make these passwords work
+    # for REST url calls and to allow you to easily copy and paste them.
+    #
+    def self.generate(length = 16)
+      seed
+      OpenSSL::Random.random_bytes(length).bytes.to_a.collect { |byte|
+        CHARS[ byte % CHARS.length ]
+      }.join
+    end
+
+    #
+    # generates a hex secret, instead of an alphanumeric on.
+    #
+    # length is in bits
+    #
+    def self.generate_hex(length = 128)
+      seed
+      OpenSSL::Random.random_bytes(length/4).bytes.to_a.collect { |byte|
+        HEX[ byte % HEX.length ]
+      }.join
+    end
+
+    private
+
+    def self.seed
+      @pid ||= 0
+      pid = $$
+      if @pid != pid
+        now = Time.now
+        ary = [now.to_i, now.nsec, @pid, pid]
+        OpenSSL::Random.seed(ary.to_s)
+        @pid = pid
+      end
+    end
+
+  end
+end; end
diff --git a/lib/leap_cli/util/x509.rb b/lib/leap_cli/util/x509.rb
new file mode 100644
index 00000000..787fdfac
--- /dev/null
+++ b/lib/leap_cli/util/x509.rb
@@ -0,0 +1,33 @@
+autoload :OpenSSL, 'openssl'
+autoload :CertificateAuthority, 'certificate_authority'
+
+require 'digest'
+require 'digest/md5'
+require 'digest/sha1'
+
+module LeapCli; module X509
+  extend self
+
+  #
+  # returns a fingerprint of a x509 certificate
+  #
+  def fingerprint(digest, cert_file)
+    if cert_file.is_a? String
+      cert = OpenSSL::X509::Certificate.new(Util.read_file!(cert_file))
+    elsif cert_file.is_a? OpenSSL::X509::Certificate
+      cert = cert_file
+    elsif cert_file.is_a? CertificateAuthority::Certificate
+      cert = cert_file.openssl_body
+    end
+    digester = case digest
+      when "MD5" then Digest::MD5.new
+      when "SHA1" then Digest::SHA1.new
+      when "SHA256" then Digest::SHA256.new
+      when "SHA384" then Digest::SHA384.new
+      when "SHA512" then Digest::SHA512.new
+    end
+    digester.hexdigest(cert.to_der)
+  end
+
+
+end; end