added ability to sync support files along with hiera.yml. this way, files don't need to be embedded in hiera.yml. this is especially useful for binary files.

8 files changed, 206 insertions, 82 deletions

diff --git a/lib/leap_cli.rb b/lib/leap_cli.rb
index f65f131..31a9d8f 100644
--- a/lib/leap_cli.rb
+++ b/lib/leap_cli.rb
@@ -18,6 +18,8 @@ require 'leap_cli/logger'
 
 require 'leap_cli/ssh_key'
 require 'leap_cli/config/object'
+require 'leap_cli/config/node'
+require 'leap_cli/config/tag'
 require 'leap_cli/config/object_list'
 require 'leap_cli/config/manager'
 
diff --git a/lib/leap_cli/commands/deploy.rb b/lib/leap_cli/commands/deploy.rb
index cc2ea96..a7f6bc3 100644
--- a/lib/leap_cli/commands/deploy.rb
+++ b/lib/leap_cli/commands/deploy.rb
@@ -1,3 +1,4 @@
+
 module LeapCli
   module Commands
 
@@ -23,13 +24,9 @@ module LeapCli
             ssh.leap.assert_initialized
           end
 
-          # sync hiera conf
-          ssh.leap.log :syching, "hiera.yaml" do
-            ssh.leap.rsync_update do |server|
-              node = manager.node(server.host)
-              ssh.leap.log Path.relative_path([:hiera, node.name]) + ' -> ' + node.name + ':/etc/leap/hiera.yaml'
-              {:source => Path.named_path([:hiera, node.name]), :dest => "/etc/leap/hiera.yaml"}
-            end
+          ssh.leap.log :syching, "configuration files" do
+            sync_hiera_config(ssh)
+            sync_support_files(ssh)
           end
 
           # sync puppet manifests and apply them
@@ -46,6 +43,37 @@ module LeapCli
 
     private
 
+    def sync_hiera_config(ssh)
+      dest_dir = manager.provider.hiera_sync_destination
+      ssh.leap.rsync_update do |server|
+        node = manager.node(server.host)
+        hiera_file = Path.relative_path([:hiera, node.name])
+        ssh.leap.log hiera_file + ' -> ' + node.name + ':' + dest_dir + '/hiera.yaml'
+        {:source => hiera_file, :dest => dest_dir + '/hiera.yaml'}
+      end
+    end
+
+    def sync_support_files(ssh)
+      dest_dir = manager.provider.hiera_sync_destination
+      ssh.leap.rsync_update do |server|
+        node = manager.node(server.host)
+        files_to_sync = node.file_paths.collect {|path| Path.relative_path(path, Path.provider) }
+        if files_to_sync.any?
+          ssh.leap.log(files_to_sync.join(', ') + ' -> ' + node.name + ':' + dest_dir)
+          {
+            :chdir => Path.provider,
+            :source => ".",
+            :dest => dest_dir,
+            :excludes => "*",
+            :includes => calculate_includes_from_files(files_to_sync),
+            :flags => "--relative --dirs --delete --delete-excluded --filter='protect hiera.yaml' --copy-links"
+          }
+        else
+          nil
+        end
+      end
+    end
+
     def init_submodules
       Dir.chdir Path.platform do
         statuses = assert_run! "git submodule status"
@@ -59,5 +87,28 @@ module LeapCli
       end
     end
 
+    def calculate_includes_from_files(files)
+      # prepend '/' (kind of like ^ for rsync)
+      includes = files.collect {|file| '/' + file}
+
+      # include all sub files of specified directories
+      includes.size.times do |i|
+        if includes[i] =~ /\/$/
+          includes << includes[i] + '**'
+        end
+      end
+
+      # include all parent directories
+      includes.size.times do |i|
+        path = File.dirname(includes[i])
+        while(path != '/')
+          includes << path unless includes.include?(path)
+          path = File.dirname(path)
+        end
+      end
+
+      return includes
+    end
+
   end
 end
\ No newline at end of file
diff --git a/lib/leap_cli/config/manager.rb b/lib/leap_cli/config/manager.rb
index 6702fc4..c860b5c 100644
--- a/lib/leap_cli/config/manager.rb
+++ b/lib/leap_cli/config/manager.rb
@@ -21,21 +21,21 @@ module LeapCli
         @provider_dir = Path.provider
 
         # load base
-        base_services = load_all_json(Path.named_path([:service_config, '*'], Path.provider_base))
-        base_tags     = load_all_json(Path.named_path([:tag_config, '*'], Path.provider_base))
-        base_common   = load_json(Path.named_path(:common_config, Path.provider_base))
-        base_provider = load_json(Path.named_path(:provider_config, Path.provider_base))
+        base_services = load_all_json(Path.named_path([:service_config, '*'], Path.provider_base), Config::Tag)
+        base_tags     = load_all_json(Path.named_path([:tag_config, '*'], Path.provider_base), Config::Tag)
+        base_common   = load_json(Path.named_path(:common_config, Path.provider_base), Config::Object)
+        base_provider = load_json(Path.named_path(:provider_config, Path.provider_base), Config::Object)
 
         # load provider
         provider_path = Path.named_path(:provider_config, @provider_dir)
         common_path = Path.named_path(:common_config, @provider_dir)
         Util::assert_files_exist!(provider_path, common_path)
-        @services = load_all_json(Path.named_path([:service_config, '*'], @provider_dir))
-        @tags     = load_all_json(Path.named_path([:tag_config, '*'],     @provider_dir))
-        @nodes    = load_all_json(Path.named_path([:node_config, '*'],    @provider_dir))
-        @common   = load_json(common_path)
-        @provider = load_json(provider_path)
-        @secrets  = load_json(Path.named_path(:secrets_config,  @provider_dir))
+        @services = load_all_json(Path.named_path([:service_config, '*'], @provider_dir), Config::Tag)
+        @tags     = load_all_json(Path.named_path([:tag_config, '*'],     @provider_dir), Config::Tag)
+        @nodes    = load_all_json(Path.named_path([:node_config, '*'],    @provider_dir), Config::Node)
+        @common   = load_json(common_path, Config::Object)
+        @provider = load_json(provider_path, Config::Object)
+        @secrets  = load_json(Path.named_path(:secrets_config,  @provider_dir), Config::Object)
 
         # inherit
         @services.inherit_from! base_services
@@ -161,10 +161,10 @@ module LeapCli
 
       private
 
-      def load_all_json(pattern)
+      def load_all_json(pattern, object_class)
         results = Config::ObjectList.new
         Dir.glob(pattern).each do |filename|
-          obj = load_json(filename)
+          obj = load_json(filename, object_class)
           if obj
             name = File.basename(filename).sub(/\.json$/,'')
             obj['name'] ||= name
@@ -174,9 +174,9 @@ module LeapCli
         results
       end
 
-      def load_json(filename)
+      def load_json(filename, object_class)
         if !File.exists?(filename)
-          return Config::Object.new(self)
+          return object_class.new(self)
         end
 
         log :loading, filename, 2
@@ -201,7 +201,7 @@ module LeapCli
           log 0, exc.to_s, :indent => 1
           return nil
         end
-        object = Config::Object.new(self)
+        object = object_class.new(self)
         object.deep_merge!(hash)
         return object
       end
@@ -226,7 +226,7 @@ module LeapCli
       # makes a node inherit options from appropriate the common, service, and tag json files.
       #
       def apply_inheritance(node)
-        new_node = Config::Object.new(self)
+        new_node = Config::Node.new(self)
         name = node.name
 
         # inherit from common
diff --git a/lib/leap_cli/config/node.rb b/lib/leap_cli/config/node.rb
new file mode 100644
index 0000000..9eea1f3
--- /dev/null
+++ b/lib/leap_cli/config/node.rb
@@ -0,0 +1,52 @@
+#
+# Configuration for a 'node' (a server in the provider's infrastructure)
+#
+
+require 'ipaddr'
+
+module LeapCli; module Config
+
+  class Node < Object
+    attr_accessor :file_paths
+
+    def initialize(manager=nil)
+      super(manager)
+      @node = self
+      @file_paths = []
+    end
+
+    #
+    # Make a copy of ourselves, except only including the specified keys.
+    #
+    # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b'
+    #
+    def pick(*keys)
+      keys.map(&:to_s).inject(self.class.new(@manager)) do |hsh, key|
+        value = self.get(key)
+        if !value.nil?
+          hsh[key.gsub('.','_')] = value
+        end
+        hsh
+      end
+    end
+
+    #
+    # returns true if this node has an ip address in the range of the vagrant network
+    #
+    def vagrant?
+      begin
+        vagrant_range = IPAddr.new @manager.provider.vagrant.network
+      rescue ArgumentError => exc
+        Util::bail! { Util::log :invalid, "ip address '#{@node.ip_address}' vagrant.network" }
+      end
+
+      begin
+        ip_address = IPAddr.new @node.get('ip_address')
+      rescue ArgumentError => exc
+        Util::log :warning, "invalid ip address '#{@node.get('ip_address')}' for node '#{@node.name}'"
+      end
+      return vagrant_range.include?(ip_address)
+    end
+  end
+
+end; end
diff --git a/lib/leap_cli/config/object.rb b/lib/leap_cli/config/object.rb
index 395ebe3..5c6cfd0 100644
--- a/lib/leap_cli/config/object.rb
+++ b/lib/leap_cli/config/object.rb
@@ -1,6 +1,5 @@
 require 'erb'
 require 'json/pure'  # pure ruby implementation is required for our sorted trick to work.
-require 'ipaddr'
 
 $KCODE = 'UTF8' unless RUBY_VERSION > "1.9.0"
 require 'ya2yaml' # pure ruby yaml
@@ -17,7 +16,6 @@ module LeapCli
 
       attr_reader :node
       attr_reader :manager
-      attr_reader :node_list
       alias :global :manager
 
       def initialize(manager=nil, node=nil)
@@ -27,9 +25,6 @@ module LeapCli
 
         # an object that is a node as @node equal to self, otherwise all the child objects point back to the top level node.
         @node = node || self
-
-        # this is only used by Config::Objects that correspond to services or tags.
-        @node_list = Config::ObjectList.new
       end
 
       #
@@ -106,21 +101,6 @@ module LeapCli
       ##
 
       #
-      # Make a copy of ourselves, except only including the specified keys.
-      #
-      # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b'
-      #
-      def pick(*keys)
-        keys.map(&:to_s).inject(Config::Object.new(@manager,@node)) do |hsh, key|
-          value = self.get(key)
-          if !value.nil?
-            hsh[key.gsub('.','_')] = value
-          end
-          hsh
-        end
-      end
-
-      #
       # a deep (recursive) merge with another Config::Object.
       #
       # if prefer_self is set to true, the value from self will be picked when there is a conflict
@@ -196,29 +176,6 @@ module LeapCli
       end
 
       ##
-      ## NODE SPECIFIC
-      ## maybe these should be moved to a Node class.
-      ##
-
-      #
-      # returns true if this node has an ip address in the range of the vagrant network
-      #
-      def vagrant?
-        begin
-          vagrant_range = IPAddr.new @manager.provider.vagrant.network
-        rescue ArgumentError => exc
-          Util::bail! { Util::log :invalid, "ip address '#{@node.ip_address}' vagrant.network" }
-        end
-
-        begin
-          ip_address = IPAddr.new @node.get('ip_address')
-        rescue ArgumentError => exc
-          Util::log :warning, "invalid ip address '#{@node.get('ip_address')}' for node '#{@node.name}'"
-        end
-        return vagrant_range.include?(ip_address)
-      end
-
-      ##
       ## MACROS
       ## these are methods used when eval'ing a value in the .json configuration
       ##
@@ -271,6 +228,43 @@ module LeapCli
       end
 
       #
+      # returns what the file path will be, once the file is rsynced to the server.
+      # an internal list of discovered file paths is saved, in order to rsync these files when needed.
+      #
+      # notes:
+      #
+      # * argument 'path' is relative to Path.provider/files or Path.provider_base/files
+      # * the path returned by this method is absolute
+      # * the path stored for use later by rsync is relative to Path.provider
+      # * if the path does not exist locally, but exists in provider_base, then the default file from
+      #   provider_base is copied locally.
+      #
+      def file_path(path)
+        if path.is_a? Symbol
+          path = [path, @node.name]
+        end
+        actual_path = Path.find_file(path)
+        if actual_path.nil?
+          nil
+        else
+          if actual_path =~ /^#{Regexp.escape(Path.provider_base)}/
+            # if file is under Path.provider_base, we must copy the default file to
+            # to Path.provider in order for rsync to be able to sync the file.
+            local_provider_path = actual_path.sub(/^#{Regexp.escape(Path.provider_base)}/, Path.provider)
+            FileUtils.cp_r actual_path, local_provider_path
+            Util.log :created, Path.relative_path(local_provider_path)
+            actual_path = local_provider_path
+          end
+          if Dir.exists?(actual_path) && actual_path !~ /\/$/
+            actual_path += '/' # ensure directories end with /, important for building rsync command
+          end
+          relative_path = Path.relative_path(actual_path)
+          @node.file_paths << relative_path
+          @node.manager.provider.hiera_sync_destination + '/' + relative_path
+        end
+      end
+
+      #
       # inserts a named secret, generating it if needed.
       #
       # manager.export_secrets should be called later to capture any newly generated secrets.
diff --git a/lib/leap_cli/config/tag.rb b/lib/leap_cli/config/tag.rb
new file mode 100644
index 0000000..e5e719d
--- /dev/null
+++ b/lib/leap_cli/config/tag.rb
@@ -0,0 +1,18 @@
+#
+#
+# A class for node services or node tags.
+#
+#
+
+module LeapCli; module Config
+
+  class Tag < Object
+    attr_reader :node_list
+
+    def initialize(manager=nil)
+      super(manager)
+      @node_list = Config::ObjectList.new
+    end
+  end
+
+end; end
diff --git a/lib/leap_cli/path.rb b/lib/leap_cli/path.rb
index 43f2edc..e7626b5 100644
--- a/lib/leap_cli/path.rb
+++ b/lib/leap_cli/path.rb
@@ -87,13 +87,14 @@ module LeapCli; module Path
   # tries to find a file somewhere
   #
   def self.find_file(arg)
-    file_path = named_path(arg, Path.provider)
-    return file_path if File.exists?(file_path)
-
-    file_path = named_path(arg, Path.provider_base)
-    return file_path if File.exists?(file_path)
-
-    # give up
+    [Path.provider, Path.provider_base].each do |base|
+      file_path = named_path(arg, base)
+      return file_path if File.exists?(file_path)
+      if arg.is_a? String
+        file_path = base + '/files/' + arg
+        return file_path if File.exists?(file_path)
+      end
+    end
     return nil
   end
 
diff --git a/lib/leap_cli/remote/plugin.rb b/lib/leap_cli/remote/plugin.rb
index 803ebf9..213c981 100644
--- a/lib/leap_cli/remote/plugin.rb
+++ b/lib/leap_cli/remote/plugin.rb
@@ -57,22 +57,28 @@ module LeapCli; module Remote; module Plugin
     # rsync to each server
     failed_servers = []
     servers.each do |server|
+      options = yield server
+      next unless options
+
       # build rsync command
-      paths       = yield server
       remote_user = server.user || fetch(:user, ENV['USER'])
+      rsync_options = {
+        :flags => options[:flags],
+        :includes => options[:includes],
+        :excludes => options[:excludes],
+        :ssh => ssh_options.merge(server.options[:ssh_options]||{})
+      }
       rsync_cmd = SupplyDrop::Rsync.command(
-        paths[:source],
-        SupplyDrop::Rsync.remote_address(remote_user, server.host, paths[:dest]),
-        {:ssh => ssh_options.merge(server.options[:ssh_options]||{})}
+        options[:source],
+        SupplyDrop::Rsync.remote_address(remote_user, server.host, options[:dest]),
+        rsync_options
       )
 
       # run command
       logger.debug rsync_cmd
-      ok = system(rsync_cmd)
-      if ok
-        logger.log 1, "rsync #{paths[:source]} #{paths[:dest]}", server.host, :color => :green
-      else
-        failed_servers << server.host
+      Dir.chdir(options[:chdir] || '.') do
+        ok = system(rsync_cmd)
+        failed_servers << server.host unless ok
       end
     end