summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorelijah <elijah@riseup.net>2013-01-13 20:27:29 -0800
committerelijah <elijah@riseup.net>2013-01-13 20:27:29 -0800
commitba301b0c8d77ae2f455d3a2d736968c981b8c757 (patch)
treefac1c2f483732d158f106d8ad7917bfc7e07c239
parentbea336480bf90f7c24737809e27b0bd224f42233 (diff)
added ability to sync support files along with hiera.yml. this way, files don't need to be embedded in hiera.yml. this is especially useful for binary files.
-rw-r--r--lib/leap_cli.rb2
-rw-r--r--lib/leap_cli/commands/deploy.rb65
-rw-r--r--lib/leap_cli/config/manager.rb32
-rw-r--r--lib/leap_cli/config/node.rb52
-rw-r--r--lib/leap_cli/config/object.rb80
-rw-r--r--lib/leap_cli/config/tag.rb18
-rw-r--r--lib/leap_cli/path.rb15
-rw-r--r--lib/leap_cli/remote/plugin.rb24
8 files changed, 206 insertions, 82 deletions
diff --git a/lib/leap_cli.rb b/lib/leap_cli.rb
index f65f131..31a9d8f 100644
--- a/lib/leap_cli.rb
+++ b/lib/leap_cli.rb
@@ -18,6 +18,8 @@ require 'leap_cli/logger'
require 'leap_cli/ssh_key'
require 'leap_cli/config/object'
+require 'leap_cli/config/node'
+require 'leap_cli/config/tag'
require 'leap_cli/config/object_list'
require 'leap_cli/config/manager'
diff --git a/lib/leap_cli/commands/deploy.rb b/lib/leap_cli/commands/deploy.rb
index cc2ea96..a7f6bc3 100644
--- a/lib/leap_cli/commands/deploy.rb
+++ b/lib/leap_cli/commands/deploy.rb
@@ -1,3 +1,4 @@
+
module LeapCli
module Commands
@@ -23,13 +24,9 @@ module LeapCli
ssh.leap.assert_initialized
end
- # sync hiera conf
- ssh.leap.log :syching, "hiera.yaml" do
- ssh.leap.rsync_update do |server|
- node = manager.node(server.host)
- ssh.leap.log Path.relative_path([:hiera, node.name]) + ' -> ' + node.name + ':/etc/leap/hiera.yaml'
- {:source => Path.named_path([:hiera, node.name]), :dest => "/etc/leap/hiera.yaml"}
- end
+ ssh.leap.log :syching, "configuration files" do
+ sync_hiera_config(ssh)
+ sync_support_files(ssh)
end
# sync puppet manifests and apply them
@@ -46,6 +43,37 @@ module LeapCli
private
+ def sync_hiera_config(ssh)
+ dest_dir = manager.provider.hiera_sync_destination
+ ssh.leap.rsync_update do |server|
+ node = manager.node(server.host)
+ hiera_file = Path.relative_path([:hiera, node.name])
+ ssh.leap.log hiera_file + ' -> ' + node.name + ':' + dest_dir + '/hiera.yaml'
+ {:source => hiera_file, :dest => dest_dir + '/hiera.yaml'}
+ end
+ end
+
+ def sync_support_files(ssh)
+ dest_dir = manager.provider.hiera_sync_destination
+ ssh.leap.rsync_update do |server|
+ node = manager.node(server.host)
+ files_to_sync = node.file_paths.collect {|path| Path.relative_path(path, Path.provider) }
+ if files_to_sync.any?
+ ssh.leap.log(files_to_sync.join(', ') + ' -> ' + node.name + ':' + dest_dir)
+ {
+ :chdir => Path.provider,
+ :source => ".",
+ :dest => dest_dir,
+ :excludes => "*",
+ :includes => calculate_includes_from_files(files_to_sync),
+ :flags => "--relative --dirs --delete --delete-excluded --filter='protect hiera.yaml' --copy-links"
+ }
+ else
+ nil
+ end
+ end
+ end
+
def init_submodules
Dir.chdir Path.platform do
statuses = assert_run! "git submodule status"
@@ -59,5 +87,28 @@ module LeapCli
end
end
+ def calculate_includes_from_files(files)
+ # prepend '/' (kind of like ^ for rsync)
+ includes = files.collect {|file| '/' + file}
+
+ # include all sub files of specified directories
+ includes.size.times do |i|
+ if includes[i] =~ /\/$/
+ includes << includes[i] + '**'
+ end
+ end
+
+ # include all parent directories
+ includes.size.times do |i|
+ path = File.dirname(includes[i])
+ while(path != '/')
+ includes << path unless includes.include?(path)
+ path = File.dirname(path)
+ end
+ end
+
+ return includes
+ end
+
end
end \ No newline at end of file
diff --git a/lib/leap_cli/config/manager.rb b/lib/leap_cli/config/manager.rb
index 6702fc4..c860b5c 100644
--- a/lib/leap_cli/config/manager.rb
+++ b/lib/leap_cli/config/manager.rb
@@ -21,21 +21,21 @@ module LeapCli
@provider_dir = Path.provider
# load base
- base_services = load_all_json(Path.named_path([:service_config, '*'], Path.provider_base))
- base_tags = load_all_json(Path.named_path([:tag_config, '*'], Path.provider_base))
- base_common = load_json(Path.named_path(:common_config, Path.provider_base))
- base_provider = load_json(Path.named_path(:provider_config, Path.provider_base))
+ base_services = load_all_json(Path.named_path([:service_config, '*'], Path.provider_base), Config::Tag)
+ base_tags = load_all_json(Path.named_path([:tag_config, '*'], Path.provider_base), Config::Tag)
+ base_common = load_json(Path.named_path(:common_config, Path.provider_base), Config::Object)
+ base_provider = load_json(Path.named_path(:provider_config, Path.provider_base), Config::Object)
# load provider
provider_path = Path.named_path(:provider_config, @provider_dir)
common_path = Path.named_path(:common_config, @provider_dir)
Util::assert_files_exist!(provider_path, common_path)
- @services = load_all_json(Path.named_path([:service_config, '*'], @provider_dir))
- @tags = load_all_json(Path.named_path([:tag_config, '*'], @provider_dir))
- @nodes = load_all_json(Path.named_path([:node_config, '*'], @provider_dir))
- @common = load_json(common_path)
- @provider = load_json(provider_path)
- @secrets = load_json(Path.named_path(:secrets_config, @provider_dir))
+ @services = load_all_json(Path.named_path([:service_config, '*'], @provider_dir), Config::Tag)
+ @tags = load_all_json(Path.named_path([:tag_config, '*'], @provider_dir), Config::Tag)
+ @nodes = load_all_json(Path.named_path([:node_config, '*'], @provider_dir), Config::Node)
+ @common = load_json(common_path, Config::Object)
+ @provider = load_json(provider_path, Config::Object)
+ @secrets = load_json(Path.named_path(:secrets_config, @provider_dir), Config::Object)
# inherit
@services.inherit_from! base_services
@@ -161,10 +161,10 @@ module LeapCli
private
- def load_all_json(pattern)
+ def load_all_json(pattern, object_class)
results = Config::ObjectList.new
Dir.glob(pattern).each do |filename|
- obj = load_json(filename)
+ obj = load_json(filename, object_class)
if obj
name = File.basename(filename).sub(/\.json$/,'')
obj['name'] ||= name
@@ -174,9 +174,9 @@ module LeapCli
results
end
- def load_json(filename)
+ def load_json(filename, object_class)
if !File.exists?(filename)
- return Config::Object.new(self)
+ return object_class.new(self)
end
log :loading, filename, 2
@@ -201,7 +201,7 @@ module LeapCli
log 0, exc.to_s, :indent => 1
return nil
end
- object = Config::Object.new(self)
+ object = object_class.new(self)
object.deep_merge!(hash)
return object
end
@@ -226,7 +226,7 @@ module LeapCli
# makes a node inherit options from appropriate the common, service, and tag json files.
#
def apply_inheritance(node)
- new_node = Config::Object.new(self)
+ new_node = Config::Node.new(self)
name = node.name
# inherit from common
diff --git a/lib/leap_cli/config/node.rb b/lib/leap_cli/config/node.rb
new file mode 100644
index 0000000..9eea1f3
--- /dev/null
+++ b/lib/leap_cli/config/node.rb
@@ -0,0 +1,52 @@
+#
+# Configuration for a 'node' (a server in the provider's infrastructure)
+#
+
+require 'ipaddr'
+
+module LeapCli; module Config
+
+ class Node < Object
+ attr_accessor :file_paths
+
+ def initialize(manager=nil)
+ super(manager)
+ @node = self
+ @file_paths = []
+ end
+
+ #
+ # Make a copy of ourselves, except only including the specified keys.
+ #
+ # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b'
+ #
+ def pick(*keys)
+ keys.map(&:to_s).inject(self.class.new(@manager)) do |hsh, key|
+ value = self.get(key)
+ if !value.nil?
+ hsh[key.gsub('.','_')] = value
+ end
+ hsh
+ end
+ end
+
+ #
+ # returns true if this node has an ip address in the range of the vagrant network
+ #
+ def vagrant?
+ begin
+ vagrant_range = IPAddr.new @manager.provider.vagrant.network
+ rescue ArgumentError => exc
+ Util::bail! { Util::log :invalid, "ip address '#{@node.ip_address}' vagrant.network" }
+ end
+
+ begin
+ ip_address = IPAddr.new @node.get('ip_address')
+ rescue ArgumentError => exc
+ Util::log :warning, "invalid ip address '#{@node.get('ip_address')}' for node '#{@node.name}'"
+ end
+ return vagrant_range.include?(ip_address)
+ end
+ end
+
+end; end
diff --git a/lib/leap_cli/config/object.rb b/lib/leap_cli/config/object.rb
index 395ebe3..5c6cfd0 100644
--- a/lib/leap_cli/config/object.rb
+++ b/lib/leap_cli/config/object.rb
@@ -1,6 +1,5 @@
require 'erb'
require 'json/pure' # pure ruby implementation is required for our sorted trick to work.
-require 'ipaddr'
$KCODE = 'UTF8' unless RUBY_VERSION > "1.9.0"
require 'ya2yaml' # pure ruby yaml
@@ -17,7 +16,6 @@ module LeapCli
attr_reader :node
attr_reader :manager
- attr_reader :node_list
alias :global :manager
def initialize(manager=nil, node=nil)
@@ -27,9 +25,6 @@ module LeapCli
# an object that is a node as @node equal to self, otherwise all the child objects point back to the top level node.
@node = node || self
-
- # this is only used by Config::Objects that correspond to services or tags.
- @node_list = Config::ObjectList.new
end
#
@@ -106,21 +101,6 @@ module LeapCli
##
#
- # Make a copy of ourselves, except only including the specified keys.
- #
- # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b'
- #
- def pick(*keys)
- keys.map(&:to_s).inject(Config::Object.new(@manager,@node)) do |hsh, key|
- value = self.get(key)
- if !value.nil?
- hsh[key.gsub('.','_')] = value
- end
- hsh
- end
- end
-
- #
# a deep (recursive) merge with another Config::Object.
#
# if prefer_self is set to true, the value from self will be picked when there is a conflict
@@ -196,29 +176,6 @@ module LeapCli
end
##
- ## NODE SPECIFIC
- ## maybe these should be moved to a Node class.
- ##
-
- #
- # returns true if this node has an ip address in the range of the vagrant network
- #
- def vagrant?
- begin
- vagrant_range = IPAddr.new @manager.provider.vagrant.network
- rescue ArgumentError => exc
- Util::bail! { Util::log :invalid, "ip address '#{@node.ip_address}' vagrant.network" }
- end
-
- begin
- ip_address = IPAddr.new @node.get('ip_address')
- rescue ArgumentError => exc
- Util::log :warning, "invalid ip address '#{@node.get('ip_address')}' for node '#{@node.name}'"
- end
- return vagrant_range.include?(ip_address)
- end
-
- ##
## MACROS
## these are methods used when eval'ing a value in the .json configuration
##
@@ -271,6 +228,43 @@ module LeapCli
end
#
+ # returns what the file path will be, once the file is rsynced to the server.
+ # an internal list of discovered file paths is saved, in order to rsync these files when needed.
+ #
+ # notes:
+ #
+ # * argument 'path' is relative to Path.provider/files or Path.provider_base/files
+ # * the path returned by this method is absolute
+ # * the path stored for use later by rsync is relative to Path.provider
+ # * if the path does not exist locally, but exists in provider_base, then the default file from
+ # provider_base is copied locally.
+ #
+ def file_path(path)
+ if path.is_a? Symbol
+ path = [path, @node.name]
+ end
+ actual_path = Path.find_file(path)
+ if actual_path.nil?
+ nil
+ else
+ if actual_path =~ /^#{Regexp.escape(Path.provider_base)}/
+ # if file is under Path.provider_base, we must copy the default file to
+ # to Path.provider in order for rsync to be able to sync the file.
+ local_provider_path = actual_path.sub(/^#{Regexp.escape(Path.provider_base)}/, Path.provider)
+ FileUtils.cp_r actual_path, local_provider_path
+ Util.log :created, Path.relative_path(local_provider_path)
+ actual_path = local_provider_path
+ end
+ if Dir.exists?(actual_path) && actual_path !~ /\/$/
+ actual_path += '/' # ensure directories end with /, important for building rsync command
+ end
+ relative_path = Path.relative_path(actual_path)
+ @node.file_paths << relative_path
+ @node.manager.provider.hiera_sync_destination + '/' + relative_path
+ end
+ end
+
+ #
# inserts a named secret, generating it if needed.
#
# manager.export_secrets should be called later to capture any newly generated secrets.
diff --git a/lib/leap_cli/config/tag.rb b/lib/leap_cli/config/tag.rb
new file mode 100644
index 0000000..e5e719d
--- /dev/null
+++ b/lib/leap_cli/config/tag.rb
@@ -0,0 +1,18 @@
+#
+#
+# A class for node services or node tags.
+#
+#
+
+module LeapCli; module Config
+
+ class Tag < Object
+ attr_reader :node_list
+
+ def initialize(manager=nil)
+ super(manager)
+ @node_list = Config::ObjectList.new
+ end
+ end
+
+end; end
diff --git a/lib/leap_cli/path.rb b/lib/leap_cli/path.rb
index 43f2edc..e7626b5 100644
--- a/lib/leap_cli/path.rb
+++ b/lib/leap_cli/path.rb
@@ -87,13 +87,14 @@ module LeapCli; module Path
# tries to find a file somewhere
#
def self.find_file(arg)
- file_path = named_path(arg, Path.provider)
- return file_path if File.exists?(file_path)
-
- file_path = named_path(arg, Path.provider_base)
- return file_path if File.exists?(file_path)
-
- # give up
+ [Path.provider, Path.provider_base].each do |base|
+ file_path = named_path(arg, base)
+ return file_path if File.exists?(file_path)
+ if arg.is_a? String
+ file_path = base + '/files/' + arg
+ return file_path if File.exists?(file_path)
+ end
+ end
return nil
end
diff --git a/lib/leap_cli/remote/plugin.rb b/lib/leap_cli/remote/plugin.rb
index 803ebf9..213c981 100644
--- a/lib/leap_cli/remote/plugin.rb
+++ b/lib/leap_cli/remote/plugin.rb
@@ -57,22 +57,28 @@ module LeapCli; module Remote; module Plugin
# rsync to each server
failed_servers = []
servers.each do |server|
+ options = yield server
+ next unless options
+
# build rsync command
- paths = yield server
remote_user = server.user || fetch(:user, ENV['USER'])
+ rsync_options = {
+ :flags => options[:flags],
+ :includes => options[:includes],
+ :excludes => options[:excludes],
+ :ssh => ssh_options.merge(server.options[:ssh_options]||{})
+ }
rsync_cmd = SupplyDrop::Rsync.command(
- paths[:source],
- SupplyDrop::Rsync.remote_address(remote_user, server.host, paths[:dest]),
- {:ssh => ssh_options.merge(server.options[:ssh_options]||{})}
+ options[:source],
+ SupplyDrop::Rsync.remote_address(remote_user, server.host, options[:dest]),
+ rsync_options
)
# run command
logger.debug rsync_cmd
- ok = system(rsync_cmd)
- if ok
- logger.log 1, "rsync #{paths[:source]} #{paths[:dest]}", server.host, :color => :green
- else
- failed_servers << server.host
+ Dir.chdir(options[:chdir] || '.') do
+ ok = system(rsync_cmd)
+ failed_servers << server.host unless ok
end
end