added gem rsync_command

11 files changed, 522 insertions, 0 deletions

diff --git a/vendor/rsync_command/.gitignore b/vendor/rsync_command/.gitignore
new file mode 100644
index 0000000..d87d4be
--- /dev/null
+++ b/vendor/rsync_command/.gitignore
@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
diff --git a/vendor/rsync_command/LICENSE.txt b/vendor/rsync_command/LICENSE.txt
new file mode 100644
index 0000000..f94ecdd
--- /dev/null
+++ b/vendor/rsync_command/LICENSE.txt
@@ -0,0 +1,22 @@
+Copyright (c) 2013 LEAP Encryption Access Project
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/vendor/rsync_command/README.md b/vendor/rsync_command/README.md
new file mode 100644
index 0000000..4b53a5c
--- /dev/null
+++ b/vendor/rsync_command/README.md
@@ -0,0 +1,25 @@
+RsyncCommand
+==================================
+
+The gem rsync_command provides a library wrapper around the rsync command line program, with additional support for parallel execution of rsync and configuration of OpenSSH options in the format understood by Capistrano (and Net::SSH).
+
+Installation
+------------------------------------
+
+    gem install rsync_command
+
+Usage
+------------------------------------
+
+    rsync   = RsyncCommand.new(:logger => logger, :ssh => {:auth_methods => 'publickey'}, :flags => '-a')
+    source  = '/source/path'
+    servers = ['red', 'green', 'blue']
+
+    rsync.asynchronously(servers) do |server|
+      dest = {:user => 'root', :host => server, :path => '/dest/path'}
+      rsync.exec(source, dest)
+    end
+
+    if rsync.failed?
+      puts rsync.failures.join
+    end
\ No newline at end of file
diff --git a/vendor/rsync_command/Rakefile b/vendor/rsync_command/Rakefile
new file mode 100644
index 0000000..03bba68
--- /dev/null
+++ b/vendor/rsync_command/Rakefile
@@ -0,0 +1,6 @@
+require "rake/testtask"
+
+Rake::TestTask.new do |t|
+  t.pattern = "test/*_test.rb"
+end
+task :default => :test
diff --git a/vendor/rsync_command/lib/rsync_command.rb b/vendor/rsync_command/lib/rsync_command.rb
new file mode 100644
index 0000000..39e5945
--- /dev/null
+++ b/vendor/rsync_command/lib/rsync_command.rb
@@ -0,0 +1,96 @@
+require "rsync_command/version"
+require "rsync_command/ssh_options"
+require "rsync_command/thread_pool"
+
+require 'monitor'
+
+class RsyncCommand
+  attr_accessor :failures, :logger
+
+  def initialize(options={})
+    @options = options.dup
+    @logger = @options.delete(:logger)
+    @flags = @options.delete(:flags)
+    @failures = []
+    @failures.extend(MonitorMixin)
+  end
+
+  #
+  # takes an Enumerable and iterates each item in the list in parallel.
+  #
+  def asynchronously(array, &block)
+    pool = ThreadPool.new
+    array.each do |item|
+      pool.schedule(item, &block)
+    end
+    pool.shutdown
+  end
+
+  #
+  # runs rsync, recording failures
+  #
+  def exec(src, dest, options={})
+    @failures.synchronize do
+      @failures.clear
+    end
+    rsync_cmd = command(src, dest, options)
+    if options[:chdir]
+      rsync_cmd = "cd '#{options[:chdir]}'; #{rsync_cmd}"
+    end
+    @logger.debug rsync_cmd if @logger
+    ok = system(rsync_cmd)
+    unless ok
+      @failures.synchronize do
+        @failures << {:source => src, :dest => dest, :options => options.dup}
+      end
+    end
+  end
+
+  #
+  # returns true if last exec returned a failure
+  #
+  def failed?
+    @failures && @failures.any?
+  end
+
+  #
+  # build rsync command
+  #
+  def command(src, dest, options={})
+    src = remote_address(src)
+    dest = remote_address(dest)
+    options = @options.merge(options)
+    flags = []
+    flags << @flags if @flags
+    flags << options[:flags] if options.has_key?(:flags)
+    flags << '--delete' if options[:delete]
+    flags << includes(options[:includes]) if options.has_key?(:includes)
+    flags << excludes(options[:excludes]) if options.has_key?(:excludes)
+    flags << SshOptions.new(options[:ssh]).to_flags if options.has_key?(:ssh)
+    "rsync #{flags.compact.join(' ')} #{src} #{dest}"
+  end
+
+  private
+
+  #
+  # Creates an rsync location if the +address+ is a hash with keys :user, :host, and :path
+  # (each component is optional). If +address+ is a string, we just pass it through.
+  #
+  def remote_address(address)
+    if address.is_a? String
+      address # assume it is already formatted.
+    elsif address.is_a? Hash
+      [[address[:user], address[:host]].compact.join('@'), address[:path]].compact.join(':')
+    end
+  end
+
+  def excludes(patterns)
+    [patterns].flatten.compact.map { |p| "--exclude='#{p}'" }
+  end
+
+  def includes(patterns)
+    [patterns].flatten.compact.map { |p| "--include='#{p}'" }
+  end
+
+end
+
diff --git a/vendor/rsync_command/lib/rsync_command/ssh_options.rb b/vendor/rsync_command/lib/rsync_command/ssh_options.rb
new file mode 100644
index 0000000..494ec9d
--- /dev/null
+++ b/vendor/rsync_command/lib/rsync_command/ssh_options.rb
@@ -0,0 +1,159 @@
+#
+# Converts capistrano-style ssh configuration (which uses Net::SSH) into a OpenSSH command line flags suitable for rsync.
+#
+# For a list of the options normally support by Net::SSH (and thus Capistrano), see
+# http://net-ssh.github.com/net-ssh/classes/Net/SSH.html#method-c-start
+#
+# Also, to see how Net::SSH does the opposite of the conversion we are doing here, check out:
+# https://github.com/net-ssh/net-ssh/blob/master/lib/net/ssh/config.rb
+#
+# API mismatch:
+#
+# * many OpenSSH options not supported
+# * some options only make sense for Net::SSH
+# * compression: for Net::SSH, this option is supposed to accept true, false, or algorithm. OpenSSH accepts 'yes' or 'no'
+#
+class RsyncCommand
+  class SshOptions
+
+    def initialize(options={})
+      @options = parse_options(options)
+    end
+
+    def to_flags
+      if @options.empty?
+        nil
+      else
+        %[-e "ssh #{@options.join(' ')}"]
+      end
+    end
+
+    private
+
+    def parse_options(options)
+      options.map do |key, value|
+        next unless value
+        # Convert Net::SSH options into OpenSSH options.
+        case key
+        when :auth_methods            then opt_auth_methods(value)
+        when :bind_address            then opt('BindAddress', value)
+        when :compression             then opt('Compression', value ? 'yes' : 'no')
+        when :compression_level       then opt('CompressionLevel', value.to_i)
+        when :config                  then "-F '#{value}'"
+        when :encryption              then opt('Ciphers', [value].flatten.join(','))
+        when :forward_agent           then opt('ForwardAgent', value)
+        when :global_known_hosts_file then opt('GlobalKnownHostsFile', value)
+        when :hmac                    then opt('MACs', [value].flatten.join(','))
+        when :host_key                then opt('HostKeyAlgorithms', [value].flatten.join(','))
+        when :host_key_alias          then opt('HostKeyAlias', value)
+        when :host_name               then opt('HostName', value)
+        when :kex                     then opt('KexAlgorithms', [value].flatten.join(','))
+        when :key_data                then nil # not supported
+        when :keys                    then [value].flatten.select { |k| File.exist?(k) }.map { |k| "-i '#{k}'" }
+        when :keys_only               then opt('IdentitiesOnly', value ? 'yes' : 'no')
+        when :languages               then nil # not applicable
+        when :logger                  then nil # not applicable
+        when :paranoid                then opt('StrictHostKeyChecking', value ? 'yes' : 'no')
+        when :passphrase              then nil # not supported
+        when :password                then nil # not supported
+        when :port                    then "-p #{value.to_i}"
+        when :properties              then nil # not applicable
+        when :proxy                   then nil # not applicable
+        when :rekey_blocks_limit      then nil # not supported
+        when :rekey_limit             then opt('RekeyLimit', reverse_interpret_size(value))
+        when :rekey_packet_limit      then nil # not supported
+        when :timeout                 then opt('ConnectTimeout', value.to_i)
+        when :user                    then "-l #{value}"
+        when :user_known_hosts_file   then multi_opt('UserKnownHostsFile', value)
+        when :verbose                 then opt('LogLevel', interpret_log_level(value))
+        end
+      end.compact
+    end
+
+    private
+
+    def opt(option_name, option_value)
+      "-o #{option_name}='#{option_value}'"
+    end
+
+    def multi_opt(option_name, option_values)
+      [option_values].flatten.map do |value|
+        opt(option_name, value)
+      end.join(' ')
+    end
+
+    #
+    # In OpenSSH, password and pubkey default to 'yes', hostbased defaults to 'no'.
+    # Regardless, if :auth_method is configured, then we explicitly set the auth method.
+    #
+    def opt_auth_methods(value)
+      value = [value].flatten
+      opts = []
+      if value.any?
+        if value.include? 'password'
+          opts << opt('PasswordAuthentication', 'yes')
+        else
+          opts << opt('PasswordAuthentication', 'no')
+        end
+        if value.include? 'publickey'
+          opts << opt('PubkeyAuthentication', 'yes')
+        else
+          opts << opt('PubkeyAuthentication', 'no')
+        end
+        if value.include? 'hostbased'
+          opts << opt('HostbasedAuthentication', 'yes')
+        else
+          opts << opt('HostbasedAuthentication', 'no')
+        end
+      end
+      if opts.any?
+        return opts.join(' ')
+      else
+        nil
+      end
+    end
+
+    #
+    # Converts the given integer size in bytes into a string with 'K', 'M', 'G' suffix, as appropriate.
+    #
+    # reverse of interpret_size in https://github.com/net-ssh/net-ssh/blob/master/lib/net/ssh/config.rb
+    #
+    def reverse_interpret_size(size)
+      size = size.to_i
+      if size < 1024
+        "#{size}"
+      elsif size < 1024 * 1024
+        "#{size/1024}K"
+      elsif size < 1024 * 1024 * 1024
+        "#{size/(1024*1024)}M"
+      else
+        "#{size/(1024*1024*1024)}G"
+      end
+    end
+
+    def interpret_log_level(level)
+      if level.is_a? Symbol
+        case level
+          when :debug then "DEBUG"
+          when :info then "INFO"
+          when :warn then "ERROR"
+          when :error then "ERROR"
+          when :fatal then "FATAL"
+          else "INFO"
+        end
+      elsif level.is_a?(Integer) && defined?(Logger)
+        case level
+          when Logger::DEBUG then "DEBUG"
+          when Logger::INFO then "INFO"
+          when Logger::WARN then "ERROR"
+          when Logger::ERROR then "ERROR"
+          when Logger::FATAL then "FATAL"
+          else "INFO"
+        end
+      else
+        "INFO"
+      end
+    end
+
+  end
+end
diff --git a/vendor/rsync_command/lib/rsync_command/thread_pool.rb b/vendor/rsync_command/lib/rsync_command/thread_pool.rb
new file mode 100644
index 0000000..c788ee2
--- /dev/null
+++ b/vendor/rsync_command/lib/rsync_command/thread_pool.rb
@@ -0,0 +1,36 @@
+require 'thread'
+
+class RsyncCommand
+  class ThreadPool
+    class << self
+      attr_accessor :default_size
+    end
+
+    def initialize(size=nil)
+      @size = size || ThreadPool.default_size || 10
+      @jobs = Queue.new
+      @retvals = []
+      @pool = Array.new(@size) do |i|
+        Thread.new do
+          Thread.current[:id] = i
+          catch(:exit) do
+            loop do
+              job, args = @jobs.pop
+              @retvals << job.call(*args)
+            end
+          end
+        end
+      end
+    end
+    def schedule(*args, &block)
+      @jobs << [block, args]
+    end
+    def shutdown
+      @size.times do
+        schedule { throw :exit }
+      end
+      @pool.map(&:join)
+      @retvals
+    end
+  end
+end
diff --git a/vendor/rsync_command/lib/rsync_command/version.rb b/vendor/rsync_command/lib/rsync_command/version.rb
new file mode 100644
index 0000000..654a308
--- /dev/null
+++ b/vendor/rsync_command/lib/rsync_command/version.rb
@@ -0,0 +1,3 @@
+class RsyncCommand
+  VERSION = "0.0.1"
+end
diff --git a/vendor/rsync_command/rsync_command.gemspec b/vendor/rsync_command/rsync_command.gemspec
new file mode 100644
index 0000000..5690490
--- /dev/null
+++ b/vendor/rsync_command/rsync_command.gemspec
@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rsync_command/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "rsync_command"
+  spec.version       = RsyncCommand::VERSION
+  spec.authors       = ["elijah"]
+  spec.email         = ["elijah@leap.se"]
+  spec.description   = %q{A library wrapper for the the rsync command.}
+  spec.summary       = %q{Includes support for Net::SSH-like configuration and asynchronous execution.}
+  spec.homepage      = "https://github.com/leapcode/rsync_command"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end
diff --git a/vendor/rsync_command/test/rsync_test.rb b/vendor/rsync_command/test/rsync_test.rb
new file mode 100644
index 0000000..7860f73
--- /dev/null
+++ b/vendor/rsync_command/test/rsync_test.rb
@@ -0,0 +1,74 @@
+require 'test/unit'
+require File.expand_path('../../lib/rsync_command', __FILE__)
+
+if RUBY_VERSION >= '1.9'
+  SimpleOrderedHash = ::Hash
+else
+  class SimpleOrderedHash < Hash
+    def each; self.keys.map(&:to_s).sort.each {|key| yield [key.to_sym, self[key.to_sym]]}; end
+  end
+end
+
+class RsyncTest < Test::Unit::TestCase
+
+  def test_build_simple_command
+    command = rsync_command('bar', 'foo')
+    assert_equal 'rsync -az bar foo', command
+  end
+
+  def test_allows_passing_delete
+    command = rsync_command('bar', 'foo', :delete => true)
+    assert_equal 'rsync -az --delete bar foo', command
+  end
+
+  def test_allows_specifying_an_exclude
+    command = rsync_command('bar', 'foo', :excludes => '.git')
+    assert_equal "rsync -az --exclude='.git' bar foo", command
+  end
+
+  def test_ssh_options_keys_only_lists_existing_files
+    command = rsync_command('.', 'foo', :ssh => { :keys => [__FILE__, "#{__FILE__}dadijofs"] })
+    assert_match /-i '#{__FILE__}'/, command
+  end
+
+  def test_ssh_options_ignores_keys_if_nil
+    command = rsync_command('.', 'foo', :ssh => { :keys => nil })
+    assert_equal 'rsync -az . foo', command
+    command = rsync_command('bar', 'foo')
+    assert_equal 'rsync -az bar foo', command
+  end
+
+  def test_ssh_options_config_adds_flag
+    command = rsync_command('.', 'foo', :ssh => { :config => __FILE__ })
+    assert_equal %Q[rsync -az -e "ssh -F '#{__FILE__}'" . foo], command
+  end
+
+  def test_ssh_options_port_adds_port
+    command = rsync_command('.', 'foo', :ssh => { :port => '30022' })
+    assert_equal %Q[rsync -az -e "ssh -p 30022" . foo], command
+  end
+
+  def test_ssh_options_ignores_config_if_nil_or_false
+    command = rsync_command('.', 'foo', :ssh => { :config => nil })
+    assert_equal 'rsync -az . foo', command
+    command = rsync_command('.', 'foo', :ssh => { :config => false })
+    assert_equal 'rsync -az . foo', command
+  end
+
+  def test_remote_address
+    cmd = rsync_command('.', {:user => 'user', :host => 'box.local', :path => '/tmp'})
+    assert_equal "rsync -az . user@box.local:/tmp", cmd
+  end
+
+  #def test_remote_address_drops_at_when_user_is_nil
+  #  assert_equal 'box.local:/tmp', SupplyDrop::Rsync.remote_address(nil, 'box.local', '/tmp')
+  #end
+
+  protected
+
+  def rsync_command(src, dest, options={})
+    rsync = RsyncCommand.new(:flags => '-az')
+    rsync.command(src, dest, options)
+  end
+
+end
\ No newline at end of file
diff --git a/vendor/rsync_command/test/ssh_options_test.rb b/vendor/rsync_command/test/ssh_options_test.rb
new file mode 100644
index 0000000..1bffa00
--- /dev/null
+++ b/vendor/rsync_command/test/ssh_options_test.rb
@@ -0,0 +1,61 @@
+require 'test/unit'
+require File.expand_path('../../lib/rsync_command', __FILE__)
+
+class SshOptionsTest < Test::Unit::TestCase
+
+  def test_simple_ssh_options
+    options = ssh_options(Hash[
+      :bind_address, '0.0.0.0',
+      :compression, true,
+      :compression_level, 1,
+      :config, '/etc/ssh/ssh_config',
+      :global_known_hosts_file, '/etc/ssh/known_hosts',
+      :host_name, 'myhost',
+      :keys_only, false,
+      :paranoid, true,
+      :port, 2222,
+      :timeout, 10000,
+      :user, 'root',
+      :user_known_hosts_file, '~/.ssh/known_hosts'
+    ])
+    assert_match /-o BindAddress='0.0.0.0'/, options
+    assert_match /-o Compression='yes'/, options
+    assert_match %r{-o CompressionLevel='1' -F '/etc/ssh/ssh_config'}, options
+    assert_match %r{-o GlobalKnownHostsFile='/etc/ssh/known_hosts'}, options
+    assert_match /-o HostName='myhost'/, options
+    assert_match /-o StrictHostKeyChecking='yes' -p 2222/, options
+    assert_match /-o ConnectTimeout='10000' -l root/, options
+    assert_match %r{-o UserKnownHostsFile='~/.ssh/known_hosts'}, options
+  end
+
+  def test_complex_ssh_options
+    options = ssh_options(Hash[
+      :auth_methods, 'publickey',
+      :encryption, ['aes256-cbc', 'aes192-cbc'],
+      :hmac, 'hmac-sha2-256',
+      :host_key, 'ecdsa-sha2-nistp256-cert-v01@openssh.com',
+      :rekey_limit, 2*1024*1024,
+      :verbose, :debug,
+      :user_known_hosts_file, ['~/.ssh/known_hosts', '~/.ssh/production_known_hosts']
+    ])
+    assert_match /PasswordAuthentication='no'/, options
+    assert_match /PubkeyAuthentication='yes'/, options
+    assert_match /HostbasedAuthentication='no'/, options
+    assert_match /-o PasswordAuthentication='no'/, options
+    assert_match /-o PubkeyAuthentication='yes'/, options
+    assert_match /-o HostbasedAuthentication='no'/, options
+    assert_match /-o Ciphers='aes256-cbc,aes192-cbc'/, options
+    assert_match /-o MACs='hmac-sha2-256'/, options
+    assert_match /-o HostKeyAlgorithms='ecdsa-sha2-nistp256-cert-v01@openssh.com'/, options
+    assert_match /-o RekeyLimit='2M'/, options
+    assert_match %r{-o UserKnownHostsFile='~/.ssh/known_hosts'}, options
+    assert_match %r{-o UserKnownHostsFile='~/.ssh/production_known_hosts'}, options
+    assert_match /-o LogLevel='DEBUG'/, options
+  end
+
+  protected
+
+  def ssh_options(options)
+    RsyncCommand::SshOptions.new(options).to_flags
+  end
+end