From b3542ab1f8a80e5674bbf367f3345a71f30cd0db Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 25 Nov 2013 00:42:46 -0800 Subject: initial test framework --- bin/run_tests | 147 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100755 bin/run_tests (limited to 'bin') diff --git a/bin/run_tests b/bin/run_tests new file mode 100755 index 00000000..19a58784 --- /dev/null +++ b/bin/run_tests @@ -0,0 +1,147 @@ +#!/usr/bin/ruby + +require 'minitest/unit' +require 'yaml' + +## +## CUSTOM TEST CLASSES +## + +class SkipTest < Exception +end + +# +# Our custom unit test class. All tests should be subclasses of this. +# +class LeapTest < MiniTest::Unit::TestCase + class Pass < MiniTest::Assertion + end + + # + # The default pass just does an `assert true`. In our case, we want to make the passes more explicit. + # + def pass + raise LeapTest::Pass + end + + # + # Always runs tests in alphanumeric order + # + def self.test_order + :alpha + end +end + +# +# Custom test runner in order to modify the output. +# +class LeapRunner < MiniTest::Unit + + attr_accessor :passes + + def initialize + @passes = 0 + super + end + + # + # call stack: + # MiniTest::Unit.new.run + # MiniTest::Unit.runner + # LeapTest._run + # + def _run args = [] + suites = LeapTest.send "test_suites" + output.sync = true + results = _run_suites suites, :test + @test_count = results.inject(0) { |sum, (tc, _)| sum + tc } + @assertion_count = results.inject(0) { |sum, (_, ac)| sum + ac } + report.each_with_index do |msg, i| + puts "%s" % msg + end + status + # return failures + errors if @test_count > 0 # or return nil... + rescue Interrupt + abort 'Interrupted' + end + + # + # override puke to change what prints out. + # + def puke(klass, meth, e) + e = case e + when MiniTest::Skip then + @skips += 1 + #if @verbose + report_line("SKIP", klass, meth, e, e.message) + #end + when LeapTest::Pass then + @passes += 1 + report_line("PASS", klass, meth) + when MiniTest::Assertion then + @failures += 1 + report_line("FAIL", klass, meth, e, e.message) + else + @errors += 1 + bt = MiniTest::filter_backtrace(e.backtrace).join "\n" + report_line("ERROR", klass, meth, e, "#{e.class}: #{e.message}\n#{bt}") + end + @report << e + return "" # disable the marching ants + end + + # + # override default status slightly + # + def status(io = self.output) + format = "%d tests, %d assertions, %d passes, %d failures, %d errors, %d skips" + io.puts format % [test_count, assertion_count, passes, failures, errors, skips] + end + + private + + # + # returns a string for a PASS, SKIP, or FAIL error + # + def report_line(prefix, klass, meth, e=nil, message=nil) + if e && message + #indent = "\n" + (" " * (prefix.length+4)) + indent = "\n " + msg_txt = indent + message.split("\n").join(indent) + "#{prefix}: #{readable(klass.name)} > #{readable(meth)} [#{File.basename(location(e))}]:#{msg_txt}\n" + else + "#{prefix}: #{readable(klass.name)} > #{readable(meth)}\n" + end + end + + # + # Converts snake_case and CamelCase to something more pleasant for humans to read. + # + def readable(str) + str.gsub(/([A-Z]+)([A-Z][a-z])/, '\1 \2'). + gsub(/([a-z])([A-Z])/, '\1 \2'). + gsub(/_/, ' '). + sub(/^test /i, ''). + downcase + end +end + +## +## RUN THE TESTS +## + +if File.exists?('/etc/leap/hiera.yaml') + $node = YAML.load_file('/etc/leap/hiera.yaml') +else + $node = {"services" => [], "dummy" => true} +end + +Dir[File.expand_path('../../tests/*.rb', __FILE__)].each do |test_file| + begin + require test_file + rescue SkipTest + end +end + +MiniTest::Unit.runner = LeapRunner.new +MiniTest::Unit.new.run -- cgit v1.2.3 From e23dc7849d1118ed2dfe7f37a6109ffbeefa959c Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 6 Dec 2013 14:59:54 -0800 Subject: added test dependencies and test halting. --- bin/run_tests | 119 ++++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 104 insertions(+), 15 deletions(-) (limited to 'bin') diff --git a/bin/run_tests b/bin/run_tests index 19a58784..91c742c5 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -1,15 +1,40 @@ #!/usr/bin/ruby +# +# this script will run the unit tests in ../tests/*.rb. +# +# Tests for the platform differ from traditional ruby unit tests in a few ways: +# +# (1) at the end of every test function, you should call 'pass()' +# (2) you can specify test dependencies by calling depends_on("TestFirst") in the test class definition. +# (3) test functions are always run in alphabetical order. +# (4) any halt or error will stop the testing unless --continue is specified. +# + require 'minitest/unit' require 'yaml' +require 'tsort' ## -## CUSTOM TEST CLASSES +## EXCEPTIONS ## +# this class is raised if a test file wants to be skipped entirely. class SkipTest < Exception end +# raised if --no-continue and there is an error +class TestError < Exception +end + +# raised if --no-continue and there is a failure +class TestFailure < Exception +end + +## +## CUSTOM UNIT TEST CLASS +## + # # Our custom unit test class. All tests should be subclasses of this. # @@ -17,6 +42,17 @@ class LeapTest < MiniTest::Unit::TestCase class Pass < MiniTest::Assertion end + # + # Test class dependencies + # + def self.depends_on(*class_names) + @dependencies ||= [] + @dependencies += class_names + end + def self.dependencies + @dependencies || [] + end + # # The default pass just does an `assert true`. In our case, we want to make the passes more explicit. # @@ -25,7 +61,7 @@ class LeapTest < MiniTest::Unit::TestCase end # - # Always runs tests in alphanumeric order + # Always runs test methods within a test class in alphanumeric order # def self.test_order :alpha @@ -53,40 +89,50 @@ class LeapRunner < MiniTest::Unit def _run args = [] suites = LeapTest.send "test_suites" output.sync = true - results = _run_suites suites, :test + suites = TestDependencyGraph.new(suites).sorted + results = _run_suites(suites, :test) @test_count = results.inject(0) { |sum, (tc, _)| sum + tc } @assertion_count = results.inject(0) { |sum, (_, ac)| sum + ac } - report.each_with_index do |msg, i| - puts "%s" % msg - end + report.each {|msg| output.puts msg} status - # return failures + errors if @test_count > 0 # or return nil... rescue Interrupt - abort 'Interrupted' + report.each {|msg| output.puts msg} + abort 'Tests halted on interrupt.' + rescue TestFailure + report.each {|msg| output.puts msg} + abort 'Tests halted on failure (because of --no-continue).' + rescue TestError + report.each {|msg| output.puts msg} + abort 'Tests halted on error (because of --no-continue).' end # # override puke to change what prints out. # def puke(klass, meth, e) - e = case e + case e when MiniTest::Skip then @skips += 1 #if @verbose - report_line("SKIP", klass, meth, e, e.message) + @report << report_line("SKIP", klass, meth, e, e.message) #end when LeapTest::Pass then @passes += 1 - report_line("PASS", klass, meth) + @report << report_line("PASS", klass, meth) when MiniTest::Assertion then @failures += 1 - report_line("FAIL", klass, meth, e, e.message) + @report << report_line("FAIL", klass, meth, e, e.message) + if $halt_on_failure + raise TestFailure.new + end else @errors += 1 bt = MiniTest::filter_backtrace(e.backtrace).join "\n" - report_line("ERROR", klass, meth, e, "#{e.class}: #{e.message}\n#{bt}") + @report << report_line("ERROR", klass, meth, e, "#{e.class}: #{e.message}\n#{bt}") + if $halt_on_failure + raise TestError.new + end end - @report << e return "" # disable the marching ants end @@ -126,16 +172,49 @@ class LeapRunner < MiniTest::Unit end end +## +## Dependency resolution +## Use a topographical sort to manage test dependencies +## + +class TestDependencyGraph + include TSort + + def initialize(test_classes) + @dependencies = {} # each key is a test class name, and the values + # are arrays of test class names that the key depends on. + test_classes.each do |test_class| + @dependencies[test_class.name] = test_class.dependencies + end + end + + def tsort_each_node(&block) + @dependencies.each_key(&block) + end + + def tsort_each_child(test_class_name, &block) + @dependencies[test_class_name].each(&block) + end + + def sorted + self.tsort.collect {|class_name| + Kernel.const_get(class_name) + } + end +end + ## ## RUN THE TESTS ## +# load node data from hiera file if File.exists?('/etc/leap/hiera.yaml') $node = YAML.load_file('/etc/leap/hiera.yaml') else - $node = {"services" => [], "dummy" => true} + $node = {"services" => ['webapp'], "dummy" => true} end +# load all test classes Dir[File.expand_path('../../tests/*.rb', __FILE__)].each do |test_file| begin require test_file @@ -143,5 +222,15 @@ Dir[File.expand_path('../../tests/*.rb', __FILE__)].each do |test_file| end end +# parse command line options +$halt_on_failure = true +loop do + case ARGV[0] + when '--continue' then ARGV.shift; $halt_on_failure = false + else break + end +end + +# run some tests already MiniTest::Unit.runner = LeapRunner.new MiniTest::Unit.new.run -- cgit v1.2.3 From 68328b6e15a38a74261ee46b06091b8e63f85499 Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 9 Dec 2013 23:06:19 -0800 Subject: added initial white-box tests for couchdb and webapp nodes --- bin/run_tests | 101 +++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 86 insertions(+), 15 deletions(-) (limited to 'bin') diff --git a/bin/run_tests b/bin/run_tests index 91c742c5..8c5fb492 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -14,6 +14,7 @@ require 'minitest/unit' require 'yaml' require 'tsort' +require 'net/http' ## ## EXCEPTIONS @@ -61,11 +62,84 @@ class LeapTest < MiniTest::Unit::TestCase end # + # the default fail() is part of the kernel and it just throws a runtime exception. for tests, + # we want the same behavior as assert(false) + # + def fail(msg=nil) + assert(false, msg) + end + # Always runs test methods within a test class in alphanumeric order # def self.test_order :alpha end + + # + # attempts a http GET on the url, yields |body, response, error| + # + def get(url, params=nil) + uri = URI(url) + if params + uri.query = URI.encode_www_form(params) + end + response = Net::HTTP.get_response(uri) + if response.is_a?(Net::HTTPSuccess) + yield response.body, response, nil + else + yield nil, response, nil + end + rescue => exc + yield nil, nil, exc + end + + def assert_get(url, params=nil) + get(url, params) do |body, response, error| + if body + yield body + elsif response + fail "Expected a 200 status code from #{url}, but got #{response.code} instead." + else + fail "Expected a response from #{url}, but got \"#{error}\" instead." + end + end + end + + # + # Matches the regexp in the file, and returns the first matched string (or fails if no match). + # + def file_match(filename, regexp) + if match = File.read(filename).match(regexp) + match.captures.first + else + fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}." + end + end + + # + # Matches the regexp in the file, and returns array of matched strings (or fails if no match). + # + def file_matches(filename, regexp) + if match = File.read(filename).match(regexp) + match.captures + else + fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}." + end + end + + # + # checks to make sure the given property path exists in $node (e.g. hiera.yaml) + # and returns the value + # + def assert_property(property) + latest = $node + property.split('.').each do |segment| + latest = latest[segment] + fail "Required node property `#{property}` is missing." if latest.nil? + end + return latest + end + end # @@ -93,16 +167,12 @@ class LeapRunner < MiniTest::Unit results = _run_suites(suites, :test) @test_count = results.inject(0) { |sum, (tc, _)| sum + tc } @assertion_count = results.inject(0) { |sum, (_, ac)| sum + ac } - report.each {|msg| output.puts msg} status rescue Interrupt - report.each {|msg| output.puts msg} abort 'Tests halted on interrupt.' rescue TestFailure - report.each {|msg| output.puts msg} abort 'Tests halted on failure (because of --no-continue).' rescue TestError - report.each {|msg| output.puts msg} abort 'Tests halted on error (because of --no-continue).' end @@ -114,21 +184,21 @@ class LeapRunner < MiniTest::Unit when MiniTest::Skip then @skips += 1 #if @verbose - @report << report_line("SKIP", klass, meth, e, e.message) + report_line("SKIP", klass, meth, e, e.message) #end when LeapTest::Pass then @passes += 1 @report << report_line("PASS", klass, meth) when MiniTest::Assertion then @failures += 1 - @report << report_line("FAIL", klass, meth, e, e.message) + report_line("FAIL", klass, meth, e, e.message) if $halt_on_failure raise TestFailure.new end else @errors += 1 bt = MiniTest::filter_backtrace(e.backtrace).join "\n" - @report << report_line("ERROR", klass, meth, e, "#{e.class}: #{e.message}\n#{bt}") + report_line("ERROR", klass, meth, e, "#{e.class}: #{e.message}\n#{bt}") if $halt_on_failure raise TestError.new end @@ -141,7 +211,7 @@ class LeapRunner < MiniTest::Unit # def status(io = self.output) format = "%d tests, %d assertions, %d passes, %d failures, %d errors, %d skips" - io.puts format % [test_count, assertion_count, passes, failures, errors, skips] + output.puts format % [test_count, assertion_count, passes, failures, errors, skips] end private @@ -151,13 +221,14 @@ class LeapRunner < MiniTest::Unit # def report_line(prefix, klass, meth, e=nil, message=nil) if e && message - #indent = "\n" + (" " * (prefix.length+4)) indent = "\n " msg_txt = indent + message.split("\n").join(indent) - "#{prefix}: #{readable(klass.name)} > #{readable(meth)} [#{File.basename(location(e))}]:#{msg_txt}\n" + output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)} [#{File.basename(location(e))}]:#{msg_txt}" else - "#{prefix}: #{readable(klass.name)} > #{readable(meth)}\n" + output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)}" end + # I don't understand at all why, but adding a very tiny sleep here will + sleep(0.0001) # keep lines from being joined together by the logger. output.flush doesn't. end # @@ -167,8 +238,8 @@ class LeapRunner < MiniTest::Unit str.gsub(/([A-Z]+)([A-Z][a-z])/, '\1 \2'). gsub(/([a-z])([A-Z])/, '\1 \2'). gsub(/_/, ' '). - sub(/^test /i, ''). - downcase + sub(/^test (\d* )?/i, ''). + downcase.capitalize end end @@ -211,11 +282,11 @@ end if File.exists?('/etc/leap/hiera.yaml') $node = YAML.load_file('/etc/leap/hiera.yaml') else - $node = {"services" => ['webapp'], "dummy" => true} + $node = {"services" => [], "dummy" => true} end # load all test classes -Dir[File.expand_path('../../tests/*.rb', __FILE__)].each do |test_file| +Dir[File.expand_path('../../tests/white-box/*.rb', __FILE__)].each do |test_file| begin require test_file rescue SkipTest -- cgit v1.2.3 From 34678e895a5a40da6f444199983fee3f8ce518ee Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 27 Dec 2013 02:43:24 -0800 Subject: added some network tests for stunnel --- bin/run_tests | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 84 insertions(+), 3 deletions(-) (limited to 'bin') diff --git a/bin/run_tests b/bin/run_tests index 8c5fb492..89fbdb24 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -93,18 +93,99 @@ class LeapTest < MiniTest::Unit::TestCase yield nil, nil, exc end - def assert_get(url, params=nil) + def assert_get(url, params=nil, options=nil) + options ||= {} get(url, params) do |body, response, error| if body yield body elsif response - fail "Expected a 200 status code from #{url}, but got #{response.code} instead." + fail ["Expected a 200 status code from #{url}, but got #{response.code} instead.", options[:error_msg]].compact.join("\n") else - fail "Expected a response from #{url}, but got \"#{error}\" instead." + fail ["Expected a response from #{url}, but got \"#{error}\" instead.", options[:error_msg]].compact.join("\n") end end end + # + # test if a socket can be connected to + # + + # + # tcp connection helper with timeout + # + def try_tcp_connect(host, port, timeout = 5) + addr = Socket.getaddrinfo(host, nil) + sockaddr = Socket.pack_sockaddr_in(port, addr[0][3]) + + Socket.new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0).tap do |socket| + socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) + begin + socket.connect_nonblock(sockaddr) + rescue IO::WaitReadable + if IO.select([socket], nil, nil, timeout) == nil + raise "Connection timeout" + else + socket.connect_nonblock(sockaddr) + end + rescue IO::WaitWritable + if IO.select(nil, [socket], nil, timeout) == nil + raise "Connection timeout" + else + socket.connect_nonblock(sockaddr) + end + end + return socket + end + end + + def try_tcp_write(socket, timeout = 5) + begin + socket.write_nonblock("\0") + rescue IO::WaitReadable + if IO.select([socket], nil, nil, timeout) == nil + raise "Write timeout" + else + retry + end + rescue IO::WaitWritable + if IO.select(nil, [socket], nil, timeout) == nil + raise "Write timeout" + else + retry + end + end + end + + def try_tcp_read(socket, timeout = 5) + begin + socket.read_nonblock(1) + rescue IO::WaitReadable + if IO.select([socket], nil, nil, timeout) == nil + raise "Read timeout" + else + retry + end + rescue IO::WaitWritable + if IO.select(nil, [socket], nil, timeout) == nil + raise "Read timeout" + else + retry + end + end + end + + def assert_tcp_socket(host, port, msg=nil) + begin + socket = try_tcp_connect(host, port, 1) + #try_tcp_write(socket,1) + #try_tcp_read(socket,1) + rescue StandardError => exc + fail ["Failed to open socket #{host}:#{port}", exc].join("\n") + ensure + socket.close if socket + end + end + # # Matches the regexp in the file, and returns the first matched string (or fails if no match). # -- cgit v1.2.3 From 0a56b656f8fbfd38ee1a9babdb93fbed39c4a973 Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 27 Dec 2013 12:27:53 -0800 Subject: improve couchdb test --- bin/run_tests | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'bin') diff --git a/bin/run_tests b/bin/run_tests index 89fbdb24..86a72f26 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -43,6 +43,11 @@ class LeapTest < MiniTest::Unit::TestCase class Pass < MiniTest::Assertion end + def initialize(name) + super(name) + io # << calling this will suppress the marching ants + end + # # Test class dependencies # @@ -69,6 +74,11 @@ class LeapTest < MiniTest::Unit::TestCase assert(false, msg) end + def warn(*msg) + method_name = caller.first.split('`').last.gsub(/(block in |')/,'') + MiniTest::Unit.runner.report_line("WARN", self.class, method_name, nil, msg.join("\n")) + end + # Always runs test methods within a test class in alphanumeric order # def self.test_order @@ -295,16 +305,18 @@ class LeapRunner < MiniTest::Unit output.puts format % [test_count, assertion_count, passes, failures, errors, skips] end - private - # # returns a string for a PASS, SKIP, or FAIL error # def report_line(prefix, klass, meth, e=nil, message=nil) - if e && message + if message indent = "\n " msg_txt = indent + message.split("\n").join(indent) + end + if e && message output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)} [#{File.basename(location(e))}]:#{msg_txt}" + elsif message + output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)}:#{msg_txt}" else output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)}" end @@ -312,6 +324,8 @@ class LeapRunner < MiniTest::Unit sleep(0.0001) # keep lines from being joined together by the logger. output.flush doesn't. end + private + # # Converts snake_case and CamelCase to something more pleasant for humans to read. # -- cgit v1.2.3 From 71f2d1b0b7db6bb35dd6bd9220795aed3174d90b Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 27 Dec 2013 16:47:57 -0800 Subject: added more network tests and pgrep test helper --- bin/run_tests | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'bin') diff --git a/bin/run_tests b/bin/run_tests index 86a72f26..f4778988 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -107,7 +107,7 @@ class LeapTest < MiniTest::Unit::TestCase options ||= {} get(url, params) do |body, response, error| if body - yield body + yield body if block_given? elsif response fail ["Expected a 200 status code from #{url}, but got #{response.code} instead.", options[:error_msg]].compact.join("\n") else @@ -231,6 +231,22 @@ class LeapTest < MiniTest::Unit::TestCase return latest end + # + # works like pgrep command line + # return an array of hashes like so [{:pid => "1234", :process => "ls"}] + # + def pgrep(match) + output = `pgrep --full --list-name '#{match}'` + output.each_line.map{|line| + pid = line.split(' ')[0] + process = line.gsub(/(#{pid} |\n)/, '') + if process =~ /pgrep --full --list-name/ + nil + else + {:pid => pid, :process => process} + end + }.compact + end end # @@ -310,6 +326,7 @@ class LeapRunner < MiniTest::Unit # def report_line(prefix, klass, meth, e=nil, message=nil) if message + message = message.sub(/http:\/\/([a-z_]+):([a-zA-Z0-9_]+)@/, "http://\\1:password@") indent = "\n " msg_txt = indent + message.split("\n").join(indent) end -- cgit v1.2.3 From 353475da8d535f2904e68977c3dafa6bb3bb483a Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 30 Dec 2013 18:04:30 -0800 Subject: tests -- added tests to check that the right processes are running --- bin/run_tests | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'bin') diff --git a/bin/run_tests b/bin/run_tests index f4778988..a44fcdcf 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -249,6 +249,10 @@ class LeapTest < MiniTest::Unit::TestCase end end +def assert_running(process) + assert pgrep(process).any?, "No running process for #{process}" +end + # # Custom test runner in order to modify the output. # -- cgit v1.2.3 From ddc190489964008ffa085685e06b93a9e86c058b Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 25 Feb 2014 17:43:29 -0800 Subject: added a checkmk mode to the script `run_tests` (run with `run_tests --checkmk`. you can symlink `/srv/leap/bin/run_tests` script to anywhere if needed) --- bin/run_tests | 58 ++++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 14 deletions(-) (limited to 'bin') diff --git a/bin/run_tests b/bin/run_tests index a44fcdcf..ee517ae5 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -318,35 +318,57 @@ class LeapRunner < MiniTest::Unit end # - # override default status slightly + # override default status summary # def status(io = self.output) - format = "%d tests, %d assertions, %d passes, %d failures, %d errors, %d skips" - output.puts format % [test_count, assertion_count, passes, failures, errors, skips] + if $output_format == :human + format = "%d tests, %d assertions, %d passes, %d failures, %d errors, %d skips" + output.puts format % [test_count, assertion_count, passes, failures, errors, skips] + end end # # returns a string for a PASS, SKIP, or FAIL error # def report_line(prefix, klass, meth, e=nil, message=nil) + msg_txt = nil if message message = message.sub(/http:\/\/([a-z_]+):([a-zA-Z0-9_]+)@/, "http://\\1:password@") - indent = "\n " - msg_txt = indent + message.split("\n").join(indent) + if $output_format == :human + indent = "\n " + msg_txt = indent + message.split("\n").join(indent) + else + msg_txt = message.gsub("\n", ' ') + end end - if e && message - output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)} [#{File.basename(location(e))}]:#{msg_txt}" - elsif message - output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)}:#{msg_txt}" - else - output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)}" + + if $output_format == :human + if e && msg_txt + output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)} [#{File.basename(location(e))}]:#{msg_txt}" + elsif msg_txt + output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)}:#{msg_txt}" + else + output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)}" + end + # I don't understand at all why, but adding a very tiny sleep here will + sleep(0.0001) # keep lines from being joined together by the logger. output.flush doesn't. + elsif $output_format == :checkmk + code = CHECKMK_CODES[prefix] + msg_txt ||= "Success" if prefix == "PASS" + if e && msg_txt + output.puts "#{code} #{klass.name}/#{machine_readable(meth)} - [#{File.basename(location(e))}]:#{msg_txt}" + elsif msg_txt + output.puts "#{code} #{klass.name}/#{machine_readable(meth)} - #{msg_txt}" + else + output.puts "#{code} #{klass.name}/#{machine_readable(meth)} - no message" + end end - # I don't understand at all why, but adding a very tiny sleep here will - sleep(0.0001) # keep lines from being joined together by the logger. output.flush doesn't. end private + CHECKMK_CODES = {"PASS" => 0, "SKIP" => 1, "FAIL" => 2, "ERROR" => 3} + # # Converts snake_case and CamelCase to something more pleasant for humans to read. # @@ -357,6 +379,11 @@ class LeapRunner < MiniTest::Unit sub(/^test (\d* )?/i, ''). downcase.capitalize end + + def machine_readable(str) + str.sub(/^test_(\d+_)?/i, '') + end + end ## @@ -402,7 +429,8 @@ else end # load all test classes -Dir[File.expand_path('../../tests/white-box/*.rb', __FILE__)].each do |test_file| +this_file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__ +Dir[File.expand_path('../../tests/white-box/*.rb', this_file)].each do |test_file| begin require test_file rescue SkipTest @@ -411,9 +439,11 @@ end # parse command line options $halt_on_failure = true +$output_format = :human loop do case ARGV[0] when '--continue' then ARGV.shift; $halt_on_failure = false + when '--checkmk' then ARGV.shift; $output_format = :checkmk; $halt_on_failure = false else break end end -- cgit v1.2.3 From e724a32316e9d183ec437a5dc2687ca547efdc75 Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 26 Feb 2014 12:45:45 -0800 Subject: run_tests: added command line options --list-tests and --test (see --help) --- bin/run_tests | 125 ++++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 100 insertions(+), 25 deletions(-) (limited to 'bin') diff --git a/bin/run_tests b/bin/run_tests index ee517ae5..2336eba8 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -59,6 +59,20 @@ class LeapTest < MiniTest::Unit::TestCase @dependencies || [] end + # + # returns all the test classes, sorted in dependency order. + # + def self.test_classes + classes = ObjectSpace.each_object(Class).select {|test_class| + test_class.ancestors.include?(self) + } + return TestDependencyGraph.new(classes).sorted + end + + def self.tests + self.instance_methods.grep(/^test_/).sort + end + # # The default pass just does an `assert true`. In our case, we want to make the passes more explicit. # @@ -272,9 +286,16 @@ class LeapRunner < MiniTest::Unit # LeapTest._run # def _run args = [] - suites = LeapTest.send "test_suites" + if $pinned_test_class + suites = [$pinned_test_class] + if $pinned_test_method + options.merge!(:filter => $pinned_test_method.to_s) + end + else + suites = LeapTest.send "test_suites" + suites = TestDependencyGraph.new(suites).sorted + end output.sync = true - suites = TestDependencyGraph.new(suites).sorted results = _run_suites(suites, :test) @test_count = results.inject(0) { |sum, (tc, _)| sum + tc } @assertion_count = results.inject(0) { |sum, (_, ac)| sum + ac } @@ -418,36 +439,90 @@ class TestDependencyGraph end ## -## RUN THE TESTS +## COMMAND LINE ACTIONS ## -# load node data from hiera file -if File.exists?('/etc/leap/hiera.yaml') - $node = YAML.load_file('/etc/leap/hiera.yaml') -else - $node = {"services" => [], "dummy" => true} +def die(test, msg) + if $output_format == :human + puts "ERROR in test `#{test}`: #{msg}" + elsif $output_format == :checkmk + puts "3 #{test} - #{msg}" + end + exit(1) +end + +def print_help + puts ["USAGE: run_tests [OPTIONS]", + " --continue Don't halt on an error, but continue to the next test.", + " --checkmk Print test results in checkmk format (must come before --test).", + " --test TEST Run only the test with name TEST.", + " --list-tests Prints the names of all available tests and exit."].join("\n") + exit(0) end -# load all test classes -this_file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__ -Dir[File.expand_path('../../tests/white-box/*.rb', this_file)].each do |test_file| - begin - require test_file - rescue SkipTest +def list_tests + LeapTest.test_classes.each do |test_class| + test_class.tests.each do |test| + puts test_class.name + "/" + test.to_s.sub(/^test_(\d+_)?/, '') + end end + exit(0) end -# parse command line options -$halt_on_failure = true -$output_format = :human -loop do - case ARGV[0] - when '--continue' then ARGV.shift; $halt_on_failure = false - when '--checkmk' then ARGV.shift; $output_format = :checkmk; $halt_on_failure = false - else break +def pin_test_name(name) + test_class, test_name = name.split('/') + $pinned_test_class = LeapTest.test_classes.detect{|c| c.name == test_class} + unless $pinned_test_class + die name, "there is no test class `#{test_class}`" + end + if test_name + $pinned_test_method = $pinned_test_class.tests.detect{|m| m.to_s =~ /^test_(\d+_)?#{test_name}$/} + unless $pinned_test_method + die name, "there is no test `#{test_name}` in class `#{test_class}`" + end + end +end + +def run_tests + MiniTest::Unit.runner = LeapRunner.new + MiniTest::Unit.new.run +end + +## +## MAIN +## + +def main + # load node data from hiera file + if File.exists?('/etc/leap/hiera.yaml') + $node = YAML.load_file('/etc/leap/hiera.yaml') + else + $node = {"services" => [], "dummy" => true} + end + + # load all test classes + this_file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__ + Dir[File.expand_path('../../tests/white-box/*.rb', this_file)].each do |test_file| + begin + require test_file + rescue SkipTest + end + end + + # parse command line options + $halt_on_failure = true + $output_format = :human + loop do + case ARGV[0] + when '--continue' then ARGV.shift; $halt_on_failure = false; + when '--checkmk' then ARGV.shift; $output_format = :checkmk; $halt_on_failure = false + when '--help' then print_help + when '--test' then ARGV.shift; pin_test_name(ARGV.shift) + when '--list-tests' then list_tests + else break + end end + run_tests end -# run some tests already -MiniTest::Unit.runner = LeapRunner.new -MiniTest::Unit.new.run +main() \ No newline at end of file -- cgit v1.2.3 From cd09ed9eb9d183123652a52651a427bab558c496 Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 18 Mar 2014 00:19:45 -0700 Subject: clean up the names of tests --- bin/run_tests | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'bin') diff --git a/bin/run_tests b/bin/run_tests index 2336eba8..f4fb0157 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -394,11 +394,9 @@ class LeapRunner < MiniTest::Unit # Converts snake_case and CamelCase to something more pleasant for humans to read. # def readable(str) - str.gsub(/([A-Z]+)([A-Z][a-z])/, '\1 \2'). - gsub(/([a-z])([A-Z])/, '\1 \2'). + str. gsub(/_/, ' '). - sub(/^test (\d* )?/i, ''). - downcase.capitalize + sub(/^test (\d* )?/i, '') end def machine_readable(str) @@ -428,7 +426,12 @@ class TestDependencyGraph end def tsort_each_child(test_class_name, &block) - @dependencies[test_class_name].each(&block) + if @dependencies[test_class_name] + @dependencies[test_class_name].each(&block) + else + puts "ERROR: bad dependency, no such class `#{test_class_name}`" + exit(1) + end end def sorted @@ -476,7 +479,7 @@ def pin_test_name(name) die name, "there is no test class `#{test_class}`" end if test_name - $pinned_test_method = $pinned_test_class.tests.detect{|m| m.to_s =~ /^test_(\d+_)?#{test_name}$/} + $pinned_test_method = $pinned_test_class.tests.detect{|m| m.to_s =~ /^test_(\d+_)?#{Regexp.escape(test_name)}$/} unless $pinned_test_method die name, "there is no test `#{test_name}` in class `#{test_class}`" end -- cgit v1.2.3 From e2fc8c9ef1934e3884fcc8ddd1ff002fcbcf85ab Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 16 Apr 2014 01:19:24 -0700 Subject: exit codes for run_tests: 0 = success, 1 = warning, 2 = failure, 3 = error. --- bin/run_tests | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 9 deletions(-) (limited to 'bin') diff --git a/bin/run_tests b/bin/run_tests index f4fb0157..cb4458f3 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -16,6 +16,26 @@ require 'yaml' require 'tsort' require 'net/http' +## +## EXIT CODES +## + +EXIT_CODES = { + :success => 0, + :warning => 1, + :failure => 2, + :error => 3 +} + +def bail(code, msg=nil) + puts msg if msg + if code.is_a? Symbol + exit(EXIT_CODES[code]) + else + exit(code) + end +end + ## ## EXCEPTIONS ## @@ -90,7 +110,7 @@ class LeapTest < MiniTest::Unit::TestCase def warn(*msg) method_name = caller.first.split('`').last.gsub(/(block in |')/,'') - MiniTest::Unit.runner.report_line("WARN", self.class, method_name, nil, msg.join("\n")) + MiniTest::Unit.runner.warn(self.class, method_name, msg.join("\n")) end # Always runs test methods within a test class in alphanumeric order @@ -272,10 +292,11 @@ end # class LeapRunner < MiniTest::Unit - attr_accessor :passes + attr_accessor :passes, :warnings def initialize @passes = 0 + @warnings = 0 super end @@ -300,12 +321,13 @@ class LeapRunner < MiniTest::Unit @test_count = results.inject(0) { |sum, (tc, _)| sum + tc } @assertion_count = results.inject(0) { |sum, (_, ac)| sum + ac } status + return exit_code() rescue Interrupt - abort 'Tests halted on interrupt.' + bail :error, 'Tests halted on interrupt.' rescue TestFailure - abort 'Tests halted on failure (because of --no-continue).' + bail :failure, 'Tests halted on failure (because of --no-continue).' rescue TestError - abort 'Tests halted on error (because of --no-continue).' + bail :error, 'Tests halted on error (because of --no-continue).' end # @@ -320,7 +342,7 @@ class LeapRunner < MiniTest::Unit #end when LeapTest::Pass then @passes += 1 - @report << report_line("PASS", klass, meth) + report_line("PASS", klass, meth) when MiniTest::Assertion then @failures += 1 report_line("FAIL", klass, meth, e, e.message) @@ -348,6 +370,21 @@ class LeapRunner < MiniTest::Unit end end + # + # return an appropriate exit_code symbol + # + def exit_code + if @errors > 0 + :error + elsif @failures > 0 + :failure + elsif @warnings > 0 + :warning + else + :success + end + end + # # returns a string for a PASS, SKIP, or FAIL error # @@ -386,6 +423,14 @@ class LeapRunner < MiniTest::Unit end end + # + # a new function used by TestCase to report warnings. + # + def warn(klass, method_name, msg) + @warnings += 1 + report_line("WARN", klass, method_name, nil, msg) + end + private CHECKMK_CODES = {"PASS" => 0, "SKIP" => 1, "FAIL" => 2, "ERROR" => 3} @@ -430,7 +475,7 @@ class TestDependencyGraph @dependencies[test_class_name].each(&block) else puts "ERROR: bad dependency, no such class `#{test_class_name}`" - exit(1) + bail :error end end @@ -451,7 +496,7 @@ def die(test, msg) elsif $output_format == :checkmk puts "3 #{test} - #{msg}" end - exit(1) + bail :error end def print_help @@ -488,7 +533,8 @@ end def run_tests MiniTest::Unit.runner = LeapRunner.new - MiniTest::Unit.new.run + exit_code = MiniTest::Unit.new.run + bail exit_code end ## -- cgit v1.2.3 From 723433a385d4d72721efb36bdf144ad89ad613a7 Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 16 Apr 2014 01:40:11 -0700 Subject: run_tests: added options --retry and --wait (to keep retrying tests if there is any problem). --- bin/run_tests | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) (limited to 'bin') diff --git a/bin/run_tests b/bin/run_tests index cb4458f3..b258e85a 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -365,8 +365,8 @@ class LeapRunner < MiniTest::Unit # def status(io = self.output) if $output_format == :human - format = "%d tests, %d assertions, %d passes, %d failures, %d errors, %d skips" - output.puts format % [test_count, assertion_count, passes, failures, errors, skips] + format = "%d tests: %d passes, %d skips, %d warnings, %d failures, %d errors" + output.puts format % [test_count, passes, skips, warnings, failures, errors] end end @@ -501,10 +501,12 @@ end def print_help puts ["USAGE: run_tests [OPTIONS]", - " --continue Don't halt on an error, but continue to the next test.", - " --checkmk Print test results in checkmk format (must come before --test).", - " --test TEST Run only the test with name TEST.", - " --list-tests Prints the names of all available tests and exit."].join("\n") + " --continue Don't halt on an error, but continue to the next test.", + " --checkmk Print test results in checkmk format (must come before --test).", + " --test TEST Run only the test with name TEST.", + " --list-tests Prints the names of all available tests and exit.", + " --retry COUNT If the tests don't pass, retry COUNT additional times (default is zero)", + " --wait SECONDS Wait for SECONDS between retries (default is 5)"].join("\n") exit(0) end @@ -531,9 +533,21 @@ def pin_test_name(name) end end +# +# run the tests, multiple times if `--repeat` and not all tests were successful. +# def run_tests - MiniTest::Unit.runner = LeapRunner.new - exit_code = MiniTest::Unit.new.run + exit_code = nil + run_count = $repeat ? $repeat + 1 : 1 + run_count.times do |i| + MiniTest::Unit.runner = LeapRunner.new + exit_code = MiniTest::Unit.new.run + if !$repeat || exit_code == :success + break + elsif i != run_count-1 + sleep $wait + end + end bail exit_code end @@ -561,6 +575,8 @@ def main # parse command line options $halt_on_failure = true $output_format = :human + $repeat = false + $wait = 5 loop do case ARGV[0] when '--continue' then ARGV.shift; $halt_on_failure = false; @@ -568,6 +584,8 @@ def main when '--help' then print_help when '--test' then ARGV.shift; pin_test_name(ARGV.shift) when '--list-tests' then list_tests + when '--repeat' then ARGV.shift; $repeat = ARGV.shift.to_i + when '--wait' then ARGV.shift; $wait = ARGV.shift.to_i else break end end -- cgit v1.2.3 From 2d64dafaf6238852d80b326ca22359da4371e912 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Thu, 17 Apr 2014 15:22:57 -0400 Subject: fix --retry argument, the help listed it correctly, but the code was using --repeat (#5119) Change-Id: I48b0ae8b3d8ab91c4ca363a2bdece46952cce5a9 --- bin/run_tests | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'bin') diff --git a/bin/run_tests b/bin/run_tests index b258e85a..9102c325 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -534,15 +534,15 @@ def pin_test_name(name) end # -# run the tests, multiple times if `--repeat` and not all tests were successful. +# run the tests, multiple times if `--retry` and not all tests were successful. # def run_tests exit_code = nil - run_count = $repeat ? $repeat + 1 : 1 + run_count = $retry ? $retry + 1 : 1 run_count.times do |i| MiniTest::Unit.runner = LeapRunner.new exit_code = MiniTest::Unit.new.run - if !$repeat || exit_code == :success + if !$retry || exit_code == :success break elsif i != run_count-1 sleep $wait @@ -575,7 +575,7 @@ def main # parse command line options $halt_on_failure = true $output_format = :human - $repeat = false + $retry = false $wait = 5 loop do case ARGV[0] @@ -584,7 +584,7 @@ def main when '--help' then print_help when '--test' then ARGV.shift; pin_test_name(ARGV.shift) when '--list-tests' then list_tests - when '--repeat' then ARGV.shift; $repeat = ARGV.shift.to_i + when '--retry' then ARGV.shift; $retry = ARGV.shift.to_i when '--wait' then ARGV.shift; $wait = ARGV.shift.to_i else break end @@ -592,4 +592,4 @@ def main run_tests end -main() \ No newline at end of file +main() -- cgit v1.2.3