summaryrefslogtreecommitdiff
path: root/bin/run_tests
diff options
context:
space:
mode:
authorMicah <micah@leap.se>2016-05-10 14:48:26 -0400
committerMicah <micah@leap.se>2016-05-10 14:48:26 -0400
commit86c85582065c391aa13c0b9b397dfd1aa2e2ac7b (patch)
tree7c027409a517d862864bf3650f4a8a66f615162d /bin/run_tests
parent70b1c648b94e6c007b9241a4661f33881e74485f (diff)
parent66b4c6b5ec6fe2f242020845fe92715ae2cdcc1e (diff)
Merge tag '0.8.0'
Release 0.8.0
Diffstat (limited to 'bin/run_tests')
-rwxr-xr-xbin/run_tests52
1 files changed, 43 insertions, 9 deletions
diff --git a/bin/run_tests b/bin/run_tests
index 8eab5286..b6784ed5 100755
--- a/bin/run_tests
+++ b/bin/run_tests
@@ -14,6 +14,7 @@
require 'minitest/unit'
require 'yaml'
require 'tsort'
+require 'timeout'
##
## CONSTANTS
@@ -33,7 +34,8 @@ HELPER_PATHS = [
]
TEST_PATHS = [
'../../tests/white-box/*.rb',
- '/srv/leap/files/tests/white-box/*.rb'
+ '/srv/leap/files/tests/white-box/*.rb',
+ '/srv/leap/tests_custom/*.rb'
]
##
@@ -80,6 +82,8 @@ end
class LeapTest < MiniTest::Unit::TestCase
class Pass < MiniTest::Assertion
end
+ class SilentPass < Pass
+ end
class Ignore < MiniTest::Assertion
end
@@ -114,6 +118,16 @@ class LeapTest < MiniTest::Unit::TestCase
end
#
+ # thrown Timeout::Error if test run
+ # takes longer than $timeout
+ #
+ def run(*args)
+ Timeout::timeout($timeout, Timeout::Error) do
+ super(*args)
+ end
+ end
+
+ #
# The default pass just does an `assert true`. In our case, we want to make the passes more explicit.
#
def pass
@@ -121,6 +135,12 @@ class LeapTest < MiniTest::Unit::TestCase
end
#
+ # This is just like pass(), but the result is normally silent, unless `run_tests --test TEST`
+ def silent_pass
+ raise LeapTest::SilentPass
+ end
+
+ #
# Called when the test should be silently ignored.
#
def ignore
@@ -143,6 +163,7 @@ class LeapTest < MiniTest::Unit::TestCase
MiniTest::Unit.runner.warn(self.class, method_name, msg.join("\n"))
end
+ #
# Always runs test methods within a test class in alphanumeric order
#
def self.test_order
@@ -208,6 +229,10 @@ class LeapRunner < MiniTest::Unit
if @verbose
report_line("IGNORE", klass, meth, e, e.message)
end
+ when LeapTest::SilentPass then
+ if $pinned_test_method || $output_format == :checkmk
+ report_line("PASS", klass, meth)
+ end
when LeapTest::Pass then
@passes += 1
report_line("PASS", klass, meth)
@@ -217,6 +242,12 @@ class LeapRunner < MiniTest::Unit
if $halt_on_failure
raise TestFailure.new
end
+ when Timeout::Error then
+ @failures += 1
+ report_line("TIMEOUT", klass, meth, nil, "Test stopped because timeout exceeded (#{$timeout} seconds).")
+ if $halt_on_failure
+ raise TestFailure.new
+ end
else
@errors += 1
bt = MiniTest::filter_backtrace(e.backtrace).join "\n"
@@ -260,7 +291,7 @@ class LeapRunner < MiniTest::Unit
def report_line(prefix, klass, meth, e=nil, message=nil)
msg_txt = nil
if message
- message = message.sub(/http:\/\/([a-z_]+):([a-zA-Z0-9_]+)@/, "http://\\1:REDACTED@")
+ message = message.gsub(/http:\/\/([a-z_]+):([a-zA-Z0-9_]+)@/, "http://\\1:REDACTED@")
if $output_format == :human
indent = "\n "
msg_txt = indent + message.split("\n").join(indent)
@@ -370,13 +401,14 @@ end
def print_help
puts ["USAGE: run_tests [OPTIONS]",
- " --continue Don't halt on an error, but continue to the next test.",
- " --checkmk Print test results in checkmk format (must come before --test).",
- " --test TEST Run only the test with name TEST.",
- " --list-tests Prints the names of all available tests and exit.",
- " --retry COUNT If the tests don't pass, retry COUNT additional times (default is zero)",
- " --wait SECONDS Wait for SECONDS between retries (default is 5)",
- " --debug Print out full stack trace on errors"].join("\n")
+ " --continue Don't halt on an error, but continue to the next test.",
+ " --checkmk Print test results in checkmk format (must come before --test).",
+ " --test TEST Run only the test with name TEST.",
+ " --list-tests Prints the names of all available tests and exit.",
+ " --retry COUNT If the tests don't pass, retry COUNT additional times (default is zero).",
+ " --timeout SECONDS Halt a test if it exceed SECONDS (default is 30).",
+ " --wait SECONDS Wait for SECONDS between retries (default is 5).",
+ " --debug Print out full stack trace on errors."].join("\n")
exit(0)
end
@@ -454,6 +486,7 @@ def main
$output_format = :human
$retry = false
$wait = 5
+ $timeout = 30
loop do
case ARGV[0]
when '--continue' then ARGV.shift; $halt_on_failure = false;
@@ -462,6 +495,7 @@ def main
when '--test' then ARGV.shift; pin_test_name(ARGV.shift)
when '--list-tests' then list_tests
when '--retry' then ARGV.shift; $retry = ARGV.shift.to_i
+ when '--timeout' then ARGV.shift; $timeout = ARGV.shift.to_i;
when '--wait' then ARGV.shift; $wait = ARGV.shift.to_i
when '--debug' then ARGV.shift
when '-d' then ARGV.shift