summaryrefslogtreecommitdiff
path: root/bin/run_tests
blob: 5733f5268c30c120f14ad68056569b38ddb49dd2 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
#!/usr/bin/ruby

#
# this script will run the unit tests in ../tests/*.rb.
#
# Tests for the platform differ from traditional ruby unit tests in a few ways:
#
# (1) at the end of every test function, you should call 'pass()'
# (2) you can specify test dependencies by calling depends_on("TestFirst") in the test class definition.
# (3) test functions are always run in alphabetical order.
# (4) any halt or error will stop the testing unless --continue is specified.
#

require 'minitest/unit'
require 'yaml'
require 'tsort'
require 'timeout'

##
## CONSTANTS
##

EXIT_CODES = {
  :success => 0,
  :warning => 1,
  :failure => 2,
  :error => 3
}

HIERA_FILE   = '/etc/leap/hiera.yaml'
HELPER_PATHS = [
  '../../tests/helpers/*.rb',
  '/srv/leap/files/tests/helpers/*.rb'
]
TEST_PATHS   = [
  '../../tests/white-box/*.rb',
  '/srv/leap/files/tests/white-box/*.rb',
  '/srv/leap/tests_custom/*.rb'
]

##
## UTILITY
##

def bail(code, msg=nil)
  puts msg if msg
  if code.is_a? Symbol
    exit(EXIT_CODES[code])
  else
    exit(code)
  end
end

def service?(service)
  $node["services"].include?(service.to_s)
end

##
## EXCEPTIONS
##

# this class is raised if a test file wants to be skipped entirely.
# (to skip an individual test, MiniTest::Skip is used instead)
class SkipTest < StandardError
end

# raised if --no-continue and there is an error
class TestError < StandardError
end

# raised if --no-continue and there is a failure
class TestFailure < StandardError
end

##
## CUSTOM UNIT TEST CLASS
##

#
# Our custom unit test class. All tests should be subclasses of this.
#
class LeapTest < MiniTest::Unit::TestCase
  class Pass < MiniTest::Assertion
  end
  class Ignore < MiniTest::Assertion
  end

  def initialize(name)
    super(name)
    io # << calling this will suppress the marching ants
  end

  #
  # Test class dependencies
  #
  def self.depends_on(*class_names)
    @dependencies ||= []
    @dependencies += class_names
  end
  def self.dependencies
    @dependencies || []
  end

  #
  # returns all the test classes, sorted in dependency order.
  #
  def self.test_classes
    classes = ObjectSpace.each_object(Class).select {|test_class|
      test_class.ancestors.include?(self)
    }
    return TestDependencyGraph.new(classes).sorted
  end

  def self.tests
    self.instance_methods.grep(/^test_/).sort
  end

  #
  # thrown Timeout::Error if test run
  # takes longer than $timeout
  #
  def run(*args)
    Timeout::timeout($timeout, Timeout::Error) do
      super(*args)
    end
  end

  #
  # The default pass just does an `assert true`. In our case, we want to make the passes more explicit.
  #
  def pass
    raise LeapTest::Pass
  end

  #
  # Called when the test should be silently ignored.
  #
  def ignore
    raise LeapTest::Ignore
  end

  #
  # the default fail() is part of the kernel and it just throws a runtime exception. for tests,
  # we want the same behavior as assert(false)
  #
  def fail(msg=nil, exception=nil)
    if DEBUG && exception && exception.respond_to?(:backtrace)
      msg += MiniTest::filter_backtrace(exception.backtrace).join "\n"
    end
    assert(false, msg)
  end

  def warn(*msg)
    method_name = caller.first.split('`').last.gsub(/(block in |')/,'')
    MiniTest::Unit.runner.warn(self.class, method_name, msg.join("\n"))
  end

  #
  # Always runs test methods within a test class in alphanumeric order
  #
  def self.test_order
    :alpha
  end

end

#
# Custom test runner in order to modify the output.
#
class LeapRunner < MiniTest::Unit

  attr_accessor :passes, :warnings

  def initialize
    @passes = 0
    @warnings = 0
    @ignores = 0
    super
  end

  #
  # call stack:
  #   MiniTest::Unit.new.run
  #     MiniTest::Unit.runner
  #       LeapTest._run
  #
  def _run args = []
    if $pinned_test_class
      suites = [$pinned_test_class]
      if $pinned_test_method
        options.merge!(:filter => $pinned_test_method.to_s)
      end
    else
      suites = LeapTest.send "test_suites"
      suites = TestDependencyGraph.new(suites).sorted
    end
    output.sync = true
    results = _run_suites(suites, :test)
    @test_count      = results.inject(0) { |sum, (tc, _)| sum + tc }
    @assertion_count = results.inject(0) { |sum, (_, ac)| sum + ac }
    status
    return exit_code()
  rescue Interrupt
    bail :error, 'Tests halted on interrupt.'
  rescue TestFailure
    bail :failure, 'Tests halted on failure (because of --no-continue).'
  rescue TestError
    bail :error, 'Tests halted on error (because of --no-continue).'
  end

  #
  # override puke to change what prints out.
  #
  def puke(klass, meth, e)
    case e
      when MiniTest::Skip then
        @skips += 1
        report_line("SKIP", klass, meth, e, e.message)
      when LeapTest::Ignore then
        @ignores += 1
        if @verbose
          report_line("IGNORE", klass, meth, e, e.message)
        end
      when LeapTest::Pass then
        @passes += 1
        report_line("PASS", klass, meth)
      when MiniTest::Assertion then
        @failures += 1
        report_line("FAIL", klass, meth, e, e.message)
        if $halt_on_failure
          raise TestFailure.new
        end
      when Timeout::Error then
        @failures += 1
        report_line("TIMEOUT", klass, meth, nil, "Test stopped because timeout exceeded (#{$timeout} seconds).")
        if $halt_on_failure
          raise TestFailure.new
        end
      else
        @errors += 1
        bt = MiniTest::filter_backtrace(e.backtrace).join "\n"
        report_line("ERROR", klass, meth, e, "#{e.class}: #{e.message}\n#{bt}")
        if $halt_on_failure
          raise TestError.new
        end
    end
    return "" # disable the marching ants
  end

  #
  # override default status summary
  #
  def status(io = self.output)
    if $output_format == :human
      format = "%d tests: %d passes, %d skips, %d warnings, %d failures, %d errors"
      output.puts format % [test_count, passes, skips, warnings, failures, errors]
    end
  end

  #
  # return an appropriate exit_code symbol
  #
  def exit_code
    if @errors > 0
      :error
    elsif @failures > 0
      :failure
    elsif @warnings > 0
      # :warning << warnings don't warrant a non-zero exit code.
      :success
    else
      :success
    end
  end

  #
  # returns a string for a PASS, SKIP, or FAIL error
  #
  def report_line(prefix, klass, meth, e=nil, message=nil)
    msg_txt = nil
    if message
      message = message.gsub(/http:\/\/([a-z_]+):([a-zA-Z0-9_]+)@/, "http://\\1:REDACTED@")
      if $output_format == :human
        indent = "\n  "
        msg_txt = indent + message.split("\n").join(indent)
      else
        msg_txt = message.gsub("\n", ' ')
      end
    end

    if $output_format == :human
      if e && msg_txt
        output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)} [#{File.basename(location(e))}]:#{msg_txt}"
      elsif msg_txt
        output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)}:#{msg_txt}"
      else
        output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)}"
      end
                      # I don't understand at all why, but adding a very tiny sleep here will
      sleep(0.0001)   # keep lines from being joined together by the logger. output.flush doesn't.
    elsif $output_format == :checkmk
      code = CHECKMK_CODES[prefix]
      msg_txt ||= "Success" if prefix == "PASS"
      if e && msg_txt
        output.puts "#{code} #{klass.name}/#{machine_readable(meth)} - [#{File.basename(location(e))}]:#{msg_txt}"
      elsif msg_txt
        output.puts "#{code} #{klass.name}/#{machine_readable(meth)} - #{msg_txt}"
      else
        output.puts "#{code} #{klass.name}/#{machine_readable(meth)} - no message"
      end
    end
  end

  #
  # a new function used by TestCase to report warnings.
  #
  def warn(klass, method_name, msg)
    @warnings += 1
    report_line("WARN", klass, method_name, nil, msg)
  end

  private

  CHECKMK_CODES = {"PASS" => 0, "SKIP" => 1, "FAIL" => 2, "ERROR" => 3}

  #
  # Converts snake_case and CamelCase to something more pleasant for humans to read.
  #
  def readable(str)
    str.
    gsub(/_/, ' ').
    sub(/^test (\d* )?/i, '')
  end

  def machine_readable(str)
    str.sub(/^test_(\d+_)?/i, '')
  end

end

##
## Dependency resolution
## Use a topographical sort to manage test dependencies
##

class TestDependencyGraph
  include TSort

  def initialize(test_classes)
    @dependencies = {}  # each key is a test class name, and the values
                        # are arrays of test class names that the key depends on.
    test_classes.each do |test_class|
      @dependencies[test_class.name] = test_class.dependencies
    end
  end

  def tsort_each_node(&block)
    @dependencies.each_key(&block)
  end

  def tsort_each_child(test_class_name, &block)
    if @dependencies[test_class_name]
      @dependencies[test_class_name].each(&block)
    else
      puts "ERROR: bad dependency, no such class `#{test_class_name}`"
      bail :error
    end
  end

  def sorted
    self.tsort.collect {|class_name|
      Kernel.const_get(class_name)
    }
  end
end

##
## COMMAND LINE ACTIONS
##

def die(test, msg)
  if $output_format == :human
    puts "ERROR in test `#{test}`: #{msg}"
  elsif $output_format == :checkmk
    puts "3 #{test} - #{msg}"
  end
  bail :error
end

def print_help
  puts ["USAGE: run_tests [OPTIONS]",
       "  --continue        Don't halt on an error, but continue to the next test.",
       "  --checkmk         Print test results in checkmk format (must come before --test).",
       "  --test TEST       Run only the test with name TEST.",
       "  --list-tests      Prints the names of all available tests and exit.",
       "  --retry COUNT     If the tests don't pass, retry COUNT additional times (default is zero).",
       "  --timeout SECONDS Halt a test if it exceed SECONDS (default is 30).",
       "  --wait SECONDS    Wait for SECONDS between retries (default is 5).",
       "  --debug           Print out full stack trace on errors."].join("\n")
  exit(0)
end

def list_tests
  LeapTest.test_classes.each do |test_class|
    test_class.tests.each do |test|
      puts test_class.name + "/" + test.to_s.sub(/^test_(\d+_)?/, '')
    end
  end
  exit(0)
end

def pin_test_name(name)
  test_class, test_name = name.split('/')
  $pinned_test_class = LeapTest.test_classes.detect{|c| c.name == test_class}
  unless $pinned_test_class
    die name, "there is no test class `#{test_class}`"
  end
  if test_name
    $pinned_test_method = $pinned_test_class.tests.detect{|m| m.to_s =~ /^test_(\d+_)?#{Regexp.escape(test_name)}$/}
    unless $pinned_test_method
      die name, "there is no test `#{test_name}` in class `#{test_class}`"
    end
  end
end

#
# run the tests, multiple times if `--retry` and not all tests were successful.
#
def run_tests
  exit_code = nil
  run_count = $retry ? $retry + 1 : 1
  run_count.times do |i|
    MiniTest::Unit.runner = LeapRunner.new
    exit_code = MiniTest::Unit.new.run
    if !$retry || exit_code == :success
      break
    elsif i != run_count-1
      sleep $wait
    end
  end
  bail exit_code
end

##
## MAIN
##

def main
  # load node data from hiera file
  if File.exists?(HIERA_FILE)
    $node = YAML.load_file(HIERA_FILE)
  else
    $node = {"services" => [], "dummy" => true}
  end

  # load all test classes
  this_file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
  HELPER_PATHS.each do |path|
    Dir[File.expand_path(path, this_file)].each do |helper|
      require helper
    end
  end
  TEST_PATHS.each do |path|
    Dir[File.expand_path(path, this_file)].each do |test_file|
      begin
        require test_file
      rescue SkipTest
      end
    end
  end

  # parse command line options
  $halt_on_failure = true
  $output_format = :human
  $retry = false
  $wait = 5
  $timeout = 30
  loop do
    case ARGV[0]
      when '--continue' then ARGV.shift; $halt_on_failure = false;
      when '--checkmk' then ARGV.shift; $output_format = :checkmk; $halt_on_failure = false
      when '--help' then print_help
      when '--test' then ARGV.shift; pin_test_name(ARGV.shift)
      when '--list-tests' then list_tests
      when '--retry' then ARGV.shift; $retry = ARGV.shift.to_i
      when '--timeout' then ARGV.shift; $timeout = ARGV.shift.to_i;
      when '--wait' then ARGV.shift; $wait = ARGV.shift.to_i
      when '--debug' then ARGV.shift
      when '-d' then ARGV.shift
      else break
    end
  end
  run_tests
end

if ARGV.include?('--debug') || ARGV.include?('-d')
  DEBUG=true
  require 'debugger'
else
  DEBUG=false
end

main()