summaryrefslogtreecommitdiff
path: root/bin/run_tests
blob: 91c742c5a5aa2323d6187aa615dd0cf9c845ba90 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
#!/usr/bin/ruby

#
# this script will run the unit tests in ../tests/*.rb.
#
# Tests for the platform differ from traditional ruby unit tests in a few ways:
#
# (1) at the end of every test function, you should call 'pass()'
# (2) you can specify test dependencies by calling depends_on("TestFirst") in the test class definition.
# (3) test functions are always run in alphabetical order.
# (4) any halt or error will stop the testing unless --continue is specified.
#

require 'minitest/unit'
require 'yaml'
require 'tsort'

##
## EXCEPTIONS
##

# this class is raised if a test file wants to be skipped entirely.
class SkipTest < Exception
end

# raised if --no-continue and there is an error
class TestError < Exception
end

# raised if --no-continue and there is a failure
class TestFailure < Exception
end

##
## CUSTOM UNIT TEST CLASS
##

#
# Our custom unit test class. All tests should be subclasses of this.
#
class LeapTest < MiniTest::Unit::TestCase
  class Pass < MiniTest::Assertion
  end

  #
  # Test class dependencies
  #
  def self.depends_on(*class_names)
    @dependencies ||= []
    @dependencies += class_names
  end
  def self.dependencies
    @dependencies || []
  end

  #
  # The default pass just does an `assert true`. In our case, we want to make the passes more explicit.
  #
  def pass
    raise LeapTest::Pass
  end

  #
  # Always runs test methods within a test class in alphanumeric order
  #
  def self.test_order
    :alpha
  end
end

#
# Custom test runner in order to modify the output.
#
class LeapRunner < MiniTest::Unit

  attr_accessor :passes

  def initialize
    @passes = 0
    super
  end

  #
  # call stack:
  #   MiniTest::Unit.new.run
  #     MiniTest::Unit.runner
  #       LeapTest._run
  #
  def _run args = []
    suites = LeapTest.send "test_suites"
    output.sync = true
    suites = TestDependencyGraph.new(suites).sorted
    results = _run_suites(suites, :test)
    @test_count      = results.inject(0) { |sum, (tc, _)| sum + tc }
    @assertion_count = results.inject(0) { |sum, (_, ac)| sum + ac }
    report.each {|msg| output.puts msg}
    status
  rescue Interrupt
    report.each {|msg| output.puts msg}
    abort 'Tests halted on interrupt.'
  rescue TestFailure
    report.each {|msg| output.puts msg}
    abort 'Tests halted on failure (because of --no-continue).'
  rescue TestError
    report.each {|msg| output.puts msg}
    abort 'Tests halted on error (because of --no-continue).'
  end

  #
  # override puke to change what prints out.
  #
  def puke(klass, meth, e)
    case e
      when MiniTest::Skip then
        @skips += 1
        #if @verbose
          @report << report_line("SKIP", klass, meth, e, e.message)
        #end
      when LeapTest::Pass then
        @passes += 1
        @report << report_line("PASS", klass, meth)
      when MiniTest::Assertion then
        @failures += 1
        @report << report_line("FAIL", klass, meth, e, e.message)
        if $halt_on_failure
          raise TestFailure.new
        end
      else
        @errors += 1
        bt = MiniTest::filter_backtrace(e.backtrace).join "\n"
        @report << report_line("ERROR", klass, meth, e, "#{e.class}: #{e.message}\n#{bt}")
        if $halt_on_failure
          raise TestError.new
        end
    end
    return "" # disable the marching ants
  end

  #
  # override default status slightly
  #
  def status(io = self.output)
    format = "%d tests, %d assertions, %d passes, %d failures, %d errors, %d skips"
    io.puts format % [test_count, assertion_count, passes, failures, errors, skips]
  end

  private

  #
  # returns a string for a PASS, SKIP, or FAIL error
  #
  def report_line(prefix, klass, meth, e=nil, message=nil)
    if e && message
      #indent = "\n" + (" " * (prefix.length+4))
      indent = "\n  "
      msg_txt = indent + message.split("\n").join(indent)
      "#{prefix}: #{readable(klass.name)} > #{readable(meth)} [#{File.basename(location(e))}]:#{msg_txt}\n"
    else
      "#{prefix}: #{readable(klass.name)} > #{readable(meth)}\n"
    end
  end

  #
  # Converts snake_case and CamelCase to something more pleasant for humans to read.
  #
  def readable(str)
    str.gsub(/([A-Z]+)([A-Z][a-z])/, '\1 \2').
    gsub(/([a-z])([A-Z])/, '\1 \2').
    gsub(/_/, ' ').
    sub(/^test /i, '').
    downcase
  end
end

##
## Dependency resolution
## Use a topographical sort to manage test dependencies
##

class TestDependencyGraph
  include TSort

  def initialize(test_classes)
    @dependencies = {}  # each key is a test class name, and the values
                        # are arrays of test class names that the key depends on.
    test_classes.each do |test_class|
      @dependencies[test_class.name] = test_class.dependencies
    end
  end

  def tsort_each_node(&block)
    @dependencies.each_key(&block)
  end

  def tsort_each_child(test_class_name, &block)
    @dependencies[test_class_name].each(&block)
  end

  def sorted
    self.tsort.collect {|class_name|
      Kernel.const_get(class_name)
    }
  end
end

##
## RUN THE TESTS
##

# load node data from hiera file
if File.exists?('/etc/leap/hiera.yaml')
  $node = YAML.load_file('/etc/leap/hiera.yaml')
else
  $node = {"services" => ['webapp'], "dummy" => true}
end

# load all test classes
Dir[File.expand_path('../../tests/*.rb', __FILE__)].each do |test_file|
  begin
    require test_file
  rescue SkipTest
  end
end

# parse command line options
$halt_on_failure = true
loop do
  case ARGV[0]
    when '--continue' then ARGV.shift; $halt_on_failure = false
    else break
  end
end

# run some tests already
MiniTest::Unit.runner = LeapRunner.new
MiniTest::Unit.new.run