diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/minitest/benchmark.rb | 330 | ||||
-rw-r--r-- | lib/minitest/mock.rb | 21 | ||||
-rw-r--r-- | lib/minitest/pride.rb | 41 | ||||
-rw-r--r-- | lib/minitest/spec.rb | 13 | ||||
-rw-r--r-- | lib/minitest/unit.rb | 281 |
5 files changed, 585 insertions, 101 deletions
diff --git a/lib/minitest/benchmark.rb b/lib/minitest/benchmark.rb new file mode 100644 index 0000000000..2bf8dd9c00 --- /dev/null +++ b/lib/minitest/benchmark.rb @@ -0,0 +1,330 @@ +############################################################ +# This file is imported from a different project. +# DO NOT make modifications in this repo. +# File a patch instead and assign it to Ryan Davis +############################################################ + +require 'minitest/unit' +require 'minitest/spec' + +class MiniTest::Unit + attr_accessor :runner + + def run_benchmarks + _run_anything :benchmark + end + + def benchmark_suite_header suite + "\n#{suite}\t#{suite.bench_range.join("\t")}" + end + + class TestCase + ## + # Returns a set of ranges stepped exponentially from +min+ to + # +max+ by powers of +base+. Eg: + # + # bench_exp(2, 16, 2) # => [2, 4, 8, 16] + + def self.bench_exp min, max, base = 10 + min = (Math.log10(min) / Math.log10(base)).to_i + max = (Math.log10(max) / Math.log10(base)).to_i + + (min..max).map { |m| base ** m }.to_a + end + + ## + # Returns a set of ranges stepped linearly from +min+ to +max+ by + # +step+. Eg: + # + # bench_linear(20, 40, 10) # => [20, 30, 40] + + def self.bench_linear min, max, step = 10 + (min..max).step(step).to_a + rescue LocalJumpError # 1.8.6 + r = []; (min..max).step(step) { |n| r << n }; r + end + + ## + # Returns the benchmark methods (methods that start with bench_) + # for that class. + + def self.benchmark_methods # :nodoc: + public_instance_methods(true).grep(/^bench_/).map { |m| m.to_s }.sort + end + + ## + # Returns all test suites that have benchmark methods. + + def self.benchmark_suites + TestCase.test_suites.reject { |s| s.benchmark_methods.empty? } + end + + ## + # Specifies the ranges used for benchmarking for that class. + # Defaults to exponential growth from 1 to 10k by powers of 10. + # Override if you need different ranges for your benchmarks. + # + # See also: ::bench_exp and ::bench_linear. + + def self.bench_range + bench_exp 1, 10_000 + end + + ## + # Runs the given +work+, gathering the times of each run. Range + # and times are then passed to a given +validation+ proc. Outputs + # the benchmark name and times in tab-separated format, making it + # easy to paste into a spreadsheet for graphing or further + # analysis. + # + # Ranges are specified by ::bench_range. + # + # Eg: + # + # def bench_algorithm + # validation = proc { |x, y| ... } + # assert_performance validation do |x| + # @obj.algorithm + # end + # end + + def assert_performance validation, &work + range = self.class.bench_range + + io.print "#{__name__}" + + times = [] + + range.each do |x| + GC.start + t0 = Time.now + instance_exec(x, &work) + t = Time.now - t0 + + io.print "\t%9.6f" % t + times << t + end + io.puts + + validation[range, times] + end + + ## + # Runs the given +work+ and asserts that the times gathered fit to + # match a constant rate (eg, linear slope == 0) within a given error + # +threshold+. + # + # Fit is calculated by #fit_constant. + # + # Ranges are specified by ::bench_range. + # + # Eg: + # + # def bench_algorithm + # assert_performance_constant 0.9999 do |x| + # @obj.algorithm + # end + # end + + def assert_performance_constant threshold = 0.99, &work + validation = proc do |range, times| + a, b, rr = fit_linear range, times + assert_in_delta 0, b, 1 - threshold + [a, b, rr] + end + + assert_performance validation, &work + end + + ## + # Runs the given +work+ and asserts that the times gathered fit to + # match a exponential curve within a given error +threshold+. + # + # Fit is calculated by #fit_exponential. + # + # Ranges are specified by ::bench_range. + # + # Eg: + # + # def bench_algorithm + # assert_performance_exponential 0.9999 do |x| + # @obj.algorithm + # end + # end + + def assert_performance_exponential threshold = 0.99, &work + assert_performance validation_for_fit(:exponential, threshold), &work + end + + ## + # Runs the given +work+ and asserts that the times gathered fit to + # match a straight line within a given error +threshold+. + # + # Fit is calculated by #fit_linear. + # + # Ranges are specified by ::bench_range. + # + # Eg: + # + # def bench_algorithm + # assert_performance_linear 0.9999 do |x| + # @obj.algorithm + # end + # end + + def assert_performance_linear threshold = 0.99, &work + assert_performance validation_for_fit(:linear, threshold), &work + end + + ## + # Runs the given +work+ and asserts that the times gathered curve + # fit to match a power curve within a given error +threshold+. + # + # Fit is calculated by #fit_power. + # + # Ranges are specified by ::bench_range. + # + # Eg: + # + # def bench_algorithm + # assert_performance_power 0.9999 do |x| + # @obj.algorithm + # end + # end + + def assert_performance_power threshold = 0.99, &work + assert_performance validation_for_fit(:power, threshold), &work + end + + ## + # Takes an array of x/y pairs and calculates the general R^2 value. + # + # See: http://en.wikipedia.org/wiki/Coefficient_of_determination + + def fit_error xys + y_bar = sigma(xys) { |x, y| y } / xys.size.to_f + ss_tot = sigma(xys) { |x, y| (y - y_bar) ** 2 } + ss_err = sigma(xys) { |x, y| (yield(x) - y) ** 2 } + + 1 - (ss_err / ss_tot) + end + + ## + # To fit a functional form: y = ae^(bx). + # + # Takes x and y values and returns [a, b, r^2]. + # + # See: http://mathworld.wolfram.com/LeastSquaresFittingExponential.html + + def fit_exponential xs, ys + n = xs.size + xys = xs.zip(ys) + sxlny = sigma(xys) { |x,y| x * Math.log(y) } + slny = sigma(xys) { |x,y| Math.log(y) } + sx2 = sigma(xys) { |x,y| x * x } + sx = sigma xs + + c = n * sx2 - sx ** 2 + a = (slny * sx2 - sx * sxlny) / c + b = ( n * sxlny - sx * slny ) / c + + return Math.exp(a), b, fit_error(xys) { |x| Math.exp(a + b * x) } + end + + ## + # Fits the functional form: a + bx. + # + # Takes x and y values and returns [a, b, r^2]. + # + # See: http://mathworld.wolfram.com/LeastSquaresFitting.html + + def fit_linear xs, ys + n = xs.size + xys = xs.zip(ys) + sx = sigma xs + sy = sigma ys + sx2 = sigma(xs) { |x| x ** 2 } + sxy = sigma(xys) { |x,y| x * y } + + c = n * sx2 - sx**2 + a = (sy * sx2 - sx * sxy) / c + b = ( n * sxy - sx * sy ) / c + + return a, b, fit_error(xys) { |x| a + b * x } + end + + ## + # To fit a functional form: y = ax^b. + # + # Takes x and y values and returns [a, b, r^2]. + # + # See: http://mathworld.wolfram.com/LeastSquaresFittingPowerLaw.html + + def fit_power xs, ys + n = xs.size + xys = xs.zip(ys) + slnxlny = sigma(xys) { |x, y| Math.log(x) * Math.log(y) } + slnx = sigma(xs) { |x | Math.log(x) } + slny = sigma(ys) { | y| Math.log(y) } + slnx2 = sigma(xs) { |x | Math.log(x) ** 2 } + + b = (n * slnxlny - slnx * slny) / (n * slnx2 - slnx ** 2); + a = (slny - b * slnx) / n + + return Math.exp(a), b, fit_error(xys) { |x| (Math.exp(a) * (x ** b)) } + end + + ## + # Enumerates over +enum+ mapping +block+ if given, returning the + # sum of the result. Eg: + # + # sigma([1, 2, 3]) # => 1 + 2 + 3 => 7 + # sigma([1, 2, 3]) { |n| n ** 2 } # => 1 + 4 + 9 => 14 + + def sigma enum, &block + enum = enum.map(&block) if block + enum.inject { |sum, n| sum + n } + end + + ## + # Returns a proc that calls the specified fit method and asserts + # that the error is within a tolerable threshold. + + def validation_for_fit msg, threshold + proc do |range, times| + a, b, rr = send "fit_#{msg}", range, times + assert_operator rr, :>=, threshold + [a, b, rr] + end + end + end +end + +class MiniTest::Spec + def self.bench name, &block + define_method "bench_#{name.gsub(/\W+/, '_')}", &block + end + + def self.bench_range &block + meta = (class << self; self; end) + meta.send :define_method, "bench_range", &block + end + + def self.bench_performance_linear name, threshold = 0.9, &work + bench name do + assert_performance_linear threshold, &work + end + end + + def self.bench_performance_constant name, threshold = 0.99, &work + bench name do + assert_performance_constant threshold, &work + end + end + + def self.bench_performance_exponential name, threshold = 0.99, &work + bench name do + assert_performance_exponential threshold, &work + end + end +end diff --git a/lib/minitest/mock.rb b/lib/minitest/mock.rb index 4871e8b183..99f7992aa9 100644 --- a/lib/minitest/mock.rb +++ b/lib/minitest/mock.rb @@ -14,14 +14,7 @@ module MiniTest end def expect(name, retval, args=[]) - n, r = name, retval # for the closure below @expected_calls[name] = { :retval => retval, :args => args } - self.class.__send__ :remove_method, name if respond_to? name - self.class.__send__(:define_method, name) { |*x| - raise ArgumentError unless @expected_calls[n][:args].size == x.size - @actual_calls[n] << { :retval => r, :args => x } - retval - } self end @@ -34,5 +27,19 @@ module MiniTest end true end + + def method_missing(sym, *args) + raise NoMethodError unless @expected_calls.has_key?(sym) + raise ArgumentError unless @expected_calls[sym][:args].size == args.size + retval = @expected_calls[sym][:retval] + @actual_calls[sym] << { :retval => retval, :args => args } + retval + end + + alias :original_respond_to? :respond_to? + def respond_to?(sym) + return true if @expected_calls.has_key?(sym) + return original_respond_to?(sym) + end end end diff --git a/lib/minitest/pride.rb b/lib/minitest/pride.rb new file mode 100644 index 0000000000..3cb4e1564c --- /dev/null +++ b/lib/minitest/pride.rb @@ -0,0 +1,41 @@ +############################################################ +# This file is imported from a different project. +# DO NOT make modifications in this repo. +# File a patch instead and assign it to Ryan Davis +############################################################ + +require "minitest/unit" + +## +# Show your testing pride! + +class PrideIO + attr_reader :io + + # stolen from /System/Library/Perl/5.10.0/Term/ANSIColor.pm + COLORS = (31..36).to_a + CHARS = ["*"] + + def initialize io + @io = io + @colors = COLORS.cycle + @chars = CHARS.cycle + end + + def print o + case o + when "." then + io.print "\e[#{@colors.next}m#{@chars.next}\e[0m" + when "E", "F" then + io.print "\e[41m\e[37m#{o}\e[0m" + else + io.print o + end + end + + def method_missing msg, *args + io.send(msg, *args) + end +end + +MiniTest::Unit.output = PrideIO.new(MiniTest::Unit.output) diff --git a/lib/minitest/spec.rb b/lib/minitest/spec.rb index 2050337015..8e2412e5ea 100644 --- a/lib/minitest/spec.rb +++ b/lib/minitest/spec.rb @@ -66,10 +66,13 @@ module Kernel def describe desc, &block stack = MiniTest::Spec.describe_stack - name = desc.to_s.split(/\W+/).map { |s| s.capitalize }.join + "Spec" - prev = stack.last - name = "#{prev == MiniTest::Spec ? nil : prev}::#{name}" - cls = Object.class_eval "class #{name} < #{prev}; end; #{name}" + name = [stack.last, desc].compact.join("::") + cls = Class.new(stack.last || MiniTest::Spec) + + # :stopdoc: + # omg this sucks + (class << cls; self; end).send(:define_method, :to_s) { name } + # :startdoc: cls.nuke_test_methods! @@ -96,7 +99,7 @@ end class MiniTest::Spec < MiniTest::Unit::TestCase - @@describe_stack = [MiniTest::Spec] + @@describe_stack = [] def self.describe_stack # :nodoc: @@describe_stack end diff --git a/lib/minitest/unit.rb b/lib/minitest/unit.rb index 80c76c45f3..65e84230dd 100644 --- a/lib/minitest/unit.rb +++ b/lib/minitest/unit.rb @@ -8,6 +8,8 @@ require 'optparse' ## # Minimal (mostly drop-in) replacement for test-unit. +# +# :include: README.txt module MiniTest @@ -27,7 +29,8 @@ module MiniTest require 'pathname' pwd = Pathname.new Dir.pwd pn = Pathname.new File.expand_path(__FILE__) - pn = File.join(".", pn.relative_path_from(pwd)) unless pn.relative? + relpath = pn.relative_path_from(pwd) rescue pn + pn = File.join ".", relpath unless pn.relative? pn.to_s else # assume both are expanded __FILE__ @@ -40,13 +43,19 @@ module MiniTest return ["No backtrace"] unless bt new_bt = [] - bt.each do |line| - break if line.rindex(MINI_DIR, 0) - new_bt << line + + unless $DEBUG then + bt.each do |line| + break if line.rindex MINI_DIR, 0 + new_bt << line + end + + new_bt = bt.reject { |line| line.rindex MINI_DIR, 0 } if new_bt.empty? + new_bt = bt.dup if new_bt.empty? + else + new_bt = bt.dup end - new_bt = bt.reject { |line| line.rindex(MINI_DIR, 0) } if new_bt.empty? - new_bt = bt.dup if new_bt.empty? new_bt end @@ -62,7 +71,7 @@ module MiniTest def mu_pp obj s = obj.inspect - s = s.force_encoding(Encoding.default_external) if defined? Encoding + s = s.force_encoding Encoding.default_external if defined? Encoding s end @@ -91,15 +100,14 @@ module MiniTest # Fails unless the block returns a true value. def assert_block msg = nil - msg = message(msg) { "Expected block to return true value" } - assert yield, msg + assert yield, "Expected block to return true value." end ## # Fails unless +obj+ is empty. def assert_empty obj, msg = nil - msg = message(msg) { "Expected #{obj.inspect} to be empty" } + msg = message(msg) { "Expected #{mu_pp(obj)} to be empty" } assert_respond_to obj, :empty? assert obj.empty?, msg end @@ -249,8 +257,8 @@ module MiniTest def assert_respond_to obj, meth, msg = nil msg = message(msg) { - "Expected #{mu_pp(obj)} (#{obj.class}) to respond to ##{meth}" - } + "Expected #{mu_pp(obj)} (#{obj.class}) to respond to ##{meth}" + } assert obj.respond_to?(meth), msg end @@ -512,28 +520,62 @@ module MiniTest end class Unit - VERSION = "1.7.2" # :nodoc: + VERSION = "2.0.0" # :nodoc: attr_accessor :report, :failures, :errors, :skips # :nodoc: attr_accessor :test_count, :assertion_count # :nodoc: attr_accessor :start_time # :nodoc: + attr_accessor :help # :nodoc: + attr_accessor :verbose # :nodoc: + attr_writer :options # :nodoc: + + def options + @options ||= {} + end @@installed_at_exit ||= false @@out = $stdout ## + # A simple hook allowing you to run a block of code after the + # tests are done. Eg: + # + # MiniTest::Unit.after_tests { p $debugging_info } + + def self.after_tests + at_exit { at_exit { yield } } + end + + ## # Registers MiniTest::Unit to run tests at process exit def self.autorun at_exit { next if $! # don't run if there was an exception - exit_code = MiniTest::Unit.new.run(ARGV) + exit_code = MiniTest::Unit.new.run ARGV exit false if exit_code && exit_code != 0 } unless @@installed_at_exit @@installed_at_exit = true end ## + # Returns the stream to use for output. + + def self.output + @@out + end + + ## + # Returns the stream to use for output. + # + # DEPRECATED: use ::output instead. + + def self.out + warn "::out deprecated, use ::output instead." if $VERBOSE + output + end + + ## # Sets MiniTest::Unit to write output to +stream+. $stdout is the default # output @@ -541,6 +583,93 @@ module MiniTest @@out = stream end + ## + # Return all plugins' run methods (methods that start with "run_"). + + def self.plugins + @@plugins ||= (["run_tests"] + + public_instance_methods(false). + grep(/^run_/).map { |s| s.to_s }).uniq + end + + def output + self.class.output + end + + def puts *a # :nodoc: + output.puts(*a) + end + + def print *a # :nodoc: + output.print(*a) + end + + def _run_anything type + suites = TestCase.send "#{type}_suites" + return if suites.empty? + + start = Time.now + + puts + puts "# Running #{type}s:" + puts + + @test_count, @assertion_count = 0, 0 + sync = output.respond_to? :"sync=" # stupid emacs + old_sync, output.sync = output.sync, true if sync + + results = _run_suites suites, type + + @test_count = results.inject(0) { |sum, (tc, ac)| sum + tc } + @assertion_count = results.inject(0) { |sum, (tc, ac)| sum + ac } + + output.sync = old_sync if sync + + t = Time.now - start + + puts + puts + puts "Finished #{type}s in %.6fs, %.4f tests/s, %.4f assertions/s." % + [t, test_count / t, assertion_count / t] + + report.each_with_index do |msg, i| + puts "\n%3d) %s" % [i + 1, msg] + end + + puts + + status + end + + def _run_suites suites, type + suites.map { |suite| _run_suite suite, type } + end + + def _run_suite suite, type + header = "#{type}_suite_header" + puts send(header, suite) if respond_to? header + + filter = options[:filter] || '/./' + filter = Regexp.new $1 if filter =~ /\/(.*)\// + + assertions = suite.send("#{type}_methods").grep(filter).map { |method| + inst = suite.new method + inst._assertions = 0 + + start_time = Time.now + result = inst.run self + time = Time.now - start_time + + print "#{suite}##{method} = %.2f s = " % time if @verbose + print result + puts if @verbose + + inst._assertions + } + + return assertions.size, assertions.inject(0) { |sum, n| sum + n } + end + def location e # :nodoc: last_before_assertion = "" e.backtrace.reverse_each do |s| @@ -564,7 +693,7 @@ module MiniTest "Failure:\n#{meth}(#{klass}) [#{location e}]:\n#{e.message}\n" else @errors += 1 - bt = MiniTest::filter_backtrace(e.backtrace).join("\n ") + bt = MiniTest::filter_backtrace(e.backtrace).join "\n " "Error:\n#{meth}(#{klass}):\n#{e.class}: #{e.message}\n #{bt}\n" end @report << e @@ -579,6 +708,7 @@ module MiniTest def process_args args = [] options = {} + orig_args = args.dup OptionParser.new do |opts| opts.banner = 'minitest options:' @@ -601,9 +731,21 @@ module MiniTest options[:filter] = a end - opts.parse args + opts.parse! args + orig_args -= args + end + + unless options[:seed] then + srand + options[:seed] = srand % 0xFFFF + orig_args << "--seed" << options[:seed].to_s end + srand options[:seed] + + self.verbose = options[:verbose] + @help = orig_args.map { |s| s =~ /[\s|&<>$()]/ ? s.inspect : s }.join " " + options end @@ -611,89 +753,40 @@ module MiniTest # Top level driver, controls all output and filtering. def run args = [] - options = process_args args - - @verbose = options[:verbose] + self.options = process_args args - filter = options[:filter] || '/./' - filter = Regexp.new $1 if filter and filter =~ /\/(.*)\// + puts "Run options: #{help}" - seed = options[:seed] - unless seed then - srand - seed = srand % 0xFFFF + self.class.plugins.each do |plugin| + send plugin + break unless report.empty? end - srand seed - - help = ["--seed", seed] - help.push "--verbose" if @verbose - help.push("--name", options[:filter].inspect) if options[:filter] - - @@out.puts "Test run options: #{help.join(" ")}" - @@out.puts - @@out.puts "Loaded suite #{$0.sub(/\.rb$/, '')}\nStarted" - - start = Time.now - run_test_suites filter - - @@out.puts - @@out.puts "Finished in #{'%.6f' % (Time.now - start)} seconds." - - @report.each_with_index do |msg, i| - @@out.puts "\n%3d) %s" % [i + 1, msg] - end - - @@out.puts - - status - - @@out.puts - - @@out.puts "Test run options: #{help.join(" ")}" - return failures + errors if @test_count > 0 # or return nil... rescue Interrupt abort 'Interrupted' end ## - # Writes status to +io+ + # Runs test suites matching +filter+. - def status io = @@out - format = "%d tests, %d assertions, %d failures, %d errors, %d skips" - io.puts format % [test_count, assertion_count, failures, errors, skips] + def run_tests + _run_anything :test end ## - # Runs test suites matching +filter+ + # Writes status to +io+ - def run_test_suites filter = /./ - @test_count, @assertion_count = 0, 0 - old_sync, @@out.sync = @@out.sync, true if @@out.respond_to? :sync= - TestCase.test_suites.each do |suite| - suite.test_methods.grep(filter).each do |test| - inst = suite.new test - inst._assertions = 0 - @@out.print "#{suite}##{test}: " if @verbose - - @start_time = Time.now - result = inst.run(self) - - @@out.print "%.2f s: " % (Time.now - @start_time) if @verbose - @@out.print result - @@out.puts if @verbose - @test_count += 1 - @assertion_count += inst._assertions - end - end - @@out.sync = old_sync if @@out.respond_to? :sync= - [@test_count, @assertion_count] + def status io = self.output + format = "%d tests, %d assertions, %d failures, %d errors, %d skips" + io.puts format % [test_count, assertion_count, failures, errors, skips] end ## - # Subclass TestCase to create your own tests. Typically you'll want a + # Subclass TestCase to create your own tests. Typically you'll want a # TestCase subclass per implementation class. + # + # See MiniTest::Assertions class TestCase attr_reader :__name__ # :nodoc: @@ -707,30 +800,31 @@ module MiniTest # Runs the tests reporting the status to +runner+ def run runner - trap 'INFO' do - warn '%s#%s %.2fs' % [self.class, self.__name__, - (Time.now - runner.start_time)] + trap "INFO" do + time = Time.now - runner.start_time + warn "%s#%s %.2fs" % [self.class, self.__name__, time] runner.status $stderr end if SUPPORTS_INFO_SIGNAL - result = '.' + result = "" begin @passed = nil self.setup self.__send__ self.__name__ + result = "." unless io? @passed = true rescue *PASSTHROUGH_EXCEPTIONS raise rescue Exception => e @passed = false - result = runner.puke(self.class, self.__name__, e) + result = runner.puke self.class, self.__name__, e ensure begin self.teardown rescue *PASSTHROUGH_EXCEPTIONS raise rescue Exception => e - result = runner.puke(self.class, self.__name__, e) + result = runner.puke self.class, self.__name__, e end trap 'INFO', 'DEFAULT' if SUPPORTS_INFO_SIGNAL end @@ -739,9 +833,19 @@ module MiniTest def initialize name # :nodoc: @__name__ = name + @__io__ = nil @passed = nil end + def io + @__io__ = true + MiniTest::Unit.output + end + + def io? + @__io__ + end + def self.reset # :nodoc: @@test_suites = {} end @@ -762,7 +866,7 @@ module MiniTest end def self.test_suites # :nodoc: - @@test_suites.keys.sort_by { |ts| ts.name } + @@test_suites.keys.sort_by { |ts| ts.name.to_s } end def self.test_methods # :nodoc: @@ -771,7 +875,7 @@ module MiniTest case self.test_order when :random then max = methods.size - methods.sort.sort_by { rand(max) } + methods.sort.sort_by { rand max } when :alpha, :sorted then methods.sort else @@ -813,4 +917,3 @@ if $DEBUG then end end end - |