From 9c66bad9f3d522d50d4a45ef8a3a92abbf93229f Mon Sep 17 00:00:00 2001 From: drbrain Date: Thu, 15 Nov 2012 21:59:37 +0000 Subject: * lib/rake*: Updated to rake 0.9.3 * test/rake*: ditto * bin/rake: ditto * NEWS: ditto git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@37664 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- ChangeLog | 9 + NEWS | 8 + bin/rake | 5 + lib/rake.rb | 1 + lib/rake/application.rb | 384 ++++++++++++++++++----------- lib/rake/backtrace.rb | 18 ++ lib/rake/clean.rb | 2 +- lib/rake/cloneable.rb | 23 +- lib/rake/contrib/ftptools.rb | 3 +- lib/rake/contrib/sys.rb | 12 +- lib/rake/dsl_definition.rb | 20 +- lib/rake/ext/module.rb | 2 +- lib/rake/ext/string.rb | 3 +- lib/rake/ext/time.rb | 3 +- lib/rake/file_list.rb | 4 +- lib/rake/file_utils_ext.rb | 7 +- lib/rake/multi_task.rb | 7 +- lib/rake/phony.rb | 13 + lib/rake/private_reader.rb | 20 ++ lib/rake/promise.rb | 99 ++++++++ lib/rake/rake_module.rb | 15 ++ lib/rake/rdoctask.rb | 2 +- lib/rake/runtest.rb | 2 +- lib/rake/task.rb | 36 ++- lib/rake/task_arguments.rb | 2 +- lib/rake/task_manager.rb | 2 +- lib/rake/testtask.rb | 6 +- lib/rake/thread_history_display.rb | 48 ++++ lib/rake/thread_pool.rb | 155 ++++++++++++ lib/rake/version.rb | 10 +- test/rake/helper.rb | 30 +++ test/rake/test_private_reader.rb | 42 ++++ test/rake/test_rake_application.rb | 13 +- test/rake/test_rake_application_options.rb | 118 ++++++++- test/rake/test_rake_backtrace.rb | 81 ++++++ test/rake/test_rake_directory_task.rb | 43 ++-- test/rake/test_rake_file_task.rb | 22 +- test/rake/test_rake_functional.rb | 26 +- test/rake/test_rake_multi_task.rb | 8 + test/rake/test_rake_rake_test_loader.rb | 2 +- test/rake/test_rake_reduce_compat.rb | 65 +++++ test/rake/test_rake_task.rb | 51 +++- test/rake/test_rake_thread_pool.rb | 123 +++++++++ test/rake/test_rake_top_level_functions.rb | 2 +- test/rake/test_thread_history_display.rb | 91 +++++++ 45 files changed, 1408 insertions(+), 230 deletions(-) create mode 100644 lib/rake/backtrace.rb create mode 100644 lib/rake/phony.rb create mode 100644 lib/rake/private_reader.rb create mode 100644 lib/rake/promise.rb create mode 100644 lib/rake/thread_history_display.rb create mode 100644 lib/rake/thread_pool.rb create mode 100644 test/rake/test_private_reader.rb create mode 100644 test/rake/test_rake_backtrace.rb create mode 100644 test/rake/test_rake_reduce_compat.rb create mode 100644 test/rake/test_rake_thread_pool.rb create mode 100644 test/rake/test_thread_history_display.rb diff --git a/ChangeLog b/ChangeLog index ee8bd21692..5d3f998ed3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,12 @@ +Fri Nov 16 06:58:52 2012 Eric Hodel + + * lib/rake*: Updated to rake 0.9.3. See + http://rake.rubyforge.org/doc/release_notes/rake-0_9_3_rdoc.html for + a list of changes in 0.9.3. + * test/rake*: ditto + * bin/rake: ditto + * NEWS: ditto + Thu Nov 15 22:39:32 2012 Yusuke Endoh * range.c (range_bsearch): fix some bugs: a documentation bug, a wrong diff --git a/NEWS b/NEWS index fe1a881120..9f27baed92 100644 --- a/NEWS +++ b/NEWS @@ -181,6 +181,14 @@ with all sufficient information, see the ChangeLog file. * extended method: * Pathname#find returns an enumerator if no block is given. +* rake + * rake has been updated to version 0.9.3. + + This version is backwards-compatible with previous rake versions and + contains many bug fixes. See + http://rake.rubyforge.org/doc/release_notes/rake-0_9_3_rdoc.html for a list + of changes in rake 0.9.3 + * resolv * new methods: * Resolv::DNS#timeouts= diff --git a/bin/rake b/bin/rake index 0de43c97ec..6aad2f0df3 100755 --- a/bin/rake +++ b/bin/rake @@ -24,9 +24,14 @@ begin require 'rubygems' + gem 'rake' rescue LoadError end +module Rake + REDUCE_COMPAT = true if ARGV.include?("--reduce-compat") +end + require 'rake' Rake.application.run diff --git a/lib/rake.rb b/lib/rake.rb index fc1a6a5165..8180a5e996 100644 --- a/lib/rake.rb +++ b/lib/rake.rb @@ -58,6 +58,7 @@ require 'rake/early_time' require 'rake/name_space' require 'rake/task_manager' require 'rake/application' +require 'rake/backtrace' $trace = false diff --git a/lib/rake/application.rb b/lib/rake/application.rb index 2079fb9be6..50930938bf 100644 --- a/lib/rake/application.rb +++ b/lib/rake/application.rb @@ -2,10 +2,14 @@ require 'shellwords' require 'optparse' require 'rake/task_manager' +require 'rake/thread_pool' +require 'rake/thread_history_display' require 'rake/win32' module Rake + CommandLineOptionError = Class.new(StandardError) + ###################################################################### # Rake main application object. When invoking +rake+ from the # command line, a Rake::Application object is created and run. @@ -54,7 +58,7 @@ module Rake # # * Initialize the command line options (+init+). # * Define the tasks (+load_rakefile+). - # * Run the top level tasks (+run_tasks+). + # * Run the top level tasks (+top_level+). # # If you wish to build a custom rake command, you should call # +init+ on your application. Then define any tasks. Finally, @@ -85,7 +89,7 @@ module Rake # Run the top level tasks of a Rake application. def top_level - standard_exception_handling do + run_with_threads do if options.show_tasks display_tasks_and_comments elsif options.show_prereqs @@ -96,6 +100,21 @@ module Rake end end + # Run the given block with the thread startup and shutdown. + def run_with_threads + thread_pool.gather_history if options.job_stats == :history + + yield + + thread_pool.join + if options.job_stats + stats = thread_pool.statistics + puts "Maximum active threads: #{stats[:max_active_threads]}" + puts "Total threads in play: #{stats[:total_threads_in_play]}" + end + ThreadHistoryDisplay.new(thread_pool.history).show if options.job_stats == :history + end + # Add a loader to handle imported files ending in the extension # +ext+. def add_loader(ext, loader) @@ -108,6 +127,11 @@ module Rake @options ||= OpenStruct.new end + # Return the thread pool used for multithreaded processing. + def thread_pool # :nodoc: + @thread_pool ||= ThreadPool.new(options.thread_pool_size||FIXNUM_MAX) + end + # private ---------------------------------------------------------------- def invoke_task(task_string) @@ -146,15 +170,15 @@ module Rake # Display the error message that caused the exception. def display_error_message(ex) - $stderr.puts "#{name} aborted!" - $stderr.puts ex.message - if options.trace - $stderr.puts ex.backtrace.join("\n") + trace "#{name} aborted!" + trace ex.message + if options.backtrace + trace ex.backtrace.join("\n") else - $stderr.puts rakefile_location(ex.backtrace) + trace Backtrace.collapse(ex.backtrace) end - $stderr.puts "Tasks: #{ex.chain}" if has_chain?(ex) - $stderr.puts "(See full trace by running task with --trace)" unless options.trace + trace "Tasks: #{ex.chain}" if has_chain?(ex) + trace "(See full trace by running task with --trace)" unless options.backtrace end # Warn about deprecated usage. @@ -180,7 +204,7 @@ module Rake def have_rakefile @rakefiles.each do |fn| if File.exist?(fn) - others = Dir.glob(fn, File::FNM_CASEFOLD) + others = Rake.glob(fn, File::FNM_CASEFOLD) return others.size == 1 ? others.first : fn elsif fn == '' return fn @@ -208,7 +232,7 @@ module Rake # Display the tasks and comments. def display_tasks_and_comments displayable_tasks = tasks.select { |t| - t.comment && t.name =~ options.show_task_pattern + (options.show_all_tasks || t.comment) && t.name =~ options.show_task_pattern } case options.show_tasks when :tasks @@ -222,7 +246,8 @@ module Rake when :describe displayable_tasks.each do |t| puts "#{name} #{t.name_with_args}" - t.full_comment.split("\n").each do |line| + comment = t.full_comment || "" + comment.split("\n").each do |line| puts " #{line}" end puts @@ -271,7 +296,9 @@ module Rake end def truncate(string, width) - if string.length <= width + if string.nil? + "" + elsif string.length <= width string else ( string[0, width-3] || "" ) + "..." @@ -286,141 +313,214 @@ module Rake end end + def trace(*str) + options.trace_output ||= $stderr + options.trace_output.puts(*str) + end + + def sort_options(options) + options.sort_by { |opt| + opt.select { |o| o =~ /^-/ }.map { |o| o.downcase }.sort.reverse + } + end + private :sort_options + # A list of all the standard options used in rake, suitable for # passing to OptionParser. def standard_rake_options - [ - ['--classic-namespace', '-C', "Put Task and FileTask in the top level namespace", - lambda { |value| - require 'rake/classic_namespace' - options.classic_namespace = true - } - ], - ['--describe', '-D [PATTERN]', "Describe the tasks (matching optional PATTERN), then exit.", - lambda { |value| - options.show_tasks = :describe - options.show_task_pattern = Regexp.new(value || '') - TaskManager.record_task_metadata = true - } - ], - ['--dry-run', '-n', "Do a dry run without executing actions.", - lambda { |value| - Rake.verbose(true) - Rake.nowrite(true) - options.dryrun = true - options.trace = true - } - ], - ['--execute', '-e CODE', "Execute some Ruby code and exit.", - lambda { |value| - eval(value) - exit - } - ], - ['--execute-print', '-p CODE', "Execute some Ruby code, print the result, then exit.", - lambda { |value| - puts eval(value) - exit - } - ], - ['--execute-continue', '-E CODE', - "Execute some Ruby code, then continue with normal task processing.", - lambda { |value| eval(value) } - ], - ['--libdir', '-I LIBDIR', "Include LIBDIR in the search path for required modules.", - lambda { |value| $:.push(value) } - ], - ['--no-search', '--nosearch', '-N', "Do not search parent directories for the Rakefile.", - lambda { |value| options.nosearch = true } - ], - ['--prereqs', '-P', "Display the tasks and dependencies, then exit.", - lambda { |value| options.show_prereqs = true } - ], - ['--quiet', '-q', "Do not log messages to standard output.", - lambda { |value| Rake.verbose(false) } - ], - ['--rakefile', '-f [FILE]', "Use FILE as the rakefile.", - lambda { |value| - value ||= '' - @rakefiles.clear - @rakefiles << value - } - ], - ['--rakelibdir', '--rakelib', '-R RAKELIBDIR', - "Auto-import any .rake files in RAKELIBDIR. (default is 'rakelib')", - # HACK Use File::PATH_SEPARATOR - lambda { |value| options.rakelib = value.split(':') } - ], - ['--require', '-r MODULE', "Require MODULE before executing rakefile.", - lambda { |value| - begin - require value - rescue LoadError => ex + sort_options( + [ + ['--all', '-A', "Show all tasks, even uncommented ones", + lambda { |value| + options.show_all_tasks = value + } + ], + ['--backtrace [OUT]', "Enable full backtrace. OUT can be stderr (default) or stdout.", + lambda { |value| + options.backtrace = true + select_trace_output(options, 'backtrace', value) + } + ], + ['--classic-namespace', '-C', "Put Task and FileTask in the top level namespace", + lambda { |value| + require 'rake/classic_namespace' + options.classic_namespace = true + } + ], + ['--comments', "Show commented tasks only", + lambda { |value| + options.show_all_tasks = !value + } + ], + ['--describe', '-D [PATTERN]', "Describe the tasks (matching optional PATTERN), then exit.", + lambda { |value| + select_tasks_to_show(options, :describe, value) + } + ], + ['--dry-run', '-n', "Do a dry run without executing actions.", + lambda { |value| + Rake.verbose(true) + Rake.nowrite(true) + options.dryrun = true + options.trace = true + } + ], + ['--execute', '-e CODE', "Execute some Ruby code and exit.", + lambda { |value| + eval(value) + exit + } + ], + ['--execute-print', '-p CODE', "Execute some Ruby code, print the result, then exit.", + lambda { |value| + puts eval(value) + exit + } + ], + ['--execute-continue', '-E CODE', + "Execute some Ruby code, then continue with normal task processing.", + lambda { |value| eval(value) } + ], + ['--jobs', '-j [NUMBER]', + "Specifies the maximum number of tasks to execute in parallel. (default:2)", + lambda { |value| options.thread_pool_size = [(value || 2).to_i,2].max } + ], + ['--job-stats [LEVEL]', + "Display job statistics. LEVEL=history displays a complete job list", + lambda { |value| + if value =~ /^history/i + options.job_stats = :history + else + options.job_stats = true + end + } + ], + ['--libdir', '-I LIBDIR', "Include LIBDIR in the search path for required modules.", + lambda { |value| $:.push(value) } + ], + ['--multitask', '-m', "Treat all tasks as multitasks.", + lambda { |value| options.always_multitask = true } + ], + ['--no-search', '--nosearch', '-N', "Do not search parent directories for the Rakefile.", + lambda { |value| options.nosearch = true } + ], + ['--prereqs', '-P', "Display the tasks and dependencies, then exit.", + lambda { |value| options.show_prereqs = true } + ], + ['--quiet', '-q', "Do not log messages to standard output.", + lambda { |value| Rake.verbose(false) } + ], + ['--rakefile', '-f [FILE]', "Use FILE as the rakefile.", + lambda { |value| + value ||= '' + @rakefiles.clear + @rakefiles << value + } + ], + ['--rakelibdir', '--rakelib', '-R RAKELIBDIR', + "Auto-import any .rake files in RAKELIBDIR. (default is 'rakelib')", + lambda { |value| options.rakelib = value.split(File::PATH_SEPARATOR) } + ], + ['--reduce-compat', "Remove DSL in Object; remove Module#const_missing which defines ::Task etc.", + # Load-time option. + # Handled in bin/rake where Rake::REDUCE_COMPAT is defined (or not). + lambda { |_| } + ], + ['--require', '-r MODULE', "Require MODULE before executing rakefile.", + lambda { |value| begin - rake_require value - rescue LoadError - raise ex + require value + rescue LoadError => ex + begin + rake_require value + rescue LoadError + raise ex + end end - end - } - ], - ['--rules', "Trace the rules resolution.", - lambda { |value| options.trace_rules = true } - ], - ['--silent', '-s', "Like --quiet, but also suppresses the 'in directory' announcement.", - lambda { |value| - Rake.verbose(false) - options.silent = true - } - ], - ['--system', '-g', - "Using system wide (global) rakefiles (usually '~/.rake/*.rake').", - lambda { |value| options.load_system = true } - ], - ['--no-system', '--nosystem', '-G', - "Use standard project Rakefile search paths, ignore system wide rakefiles.", - lambda { |value| options.ignore_system = true } - ], - ['--tasks', '-T [PATTERN]', "Display the tasks (matching optional PATTERN) with descriptions, then exit.", - lambda { |value| - options.show_tasks = :tasks - options.show_task_pattern = Regexp.new(value || '') - Rake::TaskManager.record_task_metadata = true - } - ], - ['--trace', '-t', "Turn on invoke/execute tracing, enable full backtrace.", - lambda { |value| - options.trace = true - Rake.verbose(true) - } - ], - ['--verbose', '-v', "Log message to standard output.", - lambda { |value| Rake.verbose(true) } - ], - ['--version', '-V', "Display the program version.", - lambda { |value| - puts "rake, version #{RAKEVERSION}" - exit - } - ], - ['--where', '-W [PATTERN]', "Describe the tasks (matching optional PATTERN), then exit.", - lambda { |value| - options.show_tasks = :lines - options.show_task_pattern = Regexp.new(value || '') - Rake::TaskManager.record_task_metadata = true - } - ], - ['--no-deprecation-warnings', '-X', "Disable the deprecation warnings.", - lambda { |value| - options.ignore_deprecate = true - } - ], - ] + } + ], + ['--rules', "Trace the rules resolution.", + lambda { |value| options.trace_rules = true } + ], + ['--silent', '-s', "Like --quiet, but also suppresses the 'in directory' announcement.", + lambda { |value| + Rake.verbose(false) + options.silent = true + } + ], + ['--suppress-backtrace PATTERN', "Suppress backtrace lines matching regexp PATTERN. Ignored if --trace is on.", + lambda { |value| + options.suppress_backtrace_pattern = Regexp.new(value) + } + ], + ['--system', '-g', + "Using system wide (global) rakefiles (usually '~/.rake/*.rake').", + lambda { |value| options.load_system = true } + ], + ['--no-system', '--nosystem', '-G', + "Use standard project Rakefile search paths, ignore system wide rakefiles.", + lambda { |value| options.ignore_system = true } + ], + ['--tasks', '-T [PATTERN]', "Display the tasks (matching optional PATTERN) with descriptions, then exit.", + lambda { |value| + select_tasks_to_show(options, :tasks, value) + } + ], + ['--trace', '-t [OUT]', "Turn on invoke/execute tracing, enable full backtrace. OUT can be stderr (default) or stdout.", + lambda { |value| + options.trace = true + options.backtrace = true + select_trace_output(options, 'trace', value) + Rake.verbose(true) + } + ], + ['--verbose', '-v', "Log message to standard output.", + lambda { |value| Rake.verbose(true) } + ], + ['--version', '-V', "Display the program version.", + lambda { |value| + puts "rake, version #{RAKEVERSION}" + exit + } + ], + ['--where', '-W [PATTERN]', "Describe the tasks (matching optional PATTERN), then exit.", + lambda { |value| + select_tasks_to_show(options, :lines, value) + options.show_all_tasks = true + } + ], + ['--no-deprecation-warnings', '-X', "Disable the deprecation warnings.", + lambda { |value| + options.ignore_deprecate = true + } + ], + ]) + end + + def select_tasks_to_show(options, show_tasks, value) + options.show_tasks = show_tasks + options.show_task_pattern = Regexp.new(value || '') + Rake::TaskManager.record_task_metadata = true + end + private :select_tasks_to_show + + def select_trace_output(options, trace_option, value) + value = value.strip unless value.nil? + case value + when 'stdout' + options.trace_output = $stdout + when 'stderr', nil + options.trace_output = $stderr + else + fail CommandLineOptionError, "Unrecognized --#{trace_option} option '#{value}'" + end end + private :select_trace_output # Read and handle the command line options. def handle_options options.rakelib = ['rakelib'] + options.trace_output = $stderr OptionParser.new do |opts| opts.banner = "rake [-f rakefile] {options} targets..." @@ -509,7 +609,7 @@ module Rake end def glob(path, &block) - Dir[path.gsub("\\", '/')].each(&block) + Rake.glob(path.gsub("\\", '/')).each(&block) end private :glob @@ -583,7 +683,7 @@ module Rake @const_warning = true end - def rakefile_location backtrace = caller + def rakefile_location(backtrace=caller) backtrace.map { |t| t[/([^:]+):/,1] } re = /^#{@rakefile}$/ @@ -591,5 +691,9 @@ module Rake backtrace.find { |str| str =~ re } || '' end + + private + FIXNUM_MAX = (2**(0.size * 8 - 2) - 1) # :nodoc: + end end diff --git a/lib/rake/backtrace.rb b/lib/rake/backtrace.rb new file mode 100644 index 0000000000..038ca57906 --- /dev/null +++ b/lib/rake/backtrace.rb @@ -0,0 +1,18 @@ +module Rake + module Backtrace + SUPPRESSED_PATHS = + RbConfig::CONFIG.values_at(*RbConfig::CONFIG. + keys.grep(/(prefix|libdir)/)) + [ + File.join(File.dirname(__FILE__), ".."), + ].map { |f| Regexp.quote(File.expand_path(f)) } + SUPPRESSED_PATHS.reject! { |s| s.nil? || s =~ /^ *$/ } + + SUPPRESS_PATTERN = %r!(\A#{SUPPRESSED_PATHS.join('|')}|bin/rake:\d+)!i + + def self.collapse(backtrace) + pattern = Rake.application.options.suppress_backtrace_pattern || + SUPPRESS_PATTERN + backtrace.reject { |elem| elem =~ pattern } + end + end +end diff --git a/lib/rake/clean.rb b/lib/rake/clean.rb index 5c9cbcdb24..32846d4a6d 100644 --- a/lib/rake/clean.rb +++ b/lib/rake/clean.rb @@ -16,7 +16,7 @@ require 'rake' # :stopdoc: CLEAN = Rake::FileList["**/*~", "**/*.bak", "**/core"] CLEAN.clear_exclude.exclude { |fn| - fn.pathmap("%f") == 'core' && File.directory?(fn) + fn.pathmap("%f").downcase == 'core' && File.directory?(fn) } desc "Remove any temporary products." diff --git a/lib/rake/cloneable.rb b/lib/rake/cloneable.rb index 19c780bff6..ac67471232 100644 --- a/lib/rake/cloneable.rb +++ b/lib/rake/cloneable.rb @@ -3,23 +3,14 @@ module Rake # Mixin for creating easily cloned objects. # module Cloneable - # Clone an object by making a new object and setting all the instance - # variables to the same values. - def dup - sibling = self.class.new - instance_variables.each do |ivar| - value = self.instance_variable_get(ivar) - new_value = value.clone rescue value - sibling.instance_variable_set(ivar, new_value) + # The hook that invoked by 'clone' and 'dup' methods. + def initialize_copy(source) + super + source.instance_variables.each do |var| + src_value = source.instance_variable_get(var) + value = src_value.clone rescue src_value + instance_variable_set(var, value) end - sibling.taint if tainted? - sibling - end - - def clone - sibling = dup - sibling.freeze if frozen? - sibling end end end diff --git a/lib/rake/contrib/ftptools.rb b/lib/rake/contrib/ftptools.rb index 78420c7412..eaf8885262 100644 --- a/lib/rake/contrib/ftptools.rb +++ b/lib/rake/contrib/ftptools.rb @@ -127,7 +127,8 @@ module Rake # :nodoc: # Upload all files matching +wildcard+ to the uploader's root # path. def upload_files(wildcard) - Dir[wildcard].each do |fn| + fail "OUCH" + Rake.glob(wildcard).each do |fn| upload(fn) end end diff --git a/lib/rake/contrib/sys.rb b/lib/rake/contrib/sys.rb index 41963f1fef..aefd4a1913 100644 --- a/lib/rake/contrib/sys.rb +++ b/lib/rake/contrib/sys.rb @@ -27,7 +27,7 @@ module Sys # Install all the files matching +wildcard+ into the +dest_dir+ # directory. The permission mode is set to +mode+. def install(wildcard, dest_dir, mode) - Dir[wildcard].each do |fn| + Rake.glob(wildcard).each do |fn| File.install(fn, dest_dir, mode, $verbose) end end @@ -81,7 +81,7 @@ module Sys # recursively delete directories. def delete(*wildcards) wildcards.each do |wildcard| - Dir[wildcard].each do |fn| + Rake.glob(wildcard).each do |fn| if File.directory?(fn) log "Deleting directory #{fn}" Dir.delete(fn) @@ -96,10 +96,10 @@ module Sys # Recursively delete all files and directories matching +wildcard+. def delete_all(*wildcards) wildcards.each do |wildcard| - Dir[wildcard].each do |fn| + Rake.glob(wildcard).each do |fn| next if ! File.exist?(fn) if File.directory?(fn) - Dir["#{fn}/*"].each do |subfn| + Rake.glob("#{fn}/*").each do |subfn| next if subfn=='.' || subfn=='..' delete_all(subfn) end @@ -161,7 +161,7 @@ module Sys # Perform a block with each file matching a set of wildcards. def for_files(*wildcards) wildcards.each do |wildcard| - Dir[wildcard].each do |fn| + Rake.glob(wildcard).each do |fn| yield(fn) end end @@ -172,7 +172,7 @@ module Sys private # ---------------------------------------------------------- def for_matching_files(wildcard, dest_dir) - Dir[wildcard].each do |fn| + Rake.glob(wildcard).each do |fn| dest_file = File.join(dest_dir, fn) parent = File.dirname(dest_file) makedirs(parent) if ! File.directory?(parent) diff --git a/lib/rake/dsl_definition.rb b/lib/rake/dsl_definition.rb index 6d9a6b88f3..143b0bcf7a 100644 --- a/lib/rake/dsl_definition.rb +++ b/lib/rake/dsl_definition.rb @@ -52,8 +52,8 @@ module Rake # Declare a file creation task. # (Mainly used for the directory command). - def file_create(args, &block) - Rake::FileCreationTask.define_task(args, &block) + def file_create(*args, &block) + Rake::FileCreationTask.define_task(*args, &block) end # Declare a set of files tasks to create the given directories on @@ -62,12 +62,15 @@ module Rake # Example: # directory "testdata/doc" # - def directory(dir) + def directory(*args, &block) + result = file_create(*args, &block) + dir, _ = *Rake.application.resolve_args(args) Rake.each_dir_parent(dir) do |d| file_create d do |t| mkdir_p t.name if ! File.exist?(t.name) end end + result end # Declare a task that performs its prerequisites in @@ -78,8 +81,8 @@ module Rake # Example: # multitask :deploy => [:deploy_gem, :deploy_rdoc] # - def multitask(args, &block) - Rake::MultiTask.define_task(args, &block) + def multitask(*args, &block) + Rake::MultiTask.define_task(*args, &block) end # Create a new rake namespace and use it for evaluating the given @@ -167,10 +170,13 @@ module Rake private :#{name} }, __FILE__, line end - end + end unless defined? Rake::REDUCE_COMPAT extend FileUtilsExt end +# Extend the main object with the DSL commands. This allows top-level +# calls to task, etc. to work from a Rakefile without polluting the +# object inheritance tree. self.extend Rake::DSL -include Rake::DeprecatedObjectDSL +include Rake::DeprecatedObjectDSL unless defined? Rake::REDUCE_COMPAT diff --git a/lib/rake/ext/module.rb b/lib/rake/ext/module.rb index 3f64aef6c8..fc61bea555 100644 --- a/lib/rake/ext/module.rb +++ b/lib/rake/ext/module.rb @@ -36,4 +36,4 @@ class Module rake_original_const_missing(const_name) end end -end +end unless defined? Rake::REDUCE_COMPAT diff --git a/lib/rake/ext/string.rb b/lib/rake/ext/string.rb index fb22a9deb1..be8b463e1a 100644 --- a/lib/rake/ext/string.rb +++ b/lib/rake/ext/string.rb @@ -4,6 +4,7 @@ require 'rake/ext/core' # Rake extension methods for String. # class String + rake_extension("ext") do # Replace the file extension with +newext+. If there is no extension on # the string, append the new extension to the end. If the new extension @@ -163,5 +164,5 @@ class String result end end -end # class String +end diff --git a/lib/rake/ext/time.rb b/lib/rake/ext/time.rb index 7877abf0ce..ea8b037e39 100644 --- a/lib/rake/ext/time.rb +++ b/lib/rake/ext/time.rb @@ -1,6 +1,8 @@ #-- # Extensions to time to allow comparisons with an early time class. +require 'rake/early_time' + class Time alias rake_original_time_compare :<=> def <=>(other) @@ -11,4 +13,3 @@ class Time end end end - diff --git a/lib/rake/file_list.rb b/lib/rake/file_list.rb index e49ccd0147..b74ecac4cc 100644 --- a/lib/rake/file_list.rb +++ b/lib/rake/file_list.rb @@ -286,7 +286,7 @@ module Rake matched = 0 each do |fn| begin - open(fn, "r:ascii-8bit", *options) do |inf| + open(fn, "r", *options) do |inf| count = 0 inf.each do |line| count += 1 @@ -340,7 +340,7 @@ module Rake # Add matching glob patterns. def add_matching(pattern) - Dir[pattern].each do |fn| + Rake.glob(pattern).each do |fn| self << fn unless exclude?(fn) end end diff --git a/lib/rake/file_utils_ext.rb b/lib/rake/file_utils_ext.rb index 557420df80..386af441d8 100644 --- a/lib/rake/file_utils_ext.rb +++ b/lib/rake/file_utils_ext.rb @@ -21,12 +21,13 @@ module Rake $fileutils_verbose = true $fileutils_nowrite = false - FileUtils::OPT_TABLE.each do |name, opts| + FileUtils.commands.each do |name| + opts = FileUtils.options_of name default_options = [] - if opts.include?(:verbose) || opts.include?("verbose") + if opts.include?("verbose") default_options << ':verbose => FileUtilsExt.verbose_flag' end - if opts.include?(:noop) || opts.include?("noop") + if opts.include?("noop") default_options << ':noop => FileUtilsExt.nowrite_flag' end diff --git a/lib/rake/multi_task.rb b/lib/rake/multi_task.rb index 21c8de732f..5418a7a7b0 100644 --- a/lib/rake/multi_task.rb +++ b/lib/rake/multi_task.rb @@ -5,11 +5,8 @@ module Rake # class MultiTask < Task private - def invoke_prerequisites(args, invocation_chain) - threads = @prerequisites.collect { |p| - Thread.new(p) { |r| application[r, @scope].invoke_with_call_chain(args, invocation_chain) } - } - threads.each { |t| t.join } + def invoke_prerequisites(task_args, invocation_chain) # :nodoc: + invoke_prerequisites_concurrently(task_args, invocation_chain) end end diff --git a/lib/rake/phony.rb b/lib/rake/phony.rb new file mode 100644 index 0000000000..0552c26a33 --- /dev/null +++ b/lib/rake/phony.rb @@ -0,0 +1,13 @@ +# Defines a :phony task that you can use as a dependency. This allows +# file-based tasks to use non-file-based tasks as prerequisites +# without forcing them to rebuild. +# +# See FileTask#out_of_date? and Task#timestamp for more info. + +require 'rake' + +task :phony + +def (Rake::Task[:phony]).timestamp + Time.at 0 +end diff --git a/lib/rake/private_reader.rb b/lib/rake/private_reader.rb new file mode 100644 index 0000000000..1620978576 --- /dev/null +++ b/lib/rake/private_reader.rb @@ -0,0 +1,20 @@ +module Rake + + # Include PrivateReader to use +private_reader+. + module PrivateReader # :nodoc: all + + def self.included(base) + base.extend(ClassMethods) + end + + module ClassMethods + + # Declare a list of private accessors + def private_reader(*names) + attr_reader(*names) + private(*names) + end + end + + end +end diff --git a/lib/rake/promise.rb b/lib/rake/promise.rb new file mode 100644 index 0000000000..3258b91139 --- /dev/null +++ b/lib/rake/promise.rb @@ -0,0 +1,99 @@ +module Rake + + # A Promise object represents a promise to do work (a chore) in the + # future. The promise is created with a block and a list of + # arguments for the block. Calling value will return the value of + # the promised chore. + # + # Used by ThreadPool. + # + class Promise # :nodoc: all + NOT_SET = Object.new.freeze # :nodoc: + + attr_accessor :recorder + + # Create a promise to do the chore specified by the block. + def initialize(args, &block) + @mutex = Mutex.new + @result = NOT_SET + @error = NOT_SET + @args = args.collect { |a| begin; a.dup; rescue; a; end } + @block = block + end + + # Return the value of this promise. + # + # If the promised chore is not yet complete, then do the work + # synchronously. We will wait. + def value + unless complete? + stat :sleeping_on, :item_id => object_id + @mutex.synchronize do + stat :has_lock_on, :item_id => object_id + chore + stat :releasing_lock_on, :item_id => object_id + end + end + error? ? raise(@error) : @result + end + + # If no one else is working this promise, go ahead and do the chore. + def work + stat :attempting_lock_on, :item_id => object_id + if @mutex.try_lock + stat :has_lock_on, :item_id => object_id + chore + stat :releasing_lock_on, :item_id => object_id + @mutex.unlock + else + stat :bailed_on, :item_id => object_id + end + end + + private + + # Perform the chore promised + def chore + if complete? + stat :found_completed, :item_id => object_id + return + end + stat :will_execute, :item_id => object_id + begin + @result = @block.call(*@args) + rescue Exception => e + @error = e + end + stat :did_execute, :item_id => object_id + discard + end + + # Do we have a result for the promise + def result? + ! @result.equal?(NOT_SET) + end + + # Did the promise throw an error + def error? + ! @error.equal?(NOT_SET) + end + + # Are we done with the promise + def complete? + result? || error? + end + + # free up these items for the GC + def discard + @args = nil + @block = nil + end + + # Record execution statistics if there is a recorder + def stat(*args) + @recorder.call(*args) if @recorder + end + + end + +end diff --git a/lib/rake/rake_module.rb b/lib/rake/rake_module.rb index a9d210c637..6f77d1b674 100644 --- a/lib/rake/rake_module.rb +++ b/lib/rake/rake_module.rb @@ -24,6 +24,21 @@ module Rake def load_rakefile(path) load(path) end + + # Add files to the rakelib list + def add_rakelib(*files) + application.options.rakelib ||= [] + files.each do |file| + application.options.rakelib << file + end + end + + # Get a sorted list of files matching the pattern. This method + # should be prefered to Dir[pattern] and Dir.glob[pattern] because + # the files returned are guaranteed to be sorted. + def glob(pattern, *args) + Dir.glob(pattern, *args).sort + end end end diff --git a/lib/rake/rdoctask.rb b/lib/rake/rdoctask.rb index b6ae224a9e..261fa69b4d 100644 --- a/lib/rake/rdoctask.rb +++ b/lib/rake/rdoctask.rb @@ -1,7 +1,7 @@ # rake/rdoctask is deprecated in favor of rdoc/task if Rake.application - Rake.application.deprecate('require \'rake/rdoctask\'', 'require \'rdoc/task\' (in RDoc 2.4.2+)', __FILE__) + Rake.application.deprecate('require \'rake/rdoctask\'', 'require \'rdoc/task\' (in RDoc 2.4.2+)', caller.first) end require 'rubygems' diff --git a/lib/rake/runtest.rb b/lib/rake/runtest.rb index 2b98a60cae..9c6469d45e 100644 --- a/lib/rake/runtest.rb +++ b/lib/rake/runtest.rb @@ -5,7 +5,7 @@ module Rake include Test::Unit::Assertions def run_tests(pattern='test/test*.rb', log_enabled=false) - Dir["#{pattern}"].each { |fn| + Rake.glob(pattern).each { |fn| $stderr.puts fn if log_enabled begin require fn diff --git a/lib/rake/task.rb b/lib/rake/task.rb index f977d18711..afa1b6153d 100644 --- a/lib/rake/task.rb +++ b/lib/rake/task.rb @@ -123,6 +123,7 @@ module Rake def clear clear_prerequisites clear_actions + clear_comments self end @@ -138,6 +139,13 @@ module Rake self end + # Clear the existing comments on a rake task. + def clear_comments + @full_comment = nil + @comment = nil + self + end + # Invoke the task if it is needed. Prerequisites are invoked first. def invoke(*args) task_args = TaskArguments.new(arg_names, args) @@ -150,7 +158,7 @@ module Rake new_chain = InvocationChain.append(self, invocation_chain) @lock.synchronize do if application.options.trace - $stderr.puts "** Invoke #{name} #{format_trace_flags}" + application.trace "** Invoke #{name} #{format_trace_flags}" end return if @already_invoked @already_invoked = true @@ -171,10 +179,24 @@ module Rake # Invoke all the prerequisites of a task. def invoke_prerequisites(task_args, invocation_chain) # :nodoc: - prerequisite_tasks.each { |prereq| - prereq_args = task_args.new_scope(prereq.arg_names) - prereq.invoke_with_call_chain(prereq_args, invocation_chain) - } + if application.options.always_multitask + invoke_prerequisites_concurrently(task_args, invocation_chain) + else + prerequisite_tasks.each { |prereq| + prereq_args = task_args.new_scope(prereq.arg_names) + prereq.invoke_with_call_chain(prereq_args, invocation_chain) + } + end + end + + # Invoke all the prerequisites of a task in parallel. + def invoke_prerequisites_concurrently(args, invocation_chain) # :nodoc: + futures = @prerequisites.collect do |p| + application.thread_pool.future(p) do |r| + application[r, @scope].invoke_with_call_chain(args, invocation_chain) + end + end + futures.each { |f| f.value } end # Format the trace flags for display. @@ -190,11 +212,11 @@ module Rake def execute(args=nil) args ||= EMPTY_TASK_ARGS if application.options.dryrun - $stderr.puts "** Execute (dry run) #{name}" + application.trace "** Execute (dry run) #{name}" return end if application.options.trace - $stderr.puts "** Execute #{name}" + application.trace "** Execute #{name}" end application.enhance_with_matching_rule(name) if @actions.empty? @actions.each do |act| diff --git a/lib/rake/task_arguments.rb b/lib/rake/task_arguments.rb index 02d01b99f9..4417af2f8e 100644 --- a/lib/rake/task_arguments.rb +++ b/lib/rake/task_arguments.rb @@ -47,7 +47,7 @@ module Rake keys.map { |k| lookup(k) } end - def method_missing(sym, *args, &block) + def method_missing(sym, *args) lookup(sym.to_sym) end diff --git a/lib/rake/task_manager.rb b/lib/rake/task_manager.rb index 4c3c26aa3a..5a9419d536 100644 --- a/lib/rake/task_manager.rb +++ b/lib/rake/task_manager.rb @@ -238,7 +238,7 @@ module Rake end def trace_rule(level, message) - $stderr.puts "#{" "*level}#{message}" if Rake.application.options.trace_rules + options.trace_output.puts "#{" "*level}#{message}" if Rake.application.options.trace_rules end # Attempt to create a rule given the list of prerequisites. diff --git a/lib/rake/testtask.rb b/lib/rake/testtask.rb index 04d3ae473a..99094df1c8 100644 --- a/lib/rake/testtask.rb +++ b/lib/rake/testtask.rb @@ -96,7 +96,11 @@ module Rake desc "Run tests" + (@name==:test ? "" : " for #{@name}") task @name do FileUtilsExt.verbose(@verbose) do - ruby "#{ruby_opts_string} #{run_code} #{file_list_string} #{option_list}" + ruby "#{ruby_opts_string} #{run_code} #{file_list_string} #{option_list}" do |ok, status| + if !ok && status.respond_to?(:signaled?) && status.signaled? + raise SignalException.new(status.termsig) + end + end end end self diff --git a/lib/rake/thread_history_display.rb b/lib/rake/thread_history_display.rb new file mode 100644 index 0000000000..917e951064 --- /dev/null +++ b/lib/rake/thread_history_display.rb @@ -0,0 +1,48 @@ +require 'rake/private_reader' + +module Rake + + class ThreadHistoryDisplay # :nodoc: all + include Rake::PrivateReader + + private_reader :stats, :items, :threads + + def initialize(stats) + @stats = stats + @items = { :_seq_ => 1 } + @threads = { :_seq_ => "A" } + end + + def show + puts "Job History:" + stats.each do |stat| + stat[:data] ||= {} + rename(stat, :thread, threads) + rename(stat[:data], :item_id, items) + rename(stat[:data], :new_thread, threads) + rename(stat[:data], :deleted_thread, threads) + printf("%8d %2s %-20s %s\n", + (stat[:time] * 1_000_000).round, + stat[:thread], + stat[:event], + stat[:data].map { |k,v| "#{k}:#{v}" }.join(" ")) + end + end + + private + + def rename(hash, key, renames) + if hash && hash[key] + original = hash[key] + value = renames[original] + unless value + value = renames[:_seq_] + renames[:_seq_] = renames[:_seq_].succ + renames[original] = value + end + hash[key] = value + end + end + end + +end diff --git a/lib/rake/thread_pool.rb b/lib/rake/thread_pool.rb new file mode 100644 index 0000000000..983a67a514 --- /dev/null +++ b/lib/rake/thread_pool.rb @@ -0,0 +1,155 @@ +require 'thread' +require 'set' + +require 'rake/promise' + +module Rake + + class ThreadPool # :nodoc: all + + # Creates a ThreadPool object. + # The parameter is the size of the pool. + def initialize(thread_count) + @max_active_threads = [thread_count, 0].max + @threads = Set.new + @threads_mon = Monitor.new + @queue = Queue.new + @join_cond = @threads_mon.new_cond + + @history_start_time = nil + @history = [] + @history_mon = Monitor.new + @total_threads_in_play = 0 + end + + # Creates a future executed by the +ThreadPool+. + # + # The args are passed to the block when executing (similarly to + # Thread#new) The return value is an object representing + # a future which has been created and added to the queue in the + # pool. Sending #value to the object will sleep the + # current thread until the future is finished and will return the + # result (or raise an exception thrown from the future) + def future(*args, &block) + promise = Promise.new(args, &block) + promise.recorder = lambda { |*stats| stat(*stats) } + + @queue.enq promise + stat :queued, :item_id => promise.object_id + start_thread + promise + end + + # Waits until the queue of futures is empty and all threads have exited. + def join + @threads_mon.synchronize do + begin + stat :joining + @join_cond.wait unless @threads.empty? + stat :joined + rescue Exception => e + stat :joined + $stderr.puts e + $stderr.print "Queue contains #{@queue.size} items. Thread pool contains #{@threads.count} threads\n" + $stderr.print "Current Thread #{Thread.current} status = #{Thread.current.status}\n" + $stderr.puts e.backtrace.join("\n") + @threads.each do |t| + $stderr.print "Thread #{t} status = #{t.status}\n" + # 1.8 doesn't support Thread#backtrace + $stderr.puts t.backtrace.join("\n") if t.respond_to? :backtrace + end + raise e + end + end + end + + # Enable the gathering of history events. + def gather_history #:nodoc: + @history_start_time = Time.now if @history_start_time.nil? + end + + # Return a array of history events for the thread pool. + # + # History gathering must be enabled to be able to see the events + # (see #gather_history). Best to call this when the job is + # complete (i.e. after ThreadPool#join is called). + def history # :nodoc: + @history_mon.synchronize { @history.dup }. + sort_by { |i| i[:time] }. + each { |i| i[:time] -= @history_start_time } + end + + # Return a hash of always collected statistics for the thread pool. + def statistics # :nodoc: + { + :total_threads_in_play => @total_threads_in_play, + :max_active_threads => @max_active_threads, + } + end + + private + + # processes one item on the queue. Returns true if there was an + # item to process, false if there was no item + def process_queue_item #:nodoc: + return false if @queue.empty? + + # Even though we just asked if the queue was empty, it + # still could have had an item which by this statement + # is now gone. For this reason we pass true to Queue#deq + # because we will sleep indefinitely if it is empty. + promise = @queue.deq(true) + stat :dequeued, :item_id => promise.object_id + promise.work + return true + + rescue ThreadError # this means the queue is empty + false + end + + def start_thread # :nodoc: + @threads_mon.synchronize do + next unless @threads.count < @max_active_threads + + t = Thread.new do + begin + while @threads.count <= @max_active_threads + break unless process_queue_item + end + ensure + @threads_mon.synchronize do + @threads.delete Thread.current + stat :ended, :thread_count => @threads.count + @join_cond.broadcast if @threads.empty? + end + end + end + @threads << t + stat :spawned, :new_thread => t.object_id, :thread_count => @threads.count + @total_threads_in_play = @threads.count if @threads.count > @total_threads_in_play + end + end + + def stat(event, data=nil) # :nodoc: + return if @history_start_time.nil? + info = { + :event => event, + :data => data, + :time => Time.now, + :thread => Thread.current.object_id, + } + @history_mon.synchronize { @history << info } + end + + # for testing only + + def __queue__ # :nodoc: + @queue + end + + def __threads__ # :nodoc: + @threads.dup + end + end + +end diff --git a/lib/rake/version.rb b/lib/rake/version.rb index 6c43493df9..2515e25663 100644 --- a/lib/rake/version.rb +++ b/lib/rake/version.rb @@ -1,8 +1,10 @@ module Rake - VERSION = '0.9.2.2' - module Version # :nodoc: all - MAJOR, MINOR, BUILD = VERSION.split '.' - NUMBERS = [ MAJOR, MINOR, BUILD ] + NUMBERS = [ + MAJOR = 0, + MINOR = 9, + BUILD = 3, + ] end + VERSION = Version::NUMBERS.join('.') end diff --git a/test/rake/helper.rb b/test/rake/helper.rb index 65443fc111..5a373d3043 100644 --- a/test/rake/helper.rb +++ b/test/rake/helper.rb @@ -1,4 +1,5 @@ require 'rubygems' +$:.unshift File.expand_path('../../lib', __FILE__) begin gem 'minitest' @@ -488,5 +489,34 @@ end VERBOSE end + def rakefile_test_signal + rakefile <<-TEST_SIGNAL +require 'rake/testtask' + +Rake::TestTask.new(:a) do |t| + t.test_files = ['a_test.rb'] +end + +Rake::TestTask.new(:b) do |t| + t.test_files = ['b_test.rb'] +end + +task :test do + Rake::Task[:a].invoke rescue nil + Rake::Task[:b].invoke rescue nil end +task :default => :test + TEST_SIGNAL + open 'a_test.rb', 'w' do |io| + io << 'puts "ATEST"' << "\n" + io << '$stdout.flush' << "\n" + io << 'Process.kill("TERM", $$)' << "\n" + end + open 'b_test.rb', 'w' do |io| + io << 'puts "BTEST"' << "\n" + io << '$stdout.flush' << "\n" + end + end + +end diff --git a/test/rake/test_private_reader.rb b/test/rake/test_private_reader.rb new file mode 100644 index 0000000000..f86d4249b2 --- /dev/null +++ b/test/rake/test_private_reader.rb @@ -0,0 +1,42 @@ +require File.expand_path('../helper', __FILE__) +require 'rake/private_reader' + +class TestPrivateAttrs < Rake::TestCase + + class Sample + include Rake::PrivateReader + + private_reader :reader, :a + + def initialize + @reader = :RVALUE + end + + def get_reader + reader + end + + end + + def setup + super + @sample = Sample.new + end + + def test_private_reader_is_private + assert_private do @sample.reader end + assert_private do @sample.a end + end + + def test_private_reader_returns_data + assert_equal :RVALUE, @sample.get_reader + end + + private + + def assert_private + ex = assert_raises(NoMethodError) do yield end + assert_match(/private/, ex.message) + end + +end diff --git a/test/rake/test_rake_application.rb b/test/rake/test_rake_application.rb index b5d8c652c4..eb6d148a65 100644 --- a/test/rake/test_rake_application.rb +++ b/test/rake/test_rake_application.rb @@ -385,6 +385,18 @@ class TestRakeApplication < Rake::TestCase ARGV.clear end + def test_bad_run_with_backtrace + @app.intern(Rake::Task, "default").enhance { fail } + ARGV.clear + ARGV << '-f' << '-s' << '--backtrace' + assert_raises(SystemExit) { + _, err = capture_io { @app.run } + refute_match(/see full trace/, err) + } + ensure + ARGV.clear + end + def test_run_with_bad_options @app.intern(Rake::Task, "default").enhance { fail } ARGV.clear @@ -486,4 +498,3 @@ class TestRakeApplication < Rake::TestCase end end - diff --git a/test/rake/test_rake_application_options.rb b/test/rake/test_rake_application_options.rb index c1471f443e..004b8ef3ac 100644 --- a/test/rake/test_rake_application_options.rb +++ b/test/rake/test_rake_application_options.rb @@ -29,10 +29,12 @@ class TestRakeApplicationOptions < Rake::TestCase def test_default_options opts = command_line + assert_nil opts.backtrace assert_nil opts.classic_namespace assert_nil opts.dryrun assert_nil opts.ignore_system assert_nil opts.load_system + assert_nil opts.always_multitask assert_nil opts.nosearch assert_equal ['rakelib'], opts.rakelib assert_nil opts.show_prereqs @@ -40,6 +42,7 @@ class TestRakeApplicationOptions < Rake::TestCase assert_nil opts.show_tasks assert_nil opts.silent assert_nil opts.trace + assert_nil opts.thread_pool_size assert_equal ['rakelib'], opts.rakelib assert ! Rake::FileUtilsExt.verbose_flag assert ! Rake::FileUtilsExt.nowrite_flag @@ -110,6 +113,18 @@ class TestRakeApplicationOptions < Rake::TestCase assert_equal :exit, @exit end + def test_jobs + flags(['--jobs', '4'], ['-j', '4']) do |opts| + assert_equal 4, opts.thread_pool_size + end + flags(['--jobs', 'asdas'], ['-j', 'asdas']) do |opts| + assert_equal 2, opts.thread_pool_size + end + flags('--jobs', '-j') do |opts| + assert_equal 2, opts.thread_pool_size + end + end + def test_libdir flags(['--libdir', 'xx'], ['-I', 'xx'], ['-Ixx']) do |opts| $:.include?('xx') @@ -118,6 +133,12 @@ class TestRakeApplicationOptions < Rake::TestCase $:.delete('xx') end + def test_multitask + flags('--multitask', '-m') do |opts| + assert_equal opts.always_multitask, true + end + end + def test_rakefile flags(['--rakefile', 'RF'], ['--rakefile=RF'], ['-f', 'RF'], ['-fRF']) do |opts| assert_equal ['RF'], @app.instance_eval { @rakefiles } @@ -125,7 +146,8 @@ class TestRakeApplicationOptions < Rake::TestCase end def test_rakelib - flags(['--rakelibdir', 'A:B:C'], ['--rakelibdir=A:B:C'], ['-R', 'A:B:C'], ['-RA:B:C']) do |opts| + dirs = %w(A B C).join(File::PATH_SEPARATOR) + flags(['--rakelibdir', dirs], ["--rakelibdir=#{dirs}"], ['-R', dirs], ["-R#{dirs}"]) do |opts| assert_equal ['A', 'B', 'C'], opts.rakelib end end @@ -197,12 +219,76 @@ class TestRakeApplicationOptions < Rake::TestCase def test_trace flags('--trace', '-t') do |opts| - assert opts.trace + assert opts.trace, "should enable trace option" + assert opts.backtrace, "should enabled backtrace option" + assert_equal $stderr, opts.trace_output + assert Rake::FileUtilsExt.verbose_flag + assert ! Rake::FileUtilsExt.nowrite_flag + end + end + + def test_trace_with_stdout + flags('--trace=stdout', '-tstdout', '-t stdout') do |opts| + assert opts.trace, "should enable trace option" + assert opts.backtrace, "should enabled backtrace option" + assert_equal $stdout, opts.trace_output assert Rake::FileUtilsExt.verbose_flag assert ! Rake::FileUtilsExt.nowrite_flag end end + def test_trace_with_stderr + flags('--trace=stderr', '-tstderr', '-t stderr') do |opts| + assert opts.trace, "should enable trace option" + assert opts.backtrace, "should enabled backtrace option" + assert_equal $stderr, opts.trace_output + assert Rake::FileUtilsExt.verbose_flag + assert ! Rake::FileUtilsExt.nowrite_flag + end + end + + def test_trace_with_error + ex = assert_raises(Rake::CommandLineOptionError) do + flags('--trace=xyzzy') do |opts| end + end + assert_match(/un(known|recognized).*\btrace\b.*xyzzy/i, ex.message) + end + + + def test_backtrace + flags('--backtrace') do |opts| + assert opts.backtrace, "should enable backtrace option" + assert_equal $stderr, opts.trace_output + assert ! opts.trace, "should not enable trace option" + assert ! Rake::FileUtilsExt.verbose_flag + end + end + + def test_backtrace_with_stdout + flags('--backtrace=stdout') do |opts| + assert opts.backtrace, "should enable backtrace option" + assert_equal $stdout, opts.trace_output + assert ! opts.trace, "should not enable trace option" + assert ! Rake::FileUtilsExt.verbose_flag + end + end + + def test_backtrace_with_stderr + flags('--backtrace=stderr') do |opts| + assert opts.backtrace, "should enable backtrace option" + assert_equal $stderr, opts.trace_output + assert ! opts.trace, "should not enable trace option" + assert ! Rake::FileUtilsExt.verbose_flag + end + end + + def test_backtrace_with_error + ex = assert_raises(Rake::CommandLineOptionError) do + flags('--backtrace=xyzzy') do |opts| end + end + assert_match(/un(known|recognized).*\bbacktrace\b.*xyzzy/i, ex.message) + end + def test_trace_rules flags('--rules') do |opts| assert opts.trace_rules @@ -213,10 +299,17 @@ class TestRakeApplicationOptions < Rake::TestCase flags('--tasks', '-T') do |opts| assert_equal :tasks, opts.show_tasks assert_equal(//.to_s, opts.show_task_pattern.to_s) + assert_equal nil, opts.show_all_tasks end flags(['--tasks', 'xyz'], ['-Txyz']) do |opts| assert_equal :tasks, opts.show_tasks assert_equal(/xyz/.to_s, opts.show_task_pattern.to_s) + assert_equal nil, opts.show_all_tasks + end + flags(['--tasks', 'xyz', '--comments']) do |opts| + assert_equal :tasks, opts.show_tasks + assert_equal(/xyz/.to_s, opts.show_task_pattern.to_s) + assert_equal false, opts.show_all_tasks end end @@ -224,10 +317,17 @@ class TestRakeApplicationOptions < Rake::TestCase flags('--where', '-W') do |opts| assert_equal :lines, opts.show_tasks assert_equal(//.to_s, opts.show_task_pattern.to_s) + assert_equal true, opts.show_all_tasks end flags(['--where', 'xyz'], ['-Wxyz']) do |opts| assert_equal :lines, opts.show_tasks assert_equal(/xyz/.to_s, opts.show_task_pattern.to_s) + assert_equal true, opts.show_all_tasks + end + flags(['--where', 'xyz', '--comments'], ['-Wxyz', '--comments']) do |opts| + assert_equal :lines, opts.show_tasks + assert_equal(/xyz/.to_s, opts.show_task_pattern.to_s) + assert_equal false, opts.show_all_tasks end end @@ -268,7 +368,7 @@ class TestRakeApplicationOptions < Rake::TestCase assert_equal opts.trace, $trace assert_equal opts.dryrun, $dryrun assert_equal opts.silent, $silent - end + end end assert_match(/deprecated/, err) @@ -308,6 +408,17 @@ class TestRakeApplicationOptions < Rake::TestCase assert '12', ENV['TESTKEY'] end + def test_rake_explicit_task_library + Rake.add_rakelib 'app/task', 'other' + + libs = Rake.application.options.rakelib + + assert libs.include?("app/task") + assert libs.include?("other") + end + + private + def flags(*sets) sets.each do |set| ARGV.clear @@ -332,4 +443,3 @@ class TestRakeApplicationOptions < Rake::TestCase @app.options end end - diff --git a/test/rake/test_rake_backtrace.rb b/test/rake/test_rake_backtrace.rb new file mode 100644 index 0000000000..4d84becf98 --- /dev/null +++ b/test/rake/test_rake_backtrace.rb @@ -0,0 +1,81 @@ +require File.expand_path('../helper', __FILE__) +require 'open3' + +class TestRakeBacktrace < Rake::TestCase + # TODO: factor out similar code in test_rake_functional.rb + def rake(*args) + lib = File.join(@orig_PWD, "lib") + bin_rake = File.join(@orig_PWD, "bin", "rake") + Open3.popen3(RUBY, "-I", lib, bin_rake, *args) { |_, _, err, _| err.read } + end + + def invoke(task_name) + rake task_name.to_s + end + + def test_single_collapse + rakefile %q{ + task :foo do + raise "foooo!" + end + } + + lines = invoke(:foo).split("\n") + + assert_equal "rake aborted!", lines[0] + assert_equal "foooo!", lines[1] + assert_something_matches %r!\A#{Regexp.quote Dir.pwd}/Rakefile:3!i, lines + assert_something_matches %r!\ATasks:!, lines + end + + def test_multi_collapse + rakefile %q{ + task :foo do + Rake.application.invoke_task(:bar) + end + task :bar do + raise "barrr!" + end + } + + lines = invoke(:foo).split("\n") + + assert_equal "rake aborted!", lines[0] + assert_equal "barrr!", lines[1] + assert_something_matches %r!\A#{Regexp.quote Dir.pwd}/Rakefile:6!i, lines + assert_something_matches %r!\A#{Regexp.quote Dir.pwd}/Rakefile:3!i, lines + assert_something_matches %r!\ATasks:!, lines + end + + def test_suppress_option + rakefile %q{ + task :baz do + raise "bazzz!" + end + } + + lines = rake("baz").split("\n") + assert_equal "rake aborted!", lines[0] + assert_equal "bazzz!", lines[1] + assert_something_matches %r!Rakefile!i, lines + + lines = rake("--suppress-backtrace", ".ak.file", "baz").split("\n") + assert_equal "rake aborted!", lines[0] + assert_equal "bazzz!", lines[1] + refute_match %r!Rakefile!i, lines[2] + end + + private + + # Assert that the pattern matches at least one line in +lines+. + def assert_something_matches(pattern, lines) + lines.each do |ln| + if pattern =~ ln + assert_match pattern, ln + return + end + end + flunk "expected #{pattern.inspect} to match something in:\n #{lines.join("\n ")}" + end + +end diff --git a/test/rake/test_rake_directory_task.rb b/test/rake/test_rake_directory_task.rb index 6ddb0d2870..d1154f90d5 100644 --- a/test/rake/test_rake_directory_task.rb +++ b/test/rake/test_rake_directory_task.rb @@ -27,26 +27,31 @@ class TestRakeDirectoryTask < Rake::TestCase if Rake::Win32.windows? def test_directory_win32 - drive = Dir.pwd - while drive != File.dirname(drive) - drive = File.dirname(drive) - end - drive = drive[0...-1] if drive[-1] == ?/ - desc "WIN32 DESC" - directory File.join(Dir.pwd, 'a/b/c') - assert_equal FileTask, Task[drive].class if drive[-1] == ?: - assert_equal FileCreationTask, Task[File.join(Dir.pwd, 'a')].class - assert_equal FileCreationTask, Task[File.join(Dir.pwd, 'a/b')].class - assert_equal FileCreationTask, Task[File.join(Dir.pwd, 'a/b/c')].class - assert_nil Task[drive].comment - assert_equal "WIN32 DESC", Task[File.join(Dir.pwd, 'a/b/c')].comment - assert_nil Task[File.join(Dir.pwd, 'a/b')].comment - verbose(false) { - Task[File.join(Dir.pwd, 'a/b')].invoke - } - assert File.exist?(File.join(Dir.pwd, 'a/b')) - refute File.exist?(File.join(Dir.pwd, 'a/b/c')) + directory 'c:/a/b/c' + assert_equal FileTask, Task['c:'].class + assert_equal FileCreationTask, Task['c:/a'].class + assert_equal FileCreationTask, Task['c:/a/b'].class + assert_equal FileCreationTask, Task['c:/a/b/c'].class + assert_nil Task['c:/'].comment + assert_equal "WIN32 DESC", Task['c:/a/b/c'].comment + assert_nil Task['c:/a/b'].comment end end + + def test_can_use_blocks + runlist = [] + + t1 = directory("a/b/c" => :t2) { |t| runlist << t.name } + t2 = task(:t2) { |t| runlist << t.name } + + verbose(false) { + t1.invoke + } + + assert_equal Task["a/b/c"], t1 + assert_equal FileCreationTask, Task["a/b/c"].class + assert_equal ["t2", "a/b/c"], runlist + assert File.directory?("a/b/c") + end end diff --git a/test/rake/test_rake_file_task.rb b/test/rake/test_rake_file_task.rb index e586551237..a46851bd01 100644 --- a/test/rake/test_rake_file_task.rb +++ b/test/rake/test_rake_file_task.rb @@ -41,6 +41,23 @@ class TestRakeFileTask < Rake::TestCase assert ! t1.needed?, "Should not need to rebuild new file because of old" end + def test_file_times_new_depend_on_regular_task_timestamps + load_phony + + name = "dummy" + task name + + create_timed_files(NEWFILE) + + t1 = Rake.application.intern(FileTask, NEWFILE).enhance([name]) + + assert t1.needed?, "depending on non-file task uses Time.now" + + task(name => :phony) + + assert ! t1.needed?, "unless the non-file task has a timestamp" + end + def test_file_times_old_depends_on_new create_timed_files(OLDFILE, NEWFILE) @@ -98,5 +115,8 @@ class TestRakeFileTask < Rake::TestCase assert( ! File.exist?(NEWFILE), "NEWFILE should be deleted") end -end + def load_phony + load File.join(@orig_PWD, "lib/rake/phony.rb") + end +end diff --git a/test/rake/test_rake_functional.rb b/test/rake/test_rake_functional.rb index 3764709f38..ad59f7b9f8 100644 --- a/test/rake/test_rake_functional.rb +++ b/test/rake/test_rake_functional.rb @@ -5,8 +5,8 @@ require 'open3' class TestRakeFunctional < Rake::TestCase def setup - @rake_path = File.expand_path("../../../bin/rake", __FILE__) - lib_path = File.expand_path("../../../lib", __FILE__) + @rake_path = File.expand_path("bin/rake") + lib_path = File.expand_path("lib") @ruby_options = ["-I#{lib_path}", "-I."] @verbose = ENV['VERBOSE'] @@ -417,6 +417,28 @@ class TestRakeFunctional < Rake::TestCase assert_equal "1\n", @out end + def can_detect_signals? + system "ruby -e 'Process.kill \"TERM\", $$'" + status = $? + if @verbose + puts " SIG status = #{$?.inspect}" + puts " SIG status.respond_to?(:signaled?) = #{$?.respond_to?(:signaled?).inspect}" + puts " SIG status.signaled? = #{status.signaled?}" if status.respond_to?(:signaled?) + end + status.respond_to?(:signaled?) && status.signaled? + end + + def test_signal_propagation_in_tests + if can_detect_signals? + rakefile_test_signal + rake + assert_match(/ATEST/, @out) + refute_match(/BTEST/, @out) + else + skip "Signal detect seems broken on this system" + end + end + private # Run a shell Ruby command with command line options (using the diff --git a/test/rake/test_rake_multi_task.rb b/test/rake/test_rake_multi_task.rb index cac235b9d3..22d13d27a0 100644 --- a/test/rake/test_rake_multi_task.rb +++ b/test/rake/test_rake_multi_task.rb @@ -47,5 +47,13 @@ class TestRakeMultiTask < Rake::TestCase assert @runs.index("B0") < @runs.index("B1") assert @runs.index("B1") < @runs.index("B2") end + + def test_multitasks_with_parameters + task :a, [:arg] do |t,args| add_run(args[:arg]) end + multitask :b, [:arg] => [:a] do |t,args| add_run(args[:arg]+'mt') end + Task[:b].invoke "b" + assert @runs[0] == "b" + assert @runs[1] == "bmt" + end end diff --git a/test/rake/test_rake_rake_test_loader.rb b/test/rake/test_rake_rake_test_loader.rb index be3c7da61f..5b5e81d06d 100644 --- a/test/rake/test_rake_rake_test_loader.rb +++ b/test/rake/test_rake_rake_test_loader.rb @@ -10,7 +10,7 @@ class TestRakeRakeTestLoader < Rake::TestCase ARGV.replace %w[foo.rb test_*.rb -v] - load File.expand_path('../../../lib/rake/rake_test_loader.rb', __FILE__) + load File.join(@orig_PWD, 'lib/rake/rake_test_loader.rb') assert_equal %w[-v], ARGV ensure diff --git a/test/rake/test_rake_reduce_compat.rb b/test/rake/test_rake_reduce_compat.rb new file mode 100644 index 0000000000..12872614db --- /dev/null +++ b/test/rake/test_rake_reduce_compat.rb @@ -0,0 +1,65 @@ +require File.expand_path('../helper', __FILE__) +require 'open3' + +class TestRakeReduceCompat < Rake::TestCase + # TODO: factor out similar code in test_rake_functional.rb + def rake(*args) + lib = File.join(@orig_PWD, "lib") + bin_rake = File.join(@orig_PWD, "bin", "rake") + Open3.popen3(RUBY, "-I", lib, bin_rake, *args) { |_, out, _, _| out.read } + end + + def invoke_normal(task_name) + rake task_name.to_s + end + + def invoke_reduce_compat(task_name) + rake "--reduce-compat", task_name.to_s + end + + def test_no_deprecated_dsl + rakefile %q{ + task :check_task do + Module.new { p defined?(task) } + end + + task :check_file do + Module.new { p defined?(file) } + end + } + + assert_equal %{"method"}, invoke_normal(:check_task).chomp + assert_equal %{"method"}, invoke_normal(:check_file).chomp + + assert_equal "nil", invoke_reduce_compat(:check_task).chomp + assert_equal "nil", invoke_reduce_compat(:check_file).chomp + end + + def test_no_classic_namespace + rakefile %q{ + task :check_task do + begin + Task + print "present" + rescue NameError + print "absent" + end + end + + task :check_file_task do + begin + FileTask + print "present" + rescue NameError + print "absent" + end + end + } + + assert_equal "present", invoke_normal(:check_task) + assert_equal "present", invoke_normal(:check_file_task) + + assert_equal "absent", invoke_reduce_compat(:check_task) + assert_equal "absent", invoke_reduce_compat(:check_file_task) + end +end diff --git a/test/rake/test_rake_task.rb b/test/rake/test_rake_task.rb index a5bc693237..836e930ee4 100644 --- a/test/rake/test_rake_task.rb +++ b/test/rake/test_rake_task.rb @@ -104,10 +104,12 @@ class TestRakeTask < Rake::TestCase end def test_clear + desc "a task" t = task("t" => "a") { } t.clear assert t.prerequisites.empty?, "prerequisites should be empty" assert t.actions.empty?, "actions should be empty" + assert_nil t.comment, "comments should be empty" end def test_clear_prerequisites @@ -123,6 +125,22 @@ class TestRakeTask < Rake::TestCase assert t.actions.empty?, "actions should be empty" end + def test_clear_comments + desc "the original foo" + task :foo => [:x] do + # Dummy action + end + + task(:foo).clear_comments + + desc "a slightly different foo" + task :foo + + assert_equal "a slightly different foo", task(:foo).comment + assert_equal ["x"], task(:foo).prerequisites + assert_equal 1, task(:foo).actions.size + end + def test_find task :tfind assert_equal "tfind", Task[:tfind].name @@ -223,6 +241,38 @@ class TestRakeTask < Rake::TestCase assert_in_delta now + 10, a.timestamp, 0.1, 'computer too slow?' end + def test_always_multitask + mx = Mutex.new + result = [] + + t_a = task(:a) do |t| + sleep 0.02 + mx.synchronize{ result << t.name } + end + + t_b = task(:b) do |t| + mx.synchronize{ result << t.name } + end + + t_c = task(:c => [:a,:b]) do |t| + mx.synchronize{ result << t.name } + end + + t_c.invoke + + # task should always run in order + assert_equal ['a', 'b', 'c'], result + + [t_a, t_b, t_c].each { |t| t.reenable } + result.clear + + Rake.application.options.always_multitask = true + t_c.invoke + + # with multitask, task 'b' should grab the mutex first + assert_equal ['b', 'a', 'c'], result + end + def test_investigation_output t1 = task(:t1 => [:t2, :t3]) { |t| runlist << t.name; 3321 } task(:t2) @@ -264,4 +314,3 @@ class TestRakeTask < Rake::TestCase assert_equal "HI", t.comment end end - diff --git a/test/rake/test_rake_thread_pool.rb b/test/rake/test_rake_thread_pool.rb new file mode 100644 index 0000000000..90c8bf577a --- /dev/null +++ b/test/rake/test_rake_thread_pool.rb @@ -0,0 +1,123 @@ +require File.expand_path('../helper', __FILE__) +require 'rake/thread_pool' +require 'test/unit/assertions' + +class TestRakeTestThreadPool < Rake::TestCase + include Rake + + def test_pool_executes_in_current_thread_for_zero_threads + pool = ThreadPool.new(0) + f = pool.future{Thread.current} + pool.join + assert_equal Thread.current, f.value + end + + def test_pool_executes_in_other_thread_for_pool_of_size_one + pool = ThreadPool.new(1) + f = pool.future{Thread.current} + pool.join + refute_equal Thread.current, f.value + end + + def test_pool_executes_in_two_other_threads_for_pool_of_size_two + pool = ThreadPool.new(2) + threads = 2.times.collect{ pool.future{ sleep 0.1; Thread.current } }.each{|f|f.value} + + refute_equal threads[0], threads[1] + refute_equal Thread.current, threads[0] + refute_equal Thread.current, threads[1] + end + + def test_pool_creates_the_correct_number_of_threads + pool = ThreadPool.new(2) + threads = Set.new + t_mutex = Mutex.new + 10.times.each do + pool.future do + sleep 0.02 + t_mutex.synchronize{ threads << Thread.current } + end + end + pool.join + assert_equal 2, threads.count + end + + def test_pool_future_captures_arguments + pool = ThreadPool.new(2) + a = 'a' + b = 'b' + c = 5 # 5 throws an execption with 5.dup. It should be ignored + pool.future(a,c){ |a_var,ignore| a_var.capitalize!; b.capitalize! } + pool.join + assert_equal 'a', a + assert_equal 'b'.capitalize, b + end + + def test_pool_join_empties_queue + pool = ThreadPool.new(2) + repeat = 25 + repeat.times { + pool.future do + repeat.times { + pool.future do + repeat.times { + pool.future do end + } + end + } + end + } + + pool.join + assert_equal true, pool.__send__(:__queue__).empty?, "queue should be empty" + end + + # test that throwing an exception way down in the blocks propagates + # to the top + def test_exceptions + pool = ThreadPool.new(10) + + deep_exception_block = lambda do |count| + next raise Exception.new if ( count < 1 ) + pool.future(count-1, &deep_exception_block).value + end + + assert_raises(Exception) do + pool.future(2, &deep_exception_block).value + end + + end + + def test_pool_prevents_deadlock + pool = ThreadPool.new(5) + + common_dependency_a = pool.future { sleep 0.2 } + futures_a = 10.times.collect { pool.future{ common_dependency_a.value; sleep(rand() * 0.01) } } + + common_dependency_b = pool.future { futures_a.each { |f| f.value } } + futures_b = 10.times.collect { pool.future{ common_dependency_b.value; sleep(rand() * 0.01) } } + + futures_b.each{|f|f.value} + pool.join + end + + def test_pool_reports_correct_results + pool = ThreadPool.new(7) + + a = 18 + b = 5 + c = 3 + + result = a.times.collect do + pool.future do + b.times.collect do + pool.future { sleep rand * 0.001; c } + end.inject(0) { |m,f| m+f.value } + end + end.inject(0) { |m,f| m+f.value } + + assert_equal( (a*b*c), result ) + pool.join + end + +end diff --git a/test/rake/test_rake_top_level_functions.rb b/test/rake/test_rake_top_level_functions.rb index 4cc4b4cfdd..1ed1af02e3 100644 --- a/test/rake/test_rake_top_level_functions.rb +++ b/test/rake/test_rake_top_level_functions.rb @@ -23,7 +23,7 @@ class TestRakeTopLevelFunctions < Rake::TestCase def test_namespace block = proc do end - namespace("xyz", &block) + namespace("xyz", &block) expected = [ [[:in_namespace, 'xyz'], block] diff --git a/test/rake/test_thread_history_display.rb b/test/rake/test_thread_history_display.rb new file mode 100644 index 0000000000..2ac83b0dc0 --- /dev/null +++ b/test/rake/test_thread_history_display.rb @@ -0,0 +1,91 @@ +require File.expand_path('../helper', __FILE__) + +require 'rake/thread_history_display' + +class TestThreadHistoryDisplay < Rake::TestCase + def setup + super + @time = 1000000 + @stats = [] + @display = Rake::ThreadHistoryDisplay.new(@stats) + end + + def test_banner + out, _ = capture_io do + @display.show + end + assert_match(/Job History/i, out) + end + + def test_item_queued + @stats << event(:item_queued, :item_id => 123) + out, _ = capture_io do + @display.show + end + assert_match(/^ *1000000 +A +item_queued +item_id:1$/, out) + end + + def test_item_dequeued + @stats << event(:item_dequeued, :item_id => 123) + out, _ = capture_io do + @display.show + end + assert_match(/^ *1000000 +A +item_dequeued +item_id:1$/, out) + end + + def test_multiple_items + @stats << event(:item_queued, :item_id => 123) + @stats << event(:item_queued, :item_id => 124) + out, _ = capture_io do + @display.show + end + assert_match(/^ *1000000 +A +item_queued +item_id:1$/, out) + assert_match(/^ *1000001 +A +item_queued +item_id:2$/, out) + end + + def test_waiting + @stats << event(:waiting, :item_id => 123) + out, _ = capture_io do + @display.show + end + assert_match(/^ *1000000 +A +waiting +item_id:1$/, out) + end + + def test_continue + @stats << event(:continue, :item_id => 123) + out, _ = capture_io do + @display.show + end + assert_match(/^ *1000000 +A +continue +item_id:1$/, out) + end + + def test_thread_deleted + @stats << event(:thread_deleted, :deleted_thread => 123456, :thread_count => 12) + out, _ = capture_io do + @display.show + end + assert_match(/^ *1000000 +A +thread_deleted( +deleted_thread:B| +thread_count:12){2}$/, out) + end + + def test_thread_created + @stats << event(:thread_created, :new_thread => 123456, :thread_count => 13) + out, _ = capture_io do + @display.show + end + assert_match(/^ *1000000 +A +thread_created( +new_thread:B| +thread_count:13){2}$/, out) + end + + private + + def event(type, data={}) + result = { + :event => type, + :time => @time / 1_000_000.0, + :data => data, + :thread => Thread.current.object_id + } + @time += 1 + result + end + +end -- cgit v1.2.3