Manage parallel test workers after timeout

5 files changed, 39 insertions, 13 deletions

diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb
index 23dfbd8f0f..87e05ea62e 100644
--- a/tool/lib/test/unit.rb
+++ b/tool/lib/test/unit.rb
@@ -267,7 +267,7 @@ module Test
             r.close_on_exec = true
             w.close_on_exec = true
             @jobserver = [r, w]
-            options[:parallel] ||= 1
+            options[:parallel] ||= 256 # number of tokens to acquire first
           end
         end
         @worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 180)
@@ -661,10 +661,18 @@ module Test
         @ios          = [] # Array of worker IOs
         @job_tokens   = String.new(encoding: Encoding::ASCII_8BIT) if @jobserver
         begin
-          [@tasks.size, @options[:parallel]].min.times {launch_worker}
-
           while true
-            timeout = [(@workers.filter_map {|w| w.response_at}.min&.-(Time.now) || 0) + @worker_timeout, 1].max
+            newjobs = [@tasks.size, @options[:parallel]].min - @workers.size
+            if newjobs > 0
+              if @jobserver
+                t = @jobserver[0].read_nonblock(newjobs, exception: false)
+                @job_tokens << t if String === t
+                newjobs = @job_tokens.size + 1 - @workers.size
+              end
+              newjobs.times {launch_worker}
+            end
+
+            timeout = [(@workers.filter_map {|w| w.response_at}.min&.-(Time.now) || 0), 0].max + @worker_timeout
 
             if !(_io = IO.select(@ios, nil, nil, timeout))
               timeout = Time.now - @worker_timeout
@@ -678,15 +686,9 @@ module Test
             }
               break
             end
-            break if @tasks.empty? and @workers.empty?
-            if @jobserver and @job_tokens and !@tasks.empty? and
-               ((newjobs = [@tasks.size, @options[:parallel]].min) > @workers.size or
-                !@workers.any? {|x| x.status == :ready})
-              t = @jobserver[0].read_nonblock(newjobs, exception: false)
-              if String === t
-                @job_tokens << t
-                t.size.times {launch_worker}
-              end
+            if @tasks.empty?
+              break if @workers.empty?
+              next # wait for all workers to finish
             end
           end
         rescue Interrupt => ex
diff --git a/tool/test/testunit/test_parallel.rb b/tool/test/testunit/test_parallel.rb
index 1b64afdffe..454c26a6ac 100644
--- a/tool/test/testunit/test_parallel.rb
+++ b/tool/test/testunit/test_parallel.rb
@@ -211,5 +211,12 @@ module TestParallel
       assert_match(/^Retrying hung up testcases\.+$/, buf)
       assert_match(/^2 tests,.* 0 failures,/, buf)
     end
+
+    def test_retry_workers
+      spawn_runner "--worker-timeout=1", "test4test_slow_0.rb", "test4test_slow_1.rb", jobs: "2"
+      buf = Timeout.timeout(TIMEOUT) {@test_out.read}
+      assert_match(/^Retrying hung up testcases\.+$/, buf)
+      assert_match(/^2 tests,.* 0 failures,/, buf)
+    end
   end
 end
diff --git a/tool/test/testunit/tests_for_parallel/slow_helper.rb b/tool/test/testunit/tests_for_parallel/slow_helper.rb
new file mode 100644
index 0000000000..d8372730a8
--- /dev/null
+++ b/tool/test/testunit/tests_for_parallel/slow_helper.rb
@@ -0,0 +1,7 @@
+require 'test/unit'
+
+module TestSlowTimeout
+  def test_slow
+    sleep (ENV['sec'] || 3).to_i if on_parallel_worker?
+  end
+end
diff --git a/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb b/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb
new file mode 100644
index 0000000000..a749b0e1d3
--- /dev/null
+++ b/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb
@@ -0,0 +1,5 @@
+require_relative 'slow_helper'
+
+class TestSlowV0 < Test::Unit::TestCase
+  include TestSlowTimeout
+end
diff --git a/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb b/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb
new file mode 100644
index 0000000000..924a3b11fa
--- /dev/null
+++ b/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb
@@ -0,0 +1,5 @@
+require_relative 'slow_helper'
+
+class TestSlowV1 < Test::Unit::TestCase
+  include TestSlowTimeout
+end