From d65b4d6a04c088c16a80656bf705d9ce5da730b8 Mon Sep 17 00:00:00 2001
From: Ryan Hoffman <ryan.hoffman@shopify.com>
Date: Wed, 14 Oct 2026 11:39:32 -0400
Subject: [PATCH] Don't crash retried lazy-load workers when the queue is
 almost empty

When a retried worker finds no more tests left in the queue than are
running, `minitest-queue run` skips loading tests and populating the
queue, expecting the queue to be empty by the time the application
boots. Without lazy loading, minitest runs from minitest/autorun's
at_exit hook, which loading the test files registers. No test files
load, so the worker exits 0.

In lazy-load mode the runner calls `Minitest.run` itself, and only skips
it when the queue is exhausted, which it isn't while other workers still
hold tests. The worker then polls a queue that can't resolve entries:
the first test it reserves comes back as an `UnresolvedEntry`, and
`example.run` raises NoMethodError. The worker records a worker error
and exits 42, so `report` prints "Worker N crashed" and exits 42 even
when every test passed.

In lazy-load mode, also exit 0 without running tests when the queue
wasn't prepared.
---
 ruby/lib/minitest/queue/runner.rb            | 12 +++++-
 ruby/test/integration/minitest_redis_test.rb | 44 ++++++++++++++++++++
 2 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/ruby/lib/minitest/queue/runner.rb b/ruby/lib/minitest/queue/runner.rb
index de39a04e..6fd45cdb 100644
--- a/ruby/lib/minitest/queue/runner.rb
+++ b/ruby/lib/minitest/queue/runner.rb
@@ -103,6 +103,7 @@ def run_command
         trap('TERM') { Minitest.queue.shutdown! }
         trap('INT') { Minitest.queue.shutdown! }
 
+        queue_prepared = false
         if queue.rescue_connection_errors { queue.exhausted? }
           puts green('All tests were ran already')
         else
@@ -117,9 +118,11 @@ def run_command
               puts green("Queue almost empty, exiting early...")
             else
               prepare_queue_for_execution
+              queue_prepared = true
             end
           else
             prepare_queue_for_execution
+            queue_prepared = true
           end
         end
 
@@ -132,8 +135,13 @@ def run_command
           # Re-check exhausted? after booting: slow workers may arrive after the queue
           # has been fully drained by faster workers. In that case exit cleanly (0)
           # rather than letting Minitest return false for a 0-test run.
-          if queue.rescue_connection_errors { queue.exhausted? }
-            puts green('All tests were ran already')
+          #
+          # Also exit when the queue wasn't prepared above, because it was exhausted
+          # or a retried worker found it almost empty. No tests were loaded and the
+          # queue can't resolve entries, so the first test this worker reserved would
+          # crash it.
+          if !queue_prepared || queue.rescue_connection_errors { queue.exhausted? }
+            puts green('All tests were ran already') if queue_prepared
             verify_reporters!(reporters)
             exit!(0)
           end
diff --git a/ruby/test/integration/minitest_redis_test.rb b/ruby/test/integration/minitest_redis_test.rb
index 974e4745..fb22341c 100644
--- a/ruby/test/integration/minitest_redis_test.rb
+++ b/ruby/test/integration/minitest_redis_test.rb
@@ -990,6 +990,50 @@ def test_automatic_retry
       assert_equal 'All tests were ran already', output
     end
 
+    def test_lazy_load_retry_exits_without_reserving_tests_when_queue_is_almost_empty
+      build_id = 'lazy-load-retry-queue-almost-empty'
+      test_file = File.expand_path('../../fixtures/test/passing_test.rb', __FILE__)
+      entries = 3.times.map { |i| CI::Queue::QueueEntry.format("PassingTest#test_passing_#{i}", test_file) }
+      busy_worker = CI::Queue::Redis.new(
+        @redis_url,
+        CI::Queue::Configuration.new(build_id: build_id, worker_id: '0', timeout: 5),
+      )
+      capture_io { busy_worker.stream_populate(entries) }
+      reserved_count = 0
+      busy_worker.poll do
+        reserved_count += 1
+        break if reserved_count == 2
+      end
+
+      assert_equal 1, @redis.llen("build:#{build_id}:queue")
+      assert_equal 2, @redis.zcard("build:#{build_id}:running")
+
+      out, err = capture_subprocess_io do
+        system(
+          { 'BUILDKITE_RETRY_COUNT' => '1', 'BUILDKITE_RETRY_TYPE' => 'automatic' },
+          @exe, 'run',
+          '--queue', @redis_url,
+          '--seed', 'foobar',
+          '--build', build_id,
+          '--worker', '1',
+          '--timeout', '5',
+          '--lazy-load',
+          '-Itest',
+          'test/passing_test.rb',
+          chdir: 'test/fixtures/',
+        )
+      end
+
+      assert_predicate $?, :success?
+      assert_empty filter_deprecation_warnings(err)
+      assert_equal <<~OUTPUT, normalize(out)
+        1 tests left and 2 workers running.
+        Queue almost empty, exiting early...
+      OUTPUT
+      assert_equal 1, @redis.llen("build:#{build_id}:queue")
+      assert_empty @redis.hgetall("build:#{build_id}:worker-errors")
+    end
+
     def test_automatic_retry_reruns_failed_tests_so_report_can_succeed
       # Simulate the scenario from the Sam O'Brien / Steph Sachrajda incident:
       # A test fails on the first run (flaky infra timeout). Buildkite auto-retries