diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 2e8397a8a91d32..032cb94223a7e6 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -15,6 +15,7 @@ from .findtests import findtests, split_test_packages, list_cases, collect_cases from .logger import Logger from .pgo import setup_pgo_tests +from .reporter import get_reporter from .result import TestResult from .results import TestResults, EXITCODE_INTERRUPTED from .runtests import RunTests, HuntRefleak @@ -68,6 +69,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): # Logger self.logger = Logger(self.results, self.quiet, self.pgo) + self.reporter = get_reporter() # Actions self.want_header: bool = ns.header @@ -284,8 +286,10 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: forever=False, fail_fast=False, match_tests_dict=match_tests_dict, - output_on_failure=False) + output_on_failure=False, + final_run=True) self.logger.set_tests(runtests) + self.reporter.run_started(runtests) msg = f"Re-running {len(tests)} failed tests in verbose mode" if not self.single_process: @@ -395,6 +399,7 @@ def run_test( result = run_single_test(test_name, runtests) self.results.accumulate_result(result, runtests) + self.reporter.test_result(result) return result @@ -459,7 +464,8 @@ def get_state(self) -> str: def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: from .run_workers import RunWorkers - RunWorkers(num_workers, runtests, self.logger, self.results).run() + RunWorkers(num_workers, runtests, self.logger, self.results, + self.reporter).run() def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.next_single_filename: @@ -480,6 +486,8 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.junit_filename: self.results.write_junit(self.junit_filename) + self.reporter.summary(self.results, self.get_state()) + def display_summary(self) -> None: if self.first_runtests is None: raise ValueError( @@ -517,6 +525,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), + final_run=not self.want_rerun or bool(self.python_cmd), coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, @@ -568,6 +577,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: runtests = runtests.copy(tests=case_ids, case_groups=case_groups) self.first_runtests = runtests self.logger.set_tests(runtests) + self.reporter.run_started(runtests) if (runtests.hunt_refleak is not None) and (not self.num_workers): # gh-109739: WindowsLoadTracker thread interferes with refleak check diff --git a/Lib/test/libregrtest/reporter.py b/Lib/test/libregrtest/reporter.py new file mode 100644 index 00000000000000..8b5bf7062b5b87 --- /dev/null +++ b/Lib/test/libregrtest/reporter.py @@ -0,0 +1,137 @@ +import html +import os +import pathlib +import re +from _colorize import decolor # type: ignore[import-not-found] +from typing import TextIO + +from .findtests import findtestdir +from .result import TestResult +from .results import TestResults +from .runtests import RunTests +from .utils import count + + +class Reporter: + """Report test failures, for example to a CI service. A no-op here.""" + + def run_started(self, runtests: RunTests) -> None: + """Called at the start of each run, and in the test process.""" + + def test_case_failed(self, test_case: str, traceback: str, + stream: TextIO) -> None: + """Called in the test process after a test case failed.""" + + def test_result(self, result: TestResult) -> None: + """Called in the main process after a test file completed.""" + + def summary(self, results: TestResults, state: str) -> None: + """Called in the main process after all runs.""" + + @staticmethod + def parse_traceback(traceback: str) -> tuple[str | None, str | None, str]: + """Return (filename, line, exception) of a failure.""" + # A frame is its "File" line and its source lines + frames = list(re.finditer( + r'^ *File "(.*)", line (\d+).*\n(?: .*\n)*', traceback, + re.MULTILINE)) + filename = line = None + exception = traceback + if frames: + filename, line = frames[0].groups() + exception = traceback[frames[-1].end():] + # Prefer the last frame in a test file, the assertion + for frame in frames: + if pathlib.Path(frame[1]).name.startswith("test_"): + filename, line = frame.groups() + return filename, line, exception.strip() + + +def get_reporter() -> Reporter: + if GitHubActionsReporter.enabled(): + return GitHubActionsReporter() + return Reporter() + + +class GitHubActionsReporter(Reporter): + """Annotate failures in the job log and list them in the job summary.""" + + ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"}) + ESCAPE_PROPERTY = ESCAPE_DATA | str.maketrans({":": "%3A", ",": "%2C"}) + + @staticmethod + def enabled() -> bool: + return "GITHUB_STEP_SUMMARY" in os.environ + + def run_started(self, runtests: RunTests) -> None: + self.runtests = runtests + + def test_case_failed(self, test_case: str, traceback: str, + stream: TextIO) -> None: + if not self.runtests.final_run: + return + filename, line, exception = self.parse_traceback(decolor(traceback)) + self.annotate(test_case, exception, filename, line, stream) + + def test_result(self, result: TestResult) -> None: + if not self.runtests.final_run: + return + if not result.is_failed(self.runtests.fail_env_changed): + return + # Already annotated by test_case_failed() + if result.failed_test_cases: + return + # "test.test_x.test_y" is test_x/test_y.py or test_x/test_y/__init__.py + name = result.test_name.removeprefix("test.") + path = pathlib.Path(findtestdir(self.runtests.test_dir), + *name.split(".")) + filename = None + for candidate in (path.with_suffix(".py"), path / "__init__.py"): + if candidate.exists(): + filename = str(candidate) + message = "\n".join([str(result), *(result.env_changed_reasons or ())]) + self.annotate(result.test_name, decolor(message), filename) + + def annotate(self, title: str, message: str, filename: str | None, + line: str | None = None, + stream: TextIO | None = None) -> None: + if filename: + # Make installed stdlib paths relative to the checkout + path = pathlib.Path(filename) + stdlib_dir = pathlib.Path(os.__file__).parent + if path.is_relative_to(stdlib_dir): + filename = ("Lib" / path.relative_to(stdlib_dir)).as_posix() + # GitHub needs a line to show the annotation on the file + line = line or "1" + props = {"file": filename, "line": line, "title": title} + props_text = ",".join(f"{key}={value.translate(self.ESCAPE_PROPERTY)}" + for key, value in props.items() if value) + print(f"::error {props_text}::{message.translate(self.ESCAPE_DATA)}", + file=stream, flush=True) + + def summary(self, results: TestResults, state: str) -> None: + failed = [result for result in results.rerun_results + if result.is_failed(self.runtests.fail_env_changed)] + if not failed: + return + ncases = sum(len(result.failed_test_cases) for result in failed) + blocks = [f"## {decolor(state)}: " + f"{count(len(failed), 'test file')} and " + f"{count(ncases, 'test case')} failed"] + for result in failed: + blocks.append(f"### {decolor(str(result))}") + if result.env_changed_reasons: + blocks.append("\n".join( + f"- {reason}" for reason in result.env_changed_reasons)) + for name, traceback in result.failed_test_cases: + # Expand short tracebacks if there are only a few + if ncases <= 5 and traceback.count("\n") < 30: + details = "
" + else: + details = "
" + blocks += [f"{details}{html.escape(name)}", + f"```pytb\n{decolor(traceback).rstrip()}\n```", + "
"] + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", + encoding="utf-8") as fp: + fp.write("".join(f"{block}\n\n" for block in blocks)) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 5324a46bb8cb1c..f8aa256be1ecfe 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -178,6 +178,10 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) + @property + def failed_test_cases(self) -> list[tuple[str, str]]: + return (self.errors or []) + (self.failures or []) + def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: self.state = State.ENV_CHANGED diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index c6db78e0882ba3..ad02d8dba7c23e 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -16,6 +16,7 @@ from test.support import os_helper, MS_WINDOWS from .logger import Logger +from .reporter import Reporter from .result import TestResult, State from .results import TestResults from .runtests import RunTests, WorkerRunTests, JsonFile, JsonFileType @@ -499,13 +500,15 @@ def get_running(workers: list[WorkerThread]) -> str | None: class RunWorkers: def __init__(self, num_workers: int, runtests: RunTests, - logger: Logger, results: TestResults) -> None: + logger: Logger, results: TestResults, + reporter: Reporter) -> None: self.num_workers = num_workers self.runtests = runtests self.logger = logger self.log = logger.log self.display_progress = logger.display_progress self.results: TestResults = results + self.reporter = reporter self.live_worker_count = 0 self.output: queue.Queue[QueueContent] = queue.Queue() @@ -624,6 +627,8 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) + # Report after the output: env changed warnings, crash traceback + self.reporter.test_result(result) return result diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index d86b082fb34a60..525f9573ad992d 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,6 +93,8 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool + # Failed tests are not re-run after this run + final_run: bool coverage: bool memory_limit: str | None gc_threshold: int | None diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index d62194acd9c29e..75a596c1cccc52 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -11,7 +11,9 @@ from _colorize import can_colorize # type: ignore[import-not-found] from .filter import set_match_tests +from .reporter import get_reporter from .runtests import RunTests +from .testresult import RegressionTestResult from .utils import ( setup_unraisable_hook, setup_threading_excepthook, adjust_rlimit_nofile) @@ -112,9 +114,11 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) + reporter = get_reporter() + reporter.run_started(runtests) + RegressionTestResult.reporter = reporter if runtests.use_junit: support.junit_xml_list = [] - from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 605f1f4e6a89fb..3e4243282236cf 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,10 +9,12 @@ import traceback import unittest from test import support +from test.libregrtest.reporter import Reporter from test.libregrtest.utils import sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False + reporter = Reporter() def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -130,6 +132,14 @@ def addUnexpectedSuccess(self, test): self._add_result(test, outcome='UNEXPECTED_SUCCESS') super().addUnexpectedSuccess(test) + def printErrorList(self, flavour, errors): + for test, err in errors: + super().printErrorList(flavour, [(test, err)]) + self.report_failure(test, err) + + def report_failure(self, test, err): + self.reporter.test_case_failed(str(test), err, self.stream) + def get_xml_element(self): if not self.USE_XML: raise ValueError("USE_XML is false") @@ -146,6 +156,9 @@ def __init__(self, stream, buffer=False): def run(self, test): test(self.result) + # Failures are not printed in quiet mode + for test, err in self.result.errors + self.result.failures: + self.result.report_failure(test, err) return self.result def get_test_runner_class(verbosity, buffer=False): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 53f0a2863dc4e4..adb9b72deab984 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -22,6 +22,7 @@ import tempfile import threading import textwrap +import types import unittest import unittest.mock from xml.etree import ElementTree @@ -37,7 +38,9 @@ from test.libregrtest import setup from test.libregrtest import utils from test.libregrtest.filter import get_match_tests, match_test -from test.libregrtest.result import TestStats +from test.libregrtest.reporter import GitHubActionsReporter +from test.libregrtest.result import State, TestResult, TestStats +from test.libregrtest.results import TestResults from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: @@ -628,6 +631,10 @@ def setUp(self): self.tmptestdir = tempfile.mkdtemp() self.addCleanup(os_helper.rmtree, self.tmptestdir) + # Don't report the failures of the tests to the CI running this test + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env.unset('GITHUB_STEP_SUMMARY') + def create_test(self, name=None, code=None): if not name: name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID @@ -1639,6 +1646,35 @@ def test_env_changed(self): success=True), stats=2) + def test_reporter(self): + code = textwrap.dedent(""" + import os, unittest + + class Tests(unittest.TestCase): + def test_fail(self): + self.fail() + """) + fail = self.create_test('reporter_fail', code) + crash = self.create_test('reporter_crash', + code.replace('self.fail()', 'os._exit(1)')) + summary = os.path.join(self.tmptestdir, 'summary.md') + env = dict(os.environ, GITHUB_STEP_SUMMARY=summary) + + def reported(*args, exitcode=EXITCODE_BAD_TEST): + output = self.run_tests('-j1', *args, env=env, exitcode=exitcode) + return re.findall(r'^::error file=.*,title=(\w+)', output, + re.MULTILINE) + + self.assertEqual(reported(self.create_test(), exitcode=0), []) + self.assertFalse(os.path.exists(summary)) + # Quiet mode doesn't print the failure, but reports it + self.assertEqual(reported(fail), ['test_fail']) + # Only the re-run is reported + self.assertEqual(reported('--rerun', fail, crash), + ['test_fail', crash]) + with open(summary, encoding='utf-8') as fp: + self.assertIn('2 test files and 1 test case failed', fp.read()) + def test_rerun_fail(self): # FAILURE then FAILURE code = textwrap.dedent(""" @@ -2946,7 +2982,73 @@ def test_sanitize_xml(self): 'valid t\xe9xt \u20ac') -from test.libregrtest.results import TestResults +class GitHubActionsReporterTestCase(unittest.TestCase): + def setUp(self): + self.reporter = GitHubActionsReporter() + self.reporter.run_started(types.SimpleNamespace( + final_run=True, fail_env_changed=True, test_dir=None)) + + def test_test_case_failed(self): + stdlib_dir = os.path.dirname(os.__file__) + traceback = textwrap.dedent(f"""\ + Traceback (most recent call last): + File "{stdlib_dir}/test/test_x.py", line 3, in test_x + helper() + File "{stdlib_dir}/helper.py", line 9, in helper + raise ValueError('50%') + ValueError: 50% + more + """) + stream = io.StringIO() + self.reporter.test_case_failed('test_x (a: b, c)', traceback, stream) + self.assertEqual( + stream.getvalue(), + '::error file=Lib/test/test_x.py,line=3,' + 'title=test_x (a%3A b%2C c)::ValueError: 50%25%0Amore\n') + + def test_test_result(self): + result = TestResult('test_x', state=State.ENV_CHANGED, + env_changed_reasons=['dangling thread']) + with support.captured_stdout() as stdout: + self.reporter.test_result(result) + self.assertEqual(stdout.getvalue(), + '::error title=test_x::' + 'test_x failed (env changed)%0Adangling thread\n') + + def test_summary(self): + results = TestResults() + results.rerun_results = [ + TestResult('test_x', state=State.FAILED, + failures=[('test_a ()', + 'Traceback\nAssertionError\n')]), + TestResult('test_y', state=State.ENV_CHANGED, + env_changed_reasons=['dangling thread']), + ] + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with os_helper.EnvironmentVarGuard() as env: + env['GITHUB_STEP_SUMMARY'] = os_helper.TESTFN + self.reporter.summary(results, 'FAILURE') + with open(os_helper.TESTFN, encoding='utf-8') as fp: + summary = fp.read() + self.assertEqual(summary, textwrap.dedent("""\ + ## FAILURE: 2 test files and 1 test case failed + + ### test_x failed (1 failure) + +
test_a (<T>) + + ```pytb + Traceback + AssertionError + ``` + +
+ + ### test_y failed (env changed) + + - dangling thread + + """)) class TestColorized(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst new file mode 100644 index 00000000000000..a1c9e648e5c744 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst @@ -0,0 +1,3 @@ +When run in GitHub Actions, :mod:`test.regrtest` now annotates each test +failure in the job log and writes a summary of the failures to the job +summary.