Skip to content

Commit 7cbb66d

Browse files
vstinnervstinner
committed
gh-115122: Add --bisect option to regrtest
* test.bisect_cmd now exit with code 0 on success, and code 1 on failure. Before, it was the opposite. * test.bisect_cmd now runs the test worker process with -X faulthandler. * regrtest RunTests: Add create_python_cmd() and bisect_cmd() methods.
1 parent 371c970 commit 7cbb66d

8 files changed

Lines changed: 157 additions & 28 deletions

File tree

‎Lib/test/bisect_cmd.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ def python_cmd():
5151
cmd = [sys.executable]
5252
cmd.extend(subprocess._args_from_interpreter_flags())
5353
cmd.extend(subprocess._optim_args_from_interpreter_flags())
54+
cmd.extend(('-X', 'faulthandler'))
5455
return cmd
5556

5657

@@ -77,9 +78,13 @@ def run_tests(args, tests, huntrleaks=None):
7778
write_tests(tmp, tests)
7879

7980
cmd = python_cmd()
80-
cmd.extend(['-m', 'test', '--matchfile', tmp])
81+
cmd.extend(['-u', '-m', 'test', '--matchfile', tmp])
8182
cmd.extend(args.test_args)
8283
print("+ %s" % format_shell_args(cmd))
84+
85+
sys.stdout.flush()
86+
sys.stderr.flush()
87+
8388
proc = subprocess.run(cmd)
8489
return proc.returncode
8590
finally:
@@ -137,8 +142,8 @@ def main():
137142
ntest = max(ntest // 2, 1)
138143
subtests = random.sample(tests, ntest)
139144

140-
print("[+] Iteration %s: run %s tests/%s"
141-
% (iteration, len(subtests), len(tests)))
145+
print(f"[+] Iteration {iteration}/{args.max_iter}: "
146+
f"run {len(subtests)} tests/{len(tests)}")
142147
print()
143148

144149
exitcode = run_tests(args, subtests)
@@ -170,10 +175,10 @@ def main():
170175
if len(tests) <= args.max_tests:
171176
print("Bisection completed in %s iterations and %s"
172177
% (iteration, datetime.timedelta(seconds=dt)))
173-
sys.exit(1)
174178
else:
175179
print("Bisection failed after %s iterations and %s"
176180
% (iteration, datetime.timedelta(seconds=dt)))
181+
sys.exit(1)
177182

178183

179184
if __name__ == "__main__":

‎Lib/test/libregrtest/cmdline.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,8 @@ def _create_parser():
347347
help='override the working directory for the test run')
348348
group.add_argument('--cleanup', action='store_true',
349349
help='remove old test_python_* directories')
350+
group.add_argument('--bisect', action='store_true',
351+
help='if some tests fail, run test.bisect_cmd on them')
350352
group.add_argument('--dont-add-python-opts', dest='_add_python_opts',
351353
action='store_false',
352354
help="internal option, don't use it")

‎Lib/test/libregrtest/main.py‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
import time
88
import trace
99

10-
from test import support
1110
from test.support import os_helper, MS_WINDOWS
1211

1312
from .cmdline import _parse_args, Namespace
@@ -73,6 +72,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False):
7372
self.want_cleanup: bool = ns.cleanup
7473
self.want_rerun: bool = ns.rerun
7574
self.want_run_leaks: bool = ns.runleaks
75+
self.want_bisect: bool = ns.bisect
7676

7777
self.ci_mode: bool = (ns.fast_ci or ns.slow_ci)
7878
self.want_add_python_opts: bool = (_add_python_opts
@@ -273,6 +273,47 @@ def rerun_failed_tests(self, runtests: RunTests):
273273

274274
self.display_result(rerun_runtests)
275275

276+
def _run_bisect(self, runtests: RunTests, test: str, progress: str) -> bool:
277+
print()
278+
title = f"Bisect {test}"
279+
if progress:
280+
title = f"{title} ({progress})"
281+
print(title)
282+
print("#" * len(title))
283+
print()
284+
285+
cmd = runtests.create_python_cmd()
286+
cmd.extend([
287+
"-u", "-m", "test.bisect_cmd",
288+
# Limit to 25 iterations (instead of 100) to not abuse CI resources
289+
"--max-iter", "25",
290+
"-v",
291+
# runtests.match_tests is not used (yet) for bisect_cmd -i arg
292+
])
293+
cmd.extend(runtests.bisect_cmd_args())
294+
cmd.append(test)
295+
print("+", shlex.join(cmd), flush=True)
296+
297+
import subprocess
298+
proc = subprocess.run(cmd, timeout=runtests.timeout)
299+
exitcode = proc.returncode
300+
if exitcode:
301+
print(f"Bisect failed with exit code {exitcode}")
302+
return False
303+
304+
return True
305+
306+
def run_bisect(self, runtests: RunTests) -> None:
307+
tests, _ = self.results.prepare_rerun(clear=False)
308+
309+
for index, name in enumerate(tests, 1):
310+
if len(tests) > 1:
311+
progress = f"{index}/{len(tests)}"
312+
else:
313+
progress = ""
314+
if not self._run_bisect(runtests, name, progress):
315+
return
316+
276317
def display_result(self, runtests):
277318
# If running the test suite for PGO then no one cares about results.
278319
if runtests.pgo:
@@ -466,7 +507,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
466507

467508
setup_process()
468509

469-
if self.hunt_refleak and not self.num_workers:
510+
if (runtests.hunt_refleak is not None) and (not self.num_workers):
470511
# gh-109739: WindowsLoadTracker thread interfers with refleak check
471512
use_load_tracker = False
472513
else:
@@ -486,6 +527,9 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
486527

487528
if self.want_rerun and self.results.need_rerun():
488529
self.rerun_failed_tests(runtests)
530+
531+
if self.want_bisect and self.results.need_rerun():
532+
self.run_bisect(runtests)
489533
finally:
490534
if use_load_tracker:
491535
self.logger.stop_load_tracker()

‎Lib/test/libregrtest/results.py‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ def get_coverage_results(self) -> trace.CoverageResults:
138138
def need_rerun(self):
139139
return bool(self.rerun_results)
140140

141-
def prepare_rerun(self) -> tuple[TestTuple, FilterDict]:
141+
def prepare_rerun(self, *, clear: bool = True) -> tuple[TestTuple, FilterDict]:
142142
tests: TestList = []
143143
match_tests_dict = {}
144144
for result in self.rerun_results:
@@ -149,11 +149,12 @@ def prepare_rerun(self) -> tuple[TestTuple, FilterDict]:
149149
if match_tests:
150150
match_tests_dict[result.test_name] = match_tests
151151

152-
# Clear previously failed tests
153-
self.rerun_bad.extend(self.bad)
154-
self.bad.clear()
155-
self.env_changed.clear()
156-
self.rerun_results.clear()
152+
if clear:
153+
# Clear previously failed tests
154+
self.rerun_bad.extend(self.bad)
155+
self.bad.clear()
156+
self.env_changed.clear()
157+
self.rerun_results.clear()
157158

158159
return (tuple(tests), match_tests_dict)
159160

‎Lib/test/libregrtest/runtests.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
import dataclasses
33
import json
44
import os
5+
import shlex
56
import subprocess
7+
import sys
68
from typing import Any
79

810
from test import support
@@ -67,6 +69,11 @@ class HuntRefleak:
6769
runs: int
6870
filename: StrPath
6971

72+
def bisect_cmd_args(self) -> list[str]:
73+
# Ignore filename since it can contain colon (":"),
74+
# and usually it's not used. Use the default filename.
75+
return ["-R", f"{self.warmups}:{self.runs}:"]
76+
7077

7178
@dataclasses.dataclass(slots=True, frozen=True)
7279
class RunTests:
@@ -137,6 +144,49 @@ def json_file_use_stdout(self) -> bool:
137144
or support.is_wasi
138145
)
139146

147+
def create_python_cmd(self) -> list[str]:
148+
python_opts = support.args_from_interpreter_flags()
149+
if self.python_cmd is not None:
150+
executable = self.python_cmd
151+
# Remove -E option, since --python=COMMAND can set PYTHON
152+
# environment variables, such as PYTHONPATH, in the worker
153+
# process.
154+
python_opts = [opt for opt in python_opts if opt != "-E"]
155+
else:
156+
executable = (sys.executable,)
157+
cmd = [*executable, *python_opts]
158+
if '-u' not in python_opts:
159+
cmd.append('-u') # Unbuffered stdout and stderr
160+
if self.coverage:
161+
cmd.append("-Xpresite=test.cov")
162+
return cmd
163+
164+
def bisect_cmd_args(self) -> list[str]:
165+
args = []
166+
if self.fail_fast:
167+
args.append("--failfast")
168+
if self.fail_env_changed:
169+
args.append("--fail-env-changed")
170+
if self.timeout:
171+
args.append(f"--timeout={self.timeout}")
172+
if self.hunt_refleak is not None:
173+
args.extend(self.hunt_refleak.bisect_cmd_args())
174+
if self.test_dir:
175+
args.extend(("--testdir", self.test_dir))
176+
if self.memory_limit:
177+
args.extend(("--memlimit", self.memory_limit))
178+
if self.gc_threshold:
179+
args.append(f"--threshold={self.gc_threshold}")
180+
if self.use_resources:
181+
args.extend(("-u", ','.join(self.use_resources)))
182+
if self.python_cmd:
183+
cmd = shlex.join(self.python_cmd)
184+
args.extend(("--python", cmd))
185+
if self.randomize:
186+
args.append(f"--randomize")
187+
args.append(f"--randseed={self.random_seed}")
188+
return args
189+
140190

141191
@dataclasses.dataclass(slots=True, frozen=True)
142192
class WorkerRunTests(RunTests):

‎Lib/test/libregrtest/worker.py‎

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import os
44
from typing import Any, NoReturn
55

6-
from test import support
76
from test.support import os_helper, Py_DEBUG
87

98
from .setup import setup_process, setup_test_dir
@@ -19,23 +18,10 @@
1918

2019
def create_worker_process(runtests: WorkerRunTests, output_fd: int,
2120
tmp_dir: StrPath | None = None) -> subprocess.Popen:
22-
python_cmd = runtests.python_cmd
2321
worker_json = runtests.as_json()
2422

25-
python_opts = support.args_from_interpreter_flags()
26-
if python_cmd is not None:
27-
executable = python_cmd
28-
# Remove -E option, since --python=COMMAND can set PYTHON environment
29-
# variables, such as PYTHONPATH, in the worker process.
30-
python_opts = [opt for opt in python_opts if opt != "-E"]
31-
else:
32-
executable = (sys.executable,)
33-
if runtests.coverage:
34-
python_opts.append("-Xpresite=test.cov")
35-
cmd = [*executable, *python_opts,
36-
'-u', # Unbuffered stdout and stderr
37-
'-m', 'test.libregrtest.worker',
38-
worker_json]
23+
cmd = runtests.create_python_cmd()
24+
cmd.extend(['-m', 'test.libregrtest.worker', worker_json])
3925

4026
env = dict(os.environ)
4127
if tmp_dir is not None:

‎Lib/test/test_regrtest.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -419,6 +419,7 @@ def check_ci_mode(self, args, use_resources, rerun=True):
419419
self.assertTrue(regrtest.print_slowest)
420420
self.assertTrue(regrtest.output_on_failure)
421421
self.assertEqual(sorted(regrtest.use_resources), sorted(use_resources))
422+
self.assertTrue(regrtest.want_bisect)
422423
return regrtest
423424

424425
def test_fast_ci(self):
@@ -1192,6 +1193,44 @@ def test_huntrleaks(self):
11921193
def test_huntrleaks_mp(self):
11931194
self.check_huntrleaks(run_workers=True)
11941195

1196+
@unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
1197+
def test_huntrleaks_bisect(self):
1198+
# test --huntrleaks --bisect
1199+
code = textwrap.dedent("""
1200+
import unittest
1201+
1202+
GLOBAL_LIST = []
1203+
1204+
class RefLeakTest(unittest.TestCase):
1205+
def test1(self):
1206+
pass
1207+
1208+
def test2(self):
1209+
pass
1210+
1211+
def test3(self):
1212+
GLOBAL_LIST.append(object())
1213+
1214+
def test4(self):
1215+
pass
1216+
""")
1217+
1218+
test = self.create_test('huntrleaks', code=code)
1219+
1220+
filename = 'reflog.txt'
1221+
self.addCleanup(os_helper.unlink, filename)
1222+
cmd = ['--huntrleaks', '3:3:', '--bisect', test]
1223+
output = self.run_tests(*cmd,
1224+
exitcode=EXITCODE_BAD_TEST,
1225+
stderr=subprocess.STDOUT)
1226+
1227+
# test3 is the one which leaks
1228+
self.assertIn("Bisection completed in", output)
1229+
self.assertIn(
1230+
"Tests (1):\n"
1231+
f"* {test}.RefLeakTest.test3\n",
1232+
output)
1233+
11951234
@unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
11961235
def test_huntrleaks_fd_leak(self):
11971236
# test --huntrleaks for file descriptor leak

‎Misc/NEWS.d/next/Tests/2024-02-18-14-20-52.gh-issue-115122.3rGNo9.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Add ``--bisect`` option to regrtest test runner: run failed tests with
2+
``test.bisect_cmd`` to identify failing tests. Patch by Victor Stinner.

0 commit comments

Comments
 (0)