From ca8cc20817f27ddcdfa1a0621e9f5d6e7f9af67f Mon Sep 17 00:00:00 2001 From: Kilian Lieret Date: Tue, 23 Jun 2026 15:09:10 +0000 Subject: [PATCH] Data(eval): sync dependency_ignored test updates from internal reference Port the latest test-ignore update from the internal reference harness into ProgramBench across 4 instances (abishekvashok__cmatrix, antonmedv__fx, antonmedv__walk, nuta__nsh), adding 9 `dependency_ignored` entries total and keeping all ignore metadata verbatim. These mark kept tests that pytest skips because they `@pytest.mark.dependency(depends=[X])` on an already-ignored test X. Branch-hash keys and per-branch test name-sets are unchanged, so no test blob re-upload is needed; only ignore metadata differs. Internal-reference-commit: a440a8c57c7166d11d32f21afb1069ae758057b4 --- CLAUDE.md | 4 +++ .../abishekvashok__cmatrix.5c082c6/tests.json | 15 +++++++-- .../tasks/antonmedv__fx.86d0d34/tests.json | 33 +++++++++++++++++++ .../tasks/antonmedv__walk.bf802ef/tests.json | 22 +++++++++++++ .../data/tasks/nuta__nsh.bdd0702/tests.json | 33 +++++++++++++++++++ 5 files changed, 105 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 23a9fed..4509f11 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,6 +22,10 @@ scoring regardless of reason; the id is informational. distinguish a real implementation from a stub. - `outcome_dependent_presence` — test appears in some eval runs but not others. - `slow_or_hang` — test hangs mid-call or exceeds a duration threshold. +- `dependency_ignored` — a kept test that `@pytest.mark.dependency(depends=[X])` on an + *ignored* test `X`. Because `X` is deselected at collection, `pytest-dependency` skips + the dependent, so it would count as unresolved through no fault of the submission. The + reason `note` records the prerequisite (`depends on ignored test `). - `ignored_manual` — manually excluded. ## Quick reference diff --git a/src/programbench/data/tasks/abishekvashok__cmatrix.5c082c6/tests.json b/src/programbench/data/tasks/abishekvashok__cmatrix.5c082c6/tests.json index 7b34639..108f99e 100644 --- a/src/programbench/data/tasks/abishekvashok__cmatrix.5c082c6/tests.json +++ b/src/programbench/data/tasks/abishekvashok__cmatrix.5c082c6/tests.json @@ -729,8 +729,8 @@ "name": "eval.tests.test_args_parsing.test_u_out_of_range_or_overflow_values_still_enter_render_loop[args2]", "reasons": [ { - "id": "ignored_manual", - "description": "This test passes -u 999999999999999999999999, which overflows C's 32-bit int via atoi(), wrapping to a small/negative value so the original binary enters the render loop. This is undefined behavior in C, not intentional design. A correct reimplementation in a language without integer overflow (e.g. Python) would reasonably reject or hang on this value. The test enshrines accidental C overflow semantics rather than meaningful program behavior." + "description": "This test passes -u 999999999999999999999999, which overflows C's 32-bit int via atoi(), wrapping to a small/negative value so the original binary enters the render loop. This is undefined behavior in C, not intentional design. A correct reimplementation in a language without integer overflow (e.g. Python) would reasonably reject or hang on this value. The test enshrines accidental C overflow semantics rather than meaningful program behavior.", + "id": "ignored_manual" } ] } @@ -3202,6 +3202,17 @@ } ] }, + { + "name": "eval.tests.test_tui_cmatrix.test_screen_changes_over_time", + "reasons": [ + { + "id": "dependency_ignored", + "note": "depends on ignored test test_launch_draws_nonempty_screen", + "timestamp": 1781756185, + "user": "kilian" + } + ] + }, { "name": "eval.tests.test_tui_cmatrix.test_unbound_key_does_not_crash_in_normal_mode", "reasons": [ diff --git a/src/programbench/data/tasks/antonmedv__fx.86d0d34/tests.json b/src/programbench/data/tasks/antonmedv__fx.86d0d34/tests.json index 0c0a737..ec143bb 100644 --- a/src/programbench/data/tasks/antonmedv__fx.86d0d34/tests.json +++ b/src/programbench/data/tasks/antonmedv__fx.86d0d34/tests.json @@ -13178,6 +13178,17 @@ "eval.tests.test_tui_search.test_search_prev_with_N" ], "ignored_tests": [ + { + "name": "eval.tests.test_tui_actions.test_command_line_cancel", + "reasons": [ + { + "id": "dependency_ignored", + "note": "depends on ignored test test_command_line_with_colon", + "timestamp": 1781756196, + "user": "kilian" + } + ] + }, { "name": "eval.tests.test_tui_actions.test_command_line_with_colon", "reasons": [ @@ -13242,6 +13253,17 @@ } ] }, + { + "name": "eval.tests.test_tui_search.test_search_cancel_with_esc", + "reasons": [ + { + "id": "dependency_ignored", + "note": "depends on ignored test test_search_opens_with_slash", + "timestamp": 1781756196, + "user": "kilian" + } + ] + }, { "name": "eval.tests.test_tui_search.test_search_case_sensitive", "reasons": [ @@ -13274,6 +13296,17 @@ } ] }, + { + "name": "eval.tests.test_tui_search.test_search_no_results", + "reasons": [ + { + "id": "dependency_ignored", + "note": "depends on ignored test test_search_opens_with_slash", + "timestamp": 1781756196, + "user": "kilian" + } + ] + }, { "name": "eval.tests.test_tui_search.test_search_opens_with_slash", "reasons": [ diff --git a/src/programbench/data/tasks/antonmedv__walk.bf802ef/tests.json b/src/programbench/data/tasks/antonmedv__walk.bf802ef/tests.json index 4a614d8..f028c37 100644 --- a/src/programbench/data/tasks/antonmedv__walk.bf802ef/tests.json +++ b/src/programbench/data/tasks/antonmedv__walk.bf802ef/tests.json @@ -3821,6 +3821,28 @@ } ] }, + { + "name": "eval.tests.test_tui.test_tui_shows_directories", + "reasons": [ + { + "id": "dependency_ignored", + "note": "depends on ignored test test_tui_launches", + "timestamp": 1781756196, + "user": "kilian" + } + ] + }, + { + "name": "eval.tests.test_tui.test_tui_shows_files", + "reasons": [ + { + "id": "dependency_ignored", + "note": "depends on ignored test test_tui_launches", + "timestamp": 1781756196, + "user": "kilian" + } + ] + }, { "name": "eval.tests.test_tui.test_yank_directory", "reasons": [ diff --git a/src/programbench/data/tasks/nuta__nsh.bdd0702/tests.json b/src/programbench/data/tasks/nuta__nsh.bdd0702/tests.json index b9fe5cd..be13954 100644 --- a/src/programbench/data/tasks/nuta__nsh.bdd0702/tests.json +++ b/src/programbench/data/tasks/nuta__nsh.bdd0702/tests.json @@ -5337,6 +5337,17 @@ } ] }, + { + "name": "eval.tests.test_history_mode.test_history_mode_ctrl_c_exits", + "reasons": [ + { + "id": "dependency_ignored", + "note": "depends on ignored test test_history_mode_activates", + "timestamp": 1781756408, + "user": "kilian" + } + ] + }, { "name": "eval.tests.test_history_mode.test_history_mode_empty_search", "reasons": [ @@ -5407,6 +5418,28 @@ } ] }, + { + "name": "eval.tests.test_history_mode.test_history_mode_shows_commands", + "reasons": [ + { + "id": "dependency_ignored", + "note": "depends on ignored test test_history_mode_activates", + "timestamp": 1781756408, + "user": "kilian" + } + ] + }, + { + "name": "eval.tests.test_history_mode.test_history_mode_shows_help_text", + "reasons": [ + { + "id": "dependency_ignored", + "note": "depends on ignored test test_history_mode_activates", + "timestamp": 1781756408, + "user": "kilian" + } + ] + }, { "name": "eval.tests.test_history_mode.test_history_mode_tab_edits", "reasons": [