From 0a776c2403bda2aff64301abebe9004e4d3d2511 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Sun, 19 Jul 2026 19:55:57 +0800 Subject: [PATCH] Harden benchmark submission control flow --- .../skills/submit-benchmark-result/SKILL.md | 60 +++++++-------- .../scripts/trigger-scoring.sh | 75 +++++++++++++++++++ benchmark/tests/test_docs.py | 75 ++++++++++++++++--- 3 files changed, 168 insertions(+), 42 deletions(-) create mode 100755 .agents/skills/submit-benchmark-result/scripts/trigger-scoring.sh diff --git a/.agents/skills/submit-benchmark-result/SKILL.md b/.agents/skills/submit-benchmark-result/SKILL.md index f4ee906..2689e90 100644 --- a/.agents/skills/submit-benchmark-result/SKILL.md +++ b/.agents/skills/submit-benchmark-result/SKILL.md @@ -33,14 +33,18 @@ Do not submit a result containing `run_error`. Report the error and stop. ## 2. Confirm the upload -Immediately before uploading, show: +Immediately before uploading, show this confirmation with the real values filled in: -- endpoint hostname; -- absolute file path; -- model and claimed bug count; +> Ready to officially submit this benchmark result: +> - Destination: `intake.prb-bench.workers.dev` +> - File: `` +> - Model: `` +> - Claimed bugs: `` +> +> This private submission file will leave the machine and be uploaded to the benchmark's +> private storage. Confirm submission? -State that this is an official submission and that the private file will leave the machine, -then obtain explicit confirmation. +Wait for explicit confirmation before continuing. ## 3. Prepare authentication @@ -71,11 +75,18 @@ Use the official intake: ```bash export PRB_SUBMIT_URL=https://intake.prb-bench.workers.dev/submit PRB_ACCESS_APP="${PRB_SUBMIT_URL%/submit}" -PRB_ACCESS_TOKEN="$(cloudflared access login --no-verbose --auto-close "$PRB_ACCESS_APP")" -test -n "$PRB_ACCESS_TOKEN" -PRB_ACCESS_TOKEN="$PRB_ACCESS_TOKEN" \ - python3 -m benchmark.submit --predictions -unset PRB_ACCESS_TOKEN +if PRB_ACCESS_TOKEN="$(cloudflared access login --no-verbose --auto-close "$PRB_ACCESS_APP")" \ + && [ -n "$PRB_ACCESS_TOKEN" ]; then + PRB_ACCESS_TOKEN="$PRB_ACCESS_TOKEN" \ + python3 -m benchmark.submit --predictions "" + UPLOAD_STATUS=$? + unset PRB_ACCESS_TOKEN + [ "$UPLOAD_STATUS" -eq 0 ] +else + unset PRB_ACCESS_TOKEN + echo "Cloudflare Access login failed; submission was not uploaded." >&2 + false +fi ``` The login opens GitHub in the user's browser. If no browser opens, give the user the login @@ -99,24 +110,10 @@ confirmed. ## 6. Trigger scoring After reporting an accepted submission, trigger the scoring workflow once. If GitHub CLI -is installed and authenticated, run: +is installed, run the bundled helper with the `submission_id` returned by the upload: ```bash -SUBMISSION_ID="" -RUN_URL="$(gh workflow run score-from-r2.yml \ - --repo CodingThrust/problem-reductions-benchmark \ - --ref main)" -RUN_ID="${RUN_URL##*/}" -gh run watch "$RUN_ID" \ - --repo CodingThrust/problem-reductions-benchmark \ - --exit-status --compact - -SHORT_ID="${SUBMISSION_ID:0:8}" -gh pr list \ - --repo CodingThrust/problem-reductions-benchmark \ - --state all --limit 100 \ - --json headRefName,url \ - --jq ".[] | select(.headRefName | endswith(\"--${SHORT_ID}\")) | .url" +bash /scripts/trigger-scoring.sh "" ``` If `gh` is unavailable, give the user the @@ -129,8 +126,7 @@ collaborator with Actions write permission must trigger it. Do not upload again: accepted submission remains privately queued and the daily scheduled workflow will process it if nobody triggers a run manually. -When the triggered run succeeds, return the matching PR URL. Match it using the first eight -characters of this upload's `submission_id`, which appear at the end of the leaderboard -branch name; do not return an unrelated latest PR. If the run fails, return its URL and the -failure. If it succeeds but no matching PR exists, return the run URL and explain that no PR -was created instead of guessing a link. +The helper validates or recovers the run URL, stops if the run fails, and prints the PR that +matches this upload's `submission_id` as its final line. Return that PR URL to the user. If +the helper reports a run failure or no matching PR, return its run URL and explanation +instead of guessing a link. diff --git a/.agents/skills/submit-benchmark-result/scripts/trigger-scoring.sh b/.agents/skills/submit-benchmark-result/scripts/trigger-scoring.sh new file mode 100755 index 0000000..b9e298a --- /dev/null +++ b/.agents/skills/submit-benchmark-result/scripts/trigger-scoring.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -u + +REPO="CodingThrust/problem-reductions-benchmark" +WORKFLOW="score-from-r2.yml" +RUN_PREFIX="https://github.com/${REPO}/actions/runs/" + +if [ "$#" -ne 1 ] || [ "${#1}" -lt 8 ]; then + echo "usage: trigger-scoring.sh " >&2 + exit 2 +fi +submission_id="$1" + +if ! command -v gh >/dev/null; then + echo "GitHub CLI is required to trigger and follow scoring." >&2 + exit 2 +fi +if ! gh auth status --hostname github.com >/dev/null 2>&1; then + echo "GitHub CLI is not authenticated. Run: gh auth login" >&2 + exit 2 +fi + +actor="$(gh api user --jq .login)" || exit 1 +triggered_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +if ! run_output="$(gh workflow run "$WORKFLOW" --repo "$REPO" --ref main)"; then + echo "Failed to trigger the scoring workflow." >&2 + exit 1 +fi + +run_url="$(printf '%s\n' "$run_output" \ + | sed -nE "s#.*(${RUN_PREFIX}[0-9]+).*#\1#p" \ + | tail -n 1)" + +# Older gh versions may dispatch successfully without printing the run URL. Recover the run +# created by this user after the dispatch began, allowing a short delay for API visibility. +attempt=0 +while [ -z "$run_url" ] && [ "$attempt" -lt 10 ]; do + run_url="$(gh run list --repo "$REPO" --workflow "$WORKFLOW" \ + --event workflow_dispatch --branch main --user "$actor" \ + --created ">=${triggered_at}" --limit 1 --json url --jq '.[0].url // empty')" + [ -n "$run_url" ] || sleep 2 + attempt=$((attempt + 1)) +done + +run_id="${run_url##*/}" +case "$run_url" in + "${RUN_PREFIX}"*) ;; + *) + echo "Scoring was triggered, but its run URL could not be found." >&2 + exit 1 + ;; +esac +case "$run_id" in + ""|*[!0-9]*) + echo "Scoring was triggered, but its run URL could not be found." >&2 + exit 1 + ;; +esac + +if ! gh run watch "$run_id" --repo "$REPO" --exit-status --compact; then + echo "Scoring failed: $run_url" >&2 + exit 1 +fi + +short_id="$(printf '%.8s' "$submission_id")" +pr_url="$(gh pr list --repo "$REPO" --state all --limit 100 \ + --json headRefName,url \ + --jq "[.[] | select(.headRefName | endswith(\"--${short_id}\"))][0].url // empty")" + +if [ -z "$pr_url" ]; then + echo "Scoring succeeded, but no matching PR was created. Run: $run_url" >&2 + exit 2 +fi + +printf '%s\n' "$pr_url" diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index 56dcc2b..de738f4 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -3,9 +3,12 @@ (the single run-and-submit guide). No rendering, no external calls. All tests are marked @pytest.mark.judgment. """ -import pytest +import os +import subprocess from pathlib import Path +import pytest + from benchmark.env_setup import PINNED_COMMIT, PINNED_PRED_VERSION pytestmark = pytest.mark.judgment @@ -17,6 +20,7 @@ API_SKILL = REPO_ROOT / ".agents/skills/run-api-benchmark/SKILL.md" CLI_SKILL = REPO_ROOT / ".agents/skills/run-cli-benchmark/SKILL.md" SUBMIT_SKILL = REPO_ROOT / ".agents/skills/submit-benchmark-result/SKILL.md" +TRIGGER_SCORING = SUBMIT_SKILL.parent / "scripts/trigger-scoring.sh" SCORER_WORKFLOW = REPO_ROOT / ".github/workflows/score-from-r2.yml" SUBMISSIONS_README = REPO_ROOT / "submissions/README.md" SITE_INDEX = REPO_ROOT / "site/index.html" @@ -27,6 +31,29 @@ def _text(path: Path) -> str: return path.read_text(encoding="utf-8").lower() +def _run_trigger_with_fake_gh( + tmp_path, gh_cases, + submission_id="d78aad10-e44a-40b9-b57c-093a7bb47330"): + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + gh = fake_bin / "gh" + gh.write_text( + f'''#!/bin/sh +case "$1 $2" in + "auth status") exit 0 ;; + "api user") echo submitter ;; +{gh_cases} + *) exit 1 ;; +esac +''', + encoding="utf-8") + gh.chmod(0o755) + env = os.environ | {"PATH": f"{fake_bin}:{os.environ['PATH']}"} + return subprocess.run( + ["bash", str(TRIGGER_SCORING), submission_id], + text=True, capture_output=True, env=env, check=False) + + class TestReadme: def test_readme_exists(self): assert README.exists(), "README.md missing from repo root" @@ -100,6 +127,7 @@ def test_run_skills_delegate_upload(self, skill): class TestSubmitSkill: def test_submit_skill_exists(self): assert SUBMIT_SKILL.exists(), "submit-benchmark-result skill missing" + assert os.access(TRIGGER_SCORING, os.X_OK), "trigger-scoring.sh must be executable" def test_submit_skill_validates_before_upload(self): t = _text(SUBMIT_SKILL) @@ -116,7 +144,8 @@ def test_submit_skill_uses_access_without_github_credentials(self): t = _text(SUBMIT_SKILL) assert "cloudflared access login" in t assert "prb_access_token" in t - assert 'test -n "$prb_access_token"' in t + assert '&& [ -n "$prb_access_token" ]' in t + assert "submission was not uploaded" in t assert "unset prb_access_token" in t assert "gh auth token" in t assert "github pat" in t @@ -136,16 +165,42 @@ def test_submit_skill_explains_access_denial(self): def test_submit_skill_triggers_scoring_without_reset(self): t = _text(SUBMIT_SKILL) - assert "gh workflow run score-from-r2.yml" in t - assert "--repo codingthrust/problem-reductions-benchmark" in t + assert "scripts/trigger-scoring.sh" in t assert "reset_results" not in t assert "actions write permission" in t - assert 'run_id="${run_url##*/}"' in t - assert "gh run watch" in t - assert 'short_id="${submission_id:0:8}"' in t - assert "--json headrefname,url" in t - assert "return the matching pr url" in t - assert "do not return an unrelated latest pr" in t + assert "return that pr url" in t + + def test_trigger_scoring_returns_matching_pr(self, tmp_path): + marker = tmp_path / "pr-list-called" + result = _run_trigger_with_fake_gh( + tmp_path, + f''' "workflow run") echo https://github.com/CodingThrust/problem-reductions-benchmark/actions/runs/123 ;; + "run watch") exit 0 ;; + "pr list") : > "{marker}"; echo https://github.com/CodingThrust/problem-reductions-benchmark/pull/456 ;;''') + assert result.returncode == 0 + assert result.stdout.rstrip().endswith("/pull/456") + assert marker.exists() + + def test_trigger_scoring_stops_before_pr_lookup_when_run_fails(self, tmp_path): + marker = tmp_path / "pr-list-called" + result = _run_trigger_with_fake_gh( + tmp_path, + f''' "workflow run") echo https://github.com/CodingThrust/problem-reductions-benchmark/actions/runs/123 ;; + "run watch") exit 1 ;; + "pr list") : > "{marker}"; exit 0 ;;''') + assert result.returncode == 1 + assert "actions/runs/123" in result.stderr + assert not marker.exists() + + def test_trigger_scoring_recovers_run_url_when_dispatch_prints_none(self, tmp_path): + result = _run_trigger_with_fake_gh( + tmp_path, + ''' "workflow run") exit 0 ;; + "run list") echo https://github.com/CodingThrust/problem-reductions-benchmark/actions/runs/789 ;; + "run watch") exit 0 ;; + "pr list") echo https://github.com/CodingThrust/problem-reductions-benchmark/pull/987 ;;''') + assert result.returncode == 0 + assert result.stdout.rstrip().endswith("/pull/987") @pytest.mark.parametrize( "path", [GUIDE, SUBMIT_SKILL, SUBMISSIONS_README, SITE_INDEX, INTAKE_README])