Harden benchmark submission control flow

.agents/skills/submit-benchmark-result/SKILL.md

@@ -33,14 +33,18 @@ Do not submit a result containing `run_error`. Report the error and stop.
 
 ## 2. Confirm the upload
 
-Immediately before uploading, show:
+Immediately before uploading, show this confirmation with the real values filled in:
 
-- endpoint hostname;
-- absolute file path;
-- model and claimed bug count;
+> Ready to officially submit this benchmark result:
+> - Destination: `intake.prb-bench.workers.dev`
+> - File: `<absolute path to submission.json>`
+> - Model: `<model>`
+> - Claimed bugs: `<count>`
+>
+> This private submission file will leave the machine and be uploaded to the benchmark's
+> private storage. Confirm submission?
 
-State that this is an official submission and that the private file will leave the machine,
-then obtain explicit confirmation.
+Wait for explicit confirmation before continuing.
 
 ## 3. Prepare authentication
 
@@ -71,11 +75,18 @@ Use the official intake:
 ```bash
 export PRB_SUBMIT_URL=https://intake.prb-bench.workers.dev/submit
 PRB_ACCESS_APP="${PRB_SUBMIT_URL%/submit}"
-PRB_ACCESS_TOKEN="$(cloudflared access login --no-verbose --auto-close "$PRB_ACCESS_APP")"
-test -n "$PRB_ACCESS_TOKEN"
-PRB_ACCESS_TOKEN="$PRB_ACCESS_TOKEN" \
-  python3 -m benchmark.submit --predictions <submission.json>
-unset PRB_ACCESS_TOKEN
+if PRB_ACCESS_TOKEN="$(cloudflared access login --no-verbose --auto-close "$PRB_ACCESS_APP")" \
+    && [ -n "$PRB_ACCESS_TOKEN" ]; then
+  PRB_ACCESS_TOKEN="$PRB_ACCESS_TOKEN" \
+    python3 -m benchmark.submit --predictions "<submission.json>"
+  UPLOAD_STATUS=$?
+  unset PRB_ACCESS_TOKEN
+  [ "$UPLOAD_STATUS" -eq 0 ]
+else
+  unset PRB_ACCESS_TOKEN
+  echo "Cloudflare Access login failed; submission was not uploaded." >&2
+  false
+fi
 ```
 
 The login opens GitHub in the user's browser. If no browser opens, give the user the login
@@ -99,24 +110,10 @@ confirmed.
 ## 6. Trigger scoring
 
 After reporting an accepted submission, trigger the scoring workflow once. If GitHub CLI
-is installed and authenticated, run:
+is installed, run the bundled helper with the `submission_id` returned by the upload:
 
 ```bash
-SUBMISSION_ID="<submission_id returned by the upload>"
-RUN_URL="$(gh workflow run score-from-r2.yml \
-  --repo CodingThrust/problem-reductions-benchmark \
-  --ref main)"
-RUN_ID="${RUN_URL##*/}"
-gh run watch "$RUN_ID" \
-  --repo CodingThrust/problem-reductions-benchmark \
-  --exit-status --compact
-
-SHORT_ID="${SUBMISSION_ID:0:8}"
-gh pr list \
-  --repo CodingThrust/problem-reductions-benchmark \
-  --state all --limit 100 \
-  --json headRefName,url \
-  --jq ".[] | select(.headRefName | endswith(\"--${SHORT_ID}\")) | .url"
+bash <skill-directory>/scripts/trigger-scoring.sh "<submission_id>"
 ```
 
 If `gh` is unavailable, give the user the
@@ -129,8 +126,7 @@ collaborator with Actions write permission must trigger it. Do not upload again:
 accepted submission remains privately queued and the daily scheduled workflow will process
 it if nobody triggers a run manually.
 
-When the triggered run succeeds, return the matching PR URL. Match it using the first eight
-characters of this upload's `submission_id`, which appear at the end of the leaderboard
-branch name; do not return an unrelated latest PR. If the run fails, return its URL and the
-failure. If it succeeds but no matching PR exists, return the run URL and explain that no PR
-was created instead of guessing a link.
+The helper validates or recovers the run URL, stops if the run fails, and prints the PR that
+matches this upload's `submission_id` as its final line. Return that PR URL to the user. If
+the helper reports a run failure or no matching PR, return its run URL and explanation
+instead of guessing a link.

.agents/skills/submit-benchmark-result/scripts/trigger-scoring.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+set -u
+
+REPO="CodingThrust/problem-reductions-benchmark"
+WORKFLOW="score-from-r2.yml"
+RUN_PREFIX="https://github.com/${REPO}/actions/runs/"
+
+if [ "$#" -ne 1 ] || [ "${#1}" -lt 8 ]; then
+  echo "usage: trigger-scoring.sh <submission_id>" >&2
+  exit 2
+fi
+submission_id="$1"
+
+if ! command -v gh >/dev/null; then
+  echo "GitHub CLI is required to trigger and follow scoring." >&2
+  exit 2
+fi
+if ! gh auth status --hostname github.com >/dev/null 2>&1; then
+  echo "GitHub CLI is not authenticated. Run: gh auth login" >&2
+  exit 2
+fi
+
+actor="$(gh api user --jq .login)" || exit 1
+triggered_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+if ! run_output="$(gh workflow run "$WORKFLOW" --repo "$REPO" --ref main)"; then
+  echo "Failed to trigger the scoring workflow." >&2
+  exit 1
+fi
+
+run_url="$(printf '%s\n' "$run_output" \
+  | sed -nE "s#.*(${RUN_PREFIX}[0-9]+).*#\1#p" \
+  | tail -n 1)"
+
+# Older gh versions may dispatch successfully without printing the run URL. Recover the run
+# created by this user after the dispatch began, allowing a short delay for API visibility.
+attempt=0
+while [ -z "$run_url" ] && [ "$attempt" -lt 10 ]; do
+  run_url="$(gh run list --repo "$REPO" --workflow "$WORKFLOW" \
+    --event workflow_dispatch --branch main --user "$actor" \
+    --created ">=${triggered_at}" --limit 1 --json url --jq '.[0].url // empty')"
+  [ -n "$run_url" ] || sleep 2
+  attempt=$((attempt + 1))
+done
+
+run_id="${run_url##*/}"
+case "$run_url" in
+  "${RUN_PREFIX}"*) ;;
+  *)
+    echo "Scoring was triggered, but its run URL could not be found." >&2
+    exit 1
+    ;;
+esac
+case "$run_id" in
+  ""|*[!0-9]*)
+    echo "Scoring was triggered, but its run URL could not be found." >&2
+    exit 1
+    ;;
+esac
+
+if ! gh run watch "$run_id" --repo "$REPO" --exit-status --compact; then
+  echo "Scoring failed: $run_url" >&2
+  exit 1
+fi
+
+short_id="$(printf '%.8s' "$submission_id")"
+pr_url="$(gh pr list --repo "$REPO" --state all --limit 100 \
+  --json headRefName,url \
+  --jq "[.[] | select(.headRefName | endswith(\"--${short_id}\"))][0].url // empty")"
+
+if [ -z "$pr_url" ]; then
+  echo "Scoring succeeded, but no matching PR was created. Run: $run_url" >&2
+  exit 2
+fi
+
+printf '%s\n' "$pr_url"

benchmark/tests/test_docs.py

@@ -3,9 +3,12 @@
 (the single run-and-submit guide). No rendering, no external calls.
 All tests are marked @pytest.mark.judgment.
 """
-import pytest
+import os
+import subprocess
 from pathlib import Path
 
+import pytest
+
 from benchmark.env_setup import PINNED_COMMIT, PINNED_PRED_VERSION
 
 pytestmark = pytest.mark.judgment
@@ -17,6 +20,7 @@
 API_SKILL = REPO_ROOT / ".agents/skills/run-api-benchmark/SKILL.md"
 CLI_SKILL = REPO_ROOT / ".agents/skills/run-cli-benchmark/SKILL.md"
 SUBMIT_SKILL = REPO_ROOT / ".agents/skills/submit-benchmark-result/SKILL.md"
+TRIGGER_SCORING = SUBMIT_SKILL.parent / "scripts/trigger-scoring.sh"
 SCORER_WORKFLOW = REPO_ROOT / ".github/workflows/score-from-r2.yml"
 SUBMISSIONS_README = REPO_ROOT / "submissions/README.md"
 SITE_INDEX = REPO_ROOT / "site/index.html"
@@ -27,6 +31,29 @@ def _text(path: Path) -> str:
     return path.read_text(encoding="utf-8").lower()
 
 
+def _run_trigger_with_fake_gh(
+        tmp_path, gh_cases,
+        submission_id="d78aad10-e44a-40b9-b57c-093a7bb47330"):
+    fake_bin = tmp_path / "bin"
+    fake_bin.mkdir()
+    gh = fake_bin / "gh"
+    gh.write_text(
+        f'''#!/bin/sh
+case "$1 $2" in
+  "auth status") exit 0 ;;
+  "api user") echo submitter ;;
+{gh_cases}
+  *) exit 1 ;;
+esac
+''',
+        encoding="utf-8")
+    gh.chmod(0o755)
+    env = os.environ | {"PATH": f"{fake_bin}:{os.environ['PATH']}"}
+    return subprocess.run(
+        ["bash", str(TRIGGER_SCORING), submission_id],
+        text=True, capture_output=True, env=env, check=False)
+
+
 class TestReadme:
     def test_readme_exists(self):
         assert README.exists(), "README.md missing from repo root"
@@ -100,6 +127,7 @@ def test_run_skills_delegate_upload(self, skill):
 class TestSubmitSkill:
     def test_submit_skill_exists(self):
         assert SUBMIT_SKILL.exists(), "submit-benchmark-result skill missing"
+        assert os.access(TRIGGER_SCORING, os.X_OK), "trigger-scoring.sh must be executable"
 
     def test_submit_skill_validates_before_upload(self):
         t = _text(SUBMIT_SKILL)
@@ -116,7 +144,8 @@ def test_submit_skill_uses_access_without_github_credentials(self):
         t = _text(SUBMIT_SKILL)
         assert "cloudflared access login" in t
         assert "prb_access_token" in t
-        assert 'test -n "$prb_access_token"' in t
+        assert '&& [ -n "$prb_access_token" ]' in t
+        assert "submission was not uploaded" in t
         assert "unset prb_access_token" in t
         assert "gh auth token" in t
         assert "github pat" in t
@@ -136,16 +165,42 @@ def test_submit_skill_explains_access_denial(self):
 
     def test_submit_skill_triggers_scoring_without_reset(self):
         t = _text(SUBMIT_SKILL)
-        assert "gh workflow run score-from-r2.yml" in t
-        assert "--repo codingthrust/problem-reductions-benchmark" in t
+        assert "scripts/trigger-scoring.sh" in t
         assert "reset_results" not in t
         assert "actions write permission" in t
-        assert 'run_id="${run_url##*/}"' in t
-        assert "gh run watch" in t
-        assert 'short_id="${submission_id:0:8}"' in t
-        assert "--json headrefname,url" in t
-        assert "return the matching pr url" in t
-        assert "do not return an unrelated latest pr" in t
+        assert "return that pr url" in t
+
+    def test_trigger_scoring_returns_matching_pr(self, tmp_path):
+        marker = tmp_path / "pr-list-called"
+        result = _run_trigger_with_fake_gh(
+            tmp_path,
+            f'''  "workflow run") echo https://github.com/CodingThrust/problem-reductions-benchmark/actions/runs/123 ;;
+  "run watch") exit 0 ;;
+  "pr list") : > "{marker}"; echo https://github.com/CodingThrust/problem-reductions-benchmark/pull/456 ;;''')
+        assert result.returncode == 0
+        assert result.stdout.rstrip().endswith("/pull/456")
+        assert marker.exists()
+
+    def test_trigger_scoring_stops_before_pr_lookup_when_run_fails(self, tmp_path):
+        marker = tmp_path / "pr-list-called"
+        result = _run_trigger_with_fake_gh(
+            tmp_path,
+            f'''  "workflow run") echo https://github.com/CodingThrust/problem-reductions-benchmark/actions/runs/123 ;;
+  "run watch") exit 1 ;;
+  "pr list") : > "{marker}"; exit 0 ;;''')
+        assert result.returncode == 1
+        assert "actions/runs/123" in result.stderr
+        assert not marker.exists()
+
+    def test_trigger_scoring_recovers_run_url_when_dispatch_prints_none(self, tmp_path):
+        result = _run_trigger_with_fake_gh(
+            tmp_path,
+            '''  "workflow run") exit 0 ;;
+  "run list") echo https://github.com/CodingThrust/problem-reductions-benchmark/actions/runs/789 ;;
+  "run watch") exit 0 ;;
+  "pr list") echo https://github.com/CodingThrust/problem-reductions-benchmark/pull/987 ;;''')
+        assert result.returncode == 0
+        assert result.stdout.rstrip().endswith("/pull/987")
 
     @pytest.mark.parametrize(
         "path", [GUIDE, SUBMIT_SKILL, SUBMISSIONS_README, SITE_INDEX, INTAKE_README])