From 4ffd1a11a3a25c94f9f0acbc5cfb9f19ad12ca3b Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Sun, 19 Jul 2026 18:58:35 +0800 Subject: [PATCH 1/8] Simplify benchmark submission skill --- .../skills/submit-benchmark-result/SKILL.md | 111 +++++++----------- benchmark/tests/test_docs.py | 8 +- 2 files changed, 47 insertions(+), 72 deletions(-) diff --git a/.agents/skills/submit-benchmark-result/SKILL.md b/.agents/skills/submit-benchmark-result/SKILL.md index 290c564..a29720a 100644 --- a/.agents/skills/submit-benchmark-result/SKILL.md +++ b/.agents/skills/submit-benchmark-result/SKILL.md @@ -1,101 +1,76 @@ --- name: submit-benchmark-result -description: Validate and upload an existing problem-reductions benchmark submission through the private intake. Use when a user wants to submit, upload, publish, dry-run, or test a submission.json they already produced, especially when the user is not a repository maintainer. Handles local validation, test-versus-official intent, intake authentication, upload confirmation, and submission ID reporting; routes missing results to run-benchmark instead of running a model itself. +description: Validate and upload an existing problem-reductions benchmark submission. Use when the user wants to check, test-upload, or officially submit a submission.json. If no result exists yet, route the request to $run-benchmark. --- # Submit a benchmark result -Submit an existing `submission.json` without requiring GitHub repository, R2, Worker, or -Actions access. Treat the file as confidential because it contains certificates and the -submit ledger. +Follow this workflow for an existing `submission.json`. Do not run a model or create a +submission in this skill. -## 1. Locate the authoritative result +## 1. Find and validate the file -Ask for the submission path only when it was not supplied. Accept the authoritative output -from either benchmark backend; do not reconstruct it from logs or edit its metrics, -certificates, `library_commit`, or ledger. +Ask for the path only if the user did not provide it. If no result exists, use +`$run-benchmark` instead. -If no result exists and the user wants to run the benchmark, invoke `$run-benchmark`. Do not -choose a model/backend or start a paid run inside this skill. +Treat the file as confidential. Do not print certificates, trajectories, source instances, +submit-log contents, or credentials. Do not edit the submission. -Read the current round from `README.md` and the current structure from -`benchmark/submission.schema.json`. Inspect only summary fields; never print certificates, -source instances, submit-log contents, trajectories, or credentials. Report: - -- model and `library_commit`; -- claimed bugs, token total, and submit-attempt count; -- `run_error`, when present; -- absolute submission path. - -Run the repository client as a local courtesy check: +Run: ```bash python3 -m benchmark.submit --predictions --dry-run ``` -Stop on failure. Do not repair an invalid result by hand; send it back to the producing run. - -## 2. Choose the outcome - -Ask only when the user has not already chosen: +Stop if validation fails. Otherwise report only: -> What should happen to this result? -> -> 1. Keep it local after validation. -> 2. Upload an intake test that is scored privately and never reaches the leaderboard. -> 3. Upload an official submission. +- absolute path; +- `model` and `library_commit`; +- claimed bugs, `total_tokens_k`, and number of submit attempts; +- `run_error`, if present. -For option 1, report validation and stop. For option 2, use `--test`. For option 3, never -use `--test`. A non-test result carrying `run_error` is not a clean official submission; -offer local-only or test upload instead. +## 2. Choose the outcome -## 3. Authenticate without exposing credentials +Use the user's stated choice. If they did not choose, ask: -Require `PRB_SUBMIT_URL` from the repository documentation or maintainer. Never ask the user -to paste any token into chat, print an environment variable, or commit credentials. +1. Validate only — keep the file local. +2. Test upload — process it privately without publishing it. +3. Official submission — eligible for the public leaderboard after maintainer review. -Use GitHub-backed Cloudflare Access through `PRB_ACCESS_TOKEN`: +Stop after validation for option 1. Use `--test` for option 2. Use no mode flag for option 3. +Do not officially submit a result containing `run_error`; offer options 1 or 2 instead. -1. Require `cloudflared` locally. -2. For a new session, obtain the application-scoped token only inside the confirmed upload - command: `PRB_ACCESS_TOKEN="$(cloudflared access login --no-verbose --auto-close )" `. - `cloudflared` opens the configured GitHub login. -3. A later upload may use `cloudflared access token -app=` inside the same - command substitution while the local Access session remains valid. -4. Never run either token-producing command standalone. Pass its stdout directly through - the client-supported environment variable; do not display it, save it yourself, or put it - in a command-line flag. +## 3. Confirm the upload -Do not substitute `gh auth token`, a GitHub personal access token, or `GITHUB_TOKEN`; the -intake must never receive the user's general GitHub credential. +Immediately before uploading, show: -If Cloudflare Access is not deployed or the user is not allowed by its policy, stop and ask -the maintainer to fix the Access application. There is no shared-key fallback. Never request -an intake key in chat. +- endpoint hostname; +- absolute file path; +- model and claimed bug count; +- TEST or OFFICIAL mode. -## 4. Confirm and upload once +State that the private submission will leave the machine, then obtain explicit confirmation. -Before the external write, show the endpoint hostname, absolute file path, model, claimed -bug count, and test/official mode. State that the private certificate payload will leave the -machine, then obtain explicit confirmation. +## 4. Log in and upload once -Use the repository client so validation and test marking stay consistent: +Use the official intake: ```bash -python3 -m benchmark.submit --predictions --test # private test -python3 -m benchmark.submit --predictions # official +export PRB_SUBMIT_URL=https://intake.prb-bench.workers.dev/submit +PRB_ACCESS_APP="${PRB_SUBMIT_URL%/submit}" +PRB_ACCESS_TOKEN="$(cloudflared access login --no-verbose --auto-close "$PRB_ACCESS_APP")" \ + python3 -m benchmark.submit --predictions --test ``` -Do not retry an ambiguous timeout automatically: the first request may already have reached -R2 and a retry can create a duplicate. On HTTP 401/403, re-authenticate or report the missing -Access deployment. On HTTP 413, do not trim the evidence; report the size limit. On HTTP 429, -stop and report the rate limit. +Remove `--test` for an official submission. Require `cloudflared`; the login opens GitHub in +the user's browser. Keep token acquisition inside command substitution. Never print or ask +for the token, and never substitute a GitHub PAT, `gh auth token`, or `GITHUB_TOKEN`. -## 5. Report the handoff +Upload only once. Do not retry a timeout because the first request may already be queued. +For HTTP 401/403, re-authenticate; for 413 or 429, stop and report the error. -On success, report the returned `submission_id`, model, mode, and endpoint hostname. Make it -clear that `accepted` means privately queued, not scored or published. The submitter does not -trigger Actions, inspect R2, or open the leaderboard PR; those are maintainer responsibilities. +## 5. Report the result -Never claim a score until the maintainer confirms the private scorer finished. Preserve the -local submission and logs until that confirmation. +On success, report the `submission_id`, model, mode, and endpoint. Say that `accepted` means +queued privately, not scored or published. Keep the local submission until scoring is +confirmed. diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index 6e2d3a5..82b9a01 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -108,12 +108,12 @@ def test_submit_skill_validates_before_upload(self): assert "--test" in t assert "explicit confirmation" in t - def test_submit_skill_keeps_github_credentials_out_of_intake(self): + def test_submit_skill_uses_access_without_github_credentials(self): t = _text(SUBMIT_SKILL) - assert "github-backed cloudflare access" in t + assert "cloudflared access login" in t + assert "prb_access_token" in t assert "gh auth token" in t - assert "personal access token" in t - assert "cloudflare access is not deployed" in t + assert "github pat" in t @pytest.mark.parametrize( "path", [GUIDE, SUBMIT_SKILL, SUBMISSIONS_README, SITE_INDEX, INTAKE_README]) From 8acd7ec5b6b90e516be8a7bac6f18a0f4b117896 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Sun, 19 Jul 2026 19:06:05 +0800 Subject: [PATCH 2/8] Make official submission the default --- .../skills/submit-benchmark-result/SKILL.md | 25 ++++++++++--------- benchmark/tests/test_docs.py | 5 ++++ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.agents/skills/submit-benchmark-result/SKILL.md b/.agents/skills/submit-benchmark-result/SKILL.md index a29720a..84b7150 100644 --- a/.agents/skills/submit-benchmark-result/SKILL.md +++ b/.agents/skills/submit-benchmark-result/SKILL.md @@ -1,6 +1,6 @@ --- name: submit-benchmark-result -description: Validate and upload an existing problem-reductions benchmark submission. Use when the user wants to check, test-upload, or officially submit a submission.json. If no result exists yet, route the request to $run-benchmark. +description: Validate or officially upload an existing problem-reductions benchmark submission. Use for checking or submitting a submission.json, and for an intake smoke test only when explicitly requested. If no result exists yet, route the request to $run-benchmark. --- # Submit a benchmark result @@ -29,16 +29,16 @@ Stop if validation fails. Otherwise report only: - claimed bugs, `total_tokens_k`, and number of submit attempts; - `run_error`, if present. -## 2. Choose the outcome +## 2. Decide whether to upload -Use the user's stated choice. If they did not choose, ask: +If the user asked only to validate or dry-run, stop after step 1. -1. Validate only — keep the file local. -2. Test upload — process it privately without publishing it. -3. Official submission — eligible for the public leaderboard after maintainer review. +Otherwise, prepare an official submission. Do not ask the user to choose between test and +official modes. Use `--test` only when the user explicitly asks for an intake smoke test or +end-to-end infrastructure check. -Stop after validation for option 1. Use `--test` for option 2. Use no mode flag for option 3. -Do not officially submit a result containing `run_error`; offer options 1 or 2 instead. +Do not officially submit a result containing `run_error`; report it and stop unless the user +explicitly requested an infrastructure test. ## 3. Confirm the upload @@ -59,12 +59,13 @@ Use the official intake: export PRB_SUBMIT_URL=https://intake.prb-bench.workers.dev/submit PRB_ACCESS_APP="${PRB_SUBMIT_URL%/submit}" PRB_ACCESS_TOKEN="$(cloudflared access login --no-verbose --auto-close "$PRB_ACCESS_APP")" \ - python3 -m benchmark.submit --predictions --test + python3 -m benchmark.submit --predictions ``` -Remove `--test` for an official submission. Require `cloudflared`; the login opens GitHub in -the user's browser. Keep token acquisition inside command substitution. Never print or ask -for the token, and never substitute a GitHub PAT, `gh auth token`, or `GITHUB_TOKEN`. +Add `--test` only for an explicitly requested infrastructure test. Require `cloudflared`; +the login opens GitHub in the user's browser. Keep token acquisition inside command +substitution. Never print or ask for the token, and never substitute a GitHub PAT, +`gh auth token`, or `GITHUB_TOKEN`. Upload only once. Do not retry a timeout because the first request may already be queued. For HTTP 401/403, re-authenticate; for 413 or 429, stop and report the error. diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index 82b9a01..e8eb1b3 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -108,6 +108,11 @@ def test_submit_skill_validates_before_upload(self): assert "--test" in t assert "explicit confirmation" in t + def test_submit_skill_does_not_offer_test_mode_to_submitters(self): + t = _text(SUBMIT_SKILL) + assert "test upload" not in t + assert "only when the user explicitly asks" in t + def test_submit_skill_uses_access_without_github_credentials(self): t = _text(SUBMIT_SKILL) assert "cloudflared access login" in t From 366e70b8a06a2b23f4451dce6b001084196417c4 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Sun, 19 Jul 2026 19:12:13 +0800 Subject: [PATCH 3/8] Remove defensive submission mode branching --- .../skills/submit-benchmark-result/SKILL.md | 32 +++++++------------ benchmark/tests/test_docs.py | 7 ++-- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/.agents/skills/submit-benchmark-result/SKILL.md b/.agents/skills/submit-benchmark-result/SKILL.md index 84b7150..6b0d6bf 100644 --- a/.agents/skills/submit-benchmark-result/SKILL.md +++ b/.agents/skills/submit-benchmark-result/SKILL.md @@ -1,6 +1,6 @@ --- name: submit-benchmark-result -description: Validate or officially upload an existing problem-reductions benchmark submission. Use for checking or submitting a submission.json, and for an intake smoke test only when explicitly requested. If no result exists yet, route the request to $run-benchmark. +description: Validate and officially upload an existing problem-reductions benchmark submission. Use when the user wants to submit a submission.json. If no result exists yet, route the request to $run-benchmark. --- # Submit a benchmark result @@ -29,29 +29,20 @@ Stop if validation fails. Otherwise report only: - claimed bugs, `total_tokens_k`, and number of submit attempts; - `run_error`, if present. -## 2. Decide whether to upload +Do not submit a result containing `run_error`. Report the error and stop. -If the user asked only to validate or dry-run, stop after step 1. - -Otherwise, prepare an official submission. Do not ask the user to choose between test and -official modes. Use `--test` only when the user explicitly asks for an intake smoke test or -end-to-end infrastructure check. - -Do not officially submit a result containing `run_error`; report it and stop unless the user -explicitly requested an infrastructure test. - -## 3. Confirm the upload +## 2. Confirm the upload Immediately before uploading, show: - endpoint hostname; - absolute file path; - model and claimed bug count; -- TEST or OFFICIAL mode. -State that the private submission will leave the machine, then obtain explicit confirmation. +State that this is an official submission and that the private file will leave the machine, +then obtain explicit confirmation. -## 4. Log in and upload once +## 3. Log in and upload once Use the official intake: @@ -62,16 +53,15 @@ PRB_ACCESS_TOKEN="$(cloudflared access login --no-verbose --auto-close "$PRB_ACC python3 -m benchmark.submit --predictions ``` -Add `--test` only for an explicitly requested infrastructure test. Require `cloudflared`; -the login opens GitHub in the user's browser. Keep token acquisition inside command -substitution. Never print or ask for the token, and never substitute a GitHub PAT, -`gh auth token`, or `GITHUB_TOKEN`. +Require `cloudflared`; the login opens GitHub in the user's browser. Keep token acquisition +inside command substitution. Never print or ask for the token, and never substitute a +GitHub PAT, `gh auth token`, or `GITHUB_TOKEN`. Upload only once. Do not retry a timeout because the first request may already be queued. For HTTP 401/403, re-authenticate; for 413 or 429, stop and report the error. -## 5. Report the result +## 4. Report the result -On success, report the `submission_id`, model, mode, and endpoint. Say that `accepted` means +On success, report the `submission_id`, model, and endpoint. Say that `accepted` means queued privately, not scored or published. Keep the local submission until scoring is confirmed. diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index e8eb1b3..c919f29 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -105,13 +105,12 @@ def test_submit_skill_validates_before_upload(self): t = _text(SUBMIT_SKILL) assert "python3 -m benchmark.submit" in t assert "--dry-run" in t - assert "--test" in t assert "explicit confirmation" in t - def test_submit_skill_does_not_offer_test_mode_to_submitters(self): + def test_submit_skill_has_no_intake_test_mode(self): t = _text(SUBMIT_SKILL) - assert "test upload" not in t - assert "only when the user explicitly asks" in t + assert "--test" not in t + assert "smoke test" not in t def test_submit_skill_uses_access_without_github_credentials(self): t = _text(SUBMIT_SKILL) From 6e84dd472be3b3661a604c97765ca1efff77ec74 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Sun, 19 Jul 2026 19:19:16 +0800 Subject: [PATCH 4/8] Handle missing Cloudflare Access client --- .../skills/submit-benchmark-result/SKILL.md | 39 ++++++++++++++++--- benchmark/tests/test_docs.py | 9 +++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/.agents/skills/submit-benchmark-result/SKILL.md b/.agents/skills/submit-benchmark-result/SKILL.md index 6b0d6bf..48293f2 100644 --- a/.agents/skills/submit-benchmark-result/SKILL.md +++ b/.agents/skills/submit-benchmark-result/SKILL.md @@ -42,25 +42,52 @@ Immediately before uploading, show: State that this is an official submission and that the private file will leave the machine, then obtain explicit confirmation. -## 3. Log in and upload once +## 3. Prepare authentication + +Check whether the Cloudflare Access client is already available: + +```bash +command -v cloudflared >/dev/null && cloudflared --version +``` + +If it is missing, identify the operating system and tell the user that `cloudflared` must +be installed before submission. Give the installation step from Cloudflare's official +[downloads page](https://developers.cloudflare.com/cloudflare-one/networks/connectors/cloudflare-tunnel/downloads/) +for that system. Obtain confirmation before running an installer or package-manager +command. On macOS with Homebrew, the official command is: + +```bash +brew install cloudflared +``` + +Do not improvise a download URL, use `sudo`, or change the system without confirmation. +After installation, run `cloudflared --version`. If installation cannot be completed, +report the prerequisite and stop before uploading. + +## 4. Log in and upload once Use the official intake: ```bash export PRB_SUBMIT_URL=https://intake.prb-bench.workers.dev/submit PRB_ACCESS_APP="${PRB_SUBMIT_URL%/submit}" -PRB_ACCESS_TOKEN="$(cloudflared access login --no-verbose --auto-close "$PRB_ACCESS_APP")" \ +PRB_ACCESS_TOKEN="$(cloudflared access login --no-verbose --auto-close "$PRB_ACCESS_APP")" +test -n "$PRB_ACCESS_TOKEN" +PRB_ACCESS_TOKEN="$PRB_ACCESS_TOKEN" \ python3 -m benchmark.submit --predictions +unset PRB_ACCESS_TOKEN ``` -Require `cloudflared`; the login opens GitHub in the user's browser. Keep token acquisition -inside command substitution. Never print or ask for the token, and never substitute a -GitHub PAT, `gh auth token`, or `GITHUB_TOKEN`. +The login opens GitHub in the user's browser. If no browser opens, give the user the login +URL printed by `cloudflared` and wait for them to finish. Keep token acquisition inside +command substitution. Stop if login fails or returns an empty token; do not run the upload +command. Never print or ask for the token. Do not substitute a GitHub PAT, `gh auth token`, +or `GITHUB_TOKEN`. Upload only once. Do not retry a timeout because the first request may already be queued. For HTTP 401/403, re-authenticate; for 413 or 429, stop and report the error. -## 4. Report the result +## 5. Report the result On success, report the `submission_id`, model, and endpoint. Say that `accepted` means queued privately, not scored or published. Keep the local submission until scoring is diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index c919f29..6f60d83 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -116,9 +116,18 @@ def test_submit_skill_uses_access_without_github_credentials(self): t = _text(SUBMIT_SKILL) assert "cloudflared access login" in t assert "prb_access_token" in t + assert 'test -n "$prb_access_token"' in t + assert "unset prb_access_token" in t assert "gh auth token" in t assert "github pat" in t + def test_submit_skill_prepares_cloudflared(self): + t = _text(SUBMIT_SKILL) + assert "command -v cloudflared" in t + assert "cloudflared --version" in t + assert "brew install cloudflared" in t + assert "obtain confirmation before running an installer" in t + @pytest.mark.parametrize( "path", [GUIDE, SUBMIT_SKILL, SUBMISSIONS_README, SITE_INDEX, INTAKE_README]) def test_public_submission_docs_have_no_shared_intake_key(self, path): From 9a66862c4490cc0b540638f18e86c38ba8c13724 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Sun, 19 Jul 2026 19:25:14 +0800 Subject: [PATCH 5/8] Explain intake authorization denial --- .agents/skills/submit-benchmark-result/SKILL.md | 5 ++++- benchmark/tests/test_docs.py | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.agents/skills/submit-benchmark-result/SKILL.md b/.agents/skills/submit-benchmark-result/SKILL.md index 48293f2..d79be06 100644 --- a/.agents/skills/submit-benchmark-result/SKILL.md +++ b/.agents/skills/submit-benchmark-result/SKILL.md @@ -85,7 +85,10 @@ command. Never print or ask for the token. Do not substitute a GitHub PAT, `gh a or `GITHUB_TOKEN`. Upload only once. Do not retry a timeout because the first request may already be queued. -For HTTP 401/403, re-authenticate; for 413 or 429, stop and report the error. +For HTTP 401, re-authenticate. If the login page says access is denied or the upload returns +HTTP 403, explain that the GitHub account is not authorized. Tell the user to ensure their +GitHub primary email has been added to the intake authorization list by a maintainer, then +stop. For 413 or 429, stop and report the error. ## 5. Report the result diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index 6f60d83..c1a7739 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -128,6 +128,12 @@ def test_submit_skill_prepares_cloudflared(self): assert "brew install cloudflared" in t assert "obtain confirmation before running an installer" in t + def test_submit_skill_explains_access_denial(self): + t = _text(SUBMIT_SKILL) + assert "http 403" in t + assert "github primary email" in t + assert "authorization list by a maintainer" in t + @pytest.mark.parametrize( "path", [GUIDE, SUBMIT_SKILL, SUBMISSIONS_README, SITE_INDEX, INTAKE_README]) def test_public_submission_docs_have_no_shared_intake_key(self, path): From f57cfab0832547b96ae130ef3867feeb69230e5f Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Sun, 19 Jul 2026 19:32:51 +0800 Subject: [PATCH 6/8] Trigger scoring after submission --- .../skills/submit-benchmark-result/SKILL.md | 22 +++++++++++++++++++ benchmark/tests/test_docs.py | 8 +++++++ 2 files changed, 30 insertions(+) diff --git a/.agents/skills/submit-benchmark-result/SKILL.md b/.agents/skills/submit-benchmark-result/SKILL.md index d79be06..bbf7051 100644 --- a/.agents/skills/submit-benchmark-result/SKILL.md +++ b/.agents/skills/submit-benchmark-result/SKILL.md @@ -95,3 +95,25 @@ stop. For 413 or 429, stop and report the error. On success, report the `submission_id`, model, and endpoint. Say that `accepted` means queued privately, not scored or published. Keep the local submission until scoring is confirmed. + +## 6. Trigger scoring + +After reporting an accepted submission, trigger the scoring workflow once. If GitHub CLI +is installed and authenticated, run: + +```bash +gh workflow run score-from-r2.yml \ + --repo CodingThrust/problem-reductions-benchmark \ + --ref main \ + -f reset_results=false +``` + +Never set `reset_results=true` for a submission. If `gh` is unavailable, give the user the +[workflow page](https://github.com/CodingThrust/problem-reductions-benchmark/actions/workflows/score-from-r2.yml) +and ask them to select **Run workflow** on `main` without enabling `reset_results`. + +The intake authorization list and GitHub Actions permissions are separate. If GitHub hides +the **Run workflow** button or returns 403, explain that a repository maintainer or a +collaborator with Actions write permission must trigger it. Do not upload again: the +accepted submission remains privately queued and the daily scheduled workflow will process +it if nobody triggers a run manually. diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index c1a7739..5a35ea1 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -134,6 +134,14 @@ def test_submit_skill_explains_access_denial(self): assert "github primary email" in t assert "authorization list by a maintainer" in t + def test_submit_skill_triggers_scoring_without_reset(self): + t = _text(SUBMIT_SKILL) + assert "gh workflow run score-from-r2.yml" in t + assert "--repo codingthrust/problem-reductions-benchmark" in t + assert "-f reset_results=false" in t + assert "never set `reset_results=true`" in t + assert "actions write permission" in t + @pytest.mark.parametrize( "path", [GUIDE, SUBMIT_SKILL, SUBMISSIONS_README, SITE_INDEX, INTAKE_README]) def test_public_submission_docs_have_no_shared_intake_key(self, path): From e46805da80e560e81af7166b9ac576e81896465b Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Sun, 19 Jul 2026 19:36:15 +0800 Subject: [PATCH 7/8] Return scoring pull request URL --- .../skills/submit-benchmark-result/SKILL.md | 22 +++++++++++++++++-- benchmark/tests/test_docs.py | 6 +++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/.agents/skills/submit-benchmark-result/SKILL.md b/.agents/skills/submit-benchmark-result/SKILL.md index bbf7051..0149ae2 100644 --- a/.agents/skills/submit-benchmark-result/SKILL.md +++ b/.agents/skills/submit-benchmark-result/SKILL.md @@ -102,10 +102,22 @@ After reporting an accepted submission, trigger the scoring workflow once. If Gi is installed and authenticated, run: ```bash -gh workflow run score-from-r2.yml \ +SUBMISSION_ID="" +RUN_URL="$(gh workflow run score-from-r2.yml \ --repo CodingThrust/problem-reductions-benchmark \ --ref main \ - -f reset_results=false + -f reset_results=false)" +RUN_ID="${RUN_URL##*/}" +gh run watch "$RUN_ID" \ + --repo CodingThrust/problem-reductions-benchmark \ + --exit-status --compact + +SHORT_ID="${SUBMISSION_ID:0:8}" +gh pr list \ + --repo CodingThrust/problem-reductions-benchmark \ + --state all --limit 100 \ + --json headRefName,url \ + --jq ".[] | select(.headRefName | endswith(\"--${SHORT_ID}\")) | .url" ``` Never set `reset_results=true` for a submission. If `gh` is unavailable, give the user the @@ -117,3 +129,9 @@ the **Run workflow** button or returns 403, explain that a repository maintainer collaborator with Actions write permission must trigger it. Do not upload again: the accepted submission remains privately queued and the daily scheduled workflow will process it if nobody triggers a run manually. + +When the triggered run succeeds, return the matching PR URL. Match it using the first eight +characters of this upload's `submission_id`, which appear at the end of the leaderboard +branch name; do not return an unrelated latest PR. If the run fails, return its URL and the +failure. If it succeeds but no matching PR exists, return the run URL and explain that no PR +was created instead of guessing a link. diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index 5a35ea1..d3155d6 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -141,6 +141,12 @@ def test_submit_skill_triggers_scoring_without_reset(self): assert "-f reset_results=false" in t assert "never set `reset_results=true`" in t assert "actions write permission" in t + assert 'run_id="${run_url##*/}"' in t + assert "gh run watch" in t + assert 'short_id="${submission_id:0:8}"' in t + assert "--json headrefname,url" in t + assert "return the matching pr url" in t + assert "do not return an unrelated latest pr" in t @pytest.mark.parametrize( "path", [GUIDE, SUBMIT_SKILL, SUBMISSIONS_README, SITE_INDEX, INTAKE_README]) From 2777b398eaf1f510f34ddcd66f558c07db81a354 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Sun, 19 Jul 2026 19:39:30 +0800 Subject: [PATCH 8/8] Remove destructive scoring reset input --- .agents/skills/submit-benchmark-result/SKILL.md | 7 +++---- .github/workflows/score-from-r2.yml | 11 ----------- benchmark/tests/test_docs.py | 6 ++++-- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/.agents/skills/submit-benchmark-result/SKILL.md b/.agents/skills/submit-benchmark-result/SKILL.md index 0149ae2..f4ee906 100644 --- a/.agents/skills/submit-benchmark-result/SKILL.md +++ b/.agents/skills/submit-benchmark-result/SKILL.md @@ -105,8 +105,7 @@ is installed and authenticated, run: SUBMISSION_ID="" RUN_URL="$(gh workflow run score-from-r2.yml \ --repo CodingThrust/problem-reductions-benchmark \ - --ref main \ - -f reset_results=false)" + --ref main)" RUN_ID="${RUN_URL##*/}" gh run watch "$RUN_ID" \ --repo CodingThrust/problem-reductions-benchmark \ @@ -120,9 +119,9 @@ gh pr list \ --jq ".[] | select(.headRefName | endswith(\"--${SHORT_ID}\")) | .url" ``` -Never set `reset_results=true` for a submission. If `gh` is unavailable, give the user the +If `gh` is unavailable, give the user the [workflow page](https://github.com/CodingThrust/problem-reductions-benchmark/actions/workflows/score-from-r2.yml) -and ask them to select **Run workflow** on `main` without enabling `reset_results`. +and ask them to select **Run workflow** on `main`. The intake authorization list and GitHub Actions permissions are separate. If GitHub hides the **Run workflow** button or returns 403, explain that a repository maintainer or a diff --git a/.github/workflows/score-from-r2.yml b/.github/workflows/score-from-r2.yml index e7cbcd1..7527d0e 100644 --- a/.github/workflows/score-from-r2.yml +++ b/.github/workflows/score-from-r2.yml @@ -16,11 +16,6 @@ name: Score submissions from R2 (private → aggregate) on: workflow_dispatch: # manual trigger (maintainer) - inputs: - reset_results: - description: "Wipe R2 results/ before scoring (clear stale scored files)" - type: boolean - default: false schedule: - cron: "0 16 * * *" # daily sweep of R2 for new submissions (16:00 UTC). @@ -69,12 +64,6 @@ jobs: AWS_DEFAULT_REGION: auto run: | mkdir -p incoming results/scored - # reset_results (manual): wipe prior scored detail so the board is rebuilt from - # scratch — clears stale/test entries that predate a rule change. - if [ "${{ github.event.inputs.reset_results }}" = "true" ]; then - echo "reset_results: wiping s3://$BUCKET/results" - aws s3 rm "s3://$BUCKET/results" --recursive --endpoint-url "$R2_ENDPOINT" || true - fi # Snapshot the exact queue keys before downloading. Only this immutable manifest # may be archived later; an object arriving while scoring remains in incoming/. aws s3api list-objects-v2 --bucket "$BUCKET" --prefix incoming/ \ diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index d3155d6..56dcc2b 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -138,8 +138,7 @@ def test_submit_skill_triggers_scoring_without_reset(self): t = _text(SUBMIT_SKILL) assert "gh workflow run score-from-r2.yml" in t assert "--repo codingthrust/problem-reductions-benchmark" in t - assert "-f reset_results=false" in t - assert "never set `reset_results=true`" in t + assert "reset_results" not in t assert "actions write permission" in t assert 'run_id="${run_url##*/}"' in t assert "gh run watch" in t @@ -158,3 +157,6 @@ class TestScorerWorkflow: def test_empty_r2_queue_is_valid(self): text = SCORER_WORKFLOW.read_text(encoding="utf-8") assert "jq -r '.[]? | select(endswith(\".json\"))'" in text + + def test_workflow_exposes_no_result_reset(self): + assert "reset_results" not in _text(SCORER_WORKFLOW)