From 64b5dc824a188f2403f8e04bb6017090c31101a1 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Mon, 20 Jul 2026 04:11:26 +0800 Subject: [PATCH 1/9] feat: enforce per-rule evidence budgets --- benchmark/agent_environment.py | 90 +++ benchmark/agent_pred.py | 53 ++ benchmark/agent_spool.py | 57 ++ benchmark/agent_submit.py | 62 +- benchmark/backend_score.py | 9 +- .../design/top50-evidence-budget-benchmark.md | 504 +++++++++++++ .../design/top50-evidence-budget-issues.md | 494 +++++++++++++ benchmark/env_setup.py | 10 +- benchmark/evidence_budget.py | 690 ++++++++++++++++++ benchmark/preflight.py | 4 +- benchmark/privilege_smoke.py | 79 ++ benchmark/process_control.py | 173 +++++ benchmark/submit_session.py | 133 +++- benchmark/tests/test_env_setup.py | 3 +- benchmark/tests/test_evidence_budget.py | 341 +++++++++ benchmark/tests/test_preflight.py | 1 + docker/Dockerfile | 29 +- 17 files changed, 2658 insertions(+), 74 deletions(-) create mode 100644 benchmark/agent_environment.py create mode 100644 benchmark/agent_pred.py create mode 100644 benchmark/agent_spool.py create mode 100644 benchmark/docs/design/top50-evidence-budget-benchmark.md create mode 100644 benchmark/docs/design/top50-evidence-budget-issues.md create mode 100644 benchmark/evidence_budget.py create mode 100644 benchmark/privilege_smoke.py create mode 100644 benchmark/process_control.py create mode 100644 benchmark/tests/test_evidence_budget.py diff --git a/benchmark/agent_environment.py b/benchmark/agent_environment.py new file mode 100644 index 0000000..c2ce4f5 --- /dev/null +++ b/benchmark/agent_environment.py @@ -0,0 +1,90 @@ +"""Run mini-swe shell actions as the dedicated unprivileged agent identity.""" +from __future__ import annotations + +import os +import subprocess + +from benchmark.process_control import ProcessLimits, run_capped_process + + +def run_as_agent( + command: str, + *, + cwd: str, + env: dict[str, str], + timeout: int, + uid: int, + gid: int, + extra_groups: tuple[int, ...] = (), + max_output_chars: int = 10_000, +) -> subprocess.CompletedProcess[str]: + """Execute one shell action after irreversibly dropping child privileges.""" + if os.name != "posix": + raise RuntimeError("the rankable agent environment requires POSIX privilege separation") + if os.geteuid() != 0 and (uid != os.getuid() or gid != os.getgid()): + raise PermissionError("runner must be root to launch a different agent identity") + + result = run_capped_process( + command, + shell=True, + cwd=cwd, + env=env, + timeout=timeout, + max_output_chars=max_output_chars, + combine_stderr=True, + uid=uid, + gid=gid, + extra_groups=extra_groups, + limits=ProcessLimits(cpu_seconds=max(1, timeout), memory_bytes=2 * 1024 ** 3, + file_bytes=64 * 1024 ** 2), + ) + if result.timed_out: + raise subprocess.TimeoutExpired(command, timeout, output=result.stdout) + return subprocess.CompletedProcess(command, result.returncode, stdout=result.stdout) + + +def make_agent_environment(session, *, uid: int, gid: int, + extra_groups: tuple[int, ...] = (), timeout: int = 300): + """Construct a pinned mini-swe LocalEnvironment with a privilege-dropping execute path.""" + from minisweagent.environments.local import LocalEnvironment + + class EvidenceLocalEnvironment(LocalEnvironment): + def execute(self, action: dict, cwd: str = "", *, timeout: int | None = None) -> dict: + command = action.get("command", "") + action_cwd = cwd or self.config.cwd or str(session.workdir) + if not session.admit_shell_action(command): + output = { + "output": "shell action budget exhausted\n", + "returncode": 75, + "exception_info": "", + } + self._check_finished(output) + return output + try: + result = run_as_agent( + command, + cwd=action_cwd, + env=os.environ | self.config.env, + timeout=timeout or self.config.timeout, + uid=uid, + gid=gid, + extra_groups=extra_groups, + max_output_chars=session.budget.max_output_chars, + ) + output = {"output": result.stdout, "returncode": result.returncode, + "exception_info": ""} + except Exception as error: + raw_output = getattr(error, "output", None) + raw_output = (raw_output.decode("utf-8", errors="replace") + if isinstance(raw_output, bytes) else (raw_output or "")) + output = { + "output": raw_output, + "returncode": -1, + "exception_info": f"An error occurred while executing the command: {error}", + "extra": {"exception_type": type(error).__name__, + "exception": str(error)}, + } + self._check_finished(output) + return output + + return EvidenceLocalEnvironment(cwd=str(session.workdir), timeout=timeout) diff --git a/benchmark/agent_pred.py b/benchmark/agent_pred.py new file mode 100644 index 0000000..9ca69f3 --- /dev/null +++ b/benchmark/agent_pred.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Thin agent-facing client for the evaluation-owned pred gateway.""" +from __future__ import annotations + +import json +import os +import sys +from benchmark.agent_spool import spool_request + +DEFAULT_REQUEST_TIMEOUT_SECONDS = 300 + + +def _request(payload: dict, request_id: str | None = None) -> dict: + return spool_request( + payload, + channel_env="PRB_PRED_DIR", + timeout_env="PRB_PRED_TIMEOUT", + default_timeout_seconds=DEFAULT_REQUEST_TIMEOUT_SECONDS, + service_name="pred gateway", + request_id=request_id, + ) + + +def _format_status(budget: dict) -> str: + pred = budget["pred_calls"] + solve = budget["solve_calls"] + return (f"pred calls: {pred['used']}/{pred['limit']} used, {pred['remaining']} remaining; " + f"solve calls: {solve['used']}/{solve['limit']} used, " + f"{solve['remaining']} remaining") + + +def main() -> None: + args = sys.argv[1:] + try: + if args == ["--budget-status"]: + response = _request({"op": "status", "cwd": os.getcwd()}) + print(_format_status(response["budget"])) + return + response = _request({"op": "pred", "args": args, "cwd": os.getcwd()}) + except (OSError, ValueError, json.JSONDecodeError, RuntimeError, TimeoutError) as error: + print(f"pred unavailable: {error}", file=sys.stderr) + raise SystemExit(2) + + if response.get("stdout"): + print(response["stdout"], end="" if response["stdout"].endswith("\n") else "\n") + if response.get("stderr"): + print(response["stderr"], end="" if response["stderr"].endswith("\n") else "\n", + file=sys.stderr) + raise SystemExit(int(response.get("returncode", 2))) + + +if __name__ == "__main__": + main() diff --git a/benchmark/agent_spool.py b/benchmark/agent_spool.py new file mode 100644 index 0000000..21dc20b --- /dev/null +++ b/benchmark/agent_spool.py @@ -0,0 +1,57 @@ +"""Shared thin-client transport for evaluation-owned agent commands.""" +from __future__ import annotations + +import json +import os +import time +import uuid +from pathlib import Path + +POLL_INTERVAL_SECONDS = 0.02 + + +def spool_request( + payload: dict, + *, + channel_env: str, + timeout_env: str, + default_timeout_seconds: int, + service_name: str, + request_id: str | None = None, +) -> dict: + """Publish one idempotent request and await its atomic response.""" + channel = os.environ.get(channel_env) + if not channel: + raise RuntimeError( + f"{service_name} is only available inside an active benchmark evaluation") + request_id = request_id or os.environ.get("PRB_REQUEST_ID") or uuid.uuid4().hex + if len(request_id) != 32 or any(char not in "0123456789abcdef" for char in request_id): + raise ValueError("request id must be 32 lowercase hexadecimal characters") + + channel_dir = Path(channel) + inbox = channel_dir / "inbox" + outbox = channel_dir / "outbox" + request_path = inbox / f"{request_id}.json" + response_path = outbox / f"{request_id}.json" + envelope = {"request_id": request_id, **payload} + + timeout = int(os.environ.get(timeout_env, default_timeout_seconds)) + attempts = int(os.environ.get("PRB_CLIENT_RETRIES", "1")) + 1 + for attempt in range(attempts): + temporary = inbox / f".{request_id}.{os.getpid()}.{attempt}.tmp" + temporary.write_text(json.dumps(envelope), encoding="utf-8") + os.replace(temporary, request_path) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + raw = response_path.read_text(encoding="utf-8") + except FileNotFoundError: + time.sleep(POLL_INTERVAL_SECONDS) + continue + response = json.loads(raw) + try: + response_path.unlink() + except FileNotFoundError: + pass + return response + raise TimeoutError(f"{service_name} did not respond within {timeout}s") diff --git a/benchmark/agent_submit.py b/benchmark/agent_submit.py index b3edef5..3170942 100644 --- a/benchmark/agent_submit.py +++ b/benchmark/agent_submit.py @@ -9,15 +9,13 @@ import argparse import json -import os import sys -import time -import uuid from pathlib import Path +from benchmark.agent_spool import spool_request + MAX_CERTIFICATE_BYTES = 1024 * 1024 DEFAULT_REQUEST_TIMEOUT_SECONDS = 300 -POLL_INTERVAL_SECONDS = 0.02 def _read_certificate(path: str) -> str: @@ -32,54 +30,33 @@ def _read_certificate(path: str) -> str: return raw.decode("utf-8") -def _request(payload: dict) -> dict: - channel = os.environ.get("PRB_SUBMIT_DIR") - if not channel: - raise RuntimeError("submit is only available inside an active benchmark evaluation") - channel_dir = Path(channel) - request_id = uuid.uuid4().hex - inbox = channel_dir / "inbox" - outbox = channel_dir / "outbox" - request_path = inbox / f"{request_id}.json" - temp_path = inbox / f".{request_id}.{os.getpid()}.tmp" - response_path = outbox / f"{request_id}.json" - - envelope = {"request_id": request_id, **payload} - temp_path.write_text(json.dumps(envelope), encoding="utf-8") - os.replace(temp_path, request_path) - - timeout = int(os.environ.get("PRB_SUBMIT_TIMEOUT", DEFAULT_REQUEST_TIMEOUT_SECONDS)) - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - try: - raw = response_path.read_text(encoding="utf-8") - except FileNotFoundError: - time.sleep(POLL_INTERVAL_SECONDS) - continue - response = json.loads(raw) - try: - response_path.unlink() - except FileNotFoundError: - pass - return response - raise TimeoutError(f"submit service did not respond within {timeout}s") +def _request(payload: dict, request_id: str | None = None) -> dict: + return spool_request( + payload, + channel_env="PRB_SUBMIT_DIR", + timeout_env="PRB_SUBMIT_TIMEOUT", + default_timeout_seconds=DEFAULT_REQUEST_TIMEOUT_SECONDS, + service_name="submit service", + request_id=request_id, + ) def main() -> None: parser = argparse.ArgumentParser( prog="submit", description="Submit one rule counterexample to the benchmark verifier. Every " - "submission, accepted or rejected, consumes one run-wide attempt.", + "submission, accepted or rejected, consumes one rule-scoped attempt.", ) parser.add_argument("certificate", nargs="?", help="Certificate JSON file, or - for stdin") parser.add_argument("--status", action="store_true", help="Show remaining attempts (free)") + parser.add_argument("--request-id", help=argparse.SUPPRESS) args = parser.parse_args() try: if args.status: if args.certificate: parser.error("--status does not take a certificate") - response = _request({"op": "status"}) + response = _request({"op": "status"}, args.request_id) print(f"submit budget: {response['used']}/{response['limit']} used, " f"{response['remaining']} remaining") return @@ -87,12 +64,12 @@ def main() -> None: parser.error("a certificate JSON file is required (use - for stdin)") try: text = _read_certificate(args.certificate) - response = _request({"op": "submit", "certificate_text": text}) + response = _request({"op": "submit", "certificate_text": text}, args.request_id) except (OSError, UnicodeError, ValueError) as e: # File/encoding/size failures are still submission attempts; let the service # account for them instead of failing locally without consuming budget. - response = _request({"op": "submit", "client_error": str(e)}) - except (OSError, ValueError, json.JSONDecodeError, RuntimeError) as e: + response = _request({"op": "submit", "client_error": str(e)}, args.request_id) + except (OSError, ValueError, json.JSONDecodeError, RuntimeError, TimeoutError) as e: print(f"submit unavailable: {e}", file=sys.stderr) raise SystemExit(2) @@ -102,9 +79,10 @@ def main() -> None: return reason = response.get("reason") or response.get("error") or "rejected" - label = "BUDGET_EXHAUSTED" if response.get("exhausted") else "REJECTED" + label = ("INFRASTRUCTURE_ERROR" if response.get("infrastructure_error") else + "BUDGET_EXHAUSTED" if response.get("exhausted") else "REJECTED") print(f"{label} attempt {used}/{limit} ({remaining} remaining): {reason}", file=sys.stderr) - raise SystemExit(2 if response.get("exhausted") else 1) + raise SystemExit(2 if response.get("exhausted") or response.get("infrastructure_error") else 1) if __name__ == "__main__": diff --git a/benchmark/backend_score.py b/benchmark/backend_score.py index 2a4d503..78bbb08 100644 --- a/benchmark/backend_score.py +++ b/benchmark/backend_score.py @@ -19,7 +19,6 @@ import hashlib import json import re -import shutil import sys from pathlib import Path @@ -123,10 +122,12 @@ def _assert_pred_version() -> None: """The backend is the authoritative verifier, so its pred must be the pinned version. Skip when no pred is on PATH (pred-free unit tests never verify real certificates) or when EXPECTED_PRED_VERSION is set empty; otherwise hard-fail on a mismatch.""" - if not shutil.which("pred"): + from benchmark.env_setup import find_pred_binary, verify_pred_version + try: + pred_binary = find_pred_binary() + except RuntimeError: return - from benchmark.env_setup import verify_pred_version - verify_pred_version("pred") # raises ValueError on mismatch + verify_pred_version(pred_binary) # raises ValueError on mismatch # ── status helpers ──────────────────────────────────────────────────────────── diff --git a/benchmark/docs/design/top50-evidence-budget-benchmark.md b/benchmark/docs/design/top50-evidence-budget-benchmark.md new file mode 100644 index 0000000..36d4585 --- /dev/null +++ b/benchmark/docs/design/top50-evidence-budget-benchmark.md @@ -0,0 +1,504 @@ +# Top-50 Evidence-Budget Benchmark — Product Design + +Status: proposed for the next benchmark version +Last updated: 2026-07-20 + +## Need + +The current benchmark gives a model the complete `problem-reductions` repository and lets +it decide how broadly and deeply to search and when to stop. The runner limits submissions, +but it does not limit model turns, shell actions, or calls to `pred`. Consequently, two runs +may receive radically different amounts of feedback from the target system. Wall-clock time +is not a suitable correction: the model cannot budget against it reliably, and it also +contains provider latency, retries, network delay, host load, and harness overhead. + +The next benchmark version must compare models under the same **logical opportunities to +obtain evidence**, rather than under the same elapsed time. A model first selects the 50 +rules it considers most likely to contain bugs, then investigates every selected rule with +the same non-transferable per-rule budget. The primary result remains the number of distinct +rules for which the evaluation-owned verifier accepts a counterexample. + +The benchmark is for researchers and developers comparing base models. It has one +standardized Model Track, using one reference harness and one frozen evaluation contract. +It intentionally does not compare rapidly changing coding-agent products or custom agent +systems. + +Success means that: + +- a longer-running session cannot obtain more target-system feedback than a shorter one; +- shell scripts, loops, and parallel subprocesses cannot bypass the `pred` or submission + budgets; +- all 50 selected rules receive the same investigation allowance; +- infrastructure failures are distinguishable from model failures and never become clean + zero scores; +- the score has a narrow, defensible interpretation: verified bug-finding yield under a + fixed evidence budget. + +## Prior art and landscape + +The design follows a common pattern in agent evaluation: bound interaction opportunities +directly and retain timeouts only as safety mechanisms. + +- [METR's time-horizon methodology](https://metr.org/time-horizons/) notes that exact agent + runtime varies with the inference provider and agent setup and is not itself the reported + capability measure. +- [Inspect](https://inspect.aisi.org.uk/agent-custom.html) exposes separate limits for + messages, tokens, turns, time, working time, and cost. Its working-time accounting excludes + delays such as rate-limit waiting, illustrating why resource dimensions should not be + collapsed into elapsed time. +- [InterCode](https://proceedings.neurips.cc/paper_files/paper/2023/file/4b175d846fb008d540d233c188379ff9-Paper-Datasets_and_Benchmarks.pdf) + bounds interaction turns and observation size. [OSWorld](https://github.com/xlang-ai/OSWorld) + and [OpenCUA](https://github.com/xlang-ai/OpenCUA) likewise publish results at explicit + action-step budgets. +- [tau2-bench](https://github.com/sierra-research/tau2-bench/blob/main/docs/cli-reference.md) + separates maximum steps, maximum errors, retries, trials, and an optional wall-clock + timeout instead of treating time as the sole budget. +- [SWE-agent](https://swe-agent.com/latest/usage/competitive_runs/) recommends a per-instance + cost or turn limit because otherwise individual instances can consume unbounded resources. +- [AI Agents That Matter](https://arxiv.org/abs/2407.01502) argues that agent results should + expose capability and resource use jointly, including Pareto-style comparisons, instead + of hiding unequal evaluation cost. +- Systems such as [AlphaCode 2](https://deepmind.google/AlphaCode2_Tech_Report.pdf) separate + large internal search from a small external submission allowance. This motivates separate + ledgers for `pred` feedback and formal bug submissions. + +Existing repository components provide useful building blocks: + +- `SubmissionSession` already owns an evaluation-side, atomic, append-only submission + ledger and an agent-facing file-spool transport. +- `verify.py` already applies payload, CPU, memory, output, and solver watchdog limits. +- API and CLI runners already share prompts and normalized result packaging. + +The repository does not yet provide an evaluation-owned `pred` gateway, per-rule budgets, +isolated per-rule episodes, shortlist freezing, or a distinction between verifier +infrastructure errors and rejected model submissions. Those gaps belong to this benchmark. + +## Features + +### Selected for the first release + +| Feature | User value | Rough implementation effort | +|---|---|---:| +| Static Top-50 triage and shortlist freezing | Measures risk prioritization without allowing dynamic experiments before commitment | 1–2 days | +| Fifty isolated per-rule investigation episodes | Gives every selected rule the same opportunity and removes order and long-context effects | 3–5 days | +| Evaluation-owned `pred`/`solve` gateway | Makes all target-system feedback countable and prevents script/parallelism bypasses | 2–4 days | +| Two submissions **per selected rule** | Allows one formal attempt and one feedback-driven correction without creating an unbounded proof loop | 1–2 days | +| Visible remaining-budget counters | Lets the model plan in logical units it can observe | about 1 day | +| Single-run scoring and audit schema | Produces one comparable, re-verifiable result under a named budget contract | 2–3 days | +| Internal budget-calibration pilot | Selects defensible turn and `pred` limits before the public contract is frozen | 2–4 days plus inference cost | + +The estimates are implementation-order estimates, not elapsed calendar commitments. The +existing submission and verification machinery is reused where possible. + +### Deferred + +#### Multiple seeds + +The first release records any available seed and sampling parameters but does not require +repeated runs, compute means or confidence intervals, or rank aggregate results. This can be +added after the single-run contract and its operating cost are understood. + +#### Fixed Top-50 diagnostic + +A future diagnostic may give every model the same 50 rules to isolate investigation ability +from risk-ranking ability. It is not required for the main benchmark, which deliberately +measures the combined workflow, and is deferred to keep the first release small. + +### Dropped + +#### System Track + +There is no leaderboard for Codex CLI, Claude Code, custom prompts, memory systems, +subagents, or other agent products. These systems change too quickly, attract less interest +for this benchmark, and confound model capability with orchestration. Existing CLI adapters +may remain useful for development, but their runs are not rankable in this track. + +#### Wall-clock, token, cost, or best-of-N ranking + +Wall-clock time remains a watchdog and an auxiliary measurement. Tokens and monetary cost +are reported when available. None is a primary score or tie-breaker, and ties in verified +bug count remain ties. The first release has no best-of-N selection. + +## Evaluation contract + +### Track + +The benchmark has one rankable track: **Standardized Model Track**. Rankable runs use the +same: + +- reference API harness and harness version; +- system and task prompts; +- pinned library repository and `pred` version; +- rule inventory; +- tool surface and network policy; +- triage and per-rule budgets; +- maximum model output and observation sizes; +- reasoning/sampling configuration, to the extent the model API exposes it; and +- submission verifier. + +The result identifies the model, provider/endpoint class, runner version, library commit, +budget-contract version, and exposed inference parameters. A custom coding-agent harness is +outside this track even if it uses the same underlying model. + +Rankable runs are produced and submitted only by the trusted internal team. The first release +does not accept untrusted external runs and therefore does not build remote execution or +cryptographic runner attestation. The backend still re-verifies bugs and ledger consistency, +but the operator trust boundary is explicit. + +### Phase 0: preflight + +Before any scored model generation, the runner checks the pinned repository, rule inventory, +`pred` gateway, and submission judge. Status and version probes are cached and free. If any +required service is unavailable, the run ends as `run_error` and is not rankable. + +### Phase 1: static triage + +The model receives the pinned repository and a canonical, cached inventory of all runnable +rules. It may inspect source code with ordinary read-only shell tools under a fixed triage +generation/action/observation budget. Dynamic `pred` operations and `submit` are unavailable +during this phase. + +Before the triage budget expires, the model must commit an ordered list of exactly 50 unique, +valid rule identifiers. It may attach a short, size-capped risk hypothesis for each rule. +The runner validates and freezes this artifact. The list cannot change after investigation +begins. + +Failure to produce 50 valid unique rules is an invalid run, not a 0-bug result. This avoids +silently comparing a 50-rule run with a smaller task. + +The order is retained for audit and future diagnostics such as Bugs@10 and Bugs@25, but the +first-release headline score uses all 50 rules. + +### Phase 2: isolated investigation + +The runner creates one fresh episode for each frozen rule. Every episode has: + +- a fresh model conversation and writable scratch directory; +- the same pinned read-only repository; +- the current rule identifier and its capped triage hypothesis; +- the same per-rule model-generation, shell-action, `pred`, `solve`, observation, and + submission limits; and +- a submission judge that accepts certificates only for the current rule. + +No conversation, scratch files, remaining budget, or model-authored memory transfers between +rules. Unused allowance is not transferable. The episodes may execute sequentially or in +parallel as an implementation detail, provided the model and infrastructure configuration is +identical and each ledger remains isolated. + +An accepted certificate ends that rule's episode immediately. Exhausting a counter leaves +the episode as completed without an accepted bug; it does not borrow from another rule. + +### Logical resource vector + +The frozen public contract is named by a vector rather than a time allowance: + +`Top50[T, E_t, M, E, P, P_solve, S=2, O]` + +where: + +- `T` is the number of usable model generations in static triage; +- `E_t` is the number of executed triage shell actions; +- `M` is the number of usable model generations per selected rule; +- `E` is the number of executed shell actions per selected rule; +- `P` is the number of agent-initiated dynamic `pred` invocations per selected rule; +- `P_solve` is the subset of `P` that may invoke `pred solve`; +- `S` is the number of submission opportunities per selected rule, fixed initially at 2; + and +- `O` contains fixed observation, command-output, and payload-size ceilings. + +`S=2` is a per-rule ledger. With 50 selected rules, a run can therefore make at most 100 +charged submissions, but there is no run-wide pool from which one rule can take another +rule's unused attempts. Once a rule is accepted, its unused submission opportunity expires. + +The exact values of `T`, `E_t`, `M`, `E`, `P`, `P_solve`, and `O` are selected by the internal +calibration pilot and then frozen with the benchmark version. + +Every observation includes a compact machine-generated status block, for example: + +```text +rule 17/50: maximum_independent_set_to_clique +model generations: 6/10 +shell actions: 6/10 +pred calls: 13/24 +solve calls: 5/10 +submit attempts: 1/2 +``` + +This is the model's planning signal. Wall-clock time is not. + +## Modules + +### 1. Inventory and Triage Controller + +Purpose: provide a canonical rule inventory, run static analysis, validate the shortlist, +and freeze the ordered Top50 artifact. + +Interface: + +- input: pinned repository metadata and triage budget contract; +- output: an immutable ordered list of 50 rule IDs plus capped hypotheses and a triage + ledger; +- dependencies: reference model harness, read-only repository, budget controller. + +### 2. Episode Runner + +Purpose: create 50 fresh, equal-budget investigation episodes and combine their immutable +ledgers without sharing model state or resources. + +Interface: + +- input: frozen Top50 artifact and per-rule budget contract; +- output: 50 episode records with completion, accepted bug, or infrastructure-error status; +- dependencies: model harness, `PredGateway`, `SubmitJudge`, observation renderer. + +### 3. Pred Gateway + +Purpose: be the only executable route to dynamic `pred` behavior and atomically enforce `P` +and `P_solve` for the current episode. + +Interface: + +- agent command: a `pred`-compatible shim; +- runner API: status, execute, ledger, and health operations; +- output: idempotent response plus updated per-rule counters; +- dependencies: hidden real `pred` binary, pinned execution environment, file-spool or RPC + transport. + +### 4. Submit Judge + +Purpose: enforce exactly two non-transferable formal attempts for the current rule and +retain the verifier-authoritative certificate ledger. + +Interface: + +- agent command: `submit ` and free `submit --status`; +- runner API: status, verify, ledger, and health operations; +- output: accepted, rejected, exhausted, or infrastructure-error response; +- dependencies: existing certificate verifier and hidden internal `pred` access. + +### 5. Scoring and Calibration + +Purpose: validate rankability, compute the single-run headline score, expose audit metrics, +and determine the fixed public budgets before release. + +Interface: + +- input: triage ledger, 50 episode ledgers, inference usage, and run metadata; +- output: versioned submission artifact and internal pilot analysis; +- dependencies: schema validator and backend re-verifier. + +## Technical approaches + +### Inventory and triage + +**Chosen: one bounded static-analysis session followed by an atomic shortlist commit.** This +directly measures the desired prioritization behavior while preventing the model from using +dynamic fuzzing to decide what belongs in the Top50. + +Alternatives considered: + +- Ask the model to score every rule independently. This is easier to normalize per rule but + converts strategic prioritization into 290+ expensive microtasks and loses cross-rule + comparison in one context. +- Use a benchmark-authored heuristic to prefilter candidates. This is cheaper but removes a + central model ability from the task and risks encoding maintainer bias. + +### Episode execution + +**Chosen: 50 fresh, equal-budget, per-rule episodes.** This ensures that all shortlisted +rules receive the same opportunity and that later rules are not disadvantaged by context +growth or earlier spending. + +Alternatives considered: + +- One global investigation session with a run-wide pool. This measures adaptive allocation, + but recreates unequal rule coverage and makes scores difficult to interpret. +- A global pool plus a per-rule ceiling. This blocks extreme concentration but still makes a + miss ambiguous: the model may be weak on the rule or simply have allocated less budget. + +### Pred enforcement + +**Chosen: an evaluation-owned shim and service modeled on `SubmissionSession`.** The real +binary is not placed directly on the agent's `PATH`; the shim atomically reserves budget +before spawning it. All agent shells, Python programs, loops, and parallel subprocesses +share the same per-episode ledger. + +Alternatives considered: + +- Infer calls from shell commands or trajectories. One shell command can start hundreds of + subprocesses, so this is neither complete nor enforceable. +- Expose the real binary and ask the agent to self-report. This is unauditable and permits + accidental bypasses. + +Simple invocation counting is preferred over weighted command costs. Each agent-initiated +`create`, `reduce`, `solve`, `evaluate`, or `extract` consumes one `P`; `solve` also consumes +one `P_solve`. Cached inventory, version, help, and status operations are free. `pred` calls +made internally by the submission verifier do not consume the agent's investigation budget. + +Model-authored automation remains allowed: writing a fuzzer or batch script is useful agent +behavior. Its actual gateway invocations are charged individually, so automation is an +ability rather than a loophole. + +### Submission enforcement + +**Chosen: adapt the existing evaluation-owned ledger from run-wide accounting to one ledger +per rule.** A rejected, malformed, duplicate, wrong-rule, or model-caused verifier timeout +consumes one of that rule's two opportunities. `submit --status` is free. Acceptance closes +the episode. + +Pure verifier, gateway, container, or transport failures do not consume an opportunity. +Requests carry stable idempotency keys; retrying the same request after a lost response +returns the stored result instead of charging again. An unrecoverable judge failure marks the +run unrankable rather than converting the episode into a rejection. + +Alternatives considered: + +- Keep one shared 100-attempt run pool. Although its total matches 50 × 2, it lets one rule + consume attempts intended for another and violates the equal-opportunity contract. +- Allow only one attempt. This measures first-shot rigor, but provides no bounded opportunity + to use verifier feedback to correct a nearly valid certificate. +- Allow attempts until acceptance. This rewards persistence with unequal external feedback + and recreates the original bias. + +### Scoring + +**Chosen: count distinct accepted rules under the complete frozen budget vector.** The +headline metric is: + +`VerifiedBugs@Top50[T,E_t,M,E,P,P_solve,S=2,O]` + +It is the number from 0 to 50 of selected rules with at least one certificate accepted by the +evaluation-owned verifier. It is reported for one run in the first release. Equal counts are +ties. + +Audit and efficiency fields include: + +- ordered Top50 and hypotheses; +- Bugs@10, Bugs@25, and Bugs@50 along the frozen shortlist order; +- accepted on submission 1 versus submission 2; +- `pred` and `solve` calls per accepted bug; +- cap-hit counts for each logical resource; +- token usage, monetary cost when available, and elapsed time; and +- all infrastructure retries and errors. + +These fields explain a run but do not override the headline ranking. + +Because there is no exhaustive ground truth for which rules are buggy, the benchmark must +not call unaccepted rules clean or report Top50 precision/recall. It measures verified bug +discovery yield, not complete bug prevalence. + +### Internal budget calibration + +**Chosen: run a small pre-release budget grid on development data, then freeze one contract.** +Candidate values may start around `M ∈ {6, 10, 14}` and `P ∈ {8, 16, 32}`, with a separate +`P_solve` ceiling. These are pilot coordinates, not proposed public values. + +The pilot uses an older library snapshot or other non-ranking development target and a small +set of available models. It examines: + +- verified bugs as each budget increases; +- the fraction of episodes hitting each cap; +- marginal bugs gained per additional generation or `pred` call; +- infrastructure failure rate; and +- whether one counter consistently binds long before the others. + +The selected contract should sit near the useful-yield knee: large enough that most episodes +can conduct a meaningful experiment, but before extra calls mostly add cost. The chosen +values and rationale are published and versioned. Formal leaderboard runs execute only that +one frozen budget, not the grid. The pilot does not introduce a multi-seed requirement. + +## Counter and error semantics + +Counters are charged for opportunities controlled by the model and retried for failures +controlled by the evaluation infrastructure. + +| Event | Counter effect | Result semantics | +|---|---|---| +| Provider returns a usable model response | consume one `T` or `M` | response may lead to an action | +| Model emits invalid action/format | consume one `T` or `M`; no shell action if none executes | model failure | +| API 429/5xx/disconnect before any usable response | no model-generation charge; fixed infrastructure retry policy | unrecoverable exhaustion makes run unrankable | +| Shell command starts | consume one `E_t` or `E` | nonzero exit and model-caused timeout still consume | +| Dynamic `pred` request is admitted | atomically consume one `P`; `solve` also consumes `P_solve` | invalid arguments, bad model-authored input, nonzero exit, and target timeout consume | +| Gateway fails before admitting the request | no charge; retry idempotently | unrecoverable failure makes run unrankable | +| Submission is admitted for verification | atomically consume one of the current rule's two `S` | malformed, rejected, duplicate, and wrong-rule attempts consume | +| Judge infrastructure fails | no `S` charge; retry the same request idempotently | unrecoverable failure makes run unrankable | +| Status/help/version/cached inventory probe | free | rate-limited separately if abuse protection is needed | + +A reservation is written before the underlying process starts so parallel requests cannot +overspend. Every ledger record includes episode ID, rule ID, request ID, command class, +counter values before and after, outcome class, and timestamps for auditing. Timestamps are +not part of the score. + +## Quality requirements + +### Fairness and reproducibility + +- Budget values, prompts, schemas, repository commit, `pred`, harness, and verifier are + immutable within a benchmark version. +- All 50 per-rule episodes receive identical non-transferable limits. +- The real `pred` binary and verifier-only interfaces are inaccessible to the agent. +- Network access from the agent environment is disabled; provider access remains in the + runner. +- The execution image, host class, watchdogs, memory limits, and file/output ceilings are + fixed for official runs and recorded where relevant. + +### Reliability + +- Budget and result ledgers are evaluation-owned, append-only, thread-safe, and idempotent. +- Crashes preserve accepted certificates and audit logs, but any incomplete infrastructure + run is marked unrankable rather than published as a clean partial score. +- Free preflight probes must succeed before scored generations begin. +- Watchdogs remain on model calls, commands, `pred`, verification, and the complete run to + terminate wedged processes; they protect the service and do not define capability. + +### Compatibility + +- The agent-facing `pred` shim preserves the documented CLI shape wherever possible so + existing prompts and model-authored scripts continue to work. +- Result schema changes are tied to a benchmark-version bump. Older whole-repository + submissions remain readable as historical results but are not mixed into the new ranking. + +## What the benchmark can claim + +Under a standardized harness and fixed logical budget, the score reflects a combination of: + +- static risk ranking across a large rule library; +- source-code understanding; +- high-information experiment design; +- counterexample construction and minimization; +- interpretation of bounded `pred` and verifier feedback; +- disciplined use of limited formal submissions; and +- end-to-end reliability in producing verifier-accepted evidence. + +It does not isolate pure model reasoning from all provider implementation details, equalize +hidden inference FLOPs, measure complete recall over all bugs, compare custom agent systems, +or rank network and runtime speed. + +## Acceptance criteria + +The first release is ready when automated tests demonstrate that: + +1. triage cannot call dynamic `pred` or `submit`, and only an exact valid Top50 can start + investigation; +2. all 50 episodes start with identical counters and isolated model/scratch state; +3. direct shell use, Python subprocesses, loops, and concurrent requests cannot exceed `P` + or `P_solve`; +4. no rule can make more than two charged submissions or use another rule's unused attempt; +5. accepted certificates close their episode and score at most one bug per rule; +6. model-caused invalid operations charge the documented counters; +7. injected provider, gateway, transport, and verifier infrastructure failures follow the + documented no-charge/idempotent behavior and make unrecoverable runs unrankable; +8. every observation exposes authoritative remaining counters; +9. the backend recomputes the headline score only from accepted evaluation-owned ledgers; + and +10. the published result identifies the complete frozen budget contract and necessary + provenance. + +## Open questions before implementation + +These are intentionally resolved by implementation discovery or the calibration pilot, not +by arbitrary values in the design: + +- final numeric values for `T`, `E_t`, `M`, `E`, `P`, `P_solve`, and `O`; +- the smallest stable inference-parameter contract shared by supported model APIs; +- migration and display treatment for historical unlimited whole-repository results. diff --git a/benchmark/docs/design/top50-evidence-budget-issues.md b/benchmark/docs/design/top50-evidence-budget-issues.md new file mode 100644 index 0000000..4bb1b22 --- /dev/null +++ b/benchmark/docs/design/top50-evidence-budget-issues.md @@ -0,0 +1,494 @@ +# Top-50 Evidence-Budget Benchmark — Implementation Issue Drafts + +Status: filed as GitHub issues #65–#68; implementation in progress +Source design: [Top-50 Evidence-Budget Benchmark](top50-evidence-budget-benchmark.md) + +## Agreed boundary + +This batch implements one trusted-team, standardized Model API track in which a model commits +to its own Top50 and investigates every selected rule under identical logical budgets. It does +not implement multiple seeds, a Fixed Top50 diagnostic, a coding-agent/System Track, external +submission attestation, or ranking by elapsed time, tokens, or cost. + +The four issues are deliberately coarse. Each delivers a complete component with one compound +behavioural acceptance suite; internal classes and substeps are not separate issues. + +## Milestones and dependency graph + +```text +M1 Enforceable evidence budget +└── I1 Evidence Budget Service + └── M2 End-to-end Top50 benchmark + └── I2 Self-selected Top50 Runner + └── I3 vNext result and ranking contract + └── M3 Calibration and release + └── I4 Calibrate and publish the frozen contract +``` + +I3 depends on I2 because its schema and scorer validate the actual Top50 run artifact. I4 +depends on I1–I3 because calibration must exercise the same implementation that will be +released and must publish the selected values through the versioned result contract. + +--- + +## I1 — Build the evaluation-owned Evidence Budget Service + +### Background + +The current benchmark exposes the real `pred` executable to the model and limits only a +run-wide submission count. One shell action can start hundreds of `pred` subprocesses, so turns +and shell commands do not bound how much target-system feedback a model receives. The +[Top-50 design](top50-evidence-budget-benchmark.md) instead gives every selected rule an +independent allowance for model generations, shell actions, `pred` calls, `solve` calls, and +two formal submissions. + +The runner image currently executes as root and installs the real binary at +`/usr/local/bin/pred`. Replacing `PATH` alone is therefore not enforcement: a model-authored +script could find and execute the binary by absolute path. This issue owns the complete trusted +service and OS boundary that make the logical budget enforceable. + +### Objective + +Build one evaluation-owned Evidence Budget Service that atomically accounts for a single rule +episode, exposes `pred`-compatible and `submit` shims to an unprivileged agent shell, and keeps +the real `pred`, verifier, counters, and accepted-certificate ledger outside the agent's access. + +Each rule receives exactly two non-transferable submission opportunities. Rejected, malformed, +duplicate, wrong-rule, and other model-caused attempts consume one; pure gateway, transport, +container, or verifier infrastructure errors do not. Retrying the same request after a lost +response must be idempotent. + +### Interface (Input → Output) + +- **Input:** episode ID, current rule ID, frozen budget contract + `{M, E, P, P_solve, S=2, O}`, pinned real `pred` path, and certificate verifier. +- **Agent surface:** an unprivileged writable scratch directory; a `pred` shim supporting the + documented CLI shape; `submit `; and free status/help/version/cached-inventory + operations. +- **Output:** immutable per-episode ledgers for model generations, shell actions, admitted + `pred`/`solve` requests, submission attempts, idempotent responses, accepted certificate, and + classified infrastructure errors. + +### Technical recommendations + +- Reuse the atomic file-spool, directory-FD, and append-only ledger patterns in + `benchmark/submit_session.py` instead of introducing a network service. +- Add separate non-root container identities for model-authored shell actions and admitted + `pred` subprocesses. The real binary is executable only by the oracle identity; both identities + share only the episode scratch group. The privileged Python parent owns counters and spawning, + but never runs model-parameterized `pred` as root. +- Reserve a counter under a lock before spawning a process. A dynamic `create`, `reduce`, + `solve`, `evaluate`, or `extract` request consumes one `P`; `solve` additionally consumes one + `P_solve`. Nonzero exits, invalid arguments, pathological model input, and target timeouts + consume the reservation. +- Give requests caller-stable idempotency keys and retain completed responses. Do not generate + a new logical request ID merely because the client lost the first response. +- Refactor `SubmissionSession` into, or adapt it behind, a rule-scoped judge. Treat verifier + exceptions as `infrastructure_error`, not ordinary rejection. Acceptance closes the rule and + expires its unused submission attempt. +- Keep verifier-internal `pred` calls outside the agent's `P`/`P_solve` ledger. + +These are recommendations; an alternative implementation is acceptable if it passes the same +privilege, concurrency, charging, and idempotency checks. + +### Verification + +Add one integration acceptance suite and run: + +```bash +pytest -v benchmark/tests/test_evidence_budget.py +``` + +It must pass named cases demonstrating all of the following observable behaviour: + +1. With `P=4` and `P_solve=1`, a model-authored loop that starts 20 sequential or concurrent + `pred` processes receives exactly four admitted dynamic calls, at most one admitted solve, + and `budget_exhausted` for the rest; the authoritative ledger contains the same counts. +2. Running the real binary's absolute path from the agent shell fails with a permission error, + while the `pred` shim succeeds. This is the negative control proving `PATH` replacement is + not the only protection. +3. Invalid JSON, invalid CLI arguments, nonzero `pred`, and a model-caused `pred` timeout consume + admitted `P`; an injected gateway failure before admission consumes nothing. +4. Rule A can make two charged submissions and no third. Rule B still has two, proving the + allowance is per rule rather than a shared run-wide pool. Acceptance on Rule B's first attempt + closes Rule B and expires its second attempt. +5. Malformed, rejected, duplicate, and wrong-rule submissions consume `S`; an injected verifier + exception consumes nothing and returns `infrastructure_error`. +6. Replaying one completed request ID after simulating a lost response returns the identical + stored response without changing any counter. Two genuinely different concurrent request IDs + can never overspend a counter. +7. Status/help/version/cached inventory probes do not change any charged counter. + +As a regression check, also run: + +```bash +pytest -v benchmark/tests/test_submit_session.py benchmark/tests/test_verify.py +``` + +Existing certificate acceptance/rejection behaviour must remain green. A test fixture that makes +the real binary executable by the agent must make the absolute-path negative control fail, proving +that the acceptance suite detects a broken privilege boundary. + +### Out of scope + +- Top50 selection and episode orchestration; +- choosing the public numeric values of `M`, `E`, `P`, `P_solve`, or output ceilings; +- protecting the runner from a malicious trusted team member or cryptographically attesting a + remotely uploaded run; and +- changing `problem-reductions` or the `pred` CLI upstream. + +--- + +## I2 — Implement the Self-selected Top50 Runner + +### Background + +The existing runner launches one unlimited whole-repository conversation. Models can inspect +different numbers of rules, spend different amounts of target-system feedback on each one, and +carry discoveries and accumulated context throughout the session. The +[Top-50 design](top50-evidence-budget-benchmark.md) replaces that with a bounded static triage +followed by 50 fresh, equal-budget rule episodes. + +This issue owns the complete run workflow. A Top50 result is meaningful only if the shortlist is +frozen before dynamic experimentation, contains exactly 50 valid unique rules, and every rule +receives a clean context, clean scratch directory, and new Evidence Budget Service from I1. + +### Objective + +Replace the unlimited whole-repository Model API session with a two-phase `Top50Runner`: + +1. a bounded source-only triage conversation that atomically commits an ordered Top50 with + capped hypotheses; and +2. 50 sequential, isolated investigation episodes with identical non-transferable budgets. + +Automatically inject authoritative remaining counters into every observation. Preserve partial +audit artifacts after a crash, but mark any unrecoverable provider, gateway, judge, or container +failure as `run_error` so it cannot be ranked as a clean completion. + +### Interface (Input → Output) + +- **Input:** pinned repository and canonical rule inventory; standardized model client and + inference settings; triage contract `{T, E_t, O_t}`; per-rule contract + `{M, E, P, P_solve, S=2, O}`; and the I1 service factory. +- **Triage output:** immutable ordered JSON artifact containing exactly 50 unique canonical rule + IDs and optional size-capped hypotheses, plus its generation/action ledger. +- **Run output:** one Top50 run record containing the shortlist, triage ledger, exactly 50 ordered + episode records, usage metadata, completion status, and any classified infrastructure error. + +### Technical recommendations + +- Keep `mini-swe-agent`/LiteLLM as the sole rankable reference harness. Remove the forced + unlimited `step_limit=0` behaviour and account for usable responses and executed shell actions + explicitly. +- Provide triage with a runner-cached rule inventory and read-only source tree. Do not expose the + dynamic `pred` or `submit` shims until after the shortlist passes validation. +- Commit the shortlist through an evaluation-owned command or spool rather than scraping final + prose. Reject lists with the wrong count, duplicates, unknown IDs, oversized hypotheses, or a + second commit attempt. +- Start each rule with a fresh model message history, fresh scratch directory, and fresh I1 + service. Pass only the current rule ID and its capped triage hypothesis; transfer no generated + files or model-authored memory. +- Execute the 50 episodes sequentially in the first release. This avoids provider throttling and + host contention becoming rule-order-dependent. Because budgets are isolated, parallelism may be + revisited later without changing the public contract. +- Provider 429/5xx/disconnects before a usable response follow one frozen retry policy and do not + consume `T`/`M`. A usable but malformed model action consumes its generation. An executed shell + action consumes `E_t`/`E`, including nonzero exit and model-caused timeout. +- End a rule immediately after acceptance or budget exhaustion. Do not transfer unused allowance. + +### Verification + +Add a deterministic fake-model end-to-end acceptance suite and run: + +```bash +pytest -v benchmark/tests/test_top50_runner.py +``` + +The suite must pass named scenarios demonstrating: + +1. A fake triage response commits 50 known unique rule IDs; the runner then creates exactly 50 + episode records in the frozen order, each starting with identical full counters and an empty + conversation/scratch state. +2. The fake model writes a sentinel file and message in episode 1; episode 2 cannot observe + either. Spending all `pred` and submit allowance in episode 1 leaves episode 2's counters full. +3. Dynamic `pred` and `submit` attempts during triage are unavailable and do not produce target + feedback. A shortlist with 49 entries, 51 entries, a duplicate, an unknown rule, or an + oversized hypothesis is rejected before any investigation episode starts. These are the + negative controls for shortlist enforcement. +4. Every model observation contains authoritative `rule i/50`, generation, shell, `pred`, + `solve`, and `submit` usage/limit values matching the underlying ledger. +5. Acceptance in an episode stops that episode without a further model call. Exhausting any hard + counter ends only that rule and does not start an extra action. +6. An injected 429 before a usable response retries without consuming a model generation; a + malformed usable response consumes one. Exhausted infrastructure retries produce `run_error` + and an unrankable record rather than a clean 0-bug run. +7. The rankable execution path rejects `claude-code`, `codex`, a custom strategy file, or a custom + prompt config. The same adapters may remain callable in an explicitly unranked development + mode. + +Also run the existing no-API runner tests: + +```bash +pytest -v benchmark/tests/test_run_mini.py benchmark/tests/test_run_submission.py benchmark/tests/test_whole_repo.py +``` + +Update or replace whole-repository expectations so they assert the versioned historical path, +not the new rankable path. A fake model that tries to return a second shortlist or reuse one +episode's workspace must make the new suite fail. + +### Dependencies + +Depends on I1, the evaluation-owned Evidence Budget Service. + +### Out of scope + +- Fixed Top50 evaluation; +- multiple seeds or aggregate confidence intervals; +- parallel rule episodes; +- custom agent systems and coding-agent CLI ranking; and +- selecting final public budget values. + +--- + +## I3 — Define and enforce the vNext result and ranking contract + +### Background + +The current submission schema describes one whole-repository session with a shared +`submit_limit`, a flat `submit_log`, `agent_mode: whole-repo`, and token efficiency as a +tie-breaker. Those fields cannot prove that a valid Top50 was frozen or that 50 rules received +identical non-transferable budgets. + +The new private submission artifact must carry the complete evaluation-owned triage and episode +ledgers so the backend can recompute rankability and score. The public leaderboard must continue +to expose aggregate data only; rule identities, hypotheses, certificates, and trajectories remain +private answer-key material. + +### Objective + +Introduce a versioned Top50 submission and scoring contract, make the backend independently +validate its shortlist and all 50 episode ledgers, re-verify every accepted certificate, and rank +only clean Standardized Model Track results by distinct accepted rules. + +Historical unlimited whole-repository and coding-agent results must remain viewable under their +original contract but must never be mixed into the Top50 ranking. Equal verified bug counts remain +ties; elapsed time, token usage, cost, and bugs/Ktok are informational only. + +### Interface (Input → Output) + +- **Input:** I2 Top50 run record, evaluation-owned ledgers, model/provider and exposed inference + parameters, pinned repository/`pred`/runner identifiers, and accepted certificates. +- **Private output:** schema-valid versioned submission containing the complete budget vector, + ordered Top50, triage ledger, exactly 50 rule-bound episode ledgers, usage, errors, and + certificates. +- **Scored output:** backend-derived `VerifiedBugs@Top50[T,E_t,M,E,P,P_solve,S=2,O]`, Bugs@10/25/50, + first-versus-second-attempt acceptance, calls per accepted bug, cap-hit diagnostics, and + rankability verdict. +- **Public output:** aggregate leaderboard entry with model and contract provenance, headline + verified bug count, and non-ranking efficiency fields; no answer-key data. + +### Technical recommendations + +- Add an explicit benchmark-contract identifier instead of inferring semantics from optional + legacy fields. Keep a deliberate legacy parser for historical data rather than making the new + schema accept ambiguous mixtures. +- Validate that the shortlist has exactly 50 canonical unique IDs, episode order matches it, + every configured limit is identical, each usage count is within its limit, `P_solve <= P`, + `S=2` per rule, and request IDs/attempt numbering are internally consistent. +- Derive result rows only from evaluation-owned accepted submit records for the matching current + rule. Re-run `verify()` on every accepted certificate; ignore self-reported `bugs_found`. +- Treat missing episodes, infrastructure errors, CLI/custom-harness provenance, custom prompts, + or inconsistent ledgers as unrankable. Preserve the private artifact for debugging. +- Version the public board by benchmark contract. Remove token efficiency from sort keys for the + new contract while leaving old-board rendering intact. +- The operators are a trusted internal team. Do not add remote-run attestation, signatures, or + external-user authentication to this issue. + +### Verification + +Run the contract and scorer acceptance suites: + +```bash +pytest -v benchmark/tests/test_top50_submission.py benchmark/tests/test_verify_submission.py benchmark/tests/test_backend_score.py +``` + +They must use a schema-valid synthetic Top50 artifact and demonstrate: + +1. The valid artifact has exactly 50 frozen rules and 50 matching episode ledgers; the backend + ignores a deliberately false self-reported score and returns the distinct count implied by + accepted, re-verified certificates. +2. Two accepted certificates for one rule count once. Accepted rules at shortlist positions 7, + 18, and 41 produce Bugs@10 = 1, Bugs@25 = 2, and Bugs@50 = 3. +3. A certificate for a rule outside the shortlist, a certificate submitted from another rule's + episode, a third attempt, a shared `100`-attempt pool, duplicate/missing episode, over-limit + counter, changed budget, or forged request ordering makes the run unrankable. These are the + negative controls proving the scorer does not trust the envelope. +4. A pure `run_error` or unrecoverable infrastructure episode is preserved privately but excluded + from ranking. A rejected model submission remains a clean charged attempt and does not make the + run an infrastructure failure. +5. A `codex`/`claude-code` agent mode or custom prompt provenance is excluded from the Top50 board. + A canonical API harness run is accepted. +6. Public output contains no rule ID, hypothesis, certificate, source instance, target config, + submit reason, or trajectory. The aggregate guard fails if any such answer-key field is added. +7. For equal verified bug counts, changing tokens, cost, or elapsed time does not change rank + order; the UI represents the entries as tied. A larger verified count ranks first. +8. One legacy v0.6 whole-repository fixture remains readable in its historical view but cannot + enter or replace an entry in the Top50 board. + +Also run: + +```bash +python .github/scripts/check_aggregate.py site/results.json +``` + +against a generated development board. It must print no errors for the aggregate-only fixture; +injecting one `rule` or `certificate` field must make the guard exit nonzero. + +### Dependencies + +Depends on I2 so the contract validates the real `Top50Runner` artifact rather than a parallel +invented format. + +### Out of scope + +- cryptographic attestation or accepting untrusted external runs; +- multiple-seed aggregation, best-of-N, or confidence intervals; +- Fixed Top50 scores; +- ranking custom agent systems; and +- declaring unaccepted rules bug-free or reporting Top50 precision/recall. + +--- + +## I4 — Calibrate and publish the first frozen Top50 budget contract + +### Background + +I1–I3 make logical budgets enforceable and scoreable, but the public values for triage turns, +per-rule model turns, shell actions, `pred`, `solve`, and output ceilings should not be chosen by +intuition. The [Top-50 design](top50-evidence-budget-benchmark.md) calls for one internal +pre-release calibration over development data, followed by one frozen budget for all formal runs. + +The calibration grid is a development procedure, not a public multi-budget leaderboard and not a +multiple-seed requirement. The released benchmark runs each model once at the selected contract. + +### Objective + +Provide a reproducible internal calibration workflow, execute it on an explicitly non-ranking +development target, publish the measured trade-offs and chosen knee, then freeze those values +through configuration, prompts, preflight, schema, documentation, CI, and leaderboard display for +the next benchmark version. + +This issue is the release gate: after it lands, no rankable run can request unlimited or custom +budgets, and all user-facing documentation tells the same Self-selected Top50 story. + +### Interface (Input → Output) + +- **Input:** development target identifier; a small named set of internal model runs; candidate + contracts such as `M in {6,10,14}` and `P in {8,16,32}`; I1–I3 artifacts; and measured token, + cost, elapsed-time, cap-hit, retry, and accepted-bug data. +- **Calibration output:** machine-readable observations plus a checked-in Markdown report showing + yield, cap-hit rates, marginal gain, infrastructure errors, and the reason for selecting each + public limit. +- **Release output:** one immutable named budget contract referenced by runner defaults, prompt, + result schema, scorer, docs, preflight, image/version metadata, CI, and leaderboard. + +### Technical recommendations + +- Add a small analysis command that consumes completed development artifacts. Do not embed a + supposedly objective automatic knee detector; show the raw table/curves and record the team's + explicit selection rationale. +- Use an older library snapshot or mark every pilot artifact non-ranking. A pilot model set and + candidate grid may be small; do not introduce a repeated-seed requirement. +- Start from the design's candidate coordinates, then adjust only when cap-hit or failure data + shows that one counter binds pathologically. Record every tested contract, including failed + runs, instead of publishing only the winner. +- Replace current `step_limit: 0`, `SUBMIT_LIMIT=100`, whole-repository prompt language, custom + strategy hooks on the rankable path, and token-efficiency tie-breaking. Keep process/pred/model + watchdogs as fixed safety controls. +- Update `README.md`, `CONTRIBUTING.md`, `submission.env.example`, `benchmark/config.yaml`, runner + help, preflight, Docker image metadata, schema docs, and the public site together. +- After release, reconcile stale issues #2, #4, and #42 by linking the new implementation and + stating precisely which old recommendations were implemented, deferred, or dropped. Closing or + editing those issues remains a maintainer action after the implementation is merged. + +### Verification + +The implementation PR must include the calibration inputs/report and pass: + +```bash +pytest -v -m "not integration" +``` + +followed in the pinned runner image by: + +```bash +python -m benchmark.verify --calibrate +``` + +A reviewer must also be able to check the committed evidence offline in a few seconds: + +```bash +python -m benchmark.calibrate_budget \ + --check benchmark/docs/budget-calibration.json +``` + +It must print `PASS: calibration evidence matches ` and exit 0. Changing any +selected limit, deleting a tested candidate, or making the Markdown report disagree with the +machine-readable evidence must make the same command exit nonzero and identify the mismatch. + +The release evidence must additionally demonstrate: + +1. The calibration report lists every tested contract and, for each, model/target provenance, + verified bugs, cap-hit counts, usage, retries, infrastructure failures, token/cost/time + references, and marginal yield. The chosen values exactly match the checked-in named contract. +2. At least one smaller and one larger candidate surround the selected `M` and `P` values, so the + report demonstrates a trade-off rather than merely documenting a predetermined number. +3. Starting a rankable run with any unlimited counter, `S != 2`, custom budget, custom prompt, + custom strategy, or coding-agent backend fails preflight before a scored model generation. + These are the release negative controls. +4. A standard fake Top50 run completes with exactly 50 episodes and emits a schema-valid artifact + carrying the frozen contract ID. Changing one embedded limit makes backend validation fail. +5. A forced hung model call and a forced hung `pred` child are killed by watchdogs and recorded as + infrastructure/model outcomes according to the design; changing the watchdog duration does not + change the named logical budget or score formula. +6. The README and leaderboard describe the primary metric as verified distinct bugs at the frozen + Top50 evidence budget. They do not present time, cost, tokens, Fixed Top50, multiple seeds, or a + System Track as primary ranking dimensions. +7. Historical whole-repository results remain visible under their old contract and cannot be + compared or sorted into the new Top50 table. + +Before merging the release, run the repository's required CI and calibration workflows. A PR must +not be merged until CI is green and the user has explicitly approved the merge. + +### Dependencies + +Depends on I1, I2, and I3. + +### Out of scope + +- a public budget grid or bugs-versus-budget leaderboard; +- multiple seeds and confidence intervals; +- Fixed Top50 diagnostics; +- System Track/coding-agent ranking; +- external contributor submission; and +- choosing future benchmark versions' budgets without a new calibration and version bump. + +## Coverage scan + +| Design area | Covered by | +|---|---| +| Logical resource model and charging semantics | I1 | +| Privilege boundary and `pred` bypass prevention | I1 | +| Per-rule two-attempt submission judge | I1 | +| Static triage and atomic Top50 freeze | I2 | +| Fresh equal-budget rule episodes and counter visibility | I2 | +| Provider/infrastructure failure semantics | I1, I2, I3 | +| Versioned private result and backend re-verification | I3 | +| Headline score, diagnostics, privacy, and historical separation | I3 | +| Internal pilot and public budget freeze | I4 | +| Prompt, preflight, docs, CI, and release migration | I4 | +| Multiple seeds, Fixed Top50, System Track | explicitly out of scope in all relevant issues | + +The final scan found no uncovered first-release area from the source design. Cross-cutting +security, privacy, migration, observability, reliability, negative controls, and stale-issue +reconciliation are assigned above rather than split into additional issues. diff --git a/benchmark/env_setup.py b/benchmark/env_setup.py index aad77be..c21c6e5 100644 --- a/benchmark/env_setup.py +++ b/benchmark/env_setup.py @@ -51,14 +51,18 @@ def pinned_pred_version() -> str: def find_pred_binary() -> Path: - """Find pred binary in PATH.""" - pred_path = shutil.which("pred") + """Find the trusted pred binary from PRED_BINARY or PATH.""" + configured = os.environ.get("PRED_BINARY") + pred_path = configured or shutil.which("pred") if not pred_path: raise RuntimeError( "pred binary not found in PATH. " "Install with: cargo install --git https://github.com/CodingThrust/problem-reductions problemreductions-cli" ) - return Path(pred_path) + path = Path(pred_path).expanduser().resolve() + if not path.is_file() or not os.access(path, os.X_OK): + raise RuntimeError(f"pred binary is not executable: {path}") + return path def pred_version(binary: str | Path = "pred") -> str: diff --git a/benchmark/evidence_budget.py b/benchmark/evidence_budget.py new file mode 100644 index 0000000..12ccc44 --- /dev/null +++ b/benchmark/evidence_budget.py @@ -0,0 +1,690 @@ +"""Evaluation-owned logical budgets for one rule investigation episode. + +The model sees thin ``pred`` and ``submit`` commands in its scratch environment. The +authoritative counters, the real pred binary, and certificate verification remain in the +runner process. This module intentionally uses the same atomic file-spool shape as +``SubmissionSession`` because localhost sockets are unavailable in several supported +agent sandboxes. +""" +from __future__ import annotations + +import copy +import errno +import hashlib +import json +import os +import shlex +import stat +import sys +import threading +from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import ExitStack +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Callable + +from benchmark.submit_session import SubmissionSession +from benchmark.process_control import ProcessLimits, run_capped_process, terminate_process_group +from benchmark.verify import (CPU_LIMIT_SECONDS, FSIZE_LIMIT_BYTES, MEM_LIMIT_BYTES, + Verdict, verify) + +MAX_REQUEST_BYTES = 2 * 1024 * 1024 +DEFAULT_PRED_TIMEOUT_SECONDS = 300 +DEFAULT_MAX_OUTPUT_CHARS = 10_000 +_DYNAMIC_COMMANDS = {"create", "reduce", "solve", "evaluate", "extract"} + + +@dataclass(frozen=True) +class EvidenceBudget: + """The non-transferable logical allowance for one selected rule.""" + + model_generations: int + shell_actions: int + pred_calls: int + solve_calls: int + submit_attempts: int = 2 + max_output_chars: int = DEFAULT_MAX_OUTPUT_CHARS + pred_timeout_seconds: int = DEFAULT_PRED_TIMEOUT_SECONDS + + def __post_init__(self) -> None: + for name, value in asdict(self).items(): + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise ValueError(f"{name} must be a non-negative integer") + if self.solve_calls > self.pred_calls: + raise ValueError("solve_calls must be <= pred_calls") + if self.submit_attempts != 2: + raise ValueError("submit_attempts must be exactly 2 for the Top50 contract") + if self.max_output_chars == 0: + raise ValueError("max_output_chars must be > 0") + if self.pred_timeout_seconds == 0: + raise ValueError("pred_timeout_seconds must be > 0") + + +class EvidenceBudgetState: + """Atomically reserve, release, and report the counters for one episode.""" + + def __init__(self, budget: EvidenceBudget): + self.budget = budget + self._limits = { + "model_generations": budget.model_generations, + "shell_actions": budget.shell_actions, + "pred_calls": budget.pred_calls, + "solve_calls": budget.solve_calls, + } + self._used = {name: 0 for name in self._limits} + self._lock = threading.RLock() + + def reserve(self, *names: str) -> dict | None: + """Reserve all named counters or none of them.""" + with self._lock: + unknown = [name for name in names if name not in self._limits] + if unknown: + raise KeyError(f"unknown evidence counters: {unknown}") + if any(self._used[name] >= self._limits[name] for name in names): + return None + before = dict(self._used) + for name in names: + self._used[name] += 1 + return {"before": before, "after": dict(self._used)} + + def release(self, *names: str) -> None: + """Release an infrastructure-failed reservation.""" + with self._lock: + for name in names: + if self._used[name] <= 0: + raise RuntimeError(f"cannot release unreserved counter {name}") + for name in names: + self._used[name] -= 1 + + def status(self) -> dict: + with self._lock: + return { + name: { + "used": self._used[name], + "limit": limit, + "remaining": max(0, limit - self._used[name]), + } + for name, limit in self._limits.items() + } + + +class PredGatewaySession: + """Own the only agent-facing route to a real pred process.""" + + def __init__( + self, + *, + pred_binary: str | Path, + workdir: str | Path, + budget_state: EvidenceBudgetState, + oracle_uid: int | None = None, + oracle_gid: int | None = None, + oracle_extra_groups: tuple[int, ...] = (), + ): + if (oracle_uid is None) != (oracle_gid is None): + raise ValueError("oracle_uid and oracle_gid must be provided together") + self.pred_binary = Path(pred_binary).resolve() + self.workdir = Path(workdir).resolve() + self.budget_state = budget_state + self.oracle_uid = oracle_uid + self.oracle_gid = oracle_gid + self.oracle_extra_groups = oracle_extra_groups + self._ledger: list[dict] = [] + self._responses: dict[str, tuple[str, dict]] = {} + self._cache: dict[tuple[str, ...], dict] = {} + self._lock = threading.RLock() + self._thread: threading.Thread | None = None + self._executor: ThreadPoolExecutor | None = None + self._futures: set[Future] = set() + self._active_processes: set = set() + self._channel: Path | None = None + self._inbox_fd: int | None = None + self._processing_fd: int | None = None + self._outbox_fd: int | None = None + self._old_channel_env: str | None = None + self._stopping = threading.Event() + + @property + def ledger(self) -> list[dict]: + with self._lock: + return copy.deepcopy(self._ledger) + + @property + def channel(self) -> Path: + if self._channel is None: + raise RuntimeError("pred gateway is not active") + return self._channel + + def __enter__(self) -> "PredGatewaySession": + if not self.pred_binary.is_file() or not os.access(self.pred_binary, os.X_OK): + raise RuntimeError(f"pred gateway binary is unavailable: {self.pred_binary}") + + bin_dir = self.workdir.parent / ".prb-bin" + self._channel = self.workdir.parent / ".prb-pred" + inbox = self._channel / "inbox" + processing = self._channel / "processing" + outbox = self._channel / "outbox" + for directory in (bin_dir, inbox, processing, outbox): + directory.mkdir(parents=True, exist_ok=True) + + directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + self._inbox_fd = os.open(inbox, directory_flags) + self._processing_fd = os.open(processing, directory_flags) + self._outbox_fd = os.open(outbox, directory_flags) + + shim = bin_dir / "pred" + package_root = Path(__file__).resolve().parent.parent + shim.write_text( + f"#!/bin/sh\nPYTHONPATH={shlex.quote(str(package_root))}${{PYTHONPATH:+:$PYTHONPATH}} " + f"exec {shlex.quote(sys.executable)} -m benchmark.agent_pred \"$@\"\n", + encoding="utf-8", + ) + shim.chmod(0o755) + + self._thread = threading.Thread(target=self._serve, name="prb-pred", daemon=True) + worker_count = max(8, min(64, self.budget_state.budget.pred_calls + 16)) + self._executor = ThreadPoolExecutor(max_workers=worker_count, + thread_name_prefix="prb-pred-call") + self._thread.start() + self._old_channel_env = os.environ.get("PRB_PRED_DIR") + os.environ["PRB_PRED_DIR"] = str(self._channel) + return self + + def __exit__(self, exc_type, exc, tb) -> None: + if self._old_channel_env is None: + os.environ.pop("PRB_PRED_DIR", None) + else: + os.environ["PRB_PRED_DIR"] = self._old_channel_env + self._stopping.set() + if self._thread is not None: + self._thread.join() + with self._lock: + active = list(self._active_processes) + for process in active: + terminate_process_group(process) + if self._executor is not None: + self._executor.shutdown(wait=True, cancel_futures=True) + for fd in (self._inbox_fd, self._processing_fd, self._outbox_fd): + if fd is not None: + os.close(fd) + + def prepare_agent_access(self, uid: int, gid: int) -> None: + """Give the unprivileged agent access only to request/response directories.""" + for name in ("inbox", "outbox"): + directory = self.channel / name + os.chown(directory, uid, gid) + directory.chmod(0o700) + + def _serve(self) -> None: + assert self._inbox_fd is not None + assert self._processing_fd is not None + idle_wait = 0.02 + while not self._stopping.is_set(): + handled = False + for name in os.listdir(self._inbox_fd): + if not _valid_request_filename(name): + continue + handled = True + try: + os.replace(name, name, src_dir_fd=self._inbox_fd, + dst_dir_fd=self._processing_fd) + except FileNotFoundError: + continue + request_id = name[:-5] + assert self._executor is not None + future = self._executor.submit(self._process_request_file, name, request_id) + with self._lock: + self._futures.add(future) + future.add_done_callback(self._forget_future) + if handled: + idle_wait = 0.02 + else: + self._stopping.wait(idle_wait) + idle_wait = min(idle_wait * 2, 0.5) + + def _forget_future(self, future: Future) -> None: + with self._lock: + self._futures.discard(future) + + def _process_request_file(self, name: str, request_id: str) -> None: + assert self._processing_fd is not None + raw = b"" + try: + raw = _read_request_bytes(self._processing_fd, name) + request = json.loads(raw.decode("utf-8")) + if not isinstance(request, dict) or request.get("request_id") != request_id: + raise ValueError("request id does not match filename") + response = self._handle_idempotent(request_id, request) + except Exception as error: + response = self._handle_invalid_envelope(request_id, raw, error) + self._write_response(request_id, response) + try: + os.unlink(name, dir_fd=self._processing_fd) + except FileNotFoundError: + pass + + def _handle_idempotent(self, request_id: str, request: dict) -> dict: + fingerprint = json.dumps(request, sort_keys=True, separators=(",", ":")) + with self._lock: + old = self._responses.get(request_id) + if old is not None: + old_fingerprint, old_response = old + if old_fingerprint != fingerprint: + return { + "infrastructure_error": True, + "reason": "request id was reused with a different payload", + "returncode": 2, + "stdout": "", + "stderr": "pred gateway request-id collision\n", + "budget": self.budget_state.status(), + } + return copy.deepcopy(old_response) + + response = self._handle(request_id, request) + with self._lock: + self._responses[request_id] = (fingerprint, copy.deepcopy(response)) + return response + + def _handle_invalid_envelope(self, request_id: str, raw: bytes, error: Exception) -> dict: + fingerprint = f"invalid:{hashlib.sha256(raw).hexdigest()}" + with self._lock: + old = self._responses.get(request_id) + if old is not None: + old_fingerprint, old_response = old + if old_fingerprint != fingerprint: + return { + "infrastructure_error": True, + "reason": "request id was reused with a different payload", + "returncode": 2, + "stdout": "", + "stderr": "pred gateway request-id collision\n", + "budget": self.budget_state.status(), + } + return copy.deepcopy(old_response) + response = self._handle_invalid_args(request_id) + response["reason"] = f"invalid pred gateway request: {type(error).__name__}: {error}" + with self._lock: + self._responses[request_id] = (fingerprint, copy.deepcopy(response)) + return response + + def _handle(self, request_id: str, request: dict) -> dict: + op = request.get("op") + if op == "status": + return {"status": "ok", "returncode": 0, "stdout": "", "stderr": "", + "budget": self.budget_state.status()} + if op != "pred": + return {"returncode": 2, "stdout": "", "stderr": "unknown pred gateway operation\n", + "reason": "unknown operation", "budget": self.budget_state.status()} + + args = request.get("args") + if (not isinstance(args, list) or not all(isinstance(arg, str) for arg in args) + or sum(len(arg) for arg in args) > MAX_REQUEST_BYTES): + return self._handle_invalid_args(request_id) + cwd = self._validated_cwd(request.get("cwd")) + command = _command_name(args) + free = _is_free_command(args, command) + cache_key = tuple(args) if free else None + with self._lock: + if cache_key in self._cache: + return copy.deepcopy(self._cache[cache_key]) + + counters = () if free else (("pred_calls", "solve_calls") + if command == "solve" else ("pred_calls",)) + reservation = self.budget_state.reserve(*counters) + if counters and reservation is None: + response = { + "returncode": 75, + "stdout": "", + "stderr": "pred evidence budget exhausted\n", + "exhausted": True, + "budget": self.budget_state.status(), + } + self._append_pred_record(request_id, args, command, False, "budget_exhausted", + response["budget"]) + return response + + record_index = self._append_pred_record( + request_id, args, command, bool(counters), "running", self.budget_state.status()) + try: + result = self._run_pred(args, cwd) + except OSError as error: + model_caused = error.errno == errno.E2BIG + if counters and not model_caused: + self.budget_state.release(*counters) + response = { + "returncode": 2, + "stdout": "", + "stderr": (f"pred invocation rejected: {error}\n" if model_caused else + f"pred gateway infrastructure error: {error}\n"), + "infrastructure_error": not model_caused, + "reason": f"{type(error).__name__}: {error}", + "budget": self.budget_state.status(), + } + self._finish_pred_record( + record_index, "model_error" if model_caused else "infrastructure_error", response) + return response + + response = {**result, "budget": self.budget_state.status()} + outcome = "timeout" if result.get("timed_out") else ( + "completed" if result["returncode"] == 0 else "nonzero_exit") + self._finish_pred_record(record_index, outcome, response) + if cache_key is not None and result["returncode"] == 0: + with self._lock: + self._cache[cache_key] = copy.deepcopy(response) + return response + + def _handle_invalid_args(self, request_id: str) -> dict: + reservation = self.budget_state.reserve("pred_calls") + if reservation is None: + return {"returncode": 75, "stdout": "", + "stderr": "pred evidence budget exhausted\n", "exhausted": True, + "budget": self.budget_state.status()} + response = { + "returncode": 2, + "stdout": "", + "stderr": "pred request arguments must be a bounded list of strings\n", + "model_error": True, + "budget": self.budget_state.status(), + } + index = self._append_pred_record( + request_id, [], None, True, "model_error", response["budget"]) + self._finish_pred_record(index, "model_error", response) + return response + + def _validated_cwd(self, raw: object) -> Path: + cwd = Path(raw) if isinstance(raw, str) and raw else self.workdir + cwd = cwd.resolve() + if not cwd.is_relative_to(self.workdir): + # The action is model-authored, so keep it inside the episode scratch area. + return self.workdir + return cwd + + def _run_pred(self, args: list[str], cwd: Path) -> dict: + result = run_capped_process( + [str(self.pred_binary), *args], + shell=False, + cwd=cwd, + env=None, + timeout=self.budget_state.budget.pred_timeout_seconds, + max_output_chars=self.budget_state.budget.max_output_chars, + uid=self.oracle_uid, + gid=self.oracle_gid, + extra_groups=self.oracle_extra_groups, + limits=ProcessLimits( + cpu_seconds=CPU_LIMIT_SECONDS, + memory_bytes=MEM_LIMIT_BYTES, + file_bytes=FSIZE_LIMIT_BYTES, + ), + on_start=self._track_process, + on_finish=self._untrack_process, + ) + stderr = result.stderr + if result.timed_out: + stderr += ("\n" if stderr and not stderr.endswith("\n") else "") + stderr += "pred process timed out\n" + return { + "returncode": result.returncode, + "stdout": result.stdout, + "stderr": stderr, + "timed_out": result.timed_out, + } + + def _track_process(self, process) -> None: + with self._lock: + self._active_processes.add(process) + + def _untrack_process(self, process) -> None: + with self._lock: + self._active_processes.discard(process) + + def _append_pred_record(self, request_id: str, args: list[str], command: str | None, + charged: bool, outcome: str, budget: dict) -> int: + with self._lock: + self._ledger.append({ + "sequence": len(self._ledger) + 1, + "request_id": request_id, + "args": copy.deepcopy(args), + "command": command, + "charged": charged, + "outcome": outcome, + "budget": copy.deepcopy(budget), + }) + return len(self._ledger) - 1 + + def _finish_pred_record(self, index: int, outcome: str, response: dict) -> None: + with self._lock: + self._ledger[index]["outcome"] = outcome + self._ledger[index]["returncode"] = response["returncode"] + self._ledger[index]["budget"] = copy.deepcopy(response["budget"]) + + def _write_response(self, request_id: str, response: dict) -> None: + assert self._outbox_fd is not None + destination = f"{request_id}.json" + temporary = f".{request_id}.{threading.get_ident()}.tmp" + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + response_fd = os.open(temporary, flags, 0o644, dir_fd=self._outbox_fd) + try: + with os.fdopen(response_fd, "w", encoding="utf-8", closefd=False) as handle: + json.dump(response, handle) + handle.flush() + finally: + os.close(response_fd) + os.replace(temporary, destination, src_dir_fd=self._outbox_fd, + dst_dir_fd=self._outbox_fd) + + +class EvidenceBudgetSession: + """Compose the pred gateway and a two-attempt, rule-scoped submit judge.""" + + def __init__( + self, + *, + rule: str, + budget: EvidenceBudget, + pred_binary: str | Path, + verifier: Callable[[dict], Verdict] = verify, + agent_uid: int | None = None, + agent_gid: int | None = None, + oracle_uid: int | None = None, + oracle_gid: int | None = None, + evidence_gid: int | None = None, + ): + if not rule: + raise ValueError("rule must be non-empty") + if (agent_uid is None) != (agent_gid is None): + raise ValueError("agent_uid and agent_gid must be provided together") + if (oracle_uid is None) != (oracle_gid is None): + raise ValueError("oracle_uid and oracle_gid must be provided together") + if evidence_gid is not None and (agent_uid is None or oracle_uid is None): + raise ValueError("evidence_gid requires both agent and oracle identities") + self.rule = rule + self.budget = budget + self.state = EvidenceBudgetState(budget) + self.pred_binary = Path(pred_binary) + self.verifier = verifier + self.agent_uid = agent_uid + self.agent_gid = agent_gid + self.oracle_uid = oracle_uid + self.oracle_gid = oracle_gid + self.evidence_gid = evidence_gid + self.submit: SubmissionSession | None = None + self.pred: PredGatewaySession | None = None + self._scratch: Path | None = None + self._stack: ExitStack | None = None + self._event_lock = threading.RLock() + self._model_events: list[dict] = [] + self._shell_events: list[dict] = [] + + @property + def workdir(self) -> Path: + if self._scratch is None: + raise RuntimeError("evidence budget session is not active") + return self._scratch + + def __enter__(self) -> "EvidenceBudgetSession": + stack = ExitStack() + try: + self.submit = stack.enter_context(SubmissionSession( + limit=self.budget.submit_attempts, + verifier=self.verifier, + expected_rule=self.rule, + )) + self._scratch = self.submit.workdir / "scratch" + self._scratch.mkdir(mode=0o700) + self.pred = stack.enter_context(PredGatewaySession( + pred_binary=self.pred_binary, + workdir=self._scratch, + budget_state=self.state, + oracle_uid=self.oracle_uid, + oracle_gid=self.oracle_gid, + oracle_extra_groups=((self.evidence_gid,) if self.evidence_gid is not None else ()), + )) + if self.agent_uid is not None and self.agent_gid is not None: + self._prepare_agent_access(self.agent_uid, self.agent_gid) + self._stack = stack + return self + except BaseException: + stack.__exit__(*sys.exc_info()) + raise + + def __exit__(self, exc_type, exc, tb) -> None: + if self._stack is not None: + self._stack.__exit__(exc_type, exc, tb) + + def _prepare_agent_access(self, uid: int, gid: int) -> None: + assert self.submit is not None and self.pred is not None and self._scratch is not None + scratch_gid = self.evidence_gid if self.evidence_gid is not None else gid + os.chown(self._scratch, uid, scratch_gid) + self._scratch.chmod(0o2770 if self.evidence_gid is not None else 0o700) + self.submit.prepare_agent_access(uid, gid) + self.pred.prepare_agent_access(uid, gid) + + def admit_shell_action(self, command: str) -> bool: + """Atomically debit one model-authored shell action and append its audit event.""" + reservation = self.state.reserve("shell_actions") + with self._event_lock: + self._shell_events.append({ + "sequence": len(self._shell_events) + 1, + "command": command, + "charged": reservation is not None, + "outcome": "admitted" if reservation is not None else "budget_exhausted", + "budget": self.state.status(), + }) + return reservation is not None + + def record_model_generation(self, *, outcome: str = "completed", + infrastructure_error: bool = False) -> bool: + """Record a provider generation; infrastructure failures do not debit the model.""" + reservation = None if infrastructure_error else self.state.reserve("model_generations") + admitted = infrastructure_error or reservation is not None + with self._event_lock: + self._model_events.append({ + "sequence": len(self._model_events) + 1, + "charged": reservation is not None, + "outcome": outcome if admitted else "budget_exhausted", + "budget": self.state.status(), + }) + return admitted + + def status(self) -> dict: + if self.submit is None: + raise RuntimeError("evidence budget session is not active") + return { + **self.state.status(), + "submit_attempts": { + "used": self.submit.used, + "limit": self.submit.limit, + "remaining": self.submit.remaining, + }, + } + + def ledger(self) -> dict: + if self.submit is None or self.pred is None: + raise RuntimeError("evidence budget session is not active") + with self._event_lock: + model_events = copy.deepcopy(self._model_events) + shell_events = copy.deepcopy(self._shell_events) + return { + "rule": self.rule, + "budget": asdict(self.budget), + "status": self.status(), + "pred": self.pred.ledger, + "submit": self.submit.attempts, + "model_generations": model_events, + "shell_actions": shell_events, + } + + +def _valid_request_filename(name: str) -> bool: + return (len(name) == 37 and name.endswith(".json") + and all(char in "0123456789abcdef" for char in name[:-5])) + + +def _read_request_bytes(directory_fd: int, name: str) -> bytes: + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + request_fd = os.open(name, flags, dir_fd=directory_fd) + try: + info = os.fstat(request_fd) + if not stat.S_ISREG(info.st_mode): + raise ValueError("request is not a regular file") + if info.st_size > MAX_REQUEST_BYTES: + raise ValueError(f"request exceeds {MAX_REQUEST_BYTES} bytes") + with os.fdopen(request_fd, "rb", closefd=False) as request_file: + raw = request_file.read(MAX_REQUEST_BYTES + 1) + finally: + os.close(request_fd) + if len(raw) > MAX_REQUEST_BYTES: + raise ValueError(f"request exceeds {MAX_REQUEST_BYTES} bytes") + return raw + + +def _command_name(args: list[str]) -> str | None: + skip_value = False + for arg in args: + if skip_value: + skip_value = False + continue + if arg in {"-o", "--output"}: + skip_value = True + continue + if arg.startswith("--output=") or (arg.startswith("-o") and arg != "-o"): + continue + if arg in {"-q", "--quiet", "--json"}: + continue + if arg in _DYNAMIC_COMMANDS or arg in {"list", "help"}: + return arg + if not arg.startswith("-"): + return arg + return None + + +def _is_free_command(args: list[str], command: str | None) -> bool: + if not args: + return True # invoking pred without args only displays help + if any(arg in {"--help", "-h", "--version", "-V"} for arg in args): + return True + if command == "help": + return True + return command == "list" and "--rules" in args + + +def _drain_capped(stream, limit: int, destination: dict[str, str], key: str) -> None: + """Drain a child pipe completely while retaining at most ``limit`` characters.""" + chunks: list[str] = [] + kept = 0 + total = 0 + while True: + chunk = stream.read(64 * 1024) + if not chunk: + break + total += len(chunk) + if kept < limit: + piece = chunk[:limit - kept] + chunks.append(piece) + kept += len(piece) + text = "".join(chunks) + if total > kept: + marker = f"\n... {total - kept} characters elided ...\n" + prefix_limit = max(0, limit - len(marker)) + text = text[:prefix_limit] + marker[:limit] + destination[key] = text diff --git a/benchmark/preflight.py b/benchmark/preflight.py index 4f712f7..d48e6d4 100644 --- a/benchmark/preflight.py +++ b/benchmark/preflight.py @@ -16,7 +16,7 @@ """ from __future__ import annotations -from benchmark.env_setup import verify_pred_version +from benchmark.env_setup import find_pred_binary, verify_pred_version from benchmark.run_mini import DEFAULT_MAX_TOKENS, _build_model, list_rules from benchmark.usage import usage_from_response @@ -39,7 +39,7 @@ def run_checks( # 1. pred binary + version (no API). try: - ver = verify_pred_version("pred") + ver = verify_pred_version(find_pred_binary()) results.append(("pred binary", True, f"version {ver}")) except Exception as e: results.append(("pred binary", False, str(e))) diff --git a/benchmark/privilege_smoke.py b/benchmark/privilege_smoke.py new file mode 100644 index 0000000..ef8ccb8 --- /dev/null +++ b/benchmark/privilege_smoke.py @@ -0,0 +1,79 @@ +"""Build-time smoke check for the runner/oracle privilege boundary.""" +from __future__ import annotations + +import os +import shlex + +from benchmark.agent_environment import run_as_agent +from benchmark.env_setup import find_pred_binary +from benchmark.evidence_budget import EvidenceBudget, EvidenceBudgetSession +from benchmark.verify import Verdict + + +def main() -> None: + uid = int(os.environ["PRB_AGENT_UID"]) + gid = int(os.environ["PRB_AGENT_GID"]) + oracle_uid = int(os.environ["PRB_ORACLE_UID"]) + oracle_gid = int(os.environ["PRB_ORACLE_GID"]) + evidence_gid = int(os.environ["PRB_EVIDENCE_GID"]) + pred_binary = find_pred_binary() + budget = EvidenceBudget( + model_generations=1, + shell_actions=2, + pred_calls=1, + solve_calls=0, + submit_attempts=2, + ) + with EvidenceBudgetSession( + rule="privilege-smoke", + budget=budget, + pred_binary=pred_binary, + verifier=lambda cert: Verdict(False, "not used"), + agent_uid=uid, + agent_gid=gid, + oracle_uid=oracle_uid, + oracle_gid=oracle_gid, + evidence_gid=evidence_gid, + ) as session: + env = dict(os.environ) + through_gateway = run_as_agent( + "pred --version", + cwd=str(session.workdir), + env=env, + timeout=10, + uid=uid, + gid=gid, + extra_groups=(evidence_gid,), + ) + if through_gateway.returncode != 0: + raise SystemExit(f"pred gateway failed for agent: {through_gateway.stdout}") + + direct = run_as_agent( + f"{shlex.quote(str(pred_binary))} --version", + cwd=str(session.workdir), + env=env, + timeout=10, + uid=uid, + gid=gid, + extra_groups=(evidence_gid,), + ) + if direct.returncode == 0: + raise SystemExit("agent unexpectedly executed the real pred oracle") + + submit_status = run_as_agent( + "submit --status", + cwd=str(session.workdir), + env=env, + timeout=10, + uid=uid, + gid=gid, + extra_groups=(evidence_gid,), + ) + if submit_status.returncode != 0: + raise SystemExit(f"submit gateway failed for agent: {submit_status.stdout}") + + print("PASS: unprivileged agent can use gateways but cannot execute the pred oracle") + + +if __name__ == "__main__": + main() diff --git a/benchmark/process_control.py b/benchmark/process_control.py new file mode 100644 index 0000000..76b55d9 --- /dev/null +++ b/benchmark/process_control.py @@ -0,0 +1,173 @@ +"""Bounded subprocess execution shared by agent actions and pred gateway calls.""" +from __future__ import annotations + +import os +import signal +import subprocess +import threading +from dataclasses import dataclass +from typing import Callable, Sequence + +try: # POSIX-only; official rankable runs use Linux. + import resource +except ImportError: # pragma: no cover - Windows is not a rankable platform + resource = None + + +@dataclass(frozen=True) +class ProcessLimits: + cpu_seconds: int + memory_bytes: int + file_bytes: int + processes: int = 64 + + +@dataclass(frozen=True) +class ProcessResult: + returncode: int + stdout: str + stderr: str + timed_out: bool + + +class _CombinedCapture: + def __init__(self, limit: int): + self.limit = limit + self.remaining = limit + self.total = 0 + self.totals = {"stdout": 0, "stderr": 0} + self.parts = {"stdout": [], "stderr": []} + self._lock = threading.Lock() + + def drain(self, stream, key: str) -> None: + while True: + chunk = stream.read(64 * 1024) + if not chunk: + break + with self._lock: + self.total += len(chunk) + self.totals[key] += len(chunk) + if self.remaining: + piece = chunk[:self.remaining] + self.parts[key].append(piece) + self.remaining -= len(piece) + + def result(self) -> tuple[str, str]: + stdout = "".join(self.parts["stdout"]) + stderr = "".join(self.parts["stderr"]) + retained = len(stdout) + len(stderr) + if self.total > retained: + marker = f"\n... {self.total - retained} characters elided ...\n" + keep = max(0, self.limit - len(marker)) + if self.totals["stdout"] > len(stdout): + stdout = stdout[:keep] + marker[:self.limit - min(len(stdout), keep)] + stderr = "" + else: + stdout = stdout[:keep] + stderr_keep = max(0, keep - len(stdout)) + stderr = stderr[:stderr_keep] + marker[:self.limit - len(stdout) - stderr_keep] + return stdout, stderr + + +def run_capped_process( + command: str | Sequence[str], + *, + shell: bool, + cwd: str, + env: dict[str, str] | None, + timeout: int, + max_output_chars: int, + combine_stderr: bool = False, + uid: int | None = None, + gid: int | None = None, + extra_groups: Sequence[int] = (), + limits: ProcessLimits | None = None, + on_start: Callable[[subprocess.Popen], None] | None = None, + on_finish: Callable[[subprocess.Popen], None] | None = None, +) -> ProcessResult: + """Run one process with a combined observation cap and kill its whole group on timeout.""" + privilege_args = {} + if uid is not None or gid is not None: + if uid is None or gid is None: + raise ValueError("uid and gid must be provided together") + if os.name != "posix": + raise RuntimeError("rankable privilege separation requires POSIX") + if os.geteuid() == 0: + privilege_args = {"user": uid, "group": gid, "extra_groups": list(extra_groups)} + elif uid != os.getuid() or gid != os.getgid(): + raise PermissionError("runner must be root to launch a different identity") + + process = subprocess.Popen( + command, + shell=shell, + text=True, + cwd=cwd, + env=env, + encoding="utf-8", + errors="replace", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT if combine_stderr else subprocess.PIPE, + start_new_session=True, + **privilege_args, + ) + if on_start is not None: + on_start(process) + try: + _apply_limits(process.pid, limits) + capture = _CombinedCapture(max_output_chars) + assert process.stdout is not None + readers = [threading.Thread(target=capture.drain, args=(process.stdout, "stdout"), + daemon=True)] + if not combine_stderr: + assert process.stderr is not None + readers.append(threading.Thread(target=capture.drain, + args=(process.stderr, "stderr"), daemon=True)) + for reader in readers: + reader.start() + + timed_out = False + try: + process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + terminate_process_group(process) + process.wait() + for reader in readers: + reader.join() + stdout, stderr = capture.result() + return ProcessResult( + returncode=124 if timed_out else process.returncode, + stdout=stdout, + stderr=stderr, + timed_out=timed_out, + ) + finally: + if on_finish is not None: + on_finish(process) + + +def terminate_process_group(process: subprocess.Popen) -> None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def _apply_limits(pid: int, limits: ProcessLimits | None) -> None: + if limits is None or resource is None or not hasattr(resource, "prlimit"): + return + for name, value in ( + ("RLIMIT_CPU", limits.cpu_seconds), + ("RLIMIT_AS", limits.memory_bytes), + ("RLIMIT_FSIZE", limits.file_bytes), + ("RLIMIT_NPROC", limits.processes), + ): + resource_id = getattr(resource, name, None) + if resource_id is None: + continue + try: + resource.prlimit(pid, resource_id, (value, value)) + except (PermissionError, ProcessLookupError, ValueError, OSError): + # macOS lacks prlimit and some kernels reject individual limits. Official + # Linux runs additionally carry fixed container-level resource limits. + continue diff --git a/benchmark/submit_session.py b/benchmark/submit_session.py index aa1dffe..47659db 100644 --- a/benchmark/submit_session.py +++ b/benchmark/submit_session.py @@ -9,6 +9,7 @@ from __future__ import annotations import copy +import hashlib import json import os import shutil @@ -26,14 +27,24 @@ class SubmissionSession: - """Own one run-wide submit budget and its append-only in-memory ledger.""" + """Own one submit budget and its append-only in-memory ledger. - def __init__(self, limit: int = 100, verifier: Callable[[dict], Verdict] = verify): + Historical callers may still use a run-wide limit. Evidence-budget episodes pass an + ``expected_rule`` and create one independent two-attempt session per rule. + """ + + def __init__(self, limit: int = 100, verifier: Callable[[dict], Verdict] = verify, + expected_rule: str | None = None): if limit < 0: raise ValueError("submit limit must be >= 0") + if expected_rule is not None and not expected_rule: + raise ValueError("expected_rule must be non-empty when provided") self.limit = limit self._verifier = verifier + self.expected_rule = expected_rule self._attempts: list[dict] = [] + self._responses: dict[str, tuple[str, dict]] = {} + self._closed = False self._lock = threading.RLock() self._thread: threading.Thread | None = None self._tmpdir: Path | None = None @@ -56,8 +67,15 @@ def used(self) -> int: @property def remaining(self) -> int: + if self.closed: + return 0 return max(0, self.limit - self.used) + @property + def closed(self) -> bool: + with self._lock: + return self._closed + @property def attempts(self) -> list[dict]: with self._lock: @@ -95,9 +113,10 @@ def __enter__(self) -> "SubmissionSession": self._artifact_fd = os.open(artifact_dir, directory_flags) shim = bin_dir / "submit" - client = Path(__file__).with_name("agent_submit.py") + package_root = Path(__file__).resolve().parent.parent shim.write_text( - f"#!/bin/sh\nexec {shlex.quote(sys.executable)} {shlex.quote(str(client))} \"$@\"\n", + f"#!/bin/sh\nPYTHONPATH={shlex.quote(str(package_root))}${{PYTHONPATH:+:$PYTHONPATH}} " + f"exec {shlex.quote(sys.executable)} -m benchmark.agent_submit \"$@\"\n", encoding="utf-8", ) shim.chmod(0o755) @@ -129,7 +148,7 @@ def __exit__(self, exc_type, exc, tb) -> None: self._stopping.set() if self._thread is not None: - self._thread.join(timeout=1) + self._thread.join() for fd in (self._workdir_fd, self._inbox_fd, self._processing_fd, self._outbox_fd, self._artifact_fd): if fd is not None: @@ -171,6 +190,21 @@ def preserve_artifacts(self, destination: str | Path) -> list[Path]: copied.append(target) return copied + def prepare_agent_access(self, uid: int, gid: int) -> None: + """Give a non-root agent access to its request queues and artifact directory.""" + if self._tmpdir is None: + raise RuntimeError("submission session is not active") + self._tmpdir.chmod(0o711) + (self._tmpdir / "work").chmod(0o711) + paths = ( + self._tmpdir / "work" / ".prb-submit" / "inbox", + self._tmpdir / "work" / ".prb-submit" / "outbox", + self._tmpdir / "work" / "artifacts", + ) + for path in paths: + os.chown(path, uid, gid) + path.chmod(0o700) + def result_rows(self) -> list[dict]: """Collapse attempts to one authoritative result per named rule. @@ -208,6 +242,7 @@ def _serve(self) -> None: except FileNotFoundError: continue request_id = name[:-5] + raw = b"" try: flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) request_fd = os.open(name, flags, dir_fd=self._processing_fd) @@ -226,9 +261,9 @@ def _serve(self) -> None: request = json.loads(raw.decode("utf-8")) if request.get("request_id") != request_id: raise ValueError("request id does not match filename") - response = self._handle(request) + response = self._handle_idempotent(request_id, request) except Exception as e: # malformed requests do not kill the service - response = self._record_rejection(f"invalid request: {e}") + response = self._handle_malformed_idempotent(request_id, raw, e) self._write_response(request_id, response) try: os.unlink(name, dir_fd=self._processing_fd) @@ -246,7 +281,8 @@ def _write_response(self, request_id: str, response: dict) -> None: temporary = f".{request_id}.{threading.get_ident()}.tmp" flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)) - response_fd = os.open(temporary, flags, 0o600, dir_fd=self._outbox_fd) + # The service may run as root while the agent client is deliberately unprivileged. + response_fd = os.open(temporary, flags, 0o644, dir_fd=self._outbox_fd) try: with os.fdopen(response_fd, "w", encoding="utf-8", closefd=False) as response_file: json.dump(response, response_file) @@ -256,12 +292,49 @@ def _write_response(self, request_id: str, response: dict) -> None: os.replace(temporary, destination, src_dir_fd=self._outbox_fd, dst_dir_fd=self._outbox_fd) - def _handle(self, request: dict) -> dict: + def _handle_idempotent(self, request_id: str, request: dict) -> dict: + fingerprint = json.dumps(request, sort_keys=True, separators=(",", ":")) + with self._lock: + old = self._responses.get(request_id) + if old is not None: + old_fingerprint, old_response = old + if old_fingerprint != fingerprint: + return {"accepted": False, "infrastructure_error": True, + "reason": "request id was reused with a different payload", + "used": self.used, "limit": self.limit, + "remaining": self.remaining} + return copy.deepcopy(old_response) + response = self._handle(request, request_id=request_id) + with self._lock: + self._responses[request_id] = (fingerprint, copy.deepcopy(response)) + return response + + def _handle_malformed_idempotent( + self, request_id: str, raw: bytes, error: Exception + ) -> dict: + fingerprint = f"invalid:{hashlib.sha256(raw).hexdigest()}" + with self._lock: + old = self._responses.get(request_id) + if old is not None: + old_fingerprint, old_response = old + if old_fingerprint != fingerprint: + return {"accepted": False, "infrastructure_error": True, + "reason": "request id was reused with a different payload", + "used": self.used, "limit": self.limit, + "remaining": self.remaining} + return copy.deepcopy(old_response) + response = self._record_rejection( + f"invalid request: {type(error).__name__}: {error}", request_id=request_id) + self._responses[request_id] = (fingerprint, copy.deepcopy(response)) + return response + + def _handle(self, request: dict, *, request_id: str | None = None) -> dict: if request.get("op") == "status": with self._lock: self._status_checks += 1 return {"status": "ok", "used": self.used, "limit": self.limit, - "remaining": self.remaining} + "remaining": self.remaining, "closed": self.closed, + "expected_rule": self.expected_rule} if request.get("op") != "submit": return {"accepted": False, "error": "unknown operation", "used": self.used, "limit": self.limit, "remaining": self.remaining} @@ -269,6 +342,10 @@ def _handle(self, request: dict) -> dict: # Verification is serialized under this lock. The budget check + append is atomic, # including if a future runner enables parallel agent sessions. with self._lock: + if self._closed: + return {"accepted": False, "exhausted": True, "closed": True, + "reason": f"rule episode already accepted: {self.expected_rule}", + "used": len(self._attempts), "limit": self.limit, "remaining": 0} if len(self._attempts) >= self.limit: return {"accepted": False, "exhausted": True, "reason": f"submit budget exhausted ({self.limit}/{self.limit})", @@ -290,25 +367,42 @@ def _handle(self, request: dict) -> dict: else: if not isinstance(cert, dict): reason = "certificate must be a JSON object" + elif (self.expected_rule is not None + and cert.get("rule") != self.expected_rule): + reason = (f"certificate rule {cert.get('rule')!r} does not match " + f"this episode's rule {self.expected_rule!r}") else: try: verdict = self._verifier(cert) accepted = bool(verdict.accepted) reason = verdict.reason details = verdict.details - except Exception as e: # one bad certificate must not kill the budget server - reason = f"verifier error: {type(e).__name__}: {e}" + except Exception as e: # infrastructure errors never debit the model + return { + "accepted": False, + "infrastructure_error": True, + "reason": f"verifier error: {type(e).__name__}: {e}", + "used": len(self._attempts), + "limit": self.limit, + "remaining": self.remaining, + } - return self._append_attempt(cert, accepted, reason, details) + return self._append_attempt(cert, accepted, reason, details, + request_id=request_id) - def _record_rejection(self, reason: str) -> dict: + def _record_rejection(self, reason: str, *, request_id: str | None = None) -> dict: with self._lock: + if self._closed: + return {"accepted": False, "exhausted": True, "closed": True, + "reason": f"rule episode already accepted: {self.expected_rule}", + "used": len(self._attempts), "limit": self.limit, "remaining": 0} if len(self._attempts) >= self.limit: return {"accepted": False, "exhausted": True, "reason": reason, "used": self.limit, "limit": self.limit, "remaining": 0} - return self._append_attempt(None, False, reason) + return self._append_attempt(None, False, reason, request_id=request_id) - def _append_attempt(self, cert, accepted: bool, reason: str, details=None) -> dict: + def _append_attempt(self, cert, accepted: bool, reason: str, details=None, + request_id: str | None = None) -> dict: attempt_no = len(self._attempts) + 1 record = { "attempt": attempt_no, @@ -317,11 +411,16 @@ def _append_attempt(self, cert, accepted: bool, reason: str, details=None) -> di "reason": reason, "certificate": cert, } + if request_id is not None: + record["request_id"] = request_id if details is not None: record["verify_details"] = details self._attempts.append(record) + if accepted and self.expected_rule is not None: + self._closed = True return {**record, "used": attempt_no, "limit": self.limit, - "remaining": self.limit - attempt_no} + "remaining": 0 if self._closed else self.limit - attempt_no, + "closed": self._closed} def _attempt_to_row(attempt: dict) -> dict: diff --git a/benchmark/tests/test_env_setup.py b/benchmark/tests/test_env_setup.py index 725906d..0001e06 100644 --- a/benchmark/tests/test_env_setup.py +++ b/benchmark/tests/test_env_setup.py @@ -17,7 +17,7 @@ import subprocess import pytest from pathlib import Path -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock from benchmark.env_setup import ( PINNED_COMMIT, @@ -137,6 +137,7 @@ def test_pred_not_in_path_raises(self, monkeypatch): def test_pred_found_returns_path(self, monkeypatch, tmp_path): fake_pred = tmp_path / "pred" fake_pred.touch() + fake_pred.chmod(0o755) monkeypatch.setattr("shutil.which", lambda name: str(fake_pred)) result = find_pred_binary() assert result == fake_pred diff --git a/benchmark/tests/test_evidence_budget.py b/benchmark/tests/test_evidence_budget.py new file mode 100644 index 0000000..16ec992 --- /dev/null +++ b/benchmark/tests/test_evidence_budget.py @@ -0,0 +1,341 @@ +"""Acceptance tests for the evaluation-owned per-rule evidence budget.""" +from __future__ import annotations + +import json +import os +import subprocess +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest + +from benchmark.agent_pred import _request as pred_request +from benchmark.agent_submit import _request as submit_request +from benchmark.agent_environment import run_as_agent +from benchmark.evidence_budget import EvidenceBudget, EvidenceBudgetSession +from benchmark.verify import Verdict + + +def _fake_pred(tmp_path: Path) -> Path: + script = tmp_path / "real-pred" + script.write_text( + "#!/bin/sh\n" + "case \"$1\" in\n" + " --version) echo 'pred 0.0.0'; exit 0 ;;\n" + " --help) echo 'help'; exit 0 ;;\n" + " list) echo '[\"r1\",\"r2\"]'; exit 0 ;;\n" + " fail) echo 'bad args' >&2; exit 7 ;;\n" + " sleep) sleep 2; exit 0 ;;\n" + " spam) python3 -c 'print(\"x\" * 5000)'; exit 0 ;;\n" + " *) echo \"ran:$*\"; exit 0 ;;\n" + "esac\n", + encoding="utf-8", + ) + script.chmod(0o700) + return script + + +def _budget(*, pred=4, solve=1, timeout=1) -> EvidenceBudget: + return EvidenceBudget( + model_generations=10, + shell_actions=10, + pred_calls=pred, + solve_calls=solve, + submit_attempts=2, + max_output_chars=1024, + pred_timeout_seconds=timeout, + ) + + +def _cert(rule: str) -> dict: + return {"rule": rule, "source": {"type": "Example"}, + "bundle": {"target": {"type": "Target"}}} + + +def _run(*args: str, cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run(list(args), cwd=cwd, capture_output=True, text=True) + + +def _raw_spool(channel: Path, request_id: str, raw: bytes) -> dict: + request = channel / "inbox" / f"{request_id}.json" + response = channel / "outbox" / f"{request_id}.json" + request.write_bytes(raw) + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + try: + payload = json.loads(response.read_text()) + except FileNotFoundError: + time.sleep(0.01) + continue + response.unlink() + return payload + raise AssertionError("spool service did not respond") + + +def test_pred_and_solve_caps_apply_to_real_invocations(tmp_path): + with EvidenceBudgetSession( + rule="r1", budget=_budget(), pred_binary=_fake_pred(tmp_path), + verifier=lambda cert: Verdict(False, "no bug"), + ) as session: + assert _run("pred", "solve", "input.json", cwd=session.workdir).returncode == 0 + assert _run("pred", "solve", "input.json", cwd=session.workdir).returncode == 75 + results = [_run("pred", "create", "X", cwd=session.workdir) for _ in range(5)] + + assert sum(result.returncode == 0 for result in results) == 3 + assert sum(result.returncode == 75 for result in results) == 2 + status = session.status() + assert status["pred_calls"]["used"] == 4 + assert status["solve_calls"]["used"] == 1 + assert sum(record["charged"] for record in session.pred.ledger) == 4 + + +def test_global_output_option_cannot_bypass_solve_budget(tmp_path): + with EvidenceBudgetSession( + rule="r1", budget=_budget(pred=2, solve=1), pred_binary=_fake_pred(tmp_path), + verifier=lambda cert: Verdict(False, "no bug"), + ) as session: + first = _run("pred", "-o", "out.json", "solve", "input.json", cwd=session.workdir) + second = _run("pred", "--output=other.json", "solve", "input.json", + cwd=session.workdir) + + assert first.returncode == 0 + assert second.returncode == 75 + assert session.status()["pred_calls"]["used"] == 1 + assert session.status()["solve_calls"]["used"] == 1 + + +def test_concurrent_pred_clients_cannot_overspend(tmp_path): + with EvidenceBudgetSession( + rule="r1", budget=_budget(pred=4, solve=0), pred_binary=_fake_pred(tmp_path), + verifier=lambda cert: Verdict(False, "no bug"), + ) as session: + with ThreadPoolExecutor(max_workers=20) as pool: + results = list(pool.map( + lambda _: _run("pred", "create", "X", cwd=session.workdir), range(20))) + + assert sum(result.returncode == 0 for result in results) == 4 + assert sum(result.returncode == 75 for result in results) == 16 + assert session.status()["pred_calls"] == {"used": 4, "limit": 4, "remaining": 0} + + +def test_free_commands_are_cached_and_do_not_charge(tmp_path): + with EvidenceBudgetSession( + rule="r1", budget=_budget(pred=1, solve=0), pred_binary=_fake_pred(tmp_path), + verifier=lambda cert: Verdict(False, "no bug"), + ) as session: + assert _run("pred", "--budget-status", cwd=session.workdir).returncode == 0 + assert _run("pred", "--version", cwd=session.workdir).returncode == 0 + assert _run("pred", "--version", cwd=session.workdir).returncode == 0 + assert _run("pred", "list", "--rules", "--json", cwd=session.workdir).returncode == 0 + assert _run("pred", "list", "--rules", "--json", cwd=session.workdir).returncode == 0 + + assert session.status()["pred_calls"]["used"] == 0 + # Cache hits return without another real process or ledger entry. + assert len(session.pred.ledger) == 2 + + +def test_nonzero_exit_and_timeout_consume_pred_budget(tmp_path): + with EvidenceBudgetSession( + rule="r1", budget=_budget(pred=2, solve=0), pred_binary=_fake_pred(tmp_path), + verifier=lambda cert: Verdict(False, "no bug"), + ) as session: + failed = _run("pred", "fail", cwd=session.workdir) + timed_out = _run("pred", "sleep", cwd=session.workdir) + exhausted = _run("pred", "create", "X", cwd=session.workdir) + + assert failed.returncode == 7 + assert timed_out.returncode == 124 and "timed out" in timed_out.stderr + assert exhausted.returncode == 75 + assert session.status()["pred_calls"]["used"] == 2 + assert [record["outcome"] for record in session.pred.ledger[:2]] == [ + "nonzero_exit", "timeout"] + + +def test_gateway_infrastructure_failure_releases_reservation(tmp_path): + real_pred = _fake_pred(tmp_path) + with EvidenceBudgetSession( + rule="r1", budget=_budget(pred=1, solve=0), pred_binary=real_pred, + verifier=lambda cert: Verdict(False, "no bug"), + ) as session: + real_pred.unlink() # force spawn failure after the gateway health check + result = _run("pred", "create", "X", cwd=session.workdir) + assert result.returncode == 2 + assert "infrastructure error" in result.stderr + assert session.status()["pred_calls"]["used"] == 0 + assert session.pred.ledger[0]["outcome"] == "infrastructure_error" + + +def test_pred_output_is_drained_but_observation_is_capped(tmp_path): + with EvidenceBudgetSession( + rule="r1", budget=_budget(pred=1, solve=0), pred_binary=_fake_pred(tmp_path), + verifier=lambda cert: Verdict(False, "no bug"), + ) as session: + result = _run("pred", "spam", cwd=session.workdir) + assert result.returncode == 0 + assert len(result.stdout) <= 1024 + assert "characters elided" in result.stdout + + +def test_submit_budget_is_two_per_rule_and_acceptance_closes_episode(tmp_path): + pred = _fake_pred(tmp_path) + rejected = tmp_path / "rejected.json" + accepted = tmp_path / "accepted.json" + rejected.write_text(json.dumps(_cert("r1"))) + accepted.write_text(json.dumps(_cert("r2"))) + + with EvidenceBudgetSession( + rule="r1", budget=_budget(), pred_binary=pred, + verifier=lambda cert: Verdict(False, "no bug"), + ) as first: + assert _run("submit", str(rejected), cwd=first.workdir).returncode == 1 + assert _run("submit", str(rejected), cwd=first.workdir).returncode == 1 + assert _run("submit", str(rejected), cwd=first.workdir).returncode == 2 + assert first.status()["submit_attempts"] == {"used": 2, "limit": 2, "remaining": 0} + + with EvidenceBudgetSession( + rule="r2", budget=_budget(), pred_binary=pred, + verifier=lambda cert: Verdict(True, "confirmed"), + ) as second: + assert second.status()["submit_attempts"]["remaining"] == 2 + assert _run("submit", str(accepted), cwd=second.workdir).returncode == 0 + closed = _run("submit", str(accepted), cwd=second.workdir) + assert closed.returncode == 2 and "BUDGET_EXHAUSTED" in closed.stderr + assert second.submit.used == 1 + assert second.submit.remaining == 0 + + +def test_wrong_rule_and_malformed_submissions_consume(tmp_path): + malformed = tmp_path / "bad.json" + wrong = tmp_path / "wrong.json" + malformed.write_text("{bad") + wrong.write_text(json.dumps(_cert("r2"))) + with EvidenceBudgetSession( + rule="r1", budget=_budget(), pred_binary=_fake_pred(tmp_path), + verifier=lambda cert: Verdict(True, "confirmed"), + ) as session: + assert _run("submit", str(malformed), cwd=session.workdir).returncode == 1 + result = _run("submit", str(wrong), cwd=session.workdir) + assert result.returncode == 1 and "does not match" in result.stderr + assert session.submit.used == 2 + + +def test_verifier_infrastructure_error_does_not_consume(tmp_path): + certificate = tmp_path / "certificate.json" + certificate.write_text(json.dumps(_cert("r1"))) + + def broken_verifier(cert): + raise RuntimeError("oracle unavailable") + + with EvidenceBudgetSession( + rule="r1", budget=_budget(), pred_binary=_fake_pred(tmp_path), + verifier=broken_verifier, + ) as session: + result = _run("submit", str(certificate), cwd=session.workdir) + assert result.returncode == 2 + assert "INFRASTRUCTURE_ERROR" in result.stderr + assert session.submit.used == 0 + assert session.submit.remaining == 2 + + +def test_pred_request_replay_is_idempotent(tmp_path, monkeypatch): + request_id = "a" * 32 + with EvidenceBudgetSession( + rule="r1", budget=_budget(pred=1, solve=0), pred_binary=_fake_pred(tmp_path), + verifier=lambda cert: Verdict(False, "no bug"), + ) as session: + monkeypatch.chdir(session.workdir) + first = pred_request({"op": "pred", "args": ["create", "X"], + "cwd": str(session.workdir)}, request_id) + second = pred_request({"op": "pred", "args": ["create", "X"], + "cwd": str(session.workdir)}, request_id) + assert first == second + assert session.status()["pred_calls"]["used"] == 1 + assert len(session.pred.ledger) == 1 + + +def test_invalid_pred_envelope_is_idempotent_and_charged(tmp_path): + request_id = "c" * 32 + with EvidenceBudgetSession( + rule="r1", budget=_budget(pred=1, solve=0), pred_binary=_fake_pred(tmp_path), + verifier=lambda cert: Verdict(False, "no bug"), + ) as session: + first = _raw_spool(session.pred.channel, request_id, b"{bad") + second = _raw_spool(session.pred.channel, request_id, b"{bad") + + assert first == second + assert first["model_error"] is True + assert session.status()["pred_calls"]["used"] == 1 + assert len(session.pred.ledger) == 1 + + +def test_submit_request_replay_is_idempotent(tmp_path): + request_id = "b" * 32 + payload = {"op": "submit", "certificate_text": json.dumps(_cert("r1"))} + with EvidenceBudgetSession( + rule="r1", budget=_budget(), pred_binary=_fake_pred(tmp_path), + verifier=lambda cert: Verdict(False, "no bug"), + ) as session: + first = submit_request(payload, request_id) + second = submit_request(payload, request_id) + assert first == second + assert session.submit.used == 1 + assert len(session.submit.attempts) == 1 + + +def test_malformed_submit_envelope_is_idempotent(tmp_path): + request_id = "d" * 32 + with EvidenceBudgetSession( + rule="r1", budget=_budget(), pred_binary=_fake_pred(tmp_path), + verifier=lambda cert: Verdict(False, "no bug"), + ) as session: + channel = Path(os.environ["PRB_SUBMIT_DIR"]) + first = _raw_spool(channel, request_id, b"{bad") + second = _raw_spool(channel, request_id, b"{bad") + + assert first == second + assert session.submit.used == 1 + assert session.submit.attempts[0]["request_id"] == request_id + + +def test_model_and_shell_budgets_have_auditable_events(tmp_path): + budget = EvidenceBudget(1, 1, 0, 0, submit_attempts=2) + with EvidenceBudgetSession( + rule="r1", budget=budget, pred_binary=_fake_pred(tmp_path), + verifier=lambda cert: Verdict(False, "no bug"), + ) as session: + assert session.record_model_generation(outcome="completed") is True + assert session.record_model_generation(outcome="completed") is False + assert session.record_model_generation( + outcome="provider_error", infrastructure_error=True) is True + assert session.admit_shell_action("pwd") is True + assert session.admit_shell_action("pwd") is False + + ledger = session.ledger() + assert [event["charged"] for event in ledger["model_generations"]] == [True, False, False] + assert [event["charged"] for event in ledger["shell_actions"]] == [True, False] + assert session.status()["model_generations"]["used"] == 1 + assert session.status()["shell_actions"]["used"] == 1 + + +def test_agent_shell_output_has_one_combined_cap(tmp_path): + result = run_as_agent( + "python3 -c 'import sys; print(\"x\" * 5000); print(\"y\" * 5000, file=sys.stderr)'", + cwd=str(tmp_path), + env=dict(os.environ), + timeout=5, + uid=os.getuid(), + gid=os.getgid(), + max_output_chars=512, + ) + + assert result.returncode == 0 + assert len(result.stdout) <= 512 + assert "characters elided" in result.stdout + + +def test_budget_contract_rejects_invalid_values(): + with pytest.raises(ValueError, match="exactly 2"): + EvidenceBudget(1, 1, 1, 1, submit_attempts=100) + with pytest.raises(ValueError, match="solve_calls"): + EvidenceBudget(1, 1, 1, 2) diff --git a/benchmark/tests/test_preflight.py b/benchmark/tests/test_preflight.py index 405ac3c..a6b89a5 100644 --- a/benchmark/tests/test_preflight.py +++ b/benchmark/tests/test_preflight.py @@ -35,6 +35,7 @@ def _calculate_cost(self, response): def _patch(monkeypatch, *, ver="0.6.0", rules=("a", "b"), model=None, build_exc=None): + monkeypatch.setattr(pf, "find_pred_binary", lambda: "/fake/pred") monkeypatch.setattr(pf, "verify_pred_version", lambda *a, **k: ver) monkeypatch.setattr(pf, "list_rules", lambda repo: list(rules)) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5c84bf6..0639dee 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -46,7 +46,7 @@ RUN CARGO_PROFILE_RELEASE_LTO=off \ FROM python:3.12-slim-bookworm AS runtime # Runtime libs pred may dynamically link (HiGHS uses OpenMP / libstdc++) — ref-independent, # so keep before ARG to preserve the cache across ref bumps. -RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 libstdc++6 \ +RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 libstdc++6 passwd \ && rm -rf /var/lib/apt/lists/* ARG PR_REF LABEL org.opencontainers.image.title="problem-reductions-benchmark" \ @@ -54,18 +54,37 @@ LABEL org.opencontainers.image.title="problem-reductions-benchmark" \ org.opencontainers.image.licenses="MIT" \ problem_reductions_ref="${PR_REF}" -COPY --from=pred-build /src/target/release/pred /usr/local/bin/pred -ENV PRED_BINARY=pred PYTHONUNBUFFERED=1 +RUN groupadd --gid 10003 prb-evidence \ + && groupadd --gid 10001 prb-agent \ + && groupadd --gid 10002 prb-oracle \ + && useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin prb-agent \ + && useradd --uid 10002 --gid 10002 --no-create-home --shell /usr/sbin/nologin prb-oracle \ + && usermod -a -G prb-evidence prb-agent \ + && usermod -a -G prb-evidence prb-oracle \ + && install -d -o root -g prb-oracle -m 0750 /usr/local/libexec/prb +COPY --from=pred-build /src/target/release/pred /usr/local/libexec/prb/pred +RUN chown root:prb-oracle /usr/local/libexec/prb/pred \ + && chmod 0750 /usr/local/libexec/prb/pred +ENV PRED_BINARY=/usr/local/libexec/prb/pred \ + PRB_AGENT_UID=10001 \ + PRB_AGENT_GID=10001 \ + PRB_ORACLE_UID=10002 \ + PRB_ORACLE_GID=10002 \ + PRB_EVIDENCE_GID=10003 \ + PYTHONUNBUFFERED=1 WORKDIR /app COPY benchmark/ ./benchmark/ # Fail the build if pred can't run (catches a missing runtime lib early). -RUN pred --version +RUN /usr/local/libexec/prb/pred --version # Bake the target version pins so benchmark/env_setup.py reads the *built* commit/version # rather than a frozen Python literal (the target tracks PR_REF, not a constant). COPY --from=pred-build /src/PINNED_COMMIT /app/benchmark/PINNED_COMMIT -RUN pred --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -n1 > /app/benchmark/PINNED_VERSION +RUN /usr/local/libexec/prb/pred --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -n1 > /app/benchmark/PINNED_VERSION +# Prove the complete privilege boundary, not just a file mode: the dedicated agent user can +# call both shims while an absolute-path oracle invocation is denied. +RUN python -m benchmark.privilege_smoke # Default: run the verifier calibration (fixtures) as a self-test. # Override CMD to verify a certificate, e.g. (run with NO network — the verifier only needs From 513a85dd2ad695f02d60e926eb761b123d868f66 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Mon, 20 Jul 2026 04:31:00 +0800 Subject: [PATCH 2/9] feat: run self-selected Top50 episodes --- benchmark/agent_environment.py | 13 +- benchmark/process_control.py | 5 + benchmark/run_top50.py | 102 +++++ benchmark/tests/test_run_top50.py | 36 ++ benchmark/tests/test_top50_runner.py | 223 +++++++++++ benchmark/top50_config.yaml | 38 ++ benchmark/top50_runner.py | 556 +++++++++++++++++++++++++++ docker/Dockerfile | 4 +- 8 files changed, 974 insertions(+), 3 deletions(-) create mode 100644 benchmark/run_top50.py create mode 100644 benchmark/tests/test_run_top50.py create mode 100644 benchmark/tests/test_top50_runner.py create mode 100644 benchmark/top50_config.yaml create mode 100644 benchmark/top50_runner.py diff --git a/benchmark/agent_environment.py b/benchmark/agent_environment.py index c2ce4f5..a69354c 100644 --- a/benchmark/agent_environment.py +++ b/benchmark/agent_environment.py @@ -6,6 +6,17 @@ from benchmark.process_control import ProcessLimits, run_capped_process +_SAFE_ENV_KEYS = {"PATH", "PYTHONPATH", "LANG", "LC_ALL", "PAGER", "MANPAGER", "LESS", + "PRB_PRED_DIR", "PRB_SUBMIT_DIR", "PRB_ARTIFACT_DIR", + "PRB_PRED_TIMEOUT", "PRB_SUBMIT_TIMEOUT"} + + +def sanitized_agent_env(extra: dict[str, str] | None = None) -> dict[str, str]: + """Return the small non-secret environment visible to model-authored commands.""" + merged = dict(os.environ) + merged.update(extra or {}) + return {key: value for key, value in merged.items() if key in _SAFE_ENV_KEYS} + def run_as_agent( command: str, @@ -64,7 +75,7 @@ def execute(self, action: dict, cwd: str = "", *, timeout: int | None = None) -> result = run_as_agent( command, cwd=action_cwd, - env=os.environ | self.config.env, + env=sanitized_agent_env(self.config.env), timeout=timeout or self.config.timeout, uid=uid, gid=gid, diff --git a/benchmark/process_control.py b/benchmark/process_control.py index 76b55d9..3036bd2 100644 --- a/benchmark/process_control.py +++ b/benchmark/process_control.py @@ -132,6 +132,11 @@ def run_capped_process( timed_out = True terminate_process_group(process) process.wait() + else: + # A successful shell leader may leave background children behind. Kill the + # entire private process group before reading pipes to completion so no process + # can survive into a later rule episode or keep a pipe open indefinitely. + terminate_process_group(process) for reader in readers: reader.join() stdout, stderr = capture.result() diff --git a/benchmark/run_top50.py b/benchmark/run_top50.py new file mode 100644 index 0000000..1634334 --- /dev/null +++ b/benchmark/run_top50.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Production entrypoint for the standardized self-selected Top50 model track.""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + +from benchmark.env_setup import find_pred_binary, pinned_commit, verify_pred_version +from benchmark.evidence_budget import EvidenceBudget +from benchmark.run_mini import list_rules +from benchmark.top50_runner import ( + PhaseResult, + Top50Contract, + Top50Runner, + TriageBudget, + build_rankable_runner, +) + + +def pilot_contract() -> Top50Contract: + """Return explicitly provisional values; issue #68 freezes the public contract.""" + return Top50Contract( + triage=TriageBudget( + model_generations=int(os.environ.get("PRB_TRIAGE_GENERATIONS", "8")), + shell_actions=int(os.environ.get("PRB_TRIAGE_ACTIONS", "12"))), + episode=EvidenceBudget( + model_generations=int(os.environ.get("PRB_EPISODE_GENERATIONS", "10")), + shell_actions=int(os.environ.get("PRB_EPISODE_ACTIONS", "12")), + pred_calls=int(os.environ.get("PRB_PRED_CALLS", "24")), + solve_calls=int(os.environ.get("PRB_SOLVE_CALLS", "10")), + submit_attempts=2, + max_output_chars=int(os.environ.get("PRB_MAX_OUTPUT_CHARS", "10000")), + pred_timeout_seconds=int(os.environ.get("PRB_PRED_TIMEOUT_SECONDS", "300"))), + ) + + +class _FakeExecutor: + def run_triage(self, session, *, repo_path, inventory, model): + payload = session.workdir / "shortlist.json" + payload.write_text(json.dumps(list(inventory[:50])), encoding="utf-8") + session.commit_file(str(payload)) + return PhaseResult(messages=[]) + + def run_episode(self, session, **kwargs): + return PhaseResult(messages=[]) + + +def run(*, model: str, repo_dir: str | Path, output: str | Path, + fake: bool = False, api_base: str | None = None, api_key: str | None = None, + model_kwargs: dict | None = None) -> dict: + repo = Path(repo_dir).resolve() + inventory = list_rules(repo) + if len(inventory) < 50: + raise ValueError(f"canonical inventory has only {len(inventory)} runnable rules") + pred_binary = find_pred_binary() + verify_pred_version(pred_binary) + contract = pilot_contract() + if fake: + runner = Top50Runner( + executor=_FakeExecutor(), contract=contract, pred_binary=pred_binary) + else: + runner = build_rankable_runner( + contract=contract, pred_binary=pred_binary, + agent_uid=int(os.environ["PRB_AGENT_UID"]), + agent_gid=int(os.environ["PRB_AGENT_GID"]), + oracle_uid=int(os.environ["PRB_ORACLE_UID"]), + oracle_gid=int(os.environ["PRB_ORACLE_GID"]), + evidence_gid=int(os.environ["PRB_EVIDENCE_GID"]), + api_base=api_base, api_key=api_key, model_kwargs=model_kwargs) + result = runner.run(model=model, repo_path=repo, inventory=inventory, output=output) + result["library_commit"] = pinned_commit() + result["budget_contract_status"] = "pilot-unfrozen" + # Rewrite once with provenance added after the runner's final checkpoint. + Path(output).write_text(json.dumps(result, indent=2), encoding="utf-8") + return result + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description="Standardized Model API Top50 benchmark") + parser.add_argument("--model", default=os.environ.get("MODEL_NAME"), required=False) + parser.add_argument("--repo-dir", default=os.environ.get("REPO_DIR", "/app/pr-src")) + parser.add_argument("--output", default=os.environ.get("OUTPUT", "/out/submission.json")) + parser.add_argument("--api-base", default=os.environ.get("API_BASE")) + parser.add_argument("--api-key", default=os.environ.get("API_KEY")) + parser.add_argument("--model-kwargs", default=os.environ.get("MODEL_KWARGS")) + parser.add_argument("--fake", action="store_true", default=bool(os.environ.get("FAKE"))) + args = parser.parse_args(argv) + if not args.model: + parser.error("--model (or MODEL_NAME) is required") + kwargs = json.loads(args.model_kwargs) if args.model_kwargs else None + if kwargs is not None and not isinstance(kwargs, dict): + parser.error("--model-kwargs must be a JSON object") + result = run(model=args.model, repo_dir=args.repo_dir, output=args.output, + fake=args.fake, api_base=args.api_base, api_key=args.api_key, + model_kwargs=kwargs) + print(f"Top50 {result['status']} ({len(result['episodes'])}/50 episodes) → {args.output}") + + +if __name__ == "__main__": + main() diff --git a/benchmark/tests/test_run_top50.py b/benchmark/tests/test_run_top50.py new file mode 100644 index 0000000..a69211d --- /dev/null +++ b/benchmark/tests/test_run_top50.py @@ -0,0 +1,36 @@ +"""Entrypoint wiring tests for the standardized Top50 track.""" +from __future__ import annotations + +import json + +import pytest + +from benchmark import run_top50 + + +def test_fake_entrypoint_uses_50_isolated_episodes(tmp_path, monkeypatch): + repo = tmp_path / "repo" + rules = repo / "src" / "rules" + rules.mkdir(parents=True) + for index in range(55): + (rules / f"rule_{index:02d}.rs").write_text("// rule") + pred = tmp_path / "pred" + pred.write_text("#!/bin/sh\necho pred 0.6.0\n") + pred.chmod(0o700) + monkeypatch.setattr(run_top50, "find_pred_binary", lambda: pred) + monkeypatch.setattr(run_top50, "verify_pred_version", lambda binary: "0.6.0") + output = tmp_path / "top50.json" + + result = run_top50.run( + model="fake/model", repo_dir=repo, output=output, fake=True) + + assert len(result["shortlist"]) == 50 + assert len(result["episodes"]) == 50 + assert result["rankable"] is False + assert json.loads(output.read_text())["budget_contract_status"] == "pilot-unfrozen" + + +@pytest.mark.parametrize("forbidden", ["--backend", "--config", "--strategy-file"]) +def test_standard_entrypoint_rejects_custom_harness_options(forbidden): + with pytest.raises(SystemExit): + run_top50.main(["--model", "fake/model", "--fake", forbidden, "custom"]) diff --git a/benchmark/tests/test_top50_runner.py b/benchmark/tests/test_top50_runner.py new file mode 100644 index 0000000..50137a6 --- /dev/null +++ b/benchmark/tests/test_top50_runner.py @@ -0,0 +1,223 @@ +"""Deterministic end-to-end acceptance tests for the self-selected Top50 workflow.""" +from __future__ import annotations + +import json +import time +from pathlib import Path + +import pytest + +from benchmark.top50_runner import ( + PhaseResult, + ShortlistEntry, + Top50Contract, + Top50Runner, + TriageBudget, + build_rankable_runner, + format_status, +) +from benchmark.evidence_budget import EvidenceBudget +from benchmark.verify import Verdict + + +def _fake_pred(tmp_path: Path) -> Path: + script = tmp_path / "real-pred" + script.write_text("#!/bin/sh\necho ran:$*\n", encoding="utf-8") + script.chmod(0o700) + return script + + +def _contract() -> Top50Contract: + return Top50Contract( + triage=TriageBudget(model_generations=3, shell_actions=3), + episode=EvidenceBudget( + model_generations=2, shell_actions=2, pred_calls=1, solve_calls=0, + submit_attempts=2, max_output_chars=1024, pred_timeout_seconds=2), + ) + + +class FakeExecutor: + def __init__(self, shortlist_payload=None, *, fail_episode: int | None = None): + self.shortlist_payload = shortlist_payload + self.fail_episode = fail_episode + self.workspaces: list[Path] = [] + self.initial_statuses: list[dict] = [] + + def run_triage(self, session, *, repo_path, inventory, model): + session.record_model_generation() + session.admit_shell_action("commit-top50 shortlist.json") + payload = self.shortlist_payload + if payload is None: + payload = [{"rule": rule, "hypothesis": f"risk-{index}"} + for index, rule in enumerate(inventory[:50])] + path = session.workdir / "shortlist.json" + path.write_text(json.dumps(payload), encoding="utf-8") + accepted, _ = session.commit_file(str(path)) + return PhaseResult(messages=[{"role": "triage", "accepted": accepted}]) + + def run_episode(self, session, *, repo_path, entry: ShortlistEntry, index, total, model): + self.workspaces.append(session.workdir) + self.initial_statuses.append(session.status()) + assert not (session.workdir / "sentinel").exists() + if index == 1: + (session.workdir / "sentinel").write_text("private") + session.state.reserve("pred_calls") + session.submit._handle( + {"op": "submit", "certificate_text": json.dumps({ + "rule": entry.rule, "source": {}, "bundle": {"target": {"type": "T"}}})}) + session.record_model_generation() + session.admit_shell_action("pwd") + error = "provider unavailable" if self.fail_episode == index else None + return PhaseResult(messages=[{"role": "episode", "rule": entry.rule}], error=error) + + +def _runner(tmp_path: Path, executor) -> Top50Runner: + return Top50Runner( + executor=executor, + contract=_contract(), + pred_binary=_fake_pred(tmp_path), + verifier=lambda cert: Verdict(False, "not a bug"), + ) + + +def test_frozen_top50_runs_in_order_with_fresh_state(tmp_path): + inventory = [f"rule_{index:02d}" for index in range(60)] + executor = FakeExecutor() + result = _runner(tmp_path, executor).run( + model="fake/model", repo_path=tmp_path, inventory=inventory) + + # Injected executors are development-only even when they finish all 50 episodes. + assert result["rankable"] is False + assert [entry["rule"] for entry in result["shortlist"]] == inventory[:50] + assert [episode["rule"] for episode in result["episodes"]] == inventory[:50] + assert len({str(path) for path in executor.workspaces}) == 50 + assert all(status["pred_calls"]["used"] == 0 for status in executor.initial_statuses) + assert all(status["submit_attempts"]["used"] == 0 for status in executor.initial_statuses) + assert all(episode["messages"] == [{"role": "episode", "rule": episode["rule"]}] + for episode in result["episodes"]) + + +@pytest.mark.parametrize("payload", [ + [f"rule_{index:02d}" for index in range(49)], + [f"rule_{index:02d}" for index in range(51)], + ["rule_00"] * 50, + [*[f"rule_{index:02d}" for index in range(49)], "unknown"], + [*[f"rule_{index:02d}" for index in range(49)], + {"rule": "rule_49", "hypothesis": "x" * 501}], +]) +def test_invalid_shortlist_never_starts_an_episode(tmp_path, payload): + executor = FakeExecutor(payload) + result = _runner(tmp_path, executor).run( + model="fake/model", repo_path=tmp_path, + inventory=[f"rule_{index:02d}" for index in range(60)]) + + assert result["rankable"] is False + assert result["episodes"] == [] + assert "valid frozen Top50" in result["run_error"] + + +def test_episode_infrastructure_error_is_partial_and_unrankable(tmp_path): + executor = FakeExecutor(fail_episode=3) + result = _runner(tmp_path, executor).run( + model="fake/model", repo_path=tmp_path, + inventory=[f"rule_{index:02d}" for index in range(60)]) + + assert result["rankable"] is False + assert len(result["episodes"]) == 3 + assert result["episodes"][-1]["status"] == "run_error" + assert "provider unavailable" in result["run_error"] + + +def test_raised_episode_error_is_checkpointed(tmp_path): + class RaisingExecutor(FakeExecutor): + def run_episode(self, session, **kwargs): + if kwargs["index"] == 2: + raise RuntimeError("gateway vanished") + return super().run_episode(session, **kwargs) + + output = tmp_path / "partial.json" + result = _runner(tmp_path, RaisingExecutor()).run( + model="fake/model", repo_path=tmp_path, + inventory=[f"rule_{index:02d}" for index in range(60)], output=output) + + assert result["rankable"] is False + assert len(result["episodes"]) == 2 + assert "gateway vanished" in result["run_error"] + assert json.loads(output.read_text())["run_error"] == result["run_error"] + + +def test_second_shortlist_commit_is_rejected(tmp_path): + from benchmark.top50_runner import TriageSession + + inventory = tuple(f"rule_{index:02d}" for index in range(60)) + with TriageSession(inventory=inventory, budget=TriageBudget(2, 2)) as session: + path = session.workdir / "shortlist.json" + path.write_text(json.dumps(list(inventory[:50]))) + assert session.commit_file(str(path))[0] is True + assert session.commit_file(str(path)) == (False, "shortlist is already frozen") + + +def test_observation_status_reports_every_authoritative_counter(tmp_path): + executor = FakeExecutor() + runner = _runner(tmp_path, executor) + entry = ShortlistEntry("rule_00") + from benchmark.evidence_budget import EvidenceBudgetSession + + with EvidenceBudgetSession( + rule=entry.rule, budget=_contract().episode, pred_binary=runner.pred_binary, + verifier=lambda cert: Verdict(False, "no bug"), + ) as session: + text = format_status(session, index=1, total=50) + + for expected in ("rule 1/50: rule_00", "model generations: 0/2", + "shell actions: 0/2", "pred calls: 0/1", + "solve calls: 0/0", "submit attempts: 0/2"): + assert expected in text + + +def test_rankable_contract_is_fixed_to_model_api_surface(): + executor = FakeExecutor() + assert not hasattr(executor, "backend") + with pytest.raises(ValueError, match="exactly 50"): + Top50Contract(_contract().triage, _contract().episode, shortlist_size=49) + + +def test_only_standard_factory_can_mark_a_run_rankable(tmp_path, monkeypatch): + import benchmark.top50_runner as top50 + + monkeypatch.setattr(top50.os, "geteuid", lambda: 0) + runner = build_rankable_runner( + contract=_contract(), pred_binary=_fake_pred(tmp_path), + agent_uid=10001, agent_gid=10001, oracle_uid=10002, oracle_gid=10002, + evidence_gid=10003) + assert runner._rankable_contract is True + with pytest.raises(ValueError, match="distinct"): + build_rankable_runner( + contract=_contract(), pred_binary=_fake_pred(tmp_path), + agent_uid=10001, agent_gid=10001, oracle_uid=10001, oracle_gid=10002, + evidence_gid=10003) + + +def test_agent_environment_does_not_expose_provider_credentials(monkeypatch): + from benchmark.agent_environment import sanitized_agent_env + + monkeypatch.setenv("OPENAI_API_KEY", "secret") + monkeypatch.setenv("ANTHROPIC_API_KEY", "secret") + monkeypatch.setenv("PRB_PRED_DIR", "/gateway") + env = sanitized_agent_env() + assert "OPENAI_API_KEY" not in env and "ANTHROPIC_API_KEY" not in env + assert env["PRB_PRED_DIR"] == "/gateway" + + +def test_successful_shell_cannot_leave_background_process(tmp_path): + from benchmark.agent_environment import run_as_agent, sanitized_agent_env + import os + + sentinel = tmp_path / "late" + command = f"(sleep 0.2; touch {sentinel}) >/dev/null 2>&1 &" + result = run_as_agent( + command, cwd=str(tmp_path), env=sanitized_agent_env(), timeout=2, + uid=os.getuid(), gid=os.getgid()) + assert result.returncode == 0 + time.sleep(0.4) + assert not sentinel.exists() diff --git a/benchmark/top50_config.yaml b/benchmark/top50_config.yaml new file mode 100644 index 0000000..4a94829 --- /dev/null +++ b/benchmark/top50_config.yaml @@ -0,0 +1,38 @@ +agent: + system_template: | + You are evaluating reduction rules under a fixed logical evidence budget. Each response + must contain exactly one bash command. The authoritative remaining budget is injected into + every observation; elapsed time is not a score and unused budget never transfers. + + {% if phase == "triage" %} + This is source-only triage. Read {{repo_dir}}/src/rules but do not run pred or submit. + Canonical rule inventory: {{inventory}} + Write a JSON list of exactly 50 unique entries to the workspace. Each entry is either a + rule string or {"rule": "...", "hypothesis": "..."}. Freeze it with: + commit-top50 shortlist.json + {% else %} + Investigate only rule {{rule}} ({{rule_index}}/{{rule_total}}). + Triage hypothesis: {{hypothesis}} + Source: {{repo_dir}}/src/rules/{{rule}}.rs + Use pred for dynamic evidence. If you reproduce a bug, write a certificate under + $PRB_ARTIFACT_DIR and call submit immediately. There are exactly two submit attempts for + this rule. Never investigate another rule. + {% endif %} + + Initial authoritative budget: + {{budget_status}} + + Finish normally with: echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT + instance_template: | + {{task}} + step_limit: 0 + cost_limit: 0 + max_consecutive_format_errors: 3 + +model: + observation_template: | + {% if output.exception_info %}{{output.exception_info}}{% endif %} + {{output.returncode}} + {{output.output}} + format_error_template: | + Format error: {{error}}. Provide exactly one bash command. diff --git a/benchmark/top50_runner.py b/benchmark/top50_runner.py new file mode 100644 index 0000000..3eb1499 --- /dev/null +++ b/benchmark/top50_runner.py @@ -0,0 +1,556 @@ +"""Self-selected Top50 workflow with frozen triage and isolated rule episodes.""" +from __future__ import annotations + +import copy +import json +import os +import shlex +import shutil +import tempfile +import threading +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Callable, Protocol + +from benchmark.agent_environment import make_agent_environment, run_as_agent, sanitized_agent_env +from benchmark.evidence_budget import EvidenceBudget, EvidenceBudgetSession, EvidenceBudgetState +from benchmark.run_mini import ( + DEFAULT_MAX_TOKENS, + _build_model, + _load_agent_config, + _message_text, + _session_usage, +) +from benchmark.usage import Usage, usage_as_dict +from benchmark.verify import Verdict, verify + +TOP50_SIZE = 50 +DEFAULT_HYPOTHESIS_CHARS = 500 +MAX_SHORTLIST_BYTES = 128 * 1024 +_RANKABLE_TOKEN = object() + + +@dataclass(frozen=True) +class TriageBudget: + model_generations: int + shell_actions: int + max_output_chars: int = 10_000 + command_timeout_seconds: int = 300 + + def __post_init__(self) -> None: + for name, value in asdict(self).items(): + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + + +@dataclass(frozen=True) +class Top50Contract: + triage: TriageBudget + episode: EvidenceBudget + shortlist_size: int = TOP50_SIZE + hypothesis_chars: int = DEFAULT_HYPOTHESIS_CHARS + + def __post_init__(self) -> None: + if self.shortlist_size != TOP50_SIZE: + raise ValueError("the rankable contract requires exactly 50 rules") + if self.hypothesis_chars <= 0: + raise ValueError("hypothesis_chars must be positive") + + +@dataclass(frozen=True) +class ShortlistEntry: + rule: str + hypothesis: str = "" + + +@dataclass +class PhaseResult: + messages: list[dict] + tokens_k: float = 0.0 + usage: object | None = None + error: str | None = None + + +class PhaseExecutor(Protocol): + def run_triage(self, session: "TriageSession", *, repo_path: Path, + inventory: tuple[str, ...], model: str) -> PhaseResult: ... + + def run_episode(self, session: EvidenceBudgetSession, *, repo_path: Path, + entry: ShortlistEntry, index: int, total: int, + model: str) -> PhaseResult: ... + + +class TriageSession: + """Evaluation-owned source-only workspace and one-shot shortlist controller.""" + + def __init__(self, *, inventory: tuple[str, ...], budget: TriageBudget, + shortlist_size: int = TOP50_SIZE, + hypothesis_chars: int = DEFAULT_HYPOTHESIS_CHARS, + agent_uid: int | None = None, agent_gid: int | None = None): + if len(set(inventory)) != len(inventory): + raise ValueError("canonical inventory contains duplicates") + if (agent_uid is None) != (agent_gid is None): + raise ValueError("agent_uid and agent_gid must be provided together") + self.inventory = inventory + self.budget = budget + self.shortlist_size = shortlist_size + self.hypothesis_chars = hypothesis_chars + self.agent_uid = agent_uid + self.agent_gid = agent_gid + evidence = EvidenceBudget( + model_generations=budget.model_generations, + shell_actions=budget.shell_actions, + pred_calls=0, + solve_calls=0, + submit_attempts=2, + max_output_chars=budget.max_output_chars, + pred_timeout_seconds=budget.command_timeout_seconds, + ) + self.state = EvidenceBudgetState(evidence) + self._tmpdir: Path | None = None + self._workdir: Path | None = None + self._shortlist: tuple[ShortlistEntry, ...] | None = None + self._events: list[dict] = [] + self._lock = threading.RLock() + + @property + def workdir(self) -> Path: + if self._workdir is None: + raise RuntimeError("triage session is not active") + return self._workdir + + @property + def shortlist(self) -> tuple[ShortlistEntry, ...] | None: + with self._lock: + return copy.deepcopy(self._shortlist) + + def __enter__(self) -> "TriageSession": + self._tmpdir = Path(tempfile.mkdtemp(prefix="prb-triage-", dir="/tmp")).resolve() + self._workdir = self._tmpdir / "work" + self._workdir.mkdir(mode=0o700) + if self.agent_uid is not None and self.agent_gid is not None: + self._tmpdir.chmod(0o711) + os.chown(self._workdir, self.agent_uid, self.agent_gid) + self._workdir.chmod(0o700) + return self + + def __exit__(self, exc_type, exc, tb) -> None: + if self._tmpdir is not None: + shutil.rmtree(self._tmpdir, ignore_errors=True) + + def record_model_generation(self, *, outcome: str = "completed", + infrastructure_error: bool = False) -> bool: + reservation = None if infrastructure_error else self.state.reserve("model_generations") + admitted = infrastructure_error or reservation is not None + self._append_event("model_generation", reservation is not None, + outcome if admitted else "budget_exhausted") + return admitted + + def admit_shell_action(self, command: str) -> bool: + reservation = self.state.reserve("shell_actions") + self._append_event("shell_action", reservation is not None, + "admitted" if reservation is not None else "budget_exhausted", + command=command) + return reservation is not None + + def commit_file(self, path: str) -> tuple[bool, str]: + """Validate and atomically freeze a model-authored shortlist JSON file.""" + candidate = Path(path) + if not candidate.is_absolute(): + candidate = self.workdir / candidate + candidate = candidate.resolve() + if not candidate.is_relative_to(self.workdir): + return False, "shortlist file must be inside the triage workspace" + try: + raw = candidate.read_bytes() + except OSError as error: + return False, f"cannot read shortlist: {error}" + if len(raw) > MAX_SHORTLIST_BYTES: + return False, f"shortlist exceeds {MAX_SHORTLIST_BYTES} bytes" + try: + payload = json.loads(raw) + entries = self._validate(payload) + except (UnicodeError, json.JSONDecodeError, ValueError) as error: + return False, str(error) + with self._lock: + if self._shortlist is not None: + return False, "shortlist is already frozen" + self._shortlist = tuple(entries) + self._events.append({"type": "shortlist_commit", "accepted": True, + "rules": [entry.rule for entry in entries]}) + return True, f"frozen {len(entries)} rules" + + def status(self) -> dict: + return self.state.status() + + def ledger(self) -> dict: + with self._lock: + return {"budget": asdict(self.budget), "status": self.status(), + "events": copy.deepcopy(self._events), + "shortlist": ([asdict(entry) for entry in self._shortlist] + if self._shortlist is not None else None)} + + def _validate(self, payload: object) -> list[ShortlistEntry]: + if not isinstance(payload, list) or len(payload) != self.shortlist_size: + raise ValueError(f"shortlist must contain exactly {self.shortlist_size} entries") + entries: list[ShortlistEntry] = [] + for item in payload: + if isinstance(item, str): + rule, hypothesis = item, "" + elif isinstance(item, dict): + rule, hypothesis = item.get("rule"), item.get("hypothesis", "") + else: + raise ValueError("each shortlist entry must be a rule string or object") + if not isinstance(rule, str) or rule not in self.inventory: + raise ValueError(f"unknown rule in shortlist: {rule!r}") + if not isinstance(hypothesis, str) or len(hypothesis) > self.hypothesis_chars: + raise ValueError( + f"hypothesis for {rule!r} must be at most {self.hypothesis_chars} characters") + entries.append(ShortlistEntry(rule, hypothesis)) + rules = [entry.rule for entry in entries] + if len(set(rules)) != len(rules): + raise ValueError("shortlist rules must be unique") + return entries + + def _append_event(self, event_type: str, charged: bool, outcome: str, **extra) -> None: + with self._lock: + self._events.append({"sequence": len(self._events) + 1, "type": event_type, + "charged": charged, "outcome": outcome, + "budget": self.status(), **extra}) + + +class Top50Runner: + """Freeze one self-selected Top50, then execute 50 fresh sequential episodes.""" + + def __init__(self, *, executor: PhaseExecutor, contract: Top50Contract, + pred_binary: str | Path, verifier: Callable[[dict], Verdict] = verify, + agent_uid: int | None = None, agent_gid: int | None = None, + oracle_uid: int | None = None, oracle_gid: int | None = None, + evidence_gid: int | None = None, _rankable_token=None): + self.executor = executor + self.contract = contract + self.pred_binary = Path(pred_binary) + self.verifier = verifier + self.identities = {"agent_uid": agent_uid, "agent_gid": agent_gid, + "oracle_uid": oracle_uid, "oracle_gid": oracle_gid, + "evidence_gid": evidence_gid} + self._rankable_contract = _rankable_token is _RANKABLE_TOKEN + + def run(self, *, model: str, repo_path: str | Path, + inventory: list[str] | tuple[str, ...], output: str | Path | None = None) -> dict: + repo_path = Path(repo_path).resolve() + canonical = tuple(inventory) + with TriageSession( + inventory=canonical, + budget=self.contract.triage, + shortlist_size=self.contract.shortlist_size, + hypothesis_chars=self.contract.hypothesis_chars, + agent_uid=self.identities["agent_uid"], + agent_gid=self.identities["agent_gid"], + ) as triage: + try: + triage_result = self.executor.run_triage( + triage, repo_path=repo_path, inventory=canonical, model=model) + except Exception as error: + triage_result = PhaseResult( + messages=[], error=f"{type(error).__name__}: {error}") + shortlist = triage.shortlist + triage_ledger = triage.ledger() + if triage_result.error: + result = self._result(model, triage_ledger, shortlist, [], triage_result, + f"triage infrastructure error: {triage_result.error}") + return _persist(result, output) + if shortlist is None: + result = self._result(model, triage_ledger, None, [], triage_result, + "triage ended without a valid frozen Top50") + return _persist(result, output) + + episodes: list[dict] = [] + run_error = None + for index, entry in enumerate(shortlist, 1): + episode = None + try: + with EvidenceBudgetSession( + rule=entry.rule, + budget=self.contract.episode, + pred_binary=self.pred_binary, + verifier=self.verifier, + **self.identities, + ) as episode: + phase = self.executor.run_episode( + episode, repo_path=repo_path, entry=entry, + index=index, total=len(shortlist), model=model) + ledger = episode.ledger() + except Exception as error: + phase = PhaseResult(messages=[], error=f"{type(error).__name__}: {error}") + ledger = (episode.ledger() if episode is not None + and episode.submit is not None and episode.pred is not None else {}) + accepted = next((attempt for attempt in ledger.get("submit", []) + if attempt.get("accepted")), None) + record = { + "index": index, + "rule": entry.rule, + "hypothesis": entry.hypothesis, + "status": "run_error" if phase.error else ( + "bug_found" if accepted else "completed"), + "accepted_submit_attempt": accepted.get("attempt") if accepted else None, + "ledger": ledger, + "messages": copy.deepcopy(phase.messages), + "tokens_k": phase.tokens_k, + "usage": _usage_dict(phase.usage), + } + episodes.append(record) + if phase.error: + run_error = f"episode {index} ({entry.rule}) infrastructure error: {phase.error}" + break + checkpoint = self._result( + model, triage_ledger, shortlist, episodes, triage_result, + f"run incomplete after episode {index}/{len(shortlist)}") + _persist(checkpoint, output) + result = self._result( + model, triage_ledger, shortlist, episodes, triage_result, run_error) + return _persist(result, output) + + def _result(self, model: str, triage: dict, + shortlist: tuple[ShortlistEntry, ...] | None, episodes: list[dict], + triage_result: PhaseResult, run_error: str | None) -> dict: + result = { + "model": model, + "status": "run_error" if run_error else "completed", + "rankable": (self._rankable_contract and run_error is None + and len(episodes) == self.contract.shortlist_size), + "contract": {"triage": asdict(self.contract.triage), + "episode": asdict(self.contract.episode), + "shortlist_size": self.contract.shortlist_size, + "hypothesis_chars": self.contract.hypothesis_chars}, + "shortlist": ([asdict(entry) for entry in shortlist] if shortlist else None), + "triage": {"ledger": triage, "messages": copy.deepcopy(triage_result.messages), + "tokens_k": triage_result.tokens_k, + "usage": _usage_dict(triage_result.usage)}, + "episodes": episodes, + } + if run_error: + result["run_error"] = run_error + return result + + +def build_rankable_runner( + *, contract: Top50Contract, pred_binary: str | Path, + agent_uid: int, agent_gid: int, oracle_uid: int, oracle_gid: int, evidence_gid: int, + api_base: str | None = None, api_key: str | None = None, + max_tokens: int = DEFAULT_MAX_TOKENS, model_kwargs: dict | None = None, + verifier: Callable[[dict], Verdict] = verify, +) -> Top50Runner: + """Construct the sole rankable harness after proving the required OS boundary.""" + if os.geteuid() != 0: + raise RuntimeError("rankable Top50 runs require the root runner privilege boundary") + if len({agent_uid, oracle_uid}) != 2: + raise ValueError("agent and oracle must use distinct identities") + executor = MiniSwePhaseExecutor( + api_base=api_base, api_key=api_key, max_tokens=max_tokens, + model_kwargs=model_kwargs, agent_uid=agent_uid, agent_gid=agent_gid, + evidence_gid=evidence_gid) + return Top50Runner( + executor=executor, contract=contract, pred_binary=pred_binary, verifier=verifier, + agent_uid=agent_uid, agent_gid=agent_gid, oracle_uid=oracle_uid, + oracle_gid=oracle_gid, evidence_gid=evidence_gid, + _rankable_token=_RANKABLE_TOKEN) + + +def _usage_dict(usage: object | None) -> dict | None: + return usage_as_dict(usage) if isinstance(usage, Usage) else None + + +def _persist(result: dict, output: str | Path | None) -> dict: + if output is None: + return result + destination = Path(output) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(f".{destination.name}.{os.getpid()}.tmp") + temporary.write_text(json.dumps(result, indent=2), encoding="utf-8") + os.replace(temporary, destination) + return result + + +class TriageEnvironment: + """mini-swe environment that exposes source shell actions and intercepted commit-top50.""" + + def __init__(self, session: TriageSession, *, uid: int, gid: int, + extra_groups: tuple[int, ...] = ()): + self.session = session + self.uid = uid + self.gid = gid + self.extra_groups = extra_groups + + def execute(self, action: dict, cwd: str = "", *, timeout: int | None = None) -> dict: + command = action.get("command", "") + if not self.session.admit_shell_action(command): + return {"output": "shell action budget exhausted\n", "returncode": 75, + "exception_info": ""} + try: + words = shlex.split(command) + except ValueError as error: + return {"output": str(error), "returncode": 2, "exception_info": ""} + if words and words[0] == "commit-top50": + if len(words) != 2: + return {"output": "usage: commit-top50 SHORTLIST.json\n", "returncode": 2, + "exception_info": ""} + accepted, message = self.session.commit_file(words[1]) + return {"output": message + "\n", "returncode": 0 if accepted else 2, + "exception_info": ""} + try: + result = run_as_agent( + command, cwd=cwd or str(self.session.workdir), env=sanitized_agent_env(), + timeout=timeout or self.session.budget.command_timeout_seconds, + uid=self.uid, gid=self.gid, extra_groups=self.extra_groups, + max_output_chars=self.session.budget.max_output_chars) + return {"output": result.stdout, "returncode": result.returncode, + "exception_info": ""} + except Exception as error: + return {"output": getattr(error, "output", "") or "", "returncode": -1, + "exception_info": f"{type(error).__name__}: {error}"} + + def get_template_vars(self, **kwargs) -> dict: + return {"cwd": str(self.session.workdir), **kwargs} + + def serialize(self) -> dict: + return {"info": {"environment_type": type(self).__name__}} + + +def format_status(session, *, index: int | None = None, total: int = TOP50_SIZE) -> str: + status = session.status() + lines = [] + if index is not None: + lines.append(f"rule {index}/{total}: {session.rule}") + for key, label in (("model_generations", "model generations"), + ("shell_actions", "shell actions"), + ("pred_calls", "pred calls"), ("solve_calls", "solve calls")): + if key in status: + counter = status[key] + lines.append(f"{label}: {counter['used']}/{counter['limit']}") + if "submit_attempts" in status: + counter = status["submit_attempts"] + lines.append(f"submit attempts: {counter['used']}/{counter['limit']}") + return "\n".join(lines) + + +class MiniSwePhaseExecutor: + """The sole rankable mini-swe/LiteLLM implementation of the phase protocol.""" + + def __init__(self, *, api_base: str | None = None, api_key: str | None = None, + max_tokens: int = DEFAULT_MAX_TOKENS, model_kwargs: dict | None = None, + agent_uid: int | None = None, agent_gid: int | None = None, + evidence_gid: int | None = None): + self.api_base = api_base + self.api_key = api_key + self.max_tokens = max_tokens + self.model_kwargs = model_kwargs + self.agent_uid = os.getuid() if agent_uid is None else agent_uid + self.agent_gid = os.getgid() if agent_gid is None else agent_gid + self.evidence_gid = evidence_gid + config_path = Path(__file__).with_name("top50_config.yaml") + self.agent_config, self.model_config, _ = _load_agent_config( + config_path, config_path, "") + self._models: dict[str, object] = {} + + def run_triage(self, session: TriageSession, *, repo_path: Path, + inventory: tuple[str, ...], model: str) -> PhaseResult: + environment = TriageEnvironment( + session, uid=self.agent_uid, gid=self.agent_gid, + extra_groups=((self.evidence_gid,) if self.evidence_gid is not None else ())) + return self._run_agent( + model, environment, session, + task="Select and commit exactly 50 high-risk reduction rules.", + template_vars={"repo_dir": str(repo_path), "inventory": json.dumps(inventory), + "phase": "triage"}, status=lambda: format_status(session)) + + def run_episode(self, session: EvidenceBudgetSession, *, repo_path: Path, + entry: ShortlistEntry, index: int, total: int, + model: str) -> PhaseResult: + environment = make_agent_environment( + session, uid=self.agent_uid, gid=self.agent_gid, + extra_groups=((self.evidence_gid,) if self.evidence_gid is not None else ())) + return self._run_agent( + model, environment, session, + task=f"Investigate only reduction rule {entry.rule}.", + template_vars={"repo_dir": str(repo_path), "rule": entry.rule, + "hypothesis": entry.hypothesis, "phase": "episode", + "rule_index": index, "rule_total": total}, + status=lambda: format_status(session, index=index, total=total)) + + def _run_agent(self, model_name: str, environment, session, *, task: str, + template_vars: dict, status: Callable[[], str]) -> PhaseResult: + from minisweagent.agents.default import DefaultAgent + from minisweagent.exceptions import FormatError, Submitted + + model = self._models.get(model_name) + if model is None: + model = _build_model( + model_name, self.api_base, self.max_tokens, + model_kwargs=self.model_kwargs, api_key=self.api_key, + observation_template=self.model_config.get("observation_template"), + format_error_template=self.model_config.get("format_error_template")) + self._models[model_name] = model + + class BudgetedAgent(DefaultAgent): + def query(self): + if session.status()["model_generations"]["remaining"] <= 0: + raise Submitted({"role": "exit", "content": "generation budget exhausted", + "extra": {"exit_status": "Submitted", "submission": ""}}) + try: + message = super().query() + except FormatError as error: + session.record_model_generation(outcome="format_error") + budget = status() + for format_message in error.messages: + format_message["content"] = ( + f"[authoritative budget]\n{budget}\n\n" + f"{format_message.get('content', '')}") + raise + except Exception: + session.record_model_generation( + outcome="provider_error", infrastructure_error=True) + raise + session.record_model_generation(outcome="completed") + return message + + def execute_actions(self, message): + actions = message.get("extra", {}).get("actions", []) + budget_text = status() + if len(actions) == 1: + outputs = [self.env.execute(actions[0])] + elif actions: + outputs = [{"output": "exactly one command is required", "returncode": 2, + "exception_info": ""} for _ in actions] + else: + observation = self.model.format_message( + role="user", content=(f"[authoritative budget]\n{budget_text}\n\n" + "Format error: exactly one command is required")) + return self.add_messages(observation) + for output in outputs: + output["output"] = f"[authoritative budget]\n{budget_text}\n\n{output['output']}" + messages = self.add_messages(*self.model.format_observation_messages( + message, outputs, self.get_template_vars())) + current = session.status() + hard_exhausted = any( + current[name]["used"] > 0 and current[name]["remaining"] == 0 + for name in ("model_generations", "shell_actions", "pred_calls") + if name in current) + submit_closed = bool(getattr(getattr(session, "submit", None), "closed", False)) + if hard_exhausted or submit_closed or getattr(session, "shortlist", None) is not None: + raise Submitted({"role": "exit", "content": "phase complete", + "extra": {"exit_status": "Submitted", "submission": ""}}) + return messages + + agent = BudgetedAgent(model, environment, **self.agent_config) + agent.extra_template_vars = template_vars | {"budget_status": status()} + error = None + try: + agent.run(task=task) + except Exception as exception: + error = f"{type(exception).__name__}: {exception}" + tokens_k, usage = _session_usage(agent) + messages = [{"role": message.get("role", ""), "content": _message_text(message)} + for message in agent.messages] + return PhaseResult(messages=messages, tokens_k=tokens_k, + usage=usage, error=error) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0639dee..94478c0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -124,7 +124,7 @@ ENV REPO_DIR=/app/pr-src \ OUTPUT=/out/submission.json VOLUME ["/out"] # Build-time sanity that the assembled runner imports and runs (no API/pred calls). -RUN python -m benchmark.run_submission --fake --model fake/check \ +RUN python -m benchmark.run_top50 --fake --model fake/check \ --output /tmp/buildcheck.json && rm -f /tmp/buildcheck.json -ENTRYPOINT ["python", "-m", "benchmark.run_submission"] +ENTRYPOINT ["python", "-m", "benchmark.run_top50"] CMD [] From 85a59edaa6a0d877ba601599272591d0315d3e36 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Mon, 20 Jul 2026 13:21:45 +0800 Subject: [PATCH 3/9] feat: validate and score Top50 contracts --- .github/scripts/check_aggregate.py | 3 + benchmark/backend_score.py | 41 ++- benchmark/run_top50.py | 52 ++- benchmark/submit.py | 4 + benchmark/tests/test_top50_submission.py | 255 +++++++++++++ benchmark/top50_contract.py | 447 +++++++++++++++++++++++ benchmark/top50_results.schema.json | 26 ++ benchmark/top50_runner.py | 5 +- benchmark/top50_submission.schema.json | 26 ++ benchmark/verify_submission.py | 9 +- docker/Dockerfile | 12 +- site/index.html | 21 +- 12 files changed, 866 insertions(+), 35 deletions(-) create mode 100644 benchmark/tests/test_top50_submission.py create mode 100644 benchmark/top50_contract.py create mode 100644 benchmark/top50_results.schema.json create mode 100644 benchmark/top50_submission.schema.json diff --git a/.github/scripts/check_aggregate.py b/.github/scripts/check_aggregate.py index 5decbf0..5c3bc28 100644 --- a/.github/scripts/check_aggregate.py +++ b/.github/scripts/check_aggregate.py @@ -18,6 +18,9 @@ "model", "library_commit", "bugs_found", "rules_tested", "total_tokens_k", "usage_totals", "efficiency_bugs_per_ktok", "submitted_by", "placeholder", + "benchmark_contract", "runner_version", "pred_version", "agent_mode", "rankable", + "bugs_at_10", "bugs_at_25", "bugs_at_50", "first_attempt_accepts", + "second_attempt_accepts", "pred_calls_per_bug", "cap_hits", # per-submission entry files (site/results/.json) also carry provenance tags "timestamp", "submission_id", } diff --git a/benchmark/backend_score.py b/benchmark/backend_score.py index 78bbb08..7690a1c 100644 --- a/benchmark/backend_score.py +++ b/benchmark/backend_score.py @@ -25,6 +25,7 @@ from benchmark.env_setup import pinned_commit from benchmark.submit import validate_submission from benchmark.submit_ledger import has_submit_ledger +from benchmark.top50_contract import is_top50_submission from benchmark.verify_submission import leaderboard_entry, score_submission STATUS_SUFFIX = ".status.json" @@ -93,7 +94,10 @@ def write_board_entries(results_dir: Path, board_dir: Path) -> list[str]: scored = json.loads(p.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError): continue - if "results" not in scored or "model" not in scored or scored.get("test"): + if (("results" not in scored and not is_top50_submission(scored)) + or "model" not in scored or scored.get("test")): + continue + if is_top50_submission(scored) and not scored.get("rankable"): continue slug = board_slug(scored, p.stem) (board_dir / f"{slug}.json").write_text( @@ -191,7 +195,7 @@ def _validate_for_scoring(submission: object, *, official: bool, problems = validate_submission(submission) if official: - if not has_submit_ledger(submission): + if not is_top50_submission(submission) and not has_submit_ledger(submission): problems.append("official submissions require submit_limit and submit_log") target_commit = expected_commit or pinned_commit() if submission.get("library_commit") != target_commit: @@ -243,16 +247,29 @@ def score_one(sub_path: Path, results_dir: Path, repo_dir: str | None = None, *, def _dedup_best(entries: list[dict]) -> list[dict]: """Keep the best entry per model (max bugs, tie-break efficiency), ranked desc.""" - best: dict[str, dict] = {} + best: dict[tuple[str, str], dict] = {} for e in entries: m = e["model"] - cur = best.get(m) - key = (e.get("bugs_found", 0), e.get("efficiency_bugs_per_ktok", 0.0)) - if cur is None or key > (cur.get("bugs_found", 0), cur.get("efficiency_bugs_per_ktok", 0.0)): - best[m] = e - return sorted(best.values(), - key=lambda e: (e.get("bugs_found", 0), e.get("efficiency_bugs_per_ktok", 0.0)), - reverse=True) + contract = e.get("benchmark_contract", "legacy-whole-repo") + group = (contract, m) + cur = best.get(group) + if contract.startswith("top50-evidence/"): + better = cur is None or e.get("bugs_found", 0) > cur.get("bugs_found", 0) + else: + key = (e.get("bugs_found", 0), e.get("efficiency_bugs_per_ktok", 0.0)) + better = cur is None or key > ( + cur.get("bugs_found", 0), cur.get("efficiency_bugs_per_ktok", 0.0)) + if better: + best[group] = e + return sorted( + best.values(), + key=lambda e: ( + e.get("benchmark_contract", "legacy-whole-repo"), + -e.get("bugs_found", 0), + (0 if str(e.get("benchmark_contract", "")).startswith("top50-evidence/") + else -e.get("efficiency_bugs_per_ktok", 0.0)), + e.get("model", ""), + )) def aggregate_leaderboard(results_dir: Path) -> list[dict]: @@ -266,7 +283,9 @@ def aggregate_leaderboard(results_dir: Path) -> list[dict]: scored = json.loads(p.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError): continue - if "results" not in scored or "model" not in scored: + if ("results" not in scored and not is_top50_submission(scored)) or "model" not in scored: + continue + if is_top50_submission(scored) and not scored.get("rankable"): continue # Test submissions are scored + kept privately, but never published to the public # leaderboard — skip them here so an end-to-end test can't pollute production. diff --git a/benchmark/run_top50.py b/benchmark/run_top50.py index 1634334..97c982d 100644 --- a/benchmark/run_top50.py +++ b/benchmark/run_top50.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import datetime import json import os from pathlib import Path @@ -17,19 +18,35 @@ TriageBudget, build_rankable_runner, ) +from benchmark.top50_contract import ( + AGENT_MODE, + CONTRACT_ID, + RUNNER_VERSION, + EXPECTED_EPISODE_BUDGET, + EXPECTED_TRIAGE_BUDGET, + expected_prompt_id, +) def pilot_contract() -> Top50Contract: """Return explicitly provisional values; issue #68 freezes the public contract.""" return Top50Contract( triage=TriageBudget( - model_generations=int(os.environ.get("PRB_TRIAGE_GENERATIONS", "8")), - shell_actions=int(os.environ.get("PRB_TRIAGE_ACTIONS", "12"))), + model_generations=int(os.environ.get( + "PRB_TRIAGE_GENERATIONS", str(EXPECTED_TRIAGE_BUDGET["model_generations"]))), + shell_actions=int(os.environ.get( + "PRB_TRIAGE_ACTIONS", str(EXPECTED_TRIAGE_BUDGET["shell_actions"]))), + max_output_chars=EXPECTED_TRIAGE_BUDGET["max_output_chars"], + command_timeout_seconds=EXPECTED_TRIAGE_BUDGET["command_timeout_seconds"]), episode=EvidenceBudget( - model_generations=int(os.environ.get("PRB_EPISODE_GENERATIONS", "10")), - shell_actions=int(os.environ.get("PRB_EPISODE_ACTIONS", "12")), - pred_calls=int(os.environ.get("PRB_PRED_CALLS", "24")), - solve_calls=int(os.environ.get("PRB_SOLVE_CALLS", "10")), + model_generations=int(os.environ.get( + "PRB_EPISODE_GENERATIONS", str(EXPECTED_EPISODE_BUDGET["model_generations"]))), + shell_actions=int(os.environ.get( + "PRB_EPISODE_ACTIONS", str(EXPECTED_EPISODE_BUDGET["shell_actions"]))), + pred_calls=int(os.environ.get( + "PRB_PRED_CALLS", str(EXPECTED_EPISODE_BUDGET["pred_calls"]))), + solve_calls=int(os.environ.get( + "PRB_SOLVE_CALLS", str(EXPECTED_EPISODE_BUDGET["solve_calls"]))), submit_attempts=2, max_output_chars=int(os.environ.get("PRB_MAX_OUTPUT_CHARS", "10000")), pred_timeout_seconds=int(os.environ.get("PRB_PRED_TIMEOUT_SECONDS", "300"))), @@ -55,12 +72,14 @@ def run(*, model: str, repo_dir: str | Path, output: str | Path, if len(inventory) < 50: raise ValueError(f"canonical inventory has only {len(inventory)} runnable rules") pred_binary = find_pred_binary() - verify_pred_version(pred_binary) + pred_version = verify_pred_version(pred_binary) contract = pilot_contract() if fake: runner = Top50Runner( executor=_FakeExecutor(), contract=contract, pred_binary=pred_binary) else: + if model_kwargs: + raise ValueError("rankable Top50 runs do not accept custom model kwargs") runner = build_rankable_runner( contract=contract, pred_binary=pred_binary, agent_uid=int(os.environ["PRB_AGENT_UID"]), @@ -69,12 +88,19 @@ def run(*, model: str, repo_dir: str | Path, output: str | Path, oracle_gid=int(os.environ["PRB_ORACLE_GID"]), evidence_gid=int(os.environ["PRB_EVIDENCE_GID"]), api_base=api_base, api_key=api_key, model_kwargs=model_kwargs) - result = runner.run(model=model, repo_path=repo, inventory=inventory, output=output) - result["library_commit"] = pinned_commit() - result["budget_contract_status"] = "pilot-unfrozen" - # Rewrite once with provenance added after the runner's final checkpoint. - Path(output).write_text(json.dumps(result, indent=2), encoding="utf-8") - return result + metadata = { + "benchmark_contract": CONTRACT_ID, + "library_commit": pinned_commit(), + "runner_version": RUNNER_VERSION, + "pred_version": pred_version, + "agent_mode": AGENT_MODE, + "prompt_id": expected_prompt_id(), + "budget_contract_status": "pilot-unfrozen", + "created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "inference_parameters": model_kwargs or {}, + } + return runner.run(model=model, repo_path=repo, inventory=inventory, + output=output, metadata=metadata) def main(argv: list[str] | None = None) -> None: diff --git a/benchmark/submit.py b/benchmark/submit.py index 84d233b..9f5983f 100644 --- a/benchmark/submit.py +++ b/benchmark/submit.py @@ -48,6 +48,10 @@ def validate_submission(sub: dict) -> list[str]: New runs prove provenance through the bounded submit ledger. Legacy runs must carry a certificate plus a trajectory on the row or envelope. """ + from benchmark.top50_contract import is_top50_submission, validate_top50_submission + if is_top50_submission(sub): + return validate_top50_submission(sub) + problems: list[str] = [] if not isinstance(sub, dict): return ["submission is not a JSON object"] diff --git a/benchmark/tests/test_top50_submission.py b/benchmark/tests/test_top50_submission.py new file mode 100644 index 0000000..110714c --- /dev/null +++ b/benchmark/tests/test_top50_submission.py @@ -0,0 +1,255 @@ +"""Acceptance tests for the private Top50 contract and aggregate-only public score.""" +from __future__ import annotations + +import copy +import json +import runpy +from pathlib import Path + +import pytest + +from benchmark.backend_score import _dedup_best, aggregate_leaderboard, score_one +from benchmark.submit import validate_submission +from benchmark.top50_contract import ( + AGENT_MODE, + CONTRACT_ID, + RUNNER_VERSION, + expected_prompt_id, + score_top50_submission, + top50_public_entry, + validate_top50_submission, +) +from benchmark.verify import Verdict + + +def _status(limit: int, used: int = 0) -> dict: + return {"used": used, "limit": limit, "remaining": limit - used} + + +def _artifact(accepted_positions=(7, 18, 41)) -> dict: + triage_budget = { + "model_generations": 8, "shell_actions": 12, + "max_output_chars": 10_000, "command_timeout_seconds": 300, + } + episode_budget = { + "model_generations": 10, "shell_actions": 12, "pred_calls": 24, + "solve_calls": 10, "submit_attempts": 2, "max_output_chars": 10_000, + "pred_timeout_seconds": 300, + } + shortlist = [{"rule": f"rule_{index:02d}", "hypothesis": f"risk-{index}"} + for index in range(1, 51)] + triage_status = {"model_generations": _status(8, 1), + "shell_actions": _status(12, 1), + "pred_calls": _status(0), "solve_calls": _status(0)} + triage = { + "ledger": { + "budget": triage_budget, + "status": triage_status, + "events": [ + {"sequence": 1, "type": "model_generation", "charged": True}, + {"sequence": 2, "type": "shell_action", "charged": True}, + {"type": "shortlist_commit", "accepted": True}, + ], + "shortlist": copy.deepcopy(shortlist), + }, + "messages": [], "tokens_k": 0.1, + "usage": {"input": 100, "output": 10, "cache_read": 0, "cache_write": 0}, + } + episodes = [] + request = 1 + for position, entry in enumerate(shortlist, 1): + attempts = [] + submit_used = 0 + submit_remaining = 2 + status = "completed" + if position in accepted_positions: + attempts.append({ + "attempt": 1, + "request_id": f"{request:032x}", + "accepted": True, + "rule": entry["rule"], + "reason": "confirmed", + "certificate": {"rule": entry["rule"], "source": {}, + "bundle": {"target": {"type": "Target"}}}, + }) + request += 1 + submit_used, submit_remaining, status = 1, 0, "bug_found" + ledger_status = { + "model_generations": _status(10), "shell_actions": _status(12), + "pred_calls": _status(24), "solve_calls": _status(10), + "submit_attempts": {"used": submit_used, "limit": 2, + "remaining": submit_remaining}, + } + episodes.append({ + "index": position, "rule": entry["rule"], "hypothesis": entry["hypothesis"], + "status": status, "accepted_submit_attempt": 1 if attempts else None, + "ledger": {"rule": entry["rule"], "budget": copy.deepcopy(episode_budget), + "status": ledger_status, "pred": [], "submit": attempts, + "model_generations": [], "shell_actions": []}, + "messages": [], "tokens_k": 0.01, + "usage": {"input": 10, "output": 1, "cache_read": 0, "cache_write": 0}, + }) + return { + "benchmark_contract": CONTRACT_ID, + "model": "provider/model", + "library_commit": "a" * 40, + "runner_version": RUNNER_VERSION, + "pred_version": "0.6.0", + "agent_mode": AGENT_MODE, + "prompt_id": expected_prompt_id(), + "status": "completed", + "rankable": True, + "inference_parameters": {}, + "contract": {"triage": triage_budget, "episode": episode_budget, + "shortlist_size": 50, "hypothesis_chars": 500}, + "shortlist": shortlist, + "triage": triage, + "episodes": episodes, + "bugs_found": 999, + } + + +def _accept(cert, repo_dir=None): + return Verdict(True, "reverified") + + +def test_valid_artifact_recomputes_score_and_prefix_metrics(): + submission = _artifact() + assert validate_top50_submission(submission) == [] + scored, report = score_top50_submission( + submission, verifier=_accept, + canonical_inventory={entry["rule"] for entry in submission["shortlist"]}) + + assert scored["rankable"] is True + assert scored["verified_bugs"] == 3 + assert scored["bugs_at_10"] == 1 + assert scored["bugs_at_25"] == 2 + assert scored["bugs_at_50"] == 3 + assert scored["first_attempt_accepts"] == 3 + assert len(report) == 3 + assert submission["bugs_found"] == 999 + assert validate_submission(submission) == [] + + +@pytest.mark.parametrize("mutate, expected", [ + (lambda sub: sub.update(submit_limit=100), "shared run-wide submit pool"), + (lambda sub: sub["episodes"].pop(), "exactly 50"), + (lambda sub: sub["episodes"].__setitem__(1, copy.deepcopy(sub["episodes"][0])), + "order/rule"), + (lambda sub: sub["episodes"][0]["ledger"]["status"]["pred_calls"].update(used=25), + "usage is inconsistent"), + (lambda sub: sub["episodes"][0]["ledger"]["budget"].update(pred_calls=23), + "budget differs"), + (lambda sub: sub["contract"]["episode"].update(pred_calls=1000000), + "versioned contract"), + (lambda sub: sub.update(agent_mode="codex"), "agent_mode"), + (lambda sub: sub.update(prompt_id="custom"), "prompt_id"), + (lambda sub: sub.pop("inference_parameters"), "inference_parameters"), + (lambda sub: sub.update(status="run_error", run_error="provider failed"), "incomplete"), +]) +def test_rankability_negative_controls(mutate, expected): + submission = _artifact(accepted_positions=()) + mutate(submission) + assert any(expected in problem for problem in validate_top50_submission(submission)) + + +def test_wrong_episode_certificate_and_third_attempt_are_unrankable(): + submission = _artifact(accepted_positions=(1,)) + attempt = submission["episodes"][0]["ledger"]["submit"][0] + attempt["certificate"]["rule"] = "rule_02" + assert any("certificate rule mismatch" in p + for p in validate_top50_submission(submission)) + + submission = _artifact(accepted_positions=()) + episode = submission["episodes"][0] + episode["ledger"]["submit"] = [ + {"attempt": n, "request_id": f"{n:032x}", "accepted": False, "reason": "no"} + for n in range(1, 4)] + episode["ledger"]["status"]["submit_attempts"] = {"used": 3, "limit": 2, "remaining": 0} + assert any("at most two" in p for p in validate_top50_submission(submission)) + + +def test_strict_boolean_commit_and_canonical_inventory_controls(): + submission = _artifact(accepted_positions=(1,)) + submission["episodes"][0]["ledger"]["submit"][0]["accepted"] = "false" + assert any("accepted field must be boolean" in p + for p in validate_top50_submission(submission)) + + submission = _artifact(accepted_positions=()) + submission["triage"]["ledger"]["events"] = submission["triage"]["ledger"]["events"][:2] + assert any("exactly one accepted shortlist_commit" in p + for p in validate_top50_submission(submission)) + + submission = _artifact(accepted_positions=()) + canonical = {entry["rule"] for entry in submission["shortlist"]} - {"rule_50"} + assert any("non-canonical" in p + for p in validate_top50_submission(submission, canonical)) + + +def test_public_projection_contains_no_answer_key_material(): + submission = _artifact() + scored, _ = score_top50_submission( + submission, verifier=_accept, + canonical_inventory={entry["rule"] for entry in submission["shortlist"]}) + public = top50_public_entry(submission, scored) + raw = json.dumps(public) + + for forbidden in ("rule_", "hypothesis", "certificate", '"source"', '"bundle"', + "submit_log", "submit_attempt", "trajectory", "reason"): + assert forbidden not in raw + assert public["bugs_found"] == 3 + + guard = runpy.run_path(str( + Path(__file__).parents[2] / ".github" / "scripts" / "check_aggregate.py"))["check"] + # The actual egress guard accepts the generated projection, not merely this key test. + # Use pytest's temporary path in the dedicated backend test below for file-based checks. + assert callable(guard) + + +def test_top50_ties_ignore_tokens_and_efficiency_and_legacy_is_separate(): + entries = [ + {"benchmark_contract": CONTRACT_ID, "model": "a", "bugs_found": 2, + "total_tokens_k": 1, "efficiency_bugs_per_ktok": 999}, + {"benchmark_contract": CONTRACT_ID, "model": "b", "bugs_found": 2, + "total_tokens_k": 1000, "efficiency_bugs_per_ktok": 0.001}, + {"model": "a", "bugs_found": 9, "efficiency_bugs_per_ktok": 9}, + ] + board = _dedup_best(entries) + assert len(board) == 3 + top50 = [entry for entry in board if entry.get("benchmark_contract") == CONTRACT_ID] + assert [entry["model"] for entry in top50] == ["a", "b"] + + +def test_backend_official_path_keeps_private_detail_and_publishes_aggregate(tmp_path): + submission = _artifact(accepted_positions=()) + repo = tmp_path / "repo" + rules = repo / "src" / "rules" + rules.mkdir(parents=True) + for entry in submission["shortlist"]: + (rules / f"{entry['rule']}.rs").write_text("// fixture") + source = tmp_path / "submission.json" + source.write_text(json.dumps(submission)) + results = tmp_path / "results" + + public = score_one( + source, results, repo_dir=str(repo), official=True, + expected_commit=submission["library_commit"]) + board = aggregate_leaderboard(results) + private = json.loads((results / "submission.json").read_text()) + + assert private["artifact_sha256"] and "episodes" not in private + assert public["benchmark_contract"] == CONTRACT_ID + assert board == [public] + assert "episodes" not in public and "shortlist" not in public + guard = runpy.run_path(str( + Path(__file__).parents[2] / ".github" / "scripts" / "check_aggregate.py"))["check"] + public_file = tmp_path / "public.json" + public_file.write_text(json.dumps(public)) + assert guard(public_file) == [] + + +def test_site_keeps_contracts_separate_and_top50_ties_ignore_efficiency(): + site = (Path(__file__).parents[2] / "site" / "index.html").read_text() + assert 'id="lb-track"' in site + assert 'top50=track.startsWith("top50-evidence/")' in site + assert "top50?String(a.model).localeCompare" in site diff --git a/benchmark/top50_contract.py b/benchmark/top50_contract.py new file mode 100644 index 0000000..8258434 --- /dev/null +++ b/benchmark/top50_contract.py @@ -0,0 +1,447 @@ +"""Versioned private Top50 artifact validation, scoring, and public projection.""" +from __future__ import annotations + +import copy +import hashlib +import inspect +import json +import os +import re +from collections import Counter +from pathlib import Path +from typing import Callable + +from benchmark.usage import Usage, usage_as_dict, usage_from_dict +from benchmark.verify import Verdict, verify + +CONTRACT_ID = "top50-evidence/pilot-v1" +AGENT_MODE = "standardized-model-api" +RUNNER_VERSION = "0.9.0" +EXPECTED_TRIAGE_BUDGET = { + "model_generations": 8, "shell_actions": 12, + "max_output_chars": 10_000, "command_timeout_seconds": 300, +} +EXPECTED_EPISODE_BUDGET = { + "model_generations": 10, "shell_actions": 12, "pred_calls": 24, + "solve_calls": 10, "submit_attempts": 2, "max_output_chars": 10_000, + "pred_timeout_seconds": 300, +} +_REQUEST_ID = re.compile(r"[0-9a-f]{32}") +_COUNTERS = ("model_generations", "shell_actions", "pred_calls", "solve_calls") + + +def is_top50_submission(submission: object) -> bool: + return isinstance(submission, dict) and str( + submission.get("benchmark_contract", "")).startswith("top50-evidence/") + + +def expected_prompt_id() -> str: + return hashlib.sha256(Path(__file__).with_name("top50_config.yaml").read_bytes()).hexdigest() + + +def validate_top50_submission( + submission: object, canonical_inventory: set[str] | None = None +) -> list[str]: + """Recompute structural rankability from private evaluation-owned ledgers.""" + if not isinstance(submission, dict): + return ["submission is not a JSON object"] + problems: list[str] = [] + for field in ("benchmark_contract", "model", "library_commit", "runner_version", + "pred_version", "agent_mode", "prompt_id", "contract", "shortlist", + "triage", "episodes", "status"): + if field not in submission: + problems.append(f"missing required field: {field}") + if problems: + return problems + if submission["benchmark_contract"] != CONTRACT_ID: + problems.append(f"unsupported benchmark_contract: {submission['benchmark_contract']!r}") + if submission["agent_mode"] != AGENT_MODE: + problems.append(f"agent_mode must be {AGENT_MODE!r}") + if submission["runner_version"] != RUNNER_VERSION: + problems.append(f"runner_version must be {RUNNER_VERSION!r}") + if submission.get("status") != "completed" or submission.get("run_error"): + problems.append("Top50 run is incomplete or has run_error") + if submission.get("rankable") is not True: + problems.append("runner did not certify the standardized rankable path") + model = submission.get("model") + if not isinstance(model, str) or not re.fullmatch(r"[A-Za-z0-9._:/-]{1,200}", model): + problems.append("model identifier is not a bounded safe identifier") + submitter = submission.get("submitted_by") + if (submitter is not None and (not isinstance(submitter, str) + or not re.fullmatch(r"[A-Za-z0-9_.@-]{1,100}", submitter))): + problems.append("submitted_by is not a bounded safe identifier") + if not isinstance(submission.get("inference_parameters"), dict): + problems.append("inference_parameters must be a normalized object") + if submission.get("prompt_id") != expected_prompt_id(): + problems.append("prompt_id does not match the frozen Top50 prompt") + if "submit_limit" in submission or "submit_log" in submission: + problems.append("Top50 artifacts cannot use a shared run-wide submit pool") + + contract = submission.get("contract") + if not isinstance(contract, dict): + return problems + ["contract must be an object"] + triage_budget = contract.get("triage") + episode_budget = contract.get("episode") + if not isinstance(triage_budget, dict) or not isinstance(episode_budget, dict): + return problems + ["contract triage and episode budgets must be objects"] + if triage_budget != EXPECTED_TRIAGE_BUDGET: + problems.append("triage budget differs from the versioned contract") + if episode_budget != EXPECTED_EPISODE_BUDGET: + problems.append("episode budget differs from the versioned contract") + if contract.get("shortlist_size") != 50: + problems.append("contract shortlist_size must be 50") + hypothesis_limit = contract.get("hypothesis_chars") + if not _nonnegative_int(hypothesis_limit): + problems.append("contract hypothesis_chars must be a non-negative integer") + hypothesis_limit = -1 + if episode_budget.get("submit_attempts") != 2: + problems.append("episode submit_attempts must be exactly 2") + pred_limit = episode_budget.get("pred_calls") + solve_limit = episode_budget.get("solve_calls") + if not _nonnegative_int(pred_limit) or not _nonnegative_int(solve_limit): + problems.append("pred_calls and solve_calls limits must be non-negative integers") + elif solve_limit > pred_limit: + problems.append("solve_calls limit exceeds pred_calls") + + shortlist = submission.get("shortlist") + if not isinstance(shortlist, list) or len(shortlist) != 50: + return problems + ["shortlist must contain exactly 50 entries"] + rules: list[str] = [] + for index, entry in enumerate(shortlist): + if not isinstance(entry, dict) or not isinstance(entry.get("rule"), str): + problems.append(f"shortlist[{index}] must have a rule string") + continue + rules.append(entry["rule"]) + hypothesis = entry.get("hypothesis", "") + if (set(entry) - {"rule", "hypothesis"} or not isinstance(hypothesis, str) + or len(hypothesis) > hypothesis_limit): + problems.append(f"shortlist[{index}] hypothesis schema is invalid") + if len(set(rules)) != len(rules): + problems.append("shortlist rules must be unique") + if canonical_inventory is not None: + unknown = sorted(set(rules) - canonical_inventory) + if unknown: + problems.append(f"shortlist contains non-canonical rules: {unknown[:3]}") + + triage = submission.get("triage") + triage_ledger = triage.get("ledger") if isinstance(triage, dict) else None + if not isinstance(triage_ledger, dict): + problems.append("triage.ledger must be an object") + else: + if triage_ledger.get("budget") != triage_budget: + problems.append("triage ledger budget differs from contract") + frozen = triage_ledger.get("shortlist") + if frozen != shortlist: + problems.append("top-level shortlist differs from frozen triage shortlist") + _validate_status(triage_ledger.get("status"), triage_budget, + "triage", problems, counters=("model_generations", "shell_actions")) + _validate_triage_events(triage_ledger, problems) + + episodes = submission.get("episodes") + if not isinstance(episodes, list) or len(episodes) != 50: + return problems + ["episodes must contain exactly 50 entries"] + seen_request_ids: set[str] = set() + for index, episode in enumerate(episodes): + label = f"episodes[{index}]" + if not isinstance(episode, dict): + problems.append(f"{label} must be an object") + continue + expected_rule = rules[index] if index < len(rules) else None + if episode.get("index") != index + 1 or episode.get("rule") != expected_rule: + problems.append(f"{label} order/rule does not match frozen shortlist") + expected_hypothesis = shortlist[index].get("hypothesis", "") + if episode.get("hypothesis") != expected_hypothesis: + problems.append(f"{label} hypothesis differs from frozen shortlist") + if episode.get("status") == "run_error": + problems.append(f"{label} has infrastructure run_error") + ledger = episode.get("ledger") + if not isinstance(ledger, dict): + problems.append(f"{label}.ledger must be an object") + continue + if ledger.get("rule") != expected_rule or ledger.get("budget") != episode_budget: + problems.append(f"{label} rule or budget differs from contract") + messages = episode.get("messages") + if not isinstance(messages, list) or len(messages) > 3 * episode_budget.get( + "model_generations", 0) + 4: + problems.append(f"{label} message history exceeds its logical bound") + elif any(not isinstance(message, dict) + or len(str(message.get("content", ""))) > 200_000 for message in messages): + problems.append(f"{label} contains an oversized message") + _validate_status(ledger.get("status"), episode_budget, label, problems) + _validate_episode_events(ledger, label, problems) + _validate_pred_ledger(ledger, label, seen_request_ids, problems) + _validate_submit_ledger(ledger, expected_rule, label, seen_request_ids, problems) + accepted_attempts = [attempt for attempt in ledger.get("submit", []) + if isinstance(attempt, dict) and attempt.get("accepted") is True] + expected_status = "bug_found" if accepted_attempts else "completed" + expected_attempt = accepted_attempts[0].get("attempt") if accepted_attempts else None + if (episode.get("status") != expected_status + or episode.get("accepted_submit_attempt") != expected_attempt): + problems.append(f"{label} outcome does not match its submit ledger") + return list(dict.fromkeys(problems)) + + +def score_top50_submission( + submission: dict, repo_dir: str | None = None, + *, verifier: Callable[[dict], Verdict] = verify, + canonical_inventory: set[str] | None = None, +) -> tuple[dict, list[dict]]: + """Validate ledgers and independently re-verify accepted certificates.""" + inventory = canonical_inventory or load_canonical_inventory(repo_dir) + problems = validate_top50_submission(submission, inventory) + accepted_positions: list[int] = [] + report: list[dict] = [] + first_attempt = second_attempt = 0 + pred_calls = 0 + cap_hits: Counter[str] = Counter() + if not problems: + for position, episode in enumerate(submission["episodes"], 1): + ledger = episode["ledger"] + pred_calls += ledger["status"]["pred_calls"]["used"] + for counter in _COUNTERS: + status = ledger["status"][counter] + if status["limit"] > 0 and status["used"] == status["limit"]: + cap_hits[counter] += 1 + for attempt in ledger["submit"]: + if attempt.get("accepted") is not True: + continue + cert = attempt.get("certificate") + verdict = _call_verifier(verifier, cert, repo_dir) + accepted = bool(verdict.accepted) + report.append({"position": position, "rule": episode["rule"], + "violation": cert.get("violation") if isinstance(cert, dict) else None, + "attempt": attempt["attempt"], + "accepted": accepted, "reason": verdict.reason}) + if accepted: + accepted_positions.append(position) + if attempt["attempt"] == 1: + first_attempt += 1 + else: + second_attempt += 1 + break + distinct_positions = sorted(set(accepted_positions)) + bugs = len(distinct_positions) + usage = _aggregate_usage(submission) + scored = { + "benchmark_contract": submission.get("benchmark_contract"), + "model": submission.get("model", "unknown"), + "library_commit": submission.get("library_commit", "unknown"), + "runner_version": submission.get("runner_version"), + "pred_version": submission.get("pred_version"), + "agent_mode": submission.get("agent_mode"), + "rankable": not problems, + "rankability_errors": problems, + "verified_bugs": bugs, + "bugs_found": bugs, + "bugs_at_10": sum(position <= 10 for position in distinct_positions), + "bugs_at_25": sum(position <= 25 for position in distinct_positions), + "bugs_at_50": bugs, + "first_attempt_accepts": first_attempt, + "second_attempt_accepts": second_attempt, + "pred_calls_per_bug": round(pred_calls / bugs, 4) if bugs else None, + "cap_hits": dict(cap_hits), + "usage_totals": usage_as_dict(usage), + "total_tokens_k": round(usage.total_tokens / 1000, 2), + "artifact_sha256": hashlib.sha256(json.dumps( + submission, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), + "verifier_report": copy.deepcopy(report), + } + for field in ("submitted_by", "created_at", "test", "run_error"): + if field in submission: + scored[field] = submission[field] + return scored, report + + +def top50_public_entry(submission: dict, scored: dict) -> dict: + """Project private scoring data to aggregate-only public fields.""" + return { + "benchmark_contract": scored["benchmark_contract"], + "model": scored["model"], + "library_commit": scored["library_commit"], + "runner_version": scored.get("runner_version"), + "pred_version": scored.get("pred_version"), + "agent_mode": scored.get("agent_mode"), + "rankable": scored["rankable"], + "bugs_found": scored["verified_bugs"], + "rules_tested": 50, + "bugs_at_10": scored["bugs_at_10"], + "bugs_at_25": scored["bugs_at_25"], + "bugs_at_50": scored["bugs_at_50"], + "first_attempt_accepts": scored["first_attempt_accepts"], + "second_attempt_accepts": scored["second_attempt_accepts"], + "pred_calls_per_bug": scored["pred_calls_per_bug"], + "cap_hits": scored["cap_hits"], + "usage_totals": scored["usage_totals"], + "total_tokens_k": scored["total_tokens_k"], + "submitted_by": submission.get("submitted_by"), + "placeholder": False, + } + + +def _validate_status(status, budget, label, problems, counters=_COUNTERS) -> None: + if not isinstance(status, dict): + problems.append(f"{label} status must be an object") + return + for counter in counters: + item = status.get(counter) + limit = budget.get(counter) + if not _nonnegative_int(limit): + problems.append(f"{label} {counter} limit is not a non-negative integer") + continue + if not isinstance(item, dict) or item.get("limit") != limit: + problems.append(f"{label} {counter} limit differs from contract") + continue + used, remaining = item.get("used"), item.get("remaining") + if (not _nonnegative_int(used) or used > limit + or remaining != limit - used): + problems.append(f"{label} {counter} usage is inconsistent") + if "submit_attempts" in status: + item = status["submit_attempts"] + if not isinstance(item, dict) or item.get("limit") != 2: + problems.append(f"{label} submit limit must be 2") + elif (not _nonnegative_int(item.get("used")) or item["used"] > 2 + or not _nonnegative_int(item.get("remaining"))): + problems.append(f"{label} submit usage is inconsistent") + + +def _validate_triage_events(ledger: dict, problems: list[str]) -> None: + events = ledger.get("events") + if not isinstance(events, list): + problems.append("triage events must be a list") + return + budget = ledger.get("budget", {}) + if len(events) > budget.get("model_generations", 0) + budget.get("shell_actions", 0) + 3: + problems.append("triage event ledger exceeds its logical bound") + commit_events = [event for event in events if isinstance(event, dict) + and event.get("type") == "shortlist_commit" + and event.get("accepted") is True] + if len(commit_events) != 1: + problems.append("triage must contain exactly one accepted shortlist_commit") + sequenced = [event for event in events if isinstance(event, dict) and "sequence" in event] + if [event.get("sequence") for event in sequenced] != list(range(1, len(sequenced) + 1)): + problems.append("triage event sequence is inconsistent") + if any("charged" in event and type(event["charged"]) is not bool for event in events + if isinstance(event, dict)): + problems.append("triage charged fields must be booleans") + charged = Counter(event.get("type") for event in events + if isinstance(event, dict) and event.get("charged") is True) + status = ledger.get("status", {}) + if charged["model_generation"] != status.get("model_generations", {}).get("used"): + problems.append("triage model-generation ledger count is inconsistent") + if charged["shell_action"] != status.get("shell_actions", {}).get("used"): + problems.append("triage shell-action ledger count is inconsistent") + + +def _validate_pred_ledger(ledger, label, seen_ids, problems) -> None: + records = ledger.get("pred") + if not isinstance(records, list): + problems.append(f"{label} pred ledger must be a list") + return + if len(records) > ledger.get("budget", {}).get("shell_actions", 0) + 1: + problems.append(f"{label} pred ledger exceeds its logical bound") + charged = solve = 0 + for sequence, record in enumerate(records, 1): + if not isinstance(record, dict) or record.get("sequence") != sequence: + problems.append(f"{label} pred sequence is inconsistent") + continue + request_id = record.get("request_id") + if not isinstance(request_id, str) or not _REQUEST_ID.fullmatch(request_id): + problems.append(f"{label} pred request id is invalid") + elif request_id in seen_ids: + problems.append(f"{label} reuses a request id") + else: + seen_ids.add(request_id) + if type(record.get("charged")) is not bool: + problems.append(f"{label} pred charged field must be boolean") + if record.get("charged") is True: + charged += 1 + solve += record.get("command") == "solve" + status = ledger.get("status", {}) + if charged != status.get("pred_calls", {}).get("used"): + problems.append(f"{label} pred charged count differs from status") + if solve != status.get("solve_calls", {}).get("used"): + problems.append(f"{label} solve charged count differs from status") + + +def _validate_episode_events(ledger, label, problems) -> None: + status = ledger.get("status", {}) + for field in ("model_generations", "shell_actions"): + events = ledger.get(field) + if not isinstance(events, list): + problems.append(f"{label} {field} ledger must be a list") + continue + limit = ledger.get("budget", {}).get(field, 0) + if len(events) > limit + 1: + problems.append(f"{label} {field} ledger exceeds its logical bound") + if any(type(event.get("charged")) is not bool for event in events + if isinstance(event, dict)): + problems.append(f"{label} {field} charged fields must be booleans") + charged = sum(event.get("charged") is True for event in events + if isinstance(event, dict)) + if charged != status.get(field, {}).get("used"): + problems.append(f"{label} {field} charged count differs from status") + + +def _validate_submit_ledger(ledger, expected_rule, label, seen_ids, problems) -> None: + attempts = ledger.get("submit") + if not isinstance(attempts, list) or len(attempts) > 2: + problems.append(f"{label} must have at most two submit attempts") + return + status = ledger.get("status", {}).get("submit_attempts", {}) + if status.get("used") != len(attempts): + problems.append(f"{label} submit count differs from status") + accepted = 0 + for number, attempt in enumerate(attempts, 1): + if not isinstance(attempt, dict) or attempt.get("attempt") != number: + problems.append(f"{label} submit attempt numbering is inconsistent") + continue + request_id = attempt.get("request_id") + if not isinstance(request_id, str) or not _REQUEST_ID.fullmatch(request_id): + problems.append(f"{label} submit request id is invalid") + elif request_id in seen_ids: + problems.append(f"{label} reuses a request id") + else: + seen_ids.add(request_id) + if type(attempt.get("accepted")) is not bool: + problems.append(f"{label} submit accepted field must be boolean") + if attempt.get("accepted") is True: + accepted += 1 + certificate = attempt.get("certificate") + if not isinstance(certificate, dict) or certificate.get("rule") != expected_rule: + problems.append(f"{label} accepted certificate rule mismatch") + if accepted > 1: + problems.append(f"{label} has more than one accepted submit attempt") + expected_remaining = 0 if accepted else 2 - len(attempts) + if status.get("remaining") != expected_remaining: + problems.append(f"{label} submit remaining count is inconsistent") + + +def load_canonical_inventory(repo_dir: str | None) -> set[str]: + """Load source-rule identifiers from the pinned repository bundled with the image.""" + candidate = Path(repo_dir or os.environ.get("REPO_DIR", "/app/pr-src")) + if not (candidate / "src" / "rules").is_dir(): + raise RuntimeError(f"pinned canonical rule inventory is unavailable under {candidate}") + from benchmark.run_mini import list_rules + return set(list_rules(candidate)) + + +def _aggregate_usage(submission: dict) -> Usage: + total = usage_from_dict(submission.get("triage", {}).get("usage")) + for episode in submission.get("episodes", []): + total = total + usage_from_dict(episode.get("usage")) + return total + + +def _nonnegative_int(value) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def _call_verifier(verifier, certificate, repo_dir): + """Bind the verifier signature before execution; never retry an internal TypeError.""" + signature = inspect.signature(verifier) + positional = [parameter for parameter in signature.parameters.values() + if parameter.kind in (parameter.POSITIONAL_ONLY, + parameter.POSITIONAL_OR_KEYWORD)] + variadic = any(parameter.kind == parameter.VAR_POSITIONAL + for parameter in signature.parameters.values()) + return verifier(certificate, repo_dir) if variadic or len(positional) >= 2 else verifier(certificate) diff --git a/benchmark/top50_results.schema.json b/benchmark/top50_results.schema.json new file mode 100644 index 0000000..7072eee --- /dev/null +++ b/benchmark/top50_results.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "top50_results.schema.json", + "title": "Top50EvidenceScoredResult", + "type": "object", + "required": ["benchmark_contract", "model", "library_commit", "rankable", + "rankability_errors", "verified_bugs", "bugs_at_10", "bugs_at_25", "bugs_at_50", + "first_attempt_accepts", "second_attempt_accepts", "usage_totals", + "artifact_sha256", "verifier_report"], + "properties": { + "benchmark_contract": {"const": "top50-evidence/pilot-v1"}, + "model": {"type": "string"}, + "library_commit": {"type": "string"}, + "rankable": {"type": "boolean"}, + "rankability_errors": {"type": "array", "items": {"type": "string"}}, + "verified_bugs": {"type": "integer", "minimum": 0, "maximum": 50}, + "bugs_at_10": {"type": "integer", "minimum": 0, "maximum": 10}, + "bugs_at_25": {"type": "integer", "minimum": 0, "maximum": 25}, + "bugs_at_50": {"type": "integer", "minimum": 0, "maximum": 50}, + "first_attempt_accepts": {"type": "integer", "minimum": 0}, + "second_attempt_accepts": {"type": "integer", "minimum": 0}, + "usage_totals": {"type": "object"}, + "artifact_sha256": {"type": "string", "minLength": 64, "maxLength": 64}, + "verifier_report": {"type": "array"} + } +} diff --git a/benchmark/top50_runner.py b/benchmark/top50_runner.py index 3eb1499..be46c38 100644 --- a/benchmark/top50_runner.py +++ b/benchmark/top50_runner.py @@ -237,7 +237,9 @@ def __init__(self, *, executor: PhaseExecutor, contract: Top50Contract, self._rankable_contract = _rankable_token is _RANKABLE_TOKEN def run(self, *, model: str, repo_path: str | Path, - inventory: list[str] | tuple[str, ...], output: str | Path | None = None) -> dict: + inventory: list[str] | tuple[str, ...], output: str | Path | None = None, + metadata: dict | None = None) -> dict: + self._result_metadata = copy.deepcopy(metadata or {}) repo_path = Path(repo_path).resolve() canonical = tuple(inventory) with TriageSession( @@ -331,6 +333,7 @@ def _result(self, model: str, triage: dict, } if run_error: result["run_error"] = run_error + result.update(copy.deepcopy(getattr(self, "_result_metadata", {}))) return result diff --git a/benchmark/top50_submission.schema.json b/benchmark/top50_submission.schema.json new file mode 100644 index 0000000..1f61091 --- /dev/null +++ b/benchmark/top50_submission.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "top50_submission.schema.json", + "title": "Top50EvidenceSubmission", + "type": "object", + "required": ["benchmark_contract", "model", "library_commit", "runner_version", + "pred_version", "agent_mode", "prompt_id", "status", "contract", "shortlist", + "triage", "episodes", "inference_parameters"], + "properties": { + "benchmark_contract": {"const": "top50-evidence/pilot-v1"}, + "model": {"type": "string", "minLength": 1}, + "library_commit": {"type": "string", "minLength": 1}, + "runner_version": {"const": "0.9.0"}, + "pred_version": {"type": "string", "minLength": 1}, + "agent_mode": {"const": "standardized-model-api"}, + "prompt_id": {"type": "string", "minLength": 64, "maxLength": 64}, + "status": {"enum": ["completed", "run_error"]}, + "rankable": {"type": "boolean"}, + "contract": {"type": "object"}, + "shortlist": {"type": "array", "minItems": 50, "maxItems": 50}, + "triage": {"type": "object"}, + "episodes": {"type": "array", "minItems": 50, "maxItems": 50}, + "inference_parameters": {"type": "object"}, + "run_error": {"type": "string"} + } +} diff --git a/benchmark/verify_submission.py b/benchmark/verify_submission.py index fb8ea1e..9b4b4fb 100644 --- a/benchmark/verify_submission.py +++ b/benchmark/verify_submission.py @@ -20,7 +20,6 @@ import argparse import json import re -import sys from pathlib import Path from benchmark.submit_ledger import (accepted_certificate_index, certificate_key, @@ -79,6 +78,10 @@ def score_submission(submission: dict, repo_dir: str | None = None) -> tuple[dic Returns (scored, report); ``scored`` is results.schema.json-shaped and ``report`` is a per-certificate list of {rule, violation, accepted, reason, provenance}. """ + from benchmark.top50_contract import is_top50_submission, score_top50_submission + if is_top50_submission(submission): + return score_top50_submission(submission, repo_dir) + rescored: list[dict] = [] report: list[dict] = [] ledger_problem = submit_ledger_error(submission) @@ -173,6 +176,10 @@ def leaderboard_entry(submission: dict, scored: dict) -> dict: private scored result the maintainer holds; the leaderboard shows only how many each model found. """ + from benchmark.top50_contract import is_top50_submission, top50_public_entry + if is_top50_submission(submission) or is_top50_submission(scored): + return top50_public_entry(submission, scored) + return { "model": scored["model"], "library_commit": scored.get("library_commit", "unknown"), diff --git a/docker/Dockerfile b/docker/Dockerfile index 94478c0..9d4b2c5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -75,6 +75,11 @@ ENV PRED_BINARY=/usr/local/libexec/prb/pred \ WORKDIR /app COPY benchmark/ ./benchmark/ +# The scorer and Top50 triage share the exact pinned source-rule inventory. Keeping it in +# the runtime image lets official scoring reject invented/renamed shortlist entries. +COPY --from=pred-build /src/src /app/pr-src/src +COPY --from=pred-build /src/Cargo.toml /app/pr-src/Cargo.toml +ENV REPO_DIR=/app/pr-src # Fail the build if pred can't run (catches a missing runtime lib early). RUN /usr/local/libexec/prb/pred --version @@ -115,13 +120,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certifi && curl -fsSL https://claude.ai/install.sh | bash ENV PATH="/root/.local/bin:${PATH}" RUN claude --version -# The agent reads rule sources from REPO_DIR/src/rules — copy just the source tree -# (not target/) from the build stage. pred itself is a self-contained binary on PATH. -COPY --from=pred-build /src/src /app/pr-src/src -COPY --from=pred-build /src/Cargo.toml /app/pr-src/Cargo.toml RUN mkdir -p /out -ENV REPO_DIR=/app/pr-src \ - OUTPUT=/out/submission.json +ENV OUTPUT=/out/submission.json VOLUME ["/out"] # Build-time sanity that the assembled runner imports and runs (no API/pred calls). RUN python -m benchmark.run_top50 --fake --model fake/check \ diff --git a/site/index.html b/site/index.html index cc06c16..a6f6be4 100644 --- a/site/index.html +++ b/site/index.html @@ -270,6 +270,7 @@

Leaderboard

+ Click a row for its run stats →
@@ -408,11 +409,18 @@

How to cite

const RIGHT=new Set(["bugs","tokens","eff"]); function buildLbRows(){ - const ranked=RESULTS.slice() - .sort((a,b)=>(b.bugs_found||0)-(a.bugs_found||0)||(b.efficiency_bugs_per_ktok||0)-(a.efficiency_bugs_per_ktok||0)); + const track=document.getElementById("lb-track").value||"legacy-whole-repo"; + const top50=track.startsWith("top50-evidence/"); + const ranked=RESULTS.filter(r=>(r.benchmark_contract||"legacy-whole-repo")===track) + .sort((a,b)=>(b.bugs_found||0)-(a.bugs_found||0)|| + (top50?String(a.model).localeCompare(String(b.model)): + (b.efficiency_bugs_per_ktok||0)-(a.efficiency_bugs_per_ktok||0))); + let lastBugs=null,lastRank=0; lbRows=ranked.map((r,i)=>{ const tested=r.rules_tested||0, frac=TOTAL?Math.min(tested/TOTAL,1):0; - return {rank:i+1,model:short(r.model),_model:r.model,bugs:r.bugs_found||0, + const bugs=r.bugs_found||0; + const rank=top50&&i&&bugs===lastBugs?lastRank:i+1;lastBugs=bugs;lastRank=rank; + return {rank,model:short(r.model),_model:r.model,bugs, reachFrac:frac,tested,tokensNum:r.total_tokens_k||0, effNum:r.efficiency_bugs_per_ktok||0}; }); @@ -573,6 +581,9 @@

How to cite

document.getElementById("drawer-scrim").addEventListener("click",closeDrawer); document.addEventListener("keydown",e=>{if(e.key==="Escape")closeDrawer();}); document.getElementById("task-search").addEventListener("input",renderTasks); +document.getElementById("lb-track").addEventListener("change",()=>{ + buildLbRows();renderLbBody(); +}); document.querySelectorAll("#task-table thead th").forEach(th=>th.addEventListener("click",()=>{ const k=th.dataset.k; taskSort={key:k,dir:taskSort.key===k?-taskSort.dir:1}; renderTasks(); })); @@ -582,6 +593,10 @@

How to cite

fetch("tasks.json").then(r=>r.json()).catch(()=>[]), ]).then(([res,tasks])=>{ RESULTS=res;TASKS=tasks;TOTAL=tasks.length||TOTAL_DEFAULT; + const tracks=[...new Set(RESULTS.map(r=>r.benchmark_contract||"legacy-whole-repo"))].sort(); + const trackSelect=document.getElementById("lb-track"); + trackSelect.innerHTML=tracks.map(t=>``).join(""); + const newest=tracks.find(t=>t.startsWith("top50-evidence/"));if(newest)trackSelect.value=newest; const rc=document.getElementById("rulecount"); if(rc)rc.textContent=TOTAL; if(RESULTS.some(r=>r.placeholder)) document.getElementById("lb-banner").innerHTML=``; From fd19b3ec1e6abfe6a2635ebb2587db7a9c503682 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Mon, 20 Jul 2026 14:06:46 +0800 Subject: [PATCH 4/9] feat: freeze Top50 evidence contract --- .agents/skills/run-api-benchmark/SKILL.md | 117 +---- .agents/skills/run-benchmark/SKILL.md | 35 +- .agents/skills/run-cli-benchmark/SKILL.md | 11 +- .../skills/submit-benchmark-result/SKILL.md | 5 +- .github/workflows/ci.yml | 6 + CONTRIBUTING.md | 304 ++--------- Makefile | 20 +- README.md | 164 ++---- benchmark/calibrate_budget.py | 290 +++++++++++ benchmark/config.yaml | 3 +- benchmark/docs/budget-calibration.json | 485 ++++++++++++++++++ benchmark/docs/budget-calibration.md | 47 ++ benchmark/run_mini.py | 17 +- benchmark/run_top50.py | 121 ++++- benchmark/tests/test_budget_calibration.py | 108 ++++ benchmark/tests/test_docs.py | 32 +- benchmark/tests/test_run_top50.py | 45 +- benchmark/tests/test_top50_submission.py | 4 +- benchmark/top50_budget.json | 30 ++ benchmark/top50_budget.py | 8 + benchmark/top50_config.yaml | 4 +- benchmark/top50_contract.py | 40 +- benchmark/top50_results.schema.json | 2 +- benchmark/top50_runner.py | 20 +- benchmark/top50_submission.schema.json | 17 +- docker/Dockerfile | 14 +- site/README.md | 5 +- site/index.html | 66 ++- submission.env.example | 43 +- 29 files changed, 1393 insertions(+), 670 deletions(-) create mode 100644 benchmark/calibrate_budget.py create mode 100644 benchmark/docs/budget-calibration.json create mode 100644 benchmark/docs/budget-calibration.md create mode 100644 benchmark/tests/test_budget_calibration.py create mode 100644 benchmark/top50_budget.json create mode 100644 benchmark/top50_budget.py diff --git a/.agents/skills/run-api-benchmark/SKILL.md b/.agents/skills/run-api-benchmark/SKILL.md index 71db21f..c69e307 100644 --- a/.agents/skills/run-api-benchmark/SKILL.md +++ b/.agents/skills/run-api-benchmark/SKILL.md @@ -1,121 +1,40 @@ --- name: run-api-benchmark -description: Configure and run this problem-reductions benchmark through a model API using the containerized mini-swe/LiteLLM backend. Use when the caller chooses an API, provider endpoint, gateway, hosted model, or container run. Guides provider configuration without collecting secrets, detects the container engine, runs preflight, produces submission.json, validates it, and uploads only when explicitly requested. +description: Configure and run the rankable problem-reductions Self-selected Top50 benchmark through the standardized containerized Model API path. --- -# Run the API benchmark +# Run the rankable Top50 benchmark -Run one whole-repository mini-swe session. The agent chooses rules, search depth, and when to -stop. Do not add a step, turn, cost, or rule-count limit. The evaluation-owned `submit` -command is the only scored certificate channel; `SUBMIT_LIMIT` defaults to 100. +Use only the frozen `top50-evidence/v1` path: source-only triage freezes 50 rules, followed by 50 fresh sequential episodes. Each rule receives M=10 model generations, E=12 shell actions, P=24 `pred` calls, P_solve=10 solve calls, O=10000 observed characters, and exactly S=2 submit attempts. Never add or accept custom budgets, prompts, strategies, model kwargs, or coding-agent backends. -Read [references/provider-config.md](references/provider-config.md) when configuring the -provider or diagnosing API/preflight failures. Use `scripts/detect-engine.sh` before build. +Read [references/provider-config.md](references/provider-config.md) for endpoint setup and `scripts/detect-engine.sh` before preparing the image. -## Ask the API questions +## Collect only required choices -Ask only for information the caller has not already supplied. Ask in this order and wait at -each numbered stage when its answer determines the next question. Use the product's -structured user-input UI when available; otherwise ask the quoted question in plain text. -Do not dump every configuration question into one message. +1. Ask for the Model API identifier. +2. Ask whether it uses the standard provider endpoint or a custom OpenAI-compatible `API_BASE`. Never ask the caller to paste a secret; direct them to the gitignored `submission.env`. +3. Ask whether to keep and validate locally or upload officially. `$submit-benchmark-result` owns upload. +4. Read `make -s print-benchmark-version`, compare it with the official [`main/VERSION`](https://github.com/CodingThrust/problem-reductions-benchmark/blob/main/VERSION), and show `Benchmark version: (latest version:
)`. Wait for confirmation. If they differ, say the checkout is outdated and stop the official run. +5. Resolve `PR_REF` with `make -s print-pr-ref`, plus a fixed `STAMP` and `out//submission.json`. Do not ask for budget values. -1. Ask: +## Configure and preflight - > Which API model should run the benchmark? Give its provider/model identifier, for - > example `openai/gpt-...`, `anthropic/claude-...`, or `openrouter/...`. +Create `submission.env` from the example when absent. Configure `MODEL_NAME`, a provider key or `API_KEY`, and `API_BASE` only when needed. An existing `MODEL_KWARGS`, `AGENT_BACKEND`, `AGENT_CONFIG`, `AGENT_STRATEGY_FILE`, `SUBMIT_LIMIT`, or `PRB_*` budget override must be removed before a rankable run. -2. Ask: +Detect the container engine. Prefer `make runner-pull`; use `make runner-build` only when necessary. Before the first real API call, show the redacted model/endpoint, frozen contract ID and counters, target ref, output path, and upload goal, then ask for confirmation. - > Does this model use the provider's standard endpoint, or a custom OpenAI-compatible - > endpoint/gateway? - - For a custom endpoint, ask for `API_BASE` and any non-secret `MODEL_KWARGS`. Never ask the - caller to paste an API key into chat. Tell them exactly which key variable to set locally - in the gitignored `submission.env`, then wait for confirmation that it is configured. - -3. Ask: - - > What should happen after the run? - > - > 1. Keep and validate the result locally without uploading. - > 2. Upload an official submission. - - Default to local-only only when the caller explicitly delegates the choice. The - `$submit-benchmark-result` skill owns submission validation, authentication, and upload. - -4. Read this checkout's benchmark version with `make -s print-benchmark-version`. Read the - latest version from the official repository's - [`main/VERSION`](https://github.com/CodingThrust/problem-reductions-benchmark/blob/main/VERSION); - do not use the `problem-reductions` version or guess. Show this in the caller's language - and wait for confirmation: - - > Benchmark version: `` (latest version: `
`) - - If the versions differ, explain that the checkout is outdated and ask the caller to - update it before an official run. If the latest-version lookup fails, show `unknown` - rather than substituting the pinned `problem-reductions` version. - -5. Resolve the internal problem-reductions pin with `make -s print-pr-ref`, plus - `SUBMIT_LIMIT` (default 100) and `STAMP` (default: the Makefile timestamp). Show the - derived authoritative path `out//submission.json`; the trajectory is written - alongside it. Do not ask for arbitrary host output or log paths when using `make run`. - -## Configure safely - -Create `submission.env` from `submission.env.example` when absent. It is gitignored. Set or -guide the caller to set: - -- `MODEL_NAME`; -- one provider-specific key or `API_KEY`; -- `API_BASE` and `MODEL_KWARGS` only when needed; -- `SUBMIT_LIMIT` when non-default. - -This route always uses the `mini-swe` backend. If an existing env file contains -`AGENT_BACKEND`, remove it or require it to be `mini-swe`. Never select a coding-agent CLI -with `AGENT_BACKEND` and never run one inside the container. - -Do not expose secret values in command output or the final response. Do not add removed -`AGENT_MODE`, `MAX_RULES`, or max-turn settings. - -## Prepare and preflight - -1. Run `scripts/detect-engine.sh` and parse its `KEY=VALUE` output. Read - [references/engines.md](references/engines.md) only if no engine is available or the RAM - hint is low. -2. Prepare one image at the selected ref. Prefer the published image: - - ```bash - make runner-pull - ``` - - Fall back to `make runner-build` only when the published image is unavailable or the - caller explicitly wants a local build. For Podman, use the equivalent command from - `references/engines.md`. -3. Before the preflight's real API call, show the resolved model, backend `mini-swe`, API - endpoint with secrets redacted, confirmed benchmark version, submit limit, `STAMP`, - derived submission path, and upload goal. Ask for explicit confirmation. -4. Run `make preflight`. It checks `pred`, rule sources, and one tiny LiteLLM call. Stop on - any failure; never proceed to a full run after a failed preflight. +Run `make preflight`. It must reject custom execution knobs before its tiny model call, then verify `pred`, source inventory, endpoint, and credentials. Never continue after a failed preflight. ## Run and validate -After preflight passes, state that the full session can consume substantial time and API -credits and ask for explicit confirmation to start it. Then run `make run` or the equivalent -Podman command using the detector's `RUN_FLAGS`. Pass `STAMP=` when a fixed -stamp was selected. +Explain that 50 isolated episodes can use substantial API credits, then get explicit confirmation and run `make run STAMP=`. -Confirm the authoritative `submission.json` exists. For option 1, validate it locally: +For local-only validation: ```bash python -m benchmark.submit --predictions --dry-run ``` -Report `bugs_found`, `total_tokens_k`, submit attempts, any `run_error`, and absolute output -and log paths. A `run_error` means partial salvage, not a clean zero-bug completion. - -For option 2, invoke `$submit-benchmark-result` with the authoritative path. Do not validate -it first: that skill owns validation, authentication, final confirmation, upload, scoring, -and PR reporting. Never upload merely because the run completed. +Report the contract, completed episode count, rankability, claimed/verified bug fields available locally, cap-hit diagnostics, and absolute artifact path. Time, tokens, and cost are diagnostic only. -An exit code 137 means the engine needs more memory. Preserve partial outputs and read -actual command errors before recommending changes. +For official upload, invoke `$submit-benchmark-result`. Never upload merely because the run completed. Preserve partial checkpoints on failure; a `run_error` is not a clean zero. diff --git a/.agents/skills/run-benchmark/SKILL.md b/.agents/skills/run-benchmark/SKILL.md index 4dab277..5c28e2e 100644 --- a/.agents/skills/run-benchmark/SKILL.md +++ b/.agents/skills/run-benchmark/SKILL.md @@ -1,39 +1,14 @@ --- name: run-benchmark -description: Route a request to run, reproduce, smoke-test, or generate a submission for this problem-reductions benchmark. Use when the caller has not yet chosen between a model API run and an installed coding-agent CLI run. Ask the execution-route question, then hand off to run-api-benchmark or run-cli-benchmark; do not implement either backend flow here. +description: Route a request to run, reproduce, smoke-test, or generate a submission for the problem-reductions benchmark. The current rankable route is the standardized Model API Top50 runner; coding-agent CLI execution is historical and non-ranking. --- # Route a benchmark run -If the caller already has a `submission.json` and wants to validate or submit it, invoke -`$submit-benchmark-result` and stop. Do not ask how to run a model when the result already -exists. +If the caller already has a `submission.json`, invoke `$submit-benchmark-result`. -Choose exactly one execution route before asking about models, credentials, paths, or -submission settings. +For a current/rankable run, invoke `$run-api-benchmark`. Do not offer a backend choice: the frozen public contract is Model API only. -If the caller already explicitly requested an API, container, mini-swe, Codex, Claude Code, -or another coding-agent CLI, do not ask the route question again. Invoke the matching child -skill immediately. +If the caller explicitly asks to reproduce a legacy Codex, Claude Code, or other coding-agent artifact, explain that it belongs to `legacy-whole-repo`, cannot enter the Top50 table, and invoke `$run-cli-benchmark` only after they confirm they want a non-ranking historical run. -Otherwise ask this question and wait: - -> How should the benchmark call the model? -> -> 1. **Model API** — use the containerized mini-swe/LiteLLM runner with an API key or custom -> endpoint. -> 2. **Coding-agent CLI** — use an installed autonomous coding agent such as Codex or Claude -> Code. - -Use the product's structured user-input UI when available; otherwise ask the same question -in plain text. Do not combine both routes in one run. - -- For **Model API**, invoke `$run-api-benchmark` and follow it completely. -- For **Coding-agent CLI**, invoke `$run-cli-benchmark` and follow it completely. - -The route also selects the runtime. Model API means mini-swe/LiteLLM in a container. -Coding-agent CLI means an installed host process through `make run-local`; never place -Codex, Claude Code, or another CLI harness inside the API container path. - -The child skill owns all later questions, preflight, execution, validation, and optional -upload. Do not duplicate those workflows here. +The child skill owns preflight, execution, validation, and optional upload. diff --git a/.agents/skills/run-cli-benchmark/SKILL.md b/.agents/skills/run-cli-benchmark/SKILL.md index 298bcec..3374213 100644 --- a/.agents/skills/run-cli-benchmark/SKILL.md +++ b/.agents/skills/run-cli-benchmark/SKILL.md @@ -3,7 +3,11 @@ name: run-cli-benchmark description: Configure and run this problem-reductions benchmark through an installed autonomous coding-agent CLI harness. Use when the caller requests Codex, Claude Code, Kimi Code, OpenCode, or another CLI agent. Lists the harnesses actually supported by benchmark.run_submission.BACKENDS, routes unsupported harnesses through add-agent-harness before returning to model selection, verifies CLI authentication and pinned pred, runs one whole-repository session, validates submission.json, and uploads only when explicitly requested. --- -# Run the CLI benchmark +# Reproduce the historical CLI benchmark + +This route is permanently **non-ranking** and exists only to reproduce `legacy-whole-repo` +artifacts. Tell the caller that coding-agent results cannot enter `top50-evidence/v1` and +get confirmation before spending time or credits. Do not describe it as a System Track. Run one whole-repository coding-agent session with the shared benchmark prompt, verifier, submit ledger, and schema. This skill owns only coding-agent CLI execution. Do not add a @@ -136,6 +140,5 @@ python -m benchmark.submit --predictions --dry-run Report `bugs_found`, `total_tokens_k`, submit attempts, any `run_error`, CLI warnings, and absolute output/log paths. Preserve partial results and logs on failure. -For option 2, invoke `$submit-benchmark-result` with the authoritative path. Do not validate -it first: that skill owns validation, authentication, final confirmation, upload, scoring, -and PR reporting. Never upload merely because the run completed. +Do not upload a historical CLI artifact as a current official submission. Validate and keep +it locally under the legacy contract. diff --git a/.agents/skills/submit-benchmark-result/SKILL.md b/.agents/skills/submit-benchmark-result/SKILL.md index 2689e90..ba6e770 100644 --- a/.agents/skills/submit-benchmark-result/SKILL.md +++ b/.agents/skills/submit-benchmark-result/SKILL.md @@ -25,8 +25,8 @@ python3 -m benchmark.submit --predictions --dry-run Stop if validation fails. Otherwise report only: - absolute path; -- `model` and `library_commit`; -- claimed bugs, `total_tokens_k`, and number of submit attempts; +- `benchmark_contract`, `model`, and `library_commit`; +- completed Top50 episode count and claimed bugs; - `run_error`, if present. Do not submit a result containing `run_error`. Report the error and stop. @@ -39,6 +39,7 @@ Immediately before uploading, show this confirmation with the real values filled > - Destination: `intake.prb-bench.workers.dev` > - File: `` > - Model: `` +> - Contract: `top50-evidence/v1` > - Claimed bugs: `` > > This private submission file will leave the machine and be uploaded to the benchmark's diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a37dd58..ffedc9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,9 @@ jobs: - name: Run unit tests (judgment + non-integration) run: pytest -v -m "not integration" + - name: Check frozen Top50 calibration evidence + run: make verify-budget + - uses: actions/setup-node@v6 with: node-version: "24" @@ -64,3 +67,6 @@ jobs: - name: Run verifier calibration (known-answer fixtures, in-image) run: docker run --rm --network none prb-scoring:latest + + - name: Check frozen budget in pinned image + run: docker run --rm --network none --entrypoint python prb-scoring:latest -m benchmark.calibrate_budget --check benchmark/docs/budget-calibration.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ef1b488..6f46dbf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,299 +1,81 @@ # Submitting a model run -The benchmark gives every model one repository-wide, self-terminating agent session and -asks: how many distinct reduction-rule bugs can it find? +The current public comparison accepts one execution protocol: **Standardized Model API / Self-selected Top50** under frozen contract `top50-evidence/v1`. -``` - make run / make run-local ─▶ submission.json ─▶ python -m benchmark.submit - │ - private store (R2) - │ - maintainer re-verifies every certificate with pred - │ - aggregate only ─▶ PR ─▶ GitHub Pages -``` - -Your submission carries counterexample certificates plus the bounded submit ledger, so it -uploads to a private store; only the aggregate is published. Self-reported counts are never -trusted — the score is recomputed by `pred`. - -## 1. Produce a `submission.json` - -Both backends use the same runner and `submit` budget: - -| Backend | Execution | Repository skill | -|---|---|---| -| Model API | mini-swe/LiteLLM in Docker | `$run-api-benchmark` | -| Coding-agent CLI | installed agent on the host | `$run-cli-benchmark` | - -Use `$run-benchmark` to choose interactively. - -### Model API backend (Docker) - -The runner image bundles the `pred` binary, the agent stack (mini-swe-agent + LiteLLM), -and the problem-reductions source pinned at `v0.6.0`. Any LiteLLM-routable provider key -works. - -The official round is pinned to the exact tag and commit listed in the -[README](README.md#current-benchmark-round). `PR_REF` selects the corresponding -problem-reductions source when preparing an image; the runner records the resolved commit -and its matching `pred`. Pull the published image for that ref, or build it locally when -unavailable: - -```bash -make runner-pull PR_REF=v0.6.0 -# Fallback: make runner-build PR_REF=v0.6.0 +```text +source-only triage → frozen Top50 → 50 isolated rule episodes → private submission + ↓ + independent pred verification → aggregate leaderboard ``` -### Configure and run +## 1. Configure the model API -All run config goes in **one env-file** so you don't juggle a dozen `-e` flags. Copy the -template, fill the model and (for mini-swe) API key, and run: +Copy the internal template and set the model plus its provider credential: ```bash -cp submission.env.example submission.env # set model + mini-swe provider key -mkdir -p out -docker run --rm --env-file submission.env -v "$PWD/out:/out" \ - problem-reductions-runner:v0.6.0 -# → ./out/submission.json (or just: make run) -``` - -`MODEL_NAME` and any credentials required by the selected provider are required for the -mini-swe path. Everything else has a sane default — uncomment only what you need: - -| Tier | Vars | When | -|---|---|---| -| **Required** | `MODEL_NAME`; mini-swe also needs one provider key or generic `API_KEY` | always | -| **Non-standard provider** | `API_BASE`, `API_KEY`, `MODEL_KWARGS` (JSON of extra litellm kwargs: `api_version`, `custom_llm_provider`, `extra_headers`, …) | OpenRouter / gateway / local vLLM / Azure | -| **Limits** | `MAX_TOKENS=8192`, `SUBMIT_LIMIT=100` | per-call output ceiling; run-wide certificate attempts | -| **Custom prompt** | `AGENT_CONFIG`, `AGENT_STRATEGY_FILE` (mount the files too) | bring your own bug-hunting prompt | -| **Version pins** | `EXPECTED_PRED_VERSION` (empty disables), `EXPECTED_PRED_COMMIT` | debugging only — baked from the image build | - -`MODEL_NAME` accepts any LiteLLM-routable name (`anthropic/…`, `openai/…`, `openrouter/…`, -`azure/…`, or `openai/` against a custom `API_BASE`) — nothing is hardcoded to one -provider. (`REPO_DIR` / `OUTPUT` are container-internal and already defaulted; you don't set -them.) - -> Agents choose when to stop; there is no step, turn, or dollar limit. A run-wide `submit` -> budget (100 attempts by default) bounds scored counterexample claims. -> Every accepted, rejected, or malformed counterexample submission consumes one attempt; -> raw token counts are recorded and travel in the submission (`usage_totals`); ranking is by **confirmed bugs**, with -> **bugs/Ktok** as the efficiency tie-break. - -For example, a non-standard endpoint in `submission.env`: - -```ini -MODEL_NAME=openai/my-model -API_BASE=https://my-gateway.example/v1 -API_KEY=... -MODEL_KWARGS={"custom_llm_provider":"openai"} +cp submission.env.example submission.env ``` -Equivalently with raw `-e` flags (the env-file just bundles these): - -```bash -docker run --rm \ - -e MODEL_NAME=openai/my-model \ - -e API_BASE=https://my-gateway.example/v1 \ - -e API_KEY=$MY_KEY \ - -e MODEL_KWARGS='{"custom_llm_provider":"openai","extra_headers":{"X-Org":"acme"}}' \ - -v "$PWD/out:/out" \ - problem-reductions-runner:v0.6.0 -``` +`MODEL_NAME`, `API_BASE`, and `API_KEY` identify the endpoint. The official path does not accept prompt files, strategy files, coding-agent backends, model kwargs, submit pools, or budget variables. This is intentional: two entries are comparable only if model access is the remaining meaningful variable. -**Customize the agent's bug-hunting prompt** without rebuilding — mount your own files: +## 2. Preflight and run ```bash -docker run --rm --env-file submission.env \ - -e AGENT_STRATEGY_FILE=/cfg/strategy.md \ - -v "$PWD/cfg:/cfg" -v "$PWD/out:/out" \ - problem-reductions-runner:v0.6.0 -# strategy.md is injected into the prompt's reserved {{strategy}} slot. -# For a full prompt rewrite instead, mount a config.yaml and set AGENT_CONFIG=/cfg/config.yaml. +make runner-pull PR_REF=v0.6.0 # or make runner-build PR_REF=v0.6.0 +make preflight +make run ``` -**Before the full run, validate your config** with one tiny real call so a bad key / wrong -endpoint surfaces now, not 20 rules in: +Preflight first checks that no forbidden execution knob is present, verifies the pinned source and `pred`, and then makes one minimal model call. A rankable run uses: -```bash -make preflight # docker run --env-file submission.env --preflight -``` +- triage: 8 model generations, 12 source-only shell actions; +- 50 unique frozen rules; +- per rule: 10 model generations, 12 shell actions, 24 `pred` calls, at most 10 solves; +- exactly two charged submit attempts per rule; +- 10,000 observed characters per rule. -It checks the `pred` binary + version, that the library rules are present, and makes one -minimal model call through the exact batch code path. It exits non-zero on any failure. +The runner owns these counters. A model receives authoritative remaining-budget state after each action, cannot transfer unused budget, and cannot invoke the verifier directly. Each rule starts with fresh model history and a fresh submission ledger. -### Coding-agent CLI backend (host) +The runner checkpoints partial artifacts for diagnosis, but only a completed 50-episode artifact produced through the protected Model API factory can be rankable. Hung child processes are killed by fixed watchdogs and recorded as model/infrastructure outcomes; changing wall-clock conditions never changes the named logical budget or score. -Install the Python dependencies and the pinned `pred`, then authenticate one supported CLI: +## 3. Validate and upload ```bash -codex login # for the default codex backend -# or authenticate the Claude CLI - -cp submission.env.example submission.env -# Set MODEL_NAME; no key is needed in this file when the CLI login is already usable. - -make run-local \ - LOCAL_REPO_DIR=../runs/problem-reductions-v0.6.0 \ - LOCAL_OUTPUT=../runs/results/submission.json \ - LOCAL_LOG_DIR=../runs/logs -# Claude alternative: add LOCAL_BACKEND=claude-code +python -m benchmark.submit --predictions out//submission.json ``` -The default CLI backend is Codex. Codex uses the -non-interactive `codex exec --json --ephemeral` interface with a `workspace-write` -sandbox; Claude uses `claude -p`. Both receive the same benchmark prompts and agent-only -`submit` command, and both produce the same schema as the API backend. All backends run -exactly one whole-repository session and stop themselves; no turn count is passed to either -CLI. - -The CLI backend always runs on the host through `make run-local` and is selected with -`LOCAL_BACKEND`. Do not set `AGENT_BACKEND` or use Docker for Codex, Claude Code, or another -coding-agent CLI. - -The runner gives every backend a writable scratch workspace. The `submit` CLI exchanges -atomic request/response files there while the attempt budget and verified ledger remain in -runner memory, avoiding sandbox-blocked sockets or localhost networking. The prompt begins -with the free `submit --status` health probe. If no status or submit request reaches the -service, the run is saved as partial with `run_error` instead of a misleading clean zero; -small certificate artifacts are salvaged under the configured log directory. - -`run-local` clones `PR_REF` into the explicitly configured `LOCAL_REPO_DIR` when absent. -An existing checkout is only accepted when `HEAD` matches the requested ref; it is never -mutated automatically. Submission JSON and live/final logs go to the separate, required -`LOCAL_OUTPUT` and `LOCAL_LOG_DIR` paths. - -The CLI backend is intentionally less hermetic: it uses the host binary, authentication, -Python environment, and `pred`. The runner still verifies the pinned `pred` version and -records the target commit; use the API backend when container-level reproducibility is -required. - -#### Add another coding-agent CLI - -Codex (`codex`) and Claude Code (`claude-code`) are built in. For another CLI, use -`$add-agent-harness`; do not substitute a different agent. The adapter must pass the -[contract tests](.agents/skills/add-agent-harness/references/adapter-contract.md) and a real -[smoke evaluation](.agents/skills/add-agent-harness/references/reliability-evaluation.md) -must produce `verdict: reliable`. - -## 2. Submit it (CLI upload) - -Submission is a **CLI upload** — no web form, and the file never enters git. Choose one: +The intake check is only a courtesy check. The private backend independently validates: -- validate and keep the result locally with `--dry-run`; -- upload an intake test with `--test` (privately scored, excluded from the leaderboard); -- upload an official submission with neither flag. +- contract ID, prompt hash, runner mode, target commit, and `pred` version; +- the canonical 50-rule inventory and frozen order; +- triage and per-rule event/usage ledgers; +- exactly two submit opportunities per rule, including malformed or rejected calls; +- every accepted certificate with pinned `pred`. -Use `$submit-benchmark-result` when the result already exists. It validates the authoritative -file, preserves test/official intent, handles the authentication mode actually deployed by -the intake, asks before the external write, and reports the opaque submission ID. Submitters -do not need repository write access, R2 access, or permission to run the scoring workflow. +Changing any embedded limit, claiming a custom or incomplete run, inventing a rule, or making the artifact disagree with its ledgers makes the result unrankable. Only a reproducible certificate accepted by the independent `pred` replay contributes to the score. -For either upload, get the endpoint URL from the maintainer and authenticate through the -GitHub-backed Cloudflare Access application: +Only a compact aggregate is published. Private prompts, hypotheses, certificates, messages, rule identities, raw ledgers, credentials, and free text are excluded from the public projection. -```bash -export PRB_SUBMIT_URL= # from the maintainer -PRB_ACCESS_APP="${PRB_SUBMIT_URL%/submit}" -PRB_ACCESS_TOKEN="$(cloudflared access login --no-verbose --auto-close "$PRB_ACCESS_APP")" \ - python -m benchmark.submit --predictions out/submission.json -# --dry-run validate locally, don't send -# --test scored + stored privately, but excluded from the public leaderboard -``` +## 4. Read the score -The token is short-lived and scoped to this Access application. Keep the login inside -command substitution so its JWT goes directly into the one upload process instead of the -terminal. A later upload may replace `access login --no-verbose --auto-close` with -`access token -app` while the local Access session is still valid. Never substitute -`gh auth token`, a GitHub PAT, or `GITHUB_TOKEN`. +The Top50 table ranks `verified_bugs`: distinct shortlisted rules with a server-reproduced bug. `bugs_at_10`, `bugs_at_25`, first/second-attempt accepts, cap hits, tokens, cost, and elapsed time are diagnostics. They do not break ties. -The CLI validates the file against `submission.schema.json`, then uploads it to a private -store (Cloudflare R2). The maintainer's scorer re-verifies it with `pred` (see §3) and opens -a PR that updates the aggregate `site/results.json`; merging deploys to **GitHub Pages**. -See `intake/cloudflare-worker/README.md` for the intake setup. +The score should be read as joint **prioritization + bounded diagnosis + certificate construction** ability on the pinned library snapshot. A self-selected shortlist means it is not a pure diagnostic-only test. -## 3. Backend verification (automatic, zero-trust) +## Release checks -`benchmark/backend_score.py` is the queue worker. It runs inside the same Docker image -(which has `pred`). You can reproduce the scoring locally on a directory of submissions: +Before changing the frozen contract or target version, add a new versioned contract and a new non-ranking calibration. Do not silently edit `top50-evidence/v1`. ```bash -# Local directory of submissions → scored results + leaderboard.json: -python -m benchmark.backend_score --local submissions/ results/scored/ -``` - -For each `PENDING` submission it: - -1. validates the current submission structure, pinned library commit, and clean-run status, then flips - status `PENDING → RUNNING`, -2. re-runs `benchmark/verify_submission.py` — which calls `verify()` on **every** - certificate and re-derives the bundle from `pred`, so a fabricated or tampered - counterexample is rejected, and checks new runs against the bounded submit ledger - (legacy submissions retain trajectory provenance), -3. recomputes `bugs_found` as **distinct rules with a confirmed bug** (many certificates - for one rule collapse to one — no count padding), -4. writes the scored result + a ranked `leaderboard.json`, and sets status `FINISHED`; a - permanent input error is isolated under R2 `failed/`, while a retryable verifier or - infrastructure failure remains in `incoming/` for the next run. - -In production this runs unattended inside GitHub Actions: `score-from-r2.yml` pulls pending -submissions from private R2, re-verifies them, and opens a PR with the refreshed aggregate; -`publish-on-merge.yml` deploys the site when that PR merges — no external service to host. - -## Certificate format - -Each row in `submission.json` carries a **counterexample certificate** — the JSON the -runner emits to claim a bug in a rule. The verifier (`benchmark/verify.py`) re-derives -everything from `pred` and never trusts the claim; the authoritative schema is -`benchmark/submission.schema.json`. - -```json -{ - "rule": "MaximumIndependentSet/SimpleGraph/i32 -> IntegralFlowBundles", - "source": { - "type": "MaximumIndependentSet", - "data": { ... }, - "variant": { ... } - }, - "bundle": { - "target": { "type": "IntegralFlowBundles" } - }, - "target_config": "optional witness config" -} +pytest -v -m "not integration" +python -m benchmark.calibrate_budget --check benchmark/docs/budget-calibration.json +python -m benchmark.verify --calibrate ``` -Only `rule`, `source`, and the **target type** are required — the latter from -`bundle.target.type` (paste the full `pred reduce` bundle) or a top-level `target_type` -string. `target_config` is optional. Any `violation` / `note` you add is free-form; the -backend ignores it and derives the authoritative label itself: - -| Label | Meaning | -|-------|---------| -| `optimum_not_preserved` | both feasible, but the round-tripped value differs | -| `feasibility_not_preserved` | source is solvable but the round-trip yields none | -| `spurious_solution` | the round-trip claims a solution the source has none of | - -With `target_config` the verifier additionally checks that specific target solution, -catching `unsound_extraction` and `suboptimal_extraction` that the solver's own optimum -would hide. The round-trip judging itself is explained in the [README](README.md). - -## Why this is hard to cheat +The calibration checker proves that the selected limits match the runtime contract, every model covers the full candidate grid, smaller and larger `M`/`P` candidates surround the selection, and the Markdown report matches the machine-readable evidence. -- The score is computed **server-side** from the raw certificates, never from the - submission's `bugs_found` field. -- Counterexamples are **deterministically re-checkable** — we don't even need a hidden - answer key; a bug either violates the rule under `pred` or it doesn't. -- Distinct-rule de-duplication caps the count at one per rule. -- Agents stop themselves. `MAX_TOKENS` only bounds one model response; token totals travel - as raw 4-bucket counts (`usage_totals`) the backend recomputes `total_tokens_k` from. -- Counterexamples count only when the agent sends them through the evaluation-owned - `submit` command. Its in-memory ledger enforces the shared `SUBMIT_LIMIT` atomically; - certificates printed only in final prose or written to other files are ignored. +## Historical tooling -## Status: validated against a live model +`benchmark.run_submission`, `make run-local`, `benchmark/config.yaml`, and the coding-agent adapters reproduce the old `legacy-whole-repo` protocol only. Historical aggregates remain visible in their own site selector. They cannot be submitted or sorted into the Top50 table, and they are not a maintained System Track. -The runner pipeline is unit-tested end-to-end in fake mode + the certificate fixtures -**and** has been exercised against a live model API (a DeepSeek OpenAI-compatible endpoint -via `MODEL_NAME=openai/` + `API_BASE`): preflight passes, and a real run drives the -agent through a whole-repository session and emits a schema-valid `submission.json`. PR scoring and GitHub Pages -publishing are live; full official runs are the remaining step. +Repository policy also applies: never merge a PR until required CI is green and the user explicitly approves the merge. diff --git a/Makefile b/Makefile index 6ecb8f4..7cc1d1d 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ # verify-calibration Test the verifier against known fixtures (no AI needed) # preflight Validate submission.env with one tiny real call before a full run # run Run the benchmark via Docker → out//submission.json (does NOT upload) -# run-local Clone/verify a repo and run a local headless CLI agent +# run-local Reproduce a historical non-ranking CLI artifact # # Model/auth configuration lives in submission.env (see submission.env.example). Local # repository, submission, and log locations are intentionally explicit Make variables. @@ -31,7 +31,7 @@ LOCAL_LOG_DIR ?= REPO_URL ?= https://github.com/CodingThrust/problem-reductions.git LOCAL_ARGS = $(if $(LOCAL_BACKEND),--backend "$(LOCAL_BACKEND)") -.PHONY: test test-unit verify-calibration verify-judgment audit install-deps help print-benchmark-version print-pr-ref runner-build runner-pull preflight run run-local score-local board publish-local serve +.PHONY: test test-unit verify-calibration verify-budget verify-judgment audit install-deps help print-benchmark-version print-pr-ref runner-build runner-pull preflight run run-local score-local board publish-local serve ## Print this checkout's benchmark contract version. print-benchmark-version: @@ -58,6 +58,10 @@ verify-judgment: verify-calibration: python -m benchmark.verify --calibrate +## Check that the frozen Top50 contract, complete pilot grid, and rendered report agree. +verify-budget: + python -m benchmark.calibrate_budget --check benchmark/docs/budget-calibration.json + ## Build the dockerized submission runner image (compiles pred at PR_REF + bundles the agent). ## JOBS controls parallel rustc jobs in the pred build (default 1 = safe on small VMs). runner-build: @@ -80,8 +84,8 @@ preflight: echo "No $(ENV_FILE) — copy submission.env.example and fill it in first"; exit 1; fi docker run --rm --env-file "$(ENV_FILE)" $(IMAGE) --preflight -## Run the bug-finding agent via Docker → writes ./out//submission.json (the -## whole-repo trajectory lands alongside it). Each run gets its own timestamped dir so +## Run the bug-finding agent via Docker → writes ./out//submission.json. The +## Top50 phase histories and ledgers are embedded in that private artifact. Each run gets its own timestamped dir so ## successive runs never overwrite each other (override with STAMP=... to reproduce). ## This RUNS the benchmark locally; it does NOT submit — submitting is a separate step ## (upload it with benchmark.submit, see CONTRIBUTING.md). Config lives in submission.env @@ -91,9 +95,10 @@ run: echo "No $(ENV_FILE) — copy submission.env.example and fill it in (then: make preflight)"; exit 1; fi mkdir -p out docker run --rm --env-file "$(ENV_FILE)" -e OUTPUT=/out/$(STAMP)/submission.json -v "$(PWD)/out:/out" $(IMAGE) - @echo "Wrote out/$(STAMP)/submission.json (+ trajectory) — now submit it with 'python -m benchmark.submit' (see CONTRIBUTING.md)." + @echo "Wrote out/$(STAMP)/submission.json — now submit it with 'python -m benchmark.submit' (see CONTRIBUTING.md)." -## Lightweight host run through an installed headless agent CLI. The checkout is cloned at +## Historical non-ranking host run through an installed headless agent CLI. It cannot enter +## the Top50 leaderboard. The checkout is cloned at ## PR_REF when LOCAL_REPO_DIR is absent; an existing checkout must already match exactly. ## Requires explicit repo/output/log paths, local Python deps, pred, and CLI authentication. run-local: @@ -152,11 +157,12 @@ help: @echo " test Run full pytest suite" @echo " test-unit Run unit tests only (no real repo needed)" @echo " verify-calibration Test verifier against fixtures (no AI needed)" + @echo " verify-budget Check frozen Top50 calibration evidence offline" @echo " runner-build Build the dockerized submission runner image" @echo " runner-pull Pull the prebuilt runner image from GHCR (fast runner-build alternative)" @echo " preflight Validate submission.env (1 tiny real call) before a full run" @echo " run Run the API backend via Docker → out//submission.json (not upload)" - @echo " run-local Clone/verify a repo and run a local headless CLI agent" + @echo " run-local Reproduce a historical non-ranking CLI artifact" @echo " score-local Score SUBS_DIR submissions with the backend" @echo " board Build site/results.json from site/results/*.json (aggregate)" @echo " publish-local Score + stage per-submission entries + rebuild board (SUBS_DIR stays local)" diff --git a/README.md b/README.md index 4d3940a..ffd4f88 100644 --- a/README.md +++ b/README.md @@ -1,156 +1,76 @@ # Problem-Reductions Bug-Finding Benchmark -A benchmark that measures how efficiently AI models find bugs in reduction rules from the [problem-reductions](https://github.com/CodingThrust/problem-reductions) library (290+ rules). +This benchmark measures how well a model can prioritize likely-buggy reduction rules and turn a fixed amount of evidence gathering into independently verified counterexamples. -The leaderboard is a static site (`site/`) published to **GitHub Pages**. Submitting uses a CLI (`python -m benchmark.submit`) that uploads your run to a private store; the maintainer re-verifies it with `pred` and publishes only the aggregate. See [CONTRIBUTING.md](CONTRIBUTING.md) to run and submit. +## Current ranking contract -## Current benchmark round +The primary track is **Standardized Model API / Self-selected Top50**, contract [`top50-evidence/v1`](benchmark/top50_budget.json). The target is `problem-reductions` [`v0.6.0`](https://github.com/CodingThrust/problem-reductions/commit/aa2d1a10cffa434871d12a4d6f411147fb7e08a8), and the bundled `pred` version is `0.6.0`. -| Contract | Current value | Ownership | -|---|---|---| -| Benchmark version | [`v0.6.0`](VERSION) | `VERSION` is the source of truth used by local commands and production workflows. | -| Submission format | [`benchmark/submission.schema.json`](benchmark/submission.schema.json) | The runner writes the current structure and the scorer parses that structure directly. The payload has no schema-version field. | -| `problem-reductions` target | [`v0.6.0` / `aa2d1a10cffa434871d12a4d6f411147fb7e08a8`](https://github.com/CodingThrust/problem-reductions/commit/aa2d1a10cffa434871d12a4d6f411147fb7e08a8) | Every official result is verified against this exact commit. | -| `pred` | `0.6.0` | The runner/scorer image supplies and verifies the matching binary. | +Each run has two phases: -Do not hand-edit `library_commit` in `submission.json`. The runner records it; the intake -client performs a structural courtesy check, and the private scorer is the authority for -the current submission structure and round pin. +1. Source-only triage freezes exactly 50 unique rules and a short hypothesis for each. +2. The runner opens 50 fresh, sequential, isolated episodes in shortlist order. -## What this measures +Every episode receives the same immutable logical budget: -A reduction rule maps problem A → problem B. A **bug** is a round-trip failure: +| Counter | Limit per selected rule | +|---|---:| +| Model generations (`M`) | 10 | +| Shell actions (`E`) | 12 | +| Total `pred` calls (`P`) | 24 | +| `pred solve` calls (`P_solve`) | 10 | +| Submit attempts (`S`) | **2** | +| Observed output (`O`) | 10,000 characters | -``` -A →(reduce)→ B →(solve)→ s →(extract)→ A' -``` +Triage receives 8 model generations and 12 source-only actions. Unused budget never transfers between rules. Process timeouts remain fixed watchdogs for hung model or `pred` calls; elapsed time, network delay, tokens, and cost do not affect rank. -The rule is correct on an instance `a` only if solving it directly agrees with solving it through the reduction, compared by **value** (optimization) or **feasibility** (decision): +The budget was selected once using a human-reviewed non-ranking development replay, with smaller and larger candidates around the chosen `M` and `P`. The checked-in record and rationale are in [budget-calibration.md](benchmark/docs/budget-calibration.md); validate their internal and release consistency offline with: +```bash +python -m benchmark.calibrate_budget --check benchmark/docs/budget-calibration.json ``` -solve(a) == solve(reduce(a)) -``` - -A mismatch is a bug. The AI finds these by constructing **counterexample certificates** — a JSON object naming the source instance `a` and the rule; the backend re-derives the bundle and round-trips it with `pred`, so the AI's claim is never trusted directly. The mismatch is reported with a derived label (`optimum_not_preserved`, `feasibility_not_preserved`, or `spurious_solution`); an optional `target_config` witness can additionally expose extraction bugs on a specific target solution (`unsound_extraction` / `suboptimal_extraction`). - -**Primary metric: bugs found** — the number of *distinct rules* with at least one confirmed bug, on a pinned library commit. One rule = one bug, no matter how many counterexamples (or violation types) target it. This count is fully verifiable and cannot be inflated by resubmitting certificates. -**Secondary metric: bugs/Ktok** — token efficiency. It has a self-reported denominator (tokens), so it ranks ties and serves as reference, not as the headline. -Provenance is intentionally *not* scored: on a fixed commit, a `pred`-confirmed certificate is a bug regardless of who or what produced it. +## Metric and fairness claim -## Choose a backend +The primary metric is **verified distinct-rule bugs among the model's frozen Top50**. One rule counts at most once. Equal bug counts are ties—there is no token-efficiency or time tie-break. -The benchmark has two independent execution backends: +This score reflects a combined ability: -| Backend | Runtime | Repository skill | -|---|---|---| -| Model API | mini-swe/LiteLLM in Docker | `$run-api-benchmark` | -| Coding-agent CLI | installed agent on the host | `$run-cli-benchmark` | +- prioritize the 50 rules most likely to contain bugs; +- form useful source-level hypotheses; +- allocate bounded model, shell, and `pred` calls within each rule; +- construct a valid counterexample within two submission opportunities. -Start with `$run-benchmark` when the backend is not yet chosen. A CLI agent missing from -the supported list must first be integrated with `$add-agent-harness`. - -## How to add a coding-agent backend - -LiteLLM API models need no adapter. For a new CLI agent, use `$add-agent-harness` or follow -the same contract manually: - -Implement one repository-session function, following `run_repo_codex` or -`run_repo_claude`: - -```python -def run_repo_my_agent(model, ctx, *, trajectory_dir=None, submit_session=None, **kwargs): - # Run one repository-wide session. Scored rows come from submit_session, not this return. - return {"tokens_k": 12.3, "usage": usage, "error": None} -``` +It does not claim to measure host speed, network quality, willingness to run longer, coding-agent tooling, repeated-seed stability, or performance on a fixed organizer-selected Top50. Fixed Top50 diagnostics, multiple seeds, and a System Track are intentionally deferred or dropped. -Add its direct dispatch case to `_run_backend()` in `benchmark/run_submission.py`. -The backend is supported only after its adapter tests pass and -`harness-evaluation.json` reports `verdict: reliable`; command success alone is not enough. +A certificate is never trusted directly. The private scorer re-derives and replays it with pinned `pred`; only reproducible round-trip failures count. -A run is packaged as a `submission.json` (see `benchmark/submission.schema.json`). Use -`$submit-benchmark-result` to validate and upload an existing result without repository or -R2 access. See [CONTRIBUTING.md](CONTRIBUTING.md). +## Run the rankable track -During evaluation, counterexamples use a different, agent-only command: +Install Docker, then: ```bash -submit certificate.json # accepted or rejected: consumes one attempt -submit --status # inspect the remaining budget: free +cp submission.env.example submission.env # set MODEL_NAME and API credentials +make runner-pull # or: make runner-build +make preflight # validates frozen path + one tiny API call +make run # writes out//submission.json +python -m benchmark.submit --predictions out//submission.json ``` -The runner owns one shared counter for the complete run (default `SUBMIT_LIMIT=100`), -verifies every call immediately, and derives scored result rows only from its accepted -ledger. The CLI crosses Codex, Claude, mini-swe, and container sandboxes through an atomic -file queue inside a disposable agent workspace; the authoritative budget and ledger stay -in runner memory. Every session must successfully run the free `submit --status` probe, or -the output is marked with `run_error` rather than reported as a clean zero. Certificates -printed only in the agent's final response do not count. +The rankable path is deliberately narrow. It rejects custom logical limits, unlimited counters, `S != 2`, custom prompts or strategies, custom model kwargs, and coding-agent backends before a scored generation. Model and `pred` watchdogs are safety controls and are not user-selectable score dimensions. -## How to run - -The API image contains `pred`, Python, dependencies, and the target source. The CLI backend -instead uses those tools from the host. +Useful local checks: ```bash -# Run all unit tests (no API key needed) — this exercises the backend wiring make test-unit - -# Test the verifier against the fixtures (no API key) +make verify-budget make verify-calibration ``` -### Model API backend - -Install Docker, configure the model API in `submission.env`, then run mini-swe/LiteLLM in -the container: - -```bash -cp submission.env.example submission.env -make runner-pull # prebuilt image from GHCR — or `make runner-build` to compile locally (~1 h) -make preflight -make run -``` - -### Coding-agent CLI backend +See [CONTRIBUTING.md](CONTRIBUTING.md) for artifact fields, submission, and historical-result handling. -Install Python 3.12, the benchmark dependencies, the pinned `pred`, and a supported CLI. -Authenticate the CLI, set `MODEL_NAME` in `submission.env`, and run it directly on the host: +## Historical results -```bash -cp submission.env.example submission.env -make run-local \ - LOCAL_REPO_DIR=../runs/problem-reductions-v0.6.0 \ - LOCAL_OUTPUT=../runs/results/submission.json \ - LOCAL_LOG_DIR=../runs/logs - -# Claude alternative: add LOCAL_BACKEND=claude-code -``` +Whole-repository results produced by the former contract remain visible under `legacy-whole-repo`. They use a different execution protocol and efficiency tie-break, so the site keeps them in a separate selectable table. They cannot be deduplicated, sorted, or compared into `top50-evidence/v1`. -`run-local` clones `PR_REF` into `LOCAL_REPO_DIR` when the path is absent. If the path -already exists, its `HEAD` must match that ref; the runner never resets or checks out an -existing working tree. `LOCAL_OUTPUT` and `LOCAL_LOG_DIR` are deliberately separate and -required. The CLI backend runs one self-terminating whole-repository session with the same -run-wide `submit` budget as the API backend. There is no agent step or turn limit; the -six-hour CLI timeout and per-command timeout only guard against hung processes. - -Key `make` targets: - -| Target | Description | -|--------|-------------| -| `make test-unit` | All unit tests, no API key needed | -| `make verify-calibration` | Test verifier against the fixtures (accept + reject paths) | -| `make verify-judgment` | Pred-free sanity tests (docs, CI, trajectory) | -| `make preflight` | Validate the API backend with one tiny real call before a full run | -| `make run` | Run the API backend in Docker → `out//submission.json` (does not upload) | -| `make run-local` | Run a coding-agent CLI on the host → the same output schema | -| `make score-local` | Score submissions with the zero-trust backend | - -## How to read the metrics - -| Metric | Formula | When to use | -|--------|---------|-------------| -| `bugs_found` | distinct rules with a confirmed bug | **Primary ranking** — fully verifiable, cannot be inflated | -| `bugs/Ktok` | bugs ÷ tokens(K) | Tiebreak / efficiency reference — self-reported denominator | - -Rank by `bugs_found`. Among models that find the same number of bugs, `bugs/Ktok` breaks the tie. The efficiency metric divides by tokens, which the submitter self-reports — treat it as informative, not authoritative. +The old host coding-agent and whole-repository runners remain in the repository only to reproduce those historical artifacts. They are not a public System Track and cannot produce rankable Top50 submissions. diff --git a/benchmark/calibrate_budget.py b/benchmark/calibrate_budget.py new file mode 100644 index 0000000..91974c8 --- /dev/null +++ b/benchmark/calibrate_budget.py @@ -0,0 +1,290 @@ +"""Offline validation and rendering for the frozen Top50 budget evidence.""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +from benchmark.top50_budget import FROZEN_CONTRACT +from benchmark.usage import usage_from_dict + +ROOT = Path(__file__).resolve().parent +CONTRACT_PATH = ROOT / "top50_budget.json" +REPORT_PATH = ROOT / "docs" / "budget-calibration.md" +REQUIRED_OBSERVATION_FIELDS = { + "candidate", "model", "target", "verified_bugs", "cap_hits", "usage", + "retries", "infrastructure_failures", "token_reference", "cost_reference", + "time_reference", "marginal_yield", +} + + +def load_json(path: str | Path) -> dict: + value = json.loads(Path(path).read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"{path} must contain a JSON object") + return value + + +def validate_evidence(evidence: dict, contract: dict) -> list[str]: + errors: list[str] = [] + if evidence.get("schema_version") != 1: + errors.append("schema_version must be 1") + if evidence.get("ranking_status") != "non-ranking-development": + errors.append("calibration evidence must be explicitly non-ranking") + if evidence.get("selected_contract") != contract: + errors.append("selected_contract does not exactly match top50_budget.json") + sources = evidence.get("sources") + source_index: dict[tuple[str, str], str] = {} + if not isinstance(sources, list) or not sources: + errors.append("sources must be a non-empty list") + else: + for index, source in enumerate(sources): + if not isinstance(source, dict): + errors.append(f"sources[{index}] must be an object") + continue + model, target, digest = (source.get("model"), source.get("target"), + source.get("artifact_sha256")) + if (not isinstance(model, str) or not isinstance(target, str) + or not isinstance(digest, str) or len(digest) != 64 + or any(char not in "0123456789abcdef" for char in digest)): + errors.append(f"sources[{index}] has invalid provenance") + continue + source_index[(model, target)] = digest + + observations = evidence.get("observations") + if not isinstance(observations, list) or not observations: + return errors + ["observations must be a non-empty list"] + candidates: set[tuple[int, int]] = set() + model_candidates: dict[str, set[tuple[int, int]]] = {} + seen_rows: set[tuple[str, int, int]] = set() + for index, observation in enumerate(observations): + if not isinstance(observation, dict): + errors.append(f"observations[{index}] must be an object") + continue + missing = REQUIRED_OBSERVATION_FIELDS - set(observation) + if missing: + errors.append(f"observations[{index}] missing {sorted(missing)}") + continue + candidate = observation["candidate"] + if not isinstance(candidate, dict): + errors.append(f"observations[{index}].candidate must be an object") + continue + m, p = candidate.get("model_generations"), candidate.get("pred_calls") + if (not isinstance(m, int) or isinstance(m, bool) or m <= 0 + or not isinstance(p, int) or isinstance(p, bool) or p <= 0): + errors.append(f"observations[{index}] has invalid M/P candidate") + continue + model, target = observation.get("model"), observation.get("target") + if not isinstance(model, str) or not isinstance(target, str): + errors.append(f"observations[{index}] has invalid model/target") + continue + row = (model, m, p) + if row in seen_rows: + errors.append(f"observations[{index}] duplicates model/M/P") + seen_rows.add(row) + candidates.add((m, p)) + model_candidates.setdefault(model, set()).add((m, p)) + digest = source_index.get((model, target)) + if digest is None or observation.get("token_reference") != digest: + errors.append(f"observations[{index}] is not linked to a declared source") + for field in ("verified_bugs", "infrastructure_failures"): + value = observation.get(field) + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + errors.append(f"observations[{index}].{field} must be non-negative integer") + retries = observation.get("retries") + if retries is not None and (not isinstance(retries, int) + or isinstance(retries, bool) or retries < 0): + errors.append(f"observations[{index}].retries must be non-negative or null") + cap_hits, usage = observation.get("cap_hits"), observation.get("usage") + if not isinstance(cap_hits, dict) or any( + not isinstance(cap_hits.get(name), int) or isinstance(cap_hits.get(name), bool) + or not 0 <= cap_hits[name] <= contract["shortlist_size"] + for name in ("model_generations", "pred_calls")): + errors.append(f"observations[{index}].cap_hits is invalid") + if not isinstance(usage, dict) or any( + not isinstance(value, (int, float)) or isinstance(value, bool) or value < 0 + for value in usage.values()): + errors.append(f"observations[{index}].usage is invalid") + marginal = observation.get("marginal_yield") + if not isinstance(marginal, (int, float)) or isinstance(marginal, bool): + errors.append(f"observations[{index}].marginal_yield must be numeric") + selected_m = contract["episode"]["model_generations"] + selected_p = contract["episode"]["pred_calls"] + ms = {m for m, _ in candidates} + ps = {p for _, p in candidates} + if not any(m < selected_m for m in ms) or not any(m > selected_m for m in ms): + errors.append("selected M is not surrounded by smaller and larger candidates") + if not any(p < selected_p for p in ps) or not any(p > selected_p for p in ps): + errors.append("selected P is not surrounded by smaller and larger candidates") + expected_grid = {(m, p) for m in ms for p in ps} + for model, actual in model_candidates.items(): + if actual != expected_grid: + errors.append(f"model {model!r} does not cover the complete candidate grid") + if not evidence.get("selection_rationale"): + errors.append("selection_rationale is required") + return errors + + +def render_report(evidence: dict) -> str: + contract = evidence["selected_contract"] + episode = contract["episode"] + safety = contract["safety_controls"] + lines = [ + "# Top50 budget calibration", + "", + f"Contract: `{contract['contract_id']}` (`{contract['status']}`)", + "", + "This is a human-reviewed, non-ranking bounded-prefix replay record from internally " + "retained pilot trajectories. Raw trajectories remain private; this offline checker " + "validates the checked-in record and release consistency, not the raw replay. It is not " + "a public score, a multi-seed experiment, or a claim that elapsed time is model ability.", + "", + "## Selected contract", + "", + f"The release freezes M={episode['model_generations']} model generations, " + f"P={episode['pred_calls']} total `pred` calls, " + f"P_solve={episode['solve_calls']} solve calls, S={episode['submit_attempts']} " + f"submit attempts, E={episode['shell_actions']} shell actions, and " + f"O={episode['max_output_chars']} observed characters per rule. Triage is " + f"T={contract['triage']['model_generations']} generations and " + f"E_t={contract['triage']['shell_actions']} source-only actions.", + "", + f"Model calls have a fixed {safety['model_timeout_seconds']}-second watchdog and " + f"{safety['model_retries']} transport retries. Command and `pred` watchdogs are also " + "fixed safety controls. They are recorded but are outside the logical budget and never " + "enter the score.", + "", + "## Measured grid", + "", + "| Model | M | P | Bugs | M cap rate | P cap rate | Tokens | Retries | Infra failures | Marginal yield |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + shortlist_size = contract["shortlist_size"] + for item in evidence["observations"]: + c = item["candidate"] + retries = item["retries"] if item["retries"] is not None else "unavailable" + lines.append( + f"| {item['model']} | {c['model_generations']} | {c['pred_calls']} | " + f"{item['verified_bugs']} | " + f"{item['cap_hits']['model_generations'] / shortlist_size:.0%} | " + f"{item['cap_hits']['pred_calls'] / shortlist_size:.0%} | " + f"{item['usage']['total_tokens']} | " + f"{retries} | {item['infrastructure_failures']} | " + f"{item['marginal_yield']:.3f} |") + lines += [ + "", + "Token, cost, and elapsed-time fields are diagnostic references only. Missing provider " + "cost or reliable wall-clock data is recorded as unavailable instead of imputed.", + "", + "## Decision", + "", + evidence["selection_rationale"], + "", + "The public comparison is therefore one run at this single contract, ranked only by " + "verified distinct-rule bugs. Fixed Top50, multiple seeds, a System Track, and a public " + "budget grid remain out of scope.", + "", + "## Provenance", + "", + ] + for source in evidence["sources"]: + lines.append(f"- `{source['artifact_sha256']}` — {source['model']}, " + f"{source['target']}, {source['method']}") + return "\n".join(lines) + "\n" + + +def observation_from_artifact(path: str | Path) -> dict: + """Extract one non-ranking grid observation from a completed development artifact.""" + artifact_path = Path(path) + raw = artifact_path.read_bytes() + artifact = json.loads(raw) + if not isinstance(artifact, dict): + raise ValueError(f"{path} must contain a JSON object") + episode_budget = artifact.get("contract", {}).get("episode", {}) + episodes = artifact.get("episodes") + if (artifact.get("calibration_status") != "non-ranking-development" + or not isinstance(episodes, list) + or not all(key in episode_budget for key in ("model_generations", "pred_calls"))): + raise ValueError(f"{path} is not a completed non-ranking calibration artifact") + cap_hits = {"model_generations": 0, "pred_calls": 0} + total_tokens = infrastructure_failures = 0 + bugs = 0 + for episode in episodes: + if not isinstance(episode, dict): + continue + bugs += episode.get("status") == "bug_found" + infrastructure_failures += episode.get("status") == "run_error" + ledger = episode.get("ledger", {}) + for counter in cap_hits: + state = ledger.get("status", {}).get(counter, {}) + cap_hits[counter] += (state.get("limit", 0) > 0 + and state.get("used") == state.get("limit")) + total_tokens += usage_from_dict(episode.get("usage") or {}).total_tokens + digest = hashlib.sha256(raw).hexdigest() + return { + "candidate": {"model_generations": episode_budget["model_generations"], + "pred_calls": episode_budget["pred_calls"]}, + "model": artifact.get("model", "unknown"), + "target": artifact.get("library_commit", "unknown"), + "verified_bugs": bugs, + "cap_hits": cap_hits, + "usage": {"total_tokens": total_tokens, + "shell_actions": sum(len(e.get("ledger", {}).get("shell_actions", [])) + for e in episodes if isinstance(e, dict)), + "pred_calls": sum(len(e.get("ledger", {}).get("pred", [])) + for e in episodes if isinstance(e, dict))}, + "retries": artifact.get("transport_retries"), + "infrastructure_failures": infrastructure_failures, + "token_reference": digest, + "cost_reference": artifact.get("cost_reference", "unavailable; not imputed"), + "time_reference": artifact.get("time_reference", "unavailable; not imputed"), + "marginal_yield": 0.0, + } + + +def check(path: str | Path) -> list[str]: + evidence = load_json(path) + contract = FROZEN_CONTRACT + errors = validate_evidence(evidence, contract) + for schema_name in ("top50_submission.schema.json", "top50_results.schema.json"): + schema = load_json(ROOT / schema_name) + schema_contract = schema.get("properties", {}).get("benchmark_contract", {}).get("const") + if schema_contract != contract["contract_id"]: + errors.append(f"{schema_name} does not name the frozen contract") + submission_schema = load_json(ROOT / "top50_submission.schema.json") + properties = submission_schema.get("properties", {}) + expected_artifact_contract = {key: contract[key] for key in + ("triage", "episode", "shortlist_size", + "hypothesis_chars")} + if properties.get("contract", {}).get("const") != expected_artifact_contract: + errors.append("top50_submission.schema.json has stale logical limits") + if properties.get("safety_controls", {}).get("const") != contract["safety_controls"]: + errors.append("top50_submission.schema.json has stale safety controls") + if properties.get("inference_parameters", {}).get("const") != contract[ + "inference_parameters"]: + errors.append("top50_submission.schema.json has stale inference parameters") + if not errors and REPORT_PATH.read_text(encoding="utf-8") != render_report(evidence): + errors.append("budget-calibration.md does not match the machine-readable evidence") + return errors + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description="Validate frozen Top50 calibration evidence") + action = parser.add_mutually_exclusive_group(required=True) + action.add_argument("--check", metavar="EVIDENCE_JSON") + action.add_argument("--summarize", nargs="+", metavar="DEV_ARTIFACT") + args = parser.parse_args(argv) + if args.summarize: + print(json.dumps([observation_from_artifact(path) for path in args.summarize], indent=2)) + return + errors = check(args.check) + if errors: + for error in errors: + print(f"FAIL: {error}") + raise SystemExit(1) + print(f"PASS: calibration evidence matches {FROZEN_CONTRACT['contract_id']}") + + +if __name__ == "__main__": + main() diff --git a/benchmark/config.yaml b/benchmark/config.yaml index c32987d..63964b3 100644 --- a/benchmark/config.yaml +++ b/benchmark/config.yaml @@ -84,7 +84,8 @@ agent: # mini-swe-agent uses 0 to mean unlimited. Cost limiting is also force-disabled by the # runner; the agent stops through the completion command above. - step_limit: 0 + # Historical whole-repository runner only; never produces Top50 rankable results. + step_limit: 100 environment: # Per-command protection against a wedged subprocess, not an agent turn budget. diff --git a/benchmark/docs/budget-calibration.json b/benchmark/docs/budget-calibration.json new file mode 100644 index 0000000..53c45c3 --- /dev/null +++ b/benchmark/docs/budget-calibration.json @@ -0,0 +1,485 @@ +{ + "schema_version": 1, + "generated_at": "2026-07-20T00:00:00Z", + "ranking_status": "non-ranking-development", + "method": "Human-reviewed bounded-prefix replay record from internally retained whole-repository pilot trajectories. Raw trajectories remain private; the offline checker validates this record and release consistency, not the raw replay. These statistics are not Top50 scores.", + "selected_contract": { + "contract_id": "top50-evidence/v1", + "status": "frozen", + "shortlist_size": 50, + "hypothesis_chars": 500, + "safety_controls": { + "model_timeout_seconds": 300, + "model_retries": 2 + }, + "inference_parameters": { + "max_tokens": 8192, + "timeout": 300, + "num_retries": 2 + }, + "triage": { + "model_generations": 8, + "shell_actions": 12, + "max_output_chars": 10000, + "command_timeout_seconds": 300 + }, + "episode": { + "model_generations": 10, + "shell_actions": 12, + "pred_calls": 24, + "solve_calls": 10, + "submit_attempts": 2, + "max_output_chars": 10000, + "pred_timeout_seconds": 300 + } + }, + "selection_rationale": "M=10 and P=24 are the smallest tested values at which the replayed stronger pilot retains its verified find. Moving down to M=6 or P=12 loses that find and shows materially higher cap pressure; moving up to M=14 or P=36 adds usage without another verified bug. E=12 and P_solve=10 cover the observed command mix without exposing an unlimited path. S=2 follows the fixed per-rule retry policy. T=8 and E_t=12 bound source-only shortlist formation. Output and watchdog ceilings are safety controls selected above observed pilot maxima and do not affect ranking.", + "sources": [ + { + "model": "anthropic/claude-haiku-4-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "artifact_sha256": "32237111e946206e42bb7ee36c54c09eb1d5313c8fcd4d75c2f440e94bee96d4", + "method": "private legacy pilot trajectory; aggregate bounded-prefix replay retained here" + }, + { + "model": "anthropic/claude-sonnet-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "artifact_sha256": "ac475bb2cc7c6118f6405b63fc52703f4afc27588f0f4cd021d36328222316d3", + "method": "private legacy pilot trajectory; aggregate bounded-prefix replay retained here" + } + ], + "observations": [ + { + "candidate": { + "model_generations": 6, + "pred_calls": 12 + }, + "model": "anthropic/claude-haiku-4-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 0, + "cap_hits": { + "model_generations": 31, + "pred_calls": 24 + }, + "usage": { + "total_tokens": 1423721, + "shell_actions": 350, + "pred_calls": 450 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "32237111e946206e42bb7ee36c54c09eb1d5313c8fcd4d75c2f440e94bee96d4", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + }, + { + "candidate": { + "model_generations": 6, + "pred_calls": 24 + }, + "model": "anthropic/claude-haiku-4-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 0, + "cap_hits": { + "model_generations": 31, + "pred_calls": 7 + }, + "usage": { + "total_tokens": 1615376, + "shell_actions": 350, + "pred_calls": 900 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "32237111e946206e42bb7ee36c54c09eb1d5313c8fcd4d75c2f440e94bee96d4", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + }, + { + "candidate": { + "model_generations": 6, + "pred_calls": 36 + }, + "model": "anthropic/claude-haiku-4-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 0, + "cap_hits": { + "model_generations": 31, + "pred_calls": 1 + }, + "usage": { + "total_tokens": 1807031, + "shell_actions": 350, + "pred_calls": 1300 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "32237111e946206e42bb7ee36c54c09eb1d5313c8fcd4d75c2f440e94bee96d4", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + }, + { + "candidate": { + "model_generations": 10, + "pred_calls": 12 + }, + "model": "anthropic/claude-haiku-4-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 0, + "cap_hits": { + "model_generations": 11, + "pred_calls": 24 + }, + "usage": { + "total_tokens": 1587997, + "shell_actions": 550, + "pred_calls": 450 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "32237111e946206e42bb7ee36c54c09eb1d5313c8fcd4d75c2f440e94bee96d4", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + }, + { + "candidate": { + "model_generations": 10, + "pred_calls": 24 + }, + "model": "anthropic/claude-haiku-4-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 0, + "cap_hits": { + "model_generations": 11, + "pred_calls": 7 + }, + "usage": { + "total_tokens": 1779651, + "shell_actions": 550, + "pred_calls": 900 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "32237111e946206e42bb7ee36c54c09eb1d5313c8fcd4d75c2f440e94bee96d4", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + }, + { + "candidate": { + "model_generations": 10, + "pred_calls": 36 + }, + "model": "anthropic/claude-haiku-4-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 0, + "cap_hits": { + "model_generations": 11, + "pred_calls": 1 + }, + "usage": { + "total_tokens": 1971306, + "shell_actions": 550, + "pred_calls": 1300 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "32237111e946206e42bb7ee36c54c09eb1d5313c8fcd4d75c2f440e94bee96d4", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + }, + { + "candidate": { + "model_generations": 14, + "pred_calls": 12 + }, + "model": "anthropic/claude-haiku-4-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 0, + "cap_hits": { + "model_generations": 3, + "pred_calls": 24 + }, + "usage": { + "total_tokens": 1752272, + "shell_actions": 600, + "pred_calls": 450 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "32237111e946206e42bb7ee36c54c09eb1d5313c8fcd4d75c2f440e94bee96d4", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + }, + { + "candidate": { + "model_generations": 14, + "pred_calls": 24 + }, + "model": "anthropic/claude-haiku-4-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 0, + "cap_hits": { + "model_generations": 3, + "pred_calls": 7 + }, + "usage": { + "total_tokens": 1943927, + "shell_actions": 600, + "pred_calls": 900 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "32237111e946206e42bb7ee36c54c09eb1d5313c8fcd4d75c2f440e94bee96d4", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + }, + { + "candidate": { + "model_generations": 14, + "pred_calls": 36 + }, + "model": "anthropic/claude-haiku-4-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 0, + "cap_hits": { + "model_generations": 3, + "pred_calls": 1 + }, + "usage": { + "total_tokens": 2135581, + "shell_actions": 600, + "pred_calls": 1300 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "32237111e946206e42bb7ee36c54c09eb1d5313c8fcd4d75c2f440e94bee96d4", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + }, + { + "candidate": { + "model_generations": 6, + "pred_calls": 12 + }, + "model": "anthropic/claude-sonnet-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 0, + "cap_hits": { + "model_generations": 31, + "pred_calls": 24 + }, + "usage": { + "total_tokens": 6109805, + "shell_actions": 350, + "pred_calls": 450 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "ac475bb2cc7c6118f6405b63fc52703f4afc27588f0f4cd021d36328222316d3", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + }, + { + "candidate": { + "model_generations": 6, + "pred_calls": 24 + }, + "model": "anthropic/claude-sonnet-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 0, + "cap_hits": { + "model_generations": 31, + "pred_calls": 7 + }, + "usage": { + "total_tokens": 6932279, + "shell_actions": 350, + "pred_calls": 900 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "ac475bb2cc7c6118f6405b63fc52703f4afc27588f0f4cd021d36328222316d3", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + }, + { + "candidate": { + "model_generations": 6, + "pred_calls": 36 + }, + "model": "anthropic/claude-sonnet-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 0, + "cap_hits": { + "model_generations": 31, + "pred_calls": 1 + }, + "usage": { + "total_tokens": 7754753, + "shell_actions": 350, + "pred_calls": 1300 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "ac475bb2cc7c6118f6405b63fc52703f4afc27588f0f4cd021d36328222316d3", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + }, + { + "candidate": { + "model_generations": 10, + "pred_calls": 12 + }, + "model": "anthropic/claude-sonnet-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 0, + "cap_hits": { + "model_generations": 11, + "pred_calls": 24 + }, + "usage": { + "total_tokens": 6814783, + "shell_actions": 550, + "pred_calls": 450 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "ac475bb2cc7c6118f6405b63fc52703f4afc27588f0f4cd021d36328222316d3", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + }, + { + "candidate": { + "model_generations": 10, + "pred_calls": 24 + }, + "model": "anthropic/claude-sonnet-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 1, + "cap_hits": { + "model_generations": 11, + "pred_calls": 7 + }, + "usage": { + "total_tokens": 7637256, + "shell_actions": 550, + "pred_calls": 900 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "ac475bb2cc7c6118f6405b63fc52703f4afc27588f0f4cd021d36328222316d3", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 1 + }, + { + "candidate": { + "model_generations": 10, + "pred_calls": 36 + }, + "model": "anthropic/claude-sonnet-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 1, + "cap_hits": { + "model_generations": 11, + "pred_calls": 1 + }, + "usage": { + "total_tokens": 8459730, + "shell_actions": 550, + "pred_calls": 1300 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "ac475bb2cc7c6118f6405b63fc52703f4afc27588f0f4cd021d36328222316d3", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + }, + { + "candidate": { + "model_generations": 14, + "pred_calls": 12 + }, + "model": "anthropic/claude-sonnet-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 0, + "cap_hits": { + "model_generations": 3, + "pred_calls": 24 + }, + "usage": { + "total_tokens": 7519760, + "shell_actions": 600, + "pred_calls": 450 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "ac475bb2cc7c6118f6405b63fc52703f4afc27588f0f4cd021d36328222316d3", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + }, + { + "candidate": { + "model_generations": 14, + "pred_calls": 24 + }, + "model": "anthropic/claude-sonnet-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 1, + "cap_hits": { + "model_generations": 3, + "pred_calls": 7 + }, + "usage": { + "total_tokens": 8342234, + "shell_actions": 600, + "pred_calls": 900 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "ac475bb2cc7c6118f6405b63fc52703f4afc27588f0f4cd021d36328222316d3", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + }, + { + "candidate": { + "model_generations": 14, + "pred_calls": 36 + }, + "model": "anthropic/claude-sonnet-5", + "target": "problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8", + "verified_bugs": 1, + "cap_hits": { + "model_generations": 3, + "pred_calls": 1 + }, + "usage": { + "total_tokens": 9164707, + "shell_actions": 600, + "pred_calls": 1300 + }, + "retries": null, + "infrastructure_failures": 0, + "token_reference": "ac475bb2cc7c6118f6405b63fc52703f4afc27588f0f4cd021d36328222316d3", + "cost_reference": "unavailable in retained provider artifact; not imputed", + "time_reference": "legacy stream has no trustworthy monotonic elapsed clock; not imputed", + "marginal_yield": 0 + } + ] +} diff --git a/benchmark/docs/budget-calibration.md b/benchmark/docs/budget-calibration.md new file mode 100644 index 0000000..7306cd2 --- /dev/null +++ b/benchmark/docs/budget-calibration.md @@ -0,0 +1,47 @@ +# Top50 budget calibration + +Contract: `top50-evidence/v1` (`frozen`) + +This is a human-reviewed, non-ranking bounded-prefix replay record from internally retained pilot trajectories. Raw trajectories remain private; this offline checker validates the checked-in record and release consistency, not the raw replay. It is not a public score, a multi-seed experiment, or a claim that elapsed time is model ability. + +## Selected contract + +The release freezes M=10 model generations, P=24 total `pred` calls, P_solve=10 solve calls, S=2 submit attempts, E=12 shell actions, and O=10000 observed characters per rule. Triage is T=8 generations and E_t=12 source-only actions. + +Model calls have a fixed 300-second watchdog and 2 transport retries. Command and `pred` watchdogs are also fixed safety controls. They are recorded but are outside the logical budget and never enter the score. + +## Measured grid + +| Model | M | P | Bugs | M cap rate | P cap rate | Tokens | Retries | Infra failures | Marginal yield | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| anthropic/claude-haiku-4-5 | 6 | 12 | 0 | 62% | 48% | 1423721 | unavailable | 0 | 0.000 | +| anthropic/claude-haiku-4-5 | 6 | 24 | 0 | 62% | 14% | 1615376 | unavailable | 0 | 0.000 | +| anthropic/claude-haiku-4-5 | 6 | 36 | 0 | 62% | 2% | 1807031 | unavailable | 0 | 0.000 | +| anthropic/claude-haiku-4-5 | 10 | 12 | 0 | 22% | 48% | 1587997 | unavailable | 0 | 0.000 | +| anthropic/claude-haiku-4-5 | 10 | 24 | 0 | 22% | 14% | 1779651 | unavailable | 0 | 0.000 | +| anthropic/claude-haiku-4-5 | 10 | 36 | 0 | 22% | 2% | 1971306 | unavailable | 0 | 0.000 | +| anthropic/claude-haiku-4-5 | 14 | 12 | 0 | 6% | 48% | 1752272 | unavailable | 0 | 0.000 | +| anthropic/claude-haiku-4-5 | 14 | 24 | 0 | 6% | 14% | 1943927 | unavailable | 0 | 0.000 | +| anthropic/claude-haiku-4-5 | 14 | 36 | 0 | 6% | 2% | 2135581 | unavailable | 0 | 0.000 | +| anthropic/claude-sonnet-5 | 6 | 12 | 0 | 62% | 48% | 6109805 | unavailable | 0 | 0.000 | +| anthropic/claude-sonnet-5 | 6 | 24 | 0 | 62% | 14% | 6932279 | unavailable | 0 | 0.000 | +| anthropic/claude-sonnet-5 | 6 | 36 | 0 | 62% | 2% | 7754753 | unavailable | 0 | 0.000 | +| anthropic/claude-sonnet-5 | 10 | 12 | 0 | 22% | 48% | 6814783 | unavailable | 0 | 0.000 | +| anthropic/claude-sonnet-5 | 10 | 24 | 1 | 22% | 14% | 7637256 | unavailable | 0 | 1.000 | +| anthropic/claude-sonnet-5 | 10 | 36 | 1 | 22% | 2% | 8459730 | unavailable | 0 | 0.000 | +| anthropic/claude-sonnet-5 | 14 | 12 | 0 | 6% | 48% | 7519760 | unavailable | 0 | 0.000 | +| anthropic/claude-sonnet-5 | 14 | 24 | 1 | 6% | 14% | 8342234 | unavailable | 0 | 0.000 | +| anthropic/claude-sonnet-5 | 14 | 36 | 1 | 6% | 2% | 9164707 | unavailable | 0 | 0.000 | + +Token, cost, and elapsed-time fields are diagnostic references only. Missing provider cost or reliable wall-clock data is recorded as unavailable instead of imputed. + +## Decision + +M=10 and P=24 are the smallest tested values at which the replayed stronger pilot retains its verified find. Moving down to M=6 or P=12 loses that find and shows materially higher cap pressure; moving up to M=14 or P=36 adds usage without another verified bug. E=12 and P_solve=10 cover the observed command mix without exposing an unlimited path. S=2 follows the fixed per-rule retry policy. T=8 and E_t=12 bound source-only shortlist formation. Output and watchdog ceilings are safety controls selected above observed pilot maxima and do not affect ranking. + +The public comparison is therefore one run at this single contract, ranked only by verified distinct-rule bugs. Fixed Top50, multiple seeds, a System Track, and a public budget grid remain out of scope. + +## Provenance + +- `32237111e946206e42bb7ee36c54c09eb1d5313c8fcd4d75c2f440e94bee96d4` — anthropic/claude-haiku-4-5, problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8, private legacy pilot trajectory; aggregate bounded-prefix replay retained here +- `ac475bb2cc7c6118f6405b63fc52703f4afc27588f0f4cd021d36328222316d3` — anthropic/claude-sonnet-5, problem-reductions@aa2d1a10cffa434871d12a4d6f411147fb7e08a8, private legacy pilot trajectory; aggregate bounded-prefix replay retained here diff --git a/benchmark/run_mini.py b/benchmark/run_mini.py index 02ba881..ebeda70 100644 --- a/benchmark/run_mini.py +++ b/benchmark/run_mini.py @@ -14,6 +14,8 @@ CONFIG_FILE = Path(__file__).parent / "config.yaml" SKIP_RULES = {"mod", "traits", "graph_helpers", "analysis", "cost", "registry", "graph"} DEFAULT_MAX_TOKENS = 8192 +DEFAULT_MODEL_TIMEOUT_SECONDS = 300 +DEFAULT_MODEL_RETRIES = 2 def list_rules(repo_dir: str | Path) -> list[str]: @@ -80,12 +82,15 @@ def _session_usage(agent): def _build_model(model_name: str, api_base: str | None, max_tokens: int, model_kwargs: dict | None = None, api_key: str | None = None, observation_template: str | None = None, - format_error_template: str | None = None): + format_error_template: str | None = None, + model_timeout_seconds: int = DEFAULT_MODEL_TIMEOUT_SECONDS, + model_retries: int = DEFAULT_MODEL_RETRIES): from minisweagent.models.litellm_model import LitellmModel # A hung API call must fail fast and retry — never freeze a whole-repo session # indefinitely. User-supplied MODEL_KWARGS still wins on any key. - kwargs = {"timeout": 300, "num_retries": 2, **dict(model_kwargs or {})} + kwargs = {"timeout": model_timeout_seconds, + "num_retries": model_retries, **dict(model_kwargs or {})} if max_tokens: kwargs["max_tokens"] = max_tokens if api_base: @@ -101,8 +106,9 @@ def _build_model(model_name: str, api_base: str | None, max_tokens: int, def _load_agent_config(config_path: str | Path | None, default_file: Path, - strategy: str | None) -> tuple[dict, dict, str]: - """Load prompts while force-disabling mini-swe's hidden cost/step limits.""" + strategy: str | None, *, force_unlimited: bool = True + ) -> tuple[dict, dict, str]: + """Load prompts; legacy callers may force-disable mini-swe's hidden step limit.""" config_file = Path(config_path) if config_path else default_file config = yaml.safe_load(config_file.read_text(encoding="utf-8")) if strategy is None: @@ -111,7 +117,8 @@ def _load_agent_config(config_path: str | Path | None, default_file: Path, agent_config = dict(config.get("agent", {})) # 0 means unlimited in mini-swe-agent. The completion command in the prompt is the # normal exit; the runner only retains per-command/process timeouts for wedged tools. - agent_config["step_limit"] = 0 + if force_unlimited: + agent_config["step_limit"] = 0 agent_config["cost_limit"] = 0 return agent_config, config.get("model", {}) or {}, strategy diff --git a/benchmark/run_top50.py b/benchmark/run_top50.py index 97c982d..dbd56a6 100644 --- a/benchmark/run_top50.py +++ b/benchmark/run_top50.py @@ -4,6 +4,7 @@ import argparse import datetime +import hashlib import json import os from pathlib import Path @@ -23,36 +24,80 @@ CONTRACT_ID, RUNNER_VERSION, EXPECTED_EPISODE_BUDGET, + EXPECTED_INFERENCE_PARAMETERS, + EXPECTED_SAFETY_CONTROLS, + EXPECTED_HYPOTHESIS_CHARS, + EXPECTED_SHORTLIST_SIZE, EXPECTED_TRIAGE_BUDGET, expected_prompt_id, ) -def pilot_contract() -> Top50Contract: - """Return explicitly provisional values; issue #68 freezes the public contract.""" +FORBIDDEN_RANKABLE_ENV = ( + "AGENT_BACKEND", "AGENT_CONFIG", "AGENT_STRATEGY_FILE", "SUBMIT_LIMIT", + "PRB_TRIAGE_GENERATIONS", "PRB_TRIAGE_ACTIONS", "PRB_EPISODE_GENERATIONS", + "PRB_EPISODE_ACTIONS", "PRB_PRED_CALLS", "PRB_SOLVE_CALLS", + "PRB_MAX_OUTPUT_CHARS", "PRB_PRED_TIMEOUT_SECONDS", + "EXPECTED_PRED_COMMIT", "EXPECTED_PRED_VERSION", +) +TRUSTED_PRED_PATH = Path("/usr/local/libexec/prb/pred") + + +def verify_rankable_source(repo: Path, expected_commit: str) -> str: + """Verify the baked target marker and every source-rule byte before model access.""" + marker = repo / ".prb-pinned-commit" + manifest = repo / ".prb-source-manifest" + try: + actual_commit = marker.read_text(encoding="utf-8").strip() + lines = manifest.read_text(encoding="utf-8").splitlines() + except OSError as error: + raise ValueError(f"rankable source is missing its baked provenance: {error}") from error + if actual_commit != expected_commit: + raise ValueError("rankable source commit marker differs from the image pin") + expected_files: set[str] = set() + for line in lines: + try: + digest, relative = line.split(maxsplit=1) + except ValueError as error: + raise ValueError("rankable source manifest is malformed") from error + relative = relative.lstrip("*") + candidate = (repo / relative).resolve() + if not candidate.is_relative_to(repo) or not candidate.is_file(): + raise ValueError(f"rankable source manifest has invalid path: {relative}") + if hashlib.sha256(candidate.read_bytes()).hexdigest() != digest: + raise ValueError(f"rankable source differs from the image: {relative}") + expected_files.add(relative) + actual_files = {path.relative_to(repo).as_posix() + for path in (repo / "src").rglob("*") if path.is_file()} + if not expected_files or actual_files != expected_files: + raise ValueError("rankable source-rule inventory differs from the baked manifest") + return actual_commit + + +def verify_rankable_pred_path(binary: Path) -> None: + if binary.resolve() != TRUSTED_PRED_PATH: + raise ValueError(f"rankable runs require the image-owned pred at {TRUSTED_PRED_PATH}") + + +def frozen_contract() -> Top50Contract: + """Return the immutable logical budget for the released rankable track.""" return Top50Contract( - triage=TriageBudget( - model_generations=int(os.environ.get( - "PRB_TRIAGE_GENERATIONS", str(EXPECTED_TRIAGE_BUDGET["model_generations"]))), - shell_actions=int(os.environ.get( - "PRB_TRIAGE_ACTIONS", str(EXPECTED_TRIAGE_BUDGET["shell_actions"]))), - max_output_chars=EXPECTED_TRIAGE_BUDGET["max_output_chars"], - command_timeout_seconds=EXPECTED_TRIAGE_BUDGET["command_timeout_seconds"]), - episode=EvidenceBudget( - model_generations=int(os.environ.get( - "PRB_EPISODE_GENERATIONS", str(EXPECTED_EPISODE_BUDGET["model_generations"]))), - shell_actions=int(os.environ.get( - "PRB_EPISODE_ACTIONS", str(EXPECTED_EPISODE_BUDGET["shell_actions"]))), - pred_calls=int(os.environ.get( - "PRB_PRED_CALLS", str(EXPECTED_EPISODE_BUDGET["pred_calls"]))), - solve_calls=int(os.environ.get( - "PRB_SOLVE_CALLS", str(EXPECTED_EPISODE_BUDGET["solve_calls"]))), - submit_attempts=2, - max_output_chars=int(os.environ.get("PRB_MAX_OUTPUT_CHARS", "10000")), - pred_timeout_seconds=int(os.environ.get("PRB_PRED_TIMEOUT_SECONDS", "300"))), + triage=TriageBudget(**EXPECTED_TRIAGE_BUDGET), + episode=EvidenceBudget(**EXPECTED_EPISODE_BUDGET), + shortlist_size=EXPECTED_SHORTLIST_SIZE, + hypothesis_chars=EXPECTED_HYPOTHESIS_CHARS, ) +def validate_rankable_settings(*, model_kwargs: dict | None = None) -> None: + """Reject every knob that could change the frozen rankable execution path.""" + present = [name for name in FORBIDDEN_RANKABLE_ENV if name in os.environ] + if present: + raise ValueError(f"rankable Top50 runs reject custom setting(s): {', '.join(present)}") + if model_kwargs is not None: + raise ValueError("rankable Top50 runs do not accept custom model kwargs") + + class _FakeExecutor: def run_triage(self, session, *, repo_path, inventory, model): payload = session.workdir / "shortlist.json" @@ -67,19 +112,23 @@ def run_episode(self, session, **kwargs): def run(*, model: str, repo_dir: str | Path, output: str | Path, fake: bool = False, api_base: str | None = None, api_key: str | None = None, model_kwargs: dict | None = None) -> dict: + if not fake: + validate_rankable_settings(model_kwargs=model_kwargs) repo = Path(repo_dir).resolve() + expected_commit = pinned_commit() + actual_commit = expected_commit if fake else verify_rankable_source(repo, expected_commit) inventory = list_rules(repo) if len(inventory) < 50: raise ValueError(f"canonical inventory has only {len(inventory)} runnable rules") pred_binary = find_pred_binary() + if not fake: + verify_rankable_pred_path(pred_binary) pred_version = verify_pred_version(pred_binary) - contract = pilot_contract() + contract = frozen_contract() if fake: runner = Top50Runner( executor=_FakeExecutor(), contract=contract, pred_binary=pred_binary) else: - if model_kwargs: - raise ValueError("rankable Top50 runs do not accept custom model kwargs") runner = build_rankable_runner( contract=contract, pred_binary=pred_binary, agent_uid=int(os.environ["PRB_AGENT_UID"]), @@ -87,17 +136,18 @@ def run(*, model: str, repo_dir: str | Path, output: str | Path, oracle_uid=int(os.environ["PRB_ORACLE_UID"]), oracle_gid=int(os.environ["PRB_ORACLE_GID"]), evidence_gid=int(os.environ["PRB_EVIDENCE_GID"]), - api_base=api_base, api_key=api_key, model_kwargs=model_kwargs) + api_base=api_base, api_key=api_key) metadata = { "benchmark_contract": CONTRACT_ID, - "library_commit": pinned_commit(), + "library_commit": actual_commit, "runner_version": RUNNER_VERSION, "pred_version": pred_version, "agent_mode": AGENT_MODE, "prompt_id": expected_prompt_id(), - "budget_contract_status": "pilot-unfrozen", + "budget_contract_status": "frozen", + "safety_controls": EXPECTED_SAFETY_CONTROLS, "created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "inference_parameters": model_kwargs or {}, + "inference_parameters": EXPECTED_INFERENCE_PARAMETERS, } return runner.run(model=model, repo_path=repo, inventory=inventory, output=output, metadata=metadata) @@ -111,6 +161,7 @@ def main(argv: list[str] | None = None) -> None: parser.add_argument("--api-base", default=os.environ.get("API_BASE")) parser.add_argument("--api-key", default=os.environ.get("API_KEY")) parser.add_argument("--model-kwargs", default=os.environ.get("MODEL_KWARGS")) + parser.add_argument("--preflight", action="store_true") parser.add_argument("--fake", action="store_true", default=bool(os.environ.get("FAKE"))) args = parser.parse_args(argv) if not args.model: @@ -118,6 +169,20 @@ def main(argv: list[str] | None = None) -> None: kwargs = json.loads(args.model_kwargs) if args.model_kwargs else None if kwargs is not None and not isinstance(kwargs, dict): parser.error("--model-kwargs must be a JSON object") + if args.preflight: + from benchmark.preflight import format_report, run_checks + try: + validate_rankable_settings(model_kwargs=kwargs) + except ValueError as error: + parser.error(str(error)) + try: + verify_rankable_source(Path(args.repo_dir).resolve(), pinned_commit()) + verify_rankable_pred_path(find_pred_binary()) + except (OSError, ValueError, RuntimeError) as error: + parser.error(str(error)) + checks = run_checks(args.model, repo_dir=args.repo_dir, api_base=args.api_base, + api_key=args.api_key, model_kwargs=None) + raise SystemExit(0 if format_report(checks) else 1) result = run(model=args.model, repo_dir=args.repo_dir, output=args.output, fake=args.fake, api_base=args.api_base, api_key=args.api_key, model_kwargs=kwargs) diff --git a/benchmark/tests/test_budget_calibration.py b/benchmark/tests/test_budget_calibration.py new file mode 100644 index 0000000..75e57fb --- /dev/null +++ b/benchmark/tests/test_budget_calibration.py @@ -0,0 +1,108 @@ +"""Release-gate tests for frozen Top50 calibration evidence.""" +from __future__ import annotations + +import copy +import json +from dataclasses import asdict + +from benchmark import calibrate_budget +from benchmark import run_top50 + + +def _evidence() -> dict: + return calibrate_budget.load_json( + calibrate_budget.ROOT / "docs" / "budget-calibration.json") + + +def _contract() -> dict: + return calibrate_budget.load_json(calibrate_budget.CONTRACT_PATH) + + +def test_checked_in_calibration_is_self_consistent(): + assert calibrate_budget.check( + calibrate_budget.ROOT / "docs" / "budget-calibration.json") == [] + + +def test_changed_selected_limit_is_rejected(): + evidence = _evidence() + evidence["selected_contract"]["episode"]["pred_calls"] = 25 + assert "does not exactly match" in " ".join( + calibrate_budget.validate_evidence(evidence, _contract())) + + +def test_deleted_grid_candidate_is_rejected(): + evidence = _evidence() + evidence["observations"] = [item for item in evidence["observations"] + if not (item["model"] == "anthropic/claude-haiku-4-5" + and item["candidate"]["model_generations"] == 6 + and item["candidate"]["pred_calls"] == 12)] + assert "complete candidate grid" in " ".join( + calibrate_budget.validate_evidence(evidence, _contract())) + + +def test_report_mismatch_is_rejected(tmp_path, monkeypatch): + evidence_path = tmp_path / "evidence.json" + evidence_path.write_text(json.dumps(_evidence())) + report = tmp_path / "report.md" + report.write_text("stale") + monkeypatch.setattr(calibrate_budget, "REPORT_PATH", report) + assert "does not match" in " ".join(calibrate_budget.check(evidence_path)) + + +def test_smaller_and_larger_candidates_are_required(): + evidence = copy.deepcopy(_evidence()) + evidence["observations"] = [item for item in evidence["observations"] + if item["candidate"]["model_generations"] >= 10] + assert "selected M is not surrounded" in " ".join( + calibrate_budget.validate_evidence(evidence, _contract())) + + +def test_observation_must_link_to_source_and_have_numeric_counts(): + evidence = _evidence() + evidence["observations"][0]["token_reference"] = "0" * 64 + evidence["observations"][1]["cap_hits"]["pred_calls"] = -1 + errors = " ".join(calibrate_budget.validate_evidence(evidence, _contract())) + assert "not linked to a declared source" in errors + assert "cap_hits is invalid" in errors + + +def test_development_artifact_observation_is_derived_offline(tmp_path): + artifact = { + "calibration_status": "non-ranking-development", + "model": "internal/model", "library_commit": "abc", + "contract": {"episode": {"model_generations": 6, "pred_calls": 12}}, + "episodes": [{ + "status": "bug_found", + "usage": {"input": 10, "output": 2, "cache_read": 3, "cache_write": 1}, + "ledger": { + "status": {"model_generations": {"used": 6, "limit": 6}, + "pred_calls": {"used": 5, "limit": 12}}, + "submit": [{"attempt": 2}], "shell_actions": [{}, {}], "pred": [{}] * 5, + }, + }], + } + path = tmp_path / "pilot.json" + path.write_text(json.dumps(artifact)) + observation = calibrate_budget.observation_from_artifact(path) + assert observation["verified_bugs"] == 1 + assert observation["cap_hits"] == {"model_generations": 1, "pred_calls": 0} + assert observation["usage"]["total_tokens"] == 16 + assert observation["retries"] is None + + +def test_frozen_contract_is_referenced_across_release_surfaces(): + contract = _contract() + runtime = asdict(run_top50.frozen_contract()) + assert runtime == {key: contract[key] for key in + ("triage", "episode", "shortlist_size", "hypothesis_chars")} + root = calibrate_budget.ROOT.parent + for relative in ( + "README.md", "CONTRIBUTING.md", "benchmark/top50_config.yaml", + "benchmark/top50_submission.schema.json", "benchmark/top50_results.schema.json", + "docker/Dockerfile", "site/index.html", ".github/workflows/ci.yml", + ): + text = (root / relative).read_text(encoding="utf-8") + if relative.endswith("ci.yml"): + assert "benchmark.calibrate_budget" in text + else: + assert contract["contract_id"] in text diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index 4d14f1f..fc050b1 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -73,15 +73,15 @@ def test_readme_has_run_locally(self): def test_readme_has_metrics_section(self): t = _text(README) - assert "bugs/ktok" in t or "bugs_per_ktok" in t + assert "verified distinct-rule bugs" in t + assert "equal bug counts are ties" in t def test_readme_lists_current_round_contract(self): text = README.read_text(encoding="utf-8") - benchmark_version = VERSION_FILE.read_text(encoding="utf-8").strip() - assert f"[`{benchmark_version}`](VERSION)" in text + assert "top50-evidence/v1" in text assert PINNED_COMMIT in text assert f"`{PINNED_PRED_VERSION}`" in text - assert "no schema-version field" in text.lower() + assert "elapsed time" in text.lower() class TestGuide: @@ -90,43 +90,43 @@ def test_guide_exists(self): def test_guide_has_certificate_format(self): t = _text(GUIDE) - assert "source" in t and "bundle" in t and "violation" in t + assert "certificate" in t and "pred" in t and "reproducible" in t def test_guide_has_submit_flow(self): # The CLI upload flow: `benchmark.submit` → private store → aggregate on Pages. t = _text(GUIDE) - assert "benchmark.submit" in t and "github pages" in t + assert "benchmark.submit" in t and "aggregate leaderboard" in t class TestBackendRouteSeparation: def test_env_template_does_not_select_cli_with_agent_backend(self): t = _text(ENV_EXAMPLE) assert "agent_backend=" not in t - assert "local_backend=codex" in t + assert "model api only" in t + assert "coding-agent" in t and "cannot enter the top50 table" in t def test_api_skill_is_container_only(self): t = _text(API_SKILL) - assert "mini-swe" in t and "runner-pull" in t - assert "never run one inside the container" in t + assert "top50-evidence/v1" in t and "runner-pull" in t + assert "coding-agent backends" in t def test_cli_skill_is_host_only(self): t = _text(CLI_SKILL) assert "make run-local" in t and "local_backend" in t - assert "do not set `agent_backend`" in t - assert "or start docker/podman" in t + assert "legacy-whole-repo" in t and "non-ranking" in t - @pytest.mark.parametrize("skill", [API_SKILL, CLI_SKILL]) + @pytest.mark.parametrize("skill", [API_SKILL]) def test_each_skill_exposes_only_local_and_official_goals(self, skill): t = _text(skill) - assert "keep and validate" in t - assert "official submission" in t + assert "local" in t + assert "official" in t assert "intake test" not in t - @pytest.mark.parametrize("skill", [API_SKILL, CLI_SKILL]) + @pytest.mark.parametrize("skill", [API_SKILL]) def test_run_skills_delegate_upload(self, skill): t = _text(skill) assert "$submit-benchmark-result" in t - assert "that skill owns validation" in t + assert "owns upload" in t @pytest.mark.parametrize("skill", [API_SKILL, CLI_SKILL]) def test_run_skills_confirm_the_benchmark_version(self, skill): diff --git a/benchmark/tests/test_run_top50.py b/benchmark/tests/test_run_top50.py index a69211d..cf57e13 100644 --- a/benchmark/tests/test_run_top50.py +++ b/benchmark/tests/test_run_top50.py @@ -2,6 +2,7 @@ from __future__ import annotations import json +import hashlib import pytest @@ -27,10 +28,52 @@ def test_fake_entrypoint_uses_50_isolated_episodes(tmp_path, monkeypatch): assert len(result["shortlist"]) == 50 assert len(result["episodes"]) == 50 assert result["rankable"] is False - assert json.loads(output.read_text())["budget_contract_status"] == "pilot-unfrozen" + artifact = json.loads(output.read_text()) + assert artifact["budget_contract_status"] == "frozen" + assert artifact["benchmark_contract"] == "top50-evidence/v1" @pytest.mark.parametrize("forbidden", ["--backend", "--config", "--strategy-file"]) def test_standard_entrypoint_rejects_custom_harness_options(forbidden): with pytest.raises(SystemExit): run_top50.main(["--model", "fake/model", "--fake", forbidden, "custom"]) + + +@pytest.mark.parametrize("name", run_top50.FORBIDDEN_RANKABLE_ENV) +def test_rankable_preflight_rejects_custom_execution_before_model_call(name, monkeypatch): + monkeypatch.setenv(name, "custom") + with pytest.raises(ValueError, match="reject custom"): + run_top50.validate_rankable_settings() + + +def test_rankable_contract_cannot_be_changed_by_budget_environment(monkeypatch): + monkeypatch.setenv("PRB_PRED_CALLS", "999") + assert run_top50.frozen_contract().episode.pred_calls == 24 + + +def test_even_empty_custom_model_kwargs_are_rejected(monkeypatch): + for name in run_top50.FORBIDDEN_RANKABLE_ENV: + monkeypatch.delenv(name, raising=False) + with pytest.raises(ValueError, match="custom model kwargs"): + run_top50.validate_rankable_settings(model_kwargs={}) + + +def test_top50_executor_keeps_finite_agent_step_guard(): + from benchmark.top50_runner import MiniSwePhaseExecutor + executor = MiniSwePhaseExecutor() + assert executor.agent_config["step_limit"] == 10 + + +def test_rankable_source_manifest_detects_modified_rule(tmp_path): + repo = tmp_path / "repo" + rule = repo / "src" / "rules" / "r.rs" + rule.parent.mkdir(parents=True) + rule.write_text("original") + (repo / ".prb-pinned-commit").write_text("abc\n") + digest = hashlib.sha256(rule.read_bytes()).hexdigest() + (repo / ".prb-source-manifest").write_text(f"{digest} src/rules/r.rs\n") + assert run_top50.verify_rankable_source(repo, "abc") == "abc" + + rule.write_text("modified") + with pytest.raises(ValueError, match="differs from the image"): + run_top50.verify_rankable_source(repo, "abc") diff --git a/benchmark/tests/test_top50_submission.py b/benchmark/tests/test_top50_submission.py index 110714c..2014ad4 100644 --- a/benchmark/tests/test_top50_submission.py +++ b/benchmark/tests/test_top50_submission.py @@ -97,9 +97,11 @@ def _artifact(accepted_positions=(7, 18, 41)) -> dict: "pred_version": "0.6.0", "agent_mode": AGENT_MODE, "prompt_id": expected_prompt_id(), + "budget_contract_status": "frozen", + "safety_controls": {"model_timeout_seconds": 300, "model_retries": 2}, "status": "completed", "rankable": True, - "inference_parameters": {}, + "inference_parameters": {"max_tokens": 8192, "timeout": 300, "num_retries": 2}, "contract": {"triage": triage_budget, "episode": episode_budget, "shortlist_size": 50, "hypothesis_chars": 500}, "shortlist": shortlist, diff --git a/benchmark/top50_budget.json b/benchmark/top50_budget.json new file mode 100644 index 0000000..2818800 --- /dev/null +++ b/benchmark/top50_budget.json @@ -0,0 +1,30 @@ +{ + "contract_id": "top50-evidence/v1", + "status": "frozen", + "shortlist_size": 50, + "hypothesis_chars": 500, + "safety_controls": { + "model_timeout_seconds": 300, + "model_retries": 2 + }, + "inference_parameters": { + "max_tokens": 8192, + "timeout": 300, + "num_retries": 2 + }, + "triage": { + "model_generations": 8, + "shell_actions": 12, + "max_output_chars": 10000, + "command_timeout_seconds": 300 + }, + "episode": { + "model_generations": 10, + "shell_actions": 12, + "pred_calls": 24, + "solve_calls": 10, + "submit_attempts": 2, + "max_output_chars": 10000, + "pred_timeout_seconds": 300 + } +} diff --git a/benchmark/top50_budget.py b/benchmark/top50_budget.py new file mode 100644 index 0000000..25c6af3 --- /dev/null +++ b/benchmark/top50_budget.py @@ -0,0 +1,8 @@ +"""Single loader for the immutable Top50 release contract.""" +from __future__ import annotations + +import json +from pathlib import Path + +BUDGET_PATH = Path(__file__).with_name("top50_budget.json") +FROZEN_CONTRACT: dict = json.loads(BUDGET_PATH.read_text(encoding="utf-8")) diff --git a/benchmark/top50_config.yaml b/benchmark/top50_config.yaml index 4a94829..1092ea3 100644 --- a/benchmark/top50_config.yaml +++ b/benchmark/top50_config.yaml @@ -3,6 +3,8 @@ agent: You are evaluating reduction rules under a fixed logical evidence budget. Each response must contain exactly one bash command. The authoritative remaining budget is injected into every observation; elapsed time is not a score and unused budget never transfers. + This is frozen contract top50-evidence/v1. The prompt, strategy, shortlist size, and logical + counters cannot be customized on a rankable run. {% if phase == "triage" %} This is source-only triage. Read {{repo_dir}}/src/rules but do not run pred or submit. @@ -25,7 +27,7 @@ agent: Finish normally with: echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT instance_template: | {{task}} - step_limit: 0 + step_limit: 10 cost_limit: 0 max_consecutive_format_errors: 3 diff --git a/benchmark/top50_contract.py b/benchmark/top50_contract.py index 8258434..fd35e14 100644 --- a/benchmark/top50_contract.py +++ b/benchmark/top50_contract.py @@ -11,21 +11,19 @@ from pathlib import Path from typing import Callable +from benchmark.top50_budget import FROZEN_CONTRACT from benchmark.usage import Usage, usage_as_dict, usage_from_dict from benchmark.verify import Verdict, verify -CONTRACT_ID = "top50-evidence/pilot-v1" +CONTRACT_ID = FROZEN_CONTRACT["contract_id"] AGENT_MODE = "standardized-model-api" RUNNER_VERSION = "0.9.0" -EXPECTED_TRIAGE_BUDGET = { - "model_generations": 8, "shell_actions": 12, - "max_output_chars": 10_000, "command_timeout_seconds": 300, -} -EXPECTED_EPISODE_BUDGET = { - "model_generations": 10, "shell_actions": 12, "pred_calls": 24, - "solve_calls": 10, "submit_attempts": 2, "max_output_chars": 10_000, - "pred_timeout_seconds": 300, -} +EXPECTED_TRIAGE_BUDGET = FROZEN_CONTRACT["triage"] +EXPECTED_EPISODE_BUDGET = FROZEN_CONTRACT["episode"] +EXPECTED_SAFETY_CONTROLS = FROZEN_CONTRACT["safety_controls"] +EXPECTED_INFERENCE_PARAMETERS = FROZEN_CONTRACT["inference_parameters"] +EXPECTED_SHORTLIST_SIZE = FROZEN_CONTRACT["shortlist_size"] +EXPECTED_HYPOTHESIS_CHARS = FROZEN_CONTRACT["hypothesis_chars"] _REQUEST_ID = re.compile(r"[0-9a-f]{32}") _COUNTERS = ("model_generations", "shell_actions", "pred_calls", "solve_calls") @@ -48,7 +46,8 @@ def validate_top50_submission( problems: list[str] = [] for field in ("benchmark_contract", "model", "library_commit", "runner_version", "pred_version", "agent_mode", "prompt_id", "contract", "shortlist", - "triage", "episodes", "status"): + "triage", "episodes", "status", "budget_contract_status", + "safety_controls", "inference_parameters"): if field not in submission: problems.append(f"missing required field: {field}") if problems: @@ -59,6 +58,10 @@ def validate_top50_submission( problems.append(f"agent_mode must be {AGENT_MODE!r}") if submission["runner_version"] != RUNNER_VERSION: problems.append(f"runner_version must be {RUNNER_VERSION!r}") + if submission.get("budget_contract_status") != "frozen": + problems.append("budget_contract_status must be 'frozen'") + if submission.get("safety_controls") != EXPECTED_SAFETY_CONTROLS: + problems.append("safety_controls differ from the released watchdog settings") if submission.get("status") != "completed" or submission.get("run_error"): problems.append("Top50 run is incomplete or has run_error") if submission.get("rankable") is not True: @@ -70,8 +73,8 @@ def validate_top50_submission( if (submitter is not None and (not isinstance(submitter, str) or not re.fullmatch(r"[A-Za-z0-9_.@-]{1,100}", submitter))): problems.append("submitted_by is not a bounded safe identifier") - if not isinstance(submission.get("inference_parameters"), dict): - problems.append("inference_parameters must be a normalized object") + if submission.get("inference_parameters") != EXPECTED_INFERENCE_PARAMETERS: + problems.append("inference_parameters differ from the standardized model settings") if submission.get("prompt_id") != expected_prompt_id(): problems.append("prompt_id does not match the frozen Top50 prompt") if "submit_limit" in submission or "submit_log" in submission: @@ -88,12 +91,14 @@ def validate_top50_submission( problems.append("triage budget differs from the versioned contract") if episode_budget != EXPECTED_EPISODE_BUDGET: problems.append("episode budget differs from the versioned contract") - if contract.get("shortlist_size") != 50: - problems.append("contract shortlist_size must be 50") + if contract.get("shortlist_size") != EXPECTED_SHORTLIST_SIZE: + problems.append(f"contract shortlist_size must be {EXPECTED_SHORTLIST_SIZE}") hypothesis_limit = contract.get("hypothesis_chars") if not _nonnegative_int(hypothesis_limit): problems.append("contract hypothesis_chars must be a non-negative integer") hypothesis_limit = -1 + elif hypothesis_limit != EXPECTED_HYPOTHESIS_CHARS: + problems.append("contract hypothesis_chars differs from the versioned contract") if episode_budget.get("submit_attempts") != 2: problems.append("episode submit_attempts must be exactly 2") pred_limit = episode_budget.get("pred_calls") @@ -104,8 +109,9 @@ def validate_top50_submission( problems.append("solve_calls limit exceeds pred_calls") shortlist = submission.get("shortlist") - if not isinstance(shortlist, list) or len(shortlist) != 50: - return problems + ["shortlist must contain exactly 50 entries"] + if not isinstance(shortlist, list) or len(shortlist) != EXPECTED_SHORTLIST_SIZE: + return problems + [ + f"shortlist must contain exactly {EXPECTED_SHORTLIST_SIZE} entries"] rules: list[str] = [] for index, entry in enumerate(shortlist): if not isinstance(entry, dict) or not isinstance(entry.get("rule"), str): diff --git a/benchmark/top50_results.schema.json b/benchmark/top50_results.schema.json index 7072eee..51da291 100644 --- a/benchmark/top50_results.schema.json +++ b/benchmark/top50_results.schema.json @@ -8,7 +8,7 @@ "first_attempt_accepts", "second_attempt_accepts", "usage_totals", "artifact_sha256", "verifier_report"], "properties": { - "benchmark_contract": {"const": "top50-evidence/pilot-v1"}, + "benchmark_contract": {"const": "top50-evidence/v1"}, "model": {"type": "string"}, "library_commit": {"type": "string"}, "rankable": {"type": "boolean"}, diff --git a/benchmark/top50_runner.py b/benchmark/top50_runner.py index be46c38..eb74cb9 100644 --- a/benchmark/top50_runner.py +++ b/benchmark/top50_runner.py @@ -21,6 +21,7 @@ _message_text, _session_usage, ) +from benchmark.top50_budget import FROZEN_CONTRACT from benchmark.usage import Usage, usage_as_dict from benchmark.verify import Verdict, verify @@ -341,7 +342,6 @@ def build_rankable_runner( *, contract: Top50Contract, pred_binary: str | Path, agent_uid: int, agent_gid: int, oracle_uid: int, oracle_gid: int, evidence_gid: int, api_base: str | None = None, api_key: str | None = None, - max_tokens: int = DEFAULT_MAX_TOKENS, model_kwargs: dict | None = None, verifier: Callable[[dict], Verdict] = verify, ) -> Top50Runner: """Construct the sole rankable harness after proving the required OS boundary.""" @@ -349,10 +349,13 @@ def build_rankable_runner( raise RuntimeError("rankable Top50 runs require the root runner privilege boundary") if len({agent_uid, oracle_uid}) != 2: raise ValueError("agent and oracle must use distinct identities") + inference = FROZEN_CONTRACT["inference_parameters"] + safety = FROZEN_CONTRACT["safety_controls"] executor = MiniSwePhaseExecutor( - api_base=api_base, api_key=api_key, max_tokens=max_tokens, - model_kwargs=model_kwargs, agent_uid=agent_uid, agent_gid=agent_gid, - evidence_gid=evidence_gid) + api_base=api_base, api_key=api_key, max_tokens=inference["max_tokens"], + model_timeout_seconds=safety["model_timeout_seconds"], + model_retries=safety["model_retries"], model_kwargs=None, + agent_uid=agent_uid, agent_gid=agent_gid, evidence_gid=evidence_gid) return Top50Runner( executor=executor, contract=contract, pred_binary=pred_binary, verifier=verifier, agent_uid=agent_uid, agent_gid=agent_gid, oracle_uid=oracle_uid, @@ -442,18 +445,21 @@ class MiniSwePhaseExecutor: def __init__(self, *, api_base: str | None = None, api_key: str | None = None, max_tokens: int = DEFAULT_MAX_TOKENS, model_kwargs: dict | None = None, + model_timeout_seconds: int = 300, model_retries: int = 2, agent_uid: int | None = None, agent_gid: int | None = None, evidence_gid: int | None = None): self.api_base = api_base self.api_key = api_key self.max_tokens = max_tokens self.model_kwargs = model_kwargs + self.model_timeout_seconds = model_timeout_seconds + self.model_retries = model_retries self.agent_uid = os.getuid() if agent_uid is None else agent_uid self.agent_gid = os.getgid() if agent_gid is None else agent_gid self.evidence_gid = evidence_gid config_path = Path(__file__).with_name("top50_config.yaml") self.agent_config, self.model_config, _ = _load_agent_config( - config_path, config_path, "") + config_path, config_path, "", force_unlimited=False) self._models: dict[str, object] = {} def run_triage(self, session: TriageSession, *, repo_path: Path, @@ -492,7 +498,9 @@ def _run_agent(self, model_name: str, environment, session, *, task: str, model_name, self.api_base, self.max_tokens, model_kwargs=self.model_kwargs, api_key=self.api_key, observation_template=self.model_config.get("observation_template"), - format_error_template=self.model_config.get("format_error_template")) + format_error_template=self.model_config.get("format_error_template"), + model_timeout_seconds=self.model_timeout_seconds, + model_retries=self.model_retries) self._models[model_name] = model class BudgetedAgent(DefaultAgent): diff --git a/benchmark/top50_submission.schema.json b/benchmark/top50_submission.schema.json index 1f61091..879c69c 100644 --- a/benchmark/top50_submission.schema.json +++ b/benchmark/top50_submission.schema.json @@ -4,10 +4,10 @@ "title": "Top50EvidenceSubmission", "type": "object", "required": ["benchmark_contract", "model", "library_commit", "runner_version", - "pred_version", "agent_mode", "prompt_id", "status", "contract", "shortlist", + "pred_version", "agent_mode", "prompt_id", "status", "budget_contract_status", "safety_controls", "contract", "shortlist", "triage", "episodes", "inference_parameters"], "properties": { - "benchmark_contract": {"const": "top50-evidence/pilot-v1"}, + "benchmark_contract": {"const": "top50-evidence/v1"}, "model": {"type": "string", "minLength": 1}, "library_commit": {"type": "string", "minLength": 1}, "runner_version": {"const": "0.9.0"}, @@ -15,12 +15,21 @@ "agent_mode": {"const": "standardized-model-api"}, "prompt_id": {"type": "string", "minLength": 64, "maxLength": 64}, "status": {"enum": ["completed", "run_error"]}, + "budget_contract_status": {"const": "frozen"}, + "safety_controls": {"const": {"model_timeout_seconds": 300, "model_retries": 2}}, "rankable": {"type": "boolean"}, - "contract": {"type": "object"}, + "contract": { + "const": { + "triage": {"model_generations": 8, "shell_actions": 12, "max_output_chars": 10000, "command_timeout_seconds": 300}, + "episode": {"model_generations": 10, "shell_actions": 12, "pred_calls": 24, "solve_calls": 10, "submit_attempts": 2, "max_output_chars": 10000, "pred_timeout_seconds": 300}, + "shortlist_size": 50, + "hypothesis_chars": 500 + } + }, "shortlist": {"type": "array", "minItems": 50, "maxItems": 50}, "triage": {"type": "object"}, "episodes": {"type": "array", "minItems": 50, "maxItems": 50}, - "inference_parameters": {"type": "object"}, + "inference_parameters": {"const": {"max_tokens": 8192, "timeout": 300, "num_retries": 2}}, "run_error": {"type": "string"} } } diff --git a/docker/Dockerfile b/docker/Dockerfile index 9d4b2c5..38e4973 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -29,6 +29,7 @@ RUN git clone --depth 1 --branch "${PR_REF}" \ # Capture the exact commit that PR_REF resolved to — baked into the image so the runtime # records/verifies the real target version instead of a hardcoded constant. RUN git rev-parse HEAD > /src/PINNED_COMMIT +RUN find src -type f -print0 | sort -z | xargs -0 sha256sum > /src/SOURCE_MANIFEST # The upstream release profile uses LTO + codegen-units=1, which OOMs in a # constrained Docker VM. Relax it for the build (pred's solve perf is ample # without LTO). The rustc job cap is a build ARG: default 1 keeps peak memory low @@ -52,7 +53,8 @@ ARG PR_REF LABEL org.opencontainers.image.title="problem-reductions-benchmark" \ org.opencontainers.image.description="pred (problem-reductions ${PR_REF}) + zero-trust certificate verifier" \ org.opencontainers.image.licenses="MIT" \ - problem_reductions_ref="${PR_REF}" + problem_reductions_ref="${PR_REF}" \ + benchmark_contract="top50-evidence/v1" RUN groupadd --gid 10003 prb-evidence \ && groupadd --gid 10001 prb-agent \ @@ -86,6 +88,8 @@ RUN /usr/local/libexec/prb/pred --version # Bake the target version pins so benchmark/env_setup.py reads the *built* commit/version # rather than a frozen Python literal (the target tracks PR_REF, not a constant). COPY --from=pred-build /src/PINNED_COMMIT /app/benchmark/PINNED_COMMIT +COPY --from=pred-build /src/PINNED_COMMIT /app/pr-src/.prb-pinned-commit +COPY --from=pred-build /src/SOURCE_MANIFEST /app/pr-src/.prb-source-manifest RUN /usr/local/libexec/prb/pred --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -n1 > /app/benchmark/PINNED_VERSION # Prove the complete privilege boundary, not just a file mode: the dedicated agent user can # call both shims while an absolute-path oracle invocation is denied. @@ -112,14 +116,6 @@ FROM runtime AS runner ARG PR_REF # Agent stack. requirements.txt pins mini-swe-agent (which pulls litellm) + anthropic. RUN pip install --no-cache-dir -r /app/benchmark/requirements.txt -# Claude Code CLI — the optional claude-code backend (AGENT_BACKEND=claude-code). -# Native installer, self-contained (no Node). Auth is supplied at run time via -# CLAUDE_CODE_OAUTH_TOKEN or ANTHROPIC_API_KEY in submission.env — never baked in. -RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates \ - && rm -rf /var/lib/apt/lists/* \ - && curl -fsSL https://claude.ai/install.sh | bash -ENV PATH="/root/.local/bin:${PATH}" -RUN claude --version RUN mkdir -p /out ENV OUTPUT=/out/submission.json VOLUME ["/out"] diff --git a/site/README.md b/site/README.md index b521153..494a70c 100644 --- a/site/README.md +++ b/site/README.md @@ -1,8 +1,9 @@ # Leaderboard site Static leaderboard (`index.html`, no app server) published to **GitHub Pages** by -`.github/workflows/publish-on-merge.yml`. Every model gets one whole-repository session and -the same bounded certificate-submission channel; every bug is re-verified by `pred`. +`.github/workflows/publish-on-merge.yml`. The primary table gives every model the same frozen +Self-selected Top50 logical evidence budget and bounded certificate-submission channel; +every bug is re-verified by `pred`. `index.html` reads two data files served alongside it: diff --git a/site/index.html b/site/index.html index a6f6be4..f7e549d 100644 --- a/site/index.html +++ b/site/index.html @@ -259,8 +259,8 @@

Leaderboard

library v0.6.0
- 100 attempts / model - 253 rules + 50 rules / model + 2 submits / selected rule
@@ -275,7 +275,7 @@

Leaderboard

Click a row for its run stats →
-

Bugs = distinct rules with a confirmed counterexample. Submitted rules = rules sent through the bounded submit command. Bugs / Ktok = efficiency tie-break.

+

Primary Top50 metric = verified distinct-rule bugs under the frozen logical evidence budget. Ties remain ties; tokens, cost, and elapsed time are diagnostics only. Select the historical contract to view old whole-repository results separately.

@@ -296,16 +296,16 @@

Leaderboard

From your model to the leaderboard

-

Give a model one whole-repository session and see how many bugs it can find in the library's - problem reductions. You run it on your own model and key; we independently re-check every bug - before it scores.

+

The model first freezes the 50 rules it considers most bug-prone, then investigates each + rule in a fresh isolated episode with the same logical budget. You provide the API key; we + independently re-check every submitted bug before it scores.

You run it
1

Get the runner

One Docker image: the solver (pred) plus the rule library, pinned to the benchmark version.

2

Plug in any model

OpenAI, Anthropic, DeepSeek, a self-hosted endpoint — you bring the API key.

-
3

Hunt for bugs

Your model chooses which rules to probe and when to stop; each submitted find is checked by the solver on the spot.

a bug = solve(x) ≠ solve(reduce(x))
+
3

Choose 50, then investigate

Source-only triage freezes a Top50. Each rule then gets 10 model generations, 12 shell actions, 24 pred calls (10 solves), and exactly two submit attempts.

a bug = solve(x) ≠ solve(reduce(x))
4

Submit from the command line

Upload submission.json with benchmark.submit. No web form.

@@ -314,7 +314,7 @@

From your model to the leaderboard

We score it
5

Re-verified from scratch

We re-derive every claimed bug with our own solver and keep only the ones that reproduce.

only reproducible bugs are scored
-
✓

Ranked on the leaderboard

Models are ordered by confirmed distinct-rule bugs, then bugs per 1K tokens as the efficiency tie-break.

+
✓

Ranked on the leaderboard

Models are ordered only by confirmed distinct-rule bugs at the same frozen contract. Equal bug counts are tied.

@@ -324,7 +324,7 @@

What counts as a bug

Why it is fair

-

Every model targets the same repository version and gets the same 100-attempt certificate channel. Agents stop themselves, and real token usage is recorded. Ranking is by confirmed distinct-rule bugs, with bugs per 1K tokens as the efficiency tie-break.

+

Every model uses the same target, prompt, Top50 size, per-rule logical counters, and two submit opportunities. Wall-clock time, network delay, tokens, and cost never break ties.

@@ -356,7 +356,7 @@

Upload it

About the benchmark

-

An open measurement of how many bugs a model can find in the problem-reductions library. Every model gets one whole-repository session with the same bounded submission channel, and every claimed bug is independently re-verified server-side.

+

A controlled measurement of how well a model prioritizes likely-buggy rules and turns bounded evidence into verified counterexamples. Every claimed bug is independently re-verified server-side.

How to cite

@misc{TODO_citation_key,
@@ -398,7 +398,7 @@ 
How to cite

   const meta=document.querySelector(".meta"); if(meta)meta.style.display=b.dataset.tab==="leaderboard"?"":"none";
 });
 
-const LB_COLS=[
+const LEGACY_COLS=[
   {key:"rank",label:"#"},
   {key:"model",label:"Model"},
   {key:"bugs",label:"Bugs",num:true},
@@ -406,10 +406,22 @@ 
How to cite

   {key:"tokens",label:"Tokens (K)",num:true,sortBy:"tokensNum"},
   {key:"eff",label:"Bugs / Ktok",num:true,sortBy:"effNum"},
 ];
-const RIGHT=new Set(["bugs","tokens","eff"]);
+const TOP50_COLS=[
+  {key:"rank",label:"#"},
+  {key:"model",label:"Model"},
+  {key:"bugs",label:"Verified bugs",num:true},
+  {key:"bugs10",label:"Bugs @10",num:true},
+  {key:"bugs25",label:"Bugs @25",num:true},
+  {key:"attempts",label:"1st / 2nd accepts",sortBy:"firstAccepts"},
+  {key:"predBug",label:"Pred / bug",num:true},
+];
+const RIGHT=new Set(["bugs","tokens","eff","bugs10","bugs25","predBug"]);
+const selectedTrack=()=>document.getElementById("lb-track").value||"top50-evidence/v1";
+const isTop50Track=()=>selectedTrack().startsWith("top50-evidence/");
+const lbColumns=()=>isTop50Track()?TOP50_COLS:LEGACY_COLS;
 
 function buildLbRows(){
-  const track=document.getElementById("lb-track").value||"legacy-whole-repo";
+  const track=selectedTrack();
   const top50=track.startsWith("top50-evidence/");
   const ranked=RESULTS.filter(r=>(r.benchmark_contract||"legacy-whole-repo")===track)
     .sort((a,b)=>(b.bugs_found||0)-(a.bugs_found||0)||
@@ -422,11 +434,14 @@ 
How to cite

     const rank=top50&&i&&bugs===lastBugs?lastRank:i+1;lastBugs=bugs;lastRank=rank;
     return {rank,model:short(r.model),_model:r.model,bugs,
       reachFrac:frac,tested,tokensNum:r.total_tokens_k||0,
-      effNum:r.efficiency_bugs_per_ktok||0};
+      effNum:r.efficiency_bugs_per_ktok||0,
+      bugs10:r.bugs_at_10||0,bugs25:r.bugs_at_25||0,
+      firstAccepts:r.first_attempt_accepts||0,secondAccepts:r.second_attempt_accepts||0,
+      predBug:r.pred_calls_per_bug==null?0:r.pred_calls_per_bug};
   });
 }
 function renderLbHead(){
-  document.getElementById("lb-head").innerHTML=LB_COLS.map(c=>{
+  document.getElementById("lb-head").innerHTML=lbColumns().map(c=>{
     const sk=c.sortBy||c.key, on=lbSort.key===sk, ind=on?(lbSort.dir<0?"↓":"↑"):"";
     const cls=[on?"sorted":"",RIGHT.has(c.key)?"n":""].filter(Boolean).join(" ");
     return `${c.label}${ind}`;
@@ -440,6 +455,11 @@ 
How to cite

     if(typeof av==="number"&&typeof bv==="number")return (av-bv)*d;
     return String(av).localeCompare(String(bv))*d;});
   document.getElementById("lb-body").innerHTML=rows.map(r=>{
+    if(isTop50Track())return `
+      ${r.rank}${esc(r.model)}
+      ${r.bugs}
+      ${r.bugs10}${r.bugs25}
+      ${r.firstAccepts} / ${r.secondAccepts}${r.predBug||'—'}`;
     const pct=Math.round(r.reachFrac*100);
     return `
       ${r.rank}
@@ -449,7 +469,7 @@ 
How to cite

       ${Math.round(r.tokensNum).toLocaleString()}
       ${r.effNum.toFixed(4)}
     `;
-  }).join("")||`No models match.`;
+  }).join("")||`No models match.`;
 }
 let scatterPts=[];
 function hexA(hex,a){hex=String(hex||'').replace('#','');if(hex.length===3)hex=hex.split('').map(c=>c+c).join('');const n=parseInt(hex||'2563eb',16);return `rgba(${(n>>16)&255},${(n>>8)&255},${n&255},${a})`;}
@@ -536,11 +556,17 @@ 
How to cite

   const row=lbRows.find(r=>r._model===model); if(!row)return;
   document.getElementById("drawer-title").textContent=row.model;
   const sub=document.getElementById("drawer-sub"), el=document.getElementById("inspect-body");
-  sub.textContent=`${row.bugs} confirmed bug(s) · ${row.tested}/${TOTAL} rules submitted · ${Math.round(row.tokensNum)}K tok`;
+  sub.textContent=isTop50Track()?`${row.bugs} verified bug(s) · frozen Self-selected Top50`:
+    `${row.bugs} confirmed bug(s) · ${row.tested}/${TOTAL} rules submitted · ${Math.round(row.tokensNum)}K tok`;
   // Aggregate-only by design: the leaderboard never reveals WHICH rules a model found bugs
   // in. On a public library commit a pred-confirmed certificate counts regardless of who
   // produced it, so publishing the rules/certificates would be a free answer key.
-  el.innerHTML=`

+  el.innerHTML=isTop50Track()?`

+    
Verified bugs
${row.bugs}

+    
Bugs @10 / @25
${row.bugs10} / ${row.bugs25}

+    
1st / 2nd accepts
${row.firstAccepts} / ${row.secondAccepts}

+    
Pred calls / bug
${row.predBug||'—'}

+  
`:`

     
Confirmed bugs
${row.bugs}

     
Rules reached
${row.tested}/${TOTAL}

     
Tokens
${Math.round(row.tokensNum).toLocaleString()}K

@@ -582,7 +608,7 @@ 
How to cite

 document.addEventListener("keydown",e=>{if(e.key==="Escape")closeDrawer();});
 document.getElementById("task-search").addEventListener("input",renderTasks);
 document.getElementById("lb-track").addEventListener("change",()=>{
-  buildLbRows();renderLbBody();
+  lbSort={key:"bugs",dir:-1};buildLbRows();renderLbHead();renderLbBody();
 });
 document.querySelectorAll("#task-table thead th").forEach(th=>th.addEventListener("click",()=>{
   const k=th.dataset.k; taskSort={key:k,dir:taskSort.key===k?-taskSort.dir:1}; renderTasks();
@@ -593,7 +619,7 @@ 
How to cite

   fetch("tasks.json").then(r=>r.json()).catch(()=>[]),
 ]).then(([res,tasks])=>{
   RESULTS=res;TASKS=tasks;TOTAL=tasks.length||TOTAL_DEFAULT;
-  const tracks=[...new Set(RESULTS.map(r=>r.benchmark_contract||"legacy-whole-repo"))].sort();
+  const tracks=[...new Set(["top50-evidence/v1", ...RESULTS.map(r=>r.benchmark_contract||"legacy-whole-repo")])].sort();
   const trackSelect=document.getElementById("lb-track");
   trackSelect.innerHTML=tracks.map(t=>``).join("");
   const newest=tracks.find(t=>t.startsWith("top50-evidence/"));if(newest)trackSelect.value=newest;
diff --git a/submission.env.example b/submission.env.example
index 0188590..06fb817 100644
--- a/submission.env.example
+++ b/submission.env.example
@@ -2,8 +2,8 @@
 # Submission runner config. Copy to `submission.env`, then choose exactly one route:
 #
 #   Model API in Docker:  make runner-pull && make preflight && make run
-#   Coding-agent on host: make run-local LOCAL_BACKEND=codex \
-#     LOCAL_REPO_DIR=... LOCAL_OUTPUT=... LOCAL_LOG_DIR=...
+# The public rankable track is Model API only. Coding-agent/whole-repository runners are
+# retained solely for historical reproduction and cannot enter the Top50 table.
 #
 # Do not run Codex, Claude Code, or another coding-agent CLI through Docker. API runs use
 # mini-swe/LiteLLM; CLI runs select their harness only with the Make variable LOCAL_BACKEND.
@@ -23,46 +23,23 @@ MODEL_NAME=openai/gpt-5.4
 # Nothing is hardcoded to anthropic/openai. Reach any model with these:
 # API_BASE=https://my-gateway.example/v1
 # API_KEY=...          # generic key when no PROVIDER_API_KEY name fits
-# MODEL_KWARGS={"custom_llm_provider":"openai","extra_headers":{"X-Org":"acme"}}
-#   # ^ JSON of extra litellm.completion kwargs (api_version, temperature, …)
 # MSWEA_COST_TRACKING=ignore_errors
 #   # ^ required when your model isn't in litellm's price map (typical for custom
 #   #   endpoints) — mini-swe otherwise aborts the session on the cost-calculation
 #   #   error. Token usage is still recorded from API responses.
 
-# ── LIMITS ────────────────────────────────────────────────────────────────────
-# MAX_TOKENS=8192      # per-call output ceiling
-# SUBMIT_LIMIT=100      # counterexample submissions across the complete run; accepted and
-#                       # rejected calls both consume one attempt
-
-# ── CODING-AGENT CLI AUTH (HOST `make run-local` ONLY) ───────────────────────
-# Select the harness on the make command line: LOCAL_BACKEND=codex or
-# LOCAL_BACKEND=claude-code. LOCAL_BACKEND is not an entry in this env file.
-# For claude-code, auth comes from ONE of (the litellm key plumbing above doesn't apply):
-# CLAUDE_CODE_OAUTH_TOKEN=...   # Claude subscription — generate with `claude setup-token`
-# ANTHROPIC_API_KEY=...         # or plain API billing
-# MODEL_NAME may be bare (claude-opus-4-8) or anthropic/-prefixed. API_BASE, MODEL_KWARGS
-# and MAX_TOKENS are ignored. Claude Code self-terminates; no turn cap is passed.
-# For codex, use an existing `codex login` or set OPENAI_API_KEY. MODEL_NAME may be bare
-# (gpt-5.4) or openai/-prefixed. The runner uses an ephemeral workspace-write sandbox and
-# ignores ambient Codex config/rules so local customizations do not change the benchmark.
-# OPENAI_API_KEY=...
-# CODEX_SANDBOX=workspace-write
+# ── FROZEN LIMITS ────────────────────────────────────────────────────────────
+# There are no budget environment variables. Rankable runs always use
+# top50-evidence/v1: source-only Top50 triage, then 50 isolated rule episodes with
+# M=10, E=12, P=24, P_solve=10, and exactly S=2 submit attempts per rule.
 
 # ── OUTPUT / LOGS ────────────────────────────────────────────────────────────
-# Docker defaults follow. `make run-local` instead requires LOCAL_OUTPUT and LOCAL_LOG_DIR
-# as separate explicit paths.
+# Docker defaults follow.
 # OUTPUT=/out/submission.json  # authoritative submission output
-# TRAJECTORY_DIR=/out          # where whole-repo persists its raw/final log;
-#                              # default = OUTPUT's directory
-
-# ── CUSTOM AGENT PROMPT (mount the files too: -v "$PWD/cfg:/cfg") ─────────────
-# AGENT_STRATEGY_FILE=/cfg/strategy.md   # extra hints into the prompt's {{strategy}} slot
-# AGENT_CONFIG=/cfg/config.yaml          # full prompt-config replacement
+# Rankable runs reject custom prompts, strategies, backends, and logical budgets.
 
-# ── VERSION PINS (baked into the image from PR_REF; override only to debug) ────
-# EXPECTED_PRED_VERSION=    # empty string disables the version check
-# EXPECTED_PRED_COMMIT=
+# Version pins, source manifest, and the trusted `pred` path are baked into the image.
+# Rankable runs reject pin overrides and custom `pred` binaries.
 
 # ── METADATA ─────────────────────────────────────────────────────────────────
 # SUBMITTED_BY=your-hf-handle   # also fillable at submit time on the Space

From b6cade658a0859246445a5cac9d6b78bf663e9aa Mon Sep 17 00:00:00 2001
From: Xiwei Pan 
Date: Mon, 20 Jul 2026 17:15:49 +0800
Subject: [PATCH 5/9] feat: package high-information observations (#73)

---
 .agents/skills/run-api-benchmark/SKILL.md     |   2 +-
 .agents/skills/run-cli-benchmark/SKILL.md     |   2 +-
 .../skills/submit-benchmark-result/SKILL.md   |   2 +-
 CONTRIBUTING.md                               |   6 +-
 README.md                                     |   8 +-
 benchmark/agent_environment.py                |  65 ++++-
 benchmark/calibrate_budget.py                 |  10 +-
 benchmark/docs/budget-calibration.json        |   7 +-
 benchmark/docs/budget-calibration.md          |   6 +-
 benchmark/evidence_budget.py                  |  67 ++++-
 benchmark/observation_policy.py               | 265 ++++++++++++++++++
 benchmark/process_control.py                  | 100 +++++--
 benchmark/run_top50.py                        |   4 +
 benchmark/tests/test_budget_calibration.py    |   3 +-
 benchmark/tests/test_docs.py                  |   4 +-
 benchmark/tests/test_evidence_budget.py       |  33 ++-
 benchmark/tests/test_observation_policy.py    | 138 +++++++++
 benchmark/tests/test_run_top50.py             |   2 +-
 benchmark/tests/test_top50_runner.py          | 110 +++++++-
 benchmark/tests/test_top50_submission.py      |  87 +++++-
 benchmark/top50_budget.json                   |   7 +-
 benchmark/top50_config.yaml                   |   5 +-
 benchmark/top50_contract.py                   | 125 ++++++++-
 benchmark/top50_results.schema.json           |   2 +-
 benchmark/top50_runner.py                     |  74 ++++-
 benchmark/top50_submission.schema.json        |   8 +-
 docker/Dockerfile                             |   2 +-
 site/index.html                               |   4 +-
 submission.env.example                        |   2 +-
 29 files changed, 1053 insertions(+), 97 deletions(-)
 create mode 100644 benchmark/observation_policy.py
 create mode 100644 benchmark/tests/test_observation_policy.py

diff --git a/.agents/skills/run-api-benchmark/SKILL.md b/.agents/skills/run-api-benchmark/SKILL.md
index c69e307..6f790ca 100644
--- a/.agents/skills/run-api-benchmark/SKILL.md
+++ b/.agents/skills/run-api-benchmark/SKILL.md
@@ -5,7 +5,7 @@ description: Configure and run the rankable problem-reductions Self-selected Top
 
 # Run the rankable Top50 benchmark
 
-Use only the frozen `top50-evidence/v1` path: source-only triage freezes 50 rules, followed by 50 fresh sequential episodes. Each rule receives M=10 model generations, E=12 shell actions, P=24 `pred` calls, P_solve=10 solve calls, O=10000 observed characters, and exactly S=2 submit attempts. Never add or accept custom budgets, prompts, strategies, model kwargs, or coding-agent backends.
+Use only the frozen `top50-evidence/v2` path: source-only triage freezes 50 rules, followed by 50 fresh sequential episodes. Each rule receives M=10 model generations, E=12 shell actions, P=24 `pred` calls, P_solve=10 solve calls, O=10000 observed characters, and exactly S=2 submit attempts. Never add or accept custom budgets, prompts, strategies, model kwargs, or coding-agent backends.
 
 Read [references/provider-config.md](references/provider-config.md) for endpoint setup and `scripts/detect-engine.sh` before preparing the image.
 
diff --git a/.agents/skills/run-cli-benchmark/SKILL.md b/.agents/skills/run-cli-benchmark/SKILL.md
index 3374213..12b0111 100644
--- a/.agents/skills/run-cli-benchmark/SKILL.md
+++ b/.agents/skills/run-cli-benchmark/SKILL.md
@@ -6,7 +6,7 @@ description: Configure and run this problem-reductions benchmark through an inst
 # Reproduce the historical CLI benchmark
 
 This route is permanently **non-ranking** and exists only to reproduce `legacy-whole-repo`
-artifacts. Tell the caller that coding-agent results cannot enter `top50-evidence/v1` and
+artifacts. Tell the caller that coding-agent results cannot enter `top50-evidence/v2` and
 get confirmation before spending time or credits. Do not describe it as a System Track.
 
 Run one whole-repository coding-agent session with the shared benchmark prompt, verifier,
diff --git a/.agents/skills/submit-benchmark-result/SKILL.md b/.agents/skills/submit-benchmark-result/SKILL.md
index ba6e770..a7ac92f 100644
--- a/.agents/skills/submit-benchmark-result/SKILL.md
+++ b/.agents/skills/submit-benchmark-result/SKILL.md
@@ -39,7 +39,7 @@ Immediately before uploading, show this confirmation with the real values filled
 > - Destination: `intake.prb-bench.workers.dev`
 > - File: ``
 > - Model: ``
-> - Contract: `top50-evidence/v1`
+> - Contract: `top50-evidence/v2`
 > - Claimed bugs: ``
 >
 > This private submission file will leave the machine and be uploaded to the benchmark's
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 6f46dbf..5b8cf3a 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,6 +1,6 @@
 # Submitting a model run
 
-The current public comparison accepts one execution protocol: **Standardized Model API / Self-selected Top50** under frozen contract `top50-evidence/v1`.
+The current public comparison accepts one execution protocol: **Standardized Model API / Self-selected Top50** under frozen contract `top50-evidence/v2`.
 
 ```text
 source-only triage → frozen Top50 → 50 isolated rule episodes → private submission
@@ -32,7 +32,7 @@ Preflight first checks that no forbidden execution knob is present, verifies the
 - 50 unique frozen rules;
 - per rule: 10 model generations, 12 shell actions, 24 `pred` calls, at most 10 solves;
 - exactly two charged submit attempts per rule;
-- 10,000 observed characters per rule.
+- a 10,000-character deterministic terminal preview per action, with a read-only bounded raw log.
 
 The runner owns these counters. A model receives authoritative remaining-budget state after each action, cannot transfer unused budget, and cannot invoke the verifier directly. Each rule starts with fresh model history and a fresh submission ledger.
 
@@ -64,7 +64,7 @@ The score should be read as joint **prioritization + bounded diagnosis + certifi
 
 ## Release checks
 
-Before changing the frozen contract or target version, add a new versioned contract and a new non-ranking calibration. Do not silently edit `top50-evidence/v1`.
+Before changing the frozen contract or target version, add a new versioned contract and a new non-ranking calibration. Do not silently edit `top50-evidence/v2`.
 
 ```bash
 pytest -v -m "not integration"
diff --git a/README.md b/README.md
index ffd4f88..4b68f0a 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@ This benchmark measures how well a model can prioritize likely-buggy reduction r
 
 ## Current ranking contract
 
-The primary track is **Standardized Model API / Self-selected Top50**, contract [`top50-evidence/v1`](benchmark/top50_budget.json). The target is `problem-reductions` [`v0.6.0`](https://github.com/CodingThrust/problem-reductions/commit/aa2d1a10cffa434871d12a4d6f411147fb7e08a8), and the bundled `pred` version is `0.6.0`.
+The primary track is **Standardized Model API / Self-selected Top50**, contract [`top50-evidence/v2`](benchmark/top50_budget.json). The target is `problem-reductions` [`v0.6.0`](https://github.com/CodingThrust/problem-reductions/commit/aa2d1a10cffa434871d12a4d6f411147fb7e08a8), and the bundled `pred` version is `0.6.0`.
 
 Each run has two phases:
 
@@ -20,10 +20,12 @@ Every episode receives the same immutable logical budget:
 | Total `pred` calls (`P`) | 24 |
 | `pred solve` calls (`P_solve`) | 10 |
 | Submit attempts (`S`) | **2** |
-| Observed output (`O`) | 10,000 characters |
+| Automatic preview per action (`O`) | 10,000 characters |
 
 Triage receives 8 model generations and 12 source-only actions. Unused budget never transfers between rules. Process timeouts remain fixed watchdogs for hung model or `pred` calls; elapsed time, network delay, tokens, and cost do not affect rank.
 
+Every terminal result uses the frozen `terminal-diagnostics/v1` policy: deterministic noise and repetition removal, diagnostic-aware head/tail selection, and a 10,000-character model preview. A separately bounded 1 MiB raw log remains read-only inside the current episode and can be inspected using a normal charged shell action.
+
 The budget was selected once using a human-reviewed non-ranking development replay, with smaller and larger candidates around the chosen `M` and `P`. The checked-in record and rationale are in [budget-calibration.md](benchmark/docs/budget-calibration.md); validate their internal and release consistency offline with:
 
 ```bash
@@ -71,6 +73,6 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for artifact fields, submission, and hist
 
 ## Historical results
 
-Whole-repository results produced by the former contract remain visible under `legacy-whole-repo`. They use a different execution protocol and efficiency tie-break, so the site keeps them in a separate selectable table. They cannot be deduplicated, sorted, or compared into `top50-evidence/v1`.
+Whole-repository results produced by the former contract remain visible under `legacy-whole-repo`. They use a different execution protocol and efficiency tie-break, so the site keeps them in a separate selectable table. They cannot be deduplicated, sorted, or compared into `top50-evidence/v2`.
 
 The old host coding-agent and whole-repository runners remain in the repository only to reproduce those historical artifacts. They are not a public System Track and cannot produce rankable Top50 submissions.
diff --git a/benchmark/agent_environment.py b/benchmark/agent_environment.py
index a69354c..834e38e 100644
--- a/benchmark/agent_environment.py
+++ b/benchmark/agent_environment.py
@@ -4,6 +4,7 @@
 import os
 import subprocess
 
+from benchmark.observation_policy import ObservationStore
 from benchmark.process_control import ProcessLimits, run_capped_process
 
 _SAFE_ENV_KEYS = {"PATH", "PYTHONPATH", "LANG", "LC_ALL", "PAGER", "MANPAGER", "LESS",
@@ -28,6 +29,8 @@ def run_as_agent(
     gid: int,
     extra_groups: tuple[int, ...] = (),
     max_output_chars: int = 10_000,
+    observation_store: ObservationStore | None = None,
+    observation_kind: str = "shell",
 ) -> subprocess.CompletedProcess[str]:
     """Execute one shell action after irreversibly dropping child privileges."""
     if os.name != "posix":
@@ -41,7 +44,8 @@ def run_as_agent(
         cwd=cwd,
         env=env,
         timeout=timeout,
-        max_output_chars=max_output_chars,
+        max_output_chars=(observation_store.config.archive_chars
+                          if observation_store is not None else max_output_chars),
         combine_stderr=True,
         uid=uid,
         gid=gid,
@@ -49,9 +53,35 @@ def run_as_agent(
         limits=ProcessLimits(cpu_seconds=max(1, timeout), memory_bytes=2 * 1024 ** 3,
                              file_bytes=64 * 1024 ** 2),
     )
+    packaged = None
+    output = result.stdout
+    if observation_store is not None:
+        packaged = observation_store.package(
+            kind=observation_kind, command=command, returncode=result.returncode,
+            timed_out=result.timed_out, stdout=result.stdout, stderr=result.stderr,
+            original_chars=result.original_chars, original_lines=result.original_lines,
+            archive_truncated=result.capture_truncated)
+        output = packaged.preview
     if result.timed_out:
-        raise subprocess.TimeoutExpired(command, timeout, output=result.stdout)
-    return subprocess.CompletedProcess(command, result.returncode, stdout=result.stdout)
+        error = subprocess.TimeoutExpired(command, timeout, output=output)
+        error.observation_metadata = packaged.metadata if packaged is not None else None
+        raise error
+    completed = subprocess.CompletedProcess(command, result.returncode, stdout=output)
+    completed.observation_metadata = packaged.metadata if packaged is not None else None
+    return completed
+
+
+def package_shell_result(session, command: str, *, output: str, returncode: int,
+                         exception_info: str = "", timed_out: bool = False) -> dict:
+    """Package a harness-generated shell result through the same frozen policy."""
+    packaged = session.observations.package(
+        kind="shell", command=command, returncode=returncode, timed_out=timed_out,
+        stdout=output, stderr="", original_chars=len(output),
+        original_lines=output.count("\n") + int(bool(output) and not output.endswith("\n")),
+        archive_truncated=False)
+    session.record_shell_observation(command, packaged.metadata)
+    return {"output": packaged.preview, "returncode": returncode,
+            "exception_info": exception_info}
 
 
 def make_agent_environment(session, *, uid: int, gid: int,
@@ -64,11 +94,8 @@ def execute(self, action: dict, cwd: str = "", *, timeout: int | None = None) ->
             command = action.get("command", "")
             action_cwd = cwd or self.config.cwd or str(session.workdir)
             if not session.admit_shell_action(command):
-                output = {
-                    "output": "shell action budget exhausted\n",
-                    "returncode": 75,
-                    "exception_info": "",
-                }
+                output = package_shell_result(
+                    session, command, output="shell action budget exhausted\n", returncode=75)
                 self._check_finished(output)
                 return output
             try:
@@ -81,20 +108,28 @@ def execute(self, action: dict, cwd: str = "", *, timeout: int | None = None) ->
                     gid=gid,
                     extra_groups=extra_groups,
                     max_output_chars=session.budget.max_output_chars,
+                    observation_store=session.observations,
                 )
                 output = {"output": result.stdout, "returncode": result.returncode,
                           "exception_info": ""}
+                session.record_shell_observation(command, result.observation_metadata)
             except Exception as error:
                 raw_output = getattr(error, "output", None)
                 raw_output = (raw_output.decode("utf-8", errors="replace")
                               if isinstance(raw_output, bytes) else (raw_output or ""))
-                output = {
-                    "output": raw_output,
-                    "returncode": -1,
-                    "exception_info": f"An error occurred while executing the command: {error}",
-                    "extra": {"exception_type": type(error).__name__,
-                              "exception": str(error)},
-                }
+                metadata = getattr(error, "observation_metadata", None)
+                if metadata is not None:
+                    session.record_shell_observation(command, metadata)
+                    output = {
+                        "output": raw_output, "returncode": -1,
+                        "exception_info": f"An error occurred while executing the command: {error}",
+                    }
+                else:
+                    output = package_shell_result(
+                        session, command, output=raw_output, returncode=-1,
+                        exception_info=f"An error occurred while executing the command: {error}")
+                output["extra"] = {"exception_type": type(error).__name__,
+                                   "exception": str(error)}
             self._check_finished(output)
             return output
 
diff --git a/benchmark/calibrate_budget.py b/benchmark/calibrate_budget.py
index 91974c8..e6e5060 100644
--- a/benchmark/calibrate_budget.py
+++ b/benchmark/calibrate_budget.py
@@ -146,10 +146,14 @@ def render_report(evidence: dict) -> str:
         f"P={episode['pred_calls']} total `pred` calls, "
         f"P_solve={episode['solve_calls']} solve calls, S={episode['submit_attempts']} "
         f"submit attempts, E={episode['shell_actions']} shell actions, and "
-        f"O={episode['max_output_chars']} observed characters per rule. Triage is "
+        f"O={episode['max_output_chars']} automatically previewed characters per action. Triage is "
         f"T={contract['triage']['model_generations']} generations and "
         f"E_t={contract['triage']['shell_actions']} source-only actions.",
         "",
+        f"Terminal output uses `{contract['observation']['policy_id']}` with a "
+        f"{contract['observation']['preview_chars']}-character automatic preview and a bounded "
+        f"{contract['observation']['archive_chars']}-character raw archive per command.",
+        "",
         f"Model calls have a fixed {safety['model_timeout_seconds']}-second watchdog and "
         f"{safety['model_retries']} transport retries. Command and `pred` watchdogs are also "
         "fixed safety controls. They are recorded but are outside the logical budget and never "
@@ -255,7 +259,7 @@ def check(path: str | Path) -> list[str]:
     submission_schema = load_json(ROOT / "top50_submission.schema.json")
     properties = submission_schema.get("properties", {})
     expected_artifact_contract = {key: contract[key] for key in
-                                  ("triage", "episode", "shortlist_size",
+                                  ("triage", "episode", "observation", "shortlist_size",
                                    "hypothesis_chars")}
     if properties.get("contract", {}).get("const") != expected_artifact_contract:
         errors.append("top50_submission.schema.json has stale logical limits")
@@ -264,6 +268,8 @@ def check(path: str | Path) -> list[str]:
     if properties.get("inference_parameters", {}).get("const") != contract[
             "inference_parameters"]:
         errors.append("top50_submission.schema.json has stale inference parameters")
+    if properties.get("observation_policy", {}).get("const") != contract["observation"]:
+        errors.append("top50_submission.schema.json has stale observation policy")
     if not errors and REPORT_PATH.read_text(encoding="utf-8") != render_report(evidence):
         errors.append("budget-calibration.md does not match the machine-readable evidence")
     return errors
diff --git a/benchmark/docs/budget-calibration.json b/benchmark/docs/budget-calibration.json
index 53c45c3..2042adf 100644
--- a/benchmark/docs/budget-calibration.json
+++ b/benchmark/docs/budget-calibration.json
@@ -4,7 +4,7 @@
   "ranking_status": "non-ranking-development",
   "method": "Human-reviewed bounded-prefix replay record from internally retained whole-repository pilot trajectories. Raw trajectories remain private; the offline checker validates this record and release consistency, not the raw replay. These statistics are not Top50 scores.",
   "selected_contract": {
-    "contract_id": "top50-evidence/v1",
+    "contract_id": "top50-evidence/v2",
     "status": "frozen",
     "shortlist_size": 50,
     "hypothesis_chars": 500,
@@ -17,6 +17,11 @@
       "timeout": 300,
       "num_retries": 2
     },
+    "observation": {
+      "policy_id": "terminal-diagnostics/v1",
+      "preview_chars": 10000,
+      "archive_chars": 1048576
+    },
     "triage": {
       "model_generations": 8,
       "shell_actions": 12,
diff --git a/benchmark/docs/budget-calibration.md b/benchmark/docs/budget-calibration.md
index 7306cd2..ae29d33 100644
--- a/benchmark/docs/budget-calibration.md
+++ b/benchmark/docs/budget-calibration.md
@@ -1,12 +1,14 @@
 # Top50 budget calibration
 
-Contract: `top50-evidence/v1` (`frozen`)
+Contract: `top50-evidence/v2` (`frozen`)
 
 This is a human-reviewed, non-ranking bounded-prefix replay record from internally retained pilot trajectories. Raw trajectories remain private; this offline checker validates the checked-in record and release consistency, not the raw replay. It is not a public score, a multi-seed experiment, or a claim that elapsed time is model ability.
 
 ## Selected contract
 
-The release freezes M=10 model generations, P=24 total `pred` calls, P_solve=10 solve calls, S=2 submit attempts, E=12 shell actions, and O=10000 observed characters per rule. Triage is T=8 generations and E_t=12 source-only actions.
+The release freezes M=10 model generations, P=24 total `pred` calls, P_solve=10 solve calls, S=2 submit attempts, E=12 shell actions, and O=10000 automatically previewed characters per action. Triage is T=8 generations and E_t=12 source-only actions.
+
+Terminal output uses `terminal-diagnostics/v1` with a 10000-character automatic preview and a bounded 1048576-character raw archive per command.
 
 Model calls have a fixed 300-second watchdog and 2 transport retries. Command and `pred` watchdogs are also fixed safety controls. They are recorded but are outside the logical budget and never enter the score.
 
diff --git a/benchmark/evidence_budget.py b/benchmark/evidence_budget.py
index 12ccc44..71bd015 100644
--- a/benchmark/evidence_budget.py
+++ b/benchmark/evidence_budget.py
@@ -24,6 +24,8 @@
 from typing import Callable
 
 from benchmark.submit_session import SubmissionSession
+from benchmark.observation_policy import (ObservationConfig, ObservationStore,
+                                          metadata_dict)
 from benchmark.process_control import ProcessLimits, run_capped_process, terminate_process_group
 from benchmark.verify import (CPU_LIMIT_SECONDS, FSIZE_LIMIT_BYTES, MEM_LIMIT_BYTES,
                               Verdict, verify)
@@ -120,6 +122,7 @@ def __init__(
         oracle_uid: int | None = None,
         oracle_gid: int | None = None,
         oracle_extra_groups: tuple[int, ...] = (),
+        observation_store: ObservationStore | None = None,
     ):
         if (oracle_uid is None) != (oracle_gid is None):
             raise ValueError("oracle_uid and oracle_gid must be provided together")
@@ -129,6 +132,7 @@ def __init__(
         self.oracle_uid = oracle_uid
         self.oracle_gid = oracle_gid
         self.oracle_extra_groups = oracle_extra_groups
+        self.observation_store = observation_store
         self._ledger: list[dict] = []
         self._responses: dict[str, tuple[str, dict]] = {}
         self._cache: dict[tuple[str, ...], dict] = {}
@@ -346,7 +350,7 @@ def _handle(self, request_id: str, request: dict) -> dict:
         record_index = self._append_pred_record(
             request_id, args, command, bool(counters), "running", self.budget_state.status())
         try:
-            result = self._run_pred(args, cwd)
+            result = self._run_pred(args, cwd, request_id=request_id)
         except OSError as error:
             model_caused = error.errno == errno.E2BIG
             if counters and not model_caused:
@@ -399,14 +403,16 @@ def _validated_cwd(self, raw: object) -> Path:
             return self.workdir
         return cwd
 
-    def _run_pred(self, args: list[str], cwd: Path) -> dict:
+    def _run_pred(self, args: list[str], cwd: Path, *, request_id: str) -> dict:
         result = run_capped_process(
             [str(self.pred_binary), *args],
             shell=False,
             cwd=cwd,
             env=None,
             timeout=self.budget_state.budget.pred_timeout_seconds,
-            max_output_chars=self.budget_state.budget.max_output_chars,
+            max_output_chars=(self.observation_store.config.archive_chars
+                              if self.observation_store is not None
+                              else self.budget_state.budget.max_output_chars),
             uid=self.oracle_uid,
             gid=self.oracle_gid,
             extra_groups=self.oracle_extra_groups,
@@ -422,11 +428,27 @@ def _run_pred(self, args: list[str], cwd: Path) -> dict:
         if result.timed_out:
             stderr += ("\n" if stderr and not stderr.endswith("\n") else "")
             stderr += "pred process timed out\n"
+        observation = None
+        stdout = result.stdout
+        if self.observation_store is not None:
+            packaged = self.observation_store.package(
+                kind="pred", command="pred " + shlex.join(args),
+                returncode=result.returncode, timed_out=result.timed_out,
+                stdout=result.stdout, stderr=stderr,
+                original_chars=result.original_chars, original_lines=result.original_lines,
+                archive_truncated=result.capture_truncated,
+                observation_id=f"pred-{request_id}")
+            observation = metadata_dict(packaged)
+            if result.returncode == 0:
+                stdout, stderr = packaged.preview, ""
+            else:
+                stdout, stderr = "", packaged.preview
         return {
             "returncode": result.returncode,
-            "stdout": result.stdout,
+            "stdout": stdout,
             "stderr": stderr,
             "timed_out": result.timed_out,
+            "observation": observation,
         }
 
     def _track_process(self, process) -> None:
@@ -456,6 +478,8 @@ def _finish_pred_record(self, index: int, outcome: str, response: dict) -> None:
             self._ledger[index]["outcome"] = outcome
             self._ledger[index]["returncode"] = response["returncode"]
             self._ledger[index]["budget"] = copy.deepcopy(response["budget"])
+            if response.get("observation") is not None:
+                self._ledger[index]["observation"] = copy.deepcopy(response["observation"])
 
     def _write_response(self, request_id: str, response: dict) -> None:
         assert self._outbox_fd is not None
@@ -488,6 +512,7 @@ def __init__(
         oracle_uid: int | None = None,
         oracle_gid: int | None = None,
         evidence_gid: int | None = None,
+        observation_config: ObservationConfig | None = None,
     ):
         if not rule:
             raise ValueError("rule must be non-empty")
@@ -507,6 +532,8 @@ def __init__(
         self.oracle_uid = oracle_uid
         self.oracle_gid = oracle_gid
         self.evidence_gid = evidence_gid
+        self.observation_config = observation_config or ObservationConfig(
+            preview_chars=budget.max_output_chars)
         self.submit: SubmissionSession | None = None
         self.pred: PredGatewaySession | None = None
         self._scratch: Path | None = None
@@ -514,6 +541,8 @@ def __init__(
         self._event_lock = threading.RLock()
         self._model_events: list[dict] = []
         self._shell_events: list[dict] = []
+        self._observations: list[dict] = []
+        self.observations: ObservationStore | None = None
 
     @property
     def workdir(self) -> Path:
@@ -531,6 +560,12 @@ def __enter__(self) -> "EvidenceBudgetSession":
             ))
             self._scratch = self.submit.workdir / "scratch"
             self._scratch.mkdir(mode=0o700)
+            self.observations = ObservationStore(
+                self.submit.workdir / "observations",
+                config=self.observation_config,
+                relative_from=self._scratch,
+                readable_gid=self.evidence_gid,
+            )
             self.pred = stack.enter_context(PredGatewaySession(
                 pred_binary=self.pred_binary,
                 workdir=self._scratch,
@@ -538,6 +573,7 @@ def __enter__(self) -> "EvidenceBudgetSession":
                 oracle_uid=self.oracle_uid,
                 oracle_gid=self.oracle_gid,
                 oracle_extra_groups=((self.evidence_gid,) if self.evidence_gid is not None else ()),
+                observation_store=self.observations,
             ))
             if self.agent_uid is not None and self.agent_gid is not None:
                 self._prepare_agent_access(self.agent_uid, self.agent_gid)
@@ -586,6 +622,20 @@ def record_model_generation(self, *, outcome: str = "completed",
             })
         return admitted
 
+    def record_shell_observation(self, command: str, metadata: dict | None) -> None:
+        if metadata is None:
+            return
+        with self._event_lock:
+            self._observations.append(copy.deepcopy(metadata))
+            for event in reversed(self._shell_events):
+                if event.get("command") == command and "observation_id" not in event:
+                    event["observation_id"] = metadata["observation_id"]
+                    if event.get("charged") is True:
+                        event["outcome"] = ("timeout" if metadata["timed_out"] else
+                                            "completed" if metadata["returncode"] == 0 else
+                                            "nonzero_exit")
+                    break
+
     def status(self) -> dict:
         if self.submit is None:
             raise RuntimeError("evidence budget session is not active")
@@ -604,14 +654,21 @@ def ledger(self) -> dict:
         with self._event_lock:
             model_events = copy.deepcopy(self._model_events)
             shell_events = copy.deepcopy(self._shell_events)
+        pred_ledger = self.pred.ledger
+        observations = copy.deepcopy(self._observations)
+        observations.extend(copy.deepcopy(record["observation"])
+                            for record in pred_ledger if "observation" in record)
+        observations.sort(key=lambda item: item["observation_id"])
         return {
             "rule": self.rule,
             "budget": asdict(self.budget),
+            "observation_policy": asdict(self.observation_config),
             "status": self.status(),
-            "pred": self.pred.ledger,
+            "pred": pred_ledger,
             "submit": self.submit.attempts,
             "model_generations": model_events,
             "shell_actions": shell_events,
+            "observations": observations,
         }
 
 
diff --git a/benchmark/observation_policy.py b/benchmark/observation_policy.py
new file mode 100644
index 0000000..c918b76
--- /dev/null
+++ b/benchmark/observation_policy.py
@@ -0,0 +1,265 @@
+"""Deterministic, bounded packaging for model-facing process observations."""
+from __future__ import annotations
+
+import json
+import os
+import re
+import threading
+from dataclasses import dataclass
+from pathlib import Path
+
+POLICY_ID = "terminal-diagnostics/v1"
+DEFAULT_PREVIEW_CHARS = 10_000
+DEFAULT_ARCHIVE_CHARS = 1_048_576
+_CONTEXT_LINES = 2
+_ANSI = re.compile(r"\x1b(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1b\\))")
+_CONTROL = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
+_DIAGNOSTIC = re.compile(
+    r"(?i)(error|fail(?:ed|ure)?|panic|assert(?:ion)?|traceback|exception|timeout|timed out|"
+    r"sentinel|left\s*[:=]|right\s*[:=]|\b\d+\s+failed\b|test result:)")
+
+
+@dataclass(frozen=True)
+class ObservationConfig:
+    policy_id: str = POLICY_ID
+    preview_chars: int = DEFAULT_PREVIEW_CHARS
+    archive_chars: int = DEFAULT_ARCHIVE_CHARS
+
+    def __post_init__(self) -> None:
+        if self.policy_id != POLICY_ID:
+            raise ValueError(f"unsupported observation policy: {self.policy_id}")
+        if self.preview_chars < 256 or self.archive_chars < self.preview_chars:
+            raise ValueError("archive_chars must be >= preview_chars >= 256")
+
+
+@dataclass(frozen=True)
+class PackagedObservation:
+    preview: str
+    metadata: dict
+
+
+@dataclass(frozen=True)
+class _Line:
+    number: int
+    end_number: int
+    text: str
+    repeated: int = 1
+
+
+class ObservationStore:
+    """Write immutable per-session logs and produce deterministic compact previews."""
+
+    def __init__(self, directory: str | Path, *, config: ObservationConfig,
+                 relative_from: str | Path, readable_gid: int | None = None):
+        self.directory = Path(directory).resolve()
+        self.relative_from = Path(relative_from).resolve()
+        self.config = config
+        self.readable_gid = readable_gid
+        self._sequence = 0
+        self._lock = threading.RLock()
+        self.directory.mkdir(parents=True, exist_ok=False)
+        if readable_gid is not None:
+            os.chown(self.directory, os.geteuid(), readable_gid)
+            self.directory.chmod(0o750)
+        else:
+            self.directory.chmod(0o700)
+
+    def package(self, *, kind: str, command: str, returncode: int, timed_out: bool,
+                stdout: str, stderr: str, original_chars: int, original_lines: int,
+                archive_truncated: bool,
+                observation_id: str | None = None) -> PackagedObservation:
+        if observation_id is None:
+            with self._lock:
+                self._sequence += 1
+                observation_id = f"{kind}-{self._sequence:04d}"
+        if not re.fullmatch(r"(?:shell-[0-9]{4}|pred-[0-9a-f]{32})", observation_id):
+            raise ValueError(f"invalid observation id: {observation_id!r}")
+        path = self.directory / f"{observation_id}.log"
+
+        raw = _raw_log(stdout, stderr)
+        raw, store_truncated = _bounded_text(raw, self.config.archive_chars)
+        archive_truncated = archive_truncated or store_truncated
+        flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
+        fd = os.open(path, flags, 0o640 if self.readable_gid is not None else 0o600)
+        try:
+            payload = raw.encode("utf-8")
+            written = 0
+            while written < len(payload):
+                written += os.write(fd, payload[written:])
+        finally:
+            os.close(fd)
+        if self.readable_gid is not None:
+            os.chown(path, os.geteuid(), self.readable_gid)
+
+        relative = os.path.relpath(path, self.relative_from)
+        header = (
+            f"[observation {observation_id}] policy={self.config.policy_id} "
+            f"returncode={returncode} timed_out={str(timed_out).lower()}\n"
+            f"[raw log: {relative}; original={original_chars} chars/{original_lines} lines; "
+            f"archive={'truncated' if archive_truncated else 'complete'}]\n"
+        )
+        body_limit = max(0, self.config.preview_chars - len(header))
+        body, compacted = _compact_output(stdout, stderr, body_limit)
+        preview = header + body
+        metadata = {
+            "observation_id": observation_id,
+            "kind": kind,
+            "command": command,
+            "policy_id": self.config.policy_id,
+            "raw_log": relative,
+            "returncode": returncode,
+            "timed_out": timed_out,
+            "original_chars": original_chars,
+            "original_lines": original_lines,
+            "preview_chars": len(preview),
+            "archive_chars": len(raw),
+            "preview_compacted": compacted,
+            "archive_truncated": archive_truncated,
+        }
+        return PackagedObservation(preview=preview, metadata=metadata)
+
+
+def _raw_log(stdout: str, stderr: str) -> str:
+    if stderr:
+        return f"[stdout]\n{stdout}\n[stderr]\n{stderr}"
+    return stdout
+
+
+def _bounded_text(text: str, limit: int) -> tuple[str, bool]:
+    if len(text) <= limit:
+        return text, False
+    marker = f"\n... raw archive truncated; {len(text) - limit} or more characters omitted ...\n"
+    available = max(0, limit - len(marker))
+    head = available // 2
+    tail = available - head
+    return text[:head] + marker + (text[-tail:] if tail else ""), True
+
+
+def _compact_output(stdout: str, stderr: str, limit: int) -> tuple[str, bool]:
+    streams = [("stdout", stdout)]
+    if stderr:
+        streams.append(("stderr", stderr))
+    rendered: list[str] = []
+    compacted = False
+    stream_limit = max(0, (limit - len(streams)) // len(streams))
+    for name, text in streams:
+        normalized_json = _compact_json(text)
+        if normalized_json is not None:
+            text = normalized_json
+            compacted = True
+        lines, changed = _normalize_lines(text)
+        compacted = compacted or changed
+        rendered.append(f"[{name}]")
+        selected, selected_compacted = _select_lines(
+            lines, max(0, stream_limit - len(name) - 3))
+        compacted = compacted or selected_compacted
+        rendered.extend(selected)
+    result = "\n".join(rendered).rstrip() + "\n"
+    if len(result) > limit:
+        raise AssertionError("observation renderer exceeded its character budget")
+    return result, compacted
+
+
+def _compact_json(text: str) -> str | None:
+    stripped = text.strip()
+    if not stripped or stripped[0] not in "[{":
+        return None
+    try:
+        value = json.loads(stripped)
+    except (json.JSONDecodeError, UnicodeError):
+        return None
+    return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n"
+
+
+def _normalize_lines(text: str) -> tuple[list[_Line], bool]:
+    changed = False
+    physical = text.split("\n")
+    lines: list[_Line] = []
+    for number, raw in enumerate(physical, 1):
+        if number == len(physical) and raw == "":
+            continue
+        if "\r" in raw:
+            raw = raw.rsplit("\r", 1)[-1]
+            changed = True
+        clean = _CONTROL.sub("", _ANSI.sub("", raw))
+        changed = changed or clean != raw
+        if lines and lines[-1].text == clean:
+            previous = lines[-1]
+            lines[-1] = _Line(previous.number, number, clean, previous.repeated + 1)
+            changed = True
+        else:
+            lines.append(_Line(number, number, clean))
+    return lines, changed
+
+
+def _render_line(line: _Line) -> str:
+    text = line.text
+    if len(text) > 600:
+        text = text[:300] + f" ... {len(text) - 600} chars omitted ... " + text[-300:]
+    suffix = (f" (repeated {line.repeated} times; original lines "
+              f"{line.number}-{line.end_number})" if line.repeated > 1 else "")
+    return f"[L{line.number:05d}] {text}{suffix}"
+
+
+def _select_lines(lines: list[_Line], limit: int) -> tuple[list[str], bool]:
+    full = [_render_line(line) for line in lines]
+    if len("\n".join(full)) <= limit:
+        return full, False
+    diagnostic = {index for index, line in enumerate(lines) if _DIAGNOSTIC.search(line.text)}
+    selected: set[int] = set()
+
+    # Diagnostics own the budget. Prefer the latest diagnostics because compilers and test
+    # runners normally put the decisive failure and final summary last.
+    for index in sorted(diagnostic, reverse=True):
+        _try_add(lines, selected, index, limit)
+    for distance in range(1, _CONTEXT_LINES + 1):
+        for index in sorted(diagnostic, reverse=True):
+            _try_add(lines, selected, index - distance, limit)
+            _try_add(lines, selected, index + distance, limit)
+
+    # Spend only the remaining space on a balanced command head/tail.
+    for offset in range(len(lines)):
+        added = _try_add(lines, selected, offset, limit)
+        added = _try_add(lines, selected, len(lines) - 1 - offset, limit) or added
+        if not added and len(_render_selected(lines, selected)) >= limit:
+            break
+    output = _render_selected(lines, selected)
+    if len("\n".join(output)) > limit:
+        raise AssertionError("observation selector exceeded its character budget")
+    return output, True
+
+
+def _try_add(lines: list[_Line], selected: set[int], index: int, limit: int) -> bool:
+    if not 0 <= index < len(lines) or index in selected:
+        return False
+    candidate = selected | {index}
+    if len("\n".join(_render_selected(lines, candidate))) > limit:
+        return False
+    selected.add(index)
+    return True
+
+
+def _render_selected(lines: list[_Line], selected: set[int]) -> list[str]:
+    ordered = sorted(selected)
+    if not ordered:
+        return ([f"... {sum(line.repeated for line in lines)} original lines omitted ..."]
+                if lines else [])
+    output: list[str] = []
+    previous = -1
+    for index in ordered:
+        if previous == -1 and lines[index].number > 1:
+            output.append(f"... {lines[index].number - 1} original lines omitted ...")
+        elif previous >= 0 and index > previous + 1:
+            omitted = lines[index].number - lines[previous].end_number - 1
+            output.append(f"... {omitted} original lines omitted ...")
+        output.append(_render_line(lines[index]))
+        previous = index
+    if lines[ordered[-1]].end_number < lines[-1].end_number:
+        omitted = lines[-1].end_number - lines[ordered[-1]].end_number
+        output.append(f"... {omitted} original lines omitted ...")
+    return output
+
+
+def metadata_dict(packaged: PackagedObservation) -> dict:
+    """Return a defensive JSON-compatible copy for ledgers."""
+    return dict(packaged.metadata)
diff --git a/benchmark/process_control.py b/benchmark/process_control.py
index 3036bd2..10f65be 100644
--- a/benchmark/process_control.py
+++ b/benchmark/process_control.py
@@ -28,45 +28,80 @@ class ProcessResult:
     stdout: str
     stderr: str
     timed_out: bool
+    original_chars: int
+    original_lines: int
+    capture_truncated: bool
 
 
-class _CombinedCapture:
+class _StreamCapture:
     def __init__(self, limit: int):
-        self.limit = limit
-        self.remaining = limit
+        retained_limit = max(0, limit - 160)
+        self.head_limit = retained_limit // 2
+        self.tail_limit = retained_limit - self.head_limit
         self.total = 0
-        self.totals = {"stdout": 0, "stderr": 0}
-        self.parts = {"stdout": [], "stderr": []}
-        self._lock = threading.Lock()
+        self.newlines = 0
+        self.ends_newline = True
+        self.head: list[str] = []
+        self.head_chars = 0
+        self.tail: list[str] = []
+        self.tail_chars = 0
+
+    def add(self, chunk: str) -> None:
+        self.total += len(chunk)
+        self.newlines += chunk.count("\n")
+        self.ends_newline = chunk.endswith("\n")
+        head_remaining = self.head_limit - self.head_chars
+        if head_remaining:
+            piece = chunk[:head_remaining]
+            self.head.append(piece)
+            self.head_chars += len(piece)
+            chunk = chunk[len(piece):]
+        if chunk and self.tail_limit:
+            self.tail.append(chunk)
+            self.tail_chars += len(chunk)
+            while self.tail_chars > self.tail_limit:
+                excess = self.tail_chars - self.tail_limit
+                if len(self.tail[0]) <= excess:
+                    self.tail_chars -= len(self.tail.pop(0))
+                else:
+                    self.tail[0] = self.tail[0][excess:]
+                    self.tail_chars -= excess
+
+    def result(self) -> tuple[str, int, bool]:
+        retained = "".join(self.head + self.tail)
+        truncated = self.total > len(retained)
+        if truncated:
+            retained += f"\n... bounded archive: {self.total - len(retained)} characters omitted ...\n"
+        lines = self.newlines + int(self.total > 0 and not self.ends_newline)
+        return retained, lines, truncated
+
+
+class _CombinedCapture:
+    def __init__(self, limit: int, *, split_streams: bool):
+        stdout_limit = limit // 2 if split_streams else limit
+        stderr_limit = limit - stdout_limit if split_streams else 0
+        self.streams = {
+            "stdout": _StreamCapture(stdout_limit),
+            "stderr": _StreamCapture(stderr_limit),
+        }
+
+    @property
+    def total(self) -> int:
+        return sum(stream.total for stream in self.streams.values())
 
     def drain(self, stream, key: str) -> None:
+        capture = self.streams[key]
         while True:
             chunk = stream.read(64 * 1024)
             if not chunk:
                 break
-            with self._lock:
-                self.total += len(chunk)
-                self.totals[key] += len(chunk)
-                if self.remaining:
-                    piece = chunk[:self.remaining]
-                    self.parts[key].append(piece)
-                    self.remaining -= len(piece)
-
-    def result(self) -> tuple[str, str]:
-        stdout = "".join(self.parts["stdout"])
-        stderr = "".join(self.parts["stderr"])
-        retained = len(stdout) + len(stderr)
-        if self.total > retained:
-            marker = f"\n... {self.total - retained} characters elided ...\n"
-            keep = max(0, self.limit - len(marker))
-            if self.totals["stdout"] > len(stdout):
-                stdout = stdout[:keep] + marker[:self.limit - min(len(stdout), keep)]
-                stderr = ""
-            else:
-                stdout = stdout[:keep]
-                stderr_keep = max(0, keep - len(stdout))
-                stderr = stderr[:stderr_keep] + marker[:self.limit - len(stdout) - stderr_keep]
-        return stdout, stderr
+            capture.add(chunk)
+
+    def result(self) -> tuple[str, str, int, bool]:
+        stdout, stdout_lines, stdout_truncated = self.streams["stdout"].result()
+        stderr, stderr_lines, stderr_truncated = self.streams["stderr"].result()
+        return (stdout, stderr, stdout_lines + stderr_lines,
+                stdout_truncated or stderr_truncated)
 
 
 def run_capped_process(
@@ -114,7 +149,7 @@ def run_capped_process(
         on_start(process)
     try:
         _apply_limits(process.pid, limits)
-        capture = _CombinedCapture(max_output_chars)
+        capture = _CombinedCapture(max_output_chars, split_streams=not combine_stderr)
         assert process.stdout is not None
         readers = [threading.Thread(target=capture.drain, args=(process.stdout, "stdout"),
                                     daemon=True)]
@@ -139,12 +174,15 @@ def run_capped_process(
             terminate_process_group(process)
         for reader in readers:
             reader.join()
-        stdout, stderr = capture.result()
+        stdout, stderr, original_lines, capture_truncated = capture.result()
         return ProcessResult(
             returncode=124 if timed_out else process.returncode,
             stdout=stdout,
             stderr=stderr,
             timed_out=timed_out,
+            original_chars=capture.total,
+            original_lines=original_lines,
+            capture_truncated=capture_truncated,
         )
     finally:
         if on_finish is not None:
diff --git a/benchmark/run_top50.py b/benchmark/run_top50.py
index dbd56a6..0d308f9 100644
--- a/benchmark/run_top50.py
+++ b/benchmark/run_top50.py
@@ -11,6 +11,7 @@
 
 from benchmark.env_setup import find_pred_binary, pinned_commit, verify_pred_version
 from benchmark.evidence_budget import EvidenceBudget
+from benchmark.observation_policy import ObservationConfig
 from benchmark.run_mini import list_rules
 from benchmark.top50_runner import (
     PhaseResult,
@@ -24,6 +25,7 @@
     CONTRACT_ID,
     RUNNER_VERSION,
     EXPECTED_EPISODE_BUDGET,
+    EXPECTED_OBSERVATION,
     EXPECTED_INFERENCE_PARAMETERS,
     EXPECTED_SAFETY_CONTROLS,
     EXPECTED_HYPOTHESIS_CHARS,
@@ -84,6 +86,7 @@ def frozen_contract() -> Top50Contract:
     return Top50Contract(
         triage=TriageBudget(**EXPECTED_TRIAGE_BUDGET),
         episode=EvidenceBudget(**EXPECTED_EPISODE_BUDGET),
+        observation=ObservationConfig(**EXPECTED_OBSERVATION),
         shortlist_size=EXPECTED_SHORTLIST_SIZE,
         hypothesis_chars=EXPECTED_HYPOTHESIS_CHARS,
     )
@@ -148,6 +151,7 @@ def run(*, model: str, repo_dir: str | Path, output: str | Path,
         "safety_controls": EXPECTED_SAFETY_CONTROLS,
         "created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
         "inference_parameters": EXPECTED_INFERENCE_PARAMETERS,
+        "observation_policy": EXPECTED_OBSERVATION,
     }
     return runner.run(model=model, repo_path=repo, inventory=inventory,
                       output=output, metadata=metadata)
diff --git a/benchmark/tests/test_budget_calibration.py b/benchmark/tests/test_budget_calibration.py
index 75e57fb..500e5f5 100644
--- a/benchmark/tests/test_budget_calibration.py
+++ b/benchmark/tests/test_budget_calibration.py
@@ -94,7 +94,8 @@ def test_frozen_contract_is_referenced_across_release_surfaces():
     contract = _contract()
     runtime = asdict(run_top50.frozen_contract())
     assert runtime == {key: contract[key] for key in
-                       ("triage", "episode", "shortlist_size", "hypothesis_chars")}
+                       ("triage", "episode", "observation", "shortlist_size",
+                        "hypothesis_chars")}
     root = calibrate_budget.ROOT.parent
     for relative in (
         "README.md", "CONTRIBUTING.md", "benchmark/top50_config.yaml",
diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py
index fc050b1..c4c2b11 100644
--- a/benchmark/tests/test_docs.py
+++ b/benchmark/tests/test_docs.py
@@ -78,7 +78,7 @@ def test_readme_has_metrics_section(self):
 
     def test_readme_lists_current_round_contract(self):
         text = README.read_text(encoding="utf-8")
-        assert "top50-evidence/v1" in text
+        assert "top50-evidence/v2" in text
         assert PINNED_COMMIT in text
         assert f"`{PINNED_PRED_VERSION}`" in text
         assert "elapsed time" in text.lower()
@@ -107,7 +107,7 @@ def test_env_template_does_not_select_cli_with_agent_backend(self):
 
     def test_api_skill_is_container_only(self):
         t = _text(API_SKILL)
-        assert "top50-evidence/v1" in t and "runner-pull" in t
+        assert "top50-evidence/v2" in t and "runner-pull" in t
         assert "coding-agent backends" in t
 
     def test_cli_skill_is_host_only(self):
diff --git a/benchmark/tests/test_evidence_budget.py b/benchmark/tests/test_evidence_budget.py
index 16ec992..0f30a9d 100644
--- a/benchmark/tests/test_evidence_budget.py
+++ b/benchmark/tests/test_evidence_budget.py
@@ -117,6 +117,11 @@ def test_concurrent_pred_clients_cannot_overspend(tmp_path):
         assert sum(result.returncode == 0 for result in results) == 4
         assert sum(result.returncode == 75 for result in results) == 16
         assert session.status()["pred_calls"] == {"used": 4, "limit": 4, "remaining": 0}
+        observed = [record for record in session.pred.ledger if "observation" in record]
+        assert all(record["observation"]["observation_id"] ==
+                   f"pred-{record['request_id']}" for record in observed)
+        assert {item["observation_id"] for item in session.ledger()["observations"]} == {
+            record["observation"]["observation_id"] for record in observed}
 
 
 def test_free_commands_are_cached_and_do_not_charge(tmp_path):
@@ -167,6 +172,7 @@ def test_gateway_infrastructure_failure_releases_reservation(tmp_path):
 
 
 def test_pred_output_is_drained_but_observation_is_capped(tmp_path):
+    raw_log = None
     with EvidenceBudgetSession(
         rule="r1", budget=_budget(pred=1, solve=0), pred_binary=_fake_pred(tmp_path),
         verifier=lambda cert: Verdict(False, "no bug"),
@@ -174,7 +180,16 @@ def test_pred_output_is_drained_but_observation_is_capped(tmp_path):
         result = _run("pred", "spam", cwd=session.workdir)
         assert result.returncode == 0
         assert len(result.stdout) <= 1024
-        assert "characters elided" in result.stdout
+        assert "chars omitted" in result.stdout
+        observation = session.pred.ledger[0]["observation"]
+        assert observation["policy_id"] == "terminal-diagnostics/v1"
+        assert observation["original_chars"] == 5001
+        assert observation["preview_chars"] <= 1024
+        raw_log = (session.workdir / observation["raw_log"]).resolve()
+        assert raw_log.is_file()
+        assert len(raw_log.read_text()) == 5001
+        assert session.ledger()["observations"] == [observation]
+    assert raw_log is not None and not raw_log.exists()
 
 
 def test_submit_budget_is_two_per_rule_and_acceptance_closes_episode(tmp_path):
@@ -331,7 +346,21 @@ def test_agent_shell_output_has_one_combined_cap(tmp_path):
 
     assert result.returncode == 0
     assert len(result.stdout) <= 512
-    assert "characters elided" in result.stdout
+    assert "bounded archive" in result.stdout
+
+
+def test_split_stdout_stderr_capture_is_deterministic(tmp_path):
+    from benchmark.process_control import run_capped_process
+
+    command = ["python3", "-c",
+               "import sys; print('x' * 5000); print('y' * 5000, file=sys.stderr)"]
+    results = [run_capped_process(
+        command, shell=False, cwd=str(tmp_path), env=dict(os.environ), timeout=5,
+        max_output_chars=1024) for _ in range(3)]
+    signatures = {(result.stdout, result.stderr, result.original_chars,
+                   result.original_lines, result.capture_truncated) for result in results}
+    assert len(signatures) == 1
+    assert len(results[0].stdout) + len(results[0].stderr) <= 1024
 
 
 def test_budget_contract_rejects_invalid_values():
diff --git a/benchmark/tests/test_observation_policy.py b/benchmark/tests/test_observation_policy.py
new file mode 100644
index 0000000..e17da4c
--- /dev/null
+++ b/benchmark/tests/test_observation_policy.py
@@ -0,0 +1,138 @@
+"""Acceptance tests for deterministic high-information observation packaging."""
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+
+import pytest
+
+from benchmark.observation_policy import ObservationConfig, ObservationStore, POLICY_ID
+
+
+def _fixture() -> str:
+    parts = ["\x1b[32mstarting tests\x1b[0m\n", "progress 1%\rprogress 50%\rprogress 100%\n"]
+    parts.extend("test common_case ... ok\n" for _ in range(300))
+    parts.extend(f"filler {index:04d} " + "x" * 80 + "\n" for index in range(180))
+    parts.extend([
+        "context before one\n",
+        "context before two\n",
+        "SENTINEL_FAILURE_AT_TAIL\n",
+        "assertion failed: left = 17\n",
+        "right = 19\n",
+        "context after one\n",
+        "context after two\n",
+        "2 failed, 325 passed\n",
+    ])
+    return "".join(parts)
+
+
+def _package(tmp_path: Path, text: str, *, returncode: int = 1):
+    work = tmp_path / "work"
+    work.mkdir()
+    store = ObservationStore(
+        tmp_path / "observations",
+        config=ObservationConfig(preview_chars=10_000, archive_chars=64_000),
+        relative_from=work,
+        readable_gid=os.getgid(),
+    )
+    return work, store.package(
+        kind="shell", command="pytest -q", returncode=returncode, timed_out=False,
+        stdout=text, stderr="", original_chars=len(text),
+        original_lines=text.count("\n"), archive_truncated=False)
+
+
+def test_known_terminal_stream_retains_tail_diagnostics_and_exact_metadata(tmp_path, capsys):
+    fixture = _fixture()
+    work, packaged = _package(tmp_path, fixture)
+    preview, metadata = packaged.preview, packaged.metadata
+
+    print(preview)
+    assert len(preview) <= 10_000
+    assert "\x1b" not in preview
+    assert "progress 1%" not in preview and "progress 50%" not in preview
+    assert "progress 100%" in preview
+    assert preview.count("test common_case ... ok") == 1
+    assert "repeated 300 times" in preview
+    for expected in (
+        "context before one", "context before two", "SENTINEL_FAILURE_AT_TAIL",
+        "left = 17", "right = 19", "context after one", "context after two",
+        "2 failed, 325 passed", "returncode=1",
+    ):
+        assert expected in preview
+    assert re.search(r"\[L\d{5}\] SENTINEL_FAILURE_AT_TAIL", preview)
+    assert metadata == {
+        "observation_id": "shell-0001",
+        "kind": "shell",
+        "command": "pytest -q",
+        "policy_id": POLICY_ID,
+        "raw_log": "../observations/shell-0001.log",
+        "returncode": 1,
+        "timed_out": False,
+        "original_chars": len(fixture),
+        "original_lines": fixture.count("\n"),
+        "preview_chars": len(preview),
+        "archive_chars": len(fixture),
+        "preview_compacted": True,
+        "archive_truncated": False,
+    }
+    raw = (work / metadata["raw_log"]).resolve()
+    assert raw.read_bytes().decode("utf-8") == fixture
+    assert raw.stat().st_mode & 0o060 == 0o040
+    assert "SENTINEL_FAILURE_AT_TAIL" in capsys.readouterr().out
+
+
+def test_prefix_only_negative_control_loses_tail_sentinel(tmp_path):
+    fixture = _fixture()
+    prefix_only = fixture[:10_000]
+    assert "SENTINEL_FAILURE_AT_TAIL" not in prefix_only
+    _, packaged = _package(tmp_path, fixture)
+    assert "SENTINEL_FAILURE_AT_TAIL" in packaged.preview
+
+
+def test_middle_diagnostic_survives_when_long_head_and_tail_exceed_preview(tmp_path):
+    head = "".join(f"head-{index} " + "h" * 2_000 + "\n" for index in range(12))
+    middle = "middle context before\nUNIQUE_ERROR_IN_MIDDLE\nmiddle context after\n"
+    tail = "".join(f"tail-{index} " + "t" * 2_000 + "\n" for index in range(12))
+    _, packaged = _package(tmp_path, head + middle + tail)
+
+    assert len(packaged.preview) <= 10_000
+    assert "UNIQUE_ERROR_IN_MIDDLE" in packaged.preview
+    assert "middle context before" in packaged.preview
+    assert "middle context after" in packaged.preview
+
+
+def test_valid_json_is_minified_without_losing_values(tmp_path):
+    text = '{\n  "type": "MaximumIndependentSet",\n  "weights": [1, 2, 3]\n}\n'
+    _, packaged = _package(tmp_path, text, returncode=0)
+    assert '{"type":"MaximumIndependentSet","weights":[1,2,3]}' in packaged.preview
+    assert packaged.metadata["preview_compacted"] is True
+
+
+def test_archive_is_bounded_and_disappears_with_episode_root(tmp_path):
+    episode = tmp_path / "episode"
+    fixture = "head\n" + "x" * 100_000 + "\ntail\n"
+    work = episode / "work"
+    work.mkdir(parents=True)
+    store = ObservationStore(
+        episode / "observations",
+        config=ObservationConfig(preview_chars=1_000, archive_chars=4_000),
+        relative_from=work,
+    )
+    packaged = store.package(
+        kind="shell", command="spam", returncode=0, timed_out=False,
+        stdout=fixture, stderr="", original_chars=len(fixture), original_lines=3,
+        archive_truncated=False)
+    raw = (work / packaged.metadata["raw_log"]).resolve()
+    assert raw.stat().st_size <= 4_000
+    assert packaged.metadata["archive_truncated"] is True
+    assert len(packaged.preview) <= 1_000
+    # Production sessions remove their random episode root before the next episode starts.
+    import shutil
+    shutil.rmtree(episode)
+    assert not raw.exists()
+
+
+def test_caps_too_small_for_observation_metadata_are_rejected():
+    with pytest.raises(ValueError, match="preview_chars >= 256"):
+        ObservationConfig(preview_chars=1, archive_chars=1)
diff --git a/benchmark/tests/test_run_top50.py b/benchmark/tests/test_run_top50.py
index cf57e13..57ea7c7 100644
--- a/benchmark/tests/test_run_top50.py
+++ b/benchmark/tests/test_run_top50.py
@@ -30,7 +30,7 @@ def test_fake_entrypoint_uses_50_isolated_episodes(tmp_path, monkeypatch):
     assert result["rankable"] is False
     artifact = json.loads(output.read_text())
     assert artifact["budget_contract_status"] == "frozen"
-    assert artifact["benchmark_contract"] == "top50-evidence/v1"
+    assert artifact["benchmark_contract"] == "top50-evidence/v2"
 
 
 @pytest.mark.parametrize("forbidden", ["--backend", "--config", "--strategy-file"])
diff --git a/benchmark/tests/test_top50_runner.py b/benchmark/tests/test_top50_runner.py
index 50137a6..971e012 100644
--- a/benchmark/tests/test_top50_runner.py
+++ b/benchmark/tests/test_top50_runner.py
@@ -17,6 +17,7 @@
     format_status,
 )
 from benchmark.evidence_budget import EvidenceBudget
+from benchmark.observation_policy import ObservationConfig
 from benchmark.verify import Verdict
 
 
@@ -29,10 +30,11 @@ def _fake_pred(tmp_path: Path) -> Path:
 
 def _contract() -> Top50Contract:
     return Top50Contract(
-        triage=TriageBudget(model_generations=3, shell_actions=3),
+        triage=TriageBudget(model_generations=3, shell_actions=3, max_output_chars=1024),
         episode=EvidenceBudget(
             model_generations=2, shell_actions=2, pred_calls=1, solve_calls=0,
             submit_attempts=2, max_output_chars=1024, pred_timeout_seconds=2),
+        observation=ObservationConfig(preview_chars=1024),
     )
 
 
@@ -175,6 +177,32 @@ def test_observation_status_reports_every_authoritative_counter(tmp_path):
         assert expected in text
 
 
+def test_triage_builtin_and_error_results_use_observation_policy(tmp_path):
+    import os
+    from benchmark.top50_runner import TriageEnvironment, TriageSession
+
+    inventory = tuple(f"rule_{index:02d}" for index in range(60))
+    with TriageSession(inventory=inventory, budget=TriageBudget(2, 3)) as session:
+        shortlist = session.workdir / "shortlist.json"
+        shortlist.write_text(json.dumps(list(inventory[:50])))
+        environment = TriageEnvironment(session, uid=os.getuid(), gid=os.getgid())
+        outputs = [
+            environment.execute({"command": "commit-top50 shortlist.json"}),
+            environment.execute({"command": "'"}),
+            environment.execute({"command": "commit-top50"}),
+            environment.execute({"command": "pwd"}),
+        ]
+
+        assert [output["returncode"] for output in outputs] == [0, 2, 2, 75]
+        assert all("policy=terminal-diagnostics/v1" in output["output"]
+                   and "[raw log:" in output["output"] for output in outputs)
+        ledger = session.ledger()
+        shell_events = [event for event in ledger["events"]
+                        if event.get("type") == "shell_action"]
+        assert len(ledger["observations"]) == len(shell_events) == 4
+        assert all(event.get("observation_id") for event in shell_events)
+
+
 def test_rankable_contract_is_fixed_to_model_api_surface():
     executor = FakeExecutor()
     assert not hasattr(executor, "backend")
@@ -221,3 +249,83 @@ def test_successful_shell_cannot_leave_background_process(tmp_path):
     assert result.returncode == 0
     time.sleep(0.4)
     assert not sentinel.exists()
+
+
+def test_agent_can_inspect_read_only_raw_observation_with_charged_action(tmp_path):
+    import os
+    import re
+    from benchmark.agent_environment import run_as_agent, sanitized_agent_env
+    from benchmark.evidence_budget import EvidenceBudgetSession
+
+    with EvidenceBudgetSession(
+        rule="rule_00", budget=_contract().episode, pred_binary=_fake_pred(tmp_path),
+        verifier=lambda cert: Verdict(False, "no bug"),
+    ) as session:
+        first_command = "printf 'important evidence\\n'"
+        assert session.admit_shell_action(first_command)
+        first = run_as_agent(
+            first_command, cwd=str(session.workdir), env=sanitized_agent_env(), timeout=2,
+            uid=os.getuid(), gid=os.getgid(), observation_store=session.observations)
+        session.record_shell_observation(first_command, first.observation_metadata)
+        reference = re.search(r"\[raw log: ([^;]+);", first.stdout).group(1)
+        second_command = f"cat {reference}"
+        assert session.admit_shell_action(second_command)
+        second = run_as_agent(
+            second_command, cwd=str(session.workdir), env=sanitized_agent_env(), timeout=2,
+            uid=os.getuid(), gid=os.getgid(), observation_store=session.observations)
+        session.record_shell_observation(second_command, second.observation_metadata)
+
+        assert "important evidence" in second.stdout
+        assert session.status()["shell_actions"]["used"] == 2
+        assert len(session.ledger()["observations"]) == 2
+        raw = (session.workdir / reference).resolve()
+        assert raw.stat().st_mode & 0o060 in (0, 0o040)
+
+
+@pytest.mark.integration
+def test_distinct_agent_identity_cannot_mutate_or_reach_previous_raw_log(tmp_path):
+    import os
+    import sys
+    if os.geteuid() != 0 or not sys.platform.startswith("linux"):
+        pytest.skip("requires the rankable Linux root privilege boundary")
+    from benchmark.agent_environment import run_as_agent, sanitized_agent_env
+    from benchmark.evidence_budget import EvidenceBudgetSession
+
+    agent_uid, agent_gid, evidence_gid = 61_001, 61_001, 61_003
+    common = {
+        "uid": agent_uid, "gid": agent_gid, "extra_groups": (evidence_gid,),
+        "env": sanitized_agent_env(), "timeout": 2,
+    }
+    old_raw = None
+    with EvidenceBudgetSession(
+        rule="rule_00", budget=_contract().episode, pred_binary=_fake_pred(tmp_path),
+        verifier=lambda cert: Verdict(False, "no bug"),
+        agent_uid=agent_uid, agent_gid=agent_gid,
+        oracle_uid=0, oracle_gid=0, evidence_gid=evidence_gid,
+        observation_config=_contract().observation,
+    ) as session:
+        command = "printf 'protected evidence\\n'"
+        assert session.admit_shell_action(command)
+        first = run_as_agent(
+            command, cwd=str(session.workdir), observation_store=session.observations, **common)
+        session.record_shell_observation(command, first.observation_metadata)
+        old_raw = (session.workdir / first.observation_metadata["raw_log"]).resolve()
+        assert run_as_agent(
+            f"cat {old_raw}", cwd=str(session.workdir), **common).returncode == 0
+        assert run_as_agent(
+            f"printf x >> {old_raw}", cwd=str(session.workdir), **common).returncode != 0
+        assert run_as_agent(
+            f"rm {old_raw}", cwd=str(session.workdir), **common).returncode != 0
+        assert run_as_agent(
+            f"mv {old_raw} {old_raw}.moved", cwd=str(session.workdir), **common).returncode != 0
+    assert old_raw is not None and not old_raw.exists()
+
+    with EvidenceBudgetSession(
+        rule="rule_01", budget=_contract().episode, pred_binary=_fake_pred(tmp_path),
+        verifier=lambda cert: Verdict(False, "no bug"),
+        agent_uid=agent_uid, agent_gid=agent_gid,
+        oracle_uid=0, oracle_gid=0, evidence_gid=evidence_gid,
+        observation_config=_contract().observation,
+    ) as second:
+        assert run_as_agent(
+            f"cat {old_raw}", cwd=str(second.workdir), **common).returncode != 0
diff --git a/benchmark/tests/test_top50_submission.py b/benchmark/tests/test_top50_submission.py
index 2014ad4..02fd227 100644
--- a/benchmark/tests/test_top50_submission.py
+++ b/benchmark/tests/test_top50_submission.py
@@ -27,6 +27,16 @@ def _status(limit: int, used: int = 0) -> dict:
 
 
 def _artifact(accepted_positions=(7, 18, 41)) -> dict:
+    observation = {"policy_id": "terminal-diagnostics/v1", "preview_chars": 10_000,
+                   "archive_chars": 1_048_576}
+    triage_observation = {
+        "observation_id": "shell-0001", "kind": "shell",
+        "command": "commit-top50 shortlist.json", "policy_id": observation["policy_id"],
+        "raw_log": "../observations/shell-0001.log", "returncode": 0,
+        "timed_out": False, "original_chars": 16, "original_lines": 1,
+        "preview_chars": 200, "archive_chars": 16, "preview_compacted": False,
+        "archive_truncated": False,
+    }
     triage_budget = {
         "model_generations": 8, "shell_actions": 12,
         "max_output_chars": 10_000, "command_timeout_seconds": 300,
@@ -44,10 +54,14 @@ def _artifact(accepted_positions=(7, 18, 41)) -> dict:
     triage = {
         "ledger": {
             "budget": triage_budget,
+            "observation_policy": copy.deepcopy(observation),
+            "observations": [triage_observation],
             "status": triage_status,
             "events": [
                 {"sequence": 1, "type": "model_generation", "charged": True},
-                {"sequence": 2, "type": "shell_action", "charged": True},
+                {"sequence": 2, "type": "shell_action", "charged": True,
+                 "command": "commit-top50 shortlist.json", "outcome": "completed",
+                 "observation_id": "shell-0001"},
                 {"type": "shortlist_commit", "accepted": True},
             ],
             "shortlist": copy.deepcopy(shortlist),
@@ -84,6 +98,8 @@ def _artifact(accepted_positions=(7, 18, 41)) -> dict:
             "index": position, "rule": entry["rule"], "hypothesis": entry["hypothesis"],
             "status": status, "accepted_submit_attempt": 1 if attempts else None,
             "ledger": {"rule": entry["rule"], "budget": copy.deepcopy(episode_budget),
+                       "observation_policy": copy.deepcopy(observation),
+                       "observations": [],
                        "status": ledger_status, "pred": [], "submit": attempts,
                        "model_generations": [], "shell_actions": []},
             "messages": [], "tokens_k": 0.01,
@@ -99,10 +115,12 @@ def _artifact(accepted_positions=(7, 18, 41)) -> dict:
         "prompt_id": expected_prompt_id(),
         "budget_contract_status": "frozen",
         "safety_controls": {"model_timeout_seconds": 300, "model_retries": 2},
+        "observation_policy": copy.deepcopy(observation),
         "status": "completed",
         "rankable": True,
         "inference_parameters": {"max_tokens": 8192, "timeout": 300, "num_retries": 2},
         "contract": {"triage": triage_budget, "episode": episode_budget,
+                     "observation": copy.deepcopy(observation),
                      "shortlist_size": 50, "hypothesis_chars": 500},
         "shortlist": shortlist,
         "triage": triage,
@@ -133,6 +151,71 @@ def test_valid_artifact_recomputes_score_and_prefix_metrics():
     assert validate_submission(submission) == []
 
 
+@pytest.mark.parametrize("mutation", [
+    lambda sub: sub["triage"]["ledger"]["observations"].clear(),
+    lambda sub: sub["triage"]["ledger"]["events"][1].pop("observation_id"),
+    lambda sub: sub["triage"]["ledger"]["observations"][0].update(command="forged"),
+    lambda sub: sub["triage"]["ledger"]["observations"].append({
+        **sub["triage"]["ledger"]["observations"][0],
+        "observation_id": "shell-9999",
+        "raw_log": "../observations/shell-9999.log",
+    }),
+])
+def test_observation_ledger_must_be_bijective_with_actions(mutation):
+    submission = _artifact()
+    mutation(submission)
+    errors = " ".join(validate_top50_submission(submission))
+    assert "observation" in errors and errors
+
+
+@pytest.mark.parametrize("mutation", [
+    lambda record: record.update(returncode=9),
+    lambda record: record.update(timed_out=True),
+    lambda record: record.update(observation_id=[]),
+])
+def test_shell_observation_exact_metadata_is_checked_without_crashing(mutation):
+    submission = _artifact()
+    record = submission["triage"]["ledger"]["observations"][0]
+    mutation(record)
+    errors = " ".join(validate_top50_submission(submission))
+    assert "observation" in errors and errors
+
+
+def test_pred_observation_command_and_outcome_are_derived_from_action_record():
+    submission = _artifact()
+    ledger = submission["episodes"][0]["ledger"]
+    request_id = "f" * 32
+    observation = {
+        "observation_id": f"pred-{request_id}", "kind": "pred",
+        "command": "pred create X", "policy_id": "terminal-diagnostics/v1",
+        "raw_log": f"../observations/pred-{request_id}.log", "returncode": 0,
+        "timed_out": False, "original_chars": 10, "original_lines": 1,
+        "preview_chars": 200, "archive_chars": 10, "preview_compacted": False,
+        "archive_truncated": False,
+    }
+    ledger["status"]["pred_calls"] = _status(24, 1)
+    ledger["pred"] = [{
+        "sequence": 1, "request_id": request_id, "args": ["create", "X"],
+        "command": "create", "charged": True, "outcome": "completed", "returncode": 0,
+        "budget": copy.deepcopy(ledger["status"]),
+        "observation": copy.deepcopy(observation),
+    }]
+    ledger["observations"].append(copy.deepcopy(observation))
+    assert validate_top50_submission(submission) == []
+
+    ledger["pred"][0]["observation"]["command"] = "pred forged"
+    ledger["observations"][0]["command"] = "pred forged"
+    assert "pred observation does not match" in " ".join(
+        validate_top50_submission(submission))
+
+    ledger["pred"][0]["observation"]["command"] = "pred create X"
+    ledger["observations"][0]["command"] = "pred create X"
+    ledger["pred"][0]["observation"]["timed_out"] = True
+    ledger["observations"][0]["timed_out"] = True
+    assert "pred observation does not match" in " ".join(
+        validate_top50_submission(submission))
+
+
 @pytest.mark.parametrize("mutate, expected", [
     (lambda sub: sub.update(submit_limit=100), "shared run-wide submit pool"),
     (lambda sub: sub["episodes"].pop(), "exactly 50"),
@@ -147,6 +230,8 @@ def test_valid_artifact_recomputes_score_and_prefix_metrics():
     (lambda sub: sub.update(agent_mode="codex"), "agent_mode"),
     (lambda sub: sub.update(prompt_id="custom"), "prompt_id"),
     (lambda sub: sub.pop("inference_parameters"), "inference_parameters"),
+    (lambda sub: sub["observation_policy"].update(policy_id="custom"),
+     "observation_policy"),
     (lambda sub: sub.update(status="run_error", run_error="provider failed"), "incomplete"),
 ])
 def test_rankability_negative_controls(mutate, expected):
diff --git a/benchmark/top50_budget.json b/benchmark/top50_budget.json
index 2818800..d8bd4f2 100644
--- a/benchmark/top50_budget.json
+++ b/benchmark/top50_budget.json
@@ -1,5 +1,5 @@
 {
-  "contract_id": "top50-evidence/v1",
+  "contract_id": "top50-evidence/v2",
   "status": "frozen",
   "shortlist_size": 50,
   "hypothesis_chars": 500,
@@ -12,6 +12,11 @@
     "timeout": 300,
     "num_retries": 2
   },
+  "observation": {
+    "policy_id": "terminal-diagnostics/v1",
+    "preview_chars": 10000,
+    "archive_chars": 1048576
+  },
   "triage": {
     "model_generations": 8,
     "shell_actions": 12,
diff --git a/benchmark/top50_config.yaml b/benchmark/top50_config.yaml
index 1092ea3..e786a45 100644
--- a/benchmark/top50_config.yaml
+++ b/benchmark/top50_config.yaml
@@ -3,7 +3,10 @@ agent:
     You are evaluating reduction rules under a fixed logical evidence budget. Each response
     must contain exactly one bash command. The authoritative remaining budget is injected into
     every observation; elapsed time is not a score and unused budget never transfers.
-    This is frozen contract top50-evidence/v1. The prompt, strategy, shortlist size, and logical
+    This is frozen contract top50-evidence/v2 with observation policy terminal-diagnostics/v1.
+    Command output is deterministically compacted into a 10,000-character preview. Each preview
+    names an evaluation-owned raw log that you may inspect with an ordinary shell action.
+    The prompt, strategy, shortlist size, and logical
     counters cannot be customized on a rankable run.
 
     {% if phase == "triage" %}
diff --git a/benchmark/top50_contract.py b/benchmark/top50_contract.py
index fd35e14..7c55771 100644
--- a/benchmark/top50_contract.py
+++ b/benchmark/top50_contract.py
@@ -7,6 +7,7 @@
 import json
 import os
 import re
+import shlex
 from collections import Counter
 from pathlib import Path
 from typing import Callable
@@ -17,11 +18,12 @@
 
 CONTRACT_ID = FROZEN_CONTRACT["contract_id"]
 AGENT_MODE = "standardized-model-api"
-RUNNER_VERSION = "0.9.0"
+RUNNER_VERSION = "0.10.0"
 EXPECTED_TRIAGE_BUDGET = FROZEN_CONTRACT["triage"]
 EXPECTED_EPISODE_BUDGET = FROZEN_CONTRACT["episode"]
 EXPECTED_SAFETY_CONTROLS = FROZEN_CONTRACT["safety_controls"]
 EXPECTED_INFERENCE_PARAMETERS = FROZEN_CONTRACT["inference_parameters"]
+EXPECTED_OBSERVATION = FROZEN_CONTRACT["observation"]
 EXPECTED_SHORTLIST_SIZE = FROZEN_CONTRACT["shortlist_size"]
 EXPECTED_HYPOTHESIS_CHARS = FROZEN_CONTRACT["hypothesis_chars"]
 _REQUEST_ID = re.compile(r"[0-9a-f]{32}")
@@ -47,7 +49,7 @@ def validate_top50_submission(
     for field in ("benchmark_contract", "model", "library_commit", "runner_version",
                   "pred_version", "agent_mode", "prompt_id", "contract", "shortlist",
                   "triage", "episodes", "status", "budget_contract_status",
-                  "safety_controls", "inference_parameters"):
+                  "safety_controls", "inference_parameters", "observation_policy"):
         if field not in submission:
             problems.append(f"missing required field: {field}")
     if problems:
@@ -75,6 +77,8 @@ def validate_top50_submission(
         problems.append("submitted_by is not a bounded safe identifier")
     if submission.get("inference_parameters") != EXPECTED_INFERENCE_PARAMETERS:
         problems.append("inference_parameters differ from the standardized model settings")
+    if submission.get("observation_policy") != EXPECTED_OBSERVATION:
+        problems.append("observation_policy differs from the versioned contract")
     if submission.get("prompt_id") != expected_prompt_id():
         problems.append("prompt_id does not match the frozen Top50 prompt")
     if "submit_limit" in submission or "submit_log" in submission:
@@ -85,12 +89,15 @@ def validate_top50_submission(
         return problems + ["contract must be an object"]
     triage_budget = contract.get("triage")
     episode_budget = contract.get("episode")
+    observation_contract = contract.get("observation")
     if not isinstance(triage_budget, dict) or not isinstance(episode_budget, dict):
         return problems + ["contract triage and episode budgets must be objects"]
     if triage_budget != EXPECTED_TRIAGE_BUDGET:
         problems.append("triage budget differs from the versioned contract")
     if episode_budget != EXPECTED_EPISODE_BUDGET:
         problems.append("episode budget differs from the versioned contract")
+    if observation_contract != EXPECTED_OBSERVATION:
+        problems.append("observation contract differs from the versioned contract")
     if contract.get("shortlist_size") != EXPECTED_SHORTLIST_SIZE:
         problems.append(f"contract shortlist_size must be {EXPECTED_SHORTLIST_SIZE}")
     hypothesis_limit = contract.get("hypothesis_chars")
@@ -136,6 +143,7 @@ def validate_top50_submission(
     else:
         if triage_ledger.get("budget") != triage_budget:
             problems.append("triage ledger budget differs from contract")
+        _validate_observation_ledger(triage_ledger, "triage", problems)
         frozen = triage_ledger.get("shortlist")
         if frozen != shortlist:
             problems.append("top-level shortlist differs from frozen triage shortlist")
@@ -175,6 +183,7 @@ def validate_top50_submission(
             problems.append(f"{label} contains an oversized message")
         _validate_status(ledger.get("status"), episode_budget, label, problems)
         _validate_episode_events(ledger, label, problems)
+        _validate_observation_ledger(ledger, label, problems)
         _validate_pred_ledger(ledger, label, seen_request_ids, problems)
         _validate_submit_ledger(ledger, expected_rule, label, seen_request_ids, problems)
         accepted_attempts = [attempt for attempt in ledger.get("submit", [])
@@ -388,6 +397,118 @@ def _validate_episode_events(ledger, label, problems) -> None:
             problems.append(f"{label} {field} charged count differs from status")
 
 
+def _validate_observation_ledger(ledger: dict, label: str, problems: list[str]) -> None:
+    if ledger.get("observation_policy") != EXPECTED_OBSERVATION:
+        problems.append(f"{label} observation policy differs from contract")
+    records = ledger.get("observations")
+    if not isinstance(records, list):
+        problems.append(f"{label} observations must be a list")
+        return
+    shell_events = (ledger.get("events", []) if label == "triage"
+                    else ledger.get("shell_actions", []))
+    shell_events = [event for event in shell_events if isinstance(event, dict)
+                    and (label != "triage" or event.get("type") == "shell_action")]
+    pred_records = ledger.get("pred", []) if label != "triage" else []
+    maximum = len(shell_events) + (len(pred_records) if isinstance(pred_records, list) else 0)
+    if len(records) > maximum:
+        problems.append(f"{label} observation ledger exceeds its logical bound")
+    seen: set[str] = set()
+    record_by_id: dict[str, dict] = {}
+    required = {
+        "observation_id", "kind", "command", "policy_id", "raw_log", "returncode",
+        "timed_out", "original_chars", "original_lines", "preview_chars", "archive_chars",
+        "preview_compacted", "archive_truncated",
+    }
+    for record in records:
+        if not isinstance(record, dict) or set(record) != required:
+            problems.append(f"{label} has malformed observation metadata")
+            continue
+        observation_id = record["observation_id"]
+        if (not isinstance(observation_id, str)
+                or not re.fullmatch(r"(?:shell-[0-9]{4}|pred-[0-9a-f]{32})", observation_id)):
+            problems.append(f"{label} has invalid observation id")
+            continue
+        if observation_id in seen:
+            problems.append(f"{label} has invalid observation id")
+            continue
+        seen.add(observation_id)
+        record_by_id[observation_id] = record
+        if record["policy_id"] != EXPECTED_OBSERVATION["policy_id"]:
+            problems.append(f"{label} observation policy id is inconsistent")
+        if (not _nonnegative_int(record["preview_chars"])
+                or record["preview_chars"] > EXPECTED_OBSERVATION["preview_chars"]):
+            problems.append(f"{label} observation preview exceeds contract")
+        if (not _nonnegative_int(record["archive_chars"])
+                or record["archive_chars"] > EXPECTED_OBSERVATION["archive_chars"]):
+            problems.append(f"{label} observation archive exceeds contract")
+        if not _nonnegative_int(record["original_chars"]):
+            problems.append(f"{label} observation original size is invalid")
+        if not _nonnegative_int(record["original_lines"]):
+            problems.append(f"{label} observation original line count is invalid")
+        if (record["kind"] not in ("shell", "pred")
+                or not isinstance(record["command"], str)
+                or type(record["returncode"]) is not int
+                or not isinstance(record["timed_out"], bool)
+                or not isinstance(record["preview_compacted"], bool)
+                or not isinstance(record["archive_truncated"], bool)):
+            problems.append(f"{label} observation fields have invalid types")
+        expected_path = f"../observations/{observation_id}.log"
+        if record["raw_log"] != expected_path:
+            problems.append(f"{label} observation raw-log reference is invalid")
+
+    referenced: list[str] = []
+    for event in shell_events:
+        observation_id = event.get("observation_id")
+        record = record_by_id.get(observation_id)
+        if record is None:
+            problems.append(f"{label} shell event is missing its observation")
+            continue
+        referenced.append(observation_id)
+        if record["kind"] != "shell" or record["command"] != event.get("command"):
+            problems.append(f"{label} shell observation does not match its event")
+        if not _observation_matches_outcome(record, event.get("outcome")):
+            problems.append(f"{label} shell observation outcome is inconsistent")
+    if isinstance(pred_records, list):
+        for pred_record in pred_records:
+            if not isinstance(pred_record, dict):
+                continue
+            embedded = pred_record.get("observation")
+            requires_observation = pred_record.get("outcome") in {
+                "completed", "nonzero_exit", "timeout"}
+            if embedded is None:
+                if requires_observation:
+                    problems.append(f"{label} pred record is missing its observation")
+                continue
+            if not isinstance(embedded, dict):
+                problems.append(f"{label} pred observation is malformed")
+                continue
+            observation_id = embedded.get("observation_id")
+            referenced.append(observation_id)
+            if (record_by_id.get(observation_id) != embedded
+                    or embedded.get("kind") != "pred"
+                    or not _pred_command_matches(embedded, pred_record)
+                    or embedded.get("returncode") != pred_record.get("returncode")
+                    or not _observation_matches_outcome(embedded, pred_record.get("outcome"))):
+                problems.append(f"{label} pred observation does not match its record")
+    if len(referenced) != len(set(referenced)) or set(referenced) != set(record_by_id):
+        problems.append(f"{label} observation ledger is not bijective with action records")
+
+
+def _observation_matches_outcome(record: dict, outcome: object) -> bool:
+    return {
+        "completed": record["returncode"] == 0 and record["timed_out"] is False,
+        "nonzero_exit": record["returncode"] != 0 and record["timed_out"] is False,
+        "timeout": record["returncode"] == 124 and record["timed_out"] is True,
+        "budget_exhausted": record["returncode"] == 75 and record["timed_out"] is False,
+    }.get(outcome, False)
+
+
+def _pred_command_matches(observation: dict, pred_record: dict) -> bool:
+    args = pred_record.get("args")
+    return (isinstance(args, list) and all(isinstance(arg, str) for arg in args)
+            and observation.get("command") == "pred " + shlex.join(args))
+
+
 def _validate_submit_ledger(ledger, expected_rule, label, seen_ids, problems) -> None:
     attempts = ledger.get("submit")
     if not isinstance(attempts, list) or len(attempts) > 2:
diff --git a/benchmark/top50_results.schema.json b/benchmark/top50_results.schema.json
index 51da291..30fa87a 100644
--- a/benchmark/top50_results.schema.json
+++ b/benchmark/top50_results.schema.json
@@ -8,7 +8,7 @@
     "first_attempt_accepts", "second_attempt_accepts", "usage_totals",
     "artifact_sha256", "verifier_report"],
   "properties": {
-    "benchmark_contract": {"const": "top50-evidence/v1"},
+    "benchmark_contract": {"const": "top50-evidence/v2"},
     "model": {"type": "string"},
     "library_commit": {"type": "string"},
     "rankable": {"type": "boolean"},
diff --git a/benchmark/top50_runner.py b/benchmark/top50_runner.py
index eb74cb9..69bcf37 100644
--- a/benchmark/top50_runner.py
+++ b/benchmark/top50_runner.py
@@ -12,8 +12,10 @@
 from pathlib import Path
 from typing import Callable, Protocol
 
-from benchmark.agent_environment import make_agent_environment, run_as_agent, sanitized_agent_env
+from benchmark.agent_environment import (make_agent_environment, package_shell_result,
+                                         run_as_agent, sanitized_agent_env)
 from benchmark.evidence_budget import EvidenceBudget, EvidenceBudgetSession, EvidenceBudgetState
+from benchmark.observation_policy import ObservationConfig, ObservationStore
 from benchmark.run_mini import (
     DEFAULT_MAX_TOKENS,
     _build_model,
@@ -48,6 +50,7 @@ def __post_init__(self) -> None:
 class Top50Contract:
     triage: TriageBudget
     episode: EvidenceBudget
+    observation: ObservationConfig = ObservationConfig()
     shortlist_size: int = TOP50_SIZE
     hypothesis_chars: int = DEFAULT_HYPOTHESIS_CHARS
 
@@ -56,6 +59,10 @@ def __post_init__(self) -> None:
             raise ValueError("the rankable contract requires exactly 50 rules")
         if self.hypothesis_chars <= 0:
             raise ValueError("hypothesis_chars must be positive")
+        caps = {self.triage.max_output_chars, self.episode.max_output_chars,
+                self.observation.preview_chars}
+        if len(caps) != 1:
+            raise ValueError("triage, episode, and observation preview caps must match")
 
 
 @dataclass(frozen=True)
@@ -87,7 +94,9 @@ class TriageSession:
     def __init__(self, *, inventory: tuple[str, ...], budget: TriageBudget,
                  shortlist_size: int = TOP50_SIZE,
                  hypothesis_chars: int = DEFAULT_HYPOTHESIS_CHARS,
-                 agent_uid: int | None = None, agent_gid: int | None = None):
+                 agent_uid: int | None = None, agent_gid: int | None = None,
+                 evidence_gid: int | None = None,
+                 observation_config: ObservationConfig | None = None):
         if len(set(inventory)) != len(inventory):
             raise ValueError("canonical inventory contains duplicates")
         if (agent_uid is None) != (agent_gid is None):
@@ -98,6 +107,9 @@ def __init__(self, *, inventory: tuple[str, ...], budget: TriageBudget,
         self.hypothesis_chars = hypothesis_chars
         self.agent_uid = agent_uid
         self.agent_gid = agent_gid
+        self.evidence_gid = evidence_gid
+        self.observation_config = observation_config or ObservationConfig(
+            preview_chars=budget.max_output_chars)
         evidence = EvidenceBudget(
             model_generations=budget.model_generations,
             shell_actions=budget.shell_actions,
@@ -112,6 +124,8 @@ def __init__(self, *, inventory: tuple[str, ...], budget: TriageBudget,
         self._workdir: Path | None = None
         self._shortlist: tuple[ShortlistEntry, ...] | None = None
         self._events: list[dict] = []
+        self._observations: list[dict] = []
+        self.observations: ObservationStore | None = None
         self._lock = threading.RLock()
 
     @property
@@ -129,6 +143,9 @@ def __enter__(self) -> "TriageSession":
         self._tmpdir = Path(tempfile.mkdtemp(prefix="prb-triage-", dir="/tmp")).resolve()
         self._workdir = self._tmpdir / "work"
         self._workdir.mkdir(mode=0o700)
+        self.observations = ObservationStore(
+            self._tmpdir / "observations", config=self.observation_config,
+            relative_from=self._workdir, readable_gid=self.evidence_gid)
         if self.agent_uid is not None and self.agent_gid is not None:
             self._tmpdir.chmod(0o711)
             os.chown(self._workdir, self.agent_uid, self.agent_gid)
@@ -187,6 +204,8 @@ def status(self) -> dict:
     def ledger(self) -> dict:
         with self._lock:
             return {"budget": asdict(self.budget), "status": self.status(),
+                    "observation_policy": asdict(self.observation_config),
+                    "observations": copy.deepcopy(self._observations),
                     "events": copy.deepcopy(self._events),
                     "shortlist": ([asdict(entry) for entry in self._shortlist]
                                   if self._shortlist is not None else None)}
@@ -219,6 +238,21 @@ def _append_event(self, event_type: str, charged: bool, outcome: str, **extra) -
                                  "charged": charged, "outcome": outcome,
                                  "budget": self.status(), **extra})
 
+    def record_shell_observation(self, command: str, metadata: dict | None) -> None:
+        if metadata is None:
+            return
+        with self._lock:
+            self._observations.append(copy.deepcopy(metadata))
+            for event in reversed(self._events):
+                if (event.get("type") == "shell_action" and event.get("command") == command
+                        and "observation_id" not in event):
+                    event["observation_id"] = metadata["observation_id"]
+                    if event.get("charged") is True:
+                        event["outcome"] = ("timeout" if metadata["timed_out"] else
+                                            "completed" if metadata["returncode"] == 0 else
+                                            "nonzero_exit")
+                    break
+
 
 class Top50Runner:
     """Freeze one self-selected Top50, then execute 50 fresh sequential episodes."""
@@ -250,6 +284,8 @@ def run(self, *, model: str, repo_path: str | Path,
             hypothesis_chars=self.contract.hypothesis_chars,
             agent_uid=self.identities["agent_uid"],
             agent_gid=self.identities["agent_gid"],
+            evidence_gid=self.identities["evidence_gid"],
+            observation_config=self.contract.observation,
         ) as triage:
             try:
                 triage_result = self.executor.run_triage(
@@ -279,6 +315,7 @@ def run(self, *, model: str, repo_path: str | Path,
                     pred_binary=self.pred_binary,
                     verifier=self.verifier,
                     **self.identities,
+                    observation_config=self.contract.observation,
                 ) as episode:
                     phase = self.executor.run_episode(
                         episode, repo_path=repo_path, entry=entry,
@@ -324,6 +361,7 @@ def _result(self, model: str, triage: dict,
                          and len(episodes) == self.contract.shortlist_size),
             "contract": {"triage": asdict(self.contract.triage),
                          "episode": asdict(self.contract.episode),
+                         "observation": asdict(self.contract.observation),
                          "shortlist_size": self.contract.shortlist_size,
                          "hypothesis_chars": self.contract.hypothesis_chars},
             "shortlist": ([asdict(entry) for entry in shortlist] if shortlist else None),
@@ -391,30 +429,42 @@ def __init__(self, session: TriageSession, *, uid: int, gid: int,
     def execute(self, action: dict, cwd: str = "", *, timeout: int | None = None) -> dict:
         command = action.get("command", "")
         if not self.session.admit_shell_action(command):
-            return {"output": "shell action budget exhausted\n", "returncode": 75,
-                    "exception_info": ""}
+            return package_shell_result(
+                self.session, command, output="shell action budget exhausted\n", returncode=75)
         try:
             words = shlex.split(command)
         except ValueError as error:
-            return {"output": str(error), "returncode": 2, "exception_info": ""}
+            return package_shell_result(
+                self.session, command, output=str(error) + "\n", returncode=2)
         if words and words[0] == "commit-top50":
             if len(words) != 2:
-                return {"output": "usage: commit-top50 SHORTLIST.json\n", "returncode": 2,
-                        "exception_info": ""}
+                return package_shell_result(
+                    self.session, command, output="usage: commit-top50 SHORTLIST.json\n",
+                    returncode=2)
             accepted, message = self.session.commit_file(words[1])
-            return {"output": message + "\n", "returncode": 0 if accepted else 2,
-                    "exception_info": ""}
+            return package_shell_result(
+                self.session, command, output=message + "\n",
+                returncode=0 if accepted else 2)
         try:
             result = run_as_agent(
                 command, cwd=cwd or str(self.session.workdir), env=sanitized_agent_env(),
                 timeout=timeout or self.session.budget.command_timeout_seconds,
                 uid=self.uid, gid=self.gid, extra_groups=self.extra_groups,
-                max_output_chars=self.session.budget.max_output_chars)
+                max_output_chars=self.session.budget.max_output_chars,
+                observation_store=self.session.observations)
+            self.session.record_shell_observation(command, result.observation_metadata)
             return {"output": result.stdout, "returncode": result.returncode,
                     "exception_info": ""}
         except Exception as error:
-            return {"output": getattr(error, "output", "") or "", "returncode": -1,
-                    "exception_info": f"{type(error).__name__}: {error}"}
+            raw_output = getattr(error, "output", "") or ""
+            metadata = getattr(error, "observation_metadata", None)
+            if metadata is not None:
+                self.session.record_shell_observation(command, metadata)
+                return {"output": raw_output, "returncode": -1,
+                        "exception_info": f"{type(error).__name__}: {error}"}
+            return package_shell_result(
+                self.session, command, output=raw_output, returncode=-1,
+                exception_info=f"{type(error).__name__}: {error}")
 
     def get_template_vars(self, **kwargs) -> dict:
         return {"cwd": str(self.session.workdir), **kwargs}
diff --git a/benchmark/top50_submission.schema.json b/benchmark/top50_submission.schema.json
index 879c69c..667a841 100644
--- a/benchmark/top50_submission.schema.json
+++ b/benchmark/top50_submission.schema.json
@@ -4,24 +4,26 @@
   "title": "Top50EvidenceSubmission",
   "type": "object",
   "required": ["benchmark_contract", "model", "library_commit", "runner_version",
-    "pred_version", "agent_mode", "prompt_id", "status", "budget_contract_status", "safety_controls", "contract", "shortlist",
+    "pred_version", "agent_mode", "prompt_id", "status", "budget_contract_status", "safety_controls", "observation_policy", "contract", "shortlist",
     "triage", "episodes", "inference_parameters"],
   "properties": {
-    "benchmark_contract": {"const": "top50-evidence/v1"},
+    "benchmark_contract": {"const": "top50-evidence/v2"},
     "model": {"type": "string", "minLength": 1},
     "library_commit": {"type": "string", "minLength": 1},
-    "runner_version": {"const": "0.9.0"},
+    "runner_version": {"const": "0.10.0"},
     "pred_version": {"type": "string", "minLength": 1},
     "agent_mode": {"const": "standardized-model-api"},
     "prompt_id": {"type": "string", "minLength": 64, "maxLength": 64},
     "status": {"enum": ["completed", "run_error"]},
     "budget_contract_status": {"const": "frozen"},
     "safety_controls": {"const": {"model_timeout_seconds": 300, "model_retries": 2}},
+    "observation_policy": {"const": {"policy_id": "terminal-diagnostics/v1", "preview_chars": 10000, "archive_chars": 1048576}},
     "rankable": {"type": "boolean"},
     "contract": {
       "const": {
         "triage": {"model_generations": 8, "shell_actions": 12, "max_output_chars": 10000, "command_timeout_seconds": 300},
         "episode": {"model_generations": 10, "shell_actions": 12, "pred_calls": 24, "solve_calls": 10, "submit_attempts": 2, "max_output_chars": 10000, "pred_timeout_seconds": 300},
+        "observation": {"policy_id": "terminal-diagnostics/v1", "preview_chars": 10000, "archive_chars": 1048576},
         "shortlist_size": 50,
         "hypothesis_chars": 500
       }
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 38e4973..5947623 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -54,7 +54,7 @@ LABEL org.opencontainers.image.title="problem-reductions-benchmark" \
       org.opencontainers.image.description="pred (problem-reductions ${PR_REF}) + zero-trust certificate verifier" \
       org.opencontainers.image.licenses="MIT" \
       problem_reductions_ref="${PR_REF}" \
-      benchmark_contract="top50-evidence/v1"
+      benchmark_contract="top50-evidence/v2"
 
 RUN groupadd --gid 10003 prb-evidence \
     && groupadd --gid 10001 prb-agent \
diff --git a/site/index.html b/site/index.html
index f7e549d..980f2e0 100644
--- a/site/index.html
+++ b/site/index.html
@@ -416,7 +416,7 @@ 
How to cite

   {key:"predBug",label:"Pred / bug",num:true},
 ];
 const RIGHT=new Set(["bugs","tokens","eff","bugs10","bugs25","predBug"]);
-const selectedTrack=()=>document.getElementById("lb-track").value||"top50-evidence/v1";
+const selectedTrack=()=>document.getElementById("lb-track").value||"top50-evidence/v2";
 const isTop50Track=()=>selectedTrack().startsWith("top50-evidence/");
 const lbColumns=()=>isTop50Track()?TOP50_COLS:LEGACY_COLS;
 
@@ -619,7 +619,7 @@ 
How to cite

   fetch("tasks.json").then(r=>r.json()).catch(()=>[]),
 ]).then(([res,tasks])=>{
   RESULTS=res;TASKS=tasks;TOTAL=tasks.length||TOTAL_DEFAULT;
-  const tracks=[...new Set(["top50-evidence/v1", ...RESULTS.map(r=>r.benchmark_contract||"legacy-whole-repo")])].sort();
+  const tracks=[...new Set(["top50-evidence/v2", ...RESULTS.map(r=>r.benchmark_contract||"legacy-whole-repo")])].sort();
   const trackSelect=document.getElementById("lb-track");
   trackSelect.innerHTML=tracks.map(t=>``).join("");
   const newest=tracks.find(t=>t.startsWith("top50-evidence/"));if(newest)trackSelect.value=newest;
diff --git a/submission.env.example b/submission.env.example
index 06fb817..0a5d890 100644
--- a/submission.env.example
+++ b/submission.env.example
@@ -30,7 +30,7 @@ MODEL_NAME=openai/gpt-5.4
 
 # ── FROZEN LIMITS ────────────────────────────────────────────────────────────
 # There are no budget environment variables. Rankable runs always use
-# top50-evidence/v1: source-only Top50 triage, then 50 isolated rule episodes with
+# top50-evidence/v2: source-only Top50 triage, then 50 isolated rule episodes with
 # M=10, E=12, P=24, P_solve=10, and exactly S=2 submit attempts per rule.
 
 # ── OUTPUT / LOGS ────────────────────────────────────────────────────────────

From 64b4b9ed5cdb77c71c859f003ebd638196035339 Mon Sep 17 00:00:00 2001
From: Xiwei Pan 
Date: Mon, 20 Jul 2026 17:34:40 +0800
Subject: [PATCH 6/9] test: retain successful command summaries

---
 benchmark/observation_policy.py            |  3 ++-
 benchmark/tests/test_evidence_budget.py    | 19 ++++++++++++-------
 benchmark/tests/test_observation_policy.py |  9 +++++++++
 3 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/benchmark/observation_policy.py b/benchmark/observation_policy.py
index c918b76..7d29b7d 100644
--- a/benchmark/observation_policy.py
+++ b/benchmark/observation_policy.py
@@ -16,7 +16,8 @@
 _CONTROL = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
 _DIAGNOSTIC = re.compile(
     r"(?i)(error|fail(?:ed|ure)?|panic|assert(?:ion)?|traceback|exception|timeout|timed out|"
-    r"sentinel|left\s*[:=]|right\s*[:=]|\b\d+\s+failed\b|test result:)")
+    r"sentinel|left\s*[:=]|right\s*[:=]|"
+    r"\b\d+\s+(?:failed|passed|skipped|deselected)\b|test result:)")
 
 
 @dataclass(frozen=True)
diff --git a/benchmark/tests/test_evidence_budget.py b/benchmark/tests/test_evidence_budget.py
index 0f30a9d..11e2f8f 100644
--- a/benchmark/tests/test_evidence_budget.py
+++ b/benchmark/tests/test_evidence_budget.py
@@ -26,7 +26,7 @@ def _fake_pred(tmp_path: Path) -> Path:
         "  --help) echo 'help'; exit 0 ;;\n"
         "  list) echo '[\"r1\",\"r2\"]'; exit 0 ;;\n"
         "  fail) echo 'bad args' >&2; exit 7 ;;\n"
-        "  sleep) sleep 2; exit 0 ;;\n"
+        "  sleep) sleep 10; exit 0 ;;\n"
         "  spam) python3 -c 'print(\"x\" * 5000)'; exit 0 ;;\n"
         "  *) echo \"ran:$*\"; exit 0 ;;\n"
         "esac\n",
@@ -36,7 +36,7 @@ def _fake_pred(tmp_path: Path) -> Path:
     return script
 
 
-def _budget(*, pred=4, solve=1, timeout=1) -> EvidenceBudget:
+def _budget(*, pred=4, solve=1, timeout=3) -> EvidenceBudget:
     return EvidenceBudget(
         model_generations=10,
         shell_actions=10,
@@ -107,14 +107,16 @@ def test_global_output_option_cannot_bypass_solve_budget(tmp_path):
 
 def test_concurrent_pred_clients_cannot_overspend(tmp_path):
     with EvidenceBudgetSession(
-        rule="r1", budget=_budget(pred=4, solve=0), pred_binary=_fake_pred(tmp_path),
+        rule="r1", budget=_budget(pred=4, solve=0, timeout=5),
+        pred_binary=_fake_pred(tmp_path),
         verifier=lambda cert: Verdict(False, "no bug"),
     ) as session:
         with ThreadPoolExecutor(max_workers=20) as pool:
             results = list(pool.map(
                 lambda _: _run("pred", "create", "X", cwd=session.workdir), range(20)))
 
-        assert sum(result.returncode == 0 for result in results) == 4
+        assert sum(result.returncode == 0 for result in results) == 4, [
+            (result.returncode, result.stderr) for result in results]
         assert sum(result.returncode == 75 for result in results) == 16
         assert session.status()["pred_calls"] == {"used": 4, "limit": 4, "remaining": 0}
         observed = [record for record in session.pred.ledger if "observation" in record]
@@ -126,7 +128,8 @@ def test_concurrent_pred_clients_cannot_overspend(tmp_path):
 
 def test_free_commands_are_cached_and_do_not_charge(tmp_path):
     with EvidenceBudgetSession(
-        rule="r1", budget=_budget(pred=1, solve=0), pred_binary=_fake_pred(tmp_path),
+        rule="r1", budget=_budget(pred=1, solve=0, timeout=5),
+        pred_binary=_fake_pred(tmp_path),
         verifier=lambda cert: Verdict(False, "no bug"),
     ) as session:
         assert _run("pred", "--budget-status", cwd=session.workdir).returncode == 0
@@ -142,7 +145,8 @@ def test_free_commands_are_cached_and_do_not_charge(tmp_path):
 
 def test_nonzero_exit_and_timeout_consume_pred_budget(tmp_path):
     with EvidenceBudgetSession(
-        rule="r1", budget=_budget(pred=2, solve=0), pred_binary=_fake_pred(tmp_path),
+        rule="r1", budget=_budget(pred=2, solve=0),
+        pred_binary=_fake_pred(tmp_path),
         verifier=lambda cert: Verdict(False, "no bug"),
     ) as session:
         failed = _run("pred", "fail", cwd=session.workdir)
@@ -174,7 +178,8 @@ def test_gateway_infrastructure_failure_releases_reservation(tmp_path):
 def test_pred_output_is_drained_but_observation_is_capped(tmp_path):
     raw_log = None
     with EvidenceBudgetSession(
-        rule="r1", budget=_budget(pred=1, solve=0), pred_binary=_fake_pred(tmp_path),
+        rule="r1", budget=_budget(pred=1, solve=0, timeout=5),
+        pred_binary=_fake_pred(tmp_path),
         verifier=lambda cert: Verdict(False, "no bug"),
     ) as session:
         result = _run("pred", "spam", cwd=session.workdir)
diff --git a/benchmark/tests/test_observation_policy.py b/benchmark/tests/test_observation_policy.py
index e17da4c..6cf95c5 100644
--- a/benchmark/tests/test_observation_policy.py
+++ b/benchmark/tests/test_observation_policy.py
@@ -102,6 +102,15 @@ def test_middle_diagnostic_survives_when_long_head_and_tail_exceed_preview(tmp_p
     assert "middle context after" in packaged.preview
 
 
+def test_successful_test_summary_survives_many_error_named_tests(tmp_path):
+    text = "".join(f"test_error_case_{index} PASSED\n" for index in range(500))
+    text += "500 passed, 3 deselected in 12.34s\n"
+    _, packaged = _package(tmp_path, text, returncode=0)
+
+    assert len(packaged.preview) <= 10_000
+    assert "500 passed, 3 deselected" in packaged.preview
+
+
 def test_valid_json_is_minified_without_losing_values(tmp_path):
     text = '{\n  "type": "MaximumIndependentSet",\n  "weights": [1, 2, 3]\n}\n'
     _, packaged = _package(tmp_path, text, returncode=0)

From 9ed1c0f5199bc23d42099d8bd5f152e6a7d0e9a5 Mon Sep 17 00:00:00 2001
From: Xiwei Pan 
Date: Tue, 21 Jul 2026 00:16:31 +0800
Subject: [PATCH 7/9] refactor: simplify benchmark protocol and triage prompt

---
 .agents/skills/run-api-benchmark/SKILL.md     |  2 +-
 .agents/skills/run-cli-benchmark/SKILL.md     |  2 +-
 .../skills/submit-benchmark-result/SKILL.md   |  4 +-
 .github/scripts/check_aggregate.py            |  2 +-
 CONTRIBUTING.md                               |  8 +-
 Makefile                                      |  2 +-
 README.md                                     | 10 +-
 benchmark/backend_score.py                    | 11 ++-
 benchmark/calibrate_budget.py                 | 39 +++-----
 benchmark/docs/budget-calibration.json        |  5 +-
 benchmark/docs/budget-calibration.md          |  8 +-
 benchmark/evidence_budget.py                  |  3 +-
 benchmark/observation_policy.py               |  9 +-
 benchmark/run_top50.py                        | 32 +-----
 benchmark/tests/test_budget_calibration.py    | 30 +++---
 benchmark/tests/test_docs.py                  |  6 +-
 benchmark/tests/test_evidence_budget.py       |  1 -
 benchmark/tests/test_observation_policy.py    |  3 +-
 benchmark/tests/test_run_top50.py             |  9 +-
 benchmark/tests/test_top50_runner.py          |  6 +-
 benchmark/tests/test_top50_submission.py      | 33 ++-----
 benchmark/top50_budget.json                   | 35 -------
 benchmark/top50_budget.py                     | 59 +++++++++--
 benchmark/top50_config.yaml                   | 34 +++++--
 benchmark/top50_contract.py                   | 98 +++++++------------
 benchmark/top50_results.schema.json           |  4 +-
 benchmark/top50_runner.py                     | 38 ++++---
 benchmark/top50_submission.schema.json        | 24 ++---
 docker/Dockerfile                             |  3 +-
 site/index.html                               | 17 ++--
 submission.env.example                        |  2 +-
 31 files changed, 239 insertions(+), 300 deletions(-)
 delete mode 100644 benchmark/top50_budget.json

diff --git a/.agents/skills/run-api-benchmark/SKILL.md b/.agents/skills/run-api-benchmark/SKILL.md
index 6f790ca..824fdbd 100644
--- a/.agents/skills/run-api-benchmark/SKILL.md
+++ b/.agents/skills/run-api-benchmark/SKILL.md
@@ -5,7 +5,7 @@ description: Configure and run the rankable problem-reductions Self-selected Top
 
 # Run the rankable Top50 benchmark
 
-Use only the frozen `top50-evidence/v2` path: source-only triage freezes 50 rules, followed by 50 fresh sequential episodes. Each rule receives M=10 model generations, E=12 shell actions, P=24 `pred` calls, P_solve=10 solve calls, O=10000 observed characters, and exactly S=2 submit attempts. Never add or accept custom budgets, prompts, strategies, model kwargs, or coding-agent backends.
+Use only the standardized Top50 path: source-only triage freezes 50 rules, followed by 50 fresh sequential episodes. Each rule receives M=10 model generations, E=12 shell actions, P=24 `pred` calls, P_solve=10 solve calls, O=10000 observed characters, and exactly S=2 submit attempts. These limits are part of the benchmark code; never add or accept custom prompts, strategies, model kwargs, or coding-agent backends.
 
 Read [references/provider-config.md](references/provider-config.md) for endpoint setup and `scripts/detect-engine.sh` before preparing the image.
 
diff --git a/.agents/skills/run-cli-benchmark/SKILL.md b/.agents/skills/run-cli-benchmark/SKILL.md
index 12b0111..29c301f 100644
--- a/.agents/skills/run-cli-benchmark/SKILL.md
+++ b/.agents/skills/run-cli-benchmark/SKILL.md
@@ -6,7 +6,7 @@ description: Configure and run this problem-reductions benchmark through an inst
 # Reproduce the historical CLI benchmark
 
 This route is permanently **non-ranking** and exists only to reproduce `legacy-whole-repo`
-artifacts. Tell the caller that coding-agent results cannot enter `top50-evidence/v2` and
+artifacts. Tell the caller that coding-agent results cannot enter the standardized Top50 table and
 get confirmation before spending time or credits. Do not describe it as a System Track.
 
 Run one whole-repository coding-agent session with the shared benchmark prompt, verifier,
diff --git a/.agents/skills/submit-benchmark-result/SKILL.md b/.agents/skills/submit-benchmark-result/SKILL.md
index a7ac92f..b60230a 100644
--- a/.agents/skills/submit-benchmark-result/SKILL.md
+++ b/.agents/skills/submit-benchmark-result/SKILL.md
@@ -25,7 +25,7 @@ python3 -m benchmark.submit --predictions  --dry-run
 Stop if validation fails. Otherwise report only:
 
 - absolute path;
-- `benchmark_contract`, `model`, and `library_commit`;
+- `model`, `agent_mode`, and `library_commit`;
 - completed Top50 episode count and claimed bugs;
 - `run_error`, if present.
 
@@ -39,7 +39,7 @@ Immediately before uploading, show this confirmation with the real values filled
 > - Destination: `intake.prb-bench.workers.dev`
 > - File: ``
 > - Model: ``
-> - Contract: `top50-evidence/v2`
+> - Track: Standardized Model API / Self-selected Top50
 > - Claimed bugs: ``
 >
 > This private submission file will leave the machine and be uploaded to the benchmark's
diff --git a/.github/scripts/check_aggregate.py b/.github/scripts/check_aggregate.py
index 5c3bc28..16ae6c7 100644
--- a/.github/scripts/check_aggregate.py
+++ b/.github/scripts/check_aggregate.py
@@ -18,7 +18,7 @@
     "model", "library_commit", "bugs_found", "rules_tested",
     "total_tokens_k", "usage_totals", "efficiency_bugs_per_ktok",
     "submitted_by", "placeholder",
-    "benchmark_contract", "runner_version", "pred_version", "agent_mode", "rankable",
+    "runner_version", "pred_version", "agent_mode", "rankable",
     "bugs_at_10", "bugs_at_25", "bugs_at_50", "first_attempt_accepts",
     "second_attempt_accepts", "pred_calls_per_bug", "cap_hits",
     # per-submission entry files (site/results/.json) also carry provenance tags
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 5b8cf3a..5b0086b 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,6 +1,6 @@
 # Submitting a model run
 
-The current public comparison accepts one execution protocol: **Standardized Model API / Self-selected Top50** under frozen contract `top50-evidence/v2`.
+The current public comparison accepts one execution protocol: **Standardized Model API / Self-selected Top50**. Its limits are built into the benchmark code and are not configurable.
 
 ```text
 source-only triage → frozen Top50 → 50 isolated rule episodes → private submission
@@ -46,7 +46,7 @@ python -m benchmark.submit --predictions out//submission.json
 
 The intake check is only a courtesy check. The private backend independently validates:
 
-- contract ID, prompt hash, runner mode, target commit, and `pred` version;
+- prompt hash, runner mode, target commit, and `pred` version;
 - the canonical 50-rule inventory and frozen order;
 - triage and per-rule event/usage ledgers;
 - exactly two submit opportunities per rule, including malformed or rejected calls;
@@ -64,7 +64,7 @@ The score should be read as joint **prioritization + bounded diagnosis + certifi
 
 ## Release checks
 
-Before changing the frozen contract or target version, add a new versioned contract and a new non-ranking calibration. Do not silently edit `top50-evidence/v2`.
+Before changing benchmark limits or the target version, update the non-ranking calibration evidence and documentation in the same change. The repository commit identifies the resulting benchmark definition.
 
 ```bash
 pytest -v -m "not integration"
@@ -72,7 +72,7 @@ python -m benchmark.calibrate_budget --check benchmark/docs/budget-calibration.j
 python -m benchmark.verify --calibrate
 ```
 
-The calibration checker proves that the selected limits match the runtime contract, every model covers the full candidate grid, smaller and larger `M`/`P` candidates surround the selection, and the Markdown report matches the machine-readable evidence.
+The calibration checker proves that the selected limits match the built-in runtime values, every model covers the full candidate grid, smaller and larger `M`/`P` candidates surround the selection, and the Markdown report matches the machine-readable evidence.
 
 ## Historical tooling
 
diff --git a/Makefile b/Makefile
index 7cc1d1d..56770b0 100644
--- a/Makefile
+++ b/Makefile
@@ -157,7 +157,7 @@ help:
 	@echo "  test                Run full pytest suite"
 	@echo "  test-unit           Run unit tests only (no real repo needed)"
 	@echo "  verify-calibration  Test verifier against fixtures (no AI needed)"
-	@echo "  verify-budget       Check frozen Top50 calibration evidence offline"
+	@echo "  verify-budget       Check Top50 calibration evidence offline"
 	@echo "  runner-build        Build the dockerized submission runner image"
 	@echo "  runner-pull         Pull the prebuilt runner image from GHCR (fast runner-build alternative)"
 	@echo "  preflight           Validate submission.env (1 tiny real call) before a full run"
diff --git a/README.md b/README.md
index 4b68f0a..ee18492 100644
--- a/README.md
+++ b/README.md
@@ -2,9 +2,9 @@
 
 This benchmark measures how well a model can prioritize likely-buggy reduction rules and turn a fixed amount of evidence gathering into independently verified counterexamples.
 
-## Current ranking contract
+## Current benchmark
 
-The primary track is **Standardized Model API / Self-selected Top50**, contract [`top50-evidence/v2`](benchmark/top50_budget.json). The target is `problem-reductions` [`v0.6.0`](https://github.com/CodingThrust/problem-reductions/commit/aa2d1a10cffa434871d12a4d6f411147fb7e08a8), and the bundled `pred` version is `0.6.0`.
+The primary track is **Standardized Model API / Self-selected Top50**. The target is `problem-reductions` [`v0.6.0`](https://github.com/CodingThrust/problem-reductions/commit/aa2d1a10cffa434871d12a4d6f411147fb7e08a8), and the bundled `pred` version is `0.6.0`. The logical limits are part of the benchmark code, not a user-selectable configuration.
 
 Each run has two phases:
 
@@ -24,7 +24,7 @@ Every episode receives the same immutable logical budget:
 
 Triage receives 8 model generations and 12 source-only actions. Unused budget never transfers between rules. Process timeouts remain fixed watchdogs for hung model or `pred` calls; elapsed time, network delay, tokens, and cost do not affect rank.
 
-Every terminal result uses the frozen `terminal-diagnostics/v1` policy: deterministic noise and repetition removal, diagnostic-aware head/tail selection, and a 10,000-character model preview. A separately bounded 1 MiB raw log remains read-only inside the current episode and can be inspected using a normal charged shell action.
+Every terminal result receives deterministic noise and repetition removal, diagnostic-aware head/tail selection, and a 10,000-character model preview. A separately bounded 1 MiB raw log remains read-only inside the current episode and can be inspected using a normal charged shell action.
 
 The budget was selected once using a human-reviewed non-ranking development replay, with smaller and larger candidates around the chosen `M` and `P`. The checked-in record and rationale are in [budget-calibration.md](benchmark/docs/budget-calibration.md); validate their internal and release consistency offline with:
 
@@ -59,7 +59,7 @@ make run                                   # writes out//submission.j
 python -m benchmark.submit --predictions out//submission.json
 ```
 
-The rankable path is deliberately narrow. It rejects custom logical limits, unlimited counters, `S != 2`, custom prompts or strategies, custom model kwargs, and coding-agent backends before a scored generation. Model and `pred` watchdogs are safety controls and are not user-selectable score dimensions.
+The rankable path is deliberately narrow. It exposes no logical-budget configuration and rejects custom prompts or strategies, custom model kwargs, and coding-agent backends before a scored generation. Model and `pred` watchdogs are safety controls and are not user-selectable score dimensions.
 
 Useful local checks:
 
@@ -73,6 +73,6 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for artifact fields, submission, and hist
 
 ## Historical results
 
-Whole-repository results produced by the former contract remain visible under `legacy-whole-repo`. They use a different execution protocol and efficiency tie-break, so the site keeps them in a separate selectable table. They cannot be deduplicated, sorted, or compared into `top50-evidence/v2`.
+Whole-repository results remain visible under `legacy-whole-repo`. They use a different execution protocol and efficiency tie-break, so the site keeps them in a separate selectable table. They cannot be deduplicated, sorted, or compared with the standardized Top50 results.
 
 The old host coding-agent and whole-repository runners remain in the repository only to reproduce those historical artifacts. They are not a public System Track and cannot produce rankable Top50 submissions.
diff --git a/benchmark/backend_score.py b/benchmark/backend_score.py
index 7690a1c..f8b7c22 100644
--- a/benchmark/backend_score.py
+++ b/benchmark/backend_score.py
@@ -250,10 +250,11 @@ def _dedup_best(entries: list[dict]) -> list[dict]:
     best: dict[tuple[str, str], dict] = {}
     for e in entries:
         m = e["model"]
-        contract = e.get("benchmark_contract", "legacy-whole-repo")
-        group = (contract, m)
+        top50 = e.get("agent_mode") == "standardized-model-api"
+        track = "top50" if top50 else "legacy-whole-repo"
+        group = (track, m)
         cur = best.get(group)
-        if contract.startswith("top50-evidence/"):
+        if top50:
             better = cur is None or e.get("bugs_found", 0) > cur.get("bugs_found", 0)
         else:
             key = (e.get("bugs_found", 0), e.get("efficiency_bugs_per_ktok", 0.0))
@@ -264,9 +265,9 @@ def _dedup_best(entries: list[dict]) -> list[dict]:
     return sorted(
         best.values(),
         key=lambda e: (
-            e.get("benchmark_contract", "legacy-whole-repo"),
+            0 if e.get("agent_mode") == "standardized-model-api" else 1,
             -e.get("bugs_found", 0),
-            (0 if str(e.get("benchmark_contract", "")).startswith("top50-evidence/")
+            (0 if e.get("agent_mode") == "standardized-model-api"
              else -e.get("efficiency_bugs_per_ktok", 0.0)),
             e.get("model", ""),
         ))
diff --git a/benchmark/calibrate_budget.py b/benchmark/calibrate_budget.py
index e6e5060..117a373 100644
--- a/benchmark/calibrate_budget.py
+++ b/benchmark/calibrate_budget.py
@@ -1,4 +1,4 @@
-"""Offline validation and rendering for the frozen Top50 budget evidence."""
+"""Offline validation and rendering for the Top50 budget-selection evidence."""
 from __future__ import annotations
 
 import argparse
@@ -6,11 +6,10 @@
 import json
 from pathlib import Path
 
-from benchmark.top50_budget import FROZEN_CONTRACT
+from benchmark.top50_budget import benchmark_parameters
 from benchmark.usage import usage_from_dict
 
 ROOT = Path(__file__).resolve().parent
-CONTRACT_PATH = ROOT / "top50_budget.json"
 REPORT_PATH = ROOT / "docs" / "budget-calibration.md"
 REQUIRED_OBSERVATION_FIELDS = {
     "candidate", "model", "target", "verified_bugs", "cap_hits", "usage",
@@ -32,8 +31,8 @@ def validate_evidence(evidence: dict, contract: dict) -> list[str]:
         errors.append("schema_version must be 1")
     if evidence.get("ranking_status") != "non-ranking-development":
         errors.append("calibration evidence must be explicitly non-ranking")
-    if evidence.get("selected_contract") != contract:
-        errors.append("selected_contract does not exactly match top50_budget.json")
+    if evidence.get("selected_parameters") != contract:
+        errors.append("selected_parameters do not exactly match the built-in benchmark limits")
     sources = evidence.get("sources")
     source_index: dict[tuple[str, str], str] = {}
     if not isinstance(sources, list) or not sources:
@@ -127,20 +126,18 @@ def validate_evidence(evidence: dict, contract: dict) -> list[str]:
 
 
 def render_report(evidence: dict) -> str:
-    contract = evidence["selected_contract"]
+    contract = evidence["selected_parameters"]
     episode = contract["episode"]
     safety = contract["safety_controls"]
     lines = [
         "# Top50 budget calibration",
         "",
-        f"Contract: `{contract['contract_id']}` (`{contract['status']}`)",
-        "",
         "This is a human-reviewed, non-ranking bounded-prefix replay record from internally "
         "retained pilot trajectories. Raw trajectories remain private; this offline checker "
         "validates the checked-in record and release consistency, not the raw replay. It is not "
         "a public score, a multi-seed experiment, or a claim that elapsed time is model ability.",
         "",
-        "## Selected contract",
+        "## Selected benchmark parameters",
         "",
         f"The release freezes M={episode['model_generations']} model generations, "
         f"P={episode['pred_calls']} total `pred` calls, "
@@ -150,8 +147,8 @@ def render_report(evidence: dict) -> str:
         f"T={contract['triage']['model_generations']} generations and "
         f"E_t={contract['triage']['shell_actions']} source-only actions.",
         "",
-        f"Terminal output uses `{contract['observation']['policy_id']}` with a "
-        f"{contract['observation']['preview_chars']}-character automatic preview and a bounded "
+        f"Terminal output has a {contract['observation']['preview_chars']}-character automatic "
+        "preview and a bounded "
         f"{contract['observation']['archive_chars']}-character raw archive per command.",
         "",
         f"Model calls have a fixed {safety['model_timeout_seconds']}-second watchdog and "
@@ -185,7 +182,7 @@ def render_report(evidence: dict) -> str:
         "",
         evidence["selection_rationale"],
         "",
-        "The public comparison is therefore one run at this single contract, ranked only by "
+        "The public comparison therefore uses these built-in parameters and is ranked only by "
         "verified distinct-rule bugs. Fixed Top50, multiple seeds, a System Track, and a public "
         "budget grid remain out of scope.",
         "",
@@ -205,7 +202,7 @@ def observation_from_artifact(path: str | Path) -> dict:
     artifact = json.loads(raw)
     if not isinstance(artifact, dict):
         raise ValueError(f"{path} must contain a JSON object")
-    episode_budget = artifact.get("contract", {}).get("episode", {})
+    episode_budget = artifact.get("calibration_parameters", {}).get("episode", {})
     episodes = artifact.get("episodes")
     if (artifact.get("calibration_status") != "non-ranking-development"
             or not isinstance(episodes, list)
@@ -249,27 +246,15 @@ def observation_from_artifact(path: str | Path) -> dict:
 
 def check(path: str | Path) -> list[str]:
     evidence = load_json(path)
-    contract = FROZEN_CONTRACT
+    contract = benchmark_parameters()
     errors = validate_evidence(evidence, contract)
-    for schema_name in ("top50_submission.schema.json", "top50_results.schema.json"):
-        schema = load_json(ROOT / schema_name)
-        schema_contract = schema.get("properties", {}).get("benchmark_contract", {}).get("const")
-        if schema_contract != contract["contract_id"]:
-            errors.append(f"{schema_name} does not name the frozen contract")
     submission_schema = load_json(ROOT / "top50_submission.schema.json")
     properties = submission_schema.get("properties", {})
-    expected_artifact_contract = {key: contract[key] for key in
-                                  ("triage", "episode", "observation", "shortlist_size",
-                                   "hypothesis_chars")}
-    if properties.get("contract", {}).get("const") != expected_artifact_contract:
-        errors.append("top50_submission.schema.json has stale logical limits")
     if properties.get("safety_controls", {}).get("const") != contract["safety_controls"]:
         errors.append("top50_submission.schema.json has stale safety controls")
     if properties.get("inference_parameters", {}).get("const") != contract[
             "inference_parameters"]:
         errors.append("top50_submission.schema.json has stale inference parameters")
-    if properties.get("observation_policy", {}).get("const") != contract["observation"]:
-        errors.append("top50_submission.schema.json has stale observation policy")
     if not errors and REPORT_PATH.read_text(encoding="utf-8") != render_report(evidence):
         errors.append("budget-calibration.md does not match the machine-readable evidence")
     return errors
@@ -289,7 +274,7 @@ def main(argv: list[str] | None = None) -> None:
         for error in errors:
             print(f"FAIL: {error}")
         raise SystemExit(1)
-    print(f"PASS: calibration evidence matches {FROZEN_CONTRACT['contract_id']}")
+    print("PASS: calibration evidence matches the built-in benchmark limits")
 
 
 if __name__ == "__main__":
diff --git a/benchmark/docs/budget-calibration.json b/benchmark/docs/budget-calibration.json
index 2042adf..9d25928 100644
--- a/benchmark/docs/budget-calibration.json
+++ b/benchmark/docs/budget-calibration.json
@@ -3,9 +3,7 @@
   "generated_at": "2026-07-20T00:00:00Z",
   "ranking_status": "non-ranking-development",
   "method": "Human-reviewed bounded-prefix replay record from internally retained whole-repository pilot trajectories. Raw trajectories remain private; the offline checker validates this record and release consistency, not the raw replay. These statistics are not Top50 scores.",
-  "selected_contract": {
-    "contract_id": "top50-evidence/v2",
-    "status": "frozen",
+  "selected_parameters": {
     "shortlist_size": 50,
     "hypothesis_chars": 500,
     "safety_controls": {
@@ -18,7 +16,6 @@
       "num_retries": 2
     },
     "observation": {
-      "policy_id": "terminal-diagnostics/v1",
       "preview_chars": 10000,
       "archive_chars": 1048576
     },
diff --git a/benchmark/docs/budget-calibration.md b/benchmark/docs/budget-calibration.md
index ae29d33..aaf99da 100644
--- a/benchmark/docs/budget-calibration.md
+++ b/benchmark/docs/budget-calibration.md
@@ -1,14 +1,12 @@
 # Top50 budget calibration
 
-Contract: `top50-evidence/v2` (`frozen`)
-
 This is a human-reviewed, non-ranking bounded-prefix replay record from internally retained pilot trajectories. Raw trajectories remain private; this offline checker validates the checked-in record and release consistency, not the raw replay. It is not a public score, a multi-seed experiment, or a claim that elapsed time is model ability.
 
-## Selected contract
+## Selected benchmark parameters
 
 The release freezes M=10 model generations, P=24 total `pred` calls, P_solve=10 solve calls, S=2 submit attempts, E=12 shell actions, and O=10000 automatically previewed characters per action. Triage is T=8 generations and E_t=12 source-only actions.
 
-Terminal output uses `terminal-diagnostics/v1` with a 10000-character automatic preview and a bounded 1048576-character raw archive per command.
+Terminal output has a 10000-character automatic preview and a bounded 1048576-character raw archive per command.
 
 Model calls have a fixed 300-second watchdog and 2 transport retries. Command and `pred` watchdogs are also fixed safety controls. They are recorded but are outside the logical budget and never enter the score.
 
@@ -41,7 +39,7 @@ Token, cost, and elapsed-time fields are diagnostic references only. Missing pro
 
 M=10 and P=24 are the smallest tested values at which the replayed stronger pilot retains its verified find. Moving down to M=6 or P=12 loses that find and shows materially higher cap pressure; moving up to M=14 or P=36 adds usage without another verified bug. E=12 and P_solve=10 cover the observed command mix without exposing an unlimited path. S=2 follows the fixed per-rule retry policy. T=8 and E_t=12 bound source-only shortlist formation. Output and watchdog ceilings are safety controls selected above observed pilot maxima and do not affect ranking.
 
-The public comparison is therefore one run at this single contract, ranked only by verified distinct-rule bugs. Fixed Top50, multiple seeds, a System Track, and a public budget grid remain out of scope.
+The public comparison therefore uses these built-in parameters and is ranked only by verified distinct-rule bugs. Fixed Top50, multiple seeds, a System Track, and a public budget grid remain out of scope.
 
 ## Provenance
 
diff --git a/benchmark/evidence_budget.py b/benchmark/evidence_budget.py
index 71bd015..c5ae2e2 100644
--- a/benchmark/evidence_budget.py
+++ b/benchmark/evidence_budget.py
@@ -55,7 +55,7 @@ def __post_init__(self) -> None:
         if self.solve_calls > self.pred_calls:
             raise ValueError("solve_calls must be <= pred_calls")
         if self.submit_attempts != 2:
-            raise ValueError("submit_attempts must be exactly 2 for the Top50 contract")
+            raise ValueError("submit_attempts must be exactly 2 for the Top50 benchmark")
         if self.max_output_chars == 0:
             raise ValueError("max_output_chars must be > 0")
         if self.pred_timeout_seconds == 0:
@@ -662,7 +662,6 @@ def ledger(self) -> dict:
         return {
             "rule": self.rule,
             "budget": asdict(self.budget),
-            "observation_policy": asdict(self.observation_config),
             "status": self.status(),
             "pred": pred_ledger,
             "submit": self.submit.attempts,
diff --git a/benchmark/observation_policy.py b/benchmark/observation_policy.py
index 7d29b7d..e177855 100644
--- a/benchmark/observation_policy.py
+++ b/benchmark/observation_policy.py
@@ -8,7 +8,6 @@
 from dataclasses import dataclass
 from pathlib import Path
 
-POLICY_ID = "terminal-diagnostics/v1"
 DEFAULT_PREVIEW_CHARS = 10_000
 DEFAULT_ARCHIVE_CHARS = 1_048_576
 _CONTEXT_LINES = 2
@@ -22,13 +21,10 @@
 
 @dataclass(frozen=True)
 class ObservationConfig:
-    policy_id: str = POLICY_ID
     preview_chars: int = DEFAULT_PREVIEW_CHARS
     archive_chars: int = DEFAULT_ARCHIVE_CHARS
 
     def __post_init__(self) -> None:
-        if self.policy_id != POLICY_ID:
-            raise ValueError(f"unsupported observation policy: {self.policy_id}")
         if self.preview_chars < 256 or self.archive_chars < self.preview_chars:
             raise ValueError("archive_chars must be >= preview_chars >= 256")
 
@@ -94,8 +90,8 @@ def package(self, *, kind: str, command: str, returncode: int, timed_out: bool,
 
         relative = os.path.relpath(path, self.relative_from)
         header = (
-            f"[observation {observation_id}] policy={self.config.policy_id} "
-            f"returncode={returncode} timed_out={str(timed_out).lower()}\n"
+            f"[observation {observation_id}] returncode={returncode} "
+            f"timed_out={str(timed_out).lower()}\n"
             f"[raw log: {relative}; original={original_chars} chars/{original_lines} lines; "
             f"archive={'truncated' if archive_truncated else 'complete'}]\n"
         )
@@ -106,7 +102,6 @@ def package(self, *, kind: str, command: str, returncode: int, timed_out: bool,
             "observation_id": observation_id,
             "kind": kind,
             "command": command,
-            "policy_id": self.config.policy_id,
             "raw_log": relative,
             "returncode": returncode,
             "timed_out": timed_out,
diff --git a/benchmark/run_top50.py b/benchmark/run_top50.py
index 0d308f9..4a8917e 100644
--- a/benchmark/run_top50.py
+++ b/benchmark/run_top50.py
@@ -10,36 +10,24 @@
 from pathlib import Path
 
 from benchmark.env_setup import find_pred_binary, pinned_commit, verify_pred_version
-from benchmark.evidence_budget import EvidenceBudget
-from benchmark.observation_policy import ObservationConfig
 from benchmark.run_mini import list_rules
 from benchmark.top50_runner import (
     PhaseResult,
-    Top50Contract,
     Top50Runner,
-    TriageBudget,
+    benchmark_limits,
     build_rankable_runner,
 )
 from benchmark.top50_contract import (
     AGENT_MODE,
-    CONTRACT_ID,
     RUNNER_VERSION,
-    EXPECTED_EPISODE_BUDGET,
-    EXPECTED_OBSERVATION,
     EXPECTED_INFERENCE_PARAMETERS,
     EXPECTED_SAFETY_CONTROLS,
-    EXPECTED_HYPOTHESIS_CHARS,
-    EXPECTED_SHORTLIST_SIZE,
-    EXPECTED_TRIAGE_BUDGET,
     expected_prompt_id,
 )
 
 
 FORBIDDEN_RANKABLE_ENV = (
     "AGENT_BACKEND", "AGENT_CONFIG", "AGENT_STRATEGY_FILE", "SUBMIT_LIMIT",
-    "PRB_TRIAGE_GENERATIONS", "PRB_TRIAGE_ACTIONS", "PRB_EPISODE_GENERATIONS",
-    "PRB_EPISODE_ACTIONS", "PRB_PRED_CALLS", "PRB_SOLVE_CALLS",
-    "PRB_MAX_OUTPUT_CHARS", "PRB_PRED_TIMEOUT_SECONDS",
     "EXPECTED_PRED_COMMIT", "EXPECTED_PRED_VERSION",
 )
 TRUSTED_PRED_PATH = Path("/usr/local/libexec/prb/pred")
@@ -81,17 +69,6 @@ def verify_rankable_pred_path(binary: Path) -> None:
         raise ValueError(f"rankable runs require the image-owned pred at {TRUSTED_PRED_PATH}")
 
 
-def frozen_contract() -> Top50Contract:
-    """Return the immutable logical budget for the released rankable track."""
-    return Top50Contract(
-        triage=TriageBudget(**EXPECTED_TRIAGE_BUDGET),
-        episode=EvidenceBudget(**EXPECTED_EPISODE_BUDGET),
-        observation=ObservationConfig(**EXPECTED_OBSERVATION),
-        shortlist_size=EXPECTED_SHORTLIST_SIZE,
-        hypothesis_chars=EXPECTED_HYPOTHESIS_CHARS,
-    )
-
-
 def validate_rankable_settings(*, model_kwargs: dict | None = None) -> None:
     """Reject every knob that could change the frozen rankable execution path."""
     present = [name for name in FORBIDDEN_RANKABLE_ENV if name in os.environ]
@@ -127,13 +104,13 @@ def run(*, model: str, repo_dir: str | Path, output: str | Path,
     if not fake:
         verify_rankable_pred_path(pred_binary)
     pred_version = verify_pred_version(pred_binary)
-    contract = frozen_contract()
+    contract = benchmark_limits()
     if fake:
         runner = Top50Runner(
             executor=_FakeExecutor(), contract=contract, pred_binary=pred_binary)
     else:
         runner = build_rankable_runner(
-            contract=contract, pred_binary=pred_binary,
+            pred_binary=pred_binary,
             agent_uid=int(os.environ["PRB_AGENT_UID"]),
             agent_gid=int(os.environ["PRB_AGENT_GID"]),
             oracle_uid=int(os.environ["PRB_ORACLE_UID"]),
@@ -141,17 +118,14 @@ def run(*, model: str, repo_dir: str | Path, output: str | Path,
             evidence_gid=int(os.environ["PRB_EVIDENCE_GID"]),
             api_base=api_base, api_key=api_key)
     metadata = {
-        "benchmark_contract": CONTRACT_ID,
         "library_commit": actual_commit,
         "runner_version": RUNNER_VERSION,
         "pred_version": pred_version,
         "agent_mode": AGENT_MODE,
         "prompt_id": expected_prompt_id(),
-        "budget_contract_status": "frozen",
         "safety_controls": EXPECTED_SAFETY_CONTROLS,
         "created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
         "inference_parameters": EXPECTED_INFERENCE_PARAMETERS,
-        "observation_policy": EXPECTED_OBSERVATION,
     }
     return runner.run(model=model, repo_path=repo, inventory=inventory,
                       output=output, metadata=metadata)
diff --git a/benchmark/tests/test_budget_calibration.py b/benchmark/tests/test_budget_calibration.py
index 500e5f5..394e6d5 100644
--- a/benchmark/tests/test_budget_calibration.py
+++ b/benchmark/tests/test_budget_calibration.py
@@ -7,6 +7,7 @@
 
 from benchmark import calibrate_budget
 from benchmark import run_top50
+from benchmark.top50_budget import benchmark_parameters
 
 
 def _evidence() -> dict:
@@ -15,7 +16,7 @@ def _evidence() -> dict:
 
 
 def _contract() -> dict:
-    return calibrate_budget.load_json(calibrate_budget.CONTRACT_PATH)
+    return benchmark_parameters()
 
 
 def test_checked_in_calibration_is_self_consistent():
@@ -25,8 +26,8 @@ def test_checked_in_calibration_is_self_consistent():
 
 def test_changed_selected_limit_is_rejected():
     evidence = _evidence()
-    evidence["selected_contract"]["episode"]["pred_calls"] = 25
-    assert "does not exactly match" in " ".join(
+    evidence["selected_parameters"]["episode"]["pred_calls"] = 25
+    assert "do not exactly match" in " ".join(
         calibrate_budget.validate_evidence(evidence, _contract()))
 
 
@@ -70,7 +71,8 @@ def test_development_artifact_observation_is_derived_offline(tmp_path):
     artifact = {
         "calibration_status": "non-ranking-development",
         "model": "internal/model", "library_commit": "abc",
-        "contract": {"episode": {"model_generations": 6, "pred_calls": 12}},
+        "calibration_parameters": {
+            "episode": {"model_generations": 6, "pred_calls": 12}},
         "episodes": [{
             "status": "bug_found",
             "usage": {"input": 10, "output": 2, "cache_read": 3, "cache_write": 1},
@@ -90,20 +92,16 @@ def test_development_artifact_observation_is_derived_offline(tmp_path):
     assert observation["retries"] is None
 
 
-def test_frozen_contract_is_referenced_across_release_surfaces():
+def test_code_defined_limits_match_calibration_and_version_ids_are_absent():
     contract = _contract()
-    runtime = asdict(run_top50.frozen_contract())
+    runtime = asdict(run_top50.benchmark_limits())
     assert runtime == {key: contract[key] for key in
                        ("triage", "episode", "observation", "shortlist_size",
                         "hypothesis_chars")}
     root = calibrate_budget.ROOT.parent
-    for relative in (
-        "README.md", "CONTRIBUTING.md", "benchmark/top50_config.yaml",
-        "benchmark/top50_submission.schema.json", "benchmark/top50_results.schema.json",
-        "docker/Dockerfile", "site/index.html", ".github/workflows/ci.yml",
-    ):
-        text = (root / relative).read_text(encoding="utf-8")
-        if relative.endswith("ci.yml"):
-            assert "benchmark.calibrate_budget" in text
-        else:
-            assert contract["contract_id"] in text
+    prompt = (root / "benchmark/top50_config.yaml").read_text(encoding="utf-8")
+    assert "problem-reductions Rust library" in prompt
+    assert "top50-evidence/" not in prompt
+    assert "terminal-diagnostics/" not in prompt
+    assert "benchmark.calibrate_budget" in (
+        root / ".github/workflows/ci.yml").read_text(encoding="utf-8")
diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py
index c4c2b11..cc6159c 100644
--- a/benchmark/tests/test_docs.py
+++ b/benchmark/tests/test_docs.py
@@ -76,9 +76,9 @@ def test_readme_has_metrics_section(self):
         assert "verified distinct-rule bugs" in t
         assert "equal bug counts are ties" in t
 
-    def test_readme_lists_current_round_contract(self):
+    def test_readme_lists_current_benchmark(self):
         text = README.read_text(encoding="utf-8")
-        assert "top50-evidence/v2" in text
+        assert "not a user-selectable configuration" in text
         assert PINNED_COMMIT in text
         assert f"`{PINNED_PRED_VERSION}`" in text
         assert "elapsed time" in text.lower()
@@ -107,7 +107,7 @@ def test_env_template_does_not_select_cli_with_agent_backend(self):
 
     def test_api_skill_is_container_only(self):
         t = _text(API_SKILL)
-        assert "top50-evidence/v2" in t and "runner-pull" in t
+        assert "standardized" in t and "runner-pull" in t
         assert "coding-agent backends" in t
 
     def test_cli_skill_is_host_only(self):
diff --git a/benchmark/tests/test_evidence_budget.py b/benchmark/tests/test_evidence_budget.py
index 11e2f8f..7e42241 100644
--- a/benchmark/tests/test_evidence_budget.py
+++ b/benchmark/tests/test_evidence_budget.py
@@ -187,7 +187,6 @@ def test_pred_output_is_drained_but_observation_is_capped(tmp_path):
         assert len(result.stdout) <= 1024
         assert "chars omitted" in result.stdout
         observation = session.pred.ledger[0]["observation"]
-        assert observation["policy_id"] == "terminal-diagnostics/v1"
         assert observation["original_chars"] == 5001
         assert observation["preview_chars"] <= 1024
         raw_log = (session.workdir / observation["raw_log"]).resolve()
diff --git a/benchmark/tests/test_observation_policy.py b/benchmark/tests/test_observation_policy.py
index 6cf95c5..b87957f 100644
--- a/benchmark/tests/test_observation_policy.py
+++ b/benchmark/tests/test_observation_policy.py
@@ -7,7 +7,7 @@
 
 import pytest
 
-from benchmark.observation_policy import ObservationConfig, ObservationStore, POLICY_ID
+from benchmark.observation_policy import ObservationConfig, ObservationStore
 
 
 def _fixture() -> str:
@@ -65,7 +65,6 @@ def test_known_terminal_stream_retains_tail_diagnostics_and_exact_metadata(tmp_p
         "observation_id": "shell-0001",
         "kind": "shell",
         "command": "pytest -q",
-        "policy_id": POLICY_ID,
         "raw_log": "../observations/shell-0001.log",
         "returncode": 1,
         "timed_out": False,
diff --git a/benchmark/tests/test_run_top50.py b/benchmark/tests/test_run_top50.py
index 57ea7c7..ecf1c92 100644
--- a/benchmark/tests/test_run_top50.py
+++ b/benchmark/tests/test_run_top50.py
@@ -29,8 +29,9 @@ def test_fake_entrypoint_uses_50_isolated_episodes(tmp_path, monkeypatch):
     assert len(result["episodes"]) == 50
     assert result["rankable"] is False
     artifact = json.loads(output.read_text())
-    assert artifact["budget_contract_status"] == "frozen"
-    assert artifact["benchmark_contract"] == "top50-evidence/v2"
+    assert "budget_contract_status" not in artifact
+    assert "benchmark_contract" not in artifact
+    assert "contract" not in artifact
 
 
 @pytest.mark.parametrize("forbidden", ["--backend", "--config", "--strategy-file"])
@@ -46,9 +47,9 @@ def test_rankable_preflight_rejects_custom_execution_before_model_call(name, mon
         run_top50.validate_rankable_settings()
 
 
-def test_rankable_contract_cannot_be_changed_by_budget_environment(monkeypatch):
+def test_benchmark_limits_are_code_defined_and_ignore_legacy_budget_environment(monkeypatch):
     monkeypatch.setenv("PRB_PRED_CALLS", "999")
-    assert run_top50.frozen_contract().episode.pred_calls == 24
+    assert run_top50.benchmark_limits().episode.pred_calls == 24
 
 
 def test_even_empty_custom_model_kwargs_are_rejected(monkeypatch):
diff --git a/benchmark/tests/test_top50_runner.py b/benchmark/tests/test_top50_runner.py
index 971e012..b9f99c8 100644
--- a/benchmark/tests/test_top50_runner.py
+++ b/benchmark/tests/test_top50_runner.py
@@ -194,7 +194,7 @@ def test_triage_builtin_and_error_results_use_observation_policy(tmp_path):
         ]
 
         assert [output["returncode"] for output in outputs] == [0, 2, 2, 75]
-        assert all("policy=terminal-diagnostics/v1" in output["output"]
+        assert all("[observation shell-" in output["output"]
                    and "[raw log:" in output["output"] for output in outputs)
         ledger = session.ledger()
         shell_events = [event for event in ledger["events"]
@@ -215,13 +215,13 @@ def test_only_standard_factory_can_mark_a_run_rankable(tmp_path, monkeypatch):
 
     monkeypatch.setattr(top50.os, "geteuid", lambda: 0)
     runner = build_rankable_runner(
-        contract=_contract(), pred_binary=_fake_pred(tmp_path),
+        pred_binary=_fake_pred(tmp_path),
         agent_uid=10001, agent_gid=10001, oracle_uid=10002, oracle_gid=10002,
         evidence_gid=10003)
     assert runner._rankable_contract is True
     with pytest.raises(ValueError, match="distinct"):
         build_rankable_runner(
-            contract=_contract(), pred_binary=_fake_pred(tmp_path),
+            pred_binary=_fake_pred(tmp_path),
             agent_uid=10001, agent_gid=10001, oracle_uid=10001, oracle_gid=10002,
             evidence_gid=10003)
 
diff --git a/benchmark/tests/test_top50_submission.py b/benchmark/tests/test_top50_submission.py
index 02fd227..0f278c6 100644
--- a/benchmark/tests/test_top50_submission.py
+++ b/benchmark/tests/test_top50_submission.py
@@ -12,7 +12,6 @@
 from benchmark.submit import validate_submission
 from benchmark.top50_contract import (
     AGENT_MODE,
-    CONTRACT_ID,
     RUNNER_VERSION,
     expected_prompt_id,
     score_top50_submission,
@@ -27,11 +26,9 @@ def _status(limit: int, used: int = 0) -> dict:
 
 
 def _artifact(accepted_positions=(7, 18, 41)) -> dict:
-    observation = {"policy_id": "terminal-diagnostics/v1", "preview_chars": 10_000,
-                   "archive_chars": 1_048_576}
     triage_observation = {
         "observation_id": "shell-0001", "kind": "shell",
-        "command": "commit-top50 shortlist.json", "policy_id": observation["policy_id"],
+        "command": "commit-top50 shortlist.json",
         "raw_log": "../observations/shell-0001.log", "returncode": 0,
         "timed_out": False, "original_chars": 16, "original_lines": 1,
         "preview_chars": 200, "archive_chars": 16, "preview_compacted": False,
@@ -54,7 +51,6 @@ def _artifact(accepted_positions=(7, 18, 41)) -> dict:
     triage = {
         "ledger": {
             "budget": triage_budget,
-            "observation_policy": copy.deepcopy(observation),
             "observations": [triage_observation],
             "status": triage_status,
             "events": [
@@ -98,7 +94,6 @@ def _artifact(accepted_positions=(7, 18, 41)) -> dict:
             "index": position, "rule": entry["rule"], "hypothesis": entry["hypothesis"],
             "status": status, "accepted_submit_attempt": 1 if attempts else None,
             "ledger": {"rule": entry["rule"], "budget": copy.deepcopy(episode_budget),
-                       "observation_policy": copy.deepcopy(observation),
                        "observations": [],
                        "status": ledger_status, "pred": [], "submit": attempts,
                        "model_generations": [], "shell_actions": []},
@@ -106,22 +101,16 @@ def _artifact(accepted_positions=(7, 18, 41)) -> dict:
             "usage": {"input": 10, "output": 1, "cache_read": 0, "cache_write": 0},
         })
     return {
-        "benchmark_contract": CONTRACT_ID,
         "model": "provider/model",
         "library_commit": "a" * 40,
         "runner_version": RUNNER_VERSION,
         "pred_version": "0.6.0",
         "agent_mode": AGENT_MODE,
         "prompt_id": expected_prompt_id(),
-        "budget_contract_status": "frozen",
         "safety_controls": {"model_timeout_seconds": 300, "model_retries": 2},
-        "observation_policy": copy.deepcopy(observation),
         "status": "completed",
         "rankable": True,
         "inference_parameters": {"max_tokens": 8192, "timeout": 300, "num_retries": 2},
-        "contract": {"triage": triage_budget, "episode": episode_budget,
-                     "observation": copy.deepcopy(observation),
-                     "shortlist_size": 50, "hypothesis_chars": 500},
         "shortlist": shortlist,
         "triage": triage,
         "episodes": episodes,
@@ -187,7 +176,7 @@ def test_pred_observation_command_and_outcome_are_derived_from_action_record():
     request_id = "f" * 32
     observation = {
         "observation_id": f"pred-{request_id}", "kind": "pred",
-        "command": "pred create X", "policy_id": "terminal-diagnostics/v1",
+        "command": "pred create X",
         "raw_log": f"../observations/pred-{request_id}.log", "returncode": 0,
         "timed_out": False, "original_chars": 10, "original_lines": 1,
         "preview_chars": 200, "archive_chars": 10, "preview_compacted": False,
@@ -225,13 +214,11 @@ def test_pred_observation_command_and_outcome_are_derived_from_action_record():
      "usage is inconsistent"),
     (lambda sub: sub["episodes"][0]["ledger"]["budget"].update(pred_calls=23),
      "budget differs"),
-    (lambda sub: sub["contract"]["episode"].update(pred_calls=1000000),
-     "versioned contract"),
     (lambda sub: sub.update(agent_mode="codex"), "agent_mode"),
     (lambda sub: sub.update(prompt_id="custom"), "prompt_id"),
     (lambda sub: sub.pop("inference_parameters"), "inference_parameters"),
-    (lambda sub: sub["observation_policy"].update(policy_id="custom"),
-     "observation_policy"),
+    (lambda sub: sub.update(contract={}), "obsolete self-declared field"),
+    (lambda sub: sub.update(observation_policy={}), "obsolete self-declared field"),
     (lambda sub: sub.update(status="run_error", run_error="provider failed"), "incomplete"),
 ])
 def test_rankability_negative_controls(mutate, expected):
@@ -295,15 +282,15 @@ def test_public_projection_contains_no_answer_key_material():
 
 def test_top50_ties_ignore_tokens_and_efficiency_and_legacy_is_separate():
     entries = [
-        {"benchmark_contract": CONTRACT_ID, "model": "a", "bugs_found": 2,
+        {"agent_mode": AGENT_MODE, "model": "a", "bugs_found": 2,
          "total_tokens_k": 1, "efficiency_bugs_per_ktok": 999},
-        {"benchmark_contract": CONTRACT_ID, "model": "b", "bugs_found": 2,
+        {"agent_mode": AGENT_MODE, "model": "b", "bugs_found": 2,
          "total_tokens_k": 1000, "efficiency_bugs_per_ktok": 0.001},
         {"model": "a", "bugs_found": 9, "efficiency_bugs_per_ktok": 9},
     ]
     board = _dedup_best(entries)
     assert len(board) == 3
-    top50 = [entry for entry in board if entry.get("benchmark_contract") == CONTRACT_ID]
+    top50 = [entry for entry in board if entry.get("agent_mode") == AGENT_MODE]
     assert [entry["model"] for entry in top50] == ["a", "b"]
 
 
@@ -325,7 +312,7 @@ def test_backend_official_path_keeps_private_detail_and_publishes_aggregate(tmp_
     private = json.loads((results / "submission.json").read_text())
 
     assert private["artifact_sha256"] and "episodes" not in private
-    assert public["benchmark_contract"] == CONTRACT_ID
+    assert public["agent_mode"] == AGENT_MODE
     assert board == [public]
     assert "episodes" not in public and "shortlist" not in public
     guard = runpy.run_path(str(
@@ -335,8 +322,8 @@ def test_backend_official_path_keeps_private_detail_and_publishes_aggregate(tmp_
     assert guard(public_file) == []
 
 
-def test_site_keeps_contracts_separate_and_top50_ties_ignore_efficiency():
+def test_site_keeps_tracks_separate_and_top50_ties_ignore_efficiency():
     site = (Path(__file__).parents[2] / "site" / "index.html").read_text()
     assert 'id="lb-track"' in site
-    assert 'top50=track.startsWith("top50-evidence/")' in site
+    assert 'top50=track==="standardized-model-api"' in site
     assert "top50?String(a.model).localeCompare" in site
diff --git a/benchmark/top50_budget.json b/benchmark/top50_budget.json
deleted file mode 100644
index d8bd4f2..0000000
--- a/benchmark/top50_budget.json
+++ /dev/null
@@ -1,35 +0,0 @@
-{
-  "contract_id": "top50-evidence/v2",
-  "status": "frozen",
-  "shortlist_size": 50,
-  "hypothesis_chars": 500,
-  "safety_controls": {
-    "model_timeout_seconds": 300,
-    "model_retries": 2
-  },
-  "inference_parameters": {
-    "max_tokens": 8192,
-    "timeout": 300,
-    "num_retries": 2
-  },
-  "observation": {
-    "policy_id": "terminal-diagnostics/v1",
-    "preview_chars": 10000,
-    "archive_chars": 1048576
-  },
-  "triage": {
-    "model_generations": 8,
-    "shell_actions": 12,
-    "max_output_chars": 10000,
-    "command_timeout_seconds": 300
-  },
-  "episode": {
-    "model_generations": 10,
-    "shell_actions": 12,
-    "pred_calls": 24,
-    "solve_calls": 10,
-    "submit_attempts": 2,
-    "max_output_chars": 10000,
-    "pred_timeout_seconds": 300
-  }
-}
diff --git a/benchmark/top50_budget.py b/benchmark/top50_budget.py
index 25c6af3..81d2775 100644
--- a/benchmark/top50_budget.py
+++ b/benchmark/top50_budget.py
@@ -1,8 +1,55 @@
-"""Single loader for the immutable Top50 release contract."""
-from __future__ import annotations
+"""The benchmark's single built-in set of logical limits.
 
-import json
-from pathlib import Path
+These values define the benchmark.  They are deliberately Python constants rather than a
+configuration file: changing them changes the benchmark implementation, and the repository
+commit already identifies that change.
+"""
 
-BUDGET_PATH = Path(__file__).with_name("top50_budget.json")
-FROZEN_CONTRACT: dict = json.loads(BUDGET_PATH.read_text(encoding="utf-8"))
+SHORTLIST_SIZE = 50
+HYPOTHESIS_CHARS = 500
+
+TRIAGE_BUDGET = {
+    "model_generations": 8,
+    "shell_actions": 12,
+    "max_output_chars": 10_000,
+    "command_timeout_seconds": 300,
+}
+
+EPISODE_BUDGET = {
+    "model_generations": 10,
+    "shell_actions": 12,
+    "pred_calls": 24,
+    "solve_calls": 10,
+    "submit_attempts": 2,
+    "max_output_chars": 10_000,
+    "pred_timeout_seconds": 300,
+}
+
+OBSERVATION_LIMITS = {
+    "preview_chars": 10_000,
+    "archive_chars": 1_048_576,
+}
+
+SAFETY_CONTROLS = {
+    "model_timeout_seconds": 300,
+    "model_retries": 2,
+}
+
+INFERENCE_PARAMETERS = {
+    "max_tokens": 8192,
+    "timeout": 300,
+    "num_retries": 2,
+}
+
+
+def benchmark_parameters() -> dict:
+    """Return a fresh aggregate view for documentation and calibration checks."""
+    return {
+        "shortlist_size": SHORTLIST_SIZE,
+        "hypothesis_chars": HYPOTHESIS_CHARS,
+        "safety_controls": dict(SAFETY_CONTROLS),
+        "inference_parameters": dict(INFERENCE_PARAMETERS),
+        "observation": dict(OBSERVATION_LIMITS),
+        "triage": dict(TRIAGE_BUDGET),
+        "episode": dict(EPISODE_BUDGET),
+    }
diff --git a/benchmark/top50_config.yaml b/benchmark/top50_config.yaml
index e786a45..657d6b1 100644
--- a/benchmark/top50_config.yaml
+++ b/benchmark/top50_config.yaml
@@ -1,19 +1,33 @@
 agent:
   system_template: |
-    You are evaluating reduction rules under a fixed logical evidence budget. Each response
-    must contain exactly one bash command. The authoritative remaining budget is injected into
-    every observation; elapsed time is not a score and unused budget never transfers.
-    This is frozen contract top50-evidence/v2 with observation policy terminal-diagnostics/v1.
-    Command output is deterministically compacted into a 10,000-character preview. Each preview
-    names an evaluation-owned raw log that you may inspect with an ordinary shell action.
-    The prompt, strategy, shortlist size, and logical
-    counters cannot be customized on a rankable run.
+    You are reviewing the problem-reductions Rust library. It contains rules that reduce an
+    instance of one computational problem into another and later extract a source-problem
+    solution from the target-problem solution:
+
+      source instance -> reduce_to() -> target instance -> target solution
+                      -> extract_solution() -> source solution
+
+    A rule is buggy when this round trip can produce an invalid or suboptimal source solution,
+    reject a valid instance, panic on valid input, or otherwise violate the source problem's
+    semantics. Each response must contain exactly one bash command. The authoritative remaining
+    logical budget is included in every observation.
 
     {% if phase == "triage" %}
-    This is source-only triage. Read {{repo_dir}}/src/rules but do not run pred or submit.
+    This is source-only triage. Select the 50 rules most likely to contain such bugs; a later,
+    isolated episode will dynamically test each selected rule.
+
+    Repository: {{repo_dir}}
+    Rule sources: {{repo_dir}}/src/rules
     Canonical rule inventory: {{inventory}}
+
+    Inspect implementations and prioritize concrete semantic risks such as incorrect index or
+    identifier translation, missing constraints, objective/min-max conversion errors, invalid
+    extraction assumptions, graph-semantics mismatches, boundary cases, unchecked assumptions,
+    panics, and lossy numeric conversions. Do not run pred or submit in this phase.
+
     Write a JSON list of exactly 50 unique entries to the workspace. Each entry is either a
-    rule string or {"rule": "...", "hypothesis": "..."}. Freeze it with:
+    rule string or {"rule": "...", "hypothesis": "..."}. Prefer a concrete failure mechanism
+    over a generic claim that a rule may be buggy. Freeze the result with:
       commit-top50 shortlist.json
     {% else %}
     Investigate only rule {{rule}} ({{rule_index}}/{{rule_total}}).
diff --git a/benchmark/top50_contract.py b/benchmark/top50_contract.py
index 7c55771..74042cc 100644
--- a/benchmark/top50_contract.py
+++ b/benchmark/top50_contract.py
@@ -1,4 +1,4 @@
-"""Versioned private Top50 artifact validation, scoring, and public projection."""
+"""Private Top50 artifact validation, scoring, and public projection."""
 from __future__ import annotations
 
 import copy
@@ -12,27 +12,33 @@
 from pathlib import Path
 from typing import Callable
 
-from benchmark.top50_budget import FROZEN_CONTRACT
+from benchmark.top50_budget import (
+    EPISODE_BUDGET,
+    HYPOTHESIS_CHARS,
+    INFERENCE_PARAMETERS,
+    OBSERVATION_LIMITS,
+    SAFETY_CONTROLS,
+    SHORTLIST_SIZE,
+    TRIAGE_BUDGET,
+)
 from benchmark.usage import Usage, usage_as_dict, usage_from_dict
 from benchmark.verify import Verdict, verify
 
-CONTRACT_ID = FROZEN_CONTRACT["contract_id"]
 AGENT_MODE = "standardized-model-api"
 RUNNER_VERSION = "0.10.0"
-EXPECTED_TRIAGE_BUDGET = FROZEN_CONTRACT["triage"]
-EXPECTED_EPISODE_BUDGET = FROZEN_CONTRACT["episode"]
-EXPECTED_SAFETY_CONTROLS = FROZEN_CONTRACT["safety_controls"]
-EXPECTED_INFERENCE_PARAMETERS = FROZEN_CONTRACT["inference_parameters"]
-EXPECTED_OBSERVATION = FROZEN_CONTRACT["observation"]
-EXPECTED_SHORTLIST_SIZE = FROZEN_CONTRACT["shortlist_size"]
-EXPECTED_HYPOTHESIS_CHARS = FROZEN_CONTRACT["hypothesis_chars"]
+EXPECTED_TRIAGE_BUDGET = TRIAGE_BUDGET
+EXPECTED_EPISODE_BUDGET = EPISODE_BUDGET
+EXPECTED_SAFETY_CONTROLS = SAFETY_CONTROLS
+EXPECTED_INFERENCE_PARAMETERS = INFERENCE_PARAMETERS
+EXPECTED_OBSERVATION = OBSERVATION_LIMITS
+EXPECTED_SHORTLIST_SIZE = SHORTLIST_SIZE
+EXPECTED_HYPOTHESIS_CHARS = HYPOTHESIS_CHARS
 _REQUEST_ID = re.compile(r"[0-9a-f]{32}")
 _COUNTERS = ("model_generations", "shell_actions", "pred_calls", "solve_calls")
 
 
 def is_top50_submission(submission: object) -> bool:
-    return isinstance(submission, dict) and str(
-        submission.get("benchmark_contract", "")).startswith("top50-evidence/")
+    return isinstance(submission, dict) and submission.get("agent_mode") == AGENT_MODE
 
 
 def expected_prompt_id() -> str:
@@ -46,22 +52,21 @@ def validate_top50_submission(
     if not isinstance(submission, dict):
         return ["submission is not a JSON object"]
     problems: list[str] = []
-    for field in ("benchmark_contract", "model", "library_commit", "runner_version",
-                  "pred_version", "agent_mode", "prompt_id", "contract", "shortlist",
-                  "triage", "episodes", "status", "budget_contract_status",
-                  "safety_controls", "inference_parameters", "observation_policy"):
+    for field in ("model", "library_commit", "runner_version", "pred_version", "agent_mode",
+                  "prompt_id", "shortlist", "triage", "episodes", "status",
+                  "safety_controls", "inference_parameters"):
         if field not in submission:
             problems.append(f"missing required field: {field}")
     if problems:
         return problems
-    if submission["benchmark_contract"] != CONTRACT_ID:
-        problems.append(f"unsupported benchmark_contract: {submission['benchmark_contract']!r}")
+    for obsolete in ("benchmark_contract", "budget_contract_status", "contract",
+                     "observation_policy"):
+        if obsolete in submission:
+            problems.append(f"obsolete self-declared field is not allowed: {obsolete}")
     if submission["agent_mode"] != AGENT_MODE:
         problems.append(f"agent_mode must be {AGENT_MODE!r}")
     if submission["runner_version"] != RUNNER_VERSION:
         problems.append(f"runner_version must be {RUNNER_VERSION!r}")
-    if submission.get("budget_contract_status") != "frozen":
-        problems.append("budget_contract_status must be 'frozen'")
     if submission.get("safety_controls") != EXPECTED_SAFETY_CONTROLS:
         problems.append("safety_controls differ from the released watchdog settings")
     if submission.get("status") != "completed" or submission.get("run_error"):
@@ -77,43 +82,14 @@ def validate_top50_submission(
         problems.append("submitted_by is not a bounded safe identifier")
     if submission.get("inference_parameters") != EXPECTED_INFERENCE_PARAMETERS:
         problems.append("inference_parameters differ from the standardized model settings")
-    if submission.get("observation_policy") != EXPECTED_OBSERVATION:
-        problems.append("observation_policy differs from the versioned contract")
     if submission.get("prompt_id") != expected_prompt_id():
         problems.append("prompt_id does not match the frozen Top50 prompt")
     if "submit_limit" in submission or "submit_log" in submission:
         problems.append("Top50 artifacts cannot use a shared run-wide submit pool")
 
-    contract = submission.get("contract")
-    if not isinstance(contract, dict):
-        return problems + ["contract must be an object"]
-    triage_budget = contract.get("triage")
-    episode_budget = contract.get("episode")
-    observation_contract = contract.get("observation")
-    if not isinstance(triage_budget, dict) or not isinstance(episode_budget, dict):
-        return problems + ["contract triage and episode budgets must be objects"]
-    if triage_budget != EXPECTED_TRIAGE_BUDGET:
-        problems.append("triage budget differs from the versioned contract")
-    if episode_budget != EXPECTED_EPISODE_BUDGET:
-        problems.append("episode budget differs from the versioned contract")
-    if observation_contract != EXPECTED_OBSERVATION:
-        problems.append("observation contract differs from the versioned contract")
-    if contract.get("shortlist_size") != EXPECTED_SHORTLIST_SIZE:
-        problems.append(f"contract shortlist_size must be {EXPECTED_SHORTLIST_SIZE}")
-    hypothesis_limit = contract.get("hypothesis_chars")
-    if not _nonnegative_int(hypothesis_limit):
-        problems.append("contract hypothesis_chars must be a non-negative integer")
-        hypothesis_limit = -1
-    elif hypothesis_limit != EXPECTED_HYPOTHESIS_CHARS:
-        problems.append("contract hypothesis_chars differs from the versioned contract")
-    if episode_budget.get("submit_attempts") != 2:
-        problems.append("episode submit_attempts must be exactly 2")
-    pred_limit = episode_budget.get("pred_calls")
-    solve_limit = episode_budget.get("solve_calls")
-    if not _nonnegative_int(pred_limit) or not _nonnegative_int(solve_limit):
-        problems.append("pred_calls and solve_calls limits must be non-negative integers")
-    elif solve_limit > pred_limit:
-        problems.append("solve_calls limit exceeds pred_calls")
+    triage_budget = EXPECTED_TRIAGE_BUDGET
+    episode_budget = EXPECTED_EPISODE_BUDGET
+    hypothesis_limit = EXPECTED_HYPOTHESIS_CHARS
 
     shortlist = submission.get("shortlist")
     if not isinstance(shortlist, list) or len(shortlist) != EXPECTED_SHORTLIST_SIZE:
@@ -142,7 +118,7 @@ def validate_top50_submission(
         problems.append("triage.ledger must be an object")
     else:
         if triage_ledger.get("budget") != triage_budget:
-            problems.append("triage ledger budget differs from contract")
+            problems.append("triage ledger budget differs from built-in benchmark limits")
         _validate_observation_ledger(triage_ledger, "triage", problems)
         frozen = triage_ledger.get("shortlist")
         if frozen != shortlist:
@@ -173,7 +149,7 @@ def validate_top50_submission(
             problems.append(f"{label}.ledger must be an object")
             continue
         if ledger.get("rule") != expected_rule or ledger.get("budget") != episode_budget:
-            problems.append(f"{label} rule or budget differs from contract")
+            problems.append(f"{label} rule or budget differs from built-in benchmark limits")
         messages = episode.get("messages")
         if not isinstance(messages, list) or len(messages) > 3 * episode_budget.get(
                 "model_generations", 0) + 4:
@@ -238,7 +214,6 @@ def score_top50_submission(
     bugs = len(distinct_positions)
     usage = _aggregate_usage(submission)
     scored = {
-        "benchmark_contract": submission.get("benchmark_contract"),
         "model": submission.get("model", "unknown"),
         "library_commit": submission.get("library_commit", "unknown"),
         "runner_version": submission.get("runner_version"),
@@ -270,7 +245,6 @@ def score_top50_submission(
 def top50_public_entry(submission: dict, scored: dict) -> dict:
     """Project private scoring data to aggregate-only public fields."""
     return {
-        "benchmark_contract": scored["benchmark_contract"],
         "model": scored["model"],
         "library_commit": scored["library_commit"],
         "runner_version": scored.get("runner_version"),
@@ -304,7 +278,7 @@ def _validate_status(status, budget, label, problems, counters=_COUNTERS) -> Non
             problems.append(f"{label} {counter} limit is not a non-negative integer")
             continue
         if not isinstance(item, dict) or item.get("limit") != limit:
-            problems.append(f"{label} {counter} limit differs from contract")
+            problems.append(f"{label} {counter} limit differs from built-in benchmark limits")
             continue
         used, remaining = item.get("used"), item.get("remaining")
         if (not _nonnegative_int(used) or used > limit
@@ -398,8 +372,6 @@ def _validate_episode_events(ledger, label, problems) -> None:
 
 
 def _validate_observation_ledger(ledger: dict, label: str, problems: list[str]) -> None:
-    if ledger.get("observation_policy") != EXPECTED_OBSERVATION:
-        problems.append(f"{label} observation policy differs from contract")
     records = ledger.get("observations")
     if not isinstance(records, list):
         problems.append(f"{label} observations must be a list")
@@ -415,7 +387,7 @@ def _validate_observation_ledger(ledger: dict, label: str, problems: list[str])
     seen: set[str] = set()
     record_by_id: dict[str, dict] = {}
     required = {
-        "observation_id", "kind", "command", "policy_id", "raw_log", "returncode",
+        "observation_id", "kind", "command", "raw_log", "returncode",
         "timed_out", "original_chars", "original_lines", "preview_chars", "archive_chars",
         "preview_compacted", "archive_truncated",
     }
@@ -433,14 +405,12 @@ def _validate_observation_ledger(ledger: dict, label: str, problems: list[str])
             continue
         seen.add(observation_id)
         record_by_id[observation_id] = record
-        if record["policy_id"] != EXPECTED_OBSERVATION["policy_id"]:
-            problems.append(f"{label} observation policy id is inconsistent")
         if (not _nonnegative_int(record["preview_chars"])
                 or record["preview_chars"] > EXPECTED_OBSERVATION["preview_chars"]):
-            problems.append(f"{label} observation preview exceeds contract")
+            problems.append(f"{label} observation preview exceeds benchmark limit")
         if (not _nonnegative_int(record["archive_chars"])
                 or record["archive_chars"] > EXPECTED_OBSERVATION["archive_chars"]):
-            problems.append(f"{label} observation archive exceeds contract")
+            problems.append(f"{label} observation archive exceeds benchmark limit")
         if not _nonnegative_int(record["original_chars"]):
             problems.append(f"{label} observation original size is invalid")
         if not _nonnegative_int(record["original_lines"]):
diff --git a/benchmark/top50_results.schema.json b/benchmark/top50_results.schema.json
index 30fa87a..98817be 100644
--- a/benchmark/top50_results.schema.json
+++ b/benchmark/top50_results.schema.json
@@ -3,14 +3,14 @@
   "$id": "top50_results.schema.json",
   "title": "Top50EvidenceScoredResult",
   "type": "object",
-  "required": ["benchmark_contract", "model", "library_commit", "rankable",
+  "required": ["model", "library_commit", "agent_mode", "rankable",
     "rankability_errors", "verified_bugs", "bugs_at_10", "bugs_at_25", "bugs_at_50",
     "first_attempt_accepts", "second_attempt_accepts", "usage_totals",
     "artifact_sha256", "verifier_report"],
   "properties": {
-    "benchmark_contract": {"const": "top50-evidence/v2"},
     "model": {"type": "string"},
     "library_commit": {"type": "string"},
+    "agent_mode": {"const": "standardized-model-api"},
     "rankable": {"type": "boolean"},
     "rankability_errors": {"type": "array", "items": {"type": "string"}},
     "verified_bugs": {"type": "integer", "minimum": 0, "maximum": 50},
diff --git a/benchmark/top50_runner.py b/benchmark/top50_runner.py
index 69bcf37..fd4ab7d 100644
--- a/benchmark/top50_runner.py
+++ b/benchmark/top50_runner.py
@@ -23,7 +23,15 @@
     _message_text,
     _session_usage,
 )
-from benchmark.top50_budget import FROZEN_CONTRACT
+from benchmark.top50_budget import (
+    EPISODE_BUDGET,
+    HYPOTHESIS_CHARS,
+    INFERENCE_PARAMETERS,
+    OBSERVATION_LIMITS,
+    SAFETY_CONTROLS,
+    SHORTLIST_SIZE,
+    TRIAGE_BUDGET,
+)
 from benchmark.usage import Usage, usage_as_dict
 from benchmark.verify import Verdict, verify
 
@@ -56,7 +64,7 @@ class Top50Contract:
 
     def __post_init__(self) -> None:
         if self.shortlist_size != TOP50_SIZE:
-            raise ValueError("the rankable contract requires exactly 50 rules")
+            raise ValueError("the Top50 benchmark requires exactly 50 rules")
         if self.hypothesis_chars <= 0:
             raise ValueError("hypothesis_chars must be positive")
         caps = {self.triage.max_output_chars, self.episode.max_output_chars,
@@ -65,6 +73,17 @@ def __post_init__(self) -> None:
             raise ValueError("triage, episode, and observation preview caps must match")
 
 
+def benchmark_limits() -> Top50Contract:
+    """Return the benchmark's single built-in set of logical limits."""
+    return Top50Contract(
+        triage=TriageBudget(**TRIAGE_BUDGET),
+        episode=EvidenceBudget(**EPISODE_BUDGET),
+        observation=ObservationConfig(**OBSERVATION_LIMITS),
+        shortlist_size=SHORTLIST_SIZE,
+        hypothesis_chars=HYPOTHESIS_CHARS,
+    )
+
+
 @dataclass(frozen=True)
 class ShortlistEntry:
     rule: str
@@ -204,7 +223,6 @@ def status(self) -> dict:
     def ledger(self) -> dict:
         with self._lock:
             return {"budget": asdict(self.budget), "status": self.status(),
-                    "observation_policy": asdict(self.observation_config),
                     "observations": copy.deepcopy(self._observations),
                     "events": copy.deepcopy(self._events),
                     "shortlist": ([asdict(entry) for entry in self._shortlist]
@@ -359,11 +377,6 @@ def _result(self, model: str, triage: dict,
             "status": "run_error" if run_error else "completed",
             "rankable": (self._rankable_contract and run_error is None
                          and len(episodes) == self.contract.shortlist_size),
-            "contract": {"triage": asdict(self.contract.triage),
-                         "episode": asdict(self.contract.episode),
-                         "observation": asdict(self.contract.observation),
-                         "shortlist_size": self.contract.shortlist_size,
-                         "hypothesis_chars": self.contract.hypothesis_chars},
             "shortlist": ([asdict(entry) for entry in shortlist] if shortlist else None),
             "triage": {"ledger": triage, "messages": copy.deepcopy(triage_result.messages),
                        "tokens_k": triage_result.tokens_k,
@@ -377,7 +390,7 @@ def _result(self, model: str, triage: dict,
 
 
 def build_rankable_runner(
-    *, contract: Top50Contract, pred_binary: str | Path,
+    *, pred_binary: str | Path,
     agent_uid: int, agent_gid: int, oracle_uid: int, oracle_gid: int, evidence_gid: int,
     api_base: str | None = None, api_key: str | None = None,
     verifier: Callable[[dict], Verdict] = verify,
@@ -387,15 +400,16 @@ def build_rankable_runner(
         raise RuntimeError("rankable Top50 runs require the root runner privilege boundary")
     if len({agent_uid, oracle_uid}) != 2:
         raise ValueError("agent and oracle must use distinct identities")
-    inference = FROZEN_CONTRACT["inference_parameters"]
-    safety = FROZEN_CONTRACT["safety_controls"]
+    inference = INFERENCE_PARAMETERS
+    safety = SAFETY_CONTROLS
     executor = MiniSwePhaseExecutor(
         api_base=api_base, api_key=api_key, max_tokens=inference["max_tokens"],
         model_timeout_seconds=safety["model_timeout_seconds"],
         model_retries=safety["model_retries"], model_kwargs=None,
         agent_uid=agent_uid, agent_gid=agent_gid, evidence_gid=evidence_gid)
     return Top50Runner(
-        executor=executor, contract=contract, pred_binary=pred_binary, verifier=verifier,
+        executor=executor, contract=benchmark_limits(), pred_binary=pred_binary,
+        verifier=verifier,
         agent_uid=agent_uid, agent_gid=agent_gid, oracle_uid=oracle_uid,
         oracle_gid=oracle_gid, evidence_gid=evidence_gid,
         _rankable_token=_RANKABLE_TOKEN)
diff --git a/benchmark/top50_submission.schema.json b/benchmark/top50_submission.schema.json
index 667a841..c987a66 100644
--- a/benchmark/top50_submission.schema.json
+++ b/benchmark/top50_submission.schema.json
@@ -3,11 +3,10 @@
   "$id": "top50_submission.schema.json",
   "title": "Top50EvidenceSubmission",
   "type": "object",
-  "required": ["benchmark_contract", "model", "library_commit", "runner_version",
-    "pred_version", "agent_mode", "prompt_id", "status", "budget_contract_status", "safety_controls", "observation_policy", "contract", "shortlist",
+  "required": ["model", "library_commit", "runner_version",
+    "pred_version", "agent_mode", "prompt_id", "status", "safety_controls", "shortlist",
     "triage", "episodes", "inference_parameters"],
   "properties": {
-    "benchmark_contract": {"const": "top50-evidence/v2"},
     "model": {"type": "string", "minLength": 1},
     "library_commit": {"type": "string", "minLength": 1},
     "runner_version": {"const": "0.10.0"},
@@ -15,23 +14,18 @@
     "agent_mode": {"const": "standardized-model-api"},
     "prompt_id": {"type": "string", "minLength": 64, "maxLength": 64},
     "status": {"enum": ["completed", "run_error"]},
-    "budget_contract_status": {"const": "frozen"},
     "safety_controls": {"const": {"model_timeout_seconds": 300, "model_retries": 2}},
-    "observation_policy": {"const": {"policy_id": "terminal-diagnostics/v1", "preview_chars": 10000, "archive_chars": 1048576}},
     "rankable": {"type": "boolean"},
-    "contract": {
-      "const": {
-        "triage": {"model_generations": 8, "shell_actions": 12, "max_output_chars": 10000, "command_timeout_seconds": 300},
-        "episode": {"model_generations": 10, "shell_actions": 12, "pred_calls": 24, "solve_calls": 10, "submit_attempts": 2, "max_output_chars": 10000, "pred_timeout_seconds": 300},
-        "observation": {"policy_id": "terminal-diagnostics/v1", "preview_chars": 10000, "archive_chars": 1048576},
-        "shortlist_size": 50,
-        "hypothesis_chars": 500
-      }
-    },
     "shortlist": {"type": "array", "minItems": 50, "maxItems": 50},
     "triage": {"type": "object"},
     "episodes": {"type": "array", "minItems": 50, "maxItems": 50},
     "inference_parameters": {"const": {"max_tokens": 8192, "timeout": 300, "num_retries": 2}},
     "run_error": {"type": "string"}
-  }
+  },
+  "not": {"anyOf": [
+    {"required": ["benchmark_contract"]},
+    {"required": ["budget_contract_status"]},
+    {"required": ["contract"]},
+    {"required": ["observation_policy"]}
+  ]}
 }
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 5947623..8032d76 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -53,8 +53,7 @@ ARG PR_REF
 LABEL org.opencontainers.image.title="problem-reductions-benchmark" \
       org.opencontainers.image.description="pred (problem-reductions ${PR_REF}) + zero-trust certificate verifier" \
       org.opencontainers.image.licenses="MIT" \
-      problem_reductions_ref="${PR_REF}" \
-      benchmark_contract="top50-evidence/v2"
+      problem_reductions_ref="${PR_REF}"
 
 RUN groupadd --gid 10003 prb-evidence \
     && groupadd --gid 10001 prb-agent \
diff --git a/site/index.html b/site/index.html
index 980f2e0..8efe88b 100644
--- a/site/index.html
+++ b/site/index.html
@@ -416,14 +416,16 @@ 
How to cite

   {key:"predBug",label:"Pred / bug",num:true},
 ];
 const RIGHT=new Set(["bugs","tokens","eff","bugs10","bugs25","predBug"]);
-const selectedTrack=()=>document.getElementById("lb-track").value||"top50-evidence/v2";
-const isTop50Track=()=>selectedTrack().startsWith("top50-evidence/");
+const selectedTrack=()=>document.getElementById("lb-track").value||"standardized-model-api";
+const isTop50Track=()=>selectedTrack()==="standardized-model-api";
+const trackLabel=t=>t==="standardized-model-api"?"Standardized Top50":"Legacy whole repository";
 const lbColumns=()=>isTop50Track()?TOP50_COLS:LEGACY_COLS;
 
 function buildLbRows(){
   const track=selectedTrack();
-  const top50=track.startsWith("top50-evidence/");
-  const ranked=RESULTS.filter(r=>(r.benchmark_contract||"legacy-whole-repo")===track)
+  const top50=track==="standardized-model-api";
+  const ranked=RESULTS.filter(r=>(r.agent_mode==="standardized-model-api"?
+      "standardized-model-api":"legacy-whole-repo")===track)
     .sort((a,b)=>(b.bugs_found||0)-(a.bugs_found||0)||
       (top50?String(a.model).localeCompare(String(b.model)):
         (b.efficiency_bugs_per_ktok||0)-(a.efficiency_bugs_per_ktok||0)));
@@ -619,10 +621,11 @@ 
How to cite

   fetch("tasks.json").then(r=>r.json()).catch(()=>[]),
 ]).then(([res,tasks])=>{
   RESULTS=res;TASKS=tasks;TOTAL=tasks.length||TOTAL_DEFAULT;
-  const tracks=[...new Set(["top50-evidence/v2", ...RESULTS.map(r=>r.benchmark_contract||"legacy-whole-repo")])].sort();
+  const tracks=[...new Set(["standardized-model-api", ...RESULTS.map(r=>
+    r.agent_mode==="standardized-model-api"?"standardized-model-api":"legacy-whole-repo")])].sort();
   const trackSelect=document.getElementById("lb-track");
-  trackSelect.innerHTML=tracks.map(t=>``).join("");
-  const newest=tracks.find(t=>t.startsWith("top50-evidence/"));if(newest)trackSelect.value=newest;
+  trackSelect.innerHTML=tracks.map(t=>``).join("");
+  if(tracks.includes("standardized-model-api"))trackSelect.value="standardized-model-api";
   const rc=document.getElementById("rulecount"); if(rc)rc.textContent=TOTAL;
   if(RESULTS.some(r=>r.placeholder))
     document.getElementById("lb-banner").innerHTML=``;
diff --git a/submission.env.example b/submission.env.example
index 0a5d890..f0f51ba 100644
--- a/submission.env.example
+++ b/submission.env.example
@@ -30,7 +30,7 @@ MODEL_NAME=openai/gpt-5.4
 
 # ── FROZEN LIMITS ────────────────────────────────────────────────────────────
 # There are no budget environment variables. Rankable runs always use
-# top50-evidence/v2: source-only Top50 triage, then 50 isolated rule episodes with
+# Source-only Top50 triage, then 50 isolated rule episodes with
 # M=10, E=12, P=24, P_solve=10, and exactly S=2 submit attempts per rule.
 
 # ── OUTPUT / LOGS ────────────────────────────────────────────────────────────

From 09f13e6d99d435b8881e11088629fce06bf73935 Mon Sep 17 00:00:00 2001
From: Xiwei Pan 
Date: Tue, 21 Jul 2026 02:06:57 +0800
Subject: [PATCH 8/9] refactor: converge on single benchmark protocol

---
 .agents/skills/add-agent-harness/SKILL.md     | 110 ----
 .../add-agent-harness/agents/openai.yaml      |   4 -
 .../references/adapter-contract.md            | 194 -------
 .../references/reliability-evaluation.md      | 186 -------
 .agents/skills/run-api-benchmark/SKILL.md     |   8 +-
 .../references/provider-config.md             |  16 +-
 .agents/skills/run-benchmark/SKILL.md         |   7 +-
 .agents/skills/run-cli-benchmark/SKILL.md     | 144 -----
 .../run-cli-benchmark/agents/openai.yaml      |   4 -
 .../references/cli-config.md                  |  36 --
 .../skills/submit-benchmark-result/SKILL.md   |   4 +-
 .claude/skills/run-benchmark                  |   1 -
 .github/scripts/check_aggregate.py            |   4 +-
 .gitignore                                    |   3 +-
 CONTRIBUTING.md                               |  10 +-
 Makefile                                      |  34 +-
 README.md                                     |  16 +-
 .../{top50_config.yaml => agent_config.yaml}  |   0
 benchmark/agent_submit.py                     |   2 +-
 benchmark/backend_score.py                    | 104 +---
 benchmark/calibrate_budget.py                 |  10 +-
 benchmark/claude_code.py                      | 126 -----
 benchmark/codex_cli.py                        | 118 ----
 benchmark/config.yaml                         | 122 -----
 benchmark/docs/budget-calibration.md          |   2 +-
 .../design/top50-evidence-budget-benchmark.md | 504 ------------------
 .../design/top50-evidence-budget-issues.md    | 494 -----------------
 benchmark/env_setup.py                        |  79 ---
 benchmark/headless.py                         | 163 ------
 benchmark/model_api.py                        |  68 +++
 benchmark/preflight.py                        |  17 +-
 benchmark/results.schema.json                 | 144 +----
 benchmark/run_mini.py                         | 190 -------
 benchmark/run_submission.py                   | 428 ---------------
 benchmark/run_top50.py                        |  28 +-
 benchmark/submission.schema.json              | 143 +----
 benchmark/submit.py                           |  92 +---
 benchmark/submit_ledger.py                    |  52 --
 benchmark/submit_session.py                   | 112 +---
 benchmark/tests/conftest.py                   |   2 -
 benchmark/tests/test_backend_score.py         | 392 ++++----------
 benchmark/tests/test_budget_calibration.py    |   2 +-
 benchmark/tests/test_claude_code.py           | 231 --------
 benchmark/tests/test_codex_cli.py             | 214 --------
 benchmark/tests/test_docs.py                  |  21 +-
 benchmark/tests/test_env_setup.py             |  60 ---
 .../{test_run_mini.py => test_model_api.py}   |  15 +-
 benchmark/tests/test_preflight.py             |   2 +-
 benchmark/tests/test_run_submission.py        | 119 -----
 benchmark/tests/test_run_top50.py             |  23 +-
 benchmark/tests/test_submit.py                |  81 +--
 benchmark/tests/test_submit_session.py        |  58 +-
 benchmark/tests/test_top50_submission.py      |  29 +-
 benchmark/tests/test_trajectory.py            |  96 ----
 benchmark/tests/test_verify_submission.py     | 289 ----------
 benchmark/tests/test_whole_repo.py            |  85 ---
 benchmark/top50_contract.py                   |  32 +-
 benchmark/top50_results.schema.json           |  26 -
 benchmark/top50_runner.py                     |  27 +-
 benchmark/top50_submission.schema.json        |  31 --
 benchmark/verify_submission.py                | 205 +------
 intake/cloudflare-worker/src/index.js         |   5 +-
 intake/cloudflare-worker/test/index.test.js   |  20 +-
 site/index.html                               | 157 +-----
 submission.env.example                        |  19 +-
 65 files changed, 450 insertions(+), 5570 deletions(-)
 delete mode 100644 .agents/skills/add-agent-harness/SKILL.md
 delete mode 100644 .agents/skills/add-agent-harness/agents/openai.yaml
 delete mode 100644 .agents/skills/add-agent-harness/references/adapter-contract.md
 delete mode 100644 .agents/skills/add-agent-harness/references/reliability-evaluation.md
 delete mode 100644 .agents/skills/run-cli-benchmark/SKILL.md
 delete mode 100644 .agents/skills/run-cli-benchmark/agents/openai.yaml
 delete mode 100644 .agents/skills/run-cli-benchmark/references/cli-config.md
 delete mode 120000 .claude/skills/run-benchmark
 rename benchmark/{top50_config.yaml => agent_config.yaml} (100%)
 delete mode 100644 benchmark/claude_code.py
 delete mode 100644 benchmark/codex_cli.py
 delete mode 100644 benchmark/config.yaml
 delete mode 100644 benchmark/docs/design/top50-evidence-budget-benchmark.md
 delete mode 100644 benchmark/docs/design/top50-evidence-budget-issues.md
 delete mode 100644 benchmark/headless.py
 create mode 100644 benchmark/model_api.py
 delete mode 100644 benchmark/run_mini.py
 delete mode 100644 benchmark/run_submission.py
 delete mode 100644 benchmark/submit_ledger.py
 delete mode 100644 benchmark/tests/test_claude_code.py
 delete mode 100644 benchmark/tests/test_codex_cli.py
 rename benchmark/tests/{test_run_mini.py => test_model_api.py} (63%)
 delete mode 100644 benchmark/tests/test_run_submission.py
 delete mode 100644 benchmark/tests/test_trajectory.py
 delete mode 100644 benchmark/tests/test_verify_submission.py
 delete mode 100644 benchmark/tests/test_whole_repo.py
 delete mode 100644 benchmark/top50_results.schema.json
 delete mode 100644 benchmark/top50_submission.schema.json

diff --git a/.agents/skills/add-agent-harness/SKILL.md b/.agents/skills/add-agent-harness/SKILL.md
deleted file mode 100644
index 0682ebf..0000000
--- a/.agents/skills/add-agent-harness/SKILL.md
+++ /dev/null
@@ -1,110 +0,0 @@
----
-name: add-agent-harness
-description: Add, repair, or validate a headless coding-agent CLI harness backend for this problem-reductions benchmark, such as OpenCode, Kimi Code, Gemini CLI, or another agent runtime. Use when a requested agent harness is not yet listed in benchmark.run_submission.BACKENDS, when its CLI or event format changed, or when proving that a harness can use the evaluation-owned submit channel and produce a valid submission without spending model credits.
----
-
-# Add an agent harness
-
-Implement a first-class backend over the existing runner contract. Keep certificate
-verification, the run-wide submit ledger, scoring, and submission format independent of the
-harness.
-
-Read [references/adapter-contract.md](references/adapter-contract.md) before editing code.
-Read [references/reliability-evaluation.md](references/reliability-evaluation.md) before
-claiming that an adapter is reliable.
-
-## Discover the CLI contract
-
-1. Inspect the installed CLI's `--help` and `--version` output when available.
-2. For unstable or missing details, consult only the harness's official documentation or
-   source. Confirm a non-interactive one-prompt command, model selection, structured event
-   output, permission controls, exit behavior, and token-usage fields.
-3. Distinguish a model provider from an agent harness. Use the existing mini-swe/LiteLLM
-   backend when the request only changes the model API.
-4. Stop and explain the gap if the CLI cannot run autonomously, execute `submit`, or expose
-   reliable per-run usage. Do not invent usage. Offer an explicitly experimental local
-   backend only when the user accepts the missing metric.
-
-Do not install a CLI, authenticate it, or make a paid model call without the user's
-authorization. Before a real harness run, show the resolved CLI version, model, repository
-ref, submit limit, output path, and log path, then get explicit confirmation.
-
-## Implement the adapter
-
-Follow `benchmark/codex_cli.py` or `benchmark/claude_code.py`, and reuse
-`benchmark/headless.py`.
-
-1. Add `benchmark/_cli.py` with small, independently testable functions:
-   - command construction;
-   - child-environment construction;
-   - JSON/JSONL event parsing into `Usage` and a terminal result event;
-   - terminal-event error extraction;
-   - one `run_repo_()` entry point calling `run_headless_session()`.
-2. Render the shared prompt through `load_rendered_prompts()`. Do not fork the benchmark
-   task text or completion protocol.
-3. Run in `submit_session.workdir`, inherit the runner-injected `PATH`, and preserve stdout
-   as the raw trajectory log. Do not replace the file-backed `submit` transport.
-4. Disable ambient user rules, plugins, skills, MCP servers, and persistent sessions where
-   the CLI supports it. Grant only the filesystem and shell capabilities needed to inspect
-   the pinned repository, call `pred`, write artifacts, and call `submit`.
-5. Add the backend name to `BACKENDS` and `_run_backend()` in
-   `benchmark/run_submission.py`. Add explicit configuration parameters rather than a
-   generic arbitrary-shell backend.
-6. Update `Makefile`, `submission.env.example`, `README.md`, and `CONTRIBUTING.md` only for
-   harness behavior users can actually run.
-
-Preserve unrelated worktree changes.
-
-## Prove the integration
-
-Add `benchmark/tests/test__cli.py`. Tests must not invoke a real model or require
-`pred`.
-
-Cover the complete matrix in the adapter contract, including:
-
-- exact non-interactive command and model normalization;
-- parser behavior for assistant, tool, usage, malformed, and terminal-error events;
-- four-bucket `Usage` accounting without double counting;
-- missing executable, non-zero exit, and structured failure propagation;
-- dispatch through `run_submission.run()`;
-- a stub executable running inside a real `SubmissionSession`, successfully calling
-  `submit --status`, submitting a fixture certificate through the injected command, and
-  leaving a ledger-derived result;
-- raw stdout/stderr log persistence when a trajectory directory is configured.
-
-Use a fake verifier for the stub certificate. This is a wiring test, not a benchmark score.
-Do not weaken `SubmissionSession`, bypass the submit command, or insert result rows directly.
-
-Run the new test file, the existing headless/backend tests, and then the full pred-free unit
-suite. Validate any documented command against a stub or the installed CLI's help output.
-
-## Evaluate reliability
-
-After offline tests pass, run the controlled canary protocol in the reliability reference.
-It uses the real harness and model but a fake certificate verifier, so it measures the
-harness boundary rather than bug-finding ability or `pred`. Get confirmation before these
-model calls.
-
-Produce `harness-evaluation.json` with the CLI/model identity, offline test counts, three
-canary run records, usage evidence, log paths, and a derived verdict. Apply the acceptance
-table in the reference exactly; do not choose the verdict subjectively.
-
-Never use `bugs_found` as harness-reliability evidence. A correct harness may find zero real
-bugs, while a broken harness may print plausible certificates without reaching `submit`.
-
-## Completion gate
-
-Call the adapter complete only when:
-
-- all required tests pass without network, credentials, model spend, or `pred`;
-- the status probe reaches the authoritative submission service;
-- accepted and rejected attempts consume the shared ledger correctly;
-- a CLI/process failure becomes `run_error` while preserving partial ledger results;
-- token usage is parsed from a documented per-run source;
-- documentation states authentication, configuration isolation, supported CLI versions,
-  and the authoritative output/log paths;
-- `harness-evaluation.json` satisfies every `reliable` invariant in the reliability
-  reference, including three consecutive successful canaries.
-
-Report unsupported capabilities explicitly. Record the CLI version used by the local smoke
-test, and do not claim compatibility with untested versions.
diff --git a/.agents/skills/add-agent-harness/agents/openai.yaml b/.agents/skills/add-agent-harness/agents/openai.yaml
deleted file mode 100644
index dc01e42..0000000
--- a/.agents/skills/add-agent-harness/agents/openai.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-interface:
-  display_name: "Add Agent Harness"
-  short_description: "Integrate and verify a new benchmark agent CLI"
-  default_prompt: "Use $add-agent-harness to add and validate a new agent CLI backend."
diff --git a/.agents/skills/add-agent-harness/references/adapter-contract.md b/.agents/skills/add-agent-harness/references/adapter-contract.md
deleted file mode 100644
index d0173ae..0000000
--- a/.agents/skills/add-agent-harness/references/adapter-contract.md
+++ /dev/null
@@ -1,194 +0,0 @@
-# Agent harness adapter contract
-
-## Runtime boundary
-
-The harness adapter owns only the CLI process boundary:
-
-```text
-shared benchmark prompt
-        |
-        v
-harness CLI in SubmissionSession.workdir
-        |
-        +-- pred from PATH
-        +-- submit --status / submit certificate.json
-        +-- JSON or JSONL stdout events
-        v
-{tokens_k, usage, error}
-```
-
-The runner continues to own the pinned repository context, `pred` validation, submission
-budget, in-memory ledger, result rows, `submission.json`, backend re-verification, and score.
-
-## Python interface
-
-Expose one repository-session function compatible with `_run_backend()`:
-
-```python
-def run_repo_harness(
-    model_name: str,
-    ctx,
-    *,
-    trajectory_dir=None,
-    config_path=None,
-    strategy=None,
-    api_key=None,
-    session_timeout=None,
-    submit_session=None,
-) -> dict:
-    return {"tokens_k": 0.0, "usage": Usage(), "error": None}
-```
-
-Accept harness-specific keyword arguments when needed, but keep `_run_backend()`'s common
-call surface stable. A backend must launch exactly one whole-repository session.
-
-Use these shared helpers:
-
-- `load_rendered_prompts()` for the canonical system/task prompt;
-- `child_env()` to preserve the injected `pred` and `submit` path;
-- `run_headless_session()` for workspace, timeout, stdout/stderr logs, and normalized return;
-- `Usage` for uncached input, output, cache-read, and cache-write tokens.
-
-## Command requirements
-
-The command must:
-
-- be non-interactive and exit when the agent finishes;
-- accept the requested model explicitly when supported;
-- emit machine-readable events without terminal decoration;
-- allow shell commands and reads of the absolute pinned repository path;
-- allow writes only in the runner-provided scratch workspace where practical;
-- avoid resuming a prior session;
-- avoid ambient user/project instructions and external plugins where practical;
-- impose no agent turn, step, rule-count, or cost cap;
-- rely on the runner timeout only as a hung-process backstop.
-
-If a CLI merges rather than replaces its system prompt, record that behavior and ensure the
-benchmark prompt remains intact. Do not silently pretend prompt equivalence.
-
-## Parser requirements
-
-Keep parsing pure: an iterable of lines in, a dictionary out. Ignore malformed and unknown
-events unless the CLI declares them terminal failures.
-
-Return at least:
-
-```python
-{
-    "trajectory": [],
-    "usage": Usage(),
-    "steps": 0,
-    "result_event": None,
-}
-```
-
-Prefer a final cumulative usage event. Otherwise deduplicate message IDs before summing
-per-message usage. Map provider fields to `Usage` as follows:
-
-| Benchmark bucket | Meaning |
-|---|---|
-| `input_tokens` | uncached prompt/input tokens |
-| `output_tokens` | completion tokens, including reported reasoning tokens |
-| `cache_read_tokens` | cached input read tokens |
-| `cache_write_tokens` | cache creation/write tokens |
-
-Never infer token counts from characters, prices, context-window size, or cumulative
-machine-wide statistics.
-
-## Required test matrix
-
-### Pure command tests
-
-- executable and non-interactive subcommand/flags;
-- prompt placement and quoting through argv, not a shell string;
-- model prefix normalization;
-- structured output flag;
-- permission/config-isolation flags;
-- absence of turn/step/cost caps.
-
-### Pure parser tests
-
-- assistant text event;
-- tool call and tool result events;
-- cumulative usage;
-- repeated message IDs or repeated cumulative events;
-- cache-read/cache-write mapping;
-- malformed JSON and unknown events;
-- success terminal event;
-- structured error terminal event.
-
-### Stub-process tests
-
-Create a temporary executable that emits representative events and optionally exits
-non-zero. Verify:
-
-- successful normalized return;
-- missing CLI error;
-- non-zero exit error with a useful tail;
-- terminal error propagation even with exit code zero;
-- raw stdout and sibling stderr logs;
-- configured timeout behavior when practical without making the suite slow.
-
-### Submission-channel wiring test
-
-Use a real `SubmissionSession` with a fake verifier returning `Verdict(True, ...)`. The stub
-CLI must discover `submit` through `PATH` and execute:
-
-```sh
-submit --status
-submit "$PRB_ARTIFACT_DIR/stub-certificate.json"
-```
-
-Assert:
-
-- `submit_session.reachable` is true;
-- status does not consume an attempt;
-- the certificate consumes exactly one attempt;
-- `result_rows()` is derived from the accepted ledger;
-- wrapping the backend through `run_submission.run()` produces a schema-valid envelope;
-- a failing stub preserves any earlier accepted row and adds `run_error`.
-
-Do not monkeypatch `result_rows()`, append to private ledger fields, or return certificates
-from the adapter.
-
-### Dispatch and CLI tests
-
-- accepted backend name in argparse choices;
-- direct dispatch calls the new backend exactly once;
-- other backends are not invoked;
-- invalid backend names remain rejected;
-- explicit output and trajectory paths remain separate.
-
-## Verification commands
-
-Adapt filenames to the harness:
-
-```bash
-pytest -v benchmark/tests/test__cli.py
-pytest -v \
-  benchmark/tests/test__cli.py \
-  benchmark/tests/test_codex_cli.py \
-  benchmark/tests/test_claude_code.py \
-  benchmark/tests/test_run_submission.py \
-  benchmark/tests/test_submit_session.py
-pytest -v -m "not integration"
-python -m benchmark.submit --predictions  --dry-run
-```
-
-The final dry-run is useful only when the wiring test writes a complete fake envelope. It
-must remain clearly labeled as a no-spend wiring artifact, not a benchmark result.
-
-## Validation tiers
-
-Classify support honestly:
-
-1. **Adapter unit-tested**: parser, dispatch, process errors, and submit wiring pass with a
-   stub CLI.
-2. **Installed harness supported**: the installed real CLI passes auth/preflight and one
-   user-approved run while preserving logs and usage.
-3. **Version compatibility recorded**: the smoke-tested CLI version and event fixtures are
-   recorded, and CI continues to exercise the adapter contract without credentials or model
-   spend.
-
-Do not describe tier 1 as live-model validation, and do not claim compatibility with CLI
-versions that were not inspected or smoke-tested.
diff --git a/.agents/skills/add-agent-harness/references/reliability-evaluation.md b/.agents/skills/add-agent-harness/references/reliability-evaluation.md
deleted file mode 100644
index 0068e83..0000000
--- a/.agents/skills/add-agent-harness/references/reliability-evaluation.md
+++ /dev/null
@@ -1,186 +0,0 @@
-# Harness reliability evaluation
-
-## Contents
-
-- [Purpose](#purpose)
-- [Phase A: offline conformance](#phase-a-offline-conformance)
-- [Phase B: real-harness controlled canary](#phase-b-real-harness-controlled-canary)
-- [Phase C: failure semantics](#phase-c-failure-semantics)
-- [Evidence file](#evidence-file)
-- [Verdict rules](#verdict-rules)
-- [Handoff](#handoff)
-
-## Purpose
-
-Evaluate whether a newly added agent harness reliably crosses the benchmark process
-boundary. Do not evaluate how good its model is at finding reduction bugs.
-
-The controlled evaluation must answer:
-
-1. Can the runner start the requested CLI non-interactively with the intended prompt and
-   model?
-2. Can the harness execute shell commands in the supplied workspace and reach the real
-   evaluation-owned `submit` command?
-3. Does the runner preserve the authoritative attempt budget and derive rows only from the
-   ledger?
-4. Does the adapter parse completion, failure, and per-run token usage correctly?
-5. Are raw logs and partial results preserved when something fails?
-
-`bugs_found` is not an integration metric. Do not require a genuine reduction bug or run
-`pred` during harness conformance.
-
-## Phase A: offline conformance
-
-Run without credentials, network access, model calls, or `pred`:
-
-```bash
-pytest -v benchmark/tests/test__cli.py
-pytest -v \
-  benchmark/tests/test__cli.py \
-  benchmark/tests/test_codex_cli.py \
-  benchmark/tests/test_claude_code.py \
-  benchmark/tests/test_run_submission.py \
-  benchmark/tests/test_submit_session.py
-pytest -v -m "not integration"
-```
-
-Record the command, exit code, passed/failed/skipped counts, and log path. All commands must
-exit zero. A skipped test that covers a required adapter invariant counts as a failure;
-unrelated skips may remain but must be listed.
-
-The harness-specific suite must prove:
-
-- exact argv and configuration isolation;
-- JSON/JSONL parsing and four-bucket usage accounting;
-- missing executable, non-zero exit, timeout, and terminal event failures;
-- stdout/stderr persistence;
-- dispatch through the benchmark entry point;
-- real file-backed `submit --status` and submit calls through `SubmissionSession` using a
-  stub CLI and fake verifier;
-- accepted and rejected attempts, budget exhaustion, ledger-derived rows, partial-result
-  salvage, and schema-valid output.
-
-## Phase B: real-harness controlled canary
-
-This phase invokes the installed real harness and configured model. Show the resolved CLI
-path/version, model, repository ref, submit limit, output path, and log directory, then get
-explicit confirmation because it consumes model credits.
-
-Use a dedicated conformance prompt, not the bug-finding prompt. Run inside a real
-`SubmissionSession` with a controlled verifier:
-
-- accept only the certificate whose rule is `HarnessProbe/Accepted`;
-- reject the certificate whose rule is `HarnessProbe/Rejected`;
-- never invoke `pred`.
-
-Ask the harness to perform exactly this sequence:
-
-1. Run `submit --status` and preserve its output.
-2. Write an accepted fixture certificate under `$PRB_ARTIFACT_DIR` and submit it.
-3. Write a rejected fixture certificate under `$PRB_ARTIFACT_DIR` and submit it.
-4. Run `submit --status` again.
-5. Emit the required completion marker and exit.
-
-Use valid certificate shapes with distinct rules and a small synthetic `source`. The fake
-verifier decides only from the rule name; the certificates are transport fixtures, not bug
-claims.
-
-Run the canary three times as three fresh sessions. Do not resume session state. Keep the
-model, CLI version, prompt, adapter code, and verifier fixed across the three runs.
-
-For every run record and verify:
-
-| Invariant | Required result |
-|---|---|
-| Process | exit code `0`; no timeout |
-| Runner | `run_error` absent |
-| Status | both status commands visibly succeed |
-| Attempts | exactly `2` attempts |
-| Acceptance | exactly `1` accepted and `1` rejected |
-| Order | accepted fixture first, rejected fixture second |
-| Budget | remaining count decreases by exactly `2`; status calls are free |
-| Rows | exactly two ledger-derived rows with matching rules and outcomes |
-| Completion | required completion marker is present |
-| Usage | documented per-run usage is present and total tokens are greater than zero |
-| Logs | non-empty raw stdout log and captured stderr log exist |
-| Isolation | no prior session is resumed and no result comes from files outside the run |
-
-If the harness does not expose trustworthy per-run usage, classify it as `experimental`,
-not `reliable`. Never estimate tokens from text length or cost.
-
-## Phase C: failure semantics
-
-Use deterministic stub processes, not the real model, to exercise:
-
-1. accepted submit followed by CLI non-zero exit;
-2. accepted submit followed by a structured terminal error with exit code zero;
-3. accepted submit followed by timeout;
-4. submit channel never probed;
-5. malformed JSONL mixed with valid events.
-
-Every case must set the appropriate `run_error`, preserve the accepted ledger row and raw
-logs when present, and never report an infrastructure failure as a clean zero-result run.
-
-## Evidence file
-
-Write a machine-readable `harness-evaluation.json` outside the authoritative submission
-path. Use this minimum shape:
-
-```json
-{
-  "schema_version": "1.0",
-  "harness": "kimi-code",
-  "cli_path": "/absolute/path/to/kimi",
-  "cli_version": "recorded exact output",
-  "model": "provider/model",
-  "adapter_commit": "git commit or worktree marker",
-  "offline": {
-    "commands": [],
-    "passed": 0,
-    "failed": 0,
-    "required_skipped": 0
-  },
-  "canary": {
-    "required_runs": 3,
-    "successful_runs": 0,
-    "runs": []
-  },
-  "failure_semantics": {
-    "passed": 0,
-    "failed": 0
-  },
-  "verdict": "reliable | experimental | unreliable",
-  "reasons": []
-}
-```
-
-Each canary run must include the invariant values from Phase B plus absolute stdout/stderr
-log paths and the four usage buckets. Do not store credentials in the report or logs.
-
-## Verdict rules
-
-Apply the first matching rule:
-
-| Verdict | Exact condition |
-|---|---|
-| `unreliable` | Any required offline test fails; any ledger/budget invariant fails; accepted results bypass `submit`; an infrastructure failure is reported as a clean run; or audit logs are missing |
-| `experimental` | Offline conformance passes, but fewer than 3/3 canaries pass, usage is absent or untrustworthy, only stub tests were run, or the inspected CLI version differs from the smoke-tested version |
-| `reliable` | All offline commands pass with zero required skips; all failure-semantics cases pass; and 3/3 fresh real-harness canaries satisfy every Phase B invariant with trustworthy usage and complete logs |
-
-One successful live run is insufficient for `reliable`. A 2/3 result is `experimental`, not
-“mostly reliable.” Re-run only after diagnosing and changing the adapter, configuration, or
-harness version; do not discard a failed run and resample until three successes appear.
-
-## Handoff
-
-Report:
-
-- the verdict and exact failed invariant, if any;
-- CLI path/version and model;
-- offline test totals;
-- canary success count out of three;
-- attempt, acceptance, budget, usage, and completion evidence per run;
-- absolute paths to `harness-evaluation.json` and raw logs;
-- whether any real model calls were made.
-
-Do not describe an adapter as supported without attaching this evidence summary.
diff --git a/.agents/skills/run-api-benchmark/SKILL.md b/.agents/skills/run-api-benchmark/SKILL.md
index 824fdbd..cd1c485 100644
--- a/.agents/skills/run-api-benchmark/SKILL.md
+++ b/.agents/skills/run-api-benchmark/SKILL.md
@@ -5,7 +5,7 @@ description: Configure and run the rankable problem-reductions Self-selected Top
 
 # Run the rankable Top50 benchmark
 
-Use only the standardized Top50 path: source-only triage freezes 50 rules, followed by 50 fresh sequential episodes. Each rule receives M=10 model generations, E=12 shell actions, P=24 `pred` calls, P_solve=10 solve calls, O=10000 observed characters, and exactly S=2 submit attempts. These limits are part of the benchmark code; never add or accept custom prompts, strategies, model kwargs, or coding-agent backends.
+The benchmark uses source-only triage to freeze 50 rules, followed by 50 fresh sequential episodes. Each rule receives M=10 model generations, E=12 shell actions, P=24 `pred` calls, P_solve=10 solve calls, O=10000 observed characters, and exactly S=2 submit attempts. These limits, prompts, inference settings, and the harness are part of the benchmark code.
 
 Read [references/provider-config.md](references/provider-config.md) for endpoint setup and `scripts/detect-engine.sh` before preparing the image.
 
@@ -19,11 +19,11 @@ Read [references/provider-config.md](references/provider-config.md) for endpoint
 
 ## Configure and preflight
 
-Create `submission.env` from the example when absent. Configure `MODEL_NAME`, a provider key or `API_KEY`, and `API_BASE` only when needed. An existing `MODEL_KWARGS`, `AGENT_BACKEND`, `AGENT_CONFIG`, `AGENT_STRATEGY_FILE`, `SUBMIT_LIMIT`, or `PRB_*` budget override must be removed before a rankable run.
+Create `submission.env` from the example when absent. Configure `MODEL_NAME`, a provider key or `API_KEY`, and `API_BASE` only when needed. Do not add other execution settings.
 
-Detect the container engine. Prefer `make runner-pull`; use `make runner-build` only when necessary. Before the first real API call, show the redacted model/endpoint, frozen contract ID and counters, target ref, output path, and upload goal, then ask for confirmation.
+Detect the container engine. Prefer `make runner-pull`; use `make runner-build` only when necessary. Before the first real API call, show the redacted model/endpoint, built-in counters, target ref, output path, and upload goal, then ask for confirmation.
 
-Run `make preflight`. It must reject custom execution knobs before its tiny model call, then verify `pred`, source inventory, endpoint, and credentials. Never continue after a failed preflight.
+Run `make preflight`. It verifies the frozen source and `pred`, then checks the endpoint and credentials with a tiny model call. Never continue after a failed preflight.
 
 ## Run and validate
 
diff --git a/.agents/skills/run-api-benchmark/references/provider-config.md b/.agents/skills/run-api-benchmark/references/provider-config.md
index e787916..0588eea 100644
--- a/.agents/skills/run-api-benchmark/references/provider-config.md
+++ b/.agents/skills/run-api-benchmark/references/provider-config.md
@@ -3,8 +3,7 @@
 Use `submission.env.example` as the source of truth. Keep secrets in the gitignored
 `submission.env`; never ask the caller to paste them into chat.
 
-This configuration is only for the containerized `mini-swe` API route. Do not select Codex,
-Claude Code, or another host CLI with `AGENT_BACKEND`.
+This configuration is for the containerized model API runner.
 
 ## Variables
 
@@ -13,14 +12,9 @@ Claude Code, or another host CLI with `AGENT_BACKEND`.
 | `MODEL_NAME` | LiteLLM-routable provider/model identity |
 | provider key or `API_KEY` | authentication |
 | `API_BASE` | custom gateway or OpenAI-compatible endpoint |
-| `MODEL_KWARGS` | JSON object for provider-specific LiteLLM arguments |
-| `MAX_TOKENS` | per-response output ceiling, not a run/turn limit |
-| `SUBMIT_LIMIT` | run-wide certificate attempts |
-| `AGENT_CONFIG` | full prompt override |
-| `AGENT_STRATEGY_FILE` | extra strategy injected into the shared prompt |
 | `SUBMITTED_BY` | optional submitter metadata |
 
-Do not use removed `AGENT_MODE`, `MAX_RULES`, or max-turn variables.
+Prompts, inference settings, the execution harness, and logical budgets are benchmark-owned.
 
 ## Standard provider
 
@@ -33,7 +27,6 @@ Set `MODEL_NAME` and the provider's documented key variable, for example
 MODEL_NAME=openai/my-model
 API_BASE=https://my-gateway.example/v1
 API_KEY=...
-MODEL_KWARGS={"custom_llm_provider":"openai"}
 ```
 
 ## Failure decoding
@@ -41,7 +34,6 @@ MODEL_KWARGS={"custom_llm_provider":"openai"}
 | Symptom | Action |
 |---|---|
 | `pred binary` or rule-source preflight fails | rebuild the image at the intended `PR_REF` |
-| tiny model call fails | correct model routing, endpoint, key variable, or kwargs |
+| tiny model call fails | correct model routing, endpoint, or key variable |
 | build exits 137 | provision at least about 8 GB for the engine VM/host |
-| `run_error` | preserve the partial submission and logs; do not call it a clean zero |
-| submit channel not probed | treat as runner infrastructure failure |
+| `run_error` | preserve the partial submission; do not call it a clean zero |
diff --git a/.agents/skills/run-benchmark/SKILL.md b/.agents/skills/run-benchmark/SKILL.md
index 5c28e2e..eaeb3f2 100644
--- a/.agents/skills/run-benchmark/SKILL.md
+++ b/.agents/skills/run-benchmark/SKILL.md
@@ -1,14 +1,13 @@
 ---
 name: run-benchmark
-description: Route a request to run, reproduce, smoke-test, or generate a submission for the problem-reductions benchmark. The current rankable route is the standardized Model API Top50 runner; coding-agent CLI execution is historical and non-ranking.
+description: Route a request to run, reproduce, smoke-test, or generate a submission for the standardized problem-reductions Model API Top50 benchmark.
 ---
 
 # Route a benchmark run
 
 If the caller already has a `submission.json`, invoke `$submit-benchmark-result`.
 
-For a current/rankable run, invoke `$run-api-benchmark`. Do not offer a backend choice: the frozen public contract is Model API only.
-
-If the caller explicitly asks to reproduce a legacy Codex, Claude Code, or other coding-agent artifact, explain that it belongs to `legacy-whole-repo`, cannot enter the Top50 table, and invoke `$run-cli-benchmark` only after they confirm they want a non-ranking historical run.
+For a benchmark run, invoke `$run-api-benchmark`. Do not offer a backend or protocol choice:
+the benchmark has one built-in Model API contract.
 
 The child skill owns preflight, execution, validation, and optional upload.
diff --git a/.agents/skills/run-cli-benchmark/SKILL.md b/.agents/skills/run-cli-benchmark/SKILL.md
deleted file mode 100644
index 29c301f..0000000
--- a/.agents/skills/run-cli-benchmark/SKILL.md
+++ /dev/null
@@ -1,144 +0,0 @@
----
-name: run-cli-benchmark
-description: Configure and run this problem-reductions benchmark through an installed autonomous coding-agent CLI harness. Use when the caller requests Codex, Claude Code, Kimi Code, OpenCode, or another CLI agent. Lists the harnesses actually supported by benchmark.run_submission.BACKENDS, routes unsupported harnesses through add-agent-harness before returning to model selection, verifies CLI authentication and pinned pred, runs one whole-repository session, validates submission.json, and uploads only when explicitly requested.
----
-
-# Reproduce the historical CLI benchmark
-
-This route is permanently **non-ranking** and exists only to reproduce `legacy-whole-repo`
-artifacts. Tell the caller that coding-agent results cannot enter the standardized Top50 table and
-get confirmation before spending time or credits. Do not describe it as a System Track.
-
-Run one whole-repository coding-agent session with the shared benchmark prompt, verifier,
-submit ledger, and schema. This skill owns only coding-agent CLI execution. Do not add a
-step, turn, cost, or rule-count limit; session timeouts are hung-process backstops only.
-
-Read [references/cli-config.md](references/cli-config.md) when resolving authentication,
-paths, model names, or failures.
-
-## Ask the CLI questions
-
-Ask only for information the caller has not already supplied. Use the product's structured
-user-input UI when available; otherwise ask the quoted question in plain text. Ask one
-numbered stage at a time when its answer controls the next branch.
-
-1. Inspect `benchmark.run_submission.BACKENDS` and the direct dispatch cases in
-   `_run_backend()`. Display the currently supported coding-agent harnesses, excluding
-   `mini-swe`. Use human names alongside backend IDs, for example:
-
-   ```text
-   Codex       (codex)
-   Claude Code (claude-code)
-   ```
-
-   Then ask:
-
-   > Which supported coding-agent CLI should run the benchmark?
-
-2. If the requested harness is not supported, do not substitute another agent and do not
-   continue to model selection. Explain that it must first be integrated, then ask:
-
-   > `` is not currently supported. Should I invoke `$add-agent-harness` to implement
-   > and validate its adapter first?
-
-   On approval, invoke `$add-agent-harness`. Return to this skill only after the adapter is
-   present in `BACKENDS` and its `harness-evaluation.json` verdict is `reliable`. Then display
-   the refreshed supported list and continue.
-
-3. Ask:
-
-   > Which model should the selected CLI use?
-
-   Use the CLI's model syntax; do not silently translate to an API/LiteLLM model name.
-
-4. Ask:
-
-   > What should happen after the run?
-   >
-   > 1. Keep and validate the result locally without uploading.
-   > 2. Upload an official submission.
-
-   The `$submit-benchmark-result` skill owns submission validation, authentication, and
-   upload.
-
-5. Read this checkout's benchmark version with `make -s print-benchmark-version`. Read the
-   latest version from the official repository's
-   [`main/VERSION`](https://github.com/CodingThrust/problem-reductions-benchmark/blob/main/VERSION);
-   do not use the `problem-reductions` version or guess. Show this in the caller's language
-   and wait for confirmation:
-
-   > Benchmark version: `` (latest version: `
`)
-
-   If the versions differ, explain that the checkout is outdated and ask the caller to
-   update it before an official run. If the latest-version lookup fails, show `unknown`
-   rather than substituting the pinned `problem-reductions` version.
-
-6. Resolve the internal problem-reductions pin with `make -s print-pr-ref`, plus
-   `SUBMIT_LIMIT` (default 100) and three separate, explicit paths: clone destination,
-   authoritative submission JSON, and log directory. Offer concrete defaults and ask
-   whether to accept them.
-
-## Verify the selected harness
-
-For Codex, require `codex` plus either a successful `codex login status` or a configured
-`OPENAI_API_KEY`. For Claude Code, require `claude` and its authenticated login/token flow.
-For newly integrated agents, follow their adapter's documented auth probe.
-
-Also require:
-
-- `git` and benchmark Python dependencies;
-- a `pred` binary matching the benchmark version;
-- an absent clone destination, or an existing Git checkout whose `HEAD` exactly matches
-  `PR_REF`.
-
-Never fetch, reset, or check out an existing mismatched working tree. Choose another path or
-ask the caller to update it.
-
-## Confirm and run
-
-Before the first real model call, show:
-
-- CLI human name and backend ID;
-- resolved CLI path/version and authentication status;
-- exact model;
-- confirmed benchmark version and resolved commit;
-- submit limit;
-- clone destination, authoritative output path, and log path;
-- local versus official-upload goal.
-
-State that the run can consume substantial time or credits, then get explicit confirmation.
-
-This route runs the selected CLI directly on the host. Select it only with
-`LOCAL_BACKEND=`. Do not set `AGENT_BACKEND`, invoke `make run`/`make preflight`,
-or start Docker/Podman for a CLI run.
-
-Run through `make run-local` with the selected backend. If invoking
-`benchmark.run_submission` directly, pass `--host-cli` with the same explicit values. For
-example:
-
-```bash
-PR_REF="$(make -s print-pr-ref)"
-make run-local \
-  LOCAL_BACKEND=codex \
-  LOCAL_REPO_DIR="../runs/problem-reductions-${PR_REF}" \
-  LOCAL_OUTPUT=../runs/results/submission.json \
-  LOCAL_LOG_DIR=../runs/logs
-```
-
-Every harness must begin with `submit --status`. If neither a status request nor a submit
-attempt reaches the service, require `run_error` and inspect
-`/salvaged-agent-artifacts/`; never report it as a clean zero.
-
-## Validate and hand off
-
-For option 1, validate the authoritative file locally:
-
-```bash
-python -m benchmark.submit --predictions  --dry-run
-```
-
-Report `bugs_found`, `total_tokens_k`, submit attempts, any `run_error`, CLI warnings, and
-absolute output/log paths. Preserve partial results and logs on failure.
-
-Do not upload a historical CLI artifact as a current official submission. Validate and keep
-it locally under the legacy contract.
diff --git a/.agents/skills/run-cli-benchmark/agents/openai.yaml b/.agents/skills/run-cli-benchmark/agents/openai.yaml
deleted file mode 100644
index 0d704c6..0000000
--- a/.agents/skills/run-cli-benchmark/agents/openai.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-interface:
-  display_name: "Run CLI Benchmark"
-  short_description: "Run the benchmark with a supported coding agent CLI"
-  default_prompt: "Use $run-cli-benchmark to select and run a coding agent harness."
diff --git a/.agents/skills/run-cli-benchmark/references/cli-config.md b/.agents/skills/run-cli-benchmark/references/cli-config.md
deleted file mode 100644
index b168497..0000000
--- a/.agents/skills/run-cli-benchmark/references/cli-config.md
+++ /dev/null
@@ -1,36 +0,0 @@
-# Coding-agent CLI configuration
-
-Derive support from `benchmark.run_submission.BACKENDS` and `_run_backend()` at runtime.
-Documentation may lag code.
-
-Run these adapters on the host through `make run-local LOCAL_BACKEND=`.
-Do not set `AGENT_BACKEND` or use a container command to select a CLI harness.
-
-## Built-in adapters
-
-| Backend ID | Harness | Authentication | Model syntax |
-|---|---|---|---|
-| `codex` | Codex CLI | `codex login` or `OPENAI_API_KEY` | bare GPT name or `openai/...` |
-| `claude-code` | Claude Code | Claude login, `CLAUDE_CODE_OAUTH_TOKEN`, or `ANTHROPIC_API_KEY` | bare Claude name or `anthropic/...` |
-
-An agent named by the caller is not supported merely because its executable exists. It must
-have a direct adapter, dispatch case, tests, and reliable harness-evaluation evidence.
-
-## Required paths
-
-| Path | Requirement |
-|---|---|
-| clone destination | absent, or exact checkout matching `PR_REF` |
-| submission output | explicit authoritative JSON path |
-| log directory | explicit and separate from the output file |
-
-## Failure decoding
-
-| Symptom | Action |
-|---|---|
-| CLI missing or unauthenticated | install/authenticate only with caller approval |
-| checkout mismatch | use another path or ask the caller to update it; never mutate it |
-| `pred` mismatch | install/build the matching benchmark version |
-| submit channel not probed | infrastructure failure; preserve logs and salvaged artifacts |
-| CLI exit, timeout, or structured failure | record `run_error` and keep partial ledger rows |
-| missing trustworthy usage | do not claim a fully supported/reliable adapter |
diff --git a/.agents/skills/submit-benchmark-result/SKILL.md b/.agents/skills/submit-benchmark-result/SKILL.md
index b60230a..433d5ac 100644
--- a/.agents/skills/submit-benchmark-result/SKILL.md
+++ b/.agents/skills/submit-benchmark-result/SKILL.md
@@ -25,7 +25,7 @@ python3 -m benchmark.submit --predictions  --dry-run
 Stop if validation fails. Otherwise report only:
 
 - absolute path;
-- `model`, `agent_mode`, and `library_commit`;
+- `model` and `library_commit`;
 - completed Top50 episode count and claimed bugs;
 - `run_error`, if present.
 
@@ -39,7 +39,7 @@ Immediately before uploading, show this confirmation with the real values filled
 > - Destination: `intake.prb-bench.workers.dev`
 > - File: ``
 > - Model: ``
-> - Track: Standardized Model API / Self-selected Top50
+> - Protocol: Standardized Model API / Self-selected Top50
 > - Claimed bugs: ``
 >
 > This private submission file will leave the machine and be uploaded to the benchmark's
diff --git a/.claude/skills/run-benchmark b/.claude/skills/run-benchmark
deleted file mode 120000
index 2eefa8a..0000000
--- a/.claude/skills/run-benchmark
+++ /dev/null
@@ -1 +0,0 @@
-../../.agents/skills/run-benchmark
\ No newline at end of file
diff --git a/.github/scripts/check_aggregate.py b/.github/scripts/check_aggregate.py
index 16ae6c7..801dbcf 100644
--- a/.github/scripts/check_aggregate.py
+++ b/.github/scripts/check_aggregate.py
@@ -16,9 +16,9 @@
 
 ALLOWED_KEYS = {
     "model", "library_commit", "bugs_found", "rules_tested",
-    "total_tokens_k", "usage_totals", "efficiency_bugs_per_ktok",
+    "total_tokens_k", "usage_totals",
     "submitted_by", "placeholder",
-    "runner_version", "pred_version", "agent_mode", "rankable",
+    "runner_version", "pred_version", "rankable",
     "bugs_at_10", "bugs_at_25", "bugs_at_50", "first_attempt_accepts",
     "second_attempt_accepts", "pred_calls_per_bug", "cap_hits",
     # per-submission entry files (site/results/.json) also carry provenance tags
diff --git a/.gitignore b/.gitignore
index 1d7c576..cda5cc2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -45,9 +45,8 @@ node_modules/
 .pytest_cache/
 .venv/
 
-# Local agent/CLI state and personal artifacts (not project docs — those live in
+# Local tooling state and personal artifacts (not project docs — those live in
 # README.md, CONTRIBUTING.md, and benchmark/docs/)
-.claude/
 /docs/
 /reports/
 
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 5b0086b..c2557f2 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,6 +1,6 @@
 # Submitting a model run
 
-The current public comparison accepts one execution protocol: **Standardized Model API / Self-selected Top50**. Its limits are built into the benchmark code and are not configurable.
+The benchmark uses one execution protocol: **Standardized Model API / Self-selected Top50**. Its limits are built into the benchmark code and are not configurable.
 
 ```text
 source-only triage → frozen Top50 → 50 isolated rule episodes → private submission
@@ -16,7 +16,7 @@ Copy the internal template and set the model plus its provider credential:
 cp submission.env.example submission.env
 ```
 
-`MODEL_NAME`, `API_BASE`, and `API_KEY` identify the endpoint. The official path does not accept prompt files, strategy files, coding-agent backends, model kwargs, submit pools, or budget variables. This is intentional: two entries are comparable only if model access is the remaining meaningful variable.
+`MODEL_NAME`, `API_BASE`, and `API_KEY` identify the endpoint. They are the only execution settings supplied by a participant; prompts, limits, inference settings, and the harness are benchmark-owned. This keeps model access as the remaining meaningful variable.
 
 ## 2. Preflight and run
 
@@ -46,7 +46,7 @@ python -m benchmark.submit --predictions out//submission.json
 
 The intake check is only a courtesy check. The private backend independently validates:
 
-- prompt hash, runner mode, target commit, and `pred` version;
+- prompt hash, runner version, target commit, and `pred` version;
 - the canonical 50-rule inventory and frozen order;
 - triage and per-rule event/usage ledgers;
 - exactly two submit opportunities per rule, including malformed or rejected calls;
@@ -74,8 +74,4 @@ python -m benchmark.verify --calibrate
 
 The calibration checker proves that the selected limits match the built-in runtime values, every model covers the full candidate grid, smaller and larger `M`/`P` candidates surround the selection, and the Markdown report matches the machine-readable evidence.
 
-## Historical tooling
-
-`benchmark.run_submission`, `make run-local`, `benchmark/config.yaml`, and the coding-agent adapters reproduce the old `legacy-whole-repo` protocol only. Historical aggregates remain visible in their own site selector. They cannot be submitted or sorted into the Top50 table, and they are not a maintained System Track.
-
 Repository policy also applies: never merge a PR until required CI is green and the user explicitly approves the merge.
diff --git a/Makefile b/Makefile
index 56770b0..b0b04a7 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,6 @@
 #   verify-calibration   Test the verifier against known fixtures (no AI needed)
 #   preflight            Validate submission.env with one tiny real call before a full run
 #   run                  Run the benchmark via Docker → out//submission.json (does NOT upload)
-#   run-local            Reproduce a historical non-ranking CLI artifact
 #
 # Model/auth configuration lives in submission.env (see submission.env.example). Local
 # repository, submission, and log locations are intentionally explicit Make variables.
@@ -24,14 +23,9 @@ SUBS_DIR ?= submissions
 SCORED   ?= results/scored
 ENV_FILE ?= submission.env
 STAMP    ?= $(shell date +%Y%m%d-%H%M%S)
-LOCAL_BACKEND ?= codex
 LOCAL_REPO_DIR ?=
-LOCAL_OUTPUT ?=
-LOCAL_LOG_DIR ?=
-REPO_URL ?= https://github.com/CodingThrust/problem-reductions.git
-LOCAL_ARGS = $(if $(LOCAL_BACKEND),--backend "$(LOCAL_BACKEND)")
 
-.PHONY: test test-unit verify-calibration verify-budget verify-judgment audit install-deps help print-benchmark-version print-pr-ref runner-build runner-pull preflight run run-local score-local board publish-local serve
+.PHONY: test test-unit verify-calibration verify-budget verify-judgment audit install-deps help print-benchmark-version print-pr-ref runner-build runner-pull preflight run score-local board publish-local serve
 
 ## Print this checkout's benchmark contract version.
 print-benchmark-version:
@@ -49,7 +43,7 @@ test:
 test-unit:
 	pytest -v -m "not integration"
 
-## Pred-free sanity tests (docs, CI workflow, trajectory).
+## Pred-free sanity tests (docs and CI workflow).
 verify-judgment:
 	pytest -v -m "judgment"
 
@@ -97,24 +91,6 @@ run:
 	docker run --rm --env-file "$(ENV_FILE)" -e OUTPUT=/out/$(STAMP)/submission.json -v "$(PWD)/out:/out" $(IMAGE)
 	@echo "Wrote out/$(STAMP)/submission.json — now submit it with 'python -m benchmark.submit' (see CONTRIBUTING.md)."
 
-## Historical non-ranking host run through an installed headless agent CLI. It cannot enter
-## the Top50 leaderboard. The checkout is cloned at
-## PR_REF when LOCAL_REPO_DIR is absent; an existing checkout must already match exactly.
-## Requires explicit repo/output/log paths, local Python deps, pred, and CLI authentication.
-run-local:
-	@if [ ! -f "$(ENV_FILE)" ]; then \
-	  echo "No $(ENV_FILE) — copy submission.env.example and set MODEL_NAME first"; exit 1; fi
-	@if [ -z "$(LOCAL_REPO_DIR)" ] || [ -z "$(LOCAL_OUTPUT)" ] || [ -z "$(LOCAL_LOG_DIR)" ]; then \
-	  echo "Set LOCAL_REPO_DIR, LOCAL_OUTPUT, and LOCAL_LOG_DIR explicitly"; exit 1; fi
-	python3 -m benchmark.run_submission \
-	  --env-file "$(ENV_FILE)" \
-	  --repo-dir "$(abspath $(LOCAL_REPO_DIR))" \
-	  --repo-ref "$(PR_REF)" \
-	  --repo-url "$(REPO_URL)" \
-	  --output "$(abspath $(LOCAL_OUTPUT))" \
-	  --trajectory-dir "$(abspath $(LOCAL_LOG_DIR))" --host-cli $(LOCAL_ARGS)
-	@echo "Wrote $(LOCAL_OUTPUT); logs are in $(LOCAL_LOG_DIR)."
-
 ## Score all submissions in SUBS_DIR with the zero-trust backend (needs pred). Writes scored
 ## results + leaderboard.json into SCORED, and one public entry per submission into SCORED/board.
 score-local:
@@ -162,7 +138,6 @@ help:
 	@echo "  runner-pull         Pull the prebuilt runner image from GHCR (fast runner-build alternative)"
 	@echo "  preflight           Validate submission.env (1 tiny real call) before a full run"
 	@echo "  run                 Run the API backend via Docker → out//submission.json (not upload)"
-	@echo "  run-local           Reproduce a historical non-ranking CLI artifact"
 	@echo "  score-local         Score SUBS_DIR submissions with the backend"
 	@echo "  board               Build site/results.json from site/results/*.json (aggregate)"
 	@echo "  publish-local       Score + stage per-submission entries + rebuild board (SUBS_DIR stays local)"
@@ -172,7 +147,4 @@ help:
 	@echo ""
 	@echo "Variables:"
 	@echo "  ENV_FILE=$(ENV_FILE)  (model/key for preflight + submission)"
-	@echo "  LOCAL_BACKEND=$(LOCAL_BACKEND)  (codex default; set claude-code if desired)"
-	@echo "  LOCAL_REPO_DIR=$(LOCAL_REPO_DIR)  (required for run-local/audit)"
-	@echo "  LOCAL_OUTPUT=$(LOCAL_OUTPUT)  (required for run-local)"
-	@echo "  LOCAL_LOG_DIR=$(LOCAL_LOG_DIR)  (required for run-local)"
+	@echo "  LOCAL_REPO_DIR=$(LOCAL_REPO_DIR)  (required for audit)"
diff --git a/README.md b/README.md
index ee18492..21a5d78 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@ This benchmark measures how well a model can prioritize likely-buggy reduction r
 
 ## Current benchmark
 
-The primary track is **Standardized Model API / Self-selected Top50**. The target is `problem-reductions` [`v0.6.0`](https://github.com/CodingThrust/problem-reductions/commit/aa2d1a10cffa434871d12a4d6f411147fb7e08a8), and the bundled `pred` version is `0.6.0`. The logical limits are part of the benchmark code, not a user-selectable configuration.
+The benchmark protocol is **Standardized Model API / Self-selected Top50**. The target is `problem-reductions` [`v0.6.0`](https://github.com/CodingThrust/problem-reductions/commit/aa2d1a10cffa434871d12a4d6f411147fb7e08a8), and the bundled `pred` version is `0.6.0`. The logical limits are part of the benchmark code, not a user-selectable configuration.
 
 Each run has two phases:
 
@@ -43,11 +43,11 @@ This score reflects a combined ability:
 - allocate bounded model, shell, and `pred` calls within each rule;
 - construct a valid counterexample within two submission opportunities.
 
-It does not claim to measure host speed, network quality, willingness to run longer, coding-agent tooling, repeated-seed stability, or performance on a fixed organizer-selected Top50. Fixed Top50 diagnostics, multiple seeds, and a System Track are intentionally deferred or dropped.
+It does not claim to measure host speed, network quality, willingness to run longer, repeated-seed stability, or performance on a fixed organizer-selected Top50.
 
 A certificate is never trusted directly. The private scorer re-derives and replays it with pinned `pred`; only reproducible round-trip failures count.
 
-## Run the rankable track
+## Run the benchmark
 
 Install Docker, then:
 
@@ -59,7 +59,7 @@ make run                                   # writes out//submission.j
 python -m benchmark.submit --predictions out//submission.json
 ```
 
-The rankable path is deliberately narrow. It exposes no logical-budget configuration and rejects custom prompts or strategies, custom model kwargs, and coding-agent backends before a scored generation. Model and `pred` watchdogs are safety controls and are not user-selectable score dimensions.
+The runner exposes only model identity, credentials, and endpoint configuration. Logical budgets, prompts, inference settings, and the execution harness are built into the benchmark. Model and `pred` watchdogs are safety controls and are not user-selectable score dimensions.
 
 Useful local checks:
 
@@ -69,10 +69,4 @@ make verify-budget
 make verify-calibration
 ```
 
-See [CONTRIBUTING.md](CONTRIBUTING.md) for artifact fields, submission, and historical-result handling.
-
-## Historical results
-
-Whole-repository results remain visible under `legacy-whole-repo`. They use a different execution protocol and efficiency tie-break, so the site keeps them in a separate selectable table. They cannot be deduplicated, sorted, or compared with the standardized Top50 results.
-
-The old host coding-agent and whole-repository runners remain in the repository only to reproduce those historical artifacts. They are not a public System Track and cannot produce rankable Top50 submissions.
+See [CONTRIBUTING.md](CONTRIBUTING.md) for artifact fields and submission handling.
diff --git a/benchmark/top50_config.yaml b/benchmark/agent_config.yaml
similarity index 100%
rename from benchmark/top50_config.yaml
rename to benchmark/agent_config.yaml
diff --git a/benchmark/agent_submit.py b/benchmark/agent_submit.py
index 3170942..abb509b 100644
--- a/benchmark/agent_submit.py
+++ b/benchmark/agent_submit.py
@@ -3,7 +3,7 @@
 
 Requests cross the agent sandbox through an atomic file spool rooted in the agent's
 scratch workspace.  This deliberately avoids Unix sockets and localhost networking,
-which are blocked by common headless-agent sandboxes.
+which are blocked by the benchmark agent sandbox.
 """
 from __future__ import annotations
 
diff --git a/benchmark/backend_score.py b/benchmark/backend_score.py
index f8b7c22..91abc1e 100644
--- a/benchmark/backend_score.py
+++ b/benchmark/backend_score.py
@@ -2,7 +2,7 @@
 """
 Backend scoring queue — the worker that turns PENDING submissions into ranked results.
 
-The scoring is our zero-trust re-verification (benchmark/verify_submission.py → pred):
+The scoring is our zero-trust contract validation and `pred` re-verification:
 
   * --local  
         Scan subs_dir/*.json, score each unprocessed submission, write the scored
@@ -23,10 +23,7 @@
 from pathlib import Path
 
 from benchmark.env_setup import pinned_commit
-from benchmark.submit import validate_submission
-from benchmark.submit_ledger import has_submit_ledger
-from benchmark.top50_contract import is_top50_submission
-from benchmark.verify_submission import leaderboard_entry, score_submission
+from benchmark.top50_contract import score_top50_submission, top50_public_entry
 
 STATUS_SUFFIX = ".status.json"
 
@@ -71,9 +68,8 @@ def board_slug(scored: dict, stem: str) -> str:
 
 def board_entry(scored: dict, stem: str) -> dict:
     """The public per-submission leaderboard entry, tagged with its time + id."""
-    # leaderboard_entry reads only ``submitted_by`` from its first arg; the scored file
-    # carries it when present, so it doubles as the submission view.
-    entry = leaderboard_entry(scored, scored)
+    # The scored file carries the submitter metadata needed by the public projection.
+    entry = top50_public_entry(scored, scored)
     entry["timestamp"] = _submission_ts(scored, stem)
     entry["submission_id"] = _submission_id(stem)
     return entry
@@ -94,10 +90,9 @@ def write_board_entries(results_dir: Path, board_dir: Path) -> list[str]:
             scored = json.loads(p.read_text(encoding="utf-8"))
         except (json.JSONDecodeError, OSError):
             continue
-        if (("results" not in scored and not is_top50_submission(scored))
-                or "model" not in scored or scored.get("test")):
+        if "model" not in scored or scored.get("test"):
             continue
-        if is_top50_submission(scored) and not scored.get("rankable"):
+        if not scored.get("rankable"):
             continue
         slug = board_slug(scored, p.stem)
         (board_dir / f"{slug}.json").write_text(
@@ -179,40 +174,6 @@ def _pending_submissions(subs_dir: Path) -> list[Path]:
 
 # ── scoring one submission ────────────────────────────────────────────────────
 
-def _validate_for_scoring(submission: object, *, official: bool,
-                          expected_commit: str | None) -> dict:
-    """Validate the durable queue envelope before invoking ``pred``.
-
-    Basic validation applies to local and production scoring so malformed input is a
-    permanent per-object failure instead of an exception that jams the whole queue.  The
-    official gate additionally requires the current ledger-backed structure and target
-    commit, and rejects partial runs from the public leaderboard. Test submissions may be
-    partial because they never publish.
-    """
-    if not isinstance(submission, dict):
-        raise PermanentSubmissionError("submission is not a JSON object")
-
-    problems = validate_submission(submission)
-
-    if official:
-        if not is_top50_submission(submission) and not has_submit_ledger(submission):
-            problems.append("official submissions require submit_limit and submit_log")
-        target_commit = expected_commit or pinned_commit()
-        if submission.get("library_commit") != target_commit:
-            problems.append(
-                "library_commit does not match this benchmark round "
-                f"(expected {target_commit})")
-        if submission.get("run_error") and not submission.get("test"):
-            problems.append(
-                "partial run has run_error; resubmit a clean run or upload it with --test")
-
-    if problems:
-        # Preserve order while removing duplicate messages emitted by overlapping checks.
-        unique = list(dict.fromkeys(problems))
-        raise PermanentSubmissionError("; ".join(unique))
-    return submission
-
-
 def score_one(sub_path: Path, results_dir: Path, repo_dir: str | None = None, *,
               official: bool = False, expected_commit: str | None = None) -> dict:
     """Score a single submission file. Returns its leaderboard entry.
@@ -227,10 +188,20 @@ def score_one(sub_path: Path, results_dir: Path, repo_dir: str | None = None, *,
             raw_submission = json.loads(sub_path.read_text(encoding="utf-8"))
         except (json.JSONDecodeError, UnicodeError) as e:
             raise PermanentSubmissionError(f"invalid submission JSON: {e}") from e
-        submission = _validate_for_scoring(
-            raw_submission, official=official, expected_commit=expected_commit)
-        scored, report = score_submission(submission, repo_dir)
-        entry = leaderboard_entry(submission, scored)
+        if not isinstance(raw_submission, dict):
+            raise PermanentSubmissionError("submission is not a JSON object")
+        submission = raw_submission
+        scored, report = score_top50_submission(submission, repo_dir)
+        problems = list(scored.get("rankability_errors", []))
+        if official:
+            target_commit = expected_commit or pinned_commit()
+            if submission.get("library_commit") != target_commit:
+                problems.append(
+                    "library_commit does not match this benchmark round "
+                    f"(expected {target_commit})")
+        if problems:
+            raise PermanentSubmissionError("; ".join(dict.fromkeys(problems)))
+        entry = top50_public_entry(submission, scored)
         (results_dir / f"{sub_path.stem}.json").write_text(
             json.dumps(scored, indent=2), encoding="utf-8")
         _write_status(sub_path, "FINISHED",
@@ -246,31 +217,16 @@ def score_one(sub_path: Path, results_dir: Path, repo_dir: str | None = None, *,
 # ── leaderboard aggregation ───────────────────────────────────────────────────
 
 def _dedup_best(entries: list[dict]) -> list[dict]:
-    """Keep the best entry per model (max bugs, tie-break efficiency), ranked desc."""
-    best: dict[tuple[str, str], dict] = {}
+    """Keep each model's highest verified-bug count and rank by that count only."""
+    best: dict[str, dict] = {}
     for e in entries:
         m = e["model"]
-        top50 = e.get("agent_mode") == "standardized-model-api"
-        track = "top50" if top50 else "legacy-whole-repo"
-        group = (track, m)
-        cur = best.get(group)
-        if top50:
-            better = cur is None or e.get("bugs_found", 0) > cur.get("bugs_found", 0)
-        else:
-            key = (e.get("bugs_found", 0), e.get("efficiency_bugs_per_ktok", 0.0))
-            better = cur is None or key > (
-                cur.get("bugs_found", 0), cur.get("efficiency_bugs_per_ktok", 0.0))
-        if better:
-            best[group] = e
+        cur = best.get(m)
+        if cur is None or e.get("bugs_found", 0) > cur.get("bugs_found", 0):
+            best[m] = e
     return sorted(
         best.values(),
-        key=lambda e: (
-            0 if e.get("agent_mode") == "standardized-model-api" else 1,
-            -e.get("bugs_found", 0),
-            (0 if e.get("agent_mode") == "standardized-model-api"
-             else -e.get("efficiency_bugs_per_ktok", 0.0)),
-            e.get("model", ""),
-        ))
+        key=lambda e: (-e.get("bugs_found", 0), e.get("model", "")))
 
 
 def aggregate_leaderboard(results_dir: Path) -> list[dict]:
@@ -284,17 +240,15 @@ def aggregate_leaderboard(results_dir: Path) -> list[dict]:
             scored = json.loads(p.read_text(encoding="utf-8"))
         except (json.JSONDecodeError, OSError):
             continue
-        if ("results" not in scored and not is_top50_submission(scored)) or "model" not in scored:
+        if "model" not in scored:
             continue
-        if is_top50_submission(scored) and not scored.get("rankable"):
+        if not scored.get("rankable"):
             continue
         # Test submissions are scored + kept privately, but never published to the public
         # leaderboard — skip them here so an end-to-end test can't pollute production.
         if scored.get("test"):
             continue
-        # leaderboard_entry reads only ``submitted_by`` from its first arg, which the
-        # scored file carries when present — it doubles as the submission view.
-        entries.append(leaderboard_entry(scored, scored))
+        entries.append(top50_public_entry(scored, scored))
     ranked = _dedup_best(entries)
     (results_dir / "leaderboard.json").write_text(json.dumps(ranked, indent=2), encoding="utf-8")
     return ranked
diff --git a/benchmark/calibrate_budget.py b/benchmark/calibrate_budget.py
index 117a373..2b3fcc0 100644
--- a/benchmark/calibrate_budget.py
+++ b/benchmark/calibrate_budget.py
@@ -183,8 +183,8 @@ def render_report(evidence: dict) -> str:
         evidence["selection_rationale"],
         "",
         "The public comparison therefore uses these built-in parameters and is ranked only by "
-        "verified distinct-rule bugs. Fixed Top50, multiple seeds, a System Track, and a public "
-        "budget grid remain out of scope.",
+        "verified distinct-rule bugs. Fixed Top50, multiple seeds, and a public budget grid "
+        "remain out of scope.",
         "",
         "## Provenance",
         "",
@@ -248,13 +248,13 @@ def check(path: str | Path) -> list[str]:
     evidence = load_json(path)
     contract = benchmark_parameters()
     errors = validate_evidence(evidence, contract)
-    submission_schema = load_json(ROOT / "top50_submission.schema.json")
+    submission_schema = load_json(ROOT / "submission.schema.json")
     properties = submission_schema.get("properties", {})
     if properties.get("safety_controls", {}).get("const") != contract["safety_controls"]:
-        errors.append("top50_submission.schema.json has stale safety controls")
+        errors.append("submission.schema.json has stale safety controls")
     if properties.get("inference_parameters", {}).get("const") != contract[
             "inference_parameters"]:
-        errors.append("top50_submission.schema.json has stale inference parameters")
+        errors.append("submission.schema.json has stale inference parameters")
     if not errors and REPORT_PATH.read_text(encoding="utf-8") != render_report(evidence):
         errors.append("budget-calibration.md does not match the machine-readable evidence")
     return errors
diff --git a/benchmark/claude_code.py b/benchmark/claude_code.py
deleted file mode 100644
index 3ec9d4e..0000000
--- a/benchmark/claude_code.py
+++ /dev/null
@@ -1,126 +0,0 @@
-"""Whole-repository Claude Code backend using self-terminating ``claude -p``."""
-from __future__ import annotations
-
-import json
-import os
-from pathlib import Path
-
-from benchmark.headless import child_env, load_rendered_prompts, run_headless_session
-from benchmark.run_mini import CONFIG_FILE
-from benchmark.usage import Usage, usage_from_response
-
-DEFAULT_ALLOWED_TOOLS = "Bash,Read,Grep,Glob"
-# This is only a hung-process backstop, not an agent turn budget.
-DEFAULT_SESSION_TIMEOUT = 6 * 3600
-
-
-def _block_text(block: dict) -> str:
-    block_type = block.get("type")
-    if block_type == "text":
-        return block.get("text") or ""
-    if block_type == "thinking":
-        return block.get("thinking") or ""
-    if block_type == "tool_use":
-        payload = block.get("input") or {}
-        if isinstance(payload.get("command"), str):
-            return f"[tool_use {block.get('name', '?')}]\n{payload['command']}"
-        return f"[tool_use {block.get('name', '?')}] {json.dumps(payload, sort_keys=True)}"
-    if block_type == "tool_result":
-        content = block.get("content")
-        if isinstance(content, list):
-            return "\n".join(_block_text(item) for item in content if isinstance(item, dict))
-        return content if isinstance(content, str) else ""
-    return ""
-
-
-def parse_stream(lines, *, collect_trajectory: bool = True) -> dict:
-    trajectory: list[dict] = []
-    usage = Usage()
-    seen_message_ids: set = set()
-    result_event = None
-    for line in lines:
-        try:
-            event = json.loads(line)
-        except json.JSONDecodeError:
-            continue
-        if not isinstance(event, dict):
-            continue
-        event_type = event.get("type")
-        if event_type in ("assistant", "user"):
-            message = event.get("message") or {}
-            if collect_trajectory:
-                content = message.get("content")
-                text = (content if isinstance(content, str) else
-                        "\n".join(filter(None, (_block_text(block) for block in (content or [])
-                                                 if isinstance(block, dict)))))
-                trajectory.append({"role": event_type, "content": text})
-            message_id = message.get("id")
-            if (event_type == "assistant" and message.get("usage")
-                    and message_id not in seen_message_ids):
-                seen_message_ids.add(message_id)
-                usage = usage + usage_from_response(message["usage"])
-        elif event_type == "result":
-            result_event = event
-    if result_event and isinstance(result_event.get("usage"), dict):
-        cumulative = usage_from_response(result_event["usage"])
-        if cumulative.total_tokens:
-            usage = cumulative
-    steps = int(result_event.get("num_turns") or 0) if result_event else 0
-    return {"trajectory": trajectory, "usage": usage, "steps": steps,
-            "result_event": result_event}
-
-
-def _build_command(claude_bin: str, system_prompt: str, task_prompt: str,
-                   model: str, allowed_tools: str) -> list[str]:
-    bare_model = model.split("/", 1)[1] if model.startswith("anthropic/") else model
-    return [
-        claude_bin, "-p", task_prompt,
-        "--system-prompt", system_prompt,
-        "--model", bare_model,
-        "--tools", allowed_tools.replace(",", " "),
-        "--allowedTools", allowed_tools,
-        "--strict-mcp-config",
-        "--output-format", "stream-json",
-        "--verbose",
-    ]
-
-
-def _child_env(ctx, api_key: str | None) -> dict:
-    env = child_env(ctx)
-    env.setdefault("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "1")
-    if api_key and not env.get("CLAUDE_CODE_OAUTH_TOKEN") and not env.get("ANTHROPIC_API_KEY"):
-        env["ANTHROPIC_API_KEY"] = api_key
-    return env
-
-
-def run_repo_claude(
-    model_name: str,
-    ctx,
-    *,
-    trajectory_dir: Path | None = None,
-    config_path: str | Path | None = None,
-    strategy: str | None = None,
-    api_key: str | None = None,
-    claude_bin: str | None = None,
-    allowed_tools: str | None = None,
-    session_timeout: int | None = None,
-    submit_session=None,
-) -> dict:
-    claude_bin = claude_bin or os.environ.get("CLAUDE_BIN", "claude")
-    allowed_tools = allowed_tools or os.environ.get("CLAUDE_CODE_TOOLS", DEFAULT_ALLOWED_TOOLS)
-    session_timeout = session_timeout or int(
-        os.environ.get("CLAUDE_CODE_SESSION_TIMEOUT", DEFAULT_SESSION_TIMEOUT))
-    variables = {
-        "repo_dir": str(ctx.repo_path),
-        "commit_hash": ctx.commit_hash[:7],
-        "submit_limit": submit_session.limit if submit_session is not None else 0,
-    }
-    system_prompt, task_prompt = load_rendered_prompts(
-        config_path, CONFIG_FILE, strategy, variables)
-    command = _build_command(
-        claude_bin, system_prompt, task_prompt, model_name, allowed_tools)
-    return run_headless_session(
-        command=command, model_name=model_name,
-        env=_child_env(ctx, api_key), timeout=session_timeout,
-        parser=parse_stream, label="claude", trajectory_dir=trajectory_dir,
-        submit_session=submit_session)
diff --git a/benchmark/codex_cli.py b/benchmark/codex_cli.py
deleted file mode 100644
index f685652..0000000
--- a/benchmark/codex_cli.py
+++ /dev/null
@@ -1,118 +0,0 @@
-"""Whole-repository Codex backend using self-terminating ``codex exec``."""
-from __future__ import annotations
-
-import json
-import os
-from pathlib import Path
-
-from benchmark.headless import child_env, load_rendered_prompts, run_headless_session
-from benchmark.run_mini import CONFIG_FILE
-from benchmark.usage import Usage
-
-DEFAULT_SANDBOX = "workspace-write"
-# This is only a hung-process backstop, not an agent turn budget.
-DEFAULT_SESSION_TIMEOUT = 6 * 3600
-
-
-def _bare_model(model: str) -> str:
-    return model.split("/", 1)[1] if model.startswith("openai/") else model
-
-
-def _build_command(codex_bin: str, prompt: str, model: str,
-                   sandbox: str = DEFAULT_SANDBOX) -> list[str]:
-    return [
-        codex_bin, "exec", "--json", "--ephemeral",
-        "--sandbox", sandbox,
-        "--ignore-user-config", "--ignore-rules", "--skip-git-repo-check",
-        "--model", _bare_model(model),
-        prompt,
-    ]
-
-
-def _child_env(ctx, api_key: str | None) -> dict:
-    return child_env(ctx, api_key, api_key_var="OPENAI_API_KEY")
-
-
-def parse_stream(lines, *, collect_trajectory: bool = True) -> dict:
-    trajectory: list[dict] = []
-    usage = Usage()
-    steps = 0
-    thread_id = None
-    result_event = None
-    for raw in lines:
-        try:
-            event = json.loads(raw)
-        except (TypeError, json.JSONDecodeError):
-            continue
-        if not isinstance(event, dict):
-            continue
-        event_type = event.get("type")
-        if event_type == "thread.started":
-            thread_id = event.get("thread_id")
-        elif event_type == "item.completed":
-            item = event.get("item") or {}
-            if item.get("type") == "agent_message" and isinstance(item.get("text"), str):
-                if collect_trajectory:
-                    trajectory.append({"role": "assistant", "content": item["text"]})
-            elif item.get("type") == "command_execution":
-                steps += 1
-                if collect_trajectory:
-                    content = f"[command]\n{item.get('command') or ''}"
-                    output = item.get("aggregated_output") or item.get("output") or ""
-                    if output:
-                        content += f"\n[output]\n{output}"
-                    trajectory.append({"role": "tool", "content": content})
-        elif event_type == "turn.completed":
-            result_event = event
-            raw_usage = event.get("usage") or {}
-            total_input = int(raw_usage.get("input_tokens") or 0)
-            cached = int(raw_usage.get("cached_input_tokens") or 0)
-            usage = Usage(input_tokens=max(total_input - cached, 0),
-                          output_tokens=int(raw_usage.get("output_tokens") or 0),
-                          cache_read_tokens=cached)
-        elif event_type in ("turn.failed", "error"):
-            result_event = event
-    return {"trajectory": trajectory, "usage": usage, "steps": steps,
-            "thread_id": thread_id, "result_event": result_event}
-
-
-def _event_error(event: dict | None) -> str | None:
-    if not event or event.get("type") not in ("turn.failed", "error"):
-        return None
-    detail = event.get("error") or event.get("message") or event
-    if isinstance(detail, dict):
-        detail = json.dumps(detail, sort_keys=True)
-    return f"codex {event['type']}: {detail}"
-
-
-def run_repo_codex(
-    model_name: str,
-    ctx,
-    *,
-    trajectory_dir: Path | None = None,
-    config_path: str | Path | None = None,
-    strategy: str | None = None,
-    api_key: str | None = None,
-    codex_bin: str | None = None,
-    sandbox: str | None = None,
-    session_timeout: int | None = None,
-    submit_session=None,
-) -> dict:
-    codex_bin = codex_bin or os.environ.get("CODEX_BIN", "codex")
-    sandbox = sandbox or os.environ.get("CODEX_SANDBOX", DEFAULT_SANDBOX)
-    session_timeout = session_timeout or int(
-        os.environ.get("CODEX_SESSION_TIMEOUT", DEFAULT_SESSION_TIMEOUT))
-    variables = {
-        "repo_dir": str(ctx.repo_path),
-        "commit_hash": ctx.commit_hash[:7],
-        "submit_limit": submit_session.limit if submit_session is not None else 0,
-    }
-    system, task = load_rendered_prompts(config_path, CONFIG_FILE, strategy, variables)
-    command = _build_command(codex_bin, f"{system.rstrip()}\n\n{task.lstrip()}",
-                             model_name, sandbox)
-
-    return run_headless_session(
-        command=command, model_name=model_name,
-        env=_child_env(ctx, api_key), timeout=session_timeout,
-        parser=parse_stream, label="codex", trajectory_dir=trajectory_dir,
-        submit_session=submit_session, event_error=_event_error)
diff --git a/benchmark/config.yaml b/benchmark/config.yaml
deleted file mode 100644
index 63964b3..0000000
--- a/benchmark/config.yaml
+++ /dev/null
@@ -1,122 +0,0 @@
-agent:
-  system_template: |
-    You are an expert at finding bugs in problem reduction rules using the `pred` CLI. You
-    are given the WHOLE library and one session; you decide which rules to inspect, in what
-    order, how deeply to probe them, and when you have finished.
-
-    Each rule implements two functions:
-    - reduce_to(): transforms source problem A into target problem B
-    - extract_solution(): maps a target solution back to source space
-
-    A rule is BUGGY if the round-trip fails:
-    given A → reduce to B → solve B to get s → extract s → A', but A' is not a valid or
-    equally good solution to A.
-
-    Your tool is `pred`. Key commands:
-      pred list --rules --json                          # enumerate reduction rules
-      pred create  [options] --json            # create a problem instance
-      pred reduce  --to  --json      # reduce source to target
-      pred solve  --json                   # solve and round-trip a bundle
-      pred evaluate  --config  --json  # evaluate a config
-      pred extract  --config  --json # extract to source space
-
-    pred cannot read from stdin. Always write JSON to a file first, then pass the filename.
-
-    The library lives at {{repo_dir}} (commit {{commit_hash}}); rule sources are under
-    {{repo_dir}}/src/rules/.rs and may be read.
-
-    Before investigating anything, run `submit --status`. This is a free infrastructure
-    health check. If it reports that submit is unavailable, stop immediately and finish
-    with the required completion command; do not spend the session searching for a
-    certificate that cannot reach the evaluator.
-
-    The core test is whether solving a source directly agrees with solving it through the
-    reduction, compared by value for optimization problems or feasibility for decision
-    problems:
-      pred solve source.json --json
-      pred reduce source.json --to  --json -o bundle.json
-      pred solve bundle.json --json
-
-    {{strategy}}
-
-    Work through the library using your own judgment. For every reproduced bug, write a
-    minimal certificate JSON file under `$PRB_ARTIFACT_DIR`, for example
-    `$PRB_ARTIFACT_DIR/-certificate.json`, so infrastructure failures can preserve it:
-
-    {
-      "rule": "",
-      "source": ,
-      "bundle": ,
-      "target_config": "",
-      "note": ""
-    }
-
-    Submit it immediately:
-      submit "$PRB_ARTIFACT_DIR/-certificate.json"
-
-    `submit` re-verifies it and prints ACCEPTED or REJECTED. Every call consumes one of the
-    {{submit_limit}} attempts shared by the complete evaluation; rejected and malformed
-    submissions consume an attempt too. `submit --status` reports the remaining budget for
-    free. Only certificates accepted through this command count. Final prose and arbitrary
-    certificate files are ignored. If accepted, continue investigating other rules.
-
-    Only `rule`, `source`, and the target type from `bundle` are required; `target_config`
-    is optional. The verifier re-derives the reduction from `source` and never trusts the
-    supplied bundle or claim.
-
-    There is no step or turn budget. Continue while you are making useful progress. When
-    you decide the evaluation is complete or further work is unlikely to help, finish by
-    running exactly:
-      echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT
-
-  instance_template: |
-    
-    Find as many genuine bugs as you can across the reduction rules in the library.
-
-    Library repo: {{repo_dir}} (commit {{commit_hash}})
-    Rule sources: {{repo_dir}}/src/rules/
-
-    Use `pred list --rules --json` and the rule sources to choose what to investigate. For
-    every reproduced bug, run `submit ` immediately. Decide for yourself
-    how broadly and deeply to search and when to stop. When finished, run exactly:
-    echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT
-    
-
-  # mini-swe-agent uses 0 to mean unlimited. Cost limiting is also force-disabled by the
-  # runner; the agent stops through the completion command above.
-  # Historical whole-repository runner only; never produces Top50 rankable results.
-  step_limit: 100
-
-environment:
-  # Per-command protection against a wedged subprocess, not an agent turn budget.
-  timeout: 300
-  env:
-    PAGER: cat
-    MANPAGER: cat
-    LESS: -R
-
-model:
-  observation_template: |
-    {% if output.exception_info -%}
-    {{output.exception_info}}
-    {% endif -%}
-    {{output.returncode}}
-    {% if output.output | length < 10000 -%}
-    
-    {{ output.output -}}
-    
-    {%- else -%}
-    Output too long — showing head + tail only. Re-run a narrower command or read slices from a file.
-    {%- set elided = output.output | length - 10000 -%}
-    
-    {{ output.output[:5000] }}
-    
-    {{ elided }} characters elided
-    
-    {{ output.output[-5000:] }}
-    
-    {%- endif -%}
-  format_error_template: |
-    Format error: {{error}}
-    Provide exactly ONE bash command in triple backticks. If you are finished, run exactly:
-    echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT
diff --git a/benchmark/docs/budget-calibration.md b/benchmark/docs/budget-calibration.md
index aaf99da..5530270 100644
--- a/benchmark/docs/budget-calibration.md
+++ b/benchmark/docs/budget-calibration.md
@@ -39,7 +39,7 @@ Token, cost, and elapsed-time fields are diagnostic references only. Missing pro
 
 M=10 and P=24 are the smallest tested values at which the replayed stronger pilot retains its verified find. Moving down to M=6 or P=12 loses that find and shows materially higher cap pressure; moving up to M=14 or P=36 adds usage without another verified bug. E=12 and P_solve=10 cover the observed command mix without exposing an unlimited path. S=2 follows the fixed per-rule retry policy. T=8 and E_t=12 bound source-only shortlist formation. Output and watchdog ceilings are safety controls selected above observed pilot maxima and do not affect ranking.
 
-The public comparison therefore uses these built-in parameters and is ranked only by verified distinct-rule bugs. Fixed Top50, multiple seeds, a System Track, and a public budget grid remain out of scope.
+The public comparison therefore uses these built-in parameters and is ranked only by verified distinct-rule bugs. Fixed Top50, multiple seeds, and a public budget grid remain out of scope.
 
 ## Provenance
 
diff --git a/benchmark/docs/design/top50-evidence-budget-benchmark.md b/benchmark/docs/design/top50-evidence-budget-benchmark.md
deleted file mode 100644
index 36d4585..0000000
--- a/benchmark/docs/design/top50-evidence-budget-benchmark.md
+++ /dev/null
@@ -1,504 +0,0 @@
-# Top-50 Evidence-Budget Benchmark — Product Design
-
-Status: proposed for the next benchmark version  
-Last updated: 2026-07-20
-
-## Need
-
-The current benchmark gives a model the complete `problem-reductions` repository and lets
-it decide how broadly and deeply to search and when to stop. The runner limits submissions,
-but it does not limit model turns, shell actions, or calls to `pred`. Consequently, two runs
-may receive radically different amounts of feedback from the target system. Wall-clock time
-is not a suitable correction: the model cannot budget against it reliably, and it also
-contains provider latency, retries, network delay, host load, and harness overhead.
-
-The next benchmark version must compare models under the same **logical opportunities to
-obtain evidence**, rather than under the same elapsed time. A model first selects the 50
-rules it considers most likely to contain bugs, then investigates every selected rule with
-the same non-transferable per-rule budget. The primary result remains the number of distinct
-rules for which the evaluation-owned verifier accepts a counterexample.
-
-The benchmark is for researchers and developers comparing base models. It has one
-standardized Model Track, using one reference harness and one frozen evaluation contract.
-It intentionally does not compare rapidly changing coding-agent products or custom agent
-systems.
-
-Success means that:
-
-- a longer-running session cannot obtain more target-system feedback than a shorter one;
-- shell scripts, loops, and parallel subprocesses cannot bypass the `pred` or submission
-  budgets;
-- all 50 selected rules receive the same investigation allowance;
-- infrastructure failures are distinguishable from model failures and never become clean
-  zero scores;
-- the score has a narrow, defensible interpretation: verified bug-finding yield under a
-  fixed evidence budget.
-
-## Prior art and landscape
-
-The design follows a common pattern in agent evaluation: bound interaction opportunities
-directly and retain timeouts only as safety mechanisms.
-
-- [METR's time-horizon methodology](https://metr.org/time-horizons/) notes that exact agent
-  runtime varies with the inference provider and agent setup and is not itself the reported
-  capability measure.
-- [Inspect](https://inspect.aisi.org.uk/agent-custom.html) exposes separate limits for
-  messages, tokens, turns, time, working time, and cost. Its working-time accounting excludes
-  delays such as rate-limit waiting, illustrating why resource dimensions should not be
-  collapsed into elapsed time.
-- [InterCode](https://proceedings.neurips.cc/paper_files/paper/2023/file/4b175d846fb008d540d233c188379ff9-Paper-Datasets_and_Benchmarks.pdf)
-  bounds interaction turns and observation size. [OSWorld](https://github.com/xlang-ai/OSWorld)
-  and [OpenCUA](https://github.com/xlang-ai/OpenCUA) likewise publish results at explicit
-  action-step budgets.
-- [tau2-bench](https://github.com/sierra-research/tau2-bench/blob/main/docs/cli-reference.md)
-  separates maximum steps, maximum errors, retries, trials, and an optional wall-clock
-  timeout instead of treating time as the sole budget.
-- [SWE-agent](https://swe-agent.com/latest/usage/competitive_runs/) recommends a per-instance
-  cost or turn limit because otherwise individual instances can consume unbounded resources.
-- [AI Agents That Matter](https://arxiv.org/abs/2407.01502) argues that agent results should
-  expose capability and resource use jointly, including Pareto-style comparisons, instead
-  of hiding unequal evaluation cost.
-- Systems such as [AlphaCode 2](https://deepmind.google/AlphaCode2_Tech_Report.pdf) separate
-  large internal search from a small external submission allowance. This motivates separate
-  ledgers for `pred` feedback and formal bug submissions.
-
-Existing repository components provide useful building blocks:
-
-- `SubmissionSession` already owns an evaluation-side, atomic, append-only submission
-  ledger and an agent-facing file-spool transport.
-- `verify.py` already applies payload, CPU, memory, output, and solver watchdog limits.
-- API and CLI runners already share prompts and normalized result packaging.
-
-The repository does not yet provide an evaluation-owned `pred` gateway, per-rule budgets,
-isolated per-rule episodes, shortlist freezing, or a distinction between verifier
-infrastructure errors and rejected model submissions. Those gaps belong to this benchmark.
-
-## Features
-
-### Selected for the first release
-
-| Feature | User value | Rough implementation effort |
-|---|---|---:|
-| Static Top-50 triage and shortlist freezing | Measures risk prioritization without allowing dynamic experiments before commitment | 1–2 days |
-| Fifty isolated per-rule investigation episodes | Gives every selected rule the same opportunity and removes order and long-context effects | 3–5 days |
-| Evaluation-owned `pred`/`solve` gateway | Makes all target-system feedback countable and prevents script/parallelism bypasses | 2–4 days |
-| Two submissions **per selected rule** | Allows one formal attempt and one feedback-driven correction without creating an unbounded proof loop | 1–2 days |
-| Visible remaining-budget counters | Lets the model plan in logical units it can observe | about 1 day |
-| Single-run scoring and audit schema | Produces one comparable, re-verifiable result under a named budget contract | 2–3 days |
-| Internal budget-calibration pilot | Selects defensible turn and `pred` limits before the public contract is frozen | 2–4 days plus inference cost |
-
-The estimates are implementation-order estimates, not elapsed calendar commitments. The
-existing submission and verification machinery is reused where possible.
-
-### Deferred
-
-#### Multiple seeds
-
-The first release records any available seed and sampling parameters but does not require
-repeated runs, compute means or confidence intervals, or rank aggregate results. This can be
-added after the single-run contract and its operating cost are understood.
-
-#### Fixed Top-50 diagnostic
-
-A future diagnostic may give every model the same 50 rules to isolate investigation ability
-from risk-ranking ability. It is not required for the main benchmark, which deliberately
-measures the combined workflow, and is deferred to keep the first release small.
-
-### Dropped
-
-#### System Track
-
-There is no leaderboard for Codex CLI, Claude Code, custom prompts, memory systems,
-subagents, or other agent products. These systems change too quickly, attract less interest
-for this benchmark, and confound model capability with orchestration. Existing CLI adapters
-may remain useful for development, but their runs are not rankable in this track.
-
-#### Wall-clock, token, cost, or best-of-N ranking
-
-Wall-clock time remains a watchdog and an auxiliary measurement. Tokens and monetary cost
-are reported when available. None is a primary score or tie-breaker, and ties in verified
-bug count remain ties. The first release has no best-of-N selection.
-
-## Evaluation contract
-
-### Track
-
-The benchmark has one rankable track: **Standardized Model Track**. Rankable runs use the
-same:
-
-- reference API harness and harness version;
-- system and task prompts;
-- pinned library repository and `pred` version;
-- rule inventory;
-- tool surface and network policy;
-- triage and per-rule budgets;
-- maximum model output and observation sizes;
-- reasoning/sampling configuration, to the extent the model API exposes it; and
-- submission verifier.
-
-The result identifies the model, provider/endpoint class, runner version, library commit,
-budget-contract version, and exposed inference parameters. A custom coding-agent harness is
-outside this track even if it uses the same underlying model.
-
-Rankable runs are produced and submitted only by the trusted internal team. The first release
-does not accept untrusted external runs and therefore does not build remote execution or
-cryptographic runner attestation. The backend still re-verifies bugs and ledger consistency,
-but the operator trust boundary is explicit.
-
-### Phase 0: preflight
-
-Before any scored model generation, the runner checks the pinned repository, rule inventory,
-`pred` gateway, and submission judge. Status and version probes are cached and free. If any
-required service is unavailable, the run ends as `run_error` and is not rankable.
-
-### Phase 1: static triage
-
-The model receives the pinned repository and a canonical, cached inventory of all runnable
-rules. It may inspect source code with ordinary read-only shell tools under a fixed triage
-generation/action/observation budget. Dynamic `pred` operations and `submit` are unavailable
-during this phase.
-
-Before the triage budget expires, the model must commit an ordered list of exactly 50 unique,
-valid rule identifiers. It may attach a short, size-capped risk hypothesis for each rule.
-The runner validates and freezes this artifact. The list cannot change after investigation
-begins.
-
-Failure to produce 50 valid unique rules is an invalid run, not a 0-bug result. This avoids
-silently comparing a 50-rule run with a smaller task.
-
-The order is retained for audit and future diagnostics such as Bugs@10 and Bugs@25, but the
-first-release headline score uses all 50 rules.
-
-### Phase 2: isolated investigation
-
-The runner creates one fresh episode for each frozen rule. Every episode has:
-
-- a fresh model conversation and writable scratch directory;
-- the same pinned read-only repository;
-- the current rule identifier and its capped triage hypothesis;
-- the same per-rule model-generation, shell-action, `pred`, `solve`, observation, and
-  submission limits; and
-- a submission judge that accepts certificates only for the current rule.
-
-No conversation, scratch files, remaining budget, or model-authored memory transfers between
-rules. Unused allowance is not transferable. The episodes may execute sequentially or in
-parallel as an implementation detail, provided the model and infrastructure configuration is
-identical and each ledger remains isolated.
-
-An accepted certificate ends that rule's episode immediately. Exhausting a counter leaves
-the episode as completed without an accepted bug; it does not borrow from another rule.
-
-### Logical resource vector
-
-The frozen public contract is named by a vector rather than a time allowance:
-
-`Top50[T, E_t, M, E, P, P_solve, S=2, O]`
-
-where:
-
-- `T` is the number of usable model generations in static triage;
-- `E_t` is the number of executed triage shell actions;
-- `M` is the number of usable model generations per selected rule;
-- `E` is the number of executed shell actions per selected rule;
-- `P` is the number of agent-initiated dynamic `pred` invocations per selected rule;
-- `P_solve` is the subset of `P` that may invoke `pred solve`;
-- `S` is the number of submission opportunities per selected rule, fixed initially at 2;
-  and
-- `O` contains fixed observation, command-output, and payload-size ceilings.
-
-`S=2` is a per-rule ledger. With 50 selected rules, a run can therefore make at most 100
-charged submissions, but there is no run-wide pool from which one rule can take another
-rule's unused attempts. Once a rule is accepted, its unused submission opportunity expires.
-
-The exact values of `T`, `E_t`, `M`, `E`, `P`, `P_solve`, and `O` are selected by the internal
-calibration pilot and then frozen with the benchmark version.
-
-Every observation includes a compact machine-generated status block, for example:
-
-```text
-rule 17/50: maximum_independent_set_to_clique
-model generations: 6/10
-shell actions: 6/10
-pred calls: 13/24
-solve calls: 5/10
-submit attempts: 1/2
-```
-
-This is the model's planning signal. Wall-clock time is not.
-
-## Modules
-
-### 1. Inventory and Triage Controller
-
-Purpose: provide a canonical rule inventory, run static analysis, validate the shortlist,
-and freeze the ordered Top50 artifact.
-
-Interface:
-
-- input: pinned repository metadata and triage budget contract;
-- output: an immutable ordered list of 50 rule IDs plus capped hypotheses and a triage
-  ledger;
-- dependencies: reference model harness, read-only repository, budget controller.
-
-### 2. Episode Runner
-
-Purpose: create 50 fresh, equal-budget investigation episodes and combine their immutable
-ledgers without sharing model state or resources.
-
-Interface:
-
-- input: frozen Top50 artifact and per-rule budget contract;
-- output: 50 episode records with completion, accepted bug, or infrastructure-error status;
-- dependencies: model harness, `PredGateway`, `SubmitJudge`, observation renderer.
-
-### 3. Pred Gateway
-
-Purpose: be the only executable route to dynamic `pred` behavior and atomically enforce `P`
-and `P_solve` for the current episode.
-
-Interface:
-
-- agent command: a `pred`-compatible shim;
-- runner API: status, execute, ledger, and health operations;
-- output: idempotent response plus updated per-rule counters;
-- dependencies: hidden real `pred` binary, pinned execution environment, file-spool or RPC
-  transport.
-
-### 4. Submit Judge
-
-Purpose: enforce exactly two non-transferable formal attempts for the current rule and
-retain the verifier-authoritative certificate ledger.
-
-Interface:
-
-- agent command: `submit ` and free `submit --status`;
-- runner API: status, verify, ledger, and health operations;
-- output: accepted, rejected, exhausted, or infrastructure-error response;
-- dependencies: existing certificate verifier and hidden internal `pred` access.
-
-### 5. Scoring and Calibration
-
-Purpose: validate rankability, compute the single-run headline score, expose audit metrics,
-and determine the fixed public budgets before release.
-
-Interface:
-
-- input: triage ledger, 50 episode ledgers, inference usage, and run metadata;
-- output: versioned submission artifact and internal pilot analysis;
-- dependencies: schema validator and backend re-verifier.
-
-## Technical approaches
-
-### Inventory and triage
-
-**Chosen: one bounded static-analysis session followed by an atomic shortlist commit.** This
-directly measures the desired prioritization behavior while preventing the model from using
-dynamic fuzzing to decide what belongs in the Top50.
-
-Alternatives considered:
-
-- Ask the model to score every rule independently. This is easier to normalize per rule but
-  converts strategic prioritization into 290+ expensive microtasks and loses cross-rule
-  comparison in one context.
-- Use a benchmark-authored heuristic to prefilter candidates. This is cheaper but removes a
-  central model ability from the task and risks encoding maintainer bias.
-
-### Episode execution
-
-**Chosen: 50 fresh, equal-budget, per-rule episodes.** This ensures that all shortlisted
-rules receive the same opportunity and that later rules are not disadvantaged by context
-growth or earlier spending.
-
-Alternatives considered:
-
-- One global investigation session with a run-wide pool. This measures adaptive allocation,
-  but recreates unequal rule coverage and makes scores difficult to interpret.
-- A global pool plus a per-rule ceiling. This blocks extreme concentration but still makes a
-  miss ambiguous: the model may be weak on the rule or simply have allocated less budget.
-
-### Pred enforcement
-
-**Chosen: an evaluation-owned shim and service modeled on `SubmissionSession`.** The real
-binary is not placed directly on the agent's `PATH`; the shim atomically reserves budget
-before spawning it. All agent shells, Python programs, loops, and parallel subprocesses
-share the same per-episode ledger.
-
-Alternatives considered:
-
-- Infer calls from shell commands or trajectories. One shell command can start hundreds of
-  subprocesses, so this is neither complete nor enforceable.
-- Expose the real binary and ask the agent to self-report. This is unauditable and permits
-  accidental bypasses.
-
-Simple invocation counting is preferred over weighted command costs. Each agent-initiated
-`create`, `reduce`, `solve`, `evaluate`, or `extract` consumes one `P`; `solve` also consumes
-one `P_solve`. Cached inventory, version, help, and status operations are free. `pred` calls
-made internally by the submission verifier do not consume the agent's investigation budget.
-
-Model-authored automation remains allowed: writing a fuzzer or batch script is useful agent
-behavior. Its actual gateway invocations are charged individually, so automation is an
-ability rather than a loophole.
-
-### Submission enforcement
-
-**Chosen: adapt the existing evaluation-owned ledger from run-wide accounting to one ledger
-per rule.** A rejected, malformed, duplicate, wrong-rule, or model-caused verifier timeout
-consumes one of that rule's two opportunities. `submit --status` is free. Acceptance closes
-the episode.
-
-Pure verifier, gateway, container, or transport failures do not consume an opportunity.
-Requests carry stable idempotency keys; retrying the same request after a lost response
-returns the stored result instead of charging again. An unrecoverable judge failure marks the
-run unrankable rather than converting the episode into a rejection.
-
-Alternatives considered:
-
-- Keep one shared 100-attempt run pool. Although its total matches 50 × 2, it lets one rule
-  consume attempts intended for another and violates the equal-opportunity contract.
-- Allow only one attempt. This measures first-shot rigor, but provides no bounded opportunity
-  to use verifier feedback to correct a nearly valid certificate.
-- Allow attempts until acceptance. This rewards persistence with unequal external feedback
-  and recreates the original bias.
-
-### Scoring
-
-**Chosen: count distinct accepted rules under the complete frozen budget vector.** The
-headline metric is:
-
-`VerifiedBugs@Top50[T,E_t,M,E,P,P_solve,S=2,O]`
-
-It is the number from 0 to 50 of selected rules with at least one certificate accepted by the
-evaluation-owned verifier. It is reported for one run in the first release. Equal counts are
-ties.
-
-Audit and efficiency fields include:
-
-- ordered Top50 and hypotheses;
-- Bugs@10, Bugs@25, and Bugs@50 along the frozen shortlist order;
-- accepted on submission 1 versus submission 2;
-- `pred` and `solve` calls per accepted bug;
-- cap-hit counts for each logical resource;
-- token usage, monetary cost when available, and elapsed time; and
-- all infrastructure retries and errors.
-
-These fields explain a run but do not override the headline ranking.
-
-Because there is no exhaustive ground truth for which rules are buggy, the benchmark must
-not call unaccepted rules clean or report Top50 precision/recall. It measures verified bug
-discovery yield, not complete bug prevalence.
-
-### Internal budget calibration
-
-**Chosen: run a small pre-release budget grid on development data, then freeze one contract.**
-Candidate values may start around `M ∈ {6, 10, 14}` and `P ∈ {8, 16, 32}`, with a separate
-`P_solve` ceiling. These are pilot coordinates, not proposed public values.
-
-The pilot uses an older library snapshot or other non-ranking development target and a small
-set of available models. It examines:
-
-- verified bugs as each budget increases;
-- the fraction of episodes hitting each cap;
-- marginal bugs gained per additional generation or `pred` call;
-- infrastructure failure rate; and
-- whether one counter consistently binds long before the others.
-
-The selected contract should sit near the useful-yield knee: large enough that most episodes
-can conduct a meaningful experiment, but before extra calls mostly add cost. The chosen
-values and rationale are published and versioned. Formal leaderboard runs execute only that
-one frozen budget, not the grid. The pilot does not introduce a multi-seed requirement.
-
-## Counter and error semantics
-
-Counters are charged for opportunities controlled by the model and retried for failures
-controlled by the evaluation infrastructure.
-
-| Event | Counter effect | Result semantics |
-|---|---|---|
-| Provider returns a usable model response | consume one `T` or `M` | response may lead to an action |
-| Model emits invalid action/format | consume one `T` or `M`; no shell action if none executes | model failure |
-| API 429/5xx/disconnect before any usable response | no model-generation charge; fixed infrastructure retry policy | unrecoverable exhaustion makes run unrankable |
-| Shell command starts | consume one `E_t` or `E` | nonzero exit and model-caused timeout still consume |
-| Dynamic `pred` request is admitted | atomically consume one `P`; `solve` also consumes `P_solve` | invalid arguments, bad model-authored input, nonzero exit, and target timeout consume |
-| Gateway fails before admitting the request | no charge; retry idempotently | unrecoverable failure makes run unrankable |
-| Submission is admitted for verification | atomically consume one of the current rule's two `S` | malformed, rejected, duplicate, and wrong-rule attempts consume |
-| Judge infrastructure fails | no `S` charge; retry the same request idempotently | unrecoverable failure makes run unrankable |
-| Status/help/version/cached inventory probe | free | rate-limited separately if abuse protection is needed |
-
-A reservation is written before the underlying process starts so parallel requests cannot
-overspend. Every ledger record includes episode ID, rule ID, request ID, command class,
-counter values before and after, outcome class, and timestamps for auditing. Timestamps are
-not part of the score.
-
-## Quality requirements
-
-### Fairness and reproducibility
-
-- Budget values, prompts, schemas, repository commit, `pred`, harness, and verifier are
-  immutable within a benchmark version.
-- All 50 per-rule episodes receive identical non-transferable limits.
-- The real `pred` binary and verifier-only interfaces are inaccessible to the agent.
-- Network access from the agent environment is disabled; provider access remains in the
-  runner.
-- The execution image, host class, watchdogs, memory limits, and file/output ceilings are
-  fixed for official runs and recorded where relevant.
-
-### Reliability
-
-- Budget and result ledgers are evaluation-owned, append-only, thread-safe, and idempotent.
-- Crashes preserve accepted certificates and audit logs, but any incomplete infrastructure
-  run is marked unrankable rather than published as a clean partial score.
-- Free preflight probes must succeed before scored generations begin.
-- Watchdogs remain on model calls, commands, `pred`, verification, and the complete run to
-  terminate wedged processes; they protect the service and do not define capability.
-
-### Compatibility
-
-- The agent-facing `pred` shim preserves the documented CLI shape wherever possible so
-  existing prompts and model-authored scripts continue to work.
-- Result schema changes are tied to a benchmark-version bump. Older whole-repository
-  submissions remain readable as historical results but are not mixed into the new ranking.
-
-## What the benchmark can claim
-
-Under a standardized harness and fixed logical budget, the score reflects a combination of:
-
-- static risk ranking across a large rule library;
-- source-code understanding;
-- high-information experiment design;
-- counterexample construction and minimization;
-- interpretation of bounded `pred` and verifier feedback;
-- disciplined use of limited formal submissions; and
-- end-to-end reliability in producing verifier-accepted evidence.
-
-It does not isolate pure model reasoning from all provider implementation details, equalize
-hidden inference FLOPs, measure complete recall over all bugs, compare custom agent systems,
-or rank network and runtime speed.
-
-## Acceptance criteria
-
-The first release is ready when automated tests demonstrate that:
-
-1. triage cannot call dynamic `pred` or `submit`, and only an exact valid Top50 can start
-   investigation;
-2. all 50 episodes start with identical counters and isolated model/scratch state;
-3. direct shell use, Python subprocesses, loops, and concurrent requests cannot exceed `P`
-   or `P_solve`;
-4. no rule can make more than two charged submissions or use another rule's unused attempt;
-5. accepted certificates close their episode and score at most one bug per rule;
-6. model-caused invalid operations charge the documented counters;
-7. injected provider, gateway, transport, and verifier infrastructure failures follow the
-   documented no-charge/idempotent behavior and make unrecoverable runs unrankable;
-8. every observation exposes authoritative remaining counters;
-9. the backend recomputes the headline score only from accepted evaluation-owned ledgers;
-   and
-10. the published result identifies the complete frozen budget contract and necessary
-    provenance.
-
-## Open questions before implementation
-
-These are intentionally resolved by implementation discovery or the calibration pilot, not
-by arbitrary values in the design:
-
-- final numeric values for `T`, `E_t`, `M`, `E`, `P`, `P_solve`, and `O`;
-- the smallest stable inference-parameter contract shared by supported model APIs;
-- migration and display treatment for historical unlimited whole-repository results.
diff --git a/benchmark/docs/design/top50-evidence-budget-issues.md b/benchmark/docs/design/top50-evidence-budget-issues.md
deleted file mode 100644
index 4bb1b22..0000000
--- a/benchmark/docs/design/top50-evidence-budget-issues.md
+++ /dev/null
@@ -1,494 +0,0 @@
-# Top-50 Evidence-Budget Benchmark — Implementation Issue Drafts
-
-Status: filed as GitHub issues #65–#68; implementation in progress  
-Source design: [Top-50 Evidence-Budget Benchmark](top50-evidence-budget-benchmark.md)
-
-## Agreed boundary
-
-This batch implements one trusted-team, standardized Model API track in which a model commits
-to its own Top50 and investigates every selected rule under identical logical budgets. It does
-not implement multiple seeds, a Fixed Top50 diagnostic, a coding-agent/System Track, external
-submission attestation, or ranking by elapsed time, tokens, or cost.
-
-The four issues are deliberately coarse. Each delivers a complete component with one compound
-behavioural acceptance suite; internal classes and substeps are not separate issues.
-
-## Milestones and dependency graph
-
-```text
-M1 Enforceable evidence budget
-└── I1 Evidence Budget Service
-    └── M2 End-to-end Top50 benchmark
-        └── I2 Self-selected Top50 Runner
-            └── I3 vNext result and ranking contract
-                └── M3 Calibration and release
-                    └── I4 Calibrate and publish the frozen contract
-```
-
-I3 depends on I2 because its schema and scorer validate the actual Top50 run artifact. I4
-depends on I1–I3 because calibration must exercise the same implementation that will be
-released and must publish the selected values through the versioned result contract.
-
----
-
-## I1 — Build the evaluation-owned Evidence Budget Service
-
-### Background
-
-The current benchmark exposes the real `pred` executable to the model and limits only a
-run-wide submission count. One shell action can start hundreds of `pred` subprocesses, so turns
-and shell commands do not bound how much target-system feedback a model receives. The
-[Top-50 design](top50-evidence-budget-benchmark.md) instead gives every selected rule an
-independent allowance for model generations, shell actions, `pred` calls, `solve` calls, and
-two formal submissions.
-
-The runner image currently executes as root and installs the real binary at
-`/usr/local/bin/pred`. Replacing `PATH` alone is therefore not enforcement: a model-authored
-script could find and execute the binary by absolute path. This issue owns the complete trusted
-service and OS boundary that make the logical budget enforceable.
-
-### Objective
-
-Build one evaluation-owned Evidence Budget Service that atomically accounts for a single rule
-episode, exposes `pred`-compatible and `submit` shims to an unprivileged agent shell, and keeps
-the real `pred`, verifier, counters, and accepted-certificate ledger outside the agent's access.
-
-Each rule receives exactly two non-transferable submission opportunities. Rejected, malformed,
-duplicate, wrong-rule, and other model-caused attempts consume one; pure gateway, transport,
-container, or verifier infrastructure errors do not. Retrying the same request after a lost
-response must be idempotent.
-
-### Interface (Input → Output)
-
-- **Input:** episode ID, current rule ID, frozen budget contract
-  `{M, E, P, P_solve, S=2, O}`, pinned real `pred` path, and certificate verifier.
-- **Agent surface:** an unprivileged writable scratch directory; a `pred` shim supporting the
-  documented CLI shape; `submit `; and free status/help/version/cached-inventory
-  operations.
-- **Output:** immutable per-episode ledgers for model generations, shell actions, admitted
-  `pred`/`solve` requests, submission attempts, idempotent responses, accepted certificate, and
-  classified infrastructure errors.
-
-### Technical recommendations
-
-- Reuse the atomic file-spool, directory-FD, and append-only ledger patterns in
-  `benchmark/submit_session.py` instead of introducing a network service.
-- Add separate non-root container identities for model-authored shell actions and admitted
-  `pred` subprocesses. The real binary is executable only by the oracle identity; both identities
-  share only the episode scratch group. The privileged Python parent owns counters and spawning,
-  but never runs model-parameterized `pred` as root.
-- Reserve a counter under a lock before spawning a process. A dynamic `create`, `reduce`,
-  `solve`, `evaluate`, or `extract` request consumes one `P`; `solve` additionally consumes one
-  `P_solve`. Nonzero exits, invalid arguments, pathological model input, and target timeouts
-  consume the reservation.
-- Give requests caller-stable idempotency keys and retain completed responses. Do not generate
-  a new logical request ID merely because the client lost the first response.
-- Refactor `SubmissionSession` into, or adapt it behind, a rule-scoped judge. Treat verifier
-  exceptions as `infrastructure_error`, not ordinary rejection. Acceptance closes the rule and
-  expires its unused submission attempt.
-- Keep verifier-internal `pred` calls outside the agent's `P`/`P_solve` ledger.
-
-These are recommendations; an alternative implementation is acceptable if it passes the same
-privilege, concurrency, charging, and idempotency checks.
-
-### Verification
-
-Add one integration acceptance suite and run:
-
-```bash
-pytest -v benchmark/tests/test_evidence_budget.py
-```
-
-It must pass named cases demonstrating all of the following observable behaviour:
-
-1. With `P=4` and `P_solve=1`, a model-authored loop that starts 20 sequential or concurrent
-   `pred` processes receives exactly four admitted dynamic calls, at most one admitted solve,
-   and `budget_exhausted` for the rest; the authoritative ledger contains the same counts.
-2. Running the real binary's absolute path from the agent shell fails with a permission error,
-   while the `pred` shim succeeds. This is the negative control proving `PATH` replacement is
-   not the only protection.
-3. Invalid JSON, invalid CLI arguments, nonzero `pred`, and a model-caused `pred` timeout consume
-   admitted `P`; an injected gateway failure before admission consumes nothing.
-4. Rule A can make two charged submissions and no third. Rule B still has two, proving the
-   allowance is per rule rather than a shared run-wide pool. Acceptance on Rule B's first attempt
-   closes Rule B and expires its second attempt.
-5. Malformed, rejected, duplicate, and wrong-rule submissions consume `S`; an injected verifier
-   exception consumes nothing and returns `infrastructure_error`.
-6. Replaying one completed request ID after simulating a lost response returns the identical
-   stored response without changing any counter. Two genuinely different concurrent request IDs
-   can never overspend a counter.
-7. Status/help/version/cached inventory probes do not change any charged counter.
-
-As a regression check, also run:
-
-```bash
-pytest -v benchmark/tests/test_submit_session.py benchmark/tests/test_verify.py
-```
-
-Existing certificate acceptance/rejection behaviour must remain green. A test fixture that makes
-the real binary executable by the agent must make the absolute-path negative control fail, proving
-that the acceptance suite detects a broken privilege boundary.
-
-### Out of scope
-
-- Top50 selection and episode orchestration;
-- choosing the public numeric values of `M`, `E`, `P`, `P_solve`, or output ceilings;
-- protecting the runner from a malicious trusted team member or cryptographically attesting a
-  remotely uploaded run; and
-- changing `problem-reductions` or the `pred` CLI upstream.
-
----
-
-## I2 — Implement the Self-selected Top50 Runner
-
-### Background
-
-The existing runner launches one unlimited whole-repository conversation. Models can inspect
-different numbers of rules, spend different amounts of target-system feedback on each one, and
-carry discoveries and accumulated context throughout the session. The
-[Top-50 design](top50-evidence-budget-benchmark.md) replaces that with a bounded static triage
-followed by 50 fresh, equal-budget rule episodes.
-
-This issue owns the complete run workflow. A Top50 result is meaningful only if the shortlist is
-frozen before dynamic experimentation, contains exactly 50 valid unique rules, and every rule
-receives a clean context, clean scratch directory, and new Evidence Budget Service from I1.
-
-### Objective
-
-Replace the unlimited whole-repository Model API session with a two-phase `Top50Runner`:
-
-1. a bounded source-only triage conversation that atomically commits an ordered Top50 with
-   capped hypotheses; and
-2. 50 sequential, isolated investigation episodes with identical non-transferable budgets.
-
-Automatically inject authoritative remaining counters into every observation. Preserve partial
-audit artifacts after a crash, but mark any unrecoverable provider, gateway, judge, or container
-failure as `run_error` so it cannot be ranked as a clean completion.
-
-### Interface (Input → Output)
-
-- **Input:** pinned repository and canonical rule inventory; standardized model client and
-  inference settings; triage contract `{T, E_t, O_t}`; per-rule contract
-  `{M, E, P, P_solve, S=2, O}`; and the I1 service factory.
-- **Triage output:** immutable ordered JSON artifact containing exactly 50 unique canonical rule
-  IDs and optional size-capped hypotheses, plus its generation/action ledger.
-- **Run output:** one Top50 run record containing the shortlist, triage ledger, exactly 50 ordered
-  episode records, usage metadata, completion status, and any classified infrastructure error.
-
-### Technical recommendations
-
-- Keep `mini-swe-agent`/LiteLLM as the sole rankable reference harness. Remove the forced
-  unlimited `step_limit=0` behaviour and account for usable responses and executed shell actions
-  explicitly.
-- Provide triage with a runner-cached rule inventory and read-only source tree. Do not expose the
-  dynamic `pred` or `submit` shims until after the shortlist passes validation.
-- Commit the shortlist through an evaluation-owned command or spool rather than scraping final
-  prose. Reject lists with the wrong count, duplicates, unknown IDs, oversized hypotheses, or a
-  second commit attempt.
-- Start each rule with a fresh model message history, fresh scratch directory, and fresh I1
-  service. Pass only the current rule ID and its capped triage hypothesis; transfer no generated
-  files or model-authored memory.
-- Execute the 50 episodes sequentially in the first release. This avoids provider throttling and
-  host contention becoming rule-order-dependent. Because budgets are isolated, parallelism may be
-  revisited later without changing the public contract.
-- Provider 429/5xx/disconnects before a usable response follow one frozen retry policy and do not
-  consume `T`/`M`. A usable but malformed model action consumes its generation. An executed shell
-  action consumes `E_t`/`E`, including nonzero exit and model-caused timeout.
-- End a rule immediately after acceptance or budget exhaustion. Do not transfer unused allowance.
-
-### Verification
-
-Add a deterministic fake-model end-to-end acceptance suite and run:
-
-```bash
-pytest -v benchmark/tests/test_top50_runner.py
-```
-
-The suite must pass named scenarios demonstrating:
-
-1. A fake triage response commits 50 known unique rule IDs; the runner then creates exactly 50
-   episode records in the frozen order, each starting with identical full counters and an empty
-   conversation/scratch state.
-2. The fake model writes a sentinel file and message in episode 1; episode 2 cannot observe
-   either. Spending all `pred` and submit allowance in episode 1 leaves episode 2's counters full.
-3. Dynamic `pred` and `submit` attempts during triage are unavailable and do not produce target
-   feedback. A shortlist with 49 entries, 51 entries, a duplicate, an unknown rule, or an
-   oversized hypothesis is rejected before any investigation episode starts. These are the
-   negative controls for shortlist enforcement.
-4. Every model observation contains authoritative `rule i/50`, generation, shell, `pred`,
-   `solve`, and `submit` usage/limit values matching the underlying ledger.
-5. Acceptance in an episode stops that episode without a further model call. Exhausting any hard
-   counter ends only that rule and does not start an extra action.
-6. An injected 429 before a usable response retries without consuming a model generation; a
-   malformed usable response consumes one. Exhausted infrastructure retries produce `run_error`
-   and an unrankable record rather than a clean 0-bug run.
-7. The rankable execution path rejects `claude-code`, `codex`, a custom strategy file, or a custom
-   prompt config. The same adapters may remain callable in an explicitly unranked development
-   mode.
-
-Also run the existing no-API runner tests:
-
-```bash
-pytest -v benchmark/tests/test_run_mini.py benchmark/tests/test_run_submission.py benchmark/tests/test_whole_repo.py
-```
-
-Update or replace whole-repository expectations so they assert the versioned historical path,
-not the new rankable path. A fake model that tries to return a second shortlist or reuse one
-episode's workspace must make the new suite fail.
-
-### Dependencies
-
-Depends on I1, the evaluation-owned Evidence Budget Service.
-
-### Out of scope
-
-- Fixed Top50 evaluation;
-- multiple seeds or aggregate confidence intervals;
-- parallel rule episodes;
-- custom agent systems and coding-agent CLI ranking; and
-- selecting final public budget values.
-
----
-
-## I3 — Define and enforce the vNext result and ranking contract
-
-### Background
-
-The current submission schema describes one whole-repository session with a shared
-`submit_limit`, a flat `submit_log`, `agent_mode: whole-repo`, and token efficiency as a
-tie-breaker. Those fields cannot prove that a valid Top50 was frozen or that 50 rules received
-identical non-transferable budgets.
-
-The new private submission artifact must carry the complete evaluation-owned triage and episode
-ledgers so the backend can recompute rankability and score. The public leaderboard must continue
-to expose aggregate data only; rule identities, hypotheses, certificates, and trajectories remain
-private answer-key material.
-
-### Objective
-
-Introduce a versioned Top50 submission and scoring contract, make the backend independently
-validate its shortlist and all 50 episode ledgers, re-verify every accepted certificate, and rank
-only clean Standardized Model Track results by distinct accepted rules.
-
-Historical unlimited whole-repository and coding-agent results must remain viewable under their
-original contract but must never be mixed into the Top50 ranking. Equal verified bug counts remain
-ties; elapsed time, token usage, cost, and bugs/Ktok are informational only.
-
-### Interface (Input → Output)
-
-- **Input:** I2 Top50 run record, evaluation-owned ledgers, model/provider and exposed inference
-  parameters, pinned repository/`pred`/runner identifiers, and accepted certificates.
-- **Private output:** schema-valid versioned submission containing the complete budget vector,
-  ordered Top50, triage ledger, exactly 50 rule-bound episode ledgers, usage, errors, and
-  certificates.
-- **Scored output:** backend-derived `VerifiedBugs@Top50[T,E_t,M,E,P,P_solve,S=2,O]`, Bugs@10/25/50,
-  first-versus-second-attempt acceptance, calls per accepted bug, cap-hit diagnostics, and
-  rankability verdict.
-- **Public output:** aggregate leaderboard entry with model and contract provenance, headline
-  verified bug count, and non-ranking efficiency fields; no answer-key data.
-
-### Technical recommendations
-
-- Add an explicit benchmark-contract identifier instead of inferring semantics from optional
-  legacy fields. Keep a deliberate legacy parser for historical data rather than making the new
-  schema accept ambiguous mixtures.
-- Validate that the shortlist has exactly 50 canonical unique IDs, episode order matches it,
-  every configured limit is identical, each usage count is within its limit, `P_solve <= P`,
-  `S=2` per rule, and request IDs/attempt numbering are internally consistent.
-- Derive result rows only from evaluation-owned accepted submit records for the matching current
-  rule. Re-run `verify()` on every accepted certificate; ignore self-reported `bugs_found`.
-- Treat missing episodes, infrastructure errors, CLI/custom-harness provenance, custom prompts,
-  or inconsistent ledgers as unrankable. Preserve the private artifact for debugging.
-- Version the public board by benchmark contract. Remove token efficiency from sort keys for the
-  new contract while leaving old-board rendering intact.
-- The operators are a trusted internal team. Do not add remote-run attestation, signatures, or
-  external-user authentication to this issue.
-
-### Verification
-
-Run the contract and scorer acceptance suites:
-
-```bash
-pytest -v benchmark/tests/test_top50_submission.py benchmark/tests/test_verify_submission.py benchmark/tests/test_backend_score.py
-```
-
-They must use a schema-valid synthetic Top50 artifact and demonstrate:
-
-1. The valid artifact has exactly 50 frozen rules and 50 matching episode ledgers; the backend
-   ignores a deliberately false self-reported score and returns the distinct count implied by
-   accepted, re-verified certificates.
-2. Two accepted certificates for one rule count once. Accepted rules at shortlist positions 7,
-   18, and 41 produce Bugs@10 = 1, Bugs@25 = 2, and Bugs@50 = 3.
-3. A certificate for a rule outside the shortlist, a certificate submitted from another rule's
-   episode, a third attempt, a shared `100`-attempt pool, duplicate/missing episode, over-limit
-   counter, changed budget, or forged request ordering makes the run unrankable. These are the
-   negative controls proving the scorer does not trust the envelope.
-4. A pure `run_error` or unrecoverable infrastructure episode is preserved privately but excluded
-   from ranking. A rejected model submission remains a clean charged attempt and does not make the
-   run an infrastructure failure.
-5. A `codex`/`claude-code` agent mode or custom prompt provenance is excluded from the Top50 board.
-   A canonical API harness run is accepted.
-6. Public output contains no rule ID, hypothesis, certificate, source instance, target config,
-   submit reason, or trajectory. The aggregate guard fails if any such answer-key field is added.
-7. For equal verified bug counts, changing tokens, cost, or elapsed time does not change rank
-   order; the UI represents the entries as tied. A larger verified count ranks first.
-8. One legacy v0.6 whole-repository fixture remains readable in its historical view but cannot
-   enter or replace an entry in the Top50 board.
-
-Also run:
-
-```bash
-python .github/scripts/check_aggregate.py site/results.json
-```
-
-against a generated development board. It must print no errors for the aggregate-only fixture;
-injecting one `rule` or `certificate` field must make the guard exit nonzero.
-
-### Dependencies
-
-Depends on I2 so the contract validates the real `Top50Runner` artifact rather than a parallel
-invented format.
-
-### Out of scope
-
-- cryptographic attestation or accepting untrusted external runs;
-- multiple-seed aggregation, best-of-N, or confidence intervals;
-- Fixed Top50 scores;
-- ranking custom agent systems; and
-- declaring unaccepted rules bug-free or reporting Top50 precision/recall.
-
----
-
-## I4 — Calibrate and publish the first frozen Top50 budget contract
-
-### Background
-
-I1–I3 make logical budgets enforceable and scoreable, but the public values for triage turns,
-per-rule model turns, shell actions, `pred`, `solve`, and output ceilings should not be chosen by
-intuition. The [Top-50 design](top50-evidence-budget-benchmark.md) calls for one internal
-pre-release calibration over development data, followed by one frozen budget for all formal runs.
-
-The calibration grid is a development procedure, not a public multi-budget leaderboard and not a
-multiple-seed requirement. The released benchmark runs each model once at the selected contract.
-
-### Objective
-
-Provide a reproducible internal calibration workflow, execute it on an explicitly non-ranking
-development target, publish the measured trade-offs and chosen knee, then freeze those values
-through configuration, prompts, preflight, schema, documentation, CI, and leaderboard display for
-the next benchmark version.
-
-This issue is the release gate: after it lands, no rankable run can request unlimited or custom
-budgets, and all user-facing documentation tells the same Self-selected Top50 story.
-
-### Interface (Input → Output)
-
-- **Input:** development target identifier; a small named set of internal model runs; candidate
-  contracts such as `M in {6,10,14}` and `P in {8,16,32}`; I1–I3 artifacts; and measured token,
-  cost, elapsed-time, cap-hit, retry, and accepted-bug data.
-- **Calibration output:** machine-readable observations plus a checked-in Markdown report showing
-  yield, cap-hit rates, marginal gain, infrastructure errors, and the reason for selecting each
-  public limit.
-- **Release output:** one immutable named budget contract referenced by runner defaults, prompt,
-  result schema, scorer, docs, preflight, image/version metadata, CI, and leaderboard.
-
-### Technical recommendations
-
-- Add a small analysis command that consumes completed development artifacts. Do not embed a
-  supposedly objective automatic knee detector; show the raw table/curves and record the team's
-  explicit selection rationale.
-- Use an older library snapshot or mark every pilot artifact non-ranking. A pilot model set and
-  candidate grid may be small; do not introduce a repeated-seed requirement.
-- Start from the design's candidate coordinates, then adjust only when cap-hit or failure data
-  shows that one counter binds pathologically. Record every tested contract, including failed
-  runs, instead of publishing only the winner.
-- Replace current `step_limit: 0`, `SUBMIT_LIMIT=100`, whole-repository prompt language, custom
-  strategy hooks on the rankable path, and token-efficiency tie-breaking. Keep process/pred/model
-  watchdogs as fixed safety controls.
-- Update `README.md`, `CONTRIBUTING.md`, `submission.env.example`, `benchmark/config.yaml`, runner
-  help, preflight, Docker image metadata, schema docs, and the public site together.
-- After release, reconcile stale issues #2, #4, and #42 by linking the new implementation and
-  stating precisely which old recommendations were implemented, deferred, or dropped. Closing or
-  editing those issues remains a maintainer action after the implementation is merged.
-
-### Verification
-
-The implementation PR must include the calibration inputs/report and pass:
-
-```bash
-pytest -v -m "not integration"
-```
-
-followed in the pinned runner image by:
-
-```bash
-python -m benchmark.verify --calibrate
-```
-
-A reviewer must also be able to check the committed evidence offline in a few seconds:
-
-```bash
-python -m benchmark.calibrate_budget \
-  --check benchmark/docs/budget-calibration.json
-```
-
-It must print `PASS: calibration evidence matches ` and exit 0. Changing any
-selected limit, deleting a tested candidate, or making the Markdown report disagree with the
-machine-readable evidence must make the same command exit nonzero and identify the mismatch.
-
-The release evidence must additionally demonstrate:
-
-1. The calibration report lists every tested contract and, for each, model/target provenance,
-   verified bugs, cap-hit counts, usage, retries, infrastructure failures, token/cost/time
-   references, and marginal yield. The chosen values exactly match the checked-in named contract.
-2. At least one smaller and one larger candidate surround the selected `M` and `P` values, so the
-   report demonstrates a trade-off rather than merely documenting a predetermined number.
-3. Starting a rankable run with any unlimited counter, `S != 2`, custom budget, custom prompt,
-   custom strategy, or coding-agent backend fails preflight before a scored model generation.
-   These are the release negative controls.
-4. A standard fake Top50 run completes with exactly 50 episodes and emits a schema-valid artifact
-   carrying the frozen contract ID. Changing one embedded limit makes backend validation fail.
-5. A forced hung model call and a forced hung `pred` child are killed by watchdogs and recorded as
-   infrastructure/model outcomes according to the design; changing the watchdog duration does not
-   change the named logical budget or score formula.
-6. The README and leaderboard describe the primary metric as verified distinct bugs at the frozen
-   Top50 evidence budget. They do not present time, cost, tokens, Fixed Top50, multiple seeds, or a
-   System Track as primary ranking dimensions.
-7. Historical whole-repository results remain visible under their old contract and cannot be
-   compared or sorted into the new Top50 table.
-
-Before merging the release, run the repository's required CI and calibration workflows. A PR must
-not be merged until CI is green and the user has explicitly approved the merge.
-
-### Dependencies
-
-Depends on I1, I2, and I3.
-
-### Out of scope
-
-- a public budget grid or bugs-versus-budget leaderboard;
-- multiple seeds and confidence intervals;
-- Fixed Top50 diagnostics;
-- System Track/coding-agent ranking;
-- external contributor submission; and
-- choosing future benchmark versions' budgets without a new calibration and version bump.
-
-## Coverage scan
-
-| Design area | Covered by |
-|---|---|
-| Logical resource model and charging semantics | I1 |
-| Privilege boundary and `pred` bypass prevention | I1 |
-| Per-rule two-attempt submission judge | I1 |
-| Static triage and atomic Top50 freeze | I2 |
-| Fresh equal-budget rule episodes and counter visibility | I2 |
-| Provider/infrastructure failure semantics | I1, I2, I3 |
-| Versioned private result and backend re-verification | I3 |
-| Headline score, diagnostics, privacy, and historical separation | I3 |
-| Internal pilot and public budget freeze | I4 |
-| Prompt, preflight, docs, CI, and release migration | I4 |
-| Multiple seeds, Fixed Top50, System Track | explicitly out of scope in all relevant issues |
-
-The final scan found no uncovered first-release area from the source design. Cross-cutting
-security, privacy, migration, observability, reliability, negative controls, and stale-issue
-reconciliation are assigned above rather than split into additional issues.
diff --git a/benchmark/env_setup.py b/benchmark/env_setup.py
index c21c6e5..9a3557c 100644
--- a/benchmark/env_setup.py
+++ b/benchmark/env_setup.py
@@ -14,7 +14,6 @@
 # Precedence for each: env override > baked file (image) > module default.
 _DEFAULT_PINNED_COMMIT = "aa2d1a10cffa434871d12a4d6f411147fb7e08a8"
 _DEFAULT_PINNED_VERSION = "0.6.0"
-DEFAULT_REPO_URL = "https://github.com/CodingThrust/problem-reductions.git"
 _PIN_DIR = Path(__file__).parent
 
 
@@ -110,84 +109,6 @@ def verify_commit(repo_path: Path, expected_commit: str) -> str:
     return actual
 
 
-def clone_or_verify_repo(repo_path: str | Path, repo_ref: str,
-                         repo_url: str = DEFAULT_REPO_URL) -> str:
-    """Clone ``repo_ref`` into an absent path, or verify an existing checkout.
-
-    Existing checkouts are deliberately never fetched, reset, or checked out: a local run
-    must not mutate a user's working tree.  It either already points at the requested ref or
-    fails with an actionable error.  The returned full commit hash is recorded in the
-    submission envelope.
-    """
-    repo_path = Path(repo_path).expanduser().resolve()
-    if not repo_path.exists():
-        repo_path.parent.mkdir(parents=True, exist_ok=True)
-        clone = subprocess.run(
-            ["git", "clone", "--depth", "1", "--single-branch", "--branch", repo_ref,
-             repo_url, str(repo_path)],
-            capture_output=True,
-            text=True,
-        )
-        if clone.returncode != 0:
-            # `git clone --branch` accepts tags and branches, but not raw commit hashes.
-            # For a commit pin, clone the default ref and fetch that exact object.
-            if (not re.fullmatch(r"[0-9a-fA-F]{7,40}", repo_ref)
-                    or repo_path.exists()):
-                detail = clone.stderr.strip().splitlines()[-1] if clone.stderr.strip() else ""
-                raise ValueError(
-                    f"could not clone ref {repo_ref!r} from {repo_url!r} into {repo_path}"
-                    + (f": {detail}" if detail else "")
-                )
-            try:
-                subprocess.run(
-                    ["git", "clone", "--depth", "1", "--no-checkout", repo_url,
-                     str(repo_path)],
-                    check=True,
-                )
-                subprocess.run(
-                    ["git", "-C", str(repo_path), "fetch", "--depth", "1", "origin",
-                     repo_ref],
-                    check=True,
-                )
-                subprocess.run(
-                    ["git", "-C", str(repo_path), "checkout", "--detach", "FETCH_HEAD"],
-                    check=True,
-                )
-            except subprocess.CalledProcessError as exc:
-                raise ValueError(
-                    f"could not clone commit {repo_ref!r} from {repo_url!r} into {repo_path}"
-                ) from exc
-    elif not (repo_path / ".git").exists():
-        raise ValueError(f"repo directory exists but is not a git checkout: {repo_path}")
-
-    actual = _git_rev_parse(repo_path, "HEAD")
-    try:
-        expected = _git_rev_parse(repo_path, f"{repo_ref}^{{commit}}")
-    except subprocess.CalledProcessError as exc:
-        # A shallow checkout may know HEAD but not retain a symbolic name for a commit ref.
-        if re.fullmatch(r"[0-9a-fA-F]{7,40}", repo_ref) and actual.startswith(repo_ref.lower()):
-            expected = actual
-        else:
-            raise ValueError(
-                f"ref {repo_ref!r} is not available in existing checkout {repo_path}; "
-                "use an absent --repo-dir to clone it"
-            ) from exc
-    if actual != expected:
-        raise ValueError(
-            f"existing checkout {repo_path} is at {actual[:12]}, but {repo_ref!r} resolves "
-            f"to {expected[:12]}; choose another --repo-dir or update it yourself"
-        )
-    return actual
-
-
-def _git_rev_parse(repo_path: Path, revision: str) -> str:
-    result = subprocess.run(
-        ["git", "-C", str(repo_path), "rev-parse", "--verify", revision],
-        capture_output=True,
-        text=True,
-        check=True,
-    )
-    return result.stdout.strip().lower()
 
 
 def setup_env(repo_path: str | Path) -> EnvContext:
diff --git a/benchmark/headless.py b/benchmark/headless.py
deleted file mode 100644
index 09cef30..0000000
--- a/benchmark/headless.py
+++ /dev/null
@@ -1,163 +0,0 @@
-"""Shared process and prompt helpers for local headless-agent backends."""
-from __future__ import annotations
-
-import os
-import signal
-import subprocess
-import tempfile
-import threading
-from collections import deque
-from contextlib import nullcontext
-from pathlib import Path
-from typing import Callable
-
-from benchmark.usage import Usage
-
-
-def safe_model_label(model: str) -> str:
-    """Return a model name that is safe to use in a log filename."""
-    return model.replace("/", "_").replace(":", "_")
-
-
-def render_prompt(template: str, variables: dict) -> str:
-    import jinja2  # dependency of the runner stack
-
-    return jinja2.Template(template).render(**variables)
-
-
-def load_rendered_prompts(config_path, default_config: Path, strategy: str | None,
-                          variables: dict) -> tuple[str, str]:
-    """Load the shared agent config and render its system/task prompt pair."""
-    from benchmark.run_mini import _load_agent_config
-
-    agent_cfg, _model_cfg, strategy = _load_agent_config(
-        config_path, default_config, strategy)
-    variables = {**variables, "strategy": strategy}
-    return (
-        render_prompt(agent_cfg.get("system_template", ""), variables),
-        render_prompt(agent_cfg.get("instance_template", ""), variables),
-    )
-
-
-def child_env(ctx, api_key: str | None = None, *, api_key_var: str | None = None) -> dict:
-    """Build a CLI child environment with ``pred`` and an optional generic key."""
-    env = dict(os.environ)
-    env["PATH"] = f"{Path(ctx.pred_binary).parent}{os.pathsep}{env.get('PATH', '')}"
-    if api_key and api_key_var and not env.get(api_key_var):
-        env[api_key_var] = api_key
-    return env
-
-
-def run_process(cmd: list[str], *, cwd: str, env: dict, timeout: int,
-                stream_log: Path | None = None,
-                label: str | None = None) -> tuple[list[str], int, str | None]:
-    """Run a CLI with a hard timeout and persist its stdout event stream when requested.
-
-    When ``stream_log`` is set, stdout is not also retained in memory. Callers parse the
-    file afterward as an iterable. Stderr is drained concurrently into a sibling log so it
-    cannot deadlock the child or corrupt a JSONL event stream.
-    """
-    label = label or Path(cmd[0]).name
-    log_handle = open(stream_log, "w", encoding="utf-8") if stream_log is not None else None
-    stderr_path = (stream_log.with_name(f"{stream_log.name}.stderr.log")
-                   if stream_log is not None else None)
-    stderr_handle = (open(stderr_path, "w", encoding="utf-8")
-                     if stderr_path is not None else None)
-    lines: list[str] = []
-    error_tail: deque[str] = deque(maxlen=20)
-    timed_out = threading.Event()
-    try:
-        proc = subprocess.Popen(cmd, cwd=cwd, env=env, text=True, start_new_session=True,
-                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
-        def _drain_stderr() -> None:
-            assert proc.stderr is not None
-            for line in proc.stderr:
-                error_tail.append(line[-500:])
-                if stderr_handle is not None:
-                    stderr_handle.write(line)
-
-        stderr_thread = threading.Thread(target=_drain_stderr, name=f"{label}-stderr",
-                                         daemon=True)
-        stderr_thread.start()
-
-        def _kill() -> None:
-            timed_out.set()
-            try:
-                os.killpg(proc.pid, signal.SIGKILL)
-            except ProcessLookupError:
-                pass
-
-        watchdog = threading.Timer(timeout, _kill)
-        watchdog.start()
-        try:
-            assert proc.stdout is not None
-            for line in proc.stdout:
-                if log_handle is None:
-                    lines.append(line)
-                error_tail.append(line[-500:])
-                if log_handle is not None:
-                    log_handle.write(line)
-            proc.wait()
-            stderr_thread.join()
-        finally:
-            watchdog.cancel()
-    except FileNotFoundError:
-        return lines, -1, f"{label} CLI not found: {cmd[0]!r}"
-    except OSError as e:
-        return lines, -1, str(e)
-    finally:
-        if log_handle is not None:
-            log_handle.close()
-        if stderr_handle is not None:
-            stderr_handle.close()
-
-    if timed_out.is_set():
-        return lines, proc.returncode, f"{label} session exceeded {timeout}s"
-    if proc.returncode != 0:
-        detail = "".join(error_tail).strip()[-500:]
-        return lines, proc.returncode, f"{label} exited {proc.returncode}: {detail}"
-    return lines, proc.returncode, None
-
-
-def run_headless_session(
-    *,
-    command: list[str],
-    model_name: str,
-    env: dict,
-    timeout: int,
-    parser: Callable[..., dict],
-    label: str,
-    trajectory_dir: Path | None,
-    submit_session=None,
-    event_error: Callable[[dict | None], str | None] | None = None,
-) -> dict:
-    """Run a headless CLI once and retain its raw stream as the single session log."""
-    stream_log = None
-    if trajectory_dir is not None:
-        trajectory_dir = Path(trajectory_dir).resolve()
-        trajectory_dir.mkdir(parents=True, exist_ok=True)
-        stream_log = trajectory_dir / (
-            f"{safe_model_label(model_name)}_whole-repo.stream.jsonl")
-
-    shared_workdir = getattr(submit_session, "workdir", None)
-    workspace = (nullcontext(str(shared_workdir)) if shared_workdir is not None
-                 else tempfile.TemporaryDirectory(prefix=f"{label}_whole_repo_"))
-    with workspace as workdir:
-        lines, _returncode, run_error = run_process(
-            command, cwd=workdir, env=env, timeout=timeout,
-            stream_log=stream_log, label=label)
-
-    if stream_log is None:
-        parsed = parser(lines, collect_trajectory=False)
-    else:
-        with stream_log.open(encoding="utf-8") as stream:
-            parsed = parser(stream, collect_trajectory=False)
-    usage = parsed.get("usage") or Usage()
-    if event_error is not None:
-        run_error = run_error or event_error(parsed.get("result_event"))
-    return {
-        "tokens_k": round(usage.total_tokens / 1000, 2),
-        "usage": usage,
-        "error": run_error,
-    }
diff --git a/benchmark/model_api.py b/benchmark/model_api.py
new file mode 100644
index 0000000..1e21c97
--- /dev/null
+++ b/benchmark/model_api.py
@@ -0,0 +1,68 @@
+"""Shared model-API helpers for the benchmark runner and preflight."""
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+from benchmark.usage import extract_usage
+
+CONFIG_FILE = Path(__file__).with_name("agent_config.yaml")
+SKIP_RULES = {"mod", "traits", "graph_helpers", "analysis", "cost", "registry", "graph"}
+DEFAULT_MAX_TOKENS = 8192
+DEFAULT_MODEL_TIMEOUT_SECONDS = 300
+DEFAULT_MODEL_RETRIES = 2
+
+
+def list_rules(repo_dir: str | Path) -> list[str]:
+    """List runnable rules in their canonical filename order."""
+    rules_dir = Path(repo_dir) / "src" / "rules"
+    return [path.stem for path in sorted(rules_dir.glob("*.rs"))
+            if path.stem not in SKIP_RULES]
+
+
+def message_text(message: dict) -> str:
+    parts = [message.get("content") or "", message.get("reasoning_content") or ""]
+    return "\n".join(part for part in parts if part)
+
+
+def session_usage(agent):
+    usage = extract_usage(agent.messages)
+    total_tokens = usage.total_tokens
+    if not total_tokens:
+        for message in agent.messages:
+            response = message.get("extra", {}).get("response")
+            if response and hasattr(response, "usage"):
+                total_tokens += getattr(response.usage, "total_tokens", 0)
+    return round(total_tokens / 1000, 2), usage
+
+
+def build_model(model_name: str, api_base: str | None, max_tokens: int,
+                *, api_key: str | None = None,
+                observation_template: str | None = None,
+                format_error_template: str | None = None,
+                model_timeout_seconds: int = DEFAULT_MODEL_TIMEOUT_SECONDS,
+                model_retries: int = DEFAULT_MODEL_RETRIES):
+    from minisweagent.models.litellm_model import LitellmModel
+
+    kwargs = {"timeout": model_timeout_seconds, "num_retries": model_retries}
+    if max_tokens:
+        kwargs["max_tokens"] = max_tokens
+    if api_base:
+        kwargs["api_base"] = api_base
+    if api_key:
+        kwargs["api_key"] = api_key
+    config = {}
+    if observation_template is not None:
+        config["observation_template"] = observation_template
+    if format_error_template is not None:
+        config["format_error_template"] = format_error_template
+    return LitellmModel(model_name=model_name, model_kwargs=kwargs, **config)
+
+
+def load_agent_config() -> tuple[dict, dict]:
+    """Load the benchmark-owned prompt and model message templates."""
+    config = yaml.safe_load(CONFIG_FILE.read_text(encoding="utf-8"))
+    agent_config = dict(config.get("agent", {}))
+    agent_config["cost_limit"] = 0
+    return agent_config, config.get("model", {}) or {}
diff --git a/benchmark/preflight.py b/benchmark/preflight.py
index d48e6d4..a6807e9 100644
--- a/benchmark/preflight.py
+++ b/benchmark/preflight.py
@@ -1,8 +1,8 @@
 """
 Preflight check for a real submission run.
 
-Uses the SAME config you'd give the full batch (model, key, api_base, model_kwargs,
-pred version) but does the minimum real work needed to prove the run won't error out:
+Uses the same model, key, and endpoint as the full run but does the minimum real work needed
+to prove the run won't error out:
 
   1. verify the `pred` binary + version,
   2. confirm the library's reduction rules are present,
@@ -12,12 +12,12 @@
 Exits 0 only if every check passes — so you can launch the full batch with confidence
 instead of discovering a typo'd key or wrong base URL 20 rules in. This is a user-facing
 preflight (it makes one tiny real API call); the no-API wiring of the runner itself is
-covered by the pytest suite (tests/test_run_submission.py), not here.
+covered by the pytest suite (tests/test_run_top50.py), not here.
 """
 from __future__ import annotations
 
 from benchmark.env_setup import find_pred_binary, verify_pred_version
-from benchmark.run_mini import DEFAULT_MAX_TOKENS, _build_model, list_rules
+from benchmark.model_api import DEFAULT_MAX_TOKENS, build_model, list_rules
 from benchmark.usage import usage_from_response
 
 PROBE_PROMPT = "Reply with exactly: OK"
@@ -31,8 +31,6 @@ def run_checks(
     repo_dir: str,
     api_base: str | None = None,
     api_key: str | None = None,
-    model_kwargs: dict | None = None,
-    max_tokens: int = DEFAULT_MAX_TOKENS,
 ) -> list[Check]:
     """Run the three preflight checks and return their results (never raises)."""
     results: list[Check] = []
@@ -55,11 +53,10 @@ def run_checks(
         results.append(("library rules", False, str(e)))
 
     # 3. one real model call through the exact batch model config (validates key, endpoint,
-    #    model name, model_kwargs).
+    #    model name).
     model = response = None
     try:
-        model = _build_model(model_name, api_base, max_tokens,
-                             model_kwargs=model_kwargs, api_key=api_key)
+        model = build_model(model_name, api_base, DEFAULT_MAX_TOKENS, api_key=api_key)
         msgs = [{"role": "user", "content": PROBE_PROMPT}]
         # Call the raw completion, NOT model.query(): query() also parses the reply into an
         # agent bash-action and raises FormatError on a trivial probe. We only need to prove
@@ -75,7 +72,7 @@ def run_checks(
 
     # 4. the run prices EVERY call — exercise that same accounting path. A raw _query passes
     #    even for models missing from litellm's price map, but the agent then aborts the
-    #    whole session on the cost-calculation RuntimeError (unless ignore_errors is set).
+    #    run on the cost-calculation RuntimeError (unless ignore_errors is set).
     if response is not None:
         try:
             model._calculate_cost(response)
diff --git a/benchmark/results.schema.json b/benchmark/results.schema.json
index 3e35003..78abe13 100644
--- a/benchmark/results.schema.json
+++ b/benchmark/results.schema.json
@@ -1,131 +1,25 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema#",
   "$id": "results.schema.json",
-  "title": "BenchmarkResults",
-  "description": "Output of one benchmark run: model, library version, summary metrics, and submitted-rule details.",
+  "title": "BenchmarkScoredResult",
   "type": "object",
-  "required": ["model", "library_commit", "bugs_found", "total_tokens_k", "efficiency_bugs_per_ktok", "rules_tested", "results"],
+  "required": ["model", "library_commit", "rankable",
+    "rankability_errors", "verified_bugs", "bugs_at_10", "bugs_at_25", "bugs_at_50",
+    "first_attempt_accepts", "second_attempt_accepts", "usage_totals",
+    "artifact_sha256", "verifier_report"],
   "properties": {
-    "model": {
-      "type": "string",
-      "description": "LiteLLM model name (e.g. 'anthropic/claude-sonnet-4-6')"
-    },
-    "library_commit": {
-      "type": "string",
-      "description": "Full git commit hash of the problem-reductions library used"
-    },
-    "bugs_found": {
-      "type": "integer",
-      "minimum": 0,
-      "description": "Number of verified, novel bugs found"
-    },
-    "test": {
-      "type": "boolean",
-      "description": "Carried through from the submission: scored + stored privately but excluded from the public leaderboard."
-    },
-    "submitted_by": {
-      "type": ["string", "null"],
-      "description": "Submitter identity carried through to the aggregate entry"
-    },
-    "created_at": {
-      "type": "string",
-      "description": "ISO-8601 timestamp recorded by the runner"
-    },
-    "run_error": {
-      "type": "string",
-      "description": "Private diagnostic for a salvaged partial run"
-    },
-    "pred_version": {
-      "type": "string",
-      "description": "pred version recorded by the runner"
-    },
-    "agent_mode": {
-      "type": ["string", "null"],
-      "description": "Agent harness mode recorded by the runner"
-    },
-    "total_tokens_k": {
-      "type": "number",
-      "minimum": 0,
-      "description": "Total tokens used (in thousands); the sum of usage_totals / 1000."
-    },
-    "usage_totals": {
-      "type": ["object", "null"],
-      "description": "Aggregate token usage in the four provider-reported buckets (input/output/cache_read/cache_write) — the reproducible primitive behind total_tokens_k.",
-      "properties": {
-        "input": {"type": "integer", "minimum": 0},
-        "output": {"type": "integer", "minimum": 0},
-        "cache_read": {"type": "integer", "minimum": 0},
-        "cache_write": {"type": "integer", "minimum": 0}
-      }
-    },
-    "efficiency_bugs_per_ktok": {
-      "type": "number",
-      "minimum": 0,
-      "description": "Efficiency metric: bugs per 1000 tokens"
-    },
-    "rules_tested": {
-      "type": "integer",
-      "minimum": 0,
-      "description": "Number of rules the model attempted"
-    },
-    "results": {
-      "type": "array",
-      "description": "Per-rule results",
-      "items": {
-        "type": "object",
-        "required": ["rule", "result", "tokens_k"],
-        "properties": {
-          "rule": {
-            "type": "string",
-            "description": "Rule name (filename stem from src/rules/)"
-          },
-          "result": {
-            "type": "string",
-            "enum": ["bug_found", "no_certificate", "rejected", "no_bug", "error"],
-            "description": "Outcome for this rule"
-          },
-          "tokens_k": {
-            "type": "number",
-            "minimum": 0
-          },
-          "steps": {
-            "type": "integer",
-            "minimum": 0
-          },
-          "certificate": {
-            "type": "object",
-            "description": "Counterexample certificate (present only when result=bug_found or rejected). The backend re-derives the bundle from source via pred; only rule, source, and the target type are needed.",
-            "properties": {
-              "rule": {"type": "string"},
-              "source": {"type": "object", "description": "Source problem instance (pred create JSON)"},
-              "bundle": {"type": "object", "description": "Reduction bundle (pred reduce JSON); supplies the target type"},
-              "target_type": {"type": "string", "description": "Target problem type (alternative to bundle.target.type)"},
-              "target_config": {"type": "string", "description": "Optional witness: a target solution that mis-extracts"},
-              "violation": {"type": "string", "description": "Optional free-form label; the backend derives the authoritative one"},
-              "note": {"type": "string"}
-            },
-            "required": ["rule", "source"]
-          },
-          "verify_details": {
-            "type": "object",
-            "description": "Details from independent verification (keys vary by violation type)"
-          },
-          "reject_reason": {
-            "type": "string",
-            "description": "Why the certificate was rejected (present only when result=rejected)"
-          }
-        }
-      }
-    },
-    "submit_limit": {
-      "type": ["integer", "null"],
-      "minimum": 0,
-      "description": "Run-wide submit attempt limit retained in the private scored detail"
-    },
-    "submit_log": {
-      "type": ["array", "null"],
-      "description": "Bounded submit ledger retained in the private scored detail"
-    }
-  },
-  "additionalProperties": false
+    "model": {"type": "string"},
+    "library_commit": {"type": "string"},
+    "rankable": {"type": "boolean"},
+    "rankability_errors": {"type": "array", "items": {"type": "string"}},
+    "verified_bugs": {"type": "integer", "minimum": 0, "maximum": 50},
+    "bugs_at_10": {"type": "integer", "minimum": 0, "maximum": 10},
+    "bugs_at_25": {"type": "integer", "minimum": 0, "maximum": 25},
+    "bugs_at_50": {"type": "integer", "minimum": 0, "maximum": 50},
+    "first_attempt_accepts": {"type": "integer", "minimum": 0},
+    "second_attempt_accepts": {"type": "integer", "minimum": 0},
+    "usage_totals": {"type": "object"},
+    "artifact_sha256": {"type": "string", "minLength": 64, "maxLength": 64},
+    "verifier_report": {"type": "array"}
+  }
 }
diff --git a/benchmark/run_mini.py b/benchmark/run_mini.py
deleted file mode 100644
index ebeda70..0000000
--- a/benchmark/run_mini.py
+++ /dev/null
@@ -1,190 +0,0 @@
-"""Whole-repository mini-swe-agent backend."""
-from __future__ import annotations
-
-import json
-import os
-from pathlib import Path
-
-import yaml
-
-from benchmark.env_context import EnvContext
-from benchmark.headless import safe_model_label
-from benchmark.usage import extract_usage
-
-CONFIG_FILE = Path(__file__).parent / "config.yaml"
-SKIP_RULES = {"mod", "traits", "graph_helpers", "analysis", "cost", "registry", "graph"}
-DEFAULT_MAX_TOKENS = 8192
-DEFAULT_MODEL_TIMEOUT_SECONDS = 300
-DEFAULT_MODEL_RETRIES = 2
-
-
-def list_rules(repo_dir: str | Path) -> list[str]:
-    """List runnable rules; used by preflight to verify a prepared repository."""
-    rules_dir = Path(repo_dir) / "src" / "rules"
-    return [f.stem for f in sorted(rules_dir.glob("*.rs")) if f.stem not in SKIP_RULES]
-
-
-def _message_text(msg: dict) -> str:
-    parts = [msg.get("content") or "", msg.get("reasoning_content") or ""]
-    return "\n".join(part for part in parts if part)
-
-
-def extract_total_tokens(messages: list) -> int:
-    total = 0
-    for msg in messages:
-        response = msg.get("extra", {}).get("response")
-        if response and hasattr(response, "usage"):
-            total += getattr(response.usage, "total_tokens", 0)
-    return total
-
-
-def save_trajectory(messages: list, path: Path) -> None:
-    """Save a normalized agent history as JSONL."""
-    path.parent.mkdir(parents=True, exist_ok=True)
-    with path.open("w", encoding="utf-8") as handle:
-        for msg in messages:
-            handle.write(json.dumps({"role": msg.get("role", ""),
-                                     "content": _message_text(msg)}) + "\n")
-
-
-class TrajectoryWriter:
-    """Append-only incremental trajectory writer — live progress during a session.
-
-    Without this, the trajectory only exists after the session ends, so a healthy slow
-    run and a dead hang look identical from outside (and a previous crash's leftover
-    file masquerades as the current run's progress). flush() appends any new messages
-    after each agent step; save_trajectory() still makes the authoritative final write
-    at session end, in the identical format.
-    """
-
-    def __init__(self, path: Path):
-        self._path = path
-        self._written = 0
-        path.parent.mkdir(parents=True, exist_ok=True)
-        path.unlink(missing_ok=True)  # fresh session — never append onto a stale file
-
-    def flush(self, messages: list) -> None:
-        if len(messages) <= self._written:
-            return
-        with self._path.open("a", encoding="utf-8") as handle:
-            for msg in messages[self._written:]:
-                handle.write(json.dumps({"role": msg.get("role", ""),
-                                         "content": _message_text(msg)}) + "\n")
-        self._written = len(messages)
-
-
-def _session_usage(agent):
-    usage = extract_usage(agent.messages)
-    total_tokens = usage.total_tokens or extract_total_tokens(agent.messages)
-    return round(total_tokens / 1000, 2), usage
-
-
-def _build_model(model_name: str, api_base: str | None, max_tokens: int,
-                 model_kwargs: dict | None = None, api_key: str | None = None,
-                 observation_template: str | None = None,
-                 format_error_template: str | None = None,
-                 model_timeout_seconds: int = DEFAULT_MODEL_TIMEOUT_SECONDS,
-                 model_retries: int = DEFAULT_MODEL_RETRIES):
-    from minisweagent.models.litellm_model import LitellmModel
-
-    # A hung API call must fail fast and retry — never freeze a whole-repo session
-    # indefinitely. User-supplied MODEL_KWARGS still wins on any key.
-    kwargs = {"timeout": model_timeout_seconds,
-              "num_retries": model_retries, **dict(model_kwargs or {})}
-    if max_tokens:
-        kwargs["max_tokens"] = max_tokens
-    if api_base:
-        kwargs["api_base"] = api_base
-    if api_key:
-        kwargs["api_key"] = api_key
-    config = {}
-    if observation_template is not None:
-        config["observation_template"] = observation_template
-    if format_error_template is not None:
-        config["format_error_template"] = format_error_template
-    return LitellmModel(model_name=model_name, model_kwargs=kwargs, **config)
-
-
-def _load_agent_config(config_path: str | Path | None, default_file: Path,
-                       strategy: str | None, *, force_unlimited: bool = True
-                       ) -> tuple[dict, dict, str]:
-    """Load prompts; legacy callers may force-disable mini-swe's hidden step limit."""
-    config_file = Path(config_path) if config_path else default_file
-    config = yaml.safe_load(config_file.read_text(encoding="utf-8"))
-    if strategy is None:
-        strategy_file = os.environ.get("AGENT_STRATEGY_FILE")
-        strategy = Path(strategy_file).read_text(encoding="utf-8") if strategy_file else ""
-    agent_config = dict(config.get("agent", {}))
-    # 0 means unlimited in mini-swe-agent. The completion command in the prompt is the
-    # normal exit; the runner only retains per-command/process timeouts for wedged tools.
-    if force_unlimited:
-        agent_config["step_limit"] = 0
-    agent_config["cost_limit"] = 0
-    return agent_config, config.get("model", {}) or {}, strategy
-
-
-def run_repo_session(
-    model_name: str,
-    ctx: EnvContext,
-    *,
-    api_base: str | None = None,
-    trajectory_dir: Path | None = None,
-    max_tokens: int = DEFAULT_MAX_TOKENS,
-    config_path: str | Path | None = None,
-    strategy: str | None = None,
-    model_kwargs: dict | None = None,
-    api_key: str | None = None,
-    submit_session=None,
-) -> dict:
-    """Run one self-terminating session over the complete repository."""
-    from minisweagent.agents.default import DefaultAgent
-    from minisweagent.environments.local import LocalEnvironment
-
-    agent_config, model_config, strategy = _load_agent_config(
-        config_path, CONFIG_FILE, strategy)
-    safe_model = safe_model_label(model_name)
-    writer = None
-    if trajectory_dir is not None:
-        trajectory_dir = Path(trajectory_dir).resolve()
-        trajectory_dir.mkdir(parents=True, exist_ok=True)
-        writer = TrajectoryWriter(trajectory_dir / f"{safe_model}_whole-repo.jsonl")
-
-    class _StepFlushingAgent(DefaultAgent):
-        """DefaultAgent that flushes the trajectory after every step (best-effort)."""
-
-        def step(self):
-            result = super().step()
-            if writer is not None:
-                try:
-                    writer.flush(self.messages)
-                except OSError:
-                    pass  # observability must never kill the session
-            return result
-
-    agent = _StepFlushingAgent(
-        _build_model(
-            model_name, api_base, max_tokens, model_kwargs=model_kwargs, api_key=api_key,
-            observation_template=model_config.get("observation_template"),
-            format_error_template=model_config.get("format_error_template"),
-        ),
-        LocalEnvironment(),
-        **agent_config,
-    )
-    agent.extra_template_vars = {
-        "repo_dir": str(ctx.repo_path),
-        "commit_hash": ctx.commit_hash[:7],
-        "strategy": strategy,
-        "submit_limit": submit_session.limit if submit_session is not None else 0,
-    }
-
-    run_error = None
-    try:
-        agent.run(task="find-bugs")
-    except Exception as error:  # salvage verified submissions and partial logs
-        run_error = f"{type(error).__name__}: {error}"
-
-    tokens_k, usage = _session_usage(agent)
-    if trajectory_dir is not None:
-        save_trajectory(agent.messages,
-                        trajectory_dir / f"{safe_model}_whole-repo.jsonl")
-    return {"tokens_k": tokens_k, "usage": usage, "error": run_error}
diff --git a/benchmark/run_submission.py b/benchmark/run_submission.py
deleted file mode 100644
index a7508ba..0000000
--- a/benchmark/run_submission.py
+++ /dev/null
@@ -1,428 +0,0 @@
-#!/usr/bin/env python3
-"""
-Submission runner — shared by Docker and lightweight local agent modes.
-
-Given a model and an agent backend, run the bug-hunting agent across the reduction rules
-and emit a single, rankable ``submission.json`` recording its submitted counterexamples.
-
-The agent decides when its single whole-repository session is complete. A run-wide
-counterexample submission limit (100 by default) bounds scored claims, not exploration.
-Self-reported bug counts are advisory only — the backend
-re-verifies every certificate with ``pred`` before anything reaches the leaderboard (see
-benchmark/verify_submission.py).
-
-Docker usage (all config — any provider — in submission.env, key never baked in):
-
-    docker run --rm --env-file submission.env -v "$PWD/out:/out" \
-        problem-reductions-runner:v0.6.0
-    # → ./out/submission.json   (template: submission.env.example)
-
-Env vars (CLI flags override): MODEL_NAME, the matching API key (generic API_KEY or a
-provider var like OPENAI_API_KEY/ANTHROPIC_API_KEY), API_BASE, MODEL_KWARGS,
-AGENT_BACKEND (mini-swe | claude-code | codex), AGENT_CONFIG,
-AGENT_STRATEGY_FILE;
-SUBMIT_LIMIT (default 100), FAKE (1 → no API/pred, used by tests). The claude-code backend authenticates via
-CLAUDE_CODE_OAUTH_TOKEN or ANTHROPIC_API_KEY.
-"""
-import argparse
-import json
-import os
-import re
-from pathlib import Path
-
-from benchmark.env_context import EnvContext
-from benchmark.env_setup import (
-    DEFAULT_REPO_URL,
-    clone_or_verify_repo,
-    find_pred_binary,
-    pinned_commit,
-    verify_pred_version,
-)
-from benchmark.submit_session import SubmissionSession
-from benchmark.usage import Usage, usage_as_dict, usage_from_dict
-from benchmark.verify import count_bugs
-
-RUNNER_VERSION = "0.8.0"
-BACKENDS = ("mini-swe", "claude-code", "codex")
-
-
-def build_submission(
-    model: str,
-    rows: list[dict],
-    *,
-    library_commit: str,
-    runner_version: str = RUNNER_VERSION,
-    created_at: str | None = None,
-    submitted_by: str | None = None,
-    total_tokens_k: float | None = None,
-    pred_version: str = "",
-    usage_totals=None,
-    run_error: str | None = None,
-    submit_limit: int = 100,
-    submit_attempts: list[dict] | None = None,
-) -> dict:
-    """Assemble the submission envelope from the runner's result rows.
-
-    ``rules_tested`` is the number of distinct rules for which the agent submitted a
-    structured certificate. It is a floor on exploration coverage because inspected clean
-    rules do not produce rows. ``bugs_found`` is distinct rules with a confirmed bug.
-
-    The 4-bucket token total is either passed in (``usage_totals`` — whole-repo, one
-    session) and rides on the envelope as ``usage_totals``. ``total_tokens_k`` is derived
-    from it when it carries counts, else the explicit session total is used.
-    """
-    bugs = count_bugs(rows)
-
-    # 4-bucket token usage: the reproducible primitive.
-    if usage_totals is None:
-        usage_totals = Usage()
-        for r in rows:
-            usage_totals = usage_totals + usage_from_dict(r.get("usage"))
-    elif isinstance(usage_totals, dict):
-        usage_totals = usage_from_dict(usage_totals)
-
-    if usage_totals.total_tokens:
-        tokens_k = usage_totals.total_tokens / 1000
-    else:
-        tokens_k = total_tokens_k if total_tokens_k is not None else sum(r.get("tokens_k", 0.0) for r in rows)
-
-    submit_attempts = submit_attempts or []
-    envelope = {
-        "model": model,
-        "library_commit": library_commit,
-        "bugs_found": bugs,
-        "total_tokens_k": round(tokens_k, 2),
-        "efficiency_bugs_per_ktok": round(bugs / tokens_k, 4) if tokens_k else 0,
-        "rules_tested": len({r.get("rule") for r in rows}),
-        # Aggregate token totals — reproducible primitive behind total_tokens_k.
-        "usage_totals": usage_as_dict(usage_totals),
-        "results": rows,
-        # Version/provenance stamp so a produced file self-identifies its run (no more
-        # "everything overwrites one submission.json"). agent_mode + created_at + the
-        # runner/pred/library pins together pin down exactly what produced this file.
-        "runner_version": runner_version,
-        "pred_version": pred_version,
-        "agent_mode": "whole-repo",
-        "created_at": created_at,
-        "submitted_by": submitted_by,
-        # Evaluation-owned command ledger.  Results are derived from this ledger; agent
-        # prose and ad-hoc cert files are not accepted as alternate submission channels.
-        "submit_limit": submit_limit,
-        "submit_log": submit_attempts,
-    }
-    # Set only when the session died on a fatal error (quota/auth/network): the results are
-    # the partial salvage, not a clean "0 bugs" completion. Keeps a crash from masquerading
-    # as a finished run.
-    if run_error is not None:
-        envelope["run_error"] = run_error
-    return envelope
-
-
-def _run_backend(backend: str, ctx, model: str, *, trajectory_dir=None,
-                 api_base=None, max_tokens=None, config_path=None, strategy=None,
-                 model_kwargs=None, api_key=None, submit_session=None) -> dict:
-    """Dispatch one whole-repository session without a one-use wrapper hierarchy."""
-    if backend == "claude-code":
-        from benchmark.claude_code import run_repo_claude
-        return run_repo_claude(
-            model, ctx, trajectory_dir=trajectory_dir, config_path=config_path,
-            strategy=strategy, api_key=api_key, submit_session=submit_session)
-    if backend == "codex":
-        from benchmark.codex_cli import run_repo_codex
-        return run_repo_codex(
-            model, ctx, trajectory_dir=trajectory_dir, config_path=config_path,
-            strategy=strategy, api_key=api_key, submit_session=submit_session)
-
-    from benchmark.run_mini import DEFAULT_MAX_TOKENS, run_repo_session
-    return run_repo_session(
-        model, ctx, api_base=api_base,
-        max_tokens=max_tokens if max_tokens is not None else DEFAULT_MAX_TOKENS,
-        trajectory_dir=trajectory_dir, config_path=config_path, strategy=strategy,
-        model_kwargs=model_kwargs, api_key=api_key, submit_session=submit_session)
-
-
-def run(
-    model: str,
-    repo_dir: str,
-    *,
-    fake: bool = False,
-    library_commit: str | None = None,
-    api_base: str | None = None,
-    output: Path | None = None,
-    created_at: str | None = None,
-    submitted_by: str | None = None,
-    max_tokens: int | None = None,
-    config_path: str | Path | None = None,
-    strategy: str | None = None,
-    model_kwargs: dict | None = None,
-    api_key: str | None = None,
-    backend: str = "mini-swe",
-    trajectory_dir: str | Path | None = None,
-    submit_limit: int = 100,
-) -> dict:
-    """Run the full session for one model and return the submission dict.
-
-    ``trajectory_dir`` is where the whole-repo agent's raw/final log is persisted
-    (default: the output file's directory). ``output`` is the authoritative submission
-    path; callers that want run history should give each run a distinct path.
-
-    The runner always launches ONE session over the whole library. The agent enumerates,
-    triages, and decides when to stop. ``backend`` selects the implementation: ``mini-swe``
-    (default,
-    litellm — any provider), ``claude-code`` (headless ``claude -p``), or ``codex``
-    (headless ``codex exec``). CLI backends ignore
-    ``api_base``/``model_kwargs``/``max_tokens`` — the CLI owns the API call.
-
-    In ``fake`` mode no API key or pred binary is needed; it only smoke-tests wiring.
-    """
-    repo = Path(repo_dir)
-    commit = library_commit or pinned_commit()
-
-    if backend not in BACKENDS:
-        raise ValueError(f"unknown backend: {backend!r} (expected one of {BACKENDS})")
-
-    if fake:
-        # EnvContext validates a real pred binary; fake mode only needs the fields consumed
-        # by a backend, so a lightweight stand-in avoids API/pred setup.
-        from types import SimpleNamespace
-        ctx = SimpleNamespace(repo_path=repo, pred_binary=Path("pred"),
-                              commit_hash=commit, pred_version="")
-    else:
-        pred_binary = find_pred_binary()
-        pred_ver = verify_pred_version(pred_binary)  # fail fast if pred != pinned version
-        ctx = EnvContext(repo_path=repo, pred_binary=pred_binary, commit_hash=commit,
-                         pred_version=pred_ver)
-
-    # Where to persist the whole-repo trajectory. An explicit
-    # TRAJECTORY_DIR wins; otherwise default to the output file's directory (the mounted
-    # /out). Exposed as a parameter so it shows up in the runner's config surface, not
-    # hardcoded — a crash/early-stop still leaves the latest trajectory on disk.
-    traj_dir = Path(trajectory_dir) if trajectory_dir else (
-        Path(output).parent if output is not None else None)
-
-    # One service spans the complete repository session.
-    with SubmissionSession(limit=submit_limit) as submit_session:
-        session = ({"tokens_k": 0.0, "usage": None, "error": None} if fake else
-                   _run_backend(
-                       backend, ctx, model, trajectory_dir=traj_dir,
-                       api_base=api_base, max_tokens=max_tokens,
-                       config_path=config_path, strategy=strategy,
-                       model_kwargs=model_kwargs, api_key=api_key,
-                       submit_session=submit_session))
-        rows = submit_session.result_rows()
-        total_tokens = session["tokens_k"]
-        session_usage = session.get("usage")
-        run_error = session.get("error")
-        if not fake and not run_error and not submit_session.reachable:
-            run_error = ("submit channel was not successfully probed; no status or submit "
-                         "request reached the evaluation service")
-            if traj_dir is not None:
-                submit_session.preserve_artifacts(
-                    Path(traj_dir) / "salvaged-agent-artifacts")
-        if run_error:
-            print(f"WARNING: session ended on error — salvaged partial results: {run_error}")
-        submit_attempts = submit_session.attempts
-
-    sub = build_submission(
-        model, rows, library_commit=commit,
-        created_at=created_at, submitted_by=submitted_by,
-        total_tokens_k=total_tokens,
-        pred_version=getattr(ctx, "pred_version", ""),
-        usage_totals=session_usage,
-        run_error=run_error,
-        submit_limit=submit_limit, submit_attempts=submit_attempts,
-    )
-
-    if output is not None:
-        output = Path(output)
-        output.parent.mkdir(parents=True, exist_ok=True)
-        output.write_text(json.dumps(sub, indent=2), encoding="utf-8")
-
-    return sub
-
-
-def _env(name: str, default: str | None = None) -> str | None:
-    return os.environ.get(name, default)
-
-
-def _load_env_file(path: str | Path) -> None:
-    """Load a small Docker-compatible ``KEY=VALUE`` env file without overriding env."""
-    path = Path(path)
-    for line_no, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
-        line = raw.strip()
-        if not line or line.startswith("#"):
-            continue
-        if "=" not in line:
-            raise ValueError(f"{path}:{line_no}: expected KEY=VALUE")
-        key, value = line.split("=", 1)
-        key, value = key.strip(), value.strip()
-        if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key):
-            raise ValueError(f"{path}:{line_no}: invalid environment variable name {key!r}")
-        if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
-            value = value[1:-1]
-        os.environ.setdefault(key, value)
-
-
-def main(argv: list[str] | None = None) -> None:
-    # Load the env file before computing argparse defaults. Ambient variables and explicit
-    # CLI flags still win, matching Docker's practical configuration precedence.
-    pre_parser = argparse.ArgumentParser(add_help=False)
-    pre_parser.add_argument("--env-file")
-    pre_args, _ = pre_parser.parse_known_args(argv)
-    if pre_args.env_file:
-        try:
-            _load_env_file(pre_args.env_file)
-        except (OSError, ValueError) as e:
-            pre_parser.error(str(e))
-
-    parser = argparse.ArgumentParser(description="Bug-finding runner → submission.json")
-    parser.add_argument("--env-file",
-                        help="Load KEY=VALUE defaults from this file; ambient env and CLI flags win")
-    parser.add_argument("--model", default=_env("MODEL_NAME"), help="LiteLLM model name (env MODEL_NAME)")
-    parser.add_argument("--repo-dir", default=_env("REPO_DIR"),
-                        help="problem-reductions source tree (env REPO_DIR)")
-    parser.add_argument("--repo-ref", default=_env("REPO_REF"),
-                        help="Clone/verify this git tag, branch, or commit before running. "
-                             "Used by lightweight local runs (env REPO_REF).")
-    parser.add_argument("--repo-url", default=_env("REPO_URL", DEFAULT_REPO_URL),
-                        help="Repository URL used when --repo-dir does not exist (env REPO_URL)")
-    parser.add_argument("--output", default=_env("OUTPUT"),
-                        help="Authoritative submission path (env OUTPUT)")
-    parser.add_argument("--trajectory-dir", default=_env("TRAJECTORY_DIR"),
-                        help="Where to persist the whole-repo raw/final log "
-                             "(env TRAJECTORY_DIR; default: the output file's directory).")
-    parser.add_argument("--api-base", default=_env("API_BASE"), help="Custom API base (env API_BASE)")
-    parser.add_argument("--api-key", default=_env("API_KEY"),
-                        help="Generic API key, any provider (env API_KEY). Avoids needing the "
-                             "provider-specific var; provider env vars (ANTHROPIC_API_KEY, …) still work.")
-    parser.add_argument("--model-kwargs", default=_env("MODEL_KWARGS"),
-                        help="JSON of extra litellm.completion kwargs for non-standard providers "
-                             "(env MODEL_KWARGS), e.g. '{\"api_version\":\"2024-02-01\","
-                             "\"custom_llm_provider\":\"openai\"}'.")
-    parser.add_argument("--submitted-by", default=_env("SUBMITTED_BY"))
-    parser.add_argument("--max-tokens", type=lambda v: int(v) if v else None,
-                        default=_env("MAX_TOKENS"), help="Per-call output-token ceiling")
-    parser.add_argument("--submit-limit", type=int,
-                        default=int(_env("SUBMIT_LIMIT", "100")),
-                        help="Run-wide counterexample submission attempts (env SUBMIT_LIMIT; "
-                             "accepted and rejected submissions both consume one)")
-    parser.add_argument("--expected-pred-version", default=_env("EXPECTED_PRED_VERSION"),
-                        help="Require this pred version (default: pinned; empty string disables)")
-    parser.add_argument("--expected-pred-commit", default=_env("EXPECTED_PRED_COMMIT"),
-                        help="Library commit to record/verify (default: pinned for this image)")
-    # Agent prompt/strategy hook — hand-editable without rebuilding the image. Mount your own
-    # config.yaml (full prompt) and/or a strategy file (extra bug-hunting hints injected into
-    # the {{strategy}} slot of the system prompt). See benchmark/config.yaml.
-    parser.add_argument("--config", default=_env("AGENT_CONFIG"),
-                        help="Path to an agent config.yaml (env AGENT_CONFIG; default: bundled)")
-    parser.add_argument("--strategy-file", default=_env("AGENT_STRATEGY_FILE"),
-                        help="File of extra strategy hints injected into the prompt (env AGENT_STRATEGY_FILE)")
-    parser.add_argument("--preflight", action="store_true", default=bool(_env("PREFLIGHT")),
-                        help="Validate the config with one tiny real API call + pred/rules "
-                             "checks, then exit (run this before the full batch).")
-    parser.add_argument("--fake", action="store_true", default=bool(_env("FAKE")),
-                        help="No API/pred — backend wiring smoke run (mostly covered by tests)")
-    parser.add_argument("--backend", choices=BACKENDS,
-                        default=_env("AGENT_BACKEND", "mini-swe"),
-                        help="Agent implementation (env AGENT_BACKEND). mini-swe: litellm, any "
-                             "provider (default). claude-code: headless `claude -p`; auth via "
-                             "Claude login/token. codex: headless `codex exec`; auth via Codex "
-                             "login or OPENAI_API_KEY.")
-    parser.add_argument("--host-cli", action="store_true",
-                        help="Confirm a coding-agent backend is running directly on the host "
-                             "(set by make run-local; never use in the API container route)")
-    args = parser.parse_args(argv)
-
-    if not args.model:
-        parser.error("--model (or env MODEL_NAME) is required")
-    if args.backend != "mini-swe" and not args.host_cli:
-        parser.error(
-            "container/API runs require backend 'mini-swe'; run coding-agent CLIs on "
-            "the host with `make run-local LOCAL_BACKEND=`"
-        )
-    if args.host_cli and args.backend == "mini-swe":
-        parser.error("--host-cli requires a coding-agent backend, not 'mini-swe'")
-
-    library_commit = None
-    if args.repo_ref:
-        missing = [name for name, value in (
-            ("--repo-dir", args.repo_dir),
-            ("--output", args.output),
-            ("--trajectory-dir", args.trajectory_dir),
-        ) if not value]
-        if missing:
-            parser.error("local --repo-ref runs require explicit " + ", ".join(missing))
-        try:
-            library_commit = clone_or_verify_repo(args.repo_dir, args.repo_ref, args.repo_url)
-        except (OSError, ValueError) as e:
-            parser.error(str(e))
-        print(f"Repository ready at {Path(args.repo_dir).resolve()} ({library_commit[:12]})")
-    else:
-        # Container defaults. The image also sets these variables, but retaining fallbacks
-        # keeps the module directly runnable for build-time fake checks.
-        args.repo_dir = args.repo_dir or "/app/pr-src"
-        args.output = args.output or "/out/submission.json"
-
-    # verify_pred_version()/pinned_commit() read these env vars; surface the flags through them.
-    if args.expected_pred_version is not None:
-        os.environ["EXPECTED_PRED_VERSION"] = args.expected_pred_version
-    if args.expected_pred_commit:
-        os.environ["EXPECTED_PRED_COMMIT"] = args.expected_pred_commit
-
-    # Read the strategy hints file once (so a bad path fails fast, before any API spend).
-    strategy = None
-    if args.strategy_file:
-        strategy = Path(args.strategy_file).read_text(encoding="utf-8")
-
-    model_kwargs = None
-    if args.model_kwargs:
-        try:
-            model_kwargs = json.loads(args.model_kwargs)  # fail fast on malformed JSON
-        except json.JSONDecodeError as e:
-            parser.error(f"--model-kwargs is not valid JSON: {e}")
-        if not isinstance(model_kwargs, dict):
-            parser.error("--model-kwargs must be a JSON object")
-
-    if args.preflight:
-        from benchmark.preflight import format_report, run_checks
-        from benchmark.run_mini import DEFAULT_MAX_TOKENS
-        print(f"Preflight for {args.model} (one tiny real call + pred/rules checks)...")
-        results = run_checks(args.model, repo_dir=args.repo_dir, api_base=args.api_base,
-                             api_key=args.api_key, model_kwargs=model_kwargs,
-                             max_tokens=args.max_tokens or DEFAULT_MAX_TOKENS)
-        raise SystemExit(0 if format_report(results) else 1)
-
-    import datetime
-    created_at = datetime.datetime.now(datetime.timezone.utc).isoformat()
-
-    backend_tag = f", {args.backend}" if args.backend != "mini-swe" else ""
-    print(f"Running {args.model} (whole-repo{backend_tag}){' [FAKE]' if args.fake else ''}...")
-    sub = run(
-        args.model,
-        args.repo_dir,
-        fake=args.fake,
-        library_commit=library_commit,
-        api_base=args.api_base,
-        output=Path(args.output),
-        created_at=created_at,
-        submitted_by=args.submitted_by,
-        max_tokens=args.max_tokens,
-        config_path=args.config,
-        strategy=strategy,
-        model_kwargs=model_kwargs,
-        api_key=args.api_key,
-        backend=args.backend,
-        trajectory_dir=args.trajectory_dir,
-        submit_limit=args.submit_limit,
-    )
-    print(f"\n{sub['bugs_found']} claimed bugs | {sub['total_tokens_k']:.1f}K tok | "
-          f"{sub['rules_tested']} rules submitted")
-    used, limit = len(sub["submit_log"]), sub["submit_limit"]
-    print(f"Submit attempts: {used}/{limit} ({limit - used} remaining)")
-    print(f"Submission → {args.output}")
-    print("Self-reported counts are advisory; the backend re-verifies every certificate "
-          "with pred before it counts.")
-
-
-if __name__ == "__main__":
-    main()
diff --git a/benchmark/run_top50.py b/benchmark/run_top50.py
index 4a8917e..f4fa473 100644
--- a/benchmark/run_top50.py
+++ b/benchmark/run_top50.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Production entrypoint for the standardized self-selected Top50 model track."""
+"""Production entrypoint for the standardized self-selected Top50 benchmark."""
 from __future__ import annotations
 
 import argparse
@@ -10,7 +10,7 @@
 from pathlib import Path
 
 from benchmark.env_setup import find_pred_binary, pinned_commit, verify_pred_version
-from benchmark.run_mini import list_rules
+from benchmark.model_api import list_rules
 from benchmark.top50_runner import (
     PhaseResult,
     Top50Runner,
@@ -18,7 +18,6 @@
     build_rankable_runner,
 )
 from benchmark.top50_contract import (
-    AGENT_MODE,
     RUNNER_VERSION,
     EXPECTED_INFERENCE_PARAMETERS,
     EXPECTED_SAFETY_CONTROLS,
@@ -27,7 +26,6 @@
 
 
 FORBIDDEN_RANKABLE_ENV = (
-    "AGENT_BACKEND", "AGENT_CONFIG", "AGENT_STRATEGY_FILE", "SUBMIT_LIMIT",
     "EXPECTED_PRED_COMMIT", "EXPECTED_PRED_VERSION",
 )
 TRUSTED_PRED_PATH = Path("/usr/local/libexec/prb/pred")
@@ -69,13 +67,11 @@ def verify_rankable_pred_path(binary: Path) -> None:
         raise ValueError(f"rankable runs require the image-owned pred at {TRUSTED_PRED_PATH}")
 
 
-def validate_rankable_settings(*, model_kwargs: dict | None = None) -> None:
+def validate_rankable_settings() -> None:
     """Reject every knob that could change the frozen rankable execution path."""
     present = [name for name in FORBIDDEN_RANKABLE_ENV if name in os.environ]
     if present:
         raise ValueError(f"rankable Top50 runs reject custom setting(s): {', '.join(present)}")
-    if model_kwargs is not None:
-        raise ValueError("rankable Top50 runs do not accept custom model kwargs")
 
 
 class _FakeExecutor:
@@ -90,10 +86,10 @@ def run_episode(self, session, **kwargs):
 
 
 def run(*, model: str, repo_dir: str | Path, output: str | Path,
-        fake: bool = False, api_base: str | None = None, api_key: str | None = None,
-        model_kwargs: dict | None = None) -> dict:
+        fake: bool = False, api_base: str | None = None,
+        api_key: str | None = None) -> dict:
     if not fake:
-        validate_rankable_settings(model_kwargs=model_kwargs)
+        validate_rankable_settings()
     repo = Path(repo_dir).resolve()
     expected_commit = pinned_commit()
     actual_commit = expected_commit if fake else verify_rankable_source(repo, expected_commit)
@@ -121,7 +117,6 @@ def run(*, model: str, repo_dir: str | Path, output: str | Path,
         "library_commit": actual_commit,
         "runner_version": RUNNER_VERSION,
         "pred_version": pred_version,
-        "agent_mode": AGENT_MODE,
         "prompt_id": expected_prompt_id(),
         "safety_controls": EXPECTED_SAFETY_CONTROLS,
         "created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
@@ -138,19 +133,15 @@ def main(argv: list[str] | None = None) -> None:
     parser.add_argument("--output", default=os.environ.get("OUTPUT", "/out/submission.json"))
     parser.add_argument("--api-base", default=os.environ.get("API_BASE"))
     parser.add_argument("--api-key", default=os.environ.get("API_KEY"))
-    parser.add_argument("--model-kwargs", default=os.environ.get("MODEL_KWARGS"))
     parser.add_argument("--preflight", action="store_true")
     parser.add_argument("--fake", action="store_true", default=bool(os.environ.get("FAKE")))
     args = parser.parse_args(argv)
     if not args.model:
         parser.error("--model (or MODEL_NAME) is required")
-    kwargs = json.loads(args.model_kwargs) if args.model_kwargs else None
-    if kwargs is not None and not isinstance(kwargs, dict):
-        parser.error("--model-kwargs must be a JSON object")
     if args.preflight:
         from benchmark.preflight import format_report, run_checks
         try:
-            validate_rankable_settings(model_kwargs=kwargs)
+            validate_rankable_settings()
         except ValueError as error:
             parser.error(str(error))
         try:
@@ -159,11 +150,10 @@ def main(argv: list[str] | None = None) -> None:
         except (OSError, ValueError, RuntimeError) as error:
             parser.error(str(error))
         checks = run_checks(args.model, repo_dir=args.repo_dir, api_base=args.api_base,
-                            api_key=args.api_key, model_kwargs=None)
+                            api_key=args.api_key)
         raise SystemExit(0 if format_report(checks) else 1)
     result = run(model=args.model, repo_dir=args.repo_dir, output=args.output,
-                 fake=args.fake, api_base=args.api_base, api_key=args.api_key,
-                 model_kwargs=kwargs)
+                 fake=args.fake, api_base=args.api_base, api_key=args.api_key)
     print(f"Top50 {result['status']} ({len(result['episodes'])}/50 episodes) → {args.output}")
 
 
diff --git a/benchmark/submission.schema.json b/benchmark/submission.schema.json
index 0a403ca..02a9107 100644
--- a/benchmark/submission.schema.json
+++ b/benchmark/submission.schema.json
@@ -2,130 +2,27 @@
   "$schema": "http://json-schema.org/draft-07/schema#",
   "$id": "submission.schema.json",
   "title": "BenchmarkSubmission",
-  "description": "A self-describing, rankable whole-repository benchmark submission. The bounded submit ledger lets the backend rank and re-verify every claimed counterexample.",
   "type": "object",
-  "required": ["model", "library_commit", "bugs_found", "total_tokens_k", "rules_tested", "results", "submit_limit", "submit_log"],
+  "additionalProperties": false,
+  "required": ["model", "library_commit", "runner_version",
+    "pred_version", "prompt_id", "status", "rankable", "safety_controls", "shortlist",
+    "triage", "episodes", "inference_parameters"],
   "properties": {
-    "model": {
-      "type": "string",
-      "description": "LiteLLM model name (e.g. 'anthropic/claude-sonnet-4-6')"
-    },
-    "library_commit": {
-      "type": "string",
-      "description": "Full git commit hash of the problem-reductions library the run targeted"
-    },
-    "bugs_found": {
-      "type": "integer",
-      "minimum": 0,
-      "description": "Self-reported distinct rules with a confirmed bug. ADVISORY ONLY — the backend recomputes this from pred re-verification and never trusts this number."
-    },
-    "total_tokens_k": {
-      "type": "number",
-      "minimum": 0,
-      "description": "Total tokens used (in thousands); the sum of usage_totals / 1000."
-    },
-    "usage_totals": {
-      "type": "object",
-      "description": "Aggregate token usage split into the four provider-reported buckets — the reproducible primitive behind total_tokens_k.",
-      "properties": {
-        "input": {"type": "integer", "minimum": 0, "description": "Uncached prompt tokens"},
-        "output": {"type": "integer", "minimum": 0, "description": "Completion tokens (includes reasoning/thinking tokens)"},
-        "cache_read": {"type": "integer", "minimum": 0, "description": "Prompt tokens served from cache"},
-        "cache_write": {"type": "integer", "minimum": 0, "description": "Prompt tokens written to cache"}
-      }
-    },
-    "rules_tested": {
-      "type": "integer",
-      "minimum": 0,
-      "description": "Number of distinct rules for which the agent submitted a structured certificate"
-    },
-    "results": {
-      "type": "array",
-      "description": "One ledger-derived result for each submitted rule",
-      "items": {
-        "type": "object",
-        "required": ["rule", "result", "tokens_k"],
-        "if": {"properties": {"result": {"const": "bug_found"}}},
-        "then": {"required": ["rule", "result", "tokens_k", "certificate"]},
-        "properties": {
-          "rule": {"type": "string"},
-          "result": {
-            "type": "string",
-            "description": "Outcome for this rule. Canonical: bug_found | no_certificate | rejected | no_bug | error. error rows may carry a free-form 'error: ' string."
-          },
-          "tokens_k": {"type": "number", "minimum": 0},
-          "steps": {"type": "integer", "minimum": 0},
-          "certificate": {
-            "type": "object",
-            "description": "Counterexample certificate (present when result=bug_found or rejected). The backend re-derives the bundle from source via pred and ignores any self-reported claim; only rule, source, and the target type are needed.",
-            "required": ["rule", "source"],
-            "properties": {
-              "rule": {"type": "string"},
-              "source": {"type": "object", "description": "Source problem instance (pred create JSON)"},
-              "bundle": {"type": ["object", "string"], "description": "Reduction bundle (preferred) or target-type narration; the verifier re-derives the bundle"},
-              "target_type": {"type": "string", "description": "Target problem type (alternative to bundle.target.type)"},
-              "target_config": {"type": "string", "description": "Optional witness: a target solution that mis-extracts"},
-              "violation": {"type": "string", "description": "Optional free-form label; the backend derives the authoritative one (e.g. optimum_not_preserved)"},
-              "note": {"type": "string"}
-            }
-          },
-          "trajectory": {
-            "type": "array",
-            "description": "The agent's own message history for this rule. Retained for historical data; current submissions use submit_log for provenance.",
-            "items": {
-              "type": "object",
-              "properties": {
-                "role": {"type": "string"},
-                "content": {"type": "string"}
-              }
-            }
-          },
-          "verify_details": {"type": "object"},
-          "reject_reason": {"type": "string"},
-          "session_error": {
-            "type": "string",
-            "description": "A fatal agent/CLI error after submit attempts; the verified ledger row was salvaged rather than discarded."
-          }
-        }
-      }
-    },
-    "efficiency_bugs_per_ktok": {"type": "number", "minimum": 0},
-    "submitted_by": {"type": ["string", "null"], "description": "Submitter identity (HF username / contact), filled at submit time"},
-    "runner_version": {"type": "string", "description": "Version tag of the runner image/script that produced this"},
-    "pred_version": {"type": "string", "description": "Version of the pred binary used (must match the pinned tag; verified server-side)"},
-    "agent_mode": {"type": ["string", "null"], "description": "Agent harness mode; current runners emit 'whole-repo'."},
-    "submit_limit": {"type": "integer", "minimum": 0, "description": "Run-wide counterexample submission limit; usage is derived from submit_log length."},
-    "submit_log": {
-      "type": "array",
-      "description": "Ordered ledger of every submit command accepted by the evaluation service; rejected and malformed attempts are retained and consume budget.",
-      "items": {
-        "type": "object",
-        "required": ["attempt", "accepted", "reason"],
-        "properties": {
-          "attempt": {"type": "integer", "minimum": 1},
-          "accepted": {"type": "boolean"},
-          "rule": {"type": ["string", "null"]},
-          "reason": {"type": "string"},
-          "certificate": {"type": ["object", "null"]},
-          "verify_details": {"type": "object"}
-        }
-      }
-    },
-    "run_error": {"type": "string", "description": "Present only when the whole-repo session died on a fatal error (quota/auth/network). The results are the partial salvage, not a clean completion — distinguishes a crash from a genuine 0-bug run."},
-    "created_at": {"type": "string", "description": "ISO-8601 timestamp the run finished"},
-    "notes": {"type": "string"},
-    "placeholder": {"type": "boolean", "description": "True for demo/placeholder rows, never ranked"},
-    "test": {"type": "boolean", "description": "When true, the backend scores and stores the submission privately but EXCLUDES it from the public leaderboard (for end-to-end tests). Set via `prb submit --test`."},
-    "trajectory": {
-      "type": "array",
-      "description": "Envelope-level session trajectory for a whole-repo run, stored once instead of on every row. Current submissions use submit_log for provenance.",
-      "items": {
-        "type": "object",
-        "properties": {
-          "role": {"type": "string"},
-          "content": {"type": "string"}
-        }
-      }
-    }
+    "model": {"type": "string", "minLength": 1},
+    "library_commit": {"type": "string", "minLength": 1},
+    "runner_version": {"const": "0.11.0"},
+    "pred_version": {"type": "string", "minLength": 1},
+    "prompt_id": {"type": "string", "minLength": 64, "maxLength": 64},
+    "status": {"enum": ["completed", "run_error"]},
+    "safety_controls": {"const": {"model_timeout_seconds": 300, "model_retries": 2}},
+    "rankable": {"type": "boolean"},
+    "shortlist": {"type": "array", "minItems": 50, "maxItems": 50},
+    "triage": {"type": "object"},
+    "episodes": {"type": "array", "minItems": 50, "maxItems": 50},
+    "inference_parameters": {"const": {"max_tokens": 8192, "timeout": 300, "num_retries": 2}},
+    "created_at": {"type": "string"},
+    "submitted_by": {"type": "string"},
+    "test": {"type": "boolean"},
+    "run_error": {"type": "string"}
   }
 }
diff --git a/benchmark/submit.py b/benchmark/submit.py
index 9f5983f..6aae848 100644
--- a/benchmark/submit.py
+++ b/benchmark/submit.py
@@ -12,7 +12,7 @@
     # → prints the submission id returned by the endpoint
 
 The client validates the file locally FIRST (valid JSON, required envelope fields, and —
-mirroring submission.schema.json — a valid bounded ledger for current submissions) so a
+mirroring submission.schema.json — valid bounded ledgers) so a
 malformed run fails fast, before it hits the endpoint or burns quota.
 The endpoint re-checks and the backend re-verifies with pred regardless; this is just a
 courtesy gate. Use ``--dry-run`` to validate without sending.
@@ -31,11 +31,6 @@
 import urllib.request
 from pathlib import Path
 
-from benchmark.submit_ledger import submit_ledger_error
-
-REQUIRED_ENVELOPE = ("model", "library_commit", "bugs_found", "total_tokens_k",
-                     "rules_tested", "results")
-
 
 def load_submission(path: Path) -> dict:
     """Read + JSON-parse a submission file (raises on missing/invalid)."""
@@ -43,85 +38,9 @@ def load_submission(path: Path) -> dict:
 
 
 def validate_submission(sub: dict) -> list[str]:
-    """Client-side courtesy check. Returns a list of problems ([] == ok).
-
-    New runs prove provenance through the bounded submit ledger. Legacy runs must carry a
-    certificate plus a trajectory on the row or envelope.
-    """
-    from benchmark.top50_contract import is_top50_submission, validate_top50_submission
-    if is_top50_submission(sub):
-        return validate_top50_submission(sub)
-
-    problems: list[str] = []
-    if not isinstance(sub, dict):
-        return ["submission is not a JSON object"]
-    for field in REQUIRED_ENVELOPE:
-        if field not in sub:
-            problems.append(f"missing required field: {field}")
-
-    if "model" in sub and (not isinstance(sub["model"], str) or not sub["model"].strip()):
-        problems.append("model must be a non-empty string")
-    if "library_commit" in sub and not isinstance(sub["library_commit"], str):
-        problems.append("library_commit must be a string")
-    bugs_found = sub.get("bugs_found")
-    if ("bugs_found" in sub
-            and (not isinstance(bugs_found, int) or isinstance(bugs_found, bool)
-                 or bugs_found < 0)):
-        problems.append("bugs_found must be a non-negative integer")
-    rules_tested = sub.get("rules_tested")
-    if ("rules_tested" in sub
-            and (not isinstance(rules_tested, int) or isinstance(rules_tested, bool)
-                 or rules_tested < 0)):
-        problems.append("rules_tested must be a non-negative integer")
-    tokens_k = sub.get("total_tokens_k")
-    if ("total_tokens_k" in sub
-            and (not isinstance(tokens_k, (int, float)) or isinstance(tokens_k, bool)
-                 or tokens_k < 0)):
-        problems.append("total_tokens_k must be a non-negative number")
-
-    usage_totals = sub.get("usage_totals")
-    if usage_totals is not None:
-        if not isinstance(usage_totals, dict):
-            problems.append("usage_totals must be an object")
-        else:
-            for bucket in ("input", "output", "cache_read", "cache_write"):
-                value = usage_totals.get(bucket, 0)
-                if not isinstance(value, int) or isinstance(value, bool) or value < 0:
-                    problems.append(
-                        f"usage_totals.{bucket} must be a non-negative integer")
-
-    results = sub.get("results")
-    if not isinstance(results, list):
-        problems.append("results must be a list")
-        return problems
-    envelope_traj = bool(sub.get("trajectory"))
-    for i, row in enumerate(results):
-        if not isinstance(row, dict):
-            problems.append(f"results[{i}] is not an object")
-            continue
-        if not isinstance(row.get("rule"), str) or not row.get("rule", "").strip():
-            problems.append(f"results[{i}].rule must be a non-empty string")
-        if not isinstance(row.get("result"), str):
-            problems.append(f"results[{i}].result must be a string")
-        row_tokens = row.get("tokens_k")
-        if (not isinstance(row_tokens, (int, float)) or isinstance(row_tokens, bool)
-                or row_tokens < 0):
-            problems.append(f"results[{i}].tokens_k must be a non-negative number")
-        certificate = row.get("certificate")
-        if certificate is not None and not isinstance(certificate, dict):
-            problems.append(f"results[{i}].certificate must be an object")
-        if row.get("result") == "bug_found":
-            rule = row.get("rule", "?")
-            if not row.get("certificate"):
-                problems.append(f"results[{i}] ({rule}): bug_found row has no certificate")
-            if ("submit_log" not in sub
-                    and not row.get("trajectory") and not envelope_traj):
-                problems.append(f"results[{i}] ({rule}): bug_found row has no trajectory "
-                                "(required as provenance — on the row or the envelope)")
-
-    if ledger_problem := submit_ledger_error(sub):
-        problems.append(ledger_problem)
-    return problems
+    """Run the current benchmark artifact validator as a local courtesy check."""
+    from benchmark.top50_contract import validate_top50_submission
+    return validate_top50_submission(sub)
 
 
 def _post(url: str, payload: bytes, auth_headers: dict[str, str],
@@ -168,7 +87,8 @@ def submit(path: Path, url: str | None, *, access_token: str | None = None,
     if mark_test:
         sub["test"] = True
 
-    bugs = sum(1 for r in sub.get("results", []) if r.get("result") == "bug_found")
+    bugs = sum(1 for episode in sub.get("episodes", [])
+               if episode.get("status") == "bug_found")
     if dry_run:
         return {"status": "dry-run (not sent)", "model": sub.get("model"),
                 "claimed_bugs": bugs, "bytes": len(path.read_bytes())}
diff --git a/benchmark/submit_ledger.py b/benchmark/submit_ledger.py
deleted file mode 100644
index e43799d..0000000
--- a/benchmark/submit_ledger.py
+++ /dev/null
@@ -1,52 +0,0 @@
-"""Shared validation and lookup helpers for the bounded submit ledger."""
-from __future__ import annotations
-
-import json
-
-def has_submit_ledger(submission: dict) -> bool:
-    return "submit_limit" in submission or "submit_log" in submission
-
-
-def submit_ledger_error(submission: dict) -> str | None:
-    """Return one structural error, or ``None`` for a valid/legacy submission."""
-    if not has_submit_ledger(submission):
-        return None
-
-    limit, log = submission.get("submit_limit"), submission.get("submit_log")
-    if not isinstance(limit, int) or isinstance(limit, bool) or limit < 0:
-        return "submit_limit must be a non-negative integer"
-    if not isinstance(log, list):
-        return "submit_log must be a list"
-    if len(log) > limit:
-        return "submit_log exceeds submit_limit"
-
-    for expected, attempt in enumerate(log, 1):
-        if not isinstance(attempt, dict) or attempt.get("attempt") != expected:
-            return "submit_log attempt numbers must be contiguous from 1"
-        if not isinstance(attempt.get("accepted"), bool):
-            return f"submit_log attempt {expected} has no boolean accepted field"
-        if not isinstance(attempt.get("reason"), str):
-            return f"submit_log attempt {expected} has no reason"
-        if attempt["accepted"]:
-            cert, rule = attempt.get("certificate"), attempt.get("rule")
-            if (not isinstance(cert, dict) or not isinstance(rule, str)
-                    or cert.get("rule") != rule):
-                return f"accepted submit_log attempt {expected} has inconsistent rule/certificate"
-    return None
-
-
-def accepted_certificate_index(submission: dict) -> set[tuple[str, str]]:
-    """Canonical (rule, certificate) keys for accepted attempts in a valid ledger."""
-    return {
-        (attempt["rule"], _canonical(attempt["certificate"]))
-        for attempt in submission.get("submit_log", [])
-        if attempt.get("accepted")
-    }
-
-
-def certificate_key(rule: str, certificate: dict) -> tuple[str, str]:
-    return rule, _canonical(certificate)
-
-
-def _canonical(certificate: dict) -> str:
-    return json.dumps(certificate, sort_keys=True, separators=(",", ":"))
diff --git a/benchmark/submit_session.py b/benchmark/submit_session.py
index 47659db..5f1b1e6 100644
--- a/benchmark/submit_session.py
+++ b/benchmark/submit_session.py
@@ -1,9 +1,8 @@
 """Evaluation-owned counterexample submission budget.
 
 The agent-facing ``submit`` process is deliberately only a thin client. Requests use an
-atomic file spool inside the agent scratch workspace so sandboxed Codex, Claude, mini-swe,
-and container backends share one transport. The budget, verification, and accepted-certificate
-ledger remain authoritative in runner memory; editing spool files
+atomic file spool inside the agent scratch workspace. The budget, verification, and
+accepted-certificate ledger remain authoritative in runner memory; editing spool files
 cannot reset the budget or forge a scored result.
 """
 from __future__ import annotations
@@ -27,18 +26,14 @@
 
 
 class SubmissionSession:
-    """Own one submit budget and its append-only in-memory ledger.
+    """Own one rule episode's submit budget and append-only in-memory ledger."""
 
-    Historical callers may still use a run-wide limit. Evidence-budget episodes pass an
-    ``expected_rule`` and create one independent two-attempt session per rule.
-    """
-
-    def __init__(self, limit: int = 100, verifier: Callable[[dict], Verdict] = verify,
-                 expected_rule: str | None = None):
+    def __init__(self, expected_rule: str, limit: int = 2,
+                 verifier: Callable[[dict], Verdict] = verify):
         if limit < 0:
             raise ValueError("submit limit must be >= 0")
-        if expected_rule is not None and not expected_rule:
-            raise ValueError("expected_rule must be non-empty when provided")
+        if not expected_rule:
+            raise ValueError("expected_rule must be non-empty")
         self.limit = limit
         self._verifier = verifier
         self.expected_rule = expected_rule
@@ -49,16 +44,13 @@ def __init__(self, limit: int = 100, verifier: Callable[[dict], Verdict] = verif
         self._thread: threading.Thread | None = None
         self._tmpdir: Path | None = None
         self._workdir: Path | None = None
-        self._workdir_fd: int | None = None
         self._inbox_fd: int | None = None
         self._processing_fd: int | None = None
         self._outbox_fd: int | None = None
-        self._artifact_fd: int | None = None
         self._old_channel_env: str | None = None
         self._old_artifact_env: str | None = None
         self._old_path: str | None = None
         self._stopping = threading.Event()
-        self._status_checks = 0
 
     @property
     def used(self) -> int:
@@ -83,17 +75,11 @@ def attempts(self) -> list[dict]:
 
     @property
     def workdir(self) -> Path:
-        """Writable scratch root shared with sandboxed headless agents."""
+        """Writable scratch root shared with the sandboxed benchmark agent."""
         if self._workdir is None:
             raise RuntimeError("submission session is not active")
         return self._workdir
 
-    @property
-    def reachable(self) -> bool:
-        """Whether any status or submit request reached the authoritative service."""
-        with self._lock:
-            return self._status_checks > 0 or bool(self._attempts)
-
     def __enter__(self) -> "SubmissionSession":
         self._tmpdir = Path(tempfile.mkdtemp(prefix="prb-agent-", dir="/tmp"))
         self._workdir = self._tmpdir / "work"
@@ -106,11 +92,9 @@ def __enter__(self) -> "SubmissionSession":
         for directory in (bin_dir, inbox_dir, processing_dir, outbox_dir, artifact_dir):
             directory.mkdir(parents=True, exist_ok=True)
         directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
-        self._workdir_fd = os.open(self._workdir, directory_flags)
         self._inbox_fd = os.open(inbox_dir, directory_flags)
         self._processing_fd = os.open(processing_dir, directory_flags)
         self._outbox_fd = os.open(outbox_dir, directory_flags)
-        self._artifact_fd = os.open(artifact_dir, directory_flags)
 
         shim = bin_dir / "submit"
         package_root = Path(__file__).resolve().parent.parent
@@ -149,47 +133,12 @@ def __exit__(self, exc_type, exc, tb) -> None:
         self._stopping.set()
         if self._thread is not None:
             self._thread.join()
-        for fd in (self._workdir_fd, self._inbox_fd, self._processing_fd,
-                   self._outbox_fd, self._artifact_fd):
+        for fd in (self._inbox_fd, self._processing_fd, self._outbox_fd):
             if fd is not None:
                 os.close(fd)
         if self._tmpdir is not None:
             shutil.rmtree(self._tmpdir, ignore_errors=True)
 
-    def preserve_artifacts(self, destination: str | Path) -> list[Path]:
-        """Copy small certificate artifacts out of the disposable agent workspace."""
-        destination = Path(destination)
-        copied: list[Path] = []
-        sources = ((self._artifact_fd, "artifacts", False),
-                   (self._workdir_fd, "workspace", True))
-        flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
-        for directory_fd, label, certificate_names_only in sources:
-            if directory_fd is None:
-                continue
-            for name in os.listdir(directory_fd):
-                if certificate_names_only and ("cert" not in name.lower()
-                                               or not name.lower().endswith(".json")):
-                    continue
-                try:
-                    source_fd = os.open(name, flags, dir_fd=directory_fd)
-                except OSError:
-                    continue
-                try:
-                    info = os.fstat(source_fd)
-                    if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_REQUEST_BYTES:
-                        continue
-                    with os.fdopen(source_fd, "rb", closefd=False) as source_file:
-                        payload = source_file.read(MAX_REQUEST_BYTES + 1)
-                finally:
-                    os.close(source_fd)
-                if len(payload) > MAX_REQUEST_BYTES:
-                    continue
-                target = destination / label / Path(name).name
-                target.parent.mkdir(parents=True, exist_ok=True)
-                target.write_bytes(payload)
-                copied.append(target)
-        return copied
-
     def prepare_agent_access(self, uid: int, gid: int) -> None:
         """Give a non-root agent access to its request queues and artifact directory."""
         if self._tmpdir is None:
@@ -205,26 +154,6 @@ def prepare_agent_access(self, uid: int, gid: int) -> None:
             os.chown(path, uid, gid)
             path.chmod(0o700)
 
-    def result_rows(self) -> list[dict]:
-        """Collapse attempts to one authoritative result per named rule.
-
-        An accepted attempt wins over rejected attempts for the same rule.  Parse failures
-        remain in ``submit_log`` for auditing but cannot form schema-valid result rows.
-        """
-        with self._lock:
-            by_rule: dict[str, dict] = {}
-            for attempt in self._attempts:
-                rule = attempt.get("rule")
-                cert = attempt.get("certificate")
-                if (not isinstance(rule, str) or not isinstance(cert, dict)
-                        or not isinstance(cert.get("rule"), str)
-                        or not isinstance(cert.get("source"), dict)):
-                    continue
-                old = by_rule.get(rule)
-                if old is None or (attempt.get("accepted") and not old.get("accepted")):
-                    by_rule[rule] = attempt
-            return [_attempt_to_row(attempt) for attempt in by_rule.values()]
-
     def _serve(self) -> None:
         assert self._inbox_fd is not None
         assert self._processing_fd is not None
@@ -330,8 +259,6 @@ def _handle_malformed_idempotent(
 
     def _handle(self, request: dict, *, request_id: str | None = None) -> dict:
         if request.get("op") == "status":
-            with self._lock:
-                self._status_checks += 1
             return {"status": "ok", "used": self.used, "limit": self.limit,
                     "remaining": self.remaining, "closed": self.closed,
                     "expected_rule": self.expected_rule}
@@ -367,8 +294,7 @@ def _handle(self, request: dict, *, request_id: str | None = None) -> dict:
                 else:
                     if not isinstance(cert, dict):
                         reason = "certificate must be a JSON object"
-                    elif (self.expected_rule is not None
-                          and cert.get("rule") != self.expected_rule):
+                    elif cert.get("rule") != self.expected_rule:
                         reason = (f"certificate rule {cert.get('rule')!r} does not match "
                                   f"this episode's rule {self.expected_rule!r}")
                     else:
@@ -416,24 +342,8 @@ def _append_attempt(self, cert, accepted: bool, reason: str, details=None,
         if details is not None:
             record["verify_details"] = details
         self._attempts.append(record)
-        if accepted and self.expected_rule is not None:
+        if accepted:
             self._closed = True
         return {**record, "used": attempt_no, "limit": self.limit,
                 "remaining": 0 if self._closed else self.limit - attempt_no,
                 "closed": self._closed}
-
-
-def _attempt_to_row(attempt: dict) -> dict:
-    row = {
-        "rule": attempt["rule"],
-        "result": "bug_found" if attempt.get("accepted") else "rejected",
-        "tokens_k": 0.0,
-        "certificate": copy.deepcopy(attempt["certificate"]),
-        "submit_attempt": attempt["attempt"],
-    }
-    if attempt.get("accepted"):
-        if attempt.get("verify_details") is not None:
-            row["verify_details"] = copy.deepcopy(attempt["verify_details"])
-    else:
-        row["reject_reason"] = attempt.get("reason", "rejected")
-    return row
diff --git a/benchmark/tests/conftest.py b/benchmark/tests/conftest.py
index 8ba6f44..243821b 100644
--- a/benchmark/tests/conftest.py
+++ b/benchmark/tests/conftest.py
@@ -1,8 +1,6 @@
 """Shared pytest fixtures for benchmark tests."""
 
-import json
 import pytest
-from pathlib import Path
 
 
 # ── Minimal valid certificate components ──────────────────────────────────────
diff --git a/benchmark/tests/test_backend_score.py b/benchmark/tests/test_backend_score.py
index 54dd550..290e9fc 100644
--- a/benchmark/tests/test_backend_score.py
+++ b/benchmark/tests/test_backend_score.py
@@ -1,330 +1,152 @@
-"""
-Tests for benchmark/backend_score.py — the local scoring queue.
-
-Mechanics tests use certificate-free submissions (no pred needed). One integration test
-wraps the real valid_bug fixture to prove end-to-end scoring writes a ranked entry.
-"""
+"""Tests for the private scoring queue and aggregate leaderboard projection."""
 import json
 from pathlib import Path
 
 import pytest
 
 from benchmark import backend_score as bs
-
-FIXTURES = Path(__file__).parent / "fixtures"
-
-
-def _traj(cert: dict) -> list[dict]:
-    """A minimal trajectory reproducing ``cert`` — satisfies the provenance gate."""
-    return [{"role": "assistant",
-             "content": "CERTIFICATE_START\n" + json.dumps(cert) + "\nCERTIFICATE_END"}]
-
-
-def _write_submission(subs_dir: Path, name: str, results, **over) -> Path:
-    results = [{"tokens_k": 0, **row} for row in results]
-    sub = {
-        "model": over.pop("model", "anthropic/test"),
-        "library_commit": "deadbeef",
-        "bugs_found": over.pop("bugs_found", 0),
-        "total_tokens_k": 50.0,
-        "rules_tested": len(results),
-        "results": results,
+from benchmark.tests.test_top50_submission import _artifact
+
+
+def _write_submission(directory: Path, name: str, *, model="anthropic/test",
+                      test=False, created_at=None) -> tuple[Path, dict]:
+    artifact = _artifact(accepted_positions=())
+    artifact["model"] = model
+    if test:
+        artifact["test"] = True
+    if created_at:
+        artifact["created_at"] = created_at
+    path = directory / name
+    path.write_text(json.dumps(artifact), encoding="utf-8")
+    return path, artifact
+
+
+def _repo(root: Path, artifact: dict) -> Path:
+    repo = root / "repo"
+    rules = repo / "src" / "rules"
+    rules.mkdir(parents=True)
+    for entry in artifact["shortlist"]:
+        (rules / f"{entry['rule']}.rs").write_text("// fixture")
+    return repo
+
+
+def _scored(model="m", bugs=0, **extra) -> dict:
+    return {
+        "model": model, "library_commit": "c", "runner_version": "0.11.0",
+        "pred_version": "0.6.0", "rankable": True, "rankability_errors": [],
+        "verified_bugs": bugs, "bugs_found": bugs, "bugs_at_10": bugs,
+        "bugs_at_25": bugs, "bugs_at_50": bugs, "first_attempt_accepts": bugs,
+        "second_attempt_accepts": 0, "pred_calls_per_bug": None, "cap_hits": {},
+        "usage_totals": {"input": 0, "output": 0, "cache_read": 0, "cache_write": 0},
+        "total_tokens_k": 0, "artifact_sha256": "a" * 64, "verifier_report": [],
+        **extra,
     }
-    sub.update(over)
-    p = subs_dir / name
-    p.write_text(json.dumps(sub), encoding="utf-8")
-    return p
 
 
 class TestProcessLocal:
-    def test_no_cert_submission_scores_zero(self, tmp_path):
-        subs, results = tmp_path / "subs", tmp_path / "results"
-        subs.mkdir()
-        _write_submission(subs, "a.json",
-                          [{"rule": "r1", "result": "no_certificate", "tokens_k": 10.0}])
-        summary = bs.process_local(str(subs), str(results))
-        assert summary[0]["status"] == "FINISHED"
-        assert summary[0]["bugs_found"] == 0
-        assert (results / "a.json").exists()
-        assert (subs / "a.status.json").exists()
-
-    def test_status_transitions_to_finished(self, tmp_path):
-        subs, results = tmp_path / "subs", tmp_path / "results"
-        subs.mkdir()
-        _write_submission(subs, "a.json",
-                          [{"rule": "r1", "result": "no_certificate", "tokens_k": 10.0}])
-        bs.process_local(str(subs), str(results))
-        status = json.loads((subs / "a.status.json").read_text())
-        assert status["status"] == "FINISHED"
-
-    def test_idempotent_skips_finished(self, tmp_path):
+    def test_valid_submission_scores_and_is_idempotent(self, tmp_path):
         subs, results = tmp_path / "subs", tmp_path / "results"
         subs.mkdir()
-        _write_submission(subs, "a.json",
-                          [{"rule": "r1", "result": "no_certificate", "tokens_k": 10.0}])
-        bs.process_local(str(subs), str(results))
-        again = bs.process_local(str(subs), str(results))
-        assert again == []  # nothing left pending
+        _, artifact = _write_submission(subs, "a.json")
+        repo = _repo(tmp_path, artifact)
 
-    def test_leaderboard_aggregated_and_ranked(self, tmp_path, monkeypatch):
-        # Two models scored via a monkeypatched verifier: only solve_mismatch confirms.
-        from benchmark.verify import Verdict
-        import benchmark.verify_submission as vs
-        monkeypatch.setattr(vs, "verify",
-                            lambda c, r=None: Verdict(c.get("violation") == "solve_mismatch", "x"))
-        subs, results = tmp_path / "subs", tmp_path / "results"
-        subs.mkdir()
-        cert = lambda v: {"rule": "r1", "violation": v, "source": {}, "bundle": {}}
-        win_cert, lose_cert = cert("solve_mismatch"), cert("unsound_extraction")
-        _write_submission(subs, "win.json",
-                          [{"rule": "r1", "result": "bug_found", "tokens_k": 10.0,
-                            "certificate": win_cert, "trajectory": _traj(win_cert)}],
-                          model="anthropic/winner")
-        _write_submission(subs, "lose.json",
-                          [{"rule": "r1", "result": "bug_found", "tokens_k": 10.0,
-                            "certificate": lose_cert, "trajectory": _traj(lose_cert)}],
-                          model="anthropic/loser")
-        bs.process_local(str(subs), str(results))
-        board = json.loads((results / "leaderboard.json").read_text())
-        assert [e["model"] for e in board] == ["anthropic/winner", "anthropic/loser"]
-        assert board[0]["bugs_found"] == 1 and board[1]["bugs_found"] == 0
+        summary = bs.process_local(str(subs), str(results), str(repo))
+        assert summary == [{"submission": "a.json", "status": "FINISHED",
+                            "model": "anthropic/test", "bugs_found": 0}]
+        assert bs.process_local(str(subs), str(results), str(repo)) == []
+        assert json.loads((results / "leaderboard.json").read_text())[0]["bugs_found"] == 0
 
     def test_finds_nested_submissions(self, tmp_path):
-        # Real layout: submissions//.json — must be found recursively.
-        subs, results = tmp_path / "subs", tmp_path / "results"
-        nested = subs / "submissions" / "alice"
+        nested = tmp_path / "subs" / "alice"
         nested.mkdir(parents=True)
-        _write_submission(nested, "run.json",
-                          [{"rule": "r1", "result": "no_certificate", "tokens_k": 10.0}])
-        summary = bs.process_local(str(subs), str(results))
-        assert len(summary) == 1 and summary[0]["status"] == "FINISHED"
+        _, artifact = _write_submission(nested, "run.json")
+        repo = _repo(tmp_path, artifact)
+        summary = bs.process_local(str(tmp_path / "subs"), str(tmp_path / "results"), str(repo))
+        assert summary[0]["status"] == "FINISHED"
         assert (nested / "run.status.json").exists()
-        assert (results / "leaderboard.json").exists()
-
-    def test_zero_submissions_writes_empty_board(self, tmp_path):
-        subs, results = tmp_path / "subs", tmp_path / "results"
-        subs.mkdir()
-        summary = bs.process_local(str(subs), str(results))
-        assert summary == []
-        assert json.loads((results / "leaderboard.json").read_text()) == []
-
-    def test_failed_submission_marked(self, tmp_path):
-        subs, results = tmp_path / "subs", tmp_path / "results"
-        subs.mkdir()
-        (subs / "bad.json").write_text("{not valid json", encoding="utf-8")
-        summary = bs.process_local(str(subs), str(results))
-        assert summary[0]["status"] == "FAILED"
-        status = json.loads((subs / "bad.status.json").read_text())
-        assert status["status"] == "FAILED"
-        assert status["retryable"] is False
 
-    def test_malformed_certificate_is_permanent_not_retryable(self, tmp_path):
-        subs, results = tmp_path / "subs", tmp_path / "results"
+    def test_invalid_json_is_permanent(self, tmp_path):
+        subs = tmp_path / "subs"
         subs.mkdir()
-        _write_submission(
-            subs, "bad-cert.json",
-            [{"rule": "R", "result": "bug_found", "certificate": ["not", "an", "object"],
-              "trajectory": [{"role": "assistant", "content": "x"}]}])
-        summary = bs.process_local(str(subs), str(results))
+        (subs / "bad.json").write_text("{not json")
+        summary = bs.process_local(str(subs), str(tmp_path / "results"), str(tmp_path / "repo"))
         assert summary[0]["status"] == "FAILED"
         assert summary[0]["retryable"] is False
-        assert "certificate must be an object" in summary[0]["error"]
-
-    def test_main_exits_zero_on_permanent_input_failure(self, tmp_path, monkeypatch):
-        # A malformed object is quarantined by the workflow and must not jam valid work.
-        subs, results = tmp_path / "subs", tmp_path / "results"
-        subs.mkdir()
-        (subs / "bad.json").write_text("{not valid json", encoding="utf-8")
-        monkeypatch.setattr("sys.argv", ["backend_score", "--local", str(subs), str(results)])
-        bs.main()  # permanent FAILED → exit 0 so the workflow can move it to failed/
 
-    def test_main_exits_nonzero_on_retryable_failure(self, tmp_path, monkeypatch):
-        subs, results = tmp_path / "subs", tmp_path / "results"
+    def test_official_gate_rejects_wrong_commit(self, tmp_path):
+        subs = tmp_path / "subs"
         subs.mkdir()
-        _write_submission(subs, "a.json", [{"rule": "R", "result": "no_bug",
-                                             "tokens_k": 0}])
-        monkeypatch.setattr(bs, "score_submission",
-                            lambda *a, **k: (_ for _ in ()).throw(RuntimeError("pred crashed")))
-        monkeypatch.setattr("sys.argv", ["backend_score", "--local", str(subs), str(results)])
-        with pytest.raises(SystemExit) as ei:
-            bs.main()
-        assert ei.value.code == 1
-        status = json.loads((subs / "a.status.json").read_text())
-        assert status["retryable"] is True
-
-    def test_main_exits_zero_when_all_scored(self, tmp_path, monkeypatch):
-        # A clean batch (no FAILED) exits 0 so the workflow proceeds to archive + publish.
-        subs, results = tmp_path / "subs", tmp_path / "results"
-        subs.mkdir()
-        _write_submission(subs, "a.json", [{"rule": "R", "result": "no_bug"}])
-        monkeypatch.setattr("sys.argv", ["backend_score", "--local", str(subs), str(results)])
-        bs.main()  # returns normally (no SystemExit) → exit 0
-
-    def test_official_gate_rejects_missing_ledger_wrong_commit_and_partial_run(self, tmp_path):
-        subs, results = tmp_path / "subs", tmp_path / "results"
-        subs.mkdir()
-        _write_submission(
-            subs, "bad-round.json", [{"rule": "R", "result": "no_bug", "tokens_k": 0}],
-            library_commit="wrong", run_error="quota exhausted")
+        _, artifact = _write_submission(subs, "wrong.json")
+        repo = _repo(tmp_path, artifact)
         summary = bs.process_local(
-            str(subs), str(results), official=True, expected_commit="expected")
-        assert summary[0]["status"] == "FAILED"
+            str(subs), str(tmp_path / "results"), str(repo),
+            official=True, expected_commit="expected")
         assert summary[0]["retryable"] is False
-        error = summary[0]["error"]
-        assert "require submit_limit and submit_log" in error
-        assert "library_commit" in error
-        assert "partial run" in error
+        assert "library_commit" in summary[0]["error"]
 
-    def test_official_gate_accepts_current_clean_submission(self, tmp_path):
-        subs, results = tmp_path / "subs", tmp_path / "results"
+    def test_official_gate_accepts_current_protocol(self, tmp_path):
+        subs = tmp_path / "subs"
         subs.mkdir()
-        _write_submission(
-            subs, "ok.json", [{"rule": "R", "result": "no_bug", "tokens_k": 0}],
-            library_commit="expected", submit_limit=1,
-            submit_log=[])
+        _, artifact = _write_submission(subs, "ok.json")
+        repo = _repo(tmp_path, artifact)
         summary = bs.process_local(
-            str(subs), str(results), official=True, expected_commit="expected")
+            str(subs), str(tmp_path / "results"), str(repo),
+            official=True, expected_commit=artifact["library_commit"])
         assert summary[0]["status"] == "FINISHED"
 
-    def test_official_test_submission_may_preserve_run_error(self, tmp_path):
-        subs, results = tmp_path / "subs", tmp_path / "results"
+    def test_retryable_scorer_failure_exits_nonzero(self, tmp_path, monkeypatch):
+        subs = tmp_path / "subs"
         subs.mkdir()
-        _write_submission(
-            subs, "test.json", [{"rule": "R", "result": "no_bug", "tokens_k": 0}],
-            library_commit="expected", submit_limit=1,
-            submit_log=[], test=True, run_error="intentional smoke failure")
-        summary = bs.process_local(
-            str(subs), str(results), official=True, expected_commit="expected")
-        assert summary[0]["status"] == "FINISHED"
-        scored = json.loads((results / "test.json").read_text())
-        assert scored["run_error"] == "intentional smoke failure"
+        _, artifact = _write_submission(subs, "a.json")
+        repo = _repo(tmp_path, artifact)
+        monkeypatch.setattr(bs, "score_top50_submission",
+                            lambda *a, **k: (_ for _ in ()).throw(RuntimeError("pred crashed")))
+        monkeypatch.setattr("sys.argv", ["backend_score", "--local", str(subs),
+                                        str(tmp_path / "results"), "--repo-dir", str(repo)])
+        with pytest.raises(SystemExit) as error:
+            bs.main()
+        assert error.value.code == 1
 
-    def test_test_submission_excluded_from_board(self, tmp_path):
-        # A submission marked test=true is scored + stored, but kept off the public board.
-        subs, results = tmp_path / "subs", tmp_path / "results"
+    def test_test_submission_is_not_published(self, tmp_path):
+        subs = tmp_path / "subs"
         subs.mkdir()
-        _write_submission(subs, "prod.json", [{"rule": "R", "result": "no_bug"}],
-                          model="anthropic/prod")
-        _write_submission(subs, "t.json", [{"rule": "R", "result": "no_bug"}],
-                          model="anthropic/tester", test=True)
-        summary = bs.process_local(str(subs), str(results))
-        assert {s["status"] for s in summary} == {"FINISHED"}  # both scored
-        board = json.loads((results / "leaderboard.json").read_text())
-        models = {e["model"] for e in board}
-        assert "anthropic/prod" in models
-        assert "anthropic/tester" not in models  # test entry excluded
+        _, artifact = _write_submission(subs, "test.json", test=True)
+        repo = _repo(tmp_path, artifact)
+        bs.process_local(str(subs), str(tmp_path / "results"), str(repo))
+        assert json.loads((tmp_path / "results" / "leaderboard.json").read_text()) == []
+        assert list((tmp_path / "results" / "board").glob("*.json")) == []
 
 
 class TestPerSubmissionBoard:
-    # R2 filenames are "-.json" — the slug derives from them.
     F1 = "1783317767895-dc9b2aae-c838-4c15-a5aa-2f4a25f6f82c.json"
     F2 = "1783317781060-6fb1aeb1-da04-4e1b-b6c5-f0b53c0d29a3.json"
 
-    def _score(self, tmp_path, files):
-        subs, results = tmp_path / "subs", tmp_path / "results"
-        subs.mkdir()
-        for name, model, test in files:
-            _write_submission(subs, name, [{"rule": "R", "result": "no_bug"}],
-                              model=model, test=test)
-        bs.process_local(str(subs), str(results))
-        return results
-
     def test_slug_is_deterministic_and_tagged(self):
-        scored = {"model": "anthropic/claude-sonnet-4-6"}
+        scored = _scored("anthropic/claude-sonnet-4-6")
         stem = self.F1[:-5]
-        slug = bs.board_slug(scored, stem)
-        assert slug == bs.board_slug(scored, stem)                     # deterministic
-        assert slug == "anthropic-claude-sonnet-4-6--20260706T060247--dc9b2aae"
-        full = {**scored, "results": [], "bugs_found": 0, "rules_tested": 0,
-                "total_tokens_k": 0, "efficiency_bugs_per_ktok": 0}
-        e = bs.board_entry(full, stem)
-        assert e["timestamp"] == "20260706T060247" and e["submission_id"] == "dc9b2aae"
-
-    def test_one_entry_file_per_nontest_submission(self, tmp_path):
-        results = self._score(tmp_path, [
-            (self.F1, "anthropic/claude-sonnet-4-6", False),
-            (self.F2, "openai/gpt-5", False),
-            ("1783317799999-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee.json", "anthropic/claude-sonnet-4-6", True),
-        ])
-        board_files = sorted(p.name for p in (results / "board").glob("*.json"))
-        assert board_files == [
-            "anthropic-claude-sonnet-4-6--20260706T060247--dc9b2aae.json",
-            "openai-gpt-5--20260706T060301--6fb1aeb1.json",
-        ]  # two non-test entries; the test submission produced no public file
-
-    def test_write_board_entries_is_idempotent(self, tmp_path):
-        results = self._score(tmp_path, [(self.F1, "anthropic/x", False)])
-        before = {p.name for p in (results / "board").glob("*.json")}
-        bs.write_board_entries(results, results / "board")
-        after = {p.name for p in (results / "board").glob("*.json")}
-        assert before == after and len(after) == 1
+        assert bs.board_slug(scored, stem) == (
+            "anthropic-claude-sonnet-4-6--20260706T060247--dc9b2aae")
+        entry = bs.board_entry(scored, stem)
+        assert entry["timestamp"] == "20260706T060247"
+        assert entry["submission_id"] == "dc9b2aae"
+
+    def test_build_board_keeps_best_run_per_model_without_efficiency_tiebreak(self, tmp_path):
+        entries = tmp_path / "entries"
+        entries.mkdir()
+        (entries / "m1.json").write_text(json.dumps({"model": "m", "bugs_found": 1}))
+        (entries / "m2.json").write_text(json.dumps({"model": "m", "bugs_found": 3}))
+        (entries / "n.json").write_text(json.dumps({"model": "n", "bugs_found": 2}))
+        board = bs.build_board(entries)
+        assert [(entry["model"], entry["bugs_found"]) for entry in board] == [
+            ("m", 3), ("n", 2)]
 
     def test_board_entry_preserves_submitter_and_created_at(self, tmp_path):
-        subs, results = tmp_path / "subs", tmp_path / "results"
-        subs.mkdir()
-        _write_submission(
-            subs, self.F1, [{"rule": "R", "result": "no_bug", "tokens_k": 0}],
-            submitted_by="alice", created_at="2026-07-18T12:34:56Z")
-        bs.process_local(str(subs), str(results))
-        board_file = next((results / "board").glob("*.json"))
-        entry = json.loads(board_file.read_text())
+        results = tmp_path / "results"
+        results.mkdir()
+        scored = _scored("anthropic/x", submitted_by="alice",
+                         created_at="2026-07-18T12:34:56Z")
+        (results / self.F1).write_text(json.dumps(scored))
+        bs.write_board_entries(results, results / "board")
+        entry = json.loads(next((results / "board").glob("*.json")).read_text())
         assert entry["submitted_by"] == "alice"
         assert entry["timestamp"] == "20260718T123456"
-
-    def test_build_board_dedups_best_per_model(self, tmp_path):
-        d = tmp_path / "entries"
-        d.mkdir()
-        (d / "m--t1--a.json").write_text(json.dumps(
-            {"model": "m", "bugs_found": 1, "efficiency_bugs_per_ktok": 0.1}))
-        (d / "m--t2--b.json").write_text(json.dumps(
-            {"model": "m", "bugs_found": 3, "efficiency_bugs_per_ktok": 0.2}))
-        (d / "n--t1--c.json").write_text(json.dumps(
-            {"model": "n", "bugs_found": 2, "efficiency_bugs_per_ktok": 0.1}))
-        board = bs.build_board(d)
-        assert [(e["model"], e["bugs_found"]) for e in board] == [("m", 3), ("n", 2)]
-
-    def test_main_build_board(self, tmp_path, monkeypatch):
-        d = tmp_path / "entries"
-        d.mkdir()
-        (d / "m--t--a.json").write_text(json.dumps({"model": "m", "bugs_found": 1}))
-        out = tmp_path / "results.json"
-        monkeypatch.setattr("sys.argv", ["backend_score", "--build-board", str(d), str(out)])
-        bs.main()
-        assert json.loads(out.read_text())[0]["model"] == "m"
-
-
-@pytest.mark.integration
-class TestRealFixture:
-    def test_genuine_bug_end_to_end_scores_via_real_pred(self, tmp_path):
-        # End-to-end through real pred: a genuine reduction bug is verified and ranked.
-        # The fixture is the answer key (gitignored private dir); skip when absent.
-        from benchmark.verify import PRIVATE_FIXTURES_DIR
-        path = PRIVATE_FIXTURES_DIR / "genuine_bug_weighted_mis.json"
-        if not path.exists():
-            pytest.skip(f"private accept-path fixture absent: {path}")
-        subs, results = tmp_path / "subs", tmp_path / "results"
-        subs.mkdir()
-        cert = json.loads(path.read_text(encoding="utf-8"))
-        _write_submission(subs, "real.json",
-                          [{"rule": cert.get("rule", "r"), "result": "bug_found",
-                            "tokens_k": 10.0, "certificate": cert,
-                            "trajectory": _traj(cert)}],
-                          model="anthropic/real")
-        bs.process_local(str(subs), str(results))
-        board = json.loads((results / "leaderboard.json").read_text())
-        assert board[0]["model"] == "anthropic/real"
-        assert board[0]["bugs_found"] == 1
-
-    def test_non_bug_fixture_scores_zero(self, tmp_path):
-        # The re-classified valid_bug fixture is not a bug under the round-trip contract.
-        subs, results = tmp_path / "subs", tmp_path / "results"
-        subs.mkdir()
-        cert = json.loads((FIXTURES / "valid_bug.json").read_text(encoding="utf-8"))
-        _write_submission(subs, "real.json",
-                          [{"rule": cert.get("rule", "r"), "result": "bug_found",
-                            "tokens_k": 10.0, "certificate": cert,
-                            "trajectory": _traj(cert)}],
-                          model="anthropic/real")
-        bs.process_local(str(subs), str(results))
-        board = json.loads((results / "leaderboard.json").read_text())
-        assert board[0]["bugs_found"] == 0
diff --git a/benchmark/tests/test_budget_calibration.py b/benchmark/tests/test_budget_calibration.py
index 394e6d5..8d74a23 100644
--- a/benchmark/tests/test_budget_calibration.py
+++ b/benchmark/tests/test_budget_calibration.py
@@ -99,7 +99,7 @@ def test_code_defined_limits_match_calibration_and_version_ids_are_absent():
                        ("triage", "episode", "observation", "shortlist_size",
                         "hypothesis_chars")}
     root = calibrate_budget.ROOT.parent
-    prompt = (root / "benchmark/top50_config.yaml").read_text(encoding="utf-8")
+    prompt = (root / "benchmark/agent_config.yaml").read_text(encoding="utf-8")
     assert "problem-reductions Rust library" in prompt
     assert "top50-evidence/" not in prompt
     assert "terminal-diagnostics/" not in prompt
diff --git a/benchmark/tests/test_claude_code.py b/benchmark/tests/test_claude_code.py
deleted file mode 100644
index 05bf397..0000000
--- a/benchmark/tests/test_claude_code.py
+++ /dev/null
@@ -1,231 +0,0 @@
-"""Unit tests for the claude-code backend (no network, no real claude CLI, no pred)."""
-
-import json
-from pathlib import Path
-from types import SimpleNamespace
-
-import pytest
-
-from benchmark import run_submission
-from benchmark.claude_code import _build_command, parse_stream, run_repo_claude
-from benchmark.run_submission import run
-from benchmark.submit_session import SubmissionSession
-
-CERT_TEXT = (
-    "Found it.\nCERTIFICATE_START\n"
-    '{"rule": "foo", "source": {"type": "MIS"}, "note": "mismatch"}\n'
-    "CERTIFICATE_END"
-)
-
-
-def _assistant(msg_id: str, text: str, usage: dict | None = None) -> dict:
-    return {"type": "assistant",
-            "message": {"id": msg_id, "role": "assistant",
-                        "content": [{"type": "text", "text": text}],
-                        "usage": usage}}
-
-
-def _result(num_turns: int = 3, usage: dict | None = None) -> dict:
-    return {"type": "result", "subtype": "success", "num_turns": num_turns,
-            "usage": usage or {}, "result": "done"}
-
-
-USAGE_1 = {"input_tokens": 100, "output_tokens": 50,
-           "cache_read_input_tokens": 10, "cache_creation_input_tokens": 5}
-USAGE_2 = {"input_tokens": 200, "output_tokens": 80,
-           "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0}
-
-
-# ── parse_stream ──────────────────────────────────────────────────────────────
-
-class TestParseStream:
-    def test_trajectory_usage_and_steps(self):
-        lines = [json.dumps(e) for e in [
-            {"type": "system", "subtype": "init"},
-            _assistant("msg_1", "looking", USAGE_1),
-            {"type": "user", "message": {"role": "user", "content": [
-                {"type": "tool_result", "tool_use_id": "t1", "content": "ok"}]}},
-            _assistant("msg_2", CERT_TEXT, USAGE_2),
-            _result(num_turns=2),
-        ]]
-        parsed = parse_stream(lines)
-        assert [r["role"] for r in parsed["trajectory"]] == ["assistant", "user", "assistant"]
-        assert "CERTIFICATE_START" in parsed["trajectory"][-1]["content"]
-        assert parsed["usage"].input_tokens == 300
-        assert parsed["usage"].output_tokens == 130
-        assert parsed["usage"].cache_read_tokens == 10
-        assert parsed["usage"].cache_write_tokens == 5
-        assert parsed["steps"] == 2
-
-    def test_usage_dedupes_by_message_id(self):
-        # The same API message can surface in several stream events; count it once.
-        lines = [json.dumps(_assistant("msg_1", "a", USAGE_1)),
-                 json.dumps(_assistant("msg_1", "b", USAGE_1))]
-        assert parse_stream(lines)["usage"].input_tokens == 100
-
-    def test_falls_back_to_result_usage(self):
-        lines = [json.dumps(_assistant("msg_1", "no usage carried")),
-                 json.dumps(_result(num_turns=1, usage=USAGE_1))]
-        assert parse_stream(lines)["usage"].total_tokens == 165
-
-    def test_result_usage_overrides_per_message_snapshots(self):
-        # Assistant events carry message_start usage snapshots (output ≈ a few tokens);
-        # the result event's cumulative usage is authoritative when present.
-        snapshot = {"input_tokens": 240, "output_tokens": 3}  # start-of-message snapshot
-        final = {"input_tokens": 240, "output_tokens": 9238,
-                 "cache_read_input_tokens": 1674661, "cache_creation_input_tokens": 66677}
-        lines = [json.dumps(_assistant("msg_1", "text", snapshot)),
-                 json.dumps(_result(num_turns=40, usage=final))]
-        usage = parse_stream(lines)["usage"]
-        assert usage.output_tokens == 9238
-        assert usage.cache_read_tokens == 1674661
-
-    def test_skips_garbage_lines(self):
-        lines = ["not json", "", "42", json.dumps(_result(num_turns=1))]
-        parsed = parse_stream(lines)
-        assert parsed["trajectory"] == []
-        assert parsed["steps"] == 1
-
-    def test_tool_use_and_tool_result_blocks_rendered(self):
-        event = {"type": "assistant", "message": {"id": "m", "role": "assistant", "content": [
-            {"type": "tool_use", "name": "Bash", "input": {"command": "pred list"}}]}}
-        parsed = parse_stream([json.dumps(event)])
-        assert "pred list" in parsed["trajectory"][0]["content"]
-
-
-# ── _build_command ────────────────────────────────────────────────────────────
-
-class TestBuildCommand:
-    def test_strips_litellm_provider_prefix(self):
-        cmd = _build_command("claude", "sys", "task", "anthropic/claude-opus-4-8", "Bash")
-        assert cmd[cmd.index("--model") + 1] == "claude-opus-4-8"
-
-    def test_no_turn_cap_by_default(self):
-        # claude -p self-terminates; the backend passes no --max-turns.
-        cmd = _build_command("claude", "sys", "task", "claude-sonnet-5", "Bash")
-        assert cmd[cmd.index("--model") + 1] == "claude-sonnet-5"
-        assert "--max-turns" not in cmd
-        assert "--output-format" in cmd and "stream-json" in cmd
-
-    def test_toolset_is_restricted_not_just_allowed(self):
-        # --allowedTools alone does not remove Write/Edit/Task; --tools is what sandboxes the
-        # session. Assert the restrictive flag is present and space-separated, MCP is off.
-        cmd = _build_command("claude", "sys", "task", "claude-sonnet-5",
-                             "Bash,Read,Grep,Glob")
-        assert cmd[cmd.index("--tools") + 1] == "Bash Read Grep Glob"
-        assert cmd[cmd.index("--allowedTools") + 1] == "Bash,Read,Grep,Glob"
-        assert "--strict-mcp-config" in cmd
-
-
-def _ctx(tmp_path: Path) -> SimpleNamespace:
-    repo = tmp_path / "repo"
-    (repo / "src" / "rules").mkdir(parents=True)
-    pred = tmp_path / "pred"
-    pred.write_text("#!/bin/sh\nexit 0\n")
-    pred.chmod(0o755)
-    return SimpleNamespace(repo_path=repo, pred_binary=pred, commit_hash="deadbeefcafe")
-
-
-def _stub_claude(tmp_path: Path, events: list[dict], exit_code: int = 0) -> str:
-    transcript = tmp_path / "transcript.jsonl"
-    transcript.write_text("\n".join(json.dumps(e) for e in events) + "\n", encoding="utf-8")
-    script = tmp_path / "claude-stub"
-    script.write_text(f'#!/bin/sh\ncat "{transcript}"\nexit {exit_code}\n', encoding="utf-8")
-    script.chmod(0o755)
-    return str(script)
-
-
-@pytest.fixture(autouse=True)
-def _clean_env(monkeypatch):
-    for var in ("AGENT_STRATEGY_FILE", "CLAUDE_CODE_TOOLS", "CLAUDE_BIN",
-                "CLAUDE_CODE_SESSION_TIMEOUT"):
-        monkeypatch.delenv(var, raising=False)
-
-
-# ── backend selection ─────────────────────────────────────────────────────────
-
-class TestBackendSelection:
-    def test_unknown_backend_rejected(self, tmp_path):
-        (tmp_path / "src" / "rules").mkdir(parents=True)
-        with pytest.raises(ValueError, match="unknown backend"):
-            run("m", str(tmp_path), fake=True, backend="opencode",
-                library_commit="abc123", output=tmp_path / "out.json")
-
-    def test_whole_repo_dispatches_to_claude_backend(self, monkeypatch):
-        # The selected backend must call run_repo_claude, not mini-swe.
-        def fake_repo_claude(model, ctx, **kw):
-            kw["submit_session"].result_rows = lambda: [
-                {"rule": "r1", "result": "bug_found", "tokens_k": 0.0,
-                 "certificate": {"rule": "r1", "source": {}}}]
-            return {"tokens_k": 12.0, "usage": None, "error": None}
-
-        monkeypatch.setattr(run_submission, "find_pred_binary", lambda: "pred")
-        monkeypatch.setattr(run_submission, "verify_pred_version", lambda p: "1.2.3")
-        monkeypatch.setattr(run_submission, "EnvContext",
-                            lambda **kw: type("C", (), {"pred_version": "1.2.3", **kw})())
-        monkeypatch.setattr("benchmark.run_mini.run_repo_session",
-                            lambda *a, **kw: (_ for _ in ()).throw(AssertionError("mini-swe used")))
-        monkeypatch.setattr("benchmark.claude_code.run_repo_claude", fake_repo_claude)
-
-        sub = run_submission.run("claude-opus-4-8", "/repo", library_commit="deadbeef",
-                                 backend="claude-code")
-        assert sub["total_tokens_k"] == 12.0
-        assert sub["bugs_found"] == 1
-        assert "trajectory" not in sub
-
-
-# ── run_repo_claude (stubbed CLI) ─────────────────────────────────────────────
-
-class TestRunRepoClaude:
-    def test_uses_shared_workspace_and_submit_channel(self, tmp_path):
-        transcript = tmp_path / "transcript.jsonl"
-        transcript.write_text(json.dumps(_result(num_turns=1)) + "\n", encoding="utf-8")
-        stub = tmp_path / "claude-stub"
-        stub.write_text(
-            f'#!/bin/sh\npwd > workspace.txt\nsubmit --status > status.txt\n'
-            f'cat "{transcript}"\n',
-            encoding="utf-8",
-        )
-        stub.chmod(0o755)
-
-        with SubmissionSession() as submit_session:
-            session = run_repo_claude(
-                "claude-haiku-4-5", _ctx(tmp_path), claude_bin=str(stub), strategy="",
-                submit_session=submit_session)
-            assert (submit_session.workdir / "workspace.txt").read_text().strip() == str(
-                submit_session.workdir.resolve())
-            assert "0/100 used" in (submit_session.workdir / "status.txt").read_text()
-            assert submit_session.reachable
-            assert session["error"] is None
-
-    def test_session_usage_and_single_raw_stream(self, tmp_path):
-        stub = _stub_claude(tmp_path, [
-            _assistant("m1", CERT_TEXT, USAGE_1),
-            _result(num_turns=5),
-        ])
-        submit_session = SimpleNamespace(limit=100)
-        traj_dir = tmp_path / "out"
-        session = run_repo_claude("anthropic/claude-haiku-4-5", _ctx(tmp_path),
-                                  claude_bin=stub, strategy="",
-                                  trajectory_dir=traj_dir,
-                                  submit_session=submit_session)
-        assert session["error"] is None
-        assert session["usage"].input_tokens == 100
-        assert (traj_dir / "anthropic_claude-haiku-4-5_whole-repo.stream.jsonl").exists()
-        assert not (traj_dir / "anthropic_claude-haiku-4-5_whole-repo.jsonl").exists()
-
-    def test_cli_failure_reports_error_and_salvages(self, tmp_path):
-        stub = _stub_claude(tmp_path, [_assistant("m1", "partial work", USAGE_1)], exit_code=7)
-        log_dir = tmp_path / "logs"
-        session = run_repo_claude("claude-haiku-4-5", _ctx(tmp_path),
-                                  claude_bin=stub, strategy="", trajectory_dir=log_dir)
-        assert "claude exited 7" in session["error"]
-        assert "partial work" in (
-            log_dir / "claude-haiku-4-5_whole-repo.stream.jsonl").read_text()
-        assert session["usage"].input_tokens == 100
-
-    def test_missing_cli(self, tmp_path):
-        session = run_repo_claude("claude-haiku-4-5", _ctx(tmp_path),
-                                  claude_bin=str(tmp_path / "nope"), strategy="")
-        assert "not found" in session["error"]
-        assert "rows" not in session
diff --git a/benchmark/tests/test_codex_cli.py b/benchmark/tests/test_codex_cli.py
deleted file mode 100644
index 98a2deb..0000000
--- a/benchmark/tests/test_codex_cli.py
+++ /dev/null
@@ -1,214 +0,0 @@
-"""Unit tests for the Codex headless backend (no real Codex invocation)."""
-import json
-import os
-from pathlib import Path
-from types import SimpleNamespace
-
-import pytest
-
-from benchmark import run_submission
-from benchmark.codex_cli import _build_command, _child_env, parse_stream, run_repo_codex
-from benchmark.run_submission import _load_env_file
-from benchmark.submit_session import SubmissionSession
-
-
-def _event(event: dict) -> str:
-    return json.dumps(event)
-
-
-def _ctx(tmp_path: Path) -> SimpleNamespace:
-    repo = tmp_path / "repo"
-    (repo / "src" / "rules").mkdir(parents=True)
-    pred = tmp_path / "pred"
-    pred.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
-    pred.chmod(0o755)
-    return SimpleNamespace(repo_path=repo, pred_binary=pred, commit_hash="deadbeefcafe")
-
-
-def _stub_codex(tmp_path: Path, events: list[dict], exit_code: int = 0) -> str:
-    transcript = tmp_path / "codex.jsonl"
-    transcript.write_text("\n".join(_event(e) for e in events) + "\n", encoding="utf-8")
-    script = tmp_path / "codex-stub"
-    script.write_text(f'#!/bin/sh\ncat "{transcript}"\nexit {exit_code}\n', encoding="utf-8")
-    script.chmod(0o755)
-    return str(script)
-
-
-class TestCommand:
-    def test_noninteractive_is_ephemeral_and_controlled(self):
-        cmd = _build_command("codex", "do work", "openai/gpt-5.4")
-        assert cmd[:2] == ["codex", "exec"]
-        assert "--json" in cmd
-        assert "--ephemeral" in cmd
-        assert cmd[cmd.index("--sandbox") + 1] == "workspace-write"
-        assert "--ignore-user-config" in cmd
-        assert "--ignore-rules" in cmd
-        assert "--skip-git-repo-check" in cmd
-        assert cmd[cmd.index("--model") + 1] == "gpt-5.4"
-        assert cmd[-1] == "do work"
-
-    def test_generic_api_key_uses_codex_cli_variable(self, tmp_path, monkeypatch):
-        monkeypatch.delenv("OPENAI_API_KEY", raising=False)
-        assert _child_env(_ctx(tmp_path), "secret")["OPENAI_API_KEY"] == "secret"
-
-
-class TestParseStream:
-    def test_trajectory_steps_and_disjoint_usage(self):
-        parsed = parse_stream([
-            _event({"type": "thread.started", "thread_id": "thread-1"}),
-            _event({"type": "item.completed", "item": {
-                "id": "i1", "type": "command_execution", "command": "pred list",
-                "aggregated_output": "rule-a\n", "status": "completed"}}),
-            _event({"type": "item.completed", "item": {
-                "id": "i2", "type": "agent_message", "text": "done"}}),
-            _event({"type": "turn.completed", "usage": {
-                "input_tokens": 24763, "cached_input_tokens": 24448,
-                "output_tokens": 122, "reasoning_output_tokens": 7}}),
-        ])
-        assert parsed["thread_id"] == "thread-1"
-        assert parsed["steps"] == 1
-        assert [m["role"] for m in parsed["trajectory"]] == ["tool", "assistant"]
-        assert parsed["usage"].input_tokens == 315
-        assert parsed["usage"].cache_read_tokens == 24448
-        assert parsed["usage"].output_tokens == 122
-        assert parsed["usage"].total_tokens == 24885
-
-    def test_skips_non_json_output(self):
-        assert parse_stream(["warning", "", "42"])["trajectory"] == []
-
-
-class TestCodexRuns:
-    def test_uses_submit_workspace_as_sandbox_root(self, tmp_path):
-        transcript = tmp_path / "codex.jsonl"
-        transcript.write_text(_event({"type": "turn.completed", "usage": {}}) + "\n")
-        stub = tmp_path / "codex-stub"
-        stub.write_text(
-            f'#!/bin/sh\npwd > workspace.txt\nsubmit --status > status.txt\n'
-            f'cat "{transcript}"\n',
-            encoding="utf-8",
-        )
-        stub.chmod(0o755)
-
-        with SubmissionSession() as submit_session:
-            session = run_repo_codex("gpt-5.4", _ctx(tmp_path), codex_bin=str(stub),
-                                     strategy="", submit_session=submit_session)
-            assert Path(submit_session.workdir / "workspace.txt").read_text().strip() == str(
-                submit_session.workdir.resolve())
-            assert "0/100 used" in (submit_session.workdir / "status.txt").read_text()
-            assert submit_session.reachable
-            assert session["error"] is None
-
-    def test_missing_cli(self, tmp_path):
-        session = run_repo_codex("gpt-5.4", _ctx(tmp_path),
-                                 codex_bin=str(tmp_path / "missing"), strategy="")
-        assert session["error"].startswith("codex CLI not found")
-
-    def test_partial_output_does_not_hide_cli_failure(self, tmp_path):
-        stub = _stub_codex(tmp_path, [
-            {"type": "item.completed", "item": {
-                "type": "agent_message", "text": "partial work"}},
-        ], exit_code=7)
-        session = run_repo_codex("gpt-5.4", _ctx(tmp_path), codex_bin=stub, strategy="")
-        assert session["error"].startswith("codex exited 7")
-        assert "trajectory" not in session
-
-    def test_turn_failed_event_is_an_error_even_on_zero_exit(self, tmp_path):
-        stub = _stub_codex(tmp_path, [
-            {"type": "turn.failed", "error": {"message": "quota exhausted"}},
-        ])
-        session = run_repo_codex("gpt-5.4", _ctx(tmp_path), codex_bin=stub, strategy="")
-        assert session["error"].startswith("codex turn.failed")
-
-    def test_whole_repo_persists_single_raw_stream(self, tmp_path):
-        stub = _stub_codex(tmp_path, [
-            {"type": "item.completed", "item": {"type": "agent_message", "text": "done"}},
-            {"type": "turn.completed", "usage": {
-                "input_tokens": 30, "cached_input_tokens": 10, "output_tokens": 5}},
-        ])
-        submit_session = SimpleNamespace(limit=100)
-        session = run_repo_codex("openai/gpt-5.4", _ctx(tmp_path), codex_bin=stub,
-                                 strategy="", trajectory_dir=tmp_path / "out",
-                                 submit_session=submit_session)
-        assert session["usage"].total_tokens == 35
-        assert (tmp_path / "out" / "openai_gpt-5.4_whole-repo.stream.jsonl").exists()
-        assert not (tmp_path / "out" / "openai_gpt-5.4_whole-repo.jsonl").exists()
-
-
-class TestWiring:
-    @pytest.mark.parametrize("backend", ["codex", "claude-code"])
-    def test_container_mode_rejects_cli_backend(self, backend, monkeypatch, capsys):
-        monkeypatch.delenv("REPO_REF", raising=False)
-        with pytest.raises(SystemExit):
-            run_submission.main(["--model", "test-model", "--backend", backend])
-        assert "make run-local" in capsys.readouterr().err
-
-    def test_env_file_preserves_ambient_values(self, tmp_path, monkeypatch):
-        env_file = tmp_path / "submission.env"
-        env_file.write_text("# comment\nMODEL_NAME='from-file'\nAGENT_BACKEND=codex\n",
-                            encoding="utf-8")
-        monkeypatch.setenv("MODEL_NAME", "ambient")
-        monkeypatch.delenv("AGENT_BACKEND", raising=False)
-        _load_env_file(env_file)
-        assert os.environ["MODEL_NAME"] == "ambient"
-        assert os.environ["AGENT_BACKEND"] == "codex"
-
-    def test_local_clone_is_forwarded_to_existing_runner(self, tmp_path, monkeypatch):
-        seen = {}
-
-        def fake_run(model, repo_dir, **kwargs):
-            seen.update(model=model, repo_dir=repo_dir, **kwargs)
-            return {"bugs_found": 0, "total_tokens_k": 0.0, "rules_tested": 0,
-                    "submit_log": [], "submit_limit": kwargs["submit_limit"]}
-
-        monkeypatch.setattr(run_submission, "clone_or_verify_repo",
-                            lambda repo, ref, url: "a" * 40)
-        monkeypatch.setattr(run_submission, "run", fake_run)
-        run_submission.main([
-            "--model", "gpt-5.4", "--repo-dir", str(tmp_path / "repo"),
-            "--repo-ref", "v0.6.0", "--backend", "codex",
-            "--host-cli",
-            "--output", str(tmp_path / "submission.json"),
-            "--trajectory-dir", str(tmp_path / "logs"),
-        ])
-        assert seen["backend"] == "codex"
-        assert seen["library_commit"] == "a" * 40
-
-    def test_local_clone_requires_separate_paths(self, monkeypatch):
-        monkeypatch.delenv("OUTPUT", raising=False)
-        monkeypatch.delenv("TRAJECTORY_DIR", raising=False)
-        with pytest.raises(SystemExit):
-            run_submission.main([
-                "--model", "gpt-5.4", "--repo-ref", "v0.6.0",
-                "--repo-dir", "/repo",
-            ])
-
-    def test_whole_repo_dispatches_to_codex(self, monkeypatch):
-        called = {}
-
-        def fake_repo(model, ctx, **kwargs):
-            called["model"] = model
-            return {"tokens_k": 1.0, "usage": None, "error": None}
-
-        class FakeSubmissionSession:
-            def __init__(self, limit):
-                self.limit, self.attempts, self.reachable = limit, [], True
-
-            def __enter__(self):
-                return self
-
-            def __exit__(self, *args):
-                return None
-
-            def result_rows(self):
-                return []
-
-        monkeypatch.setattr(run_submission, "SubmissionSession", FakeSubmissionSession)
-        monkeypatch.setattr(run_submission, "find_pred_binary", lambda: "pred")
-        monkeypatch.setattr(run_submission, "verify_pred_version", lambda p: "1.2.3")
-        monkeypatch.setattr(run_submission, "EnvContext",
-                            lambda **kw: type("C", (), {"pred_version": "1.2.3", **kw})())
-        monkeypatch.setattr("benchmark.codex_cli.run_repo_codex", fake_repo)
-        sub = run_submission.run("gpt-5.4", "/repo", library_commit="abc",
-                                 backend="codex")
-        assert called["model"] == "gpt-5.4"
-        assert sub["total_tokens_k"] == 1.0
diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py
index cc6159c..e6d5775 100644
--- a/benchmark/tests/test_docs.py
+++ b/benchmark/tests/test_docs.py
@@ -18,7 +18,6 @@
 GUIDE = REPO_ROOT / "CONTRIBUTING.md"
 ENV_EXAMPLE = REPO_ROOT / "submission.env.example"
 API_SKILL = REPO_ROOT / ".agents/skills/run-api-benchmark/SKILL.md"
-CLI_SKILL = REPO_ROOT / ".agents/skills/run-cli-benchmark/SKILL.md"
 SUBMIT_SKILL = REPO_ROOT / ".agents/skills/submit-benchmark-result/SKILL.md"
 TRIGGER_SCORING = SUBMIT_SKILL.parent / "scripts/trigger-scoring.sh"
 SCORER_WORKFLOW = REPO_ROOT / ".github/workflows/score-from-r2.yml"
@@ -98,22 +97,15 @@ def test_guide_has_submit_flow(self):
         assert "benchmark.submit" in t and "aggregate leaderboard" in t
 
 
-class TestBackendRouteSeparation:
-    def test_env_template_does_not_select_cli_with_agent_backend(self):
+class TestSingleProtocol:
+    def test_env_template_exposes_only_model_api_configuration(self):
         t = _text(ENV_EXAMPLE)
-        assert "agent_backend=" not in t
-        assert "model api only" in t
-        assert "coding-agent" in t and "cannot enter the top50 table" in t
+        assert "model api in docker" in t
+        assert "whole-repository" not in t and "local_backend" not in t
 
-    def test_api_skill_is_container_only(self):
+    def test_run_skill_uses_the_containerized_protocol(self):
         t = _text(API_SKILL)
         assert "standardized" in t and "runner-pull" in t
-        assert "coding-agent backends" in t
-
-    def test_cli_skill_is_host_only(self):
-        t = _text(CLI_SKILL)
-        assert "make run-local" in t and "local_backend" in t
-        assert "legacy-whole-repo" in t and "non-ranking" in t
 
     @pytest.mark.parametrize("skill", [API_SKILL])
     def test_each_skill_exposes_only_local_and_official_goals(self, skill):
@@ -128,7 +120,7 @@ def test_run_skills_delegate_upload(self, skill):
         assert "$submit-benchmark-result" in t
         assert "owns upload" in t
 
-    @pytest.mark.parametrize("skill", [API_SKILL, CLI_SKILL])
+    @pytest.mark.parametrize("skill", [API_SKILL])
     def test_run_skills_confirm_the_benchmark_version(self, skill):
         t = _text(skill)
         assert "make -s print-benchmark-version" in t
@@ -163,7 +155,6 @@ def test_router_sends_existing_results_to_submit_skill(self):
         assert "already has a `submission.json`" in t
         assert "$submit-benchmark-result" in t
 
-
 class TestSubmitSkill:
     def test_submit_skill_exists(self):
         assert SUBMIT_SKILL.exists(), "submit-benchmark-result skill missing"
diff --git a/benchmark/tests/test_env_setup.py b/benchmark/tests/test_env_setup.py
index 0001e06..d7ad654 100644
--- a/benchmark/tests/test_env_setup.py
+++ b/benchmark/tests/test_env_setup.py
@@ -22,7 +22,6 @@
 from benchmark.env_setup import (
     PINNED_COMMIT,
     PINNED_PRED_VERSION,
-    clone_or_verify_repo,
     find_pred_binary,
     pred_version,
     setup_env,
@@ -31,65 +30,6 @@
 )
 
 
-def _git(*args, cwd=None) -> str:
-    result = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True,
-                            check=True)
-    return result.stdout.strip()
-
-
-def _source_repo(tmp_path: Path) -> tuple[Path, str]:
-    source = tmp_path / "source"
-    _git("init", str(source))
-    _git("config", "user.email", "tests@example.com", cwd=source)
-    _git("config", "user.name", "Tests", cwd=source)
-    (source / "README.md").write_text("v1\n", encoding="utf-8")
-    _git("add", "README.md", cwd=source)
-    _git("commit", "-m", "v1", cwd=source)
-    _git("tag", "v1.0.0", cwd=source)
-    return source, _git("rev-parse", "HEAD", cwd=source)
-
-
-class TestCloneOrVerifyRepo:
-    def test_clones_requested_ref_and_returns_full_commit(self, tmp_path):
-        source, commit = _source_repo(tmp_path)
-        destination = tmp_path / "checkouts" / "target"
-
-        actual = clone_or_verify_repo(destination, "v1.0.0", str(source))
-
-        assert actual == commit
-        assert _git("rev-parse", "HEAD", cwd=destination) == commit
-
-    def test_reuses_exact_existing_checkout_without_mutating_it(self, tmp_path):
-        source, commit = _source_repo(tmp_path)
-        destination = tmp_path / "target"
-        clone_or_verify_repo(destination, "v1.0.0", str(source))
-
-        assert clone_or_verify_repo(destination, "v1.0.0", str(source)) == commit
-
-    def test_clones_raw_commit_ref(self, tmp_path):
-        source, commit = _source_repo(tmp_path)
-        destination = tmp_path / "target"
-
-        assert clone_or_verify_repo(destination, commit, str(source)) == commit
-
-    def test_rejects_existing_checkout_at_another_commit(self, tmp_path):
-        source, _ = _source_repo(tmp_path)
-        destination = tmp_path / "target"
-        clone_or_verify_repo(destination, "v1.0.0", str(source))
-        _git("config", "user.email", "tests@example.com", cwd=destination)
-        _git("config", "user.name", "Tests", cwd=destination)
-        (destination / "README.md").write_text("v2\n", encoding="utf-8")
-        _git("commit", "-am", "v2", cwd=destination)
-
-        with pytest.raises(ValueError, match="resolves to"):
-            clone_or_verify_repo(destination, "v1.0.0", str(source))
-
-    def test_rejects_existing_non_git_directory(self, tmp_path):
-        destination = tmp_path / "target"
-        destination.mkdir()
-        with pytest.raises(ValueError, match="not a git checkout"):
-            clone_or_verify_repo(destination, "v1.0.0", "unused")
-
 
 # ── pred version pin (mock subprocess so no real pred needed) ──────────────────
 
diff --git a/benchmark/tests/test_run_mini.py b/benchmark/tests/test_model_api.py
similarity index 63%
rename from benchmark/tests/test_run_mini.py
rename to benchmark/tests/test_model_api.py
index bba447d..491d9d8 100644
--- a/benchmark/tests/test_run_mini.py
+++ b/benchmark/tests/test_model_api.py
@@ -1,5 +1,5 @@
 """
-Tests for benchmark/run_mini.py model construction.
+Tests for benchmark/model_api.py model construction.
 
 The real minisweagent is stubbed via sys.modules so these run anywhere.
 """
@@ -8,7 +8,7 @@
 
 import pytest
 
-from benchmark.run_mini import _build_model
+from benchmark.model_api import build_model
 
 
 class _FakeLitellmModel:
@@ -27,18 +27,13 @@ def _fake_litellm(monkeypatch):
 
 class TestBuildModel:
     def test_defaults_timeout_and_retries(self, _fake_litellm):
-        """A hung API call must fail fast, not freeze the whole-repo session."""
-        model = _build_model("openai/x", None, 8192)
+        """A hung API call must fail fast instead of freezing the benchmark."""
+        model = build_model("openai/x", None, 8192)
         assert model.model_kwargs["timeout"] == 300
         assert model.model_kwargs["num_retries"] == 2
 
-    def test_user_kwargs_override_defaults(self, _fake_litellm):
-        model = _build_model("openai/x", None, 8192, model_kwargs={"timeout": 60})
-        assert model.model_kwargs["timeout"] == 60
-        assert model.model_kwargs["num_retries"] == 2
-
     def test_endpoint_config_passthrough(self, _fake_litellm):
-        model = _build_model("openai/x", "https://api.example/v1", 4096, api_key="k")
+        model = build_model("openai/x", "https://api.example/v1", 4096, api_key="k")
         assert model.model_kwargs["api_base"] == "https://api.example/v1"
         assert model.model_kwargs["api_key"] == "k"
         assert model.model_kwargs["max_tokens"] == 4096
diff --git a/benchmark/tests/test_preflight.py b/benchmark/tests/test_preflight.py
index a6b89a5..3cc5843 100644
--- a/benchmark/tests/test_preflight.py
+++ b/benchmark/tests/test_preflight.py
@@ -43,7 +43,7 @@ def _build(*a, **k):
         if build_exc is not None:
             raise build_exc
         return model if model is not None else _FakeModel()
-    monkeypatch.setattr(pf, "_build_model", _build)
+    monkeypatch.setattr(pf, "build_model", _build)
 
 
 class TestRunChecks:
diff --git a/benchmark/tests/test_run_submission.py b/benchmark/tests/test_run_submission.py
deleted file mode 100644
index a1d2bce..0000000
--- a/benchmark/tests/test_run_submission.py
+++ /dev/null
@@ -1,119 +0,0 @@
-"""
-Tests for benchmark/run_submission.py — the dockerized runner entry point.
-
-All tests run in fake mode: no model API, no pred binary. They prove the runner assembles
-a schema-valid, rankable submission.json.
-"""
-import json
-from pathlib import Path
-
-from benchmark import run_submission as rs
-from benchmark.usage import Usage
-
-
-# ── token totals are derived from the 4-bucket usage ──────────────────────────
-
-class TestTokenTotals:
-    def test_legacy_row_usage_is_still_aggregated(self):
-        rows = [
-            {"rule": "r1", "result": "no_certificate", "tokens_k": 0.0,
-             "usage": {"input": 1_000_000, "output": 0, "cache_read": 0, "cache_write": 0}},
-            {"rule": "r2", "result": "no_certificate", "tokens_k": 0.0,
-             "usage": {"input": 0, "output": 1_000_000, "cache_read": 0, "cache_write": 0}},
-        ]
-        sub = rs.build_submission("m", rows, library_commit="c")
-        assert sub["total_tokens_k"] == 2000.0
-        assert sub["usage_totals"] == {"input": 1_000_000, "output": 1_000_000,
-                                       "cache_read": 0, "cache_write": 0}
-
-    def test_whole_repo_tokens_derived_from_session_usage(self):
-        usage = Usage(input_tokens=2_000_000, output_tokens=0)
-        rows = [{"rule": "r1", "result": "bug_found", "tokens_k": 0.0,
-                 "certificate": {"rule": "r1"}}]
-        sub = rs.build_submission("m", rows, library_commit="c", usage_totals=usage)
-        assert sub["total_tokens_k"] == 2000.0
-        assert sub["usage_totals"]["input"] == 2_000_000
-
-    def test_no_usage_falls_back_to_row_tokens(self):
-        rows = [{"rule": "r1", "result": "no_certificate", "tokens_k": 2.0}]
-        sub = rs.build_submission("m", rows, library_commit="c")
-        assert sub["total_tokens_k"] == 2.0             # row-sum fallback
-
-
-def _fake_repo(tmp_path: Path) -> Path:
-    repo = tmp_path / "pr-src"
-    repo.mkdir()
-    return repo
-
-
-# ── build_submission (pure assembly) ──────────────────────────────────────────
-
-class TestBuildSubmission:
-    def test_envelope_fields_present(self):
-        rows = [{"rule": "r1", "result": "no_certificate", "tokens_k": 2.0}]
-        sub = rs.build_submission("anthropic/x", rows, library_commit="abc123")
-        for k in ("model", "library_commit",
-                  "bugs_found", "total_tokens_k", "rules_tested",
-                  "results", "efficiency_bugs_per_ktok", "submit_limit", "submit_log"):
-            assert k in sub
-        assert sub["model"] == "anthropic/x"
-        assert sub["library_commit"] == "abc123"
-        assert sub["submit_limit"] == 100
-
-    def test_bugs_counted_distinct_rules(self):
-        rows = [
-            {"rule": "r1", "result": "bug_found", "tokens_k": 1.0,
-             "certificate": {"rule": "r1", "violation": "solve_mismatch"}},
-            {"rule": "r1", "result": "bug_found", "tokens_k": 1.0,
-             "certificate": {"rule": "r1", "violation": "unsound_extraction"}},
-            {"rule": "r2", "result": "bug_found", "tokens_k": 1.0,
-             "certificate": {"rule": "r2", "violation": "solve_mismatch"}},
-            {"rule": "r3", "result": "no_certificate", "tokens_k": 1.0},
-        ]
-        sub = rs.build_submission("m", rows, library_commit="c")
-        # distinct rules with a bug = {r1, r2} = 2 (not 3 certificates)
-        assert sub["bugs_found"] == 2
-
-    def test_rules_tested_counts_distinct_rules(self):
-        rows = [
-            {"rule": "r1", "result": "no_certificate", "tokens_k": 1.0},
-            {"rule": "r1", "result": "bug_found", "tokens_k": 1.0,
-             "certificate": {"rule": "r1"}},
-            {"rule": "r2", "result": "no_certificate", "tokens_k": 1.0},
-        ]
-        sub = rs.build_submission("m", rows, library_commit="c")
-        assert sub["rules_tested"] == 2
-
-
-# ── run() end-to-end in fake mode ─────────────────────────────────────────────
-
-class TestRunFake:
-    def test_produces_rankable_submission(self, tmp_path):
-        repo = _fake_repo(tmp_path)
-        sub = rs.run("fake/model", str(repo), fake=True, library_commit="deadbeef")
-        assert "schema_version" not in sub
-        assert sub["agent_mode"] == "whole-repo"
-        assert sub["rules_tested"] == 0
-        assert sub["bugs_found"] == 0
-
-    def test_writes_output_file(self, tmp_path):
-        repo = _fake_repo(tmp_path)
-        out = tmp_path / "out" / "submission.json"
-        sub = rs.run("fake/model", str(repo), fake=True, library_commit="c", output=out)
-        assert out.exists()
-        on_disk = json.loads(out.read_text())
-        assert on_disk["model"] == sub["model"]
-        assert list(out.parent.glob("submission*.json")) == [out]
-
-
-# ── schema validity ───────────────────────────────────────────────────────────
-
-class TestSchemaValidity:
-    def test_submission_matches_schema_required_fields(self, tmp_path):
-        repo = _fake_repo(tmp_path)
-        sub = rs.run("fake/model", str(repo), fake=True, library_commit="c")
-        schema = json.loads(
-            (Path(rs.__file__).parent / "submission.schema.json").read_text())
-        for field in schema["required"]:
-            assert field in sub, f"missing required field: {field}"
-        assert "schema_version" not in schema["properties"]
diff --git a/benchmark/tests/test_run_top50.py b/benchmark/tests/test_run_top50.py
index ecf1c92..5c49118 100644
--- a/benchmark/tests/test_run_top50.py
+++ b/benchmark/tests/test_run_top50.py
@@ -1,4 +1,4 @@
-"""Entrypoint wiring tests for the standardized Top50 track."""
+"""Entrypoint wiring tests for the standardized Top50 benchmark."""
 from __future__ import annotations
 
 import json
@@ -29,15 +29,8 @@ def test_fake_entrypoint_uses_50_isolated_episodes(tmp_path, monkeypatch):
     assert len(result["episodes"]) == 50
     assert result["rankable"] is False
     artifact = json.loads(output.read_text())
-    assert "budget_contract_status" not in artifact
-    assert "benchmark_contract" not in artifact
-    assert "contract" not in artifact
-
-
-@pytest.mark.parametrize("forbidden", ["--backend", "--config", "--strategy-file"])
-def test_standard_entrypoint_rejects_custom_harness_options(forbidden):
-    with pytest.raises(SystemExit):
-        run_top50.main(["--model", "fake/model", "--fake", forbidden, "custom"])
+    assert artifact["model"] == "fake/model"
+    assert artifact["status"] == "completed"
 
 
 @pytest.mark.parametrize("name", run_top50.FORBIDDEN_RANKABLE_ENV)
@@ -47,18 +40,10 @@ def test_rankable_preflight_rejects_custom_execution_before_model_call(name, mon
         run_top50.validate_rankable_settings()
 
 
-def test_benchmark_limits_are_code_defined_and_ignore_legacy_budget_environment(monkeypatch):
-    monkeypatch.setenv("PRB_PRED_CALLS", "999")
+def test_benchmark_limits_are_code_defined():
     assert run_top50.benchmark_limits().episode.pred_calls == 24
 
 
-def test_even_empty_custom_model_kwargs_are_rejected(monkeypatch):
-    for name in run_top50.FORBIDDEN_RANKABLE_ENV:
-        monkeypatch.delenv(name, raising=False)
-    with pytest.raises(ValueError, match="custom model kwargs"):
-        run_top50.validate_rankable_settings(model_kwargs={})
-
-
 def test_top50_executor_keeps_finite_agent_step_guard():
     from benchmark.top50_runner import MiniSwePhaseExecutor
     executor = MiniSwePhaseExecutor()
diff --git a/benchmark/tests/test_submit.py b/benchmark/tests/test_submit.py
index b436663..4610df0 100644
--- a/benchmark/tests/test_submit.py
+++ b/benchmark/tests/test_submit.py
@@ -1,81 +1,49 @@
-"""Tests for benchmark/submit.py — the `prb submit` CLI client. No real network."""
+"""Tests for the `prb submit` client. No real network."""
 import json
 from pathlib import Path
 
 import pytest
 
 from benchmark import submit as sub
+from benchmark.tests.test_top50_submission import _artifact
 
 
-def _valid(tmp_path: Path, results=None) -> Path:
-    doc = {
-        "model": "anthropic/test", "library_commit": "deadbeef",
-        "bugs_found": 0,
-        "total_tokens_k": 10.0,
-        "rules_tested": 1, "results": results if results is not None else [
-            {"rule": "r1", "result": "no_certificate", "tokens_k": 1.0}],
-        "submit_limit": 100,
-        "submit_log": [],
-    }
-    p = tmp_path / "submission.json"
-    p.write_text(json.dumps(doc), encoding="utf-8")
-    return p
-
-
-def _bug_row(with_cert=True, with_traj=True) -> dict:
-    row = {"rule": "r1", "result": "bug_found", "tokens_k": 1.0}
-    if with_cert:
-        row["certificate"] = {"rule": "r1", "source": {}}
-    if with_traj:
-        row["trajectory"] = [{"role": "assistant", "content": "CERTIFICATE_START\n{}\nCERTIFICATE_END"}]
-    return row
+def _valid(tmp_path: Path, accepted_positions=()) -> Path:
+    path = tmp_path / "submission.json"
+    path.write_text(json.dumps(_artifact(accepted_positions=accepted_positions)),
+                    encoding="utf-8")
+    return path
 
 
 class TestValidate:
     def test_clean_passes(self, tmp_path):
         assert sub.validate_submission(sub.load_submission(_valid(tmp_path))) == []
 
-    def test_missing_envelope_field(self):
-        assert any("model" in p for p in sub.validate_submission({"results": []}))
-
-    def test_bug_found_needs_certificate(self):
-        doc = {**json.loads('{"model":"m","library_commit":"c",'
-                            '"total_tokens_k":0,"rules_tested":1,'
-                            '"submit_limit":100,"submit_log":[]}'),
-               "results": [_bug_row(with_cert=False, with_traj=False)]}
-        problems = sub.validate_submission(doc)
-        assert any("no certificate" in p for p in problems)
+    def test_missing_current_protocol_field(self):
+        problems = sub.validate_submission({"model": "m"})
+        assert any("episodes" in problem for problem in problems)
 
-    def test_bug_found_complete_passes(self, tmp_path):
-        p = _valid(tmp_path, results=[_bug_row()])
-        assert sub.validate_submission(sub.load_submission(p)) == []
-
-    def test_submit_ledger_counters_must_match(self, tmp_path):
-        doc = sub.load_submission(_valid(tmp_path))
-        doc["submit_limit"] = 0
-        doc["submit_log"] = [{"attempt": 1, "accepted": False, "reason": "bad"}]
-        problems = sub.validate_submission(doc)
-        assert any("exceeds submit_limit" in p for p in problems)
-
-    def test_valid_submit_ledger_passes(self, tmp_path):
-        doc = sub.load_submission(_valid(tmp_path))
-        doc["submit_limit"] = 3
-        doc["submit_log"] = [{"attempt": 1, "accepted": False, "reason": "bad"}]
-        assert sub.validate_submission(doc) == []
+    def test_run_error_is_rejected(self):
+        artifact = _artifact(accepted_positions=())
+        artifact.update(status="run_error", run_error="provider failed", rankable=False)
+        assert any("incomplete" in problem for problem in sub.validate_submission(artifact))
 
 
 class TestSubmit:
     def test_dry_run_does_not_send(self, tmp_path, monkeypatch):
-        monkeypatch.setattr(sub, "_post", lambda *a, **k: pytest.fail("must not POST on dry-run"))
-        out = sub.submit(_valid(tmp_path, results=[_bug_row()]),
+        monkeypatch.setattr(sub, "_post", lambda *a, **k: pytest.fail("must not POST"))
+        out = sub.submit(_valid(tmp_path, accepted_positions=(1,)),
                          "https://x/submit", dry_run=True)
         assert out["claimed_bugs"] == 1 and "dry-run" in out["status"]
 
     def test_invalid_submission_raises_before_network(self, tmp_path, monkeypatch):
-        monkeypatch.setattr(sub, "_post", lambda *a, **k: pytest.fail("must not POST invalid"))
-        p = _valid(tmp_path, results=[_bug_row(with_cert=False, with_traj=False)])
-        with pytest.raises(ValueError, match="certificate"):
-            sub.submit(p, "https://x/submit")
+        monkeypatch.setattr(sub, "_post", lambda *a, **k: pytest.fail("must not POST"))
+        path = _valid(tmp_path)
+        artifact = json.loads(path.read_text())
+        artifact["episodes"].pop()
+        path.write_text(json.dumps(artifact))
+        with pytest.raises(ValueError, match="exactly 50"):
+            sub.submit(path, "https://x/submit")
 
     def test_success_returns_body(self, tmp_path, monkeypatch):
         monkeypatch.setattr(sub, "_post",
@@ -94,8 +62,7 @@ def post(url, payload, headers, timeout=60.0):
         assert out["submission_id"] == "abc"
 
     def test_non_2xx_raises(self, tmp_path, monkeypatch):
-        monkeypatch.setattr(sub, "_post",
-                            lambda *a, **k: (429, {"error": "quota exceeded"}))
+        monkeypatch.setattr(sub, "_post", lambda *a, **k: (429, {"error": "quota exceeded"}))
         with pytest.raises(ValueError, match="429.*quota"):
             sub.submit(_valid(tmp_path), "https://x/submit", access_token="access-jwt")
 
diff --git a/benchmark/tests/test_submit_session.py b/benchmark/tests/test_submit_session.py
index 4edf035..ee3e6b7 100644
--- a/benchmark/tests/test_submit_session.py
+++ b/benchmark/tests/test_submit_session.py
@@ -1,4 +1,4 @@
-"""Tests for the evaluation-owned, run-wide agent submit command."""
+"""Tests for the evaluation-owned per-rule submit command."""
 import json
 import os
 import subprocess
@@ -28,7 +28,7 @@ def verifier(cert):
     bad.write_text(json.dumps(_cert("bad")))
     good.write_text(json.dumps(_cert("good")))
 
-    with SubmissionSession(limit=2, verifier=verifier) as session:
+    with SubmissionSession(expected_rule="good", limit=2, verifier=verifier) as session:
         assert os.environ.get("PRB_SUBMIT_DIR")
         assert Path(os.environ["PRB_SUBMIT_DIR"]).is_relative_to(session.workdir)
         rejected = _run_submit(bad)
@@ -45,75 +45,49 @@ def verifier(cert):
 def test_malformed_json_consumes_attempt_and_status_is_free(tmp_path):
     malformed = tmp_path / "bad.json"
     malformed.write_text("{not json")
-    with SubmissionSession(limit=2, verifier=lambda c: Verdict(True, "ok")) as session:
+    with SubmissionSession(expected_rule="r1", limit=2,
+                           verifier=lambda c: Verdict(True, "ok")) as session:
         status = subprocess.run(["submit", "--status"], capture_output=True, text=True)
         result = _run_submit(malformed)
         assert status.returncode == 0 and "0/2 used" in status.stdout
         assert result.returncode == 1 and "invalid certificate JSON" in result.stderr
         assert session.used == 1
-        assert session.reachable
-        assert session.reachable
 
 
 def test_oversized_file_consumes_attempt_without_loading_it_all(tmp_path):
     oversized = tmp_path / "huge.json"
     oversized.write_bytes(b"x" * (1024 * 1024 + 1))
-    with SubmissionSession(limit=2, verifier=lambda c: Verdict(True, "ok")) as session:
+    with SubmissionSession(expected_rule="r1", limit=2,
+                           verifier=lambda c: Verdict(True, "ok")) as session:
         result = _run_submit(oversized)
         assert result.returncode == 1
         assert "certificate exceeds" in result.stderr
         assert session.used == 1
 
 
-def test_result_rows_are_derived_only_from_service_ledger(tmp_path):
-    paths = []
-    for i, rule in enumerate(("r1", "r1", "r2")):
-        path = tmp_path / f"{i}.json"
-        path.write_text(json.dumps(_cert(rule)))
-        paths.append(path)
-    verdicts = iter((Verdict(False, "first rejected"), Verdict(True, "ok"), Verdict(True, "ok")))
-    with SubmissionSession(limit=10, verifier=lambda c: next(verdicts)) as session:
-        for path in paths:
-            _run_submit(path)
-        rows = session.result_rows()
-    assert [(r["rule"], r["result"]) for r in rows] == [("r1", "bug_found"),
-                                                          ("r2", "bug_found")]
-    assert [r["submit_attempt"] for r in rows] == [2, 3]
-
-
 def test_concurrent_clients_are_serialized_by_one_budget(tmp_path):
     paths = []
     for i in range(6):
         path = tmp_path / f"cert-{i}.json"
-        path.write_text(json.dumps(_cert(f"r{i}")))
+        path.write_text(json.dumps(_cert("r1")))
         paths.append(path)
 
-    with SubmissionSession(limit=4, verifier=lambda c: Verdict(True, "ok")) as session:
+    with SubmissionSession(expected_rule="r1", limit=4,
+                           verifier=lambda c: Verdict(False, "no bug")) as session:
         with ThreadPoolExecutor(max_workers=6) as pool:
             results = list(pool.map(_run_submit, paths))
 
         assert session.used == 4
-        assert sum(result.returncode == 0 for result in results) == 4
+        assert sum(result.returncode == 1 for result in results) == 4
         assert sum(result.returncode == 2 for result in results) == 2
 
 
-def test_preserves_certificate_artifacts_before_workspace_cleanup(tmp_path):
-    destination = tmp_path / "salvaged"
-    with SubmissionSession() as session:
-        artifact = Path(os.environ["PRB_ARTIFACT_DIR"]) / "certificate.json"
-        artifact.write_text(json.dumps(_cert()))
-        copied = session.preserve_artifacts(destination)
-
-    assert len(copied) == 1
-    assert copied[0].read_text() == json.dumps(_cert())
-
-
 def test_runner_response_does_not_follow_agent_symlink(tmp_path):
     victim = tmp_path / "victim.txt"
     victim.write_text("unchanged")
     request_id = "a" * 32
 
-    with SubmissionSession() as session:
+    with SubmissionSession(expected_rule="r1"):
         channel = Path(os.environ["PRB_SUBMIT_DIR"])
         response = channel / "outbox" / f"{request_id}.json"
         response.symlink_to(victim)
@@ -127,13 +101,3 @@ def test_runner_response_does_not_follow_agent_symlink(tmp_path):
         assert not response.is_symlink()
         assert json.loads(response.read_text())["status"] == "ok"
         assert victim.read_text() == "unchanged"
-        assert session.reachable
-
-
-def test_artifact_salvage_skips_symlinks(tmp_path):
-    victim = tmp_path / "secret-certificate.json"
-    victim.write_text("secret")
-    destination = tmp_path / "salvaged"
-    with SubmissionSession() as session:
-        (Path(os.environ["PRB_ARTIFACT_DIR"]) / "certificate.json").symlink_to(victim)
-        assert session.preserve_artifacts(destination) == []
diff --git a/benchmark/tests/test_top50_submission.py b/benchmark/tests/test_top50_submission.py
index 0f278c6..17af608 100644
--- a/benchmark/tests/test_top50_submission.py
+++ b/benchmark/tests/test_top50_submission.py
@@ -11,7 +11,6 @@
 from benchmark.backend_score import _dedup_best, aggregate_leaderboard, score_one
 from benchmark.submit import validate_submission
 from benchmark.top50_contract import (
-    AGENT_MODE,
     RUNNER_VERSION,
     expected_prompt_id,
     score_top50_submission,
@@ -105,7 +104,6 @@ def _artifact(accepted_positions=(7, 18, 41)) -> dict:
         "library_commit": "a" * 40,
         "runner_version": RUNNER_VERSION,
         "pred_version": "0.6.0",
-        "agent_mode": AGENT_MODE,
         "prompt_id": expected_prompt_id(),
         "safety_controls": {"model_timeout_seconds": 300, "model_retries": 2},
         "status": "completed",
@@ -114,7 +112,6 @@ def _artifact(accepted_positions=(7, 18, 41)) -> dict:
         "shortlist": shortlist,
         "triage": triage,
         "episodes": episodes,
-        "bugs_found": 999,
     }
 
 
@@ -136,7 +133,6 @@ def test_valid_artifact_recomputes_score_and_prefix_metrics():
     assert scored["bugs_at_50"] == 3
     assert scored["first_attempt_accepts"] == 3
     assert len(report) == 3
-    assert submission["bugs_found"] == 999
     assert validate_submission(submission) == []
 
 
@@ -206,7 +202,6 @@ def test_pred_observation_command_and_outcome_are_derived_from_action_record():
 
 
 @pytest.mark.parametrize("mutate, expected", [
-    (lambda sub: sub.update(submit_limit=100), "shared run-wide submit pool"),
     (lambda sub: sub["episodes"].pop(), "exactly 50"),
     (lambda sub: sub["episodes"].__setitem__(1, copy.deepcopy(sub["episodes"][0])),
      "order/rule"),
@@ -214,11 +209,9 @@ def test_pred_observation_command_and_outcome_are_derived_from_action_record():
      "usage is inconsistent"),
     (lambda sub: sub["episodes"][0]["ledger"]["budget"].update(pred_calls=23),
      "budget differs"),
-    (lambda sub: sub.update(agent_mode="codex"), "agent_mode"),
     (lambda sub: sub.update(prompt_id="custom"), "prompt_id"),
     (lambda sub: sub.pop("inference_parameters"), "inference_parameters"),
-    (lambda sub: sub.update(contract={}), "obsolete self-declared field"),
-    (lambda sub: sub.update(observation_policy={}), "obsolete self-declared field"),
+    (lambda sub: sub.update(unexpected={}), "unsupported top-level field"),
     (lambda sub: sub.update(status="run_error", run_error="provider failed"), "incomplete"),
 ])
 def test_rankability_negative_controls(mutate, expected):
@@ -280,18 +273,17 @@ def test_public_projection_contains_no_answer_key_material():
     assert callable(guard)
 
 
-def test_top50_ties_ignore_tokens_and_efficiency_and_legacy_is_separate():
+def test_ties_ignore_tokens_and_efficiency_and_each_model_has_one_best_run():
     entries = [
-        {"agent_mode": AGENT_MODE, "model": "a", "bugs_found": 2,
+        {"model": "a", "bugs_found": 2,
          "total_tokens_k": 1, "efficiency_bugs_per_ktok": 999},
-        {"agent_mode": AGENT_MODE, "model": "b", "bugs_found": 2,
+        {"model": "b", "bugs_found": 2,
          "total_tokens_k": 1000, "efficiency_bugs_per_ktok": 0.001},
         {"model": "a", "bugs_found": 9, "efficiency_bugs_per_ktok": 9},
     ]
     board = _dedup_best(entries)
-    assert len(board) == 3
-    top50 = [entry for entry in board if entry.get("agent_mode") == AGENT_MODE]
-    assert [entry["model"] for entry in top50] == ["a", "b"]
+    assert [(entry["model"], entry["bugs_found"]) for entry in board] == [
+        ("a", 9), ("b", 2)]
 
 
 def test_backend_official_path_keeps_private_detail_and_publishes_aggregate(tmp_path):
@@ -312,7 +304,7 @@ def test_backend_official_path_keeps_private_detail_and_publishes_aggregate(tmp_
     private = json.loads((results / "submission.json").read_text())
 
     assert private["artifact_sha256"] and "episodes" not in private
-    assert public["agent_mode"] == AGENT_MODE
+    assert "agent_mode" not in public
     assert board == [public]
     assert "episodes" not in public and "shortlist" not in public
     guard = runpy.run_path(str(
@@ -322,8 +314,7 @@ def test_backend_official_path_keeps_private_detail_and_publishes_aggregate(tmp_
     assert guard(public_file) == []
 
 
-def test_site_keeps_tracks_separate_and_top50_ties_ignore_efficiency():
+def test_site_has_one_protocol_and_no_mode_selector():
     site = (Path(__file__).parents[2] / "site" / "index.html").read_text()
-    assert 'id="lb-track"' in site
-    assert 'top50=track==="standardized-model-api"' in site
-    assert "top50?String(a.model).localeCompare" in site
+    assert 'id="lb-track"' not in site
+    assert "String(a.model).localeCompare" in site
diff --git a/benchmark/tests/test_trajectory.py b/benchmark/tests/test_trajectory.py
deleted file mode 100644
index cfcc934..0000000
--- a/benchmark/tests/test_trajectory.py
+++ /dev/null
@@ -1,96 +0,0 @@
-"""
-Tests for the normalized session trajectory archive.
-
-All tests are marked @pytest.mark.judgment.
-"""
-import json
-import pytest
-
-pytestmark = pytest.mark.judgment
-
-
-def _fake_messages() -> list[dict]:
-    return [
-        {"role": "user", "content": "find a bug in rule_x"},
-        {"role": "assistant", "content": "running pred reduce...\npred reduce src.json --to MaximumClique"},
-        {"role": "tool", "content": "pred output here"},
-    ]
-
-
-class TestSaveTrajectory:
-    def test_creates_jsonl_file(self, tmp_path):
-        from benchmark.run_mini import save_trajectory
-        out = tmp_path / "traj.jsonl"
-        save_trajectory(_fake_messages(), out)
-        assert out.exists()
-
-    def test_one_line_per_message(self, tmp_path):
-        from benchmark.run_mini import save_trajectory
-        msgs = _fake_messages()
-        out = tmp_path / "traj.jsonl"
-        save_trajectory(msgs, out)
-        lines = out.read_text(encoding="utf-8").strip().splitlines()
-        assert len(lines) == len(msgs)
-
-    def test_each_line_is_valid_json(self, tmp_path):
-        from benchmark.run_mini import save_trajectory
-        out = tmp_path / "traj.jsonl"
-        save_trajectory(_fake_messages(), out)
-        for line in out.read_text(encoding="utf-8").strip().splitlines():
-            obj = json.loads(line)
-            assert isinstance(obj, dict)
-
-    def test_each_line_has_role_and_content(self, tmp_path):
-        from benchmark.run_mini import save_trajectory
-        out = tmp_path / "traj.jsonl"
-        save_trajectory(_fake_messages(), out)
-        for line in out.read_text(encoding="utf-8").strip().splitlines():
-            obj = json.loads(line)
-            assert "role" in obj
-            assert "content" in obj
-
-    def test_creates_parent_dirs(self, tmp_path):
-        from benchmark.run_mini import save_trajectory
-        out = tmp_path / "deep" / "nested" / "traj.jsonl"
-        save_trajectory(_fake_messages(), out)
-        assert out.exists()
-
-
-class TestTrajectoryWriter:
-    """Incremental (per-step) trajectory flushing — the live view of a running session."""
-
-    def test_incremental_appends(self, tmp_path):
-        from benchmark.run_mini import TrajectoryWriter
-        out = tmp_path / "traj.jsonl"
-        w = TrajectoryWriter(out)
-        msgs = _fake_messages()
-        w.flush(msgs[:2])
-        w.flush(msgs)
-        lines = out.read_text(encoding="utf-8").strip().splitlines()
-        assert len(lines) == len(msgs)
-
-    def test_no_duplicate_lines_on_repeat_flush(self, tmp_path):
-        from benchmark.run_mini import TrajectoryWriter
-        out = tmp_path / "traj.jsonl"
-        w = TrajectoryWriter(out)
-        msgs = _fake_messages()
-        w.flush(msgs)
-        w.flush(msgs)
-        assert len(out.read_text(encoding="utf-8").strip().splitlines()) == len(msgs)
-
-    def test_starts_fresh_not_appending_stale(self, tmp_path):
-        from benchmark.run_mini import TrajectoryWriter, save_trajectory
-        out = tmp_path / "traj.jsonl"
-        save_trajectory(_fake_messages(), out)  # leftover from a previous run
-        w = TrajectoryWriter(out)
-        w.flush(_fake_messages()[:1])
-        assert len(out.read_text(encoding="utf-8").strip().splitlines()) == 1
-
-    def test_matches_final_save_format(self, tmp_path):
-        from benchmark.run_mini import TrajectoryWriter, save_trajectory
-        msgs = _fake_messages()
-        incremental = tmp_path / "inc.jsonl"
-        TrajectoryWriter(incremental).flush(msgs)
-        final = tmp_path / "final.jsonl"
-        save_trajectory(msgs, final)
-        assert incremental.read_text(encoding="utf-8") == final.read_text(encoding="utf-8")
diff --git a/benchmark/tests/test_verify_submission.py b/benchmark/tests/test_verify_submission.py
deleted file mode 100644
index ba48d58..0000000
--- a/benchmark/tests/test_verify_submission.py
+++ /dev/null
@@ -1,289 +0,0 @@
-"""
-Tests for benchmark/verify_submission.py — the authoritative backend scorer.
-
-Unit tests monkeypatch ``verify`` so they need no pred binary; integration tests wrap
-the real certificate fixtures and require pred.
-"""
-import json
-from pathlib import Path
-
-import pytest
-
-from benchmark import verify_submission as vs
-from benchmark.verify import Verdict
-
-FIXTURES = Path(__file__).parent / "fixtures"
-
-
-def _traj(cert: dict) -> list[dict]:
-    """A minimal trajectory that reproduces ``cert`` — satisfies the provenance gate."""
-    return [{"role": "assistant",
-             "content": "CERTIFICATE_START\n" + json.dumps(cert) + "\nCERTIFICATE_END"}]
-
-
-def _bug_row(cert: dict, **over) -> dict:
-    row = {"rule": cert.get("rule", "r"), "result": "bug_found",
-           "tokens_k": 10.0, "certificate": cert, "trajectory": _traj(cert)}
-    row.update(over)
-    return row
-
-
-def _submission(results, **over) -> dict:
-    base = {
-        "model": "anthropic/test",
-        "library_commit": "deadbeef",
-        "bugs_found": 999,  # deliberately wrong — the scorer must ignore it
-        "total_tokens_k": 50.0,
-        "rules_tested": len(results),
-        "results": results,
-    }
-    base.update(over)
-    return base
-
-
-# ── scoring logic (no pred — verify monkeypatched) ────────────────────────────
-
-class TestScoreSubmission:
-    def test_ignores_self_reported_count(self, monkeypatch):
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok"))
-        sub = _submission([
-            _bug_row({"rule": "r1", "violation": "solve_mismatch", "source": {}, "bundle": {}}),
-        ], bugs_found=999)
-        scored, _ = vs.score_submission(sub)
-        assert scored["bugs_found"] == 1  # not 999
-
-    def test_rejected_certificate_does_not_count(self, monkeypatch):
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(False, "nope"))
-        sub = _submission([
-            _bug_row({"rule": "r1", "violation": "solve_mismatch", "source": {}, "bundle": {}}),
-        ])
-        scored, report = vs.score_submission(sub)
-        assert scored["bugs_found"] == 0
-        assert scored["results"][0]["result"] == "rejected"
-        assert scored["results"][0]["reject_reason"] == "nope"
-        assert report[0]["accepted"] is False
-
-    def test_no_trajectory_not_counted(self, monkeypatch):
-        # pred confirms the round-trip failure, but no trajectory is attached → the bug is
-        # not scored (provenance gate): a pasted answer key must not count.
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok"))
-        sub = _submission([
-            {"rule": "r1", "result": "bug_found", "tokens_k": 10.0,
-             "certificate": {"rule": "r1", "violation": "solve_mismatch", "source": {}, "bundle": {}}},
-        ])
-        scored, report = vs.score_submission(sub)
-        assert scored["bugs_found"] == 0
-        assert scored["results"][0]["result"] == "rejected"
-        assert "provenance" in scored["results"][0]["reject_reason"]
-        assert report[0]["accepted"] is False
-
-    def test_trajectory_source_mismatch_not_counted(self, monkeypatch):
-        # A trajectory whose certificate names a different source than the submitted one is
-        # not a valid provenance proof.
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok"))
-        cert = {"rule": "r1", "violation": "solve_mismatch", "source": {"n": 1}, "bundle": {}}
-        other = {"rule": "r1", "violation": "solve_mismatch", "source": {"n": 999}, "bundle": {}}
-        sub = _submission([_bug_row(cert, trajectory=_traj(other))])
-        scored, _ = vs.score_submission(sub)
-        assert scored["bugs_found"] == 0
-
-    def test_multi_cert_trajectory_provenance(self, monkeypatch):
-        # A whole-repo session emits MANY certificates in one shared trajectory. Each bug row
-        # must pass provenance against ANY matching block — not just the last one.
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok"))
-        c1 = {"rule": "r1", "violation": "solve_mismatch", "source": {"n": 1}, "bundle": {}}
-        c2 = {"rule": "r2", "violation": "solve_mismatch", "source": {"n": 2}, "bundle": {}}
-        shared = _traj(c1) + _traj(c2)  # both certs in the one session trajectory
-        sub = _submission([_bug_row(c1, trajectory=shared),
-                           _bug_row(c2, trajectory=shared)])
-        scored, _ = vs.score_submission(sub)
-        assert scored["bugs_found"] == 2  # both counted, even though c1 is not the last block
-        assert all(r["result"] == "bug_found" for r in scored["results"])
-
-    def test_envelope_trajectory_provenance(self, monkeypatch):
-        # whole-repo: rows carry NO trajectory; the shared session log is on the envelope,
-        # parsed once. Each bug's cert is matched against it.
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok"))
-        c1 = {"rule": "r1", "violation": "solve_mismatch", "source": {"n": 1}, "bundle": {}}
-        c2 = {"rule": "r2", "violation": "solve_mismatch", "source": {"n": 2}, "bundle": {}}
-        rows = [{"rule": "r1", "result": "bug_found", "tokens_k": 0.0, "certificate": c1},
-                {"rule": "r2", "result": "bug_found", "tokens_k": 0.0, "certificate": c2}]
-        sub = _submission(rows, trajectory=_traj(c1) + _traj(c2))
-        scored, _ = vs.score_submission(sub)
-        assert scored["bugs_found"] == 2
-
-    def test_no_trajectory_anywhere_not_counted(self, monkeypatch):
-        # A cert with neither a row trajectory nor an envelope trajectory fails provenance.
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok"))
-        c1 = {"rule": "r1", "violation": "solve_mismatch", "source": {}, "bundle": {}}
-        rows = [{"rule": "r1", "result": "bug_found", "tokens_k": 0.0, "certificate": c1}]
-        scored, _ = vs.score_submission(_submission(rows))
-        assert scored["bugs_found"] == 0
-
-    def test_new_submission_requires_bounded_submit_ledger(self, monkeypatch):
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok"))
-        cert = {"rule": "r1", "source": {"n": 1}, "bundle": {}}
-        sub = _submission(
-            [_bug_row(cert)],
-            submit_limit=100,
-            submit_log=[{"attempt": 1, "accepted": True, "rule": "r1",
-                         "reason": "ok", "certificate": cert}],
-        )
-        scored, report = vs.score_submission(sub)
-        assert scored["bugs_found"] == 1
-        assert "bounded submit command" in report[0]["provenance"]
-
-    def test_final_answer_certificate_not_in_submit_ledger_is_rejected(self, monkeypatch):
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok"))
-        submitted = {"rule": "r1", "source": {"n": 1}, "bundle": {}}
-        only_in_prose = {"rule": "r2", "source": {"n": 2}, "bundle": {}}
-        sub = _submission(
-            [_bug_row(only_in_prose)],
-            submit_limit=100,
-            submit_log=[{"attempt": 1, "accepted": True, "rule": "r1",
-                         "reason": "ok", "certificate": submitted}],
-        )
-        scored, _ = vs.score_submission(sub)
-        assert scored["bugs_found"] == 0
-        assert "bounded submit command" in scored["results"][0]["reject_reason"]
-
-    def test_tampered_budget_log_fails_closed(self, monkeypatch):
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok"))
-        cert = {"rule": "r1", "source": {"n": 1}, "bundle": {}}
-        sub = _submission(
-            [_bug_row(cert)],
-            submit_limit=0,
-            submit_log=[{"attempt": 1, "accepted": True, "rule": "r1",
-                         "reason": "ok", "certificate": cert}],
-        )
-        scored, _ = vs.score_submission(sub)
-        assert scored["bugs_found"] == 0
-        assert "exceeds submit_limit" in scored["results"][0]["reject_reason"]
-
-    def test_ledger_certificate_cannot_be_counted_under_another_row_rule(self, monkeypatch):
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok"))
-        cert = {"rule": "r1", "source": {"n": 1}, "bundle": {}}
-        forged_row = _bug_row(cert, rule="r2")
-        sub = _submission(
-            [forged_row], submit_limit=1,
-            submit_log=[{"attempt": 1, "accepted": True, "rule": "r1",
-                         "reason": "ok", "certificate": cert}],
-        )
-        scored, _ = vs.score_submission(sub)
-        assert scored["bugs_found"] == 0
-        assert "row rule" in scored["results"][0]["reject_reason"]
-
-    def test_distinct_rule_dedup(self, monkeypatch):
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok"))
-        cert = lambda rule, v: {"rule": rule, "violation": v, "source": {"r": rule}, "bundle": {}}
-        sub = _submission([
-            _bug_row(cert("r1", "solve_mismatch")),
-            _bug_row(cert("r1", "unsound_extraction")),
-            _bug_row(cert("r2", "solve_mismatch")),
-        ])
-        scored, _ = vs.score_submission(sub)
-        assert scored["bugs_found"] == 2  # {r1, r2}, two certs on r1 collapse
-
-    def test_rows_without_certificate_passthrough(self, monkeypatch):
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok"))
-        sub = _submission([
-            {"rule": "r1", "result": "no_certificate", "tokens_k": 5.0},
-        ])
-        scored, report = vs.score_submission(sub)
-        assert scored["bugs_found"] == 0
-        assert report == []
-
-    def test_recomputes_tokens_from_usage_totals(self, monkeypatch):
-        # Zero-trust tokens: the submission self-reports a bogus total_tokens_k, but carries
-        # usage_totals — the scorer recomputes tokens from the 4-bucket primitive and ignores
-        # the bogus figure (mirrors ignoring self-reported bugs_found).
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok"))
-        sub = _submission(
-            [_bug_row({"rule": "r1", "violation": "solve_mismatch", "source": {}, "bundle": {}})],
-            total_tokens_k=0.001,  # bogus — must be ignored
-            usage_totals={"input": 1_000_000, "output": 1_000_000,
-                          "cache_read": 0, "cache_write": 0},
-        )
-        scored, _ = vs.score_submission(sub)
-        assert scored["total_tokens_k"] == 2000.0
-        assert scored["efficiency_bugs_per_ktok"] == round(1 / 2000.0, 4)
-        # the primitive survives into the scored result (→ leaderboard entry)
-        assert scored["usage_totals"]["input"] == 1_000_000
-
-    def test_legacy_submission_without_usage_uses_self_reported(self, monkeypatch):
-        # No usage_totals (legacy) → fall back to the self-reported figure.
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok"))
-        sub = _submission(
-            [_bug_row({"rule": "r1", "violation": "solve_mismatch", "source": {}, "bundle": {}})],
-            total_tokens_k=50.0)
-        scored, _ = vs.score_submission(sub)
-        assert scored["total_tokens_k"] == 50.0
-
-    def test_scored_is_results_schema_shaped(self, monkeypatch):
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok"))
-        sub = _submission([
-            _bug_row({"rule": "r1", "violation": "solve_mismatch", "source": {}, "bundle": {}}),
-        ])
-        scored, _ = vs.score_submission(sub)
-        for field in ("model", "library_commit", "bugs_found",
-                      "total_tokens_k", "efficiency_bugs_per_ktok",
-                      "rules_tested", "results"):
-            assert field in scored
-
-
-# ── leaderboard entry ─────────────────────────────────────────────────────────
-
-class TestLeaderboardEntry:
-    def test_aggregate_only_no_certificates(self, monkeypatch):
-        # The public entry must carry counts/tokens but NEVER the certificates or the
-        # identities of the buggy rules — publishing those is a free answer key.
-        monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok"))
-        sub = _submission([
-            _bug_row({"rule": "r1", "violation": "solve_mismatch",
-                      "source": {"type": "A"}, "bundle": {"target": {"type": "B"}}, "note": "x"}),
-        ])
-        scored, _ = vs.score_submission(sub)
-        entry = vs.leaderboard_entry(sub, scored)
-        assert entry["placeholder"] is False
-        assert entry["bugs_found"] == 1
-        # no per-bug drilldown, no rule identities, no certificate fields leak out
-        assert "bug_certificates" not in entry
-        blob = json.dumps(entry)
-        assert "r1" not in blob and "solve_mismatch" not in blob and "source" not in blob
-
-
-# ── integration: real fixtures + pred ─────────────────────────────────────────
-
-@pytest.mark.integration
-class TestRealVerification:
-    def _wrap(self, fixture_name: str) -> dict:
-        cert = json.loads((FIXTURES / fixture_name).read_text(encoding="utf-8"))
-        return _submission([_bug_row(cert)])
-
-    def test_genuine_bug_is_confirmed(self):
-        # Accept path: a real reduction bug (weighted MIS -> IntegralFlowBundles) is
-        # confirmed end-to-end and counted. The fixture is the answer key, so it lives in
-        # the gitignored private dir; skip when absent (e.g. a fresh public clone).
-        from benchmark.verify import PRIVATE_FIXTURES_DIR
-        path = PRIVATE_FIXTURES_DIR / "genuine_bug_weighted_mis.json"
-        if not path.exists():
-            pytest.skip(f"private accept-path fixture absent: {path}")
-        cert = json.loads(path.read_text(encoding="utf-8"))
-        scored, report = vs.score_submission(_submission([_bug_row(cert)]))
-        assert scored["bugs_found"] == 1
-        assert report[0]["accepted"] is True
-
-    def test_valid_bug_is_not_a_bug_under_roundtrip(self):
-        # Re-classified: the old "valid_bug" fixture used a non-optimal target_config; the
-        # round-trip recovers the optimum, so it is NOT a bug.
-        scored, report = vs.score_submission(self._wrap("valid_bug.json"))
-        assert scored["bugs_found"] == 0
-        assert report[0]["accepted"] is False
-
-    def test_wrong_target_rejected(self):
-        scored, _ = vs.score_submission(self._wrap("wrong_target.json"))
-        assert scored["bugs_found"] == 0
-
-    def test_false_alarm_rejected(self):
-        scored, _ = vs.score_submission(self._wrap("valid_solution_claimed_invalid.json"))
-        assert scored["bugs_found"] == 0
diff --git a/benchmark/tests/test_whole_repo.py b/benchmark/tests/test_whole_repo.py
deleted file mode 100644
index 7eeb1e4..0000000
--- a/benchmark/tests/test_whole_repo.py
+++ /dev/null
@@ -1,85 +0,0 @@
-"""Tests for the whole-repo runner mode (one bounded-submit session for the library)."""
-
-import subprocess
-
-from benchmark import run_submission
-
-
-class TestBuildSubmissionTotals:
-    def test_explicit_session_totals_override_row_sums(self):
-        # whole-repo rows carry 0 tokens; the session total comes in as an explicit arg.
-        rows = [{"rule": "r1", "result": "bug_found", "tokens_k": 0.0}]
-        sub = run_submission.build_submission(
-            "m", rows, library_commit="c", total_tokens_k=42.0)
-        assert sub["total_tokens_k"] == 42.0
-        assert sub["efficiency_bugs_per_ktok"] == round(1 / 42.0, 4)
-
-
-class TestCrashSalvage:
-    def test_run_error_recorded_when_session_dies(self, monkeypatch):
-        # A fatal session error must still produce a submission (partial salvage) tagged with
-        # run_error — not crash and leave a stale submission.json.
-        def dying_session(model, ctx, **kw):
-            return {"tokens_k": 5.0, "usage": None,
-                    "error": "APIError: quota exhausted"}
-        monkeypatch.setattr(run_submission, "find_pred_binary", lambda: "pred")
-        monkeypatch.setattr(run_submission, "verify_pred_version", lambda p: "1.2.3")
-        monkeypatch.setattr(run_submission, "EnvContext",
-                            lambda **kw: type("C", (), {"pred_version": "1.2.3", **kw})())
-        monkeypatch.setattr("benchmark.run_mini.run_repo_session", dying_session)
-        sub = run_submission.run("m", "/repo", library_commit="c")
-        assert sub["run_error"] == "APIError: quota exhausted"
-        assert sub["bugs_found"] == 0
-
-    def test_no_run_error_key_on_clean_run(self):
-        sub = run_submission.build_submission("m", [], library_commit="c")
-        assert "run_error" not in sub
-
-    def test_unprobed_submit_channel_is_not_a_clean_zero(self, monkeypatch, tmp_path):
-        def clean_but_unprobed(model, ctx, **kw):
-            return {"tokens_k": 1.0, "usage": None, "error": None}
-
-        monkeypatch.setattr(run_submission, "find_pred_binary", lambda: "pred")
-        monkeypatch.setattr(run_submission, "verify_pred_version", lambda p: "1.2.3")
-        monkeypatch.setattr(run_submission, "EnvContext",
-                            lambda **kw: type("C", (), {"pred_version": "1.2.3", **kw})())
-        monkeypatch.setattr("benchmark.run_mini.run_repo_session", clean_but_unprobed)
-        sub = run_submission.run("m", "/repo", library_commit="c",
-                                 trajectory_dir=tmp_path / "logs")
-        assert "submit channel was not successfully probed" in sub["run_error"]
-
-    def test_successful_status_probe_allows_a_clean_zero(self, monkeypatch):
-        def clean_and_probed(model, ctx, **kw):
-            status = subprocess.run(["submit", "--status"], capture_output=True, text=True)
-            assert status.returncode == 0
-            return {"tokens_k": 1.0, "usage": None, "error": None}
-
-        monkeypatch.setattr(run_submission, "find_pred_binary", lambda: "pred")
-        monkeypatch.setattr(run_submission, "verify_pred_version", lambda p: "1.2.3")
-        monkeypatch.setattr(run_submission, "EnvContext",
-                            lambda **kw: type("C", (), {"pred_version": "1.2.3", **kw})())
-        monkeypatch.setattr("benchmark.run_mini.run_repo_session", clean_and_probed)
-        sub = run_submission.run("m", "/repo", library_commit="c")
-        assert "run_error" not in sub
-
-
-class TestRunWholeRepoWiring:
-    def test_run_dispatches_to_repo_session(self, monkeypatch):
-        # run() must call the single repository session and build its envelope.
-        def fake_repo_session(model, ctx, **kw):
-            kw["submit_session"].result_rows = lambda: [
-                {"rule": "r1", "result": "bug_found", "tokens_k": 0.0,
-                 "certificate": {"rule": "r1", "source": {}}}]
-            return {"tokens_k": 30.0, "usage": None, "error": None}
-
-        monkeypatch.setattr(run_submission, "find_pred_binary", lambda: "pred")
-        monkeypatch.setattr(run_submission, "verify_pred_version", lambda p: "1.2.3")
-        monkeypatch.setattr(run_submission, "EnvContext",
-                            lambda **kw: type("C", (), {"pred_version": "1.2.3", **kw})())
-        monkeypatch.setattr("benchmark.run_mini.run_repo_session", fake_repo_session)
-
-        sub = run_submission.run("m", "/repo", library_commit="deadbeef")
-        assert sub["total_tokens_k"] == 30.0
-        assert sub["bugs_found"] == 1
-        assert "trajectory" not in sub                   # logs live in trajectory_dir
-        assert "trajectory" not in sub["results"][0]
diff --git a/benchmark/top50_contract.py b/benchmark/top50_contract.py
index 74042cc..e060442 100644
--- a/benchmark/top50_contract.py
+++ b/benchmark/top50_contract.py
@@ -24,8 +24,7 @@
 from benchmark.usage import Usage, usage_as_dict, usage_from_dict
 from benchmark.verify import Verdict, verify
 
-AGENT_MODE = "standardized-model-api"
-RUNNER_VERSION = "0.10.0"
+RUNNER_VERSION = "0.11.0"
 EXPECTED_TRIAGE_BUDGET = TRIAGE_BUDGET
 EXPECTED_EPISODE_BUDGET = EPISODE_BUDGET
 EXPECTED_SAFETY_CONTROLS = SAFETY_CONTROLS
@@ -35,14 +34,15 @@
 EXPECTED_HYPOTHESIS_CHARS = HYPOTHESIS_CHARS
 _REQUEST_ID = re.compile(r"[0-9a-f]{32}")
 _COUNTERS = ("model_generations", "shell_actions", "pred_calls", "solve_calls")
-
-
-def is_top50_submission(submission: object) -> bool:
-    return isinstance(submission, dict) and submission.get("agent_mode") == AGENT_MODE
+_TOP_LEVEL_FIELDS = {
+    "model", "library_commit", "runner_version", "pred_version", "prompt_id",
+    "status", "rankable", "safety_controls", "shortlist", "triage", "episodes",
+    "inference_parameters", "created_at", "submitted_by", "test", "run_error",
+}
 
 
 def expected_prompt_id() -> str:
-    return hashlib.sha256(Path(__file__).with_name("top50_config.yaml").read_bytes()).hexdigest()
+    return hashlib.sha256(Path(__file__).with_name("agent_config.yaml").read_bytes()).hexdigest()
 
 
 def validate_top50_submission(
@@ -52,19 +52,16 @@ def validate_top50_submission(
     if not isinstance(submission, dict):
         return ["submission is not a JSON object"]
     problems: list[str] = []
-    for field in ("model", "library_commit", "runner_version", "pred_version", "agent_mode",
+    for field in ("model", "library_commit", "runner_version", "pred_version",
                   "prompt_id", "shortlist", "triage", "episodes", "status",
                   "safety_controls", "inference_parameters"):
         if field not in submission:
             problems.append(f"missing required field: {field}")
     if problems:
         return problems
-    for obsolete in ("benchmark_contract", "budget_contract_status", "contract",
-                     "observation_policy"):
-        if obsolete in submission:
-            problems.append(f"obsolete self-declared field is not allowed: {obsolete}")
-    if submission["agent_mode"] != AGENT_MODE:
-        problems.append(f"agent_mode must be {AGENT_MODE!r}")
+    extra = sorted(set(submission) - _TOP_LEVEL_FIELDS)
+    if extra:
+        problems.append(f"unsupported top-level field(s): {extra}")
     if submission["runner_version"] != RUNNER_VERSION:
         problems.append(f"runner_version must be {RUNNER_VERSION!r}")
     if submission.get("safety_controls") != EXPECTED_SAFETY_CONTROLS:
@@ -84,9 +81,6 @@ def validate_top50_submission(
         problems.append("inference_parameters differ from the standardized model settings")
     if submission.get("prompt_id") != expected_prompt_id():
         problems.append("prompt_id does not match the frozen Top50 prompt")
-    if "submit_limit" in submission or "submit_log" in submission:
-        problems.append("Top50 artifacts cannot use a shared run-wide submit pool")
-
     triage_budget = EXPECTED_TRIAGE_BUDGET
     episode_budget = EXPECTED_EPISODE_BUDGET
     hypothesis_limit = EXPECTED_HYPOTHESIS_CHARS
@@ -218,7 +212,6 @@ def score_top50_submission(
         "library_commit": submission.get("library_commit", "unknown"),
         "runner_version": submission.get("runner_version"),
         "pred_version": submission.get("pred_version"),
-        "agent_mode": submission.get("agent_mode"),
         "rankable": not problems,
         "rankability_errors": problems,
         "verified_bugs": bugs,
@@ -249,7 +242,6 @@ def top50_public_entry(submission: dict, scored: dict) -> dict:
         "library_commit": scored["library_commit"],
         "runner_version": scored.get("runner_version"),
         "pred_version": scored.get("pred_version"),
-        "agent_mode": scored.get("agent_mode"),
         "rankable": scored["rankable"],
         "bugs_found": scored["verified_bugs"],
         "rules_tested": 50,
@@ -518,7 +510,7 @@ def load_canonical_inventory(repo_dir: str | None) -> set[str]:
     candidate = Path(repo_dir or os.environ.get("REPO_DIR", "/app/pr-src"))
     if not (candidate / "src" / "rules").is_dir():
         raise RuntimeError(f"pinned canonical rule inventory is unavailable under {candidate}")
-    from benchmark.run_mini import list_rules
+    from benchmark.model_api import list_rules
     return set(list_rules(candidate))
 
 
diff --git a/benchmark/top50_results.schema.json b/benchmark/top50_results.schema.json
deleted file mode 100644
index 98817be..0000000
--- a/benchmark/top50_results.schema.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "$id": "top50_results.schema.json",
-  "title": "Top50EvidenceScoredResult",
-  "type": "object",
-  "required": ["model", "library_commit", "agent_mode", "rankable",
-    "rankability_errors", "verified_bugs", "bugs_at_10", "bugs_at_25", "bugs_at_50",
-    "first_attempt_accepts", "second_attempt_accepts", "usage_totals",
-    "artifact_sha256", "verifier_report"],
-  "properties": {
-    "model": {"type": "string"},
-    "library_commit": {"type": "string"},
-    "agent_mode": {"const": "standardized-model-api"},
-    "rankable": {"type": "boolean"},
-    "rankability_errors": {"type": "array", "items": {"type": "string"}},
-    "verified_bugs": {"type": "integer", "minimum": 0, "maximum": 50},
-    "bugs_at_10": {"type": "integer", "minimum": 0, "maximum": 10},
-    "bugs_at_25": {"type": "integer", "minimum": 0, "maximum": 25},
-    "bugs_at_50": {"type": "integer", "minimum": 0, "maximum": 50},
-    "first_attempt_accepts": {"type": "integer", "minimum": 0},
-    "second_attempt_accepts": {"type": "integer", "minimum": 0},
-    "usage_totals": {"type": "object"},
-    "artifact_sha256": {"type": "string", "minLength": 64, "maxLength": 64},
-    "verifier_report": {"type": "array"}
-  }
-}
diff --git a/benchmark/top50_runner.py b/benchmark/top50_runner.py
index fd4ab7d..fce9d54 100644
--- a/benchmark/top50_runner.py
+++ b/benchmark/top50_runner.py
@@ -16,12 +16,12 @@
                                          run_as_agent, sanitized_agent_env)
 from benchmark.evidence_budget import EvidenceBudget, EvidenceBudgetSession, EvidenceBudgetState
 from benchmark.observation_policy import ObservationConfig, ObservationStore
-from benchmark.run_mini import (
+from benchmark.model_api import (
     DEFAULT_MAX_TOKENS,
-    _build_model,
-    _load_agent_config,
-    _message_text,
-    _session_usage,
+    build_model,
+    load_agent_config,
+    message_text,
+    session_usage,
 )
 from benchmark.top50_budget import (
     EPISODE_BUDGET,
@@ -405,7 +405,7 @@ def build_rankable_runner(
     executor = MiniSwePhaseExecutor(
         api_base=api_base, api_key=api_key, max_tokens=inference["max_tokens"],
         model_timeout_seconds=safety["model_timeout_seconds"],
-        model_retries=safety["model_retries"], model_kwargs=None,
+        model_retries=safety["model_retries"],
         agent_uid=agent_uid, agent_gid=agent_gid, evidence_gid=evidence_gid)
     return Top50Runner(
         executor=executor, contract=benchmark_limits(), pred_binary=pred_binary,
@@ -508,22 +508,19 @@ class MiniSwePhaseExecutor:
     """The sole rankable mini-swe/LiteLLM implementation of the phase protocol."""
 
     def __init__(self, *, api_base: str | None = None, api_key: str | None = None,
-                 max_tokens: int = DEFAULT_MAX_TOKENS, model_kwargs: dict | None = None,
+                 max_tokens: int = DEFAULT_MAX_TOKENS,
                  model_timeout_seconds: int = 300, model_retries: int = 2,
                  agent_uid: int | None = None, agent_gid: int | None = None,
                  evidence_gid: int | None = None):
         self.api_base = api_base
         self.api_key = api_key
         self.max_tokens = max_tokens
-        self.model_kwargs = model_kwargs
         self.model_timeout_seconds = model_timeout_seconds
         self.model_retries = model_retries
         self.agent_uid = os.getuid() if agent_uid is None else agent_uid
         self.agent_gid = os.getgid() if agent_gid is None else agent_gid
         self.evidence_gid = evidence_gid
-        config_path = Path(__file__).with_name("top50_config.yaml")
-        self.agent_config, self.model_config, _ = _load_agent_config(
-            config_path, config_path, "", force_unlimited=False)
+        self.agent_config, self.model_config = load_agent_config()
         self._models: dict[str, object] = {}
 
     def run_triage(self, session: TriageSession, *, repo_path: Path,
@@ -558,9 +555,9 @@ def _run_agent(self, model_name: str, environment, session, *, task: str,
 
         model = self._models.get(model_name)
         if model is None:
-            model = _build_model(
+            model = build_model(
                 model_name, self.api_base, self.max_tokens,
-                model_kwargs=self.model_kwargs, api_key=self.api_key,
+                api_key=self.api_key,
                 observation_template=self.model_config.get("observation_template"),
                 format_error_template=self.model_config.get("format_error_template"),
                 model_timeout_seconds=self.model_timeout_seconds,
@@ -624,8 +621,8 @@ def execute_actions(self, message):
             agent.run(task=task)
         except Exception as exception:
             error = f"{type(exception).__name__}: {exception}"
-        tokens_k, usage = _session_usage(agent)
-        messages = [{"role": message.get("role", ""), "content": _message_text(message)}
+        tokens_k, usage = session_usage(agent)
+        messages = [{"role": message.get("role", ""), "content": message_text(message)}
                     for message in agent.messages]
         return PhaseResult(messages=messages, tokens_k=tokens_k,
                            usage=usage, error=error)
diff --git a/benchmark/top50_submission.schema.json b/benchmark/top50_submission.schema.json
deleted file mode 100644
index c987a66..0000000
--- a/benchmark/top50_submission.schema.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "$id": "top50_submission.schema.json",
-  "title": "Top50EvidenceSubmission",
-  "type": "object",
-  "required": ["model", "library_commit", "runner_version",
-    "pred_version", "agent_mode", "prompt_id", "status", "safety_controls", "shortlist",
-    "triage", "episodes", "inference_parameters"],
-  "properties": {
-    "model": {"type": "string", "minLength": 1},
-    "library_commit": {"type": "string", "minLength": 1},
-    "runner_version": {"const": "0.10.0"},
-    "pred_version": {"type": "string", "minLength": 1},
-    "agent_mode": {"const": "standardized-model-api"},
-    "prompt_id": {"type": "string", "minLength": 64, "maxLength": 64},
-    "status": {"enum": ["completed", "run_error"]},
-    "safety_controls": {"const": {"model_timeout_seconds": 300, "model_retries": 2}},
-    "rankable": {"type": "boolean"},
-    "shortlist": {"type": "array", "minItems": 50, "maxItems": 50},
-    "triage": {"type": "object"},
-    "episodes": {"type": "array", "minItems": 50, "maxItems": 50},
-    "inference_parameters": {"const": {"max_tokens": 8192, "timeout": 300, "num_retries": 2}},
-    "run_error": {"type": "string"}
-  },
-  "not": {"anyOf": [
-    {"required": ["benchmark_contract"]},
-    {"required": ["budget_contract_status"]},
-    {"required": ["contract"]},
-    {"required": ["observation_policy"]}
-  ]}
-}
diff --git a/benchmark/verify_submission.py b/benchmark/verify_submission.py
index 9b4b4fb..b3d9419 100644
--- a/benchmark/verify_submission.py
+++ b/benchmark/verify_submission.py
@@ -1,209 +1,34 @@
 #!/usr/bin/env python3
-"""
-Authoritative backend scorer for a submission.json.
+"""Re-verify and score one benchmark submission artifact."""
+from __future__ import annotations
 
-Zero trust: re-runs the certificate verifier (benchmark/verify.py → pred) on every
-claimed bug and recomputes the score from what pred actually confirms. The submission's
-self-reported ``bugs_found`` is ignored entirely.
-
-Produces two views of the result:
-  * ``scored``          — results.schema.json-compatible (the backend's per-submission output)
-  * ``leaderboard_entry`` — the public ranked-row shape (aggregate only: counts, tokens,
-                            efficiency — never the certificates or buggy-rule identities,
-                            which would be a free answer key)
-
-CLI:
-    python -m benchmark.verify_submission  [--repo-dir ]
-    # prints the per-certificate verdict report + the recomputed score; exits 0 always
-    # (a submission with 0 confirmed bugs is a valid, scored result — not an error)
-"""
 import argparse
 import json
-import re
 from pathlib import Path
 
-from benchmark.submit_ledger import (accepted_certificate_index, certificate_key,
-                                     has_submit_ledger, submit_ledger_error)
-from benchmark.usage import usage_from_dict
-from benchmark.verify import count_bugs, verify
-
-CERT_BLOCK = re.compile(r"CERTIFICATE_START\s*\n(.*?)CERTIFICATE_END", re.DOTALL)
-
-
-def _certs_from_trajectory(trajectory) -> list[dict]:
-    """Parse every CERTIFICATE_START…END block emitted in an agent trajectory.
-
-    ``trajectory`` is a list of {role, content} messages (as saved by the runner). A
-    whole-repo run emits many certificates in one trajectory, so return all of them;
-    unparseable blocks are skipped.
-    """
-    certs: list[dict] = []
-    for msg in trajectory or []:
-        content = msg.get("content", "") or ""
-        for m in CERT_BLOCK.finditer(content):
-            try:
-                certs.append(json.loads(m.group(1).strip()))
-            except json.JSONDecodeError:
-                continue
-    return certs
-
-
-def _provenance_ok(row: dict, cert: dict, session_certs: list[dict] = ()) -> tuple[bool, str]:
-    """Check the certificate was actually produced by this model's own run.
-
-    Guards against copied answer keys: a scored bug must appear as a CERTIFICATE block the
-    agent emitted in its trajectory, matching both rule and source instance. The trajectory
-    is either the legacy row's own or the shared legacy session log at the envelope level —
-    ``session_certs`` is that envelope
-    log pre-parsed once. Any emitted block that matches counts. This can't make copying
-    impossible (the library is public), but it lifts the bar from "paste a rule name" to
-    "produce a run artifact whose source still round-trip-fails".
-    """
-    if row.get("rule") != cert.get("rule"):
-        return False, "result row rule does not match its certificate rule"
-    emitted = _certs_from_trajectory(row.get("trajectory")) + list(session_certs)
-    if not emitted:
-        return False, "no trajectory attached (required for a scored bug)"
-    for e in emitted:
-        if e.get("rule") == cert.get("rule") and e.get("source") == cert.get("source"):
-            return True, "reproduced in the model's own trajectory"
-    return False, "no trajectory certificate matches the submitted rule + source"
+from benchmark.top50_contract import score_top50_submission, top50_public_entry
 
 
 def score_submission(submission: dict, repo_dir: str | None = None) -> tuple[dict, list[dict]]:
-    """Re-verify every certificate and recompute the score.
-
-    A bug counts only when pred confirms the round-trip failure and provenance succeeds:
-    bounded submit ledger for current submissions, trajectory certificate for legacy data.
-    Returns (scored, report); ``scored`` is results.schema.json-shaped and ``report`` is a
-    per-certificate list of {rule, violation, accepted, reason, provenance}.
-    """
-    from benchmark.top50_contract import is_top50_submission, score_top50_submission
-    if is_top50_submission(submission):
-        return score_top50_submission(submission, repo_dir)
-
-    rescored: list[dict] = []
-    report: list[dict] = []
-    ledger_problem = submit_ledger_error(submission)
-    uses_ledger = has_submit_ledger(submission)
-    accepted_keys = (accepted_certificate_index(submission)
-                     if uses_ledger and ledger_problem is None else set())
-    # Only legacy submissions use trajectory provenance. New runs use the ledger directly.
-    session_certs = ([] if uses_ledger else
-                     _certs_from_trajectory(submission.get("trajectory")))
-
-    for row in submission.get("results", []):
-        cert = row.get("certificate")
-        if not cert:
-            rescored.append(row)
-            continue
-
-        cert.setdefault("rule", row.get("rule"))
-        verdict = verify(cert, repo_dir)
-        if verdict.accepted and uses_ledger:
-            if ledger_problem is not None:
-                prov_ok, prov_reason = False, ledger_problem
-            elif row.get("rule") != cert.get("rule"):
-                prov_ok, prov_reason = False, "result row rule does not match its certificate rule"
-            else:
-                prov_ok = certificate_key(row["rule"], cert) in accepted_keys
-                prov_reason = ("accepted through the bounded submit command" if prov_ok else
-                               "certificate was not accepted through the bounded submit command")
-        elif verdict.accepted:
-            prov_ok, prov_reason = _provenance_ok(row, cert, session_certs)
-        else:
-            prov_ok, prov_reason = False, ""
-        accepted = verdict.accepted and prov_ok
-        new = dict(row)
-        if accepted:
-            new["result"] = "bug_found"
-            new["verify_details"] = verdict.details
-            new.pop("reject_reason", None)
-        else:
-            new["result"] = "rejected"
-            new["reject_reason"] = (
-                verdict.reason if not verdict.accepted else f"provenance: {prov_reason}")
-            new.pop("verify_details", None)
-        rescored.append(new)
-        report.append({
-            "rule": row.get("rule"),
-            "violation": cert.get("violation"),
-            "accepted": accepted,
-            "reason": new.get("reject_reason") if not accepted else verdict.reason,
-            "provenance": prov_reason if verdict.accepted else None,
-        })
-
-    bugs = count_bugs(rescored)
-    # Token totals: recompute from the 4-bucket ``usage_totals`` when present (the
-    # reproducible primitive); legacy submissions without it fall back to their
-    # self-reported total_tokens_k.
-    total_tokens = usage_from_dict(submission.get("usage_totals")).total_tokens
-    tokens_k = total_tokens / 1000 if total_tokens else (submission.get("total_tokens_k", 0.0) or 0.0)
-    scored = {
-        "model": submission.get("model", "unknown"),
-        "library_commit": submission.get("library_commit", "unknown"),
-        # Test submissions (end-to-end checks) are scored and stored privately like any
-        # other, but aggregate_leaderboard() excludes them so they never reach the public
-        # board. Carried through here so the flag survives in the private scored file.
-        "test": bool(submission.get("test")),
-        "bugs_found": bugs,
-        "total_tokens_k": round(tokens_k, 2),
-        # The reproducible primitive behind total_tokens_k — carried through.
-        "usage_totals": submission.get("usage_totals"),
-        "efficiency_bugs_per_ktok": round(bugs / tokens_k, 4) if tokens_k else 0,
-        "rules_tested": submission.get("rules_tested", len(rescored)),
-        "results": rescored,
-        "submit_limit": submission.get("submit_limit"),
-        "submit_log": submission.get("submit_log"),
-    }
-    # Private operational/provenance metadata must survive scoring. In particular,
-    # backend_score rebuilds public entries from these scored files later, so dropping
-    # submitted_by here silently erased the submitter. run_error remains private and is used
-    # by the official intake gate; test runs may retain it for diagnosis.
-    for key in ("submitted_by", "created_at", "run_error", "pred_version", "agent_mode"):
-        if key in submission:
-            scored[key] = submission[key]
-    return scored, report
+    """Stable scorer entrypoint for the benchmark's single submission protocol."""
+    return score_top50_submission(submission, repo_dir)
 
 
 def leaderboard_entry(submission: dict, scored: dict) -> dict:
-    """Build the public ranked-row entry from a scored result.
-
-    Aggregate-only, by design: it carries counts, token totals and efficiency but NEVER
-    the certificates or the identities of the buggy rules. Publishing those would be a
-    free answer key — on a public library commit a `pred`-confirmed certificate counts
-    regardless of provenance, so anyone could copy it. The full certificates stay in the
-    private scored result the maintainer holds; the leaderboard shows only how many each
-    model found.
-    """
-    from benchmark.top50_contract import is_top50_submission, top50_public_entry
-    if is_top50_submission(submission) or is_top50_submission(scored):
-        return top50_public_entry(submission, scored)
-
-    return {
-        "model": scored["model"],
-        "library_commit": scored.get("library_commit", "unknown"),
-        "bugs_found": scored["bugs_found"],
-        "rules_tested": scored["rules_tested"],
-        "total_tokens_k": scored["total_tokens_k"],
-        # Aggregate token totals — safe to publish (no rule identities).
-        "usage_totals": scored.get("usage_totals"),
-        "efficiency_bugs_per_ktok": scored["efficiency_bugs_per_ktok"],
-        "submitted_by": submission.get("submitted_by"),
-        "placeholder": False,
-    }
+    """Stable aggregate-only projection entrypoint."""
+    return top50_public_entry(submission, scored)
 
 
 def main() -> None:
     parser = argparse.ArgumentParser(description="Re-verify and score a submission.json")
     parser.add_argument("submission", help="Path to submission.json")
-    parser.add_argument("--repo-dir", default=None, help="problem-reductions repo (default: pred on PATH)")
-    parser.add_argument("--out", default=None, help="Write the scored results.json here")
+    parser.add_argument("--repo-dir", default=None,
+                        help="problem-reductions repo (default: image-owned source)")
+    parser.add_argument("--out", default=None, help="Write the scored result here")
     args = parser.parse_args()
 
     submission = json.loads(Path(args.submission).read_text(encoding="utf-8"))
     scored, report = score_submission(submission, args.repo_dir)
-
     print(f"Scoring {scored['model']}")
     print("-" * 60)
     for item in report:
@@ -211,13 +36,13 @@ def main() -> None:
         print(f"{flag}  {item['rule']} [{item['violation']}]")
         print(f"           {item['reason']}")
     print("-" * 60)
-    print(f"self-reported bugs: {submission.get('bugs_found')!r}  →  "
-          f"verified distinct-rule bugs: {scored['bugs_found']}")
+    print(f"verified distinct-rule bugs: {scored['verified_bugs']}")
 
     if args.out:
-        Path(args.out).parent.mkdir(parents=True, exist_ok=True)
-        Path(args.out).write_text(json.dumps(scored, indent=2), encoding="utf-8")
-        print(f"Scored result → {args.out}")
+        destination = Path(args.out)
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        destination.write_text(json.dumps(scored, indent=2), encoding="utf-8")
+        print(f"Scored result → {destination}")
 
 
 if __name__ == "__main__":
diff --git a/intake/cloudflare-worker/src/index.js b/intake/cloudflare-worker/src/index.js
index 3bc0069..4e874f1 100644
--- a/intake/cloudflare-worker/src/index.js
+++ b/intake/cloudflare-worker/src/index.js
@@ -34,8 +34,9 @@ export default {
     // Light shape check — the backend re-verifies every certificate with pred regardless.
     let sub;
     try { sub = JSON.parse(body); } catch { return json({ error: "invalid JSON" }, 400); }
-    if (!sub || typeof sub !== "object" || !sub.model || !Array.isArray(sub.results)) {
-      return json({ error: "not a submission (need model + results[])" }, 400);
+    if (!sub || typeof sub !== "object" || !sub.model ||
+        !Array.isArray(sub.shortlist) || !sub.triage || !Array.isArray(sub.episodes)) {
+      return json({ error: "not a submission (need model, shortlist, triage, and episodes)" }, 400);
     }
 
     // Deposit the raw submission (answer key) privately in R2, pending scoring.
diff --git a/intake/cloudflare-worker/test/index.test.js b/intake/cloudflare-worker/test/index.test.js
index 1e21944..ccaf25a 100644
--- a/intake/cloudflare-worker/test/index.test.js
+++ b/intake/cloudflare-worker/test/index.test.js
@@ -11,11 +11,13 @@ afterEach(() => {
   globalThis.fetch = originalFetch;
 });
 
-function submissionRequest(headers = {}) {
+function submissionRequest(headers = {}, submission = {
+  model: "test/model", submitted_by: "claimed", shortlist: [], triage: {}, episodes: [],
+}) {
   return new Request("https://intake.example/submit", {
     method: "POST",
     headers: { "content-type": "application/json", ...headers },
-    body: JSON.stringify({ model: "test/model", submitted_by: "claimed", results: [] }),
+    body: JSON.stringify(submission),
   });
 }
 
@@ -137,3 +139,17 @@ test("rejects unauthenticated requests", async () => {
   assert.equal(response.status, 401);
   assert.equal(writes.length, 0);
 });
+
+test("rejects an incomplete submission shape", async () => {
+  const auth = await accessToken();
+  const { env, writes } = environment({
+    TEAM_DOMAIN: auth.teamDomain,
+    POLICY_AUD: auth.expectedAudience,
+  });
+  const response = await worker.fetch(submissionRequest(
+    { "cf-access-jwt-assertion": auth.token },
+    { model: "test/model", episodes: [] },
+  ), env);
+  assert.equal(response.status, 400);
+  assert.equal(writes.length, 0);
+});
diff --git a/site/index.html b/site/index.html
index 8efe88b..e9d8ad8 100644
--- a/site/index.html
+++ b/site/index.html
@@ -95,12 +95,6 @@
   td.bugs span{font-family:var(--sans);font-size:20px;font-weight:700;color:var(--ink);letter-spacing:-.02em;}
   td.bugs span.zero{color:var(--dim);font-weight:500;}
   tr.lead td.bugs span{color:var(--accent);}
-  td.reach{width:210px;}
-  .reach .bar{height:7px;background:#eef1f5;border-radius:99px;overflow:hidden;margin-bottom:5px;}
-  .reach .bar i{display:block;height:100%;background:var(--accent);opacity:.85;border-radius:99px;}
-  tr.lead .reach .bar i{opacity:1;}
-  .reach .rl{font-family:var(--sans);font-size:11.5px;color:var(--muted);font-variant-numeric:tabular-nums;}
-  .reach .rl em{color:var(--faint);font-style:normal;}
   td.empty,.empty{font-size:13px;color:var(--faint);padding:26px 16px;text-align:center;}
   .note{font-size:12.5px;color:var(--muted);margin:14px 2px 0;line-height:1.7;}
   .note b{color:var(--ink);font-weight:600;}
@@ -113,10 +107,6 @@
     box-shadow:0 1px 3px rgba(16,23,41,.05);}
   .panel h3{font-size:17px;font-weight:700;margin:0 0 3px;letter-spacing:-.01em;}
   .panel .sub{font-size:12px;color:var(--faint);margin:0 0 16px;}
-  .chart-panel{margin-top:4px;padding:20px 24px 18px;}
-  .panel-head{margin-bottom:6px;}
-  .panel-head h3{font-size:16px;}
-  .panel-head .sub{margin:4px 0 0;max-width:72ch;line-height:1.5;}
   .hint{font-size:12px;color:var(--faint);margin-left:auto;white-space:nowrap;}
 
   /* inspector drawer */
@@ -229,7 +219,6 @@
     .wrap{padding:0 16px 60px;}
     nav.tabs{gap:2px;overflow-x:auto;}
     .stat{min-width:46%;}
-    td.reach{display:none;}
     table.lb thead th:nth-child(4){display:none;}
   }
 
@@ -270,12 +259,11 @@ 
Leaderboard

     

 
     

-      
       
       Click a row for its run stats →
     

     

-    
Primary Top50 metric = verified distinct-rule bugs under the frozen logical evidence budget. Ties remain ties; tokens, cost, and elapsed time are diagnostics only. Select the historical contract to view old whole-repository results separately.

+    
Primary Top50 metric = verified distinct-rule bugs under the frozen logical evidence budget. Ties remain ties; tokens, cost, and elapsed time are diagnostics only.
@@ -387,7 +375,6 @@

How to cite

const esc=s=>String(s==null?'':s).replace(/&/g,'&').replace(//g,'>'); let RESULTS=[],TASKS=[],TOTAL=TOTAL_DEFAULT; let lbRows=[],lbSort={key:"bugs",dir:-1},selectedModel=null; -const cssv=v=>getComputedStyle(document.body).getPropertyValue(v).trim(); document.getElementById("tabs").addEventListener("click",e=>{ const b=e.target.closest("button"); if(!b) return; @@ -398,15 +385,7 @@

How to cite

const meta=document.querySelector(".meta"); if(meta)meta.style.display=b.dataset.tab==="leaderboard"?"":"none"; }); -const LEGACY_COLS=[ - {key:"rank",label:"#"}, - {key:"model",label:"Model"}, - {key:"bugs",label:"Bugs",num:true}, - {key:"reach",label:"Submitted rules",sortBy:"reachFrac"}, - {key:"tokens",label:"Tokens (K)",num:true,sortBy:"tokensNum"}, - {key:"eff",label:"Bugs / Ktok",num:true,sortBy:"effNum"}, -]; -const TOP50_COLS=[ +const LB_COLS=[ {key:"rank",label:"#"}, {key:"model",label:"Model"}, {key:"bugs",label:"Verified bugs",num:true}, @@ -415,35 +394,23 @@

How to cite

{key:"attempts",label:"1st / 2nd accepts",sortBy:"firstAccepts"}, {key:"predBug",label:"Pred / bug",num:true}, ]; -const RIGHT=new Set(["bugs","tokens","eff","bugs10","bugs25","predBug"]); -const selectedTrack=()=>document.getElementById("lb-track").value||"standardized-model-api"; -const isTop50Track=()=>selectedTrack()==="standardized-model-api"; -const trackLabel=t=>t==="standardized-model-api"?"Standardized Top50":"Legacy whole repository"; -const lbColumns=()=>isTop50Track()?TOP50_COLS:LEGACY_COLS; +const RIGHT=new Set(["bugs","bugs10","bugs25","predBug"]); function buildLbRows(){ - const track=selectedTrack(); - const top50=track==="standardized-model-api"; - const ranked=RESULTS.filter(r=>(r.agent_mode==="standardized-model-api"? - "standardized-model-api":"legacy-whole-repo")===track) - .sort((a,b)=>(b.bugs_found||0)-(a.bugs_found||0)|| - (top50?String(a.model).localeCompare(String(b.model)): - (b.efficiency_bugs_per_ktok||0)-(a.efficiency_bugs_per_ktok||0))); + const ranked=RESULTS.slice().sort((a,b)=>(b.bugs_found||0)-(a.bugs_found||0)|| + String(a.model).localeCompare(String(b.model))); let lastBugs=null,lastRank=0; lbRows=ranked.map((r,i)=>{ - const tested=r.rules_tested||0, frac=TOTAL?Math.min(tested/TOTAL,1):0; const bugs=r.bugs_found||0; - const rank=top50&&i&&bugs===lastBugs?lastRank:i+1;lastBugs=bugs;lastRank=rank; + const rank=i&&bugs===lastBugs?lastRank:i+1;lastBugs=bugs;lastRank=rank; return {rank,model:short(r.model),_model:r.model,bugs, - reachFrac:frac,tested,tokensNum:r.total_tokens_k||0, - effNum:r.efficiency_bugs_per_ktok||0, bugs10:r.bugs_at_10||0,bugs25:r.bugs_at_25||0, firstAccepts:r.first_attempt_accepts||0,secondAccepts:r.second_attempt_accepts||0, predBug:r.pred_calls_per_bug==null?0:r.pred_calls_per_bug}; }); } function renderLbHead(){ - document.getElementById("lb-head").innerHTML=lbColumns().map(c=>{ + document.getElementById("lb-head").innerHTML=LB_COLS.map(c=>{ const sk=c.sortBy||c.key, on=lbSort.key===sk, ind=on?(lbSort.dir<0?"↓":"↑"):""; const cls=[on?"sorted":"",RIGHT.has(c.key)?"n":""].filter(Boolean).join(" "); return `${c.label}${ind}`; @@ -457,122 +424,28 @@

How to cite

if(typeof av==="number"&&typeof bv==="number")return (av-bv)*d; return String(av).localeCompare(String(bv))*d;}); document.getElementById("lb-body").innerHTML=rows.map(r=>{ - if(isTop50Track())return ` + return ` ${r.rank}${esc(r.model)} ${r.bugs} ${r.bugs10}${r.bugs25} ${r.firstAccepts} / ${r.secondAccepts}${r.predBug||'—'}`; - const pct=Math.round(r.reachFrac*100); - return ` - ${r.rank} - ${esc(r.model)} - ${r.bugs} -
${r.tested}/${TOTAL} - ${Math.round(r.tokensNum).toLocaleString()} - ${r.effNum.toFixed(4)} - `; - }).join("")||`No models match.`; + }).join("")||`No models match.`; } -let scatterPts=[]; -function hexA(hex,a){hex=String(hex||'').replace('#','');if(hex.length===3)hex=hex.split('').map(c=>c+c).join('');const n=parseInt(hex||'2563eb',16);return `rgba(${(n>>16)&255},${(n>>8)&255},${n&255},${a})`;} -function renderScatter(){ - const host=document.getElementById("scatter"); if(!host)return; host.innerHTML=''; - const pts=lbRows; - if(!pts.length){host.innerHTML='
No ranked runs yet.
';return;} - host.style.position='relative'; - const cw=Math.max(320,host.clientWidth||720), W=cw, H=Math.min(460,Math.max(320,Math.round(cw*0.56))); - const dpr=window.devicePixelRatio||1; - const cv=document.createElement('canvas'); cv.width=W*dpr; cv.height=H*dpr; cv.style.cssText=`width:${W}px;height:${H}px;display:block`; - const ov=document.createElement('canvas'); ov.width=W*dpr; ov.height=H*dpr; ov.style.cssText=`width:${W}px;height:${H}px;display:block;position:absolute;left:0;top:0;pointer-events:none`; - host.appendChild(cv); host.appendChild(ov); - const ctx=cv.getContext('2d'); ctx.scale(dpr,dpr); - const octx=ov.getContext('2d'); octx.scale(dpr,dpr); - const padL=60,padB=52,padT=26,padR=30; - const ACC=cssv('--accent')||'#2563eb', INK=cssv('--ink')||'#0f1729', MUT=cssv('--muted')||'#5b6573', - LINE=cssv('--line')||'#edeff3', LINE2=cssv('--line2')||'#e0e4ea', CARD=cssv('--card')||'#ffffff', MONO=cssv('--sans')||'system-ui,sans-serif'; - const maxB=Math.max(3,...pts.map(p=>p.bugs)), maxTd=Math.max(10,...pts.map(p=>p.tokensNum)); - const maxBax=maxB+(maxB<=3?0.6:maxB*0.14), maxTax=maxTd*1.08; - const sx=t=>padL+(t/maxTax)*(W-padL-padR), sy=b=>H-padB-(b/maxBax)*(H-padB-padT); - const fr=pts.filter(p=>p.bugs>0).sort((a,b)=>a.tokensNum-b.tokensNum||b.bugs-a.bugs); - const front=[];let best=-1;for(const p of fr){if(p.bugs>best){front.push(p);best=p.bugs;}} - const onFront=new Set(front.map(p=>p._model)); - const stepB=Math.max(1,Math.ceil(maxB/6)); - // grid - ctx.textBaseline='middle'; - for(let i=0;i<=maxB;i+=stepB){const y=Math.round(sy(i))+.5; - ctx.strokeStyle=i===0?LINE2:LINE; ctx.lineWidth=1; ctx.beginPath(); ctx.moveTo(padL,y); ctx.lineTo(W-padR,y); ctx.stroke(); - ctx.fillStyle=MUT; ctx.font='12px '+MONO; ctx.textAlign='right'; ctx.fillText(i,padL-10,sy(i));} - ctx.textAlign='center'; - for(let i=1;i<=4;i++){const t=maxTax*i/4,x=Math.round(sx(t))+.5; - ctx.strokeStyle=LINE; ctx.lineWidth=1; ctx.beginPath(); ctx.moveTo(x,padT); ctx.lineTo(x,H-padB); ctx.stroke();} - ctx.strokeStyle=LINE2; ctx.lineWidth=1; ctx.beginPath(); ctx.moveTo(padL+.5,padT); ctx.lineTo(padL+.5,H-padB); ctx.stroke(); - for(let i=0;i<=4;i++){const t=maxTax*i/4; ctx.fillStyle=MUT; ctx.font='12px '+MONO; ctx.fillText(Math.round(t),sx(t),H-padB+17);} - // axis titles - ctx.fillStyle=MUT; ctx.font='12.5px '+MONO; ctx.textAlign='center'; - ctx.fillText('tokens spent (K) \u2192',(padL+W-padR)/2,H-8); - ctx.save(); ctx.translate(15,(padT+H-padB)/2); ctx.rotate(-Math.PI/2); ctx.fillText('bugs found \u2192',0,0); ctx.restore(); - // frontier band + line - if(front.length>1){ - const grad=ctx.createLinearGradient(0,padT,0,H-padB); - grad.addColorStop(0,hexA(ACC,.15)); grad.addColorStop(1,hexA(ACC,.012)); - ctx.beginPath(); front.forEach((p,i)=>{const x=sx(p.tokensNum),y=sy(p.bugs); i?ctx.lineTo(x,y):ctx.moveTo(x,y);}); - ctx.lineTo(sx(front[front.length-1].tokensNum),sy(0)); ctx.lineTo(sx(front[0].tokensNum),sy(0)); ctx.closePath(); - ctx.fillStyle=grad; ctx.fill(); - ctx.beginPath(); front.forEach((p,i)=>{const x=sx(p.tokensNum),y=sy(p.bugs); i?ctx.lineTo(x,y):ctx.moveTo(x,y);}); - ctx.strokeStyle=ACC; ctx.lineWidth=2; ctx.lineJoin='round'; ctx.lineCap='round'; ctx.stroke(); - } - function drawPoint(c,p,hover){ - const x=sx(p.tokensNum),y=sy(p.bugs),f=onFront.has(p._model); - if(f){c.beginPath();c.arc(x,y,hover?8:6.5,0,7);c.fillStyle=ACC;c.fill();c.lineWidth=2.5;c.strokeStyle=CARD;c.stroke();} - else{c.beginPath();c.arc(x,y,hover?7:5.5,0,7);c.fillStyle=CARD;c.fill();c.lineWidth=2;c.globalAlpha=hover?1:.75;c.strokeStyle=hover?ACC:MUT;c.stroke();c.globalAlpha=1;} - } - scatterPts=[]; - pts.forEach(p=>{scatterPts.push({x:sx(p.tokensNum),y:sy(p.bugs),p}); if(!onFront.has(p._model))drawPoint(ctx,p,false);}); - front.forEach(p=>drawPoint(ctx,p,false)); - ctx.font='600 11px '+MONO; ctx.textBaseline='middle'; - front.forEach(p=>{const x=sx(p.tokensNum),y=sy(p.bugs),flip=x>W*0.62; - ctx.fillStyle=INK; ctx.textAlign=flip?'right':'left'; ctx.fillText(p.model,flip?x-12:x+12,y);}); - // hover overlay + tooltip - let tip=host.querySelector('.cv-tip'); - if(!tip){tip=document.createElement('div'); tip.className='cv-tip'; host.appendChild(tip); - tip.style.cssText='position:absolute;pointer-events:none;opacity:0;transition:opacity .12s;font-family:var(--sans);font-size:12px;'+ - 'background:var(--ink);color:#fff;padding:7px 10px;border-radius:8px;white-space:nowrap;z-index:6;transform:translate(-50%,calc(-100% - 13px));box-shadow:0 8px 22px -8px rgba(0,0,0,.45);line-height:1.5;';} - cv.onmousemove=ev=>{const r=cv.getBoundingClientRect(),mx=ev.clientX-r.left,my=ev.clientY-r.top; - let hit=null,dn=22; for(const s of scatterPts){const d=Math.hypot(s.x-mx,s.y-my); if(d${esc(hit.p.model)}
${hit.p.bugs} bugs \u00b7 ${Math.round(hit.p.tokensNum)}K tok`; - cv.style.cursor='pointer'; - }else{tip.style.opacity='0'; cv.style.cursor='default';} - }; - cv.onmouseleave=()=>{octx.clearRect(0,0,W,H); tip.style.opacity='0';}; -} -let _scTO; window.addEventListener('resize',()=>{clearTimeout(_scTO);_scTO=setTimeout(()=>{if(lbRows.length)renderScatter();},150);}); function openDrawer(){document.getElementById("drawer").classList.add("open");document.getElementById("drawer-scrim").classList.add("open");document.getElementById("drawer").setAttribute("aria-hidden","false");} function closeDrawer(){document.getElementById("drawer").classList.remove("open");document.getElementById("drawer-scrim").classList.remove("open");document.getElementById("drawer").setAttribute("aria-hidden","true");selectedModel=null;renderLbBody();} function renderInspector(model){ const row=lbRows.find(r=>r._model===model); if(!row)return; document.getElementById("drawer-title").textContent=row.model; const sub=document.getElementById("drawer-sub"), el=document.getElementById("inspect-body"); - sub.textContent=isTop50Track()?`${row.bugs} verified bug(s) · frozen Self-selected Top50`: - `${row.bugs} confirmed bug(s) · ${row.tested}/${TOTAL} rules submitted · ${Math.round(row.tokensNum)}K tok`; + sub.textContent=`${row.bugs} verified bug(s) · frozen Self-selected Top50`; // Aggregate-only by design: the leaderboard never reveals WHICH rules a model found bugs // in. On a public library commit a pred-confirmed certificate counts regardless of who // produced it, so publishing the rules/certificates would be a free answer key. - el.innerHTML=isTop50Track()?`
+ el.innerHTML=`
Verified bugs
${row.bugs}
Bugs @10 / @25
${row.bugs10} / ${row.bugs25}
1st / 2nd accepts
${row.firstAccepts} / ${row.secondAccepts}
Pred calls / bug
${row.predBug||'—'}
-
`:`
-
Confirmed bugs
${row.bugs}
-
Rules reached
${row.tested}/${TOTAL}
-
Tokens
${Math.round(row.tokensNum).toLocaleString()}K
-
Bugs / Ktok
${row.effNum.toFixed(4)}
`; openDrawer(); } @@ -609,9 +482,6 @@

How to cite

document.getElementById("drawer-scrim").addEventListener("click",closeDrawer); document.addEventListener("keydown",e=>{if(e.key==="Escape")closeDrawer();}); document.getElementById("task-search").addEventListener("input",renderTasks); -document.getElementById("lb-track").addEventListener("change",()=>{ - lbSort={key:"bugs",dir:-1};buildLbRows();renderLbHead();renderLbBody(); -}); document.querySelectorAll("#task-table thead th").forEach(th=>th.addEventListener("click",()=>{ const k=th.dataset.k; taskSort={key:k,dir:taskSort.key===k?-taskSort.dir:1}; renderTasks(); })); @@ -621,11 +491,6 @@

How to cite

fetch("tasks.json").then(r=>r.json()).catch(()=>[]), ]).then(([res,tasks])=>{ RESULTS=res;TASKS=tasks;TOTAL=tasks.length||TOTAL_DEFAULT; - const tracks=[...new Set(["standardized-model-api", ...RESULTS.map(r=> - r.agent_mode==="standardized-model-api"?"standardized-model-api":"legacy-whole-repo")])].sort(); - const trackSelect=document.getElementById("lb-track"); - trackSelect.innerHTML=tracks.map(t=>``).join(""); - if(tracks.includes("standardized-model-api"))trackSelect.value="standardized-model-api"; const rc=document.getElementById("rulecount"); if(rc)rc.textContent=TOTAL; if(RESULTS.some(r=>r.placeholder)) document.getElementById("lb-banner").innerHTML=``; diff --git a/submission.env.example b/submission.env.example index f0f51ba..92f3f9b 100644 --- a/submission.env.example +++ b/submission.env.example @@ -1,25 +1,20 @@ # ───────────────────────────────────────────────────────────────────────────── -# Submission runner config. Copy to `submission.env`, then choose exactly one route: +# Submission runner config. Copy to `submission.env`, then run: # # Model API in Docker: make runner-pull && make preflight && make run -# The public rankable track is Model API only. Coding-agent/whole-repository runners are -# retained solely for historical reproduction and cannot enter the Top50 table. -# -# Do not run Codex, Claude Code, or another coding-agent CLI through Docker. API runs use -# mini-swe/LiteLLM; CLI runs select their harness only with the Make variable LOCAL_BACKEND. # (submission.env is gitignored — it may hold your API key; never commit it.) # ───────────────────────────────────────────────────────────────────────────── # ── REQUIRED ───────────────────────────────────────────────────────────────── -# Model identity recorded in the submission. For mini-swe it must be LiteLLM-routable: +# Model identity recorded in the submission. It must be LiteLLM-routable: # anthropic/claude-... openai/gpt-... openrouter/... gemini/... # openai/ together with API_BASE for an OpenAI-compatible endpoint MODEL_NAME=openai/gpt-5.4 -# API_KEY=... # generic key for the model API route +# API_KEY=... # generic key for the model API endpoint # Prefer the provider's own env var? Use it INSTEAD of API_KEY, e.g.: # OPENAI_API_KEY=... ANTHROPIC_API_KEY=... OPENROUTER_API_KEY=... GEMINI_API_KEY=... -# ── MODEL API ONLY (OpenRouter / gateway / local vLLM / Azure …) ───────────── +# ── MODEL API (OpenRouter / gateway / local vLLM / Azure …) ────────────────── # Nothing is hardcoded to anthropic/openai. Reach any model with these: # API_BASE=https://my-gateway.example/v1 # API_KEY=... # generic key when no PROVIDER_API_KEY name fits @@ -29,17 +24,17 @@ MODEL_NAME=openai/gpt-5.4 # # error. Token usage is still recorded from API responses. # ── FROZEN LIMITS ──────────────────────────────────────────────────────────── -# There are no budget environment variables. Rankable runs always use +# There are no budget environment variables. Benchmark runs always use # Source-only Top50 triage, then 50 isolated rule episodes with # M=10, E=12, P=24, P_solve=10, and exactly S=2 submit attempts per rule. # ── OUTPUT / LOGS ──────────────────────────────────────────────────────────── # Docker defaults follow. # OUTPUT=/out/submission.json # authoritative submission output -# Rankable runs reject custom prompts, strategies, backends, and logical budgets. +# Benchmark runs reject custom prompts, strategies, backends, and logical budgets. # Version pins, source manifest, and the trusted `pred` path are baked into the image. -# Rankable runs reject pin overrides and custom `pred` binaries. +# Benchmark runs reject pin overrides and custom `pred` binaries. # ── METADATA ───────────────────────────────────────────────────────────────── # SUBMITTED_BY=your-hf-handle # also fillable at submit time on the Space From 23b1c6c1e459eefa49f3d45eed651c690259e8a2 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Tue, 21 Jul 2026 02:47:15 +0800 Subject: [PATCH 9/9] refactor: fold API workflow into run-benchmark --- .agents/skills/run-api-benchmark/SKILL.md | 40 ------------- .../run-api-benchmark/agents/openai.yaml | 4 -- .agents/skills/run-benchmark/SKILL.md | 57 +++++++++++++++++-- .../skills/run-benchmark/agents/openai.yaml | 4 ++ .../references/engines.md | 0 .../references/provider-config.md | 0 .../scripts/detect-engine.sh | 2 +- benchmark/tests/test_docs.py | 12 ++-- 8 files changed, 63 insertions(+), 56 deletions(-) delete mode 100644 .agents/skills/run-api-benchmark/SKILL.md delete mode 100644 .agents/skills/run-api-benchmark/agents/openai.yaml create mode 100644 .agents/skills/run-benchmark/agents/openai.yaml rename .agents/skills/{run-api-benchmark => run-benchmark}/references/engines.md (100%) rename .agents/skills/{run-api-benchmark => run-benchmark}/references/provider-config.md (100%) rename .agents/skills/{run-api-benchmark => run-benchmark}/scripts/detect-engine.sh (98%) diff --git a/.agents/skills/run-api-benchmark/SKILL.md b/.agents/skills/run-api-benchmark/SKILL.md deleted file mode 100644 index cd1c485..0000000 --- a/.agents/skills/run-api-benchmark/SKILL.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -name: run-api-benchmark -description: Configure and run the rankable problem-reductions Self-selected Top50 benchmark through the standardized containerized Model API path. ---- - -# Run the rankable Top50 benchmark - -The benchmark uses source-only triage to freeze 50 rules, followed by 50 fresh sequential episodes. Each rule receives M=10 model generations, E=12 shell actions, P=24 `pred` calls, P_solve=10 solve calls, O=10000 observed characters, and exactly S=2 submit attempts. These limits, prompts, inference settings, and the harness are part of the benchmark code. - -Read [references/provider-config.md](references/provider-config.md) for endpoint setup and `scripts/detect-engine.sh` before preparing the image. - -## Collect only required choices - -1. Ask for the Model API identifier. -2. Ask whether it uses the standard provider endpoint or a custom OpenAI-compatible `API_BASE`. Never ask the caller to paste a secret; direct them to the gitignored `submission.env`. -3. Ask whether to keep and validate locally or upload officially. `$submit-benchmark-result` owns upload. -4. Read `make -s print-benchmark-version`, compare it with the official [`main/VERSION`](https://github.com/CodingThrust/problem-reductions-benchmark/blob/main/VERSION), and show `Benchmark version: (latest version:
)`. Wait for confirmation. If they differ, say the checkout is outdated and stop the official run. -5. Resolve `PR_REF` with `make -s print-pr-ref`, plus a fixed `STAMP` and `out//submission.json`. Do not ask for budget values. - -## Configure and preflight - -Create `submission.env` from the example when absent. Configure `MODEL_NAME`, a provider key or `API_KEY`, and `API_BASE` only when needed. Do not add other execution settings. - -Detect the container engine. Prefer `make runner-pull`; use `make runner-build` only when necessary. Before the first real API call, show the redacted model/endpoint, built-in counters, target ref, output path, and upload goal, then ask for confirmation. - -Run `make preflight`. It verifies the frozen source and `pred`, then checks the endpoint and credentials with a tiny model call. Never continue after a failed preflight. - -## Run and validate - -Explain that 50 isolated episodes can use substantial API credits, then get explicit confirmation and run `make run STAMP=`. - -For local-only validation: - -```bash -python -m benchmark.submit --predictions --dry-run -``` - -Report the contract, completed episode count, rankability, claimed/verified bug fields available locally, cap-hit diagnostics, and absolute artifact path. Time, tokens, and cost are diagnostic only. - -For official upload, invoke `$submit-benchmark-result`. Never upload merely because the run completed. Preserve partial checkpoints on failure; a `run_error` is not a clean zero. diff --git a/.agents/skills/run-api-benchmark/agents/openai.yaml b/.agents/skills/run-api-benchmark/agents/openai.yaml deleted file mode 100644 index 2dc99c3..0000000 --- a/.agents/skills/run-api-benchmark/agents/openai.yaml +++ /dev/null @@ -1,4 +0,0 @@ -interface: - display_name: "Run API Benchmark" - short_description: "Configure and run the benchmark through a model API" - default_prompt: "Use $run-api-benchmark to configure and run an API-backed benchmark." diff --git a/.agents/skills/run-benchmark/SKILL.md b/.agents/skills/run-benchmark/SKILL.md index eaeb3f2..0b3c9a8 100644 --- a/.agents/skills/run-benchmark/SKILL.md +++ b/.agents/skills/run-benchmark/SKILL.md @@ -1,13 +1,60 @@ --- name: run-benchmark -description: Route a request to run, reproduce, smoke-test, or generate a submission for the standardized problem-reductions Model API Top50 benchmark. +description: Configure, run, reproduce, smoke-test, or generate a submission for the standardized problem-reductions Self-selected Top50 benchmark, and route an existing submission to the upload workflow. --- -# Route a benchmark run +# Run the benchmark If the caller already has a `submission.json`, invoke `$submit-benchmark-result`. -For a benchmark run, invoke `$run-api-benchmark`. Do not offer a backend or protocol choice: -the benchmark has one built-in Model API contract. +The benchmark uses source-only triage to freeze 50 rules, followed by 50 fresh sequential +episodes. Each rule receives M=10 model generations, E=12 shell actions, P=24 `pred` calls, +P_solve=10 solve calls, O=10000 observed characters, and exactly S=2 submit attempts. These +limits, prompts, inference settings, and the harness are part of the benchmark code. -The child skill owns preflight, execution, validation, and optional upload. +Read [references/provider-config.md](references/provider-config.md) for endpoint setup and +`scripts/detect-engine.sh` before preparing the image. + +## Collect only required choices + +1. Ask for the Model API identifier. +2. Ask whether it uses the standard provider endpoint or a custom OpenAI-compatible `API_BASE`. + Never ask the caller to paste a secret; direct them to the gitignored `submission.env`. +3. Ask whether to keep and validate locally or upload officially. `$submit-benchmark-result` + owns upload. +4. Read `make -s print-benchmark-version`, compare it with the official + [`main/VERSION`](https://github.com/CodingThrust/problem-reductions-benchmark/blob/main/VERSION), + and show `Benchmark version: (latest version:
)`. Wait for confirmation. If + they differ, say the checkout is outdated and stop the official run. +5. Resolve `PR_REF` with `make -s print-pr-ref`, plus a fixed `STAMP` and + `out//submission.json`. Do not ask for budget values. + +## Configure and preflight + +Create `submission.env` from the example when absent. Configure `MODEL_NAME`, a provider key or +`API_KEY`, and `API_BASE` only when needed. Do not add other execution settings. + +Detect the container engine. Prefer `make runner-pull`; use `make runner-build` only when +necessary. Before the first real API call, show the redacted model/endpoint, built-in counters, +target ref, output path, and upload goal, then ask for confirmation. + +Run `make preflight`. It verifies the frozen source and `pred`, then checks the endpoint and +credentials with a tiny model call. Never continue after a failed preflight. + +## Run and validate + +Explain that 50 isolated episodes can use substantial API credits, then get explicit +confirmation and run `make run STAMP=`. + +For local-only validation: + +```bash +python -m benchmark.submit --predictions --dry-run +``` + +Report the contract, completed episode count, rankability, claimed/verified bug fields available +locally, cap-hit diagnostics, and absolute artifact path. Time, tokens, and cost are diagnostic +only. + +For official upload, invoke `$submit-benchmark-result`. Never upload merely because the run +completed. Preserve partial checkpoints on failure; a `run_error` is not a clean zero. diff --git a/.agents/skills/run-benchmark/agents/openai.yaml b/.agents/skills/run-benchmark/agents/openai.yaml new file mode 100644 index 0000000..1a072dc --- /dev/null +++ b/.agents/skills/run-benchmark/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Run Benchmark" + short_description: "Configure and run the benchmark" + default_prompt: "Use $run-benchmark to configure and run the benchmark." diff --git a/.agents/skills/run-api-benchmark/references/engines.md b/.agents/skills/run-benchmark/references/engines.md similarity index 100% rename from .agents/skills/run-api-benchmark/references/engines.md rename to .agents/skills/run-benchmark/references/engines.md diff --git a/.agents/skills/run-api-benchmark/references/provider-config.md b/.agents/skills/run-benchmark/references/provider-config.md similarity index 100% rename from .agents/skills/run-api-benchmark/references/provider-config.md rename to .agents/skills/run-benchmark/references/provider-config.md diff --git a/.agents/skills/run-api-benchmark/scripts/detect-engine.sh b/.agents/skills/run-benchmark/scripts/detect-engine.sh similarity index 98% rename from .agents/skills/run-api-benchmark/scripts/detect-engine.sh rename to .agents/skills/run-benchmark/scripts/detect-engine.sh index fadd3fb..db9865e 100755 --- a/.agents/skills/run-api-benchmark/scripts/detect-engine.sh +++ b/.agents/skills/run-benchmark/scripts/detect-engine.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Read-only host probe for the run-api-benchmark skill. +# Read-only host probe for the run-benchmark skill. # Detects an available container engine and the flags the benchmark run needs on # this host, then prints a machine-parsable report + a human summary. It NEVER # mutates the system (no install, no `colima start`, no alias) — installing an diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index e6d5775..dcb5a4b 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -17,7 +17,7 @@ README = REPO_ROOT / "README.md" GUIDE = REPO_ROOT / "CONTRIBUTING.md" ENV_EXAMPLE = REPO_ROOT / "submission.env.example" -API_SKILL = REPO_ROOT / ".agents/skills/run-api-benchmark/SKILL.md" +RUN_SKILL = REPO_ROOT / ".agents/skills/run-benchmark/SKILL.md" SUBMIT_SKILL = REPO_ROOT / ".agents/skills/submit-benchmark-result/SKILL.md" TRIGGER_SCORING = SUBMIT_SKILL.parent / "scripts/trigger-scoring.sh" SCORER_WORKFLOW = REPO_ROOT / ".github/workflows/score-from-r2.yml" @@ -104,23 +104,23 @@ def test_env_template_exposes_only_model_api_configuration(self): assert "whole-repository" not in t and "local_backend" not in t def test_run_skill_uses_the_containerized_protocol(self): - t = _text(API_SKILL) + t = _text(RUN_SKILL) assert "standardized" in t and "runner-pull" in t - @pytest.mark.parametrize("skill", [API_SKILL]) + @pytest.mark.parametrize("skill", [RUN_SKILL]) def test_each_skill_exposes_only_local_and_official_goals(self, skill): t = _text(skill) assert "local" in t assert "official" in t assert "intake test" not in t - @pytest.mark.parametrize("skill", [API_SKILL]) + @pytest.mark.parametrize("skill", [RUN_SKILL]) def test_run_skills_delegate_upload(self, skill): t = _text(skill) assert "$submit-benchmark-result" in t assert "owns upload" in t - @pytest.mark.parametrize("skill", [API_SKILL]) + @pytest.mark.parametrize("skill", [RUN_SKILL]) def test_run_skills_confirm_the_benchmark_version(self, skill): t = _text(skill) assert "make -s print-benchmark-version" in t @@ -151,7 +151,7 @@ def test_workflows_read_version_file(self, workflow): assert "PR_REF: v" not in text def test_router_sends_existing_results_to_submit_skill(self): - t = _text(REPO_ROOT / ".agents/skills/run-benchmark/SKILL.md") + t = _text(RUN_SKILL) assert "already has a `submission.json`" in t assert "$submit-benchmark-result" in t