feat: validate and score Top50 contracts

.github/scripts/check_aggregate.py

@@ -18,6 +18,9 @@
     "model", "library_commit", "bugs_found", "rules_tested",
     "total_tokens_k", "usage_totals", "efficiency_bugs_per_ktok",
     "submitted_by", "placeholder",
+    "benchmark_contract", "runner_version", "pred_version", "agent_mode", "rankable",
+    "bugs_at_10", "bugs_at_25", "bugs_at_50", "first_attempt_accepts",
+    "second_attempt_accepts", "pred_calls_per_bug", "cap_hits",
     # per-submission entry files (site/results/<slug>.json) also carry provenance tags
     "timestamp", "submission_id",
 }

benchmark/backend_score.py

@@ -25,6 +25,7 @@
 from benchmark.env_setup import pinned_commit
 from benchmark.submit import validate_submission
 from benchmark.submit_ledger import has_submit_ledger
+from benchmark.top50_contract import is_top50_submission
 from benchmark.verify_submission import leaderboard_entry, score_submission
 
 STATUS_SUFFIX = ".status.json"
@@ -93,7 +94,10 @@ def write_board_entries(results_dir: Path, board_dir: Path) -> list[str]:
             scored = json.loads(p.read_text(encoding="utf-8"))
         except (json.JSONDecodeError, OSError):
             continue
-        if "results" not in scored or "model" not in scored or scored.get("test"):
+        if (("results" not in scored and not is_top50_submission(scored))
+                or "model" not in scored or scored.get("test")):
+            continue
+        if is_top50_submission(scored) and not scored.get("rankable"):
             continue
         slug = board_slug(scored, p.stem)
         (board_dir / f"{slug}.json").write_text(
@@ -191,7 +195,7 @@ def _validate_for_scoring(submission: object, *, official: bool,
     problems = validate_submission(submission)
 
     if official:
-        if not has_submit_ledger(submission):
+        if not is_top50_submission(submission) and not has_submit_ledger(submission):
             problems.append("official submissions require submit_limit and submit_log")
         target_commit = expected_commit or pinned_commit()
         if submission.get("library_commit") != target_commit:
@@ -243,16 +247,29 @@ def score_one(sub_path: Path, results_dir: Path, repo_dir: str | None = None, *,
 
 def _dedup_best(entries: list[dict]) -> list[dict]:
     """Keep the best entry per model (max bugs, tie-break efficiency), ranked desc."""
-    best: dict[str, dict] = {}
+    best: dict[tuple[str, str], dict] = {}
     for e in entries:
         m = e["model"]
-        cur = best.get(m)
-        key = (e.get("bugs_found", 0), e.get("efficiency_bugs_per_ktok", 0.0))
-        if cur is None or key > (cur.get("bugs_found", 0), cur.get("efficiency_bugs_per_ktok", 0.0)):
-            best[m] = e
-    return sorted(best.values(),
-                  key=lambda e: (e.get("bugs_found", 0), e.get("efficiency_bugs_per_ktok", 0.0)),
-                  reverse=True)
+        contract = e.get("benchmark_contract", "legacy-whole-repo")
+        group = (contract, m)
+        cur = best.get(group)
+        if contract.startswith("top50-evidence/"):
+            better = cur is None or e.get("bugs_found", 0) > cur.get("bugs_found", 0)
+        else:
+            key = (e.get("bugs_found", 0), e.get("efficiency_bugs_per_ktok", 0.0))
+            better = cur is None or key > (
+                cur.get("bugs_found", 0), cur.get("efficiency_bugs_per_ktok", 0.0))
+        if better:
+            best[group] = e
+    return sorted(
+        best.values(),
+        key=lambda e: (
+            e.get("benchmark_contract", "legacy-whole-repo"),
+            -e.get("bugs_found", 0),
+            (0 if str(e.get("benchmark_contract", "")).startswith("top50-evidence/")
+             else -e.get("efficiency_bugs_per_ktok", 0.0)),
+            e.get("model", ""),
+        ))
 
 
 def aggregate_leaderboard(results_dir: Path) -> list[dict]:
@@ -266,7 +283,9 @@ def aggregate_leaderboard(results_dir: Path) -> list[dict]:
             scored = json.loads(p.read_text(encoding="utf-8"))
         except (json.JSONDecodeError, OSError):
             continue
-        if "results" not in scored or "model" not in scored:
+        if ("results" not in scored and not is_top50_submission(scored)) or "model" not in scored:
+            continue
+        if is_top50_submission(scored) and not scored.get("rankable"):
             continue
         # Test submissions are scored + kept privately, but never published to the public
         # leaderboard — skip them here so an end-to-end test can't pollute production.

benchmark/run_top50.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import argparse
+import datetime
 import json
 import os
 from pathlib import Path
@@ -17,19 +18,35 @@
     TriageBudget,
     build_rankable_runner,
 )
+from benchmark.top50_contract import (
+    AGENT_MODE,
+    CONTRACT_ID,
+    RUNNER_VERSION,
+    EXPECTED_EPISODE_BUDGET,
+    EXPECTED_TRIAGE_BUDGET,
+    expected_prompt_id,
+)
 
 
 def pilot_contract() -> Top50Contract:
     """Return explicitly provisional values; issue #68 freezes the public contract."""
     return Top50Contract(
         triage=TriageBudget(
-            model_generations=int(os.environ.get("PRB_TRIAGE_GENERATIONS", "8")),
-            shell_actions=int(os.environ.get("PRB_TRIAGE_ACTIONS", "12"))),
+            model_generations=int(os.environ.get(
+                "PRB_TRIAGE_GENERATIONS", str(EXPECTED_TRIAGE_BUDGET["model_generations"]))),
+            shell_actions=int(os.environ.get(
+                "PRB_TRIAGE_ACTIONS", str(EXPECTED_TRIAGE_BUDGET["shell_actions"]))),
+            max_output_chars=EXPECTED_TRIAGE_BUDGET["max_output_chars"],
+            command_timeout_seconds=EXPECTED_TRIAGE_BUDGET["command_timeout_seconds"]),
         episode=EvidenceBudget(
-            model_generations=int(os.environ.get("PRB_EPISODE_GENERATIONS", "10")),
-            shell_actions=int(os.environ.get("PRB_EPISODE_ACTIONS", "12")),
-            pred_calls=int(os.environ.get("PRB_PRED_CALLS", "24")),
-            solve_calls=int(os.environ.get("PRB_SOLVE_CALLS", "10")),
+            model_generations=int(os.environ.get(
+                "PRB_EPISODE_GENERATIONS", str(EXPECTED_EPISODE_BUDGET["model_generations"]))),
+            shell_actions=int(os.environ.get(
+                "PRB_EPISODE_ACTIONS", str(EXPECTED_EPISODE_BUDGET["shell_actions"]))),
+            pred_calls=int(os.environ.get(
+                "PRB_PRED_CALLS", str(EXPECTED_EPISODE_BUDGET["pred_calls"]))),
+            solve_calls=int(os.environ.get(
+                "PRB_SOLVE_CALLS", str(EXPECTED_EPISODE_BUDGET["solve_calls"]))),
             submit_attempts=2,
             max_output_chars=int(os.environ.get("PRB_MAX_OUTPUT_CHARS", "10000")),
             pred_timeout_seconds=int(os.environ.get("PRB_PRED_TIMEOUT_SECONDS", "300"))),
@@ -55,12 +72,14 @@ def run(*, model: str, repo_dir: str | Path, output: str | Path,
     if len(inventory) < 50:
         raise ValueError(f"canonical inventory has only {len(inventory)} runnable rules")
     pred_binary = find_pred_binary()
-    verify_pred_version(pred_binary)
+    pred_version = verify_pred_version(pred_binary)
     contract = pilot_contract()
     if fake:
         runner = Top50Runner(
             executor=_FakeExecutor(), contract=contract, pred_binary=pred_binary)
     else:
+        if model_kwargs:
+            raise ValueError("rankable Top50 runs do not accept custom model kwargs")
         runner = build_rankable_runner(
             contract=contract, pred_binary=pred_binary,
             agent_uid=int(os.environ["PRB_AGENT_UID"]),
@@ -69,12 +88,19 @@ def run(*, model: str, repo_dir: str | Path, output: str | Path,
             oracle_gid=int(os.environ["PRB_ORACLE_GID"]),
             evidence_gid=int(os.environ["PRB_EVIDENCE_GID"]),
             api_base=api_base, api_key=api_key, model_kwargs=model_kwargs)
-    result = runner.run(model=model, repo_path=repo, inventory=inventory, output=output)
-    result["library_commit"] = pinned_commit()
-    result["budget_contract_status"] = "pilot-unfrozen"
-    # Rewrite once with provenance added after the runner's final checkpoint.
-    Path(output).write_text(json.dumps(result, indent=2), encoding="utf-8")
-    return result
+    metadata = {
+        "benchmark_contract": CONTRACT_ID,
+        "library_commit": pinned_commit(),
+        "runner_version": RUNNER_VERSION,
+        "pred_version": pred_version,
+        "agent_mode": AGENT_MODE,
+        "prompt_id": expected_prompt_id(),
+        "budget_contract_status": "pilot-unfrozen",
+        "created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+        "inference_parameters": model_kwargs or {},
+    }
+    return runner.run(model=model, repo_path=repo, inventory=inventory,
+                      output=output, metadata=metadata)
 
 
 def main(argv: list[str] | None = None) -> None:

benchmark/submit.py

@@ -48,6 +48,10 @@ def validate_submission(sub: dict) -> list[str]:
     New runs prove provenance through the bounded submit ledger. Legacy runs must carry a
     certificate plus a trajectory on the row or envelope.
     """
+    from benchmark.top50_contract import is_top50_submission, validate_top50_submission
+    if is_top50_submission(sub):
+        return validate_top50_submission(sub)
+
     problems: list[str] = []
     if not isinstance(sub, dict):
         return ["submission is not a JSON object"]