From 85a59edaa6a0d877ba601599272591d0315d3e36 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Mon, 20 Jul 2026 13:21:45 +0800 Subject: [PATCH] feat: validate and score Top50 contracts --- .github/scripts/check_aggregate.py | 3 + benchmark/backend_score.py | 41 ++- benchmark/run_top50.py | 52 ++- benchmark/submit.py | 4 + benchmark/tests/test_top50_submission.py | 255 +++++++++++++ benchmark/top50_contract.py | 447 +++++++++++++++++++++++ benchmark/top50_results.schema.json | 26 ++ benchmark/top50_runner.py | 5 +- benchmark/top50_submission.schema.json | 26 ++ benchmark/verify_submission.py | 9 +- docker/Dockerfile | 12 +- site/index.html | 21 +- 12 files changed, 866 insertions(+), 35 deletions(-) create mode 100644 benchmark/tests/test_top50_submission.py create mode 100644 benchmark/top50_contract.py create mode 100644 benchmark/top50_results.schema.json create mode 100644 benchmark/top50_submission.schema.json diff --git a/.github/scripts/check_aggregate.py b/.github/scripts/check_aggregate.py index 5decbf0..5c3bc28 100644 --- a/.github/scripts/check_aggregate.py +++ b/.github/scripts/check_aggregate.py @@ -18,6 +18,9 @@ "model", "library_commit", "bugs_found", "rules_tested", "total_tokens_k", "usage_totals", "efficiency_bugs_per_ktok", "submitted_by", "placeholder", + "benchmark_contract", "runner_version", "pred_version", "agent_mode", "rankable", + "bugs_at_10", "bugs_at_25", "bugs_at_50", "first_attempt_accepts", + "second_attempt_accepts", "pred_calls_per_bug", "cap_hits", # per-submission entry files (site/results/.json) also carry provenance tags "timestamp", "submission_id", } diff --git a/benchmark/backend_score.py b/benchmark/backend_score.py index 78bbb08..7690a1c 100644 --- a/benchmark/backend_score.py +++ b/benchmark/backend_score.py @@ -25,6 +25,7 @@ from benchmark.env_setup import pinned_commit from benchmark.submit import validate_submission from benchmark.submit_ledger import has_submit_ledger +from benchmark.top50_contract import is_top50_submission from benchmark.verify_submission import leaderboard_entry, score_submission STATUS_SUFFIX = ".status.json" @@ -93,7 +94,10 @@ def write_board_entries(results_dir: Path, board_dir: Path) -> list[str]: scored = json.loads(p.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError): continue - if "results" not in scored or "model" not in scored or scored.get("test"): + if (("results" not in scored and not is_top50_submission(scored)) + or "model" not in scored or scored.get("test")): + continue + if is_top50_submission(scored) and not scored.get("rankable"): continue slug = board_slug(scored, p.stem) (board_dir / f"{slug}.json").write_text( @@ -191,7 +195,7 @@ def _validate_for_scoring(submission: object, *, official: bool, problems = validate_submission(submission) if official: - if not has_submit_ledger(submission): + if not is_top50_submission(submission) and not has_submit_ledger(submission): problems.append("official submissions require submit_limit and submit_log") target_commit = expected_commit or pinned_commit() if submission.get("library_commit") != target_commit: @@ -243,16 +247,29 @@ def score_one(sub_path: Path, results_dir: Path, repo_dir: str | None = None, *, def _dedup_best(entries: list[dict]) -> list[dict]: """Keep the best entry per model (max bugs, tie-break efficiency), ranked desc.""" - best: dict[str, dict] = {} + best: dict[tuple[str, str], dict] = {} for e in entries: m = e["model"] - cur = best.get(m) - key = (e.get("bugs_found", 0), e.get("efficiency_bugs_per_ktok", 0.0)) - if cur is None or key > (cur.get("bugs_found", 0), cur.get("efficiency_bugs_per_ktok", 0.0)): - best[m] = e - return sorted(best.values(), - key=lambda e: (e.get("bugs_found", 0), e.get("efficiency_bugs_per_ktok", 0.0)), - reverse=True) + contract = e.get("benchmark_contract", "legacy-whole-repo") + group = (contract, m) + cur = best.get(group) + if contract.startswith("top50-evidence/"): + better = cur is None or e.get("bugs_found", 0) > cur.get("bugs_found", 0) + else: + key = (e.get("bugs_found", 0), e.get("efficiency_bugs_per_ktok", 0.0)) + better = cur is None or key > ( + cur.get("bugs_found", 0), cur.get("efficiency_bugs_per_ktok", 0.0)) + if better: + best[group] = e + return sorted( + best.values(), + key=lambda e: ( + e.get("benchmark_contract", "legacy-whole-repo"), + -e.get("bugs_found", 0), + (0 if str(e.get("benchmark_contract", "")).startswith("top50-evidence/") + else -e.get("efficiency_bugs_per_ktok", 0.0)), + e.get("model", ""), + )) def aggregate_leaderboard(results_dir: Path) -> list[dict]: @@ -266,7 +283,9 @@ def aggregate_leaderboard(results_dir: Path) -> list[dict]: scored = json.loads(p.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError): continue - if "results" not in scored or "model" not in scored: + if ("results" not in scored and not is_top50_submission(scored)) or "model" not in scored: + continue + if is_top50_submission(scored) and not scored.get("rankable"): continue # Test submissions are scored + kept privately, but never published to the public # leaderboard — skip them here so an end-to-end test can't pollute production. diff --git a/benchmark/run_top50.py b/benchmark/run_top50.py index 1634334..97c982d 100644 --- a/benchmark/run_top50.py +++ b/benchmark/run_top50.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import datetime import json import os from pathlib import Path @@ -17,19 +18,35 @@ TriageBudget, build_rankable_runner, ) +from benchmark.top50_contract import ( + AGENT_MODE, + CONTRACT_ID, + RUNNER_VERSION, + EXPECTED_EPISODE_BUDGET, + EXPECTED_TRIAGE_BUDGET, + expected_prompt_id, +) def pilot_contract() -> Top50Contract: """Return explicitly provisional values; issue #68 freezes the public contract.""" return Top50Contract( triage=TriageBudget( - model_generations=int(os.environ.get("PRB_TRIAGE_GENERATIONS", "8")), - shell_actions=int(os.environ.get("PRB_TRIAGE_ACTIONS", "12"))), + model_generations=int(os.environ.get( + "PRB_TRIAGE_GENERATIONS", str(EXPECTED_TRIAGE_BUDGET["model_generations"]))), + shell_actions=int(os.environ.get( + "PRB_TRIAGE_ACTIONS", str(EXPECTED_TRIAGE_BUDGET["shell_actions"]))), + max_output_chars=EXPECTED_TRIAGE_BUDGET["max_output_chars"], + command_timeout_seconds=EXPECTED_TRIAGE_BUDGET["command_timeout_seconds"]), episode=EvidenceBudget( - model_generations=int(os.environ.get("PRB_EPISODE_GENERATIONS", "10")), - shell_actions=int(os.environ.get("PRB_EPISODE_ACTIONS", "12")), - pred_calls=int(os.environ.get("PRB_PRED_CALLS", "24")), - solve_calls=int(os.environ.get("PRB_SOLVE_CALLS", "10")), + model_generations=int(os.environ.get( + "PRB_EPISODE_GENERATIONS", str(EXPECTED_EPISODE_BUDGET["model_generations"]))), + shell_actions=int(os.environ.get( + "PRB_EPISODE_ACTIONS", str(EXPECTED_EPISODE_BUDGET["shell_actions"]))), + pred_calls=int(os.environ.get( + "PRB_PRED_CALLS", str(EXPECTED_EPISODE_BUDGET["pred_calls"]))), + solve_calls=int(os.environ.get( + "PRB_SOLVE_CALLS", str(EXPECTED_EPISODE_BUDGET["solve_calls"]))), submit_attempts=2, max_output_chars=int(os.environ.get("PRB_MAX_OUTPUT_CHARS", "10000")), pred_timeout_seconds=int(os.environ.get("PRB_PRED_TIMEOUT_SECONDS", "300"))), @@ -55,12 +72,14 @@ def run(*, model: str, repo_dir: str | Path, output: str | Path, if len(inventory) < 50: raise ValueError(f"canonical inventory has only {len(inventory)} runnable rules") pred_binary = find_pred_binary() - verify_pred_version(pred_binary) + pred_version = verify_pred_version(pred_binary) contract = pilot_contract() if fake: runner = Top50Runner( executor=_FakeExecutor(), contract=contract, pred_binary=pred_binary) else: + if model_kwargs: + raise ValueError("rankable Top50 runs do not accept custom model kwargs") runner = build_rankable_runner( contract=contract, pred_binary=pred_binary, agent_uid=int(os.environ["PRB_AGENT_UID"]), @@ -69,12 +88,19 @@ def run(*, model: str, repo_dir: str | Path, output: str | Path, oracle_gid=int(os.environ["PRB_ORACLE_GID"]), evidence_gid=int(os.environ["PRB_EVIDENCE_GID"]), api_base=api_base, api_key=api_key, model_kwargs=model_kwargs) - result = runner.run(model=model, repo_path=repo, inventory=inventory, output=output) - result["library_commit"] = pinned_commit() - result["budget_contract_status"] = "pilot-unfrozen" - # Rewrite once with provenance added after the runner's final checkpoint. - Path(output).write_text(json.dumps(result, indent=2), encoding="utf-8") - return result + metadata = { + "benchmark_contract": CONTRACT_ID, + "library_commit": pinned_commit(), + "runner_version": RUNNER_VERSION, + "pred_version": pred_version, + "agent_mode": AGENT_MODE, + "prompt_id": expected_prompt_id(), + "budget_contract_status": "pilot-unfrozen", + "created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "inference_parameters": model_kwargs or {}, + } + return runner.run(model=model, repo_path=repo, inventory=inventory, + output=output, metadata=metadata) def main(argv: list[str] | None = None) -> None: diff --git a/benchmark/submit.py b/benchmark/submit.py index 84d233b..9f5983f 100644 --- a/benchmark/submit.py +++ b/benchmark/submit.py @@ -48,6 +48,10 @@ def validate_submission(sub: dict) -> list[str]: New runs prove provenance through the bounded submit ledger. Legacy runs must carry a certificate plus a trajectory on the row or envelope. """ + from benchmark.top50_contract import is_top50_submission, validate_top50_submission + if is_top50_submission(sub): + return validate_top50_submission(sub) + problems: list[str] = [] if not isinstance(sub, dict): return ["submission is not a JSON object"] diff --git a/benchmark/tests/test_top50_submission.py b/benchmark/tests/test_top50_submission.py new file mode 100644 index 0000000..110714c --- /dev/null +++ b/benchmark/tests/test_top50_submission.py @@ -0,0 +1,255 @@ +"""Acceptance tests for the private Top50 contract and aggregate-only public score.""" +from __future__ import annotations + +import copy +import json +import runpy +from pathlib import Path + +import pytest + +from benchmark.backend_score import _dedup_best, aggregate_leaderboard, score_one +from benchmark.submit import validate_submission +from benchmark.top50_contract import ( + AGENT_MODE, + CONTRACT_ID, + RUNNER_VERSION, + expected_prompt_id, + score_top50_submission, + top50_public_entry, + validate_top50_submission, +) +from benchmark.verify import Verdict + + +def _status(limit: int, used: int = 0) -> dict: + return {"used": used, "limit": limit, "remaining": limit - used} + + +def _artifact(accepted_positions=(7, 18, 41)) -> dict: + triage_budget = { + "model_generations": 8, "shell_actions": 12, + "max_output_chars": 10_000, "command_timeout_seconds": 300, + } + episode_budget = { + "model_generations": 10, "shell_actions": 12, "pred_calls": 24, + "solve_calls": 10, "submit_attempts": 2, "max_output_chars": 10_000, + "pred_timeout_seconds": 300, + } + shortlist = [{"rule": f"rule_{index:02d}", "hypothesis": f"risk-{index}"} + for index in range(1, 51)] + triage_status = {"model_generations": _status(8, 1), + "shell_actions": _status(12, 1), + "pred_calls": _status(0), "solve_calls": _status(0)} + triage = { + "ledger": { + "budget": triage_budget, + "status": triage_status, + "events": [ + {"sequence": 1, "type": "model_generation", "charged": True}, + {"sequence": 2, "type": "shell_action", "charged": True}, + {"type": "shortlist_commit", "accepted": True}, + ], + "shortlist": copy.deepcopy(shortlist), + }, + "messages": [], "tokens_k": 0.1, + "usage": {"input": 100, "output": 10, "cache_read": 0, "cache_write": 0}, + } + episodes = [] + request = 1 + for position, entry in enumerate(shortlist, 1): + attempts = [] + submit_used = 0 + submit_remaining = 2 + status = "completed" + if position in accepted_positions: + attempts.append({ + "attempt": 1, + "request_id": f"{request:032x}", + "accepted": True, + "rule": entry["rule"], + "reason": "confirmed", + "certificate": {"rule": entry["rule"], "source": {}, + "bundle": {"target": {"type": "Target"}}}, + }) + request += 1 + submit_used, submit_remaining, status = 1, 0, "bug_found" + ledger_status = { + "model_generations": _status(10), "shell_actions": _status(12), + "pred_calls": _status(24), "solve_calls": _status(10), + "submit_attempts": {"used": submit_used, "limit": 2, + "remaining": submit_remaining}, + } + episodes.append({ + "index": position, "rule": entry["rule"], "hypothesis": entry["hypothesis"], + "status": status, "accepted_submit_attempt": 1 if attempts else None, + "ledger": {"rule": entry["rule"], "budget": copy.deepcopy(episode_budget), + "status": ledger_status, "pred": [], "submit": attempts, + "model_generations": [], "shell_actions": []}, + "messages": [], "tokens_k": 0.01, + "usage": {"input": 10, "output": 1, "cache_read": 0, "cache_write": 0}, + }) + return { + "benchmark_contract": CONTRACT_ID, + "model": "provider/model", + "library_commit": "a" * 40, + "runner_version": RUNNER_VERSION, + "pred_version": "0.6.0", + "agent_mode": AGENT_MODE, + "prompt_id": expected_prompt_id(), + "status": "completed", + "rankable": True, + "inference_parameters": {}, + "contract": {"triage": triage_budget, "episode": episode_budget, + "shortlist_size": 50, "hypothesis_chars": 500}, + "shortlist": shortlist, + "triage": triage, + "episodes": episodes, + "bugs_found": 999, + } + + +def _accept(cert, repo_dir=None): + return Verdict(True, "reverified") + + +def test_valid_artifact_recomputes_score_and_prefix_metrics(): + submission = _artifact() + assert validate_top50_submission(submission) == [] + scored, report = score_top50_submission( + submission, verifier=_accept, + canonical_inventory={entry["rule"] for entry in submission["shortlist"]}) + + assert scored["rankable"] is True + assert scored["verified_bugs"] == 3 + assert scored["bugs_at_10"] == 1 + assert scored["bugs_at_25"] == 2 + assert scored["bugs_at_50"] == 3 + assert scored["first_attempt_accepts"] == 3 + assert len(report) == 3 + assert submission["bugs_found"] == 999 + assert validate_submission(submission) == [] + + +@pytest.mark.parametrize("mutate, expected", [ + (lambda sub: sub.update(submit_limit=100), "shared run-wide submit pool"), + (lambda sub: sub["episodes"].pop(), "exactly 50"), + (lambda sub: sub["episodes"].__setitem__(1, copy.deepcopy(sub["episodes"][0])), + "order/rule"), + (lambda sub: sub["episodes"][0]["ledger"]["status"]["pred_calls"].update(used=25), + "usage is inconsistent"), + (lambda sub: sub["episodes"][0]["ledger"]["budget"].update(pred_calls=23), + "budget differs"), + (lambda sub: sub["contract"]["episode"].update(pred_calls=1000000), + "versioned contract"), + (lambda sub: sub.update(agent_mode="codex"), "agent_mode"), + (lambda sub: sub.update(prompt_id="custom"), "prompt_id"), + (lambda sub: sub.pop("inference_parameters"), "inference_parameters"), + (lambda sub: sub.update(status="run_error", run_error="provider failed"), "incomplete"), +]) +def test_rankability_negative_controls(mutate, expected): + submission = _artifact(accepted_positions=()) + mutate(submission) + assert any(expected in problem for problem in validate_top50_submission(submission)) + + +def test_wrong_episode_certificate_and_third_attempt_are_unrankable(): + submission = _artifact(accepted_positions=(1,)) + attempt = submission["episodes"][0]["ledger"]["submit"][0] + attempt["certificate"]["rule"] = "rule_02" + assert any("certificate rule mismatch" in p + for p in validate_top50_submission(submission)) + + submission = _artifact(accepted_positions=()) + episode = submission["episodes"][0] + episode["ledger"]["submit"] = [ + {"attempt": n, "request_id": f"{n:032x}", "accepted": False, "reason": "no"} + for n in range(1, 4)] + episode["ledger"]["status"]["submit_attempts"] = {"used": 3, "limit": 2, "remaining": 0} + assert any("at most two" in p for p in validate_top50_submission(submission)) + + +def test_strict_boolean_commit_and_canonical_inventory_controls(): + submission = _artifact(accepted_positions=(1,)) + submission["episodes"][0]["ledger"]["submit"][0]["accepted"] = "false" + assert any("accepted field must be boolean" in p + for p in validate_top50_submission(submission)) + + submission = _artifact(accepted_positions=()) + submission["triage"]["ledger"]["events"] = submission["triage"]["ledger"]["events"][:2] + assert any("exactly one accepted shortlist_commit" in p + for p in validate_top50_submission(submission)) + + submission = _artifact(accepted_positions=()) + canonical = {entry["rule"] for entry in submission["shortlist"]} - {"rule_50"} + assert any("non-canonical" in p + for p in validate_top50_submission(submission, canonical)) + + +def test_public_projection_contains_no_answer_key_material(): + submission = _artifact() + scored, _ = score_top50_submission( + submission, verifier=_accept, + canonical_inventory={entry["rule"] for entry in submission["shortlist"]}) + public = top50_public_entry(submission, scored) + raw = json.dumps(public) + + for forbidden in ("rule_", "hypothesis", "certificate", '"source"', '"bundle"', + "submit_log", "submit_attempt", "trajectory", "reason"): + assert forbidden not in raw + assert public["bugs_found"] == 3 + + guard = runpy.run_path(str( + Path(__file__).parents[2] / ".github" / "scripts" / "check_aggregate.py"))["check"] + # The actual egress guard accepts the generated projection, not merely this key test. + # Use pytest's temporary path in the dedicated backend test below for file-based checks. + assert callable(guard) + + +def test_top50_ties_ignore_tokens_and_efficiency_and_legacy_is_separate(): + entries = [ + {"benchmark_contract": CONTRACT_ID, "model": "a", "bugs_found": 2, + "total_tokens_k": 1, "efficiency_bugs_per_ktok": 999}, + {"benchmark_contract": CONTRACT_ID, "model": "b", "bugs_found": 2, + "total_tokens_k": 1000, "efficiency_bugs_per_ktok": 0.001}, + {"model": "a", "bugs_found": 9, "efficiency_bugs_per_ktok": 9}, + ] + board = _dedup_best(entries) + assert len(board) == 3 + top50 = [entry for entry in board if entry.get("benchmark_contract") == CONTRACT_ID] + assert [entry["model"] for entry in top50] == ["a", "b"] + + +def test_backend_official_path_keeps_private_detail_and_publishes_aggregate(tmp_path): + submission = _artifact(accepted_positions=()) + repo = tmp_path / "repo" + rules = repo / "src" / "rules" + rules.mkdir(parents=True) + for entry in submission["shortlist"]: + (rules / f"{entry['rule']}.rs").write_text("// fixture") + source = tmp_path / "submission.json" + source.write_text(json.dumps(submission)) + results = tmp_path / "results" + + public = score_one( + source, results, repo_dir=str(repo), official=True, + expected_commit=submission["library_commit"]) + board = aggregate_leaderboard(results) + private = json.loads((results / "submission.json").read_text()) + + assert private["artifact_sha256"] and "episodes" not in private + assert public["benchmark_contract"] == CONTRACT_ID + assert board == [public] + assert "episodes" not in public and "shortlist" not in public + guard = runpy.run_path(str( + Path(__file__).parents[2] / ".github" / "scripts" / "check_aggregate.py"))["check"] + public_file = tmp_path / "public.json" + public_file.write_text(json.dumps(public)) + assert guard(public_file) == [] + + +def test_site_keeps_contracts_separate_and_top50_ties_ignore_efficiency(): + site = (Path(__file__).parents[2] / "site" / "index.html").read_text() + assert 'id="lb-track"' in site + assert 'top50=track.startsWith("top50-evidence/")' in site + assert "top50?String(a.model).localeCompare" in site diff --git a/benchmark/top50_contract.py b/benchmark/top50_contract.py new file mode 100644 index 0000000..8258434 --- /dev/null +++ b/benchmark/top50_contract.py @@ -0,0 +1,447 @@ +"""Versioned private Top50 artifact validation, scoring, and public projection.""" +from __future__ import annotations + +import copy +import hashlib +import inspect +import json +import os +import re +from collections import Counter +from pathlib import Path +from typing import Callable + +from benchmark.usage import Usage, usage_as_dict, usage_from_dict +from benchmark.verify import Verdict, verify + +CONTRACT_ID = "top50-evidence/pilot-v1" +AGENT_MODE = "standardized-model-api" +RUNNER_VERSION = "0.9.0" +EXPECTED_TRIAGE_BUDGET = { + "model_generations": 8, "shell_actions": 12, + "max_output_chars": 10_000, "command_timeout_seconds": 300, +} +EXPECTED_EPISODE_BUDGET = { + "model_generations": 10, "shell_actions": 12, "pred_calls": 24, + "solve_calls": 10, "submit_attempts": 2, "max_output_chars": 10_000, + "pred_timeout_seconds": 300, +} +_REQUEST_ID = re.compile(r"[0-9a-f]{32}") +_COUNTERS = ("model_generations", "shell_actions", "pred_calls", "solve_calls") + + +def is_top50_submission(submission: object) -> bool: + return isinstance(submission, dict) and str( + submission.get("benchmark_contract", "")).startswith("top50-evidence/") + + +def expected_prompt_id() -> str: + return hashlib.sha256(Path(__file__).with_name("top50_config.yaml").read_bytes()).hexdigest() + + +def validate_top50_submission( + submission: object, canonical_inventory: set[str] | None = None +) -> list[str]: + """Recompute structural rankability from private evaluation-owned ledgers.""" + if not isinstance(submission, dict): + return ["submission is not a JSON object"] + problems: list[str] = [] + for field in ("benchmark_contract", "model", "library_commit", "runner_version", + "pred_version", "agent_mode", "prompt_id", "contract", "shortlist", + "triage", "episodes", "status"): + if field not in submission: + problems.append(f"missing required field: {field}") + if problems: + return problems + if submission["benchmark_contract"] != CONTRACT_ID: + problems.append(f"unsupported benchmark_contract: {submission['benchmark_contract']!r}") + if submission["agent_mode"] != AGENT_MODE: + problems.append(f"agent_mode must be {AGENT_MODE!r}") + if submission["runner_version"] != RUNNER_VERSION: + problems.append(f"runner_version must be {RUNNER_VERSION!r}") + if submission.get("status") != "completed" or submission.get("run_error"): + problems.append("Top50 run is incomplete or has run_error") + if submission.get("rankable") is not True: + problems.append("runner did not certify the standardized rankable path") + model = submission.get("model") + if not isinstance(model, str) or not re.fullmatch(r"[A-Za-z0-9._:/-]{1,200}", model): + problems.append("model identifier is not a bounded safe identifier") + submitter = submission.get("submitted_by") + if (submitter is not None and (not isinstance(submitter, str) + or not re.fullmatch(r"[A-Za-z0-9_.@-]{1,100}", submitter))): + problems.append("submitted_by is not a bounded safe identifier") + if not isinstance(submission.get("inference_parameters"), dict): + problems.append("inference_parameters must be a normalized object") + if submission.get("prompt_id") != expected_prompt_id(): + problems.append("prompt_id does not match the frozen Top50 prompt") + if "submit_limit" in submission or "submit_log" in submission: + problems.append("Top50 artifacts cannot use a shared run-wide submit pool") + + contract = submission.get("contract") + if not isinstance(contract, dict): + return problems + ["contract must be an object"] + triage_budget = contract.get("triage") + episode_budget = contract.get("episode") + if not isinstance(triage_budget, dict) or not isinstance(episode_budget, dict): + return problems + ["contract triage and episode budgets must be objects"] + if triage_budget != EXPECTED_TRIAGE_BUDGET: + problems.append("triage budget differs from the versioned contract") + if episode_budget != EXPECTED_EPISODE_BUDGET: + problems.append("episode budget differs from the versioned contract") + if contract.get("shortlist_size") != 50: + problems.append("contract shortlist_size must be 50") + hypothesis_limit = contract.get("hypothesis_chars") + if not _nonnegative_int(hypothesis_limit): + problems.append("contract hypothesis_chars must be a non-negative integer") + hypothesis_limit = -1 + if episode_budget.get("submit_attempts") != 2: + problems.append("episode submit_attempts must be exactly 2") + pred_limit = episode_budget.get("pred_calls") + solve_limit = episode_budget.get("solve_calls") + if not _nonnegative_int(pred_limit) or not _nonnegative_int(solve_limit): + problems.append("pred_calls and solve_calls limits must be non-negative integers") + elif solve_limit > pred_limit: + problems.append("solve_calls limit exceeds pred_calls") + + shortlist = submission.get("shortlist") + if not isinstance(shortlist, list) or len(shortlist) != 50: + return problems + ["shortlist must contain exactly 50 entries"] + rules: list[str] = [] + for index, entry in enumerate(shortlist): + if not isinstance(entry, dict) or not isinstance(entry.get("rule"), str): + problems.append(f"shortlist[{index}] must have a rule string") + continue + rules.append(entry["rule"]) + hypothesis = entry.get("hypothesis", "") + if (set(entry) - {"rule", "hypothesis"} or not isinstance(hypothesis, str) + or len(hypothesis) > hypothesis_limit): + problems.append(f"shortlist[{index}] hypothesis schema is invalid") + if len(set(rules)) != len(rules): + problems.append("shortlist rules must be unique") + if canonical_inventory is not None: + unknown = sorted(set(rules) - canonical_inventory) + if unknown: + problems.append(f"shortlist contains non-canonical rules: {unknown[:3]}") + + triage = submission.get("triage") + triage_ledger = triage.get("ledger") if isinstance(triage, dict) else None + if not isinstance(triage_ledger, dict): + problems.append("triage.ledger must be an object") + else: + if triage_ledger.get("budget") != triage_budget: + problems.append("triage ledger budget differs from contract") + frozen = triage_ledger.get("shortlist") + if frozen != shortlist: + problems.append("top-level shortlist differs from frozen triage shortlist") + _validate_status(triage_ledger.get("status"), triage_budget, + "triage", problems, counters=("model_generations", "shell_actions")) + _validate_triage_events(triage_ledger, problems) + + episodes = submission.get("episodes") + if not isinstance(episodes, list) or len(episodes) != 50: + return problems + ["episodes must contain exactly 50 entries"] + seen_request_ids: set[str] = set() + for index, episode in enumerate(episodes): + label = f"episodes[{index}]" + if not isinstance(episode, dict): + problems.append(f"{label} must be an object") + continue + expected_rule = rules[index] if index < len(rules) else None + if episode.get("index") != index + 1 or episode.get("rule") != expected_rule: + problems.append(f"{label} order/rule does not match frozen shortlist") + expected_hypothesis = shortlist[index].get("hypothesis", "") + if episode.get("hypothesis") != expected_hypothesis: + problems.append(f"{label} hypothesis differs from frozen shortlist") + if episode.get("status") == "run_error": + problems.append(f"{label} has infrastructure run_error") + ledger = episode.get("ledger") + if not isinstance(ledger, dict): + problems.append(f"{label}.ledger must be an object") + continue + if ledger.get("rule") != expected_rule or ledger.get("budget") != episode_budget: + problems.append(f"{label} rule or budget differs from contract") + messages = episode.get("messages") + if not isinstance(messages, list) or len(messages) > 3 * episode_budget.get( + "model_generations", 0) + 4: + problems.append(f"{label} message history exceeds its logical bound") + elif any(not isinstance(message, dict) + or len(str(message.get("content", ""))) > 200_000 for message in messages): + problems.append(f"{label} contains an oversized message") + _validate_status(ledger.get("status"), episode_budget, label, problems) + _validate_episode_events(ledger, label, problems) + _validate_pred_ledger(ledger, label, seen_request_ids, problems) + _validate_submit_ledger(ledger, expected_rule, label, seen_request_ids, problems) + accepted_attempts = [attempt for attempt in ledger.get("submit", []) + if isinstance(attempt, dict) and attempt.get("accepted") is True] + expected_status = "bug_found" if accepted_attempts else "completed" + expected_attempt = accepted_attempts[0].get("attempt") if accepted_attempts else None + if (episode.get("status") != expected_status + or episode.get("accepted_submit_attempt") != expected_attempt): + problems.append(f"{label} outcome does not match its submit ledger") + return list(dict.fromkeys(problems)) + + +def score_top50_submission( + submission: dict, repo_dir: str | None = None, + *, verifier: Callable[[dict], Verdict] = verify, + canonical_inventory: set[str] | None = None, +) -> tuple[dict, list[dict]]: + """Validate ledgers and independently re-verify accepted certificates.""" + inventory = canonical_inventory or load_canonical_inventory(repo_dir) + problems = validate_top50_submission(submission, inventory) + accepted_positions: list[int] = [] + report: list[dict] = [] + first_attempt = second_attempt = 0 + pred_calls = 0 + cap_hits: Counter[str] = Counter() + if not problems: + for position, episode in enumerate(submission["episodes"], 1): + ledger = episode["ledger"] + pred_calls += ledger["status"]["pred_calls"]["used"] + for counter in _COUNTERS: + status = ledger["status"][counter] + if status["limit"] > 0 and status["used"] == status["limit"]: + cap_hits[counter] += 1 + for attempt in ledger["submit"]: + if attempt.get("accepted") is not True: + continue + cert = attempt.get("certificate") + verdict = _call_verifier(verifier, cert, repo_dir) + accepted = bool(verdict.accepted) + report.append({"position": position, "rule": episode["rule"], + "violation": cert.get("violation") if isinstance(cert, dict) else None, + "attempt": attempt["attempt"], + "accepted": accepted, "reason": verdict.reason}) + if accepted: + accepted_positions.append(position) + if attempt["attempt"] == 1: + first_attempt += 1 + else: + second_attempt += 1 + break + distinct_positions = sorted(set(accepted_positions)) + bugs = len(distinct_positions) + usage = _aggregate_usage(submission) + scored = { + "benchmark_contract": submission.get("benchmark_contract"), + "model": submission.get("model", "unknown"), + "library_commit": submission.get("library_commit", "unknown"), + "runner_version": submission.get("runner_version"), + "pred_version": submission.get("pred_version"), + "agent_mode": submission.get("agent_mode"), + "rankable": not problems, + "rankability_errors": problems, + "verified_bugs": bugs, + "bugs_found": bugs, + "bugs_at_10": sum(position <= 10 for position in distinct_positions), + "bugs_at_25": sum(position <= 25 for position in distinct_positions), + "bugs_at_50": bugs, + "first_attempt_accepts": first_attempt, + "second_attempt_accepts": second_attempt, + "pred_calls_per_bug": round(pred_calls / bugs, 4) if bugs else None, + "cap_hits": dict(cap_hits), + "usage_totals": usage_as_dict(usage), + "total_tokens_k": round(usage.total_tokens / 1000, 2), + "artifact_sha256": hashlib.sha256(json.dumps( + submission, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), + "verifier_report": copy.deepcopy(report), + } + for field in ("submitted_by", "created_at", "test", "run_error"): + if field in submission: + scored[field] = submission[field] + return scored, report + + +def top50_public_entry(submission: dict, scored: dict) -> dict: + """Project private scoring data to aggregate-only public fields.""" + return { + "benchmark_contract": scored["benchmark_contract"], + "model": scored["model"], + "library_commit": scored["library_commit"], + "runner_version": scored.get("runner_version"), + "pred_version": scored.get("pred_version"), + "agent_mode": scored.get("agent_mode"), + "rankable": scored["rankable"], + "bugs_found": scored["verified_bugs"], + "rules_tested": 50, + "bugs_at_10": scored["bugs_at_10"], + "bugs_at_25": scored["bugs_at_25"], + "bugs_at_50": scored["bugs_at_50"], + "first_attempt_accepts": scored["first_attempt_accepts"], + "second_attempt_accepts": scored["second_attempt_accepts"], + "pred_calls_per_bug": scored["pred_calls_per_bug"], + "cap_hits": scored["cap_hits"], + "usage_totals": scored["usage_totals"], + "total_tokens_k": scored["total_tokens_k"], + "submitted_by": submission.get("submitted_by"), + "placeholder": False, + } + + +def _validate_status(status, budget, label, problems, counters=_COUNTERS) -> None: + if not isinstance(status, dict): + problems.append(f"{label} status must be an object") + return + for counter in counters: + item = status.get(counter) + limit = budget.get(counter) + if not _nonnegative_int(limit): + problems.append(f"{label} {counter} limit is not a non-negative integer") + continue + if not isinstance(item, dict) or item.get("limit") != limit: + problems.append(f"{label} {counter} limit differs from contract") + continue + used, remaining = item.get("used"), item.get("remaining") + if (not _nonnegative_int(used) or used > limit + or remaining != limit - used): + problems.append(f"{label} {counter} usage is inconsistent") + if "submit_attempts" in status: + item = status["submit_attempts"] + if not isinstance(item, dict) or item.get("limit") != 2: + problems.append(f"{label} submit limit must be 2") + elif (not _nonnegative_int(item.get("used")) or item["used"] > 2 + or not _nonnegative_int(item.get("remaining"))): + problems.append(f"{label} submit usage is inconsistent") + + +def _validate_triage_events(ledger: dict, problems: list[str]) -> None: + events = ledger.get("events") + if not isinstance(events, list): + problems.append("triage events must be a list") + return + budget = ledger.get("budget", {}) + if len(events) > budget.get("model_generations", 0) + budget.get("shell_actions", 0) + 3: + problems.append("triage event ledger exceeds its logical bound") + commit_events = [event for event in events if isinstance(event, dict) + and event.get("type") == "shortlist_commit" + and event.get("accepted") is True] + if len(commit_events) != 1: + problems.append("triage must contain exactly one accepted shortlist_commit") + sequenced = [event for event in events if isinstance(event, dict) and "sequence" in event] + if [event.get("sequence") for event in sequenced] != list(range(1, len(sequenced) + 1)): + problems.append("triage event sequence is inconsistent") + if any("charged" in event and type(event["charged"]) is not bool for event in events + if isinstance(event, dict)): + problems.append("triage charged fields must be booleans") + charged = Counter(event.get("type") for event in events + if isinstance(event, dict) and event.get("charged") is True) + status = ledger.get("status", {}) + if charged["model_generation"] != status.get("model_generations", {}).get("used"): + problems.append("triage model-generation ledger count is inconsistent") + if charged["shell_action"] != status.get("shell_actions", {}).get("used"): + problems.append("triage shell-action ledger count is inconsistent") + + +def _validate_pred_ledger(ledger, label, seen_ids, problems) -> None: + records = ledger.get("pred") + if not isinstance(records, list): + problems.append(f"{label} pred ledger must be a list") + return + if len(records) > ledger.get("budget", {}).get("shell_actions", 0) + 1: + problems.append(f"{label} pred ledger exceeds its logical bound") + charged = solve = 0 + for sequence, record in enumerate(records, 1): + if not isinstance(record, dict) or record.get("sequence") != sequence: + problems.append(f"{label} pred sequence is inconsistent") + continue + request_id = record.get("request_id") + if not isinstance(request_id, str) or not _REQUEST_ID.fullmatch(request_id): + problems.append(f"{label} pred request id is invalid") + elif request_id in seen_ids: + problems.append(f"{label} reuses a request id") + else: + seen_ids.add(request_id) + if type(record.get("charged")) is not bool: + problems.append(f"{label} pred charged field must be boolean") + if record.get("charged") is True: + charged += 1 + solve += record.get("command") == "solve" + status = ledger.get("status", {}) + if charged != status.get("pred_calls", {}).get("used"): + problems.append(f"{label} pred charged count differs from status") + if solve != status.get("solve_calls", {}).get("used"): + problems.append(f"{label} solve charged count differs from status") + + +def _validate_episode_events(ledger, label, problems) -> None: + status = ledger.get("status", {}) + for field in ("model_generations", "shell_actions"): + events = ledger.get(field) + if not isinstance(events, list): + problems.append(f"{label} {field} ledger must be a list") + continue + limit = ledger.get("budget", {}).get(field, 0) + if len(events) > limit + 1: + problems.append(f"{label} {field} ledger exceeds its logical bound") + if any(type(event.get("charged")) is not bool for event in events + if isinstance(event, dict)): + problems.append(f"{label} {field} charged fields must be booleans") + charged = sum(event.get("charged") is True for event in events + if isinstance(event, dict)) + if charged != status.get(field, {}).get("used"): + problems.append(f"{label} {field} charged count differs from status") + + +def _validate_submit_ledger(ledger, expected_rule, label, seen_ids, problems) -> None: + attempts = ledger.get("submit") + if not isinstance(attempts, list) or len(attempts) > 2: + problems.append(f"{label} must have at most two submit attempts") + return + status = ledger.get("status", {}).get("submit_attempts", {}) + if status.get("used") != len(attempts): + problems.append(f"{label} submit count differs from status") + accepted = 0 + for number, attempt in enumerate(attempts, 1): + if not isinstance(attempt, dict) or attempt.get("attempt") != number: + problems.append(f"{label} submit attempt numbering is inconsistent") + continue + request_id = attempt.get("request_id") + if not isinstance(request_id, str) or not _REQUEST_ID.fullmatch(request_id): + problems.append(f"{label} submit request id is invalid") + elif request_id in seen_ids: + problems.append(f"{label} reuses a request id") + else: + seen_ids.add(request_id) + if type(attempt.get("accepted")) is not bool: + problems.append(f"{label} submit accepted field must be boolean") + if attempt.get("accepted") is True: + accepted += 1 + certificate = attempt.get("certificate") + if not isinstance(certificate, dict) or certificate.get("rule") != expected_rule: + problems.append(f"{label} accepted certificate rule mismatch") + if accepted > 1: + problems.append(f"{label} has more than one accepted submit attempt") + expected_remaining = 0 if accepted else 2 - len(attempts) + if status.get("remaining") != expected_remaining: + problems.append(f"{label} submit remaining count is inconsistent") + + +def load_canonical_inventory(repo_dir: str | None) -> set[str]: + """Load source-rule identifiers from the pinned repository bundled with the image.""" + candidate = Path(repo_dir or os.environ.get("REPO_DIR", "/app/pr-src")) + if not (candidate / "src" / "rules").is_dir(): + raise RuntimeError(f"pinned canonical rule inventory is unavailable under {candidate}") + from benchmark.run_mini import list_rules + return set(list_rules(candidate)) + + +def _aggregate_usage(submission: dict) -> Usage: + total = usage_from_dict(submission.get("triage", {}).get("usage")) + for episode in submission.get("episodes", []): + total = total + usage_from_dict(episode.get("usage")) + return total + + +def _nonnegative_int(value) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def _call_verifier(verifier, certificate, repo_dir): + """Bind the verifier signature before execution; never retry an internal TypeError.""" + signature = inspect.signature(verifier) + positional = [parameter for parameter in signature.parameters.values() + if parameter.kind in (parameter.POSITIONAL_ONLY, + parameter.POSITIONAL_OR_KEYWORD)] + variadic = any(parameter.kind == parameter.VAR_POSITIONAL + for parameter in signature.parameters.values()) + return verifier(certificate, repo_dir) if variadic or len(positional) >= 2 else verifier(certificate) diff --git a/benchmark/top50_results.schema.json b/benchmark/top50_results.schema.json new file mode 100644 index 0000000..7072eee --- /dev/null +++ b/benchmark/top50_results.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "top50_results.schema.json", + "title": "Top50EvidenceScoredResult", + "type": "object", + "required": ["benchmark_contract", "model", "library_commit", "rankable", + "rankability_errors", "verified_bugs", "bugs_at_10", "bugs_at_25", "bugs_at_50", + "first_attempt_accepts", "second_attempt_accepts", "usage_totals", + "artifact_sha256", "verifier_report"], + "properties": { + "benchmark_contract": {"const": "top50-evidence/pilot-v1"}, + "model": {"type": "string"}, + "library_commit": {"type": "string"}, + "rankable": {"type": "boolean"}, + "rankability_errors": {"type": "array", "items": {"type": "string"}}, + "verified_bugs": {"type": "integer", "minimum": 0, "maximum": 50}, + "bugs_at_10": {"type": "integer", "minimum": 0, "maximum": 10}, + "bugs_at_25": {"type": "integer", "minimum": 0, "maximum": 25}, + "bugs_at_50": {"type": "integer", "minimum": 0, "maximum": 50}, + "first_attempt_accepts": {"type": "integer", "minimum": 0}, + "second_attempt_accepts": {"type": "integer", "minimum": 0}, + "usage_totals": {"type": "object"}, + "artifact_sha256": {"type": "string", "minLength": 64, "maxLength": 64}, + "verifier_report": {"type": "array"} + } +} diff --git a/benchmark/top50_runner.py b/benchmark/top50_runner.py index 3eb1499..be46c38 100644 --- a/benchmark/top50_runner.py +++ b/benchmark/top50_runner.py @@ -237,7 +237,9 @@ def __init__(self, *, executor: PhaseExecutor, contract: Top50Contract, self._rankable_contract = _rankable_token is _RANKABLE_TOKEN def run(self, *, model: str, repo_path: str | Path, - inventory: list[str] | tuple[str, ...], output: str | Path | None = None) -> dict: + inventory: list[str] | tuple[str, ...], output: str | Path | None = None, + metadata: dict | None = None) -> dict: + self._result_metadata = copy.deepcopy(metadata or {}) repo_path = Path(repo_path).resolve() canonical = tuple(inventory) with TriageSession( @@ -331,6 +333,7 @@ def _result(self, model: str, triage: dict, } if run_error: result["run_error"] = run_error + result.update(copy.deepcopy(getattr(self, "_result_metadata", {}))) return result diff --git a/benchmark/top50_submission.schema.json b/benchmark/top50_submission.schema.json new file mode 100644 index 0000000..1f61091 --- /dev/null +++ b/benchmark/top50_submission.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "top50_submission.schema.json", + "title": "Top50EvidenceSubmission", + "type": "object", + "required": ["benchmark_contract", "model", "library_commit", "runner_version", + "pred_version", "agent_mode", "prompt_id", "status", "contract", "shortlist", + "triage", "episodes", "inference_parameters"], + "properties": { + "benchmark_contract": {"const": "top50-evidence/pilot-v1"}, + "model": {"type": "string", "minLength": 1}, + "library_commit": {"type": "string", "minLength": 1}, + "runner_version": {"const": "0.9.0"}, + "pred_version": {"type": "string", "minLength": 1}, + "agent_mode": {"const": "standardized-model-api"}, + "prompt_id": {"type": "string", "minLength": 64, "maxLength": 64}, + "status": {"enum": ["completed", "run_error"]}, + "rankable": {"type": "boolean"}, + "contract": {"type": "object"}, + "shortlist": {"type": "array", "minItems": 50, "maxItems": 50}, + "triage": {"type": "object"}, + "episodes": {"type": "array", "minItems": 50, "maxItems": 50}, + "inference_parameters": {"type": "object"}, + "run_error": {"type": "string"} + } +} diff --git a/benchmark/verify_submission.py b/benchmark/verify_submission.py index fb8ea1e..9b4b4fb 100644 --- a/benchmark/verify_submission.py +++ b/benchmark/verify_submission.py @@ -20,7 +20,6 @@ import argparse import json import re -import sys from pathlib import Path from benchmark.submit_ledger import (accepted_certificate_index, certificate_key, @@ -79,6 +78,10 @@ def score_submission(submission: dict, repo_dir: str | None = None) -> tuple[dic Returns (scored, report); ``scored`` is results.schema.json-shaped and ``report`` is a per-certificate list of {rule, violation, accepted, reason, provenance}. """ + from benchmark.top50_contract import is_top50_submission, score_top50_submission + if is_top50_submission(submission): + return score_top50_submission(submission, repo_dir) + rescored: list[dict] = [] report: list[dict] = [] ledger_problem = submit_ledger_error(submission) @@ -173,6 +176,10 @@ def leaderboard_entry(submission: dict, scored: dict) -> dict: private scored result the maintainer holds; the leaderboard shows only how many each model found. """ + from benchmark.top50_contract import is_top50_submission, top50_public_entry + if is_top50_submission(submission) or is_top50_submission(scored): + return top50_public_entry(submission, scored) + return { "model": scored["model"], "library_commit": scored.get("library_commit", "unknown"), diff --git a/docker/Dockerfile b/docker/Dockerfile index 94478c0..9d4b2c5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -75,6 +75,11 @@ ENV PRED_BINARY=/usr/local/libexec/prb/pred \ WORKDIR /app COPY benchmark/ ./benchmark/ +# The scorer and Top50 triage share the exact pinned source-rule inventory. Keeping it in +# the runtime image lets official scoring reject invented/renamed shortlist entries. +COPY --from=pred-build /src/src /app/pr-src/src +COPY --from=pred-build /src/Cargo.toml /app/pr-src/Cargo.toml +ENV REPO_DIR=/app/pr-src # Fail the build if pred can't run (catches a missing runtime lib early). RUN /usr/local/libexec/prb/pred --version @@ -115,13 +120,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certifi && curl -fsSL https://claude.ai/install.sh | bash ENV PATH="/root/.local/bin:${PATH}" RUN claude --version -# The agent reads rule sources from REPO_DIR/src/rules — copy just the source tree -# (not target/) from the build stage. pred itself is a self-contained binary on PATH. -COPY --from=pred-build /src/src /app/pr-src/src -COPY --from=pred-build /src/Cargo.toml /app/pr-src/Cargo.toml RUN mkdir -p /out -ENV REPO_DIR=/app/pr-src \ - OUTPUT=/out/submission.json +ENV OUTPUT=/out/submission.json VOLUME ["/out"] # Build-time sanity that the assembled runner imports and runs (no API/pred calls). RUN python -m benchmark.run_top50 --fake --model fake/check \ diff --git a/site/index.html b/site/index.html index cc06c16..a6f6be4 100644 --- a/site/index.html +++ b/site/index.html @@ -270,6 +270,7 @@

Leaderboard

+ Click a row for its run stats →
@@ -408,11 +409,18 @@

How to cite

const RIGHT=new Set(["bugs","tokens","eff"]); function buildLbRows(){ - const ranked=RESULTS.slice() - .sort((a,b)=>(b.bugs_found||0)-(a.bugs_found||0)||(b.efficiency_bugs_per_ktok||0)-(a.efficiency_bugs_per_ktok||0)); + const track=document.getElementById("lb-track").value||"legacy-whole-repo"; + const top50=track.startsWith("top50-evidence/"); + const ranked=RESULTS.filter(r=>(r.benchmark_contract||"legacy-whole-repo")===track) + .sort((a,b)=>(b.bugs_found||0)-(a.bugs_found||0)|| + (top50?String(a.model).localeCompare(String(b.model)): + (b.efficiency_bugs_per_ktok||0)-(a.efficiency_bugs_per_ktok||0))); + let lastBugs=null,lastRank=0; lbRows=ranked.map((r,i)=>{ const tested=r.rules_tested||0, frac=TOTAL?Math.min(tested/TOTAL,1):0; - return {rank:i+1,model:short(r.model),_model:r.model,bugs:r.bugs_found||0, + const bugs=r.bugs_found||0; + const rank=top50&&i&&bugs===lastBugs?lastRank:i+1;lastBugs=bugs;lastRank=rank; + return {rank,model:short(r.model),_model:r.model,bugs, reachFrac:frac,tested,tokensNum:r.total_tokens_k||0, effNum:r.efficiency_bugs_per_ktok||0}; }); @@ -573,6 +581,9 @@

How to cite

document.getElementById("drawer-scrim").addEventListener("click",closeDrawer); document.addEventListener("keydown",e=>{if(e.key==="Escape")closeDrawer();}); document.getElementById("task-search").addEventListener("input",renderTasks); +document.getElementById("lb-track").addEventListener("change",()=>{ + buildLbRows();renderLbBody(); +}); document.querySelectorAll("#task-table thead th").forEach(th=>th.addEventListener("click",()=>{ const k=th.dataset.k; taskSort={key:k,dir:taskSort.key===k?-taskSort.dir:1}; renderTasks(); })); @@ -582,6 +593,10 @@

How to cite

fetch("tasks.json").then(r=>r.json()).catch(()=>[]), ]).then(([res,tasks])=>{ RESULTS=res;TASKS=tasks;TOTAL=tasks.length||TOTAL_DEFAULT; + const tracks=[...new Set(RESULTS.map(r=>r.benchmark_contract||"legacy-whole-repo"))].sort(); + const trackSelect=document.getElementById("lb-track"); + trackSelect.innerHTML=tracks.map(t=>``).join(""); + const newest=tracks.find(t=>t.startsWith("top50-evidence/"));if(newest)trackSelect.value=newest; const rc=document.getElementById("rulecount"); if(rc)rc.textContent=TOTAL; if(RESULTS.some(r=>r.placeholder)) document.getElementById("lb-banner").innerHTML=``;