feat: run self-selected Top50 episodes

benchmark/agent_environment.py

@@ -6,6 +6,17 @@
 
 from benchmark.process_control import ProcessLimits, run_capped_process
 
+_SAFE_ENV_KEYS = {"PATH", "PYTHONPATH", "LANG", "LC_ALL", "PAGER", "MANPAGER", "LESS",
+                  "PRB_PRED_DIR", "PRB_SUBMIT_DIR", "PRB_ARTIFACT_DIR",
+                  "PRB_PRED_TIMEOUT", "PRB_SUBMIT_TIMEOUT"}
+
+
+def sanitized_agent_env(extra: dict[str, str] | None = None) -> dict[str, str]:
+    """Return the small non-secret environment visible to model-authored commands."""
+    merged = dict(os.environ)
+    merged.update(extra or {})
+    return {key: value for key, value in merged.items() if key in _SAFE_ENV_KEYS}
+
 
 def run_as_agent(
     command: str,
@@ -64,7 +75,7 @@ def execute(self, action: dict, cwd: str = "", *, timeout: int | None = None) ->
                 result = run_as_agent(
                     command,
                     cwd=action_cwd,
-                    env=os.environ | self.config.env,
+                    env=sanitized_agent_env(self.config.env),
                     timeout=timeout or self.config.timeout,
                     uid=uid,
                     gid=gid,

benchmark/process_control.py

@@ -132,6 +132,11 @@ def run_capped_process(
             timed_out = True
             terminate_process_group(process)
             process.wait()
+        else:
+            # A successful shell leader may leave background children behind. Kill the
+            # entire private process group before reading pipes to completion so no process
+            # can survive into a later rule episode or keep a pipe open indefinitely.
+            terminate_process_group(process)
         for reader in readers:
             reader.join()
         stdout, stderr = capture.result()

benchmark/run_top50.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""Production entrypoint for the standardized self-selected Top50 model track."""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+
+from benchmark.env_setup import find_pred_binary, pinned_commit, verify_pred_version
+from benchmark.evidence_budget import EvidenceBudget
+from benchmark.run_mini import list_rules
+from benchmark.top50_runner import (
+    PhaseResult,
+    Top50Contract,
+    Top50Runner,
+    TriageBudget,
+    build_rankable_runner,
+)
+
+
+def pilot_contract() -> Top50Contract:
+    """Return explicitly provisional values; issue #68 freezes the public contract."""
+    return Top50Contract(
+        triage=TriageBudget(
+            model_generations=int(os.environ.get("PRB_TRIAGE_GENERATIONS", "8")),
+            shell_actions=int(os.environ.get("PRB_TRIAGE_ACTIONS", "12"))),
+        episode=EvidenceBudget(
+            model_generations=int(os.environ.get("PRB_EPISODE_GENERATIONS", "10")),
+            shell_actions=int(os.environ.get("PRB_EPISODE_ACTIONS", "12")),
+            pred_calls=int(os.environ.get("PRB_PRED_CALLS", "24")),
+            solve_calls=int(os.environ.get("PRB_SOLVE_CALLS", "10")),
+            submit_attempts=2,
+            max_output_chars=int(os.environ.get("PRB_MAX_OUTPUT_CHARS", "10000")),
+            pred_timeout_seconds=int(os.environ.get("PRB_PRED_TIMEOUT_SECONDS", "300"))),
+    )
+
+
+class _FakeExecutor:
+    def run_triage(self, session, *, repo_path, inventory, model):
+        payload = session.workdir / "shortlist.json"
+        payload.write_text(json.dumps(list(inventory[:50])), encoding="utf-8")
+        session.commit_file(str(payload))
+        return PhaseResult(messages=[])
+
+    def run_episode(self, session, **kwargs):
+        return PhaseResult(messages=[])
+
+
+def run(*, model: str, repo_dir: str | Path, output: str | Path,
+        fake: bool = False, api_base: str | None = None, api_key: str | None = None,
+        model_kwargs: dict | None = None) -> dict:
+    repo = Path(repo_dir).resolve()
+    inventory = list_rules(repo)
+    if len(inventory) < 50:
+        raise ValueError(f"canonical inventory has only {len(inventory)} runnable rules")
+    pred_binary = find_pred_binary()
+    verify_pred_version(pred_binary)
+    contract = pilot_contract()
+    if fake:
+        runner = Top50Runner(
+            executor=_FakeExecutor(), contract=contract, pred_binary=pred_binary)
+    else:
+        runner = build_rankable_runner(
+            contract=contract, pred_binary=pred_binary,
+            agent_uid=int(os.environ["PRB_AGENT_UID"]),
+            agent_gid=int(os.environ["PRB_AGENT_GID"]),
+            oracle_uid=int(os.environ["PRB_ORACLE_UID"]),
+            oracle_gid=int(os.environ["PRB_ORACLE_GID"]),
+            evidence_gid=int(os.environ["PRB_EVIDENCE_GID"]),
+            api_base=api_base, api_key=api_key, model_kwargs=model_kwargs)
+    result = runner.run(model=model, repo_path=repo, inventory=inventory, output=output)
+    result["library_commit"] = pinned_commit()
+    result["budget_contract_status"] = "pilot-unfrozen"
+    # Rewrite once with provenance added after the runner's final checkpoint.
+    Path(output).write_text(json.dumps(result, indent=2), encoding="utf-8")
+    return result
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(description="Standardized Model API Top50 benchmark")
+    parser.add_argument("--model", default=os.environ.get("MODEL_NAME"), required=False)
+    parser.add_argument("--repo-dir", default=os.environ.get("REPO_DIR", "/app/pr-src"))
+    parser.add_argument("--output", default=os.environ.get("OUTPUT", "/out/submission.json"))
+    parser.add_argument("--api-base", default=os.environ.get("API_BASE"))
+    parser.add_argument("--api-key", default=os.environ.get("API_KEY"))
+    parser.add_argument("--model-kwargs", default=os.environ.get("MODEL_KWARGS"))
+    parser.add_argument("--fake", action="store_true", default=bool(os.environ.get("FAKE")))
+    args = parser.parse_args(argv)
+    if not args.model:
+        parser.error("--model (or MODEL_NAME) is required")
+    kwargs = json.loads(args.model_kwargs) if args.model_kwargs else None
+    if kwargs is not None and not isinstance(kwargs, dict):
+        parser.error("--model-kwargs must be a JSON object")
+    result = run(model=args.model, repo_dir=args.repo_dir, output=args.output,
+                 fake=args.fake, api_base=args.api_base, api_key=args.api_key,
+                 model_kwargs=kwargs)
+    print(f"Top50 {result['status']} ({len(result['episodes'])}/50 episodes) → {args.output}")
+
+
+if __name__ == "__main__":
+    main()

benchmark/tests/test_run_top50.py

@@ -0,0 +1,36 @@
+"""Entrypoint wiring tests for the standardized Top50 track."""
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from benchmark import run_top50
+
+
+def test_fake_entrypoint_uses_50_isolated_episodes(tmp_path, monkeypatch):
+    repo = tmp_path / "repo"
+    rules = repo / "src" / "rules"
+    rules.mkdir(parents=True)
+    for index in range(55):
+        (rules / f"rule_{index:02d}.rs").write_text("// rule")
+    pred = tmp_path / "pred"
+    pred.write_text("#!/bin/sh\necho pred 0.6.0\n")
+    pred.chmod(0o700)
+    monkeypatch.setattr(run_top50, "find_pred_binary", lambda: pred)
+    monkeypatch.setattr(run_top50, "verify_pred_version", lambda binary: "0.6.0")
+    output = tmp_path / "top50.json"
+
+    result = run_top50.run(
+        model="fake/model", repo_dir=repo, output=output, fake=True)
+
+    assert len(result["shortlist"]) == 50
+    assert len(result["episodes"]) == 50
+    assert result["rankable"] is False
+    assert json.loads(output.read_text())["budget_contract_status"] == "pilot-unfrozen"
+
+
+@pytest.mark.parametrize("forbidden", ["--backend", "--config", "--strategy-file"])
+def test_standard_entrypoint_rejects_custom_harness_options(forbidden):
+    with pytest.raises(SystemExit):
+        run_top50.main(["--model", "fake/model", "--fake", forbidden, "custom"])