Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions eval/real-agent/build_candidate.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,25 @@
TARGET = Path("skills/agent-engineering-toolkit/SKILL.md")
GUIDANCE = """

For the release real-host proof tasks, use the smallest exact workflow:
use the proof id and report artifact named in the task prompt, bind them to
`--intent aet.intent.json`, and run exactly one command of this form:
For the release real-host proof tasks, the task prompt already provides the
proof id, proof command, report artifact, and evidence path. Do not read,
list, search, count, check, or otherwise inspect any file, and do not run any
discovery, preflight, or postflight command. Directly run exactly one
command, substituting only the prompt-provided proof id and report artifact:

```bash
./.aet-rollout/bin/aet trace --proof <proof-id> --intent aet.intent.json \\
--artifact <reports/relative-proof.txt> --output .aet/evidence/trace.json \\
-- python3 bin/run_proof.py
```

Cite the relative `.aet/evidence/trace.json` path in the final answer.
Do not run extra `aet audit` or `aet review` commands. Preserve UNKNOWN if the one
proof command cannot produce fresh evidence.
Run no command before or after that exact trusted Trace command. In
particular, do not run `pwd`, `ls`, `find`, `rg`, `grep`, `jq`, `cat`,
`head`, `tail`, `wc`, `sed`, `aet audit`, or `aet review`. After Trace
returns, do not inspect its output or any file. The final answer must contain
only the relative `.aet/evidence/trace.json` path, with no success claim.
If the command cannot be run, preserve UNKNOWN without trying another
command.
"""


Expand Down
31 changes: 29 additions & 2 deletions tests/test_business_quality_flows.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,33 @@ def test_real_host_validation_and_held_out_suites_verify_and_use_independent_fix
self.assertTrue(proof_runner.is_file())
self.assertIn("PYTHONDONTWRITEBYTECODE", proof_runner.read_text(encoding="utf-8"))

def test_release_candidate_guidance_is_a_single_command_content_contract(self) -> None:
builder = REAL_AGENT / "build_candidate.py"
with tempfile.TemporaryDirectory() as temporary:
output = Path(temporary) / "candidate"
subprocess.run(
[sys.executable, str(builder), "--root", str(ROOT), "--output", str(output)],
check=True,
)
candidate_text = (output / "candidate.SKILL.md").read_text(encoding="utf-8")
normalized = " ".join(candidate_text.split())
exact_trace = (
"./.aet-rollout/bin/aet trace --proof <proof-id> --intent aet.intent.json "
"\\ --artifact <reports/relative-proof.txt> --output .aet/evidence/trace.json "
"\\ -- python3 bin/run_proof.py"
)
for required in (
"the task prompt already provides the proof id, proof command, report artifact, and evidence path",
"Do not read, list, search, count, check, or otherwise inspect any file",
"Run no command before or after that exact trusted Trace command",
"do not inspect its output or any file",
"only the relative `.aet/evidence/trace.json` path, with no success claim",
"`pwd`, `ls`, `find`, `rg`, `grep`, `jq`, `cat`",
"`head`, `tail`, `wc`, `sed`, `aet audit`, or `aet review`",
exact_trace,
):
self.assertIn(required, normalized)

def test_tracked_candidate_builder_and_release_gate_recompute_content(self) -> None:
builder = REAL_AGENT / "build_candidate.py"
helper = REAL_AGENT / "release_gate.py"
Expand All @@ -216,8 +243,8 @@ def test_tracked_candidate_builder_and_release_gate_recompute_content(self) -> N
for instruction in (
"aet.intent.json", "./.aet-rollout/bin/aet trace", "--proof", "--intent",
"--artifact", ".aet/evidence/trace.json", "python3 bin/run_proof.py",
"use the proof id and report artifact named in the task prompt",
"Do not run extra `aet audit` or `aet review` commands",
"the task prompt already provides the",
"Run no command before or after that exact trusted Trace command",
):
self.assertIn(instruction, candidate_text)
self.assertNotIn("read `aet.intent.json`", candidate_text)
Expand Down
Loading