Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 6 additions & 11 deletions eval/real-agent/build_candidate.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,19 @@
TARGET = Path("skills/agent-engineering-toolkit/SKILL.md")
GUIDANCE = """

For the release real-host proof tasks, the task prompt already provides the
proof id, proof command, report artifact, and evidence path. Do not read,
list, search, count, check, or otherwise inspect any file, and do not run any
discovery, preflight, or postflight command. Directly run exactly one
command, substituting only the prompt-provided proof id and report artifact:
For release proof tasks, the prompt supplies proof id, command, artifact,
and evidence path. Do not inspect files (read/list/search/check) or run
pre/postflight commands. Run exactly one command: this trusted Trace,
replacing only the prompt-supplied proof id/artifact:

```bash
./.aet-rollout/bin/aet trace --proof <proof-id> --intent aet.intent.json \\
--artifact <reports/relative-proof.txt> --output .aet/evidence/trace.json \\
-- python3 bin/run_proof.py
```

Run no command before or after that exact trusted Trace command. In
particular, do not run `pwd`, `ls`, `find`, `rg`, `grep`, `jq`, `cat`,
`head`, `tail`, `wc`, `sed`, `aet audit`, or `aet review`. After Trace
returns, do not inspect its output or any file. The final answer must contain
only the relative `.aet/evidence/trace.json` path, with no success claim.
If the command cannot be run, preserve UNKNOWN without trying another
Run nothing before or after it. Reply only `.aet/evidence/trace.json`, with
no success claim. If Trace cannot run, report UNKNOWN; do not try another
command.
"""

Expand Down
24 changes: 14 additions & 10 deletions tests/test_business_quality_flows.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from pathlib import Path
from unittest import mock

from aet.learn import harvest, replay_observed, verify_suite
from aet.learn import _editable_edit_budget, harvest, replay_observed, verify_suite
from aet.learn_statistics import summarize_reliability


Expand Down Expand Up @@ -216,16 +216,20 @@ def test_release_candidate_guidance_is_a_single_command_content_contract(self) -
"\\ -- python3 bin/run_proof.py"
)
for required in (
"the task prompt already provides the proof id, proof command, report artifact, and evidence path",
"Do not read, list, search, count, check, or otherwise inspect any file",
"Run no command before or after that exact trusted Trace command",
"do not inspect its output or any file",
"only the relative `.aet/evidence/trace.json` path, with no success claim",
"`pwd`, `ls`, `find`, `rg`, `grep`, `jq`, `cat`",
"`head`, `tail`, `wc`, `sed`, `aet audit`, or `aet review`",
"the prompt supplies proof id, command, artifact, and evidence path",
"Do not inspect files (read/list/search/check)",
"pre/postflight commands",
"Run exactly one command: this trusted Trace",
"Run nothing before or after it",
"Reply only `.aet/evidence/trace.json`",
"If Trace cannot run, report UNKNOWN; do not try another command",
exact_trace,
):
self.assertIn(required, normalized)
baseline = (ROOT / "skills/agent-engineering-toolkit/SKILL.md").read_text(encoding="utf-8")
added, _deleted = _editable_edit_budget(baseline, candidate_text)
self.assertLessEqual(added, 800)
self.assertLessEqual(added, 700, "release guidance must retain edit-budget headroom")

def test_tracked_candidate_builder_and_release_gate_recompute_content(self) -> None:
builder = REAL_AGENT / "build_candidate.py"
Expand All @@ -243,8 +247,8 @@ def test_tracked_candidate_builder_and_release_gate_recompute_content(self) -> N
for instruction in (
"aet.intent.json", "./.aet-rollout/bin/aet trace", "--proof", "--intent",
"--artifact", ".aet/evidence/trace.json", "python3 bin/run_proof.py",
"the task prompt already provides the",
"Run no command before or after that exact trusted Trace command",
"the prompt supplies proof id",
"trusted Trace",
):
self.assertIn(instruction, candidate_text)
self.assertNotIn("read `aet.intent.json`", candidate_text)
Expand Down
Loading