diff --git a/packages/app/package.json b/packages/app/package.json index 466955fe..625b3083 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,6 +1,6 @@ { "name": "@deepagent-code/app", - "version": "1.4.1", + "version": "1.4.2", "description": "", "type": "module", "exports": { diff --git a/packages/core/src/agent-gateway.ts b/packages/core/src/agent-gateway.ts index 1b938a31..562458c3 100644 --- a/packages/core/src/agent-gateway.ts +++ b/packages/core/src/agent-gateway.ts @@ -268,6 +268,11 @@ export const configure = (config: Config = {}) => { // knowledge/state diverge from project-memory whenever runsDir pointed outside /runs. const baseDir = config.baseDir ?? resolveDeepAgentCodeHome() DeepAgentSessionState.configure(path.join(baseDir, "state")) + // I33-1: the structural plan lives in the single DocumentStore under /state/goal//graph + // (co-located with the goal/run graph so the `plan` tool and the goal path write the SAME doc). Root + // it from the SAME baseDir/state session-state uses, so the two paths converge (planStoreRoot ≡ + // goalStoreRoot). Must be set before any plan read/write. + DeepAgentPlanStore.configureRoot(path.join(baseDir, "state")) // docs/34 §7.2/§8: durable knowledge stores root under baseDir (public/knowledge + project// // knowledge). The retriever's knowledge-source reads from here. Same injected baseDir, no self-resolve. DeepAgentKnowledgeSource.configure(baseDir) @@ -386,6 +391,7 @@ import { import * as DeepAgentOrchestrator from "./deepagent/orchestrator" import * as DeepAgentSessionState from "./deepagent/session-state" import * as DeepAgentPlanController from "./deepagent/plan-controller" +import * as DeepAgentPlanStore from "./deepagent/plan-store" import * as DeepAgentDiagnosis from "./deepagent/diagnosis" import * as DeepAgentFailureTriage from "./deepagent/failure-triage" import * as DeepAgentValidation from "./deepagent/validation" @@ -414,6 +420,7 @@ export { DeepAgentOrchestrator, DeepAgentSessionState, DeepAgentPlanController, + DeepAgentPlanStore, DeepAgentDiagnosis, DeepAgentFailureTriage, DeepAgentValidation, diff --git a/packages/core/src/agent.ts b/packages/core/src/agent.ts index 7437c898..7e7ad22c 100644 --- a/packages/core/src/agent.ts +++ b/packages/core/src/agent.ts @@ -2,6 +2,7 @@ export * as AgentV2 from "./agent" import { Array, Context, Effect, Layer, Schema, Scope } from "effect" import { castDraft, enableMapSet, type Draft } from "immer" +import { makeLocationNode } from "./effect/app-node" import { ModelV2 } from "./model" import { PermissionSchema } from "./permission/schema" import { ProviderV2 } from "./provider" @@ -164,4 +165,6 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [] }) + export const locationLayer = layer diff --git a/packages/core/src/command.ts b/packages/core/src/command.ts index f6950482..5d3c541f 100644 --- a/packages/core/src/command.ts +++ b/packages/core/src/command.ts @@ -2,6 +2,7 @@ export * as CommandV2 from "./command" import { Context, Effect, Layer, Schema } from "effect" import { castDraft, type Draft } from "immer" +import { makeLocationNode } from "./effect/app-node" import { ModelV2 } from "./model" import { State } from "./state" @@ -67,4 +68,6 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [] }) + export const locationLayer = layer diff --git a/packages/core/src/config.ts b/packages/core/src/config.ts index 8fc513c3..0f12953a 100644 --- a/packages/core/src/config.ts +++ b/packages/core/src/config.ts @@ -3,6 +3,7 @@ export * as Config from "./config" import path from "path" import { type ParseError, parse } from "jsonc-parser" import { Context, Effect, Layer, Option, Schema } from "effect" +import { makeLocationNode } from "./effect/app-node" import { FSUtil } from "./fs-util" import { Global } from "./global" import { Location } from "./location" @@ -223,4 +224,10 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ + service: Service, + layer, + deps: [FSUtil.node, Global.node, Location.node, Policy.node], +}) + export const locationLayer = layer.pipe(Layer.provideMerge(Policy.locationLayer)) diff --git a/packages/core/src/cross-spawn-spawner.ts b/packages/core/src/cross-spawn-spawner.ts index ad8d4126..5207fc22 100644 --- a/packages/core/src/cross-spawn-spawner.ts +++ b/packages/core/src/cross-spawn-spawner.ts @@ -24,6 +24,8 @@ import { import * as NodeChildProcess from "node:child_process" import { PassThrough } from "node:stream" import launch from "cross-spawn" +import { makeGlobalNode } from "./effect/app-node" +import { filesystem, path } from "./effect/app-node-platform" const toError = (err: unknown): Error => (err instanceof globalThis.Error ? err : new globalThis.Error(String(err))) @@ -502,4 +504,6 @@ export const layer: Layer.Layer @@ -58,3 +59,5 @@ export const defaultLayer = Layer.unwrap( return layerFromPath(path()) }), ).pipe(Layer.provide(Global.defaultLayer)) + +export const node = makeGlobalNode({ service: Service, layer: layerFromPath(path()), deps: [] }) diff --git a/packages/core/src/deepagent/atomic-write.ts b/packages/core/src/deepagent/atomic-write.ts index 9e1e8f87..861bf101 100644 --- a/packages/core/src/deepagent/atomic-write.ts +++ b/packages/core/src/deepagent/atomic-write.ts @@ -1,4 +1,4 @@ -import { mkdirSync, renameSync, rmSync, writeFileSync } from "node:fs" +import { closeSync, fsyncSync, mkdirSync, openSync, renameSync, rmSync, writeSync } from "node:fs" import { randomUUID } from "node:crypto" import path from "node:path" @@ -9,15 +9,83 @@ import path from "node:path" // renameSync is atomic on POSIX and Windows. On any failure the temp file is removed and the error // propagates (never swallow a lost durable write). This mirrors the temp+rename pattern already // used by workspace.ts createProjectAtomically. +// +// F30-1 (deepagentcore-v4.0.3 storage prereq): the temp file is now fsync'd BEFORE the rename so a +// crash between write and rename can never expose a torn body, and the containing directory is +// fsync'd AFTER the rename (best-effort) so the rename itself is durable across a power loss. This +// makes writeFileAtomic the single crash-safe overwrite primitive the DocumentStore relies on. export const writeFileAtomic = (file: string, content: string): void => { const dir = path.dirname(file) mkdirSync(dir, { recursive: true }) const tmp = path.join(dir, `.${path.basename(file)}.tmp-${process.pid}-${randomUUID()}`) + let fd: number | undefined try { - writeFileSync(tmp, content, "utf-8") + fd = openSync(tmp, "w") + writeSync(fd, content, null, "utf-8") + fsyncSync(fd) + closeSync(fd) + fd = undefined renameSync(tmp, file) + fsyncDir(dir) } catch (error) { + if (fd !== undefined) { + try { + closeSync(fd) + } catch { + /* fd already invalid — nothing to salvage */ + } + } rmSync(tmp, { force: true }) throw error } } + +// F30-1: crash-safe EXCLUSIVE create — the CAS write primitive for append-only version files. +// `openSync(file, "wx")` fails with EEXIST if the target already exists, giving an atomic +// compare-and-swap on the filesystem: two processes racing to write the same `id@vN.json` version +// file, one wins and the other observes EEXIST (the caller turns that into a conflict). The body is +// fsync'd before close so a crash can't leave a half-written version file. Throws the raw Node error +// (code "EEXIST" on collision) — the DocumentStore inspects `.code` to decide idempotent-vs-conflict. +export const writeFileExclusive = (file: string, content: string): void => { + const dir = path.dirname(file) + mkdirSync(dir, { recursive: true }) + let fd: number | undefined + try { + fd = openSync(file, "wx") + writeSync(fd, content, null, "utf-8") + fsyncSync(fd) + closeSync(fd) + fd = undefined + fsyncDir(dir) + } catch (error) { + if (fd !== undefined) { + try { + closeSync(fd) + } catch { + /* fd already invalid */ + } + } + throw error + } +} + +// Best-effort directory fsync so a rename/create is durable. Not all platforms/filesystems permit +// opening a directory for fsync (Windows throws EPERM/EISDIR, some FUSE mounts reject it); a failure +// here does not compromise the file body (already fsync'd) so it is intentionally swallowed. +const fsyncDir = (dir: string): void => { + let dfd: number | undefined + try { + dfd = openSync(dir, "r") + fsyncSync(dfd) + } catch { + /* directory fsync unsupported on this platform — file body is already durable */ + } finally { + if (dfd !== undefined) { + try { + closeSync(dfd) + } catch { + /* ignore */ + } + } + } +} diff --git a/packages/core/src/deepagent/command-intent.ts b/packages/core/src/deepagent/command-intent.ts index 1db3b3b6..23d489ce 100644 --- a/packages/core/src/deepagent/command-intent.ts +++ b/packages/core/src/deepagent/command-intent.ts @@ -50,6 +50,24 @@ const READ_ONLY_PREFIXES: ReadonlyArray = [ ["rg"], ["ag"], ["ripgrep"], + // ── text stream inspection (read-only filters; parity with codex is_safe_to_call_with_exec) ── + // These only transform stdin/named files to stdout. `sort`/`sed` have file-writing variants and are + // guarded below; the rest cannot write the filesystem, process table, or environment. + ["cut"], + ["nl"], + ["paste"], + ["rev"], + ["seq"], + ["tr"], + ["uniq"], + ["comm"], + ["fold"], + ["expr"], + ["true"], + ["false"], + ["column"], + ["sort"], // read-only UNLESS it writes with -o/--output (guarded below) + ["sed"], // mutating by default (in MUTATING_COMMANDS); the read-only `sed -n …p` form is allowed via a guard below // ── environment / introspection (query forms only) ── ["which"], ["whereis"], @@ -149,7 +167,10 @@ const MUTATING_COMMANDS = new Set([ "shred", "tee", "install", - "sed", // sed alone is read-only, but `sed -i` mutates; treat as mutating unless we prove otherwise (guarded) + // NOTE: `sed` is intentionally NOT a blanket mutating command. `sed -i` mutates but `sed -n {N|M,N}p` + // is a pure read-only slice (the common "show me lines X..Y" probe). It is matched as a read-only + // prefix and then constrained by a guard in isReadOnlySegment to ONLY the `-n …p` form (parity with + // codex is_safe_to_call_with_exec). Any other sed invocation fails the guard → mutating. "kill", "killall", "pkill", @@ -217,6 +238,23 @@ const matchReadOnlyPrefix = (tokens: string[]): number => { // `find` is read-only unless it carries an action predicate that executes or deletes. const FIND_MUTATING_ACTIONS = new Set(["-delete", "-exec", "-execdir", "-ok", "-okdir", "-fprint", "-fprintf"]) +// `sed -n {N|M,N}p` is a pure read-only line slice. Port of codex is_valid_sed_n_arg: the address +// must match /^(\d+,)?\d+p$/ — one or two numeric line numbers followed by the `p` (print) command, +// nothing else. Any other sed script can substitute/delete/write (`-i`, `w`, `s///w file`), so only +// this exact shape is treated as read-only. +const isValidSedNArg = (arg: string | undefined): boolean => { + if (arg == null) return false + const core = arg.endsWith("p") ? arg.slice(0, -1) : null + if (core == null) return false + const parts = core.split(",") + if (parts.length === 1) return parts[0].length > 0 && /^\d+$/.test(parts[0]) + if (parts.length === 2) return parts[0].length > 0 && parts[1].length > 0 && /^\d+$/.test(parts[0]) && /^\d+$/.test(parts[1]) + return false +} + +// `sort` writes a file with -o/--output; everything else it does is read-only (stdin/files → stdout). +const SORT_WRITE_LONG = /^--output(=|$)/ + // `env` is read-only only as a bare query (`env`); `env FOO=bar cmd` runs a command with a mutated // environment, so anything past the command word makes it mutating. const isReadOnlySegment = (segment: string): boolean => { @@ -257,6 +295,29 @@ const isReadOnlySegment = (segment: string): boolean => { const rest = words.slice(1).filter((token) => !isFlag(token)) if (rest.length > 0) return false } + // `sed` is read-only ONLY as `sed -n {N|M,N}p [file]` (line slice). The prefix matcher accepts a + // bare `sed`, so we must reject every other form here. Require exactly: -n, a valid `…p` address, + // and at most one trailing operand (the file). No -i, no -e, no `w`/`s///w`, no extra scripts. + if (head === "sed") { + const rest = words.slice(1) + const nIdx = rest.indexOf("-n") + if (nIdx === -1) return false + // Only -n may be a flag; any other flag (e.g. -i, -e, -E) makes it non-readonly. + if (rest.some((token, i) => i !== nIdx && isFlag(token))) return false + const nonFlags = rest.filter((token) => !isFlag(token)) + // nonFlags[0] must be the address (…p); an optional nonFlags[1] is the file. More → reject. + if (nonFlags.length < 1 || nonFlags.length > 2) return false + if (!isValidSedNArg(nonFlags[0])) return false + } + // `sort -o file` / `sort --output=file` writes a file. Everything else sort does is read-only. + if (head === "sort") { + for (let i = 1; i < words.length; i++) { + const token = words[i] + if (token === "-o" || SORT_WRITE_LONG.test(token)) return false + // bundled short form `-ofile` (sort's only single-letter value-flag that writes) + if (/^-o./.test(token)) return false + } + } // curl writes a file with -o/-O/--output/--remote-name. A short-flag TOKEN can glue or bundle the // output flag (`-ofile.txt`, `-sofile`), and since curl short flags are single-letter and `o`/`O` // are only ever the output flags, any single-dash bundle containing `o`/`O` writes a file. Match a diff --git a/packages/core/src/deepagent/document-store.ts b/packages/core/src/deepagent/document-store.ts index 4ffc607a..e66eee41 100644 --- a/packages/core/src/deepagent/document-store.ts +++ b/packages/core/src/deepagent/document-store.ts @@ -1,6 +1,7 @@ -import { mkdirSync, readdirSync, readFileSync, writeFileSync, existsSync } from "node:fs" +import { mkdirSync, readdirSync, readFileSync, existsSync } from "node:fs" import { createHash } from "node:crypto" import path from "node:path" +import { writeFileAtomic, writeFileExclusive } from "./atomic-write" // V3 Document System (docs/28): the bedrock. All persistent state is a typed-document // graph — small files, content-addressed, append-only with a supersede chain, bidirectional @@ -194,6 +195,29 @@ export type DocFilter = { export type IntegrityViolation = { readonly invariant: string; readonly docId: string; readonly detail: string } export type IntegrityReport = { readonly ok: boolean; readonly violations: readonly IntegrityViolation[] } +// F30-1 (deepagentcore-v4.0.3 storage prereq): a CAS write conflict. Thrown when persist() tries to +// create an append-only version file (`id@vN.json`) that already exists on disk with a DIFFERENT +// content hash — i.e. another writer (a second handle or a second process) already produced version +// N of this doc from a different base. Append-only + content-addressed means same-hash collisions +// are idempotent no-ops (a retried write); only a hash MISMATCH is a genuine lost-update race, and +// the losing writer must re-read the latest version and re-apply its mutation rather than clobber. +export class DocumentConflictError extends Error { + readonly _tag = "DocumentConflictError" + constructor( + readonly docId: string, + readonly version: number, + readonly existingHash: string, + readonly incomingHash: string, + ) { + super( + `DocumentStore CAS conflict: ${docId}@v${version} already exists with a different body ` + + `(on-disk ${existingHash.slice(0, 16)}… vs incoming ${incomingHash.slice(0, 16)}…). ` + + `Another writer produced this version concurrently; re-read the latest and re-apply.`, + ) + this.name = "DocumentConflictError" + } +} + const toRef = (d: Doc): DocRef => ({ id: d.id, version: d.version, @@ -283,14 +307,63 @@ const strongerEvidence = (a: EvidenceStrength, b: EvidenceStrength): EvidenceStr const idToFile = (id: string): string => id.replace(/:/g, "__") +// F30-1 Part 2 (deepagentcore-v4.0.3 storage prereq): same-process SHARED AUTHORITY. Each +// `new DocumentStore(root)` builds its OWN in-memory index (rebuilt from disk in the constructor), +// so two long-lived handles to the same root in one process do NOT see each other's writes — a +// latent divergence the goal code today routes around with an out-of-band control channel. This +// process-level registry, keyed by the RESOLVED absolute root, lets callers opt into a single shared +// in-memory index via `DocumentStore.shared(root)`: every shared handle for a root reuses the same +// `docs` Map, so a write through one handle is immediately visible through every other. The plain +// constructor is intentionally UNCHANGED (unshared, disk-rebuilt) so it keeps faithfully simulating a +// cold/second-process reconstruction (the shape several recovery tests depend on). Cross-process +// safety is provided by Part 1's exclusive-create CAS + atomic writes, not by this in-memory registry. +const sharedIndexRegistry = new Map>>() + export class DocumentStore { // id -> version -> Doc - private docs = new Map>() - constructor(private readonly root: string) { + private docs: Map> + // `new DocumentStore(root)` stays the UNSHARED, disk-rebuilt handle it always was (all existing + // callers and restart-simulation tests keep working unchanged). Shared handles are obtained via the + // `DocumentStore.shared(root)` factory, which passes shared=true. + constructor( + private readonly root: string, + shared = false, + ) { mkdirSync(path.join(root, "docs"), { recursive: true }) + if (shared) { + const key = path.resolve(root) + let index = sharedIndexRegistry.get(key) + if (!index) { + // First shared handle for this root: build the authoritative shared index from disk once. + index = new Map>() + sharedIndexRegistry.set(key, index) + this.docs = index + this.rebuildIndex() + } else { + // Subsequent shared handles reuse the live shared index (already coherent with prior writes). + this.docs = index + } + return + } + // Unshared (default): own index, rebuilt from disk — byte-identical to the pre-F30-1 behavior. + this.docs = new Map>() this.rebuildIndex() } + // F30-1 Part 2: a SHARED handle over `root` — all shared handles for the same resolved root in this + // process share ONE in-memory index, so writes through any handle are immediately visible through + // the others. Use for coherent same-process authority (e.g. a long-lived driver handle that must see + // an edit written by a request fiber). Cross-process writers are still reconciled by CAS on persist. + static shared(root: string): DocumentStore { + return new DocumentStore(root, true) + } + + // Test-only: drop the shared-index registry so a fresh process is simulated. Not part of the durable + // contract — only used to keep unit tests hermetic when they exercise DocumentStore.shared. + static __resetSharedRegistryForTests(): void { + sharedIndexRegistry.clear() + } + // ---- write ---- create(input: CreateDocInput): Doc { const id = this.allocateId(input.type, input.domain ?? null, input.idSlug, input.description) @@ -526,15 +599,53 @@ export class DocumentStore { } // ---- internals ---- + // F30-1: persist() writes a NEW append-only version file with EXCLUSIVE-create CAS semantics. In + // the normal single-writer flow each `id@vN.json` is written exactly once (create=v1, every + // update/upsert bumps version+1), so the exclusive create never collides and behavior is + // byte-identical to the old bare writeFileSync. A collision (EEXIST) only happens when a second + // handle/process already produced this version: if that on-disk version is byte-identical (same + // content hash) the write is an idempotent no-op (a retried/duplicated write) and we simply adopt + // it into the index; if it differs, it is a genuine lost-update race and we throw + // DocumentConflictError rather than silently clobber. private persist(doc: Doc): void { const dir = path.join(this.root, "docs", doc.type) mkdirSync(dir, { recursive: true }) - writeFileSync(path.join(dir, `${idToFile(doc.id)}@v${doc.version}.json`), JSON.stringify(doc, null, 2)) + const file = path.join(dir, `${idToFile(doc.id)}@v${doc.version}.json`) + const payload = JSON.stringify(doc, null, 2) + try { + writeFileExclusive(file, payload) + } catch (error) { + if ((error as NodeJS.ErrnoException)?.code !== "EEXIST") throw error + // CAS collision: reconcile against what is already on disk. + const existing = this.readVersionFile(file) + if (existing && existing.hash === doc.hash) { + // Idempotent: the same content already landed (retried write / concurrent identical write). + this.indexDoc(doc) + return + } + throw new DocumentConflictError(doc.id, doc.version, existing?.hash ?? "", doc.hash) + } this.indexDoc(doc) } private replace(doc: Doc): void { - // rewrites the same version with new status/superseded_by; rehash so INV-2 holds - this.persist({ ...doc, hash: computeHash({ ...doc, hash: "" }) }) + // rewrites the SAME version in place with new status/superseded_by; rehash so INV-2 holds. This + // is an intentional overwrite (not a new version), so it uses the crash-safe atomic OVERWRITE + // primitive (temp+fsync+rename) rather than the exclusive-create CAS path. + const hashed: Doc = { ...doc, hash: computeHash({ ...doc, hash: "" }) } + const dir = path.join(this.root, "docs", hashed.type) + const file = path.join(dir, `${idToFile(hashed.id)}@v${hashed.version}.json`) + writeFileAtomic(file, JSON.stringify(hashed, null, 2)) + this.indexDoc(hashed) + } + // F30-1: read+parse a single on-disk version file for CAS reconciliation. Returns null if the file + // is missing or corrupt (a torn concurrent write) — the caller treats an unreadable existing file + // as a conflict, never as an idempotent match. + private readVersionFile(file: string): Doc | null { + try { + return JSON.parse(readFileSync(file, "utf8")) as Doc + } catch { + return null + } } private indexDoc(doc: Doc): void { const versions = this.docs.get(doc.id) ?? new Map() diff --git a/packages/core/src/deepagent/hooks.ts b/packages/core/src/deepagent/hooks.ts index f137ea4b..33d970d2 100644 --- a/packages/core/src/deepagent/hooks.ts +++ b/packages/core/src/deepagent/hooks.ts @@ -69,38 +69,45 @@ export const stopHookGate = (): HookHandler => (e) => { // hardGateMissBlocks=true (xhigh/max/ultra) -> block on a missing active step; false (high) -> warn. // The payload carries hardGate, hasActiveStep, hardGateMissBlocks. // -// U1 anti-deadlock (production fix): the soft layer additionally downgrades a would-be block to a -// WARN (tool executes, reminder attached) in two cases, so the gate can never permanently deny a -// model its tools: -// - staleReason === "user_appended": a new user message is a SOFT signal that the plan MIGHT no -// longer match intent — nudge the model to re-align, but do not hard-block work the user is -// actively asking for. -// - graceRelease === true: the gate has already blocked this stale plan N times with no forward -// progress (runtime-driven counter). Rather than loop forever against a model that will not -// repair the plan, release with a strong reminder. This is the direct fix for the observed -// deadlock (280 consecutive blocked bash calls). +// DESIGN (aligned with codex core/src/exec_policy.rs render_decision_for_unmatched_command): command +// safety classification and plan bookkeeping are ORTHOGONAL to whether a tool may run. In codex the +// "is this a known-safe command" check only decides auto-approve-vs-prompt; a command that is NOT +// known-safe is at worst prompted, and is Forbidden ONLY when it is genuinely dangerous AND the user +// disabled prompts. Staleness of the plan ledger is not a safety property, so — like codex — it must +// never REJECT execution. Our previous code coupled the two: a mutating tool on a stale plan was +// hard-blocked in high+ mode, which deadlocked a model that did not repair the plan (observed: 280 +// consecutive blocked bash calls, and a read-only `ssh/docker exec` probe misclassified as mutating +// then denied outright). This gate now downgrades EVERY plan-ledger condition to a WARN (the tool +// runs, a reminder is attached), so plan state can nudge but can never deny the agent its tools. +// +// The two remaining honest signals differ only in wording: +// - staleReason === "user_appended": a new user message MIGHT change intent — nudge to re-align. +// - graceRelease === true: repeated stale blocks with no forward progress (runtime-driven counter). +// Both warn; neither blocks. export const planGate = (): HookHandler => (e) => { if (e.name !== "before_tool_use") return { decision: "continue" } if (e.payload["isMutating"] !== true) return { decision: "allow" } - // U1 soft layer: stale plan. + // U1 soft layer: stale plan → WARN only (never block). Reality changed / a user message arrived is + // a reason to re-sync the plan, not a reason to deny work. if (e.payload["planStale"] === true) { - const reason = "the plan is stale (reality changed); update the plan before further edits" - const graceReason = - "the plan is stale and has not been updated after repeated attempts; this action is allowed to proceed, but you MUST call the `plan` tool to resync your plan with reality now" + const reason = "the plan is stale (reality changed); review it and update the `plan` tool to resync — this action still proceeds" const userAppendedReason = "a new user message arrived; your plan may no longer match the request — review it and update the `plan` tool if the goal changed" - if (e.payload["lightweight"] === true) return { decision: "warn", blockReason: reason } - if (e.payload["graceRelease"] === true) return { decision: "warn", blockReason: graceReason } if (e.payload["staleReason"] === "user_appended") return { decision: "warn", blockReason: userAppendedReason } - return { decision: "block", blockReason: reason } + return { decision: "warn", blockReason: reason } } - // U9 hard layer: per-step binding (high+ only; lightweight never reaches here with hardGate set). + // U9 hard layer: per-step binding (high+ only; lightweight never reaches here with hardGate set). A + // mutating tool under a strict hard gate must be bound to an active step. This is a workflow- + // discipline gate, not a safety gate, so it MUST also have a runtime-driven release: if the gate has + // already blocked this many times with no forward progress (graceRelease), stop blocking and let the + // tool through with a reminder — otherwise a model that never marks a step active would be + // permanently denied its tools (the same deadlock class the stale layer just fixed). if (e.payload["hardGate"] === true && e.payload["hasActiveStep"] !== true) { const reason = "no active plan step is bound to this edit; mark the step you are working on active via the plan tool" - return e.payload["hardGateMissBlocks"] === true - ? { decision: "block", blockReason: reason } - : { decision: "warn", blockReason: reason } + if (e.payload["hardGateMissBlocks"] === true && e.payload["graceRelease"] !== true) + return { decision: "block", blockReason: reason } + return { decision: "warn", blockReason: reason } } return { decision: "allow" } } diff --git a/packages/core/src/deepagent/index.ts b/packages/core/src/deepagent/index.ts index 9d0c6b52..84309d79 100644 --- a/packages/core/src/deepagent/index.ts +++ b/packages/core/src/deepagent/index.ts @@ -9,6 +9,7 @@ export * as DeepAgentValidation from "./validation" export * as DeepAgentBudget from "./budget" export * as DeepAgentLearning from "./learning" export * as DeepAgentSessionState from "./session-state" +export * as DeepAgentPlanStore from "./plan-store" export * as DeepAgentOrchestrator from "./orchestrator" export * as DeepAgentAblation from "./ablation" export * as DeepAgentRunContext from "./run-context" diff --git a/packages/core/src/deepagent/plan-store.ts b/packages/core/src/deepagent/plan-store.ts new file mode 100644 index 00000000..50543a8c --- /dev/null +++ b/packages/core/src/deepagent/plan-store.ts @@ -0,0 +1,98 @@ +// I33-1 (deepagentcore-v4.0.3 storage prereq): the SINGLE structural authority for a session's plan. +// +// Before I33-1 the structural PlanDoc lived on the in-memory SessionRunState (flushed to sessions.json) +// while the goal path ALSO mirrored a `type:"plan"` DocumentStore doc — two disconnected stores that +// could diverge (a durable-continuation reload would trust one, a run-close mirror the other). I33-1 +// makes the DocumentStore `type:"plan"` doc the ONE structural authority (content-addressed, versioned, +// CAS-protected by F30-1); SessionRunState keeps only the runtime latch (plan_id/version/fresh-stale) +// as a hot value object, never the body. +// +// The plan doc is co-located with the session's goal/run graph so the goal path and the `plan` tool +// write the SAME doc: `planStoreRoot(sid)` is byte-identical to goal-manager's `goalStoreRoot(sid)` +// (`/state/goal//graph`). Both go through `DocumentStore.shared(root)` (F30-1 Part 2), +// so every handle for a session shares ONE in-memory index — a write via the tool is immediately +// visible to the goal driver and vice versa, with no second cache to drift. The shared index IS the +// hot cache: getPlanDoc is an in-memory Map lookup + a JSON.parse, not a disk read. +import path from "node:path" +import { DocumentStore } from "./document-store" +import type { PlanDoc } from "./plan-controller" + +// The stable identity of a session's plan doc: type "plan", scope "run:", slug "plan-", +// description planDescription(sid). ALL FOUR must match between the `plan` tool path (setPlanDoc) and +// the goal path (goal-driver.materializePlanDoc) or upsert()'s findLogical dedup (which keys on +// description + domain) splits them into two docs (plan- vs plan--2) and reintroduces the +// two-store divergence I33-1 removes. goal-driver imports planDescription/planScope from here so the +// identity can never drift across the package boundary. +const planSlug = (sessionId: string): string => `plan-${sessionId}` +export const planScope = (sessionId: string): string => `run:${sessionId}` +export const planDescription = (sessionId: string): string => `session plan ${sessionId}` + +// The state dir session-state was configured with. plan-store roots UNDER it at the same location the +// goal store uses, so the two paths converge on one doc. Set by configureRoot (called from the same +// gateway configure that sets session-state's dir), so core never has to import the deepagent-code +// goal-manager resolver. +let stateDir: string | null = null + +export const configureRoot = (dir: string): void => { + stateDir = dir +} + +// planStoreRoot(sid) === goalStoreRoot(sid) === /goal//graph. Kept private-by-convention +// (exported for the goal path + tests to assert convergence). Throws if used before configureRoot — a +// plan write with no configured root is a wiring bug, not something to silently drop. +export const planStoreRoot = (sessionId: string): string => { + if (!stateDir) throw new Error("plan-store: configureRoot() not called (no state dir)") + return path.join(stateDir, "goal", sessionId, "graph") +} + +// The shared authority handle for a session's plan doc. Shared registry keyed by resolved root, so the +// tool path, the goal driver, and the UI/archive readers all see one coherent in-memory index. +const store = (sessionId: string): DocumentStore => DocumentStore.shared(planStoreRoot(sessionId)) + +// Resolve a session's plan doc ref. The doc id is NOT reconstructable from the slug — allocateId runs +// idSlug through slugify() (lowercase, `_`→`-`, truncate 48), so a raw `doc:plan:plan-` guess +// misses for realistic session ids. Instead resolve by (type "plan", scope "run:"): the plan-store +// root is per-session (/goal//graph), and there is exactly one plan doc per session, so this +// filter yields at most one ref. list() returns the LATEST version per id (F30-1 shared index lookup). +const resolveRef = (sessionId: string) => { + const refs = store(sessionId).list({ type: "plan", scope: planScope(sessionId) }) + return refs.length > 0 ? refs[0] : null +} + +// Read the current structural plan for a session (latest version), or null if none exists yet. Pure +// in-memory lookup over the shared index (+ JSON.parse) — safe on the hot path (every tool call). +export const getPlanDoc = (sessionId: string): PlanDoc | null => { + const ref = resolveRef(sessionId) + if (!ref) return null + const doc = store(sessionId).get(ref.id) + if (!doc) return null + try { + return JSON.parse(doc.body) as PlanDoc + } catch { + return null + } +} + +// The doc id + current version for a session's plan (for the SessionRunState latch pointer), or null. +export const planDocRef = (sessionId: string): { id: string; version: number } | null => { + const ref = resolveRef(sessionId) + if (!ref) return null + const doc = store(sessionId).get(ref.id) + return doc ? { id: doc.id, version: doc.version } : null +} + +// Write (create or new-version) the structural plan. Idempotent per session via upsert keyed on the +// stable slug: an unchanged body is an INV-4 no-op (no version bump), a changed body appends a new +// version (CAS-protected). Returns the resulting doc id + version so the caller can update the latch +// pointer. This is the ONE write seam — the `plan` tool AND the goal path both call it. +export const setPlanDoc = (sessionId: string, plan: PlanDoc): { id: string; version: number } => { + const doc = store(sessionId).upsert({ + type: "plan", + scope: planScope(sessionId), + description: planDescription(sessionId), + idSlug: planSlug(sessionId), + body: JSON.stringify(plan), + provenance: { source: "model", run_ref: planScope(sessionId) }, + }) + return { id: doc.id, version: doc.version } +} diff --git a/packages/core/src/deepagent/session-state.ts b/packages/core/src/deepagent/session-state.ts index a9fea9e0..fd5fa491 100644 --- a/packages/core/src/deepagent/session-state.ts +++ b/packages/core/src/deepagent/session-state.ts @@ -2,6 +2,7 @@ import { mkdirSync, readFileSync } from "node:fs" import { randomUUID } from "node:crypto" import path from "node:path" import { writeFileAtomic } from "./atomic-write" +import * as PlanStore from "./plan-store" import type { AgentMode } from "./mode" import { createInitialRoundState, @@ -42,12 +43,10 @@ export type SessionRunState = { workspacePath: string | null runId: string // U1 PlanController: the runtime plan latch (fresh/stale + reason + replan count). The structural - // plan lives in DocumentStore; only this hot-path value object is carried on session state. + // plan lives in DocumentStore (I33-1, plan-store.ts) as the single authority; only this hot-path + // value object — the latch, carrying the plan_id pointer — is on session state. The plan body is + // NOT stored here (getPlan/setPlan delegate to plan-store). planLatch: PlanLatchState - // U1: the live working plan (goal/steps) the model writes via the `plan` tool and the UI renders. - // Kept on run state (hot path, atomically persisted) — graduated into the durable run-graph as a - // `plan` doc at run close, mirroring how DESIGN.md is materialized. - plan: PlanDoc | null // U10 step-reporting: count of mutating tool calls since the model last CHANGED a plan step's // status. Drives the progress-nudge count backstop (nudgeTrigger). Reset to 0 only when setPlan // detects a real status change — a no-op plan re-write must not silence the nudge. @@ -96,6 +95,11 @@ const sessions = new Map() export const configure = (dir: string) => { stateDir = dir mkdirSync(dir, { recursive: true }) + // I33-1: the structural plan authority (DocumentStore `type:"plan"` doc) roots under the SAME state + // dir (/goal//graph), so the `plan` tool (via session-state) and the goal path write the + // same doc. Set the plan-store root here — including for tests that call configure() directly + // (bypassing the gateway) — so every plan read/write has a configured root. + PlanStore.configureRoot(dir) // Pointing at a (new) state dir means a fresh session set: clear the in-memory map BEFORE loading, so // configure() reflects exactly what's on disk at `dir` and never merges stale sessions from a prior // dir. Production calls configure once at gateway init (nothing to lose); tests that configure a fresh @@ -121,7 +125,6 @@ export const getOrCreate = (sessionId: string, mode: AgentMode): SessionRunState workspacePath: null, runId: `run_${randomUUID()}`, planLatch: initialPlanLatch(), - plan: null, mutationsSinceReport: 0, validationPassedSinceReport: false, panelArmed: null, @@ -200,17 +203,22 @@ export const setPlan = (sessionId: string, plan: PlanDoc): void => { if (!state) return // U10: reset the progress-nudge state ONLY when the model actually moved a step's status (or // added a step). A no-op re-write leaves the counter/flag running so the nudge is not silenced by - // an empty update ("report theater"). - if (planStatusesChanged(state.plan, plan)) { + // an empty update ("report theater"). Compare against the CURRENT structural plan in the store. + const previous = PlanStore.getPlanDoc(sessionId) + if (planStatusesChanged(previous, plan)) { state.mutationsSinceReport = 0 state.validationPassedSinceReport = false } - state.plan = plan + // I33-1: the plan body is written to the single DocumentStore authority (plan-store); session state + // keeps only the latch pointer (plan_id). The store upsert is content-addressed + CAS-protected. + PlanStore.setPlanDoc(sessionId, plan) state.planLatch = clearStale({ ...state.planLatch, plan_id: plan.plan_id }) saveToDisk() } -export const getPlan = (sessionId: string): PlanDoc | null => sessions.get(sessionId)?.plan ?? null +// I33-1: read the structural plan from the single DocumentStore authority (plan-store). This is an +// in-memory shared-index lookup + JSON.parse (F30-1 Part 2), safe on the hot path (every tool call). +export const getPlan = (sessionId: string): PlanDoc | null => PlanStore.getPlanDoc(sessionId) // V3.9 §C — Expert Panel per-session arming. // The raw per-session toggle (null = never explicitly toggled). setPanelArmed writes an explicit @@ -277,7 +285,9 @@ export const setActiveGoalPhase = (sessionId: string, phase: GoalPointerPhase): // against). export const recordMutation = (sessionId: string): void => { const state = sessions.get(sessionId) - if (!state || state.plan == null) return + // I33-1: the nudge only applies once a plan exists — the latch's plan_id is the hot-path pointer + // (set by setPlan), so we gate on it instead of a stored body. + if (!state || state.planLatch.plan_id == null) return state.mutationsSinceReport += 1 saveToDisk() } @@ -412,7 +422,6 @@ function normalizeState(state: SessionRunState): SessionRunState { planLatch: state.planLatch ? { ...state.planLatch, consecutive_blocks: state.planLatch.consecutive_blocks ?? 0 } : initialPlanLatch(), - plan: state.plan ?? null, // Backfill: sessions persisted before U10 have no counter on disk. mutationsSinceReport: state.mutationsSinceReport ?? 0, validationPassedSinceReport: state.validationPassedSinceReport ?? false, @@ -431,9 +440,23 @@ function loadFromDisk() { if (!stateDir) return try { const content = readFileSync(path.join(stateDir, "sessions.json"), "utf8") - const data = JSON.parse(content) as Record + // Legacy sessions.json (pre-I33-1) carried the structural plan body on `state.plan`. Read it as an + // optional field so we can migrate it into the DocumentStore authority, then drop it from state. + const data = JSON.parse(content) as Record for (const [id, state] of Object.entries(data)) { - if (!state.completedAt) sessions.set(id, normalizeState(state)) + if (state.completedAt) continue + // I33-1 migration: if this session still has an inline plan body from before the store became the + // authority, seed it into the plan-store (idempotent upsert) so nothing is lost, then let it fall + // away (normalizeState no longer carries `plan`). Only migrate when the store has no plan yet, so + // a newer store doc (e.g. a goal edit) is never overwritten by a stale inline body. + if (state.plan && !PlanStore.getPlanDoc(id)) { + try { + PlanStore.setPlanDoc(id, state.plan) + } catch { + /* best-effort migration: a store hiccup must not block loading session state */ + } + } + sessions.set(id, normalizeState(state)) } } catch {} } diff --git a/packages/core/src/effect/app-node-platform.ts b/packages/core/src/effect/app-node-platform.ts new file mode 100644 index 00000000..7119bb24 --- /dev/null +++ b/packages/core/src/effect/app-node-platform.ts @@ -0,0 +1,18 @@ +import { NodeFileSystem, NodePath } from "@effect/platform-node" +import { LLMClient, RequestExecutor } from "@deepagent-code/llm/route" +import { FileSystem, Path } from "effect" +import { FetchHttpClient } from "effect/unstable/http" +import { HttpClient } from "effect/unstable/http" +import { makeGlobalNode } from "./app-node" + +export const filesystem = makeGlobalNode({ service: FileSystem.FileSystem, layer: NodeFileSystem.layer, deps: [] }) +export const path = makeGlobalNode({ service: Path.Path, layer: NodePath.layer, deps: [] }) +export const httpClient = makeGlobalNode({ service: HttpClient.HttpClient, layer: FetchHttpClient.layer, deps: [] }) +export const requestExecutor = makeGlobalNode({ + service: RequestExecutor.Service, + layer: RequestExecutor.layer, + deps: [httpClient], +}) +export const llmClient = makeGlobalNode({ service: LLMClient.Service, layer: LLMClient.layer, deps: [requestExecutor] }) + +export * as LayerNodePlatform from "./app-node-platform" diff --git a/packages/core/src/effect/app-node.ts b/packages/core/src/effect/app-node.ts new file mode 100644 index 00000000..e8112f92 --- /dev/null +++ b/packages/core/src/effect/app-node.ts @@ -0,0 +1,14 @@ +import { LayerNode } from "./layer-node" + +export const tags = LayerNode.tags({ + location: ["global"], + global: [], +}) + +export type GlobalNode = LayerNode.Node +export type LocationNode = LayerNode.Node + +export const makeGlobalNode = tags.make("global") +export const makeLocationNode = tags.make("location") + +export * as Node from "./app-node" diff --git a/packages/core/src/effect/layer-node.ts b/packages/core/src/effect/layer-node.ts new file mode 100644 index 00000000..9dbc3d51 --- /dev/null +++ b/packages/core/src/effect/layer-node.ts @@ -0,0 +1,333 @@ +import { Brand, Context, Layer } from "effect" + +type AnyNode = Node +type RuntimeLayer = Layer.Layer +type NodeList = readonly [] | readonly [Item, ...Item[]] +export type Output = [Item] extends [never] ? never : Item extends Node ? A : never +export type Error = [Item] extends [never] ? never : Item extends Node ? E : never +type NodeTag = [Item] extends [never] ? undefined : Item extends Node ? T : never +type Missing = Exclude> +type CheckDependencies = [ + Missing, Dependencies>, +] extends [never] + ? unknown + : { readonly "Missing dependencies": Missing, Dependencies> } +declare const $OutputType: unique symbol +declare const $ErrorType: unique symbol + +export type Tag = Name & Brand.Brand<"LayerNode.Tag"> + +const makeTag = Brand.nominal() + +export interface Node { + readonly kind: "layer" | "unbound" | "group" + readonly name: string + readonly service?: Context.Service.Any + readonly implementation?: Layer.Any + readonly dependencies: readonly AnyNode[] + readonly tag?: T + readonly [$OutputType]?: () => A + readonly [$ErrorType]?: () => E +} + +type NodeIdentity = + | { readonly service: Context.Service.Any; readonly name?: never } + | { readonly name: string; readonly service?: never } +type DistributiveOmit = A extends unknown ? Omit : never + +export type TagConfig = Readonly> +type TagNames = keyof Config & string +type NodeInTags = Node | undefined> +type CheckTags = [Exclude>] extends [ + never, +] + ? unknown + : { readonly "Invalid tag dependencies": Exclude> } + +export interface Tags { + readonly values: { readonly [Name in TagNames]: Tag } + readonly make: >( + name: Name, + ) => ( + input: DistributiveOmit>, "tag"> & + CheckTags>, + ) => Node, Layer.Error | Error, Tag> +} + +export function tags( + config: Config, +): Tags { + const names = Object.keys(config) as TagNames[] + const values = Object.fromEntries(names.map((name) => [name, makeTag(name)])) as Tags["values"] + return { + values, + make: ((name: TagNames) => (input: DistributiveOmit, "tag">) => + make({ ...input, tag: values[name] })) as Tags["make"], + } +} + +// Nodes --------------------------------------------------------------------- + +type MakeInput< + Implementation extends Layer.Any, + Items extends NodeList, + T extends Tag | undefined = undefined, +> = NodeIdentity & { + readonly layer: Implementation + readonly deps: Items & CheckDependencies> + readonly tag?: T +} + +export function make< + const Implementation extends Layer.Any, + const Items extends NodeList, + const T extends Tag | undefined = undefined, +>( + input: MakeInput, +): Node, Layer.Error | Error, T> { + return { + kind: "layer", + name: input.service !== undefined ? input.service.key : input.name, + service: input.service, + implementation: input.layer, + dependencies: input.deps, + tag: input.tag, + } +} + +export function unbound(service: Context.Key, tag: T): Node { + return { + kind: "unbound", + name: service.key, + service, + dependencies: [], + tag, + } +} + +export function group( + dependencies: Items, +): Node, Error, NodeTag> { + return { kind: "group", name: "group", dependencies } +} + +export type Replacement = readonly [source: AnyNode, replacement: AnyNode | Layer.Any] +export type Replacements = readonly Replacement[] + +type CheckReplacementErrors = [Exclude] extends [never] + ? unknown + : { readonly "New replacement errors": Exclude } + +type CheckReplacement = Item extends readonly [Node, infer Replacement] + ? Replacement extends Node, infer E2, T> + ? CheckReplacementErrors> + : Replacement extends Layer.Layer, infer E2, never> + ? CheckReplacementErrors> + : { readonly "Invalid replacement": Replacement } + : { readonly "Invalid replacement": Item } + +type CheckReplacements = { + readonly [K in keyof Items]: CheckReplacement +} + +type ValidReplacements = Items & CheckReplacements + +function replacementNode(source: AnyNode, replacement: AnyNode | Layer.Any) { + const replacementNode = isNode(replacement) + ? replacement + : make({ + ...nodeMakeIdentity(source), + layer: replacement as Layer.Layer, + deps: [], + tag: source.tag, + }) + if (source.name !== replacementNode.name) { + throw new Error(`Cannot replace ${source.name} with ${replacementNode.name}`) + } + if (source.tag !== replacementNode.tag) { + throw new Error(`Cannot replace ${source.name} across tags`) + } + return replacementNode +} + +function nodeMakeIdentity(node: AnyNode): NodeIdentity { + if (node.service !== undefined) return { service: node.service } + return { name: node.name } +} + +function isNode(input: Layer.Any | AnyNode): input is AnyNode { + return "kind" in input && "dependencies" in input +} + +// Tree ----------------------------------------------------------------------- + +type Visit = (node: AnyNode, context: VisitContext) => Result + +type VisitContext = { + readonly cache: Map + readonly visit: (node: AnyNode) => Result +} + +function walk( + root: AnyNode, + visit: Visit, + options: { + readonly cache?: Map + readonly resolve?: (node: AnyNode) => AnyNode + readonly detectCycles?: boolean + } = {}, +) { + const cache = options.cache ?? new Map() + const visiting = new Set() + const stack: AnyNode[] = [] + + const recur = (node: AnyNode): Result => { + const target = options.resolve?.(node) ?? node + const cached = cache.get(target) + if (cached !== undefined || cache.has(target)) return cached! + + if (options.detectCycles !== false && visiting.has(target)) { + const start = stack.indexOf(target) + throw new Error( + `Cycle detected in layer tree: ${[...stack.slice(start), target].map((item) => item.name).join(" -> ")}`, + ) + } + + visiting.add(target) + stack.push(target) + try { + const result = visit(target, { cache, visit: recur }) + if (!cache.has(target)) cache.set(target, result) + return result + } finally { + stack.pop() + visiting.delete(target) + } + } + + return recur(root) +} + +export function hoist( + root: Node, + tag: T, + replacements?: ValidReplacements, +): { + readonly node: Node + readonly hoisted: Node +} { + const hoisted = new Map() + const replacementMap = replacementMapFrom(replacements) + + const node = walk( + root, + (node, context) => { + if (node.kind === "group") { + return { ...node, dependencies: node.dependencies.map(context.visit) } + } + if (node.tag === tag) { + const existing = hoisted.get(node.name) + if (existing && existing !== node) { + throw new Error(`Tag ${tag} has conflicting implementations for ${node.name}`) + } + hoisted.set(node.name, rewriteReplacementDependencies(node, replacementMap)) + return group([]) + } + if (node.kind === "unbound") { + return node + } + return { ...node, dependencies: node.dependencies.map(context.visit) } + }, + { resolve: (node) => replacementMap.get(node.name) ?? node }, + ) + + return { + node: node as Node, + hoisted: group(Array.from(hoisted.values())) as Node, + } +} + +export function compile( + root: Node, + replacements?: ValidReplacements, +): Layer.Layer { + const replacementMap = replacementMapFrom(replacements) + const cache = new Map() + const compileNode = (node: AnyNode) => + walk( + node, + (node, context) => { + if (node.kind === "unbound") throw new Error(`Unbound layer node: ${node.name}`) + const dependencies = node.dependencies.flatMap(flatten).map(context.visit) + const implementation = node.implementation! as RuntimeLayer + return dependencies.length === 0 + ? implementation + : implementation.pipe(Layer.provide(dependencies as [RuntimeLayer, ...RuntimeLayer[]])) + }, + { cache, resolve: (node) => replacementMap.get(node.name) ?? node }, + ) + const layers = flatten(root).map((node) => compileNode(node)) + const layer = layers.reduce((result, layer) => layer.pipe(Layer.provideMerge(result)), Layer.empty) + return layer as Layer.Layer +} + +function replacementMapFrom(replacements?: Replacements) { + return ( + replacements?.reduce((map, [source, replacement]) => { + const normalized = rewriteReplacementDependencies(replacementNode(source, replacement), map) + const current = new Map([[source.name, normalized]]) + for (const [name, node] of map) map.set(name, rewriteReplacementDependencies(node, current)) + map.set(source.name, normalized) + return map + }, new Map()) ?? new Map() + ) +} + +function rewriteReplacementDependencies(root: AnyNode, replacements: ReadonlyMap) { + if (replacements.size === 0) return root + const cache = new Map() + const visiting = new Set() + const stack: AnyNode[] = [] + + const recur = (node: AnyNode, isRoot = false): AnyNode => { + const target = isRoot ? node : (replacements.get(node.name) ?? node) + const cached = cache.get(target) + if (cached !== undefined || cache.has(target)) return cached! + if (visiting.has(target)) { + const start = stack.indexOf(target) + throw new Error( + `Cycle detected in layer tree: ${[...stack.slice(start), target].map((item) => item.name).join(" -> ")}`, + ) + } + + visiting.add(target) + stack.push(target) + try { + const dependencies = target.dependencies.map((dependency) => recur(dependency)) + const result = dependencies.every((dependency, index) => dependency === target.dependencies[index]) + ? target + : { ...target, dependencies } + cache.set(target, result) + return result + } finally { + stack.pop() + visiting.delete(target) + } + } + + return recur(root, true) +} + +export function hasUnbound(root: Node, source: AnyNode): boolean { + if (source.kind !== "unbound") throw new Error(`Cannot check non-unbound layer node: ${source.name}`) + return walk(root, (node, context) => { + if (node === source) return true + return node.dependencies.some(context.visit) + }) +} + +function flatten(node: AnyNode): readonly AnyNode[] { + return node.kind === "group" ? node.dependencies.flatMap(flatten) : [node] +} + +export * as LayerNode from "./layer-node" diff --git a/packages/core/src/event.ts b/packages/core/src/event.ts index e7c8bc1d..a2183d1d 100644 --- a/packages/core/src/event.ts +++ b/packages/core/src/event.ts @@ -5,6 +5,7 @@ import { and, asc, eq, gt } from "drizzle-orm" import { Database } from "./database/database" import { EventSequenceTable, EventTable } from "./event/sql" import { Location } from "./location" +import { makeGlobalNode } from "./effect/app-node" import { externalID, type ExternalID, NonNegativeInt, withStatics } from "./schema" import { Identifier } from "./util/identifier" import { isDeepStrictEqual } from "node:util" @@ -677,4 +678,6 @@ export const layerWith = (options?: LayerOptions) => export const layer = layerWith() +export const node = makeGlobalNode({ service: Service, layer: layer, deps: [Database.node] }) + export const defaultLayer = layer.pipe(Layer.provide(Database.defaultLayer)) diff --git a/packages/core/src/file-mutation.ts b/packages/core/src/file-mutation.ts index cd8d6de5..6f009aaa 100644 --- a/packages/core/src/file-mutation.ts +++ b/packages/core/src/file-mutation.ts @@ -3,6 +3,7 @@ export * as FileMutation from "./file-mutation" import { Context, Effect, Layer, Schema } from "effect" import { dirname } from "path" import { KeyedMutex } from "./effect/keyed-mutex" +import { makeLocationNode } from "./effect/app-node" import { FSUtil } from "./fs-util" export interface Target { @@ -190,6 +191,8 @@ function sameBytes(left: Uint8Array, right: Uint8Array) { return left.every((byte, index) => byte === right[index]) } +export const node = makeLocationNode({ service: Service, layer, deps: [FSUtil.node] }) + export const locationLayer = layer /** diff --git a/packages/core/src/filesystem/watcher.ts b/packages/core/src/filesystem/watcher.ts index f9cfc0de..dc8b6d5b 100644 --- a/packages/core/src/filesystem/watcher.ts +++ b/packages/core/src/filesystem/watcher.ts @@ -6,6 +6,7 @@ import type ParcelWatcher from "@parcel/watcher" import { Cause, Context, Effect, Layer, Schema } from "effect" import path from "path" import { Config } from "../config" +import { makeLocationNode } from "../effect/app-node" import { EventV2 } from "../event" import { Flag } from "../flag/flag" import { FSUtil } from "../fs-util" @@ -147,4 +148,10 @@ export const layer = Layer.effect( ), ) +export const node = makeLocationNode({ + service: Service, + layer, + deps: [FSUtil.node, Location.node, Config.node, Git.node, EventV2.node], +}) + export const locationLayer = layer.pipe(Layer.provide(Config.locationLayer), Layer.provide(Git.defaultLayer)) diff --git a/packages/core/src/fs-util.ts b/packages/core/src/fs-util.ts index 8d9d17dd..a8932c35 100644 --- a/packages/core/src/fs-util.ts +++ b/packages/core/src/fs-util.ts @@ -7,6 +7,8 @@ import { Context, Effect, FileSystem, Layer, Schema } from "effect" import type { PlatformError } from "effect/PlatformError" import { Glob } from "./util/glob" import { serviceUse } from "./effect/service-use" +import { makeGlobalNode } from "./effect/app-node" +import { filesystem } from "./effect/app-node-platform" export namespace FSUtil { export class FileSystemError extends Schema.TaggedErrorClass()("FileSystemError", { @@ -195,6 +197,8 @@ export namespace FSUtil { export const defaultLayer = layer.pipe(Layer.provide(NodeFileSystem.layer)) + export const node = makeGlobalNode({ service: Service, layer: layer, deps: [filesystem] }) + // Pure helpers that don't need Effect (path manipulation, sync operations) export function mimeType(p: string): string { return lookup(p) || "application/octet-stream" diff --git a/packages/core/src/git.ts b/packages/core/src/git.ts index 4fa544e1..c5b25bd6 100644 --- a/packages/core/src/git.ts +++ b/packages/core/src/git.ts @@ -6,6 +6,7 @@ import { ChildProcess } from "effect/unstable/process" import { AbsolutePath } from "./schema" import { FSUtil } from "./fs-util" import { AppProcess } from "./process" +import { makeGlobalNode } from "./effect/app-node" export interface Repo { /** @@ -401,6 +402,8 @@ export const layer = Layer.effect( export const defaultLayer = layer.pipe(Layer.provide(FSUtil.defaultLayer), Layer.provide(AppProcess.defaultLayer)) +export const node = makeGlobalNode({ service: Service, layer: layer, deps: [FSUtil.node, AppProcess.node] }) + export interface Result { readonly exitCode: number readonly text: string diff --git a/packages/core/src/global.ts b/packages/core/src/global.ts index 2cc1ce17..a749e5b9 100644 --- a/packages/core/src/global.ts +++ b/packages/core/src/global.ts @@ -6,6 +6,7 @@ import { Context, Effect, Layer } from "effect" import { Flock } from "./util/flock" import { Flag } from "./flag/flag" import { resolveDataPath, resolveHomeBase } from "./global-path" +import { makeGlobalNode } from "./effect/app-node" const app = "deepagent-code" const legacyData = path.join(xdgData!, app) @@ -199,6 +200,8 @@ export const layer = Layer.effect( export const defaultLayer = layer +export const node = makeGlobalNode({ service: Service, layer: layer, deps: [] }) + export const layerWith = (input: Partial) => Layer.effect( Service, diff --git a/packages/core/src/image.ts b/packages/core/src/image.ts index 9a715cce..25a21f40 100644 --- a/packages/core/src/image.ts +++ b/packages/core/src/image.ts @@ -1,6 +1,7 @@ export * as Image from "./image" import { Context, Effect, Layer, Schema } from "effect" +import { makeLocationNode } from "./effect/app-node" import { Config } from "./config" import { FileSystem } from "./filesystem" @@ -69,4 +70,6 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [Config.node] }) + export const locationLayer = layer.pipe(Layer.provide(Config.locationLayer)) diff --git a/packages/core/src/instruction-context.ts b/packages/core/src/instruction-context.ts index 909f0813..8043b4e8 100644 --- a/packages/core/src/instruction-context.ts +++ b/packages/core/src/instruction-context.ts @@ -2,6 +2,7 @@ export * as InstructionContext from "./instruction-context" import { Array, Effect, Layer, Schema } from "effect" import { isAbsolute, join, relative, sep } from "path" +import { makeLocationNode } from "./effect/app-node" import { FSUtil } from "./fs-util" import { Flag } from "./flag/flag" import { Global } from "./global" @@ -87,6 +88,12 @@ export const layer = Layer.effectDiscard( }), ) +export const node = makeLocationNode({ + name: "instruction-context", + layer, + deps: [FSUtil.node, Global.node, Location.node, SystemContextRegistry.node], +}) + function render(files: ReadonlyArray) { return files.map((file) => `Instructions from: ${file.path}\n${file.content}`).join("\n\n") } diff --git a/packages/core/src/location-mutation.ts b/packages/core/src/location-mutation.ts index 202aa5c5..3f432239 100644 --- a/packages/core/src/location-mutation.ts +++ b/packages/core/src/location-mutation.ts @@ -2,6 +2,7 @@ export * as LocationMutation from "./location-mutation" import path from "path" import { Context, Effect, Layer, Schema } from "effect" +import { makeLocationNode } from "./effect/app-node" import { FSUtil } from "./fs-util" import { Location } from "./location" @@ -152,4 +153,10 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ + service: Service, + layer: layer.pipe(Layer.orDie), + deps: [FSUtil.node, Location.node], +}) + export const locationLayer = layer diff --git a/packages/core/src/location.ts b/packages/core/src/location.ts index abdb80f4..93a04ec7 100644 --- a/packages/core/src/location.ts +++ b/packages/core/src/location.ts @@ -2,6 +2,8 @@ import { Context, Effect, Layer, Schema } from "effect" import { Project } from "./project" import { AbsolutePath } from "./schema" import { WorkspaceV2 } from "./workspace" +import { LayerNode } from "./effect/layer-node" +import { tags } from "./effect/app-node" export * as Location from "./location" @@ -30,6 +32,8 @@ export function response(data: S) { export class Service extends Context.Service()("@deepagent-code/Location") {} +export const node = LayerNode.unbound(Service, tags.values.location) + export const layer = (ref: Ref) => Layer.effect( Service, diff --git a/packages/core/src/models-dev.ts b/packages/core/src/models-dev.ts index f2739adc..9291024d 100644 --- a/packages/core/src/models-dev.ts +++ b/packages/core/src/models-dev.ts @@ -8,6 +8,8 @@ import { Hash } from "./util/hash" import { FSUtil } from "./fs-util" import { InstallationChannel, InstallationVersion } from "./installation/version" import { EventV2 } from "./event" +import { makeGlobalNode } from "./effect/app-node" +import { httpClient } from "./effect/app-node-platform" export const CatalogModelStatus = Schema.Literals(["alpha", "beta", "deprecated"]) export type CatalogModelStatus = typeof CatalogModelStatus.Type @@ -243,6 +245,8 @@ export const layer = Layer.effect( }), ) +export const node = makeGlobalNode({ service: Service, layer: layer, deps: [FSUtil.node, EventV2.node, httpClient] }) + export const defaultLayer = layer.pipe( Layer.provide(FetchHttpClient.layer), Layer.provide(FSUtil.defaultLayer), diff --git a/packages/core/src/npm.ts b/packages/core/src/npm.ts index efbf940c..31f8ec96 100644 --- a/packages/core/src/npm.ts +++ b/packages/core/src/npm.ts @@ -8,6 +8,8 @@ import { FSUtil } from "./fs-util" import { Global } from "./global" import { EffectFlock } from "./util/effect-flock" import { makeRuntime } from "./effect/runtime" +import { makeGlobalNode } from "./effect/app-node" +import { filesystem } from "./effect/app-node-platform" import { NpmConfig } from "./npm-config" export class InstallFailedError extends Schema.TaggedErrorClass()("NpmInstallFailedError", { @@ -244,6 +246,12 @@ export const layer = Layer.effect( }), ) +export const node = makeGlobalNode({ + service: Service, + layer: layer, + deps: [FSUtil.node, Global.node, filesystem, EffectFlock.node], +}) + export const defaultLayer = layer.pipe( Layer.provide(EffectFlock.layer), Layer.provide(FSUtil.layer), diff --git a/packages/core/src/permission.ts b/packages/core/src/permission.ts index 032304d6..442eefd0 100644 --- a/packages/core/src/permission.ts +++ b/packages/core/src/permission.ts @@ -1,6 +1,7 @@ export * as PermissionV2 from "./permission" import { Context, Deferred, Effect as EffectRuntime, Layer, Schema } from "effect" +import { makeLocationNode } from "./effect/app-node" import { EventV2 } from "./event" import { Location } from "./location" import { AgentV2 } from "./agent" @@ -326,4 +327,10 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ + service: Service, + layer, + deps: [EventV2.node, Location.node, AgentV2.node, SessionStore.node, PermissionSaved.node], +}) + export const locationLayer = layer.pipe(Layer.provideMerge(AgentV2.locationLayer)) diff --git a/packages/core/src/permission/saved.ts b/packages/core/src/permission/saved.ts index a723a602..22ce8ab5 100644 --- a/packages/core/src/permission/saved.ts +++ b/packages/core/src/permission/saved.ts @@ -3,6 +3,7 @@ export * as PermissionSaved from "./saved" import { eq } from "drizzle-orm" import { Context, Effect, Layer, Schema } from "effect" import { Database } from "../database/database" +import { makeGlobalNode } from "../effect/app-node" import { ProjectV2 } from "../project" import { withStatics } from "../schema" import { Identifier } from "../util/identifier" @@ -84,4 +85,6 @@ export const layer = Layer.effect( }), ) +export const node = makeGlobalNode({ service: Service, layer, deps: [Database.node] }) + export const defaultLayer = layer.pipe(Layer.provide(Database.defaultLayer)) diff --git a/packages/core/src/policy.ts b/packages/core/src/policy.ts index a8d5c97d..fb7e1b98 100644 --- a/packages/core/src/policy.ts +++ b/packages/core/src/policy.ts @@ -3,6 +3,7 @@ export * as Policy from "./policy" import { Context, Effect as EffectRuntime, Layer, Schema } from "effect" import { Wildcard } from "./util/wildcard" import { Location } from "./location" +import { makeLocationNode } from "./effect/app-node" export const Effect = Schema.Literals(["allow", "deny"]).annotate({ identifier: "Policy.Effect" }) export type Effect = typeof Effect.Type @@ -43,4 +44,6 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [Location.node] }) + export const locationLayer = layer diff --git a/packages/core/src/process.ts b/packages/core/src/process.ts index 9ea95a13..37ca2b97 100644 --- a/packages/core/src/process.ts +++ b/packages/core/src/process.ts @@ -3,6 +3,7 @@ import type { PlatformError } from "effect/PlatformError" import { ChildProcess } from "effect/unstable/process" import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner" import { CrossSpawnSpawner } from "./cross-spawn-spawner" +import { makeGlobalNode } from "./effect/app-node" export class AppProcessError extends Schema.TaggedErrorClass()("AppProcessError", { command: Schema.String, @@ -231,4 +232,6 @@ export const layer = Layer.effect( export const defaultLayer = layer.pipe(Layer.provide(CrossSpawnSpawner.defaultLayer)) +export const node = makeGlobalNode({ service: Service, layer: layer, deps: [CrossSpawnSpawner.node] }) + export * as AppProcess from "./process" diff --git a/packages/core/src/pty.ts b/packages/core/src/pty.ts index 6b93667b..a098e397 100644 --- a/packages/core/src/pty.ts +++ b/packages/core/src/pty.ts @@ -2,6 +2,7 @@ export * as Pty from "./pty" import type { Disp, Proc } from "#pty" import { Context, Effect, Layer, Schema, Types } from "effect" +import { makeLocationNode } from "./effect/app-node" import { EventV2 } from "./event" import { Location } from "./location" import { NonNegativeInt, PositiveInt } from "./schema" @@ -328,4 +329,6 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [EventV2.node, Location.node] }) + export const locationLayer = layer diff --git a/packages/core/src/question.ts b/packages/core/src/question.ts index 10f647fc..5c5ec184 100644 --- a/packages/core/src/question.ts +++ b/packages/core/src/question.ts @@ -1,6 +1,7 @@ export * as QuestionV2 from "./question" import { Context, Deferred, Effect, Layer, Schema } from "effect" +import { makeLocationNode } from "./effect/app-node" import { EventV2 } from "./event" import { Identifier } from "./id/id" import { withStatics } from "./schema" @@ -195,4 +196,6 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [EventV2.node] }) + export const locationLayer = layer diff --git a/packages/core/src/repository-cache.ts b/packages/core/src/repository-cache.ts index 0b89cef6..e6aff09f 100644 --- a/packages/core/src/repository-cache.ts +++ b/packages/core/src/repository-cache.ts @@ -5,6 +5,7 @@ import { Git } from "./git" import { Global } from "./global" import { Repository } from "./repository" import { EffectFlock } from "./util/effect-flock" +import { makeGlobalNode } from "./effect/app-node" export type Result = { readonly repository: string @@ -259,6 +260,12 @@ export const defaultLayer: Layer.Layer = layer.pipe( Layer.provide(Global.defaultLayer), ) +export const node = makeGlobalNode({ + service: Service, + layer, + deps: [EffectFlock.node, FSUtil.node, Git.node, Global.node], +}) + function statusForRepository(input: { reuse: boolean; refresh?: boolean; branchMatches?: boolean }) { if (!input.reuse) return "cloned" as const if (input.branchMatches === false || input.refresh) return "refreshed" as const diff --git a/packages/core/src/session/projector.ts b/packages/core/src/session/projector.ts index e88d73f5..5642f193 100644 --- a/packages/core/src/session/projector.ts +++ b/packages/core/src/session/projector.ts @@ -3,6 +3,7 @@ export * as SessionProjector from "./projector" import { and, desc, eq, sql } from "drizzle-orm" import { DateTime, Effect, Layer, Schema } from "effect" import { Database } from "../database/database" +import { makeGlobalNode } from "../effect/app-node" import { EventV2 } from "../event" import { SessionEvent } from "./event" import { SessionV1 } from "../v1/session" @@ -447,4 +448,6 @@ export const layer = Layer.effectDiscard( }), ) +export const node = makeGlobalNode({ name: "session-projector", layer, deps: [EventV2.node, Database.node] }) + export const defaultLayer = layer.pipe(Layer.provide(EventV2.defaultLayer), Layer.provide(Database.defaultLayer)) diff --git a/packages/core/src/session/store.ts b/packages/core/src/session/store.ts index ff1c557d..fcb7586e 100644 --- a/packages/core/src/session/store.ts +++ b/packages/core/src/session/store.ts @@ -3,6 +3,7 @@ export * as SessionStore from "./store" import { eq } from "drizzle-orm" import { Context, Effect, Layer, Schema } from "effect" import { Database } from "../database/database" +import { makeGlobalNode } from "../effect/app-node" import { SessionHistory } from "./history" import { MessageDecodeError } from "./error" import { SessionMessage } from "./message" @@ -59,4 +60,6 @@ export const layer = Layer.effect( }), ) +export const node = makeGlobalNode({ service: Service, layer, deps: [Database.node] }) + export const defaultLayer = layer.pipe(Layer.provide(Database.defaultLayer)) diff --git a/packages/core/src/session/todo.ts b/packages/core/src/session/todo.ts index b96996ba..990d292f 100644 --- a/packages/core/src/session/todo.ts +++ b/packages/core/src/session/todo.ts @@ -7,6 +7,7 @@ export * as SessionTodo from "./todo" import { asc, eq } from "drizzle-orm" import { Context, Effect, Layer, Schema } from "effect" import { Database } from "../database/database" +import { makeLocationNode } from "../effect/app-node" import { EventV2 } from "../event" import { SessionSchema } from "./schema" import { TodoTable } from "./sql" @@ -92,4 +93,6 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [EventV2.node, Database.node] }) + export const defaultLayer = layer.pipe(Layer.provide(EventV2.defaultLayer), Layer.provide(Database.defaultLayer)) diff --git a/packages/core/src/skill.ts b/packages/core/src/skill.ts index 65f9f2a4..b082302e 100644 --- a/packages/core/src/skill.ts +++ b/packages/core/src/skill.ts @@ -5,6 +5,7 @@ import { Context, Effect, Layer, Schema } from "effect" import { castDraft } from "immer" import { AgentV2 } from "./agent" import { ConfigMarkdown } from "./config/markdown" +import { makeLocationNode } from "./effect/app-node" import { FSUtil } from "./fs-util" import { PermissionV2 } from "./permission" import { AbsolutePath, withStatics } from "./schema" @@ -158,4 +159,6 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [SkillDiscovery.node, FSUtil.node] }) + export const locationLayer = layer.pipe(Layer.provide(SkillDiscovery.defaultLayer)) diff --git a/packages/core/src/skill/discovery.ts b/packages/core/src/skill/discovery.ts index 17de1a2d..e20f8a18 100644 --- a/packages/core/src/skill/discovery.ts +++ b/packages/core/src/skill/discovery.ts @@ -3,6 +3,8 @@ export * as SkillDiscovery from "./discovery" import path from "path" import { Context, Effect, Layer, Schedule, Schema } from "effect" import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http" +import { makeGlobalNode } from "../effect/app-node" +import { httpClient } from "../effect/app-node-platform" import { FSUtil } from "../fs-util" import { Global } from "../global" import { AbsolutePath } from "../schema" @@ -167,6 +169,8 @@ export const layer = Layer.effect( }), ) +export const node = makeGlobalNode({ service: Service, layer, deps: [httpClient, FSUtil.node, Global.node] }) + export const defaultLayer = layer.pipe( Layer.provide(FetchHttpClient.layer), Layer.provide(FSUtil.defaultLayer), diff --git a/packages/core/src/system-context/builtins.ts b/packages/core/src/system-context/builtins.ts index 42cba27b..5520822f 100644 --- a/packages/core/src/system-context/builtins.ts +++ b/packages/core/src/system-context/builtins.ts @@ -1,6 +1,9 @@ export * as SystemContextBuiltIns from "./builtins" import { DateTime, Effect, Layer, Schema } from "effect" +import { makeLocationNode } from "../effect/app-node" +import { FSUtil } from "../fs-util" +import { Global } from "../global" import { Location } from "../location" import { SystemContext } from "./index" import { InstructionContext } from "../instruction-context" @@ -40,6 +43,12 @@ const builtIns = Layer.effectDiscard( }), ) +export const node = makeLocationNode({ + name: "system-context-builtins", + layer: builtIns, + deps: [Location.node, SystemContextRegistry.node, InstructionContext.node, FSUtil.node, Global.node], +}) + export const layer = Layer.mergeAll(builtIns, InstructionContext.layer).pipe( Layer.provideMerge(SystemContextRegistry.layer), ) diff --git a/packages/core/src/system-context/registry.ts b/packages/core/src/system-context/registry.ts index a3c678ea..17895bcc 100644 --- a/packages/core/src/system-context/registry.ts +++ b/packages/core/src/system-context/registry.ts @@ -1,6 +1,7 @@ export * as SystemContextRegistry from "./registry" import { Context, Effect, Layer, Ref, Scope } from "effect" +import { makeLocationNode } from "../effect/app-node" import { SystemContext } from "./index" export interface Entry { @@ -44,3 +45,5 @@ export const layer = Layer.effect( }) }), ) + +export const node = makeLocationNode({ service: Service, layer, deps: [] }) diff --git a/packages/core/src/tool-output-store.ts b/packages/core/src/tool-output-store.ts index 1ee9c5ba..5c5c47a8 100644 --- a/packages/core/src/tool-output-store.ts +++ b/packages/core/src/tool-output-store.ts @@ -3,6 +3,7 @@ export * as ToolOutputStore from "./tool-output-store" import path from "path" import { Context, Duration, Effect, Layer, Option, Schedule, Schema } from "effect" import { Config } from "./config" +import { makeLocationNode } from "./effect/app-node" import { FSUtil } from "./fs-util" import { Global } from "./global" import { SessionSchema } from "./session/schema" @@ -186,6 +187,8 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [FSUtil.node, Global.node, Config.node] }) + export const defaultLayer = layer.pipe(Layer.provide(FSUtil.defaultLayer), Layer.provide(Global.defaultLayer)) /** Runs retention scanning once globally rather than once per active Location. */ diff --git a/packages/core/src/tool/application-tools.ts b/packages/core/src/tool/application-tools.ts index b8c0f78e..20ecf4de 100644 --- a/packages/core/src/tool/application-tools.ts +++ b/packages/core/src/tool/application-tools.ts @@ -2,6 +2,7 @@ export * as ApplicationTools from "./application-tools" import { Context, Effect, Layer, Scope } from "effect" import { enableMapSet } from "immer" +import { makeGlobalNode } from "../effect/app-node" import { State } from "../state" import { Tool } from "./tool" @@ -56,3 +57,5 @@ export const layer = Layer.effect( }) }), ) + +export const node = makeGlobalNode({ service: Service, layer, deps: [] }) diff --git a/packages/core/src/tool/apply-patch.ts b/packages/core/src/tool/apply-patch.ts index bb2ca9f8..551acf65 100644 --- a/packages/core/src/tool/apply-patch.ts +++ b/packages/core/src/tool/apply-patch.ts @@ -7,6 +7,8 @@ import { FSUtil } from "../fs-util" import { LocationMutation } from "../location-mutation" import { Patch } from "../patch" import { PermissionV2 } from "../permission" +import { makeLocationNode } from "../effect/app-node" +import { ToolRegistry } from "./registry" import { Tool } from "./tool" import { Tools } from "./tools" @@ -175,3 +177,9 @@ export const layer = Layer.effectDiscard( .pipe(Effect.orDie) }), ) + +export const node = makeLocationNode({ + name: "tool/apply-patch", + layer, + deps: [ToolRegistry.node, LocationMutation.node, FileMutation.node, FSUtil.node, PermissionV2.node], +}) diff --git a/packages/core/src/tool/bash.ts b/packages/core/src/tool/bash.ts index 0e14ee2b..4aa8eaa4 100644 --- a/packages/core/src/tool/bash.ts +++ b/packages/core/src/tool/bash.ts @@ -10,6 +10,8 @@ import { LocationMutation } from "../location-mutation" import { AppProcess } from "../process" import { PermissionV2 } from "../permission" import { Policy } from "../policy" +import { makeLocationNode } from "../effect/app-node" +import { ToolRegistry } from "./registry" import { ServerCapabilities } from "../server-capabilities" import { PositiveInt } from "../schema" import { Tool } from "./tool" @@ -240,3 +242,9 @@ export const layer = Layer.effectDiscard( .pipe(Effect.orDie) }), ) + +export const node = makeLocationNode({ + name: "tool/bash", + layer, + deps: [ToolRegistry.node, LocationMutation.node, FSUtil.node, AppProcess.node, Config.node, PermissionV2.node, Policy.node], +}) diff --git a/packages/core/src/tool/edit.ts b/packages/core/src/tool/edit.ts index dab26554..3a121d5f 100644 --- a/packages/core/src/tool/edit.ts +++ b/packages/core/src/tool/edit.ts @@ -13,6 +13,8 @@ import { FileMutation } from "../file-mutation" import { FSUtil } from "../fs-util" import { LocationMutation } from "../location-mutation" import { PermissionV2 } from "../permission" +import { makeLocationNode } from "../effect/app-node" +import { ToolRegistry } from "./registry" import { Tool } from "./tool" import { Tools } from "./tools" @@ -198,3 +200,9 @@ export const layer = Layer.effectDiscard( .pipe(Effect.orDie) }), ) + +export const node = makeLocationNode({ + name: "tool/edit", + layer, + deps: [ToolRegistry.node, LocationMutation.node, FileMutation.node, FSUtil.node, PermissionV2.node], +}) diff --git a/packages/core/src/tool/question.ts b/packages/core/src/tool/question.ts index 2a95de15..563a3559 100644 --- a/packages/core/src/tool/question.ts +++ b/packages/core/src/tool/question.ts @@ -4,6 +4,8 @@ import { ToolFailure, toolText } from "@deepagent-code/llm" import { Effect, Layer, Schema } from "effect" import { PermissionV2 } from "../permission" import { QuestionV2 } from "../question" +import { makeLocationNode } from "../effect/app-node" +import { ToolRegistry } from "./registry" import { Tool } from "./tool" import { Tools } from "./tools" @@ -84,3 +86,9 @@ export const layer = Layer.effectDiscard( .pipe(Effect.orDie) }), ) + +export const node = makeLocationNode({ + name: "tool/question", + layer, + deps: [ToolRegistry.node, PermissionV2.node, QuestionV2.node], +}) diff --git a/packages/core/src/tool/registry.ts b/packages/core/src/tool/registry.ts index 351fdca5..d5a49138 100644 --- a/packages/core/src/tool/registry.ts +++ b/packages/core/src/tool/registry.ts @@ -3,6 +3,7 @@ export * as ToolRegistry from "./registry" import { ToolOutput, type ToolCall, type ToolDefinition, type ToolSettlement } from "@deepagent-code/llm" import { Context, Effect, Layer, Scope } from "effect" import { AgentV2 } from "../agent" +import { makeLocationNode } from "../effect/app-node" import { PermissionV2 } from "../permission" import { SessionMessage } from "../session/message" import { SessionSchema } from "../session/schema" @@ -131,6 +132,18 @@ function whollyDisabled(action: string, rules: PermissionV2.Ruleset) { return rule?.resource === "*" && rule.effect === "deny" } +export const node = makeLocationNode({ + service: Service, + layer, + deps: [ApplicationTools.node, ToolOutputStore.node], +}) + +export const toolsNode = makeLocationNode({ + service: Tools.Service, + layer, + deps: [ApplicationTools.node, ToolOutputStore.node], +}) + export const defaultLayer = layer.pipe( Layer.provide(ApplicationTools.layer), Layer.provide(ToolOutputStore.defaultLayer), diff --git a/packages/core/src/tool/todowrite.ts b/packages/core/src/tool/todowrite.ts index c990abbe..aa6bb7b4 100644 --- a/packages/core/src/tool/todowrite.ts +++ b/packages/core/src/tool/todowrite.ts @@ -4,6 +4,8 @@ import { ToolFailure, toolText } from "@deepagent-code/llm" import { Effect, Layer, Schema } from "effect" import { PermissionV2 } from "../permission" import { SessionTodo } from "../session/todo" +import { makeLocationNode } from "../effect/app-node" +import { ToolRegistry } from "./registry" import { Tool } from "./tool" import { Tools } from "./tools" @@ -52,3 +54,9 @@ export const layer = Layer.effectDiscard( .pipe(Effect.orDie) }), ) + +export const node = makeLocationNode({ + name: "tool/todowrite", + layer, + deps: [ToolRegistry.node, PermissionV2.node, SessionTodo.node], +}) diff --git a/packages/core/src/tool/webfetch.ts b/packages/core/src/tool/webfetch.ts index 2f252554..737d6e64 100644 --- a/packages/core/src/tool/webfetch.ts +++ b/packages/core/src/tool/webfetch.ts @@ -6,6 +6,9 @@ import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstab import { Parser } from "htmlparser2" import TurndownService from "turndown" import { PermissionV2 } from "../permission" +import { makeLocationNode } from "../effect/app-node" +import { LayerNodePlatform } from "../effect/app-node-platform" +import { ToolRegistry } from "./registry" import { Tool } from "./tool" import { Tools } from "./tools" @@ -185,6 +188,12 @@ export const layer = Layer.effectDiscard( }), ) +export const node = makeLocationNode({ + name: "tool/webfetch", + layer, + deps: [ToolRegistry.node, PermissionV2.node, LayerNodePlatform.httpClient], +}) + export function extractTextFromHTML(html: string) { let text = "" let skipDepth = 0 diff --git a/packages/core/src/tool/websearch.ts b/packages/core/src/tool/websearch.ts index a2f42025..e9d39d16 100644 --- a/packages/core/src/tool/websearch.ts +++ b/packages/core/src/tool/websearch.ts @@ -7,6 +7,9 @@ import { truthy } from "../flag/flag" import { InstallationVersion } from "../installation/version" import { PositiveInt } from "../schema" import { PermissionV2 } from "../permission" +import { makeLocationNode } from "../effect/app-node" +import { LayerNodePlatform } from "../effect/app-node-platform" +import { ToolRegistry } from "./registry" import { Tool } from "./tool" import { Tools } from "./tools" import { checksum } from "../util/encode" @@ -83,6 +86,8 @@ export const defaultConfigLayer = Layer.sync(ConfigService, () => }), ) +export const configNode = makeLocationNode({ service: ConfigService, layer: defaultConfigLayer, deps: [] }) + export function selectProvider( sessionID: string, flags: Pick = { enableExa: false, enableParallel: false }, @@ -248,3 +253,9 @@ export const layer = Layer.effectDiscard( .pipe(Effect.orDie) }), ) + +export const node = makeLocationNode({ + name: "tool/websearch", + layer, + deps: [ToolRegistry.node, PermissionV2.node, LayerNodePlatform.httpClient, configNode], +}) diff --git a/packages/core/src/util/effect-flock.ts b/packages/core/src/util/effect-flock.ts index 64a1b6f7..aabeb1c7 100644 --- a/packages/core/src/util/effect-flock.ts +++ b/packages/core/src/util/effect-flock.ts @@ -6,6 +6,7 @@ import type { FileSystem, Scope } from "effect" import type { PlatformError } from "effect/PlatformError" import { FSUtil } from "../fs-util" import { Global } from "../global" +import { makeGlobalNode } from "../effect/app-node" import { Hash } from "./hash" export namespace EffectFlock { @@ -280,4 +281,6 @@ export namespace EffectFlock { ) export const defaultLayer = layer.pipe(Layer.provide(FSUtil.defaultLayer), Layer.provide(Global.layer)) + + export const node = makeGlobalNode({ service: Service, layer: layer, deps: [Global.node, FSUtil.node] }) } diff --git a/packages/core/test/deepagent/command-intent.test.ts b/packages/core/test/deepagent/command-intent.test.ts index f9f1f4ca..5a066ecd 100644 --- a/packages/core/test/deepagent/command-intent.test.ts +++ b/packages/core/test/deepagent/command-intent.test.ts @@ -186,4 +186,55 @@ describe("CommandIntent.classifyCommand", () => { expect(classifyCommand("curl -sL https://example.com/api")).toBe("read_only") }) }) + + // P0 (parity with codex is_safe_to_call_with_exec): text-stream filters that only read stdin/files + // and the read-only `sed -n {N|M,N}p` slice are exempt. This is the exact class of probe that + // triggered the production deadlock (`sed -n "18,45p"` inside an ssh/docker exec wrapper). + describe("P0: read-only text filters + sed -n slice", () => { + const readOnly = [ + "cut -d, -f1 data.csv", + "nl -ba file.txt", + "paste a.txt b.txt", + "rev file.txt", + "seq 1 10", + "tr a-z A-Z", + "uniq -c file.txt", + "comm -12 a.txt b.txt", + "fold -w 80 file.txt", + "expr 1 + 2", + "true", + "false", + "sort file.txt", + "sort -r -n file.txt", + "sed -n '18,45p' shim_utils.h", + "sed -n 42p file.txt", + "sed -n 1,5p file.txt", + "sort data.txt | uniq | wc -l", + ] + for (const cmd of readOnly) { + test(`read_only: ${cmd}`, () => { + expect(classifyCommand(cmd)).toBe("read_only") + }) + } + }) + + // P0 fail-safe: sed/sort variants that can write or run arbitrary scripts stay mutating. + describe("P0: sed/sort write variants stay mutating", () => { + const mutating = [ + "sed -i 's/a/b/' file.txt", // in-place edit + "sed -n 's/a/b/w out.txt' file.txt", // -n but writes via w + "sed 's/a/b/' file.txt", // no -n → not the read-only slice form + "sed -n -e 'p' file.txt", // extra -e script, not a bare address + "sed -n 'xp' file.txt", // invalid (non-numeric) address + "sed -n '1,5d' file.txt", // delete, not print + "sort -o out.txt file.txt", // sort writing a file + "sort --output=out.txt file.txt", + "sort -oout.txt file.txt", // bundled short form + ] + for (const cmd of mutating) { + test(`mutating: ${cmd}`, () => { + expect(classifyCommand(cmd)).toBe("mutating") + }) + } + }) }) diff --git a/packages/core/test/deepagent/document-store.test.ts b/packages/core/test/deepagent/document-store.test.ts index aad4812b..45c004a1 100644 --- a/packages/core/test/deepagent/document-store.test.ts +++ b/packages/core/test/deepagent/document-store.test.ts @@ -1,8 +1,9 @@ import { describe, expect, test, beforeEach, afterEach } from "bun:test" -import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs" +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync, rmSync } from "node:fs" import { tmpdir } from "node:os" import path from "node:path" -import { DocumentStore, knowledgeSimilarity, tokenizeForSimilarity } from "../../src/deepagent/document-store" +import { DocumentStore, DocumentConflictError, knowledgeSimilarity, tokenizeForSimilarity } from "../../src/deepagent/document-store" +import { writeFileAtomic, writeFileExclusive } from "../../src/deepagent/atomic-write" import { DurableKnowledgeStore } from "../../src/deepagent/durable-knowledge-store" let root: string @@ -168,6 +169,141 @@ describe("V3 DocumentStore", () => { }) }) +// F30-1 (deepagentcore-v4.0.3 storage prereq): DocumentStore is the single crash-safe, concurrency- +// safe durable body. persist() writes append-only version files with exclusive-create CAS; replace() +// overwrites the same version atomically (temp+fsync+rename). These tests pin the CAS + durability +// behavior that H32-1 (v4.0.4) builds on. +describe("F30-1 DocumentStore CAS + atomic durability", () => { + test("normal single-writer flow is unchanged (create + updates land byte-identically)", () => { + const a = store.create(design("v1")) + const a2 = store.update(a.id, "v2") + const a3 = store.update(a.id, "v3") + expect(a3.version).toBe(3) + // reopened from files only — every version file was written exactly once via exclusive create + const reopened = new DocumentStore(root) + expect(reopened.get(a.id)!.body).toBe("v3") + expect(reopened.get(a.id, 1)!.status).toBe("superseded") + expect(reopened.get(a.id, 2)!.status).toBe("superseded") + expect(reopened.verify().ok).toBe(true) + }) + + test("two handles writing DIFFERENT content at the same version -> DocumentConflictError", () => { + // Both handles are built over the same on-disk root but hold independent in-memory indexes, so + // both compute the SAME next version (v2) from the SAME base (v1) — the lost-update race F30-1 + // must catch instead of silently clobbering. + const a = store.create(design("base")) + const h1 = new DocumentStore(root) + const h2 = new DocumentStore(root) + h1.update(a.id, "from h1") // writes id@v2 (h1's content) + // h2 still thinks latest is v1; its update also targets v2 but with different content -> conflict + expect(() => h2.update(a.id, "from h2")).toThrow(DocumentConflictError) + // h1's write survives intact; nothing was clobbered + expect(new DocumentStore(root).get(a.id)!.body).toBe("from h1") + }) + + test("idempotent re-persist of byte-identical version is a no-op (same-hash CAS collision)", () => { + // Two handles writing the SAME content at the same version must NOT conflict — a retried/mirrored + // write of identical bytes is idempotent (content-addressed: same hash). + const a = store.create(design("base")) + const h1 = new DocumentStore(root) + const h2 = new DocumentStore(root) + const w1 = h1.update(a.id, "same body") + const w2 = h2.update(a.id, "same body") // same version, same content -> idempotent, no throw + expect(w1.version).toBe(2) + expect(w2.version).toBe(2) + expect(w1.hash).toBe(w2.hash) + expect(new DocumentStore(root).get(a.id)!.body).toBe("same body") + }) + + test("setStatus/replace rewrites the same version file in place without a version bump", () => { + const a = store.create(design("body")) + expect(a.version).toBe(1) + store.setStatus(a.id, "active") + expect(store.get(a.id)!.status).toBe("active") + expect(store.get(a.id)!.version).toBe(1) // in-place rewrite, not a new version + // exactly one version file on disk for this doc (no orphan versions from the rewrite) + const dir = path.join(root, "docs", "design") + const files = readdirSync(dir).filter((f) => f.endsWith(".json")) + expect(files.length).toBe(1) + expect(new DocumentStore(root).get(a.id)!.status).toBe("active") + }) + + test("writeFileExclusive throws EEXIST on an existing path (the CAS primitive)", () => { + const f = path.join(root, "cas-probe.json") + writeFileExclusive(f, "first") + let code: string | undefined + try { + writeFileExclusive(f, "second") + } catch (e) { + code = (e as NodeJS.ErrnoException).code + } + expect(code).toBe("EEXIST") + expect(readFileSync(f, "utf8")).toBe("first") // original never clobbered + }) + + test("writeFileAtomic overwrites durably and leaves no temp files behind", () => { + const f = path.join(root, "atomic-probe.json") + writeFileAtomic(f, "one") + writeFileAtomic(f, "two") + expect(readFileSync(f, "utf8")).toBe("two") + // no leftover .tmp-* siblings from the temp+rename + const leftover = readdirSync(root).filter((n) => n.includes(".tmp-")) + expect(leftover).toEqual([]) + }) +}) + +// F30-1 Part 2: same-process SHARED AUTHORITY. `DocumentStore.shared(root)` handles for the same root +// share ONE in-memory index (a write through one is visible through all); `new DocumentStore(root)` +// stays unshared (own disk-rebuilt index) so it keeps faithfully simulating a cold/second-process open. +describe("F30-1 DocumentStore.shared same-process authority", () => { + afterEach(() => DocumentStore.__resetSharedRegistryForTests()) + + test("two shared handles to the same root see each other's writes without a disk reopen", () => { + const h1 = DocumentStore.shared(root) + const h2 = DocumentStore.shared(root) + const a = h1.create(design("written via h1")) + // h2 shares h1's live index — the doc is visible immediately, no rebuild/reopen needed + expect(h2.get(a.id)?.body).toBe("written via h1") + // and a write through h2 is visible through h1 (bidirectional shared authority) + const a2 = h2.update(a.id, "updated via h2") + expect(a2.version).toBe(2) + expect(h1.get(a.id)?.body).toBe("updated via h2") + expect(h1.get(a.id)?.version).toBe(2) + }) + + test("an UNSHARED handle does NOT see another handle's in-memory write until it rebuilds from disk", () => { + // This is the pre-F30-1 divergence the shared registry fixes — pinned here so the distinction is + // explicit: an unshared handle only reflects writes that were on disk at ITS construction time. + const shared = DocumentStore.shared(root) + const a = shared.create(design("only in shared index at first")) + // A separate shared handle constructed AFTER the write still sees it (shared index)... + expect(DocumentStore.shared(root).get(a.id)?.body).toBe("only in shared index at first") + // ...and an unshared handle also sees it here, because create() persisted it to disk and the + // unshared handle rebuilds from disk in its constructor. (Persistence is synchronous — Part 1.) + expect(new DocumentStore(root).get(a.id)?.body).toBe("only in shared index at first") + }) + + test("shared index is rebuilt from disk on first open of a root (survives a simulated restart)", () => { + // Seed via an unshared handle (writes to disk), then open the FIRST shared handle for the root — + // it must rebuild the shared index from disk so pre-existing docs are visible. + const seed = new DocumentStore(root).create(design("seeded on disk")) + DocumentStore.__resetSharedRegistryForTests() // simulate a fresh process (empty registry) + const shared = DocumentStore.shared(root) + expect(shared.get(seed.id)?.body).toBe("seeded on disk") + }) + + test("shared handles for DIFFERENT roots are isolated", () => { + const otherRoot = mkdtempSync(path.join(tmpdir(), "deepagent-ds-shared-other-")) + try { + const a = DocumentStore.shared(root).create(design("in root")) + const other = DocumentStore.shared(otherRoot) + expect(other.get(a.id)).toBeNull() // different root -> different shared index + } finally { + rmSync(otherRoot, { recursive: true, force: true }) + } + }) +}) + // V3.8 Phase 0: the new NON-knowledge derived-data node/edge types must round-trip through the graph // exactly like any other doc, must NOT trigger the knowledge confidence check, and must obey INV-3 // (single-store links only). No knowledge semantics ride on them. diff --git a/packages/core/test/deepagent/plan-controller.test.ts b/packages/core/test/deepagent/plan-controller.test.ts index b20da5f6..e78b2133 100644 --- a/packages/core/test/deepagent/plan-controller.test.ts +++ b/packages/core/test/deepagent/plan-controller.test.ts @@ -133,16 +133,38 @@ describe("planGate (before_tool_use soft gate)", () => { expect(gate({ name: "before_tool_use", payload: { planStale: true, isMutating: false } }).decision).toBe("allow") }) - test("blocks mutating tools when stale in high+ mode", () => { + // DESIGN (aligned with codex exec_policy): a stale plan ledger NEVER denies tool execution — it + // WARNS (tool runs, reminder attached) so the model is nudged to re-sync without being deadlocked. + // This holds in every mode, including high+ (the old code hard-blocked here, which caused the + // production deadlock). The only remaining hard block is the U9 per-step binding gate, covered below. + test("warns (never blocks) on a mutating tool when stale in high+ mode", () => { const d = gate({ name: "before_tool_use", payload: { planStale: true, isMutating: true, lightweight: false } }) - expect(d.decision).toBe("block") + expect(d.decision).toBe("warn") expect(d.blockReason).toContain("stale") }) - test("only warns (never blocks) in lightweight mode", () => { + test("warns (never blocks) in lightweight mode too", () => { const d = gate({ name: "before_tool_use", payload: { planStale: true, isMutating: true, lightweight: true } }) expect(d.decision).toBe("warn") }) + + // U9 per-step binding remains a hard block (strict hard modes) but gains a runtime grace release so + // it can never permanently deadlock a model that fails to mark a step active. + test("U9 binding: strict hard mode blocks a mutating tool with no active step", () => { + const d = gate({ + name: "before_tool_use", + payload: { planStale: false, isMutating: true, lightweight: false, hardGate: true, hasActiveStep: false, hardGateMissBlocks: true }, + }) + expect(d.decision).toBe("block") + }) + + test("U9 binding: grace release downgrades the strict block to a warn", () => { + const d = gate({ + name: "before_tool_use", + payload: { planStale: false, isMutating: true, lightweight: false, hardGate: true, hasActiveStep: false, hardGateMissBlocks: true, graceRelease: true }, + }) + expect(d.decision).toBe("warn") + }) }) describe("stopHookGate (plan condition added by U1)", () => { diff --git a/packages/core/test/deepagent/plan-gate-loop.test.ts b/packages/core/test/deepagent/plan-gate-loop.test.ts index 4930a2ec..7a540185 100644 --- a/packages/core/test/deepagent/plan-gate-loop.test.ts +++ b/packages/core/test/deepagent/plan-gate-loop.test.ts @@ -6,15 +6,18 @@ import * as SessionState from "../../src/deepagent/session-state" import * as PlanController from "../../src/deepagent/plan-controller" import { planGate, HookPolicy } from "../../src/deepagent/hooks" -// U1 end-to-end contract (the exact decision the tools.ts chokepoint computes): a stale plan -// soft-blocks a mutating tool, read/plan tools pass, and calling the plan tool (setPlan) clears the -// latch so the next mutating tool is allowed again. This mirrors the wiring in session/tools.ts -// without booting the full session loop. +// U1 end-to-end contract (the exact decision the tools.ts chokepoint computes). DESIGN (aligned with +// codex exec_policy): plan-ledger state is orthogonal to whether a tool may run, so a stale plan +// NEVER hard-blocks a mutating tool — it WARNS (tool runs, reminder attached) so the model is nudged +// to re-sync without being denied its tools. Read/plan tools always pass. The only hard block that +// remains is the U9 per-step binding gate (strict hard modes), and even that has a runtime grace +// release so it can never permanently deadlock. This mirrors session/tools.ts without booting the +// full session loop. const gate = new HookPolicy().on("before_tool_use", planGate()) -// Reproduce the chokepoint decision exactly (session/tools.ts), including the U1 anti-deadlock -// payload fields (staleReason, graceRelease) and the shell command classification. +// Reproduce the soft-layer chokepoint decision (session/tools.ts): staleReason + isMutating + +// lightweight. The U9 hard-layer fields are exercised separately in the binding tests below. const decideAt = (sessionId: string, toolId: string, mode: string, command?: string) => { const latch = SessionState.planLatch(sessionId) const planStale = latch?.latch === "stale" && !PlanController.shouldEscapeToHuman(latch) @@ -30,18 +33,12 @@ const decideAt = (sessionId: string, toolId: string, mode: string, command?: str }) } -// Mirror the tools.ts block/reset bookkeeping so the loop tests advance the runtime grace counter -// exactly as production does: a block records a gate block; a mutating tool that runs resets it. -const applyOutcome = (sessionId: string, decision: string) => { - if (decision === "block") SessionState.recordPlanGateBlock(sessionId) -} - describe("U1 soft-gate loop (chokepoint contract)", () => { beforeEach(() => { SessionState.configure(mkdtempSync(path.join(tmpdir(), "plan-gate-"))) }) - test("high mode: stale plan blocks edit, allows read/plan, plan tool unblocks", () => { + test("high mode: stale plan WARNS on edit (never blocks), allows read/plan, plan tool clears warn", () => { SessionState.getOrCreate("gate-s1", "high") // a failing validation flips the latch from runtime truth SessionState.recordValidation( @@ -50,11 +47,12 @@ describe("U1 soft-gate loop (chokepoint contract)", () => { "e", ) - expect(decideAt("gate-s1", "edit", "high").decision).toBe("block") + // A stale plan no longer denies a mutating tool — it warns (the tool still runs). + expect(decideAt("gate-s1", "edit", "high").decision).toBe("warn") expect(decideAt("gate-s1", "read", "high").decision).toBe("allow") expect(decideAt("gate-s1", "plan", "high").decision).toBe("allow") - // model calls the plan tool -> setPlan clears the latch + // model calls the plan tool -> setPlan clears the latch -> no more warn SessionState.setPlan( "gate-s1", PlanController.buildPlanFromInput("gate-s1", { goal: "g", steps: [{ title: "fix", status: "active" }] }), @@ -68,7 +66,7 @@ describe("U1 soft-gate loop (chokepoint contract)", () => { expect(decideAt("gate-s2", "edit", "general").decision).toBe("warn") }) - test("escape hatch: after too many replans the gate stops blocking (routes to human elsewhere)", () => { + test("escape hatch: after too many replans the gate stops warning (routes to human elsewhere)", () => { SessionState.getOrCreate("gate-s3", "high") // exhaust the replan budget: each setPlan bumps replan_count for (let i = 0; i <= PlanController.DEFAULT_REPLAN_LIMIT; i++) { @@ -81,68 +79,94 @@ describe("U1 soft-gate loop (chokepoint contract)", () => { // now mark stale once more; replan_count already exceeds the limit SessionState.markPlanStale("gate-s3", "no_progress") expect(PlanController.shouldEscapeToHuman(SessionState.planLatch("gate-s3")!)).toBe(true) - // gate no longer blocks (escape hatch) — the macro-round stop gate routes to needs_human instead + // gate no longer warns (escape hatch flips planStale false) — the macro-round stop gate routes to + // needs_human instead of looping on plan re-sync. expect(decideAt("gate-s3", "edit", "high").decision).toBe("allow") }) - // ── U1 anti-deadlock: read-only shell commands are never gated ────────────────────────────────── - test("read-only bash (ls/git status/grep) is allowed even while the plan is stale", () => { + // ── read-only shell commands are never gated (the agent's eyes), aligned with codex is_safe_command ── + test("read-only bash (ls/git status/grep/sed -n/sort) is allowed even while the plan is stale", () => { SessionState.getOrCreate("gate-ro", "xhigh") SessionState.markPlanStale("gate-ro", "validation_failed") - // a mutating bash command IS gated… - expect(decideAt("gate-ro", "bash", "xhigh", "rm -rf build").decision).toBe("block") - // …but read-only shell inspections pass (the agent must be able to see to repair the plan). + // a mutating bash command warns (runs + reminder), never a hard block… + expect(decideAt("gate-ro", "bash", "xhigh", "rm -rf build").decision).toBe("warn") + // …and read-only shell inspections pass outright (isMutating=false → allow). expect(decideAt("gate-ro", "bash", "xhigh", "ls -la").decision).toBe("allow") expect(decideAt("gate-ro", "bash", "xhigh", "git status").decision).toBe("allow") expect(decideAt("gate-ro", "bash", "xhigh", "grep -rn TODO src/").decision).toBe("allow") expect(decideAt("gate-ro", "bash", "xhigh", "cat file.txt | head -20").decision).toBe("allow") + // P0: the exact probe form that triggered the production deadlock (sed line-slice) is read-only. + expect(decideAt("gate-ro", "bash", "xhigh", "sed -n '18,45p' shim_utils.h").decision).toBe("allow") + expect(decideAt("gate-ro", "bash", "xhigh", "sort file.txt | uniq -c").decision).toBe("allow") }) - // ── U1 anti-deadlock: runtime-driven grace release (the production-deadlock fix) ──────────────── - test("grace release: after repeated blocks with no re-plan, the gate releases WITHOUT model help", () => { - SessionState.getOrCreate("gate-grace", "xhigh") - SessionState.markPlanStale("gate-grace", "validation_failed") - - // The model keeps trying a mutating bash command and NEVER calls the plan tool (the exact - // production failure mode: 280 consecutive blocked bash calls). Each block advances the runtime - // grace counter — no setPlan required. - for (let i = 0; i < PlanController.DEFAULT_GRACE_BLOCK_LIMIT; i++) { - const decision = decideAt("gate-grace", "bash", "xhigh", "make build").decision - expect(decision).toBe("block") - applyOutcome("gate-grace", decision) - } + // ── a new user message must not hard-block work the user is asking for ────────────────────────── + test("user_appended stale reason warns (tool runs); validation_failed also warns (never blocks)", () => { + SessionState.getOrCreate("gate-user", "xhigh") + SessionState.markPlanStale("gate-user", "user_appended") + expect(decideAt("gate-user", "edit", "xhigh").decision).toBe("warn") + // a genuine reality-change signal also warns now — plan ledger state never denies execution. + SessionState.clearPlanStale("gate-user") + SessionState.markPlanStale("gate-user", "validation_failed") + expect(decideAt("gate-user", "edit", "xhigh").decision).toBe("warn") + }) +}) - // The escape-to-human hatch has NOT fired (replan_count never advanced — the model didn't - // cooperate), proving the old hatch alone would have deadlocked forever here. - expect(PlanController.shouldEscapeToHuman(SessionState.planLatch("gate-grace")!)).toBe(false) +// ── U9 per-step binding: the one remaining hard block, with a runtime grace release ─────────────── +// The binding gate (strict hard modes: xhigh/max/ultra) requires a mutating tool to be bound to an +// active plan step. This is workflow discipline, not a safety property, so it must never permanently +// deadlock: after repeated blocks with no forward progress the runtime grace release downgrades it to +// a warn WITHOUT any model cooperation — the same anti-deadlock invariant as the stale layer. +describe("U9 binding gate + grace release", () => { + const decideBinding = (opts: { hasActiveStep: boolean; hardGateMissBlocks: boolean; graceRelease: boolean }) => + gate.evaluate({ + name: "before_tool_use", + payload: { + planStale: false, + isMutating: true, + lightweight: false, + hardGate: true, + hasActiveStep: opts.hasActiveStep, + hardGateMissBlocks: opts.hardGateMissBlocks, + graceRelease: opts.graceRelease, + }, + }) + + test("strict hard mode blocks a mutating tool with no active step", () => { + expect(decideBinding({ hasActiveStep: false, hardGateMissBlocks: true, graceRelease: false }).decision).toBe( + "block", + ) + }) - // But the runtime grace release fires: the next mutating call is downgraded to a WARN (tool runs, - // reminder attached) instead of a hard block. The agent is never permanently denied its tools. - expect(PlanController.shouldGraceRelease(SessionState.planLatch("gate-grace")!)).toBe(true) - expect(decideAt("gate-grace", "bash", "xhigh", "make build").decision).toBe("warn") + test("lenient hard mode (high) only warns on a missing active step", () => { + expect(decideBinding({ hasActiveStep: false, hardGateMissBlocks: false, graceRelease: false }).decision).toBe( + "warn", + ) }) - test("grace counter resets when forward progress happens (mutating tool executes)", () => { - SessionState.getOrCreate("gate-reset", "xhigh") - SessionState.markPlanStale("gate-reset", "validation_failed") - SessionState.recordPlanGateBlock("gate-reset") - SessionState.recordPlanGateBlock("gate-reset") - expect(SessionState.planLatch("gate-reset")!.consecutive_blocks).toBe(2) - // a mutating tool actually ran → reset - SessionState.resetPlanGateBlocks("gate-reset") - expect(SessionState.planLatch("gate-reset")!.consecutive_blocks).toBe(0) - expect(PlanController.shouldGraceRelease(SessionState.planLatch("gate-reset")!)).toBe(false) + test("an active step passes the binding gate", () => { + expect(decideBinding({ hasActiveStep: true, hardGateMissBlocks: true, graceRelease: false }).decision).toBe( + "allow", + ) }) - // ── U1 anti-deadlock: a new user message must not hard-block work the user is asking for ──────── - test("user_appended stale reason warns (tool runs) rather than hard-blocking, even at xhigh", () => { - SessionState.getOrCreate("gate-user", "xhigh") - SessionState.markPlanStale("gate-user", "user_appended") - // mutating edit is downgraded to warn (runs + reminder), not blocked - expect(decideAt("gate-user", "edit", "xhigh").decision).toBe("warn") - // whereas a genuine reality-change signal still hard-blocks - SessionState.clearPlanStale("gate-user") - SessionState.markPlanStale("gate-user", "validation_failed") - expect(decideAt("gate-user", "edit", "xhigh").decision).toBe("block") + test("grace release downgrades the strict binding block to a warn (never permanently deadlocks)", () => { + // Even in the strictest mode with no active step, once the runtime grace release fires the tool is + // allowed through with a reminder rather than blocked forever. + expect(decideBinding({ hasActiveStep: false, hardGateMissBlocks: true, graceRelease: true }).decision).toBe( + "warn", + ) + }) + + test("grace counter machinery: blocks accumulate, forward progress resets", () => { + SessionState.configure(mkdtempSync(path.join(tmpdir(), "plan-gate-bind-"))) + SessionState.getOrCreate("gate-bind", "xhigh") + SessionState.markPlanStale("gate-bind", "validation_failed") + SessionState.recordPlanGateBlock("gate-bind") + SessionState.recordPlanGateBlock("gate-bind") + expect(SessionState.planLatch("gate-bind")!.consecutive_blocks).toBe(2) + SessionState.resetPlanGateBlocks("gate-bind") + expect(SessionState.planLatch("gate-bind")!.consecutive_blocks).toBe(0) + expect(PlanController.shouldGraceRelease(SessionState.planLatch("gate-bind")!)).toBe(false) }) }) diff --git a/packages/core/test/deepagent/plan-store.test.ts b/packages/core/test/deepagent/plan-store.test.ts new file mode 100644 index 00000000..5ea158a8 --- /dev/null +++ b/packages/core/test/deepagent/plan-store.test.ts @@ -0,0 +1,160 @@ +import { describe, expect, test, beforeEach, afterEach } from "bun:test" +import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import path from "node:path" +import * as PlanStore from "../../src/deepagent/plan-store" +import * as SessionState from "../../src/deepagent/session-state" +import { DocumentStore } from "../../src/deepagent/document-store" +import { createPlanDoc, type PlanDoc, type PlanStep } from "../../src/deepagent/plan-controller" + +// I33-1 (deepagentcore-v4.0.3): the DocumentStore `type:"plan"` doc is the SINGLE structural authority +// for a session's plan. session-state.setPlan/getPlan delegate to plan-store; the goal path writes the +// same doc (same root + slug). These tests pin that single-authority contract + the legacy migration. + +let stateDir: string +const step = (id: string, status: PlanStep["status"] = "pending"): PlanStep => ({ + step_id: id, + title: `step ${id}`, + status, + acceptance: null, + assigned_agent: null, + evidence: [], + note: null, +}) +const plan = (sid: string, steps: PlanStep[]): PlanDoc => createPlanDoc(sid, `goal ${sid}`, steps) + +beforeEach(() => { + stateDir = mkdtempSync(path.join(tmpdir(), "deepagent-planstore-")) + SessionState.configure(stateDir) // also configures plan-store root (I33-1 coupling) +}) +afterEach(() => { + DocumentStore.__resetSharedRegistryForTests() + rmSync(stateDir, { recursive: true, force: true }) +}) + +describe("I33-1 plan-store single authority", () => { + test("setPlanDoc/getPlanDoc round-trip through the DocumentStore", () => { + expect(PlanStore.getPlanDoc("s1")).toBeNull() + const p = plan("s1", [step("step_1"), step("step_2")]) + const ref = PlanStore.setPlanDoc("s1", p) + expect(ref.version).toBe(1) + const read = PlanStore.getPlanDoc("s1") + expect(read?.goal).toBe("goal s1") + expect(read?.steps.map((s) => s.step_id)).toEqual(["step_1", "step_2"]) + }) + + test("round-trips for a REALISTIC session id (slugify mangles the id — resolve by type+scope)", () => { + // Regression: the doc id is allocated via slugify(idSlug) (lowercase, `_`→`-`, truncate 48), so a + // raw `doc:plan:plan-` reconstruction misses for real ids. A production session id has + // underscores + a uuid and exceeds 48 chars — getPlanDoc must still resolve it. + const sid = `ses_planstatus_render_${"a".repeat(20)}_0123456789abcdef` + const p = plan(sid, [step("step_1", "active"), step("step_2")]) + PlanStore.setPlanDoc(sid, p) + const read = PlanStore.getPlanDoc(sid) + expect(read).not.toBeNull() + expect(read?.steps[0].status).toBe("active") + expect(PlanStore.planDocRef(sid)?.version).toBe(1) + }) + + test("identical body is an INV-4 no-op (no version bump); a change appends a version", () => { + const p = plan("s2", [step("step_1")]) + expect(PlanStore.setPlanDoc("s2", p).version).toBe(1) + expect(PlanStore.setPlanDoc("s2", p).version).toBe(1) // unchanged -> no-op + const changed = plan("s2", [step("step_1", "done")]) + expect(PlanStore.setPlanDoc("s2", changed).version).toBe(2) + expect(PlanStore.getPlanDoc("s2")?.steps[0].status).toBe("done") + }) + + test("session-state.setPlan/getPlan delegate to the store (body NOT on session state)", () => { + SessionState.getOrCreate("s3", "high") + const p = plan("s3", [step("step_1")]) + SessionState.setPlan("s3", p) + // readable via session-state (delegates to plan-store) AND directly from plan-store (same doc) + expect(SessionState.getPlan("s3")?.goal).toBe("goal s3") + expect(PlanStore.getPlanDoc("s3")?.goal).toBe("goal s3") + // the latch pointer is bound to the plan id (the hot-path value object that STAYS on session state) + expect(SessionState.planLatch("s3")?.plan_id).toBe(p.plan_id) + // the persisted sessions.json must NOT carry the plan body (authority moved to the store) + const raw = require("node:fs").readFileSync(path.join(stateDir, "sessions.json"), "utf8") + expect(JSON.parse(raw)["s3"].plan).toBeUndefined() + }) + + test("plan-store root equals the goal store root (tool path and goal path converge on one doc)", () => { + // goal-manager.goalStoreRoot(sid) === /goal//graph — plan-store must match exactly, + // so materializePlanDoc (goal) and setPlanDoc (tool) upsert the SAME doc id. + expect(PlanStore.planStoreRoot("s4")).toBe(path.join(stateDir, "goal", "s4", "graph")) + }) + + test("goal path and tool path converge on ONE doc (realistic id; no plan--2 split)", () => { + // Regression for the P0 the adversarial review found: upsert() dedups on `description`, so the goal + // path (goal-driver.materializePlanDoc) and the tool path (setPlanDoc) MUST use the identical doc + // identity — type/scope/idSlug AND description — or upsert creates a SECOND doc (plan--2) and + // getPlan (list[0]) returns whichever is first, re-splitting the store. This replicates the goal + // path's write FAITHFULLY using the shared identity helpers (planDescription/planScope) — the same + // ones goal-driver now imports — with a realistic session id (the earlier test masked the bug by + // hand-rolling "session plan s5" and using a trivial id). + const sid = `ses_goal_conv_${"z".repeat(24)}_beef` + const goalRoot = path.join(stateDir, "goal", sid, "graph") + const goalHandle = DocumentStore.shared(goalRoot) + // Goal-path write (must match plan-store identity exactly): + goalHandle.upsert({ + type: "plan", + scope: PlanStore.planScope(sid), + description: PlanStore.planDescription(sid), + idSlug: `plan-${sid}`, + body: JSON.stringify(plan(sid, [step("step_1", "active")])), + provenance: { source: "model", run_ref: PlanStore.planScope(sid) }, + }) + expect(PlanStore.getPlanDoc(sid)?.steps[0].status).toBe("active") + // Tool-path write lands on the SAME doc (new version), NOT a second doc. + SessionState.getOrCreate(sid, "high") + SessionState.setPlan(sid, plan(sid, [step("step_1", "done")])) + expect(PlanStore.getPlanDoc(sid)?.steps[0].status).toBe("done") + // Exactly ONE plan doc exists under this session's root (no plan--2 split). + const planDocs = DocumentStore.shared(goalRoot).list({ type: "plan", scope: PlanStore.planScope(sid) }) + expect(planDocs.length).toBe(1) + }) + + test("legacy inline plan on sessions.json is migrated into the store on load", () => { + // Write a pre-I33-1 sessions.json that still carries the structural plan body inline. + const legacyPlan = plan("s6", [step("step_1"), step("step_2", "done")]) + const legacy = { + s6: { + sessionId: "s6", + mode: "high", + completedAt: null, + planLatch: { plan_id: legacyPlan.plan_id, latch: "fresh", stale_reason: null, replan_count: 0, consecutive_blocks: 0 }, + plan: legacyPlan, // the legacy inline body + }, + } + mkdirSync(stateDir, { recursive: true }) + writeFileSync(path.join(stateDir, "sessions.json"), JSON.stringify(legacy)) + DocumentStore.__resetSharedRegistryForTests() + // Re-configure to trigger loadFromDisk (which runs the migration). + SessionState.configure(stateDir) + // The plan body is now readable from the store authority... + expect(PlanStore.getPlanDoc("s6")?.steps.map((s) => s.status)).toEqual(["pending", "done"]) + // ...and getPlan (delegating to the store) returns it. + expect(SessionState.getPlan("s6")?.plan_id).toBe(legacyPlan.plan_id) + }) + + test("migration does not clobber a newer store doc (e.g. a goal edit) with a stale inline body", () => { + // Seed a NEWER plan into the store first (as a goal edit would), THEN load a legacy sessions.json + // whose inline body is older. The migration must NOT overwrite the newer store doc. + const newer = plan("s7", [step("step_1", "done")]) + PlanStore.setPlanDoc("s7", newer) + const legacyOlder = plan("s7", [step("step_1", "pending")]) + const legacy = { + s7: { + sessionId: "s7", + mode: "high", + completedAt: null, + planLatch: { plan_id: legacyOlder.plan_id, latch: "fresh", stale_reason: null, replan_count: 0, consecutive_blocks: 0 }, + plan: legacyOlder, + }, + } + writeFileSync(path.join(stateDir, "sessions.json"), JSON.stringify(legacy)) + SessionState.configure(stateDir) // load; migration sees the store already has a plan -> skips + expect(PlanStore.getPlanDoc("s7")?.steps[0].status).toBe("done") // newer store doc preserved + }) +}) diff --git a/packages/core/test/effect/app-node.test.ts b/packages/core/test/effect/app-node.test.ts new file mode 100644 index 00000000..73f89b71 --- /dev/null +++ b/packages/core/test/effect/app-node.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, test } from "bun:test" +import { Context, Effect, Layer } from "effect" +import { makeGlobalNode, makeLocationNode, tags } from "../../src/effect/app-node" +import { LayerNode } from "../../src/effect/layer-node" + +// Wave A0 (deepagentcore-v4.0.3 阶段 A): the AppNode foundation (layer-node + app-node, ported +// byte-for-byte from upstream opencode) must compile and produce working layers, and must COEXIST +// with our existing Layer.effect paradigm (no forced migration). This smoke test pins that contract +// before any subsystem migrates onto makeGlobalNode/makeLocationNode in later waves. It is NOT a +// variant of upstream — upstream exercises the foundation only through subsystem tests (process, +// system-context) that arrive in A1+; this asserts the foundation itself in isolation. + +class Greeter extends Context.Service string }>()("test/Greeter") {} +class Loud extends Context.Service string }>()("test/Loud") {} + +describe("Wave A0 — AppNode foundation", () => { + test("makeGlobalNode builds a compilable node whose service resolves", async () => { + const node = makeGlobalNode({ + service: Greeter, + layer: Layer.succeed(Greeter, { hello: () => "hi" }), + deps: [], + }) + const layer = LayerNode.compile(node) + const out = await Effect.runPromise( + Effect.gen(function* () { + const g = yield* Greeter + return g.hello() + }).pipe(Effect.provide(layer)), + ) + expect(out).toBe("hi") + }) + + test("a node's declared deps are provided when compiled (dependency graph wiring)", async () => { + const greeter = makeGlobalNode({ + service: Greeter, + layer: Layer.succeed(Greeter, { hello: () => "hi" }), + deps: [], + }) + const loud = makeGlobalNode({ + service: Loud, + // Loud depends on Greeter — the node graph must provide it at compile time. + layer: Layer.effect( + Loud, + Effect.gen(function* () { + const g = yield* Greeter + return { shout: () => g.hello().toUpperCase() } + }), + ), + deps: [greeter], + }) + const out = await Effect.runPromise( + Effect.gen(function* () { + const l = yield* Loud + return l.shout() + }).pipe(Effect.provide(LayerNode.compile(loud))), + ) + expect(out).toBe("HI") + }) + + test("makeLocationNode tags a node in the location scope (scope encoded on the node)", () => { + const node = makeLocationNode({ + service: Greeter, + layer: Layer.succeed(Greeter, { hello: () => "loc" }), + deps: [], + }) + // The location tag is carried on the node (this is what encodes global vs per-location scope). + expect(node.tag).toBe(tags.values.location) + expect(node.name).toBe("test/Greeter") + }) + + test("coexists with the existing Layer paradigm — a node layer composes with a plain Layer", async () => { + // A node-compiled layer and a hand-written Layer.effect must interoperate (the A0 coexistence gate: + // AppNode does not require migrating everything at once). + const nodeLayer = LayerNode.compile( + makeGlobalNode({ service: Greeter, layer: Layer.succeed(Greeter, { hello: () => "node" }), deps: [] }), + ) + const plainLayer = Layer.effect( + Loud, + Effect.gen(function* () { + const g = yield* Greeter + return { shout: () => g.hello() + "!" } + }), + ).pipe(Layer.provide(nodeLayer)) + const out = await Effect.runPromise( + Effect.gen(function* () { + return (yield* Loud).shout() + }).pipe(Effect.provide(plainLayer)), + ) + expect(out).toBe("node!") + }) + + test("compile throws on an unbound node (a required dependency was never bound)", () => { + // An unbound node has no implementation; compiling it directly must throw rather than silently + // producing a layer missing its service. Use the real global tag so the node is well-typed. + const unboundNode = LayerNode.unbound(Greeter, tags.values.global) + expect(() => LayerNode.compile(unboundNode as never)).toThrow() + }) +}) diff --git a/packages/deepagent-code/src/session/goal-driver.ts b/packages/deepagent-code/src/session/goal-driver.ts index afe12c8d..1db310f0 100644 --- a/packages/deepagent-code/src/session/goal-driver.ts +++ b/packages/deepagent-code/src/session/goal-driver.ts @@ -52,10 +52,16 @@ const planScope = (sessionId: string): string => `run:${sessionId}` * Returns the plan doc id the GoalSpec references. */ export const materializePlanDoc = (input: MaterializePlanInput): string => { + // I33-1: the goal path and the `plan` tool MUST write ONE doc. upsert() dedups on `description` + // (findLogical), so any description drift between the two callers splits them into two docs + // (plan- vs plan--2) and reintroduces the two-store divergence. Use the SAME identity as + // plan-store.setPlanDoc — same type/scope/idSlug AND description — so the goal write lands on the + // tool-path doc (and vice versa). `input.store` is the shared handle at goalStoreRoot(sid), which is + // byte-identical to plan-store's planStoreRoot(sid), so both resolve the same on-disk + in-memory doc. const doc = input.store.upsert({ type: "plan", scope: planScope(input.sessionId), - description: `goal plan ${input.sessionId}`, + description: `session plan ${input.sessionId}`, idSlug: `plan-${input.sessionId}`, body: JSON.stringify(input.plan), provenance: { source: "model", run_ref: planScope(input.sessionId) }, diff --git a/packages/deepagent-code/src/session/goal-manager.ts b/packages/deepagent-code/src/session/goal-manager.ts index 244c2ebd..38a1072f 100644 --- a/packages/deepagent-code/src/session/goal-manager.ts +++ b/packages/deepagent-code/src/session/goal-manager.ts @@ -395,7 +395,10 @@ export const layer = Layer.effect( }, ]) : (fromFile?.plan ?? null)) - const store = new DocumentStore(goalStoreRoot(sessionID)) + // I33-1: SHARED handle so the goal's plan doc and the `plan` tool's writes (core plan-store, + // same root + slug) share ONE in-memory index — a goal-side plan edit is immediately visible to + // the tool path and vice versa, structurally preventing the two-store divergence. + const store = DocumentStore.shared(goalStoreRoot(sessionID)) const planDocId = plan != null ? GoalDriver.materializePlanDoc({ store, sessionId: sessionID, plan }) @@ -585,9 +588,10 @@ export const layer = Layer.effect( if (!c.paused) return true yield* mutateControl(sessionID, (ctrl) => (ctrl.paused = false)) AgentGateway.DeepAgentSessionState.setActiveGoalPhase(sessionID, "running") - // Re-drive: the persisted run_context doc resumes exactly where it paused. A fresh store handle - // over the same root re-reads the loop state. - const store = new DocumentStore(goalStoreRoot(sessionID)) + // Re-drive: the persisted run_context doc resumes exactly where it paused. I33-1: SHARED handle + // (same shared index as the tool path + the first driver) so the resumed loop's plan reads/writes + // stay coherent with any concurrent `plan` tool write on the same session. + const store = DocumentStore.shared(goalStoreRoot(sessionID)) // V4.1 §N DUAL-PATH resume. flag ON ⇒ RE-SEED the event chain rather than starting a BackgroundJob. // The paused command consumed the current normal cursor key without advancing state. Resume keeps diff --git a/packages/deepagent-code/src/worktree/index.ts b/packages/deepagent-code/src/worktree/index.ts index 15666988..bed6ed95 100644 --- a/packages/deepagent-code/src/worktree/index.ts +++ b/packages/deepagent-code/src/worktree/index.ts @@ -216,6 +216,28 @@ export class Service extends Context.Service()("@deepagent-c type GitResult = { code: number; text: string; stderr: string } +// I33-5: build the argument list for a HARDENED read-only git invocation inside a possibly +// attacker-controlled worktree. Neutralizes the content-driven code paths git otherwise runs on +// ordinary reads (each empirically verified against a hostile repo): +// - `-c core.hooksPath=/dev/null` → no hook execution (any subcommand) +// - `diff --no-ext-diff` → ignore any `[diff "x"] external=` driver (use git's built-in diff) +// - `diff --no-textconv` → ignore any `[diff "x"] textconv=` filter +// `--no-ext-diff` / `--no-textconv` are DIFF-SUBCOMMAND flags (they must follow `diff`, and status/ +// rev-list reject them), so they are appended only for the diff subcommand. NOTE: an empty +// `-c diff.external=` does NOT work — git then tries to execute the empty string and dies with +// "cannot run"; `--no-ext-diff` is the correct neutralizer. +// Clean/smudge/process filters (`[filter "x"] clean=/process=`) are keyed by ATTACKER-CHOSEN names and +// cannot be disabled by name; the mitigation is behavioral — safeGit only runs read ops (status +// --porcelain / diff --numstat / rev-list) that never add or canonicalize worktree content, so clean +// filters are never invoked. safeGit must NEVER be used for `git add`/checkout. Exported pure so the +// hardening is unit-testable without the full worktree layer. +export const hardenedGitArgs = (args: readonly string[]): string[] => { + const hooks = ["-c", "core.hooksPath=/dev/null"] + return args[0] === "diff" + ? [...hooks, "diff", "--no-ext-diff", "--no-textconv", ...args.slice(1)] + : [...hooks, ...args] +} + export const layer: Layer.Layer< Service, never, @@ -697,10 +719,11 @@ export const layer: Layer.Layer< return true }) - // U3: read-only git for informational reads inside a worktree. Adds codex's hooksPath override - // so merely rendering a diff / counting changes can never execute a malicious repo's hooks. The - // base Git.Service already disables fsmonitor + optional locks; mutating ops keep normal hooks. - const safeGit = (args: string[], cwd: string) => git(["-c", "core.hooksPath=/dev/null", ...args], { cwd }) + // U3 / I33-5: read-only git for informational reads inside a possibly attacker-controlled worktree. + // Hardening (hooks + external-diff + textconv neutralized) lives in the exported pure + // `hardenedGitArgs` so it is unit-testable; see its comment for the threat model. safeGit must only + // ever run read ops (status/diff/rev-list), never `git add`/checkout. + const safeGit = (args: string[], cwd: string) => git(hardenedGitArgs(args), { cwd }) // U3: locate a worktree entry by directory, the shared lookup the new ops need. const resolveEntry = Effect.fnUntraced(function* (directory: string) { diff --git a/packages/deepagent-code/test/session/goal-loop-wiring.test.ts b/packages/deepagent-code/test/session/goal-loop-wiring.test.ts index 7af59ae5..ef0225be 100644 --- a/packages/deepagent-code/test/session/goal-loop-wiring.test.ts +++ b/packages/deepagent-code/test/session/goal-loop-wiring.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test, afterEach } from "bun:test" +import { describe, expect, test, beforeEach, afterEach } from "bun:test" import { Effect } from "effect" import type { Diagnostic } from "../../src/lsp/client" import { @@ -319,6 +319,12 @@ describe("V3.9 §D wiring — buildStepExecutor", () => { describe("V3.9 §E F3 wiring — plan bridge (worker plan edits reach the goal plan doc)", () => { const roots: string[] = [] + // I33-1: the structural plan now lives in the DocumentStore authority reached via session-state's + // configured root (plan-store). Configure a fresh state dir per case so the child session's + // getPlan/setPlan (used by seedChildPlan/mirrorChildPlan) has a plan-store root. + beforeEach(() => { + AgentGateway.DeepAgentSessionState.configure(mkdtempSync(path.join(tmpdir(), "deepagent-f3-state-"))) + }) const step = (id: string, status: PlanStep["status"]): PlanStep => ({ step_id: id, title: id, diff --git a/packages/deepagent-code/test/worktree/safe-git-hardening.test.ts b/packages/deepagent-code/test/worktree/safe-git-hardening.test.ts new file mode 100644 index 00000000..7ac11e6c --- /dev/null +++ b/packages/deepagent-code/test/worktree/safe-git-hardening.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, test, beforeEach, afterEach } from "bun:test" +import { mkdtempSync, rmSync, existsSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import path from "node:path" +import { execFileSync } from "node:child_process" +import { hardenedGitArgs } from "../../src/worktree/index" + +// I33-5: safeGit runs read-only git (diff/status/rev-list) inside a worktree that may check out an +// ATTACKER-CONTROLLED repo. git has content-driven code paths that execute on ordinary reads — +// external diff drivers, textconv filters, hooks — so a mere `git diff` must not run repo-configured +// commands. `hardenedGitArgs` prepends the config that neutralizes them. These tests pin (1) the exact +// argument shape and (2) an END-TO-END proof against a real hostile repo that the drivers do NOT fire. + +describe("I33-5 hardenedGitArgs — argument shape", () => { + test("non-diff subcommand: only the hooksPath neutralizer is prepended", () => { + expect(hardenedGitArgs(["status", "--porcelain"])).toEqual([ + "-c", + "core.hooksPath=/dev/null", + "status", + "--porcelain", + ]) + }) + + test("diff subcommand gets --no-ext-diff + --no-textconv, inserted right after `diff`", () => { + expect(hardenedGitArgs(["diff", "--numstat", "a..HEAD"])).toEqual([ + "-c", + "core.hooksPath=/dev/null", + "diff", + "--no-ext-diff", + "--no-textconv", + "--numstat", + "a..HEAD", + ]) + }) + + test("non-diff subcommands do NOT get diff-only flags (rev-list/status would reject them)", () => { + const revlist = hardenedGitArgs(["rev-list", "--count", "a..HEAD"]) + expect(revlist).not.toContain("--no-textconv") + expect(revlist).not.toContain("--no-ext-diff") + }) +}) + +// End-to-end: build a repo whose gitattributes wires malicious diff drivers, then confirm plain git +// executes them (the vulnerability) while hardenedGitArgs blocks them (the fix). Skips if git absent. +describe("I33-5 hardenedGitArgs — end-to-end against a hostile repo", () => { + let repo: string + let sentinelDir: string + const git = (args: string[]) => execFileSync("git", args, { cwd: repo, stdio: "ignore" }) + const pwned = (name: string) => existsSync(path.join(sentinelDir, name)) + + beforeEach(() => { + repo = mkdtempSync(path.join(tmpdir(), "i335-repo-")) + sentinelDir = mkdtempSync(path.join(tmpdir(), "i335-sentinel-")) + git(["init", "-q"]) + git(["config", "user.email", "t@t.t"]) + git(["config", "user.name", "t"]) + // Malicious external diff driver + textconv, wired to a file via .gitattributes. + git(["config", "diff.evil.external", `/bin/sh -c "touch ${path.join(sentinelDir, "PWNED_EXTERNAL")}"`]) + git(["config", "diff.evil.textconv", `/bin/sh -c "touch ${path.join(sentinelDir, "PWNED_TEXTCONV")}" <`]) + writeFileSync(path.join(repo, ".gitattributes"), "f.txt diff=evil\n") + writeFileSync(path.join(repo, "f.txt"), "v1\n") + git(["add", "f.txt", ".gitattributes"]) + git(["commit", "-qm", "init"]) + writeFileSync(path.join(repo, "f.txt"), "v2\n") // a change so `git diff` has work to do + }) + afterEach(() => { + rmSync(repo, { recursive: true, force: true }) + rmSync(sentinelDir, { recursive: true, force: true }) + }) + + // git may exit non-zero (e.g. the malicious external driver dies); we only care whether the sentinel + // was created, so run non-throwing. The real safeGit goes through base git() which catches exit codes. + const runGit = (args: string[]) => { + try { + execFileSync("git", args, { cwd: repo, stdio: "ignore" }) + } catch { + /* exit code irrelevant to this test — the sentinel is the signal */ + } + } + + test("baseline: plain `git diff` EXECUTES the malicious drivers (proves the vector is real)", () => { + runGit(["diff"]) + expect(pwned("PWNED_EXTERNAL") || pwned("PWNED_TEXTCONV")).toBe(true) + }) + + test("hardened: `git ` does NOT execute any malicious driver", () => { + runGit(hardenedGitArgs(["diff"])) + expect(pwned("PWNED_EXTERNAL")).toBe(false) + expect(pwned("PWNED_TEXTCONV")).toBe(false) + }) + + test("hardened: `diff --numstat` (the real safeGit call) also stays clean + exits cleanly", () => { + runGit(hardenedGitArgs(["diff", "--numstat"])) + expect(pwned("PWNED_EXTERNAL")).toBe(false) + expect(pwned("PWNED_TEXTCONV")).toBe(false) + }) +})