From 024d3646a24f5f81bdb982b5803357e9a293f4af Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Fri, 17 Jul 2026 20:18:38 +0800 Subject: [PATCH] Core v4.0.3 (#71) ### Issue for this PR Closes # ### Type of change - [ ] Bug fix - [ ] New feature - [ ] Refactor / code improvement - [ ] Documentation ### What does this PR do? Please provide a description of the issue, the changes you made to fix it, and why they work. It is expected that you understand why your changes work and if you do not understand why at least say as much so a maintainer knows how much to value the PR. **If you paste a large clearly AI generated description here your PR may be IGNORED or CLOSED!** ### How did you verify your code works? ### Screenshots / recordings _If this is a UI change, please include a screenshot or recording._ ### Checklist - [ ] I have tested my changes locally - [ ] I have not included unrelated changes in this PR _If you do not follow this template your PR will be automatically rejected._ --------- Co-authored-by: Claude Opus 4.8 (1M context) --- packages/app/package.json | 2 +- packages/core/src/agent-gateway.ts | 7 + packages/core/src/agent.ts | 3 + packages/core/src/command.ts | 3 + packages/core/src/config.ts | 7 + packages/core/src/cross-spawn-spawner.ts | 4 + packages/core/src/database/database.ts | 3 + packages/core/src/deepagent/atomic-write.ts | 72 +++- packages/core/src/deepagent/command-intent.ts | 63 +++- packages/core/src/deepagent/document-store.ts | 123 ++++++- packages/core/src/deepagent/hooks.ts | 49 +-- packages/core/src/deepagent/index.ts | 1 + packages/core/src/deepagent/plan-store.ts | 98 ++++++ packages/core/src/deepagent/session-state.ts | 51 ++- packages/core/src/effect/app-node-platform.ts | 18 + packages/core/src/effect/app-node.ts | 14 + packages/core/src/effect/layer-node.ts | 333 ++++++++++++++++++ packages/core/src/event.ts | 3 + packages/core/src/file-mutation.ts | 3 + packages/core/src/filesystem/watcher.ts | 7 + packages/core/src/fs-util.ts | 4 + packages/core/src/git.ts | 3 + packages/core/src/global.ts | 3 + packages/core/src/image.ts | 3 + packages/core/src/instruction-context.ts | 7 + packages/core/src/location-mutation.ts | 7 + packages/core/src/location.ts | 4 + packages/core/src/models-dev.ts | 4 + packages/core/src/npm.ts | 8 + packages/core/src/permission.ts | 7 + packages/core/src/permission/saved.ts | 3 + packages/core/src/policy.ts | 3 + packages/core/src/process.ts | 3 + packages/core/src/pty.ts | 3 + packages/core/src/question.ts | 3 + packages/core/src/repository-cache.ts | 7 + packages/core/src/session/projector.ts | 3 + packages/core/src/session/store.ts | 3 + packages/core/src/session/todo.ts | 3 + packages/core/src/skill.ts | 3 + packages/core/src/skill/discovery.ts | 4 + packages/core/src/system-context/builtins.ts | 9 + packages/core/src/system-context/registry.ts | 3 + packages/core/src/tool-output-store.ts | 3 + packages/core/src/tool/application-tools.ts | 3 + packages/core/src/tool/apply-patch.ts | 8 + packages/core/src/tool/bash.ts | 8 + packages/core/src/tool/edit.ts | 8 + packages/core/src/tool/question.ts | 8 + packages/core/src/tool/registry.ts | 13 + packages/core/src/tool/todowrite.ts | 8 + packages/core/src/tool/webfetch.ts | 9 + packages/core/src/tool/websearch.ts | 11 + packages/core/src/util/effect-flock.ts | 3 + .../test/deepagent/command-intent.test.ts | 51 +++ .../test/deepagent/document-store.test.ts | 140 +++++++- .../test/deepagent/plan-controller.test.ts | 28 +- .../test/deepagent/plan-gate-loop.test.ts | 148 ++++---- .../core/test/deepagent/plan-store.test.ts | 160 +++++++++ packages/core/test/effect/app-node.test.ts | 98 ++++++ .../deepagent-code/src/session/goal-driver.ts | 8 +- .../src/session/goal-manager.ts | 12 +- packages/deepagent-code/src/worktree/index.ts | 31 +- .../test/session/goal-loop-wiring.test.ts | 8 +- .../test/worktree/safe-git-hardening.test.ts | 97 +++++ 65 files changed, 1705 insertions(+), 122 deletions(-) create mode 100644 packages/core/src/deepagent/plan-store.ts create mode 100644 packages/core/src/effect/app-node-platform.ts create mode 100644 packages/core/src/effect/app-node.ts create mode 100644 packages/core/src/effect/layer-node.ts create mode 100644 packages/core/test/deepagent/plan-store.test.ts create mode 100644 packages/core/test/effect/app-node.test.ts create mode 100644 packages/deepagent-code/test/worktree/safe-git-hardening.test.ts diff --git a/packages/app/package.json b/packages/app/package.json index 466955fe..625b3083 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,6 +1,6 @@ { "name": "@deepagent-code/app", - "version": "1.4.1", + "version": "1.4.2", "description": "", "type": "module", "exports": { diff --git a/packages/core/src/agent-gateway.ts b/packages/core/src/agent-gateway.ts index 1b938a31..562458c3 100644 --- a/packages/core/src/agent-gateway.ts +++ b/packages/core/src/agent-gateway.ts @@ -268,6 +268,11 @@ export const configure = (config: Config = {}) => { // knowledge/state diverge from project-memory whenever runsDir pointed outside /runs. const baseDir = config.baseDir ?? resolveDeepAgentCodeHome() DeepAgentSessionState.configure(path.join(baseDir, "state")) + // I33-1: the structural plan lives in the single DocumentStore under /state/goal//graph + // (co-located with the goal/run graph so the `plan` tool and the goal path write the SAME doc). Root + // it from the SAME baseDir/state session-state uses, so the two paths converge (planStoreRoot ≡ + // goalStoreRoot). Must be set before any plan read/write. + DeepAgentPlanStore.configureRoot(path.join(baseDir, "state")) // docs/34 §7.2/§8: durable knowledge stores root under baseDir (public/knowledge + project// // knowledge). The retriever's knowledge-source reads from here. Same injected baseDir, no self-resolve. DeepAgentKnowledgeSource.configure(baseDir) @@ -386,6 +391,7 @@ import { import * as DeepAgentOrchestrator from "./deepagent/orchestrator" import * as DeepAgentSessionState from "./deepagent/session-state" import * as DeepAgentPlanController from "./deepagent/plan-controller" +import * as DeepAgentPlanStore from "./deepagent/plan-store" import * as DeepAgentDiagnosis from "./deepagent/diagnosis" import * as DeepAgentFailureTriage from "./deepagent/failure-triage" import * as DeepAgentValidation from "./deepagent/validation" @@ -414,6 +420,7 @@ export { DeepAgentOrchestrator, DeepAgentSessionState, DeepAgentPlanController, + DeepAgentPlanStore, DeepAgentDiagnosis, DeepAgentFailureTriage, DeepAgentValidation, diff --git a/packages/core/src/agent.ts b/packages/core/src/agent.ts index 7437c898..7e7ad22c 100644 --- a/packages/core/src/agent.ts +++ b/packages/core/src/agent.ts @@ -2,6 +2,7 @@ export * as AgentV2 from "./agent" import { Array, Context, Effect, Layer, Schema, Scope } from "effect" import { castDraft, enableMapSet, type Draft } from "immer" +import { makeLocationNode } from "./effect/app-node" import { ModelV2 } from "./model" import { PermissionSchema } from "./permission/schema" import { ProviderV2 } from "./provider" @@ -164,4 +165,6 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [] }) + export const locationLayer = layer diff --git a/packages/core/src/command.ts b/packages/core/src/command.ts index f6950482..5d3c541f 100644 --- a/packages/core/src/command.ts +++ b/packages/core/src/command.ts @@ -2,6 +2,7 @@ export * as CommandV2 from "./command" import { Context, Effect, Layer, Schema } from "effect" import { castDraft, type Draft } from "immer" +import { makeLocationNode } from "./effect/app-node" import { ModelV2 } from "./model" import { State } from "./state" @@ -67,4 +68,6 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [] }) + export const locationLayer = layer diff --git a/packages/core/src/config.ts b/packages/core/src/config.ts index 8fc513c3..0f12953a 100644 --- a/packages/core/src/config.ts +++ b/packages/core/src/config.ts @@ -3,6 +3,7 @@ export * as Config from "./config" import path from "path" import { type ParseError, parse } from "jsonc-parser" import { Context, Effect, Layer, Option, Schema } from "effect" +import { makeLocationNode } from "./effect/app-node" import { FSUtil } from "./fs-util" import { Global } from "./global" import { Location } from "./location" @@ -223,4 +224,10 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ + service: Service, + layer, + deps: [FSUtil.node, Global.node, Location.node, Policy.node], +}) + export const locationLayer = layer.pipe(Layer.provideMerge(Policy.locationLayer)) diff --git a/packages/core/src/cross-spawn-spawner.ts b/packages/core/src/cross-spawn-spawner.ts index ad8d4126..5207fc22 100644 --- a/packages/core/src/cross-spawn-spawner.ts +++ b/packages/core/src/cross-spawn-spawner.ts @@ -24,6 +24,8 @@ import { import * as NodeChildProcess from "node:child_process" import { PassThrough } from "node:stream" import launch from "cross-spawn" +import { makeGlobalNode } from "./effect/app-node" +import { filesystem, path } from "./effect/app-node-platform" const toError = (err: unknown): Error => (err instanceof globalThis.Error ? err : new globalThis.Error(String(err))) @@ -502,4 +504,6 @@ export const layer: Layer.Layer @@ -58,3 +59,5 @@ export const defaultLayer = Layer.unwrap( return layerFromPath(path()) }), ).pipe(Layer.provide(Global.defaultLayer)) + +export const node = makeGlobalNode({ service: Service, layer: layerFromPath(path()), deps: [] }) diff --git a/packages/core/src/deepagent/atomic-write.ts b/packages/core/src/deepagent/atomic-write.ts index 9e1e8f87..861bf101 100644 --- a/packages/core/src/deepagent/atomic-write.ts +++ b/packages/core/src/deepagent/atomic-write.ts @@ -1,4 +1,4 @@ -import { mkdirSync, renameSync, rmSync, writeFileSync } from "node:fs" +import { closeSync, fsyncSync, mkdirSync, openSync, renameSync, rmSync, writeSync } from "node:fs" import { randomUUID } from "node:crypto" import path from "node:path" @@ -9,15 +9,83 @@ import path from "node:path" // renameSync is atomic on POSIX and Windows. On any failure the temp file is removed and the error // propagates (never swallow a lost durable write). This mirrors the temp+rename pattern already // used by workspace.ts createProjectAtomically. +// +// F30-1 (deepagentcore-v4.0.3 storage prereq): the temp file is now fsync'd BEFORE the rename so a +// crash between write and rename can never expose a torn body, and the containing directory is +// fsync'd AFTER the rename (best-effort) so the rename itself is durable across a power loss. This +// makes writeFileAtomic the single crash-safe overwrite primitive the DocumentStore relies on. export const writeFileAtomic = (file: string, content: string): void => { const dir = path.dirname(file) mkdirSync(dir, { recursive: true }) const tmp = path.join(dir, `.${path.basename(file)}.tmp-${process.pid}-${randomUUID()}`) + let fd: number | undefined try { - writeFileSync(tmp, content, "utf-8") + fd = openSync(tmp, "w") + writeSync(fd, content, null, "utf-8") + fsyncSync(fd) + closeSync(fd) + fd = undefined renameSync(tmp, file) + fsyncDir(dir) } catch (error) { + if (fd !== undefined) { + try { + closeSync(fd) + } catch { + /* fd already invalid — nothing to salvage */ + } + } rmSync(tmp, { force: true }) throw error } } + +// F30-1: crash-safe EXCLUSIVE create — the CAS write primitive for append-only version files. +// `openSync(file, "wx")` fails with EEXIST if the target already exists, giving an atomic +// compare-and-swap on the filesystem: two processes racing to write the same `id@vN.json` version +// file, one wins and the other observes EEXIST (the caller turns that into a conflict). The body is +// fsync'd before close so a crash can't leave a half-written version file. Throws the raw Node error +// (code "EEXIST" on collision) — the DocumentStore inspects `.code` to decide idempotent-vs-conflict. +export const writeFileExclusive = (file: string, content: string): void => { + const dir = path.dirname(file) + mkdirSync(dir, { recursive: true }) + let fd: number | undefined + try { + fd = openSync(file, "wx") + writeSync(fd, content, null, "utf-8") + fsyncSync(fd) + closeSync(fd) + fd = undefined + fsyncDir(dir) + } catch (error) { + if (fd !== undefined) { + try { + closeSync(fd) + } catch { + /* fd already invalid */ + } + } + throw error + } +} + +// Best-effort directory fsync so a rename/create is durable. Not all platforms/filesystems permit +// opening a directory for fsync (Windows throws EPERM/EISDIR, some FUSE mounts reject it); a failure +// here does not compromise the file body (already fsync'd) so it is intentionally swallowed. +const fsyncDir = (dir: string): void => { + let dfd: number | undefined + try { + dfd = openSync(dir, "r") + fsyncSync(dfd) + } catch { + /* directory fsync unsupported on this platform — file body is already durable */ + } finally { + if (dfd !== undefined) { + try { + closeSync(dfd) + } catch { + /* ignore */ + } + } + } +} diff --git a/packages/core/src/deepagent/command-intent.ts b/packages/core/src/deepagent/command-intent.ts index 1db3b3b6..23d489ce 100644 --- a/packages/core/src/deepagent/command-intent.ts +++ b/packages/core/src/deepagent/command-intent.ts @@ -50,6 +50,24 @@ const READ_ONLY_PREFIXES: ReadonlyArray = [ ["rg"], ["ag"], ["ripgrep"], + // ── text stream inspection (read-only filters; parity with codex is_safe_to_call_with_exec) ── + // These only transform stdin/named files to stdout. `sort`/`sed` have file-writing variants and are + // guarded below; the rest cannot write the filesystem, process table, or environment. + ["cut"], + ["nl"], + ["paste"], + ["rev"], + ["seq"], + ["tr"], + ["uniq"], + ["comm"], + ["fold"], + ["expr"], + ["true"], + ["false"], + ["column"], + ["sort"], // read-only UNLESS it writes with -o/--output (guarded below) + ["sed"], // mutating by default (in MUTATING_COMMANDS); the read-only `sed -n …p` form is allowed via a guard below // ── environment / introspection (query forms only) ── ["which"], ["whereis"], @@ -149,7 +167,10 @@ const MUTATING_COMMANDS = new Set([ "shred", "tee", "install", - "sed", // sed alone is read-only, but `sed -i` mutates; treat as mutating unless we prove otherwise (guarded) + // NOTE: `sed` is intentionally NOT a blanket mutating command. `sed -i` mutates but `sed -n {N|M,N}p` + // is a pure read-only slice (the common "show me lines X..Y" probe). It is matched as a read-only + // prefix and then constrained by a guard in isReadOnlySegment to ONLY the `-n …p` form (parity with + // codex is_safe_to_call_with_exec). Any other sed invocation fails the guard → mutating. "kill", "killall", "pkill", @@ -217,6 +238,23 @@ const matchReadOnlyPrefix = (tokens: string[]): number => { // `find` is read-only unless it carries an action predicate that executes or deletes. const FIND_MUTATING_ACTIONS = new Set(["-delete", "-exec", "-execdir", "-ok", "-okdir", "-fprint", "-fprintf"]) +// `sed -n {N|M,N}p` is a pure read-only line slice. Port of codex is_valid_sed_n_arg: the address +// must match /^(\d+,)?\d+p$/ — one or two numeric line numbers followed by the `p` (print) command, +// nothing else. Any other sed script can substitute/delete/write (`-i`, `w`, `s///w file`), so only +// this exact shape is treated as read-only. +const isValidSedNArg = (arg: string | undefined): boolean => { + if (arg == null) return false + const core = arg.endsWith("p") ? arg.slice(0, -1) : null + if (core == null) return false + const parts = core.split(",") + if (parts.length === 1) return parts[0].length > 0 && /^\d+$/.test(parts[0]) + if (parts.length === 2) return parts[0].length > 0 && parts[1].length > 0 && /^\d+$/.test(parts[0]) && /^\d+$/.test(parts[1]) + return false +} + +// `sort` writes a file with -o/--output; everything else it does is read-only (stdin/files → stdout). +const SORT_WRITE_LONG = /^--output(=|$)/ + // `env` is read-only only as a bare query (`env`); `env FOO=bar cmd` runs a command with a mutated // environment, so anything past the command word makes it mutating. const isReadOnlySegment = (segment: string): boolean => { @@ -257,6 +295,29 @@ const isReadOnlySegment = (segment: string): boolean => { const rest = words.slice(1).filter((token) => !isFlag(token)) if (rest.length > 0) return false } + // `sed` is read-only ONLY as `sed -n {N|M,N}p [file]` (line slice). The prefix matcher accepts a + // bare `sed`, so we must reject every other form here. Require exactly: -n, a valid `…p` address, + // and at most one trailing operand (the file). No -i, no -e, no `w`/`s///w`, no extra scripts. + if (head === "sed") { + const rest = words.slice(1) + const nIdx = rest.indexOf("-n") + if (nIdx === -1) return false + // Only -n may be a flag; any other flag (e.g. -i, -e, -E) makes it non-readonly. + if (rest.some((token, i) => i !== nIdx && isFlag(token))) return false + const nonFlags = rest.filter((token) => !isFlag(token)) + // nonFlags[0] must be the address (…p); an optional nonFlags[1] is the file. More → reject. + if (nonFlags.length < 1 || nonFlags.length > 2) return false + if (!isValidSedNArg(nonFlags[0])) return false + } + // `sort -o file` / `sort --output=file` writes a file. Everything else sort does is read-only. + if (head === "sort") { + for (let i = 1; i < words.length; i++) { + const token = words[i] + if (token === "-o" || SORT_WRITE_LONG.test(token)) return false + // bundled short form `-ofile` (sort's only single-letter value-flag that writes) + if (/^-o./.test(token)) return false + } + } // curl writes a file with -o/-O/--output/--remote-name. A short-flag TOKEN can glue or bundle the // output flag (`-ofile.txt`, `-sofile`), and since curl short flags are single-letter and `o`/`O` // are only ever the output flags, any single-dash bundle containing `o`/`O` writes a file. Match a diff --git a/packages/core/src/deepagent/document-store.ts b/packages/core/src/deepagent/document-store.ts index 4ffc607a..e66eee41 100644 --- a/packages/core/src/deepagent/document-store.ts +++ b/packages/core/src/deepagent/document-store.ts @@ -1,6 +1,7 @@ -import { mkdirSync, readdirSync, readFileSync, writeFileSync, existsSync } from "node:fs" +import { mkdirSync, readdirSync, readFileSync, existsSync } from "node:fs" import { createHash } from "node:crypto" import path from "node:path" +import { writeFileAtomic, writeFileExclusive } from "./atomic-write" // V3 Document System (docs/28): the bedrock. All persistent state is a typed-document // graph — small files, content-addressed, append-only with a supersede chain, bidirectional @@ -194,6 +195,29 @@ export type DocFilter = { export type IntegrityViolation = { readonly invariant: string; readonly docId: string; readonly detail: string } export type IntegrityReport = { readonly ok: boolean; readonly violations: readonly IntegrityViolation[] } +// F30-1 (deepagentcore-v4.0.3 storage prereq): a CAS write conflict. Thrown when persist() tries to +// create an append-only version file (`id@vN.json`) that already exists on disk with a DIFFERENT +// content hash — i.e. another writer (a second handle or a second process) already produced version +// N of this doc from a different base. Append-only + content-addressed means same-hash collisions +// are idempotent no-ops (a retried write); only a hash MISMATCH is a genuine lost-update race, and +// the losing writer must re-read the latest version and re-apply its mutation rather than clobber. +export class DocumentConflictError extends Error { + readonly _tag = "DocumentConflictError" + constructor( + readonly docId: string, + readonly version: number, + readonly existingHash: string, + readonly incomingHash: string, + ) { + super( + `DocumentStore CAS conflict: ${docId}@v${version} already exists with a different body ` + + `(on-disk ${existingHash.slice(0, 16)}… vs incoming ${incomingHash.slice(0, 16)}…). ` + + `Another writer produced this version concurrently; re-read the latest and re-apply.`, + ) + this.name = "DocumentConflictError" + } +} + const toRef = (d: Doc): DocRef => ({ id: d.id, version: d.version, @@ -283,14 +307,63 @@ const strongerEvidence = (a: EvidenceStrength, b: EvidenceStrength): EvidenceStr const idToFile = (id: string): string => id.replace(/:/g, "__") +// F30-1 Part 2 (deepagentcore-v4.0.3 storage prereq): same-process SHARED AUTHORITY. Each +// `new DocumentStore(root)` builds its OWN in-memory index (rebuilt from disk in the constructor), +// so two long-lived handles to the same root in one process do NOT see each other's writes — a +// latent divergence the goal code today routes around with an out-of-band control channel. This +// process-level registry, keyed by the RESOLVED absolute root, lets callers opt into a single shared +// in-memory index via `DocumentStore.shared(root)`: every shared handle for a root reuses the same +// `docs` Map, so a write through one handle is immediately visible through every other. The plain +// constructor is intentionally UNCHANGED (unshared, disk-rebuilt) so it keeps faithfully simulating a +// cold/second-process reconstruction (the shape several recovery tests depend on). Cross-process +// safety is provided by Part 1's exclusive-create CAS + atomic writes, not by this in-memory registry. +const sharedIndexRegistry = new Map>>() + export class DocumentStore { // id -> version -> Doc - private docs = new Map>() - constructor(private readonly root: string) { + private docs: Map> + // `new DocumentStore(root)` stays the UNSHARED, disk-rebuilt handle it always was (all existing + // callers and restart-simulation tests keep working unchanged). Shared handles are obtained via the + // `DocumentStore.shared(root)` factory, which passes shared=true. + constructor( + private readonly root: string, + shared = false, + ) { mkdirSync(path.join(root, "docs"), { recursive: true }) + if (shared) { + const key = path.resolve(root) + let index = sharedIndexRegistry.get(key) + if (!index) { + // First shared handle for this root: build the authoritative shared index from disk once. + index = new Map>() + sharedIndexRegistry.set(key, index) + this.docs = index + this.rebuildIndex() + } else { + // Subsequent shared handles reuse the live shared index (already coherent with prior writes). + this.docs = index + } + return + } + // Unshared (default): own index, rebuilt from disk — byte-identical to the pre-F30-1 behavior. + this.docs = new Map>() this.rebuildIndex() } + // F30-1 Part 2: a SHARED handle over `root` — all shared handles for the same resolved root in this + // process share ONE in-memory index, so writes through any handle are immediately visible through + // the others. Use for coherent same-process authority (e.g. a long-lived driver handle that must see + // an edit written by a request fiber). Cross-process writers are still reconciled by CAS on persist. + static shared(root: string): DocumentStore { + return new DocumentStore(root, true) + } + + // Test-only: drop the shared-index registry so a fresh process is simulated. Not part of the durable + // contract — only used to keep unit tests hermetic when they exercise DocumentStore.shared. + static __resetSharedRegistryForTests(): void { + sharedIndexRegistry.clear() + } + // ---- write ---- create(input: CreateDocInput): Doc { const id = this.allocateId(input.type, input.domain ?? null, input.idSlug, input.description) @@ -526,15 +599,53 @@ export class DocumentStore { } // ---- internals ---- + // F30-1: persist() writes a NEW append-only version file with EXCLUSIVE-create CAS semantics. In + // the normal single-writer flow each `id@vN.json` is written exactly once (create=v1, every + // update/upsert bumps version+1), so the exclusive create never collides and behavior is + // byte-identical to the old bare writeFileSync. A collision (EEXIST) only happens when a second + // handle/process already produced this version: if that on-disk version is byte-identical (same + // content hash) the write is an idempotent no-op (a retried/duplicated write) and we simply adopt + // it into the index; if it differs, it is a genuine lost-update race and we throw + // DocumentConflictError rather than silently clobber. private persist(doc: Doc): void { const dir = path.join(this.root, "docs", doc.type) mkdirSync(dir, { recursive: true }) - writeFileSync(path.join(dir, `${idToFile(doc.id)}@v${doc.version}.json`), JSON.stringify(doc, null, 2)) + const file = path.join(dir, `${idToFile(doc.id)}@v${doc.version}.json`) + const payload = JSON.stringify(doc, null, 2) + try { + writeFileExclusive(file, payload) + } catch (error) { + if ((error as NodeJS.ErrnoException)?.code !== "EEXIST") throw error + // CAS collision: reconcile against what is already on disk. + const existing = this.readVersionFile(file) + if (existing && existing.hash === doc.hash) { + // Idempotent: the same content already landed (retried write / concurrent identical write). + this.indexDoc(doc) + return + } + throw new DocumentConflictError(doc.id, doc.version, existing?.hash ?? "", doc.hash) + } this.indexDoc(doc) } private replace(doc: Doc): void { - // rewrites the same version with new status/superseded_by; rehash so INV-2 holds - this.persist({ ...doc, hash: computeHash({ ...doc, hash: "" }) }) + // rewrites the SAME version in place with new status/superseded_by; rehash so INV-2 holds. This + // is an intentional overwrite (not a new version), so it uses the crash-safe atomic OVERWRITE + // primitive (temp+fsync+rename) rather than the exclusive-create CAS path. + const hashed: Doc = { ...doc, hash: computeHash({ ...doc, hash: "" }) } + const dir = path.join(this.root, "docs", hashed.type) + const file = path.join(dir, `${idToFile(hashed.id)}@v${hashed.version}.json`) + writeFileAtomic(file, JSON.stringify(hashed, null, 2)) + this.indexDoc(hashed) + } + // F30-1: read+parse a single on-disk version file for CAS reconciliation. Returns null if the file + // is missing or corrupt (a torn concurrent write) — the caller treats an unreadable existing file + // as a conflict, never as an idempotent match. + private readVersionFile(file: string): Doc | null { + try { + return JSON.parse(readFileSync(file, "utf8")) as Doc + } catch { + return null + } } private indexDoc(doc: Doc): void { const versions = this.docs.get(doc.id) ?? new Map() diff --git a/packages/core/src/deepagent/hooks.ts b/packages/core/src/deepagent/hooks.ts index f137ea4b..33d970d2 100644 --- a/packages/core/src/deepagent/hooks.ts +++ b/packages/core/src/deepagent/hooks.ts @@ -69,38 +69,45 @@ export const stopHookGate = (): HookHandler => (e) => { // hardGateMissBlocks=true (xhigh/max/ultra) -> block on a missing active step; false (high) -> warn. // The payload carries hardGate, hasActiveStep, hardGateMissBlocks. // -// U1 anti-deadlock (production fix): the soft layer additionally downgrades a would-be block to a -// WARN (tool executes, reminder attached) in two cases, so the gate can never permanently deny a -// model its tools: -// - staleReason === "user_appended": a new user message is a SOFT signal that the plan MIGHT no -// longer match intent — nudge the model to re-align, but do not hard-block work the user is -// actively asking for. -// - graceRelease === true: the gate has already blocked this stale plan N times with no forward -// progress (runtime-driven counter). Rather than loop forever against a model that will not -// repair the plan, release with a strong reminder. This is the direct fix for the observed -// deadlock (280 consecutive blocked bash calls). +// DESIGN (aligned with codex core/src/exec_policy.rs render_decision_for_unmatched_command): command +// safety classification and plan bookkeeping are ORTHOGONAL to whether a tool may run. In codex the +// "is this a known-safe command" check only decides auto-approve-vs-prompt; a command that is NOT +// known-safe is at worst prompted, and is Forbidden ONLY when it is genuinely dangerous AND the user +// disabled prompts. Staleness of the plan ledger is not a safety property, so — like codex — it must +// never REJECT execution. Our previous code coupled the two: a mutating tool on a stale plan was +// hard-blocked in high+ mode, which deadlocked a model that did not repair the plan (observed: 280 +// consecutive blocked bash calls, and a read-only `ssh/docker exec` probe misclassified as mutating +// then denied outright). This gate now downgrades EVERY plan-ledger condition to a WARN (the tool +// runs, a reminder is attached), so plan state can nudge but can never deny the agent its tools. +// +// The two remaining honest signals differ only in wording: +// - staleReason === "user_appended": a new user message MIGHT change intent — nudge to re-align. +// - graceRelease === true: repeated stale blocks with no forward progress (runtime-driven counter). +// Both warn; neither blocks. export const planGate = (): HookHandler => (e) => { if (e.name !== "before_tool_use") return { decision: "continue" } if (e.payload["isMutating"] !== true) return { decision: "allow" } - // U1 soft layer: stale plan. + // U1 soft layer: stale plan → WARN only (never block). Reality changed / a user message arrived is + // a reason to re-sync the plan, not a reason to deny work. if (e.payload["planStale"] === true) { - const reason = "the plan is stale (reality changed); update the plan before further edits" - const graceReason = - "the plan is stale and has not been updated after repeated attempts; this action is allowed to proceed, but you MUST call the `plan` tool to resync your plan with reality now" + const reason = "the plan is stale (reality changed); review it and update the `plan` tool to resync — this action still proceeds" const userAppendedReason = "a new user message arrived; your plan may no longer match the request — review it and update the `plan` tool if the goal changed" - if (e.payload["lightweight"] === true) return { decision: "warn", blockReason: reason } - if (e.payload["graceRelease"] === true) return { decision: "warn", blockReason: graceReason } if (e.payload["staleReason"] === "user_appended") return { decision: "warn", blockReason: userAppendedReason } - return { decision: "block", blockReason: reason } + return { decision: "warn", blockReason: reason } } - // U9 hard layer: per-step binding (high+ only; lightweight never reaches here with hardGate set). + // U9 hard layer: per-step binding (high+ only; lightweight never reaches here with hardGate set). A + // mutating tool under a strict hard gate must be bound to an active step. This is a workflow- + // discipline gate, not a safety gate, so it MUST also have a runtime-driven release: if the gate has + // already blocked this many times with no forward progress (graceRelease), stop blocking and let the + // tool through with a reminder — otherwise a model that never marks a step active would be + // permanently denied its tools (the same deadlock class the stale layer just fixed). if (e.payload["hardGate"] === true && e.payload["hasActiveStep"] !== true) { const reason = "no active plan step is bound to this edit; mark the step you are working on active via the plan tool" - return e.payload["hardGateMissBlocks"] === true - ? { decision: "block", blockReason: reason } - : { decision: "warn", blockReason: reason } + if (e.payload["hardGateMissBlocks"] === true && e.payload["graceRelease"] !== true) + return { decision: "block", blockReason: reason } + return { decision: "warn", blockReason: reason } } return { decision: "allow" } } diff --git a/packages/core/src/deepagent/index.ts b/packages/core/src/deepagent/index.ts index 9d0c6b52..84309d79 100644 --- a/packages/core/src/deepagent/index.ts +++ b/packages/core/src/deepagent/index.ts @@ -9,6 +9,7 @@ export * as DeepAgentValidation from "./validation" export * as DeepAgentBudget from "./budget" export * as DeepAgentLearning from "./learning" export * as DeepAgentSessionState from "./session-state" +export * as DeepAgentPlanStore from "./plan-store" export * as DeepAgentOrchestrator from "./orchestrator" export * as DeepAgentAblation from "./ablation" export * as DeepAgentRunContext from "./run-context" diff --git a/packages/core/src/deepagent/plan-store.ts b/packages/core/src/deepagent/plan-store.ts new file mode 100644 index 00000000..50543a8c --- /dev/null +++ b/packages/core/src/deepagent/plan-store.ts @@ -0,0 +1,98 @@ +// I33-1 (deepagentcore-v4.0.3 storage prereq): the SINGLE structural authority for a session's plan. +// +// Before I33-1 the structural PlanDoc lived on the in-memory SessionRunState (flushed to sessions.json) +// while the goal path ALSO mirrored a `type:"plan"` DocumentStore doc — two disconnected stores that +// could diverge (a durable-continuation reload would trust one, a run-close mirror the other). I33-1 +// makes the DocumentStore `type:"plan"` doc the ONE structural authority (content-addressed, versioned, +// CAS-protected by F30-1); SessionRunState keeps only the runtime latch (plan_id/version/fresh-stale) +// as a hot value object, never the body. +// +// The plan doc is co-located with the session's goal/run graph so the goal path and the `plan` tool +// write the SAME doc: `planStoreRoot(sid)` is byte-identical to goal-manager's `goalStoreRoot(sid)` +// (`/state/goal//graph`). Both go through `DocumentStore.shared(root)` (F30-1 Part 2), +// so every handle for a session shares ONE in-memory index — a write via the tool is immediately +// visible to the goal driver and vice versa, with no second cache to drift. The shared index IS the +// hot cache: getPlanDoc is an in-memory Map lookup + a JSON.parse, not a disk read. +import path from "node:path" +import { DocumentStore } from "./document-store" +import type { PlanDoc } from "./plan-controller" + +// The stable identity of a session's plan doc: type "plan", scope "run:", slug "plan-", +// description planDescription(sid). ALL FOUR must match between the `plan` tool path (setPlanDoc) and +// the goal path (goal-driver.materializePlanDoc) or upsert()'s findLogical dedup (which keys on +// description + domain) splits them into two docs (plan- vs plan--2) and reintroduces the +// two-store divergence I33-1 removes. goal-driver imports planDescription/planScope from here so the +// identity can never drift across the package boundary. +const planSlug = (sessionId: string): string => `plan-${sessionId}` +export const planScope = (sessionId: string): string => `run:${sessionId}` +export const planDescription = (sessionId: string): string => `session plan ${sessionId}` + +// The state dir session-state was configured with. plan-store roots UNDER it at the same location the +// goal store uses, so the two paths converge on one doc. Set by configureRoot (called from the same +// gateway configure that sets session-state's dir), so core never has to import the deepagent-code +// goal-manager resolver. +let stateDir: string | null = null + +export const configureRoot = (dir: string): void => { + stateDir = dir +} + +// planStoreRoot(sid) === goalStoreRoot(sid) === /goal//graph. Kept private-by-convention +// (exported for the goal path + tests to assert convergence). Throws if used before configureRoot — a +// plan write with no configured root is a wiring bug, not something to silently drop. +export const planStoreRoot = (sessionId: string): string => { + if (!stateDir) throw new Error("plan-store: configureRoot() not called (no state dir)") + return path.join(stateDir, "goal", sessionId, "graph") +} + +// The shared authority handle for a session's plan doc. Shared registry keyed by resolved root, so the +// tool path, the goal driver, and the UI/archive readers all see one coherent in-memory index. +const store = (sessionId: string): DocumentStore => DocumentStore.shared(planStoreRoot(sessionId)) + +// Resolve a session's plan doc ref. The doc id is NOT reconstructable from the slug — allocateId runs +// idSlug through slugify() (lowercase, `_`→`-`, truncate 48), so a raw `doc:plan:plan-` guess +// misses for realistic session ids. Instead resolve by (type "plan", scope "run:"): the plan-store +// root is per-session (/goal//graph), and there is exactly one plan doc per session, so this +// filter yields at most one ref. list() returns the LATEST version per id (F30-1 shared index lookup). +const resolveRef = (sessionId: string) => { + const refs = store(sessionId).list({ type: "plan", scope: planScope(sessionId) }) + return refs.length > 0 ? refs[0] : null +} + +// Read the current structural plan for a session (latest version), or null if none exists yet. Pure +// in-memory lookup over the shared index (+ JSON.parse) — safe on the hot path (every tool call). +export const getPlanDoc = (sessionId: string): PlanDoc | null => { + const ref = resolveRef(sessionId) + if (!ref) return null + const doc = store(sessionId).get(ref.id) + if (!doc) return null + try { + return JSON.parse(doc.body) as PlanDoc + } catch { + return null + } +} + +// The doc id + current version for a session's plan (for the SessionRunState latch pointer), or null. +export const planDocRef = (sessionId: string): { id: string; version: number } | null => { + const ref = resolveRef(sessionId) + if (!ref) return null + const doc = store(sessionId).get(ref.id) + return doc ? { id: doc.id, version: doc.version } : null +} + +// Write (create or new-version) the structural plan. Idempotent per session via upsert keyed on the +// stable slug: an unchanged body is an INV-4 no-op (no version bump), a changed body appends a new +// version (CAS-protected). Returns the resulting doc id + version so the caller can update the latch +// pointer. This is the ONE write seam — the `plan` tool AND the goal path both call it. +export const setPlanDoc = (sessionId: string, plan: PlanDoc): { id: string; version: number } => { + const doc = store(sessionId).upsert({ + type: "plan", + scope: planScope(sessionId), + description: planDescription(sessionId), + idSlug: planSlug(sessionId), + body: JSON.stringify(plan), + provenance: { source: "model", run_ref: planScope(sessionId) }, + }) + return { id: doc.id, version: doc.version } +} diff --git a/packages/core/src/deepagent/session-state.ts b/packages/core/src/deepagent/session-state.ts index a9fea9e0..fd5fa491 100644 --- a/packages/core/src/deepagent/session-state.ts +++ b/packages/core/src/deepagent/session-state.ts @@ -2,6 +2,7 @@ import { mkdirSync, readFileSync } from "node:fs" import { randomUUID } from "node:crypto" import path from "node:path" import { writeFileAtomic } from "./atomic-write" +import * as PlanStore from "./plan-store" import type { AgentMode } from "./mode" import { createInitialRoundState, @@ -42,12 +43,10 @@ export type SessionRunState = { workspacePath: string | null runId: string // U1 PlanController: the runtime plan latch (fresh/stale + reason + replan count). The structural - // plan lives in DocumentStore; only this hot-path value object is carried on session state. + // plan lives in DocumentStore (I33-1, plan-store.ts) as the single authority; only this hot-path + // value object — the latch, carrying the plan_id pointer — is on session state. The plan body is + // NOT stored here (getPlan/setPlan delegate to plan-store). planLatch: PlanLatchState - // U1: the live working plan (goal/steps) the model writes via the `plan` tool and the UI renders. - // Kept on run state (hot path, atomically persisted) — graduated into the durable run-graph as a - // `plan` doc at run close, mirroring how DESIGN.md is materialized. - plan: PlanDoc | null // U10 step-reporting: count of mutating tool calls since the model last CHANGED a plan step's // status. Drives the progress-nudge count backstop (nudgeTrigger). Reset to 0 only when setPlan // detects a real status change — a no-op plan re-write must not silence the nudge. @@ -96,6 +95,11 @@ const sessions = new Map() export const configure = (dir: string) => { stateDir = dir mkdirSync(dir, { recursive: true }) + // I33-1: the structural plan authority (DocumentStore `type:"plan"` doc) roots under the SAME state + // dir (/goal//graph), so the `plan` tool (via session-state) and the goal path write the + // same doc. Set the plan-store root here — including for tests that call configure() directly + // (bypassing the gateway) — so every plan read/write has a configured root. + PlanStore.configureRoot(dir) // Pointing at a (new) state dir means a fresh session set: clear the in-memory map BEFORE loading, so // configure() reflects exactly what's on disk at `dir` and never merges stale sessions from a prior // dir. Production calls configure once at gateway init (nothing to lose); tests that configure a fresh @@ -121,7 +125,6 @@ export const getOrCreate = (sessionId: string, mode: AgentMode): SessionRunState workspacePath: null, runId: `run_${randomUUID()}`, planLatch: initialPlanLatch(), - plan: null, mutationsSinceReport: 0, validationPassedSinceReport: false, panelArmed: null, @@ -200,17 +203,22 @@ export const setPlan = (sessionId: string, plan: PlanDoc): void => { if (!state) return // U10: reset the progress-nudge state ONLY when the model actually moved a step's status (or // added a step). A no-op re-write leaves the counter/flag running so the nudge is not silenced by - // an empty update ("report theater"). - if (planStatusesChanged(state.plan, plan)) { + // an empty update ("report theater"). Compare against the CURRENT structural plan in the store. + const previous = PlanStore.getPlanDoc(sessionId) + if (planStatusesChanged(previous, plan)) { state.mutationsSinceReport = 0 state.validationPassedSinceReport = false } - state.plan = plan + // I33-1: the plan body is written to the single DocumentStore authority (plan-store); session state + // keeps only the latch pointer (plan_id). The store upsert is content-addressed + CAS-protected. + PlanStore.setPlanDoc(sessionId, plan) state.planLatch = clearStale({ ...state.planLatch, plan_id: plan.plan_id }) saveToDisk() } -export const getPlan = (sessionId: string): PlanDoc | null => sessions.get(sessionId)?.plan ?? null +// I33-1: read the structural plan from the single DocumentStore authority (plan-store). This is an +// in-memory shared-index lookup + JSON.parse (F30-1 Part 2), safe on the hot path (every tool call). +export const getPlan = (sessionId: string): PlanDoc | null => PlanStore.getPlanDoc(sessionId) // V3.9 §C — Expert Panel per-session arming. // The raw per-session toggle (null = never explicitly toggled). setPanelArmed writes an explicit @@ -277,7 +285,9 @@ export const setActiveGoalPhase = (sessionId: string, phase: GoalPointerPhase): // against). export const recordMutation = (sessionId: string): void => { const state = sessions.get(sessionId) - if (!state || state.plan == null) return + // I33-1: the nudge only applies once a plan exists — the latch's plan_id is the hot-path pointer + // (set by setPlan), so we gate on it instead of a stored body. + if (!state || state.planLatch.plan_id == null) return state.mutationsSinceReport += 1 saveToDisk() } @@ -412,7 +422,6 @@ function normalizeState(state: SessionRunState): SessionRunState { planLatch: state.planLatch ? { ...state.planLatch, consecutive_blocks: state.planLatch.consecutive_blocks ?? 0 } : initialPlanLatch(), - plan: state.plan ?? null, // Backfill: sessions persisted before U10 have no counter on disk. mutationsSinceReport: state.mutationsSinceReport ?? 0, validationPassedSinceReport: state.validationPassedSinceReport ?? false, @@ -431,9 +440,23 @@ function loadFromDisk() { if (!stateDir) return try { const content = readFileSync(path.join(stateDir, "sessions.json"), "utf8") - const data = JSON.parse(content) as Record + // Legacy sessions.json (pre-I33-1) carried the structural plan body on `state.plan`. Read it as an + // optional field so we can migrate it into the DocumentStore authority, then drop it from state. + const data = JSON.parse(content) as Record for (const [id, state] of Object.entries(data)) { - if (!state.completedAt) sessions.set(id, normalizeState(state)) + if (state.completedAt) continue + // I33-1 migration: if this session still has an inline plan body from before the store became the + // authority, seed it into the plan-store (idempotent upsert) so nothing is lost, then let it fall + // away (normalizeState no longer carries `plan`). Only migrate when the store has no plan yet, so + // a newer store doc (e.g. a goal edit) is never overwritten by a stale inline body. + if (state.plan && !PlanStore.getPlanDoc(id)) { + try { + PlanStore.setPlanDoc(id, state.plan) + } catch { + /* best-effort migration: a store hiccup must not block loading session state */ + } + } + sessions.set(id, normalizeState(state)) } } catch {} } diff --git a/packages/core/src/effect/app-node-platform.ts b/packages/core/src/effect/app-node-platform.ts new file mode 100644 index 00000000..7119bb24 --- /dev/null +++ b/packages/core/src/effect/app-node-platform.ts @@ -0,0 +1,18 @@ +import { NodeFileSystem, NodePath } from "@effect/platform-node" +import { LLMClient, RequestExecutor } from "@deepagent-code/llm/route" +import { FileSystem, Path } from "effect" +import { FetchHttpClient } from "effect/unstable/http" +import { HttpClient } from "effect/unstable/http" +import { makeGlobalNode } from "./app-node" + +export const filesystem = makeGlobalNode({ service: FileSystem.FileSystem, layer: NodeFileSystem.layer, deps: [] }) +export const path = makeGlobalNode({ service: Path.Path, layer: NodePath.layer, deps: [] }) +export const httpClient = makeGlobalNode({ service: HttpClient.HttpClient, layer: FetchHttpClient.layer, deps: [] }) +export const requestExecutor = makeGlobalNode({ + service: RequestExecutor.Service, + layer: RequestExecutor.layer, + deps: [httpClient], +}) +export const llmClient = makeGlobalNode({ service: LLMClient.Service, layer: LLMClient.layer, deps: [requestExecutor] }) + +export * as LayerNodePlatform from "./app-node-platform" diff --git a/packages/core/src/effect/app-node.ts b/packages/core/src/effect/app-node.ts new file mode 100644 index 00000000..e8112f92 --- /dev/null +++ b/packages/core/src/effect/app-node.ts @@ -0,0 +1,14 @@ +import { LayerNode } from "./layer-node" + +export const tags = LayerNode.tags({ + location: ["global"], + global: [], +}) + +export type GlobalNode = LayerNode.Node +export type LocationNode = LayerNode.Node + +export const makeGlobalNode = tags.make("global") +export const makeLocationNode = tags.make("location") + +export * as Node from "./app-node" diff --git a/packages/core/src/effect/layer-node.ts b/packages/core/src/effect/layer-node.ts new file mode 100644 index 00000000..9dbc3d51 --- /dev/null +++ b/packages/core/src/effect/layer-node.ts @@ -0,0 +1,333 @@ +import { Brand, Context, Layer } from "effect" + +type AnyNode = Node +type RuntimeLayer = Layer.Layer +type NodeList = readonly [] | readonly [Item, ...Item[]] +export type Output = [Item] extends [never] ? never : Item extends Node ? A : never +export type Error = [Item] extends [never] ? never : Item extends Node ? E : never +type NodeTag = [Item] extends [never] ? undefined : Item extends Node ? T : never +type Missing = Exclude> +type CheckDependencies = [ + Missing, Dependencies>, +] extends [never] + ? unknown + : { readonly "Missing dependencies": Missing, Dependencies> } +declare const $OutputType: unique symbol +declare const $ErrorType: unique symbol + +export type Tag = Name & Brand.Brand<"LayerNode.Tag"> + +const makeTag = Brand.nominal() + +export interface Node { + readonly kind: "layer" | "unbound" | "group" + readonly name: string + readonly service?: Context.Service.Any + readonly implementation?: Layer.Any + readonly dependencies: readonly AnyNode[] + readonly tag?: T + readonly [$OutputType]?: () => A + readonly [$ErrorType]?: () => E +} + +type NodeIdentity = + | { readonly service: Context.Service.Any; readonly name?: never } + | { readonly name: string; readonly service?: never } +type DistributiveOmit = A extends unknown ? Omit : never + +export type TagConfig = Readonly> +type TagNames = keyof Config & string +type NodeInTags = Node | undefined> +type CheckTags = [Exclude>] extends [ + never, +] + ? unknown + : { readonly "Invalid tag dependencies": Exclude> } + +export interface Tags { + readonly values: { readonly [Name in TagNames]: Tag } + readonly make: >( + name: Name, + ) => ( + input: DistributiveOmit>, "tag"> & + CheckTags>, + ) => Node, Layer.Error | Error, Tag> +} + +export function tags( + config: Config, +): Tags { + const names = Object.keys(config) as TagNames[] + const values = Object.fromEntries(names.map((name) => [name, makeTag(name)])) as Tags["values"] + return { + values, + make: ((name: TagNames) => (input: DistributiveOmit, "tag">) => + make({ ...input, tag: values[name] })) as Tags["make"], + } +} + +// Nodes --------------------------------------------------------------------- + +type MakeInput< + Implementation extends Layer.Any, + Items extends NodeList, + T extends Tag | undefined = undefined, +> = NodeIdentity & { + readonly layer: Implementation + readonly deps: Items & CheckDependencies> + readonly tag?: T +} + +export function make< + const Implementation extends Layer.Any, + const Items extends NodeList, + const T extends Tag | undefined = undefined, +>( + input: MakeInput, +): Node, Layer.Error | Error, T> { + return { + kind: "layer", + name: input.service !== undefined ? input.service.key : input.name, + service: input.service, + implementation: input.layer, + dependencies: input.deps, + tag: input.tag, + } +} + +export function unbound(service: Context.Key, tag: T): Node { + return { + kind: "unbound", + name: service.key, + service, + dependencies: [], + tag, + } +} + +export function group( + dependencies: Items, +): Node, Error, NodeTag> { + return { kind: "group", name: "group", dependencies } +} + +export type Replacement = readonly [source: AnyNode, replacement: AnyNode | Layer.Any] +export type Replacements = readonly Replacement[] + +type CheckReplacementErrors = [Exclude] extends [never] + ? unknown + : { readonly "New replacement errors": Exclude } + +type CheckReplacement = Item extends readonly [Node, infer Replacement] + ? Replacement extends Node, infer E2, T> + ? CheckReplacementErrors> + : Replacement extends Layer.Layer, infer E2, never> + ? CheckReplacementErrors> + : { readonly "Invalid replacement": Replacement } + : { readonly "Invalid replacement": Item } + +type CheckReplacements = { + readonly [K in keyof Items]: CheckReplacement +} + +type ValidReplacements = Items & CheckReplacements + +function replacementNode(source: AnyNode, replacement: AnyNode | Layer.Any) { + const replacementNode = isNode(replacement) + ? replacement + : make({ + ...nodeMakeIdentity(source), + layer: replacement as Layer.Layer, + deps: [], + tag: source.tag, + }) + if (source.name !== replacementNode.name) { + throw new Error(`Cannot replace ${source.name} with ${replacementNode.name}`) + } + if (source.tag !== replacementNode.tag) { + throw new Error(`Cannot replace ${source.name} across tags`) + } + return replacementNode +} + +function nodeMakeIdentity(node: AnyNode): NodeIdentity { + if (node.service !== undefined) return { service: node.service } + return { name: node.name } +} + +function isNode(input: Layer.Any | AnyNode): input is AnyNode { + return "kind" in input && "dependencies" in input +} + +// Tree ----------------------------------------------------------------------- + +type Visit = (node: AnyNode, context: VisitContext) => Result + +type VisitContext = { + readonly cache: Map + readonly visit: (node: AnyNode) => Result +} + +function walk( + root: AnyNode, + visit: Visit, + options: { + readonly cache?: Map + readonly resolve?: (node: AnyNode) => AnyNode + readonly detectCycles?: boolean + } = {}, +) { + const cache = options.cache ?? new Map() + const visiting = new Set() + const stack: AnyNode[] = [] + + const recur = (node: AnyNode): Result => { + const target = options.resolve?.(node) ?? node + const cached = cache.get(target) + if (cached !== undefined || cache.has(target)) return cached! + + if (options.detectCycles !== false && visiting.has(target)) { + const start = stack.indexOf(target) + throw new Error( + `Cycle detected in layer tree: ${[...stack.slice(start), target].map((item) => item.name).join(" -> ")}`, + ) + } + + visiting.add(target) + stack.push(target) + try { + const result = visit(target, { cache, visit: recur }) + if (!cache.has(target)) cache.set(target, result) + return result + } finally { + stack.pop() + visiting.delete(target) + } + } + + return recur(root) +} + +export function hoist( + root: Node, + tag: T, + replacements?: ValidReplacements, +): { + readonly node: Node + readonly hoisted: Node +} { + const hoisted = new Map() + const replacementMap = replacementMapFrom(replacements) + + const node = walk( + root, + (node, context) => { + if (node.kind === "group") { + return { ...node, dependencies: node.dependencies.map(context.visit) } + } + if (node.tag === tag) { + const existing = hoisted.get(node.name) + if (existing && existing !== node) { + throw new Error(`Tag ${tag} has conflicting implementations for ${node.name}`) + } + hoisted.set(node.name, rewriteReplacementDependencies(node, replacementMap)) + return group([]) + } + if (node.kind === "unbound") { + return node + } + return { ...node, dependencies: node.dependencies.map(context.visit) } + }, + { resolve: (node) => replacementMap.get(node.name) ?? node }, + ) + + return { + node: node as Node, + hoisted: group(Array.from(hoisted.values())) as Node, + } +} + +export function compile( + root: Node, + replacements?: ValidReplacements, +): Layer.Layer { + const replacementMap = replacementMapFrom(replacements) + const cache = new Map() + const compileNode = (node: AnyNode) => + walk( + node, + (node, context) => { + if (node.kind === "unbound") throw new Error(`Unbound layer node: ${node.name}`) + const dependencies = node.dependencies.flatMap(flatten).map(context.visit) + const implementation = node.implementation! as RuntimeLayer + return dependencies.length === 0 + ? implementation + : implementation.pipe(Layer.provide(dependencies as [RuntimeLayer, ...RuntimeLayer[]])) + }, + { cache, resolve: (node) => replacementMap.get(node.name) ?? node }, + ) + const layers = flatten(root).map((node) => compileNode(node)) + const layer = layers.reduce((result, layer) => layer.pipe(Layer.provideMerge(result)), Layer.empty) + return layer as Layer.Layer +} + +function replacementMapFrom(replacements?: Replacements) { + return ( + replacements?.reduce((map, [source, replacement]) => { + const normalized = rewriteReplacementDependencies(replacementNode(source, replacement), map) + const current = new Map([[source.name, normalized]]) + for (const [name, node] of map) map.set(name, rewriteReplacementDependencies(node, current)) + map.set(source.name, normalized) + return map + }, new Map()) ?? new Map() + ) +} + +function rewriteReplacementDependencies(root: AnyNode, replacements: ReadonlyMap) { + if (replacements.size === 0) return root + const cache = new Map() + const visiting = new Set() + const stack: AnyNode[] = [] + + const recur = (node: AnyNode, isRoot = false): AnyNode => { + const target = isRoot ? node : (replacements.get(node.name) ?? node) + const cached = cache.get(target) + if (cached !== undefined || cache.has(target)) return cached! + if (visiting.has(target)) { + const start = stack.indexOf(target) + throw new Error( + `Cycle detected in layer tree: ${[...stack.slice(start), target].map((item) => item.name).join(" -> ")}`, + ) + } + + visiting.add(target) + stack.push(target) + try { + const dependencies = target.dependencies.map((dependency) => recur(dependency)) + const result = dependencies.every((dependency, index) => dependency === target.dependencies[index]) + ? target + : { ...target, dependencies } + cache.set(target, result) + return result + } finally { + stack.pop() + visiting.delete(target) + } + } + + return recur(root, true) +} + +export function hasUnbound(root: Node, source: AnyNode): boolean { + if (source.kind !== "unbound") throw new Error(`Cannot check non-unbound layer node: ${source.name}`) + return walk(root, (node, context) => { + if (node === source) return true + return node.dependencies.some(context.visit) + }) +} + +function flatten(node: AnyNode): readonly AnyNode[] { + return node.kind === "group" ? node.dependencies.flatMap(flatten) : [node] +} + +export * as LayerNode from "./layer-node" diff --git a/packages/core/src/event.ts b/packages/core/src/event.ts index e7c8bc1d..a2183d1d 100644 --- a/packages/core/src/event.ts +++ b/packages/core/src/event.ts @@ -5,6 +5,7 @@ import { and, asc, eq, gt } from "drizzle-orm" import { Database } from "./database/database" import { EventSequenceTable, EventTable } from "./event/sql" import { Location } from "./location" +import { makeGlobalNode } from "./effect/app-node" import { externalID, type ExternalID, NonNegativeInt, withStatics } from "./schema" import { Identifier } from "./util/identifier" import { isDeepStrictEqual } from "node:util" @@ -677,4 +678,6 @@ export const layerWith = (options?: LayerOptions) => export const layer = layerWith() +export const node = makeGlobalNode({ service: Service, layer: layer, deps: [Database.node] }) + export const defaultLayer = layer.pipe(Layer.provide(Database.defaultLayer)) diff --git a/packages/core/src/file-mutation.ts b/packages/core/src/file-mutation.ts index cd8d6de5..6f009aaa 100644 --- a/packages/core/src/file-mutation.ts +++ b/packages/core/src/file-mutation.ts @@ -3,6 +3,7 @@ export * as FileMutation from "./file-mutation" import { Context, Effect, Layer, Schema } from "effect" import { dirname } from "path" import { KeyedMutex } from "./effect/keyed-mutex" +import { makeLocationNode } from "./effect/app-node" import { FSUtil } from "./fs-util" export interface Target { @@ -190,6 +191,8 @@ function sameBytes(left: Uint8Array, right: Uint8Array) { return left.every((byte, index) => byte === right[index]) } +export const node = makeLocationNode({ service: Service, layer, deps: [FSUtil.node] }) + export const locationLayer = layer /** diff --git a/packages/core/src/filesystem/watcher.ts b/packages/core/src/filesystem/watcher.ts index f9cfc0de..dc8b6d5b 100644 --- a/packages/core/src/filesystem/watcher.ts +++ b/packages/core/src/filesystem/watcher.ts @@ -6,6 +6,7 @@ import type ParcelWatcher from "@parcel/watcher" import { Cause, Context, Effect, Layer, Schema } from "effect" import path from "path" import { Config } from "../config" +import { makeLocationNode } from "../effect/app-node" import { EventV2 } from "../event" import { Flag } from "../flag/flag" import { FSUtil } from "../fs-util" @@ -147,4 +148,10 @@ export const layer = Layer.effect( ), ) +export const node = makeLocationNode({ + service: Service, + layer, + deps: [FSUtil.node, Location.node, Config.node, Git.node, EventV2.node], +}) + export const locationLayer = layer.pipe(Layer.provide(Config.locationLayer), Layer.provide(Git.defaultLayer)) diff --git a/packages/core/src/fs-util.ts b/packages/core/src/fs-util.ts index 8d9d17dd..a8932c35 100644 --- a/packages/core/src/fs-util.ts +++ b/packages/core/src/fs-util.ts @@ -7,6 +7,8 @@ import { Context, Effect, FileSystem, Layer, Schema } from "effect" import type { PlatformError } from "effect/PlatformError" import { Glob } from "./util/glob" import { serviceUse } from "./effect/service-use" +import { makeGlobalNode } from "./effect/app-node" +import { filesystem } from "./effect/app-node-platform" export namespace FSUtil { export class FileSystemError extends Schema.TaggedErrorClass()("FileSystemError", { @@ -195,6 +197,8 @@ export namespace FSUtil { export const defaultLayer = layer.pipe(Layer.provide(NodeFileSystem.layer)) + export const node = makeGlobalNode({ service: Service, layer: layer, deps: [filesystem] }) + // Pure helpers that don't need Effect (path manipulation, sync operations) export function mimeType(p: string): string { return lookup(p) || "application/octet-stream" diff --git a/packages/core/src/git.ts b/packages/core/src/git.ts index 4fa544e1..c5b25bd6 100644 --- a/packages/core/src/git.ts +++ b/packages/core/src/git.ts @@ -6,6 +6,7 @@ import { ChildProcess } from "effect/unstable/process" import { AbsolutePath } from "./schema" import { FSUtil } from "./fs-util" import { AppProcess } from "./process" +import { makeGlobalNode } from "./effect/app-node" export interface Repo { /** @@ -401,6 +402,8 @@ export const layer = Layer.effect( export const defaultLayer = layer.pipe(Layer.provide(FSUtil.defaultLayer), Layer.provide(AppProcess.defaultLayer)) +export const node = makeGlobalNode({ service: Service, layer: layer, deps: [FSUtil.node, AppProcess.node] }) + export interface Result { readonly exitCode: number readonly text: string diff --git a/packages/core/src/global.ts b/packages/core/src/global.ts index 2cc1ce17..a749e5b9 100644 --- a/packages/core/src/global.ts +++ b/packages/core/src/global.ts @@ -6,6 +6,7 @@ import { Context, Effect, Layer } from "effect" import { Flock } from "./util/flock" import { Flag } from "./flag/flag" import { resolveDataPath, resolveHomeBase } from "./global-path" +import { makeGlobalNode } from "./effect/app-node" const app = "deepagent-code" const legacyData = path.join(xdgData!, app) @@ -199,6 +200,8 @@ export const layer = Layer.effect( export const defaultLayer = layer +export const node = makeGlobalNode({ service: Service, layer: layer, deps: [] }) + export const layerWith = (input: Partial) => Layer.effect( Service, diff --git a/packages/core/src/image.ts b/packages/core/src/image.ts index 9a715cce..25a21f40 100644 --- a/packages/core/src/image.ts +++ b/packages/core/src/image.ts @@ -1,6 +1,7 @@ export * as Image from "./image" import { Context, Effect, Layer, Schema } from "effect" +import { makeLocationNode } from "./effect/app-node" import { Config } from "./config" import { FileSystem } from "./filesystem" @@ -69,4 +70,6 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [Config.node] }) + export const locationLayer = layer.pipe(Layer.provide(Config.locationLayer)) diff --git a/packages/core/src/instruction-context.ts b/packages/core/src/instruction-context.ts index 909f0813..8043b4e8 100644 --- a/packages/core/src/instruction-context.ts +++ b/packages/core/src/instruction-context.ts @@ -2,6 +2,7 @@ export * as InstructionContext from "./instruction-context" import { Array, Effect, Layer, Schema } from "effect" import { isAbsolute, join, relative, sep } from "path" +import { makeLocationNode } from "./effect/app-node" import { FSUtil } from "./fs-util" import { Flag } from "./flag/flag" import { Global } from "./global" @@ -87,6 +88,12 @@ export const layer = Layer.effectDiscard( }), ) +export const node = makeLocationNode({ + name: "instruction-context", + layer, + deps: [FSUtil.node, Global.node, Location.node, SystemContextRegistry.node], +}) + function render(files: ReadonlyArray) { return files.map((file) => `Instructions from: ${file.path}\n${file.content}`).join("\n\n") } diff --git a/packages/core/src/location-mutation.ts b/packages/core/src/location-mutation.ts index 202aa5c5..3f432239 100644 --- a/packages/core/src/location-mutation.ts +++ b/packages/core/src/location-mutation.ts @@ -2,6 +2,7 @@ export * as LocationMutation from "./location-mutation" import path from "path" import { Context, Effect, Layer, Schema } from "effect" +import { makeLocationNode } from "./effect/app-node" import { FSUtil } from "./fs-util" import { Location } from "./location" @@ -152,4 +153,10 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ + service: Service, + layer: layer.pipe(Layer.orDie), + deps: [FSUtil.node, Location.node], +}) + export const locationLayer = layer diff --git a/packages/core/src/location.ts b/packages/core/src/location.ts index abdb80f4..93a04ec7 100644 --- a/packages/core/src/location.ts +++ b/packages/core/src/location.ts @@ -2,6 +2,8 @@ import { Context, Effect, Layer, Schema } from "effect" import { Project } from "./project" import { AbsolutePath } from "./schema" import { WorkspaceV2 } from "./workspace" +import { LayerNode } from "./effect/layer-node" +import { tags } from "./effect/app-node" export * as Location from "./location" @@ -30,6 +32,8 @@ export function response(data: S) { export class Service extends Context.Service()("@deepagent-code/Location") {} +export const node = LayerNode.unbound(Service, tags.values.location) + export const layer = (ref: Ref) => Layer.effect( Service, diff --git a/packages/core/src/models-dev.ts b/packages/core/src/models-dev.ts index f2739adc..9291024d 100644 --- a/packages/core/src/models-dev.ts +++ b/packages/core/src/models-dev.ts @@ -8,6 +8,8 @@ import { Hash } from "./util/hash" import { FSUtil } from "./fs-util" import { InstallationChannel, InstallationVersion } from "./installation/version" import { EventV2 } from "./event" +import { makeGlobalNode } from "./effect/app-node" +import { httpClient } from "./effect/app-node-platform" export const CatalogModelStatus = Schema.Literals(["alpha", "beta", "deprecated"]) export type CatalogModelStatus = typeof CatalogModelStatus.Type @@ -243,6 +245,8 @@ export const layer = Layer.effect( }), ) +export const node = makeGlobalNode({ service: Service, layer: layer, deps: [FSUtil.node, EventV2.node, httpClient] }) + export const defaultLayer = layer.pipe( Layer.provide(FetchHttpClient.layer), Layer.provide(FSUtil.defaultLayer), diff --git a/packages/core/src/npm.ts b/packages/core/src/npm.ts index efbf940c..31f8ec96 100644 --- a/packages/core/src/npm.ts +++ b/packages/core/src/npm.ts @@ -8,6 +8,8 @@ import { FSUtil } from "./fs-util" import { Global } from "./global" import { EffectFlock } from "./util/effect-flock" import { makeRuntime } from "./effect/runtime" +import { makeGlobalNode } from "./effect/app-node" +import { filesystem } from "./effect/app-node-platform" import { NpmConfig } from "./npm-config" export class InstallFailedError extends Schema.TaggedErrorClass()("NpmInstallFailedError", { @@ -244,6 +246,12 @@ export const layer = Layer.effect( }), ) +export const node = makeGlobalNode({ + service: Service, + layer: layer, + deps: [FSUtil.node, Global.node, filesystem, EffectFlock.node], +}) + export const defaultLayer = layer.pipe( Layer.provide(EffectFlock.layer), Layer.provide(FSUtil.layer), diff --git a/packages/core/src/permission.ts b/packages/core/src/permission.ts index 032304d6..442eefd0 100644 --- a/packages/core/src/permission.ts +++ b/packages/core/src/permission.ts @@ -1,6 +1,7 @@ export * as PermissionV2 from "./permission" import { Context, Deferred, Effect as EffectRuntime, Layer, Schema } from "effect" +import { makeLocationNode } from "./effect/app-node" import { EventV2 } from "./event" import { Location } from "./location" import { AgentV2 } from "./agent" @@ -326,4 +327,10 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ + service: Service, + layer, + deps: [EventV2.node, Location.node, AgentV2.node, SessionStore.node, PermissionSaved.node], +}) + export const locationLayer = layer.pipe(Layer.provideMerge(AgentV2.locationLayer)) diff --git a/packages/core/src/permission/saved.ts b/packages/core/src/permission/saved.ts index a723a602..22ce8ab5 100644 --- a/packages/core/src/permission/saved.ts +++ b/packages/core/src/permission/saved.ts @@ -3,6 +3,7 @@ export * as PermissionSaved from "./saved" import { eq } from "drizzle-orm" import { Context, Effect, Layer, Schema } from "effect" import { Database } from "../database/database" +import { makeGlobalNode } from "../effect/app-node" import { ProjectV2 } from "../project" import { withStatics } from "../schema" import { Identifier } from "../util/identifier" @@ -84,4 +85,6 @@ export const layer = Layer.effect( }), ) +export const node = makeGlobalNode({ service: Service, layer, deps: [Database.node] }) + export const defaultLayer = layer.pipe(Layer.provide(Database.defaultLayer)) diff --git a/packages/core/src/policy.ts b/packages/core/src/policy.ts index a8d5c97d..fb7e1b98 100644 --- a/packages/core/src/policy.ts +++ b/packages/core/src/policy.ts @@ -3,6 +3,7 @@ export * as Policy from "./policy" import { Context, Effect as EffectRuntime, Layer, Schema } from "effect" import { Wildcard } from "./util/wildcard" import { Location } from "./location" +import { makeLocationNode } from "./effect/app-node" export const Effect = Schema.Literals(["allow", "deny"]).annotate({ identifier: "Policy.Effect" }) export type Effect = typeof Effect.Type @@ -43,4 +44,6 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [Location.node] }) + export const locationLayer = layer diff --git a/packages/core/src/process.ts b/packages/core/src/process.ts index 9ea95a13..37ca2b97 100644 --- a/packages/core/src/process.ts +++ b/packages/core/src/process.ts @@ -3,6 +3,7 @@ import type { PlatformError } from "effect/PlatformError" import { ChildProcess } from "effect/unstable/process" import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner" import { CrossSpawnSpawner } from "./cross-spawn-spawner" +import { makeGlobalNode } from "./effect/app-node" export class AppProcessError extends Schema.TaggedErrorClass()("AppProcessError", { command: Schema.String, @@ -231,4 +232,6 @@ export const layer = Layer.effect( export const defaultLayer = layer.pipe(Layer.provide(CrossSpawnSpawner.defaultLayer)) +export const node = makeGlobalNode({ service: Service, layer: layer, deps: [CrossSpawnSpawner.node] }) + export * as AppProcess from "./process" diff --git a/packages/core/src/pty.ts b/packages/core/src/pty.ts index 6b93667b..a098e397 100644 --- a/packages/core/src/pty.ts +++ b/packages/core/src/pty.ts @@ -2,6 +2,7 @@ export * as Pty from "./pty" import type { Disp, Proc } from "#pty" import { Context, Effect, Layer, Schema, Types } from "effect" +import { makeLocationNode } from "./effect/app-node" import { EventV2 } from "./event" import { Location } from "./location" import { NonNegativeInt, PositiveInt } from "./schema" @@ -328,4 +329,6 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [EventV2.node, Location.node] }) + export const locationLayer = layer diff --git a/packages/core/src/question.ts b/packages/core/src/question.ts index 10f647fc..5c5ec184 100644 --- a/packages/core/src/question.ts +++ b/packages/core/src/question.ts @@ -1,6 +1,7 @@ export * as QuestionV2 from "./question" import { Context, Deferred, Effect, Layer, Schema } from "effect" +import { makeLocationNode } from "./effect/app-node" import { EventV2 } from "./event" import { Identifier } from "./id/id" import { withStatics } from "./schema" @@ -195,4 +196,6 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [EventV2.node] }) + export const locationLayer = layer diff --git a/packages/core/src/repository-cache.ts b/packages/core/src/repository-cache.ts index 0b89cef6..e6aff09f 100644 --- a/packages/core/src/repository-cache.ts +++ b/packages/core/src/repository-cache.ts @@ -5,6 +5,7 @@ import { Git } from "./git" import { Global } from "./global" import { Repository } from "./repository" import { EffectFlock } from "./util/effect-flock" +import { makeGlobalNode } from "./effect/app-node" export type Result = { readonly repository: string @@ -259,6 +260,12 @@ export const defaultLayer: Layer.Layer = layer.pipe( Layer.provide(Global.defaultLayer), ) +export const node = makeGlobalNode({ + service: Service, + layer, + deps: [EffectFlock.node, FSUtil.node, Git.node, Global.node], +}) + function statusForRepository(input: { reuse: boolean; refresh?: boolean; branchMatches?: boolean }) { if (!input.reuse) return "cloned" as const if (input.branchMatches === false || input.refresh) return "refreshed" as const diff --git a/packages/core/src/session/projector.ts b/packages/core/src/session/projector.ts index e88d73f5..5642f193 100644 --- a/packages/core/src/session/projector.ts +++ b/packages/core/src/session/projector.ts @@ -3,6 +3,7 @@ export * as SessionProjector from "./projector" import { and, desc, eq, sql } from "drizzle-orm" import { DateTime, Effect, Layer, Schema } from "effect" import { Database } from "../database/database" +import { makeGlobalNode } from "../effect/app-node" import { EventV2 } from "../event" import { SessionEvent } from "./event" import { SessionV1 } from "../v1/session" @@ -447,4 +448,6 @@ export const layer = Layer.effectDiscard( }), ) +export const node = makeGlobalNode({ name: "session-projector", layer, deps: [EventV2.node, Database.node] }) + export const defaultLayer = layer.pipe(Layer.provide(EventV2.defaultLayer), Layer.provide(Database.defaultLayer)) diff --git a/packages/core/src/session/store.ts b/packages/core/src/session/store.ts index ff1c557d..fcb7586e 100644 --- a/packages/core/src/session/store.ts +++ b/packages/core/src/session/store.ts @@ -3,6 +3,7 @@ export * as SessionStore from "./store" import { eq } from "drizzle-orm" import { Context, Effect, Layer, Schema } from "effect" import { Database } from "../database/database" +import { makeGlobalNode } from "../effect/app-node" import { SessionHistory } from "./history" import { MessageDecodeError } from "./error" import { SessionMessage } from "./message" @@ -59,4 +60,6 @@ export const layer = Layer.effect( }), ) +export const node = makeGlobalNode({ service: Service, layer, deps: [Database.node] }) + export const defaultLayer = layer.pipe(Layer.provide(Database.defaultLayer)) diff --git a/packages/core/src/session/todo.ts b/packages/core/src/session/todo.ts index b96996ba..990d292f 100644 --- a/packages/core/src/session/todo.ts +++ b/packages/core/src/session/todo.ts @@ -7,6 +7,7 @@ export * as SessionTodo from "./todo" import { asc, eq } from "drizzle-orm" import { Context, Effect, Layer, Schema } from "effect" import { Database } from "../database/database" +import { makeLocationNode } from "../effect/app-node" import { EventV2 } from "../event" import { SessionSchema } from "./schema" import { TodoTable } from "./sql" @@ -92,4 +93,6 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [EventV2.node, Database.node] }) + export const defaultLayer = layer.pipe(Layer.provide(EventV2.defaultLayer), Layer.provide(Database.defaultLayer)) diff --git a/packages/core/src/skill.ts b/packages/core/src/skill.ts index 65f9f2a4..b082302e 100644 --- a/packages/core/src/skill.ts +++ b/packages/core/src/skill.ts @@ -5,6 +5,7 @@ import { Context, Effect, Layer, Schema } from "effect" import { castDraft } from "immer" import { AgentV2 } from "./agent" import { ConfigMarkdown } from "./config/markdown" +import { makeLocationNode } from "./effect/app-node" import { FSUtil } from "./fs-util" import { PermissionV2 } from "./permission" import { AbsolutePath, withStatics } from "./schema" @@ -158,4 +159,6 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [SkillDiscovery.node, FSUtil.node] }) + export const locationLayer = layer.pipe(Layer.provide(SkillDiscovery.defaultLayer)) diff --git a/packages/core/src/skill/discovery.ts b/packages/core/src/skill/discovery.ts index 17de1a2d..e20f8a18 100644 --- a/packages/core/src/skill/discovery.ts +++ b/packages/core/src/skill/discovery.ts @@ -3,6 +3,8 @@ export * as SkillDiscovery from "./discovery" import path from "path" import { Context, Effect, Layer, Schedule, Schema } from "effect" import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http" +import { makeGlobalNode } from "../effect/app-node" +import { httpClient } from "../effect/app-node-platform" import { FSUtil } from "../fs-util" import { Global } from "../global" import { AbsolutePath } from "../schema" @@ -167,6 +169,8 @@ export const layer = Layer.effect( }), ) +export const node = makeGlobalNode({ service: Service, layer, deps: [httpClient, FSUtil.node, Global.node] }) + export const defaultLayer = layer.pipe( Layer.provide(FetchHttpClient.layer), Layer.provide(FSUtil.defaultLayer), diff --git a/packages/core/src/system-context/builtins.ts b/packages/core/src/system-context/builtins.ts index 42cba27b..5520822f 100644 --- a/packages/core/src/system-context/builtins.ts +++ b/packages/core/src/system-context/builtins.ts @@ -1,6 +1,9 @@ export * as SystemContextBuiltIns from "./builtins" import { DateTime, Effect, Layer, Schema } from "effect" +import { makeLocationNode } from "../effect/app-node" +import { FSUtil } from "../fs-util" +import { Global } from "../global" import { Location } from "../location" import { SystemContext } from "./index" import { InstructionContext } from "../instruction-context" @@ -40,6 +43,12 @@ const builtIns = Layer.effectDiscard( }), ) +export const node = makeLocationNode({ + name: "system-context-builtins", + layer: builtIns, + deps: [Location.node, SystemContextRegistry.node, InstructionContext.node, FSUtil.node, Global.node], +}) + export const layer = Layer.mergeAll(builtIns, InstructionContext.layer).pipe( Layer.provideMerge(SystemContextRegistry.layer), ) diff --git a/packages/core/src/system-context/registry.ts b/packages/core/src/system-context/registry.ts index a3c678ea..17895bcc 100644 --- a/packages/core/src/system-context/registry.ts +++ b/packages/core/src/system-context/registry.ts @@ -1,6 +1,7 @@ export * as SystemContextRegistry from "./registry" import { Context, Effect, Layer, Ref, Scope } from "effect" +import { makeLocationNode } from "../effect/app-node" import { SystemContext } from "./index" export interface Entry { @@ -44,3 +45,5 @@ export const layer = Layer.effect( }) }), ) + +export const node = makeLocationNode({ service: Service, layer, deps: [] }) diff --git a/packages/core/src/tool-output-store.ts b/packages/core/src/tool-output-store.ts index 1ee9c5ba..5c5c47a8 100644 --- a/packages/core/src/tool-output-store.ts +++ b/packages/core/src/tool-output-store.ts @@ -3,6 +3,7 @@ export * as ToolOutputStore from "./tool-output-store" import path from "path" import { Context, Duration, Effect, Layer, Option, Schedule, Schema } from "effect" import { Config } from "./config" +import { makeLocationNode } from "./effect/app-node" import { FSUtil } from "./fs-util" import { Global } from "./global" import { SessionSchema } from "./session/schema" @@ -186,6 +187,8 @@ export const layer = Layer.effect( }), ) +export const node = makeLocationNode({ service: Service, layer, deps: [FSUtil.node, Global.node, Config.node] }) + export const defaultLayer = layer.pipe(Layer.provide(FSUtil.defaultLayer), Layer.provide(Global.defaultLayer)) /** Runs retention scanning once globally rather than once per active Location. */ diff --git a/packages/core/src/tool/application-tools.ts b/packages/core/src/tool/application-tools.ts index b8c0f78e..20ecf4de 100644 --- a/packages/core/src/tool/application-tools.ts +++ b/packages/core/src/tool/application-tools.ts @@ -2,6 +2,7 @@ export * as ApplicationTools from "./application-tools" import { Context, Effect, Layer, Scope } from "effect" import { enableMapSet } from "immer" +import { makeGlobalNode } from "../effect/app-node" import { State } from "../state" import { Tool } from "./tool" @@ -56,3 +57,5 @@ export const layer = Layer.effect( }) }), ) + +export const node = makeGlobalNode({ service: Service, layer, deps: [] }) diff --git a/packages/core/src/tool/apply-patch.ts b/packages/core/src/tool/apply-patch.ts index bb2ca9f8..551acf65 100644 --- a/packages/core/src/tool/apply-patch.ts +++ b/packages/core/src/tool/apply-patch.ts @@ -7,6 +7,8 @@ import { FSUtil } from "../fs-util" import { LocationMutation } from "../location-mutation" import { Patch } from "../patch" import { PermissionV2 } from "../permission" +import { makeLocationNode } from "../effect/app-node" +import { ToolRegistry } from "./registry" import { Tool } from "./tool" import { Tools } from "./tools" @@ -175,3 +177,9 @@ export const layer = Layer.effectDiscard( .pipe(Effect.orDie) }), ) + +export const node = makeLocationNode({ + name: "tool/apply-patch", + layer, + deps: [ToolRegistry.node, LocationMutation.node, FileMutation.node, FSUtil.node, PermissionV2.node], +}) diff --git a/packages/core/src/tool/bash.ts b/packages/core/src/tool/bash.ts index 0e14ee2b..4aa8eaa4 100644 --- a/packages/core/src/tool/bash.ts +++ b/packages/core/src/tool/bash.ts @@ -10,6 +10,8 @@ import { LocationMutation } from "../location-mutation" import { AppProcess } from "../process" import { PermissionV2 } from "../permission" import { Policy } from "../policy" +import { makeLocationNode } from "../effect/app-node" +import { ToolRegistry } from "./registry" import { ServerCapabilities } from "../server-capabilities" import { PositiveInt } from "../schema" import { Tool } from "./tool" @@ -240,3 +242,9 @@ export const layer = Layer.effectDiscard( .pipe(Effect.orDie) }), ) + +export const node = makeLocationNode({ + name: "tool/bash", + layer, + deps: [ToolRegistry.node, LocationMutation.node, FSUtil.node, AppProcess.node, Config.node, PermissionV2.node, Policy.node], +}) diff --git a/packages/core/src/tool/edit.ts b/packages/core/src/tool/edit.ts index dab26554..3a121d5f 100644 --- a/packages/core/src/tool/edit.ts +++ b/packages/core/src/tool/edit.ts @@ -13,6 +13,8 @@ import { FileMutation } from "../file-mutation" import { FSUtil } from "../fs-util" import { LocationMutation } from "../location-mutation" import { PermissionV2 } from "../permission" +import { makeLocationNode } from "../effect/app-node" +import { ToolRegistry } from "./registry" import { Tool } from "./tool" import { Tools } from "./tools" @@ -198,3 +200,9 @@ export const layer = Layer.effectDiscard( .pipe(Effect.orDie) }), ) + +export const node = makeLocationNode({ + name: "tool/edit", + layer, + deps: [ToolRegistry.node, LocationMutation.node, FileMutation.node, FSUtil.node, PermissionV2.node], +}) diff --git a/packages/core/src/tool/question.ts b/packages/core/src/tool/question.ts index 2a95de15..563a3559 100644 --- a/packages/core/src/tool/question.ts +++ b/packages/core/src/tool/question.ts @@ -4,6 +4,8 @@ import { ToolFailure, toolText } from "@deepagent-code/llm" import { Effect, Layer, Schema } from "effect" import { PermissionV2 } from "../permission" import { QuestionV2 } from "../question" +import { makeLocationNode } from "../effect/app-node" +import { ToolRegistry } from "./registry" import { Tool } from "./tool" import { Tools } from "./tools" @@ -84,3 +86,9 @@ export const layer = Layer.effectDiscard( .pipe(Effect.orDie) }), ) + +export const node = makeLocationNode({ + name: "tool/question", + layer, + deps: [ToolRegistry.node, PermissionV2.node, QuestionV2.node], +}) diff --git a/packages/core/src/tool/registry.ts b/packages/core/src/tool/registry.ts index 351fdca5..d5a49138 100644 --- a/packages/core/src/tool/registry.ts +++ b/packages/core/src/tool/registry.ts @@ -3,6 +3,7 @@ export * as ToolRegistry from "./registry" import { ToolOutput, type ToolCall, type ToolDefinition, type ToolSettlement } from "@deepagent-code/llm" import { Context, Effect, Layer, Scope } from "effect" import { AgentV2 } from "../agent" +import { makeLocationNode } from "../effect/app-node" import { PermissionV2 } from "../permission" import { SessionMessage } from "../session/message" import { SessionSchema } from "../session/schema" @@ -131,6 +132,18 @@ function whollyDisabled(action: string, rules: PermissionV2.Ruleset) { return rule?.resource === "*" && rule.effect === "deny" } +export const node = makeLocationNode({ + service: Service, + layer, + deps: [ApplicationTools.node, ToolOutputStore.node], +}) + +export const toolsNode = makeLocationNode({ + service: Tools.Service, + layer, + deps: [ApplicationTools.node, ToolOutputStore.node], +}) + export const defaultLayer = layer.pipe( Layer.provide(ApplicationTools.layer), Layer.provide(ToolOutputStore.defaultLayer), diff --git a/packages/core/src/tool/todowrite.ts b/packages/core/src/tool/todowrite.ts index c990abbe..aa6bb7b4 100644 --- a/packages/core/src/tool/todowrite.ts +++ b/packages/core/src/tool/todowrite.ts @@ -4,6 +4,8 @@ import { ToolFailure, toolText } from "@deepagent-code/llm" import { Effect, Layer, Schema } from "effect" import { PermissionV2 } from "../permission" import { SessionTodo } from "../session/todo" +import { makeLocationNode } from "../effect/app-node" +import { ToolRegistry } from "./registry" import { Tool } from "./tool" import { Tools } from "./tools" @@ -52,3 +54,9 @@ export const layer = Layer.effectDiscard( .pipe(Effect.orDie) }), ) + +export const node = makeLocationNode({ + name: "tool/todowrite", + layer, + deps: [ToolRegistry.node, PermissionV2.node, SessionTodo.node], +}) diff --git a/packages/core/src/tool/webfetch.ts b/packages/core/src/tool/webfetch.ts index 2f252554..737d6e64 100644 --- a/packages/core/src/tool/webfetch.ts +++ b/packages/core/src/tool/webfetch.ts @@ -6,6 +6,9 @@ import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstab import { Parser } from "htmlparser2" import TurndownService from "turndown" import { PermissionV2 } from "../permission" +import { makeLocationNode } from "../effect/app-node" +import { LayerNodePlatform } from "../effect/app-node-platform" +import { ToolRegistry } from "./registry" import { Tool } from "./tool" import { Tools } from "./tools" @@ -185,6 +188,12 @@ export const layer = Layer.effectDiscard( }), ) +export const node = makeLocationNode({ + name: "tool/webfetch", + layer, + deps: [ToolRegistry.node, PermissionV2.node, LayerNodePlatform.httpClient], +}) + export function extractTextFromHTML(html: string) { let text = "" let skipDepth = 0 diff --git a/packages/core/src/tool/websearch.ts b/packages/core/src/tool/websearch.ts index a2f42025..e9d39d16 100644 --- a/packages/core/src/tool/websearch.ts +++ b/packages/core/src/tool/websearch.ts @@ -7,6 +7,9 @@ import { truthy } from "../flag/flag" import { InstallationVersion } from "../installation/version" import { PositiveInt } from "../schema" import { PermissionV2 } from "../permission" +import { makeLocationNode } from "../effect/app-node" +import { LayerNodePlatform } from "../effect/app-node-platform" +import { ToolRegistry } from "./registry" import { Tool } from "./tool" import { Tools } from "./tools" import { checksum } from "../util/encode" @@ -83,6 +86,8 @@ export const defaultConfigLayer = Layer.sync(ConfigService, () => }), ) +export const configNode = makeLocationNode({ service: ConfigService, layer: defaultConfigLayer, deps: [] }) + export function selectProvider( sessionID: string, flags: Pick = { enableExa: false, enableParallel: false }, @@ -248,3 +253,9 @@ export const layer = Layer.effectDiscard( .pipe(Effect.orDie) }), ) + +export const node = makeLocationNode({ + name: "tool/websearch", + layer, + deps: [ToolRegistry.node, PermissionV2.node, LayerNodePlatform.httpClient, configNode], +}) diff --git a/packages/core/src/util/effect-flock.ts b/packages/core/src/util/effect-flock.ts index 64a1b6f7..aabeb1c7 100644 --- a/packages/core/src/util/effect-flock.ts +++ b/packages/core/src/util/effect-flock.ts @@ -6,6 +6,7 @@ import type { FileSystem, Scope } from "effect" import type { PlatformError } from "effect/PlatformError" import { FSUtil } from "../fs-util" import { Global } from "../global" +import { makeGlobalNode } from "../effect/app-node" import { Hash } from "./hash" export namespace EffectFlock { @@ -280,4 +281,6 @@ export namespace EffectFlock { ) export const defaultLayer = layer.pipe(Layer.provide(FSUtil.defaultLayer), Layer.provide(Global.layer)) + + export const node = makeGlobalNode({ service: Service, layer: layer, deps: [Global.node, FSUtil.node] }) } diff --git a/packages/core/test/deepagent/command-intent.test.ts b/packages/core/test/deepagent/command-intent.test.ts index f9f1f4ca..5a066ecd 100644 --- a/packages/core/test/deepagent/command-intent.test.ts +++ b/packages/core/test/deepagent/command-intent.test.ts @@ -186,4 +186,55 @@ describe("CommandIntent.classifyCommand", () => { expect(classifyCommand("curl -sL https://example.com/api")).toBe("read_only") }) }) + + // P0 (parity with codex is_safe_to_call_with_exec): text-stream filters that only read stdin/files + // and the read-only `sed -n {N|M,N}p` slice are exempt. This is the exact class of probe that + // triggered the production deadlock (`sed -n "18,45p"` inside an ssh/docker exec wrapper). + describe("P0: read-only text filters + sed -n slice", () => { + const readOnly = [ + "cut -d, -f1 data.csv", + "nl -ba file.txt", + "paste a.txt b.txt", + "rev file.txt", + "seq 1 10", + "tr a-z A-Z", + "uniq -c file.txt", + "comm -12 a.txt b.txt", + "fold -w 80 file.txt", + "expr 1 + 2", + "true", + "false", + "sort file.txt", + "sort -r -n file.txt", + "sed -n '18,45p' shim_utils.h", + "sed -n 42p file.txt", + "sed -n 1,5p file.txt", + "sort data.txt | uniq | wc -l", + ] + for (const cmd of readOnly) { + test(`read_only: ${cmd}`, () => { + expect(classifyCommand(cmd)).toBe("read_only") + }) + } + }) + + // P0 fail-safe: sed/sort variants that can write or run arbitrary scripts stay mutating. + describe("P0: sed/sort write variants stay mutating", () => { + const mutating = [ + "sed -i 's/a/b/' file.txt", // in-place edit + "sed -n 's/a/b/w out.txt' file.txt", // -n but writes via w + "sed 's/a/b/' file.txt", // no -n → not the read-only slice form + "sed -n -e 'p' file.txt", // extra -e script, not a bare address + "sed -n 'xp' file.txt", // invalid (non-numeric) address + "sed -n '1,5d' file.txt", // delete, not print + "sort -o out.txt file.txt", // sort writing a file + "sort --output=out.txt file.txt", + "sort -oout.txt file.txt", // bundled short form + ] + for (const cmd of mutating) { + test(`mutating: ${cmd}`, () => { + expect(classifyCommand(cmd)).toBe("mutating") + }) + } + }) }) diff --git a/packages/core/test/deepagent/document-store.test.ts b/packages/core/test/deepagent/document-store.test.ts index aad4812b..45c004a1 100644 --- a/packages/core/test/deepagent/document-store.test.ts +++ b/packages/core/test/deepagent/document-store.test.ts @@ -1,8 +1,9 @@ import { describe, expect, test, beforeEach, afterEach } from "bun:test" -import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs" +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync, rmSync } from "node:fs" import { tmpdir } from "node:os" import path from "node:path" -import { DocumentStore, knowledgeSimilarity, tokenizeForSimilarity } from "../../src/deepagent/document-store" +import { DocumentStore, DocumentConflictError, knowledgeSimilarity, tokenizeForSimilarity } from "../../src/deepagent/document-store" +import { writeFileAtomic, writeFileExclusive } from "../../src/deepagent/atomic-write" import { DurableKnowledgeStore } from "../../src/deepagent/durable-knowledge-store" let root: string @@ -168,6 +169,141 @@ describe("V3 DocumentStore", () => { }) }) +// F30-1 (deepagentcore-v4.0.3 storage prereq): DocumentStore is the single crash-safe, concurrency- +// safe durable body. persist() writes append-only version files with exclusive-create CAS; replace() +// overwrites the same version atomically (temp+fsync+rename). These tests pin the CAS + durability +// behavior that H32-1 (v4.0.4) builds on. +describe("F30-1 DocumentStore CAS + atomic durability", () => { + test("normal single-writer flow is unchanged (create + updates land byte-identically)", () => { + const a = store.create(design("v1")) + const a2 = store.update(a.id, "v2") + const a3 = store.update(a.id, "v3") + expect(a3.version).toBe(3) + // reopened from files only — every version file was written exactly once via exclusive create + const reopened = new DocumentStore(root) + expect(reopened.get(a.id)!.body).toBe("v3") + expect(reopened.get(a.id, 1)!.status).toBe("superseded") + expect(reopened.get(a.id, 2)!.status).toBe("superseded") + expect(reopened.verify().ok).toBe(true) + }) + + test("two handles writing DIFFERENT content at the same version -> DocumentConflictError", () => { + // Both handles are built over the same on-disk root but hold independent in-memory indexes, so + // both compute the SAME next version (v2) from the SAME base (v1) — the lost-update race F30-1 + // must catch instead of silently clobbering. + const a = store.create(design("base")) + const h1 = new DocumentStore(root) + const h2 = new DocumentStore(root) + h1.update(a.id, "from h1") // writes id@v2 (h1's content) + // h2 still thinks latest is v1; its update also targets v2 but with different content -> conflict + expect(() => h2.update(a.id, "from h2")).toThrow(DocumentConflictError) + // h1's write survives intact; nothing was clobbered + expect(new DocumentStore(root).get(a.id)!.body).toBe("from h1") + }) + + test("idempotent re-persist of byte-identical version is a no-op (same-hash CAS collision)", () => { + // Two handles writing the SAME content at the same version must NOT conflict — a retried/mirrored + // write of identical bytes is idempotent (content-addressed: same hash). + const a = store.create(design("base")) + const h1 = new DocumentStore(root) + const h2 = new DocumentStore(root) + const w1 = h1.update(a.id, "same body") + const w2 = h2.update(a.id, "same body") // same version, same content -> idempotent, no throw + expect(w1.version).toBe(2) + expect(w2.version).toBe(2) + expect(w1.hash).toBe(w2.hash) + expect(new DocumentStore(root).get(a.id)!.body).toBe("same body") + }) + + test("setStatus/replace rewrites the same version file in place without a version bump", () => { + const a = store.create(design("body")) + expect(a.version).toBe(1) + store.setStatus(a.id, "active") + expect(store.get(a.id)!.status).toBe("active") + expect(store.get(a.id)!.version).toBe(1) // in-place rewrite, not a new version + // exactly one version file on disk for this doc (no orphan versions from the rewrite) + const dir = path.join(root, "docs", "design") + const files = readdirSync(dir).filter((f) => f.endsWith(".json")) + expect(files.length).toBe(1) + expect(new DocumentStore(root).get(a.id)!.status).toBe("active") + }) + + test("writeFileExclusive throws EEXIST on an existing path (the CAS primitive)", () => { + const f = path.join(root, "cas-probe.json") + writeFileExclusive(f, "first") + let code: string | undefined + try { + writeFileExclusive(f, "second") + } catch (e) { + code = (e as NodeJS.ErrnoException).code + } + expect(code).toBe("EEXIST") + expect(readFileSync(f, "utf8")).toBe("first") // original never clobbered + }) + + test("writeFileAtomic overwrites durably and leaves no temp files behind", () => { + const f = path.join(root, "atomic-probe.json") + writeFileAtomic(f, "one") + writeFileAtomic(f, "two") + expect(readFileSync(f, "utf8")).toBe("two") + // no leftover .tmp-* siblings from the temp+rename + const leftover = readdirSync(root).filter((n) => n.includes(".tmp-")) + expect(leftover).toEqual([]) + }) +}) + +// F30-1 Part 2: same-process SHARED AUTHORITY. `DocumentStore.shared(root)` handles for the same root +// share ONE in-memory index (a write through one is visible through all); `new DocumentStore(root)` +// stays unshared (own disk-rebuilt index) so it keeps faithfully simulating a cold/second-process open. +describe("F30-1 DocumentStore.shared same-process authority", () => { + afterEach(() => DocumentStore.__resetSharedRegistryForTests()) + + test("two shared handles to the same root see each other's writes without a disk reopen", () => { + const h1 = DocumentStore.shared(root) + const h2 = DocumentStore.shared(root) + const a = h1.create(design("written via h1")) + // h2 shares h1's live index — the doc is visible immediately, no rebuild/reopen needed + expect(h2.get(a.id)?.body).toBe("written via h1") + // and a write through h2 is visible through h1 (bidirectional shared authority) + const a2 = h2.update(a.id, "updated via h2") + expect(a2.version).toBe(2) + expect(h1.get(a.id)?.body).toBe("updated via h2") + expect(h1.get(a.id)?.version).toBe(2) + }) + + test("an UNSHARED handle does NOT see another handle's in-memory write until it rebuilds from disk", () => { + // This is the pre-F30-1 divergence the shared registry fixes — pinned here so the distinction is + // explicit: an unshared handle only reflects writes that were on disk at ITS construction time. + const shared = DocumentStore.shared(root) + const a = shared.create(design("only in shared index at first")) + // A separate shared handle constructed AFTER the write still sees it (shared index)... + expect(DocumentStore.shared(root).get(a.id)?.body).toBe("only in shared index at first") + // ...and an unshared handle also sees it here, because create() persisted it to disk and the + // unshared handle rebuilds from disk in its constructor. (Persistence is synchronous — Part 1.) + expect(new DocumentStore(root).get(a.id)?.body).toBe("only in shared index at first") + }) + + test("shared index is rebuilt from disk on first open of a root (survives a simulated restart)", () => { + // Seed via an unshared handle (writes to disk), then open the FIRST shared handle for the root — + // it must rebuild the shared index from disk so pre-existing docs are visible. + const seed = new DocumentStore(root).create(design("seeded on disk")) + DocumentStore.__resetSharedRegistryForTests() // simulate a fresh process (empty registry) + const shared = DocumentStore.shared(root) + expect(shared.get(seed.id)?.body).toBe("seeded on disk") + }) + + test("shared handles for DIFFERENT roots are isolated", () => { + const otherRoot = mkdtempSync(path.join(tmpdir(), "deepagent-ds-shared-other-")) + try { + const a = DocumentStore.shared(root).create(design("in root")) + const other = DocumentStore.shared(otherRoot) + expect(other.get(a.id)).toBeNull() // different root -> different shared index + } finally { + rmSync(otherRoot, { recursive: true, force: true }) + } + }) +}) + // V3.8 Phase 0: the new NON-knowledge derived-data node/edge types must round-trip through the graph // exactly like any other doc, must NOT trigger the knowledge confidence check, and must obey INV-3 // (single-store links only). No knowledge semantics ride on them. diff --git a/packages/core/test/deepagent/plan-controller.test.ts b/packages/core/test/deepagent/plan-controller.test.ts index b20da5f6..e78b2133 100644 --- a/packages/core/test/deepagent/plan-controller.test.ts +++ b/packages/core/test/deepagent/plan-controller.test.ts @@ -133,16 +133,38 @@ describe("planGate (before_tool_use soft gate)", () => { expect(gate({ name: "before_tool_use", payload: { planStale: true, isMutating: false } }).decision).toBe("allow") }) - test("blocks mutating tools when stale in high+ mode", () => { + // DESIGN (aligned with codex exec_policy): a stale plan ledger NEVER denies tool execution — it + // WARNS (tool runs, reminder attached) so the model is nudged to re-sync without being deadlocked. + // This holds in every mode, including high+ (the old code hard-blocked here, which caused the + // production deadlock). The only remaining hard block is the U9 per-step binding gate, covered below. + test("warns (never blocks) on a mutating tool when stale in high+ mode", () => { const d = gate({ name: "before_tool_use", payload: { planStale: true, isMutating: true, lightweight: false } }) - expect(d.decision).toBe("block") + expect(d.decision).toBe("warn") expect(d.blockReason).toContain("stale") }) - test("only warns (never blocks) in lightweight mode", () => { + test("warns (never blocks) in lightweight mode too", () => { const d = gate({ name: "before_tool_use", payload: { planStale: true, isMutating: true, lightweight: true } }) expect(d.decision).toBe("warn") }) + + // U9 per-step binding remains a hard block (strict hard modes) but gains a runtime grace release so + // it can never permanently deadlock a model that fails to mark a step active. + test("U9 binding: strict hard mode blocks a mutating tool with no active step", () => { + const d = gate({ + name: "before_tool_use", + payload: { planStale: false, isMutating: true, lightweight: false, hardGate: true, hasActiveStep: false, hardGateMissBlocks: true }, + }) + expect(d.decision).toBe("block") + }) + + test("U9 binding: grace release downgrades the strict block to a warn", () => { + const d = gate({ + name: "before_tool_use", + payload: { planStale: false, isMutating: true, lightweight: false, hardGate: true, hasActiveStep: false, hardGateMissBlocks: true, graceRelease: true }, + }) + expect(d.decision).toBe("warn") + }) }) describe("stopHookGate (plan condition added by U1)", () => { diff --git a/packages/core/test/deepagent/plan-gate-loop.test.ts b/packages/core/test/deepagent/plan-gate-loop.test.ts index 4930a2ec..7a540185 100644 --- a/packages/core/test/deepagent/plan-gate-loop.test.ts +++ b/packages/core/test/deepagent/plan-gate-loop.test.ts @@ -6,15 +6,18 @@ import * as SessionState from "../../src/deepagent/session-state" import * as PlanController from "../../src/deepagent/plan-controller" import { planGate, HookPolicy } from "../../src/deepagent/hooks" -// U1 end-to-end contract (the exact decision the tools.ts chokepoint computes): a stale plan -// soft-blocks a mutating tool, read/plan tools pass, and calling the plan tool (setPlan) clears the -// latch so the next mutating tool is allowed again. This mirrors the wiring in session/tools.ts -// without booting the full session loop. +// U1 end-to-end contract (the exact decision the tools.ts chokepoint computes). DESIGN (aligned with +// codex exec_policy): plan-ledger state is orthogonal to whether a tool may run, so a stale plan +// NEVER hard-blocks a mutating tool — it WARNS (tool runs, reminder attached) so the model is nudged +// to re-sync without being denied its tools. Read/plan tools always pass. The only hard block that +// remains is the U9 per-step binding gate (strict hard modes), and even that has a runtime grace +// release so it can never permanently deadlock. This mirrors session/tools.ts without booting the +// full session loop. const gate = new HookPolicy().on("before_tool_use", planGate()) -// Reproduce the chokepoint decision exactly (session/tools.ts), including the U1 anti-deadlock -// payload fields (staleReason, graceRelease) and the shell command classification. +// Reproduce the soft-layer chokepoint decision (session/tools.ts): staleReason + isMutating + +// lightweight. The U9 hard-layer fields are exercised separately in the binding tests below. const decideAt = (sessionId: string, toolId: string, mode: string, command?: string) => { const latch = SessionState.planLatch(sessionId) const planStale = latch?.latch === "stale" && !PlanController.shouldEscapeToHuman(latch) @@ -30,18 +33,12 @@ const decideAt = (sessionId: string, toolId: string, mode: string, command?: str }) } -// Mirror the tools.ts block/reset bookkeeping so the loop tests advance the runtime grace counter -// exactly as production does: a block records a gate block; a mutating tool that runs resets it. -const applyOutcome = (sessionId: string, decision: string) => { - if (decision === "block") SessionState.recordPlanGateBlock(sessionId) -} - describe("U1 soft-gate loop (chokepoint contract)", () => { beforeEach(() => { SessionState.configure(mkdtempSync(path.join(tmpdir(), "plan-gate-"))) }) - test("high mode: stale plan blocks edit, allows read/plan, plan tool unblocks", () => { + test("high mode: stale plan WARNS on edit (never blocks), allows read/plan, plan tool clears warn", () => { SessionState.getOrCreate("gate-s1", "high") // a failing validation flips the latch from runtime truth SessionState.recordValidation( @@ -50,11 +47,12 @@ describe("U1 soft-gate loop (chokepoint contract)", () => { "e", ) - expect(decideAt("gate-s1", "edit", "high").decision).toBe("block") + // A stale plan no longer denies a mutating tool — it warns (the tool still runs). + expect(decideAt("gate-s1", "edit", "high").decision).toBe("warn") expect(decideAt("gate-s1", "read", "high").decision).toBe("allow") expect(decideAt("gate-s1", "plan", "high").decision).toBe("allow") - // model calls the plan tool -> setPlan clears the latch + // model calls the plan tool -> setPlan clears the latch -> no more warn SessionState.setPlan( "gate-s1", PlanController.buildPlanFromInput("gate-s1", { goal: "g", steps: [{ title: "fix", status: "active" }] }), @@ -68,7 +66,7 @@ describe("U1 soft-gate loop (chokepoint contract)", () => { expect(decideAt("gate-s2", "edit", "general").decision).toBe("warn") }) - test("escape hatch: after too many replans the gate stops blocking (routes to human elsewhere)", () => { + test("escape hatch: after too many replans the gate stops warning (routes to human elsewhere)", () => { SessionState.getOrCreate("gate-s3", "high") // exhaust the replan budget: each setPlan bumps replan_count for (let i = 0; i <= PlanController.DEFAULT_REPLAN_LIMIT; i++) { @@ -81,68 +79,94 @@ describe("U1 soft-gate loop (chokepoint contract)", () => { // now mark stale once more; replan_count already exceeds the limit SessionState.markPlanStale("gate-s3", "no_progress") expect(PlanController.shouldEscapeToHuman(SessionState.planLatch("gate-s3")!)).toBe(true) - // gate no longer blocks (escape hatch) — the macro-round stop gate routes to needs_human instead + // gate no longer warns (escape hatch flips planStale false) — the macro-round stop gate routes to + // needs_human instead of looping on plan re-sync. expect(decideAt("gate-s3", "edit", "high").decision).toBe("allow") }) - // ── U1 anti-deadlock: read-only shell commands are never gated ────────────────────────────────── - test("read-only bash (ls/git status/grep) is allowed even while the plan is stale", () => { + // ── read-only shell commands are never gated (the agent's eyes), aligned with codex is_safe_command ── + test("read-only bash (ls/git status/grep/sed -n/sort) is allowed even while the plan is stale", () => { SessionState.getOrCreate("gate-ro", "xhigh") SessionState.markPlanStale("gate-ro", "validation_failed") - // a mutating bash command IS gated… - expect(decideAt("gate-ro", "bash", "xhigh", "rm -rf build").decision).toBe("block") - // …but read-only shell inspections pass (the agent must be able to see to repair the plan). + // a mutating bash command warns (runs + reminder), never a hard block… + expect(decideAt("gate-ro", "bash", "xhigh", "rm -rf build").decision).toBe("warn") + // …and read-only shell inspections pass outright (isMutating=false → allow). expect(decideAt("gate-ro", "bash", "xhigh", "ls -la").decision).toBe("allow") expect(decideAt("gate-ro", "bash", "xhigh", "git status").decision).toBe("allow") expect(decideAt("gate-ro", "bash", "xhigh", "grep -rn TODO src/").decision).toBe("allow") expect(decideAt("gate-ro", "bash", "xhigh", "cat file.txt | head -20").decision).toBe("allow") + // P0: the exact probe form that triggered the production deadlock (sed line-slice) is read-only. + expect(decideAt("gate-ro", "bash", "xhigh", "sed -n '18,45p' shim_utils.h").decision).toBe("allow") + expect(decideAt("gate-ro", "bash", "xhigh", "sort file.txt | uniq -c").decision).toBe("allow") }) - // ── U1 anti-deadlock: runtime-driven grace release (the production-deadlock fix) ──────────────── - test("grace release: after repeated blocks with no re-plan, the gate releases WITHOUT model help", () => { - SessionState.getOrCreate("gate-grace", "xhigh") - SessionState.markPlanStale("gate-grace", "validation_failed") - - // The model keeps trying a mutating bash command and NEVER calls the plan tool (the exact - // production failure mode: 280 consecutive blocked bash calls). Each block advances the runtime - // grace counter — no setPlan required. - for (let i = 0; i < PlanController.DEFAULT_GRACE_BLOCK_LIMIT; i++) { - const decision = decideAt("gate-grace", "bash", "xhigh", "make build").decision - expect(decision).toBe("block") - applyOutcome("gate-grace", decision) - } + // ── a new user message must not hard-block work the user is asking for ────────────────────────── + test("user_appended stale reason warns (tool runs); validation_failed also warns (never blocks)", () => { + SessionState.getOrCreate("gate-user", "xhigh") + SessionState.markPlanStale("gate-user", "user_appended") + expect(decideAt("gate-user", "edit", "xhigh").decision).toBe("warn") + // a genuine reality-change signal also warns now — plan ledger state never denies execution. + SessionState.clearPlanStale("gate-user") + SessionState.markPlanStale("gate-user", "validation_failed") + expect(decideAt("gate-user", "edit", "xhigh").decision).toBe("warn") + }) +}) - // The escape-to-human hatch has NOT fired (replan_count never advanced — the model didn't - // cooperate), proving the old hatch alone would have deadlocked forever here. - expect(PlanController.shouldEscapeToHuman(SessionState.planLatch("gate-grace")!)).toBe(false) +// ── U9 per-step binding: the one remaining hard block, with a runtime grace release ─────────────── +// The binding gate (strict hard modes: xhigh/max/ultra) requires a mutating tool to be bound to an +// active plan step. This is workflow discipline, not a safety property, so it must never permanently +// deadlock: after repeated blocks with no forward progress the runtime grace release downgrades it to +// a warn WITHOUT any model cooperation — the same anti-deadlock invariant as the stale layer. +describe("U9 binding gate + grace release", () => { + const decideBinding = (opts: { hasActiveStep: boolean; hardGateMissBlocks: boolean; graceRelease: boolean }) => + gate.evaluate({ + name: "before_tool_use", + payload: { + planStale: false, + isMutating: true, + lightweight: false, + hardGate: true, + hasActiveStep: opts.hasActiveStep, + hardGateMissBlocks: opts.hardGateMissBlocks, + graceRelease: opts.graceRelease, + }, + }) + + test("strict hard mode blocks a mutating tool with no active step", () => { + expect(decideBinding({ hasActiveStep: false, hardGateMissBlocks: true, graceRelease: false }).decision).toBe( + "block", + ) + }) - // But the runtime grace release fires: the next mutating call is downgraded to a WARN (tool runs, - // reminder attached) instead of a hard block. The agent is never permanently denied its tools. - expect(PlanController.shouldGraceRelease(SessionState.planLatch("gate-grace")!)).toBe(true) - expect(decideAt("gate-grace", "bash", "xhigh", "make build").decision).toBe("warn") + test("lenient hard mode (high) only warns on a missing active step", () => { + expect(decideBinding({ hasActiveStep: false, hardGateMissBlocks: false, graceRelease: false }).decision).toBe( + "warn", + ) }) - test("grace counter resets when forward progress happens (mutating tool executes)", () => { - SessionState.getOrCreate("gate-reset", "xhigh") - SessionState.markPlanStale("gate-reset", "validation_failed") - SessionState.recordPlanGateBlock("gate-reset") - SessionState.recordPlanGateBlock("gate-reset") - expect(SessionState.planLatch("gate-reset")!.consecutive_blocks).toBe(2) - // a mutating tool actually ran → reset - SessionState.resetPlanGateBlocks("gate-reset") - expect(SessionState.planLatch("gate-reset")!.consecutive_blocks).toBe(0) - expect(PlanController.shouldGraceRelease(SessionState.planLatch("gate-reset")!)).toBe(false) + test("an active step passes the binding gate", () => { + expect(decideBinding({ hasActiveStep: true, hardGateMissBlocks: true, graceRelease: false }).decision).toBe( + "allow", + ) }) - // ── U1 anti-deadlock: a new user message must not hard-block work the user is asking for ──────── - test("user_appended stale reason warns (tool runs) rather than hard-blocking, even at xhigh", () => { - SessionState.getOrCreate("gate-user", "xhigh") - SessionState.markPlanStale("gate-user", "user_appended") - // mutating edit is downgraded to warn (runs + reminder), not blocked - expect(decideAt("gate-user", "edit", "xhigh").decision).toBe("warn") - // whereas a genuine reality-change signal still hard-blocks - SessionState.clearPlanStale("gate-user") - SessionState.markPlanStale("gate-user", "validation_failed") - expect(decideAt("gate-user", "edit", "xhigh").decision).toBe("block") + test("grace release downgrades the strict binding block to a warn (never permanently deadlocks)", () => { + // Even in the strictest mode with no active step, once the runtime grace release fires the tool is + // allowed through with a reminder rather than blocked forever. + expect(decideBinding({ hasActiveStep: false, hardGateMissBlocks: true, graceRelease: true }).decision).toBe( + "warn", + ) + }) + + test("grace counter machinery: blocks accumulate, forward progress resets", () => { + SessionState.configure(mkdtempSync(path.join(tmpdir(), "plan-gate-bind-"))) + SessionState.getOrCreate("gate-bind", "xhigh") + SessionState.markPlanStale("gate-bind", "validation_failed") + SessionState.recordPlanGateBlock("gate-bind") + SessionState.recordPlanGateBlock("gate-bind") + expect(SessionState.planLatch("gate-bind")!.consecutive_blocks).toBe(2) + SessionState.resetPlanGateBlocks("gate-bind") + expect(SessionState.planLatch("gate-bind")!.consecutive_blocks).toBe(0) + expect(PlanController.shouldGraceRelease(SessionState.planLatch("gate-bind")!)).toBe(false) }) }) diff --git a/packages/core/test/deepagent/plan-store.test.ts b/packages/core/test/deepagent/plan-store.test.ts new file mode 100644 index 00000000..5ea158a8 --- /dev/null +++ b/packages/core/test/deepagent/plan-store.test.ts @@ -0,0 +1,160 @@ +import { describe, expect, test, beforeEach, afterEach } from "bun:test" +import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import path from "node:path" +import * as PlanStore from "../../src/deepagent/plan-store" +import * as SessionState from "../../src/deepagent/session-state" +import { DocumentStore } from "../../src/deepagent/document-store" +import { createPlanDoc, type PlanDoc, type PlanStep } from "../../src/deepagent/plan-controller" + +// I33-1 (deepagentcore-v4.0.3): the DocumentStore `type:"plan"` doc is the SINGLE structural authority +// for a session's plan. session-state.setPlan/getPlan delegate to plan-store; the goal path writes the +// same doc (same root + slug). These tests pin that single-authority contract + the legacy migration. + +let stateDir: string +const step = (id: string, status: PlanStep["status"] = "pending"): PlanStep => ({ + step_id: id, + title: `step ${id}`, + status, + acceptance: null, + assigned_agent: null, + evidence: [], + note: null, +}) +const plan = (sid: string, steps: PlanStep[]): PlanDoc => createPlanDoc(sid, `goal ${sid}`, steps) + +beforeEach(() => { + stateDir = mkdtempSync(path.join(tmpdir(), "deepagent-planstore-")) + SessionState.configure(stateDir) // also configures plan-store root (I33-1 coupling) +}) +afterEach(() => { + DocumentStore.__resetSharedRegistryForTests() + rmSync(stateDir, { recursive: true, force: true }) +}) + +describe("I33-1 plan-store single authority", () => { + test("setPlanDoc/getPlanDoc round-trip through the DocumentStore", () => { + expect(PlanStore.getPlanDoc("s1")).toBeNull() + const p = plan("s1", [step("step_1"), step("step_2")]) + const ref = PlanStore.setPlanDoc("s1", p) + expect(ref.version).toBe(1) + const read = PlanStore.getPlanDoc("s1") + expect(read?.goal).toBe("goal s1") + expect(read?.steps.map((s) => s.step_id)).toEqual(["step_1", "step_2"]) + }) + + test("round-trips for a REALISTIC session id (slugify mangles the id — resolve by type+scope)", () => { + // Regression: the doc id is allocated via slugify(idSlug) (lowercase, `_`→`-`, truncate 48), so a + // raw `doc:plan:plan-` reconstruction misses for real ids. A production session id has + // underscores + a uuid and exceeds 48 chars — getPlanDoc must still resolve it. + const sid = `ses_planstatus_render_${"a".repeat(20)}_0123456789abcdef` + const p = plan(sid, [step("step_1", "active"), step("step_2")]) + PlanStore.setPlanDoc(sid, p) + const read = PlanStore.getPlanDoc(sid) + expect(read).not.toBeNull() + expect(read?.steps[0].status).toBe("active") + expect(PlanStore.planDocRef(sid)?.version).toBe(1) + }) + + test("identical body is an INV-4 no-op (no version bump); a change appends a version", () => { + const p = plan("s2", [step("step_1")]) + expect(PlanStore.setPlanDoc("s2", p).version).toBe(1) + expect(PlanStore.setPlanDoc("s2", p).version).toBe(1) // unchanged -> no-op + const changed = plan("s2", [step("step_1", "done")]) + expect(PlanStore.setPlanDoc("s2", changed).version).toBe(2) + expect(PlanStore.getPlanDoc("s2")?.steps[0].status).toBe("done") + }) + + test("session-state.setPlan/getPlan delegate to the store (body NOT on session state)", () => { + SessionState.getOrCreate("s3", "high") + const p = plan("s3", [step("step_1")]) + SessionState.setPlan("s3", p) + // readable via session-state (delegates to plan-store) AND directly from plan-store (same doc) + expect(SessionState.getPlan("s3")?.goal).toBe("goal s3") + expect(PlanStore.getPlanDoc("s3")?.goal).toBe("goal s3") + // the latch pointer is bound to the plan id (the hot-path value object that STAYS on session state) + expect(SessionState.planLatch("s3")?.plan_id).toBe(p.plan_id) + // the persisted sessions.json must NOT carry the plan body (authority moved to the store) + const raw = require("node:fs").readFileSync(path.join(stateDir, "sessions.json"), "utf8") + expect(JSON.parse(raw)["s3"].plan).toBeUndefined() + }) + + test("plan-store root equals the goal store root (tool path and goal path converge on one doc)", () => { + // goal-manager.goalStoreRoot(sid) === /goal//graph — plan-store must match exactly, + // so materializePlanDoc (goal) and setPlanDoc (tool) upsert the SAME doc id. + expect(PlanStore.planStoreRoot("s4")).toBe(path.join(stateDir, "goal", "s4", "graph")) + }) + + test("goal path and tool path converge on ONE doc (realistic id; no plan--2 split)", () => { + // Regression for the P0 the adversarial review found: upsert() dedups on `description`, so the goal + // path (goal-driver.materializePlanDoc) and the tool path (setPlanDoc) MUST use the identical doc + // identity — type/scope/idSlug AND description — or upsert creates a SECOND doc (plan--2) and + // getPlan (list[0]) returns whichever is first, re-splitting the store. This replicates the goal + // path's write FAITHFULLY using the shared identity helpers (planDescription/planScope) — the same + // ones goal-driver now imports — with a realistic session id (the earlier test masked the bug by + // hand-rolling "session plan s5" and using a trivial id). + const sid = `ses_goal_conv_${"z".repeat(24)}_beef` + const goalRoot = path.join(stateDir, "goal", sid, "graph") + const goalHandle = DocumentStore.shared(goalRoot) + // Goal-path write (must match plan-store identity exactly): + goalHandle.upsert({ + type: "plan", + scope: PlanStore.planScope(sid), + description: PlanStore.planDescription(sid), + idSlug: `plan-${sid}`, + body: JSON.stringify(plan(sid, [step("step_1", "active")])), + provenance: { source: "model", run_ref: PlanStore.planScope(sid) }, + }) + expect(PlanStore.getPlanDoc(sid)?.steps[0].status).toBe("active") + // Tool-path write lands on the SAME doc (new version), NOT a second doc. + SessionState.getOrCreate(sid, "high") + SessionState.setPlan(sid, plan(sid, [step("step_1", "done")])) + expect(PlanStore.getPlanDoc(sid)?.steps[0].status).toBe("done") + // Exactly ONE plan doc exists under this session's root (no plan--2 split). + const planDocs = DocumentStore.shared(goalRoot).list({ type: "plan", scope: PlanStore.planScope(sid) }) + expect(planDocs.length).toBe(1) + }) + + test("legacy inline plan on sessions.json is migrated into the store on load", () => { + // Write a pre-I33-1 sessions.json that still carries the structural plan body inline. + const legacyPlan = plan("s6", [step("step_1"), step("step_2", "done")]) + const legacy = { + s6: { + sessionId: "s6", + mode: "high", + completedAt: null, + planLatch: { plan_id: legacyPlan.plan_id, latch: "fresh", stale_reason: null, replan_count: 0, consecutive_blocks: 0 }, + plan: legacyPlan, // the legacy inline body + }, + } + mkdirSync(stateDir, { recursive: true }) + writeFileSync(path.join(stateDir, "sessions.json"), JSON.stringify(legacy)) + DocumentStore.__resetSharedRegistryForTests() + // Re-configure to trigger loadFromDisk (which runs the migration). + SessionState.configure(stateDir) + // The plan body is now readable from the store authority... + expect(PlanStore.getPlanDoc("s6")?.steps.map((s) => s.status)).toEqual(["pending", "done"]) + // ...and getPlan (delegating to the store) returns it. + expect(SessionState.getPlan("s6")?.plan_id).toBe(legacyPlan.plan_id) + }) + + test("migration does not clobber a newer store doc (e.g. a goal edit) with a stale inline body", () => { + // Seed a NEWER plan into the store first (as a goal edit would), THEN load a legacy sessions.json + // whose inline body is older. The migration must NOT overwrite the newer store doc. + const newer = plan("s7", [step("step_1", "done")]) + PlanStore.setPlanDoc("s7", newer) + const legacyOlder = plan("s7", [step("step_1", "pending")]) + const legacy = { + s7: { + sessionId: "s7", + mode: "high", + completedAt: null, + planLatch: { plan_id: legacyOlder.plan_id, latch: "fresh", stale_reason: null, replan_count: 0, consecutive_blocks: 0 }, + plan: legacyOlder, + }, + } + writeFileSync(path.join(stateDir, "sessions.json"), JSON.stringify(legacy)) + SessionState.configure(stateDir) // load; migration sees the store already has a plan -> skips + expect(PlanStore.getPlanDoc("s7")?.steps[0].status).toBe("done") // newer store doc preserved + }) +}) diff --git a/packages/core/test/effect/app-node.test.ts b/packages/core/test/effect/app-node.test.ts new file mode 100644 index 00000000..73f89b71 --- /dev/null +++ b/packages/core/test/effect/app-node.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, test } from "bun:test" +import { Context, Effect, Layer } from "effect" +import { makeGlobalNode, makeLocationNode, tags } from "../../src/effect/app-node" +import { LayerNode } from "../../src/effect/layer-node" + +// Wave A0 (deepagentcore-v4.0.3 阶段 A): the AppNode foundation (layer-node + app-node, ported +// byte-for-byte from upstream opencode) must compile and produce working layers, and must COEXIST +// with our existing Layer.effect paradigm (no forced migration). This smoke test pins that contract +// before any subsystem migrates onto makeGlobalNode/makeLocationNode in later waves. It is NOT a +// variant of upstream — upstream exercises the foundation only through subsystem tests (process, +// system-context) that arrive in A1+; this asserts the foundation itself in isolation. + +class Greeter extends Context.Service string }>()("test/Greeter") {} +class Loud extends Context.Service string }>()("test/Loud") {} + +describe("Wave A0 — AppNode foundation", () => { + test("makeGlobalNode builds a compilable node whose service resolves", async () => { + const node = makeGlobalNode({ + service: Greeter, + layer: Layer.succeed(Greeter, { hello: () => "hi" }), + deps: [], + }) + const layer = LayerNode.compile(node) + const out = await Effect.runPromise( + Effect.gen(function* () { + const g = yield* Greeter + return g.hello() + }).pipe(Effect.provide(layer)), + ) + expect(out).toBe("hi") + }) + + test("a node's declared deps are provided when compiled (dependency graph wiring)", async () => { + const greeter = makeGlobalNode({ + service: Greeter, + layer: Layer.succeed(Greeter, { hello: () => "hi" }), + deps: [], + }) + const loud = makeGlobalNode({ + service: Loud, + // Loud depends on Greeter — the node graph must provide it at compile time. + layer: Layer.effect( + Loud, + Effect.gen(function* () { + const g = yield* Greeter + return { shout: () => g.hello().toUpperCase() } + }), + ), + deps: [greeter], + }) + const out = await Effect.runPromise( + Effect.gen(function* () { + const l = yield* Loud + return l.shout() + }).pipe(Effect.provide(LayerNode.compile(loud))), + ) + expect(out).toBe("HI") + }) + + test("makeLocationNode tags a node in the location scope (scope encoded on the node)", () => { + const node = makeLocationNode({ + service: Greeter, + layer: Layer.succeed(Greeter, { hello: () => "loc" }), + deps: [], + }) + // The location tag is carried on the node (this is what encodes global vs per-location scope). + expect(node.tag).toBe(tags.values.location) + expect(node.name).toBe("test/Greeter") + }) + + test("coexists with the existing Layer paradigm — a node layer composes with a plain Layer", async () => { + // A node-compiled layer and a hand-written Layer.effect must interoperate (the A0 coexistence gate: + // AppNode does not require migrating everything at once). + const nodeLayer = LayerNode.compile( + makeGlobalNode({ service: Greeter, layer: Layer.succeed(Greeter, { hello: () => "node" }), deps: [] }), + ) + const plainLayer = Layer.effect( + Loud, + Effect.gen(function* () { + const g = yield* Greeter + return { shout: () => g.hello() + "!" } + }), + ).pipe(Layer.provide(nodeLayer)) + const out = await Effect.runPromise( + Effect.gen(function* () { + return (yield* Loud).shout() + }).pipe(Effect.provide(plainLayer)), + ) + expect(out).toBe("node!") + }) + + test("compile throws on an unbound node (a required dependency was never bound)", () => { + // An unbound node has no implementation; compiling it directly must throw rather than silently + // producing a layer missing its service. Use the real global tag so the node is well-typed. + const unboundNode = LayerNode.unbound(Greeter, tags.values.global) + expect(() => LayerNode.compile(unboundNode as never)).toThrow() + }) +}) diff --git a/packages/deepagent-code/src/session/goal-driver.ts b/packages/deepagent-code/src/session/goal-driver.ts index afe12c8d..1db310f0 100644 --- a/packages/deepagent-code/src/session/goal-driver.ts +++ b/packages/deepagent-code/src/session/goal-driver.ts @@ -52,10 +52,16 @@ const planScope = (sessionId: string): string => `run:${sessionId}` * Returns the plan doc id the GoalSpec references. */ export const materializePlanDoc = (input: MaterializePlanInput): string => { + // I33-1: the goal path and the `plan` tool MUST write ONE doc. upsert() dedups on `description` + // (findLogical), so any description drift between the two callers splits them into two docs + // (plan- vs plan--2) and reintroduces the two-store divergence. Use the SAME identity as + // plan-store.setPlanDoc — same type/scope/idSlug AND description — so the goal write lands on the + // tool-path doc (and vice versa). `input.store` is the shared handle at goalStoreRoot(sid), which is + // byte-identical to plan-store's planStoreRoot(sid), so both resolve the same on-disk + in-memory doc. const doc = input.store.upsert({ type: "plan", scope: planScope(input.sessionId), - description: `goal plan ${input.sessionId}`, + description: `session plan ${input.sessionId}`, idSlug: `plan-${input.sessionId}`, body: JSON.stringify(input.plan), provenance: { source: "model", run_ref: planScope(input.sessionId) }, diff --git a/packages/deepagent-code/src/session/goal-manager.ts b/packages/deepagent-code/src/session/goal-manager.ts index 244c2ebd..38a1072f 100644 --- a/packages/deepagent-code/src/session/goal-manager.ts +++ b/packages/deepagent-code/src/session/goal-manager.ts @@ -395,7 +395,10 @@ export const layer = Layer.effect( }, ]) : (fromFile?.plan ?? null)) - const store = new DocumentStore(goalStoreRoot(sessionID)) + // I33-1: SHARED handle so the goal's plan doc and the `plan` tool's writes (core plan-store, + // same root + slug) share ONE in-memory index — a goal-side plan edit is immediately visible to + // the tool path and vice versa, structurally preventing the two-store divergence. + const store = DocumentStore.shared(goalStoreRoot(sessionID)) const planDocId = plan != null ? GoalDriver.materializePlanDoc({ store, sessionId: sessionID, plan }) @@ -585,9 +588,10 @@ export const layer = Layer.effect( if (!c.paused) return true yield* mutateControl(sessionID, (ctrl) => (ctrl.paused = false)) AgentGateway.DeepAgentSessionState.setActiveGoalPhase(sessionID, "running") - // Re-drive: the persisted run_context doc resumes exactly where it paused. A fresh store handle - // over the same root re-reads the loop state. - const store = new DocumentStore(goalStoreRoot(sessionID)) + // Re-drive: the persisted run_context doc resumes exactly where it paused. I33-1: SHARED handle + // (same shared index as the tool path + the first driver) so the resumed loop's plan reads/writes + // stay coherent with any concurrent `plan` tool write on the same session. + const store = DocumentStore.shared(goalStoreRoot(sessionID)) // V4.1 §N DUAL-PATH resume. flag ON ⇒ RE-SEED the event chain rather than starting a BackgroundJob. // The paused command consumed the current normal cursor key without advancing state. Resume keeps diff --git a/packages/deepagent-code/src/worktree/index.ts b/packages/deepagent-code/src/worktree/index.ts index 15666988..bed6ed95 100644 --- a/packages/deepagent-code/src/worktree/index.ts +++ b/packages/deepagent-code/src/worktree/index.ts @@ -216,6 +216,28 @@ export class Service extends Context.Service()("@deepagent-c type GitResult = { code: number; text: string; stderr: string } +// I33-5: build the argument list for a HARDENED read-only git invocation inside a possibly +// attacker-controlled worktree. Neutralizes the content-driven code paths git otherwise runs on +// ordinary reads (each empirically verified against a hostile repo): +// - `-c core.hooksPath=/dev/null` → no hook execution (any subcommand) +// - `diff --no-ext-diff` → ignore any `[diff "x"] external=` driver (use git's built-in diff) +// - `diff --no-textconv` → ignore any `[diff "x"] textconv=` filter +// `--no-ext-diff` / `--no-textconv` are DIFF-SUBCOMMAND flags (they must follow `diff`, and status/ +// rev-list reject them), so they are appended only for the diff subcommand. NOTE: an empty +// `-c diff.external=` does NOT work — git then tries to execute the empty string and dies with +// "cannot run"; `--no-ext-diff` is the correct neutralizer. +// Clean/smudge/process filters (`[filter "x"] clean=/process=`) are keyed by ATTACKER-CHOSEN names and +// cannot be disabled by name; the mitigation is behavioral — safeGit only runs read ops (status +// --porcelain / diff --numstat / rev-list) that never add or canonicalize worktree content, so clean +// filters are never invoked. safeGit must NEVER be used for `git add`/checkout. Exported pure so the +// hardening is unit-testable without the full worktree layer. +export const hardenedGitArgs = (args: readonly string[]): string[] => { + const hooks = ["-c", "core.hooksPath=/dev/null"] + return args[0] === "diff" + ? [...hooks, "diff", "--no-ext-diff", "--no-textconv", ...args.slice(1)] + : [...hooks, ...args] +} + export const layer: Layer.Layer< Service, never, @@ -697,10 +719,11 @@ export const layer: Layer.Layer< return true }) - // U3: read-only git for informational reads inside a worktree. Adds codex's hooksPath override - // so merely rendering a diff / counting changes can never execute a malicious repo's hooks. The - // base Git.Service already disables fsmonitor + optional locks; mutating ops keep normal hooks. - const safeGit = (args: string[], cwd: string) => git(["-c", "core.hooksPath=/dev/null", ...args], { cwd }) + // U3 / I33-5: read-only git for informational reads inside a possibly attacker-controlled worktree. + // Hardening (hooks + external-diff + textconv neutralized) lives in the exported pure + // `hardenedGitArgs` so it is unit-testable; see its comment for the threat model. safeGit must only + // ever run read ops (status/diff/rev-list), never `git add`/checkout. + const safeGit = (args: string[], cwd: string) => git(hardenedGitArgs(args), { cwd }) // U3: locate a worktree entry by directory, the shared lookup the new ops need. const resolveEntry = Effect.fnUntraced(function* (directory: string) { diff --git a/packages/deepagent-code/test/session/goal-loop-wiring.test.ts b/packages/deepagent-code/test/session/goal-loop-wiring.test.ts index 7af59ae5..ef0225be 100644 --- a/packages/deepagent-code/test/session/goal-loop-wiring.test.ts +++ b/packages/deepagent-code/test/session/goal-loop-wiring.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test, afterEach } from "bun:test" +import { describe, expect, test, beforeEach, afterEach } from "bun:test" import { Effect } from "effect" import type { Diagnostic } from "../../src/lsp/client" import { @@ -319,6 +319,12 @@ describe("V3.9 §D wiring — buildStepExecutor", () => { describe("V3.9 §E F3 wiring — plan bridge (worker plan edits reach the goal plan doc)", () => { const roots: string[] = [] + // I33-1: the structural plan now lives in the DocumentStore authority reached via session-state's + // configured root (plan-store). Configure a fresh state dir per case so the child session's + // getPlan/setPlan (used by seedChildPlan/mirrorChildPlan) has a plan-store root. + beforeEach(() => { + AgentGateway.DeepAgentSessionState.configure(mkdtempSync(path.join(tmpdir(), "deepagent-f3-state-"))) + }) const step = (id: string, status: PlanStep["status"]): PlanStep => ({ step_id: id, title: id, diff --git a/packages/deepagent-code/test/worktree/safe-git-hardening.test.ts b/packages/deepagent-code/test/worktree/safe-git-hardening.test.ts new file mode 100644 index 00000000..7ac11e6c --- /dev/null +++ b/packages/deepagent-code/test/worktree/safe-git-hardening.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, test, beforeEach, afterEach } from "bun:test" +import { mkdtempSync, rmSync, existsSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import path from "node:path" +import { execFileSync } from "node:child_process" +import { hardenedGitArgs } from "../../src/worktree/index" + +// I33-5: safeGit runs read-only git (diff/status/rev-list) inside a worktree that may check out an +// ATTACKER-CONTROLLED repo. git has content-driven code paths that execute on ordinary reads — +// external diff drivers, textconv filters, hooks — so a mere `git diff` must not run repo-configured +// commands. `hardenedGitArgs` prepends the config that neutralizes them. These tests pin (1) the exact +// argument shape and (2) an END-TO-END proof against a real hostile repo that the drivers do NOT fire. + +describe("I33-5 hardenedGitArgs — argument shape", () => { + test("non-diff subcommand: only the hooksPath neutralizer is prepended", () => { + expect(hardenedGitArgs(["status", "--porcelain"])).toEqual([ + "-c", + "core.hooksPath=/dev/null", + "status", + "--porcelain", + ]) + }) + + test("diff subcommand gets --no-ext-diff + --no-textconv, inserted right after `diff`", () => { + expect(hardenedGitArgs(["diff", "--numstat", "a..HEAD"])).toEqual([ + "-c", + "core.hooksPath=/dev/null", + "diff", + "--no-ext-diff", + "--no-textconv", + "--numstat", + "a..HEAD", + ]) + }) + + test("non-diff subcommands do NOT get diff-only flags (rev-list/status would reject them)", () => { + const revlist = hardenedGitArgs(["rev-list", "--count", "a..HEAD"]) + expect(revlist).not.toContain("--no-textconv") + expect(revlist).not.toContain("--no-ext-diff") + }) +}) + +// End-to-end: build a repo whose gitattributes wires malicious diff drivers, then confirm plain git +// executes them (the vulnerability) while hardenedGitArgs blocks them (the fix). Skips if git absent. +describe("I33-5 hardenedGitArgs — end-to-end against a hostile repo", () => { + let repo: string + let sentinelDir: string + const git = (args: string[]) => execFileSync("git", args, { cwd: repo, stdio: "ignore" }) + const pwned = (name: string) => existsSync(path.join(sentinelDir, name)) + + beforeEach(() => { + repo = mkdtempSync(path.join(tmpdir(), "i335-repo-")) + sentinelDir = mkdtempSync(path.join(tmpdir(), "i335-sentinel-")) + git(["init", "-q"]) + git(["config", "user.email", "t@t.t"]) + git(["config", "user.name", "t"]) + // Malicious external diff driver + textconv, wired to a file via .gitattributes. + git(["config", "diff.evil.external", `/bin/sh -c "touch ${path.join(sentinelDir, "PWNED_EXTERNAL")}"`]) + git(["config", "diff.evil.textconv", `/bin/sh -c "touch ${path.join(sentinelDir, "PWNED_TEXTCONV")}" <`]) + writeFileSync(path.join(repo, ".gitattributes"), "f.txt diff=evil\n") + writeFileSync(path.join(repo, "f.txt"), "v1\n") + git(["add", "f.txt", ".gitattributes"]) + git(["commit", "-qm", "init"]) + writeFileSync(path.join(repo, "f.txt"), "v2\n") // a change so `git diff` has work to do + }) + afterEach(() => { + rmSync(repo, { recursive: true, force: true }) + rmSync(sentinelDir, { recursive: true, force: true }) + }) + + // git may exit non-zero (e.g. the malicious external driver dies); we only care whether the sentinel + // was created, so run non-throwing. The real safeGit goes through base git() which catches exit codes. + const runGit = (args: string[]) => { + try { + execFileSync("git", args, { cwd: repo, stdio: "ignore" }) + } catch { + /* exit code irrelevant to this test — the sentinel is the signal */ + } + } + + test("baseline: plain `git diff` EXECUTES the malicious drivers (proves the vector is real)", () => { + runGit(["diff"]) + expect(pwned("PWNED_EXTERNAL") || pwned("PWNED_TEXTCONV")).toBe(true) + }) + + test("hardened: `git ` does NOT execute any malicious driver", () => { + runGit(hardenedGitArgs(["diff"])) + expect(pwned("PWNED_EXTERNAL")).toBe(false) + expect(pwned("PWNED_TEXTCONV")).toBe(false) + }) + + test("hardened: `diff --numstat` (the real safeGit call) also stays clean + exits cleanly", () => { + runGit(hardenedGitArgs(["diff", "--numstat"])) + expect(pwned("PWNED_EXTERNAL")).toBe(false) + expect(pwned("PWNED_TEXTCONV")).toBe(false) + }) +})